diff --git a/fluxer_common/src/config.rs b/fluxer_common/src/config.rs index fe945d8f7..b051203a8 100644 --- a/fluxer_common/src/config.rs +++ b/fluxer_common/src/config.rs @@ -1,6 +1,6 @@ // SPDX-License-Identifier: AGPL-3.0-or-later -use std::{env, path::Path}; +use std::{env, fs, path::Path}; #[derive(Clone, Debug, Eq, PartialEq)] pub enum GeoipSourceConfig { @@ -124,7 +124,59 @@ fn percent_decode(value: &str) -> String { } pub fn env_value(name: &str) -> Option { - env::var(name).ok().filter(|value| !value.trim().is_empty()) + resolve_env_value(name, |key| env::var(key).ok()).unwrap_or_else(|error| panic!("{error}")) +} + +pub fn resolve_env_value(name: &str, get: F) -> Result, String> +where + F: Fn(&str) -> Option, +{ + let non_blank = |value: Option| value.filter(|value| !value.trim().is_empty()); + let value = non_blank(get(name)); + let Some(path) = non_blank(get(&format!("{name}_FILE"))) else { + return Ok(value); + }; + if value.is_some() { + return Err(format!("{name} and {name}_FILE are both set, set only one")); + } + let contents = fs::read_to_string(&path) + .map_err(|error| format!("{name}_FILE could not read {path} ({error})"))?; + let contents = contents + .strip_suffix('\n') + .map_or(contents.as_str(), |rest| { + rest.strip_suffix('\r').unwrap_or(rest) + }); + Ok(non_blank(Some(contents.to_owned()))) +} + +pub fn resolve_env_files(vars: I) -> Result, String> +where + I: IntoIterator, +{ + let vars: Vec<(String, String)> = vars.into_iter().collect(); + let get = |key: &str| { + vars.iter() + .find_map(|(name, value)| (name == key).then(|| value.clone())) + }; + let mut resolved = Vec::new(); + for (key, _) in &vars { + let Some(name) = key + .strip_suffix("_FILE") + .filter(|name| name.starts_with("FLUXER_")) + else { + continue; + }; + if let Some(value) = resolve_env_value(name, get)? { + resolved.push((name.to_owned(), value)); + } + } + let rest: Vec<(String, String)> = vars + .iter() + .filter(|(key, _)| !resolved.iter().any(|(name, _)| name == key)) + .cloned() + .collect(); + resolved.extend(rest); + Ok(resolved) } pub fn read_env(name: &str, fallback: &str) -> String { @@ -763,6 +815,106 @@ mod tests { assert_eq!(None, port); } + fn secret_file(dir: &tempfile::TempDir, name: &str, contents: &str) -> String { + let path = dir.path().join(name); + fs::write(&path, contents).expect("write secret file"); + path.to_string_lossy().into_owned() + } + + fn pairs_reader(pairs: Vec<(String, String)>) -> impl Fn(&str) -> Option { + move |key| { + pairs + .iter() + .find_map(|(name, value)| (name == key).then(|| value.clone())) + } + } + + #[test] + fn env_value_reads_name_file_when_name_is_blank() { + let dir = tempfile::tempdir().expect("tempdir"); + let path = secret_file(&dir, "secret", "from-file\r\n"); + let unset = pairs_reader(vec![("X_FILE".to_owned(), path.clone())]); + let blank = pairs_reader(vec![ + ("X".to_owned(), " ".to_owned()), + ("X_FILE".to_owned(), path), + ]); + assert_eq!( + Ok(Some("from-file".to_owned())), + resolve_env_value("X", unset) + ); + assert_eq!( + Ok(Some("from-file".to_owned())), + resolve_env_value("X", blank) + ); + } + + #[test] + fn env_value_trims_only_one_trailing_newline() { + let dir = tempfile::tempdir().expect("tempdir"); + let pem = secret_file(&dir, "pem", "-----BEGIN-----\nabc\n-----END-----\n\n"); + let empty = secret_file(&dir, "empty", "\n"); + assert_eq!( + Ok(Some("-----BEGIN-----\nabc\n-----END-----\n".to_owned())), + resolve_env_value("X", pairs_reader(vec![("X_FILE".to_owned(), pem)])) + ); + assert_eq!( + Ok(None), + resolve_env_value("X", pairs_reader(vec![("X_FILE".to_owned(), empty)])) + ); + } + + #[test] + fn env_value_rejects_name_and_name_file_together() { + let reader = pairs_reader(vec![ + ("X".to_owned(), "direct".to_owned()), + ("X_FILE".to_owned(), "/run/secrets/x".to_owned()), + ]); + assert_eq!( + Err("X and X_FILE are both set, set only one".to_owned()), + resolve_env_value("X", reader) + ); + } + + #[test] + fn env_value_names_the_file_when_it_is_missing() { + let dir = tempfile::tempdir().expect("tempdir"); + let path = dir.path().join("missing").to_string_lossy().into_owned(); + let error = resolve_env_value("X", pairs_reader(vec![("X_FILE".to_owned(), path.clone())])) + .expect_err("missing file fails"); + assert!(error.starts_with(&format!("X_FILE could not read {path} ("))); + } + + #[test] + fn resolve_env_files_replaces_fluxer_names_with_file_values() { + let dir = tempfile::tempdir().expect("tempdir"); + let path = secret_file(&dir, "secret", "from-file\n"); + let missing = dir.path().join("missing").to_string_lossy().into_owned(); + let resolved = resolve_env_files(vec![ + ("FLUXER_X".to_owned(), String::new()), + ("FLUXER_X_FILE".to_owned(), path), + ("FLUXER_Y".to_owned(), "plain".to_owned()), + ("FLUXER_Y_FILE".to_owned(), String::new()), + ("SSL_CERT_FILE".to_owned(), missing), + ]) + .expect("resolves"); + let values = |key: &str| { + resolved + .iter() + .filter(|(name, _)| name == key) + .map(|(_, value)| value.as_str()) + .collect::>() + }; + assert_eq!(vec!["from-file"], values("FLUXER_X")); + assert_eq!(vec!["plain"], values("FLUXER_Y")); + assert!( + resolve_env_files(vec![ + ("FLUXER_X".to_owned(), "direct".to_owned()), + ("FLUXER_X_FILE".to_owned(), "/run/secrets/x".to_owned()), + ]) + .is_err() + ); + } + #[test] fn a_blank_value_reads_as_unset() { unsafe { diff --git a/fluxer_docs/src/content/docs/operator/configuration.mdx b/fluxer_docs/src/content/docs/operator/configuration.mdx index 2b9ad36e1..1df64841f 100644 --- a/fluxer_docs/src/content/docs/operator/configuration.mdx +++ b/fluxer_docs/src/content/docs/operator/configuration.mdx @@ -36,6 +36,34 @@ Precedence, highest first: Use `true` or `false` for booleans, decimal integers for integer settings, and the specified object or array for JSON settings. Defaults and accepted values are listed below. Every service built from this version of the stack files or later reads an empty or blank value the same as an unset one, so an empty value restores the default. Older images do not, which [Match the images to the stack files](/operator/upgrading/#match-the-images-to-the-stack-files) covers. +### Reading a secret from a file + +Any setting a Fluxer service reads for its configuration can come from a file instead, such as a Docker secret. Set the variable's name with `_FILE` appended to the file's path, and leave the variable itself empty. The service reads the file at startup and drops one trailing newline. Startup fails, naming both variables, when the variable and its `_FILE` form are both set. It also fails, naming the path, when the file is missing, unreadable or not valid UTF-8. The name to use is the one the service sees, so `POSTGRES_PASSWORD` in `.env` becomes `FLUXER_POSTGRES_PASSWORD_FILE`. + +Add the secrets through a local Compose override. `FLUXER_SUDO_MODE_SECRET` is read by `api` and `worker` only: + +```yaml +secrets: + sudo_mode_secret: + file: ./secrets/sudo_mode_secret + +services: + api: + secrets: [sudo_mode_secret] + environment: + FLUXER_SUDO_MODE_SECRET: '' + FLUXER_SUDO_MODE_SECRET_FILE: /run/secrets/sudo_mode_secret + worker: + secrets: [sudo_mode_secret] + environment: + FLUXER_SUDO_MODE_SECRET: '' + FLUXER_SUDO_MODE_SECRET_FILE: /run/secrets/sudo_mode_secret +``` + +Compose still checks the required names in `.env` before it applies the override, so keep a placeholder value there. Every Fluxer service gets the same shared settings, so a secret several services read, such as `FLUXER_POSTGRES_PASSWORD`, needs the same two entries on each of them: `api`, `worker`, `gateway`, `media-proxy`, `push`, `admin`, and `snowflakes`, `users`, `gifs`, `messages` and `unfurl` with their `-shard` services. Any service left out keeps the placeholder. The secret file must be readable by the user the container runs as. + +The bundled backing containers read their own settings. The `postgres` container takes `POSTGRES_PASSWORD: ''` and `POSTGRES_PASSWORD_FILE` in its own `environment`, and on a fresh volume it creates the database with that password. `seaweedfs-init`, `meilisearch` and `livekit` have no `_FILE` form, so a file-based S3, Meilisearch or LiveKit secret still needs its real value in `.env` for them. `FLUXER_ENV`, `LOG_LEVEL`, `FLUXER_DISABLE_RATE_LIMITS` and the `FLUXER_ERLANG_*` settings other than `FLUXER_ERLANG_COOKIE` are read directly, so set them as plain values. + ## Core identity and public address `FLUXER_DOMAIN` is required. Everything else here is optional. diff --git a/fluxer_gateway/scripts/docker_entrypoint.sh b/fluxer_gateway/scripts/docker_entrypoint.sh index 3b04738b4..03ea91476 100755 --- a/fluxer_gateway/scripts/docker_entrypoint.sh +++ b/fluxer_gateway/scripts/docker_entrypoint.sh @@ -70,6 +70,21 @@ else rm -f "$node_name_file" fi +case "${FLUXER_ERLANG_COOKIE_FILE:-}" in +*[![:space:]]*) + case "${FLUXER_ERLANG_COOKIE:-}" in + *[![:space:]]*) + echo 'FLUXER_ERLANG_COOKIE and FLUXER_ERLANG_COOKIE_FILE are both set, set only one.' >&2 + exit 1 + ;; + esac + if ! FLUXER_ERLANG_COOKIE="$(cat -- "$FLUXER_ERLANG_COOKIE_FILE")"; then + echo "FLUXER_ERLANG_COOKIE_FILE could not read $FLUXER_ERLANG_COOKIE_FILE." >&2 + exit 1 + fi + ;; +esac + if [ -z "${FLUXER_ERLANG_COOKIE:-}" ]; then echo 'FLUXER_ERLANG_COOKIE is required.' >&2 exit 1 diff --git a/fluxer_gateway/scripts/fluxer_gateway_release_wrapper.sh b/fluxer_gateway/scripts/fluxer_gateway_release_wrapper.sh index 244bedb71..6764395dc 100644 --- a/fluxer_gateway/scripts/fluxer_gateway_release_wrapper.sh +++ b/fluxer_gateway/scripts/fluxer_gateway_release_wrapper.sh @@ -18,4 +18,8 @@ if [ -r "$node_name_file" ]; then FLUXER_ERLANG_NODE_NAME="$(cat "$node_name_file")" export FLUXER_ERLANG_NODE_NAME fi +if [ -r "${FLUXER_ERLANG_COOKIE_FILE:-}" ] && [ -z "$(printf '%s' "${FLUXER_ERLANG_COOKIE:-}" | tr -d '[:space:]')" ]; then + FLUXER_ERLANG_COOKIE="$(cat "$FLUXER_ERLANG_COOKIE_FILE")" + export FLUXER_ERLANG_COOKIE +fi exec "$script_dir/fluxer_gateway.real" "$@" diff --git a/fluxer_gateway/src/gateway/fluxer_gateway_config.erl b/fluxer_gateway/src/gateway/fluxer_gateway_config.erl index f65069135..9ca9752a1 100644 --- a/fluxer_gateway/src/gateway/fluxer_gateway_config.erl +++ b/fluxer_gateway/src/gateway/fluxer_gateway_config.erl @@ -305,11 +305,44 @@ env_string(Name, Default) -> -spec env_value(string()) -> string() | undefined. env_value(Name) -> + Value = non_blank_env(Name), + FileName = Name ++ "_FILE", + case non_blank_env(FileName) of + undefined -> Value; + Path when Value =:= undefined -> read_env_file(FileName, Path); + _ -> erlang:error({ambiguous_env, Name, FileName}) + end. + +-spec non_blank_env(string()) -> string() | undefined. +non_blank_env(Name) -> case os:getenv(Name) of false -> undefined; Value -> non_blank(Value) end. +-spec read_env_file(string(), string()) -> string() | undefined. +read_env_file(FileName, Path) -> + case file:read_file(Path) of + {ok, Contents} -> env_file_value(FileName, Path, strip_newline(Contents)); + {error, Reason} -> erlang:error({unreadable_env_file, FileName, Path, Reason}) + end. + +-spec env_file_value(string(), string(), binary()) -> string() | undefined. +env_file_value(FileName, Path, Contents) -> + case unicode:characters_to_list(Contents) of + Value when is_list(Value) -> non_blank(Value); + _ -> erlang:error({invalid_env_file, FileName, Path}) + end. + +-spec strip_newline(binary()) -> binary(). +strip_newline(Contents) -> + Size = byte_size(Contents), + case Contents of + <> -> Rest; + <> -> Rest; + _ -> Contents + end. + -spec non_blank(string()) -> string() | undefined. non_blank(Value) -> case string:trim(Value) of diff --git a/fluxer_gateway/test/fluxer_gateway_config_tests.erl b/fluxer_gateway/test/fluxer_gateway_config_tests.erl index 4bae2d57c..d9b6e1f91 100644 --- a/fluxer_gateway/test/fluxer_gateway_config_tests.erl +++ b/fluxer_gateway/test/fluxer_gateway_config_tests.erl @@ -272,6 +272,64 @@ public_endpoints_defaults_test() -> ?assertEqual(undefined, maps:get(media_proxy_endpoint, Config)), ?assertEqual(<<"http://localhost:8088">>, maps:get(static_cdn_endpoint, Config)). +env_value_reads_name_file_test() -> + with_env_file(<<"from-file\r\n">>, fun(Path) -> + with_envs( + [{"FLUXER_GATEWAY_TEST_SECRET", ""}, {"FLUXER_GATEWAY_TEST_SECRET_FILE", Path}], + fun() -> + ?assertEqual( + "from-file", fluxer_gateway_config:env_value("FLUXER_GATEWAY_TEST_SECRET") + ) + end + ) + end). + +env_value_trims_only_one_newline_test() -> + with_env_file(<<"line1\nline2\n\n">>, fun(Path) -> + with_env("FLUXER_GATEWAY_TEST_SECRET_FILE", Path, fun() -> + ?assertEqual( + "line1\nline2\n", fluxer_gateway_config:env_value("FLUXER_GATEWAY_TEST_SECRET") + ) + end) + end). + +env_value_rejects_name_and_name_file_test() -> + with_envs( + [ + {"FLUXER_GATEWAY_TEST_SECRET", "direct"}, + {"FLUXER_GATEWAY_TEST_SECRET_FILE", "/run/secrets/x"} + ], + fun() -> + ?assertError( + {ambiguous_env, "FLUXER_GATEWAY_TEST_SECRET", + "FLUXER_GATEWAY_TEST_SECRET_FILE"}, + fluxer_gateway_config:env_value("FLUXER_GATEWAY_TEST_SECRET") + ) + end + ). + +env_value_rejects_missing_name_file_test() -> + Path = "/nonexistent/fluxer-gateway-test-secret", + with_env("FLUXER_GATEWAY_TEST_SECRET_FILE", Path, fun() -> + ?assertError( + {unreadable_env_file, "FLUXER_GATEWAY_TEST_SECRET_FILE", Path, enoent}, + fluxer_gateway_config:env_value("FLUXER_GATEWAY_TEST_SECRET") + ) + end). + +with_env_file(Contents, Fun) -> + Path = filename:join( + filename:basedir(user_cache, "fluxer_gateway_tests"), + integer_to_list(erlang:unique_integer([positive])) + ), + ok = filelib:ensure_dir(Path), + ok = file:write_file(Path, Contents), + try + Fun(Path) + after + file:delete(Path) + end. + with_envs([], Fun) -> Fun(); with_envs([{Name, Value} | Rest], Fun) -> diff --git a/fluxer_media_proxy/src/cli.rs b/fluxer_media_proxy/src/cli.rs index 1ab635bac..4e96a310d 100644 --- a/fluxer_media_proxy/src/cli.rs +++ b/fluxer_media_proxy/src/cli.rs @@ -48,7 +48,9 @@ pub enum StorageBackendArg { } pub fn load_config(args: &Args) -> anyhow::Result { - load_config_from_iter(args, std::env::vars()) + let vars = + fluxer_common::config::resolve_env_files(std::env::vars()).map_err(anyhow::Error::msg)?; + load_config_from_iter(args, vars) } pub fn load_config_from_iter(args: &Args, vars: I) -> anyhow::Result diff --git a/fluxer_media_proxy/src/config/mod.rs b/fluxer_media_proxy/src/config/mod.rs index ff971261c..d341c4d3f 100644 --- a/fluxer_media_proxy/src/config/mod.rs +++ b/fluxer_media_proxy/src/config/mod.rs @@ -13,7 +13,7 @@ use parse::{ parse_mode_env, parse_policy_mode, parse_storage_backend, parse_u16, parse_u64, parse_usize, validate_read_endpoint, }; -use std::{env, path::PathBuf}; +use std::path::PathBuf; #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub enum StorageBackend { @@ -144,10 +144,6 @@ pub struct Config { } impl Config { - pub fn load_from_env() -> anyhow::Result { - Self::load_from_iter(env::vars()) - } - pub fn load_from_iter(vars: I) -> anyhow::Result where I: IntoIterator, diff --git a/fluxer_push/src/cli.rs b/fluxer_push/src/cli.rs index ff7f94243..3925c2db9 100644 --- a/fluxer_push/src/cli.rs +++ b/fluxer_push/src/cli.rs @@ -27,7 +27,9 @@ pub enum Command { } pub fn load_config(args: &Args) -> anyhow::Result { - load_config_from_iter(args, std::env::vars()) + let vars = + fluxer_svc::config::resolve_env_files(std::env::vars()).map_err(anyhow::Error::msg)?; + load_config_from_iter(args, vars) } fn load_config_from_iter(args: &Args, vars: I) -> anyhow::Result diff --git a/fluxer_svc/src/config.rs b/fluxer_svc/src/config.rs index 508e43844..12ccf878a 100644 --- a/fluxer_svc/src/config.rs +++ b/fluxer_svc/src/config.rs @@ -1,6 +1,7 @@ // SPDX-License-Identifier: AGPL-3.0-or-later use std::env; +use std::fs; use std::net::{IpAddr, SocketAddr}; use std::time::Duration; @@ -52,7 +53,7 @@ pub enum DatabaseBackend { impl ServiceConfig { pub fn from_env() -> anyhow::Result { - Self::from_env_reader(|name| env::var(name).ok()) + Self::from_env_reader(env_var) } fn from_env_reader(get: F) -> anyhow::Result @@ -203,7 +204,63 @@ fn default_max_concurrent_requests(service_name: &str) -> usize { } pub fn optional_env(name: &str) -> Option { - optional_from(&|key| env::var(key).ok(), name) + env_var(name) +} + +fn env_var(name: &str) -> Option { + resolve_env_value(name, |key| env::var(key).ok()).unwrap_or_else(|error| panic!("{error}")) +} + +fn resolve_env_value(name: &str, get: F) -> Result, String> +where + F: Fn(&str) -> Option, +{ + let non_blank = |value: Option| value.filter(|value| !value.trim().is_empty()); + let value = non_blank(get(name)); + let Some(path) = non_blank(get(&format!("{name}_FILE"))) else { + return Ok(value); + }; + if value.is_some() { + return Err(format!("{name} and {name}_FILE are both set, set only one")); + } + let contents = fs::read_to_string(&path) + .map_err(|error| format!("{name}_FILE could not read {path} ({error})"))?; + let contents = contents + .strip_suffix('\n') + .map_or(contents.as_str(), |rest| { + rest.strip_suffix('\r').unwrap_or(rest) + }); + Ok(non_blank(Some(contents.to_owned()))) +} + +pub fn resolve_env_files(vars: I) -> Result, String> +where + I: IntoIterator, +{ + let vars: Vec<(String, String)> = vars.into_iter().collect(); + let get = |key: &str| { + vars.iter() + .find_map(|(name, value)| (name == key).then(|| value.clone())) + }; + let mut resolved = Vec::new(); + for (key, _) in &vars { + let Some(name) = key + .strip_suffix("_FILE") + .filter(|name| name.starts_with("FLUXER_")) + else { + continue; + }; + if let Some(value) = resolve_env_value(name, get)? { + resolved.push((name.to_owned(), value)); + } + } + let rest: Vec<(String, String)> = vars + .iter() + .filter(|(key, _)| !resolved.iter().any(|(name, _)| name == key)) + .cloned() + .collect(); + resolved.extend(rest); + Ok(resolved) } fn optional_from(get: &F, name: &str) -> Option @@ -276,6 +333,46 @@ mod tests { .unwrap() } + #[test] + fn resolves_name_file_entries() { + let dir = env::temp_dir().join(format!("fluxer-svc-env-file-{}", std::process::id())); + fs::create_dir_all(&dir).unwrap(); + let path = dir.join("token"); + fs::write(&path, "from-file\r\n").unwrap(); + let path = path.to_string_lossy().into_owned(); + let missing = dir.join("missing").to_string_lossy().into_owned(); + let resolved = resolve_env_files(vec![ + ("FLUXER_NATS_AUTH_TOKEN".to_owned(), String::new()), + ("FLUXER_NATS_AUTH_TOKEN_FILE".to_owned(), path.clone()), + ]) + .unwrap(); + let both = resolve_env_files(vec![ + ("FLUXER_X".to_owned(), "direct".to_owned()), + ("FLUXER_X_FILE".to_owned(), path), + ]); + let unreadable = resolve_env_files(vec![("FLUXER_X_FILE".to_owned(), missing.clone())]); + let unrelated = resolve_env_files(vec![("SSL_CERT_FILE".to_owned(), missing.clone())]); + fs::remove_dir_all(&dir).unwrap(); + assert_eq!( + vec!["from-file"], + resolved + .iter() + .filter(|(name, _)| name == "FLUXER_NATS_AUTH_TOKEN") + .map(|(_, value)| value.as_str()) + .collect::>() + ); + assert_eq!( + Err("FLUXER_X and FLUXER_X_FILE are both set, set only one".to_owned()), + both + ); + assert!( + unreadable + .unwrap_err() + .starts_with(&format!("FLUXER_X_FILE could not read {missing} (")) + ); + assert_eq!(Ok(vec![("SSL_CERT_FILE".to_owned(), missing)]), unrelated); + } + #[test] fn extracts_statefulset_ordinal() { assert_eq!(shard_id_from_pod_name("my-service-0"), Some(0)); diff --git a/fluxer_unfurl/src/resolvers/klipy.rs b/fluxer_unfurl/src/resolvers/klipy.rs index 0c8cc2e52..50f2fe1a9 100644 --- a/fluxer_unfurl/src/resolvers/klipy.rs +++ b/fluxer_unfurl/src/resolvers/klipy.rs @@ -4,7 +4,6 @@ use super::{ResolveContext, Resolver, ResolverResult}; use crate::http_fetch; use crate::media_proxy::{MediaMetadata, embed_media_flags}; use crate::types::{EmbedMedia, EmbedProvider, MessageEmbed}; -use fluxer_svc::config::optional_env; use std::future::Future; use std::pin::Pin; use std::time::Duration; @@ -49,7 +48,7 @@ impl Resolver for KlipyResolver { ctx: &'a ResolveContext<'_>, ) -> Pin> + Send + 'a>> { Box::pin(async move { - let Some(api_key) = ctx.klipy_api_key.clone().or_else(klipy_api_key) else { + let Some(api_key) = ctx.klipy_api_key.clone() else { return Ok(ResolverResult { embeds: vec![] }); }; let formats = match resolve_media_via_api(ctx, &api_key).await { @@ -136,10 +135,6 @@ fn klipy_resource(kind: &str) -> &'static str { } } -fn klipy_api_key() -> Option { - optional_env("FLUXER_KLIPY_API_KEY").or_else(|| optional_env("KLIPY_API_KEY")) -} - async fn resolve_media_via_api( ctx: &ResolveContext<'_>, api_key: &str, diff --git a/fluxer_unfurl/src/resolvers/youtube.rs b/fluxer_unfurl/src/resolvers/youtube.rs index 634116e2a..495a1068c 100644 --- a/fluxer_unfurl/src/resolvers/youtube.rs +++ b/fluxer_unfurl/src/resolvers/youtube.rs @@ -5,7 +5,6 @@ use crate::http_fetch; use crate::media_proxy::embed_media_flags; use crate::text_limits; use crate::types::{EmbedAuthor, EmbedMedia, EmbedProvider, MessageEmbed}; -use fluxer_svc::config::optional_env; use serde::Deserialize; use std::future::Future; use std::pin::Pin; @@ -61,7 +60,7 @@ impl Resolver for YouTubeResolver { } }; - let Some(api_key) = ctx.youtube_api_key.clone().or_else(youtube_api_key) else { + let Some(api_key) = ctx.youtube_api_key.clone() else { tracing::debug!("No YouTube API key configured"); return Ok(ResolverResult { embeds: vec![] }); }; @@ -251,10 +250,6 @@ struct YouTubeThumbnail { height: Option, } -fn youtube_api_key() -> Option { - optional_env("FLUXER_YOUTUBE_API_KEY").or_else(|| optional_env("YOUTUBE_API_KEY")) -} - fn build_youtube_api_url(video_id: &str, api_key: &str) -> anyhow::Result { let mut url = Url::parse(YOUTUBE_API_BASE)?; url.query_pairs_mut() diff --git a/fluxer_unfurl/src/shard_impl.rs b/fluxer_unfurl/src/shard_impl.rs index 7413c01b7..2a07f0bad 100644 --- a/fluxer_unfurl/src/shard_impl.rs +++ b/fluxer_unfurl/src/shard_impl.rs @@ -24,6 +24,8 @@ pub struct UnfurlShard { resolvers: Vec>, media_proxy: MediaProxyClient, self_hosted: bool, + youtube_api_key: Option, + klipy_api_key: Option, } impl UnfurlShard { @@ -78,6 +80,10 @@ impl UnfurlShard { resolvers, media_proxy, self_hosted, + youtube_api_key: optional_env("FLUXER_YOUTUBE_API_KEY") + .or_else(|| optional_env("YOUTUBE_API_KEY")), + klipy_api_key: optional_env("FLUXER_KLIPY_API_KEY") + .or_else(|| optional_env("KLIPY_API_KEY")), } } @@ -100,6 +106,8 @@ impl UnfurlShard { internal_http_client(), ), self_hosted: false, + youtube_api_key: None, + klipy_api_key: None, } } @@ -125,8 +133,12 @@ impl UnfurlShard { nsfw_mode, media_proxy: &self.media_proxy, self_hosted: self.self_hosted, - youtube_api_key: youtube_api_key.map(str::to_owned), - klipy_api_key: klipy_api_key.map(str::to_owned), + youtube_api_key: youtube_api_key + .map(str::to_owned) + .or_else(|| self.youtube_api_key.clone()), + klipy_api_key: klipy_api_key + .map(str::to_owned) + .or_else(|| self.klipy_api_key.clone()), }; if let Some(idx) = matched_resolver_idx { diff --git a/fluxer_users/src/pseudonym.rs b/fluxer_users/src/pseudonym.rs index 604532f02..6e140051c 100644 --- a/fluxer_users/src/pseudonym.rs +++ b/fluxer_users/src/pseudonym.rs @@ -1,5 +1,6 @@ // SPDX-License-Identifier: AGPL-3.0-or-later +use fluxer_svc::config::optional_env; use hmac::{KeyInit, Mac}; use std::sync::{LazyLock, OnceLock}; @@ -46,8 +47,8 @@ pub fn configure(secret: Option, environment: Option<&str>) -> anyhow::R pub fn configure_from_env() -> anyhow::Result<()> { configure( - std::env::var(SECRET_ENV).ok(), - std::env::var("FLUXER_ENV").ok().as_deref(), + optional_env(SECRET_ENV), + optional_env("FLUXER_ENV").as_deref(), ) } diff --git a/packages/config/src/__tests__/ConfigLoader.test.ts b/packages/config/src/__tests__/ConfigLoader.test.ts index 2d4e8a7e6..1e79662f7 100644 --- a/packages/config/src/__tests__/ConfigLoader.test.ts +++ b/packages/config/src/__tests__/ConfigLoader.test.ts @@ -1,6 +1,9 @@ // SPDX-License-Identifier: AGPL-3.0-or-later import {generateKeyPairSync} from 'node:crypto'; +import {mkdtempSync, rmSync, writeFileSync} from 'node:fs'; +import {tmpdir} from 'node:os'; +import {join} from 'node:path'; import {getConfig, loadConfig, resetConfig} from '@fluxer/config/src/ConfigLoader'; import {afterEach, beforeEach, describe, expect, test, vi} from 'vitest'; @@ -80,6 +83,24 @@ describe('ConfigLoader', () => { expect((await loadConfig()).domain.base_domain).toBe('localhost'); }); + test('loadConfig reads secrets from NAME_FILE', async () => { + const dir = mkdtempSync(join(tmpdir(), 'fluxer-config-file-')); + try { + const path = join(dir, 'postgres_password'); + writeFileSync(path, 'from-secret-file\n'); + stubMinimalEnv({FLUXER_POSTGRES_PASSWORD: '', FLUXER_POSTGRES_PASSWORD_FILE: path}); + const config = await loadConfig(); + expect(config.database.postgres.password).toBe('from-secret-file'); + } finally { + rmSync(dir, {recursive: true, force: true}); + } + }); + + test('loadConfig rejects NAME and NAME_FILE together', async () => { + stubMinimalEnv({FLUXER_SUDO_MODE_SECRET_FILE: '/run/secrets/sudo'}); + await expect(loadConfig()).rejects.toThrow('FLUXER_SUDO_MODE_SECRET and FLUXER_SUDO_MODE_SECRET_FILE are both set'); + }); + test('getConfig throws when config is not loaded', () => { expect(() => getConfig()).toThrow('Config not loaded'); }); diff --git a/packages/config/src/__tests__/EnvironmentOverrides.test.ts b/packages/config/src/__tests__/EnvironmentOverrides.test.ts index f35f402ea..54e4438eb 100644 --- a/packages/config/src/__tests__/EnvironmentOverrides.test.ts +++ b/packages/config/src/__tests__/EnvironmentOverrides.test.ts @@ -1,7 +1,14 @@ // SPDX-License-Identifier: AGPL-3.0-or-later -import {buildNamedFluxerEnvOverrides, setNestedValue} from '@fluxer/config/src/config_loader/EnvironmentOverrides'; -import {describe, expect, test} from 'vitest'; +import {mkdtempSync, rmSync, writeFileSync} from 'node:fs'; +import {tmpdir} from 'node:os'; +import {join} from 'node:path'; +import { + buildNamedFluxerEnvOverrides, + readEnvValue, + setNestedValue, +} from '@fluxer/config/src/config_loader/EnvironmentOverrides'; +import {afterAll, describe, expect, test} from 'vitest'; describe('setNestedValue', () => { test('sets a top-level key', () => { @@ -316,3 +323,69 @@ describe('buildNamedFluxerEnvOverrides', () => { ); }); }); + +describe('readEnvValue with NAME_FILE', () => { + const dir = mkdtempSync(join(tmpdir(), 'fluxer-env-file-')); + afterAll(() => rmSync(dir, {recursive: true, force: true})); + + function secretFile(name: string, contents: string): string { + const path = join(dir, name); + writeFileSync(path, contents); + return path; + } + + test('reads the value from NAME_FILE when NAME is unset', () => { + const path = secretFile('plain', 'from-file\n'); + expect(readEnvValue({FLUXER_SUDO_MODE_SECRET_FILE: path}, 'FLUXER_SUDO_MODE_SECRET')).toBe('from-file'); + }); + + test('reads the value from NAME_FILE when NAME is blank', () => { + const path = secretFile('blank', 'from-file'); + expect( + readEnvValue({FLUXER_SUDO_MODE_SECRET: ' ', FLUXER_SUDO_MODE_SECRET_FILE: path}, 'FLUXER_SUDO_MODE_SECRET'), + ).toBe('from-file'); + }); + + test('trims only one trailing newline', () => { + const crlf = secretFile('crlf', 'value\r\n'); + const multi = secretFile('multi', '-----BEGIN-----\nabc\n-----END-----\n\n'); + expect(readEnvValue({X_FILE: crlf}, 'X')).toBe('value'); + expect(readEnvValue({X_FILE: multi}, 'X')).toBe('-----BEGIN-----\nabc\n-----END-----\n'); + }); + + test('treats an empty file as unset', () => { + const path = secretFile('empty', '\n'); + expect(readEnvValue({X_FILE: path}, 'X')).toBeUndefined(); + }); + + test('keeps NAME when NAME_FILE is blank', () => { + expect(readEnvValue({X: 'direct', X_FILE: ''}, 'X')).toBe('direct'); + }); + + test('rejects NAME and NAME_FILE together', () => { + const path = secretFile('both', 'from-file'); + expect(() => readEnvValue({X: 'direct', X_FILE: path}, 'X')).toThrow('X and X_FILE are both set, set only one'); + }); + + test('names NAME_FILE and the path when the file is missing', () => { + const path = join(dir, 'missing'); + expect(() => readEnvValue({X_FILE: path}, 'X')).toThrow(`X_FILE could not read ${path} (ENOENT)`); + }); + + test('rejects a file that is not valid UTF-8', () => { + const path = join(dir, 'binary'); + writeFileSync(path, Buffer.from([0xff, 0x61])); + expect(() => readEnvValue({X_FILE: path}, 'X')).toThrow(`X_FILE could not read ${path} (`); + }); + + test('feeds named overrides and aliases', () => { + const secret = secretFile('stripe', 'sk_test_file\n'); + const nats = secretFile('nats', 'nats://nats:4222\n'); + expect( + buildNamedFluxerEnvOverrides({FLUXER_STRIPE_SECRET_KEY_FILE: secret, FLUXER_NATS_CORE_URL_FILE: nats}), + ).toMatchObject({ + integrations: {stripe: {secret_key: 'sk_test_file'}}, + services: {nats: {core_url: 'nats://nats:4222'}}, + }); + }); +}); diff --git a/packages/config/src/config_loader/EnvironmentOverrides.ts b/packages/config/src/config_loader/EnvironmentOverrides.ts index 393874df5..cf5983a89 100644 --- a/packages/config/src/config_loader/EnvironmentOverrides.ts +++ b/packages/config/src/config_loader/EnvironmentOverrides.ts @@ -1,5 +1,6 @@ // SPDX-License-Identifier: AGPL-3.0-or-later +import {readFileSync} from 'node:fs'; import {type ConfigObject, isConfigObject} from '@fluxer/config/src/config_loader/ConfigObject'; type ConfigPathKey = string | number; @@ -420,11 +421,29 @@ const NAMED_FLUXER_ENV_ALIASES: Record = { export const NAMED_FLUXER_ENV_NAMES = Object.keys(NAMED_FLUXER_ENV_OVERRIDES); -export function readEnvValue(env: NodeJS.ProcessEnv, name: string): string | undefined { - const value = env[name]; +function nonBlank(value: string | undefined): string | undefined { return value === undefined || value.trim().length === 0 ? undefined : value; } +export function readEnvValue(env: NodeJS.ProcessEnv, name: string): string | undefined { + const value = nonBlank(env[name]); + const filePath = nonBlank(env[`${name}_FILE`]); + if (filePath === undefined) { + return value; + } + if (value !== undefined) { + throw new Error(`${name} and ${name}_FILE are both set, set only one`); + } + let contents: string; + try { + contents = new TextDecoder('utf-8', {fatal: true}).decode(readFileSync(filePath)); + } catch (error) { + const reason = error instanceof Error && 'code' in error ? String(error.code) : 'unreadable'; + throw new Error(`${name}_FILE could not read ${filePath} (${reason})`); + } + return nonBlank(contents.replace(/\r?\n$/, '')); +} + export function buildNamedFluxerEnvOverrides(env: NodeJS.ProcessEnv): ConfigObject { const overrides: ConfigObject = {}; for (const [envKey, mapping] of Object.entries(NAMED_FLUXER_ENV_OVERRIDES)) {