feat(config): read secrets from NAME_FILE variables (#1421)

Co-authored-by: Hampus <[email protected]>

fluxer_common/src/config.rs

@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: AGPL-3.0-or-later
 
-use std::{env, path::Path};
+use std::{env, fs, path::Path};
 
 #[derive(Clone, Debug, Eq, PartialEq)]
 pub enum GeoipSourceConfig {
@@ -124,7 +124,59 @@ fn percent_decode(value: &str) -> String {
 }
 
 pub fn env_value(name: &str) -> Option<String> {
-    env::var(name).ok().filter(|value| !value.trim().is_empty())
+    resolve_env_value(name, |key| env::var(key).ok()).unwrap_or_else(|error| panic!("{error}"))
+}
+
+pub fn resolve_env_value<F>(name: &str, get: F) -> Result<Option<String>, String>
+where
+    F: Fn(&str) -> Option<String>,
+{
+    let non_blank = |value: Option<String>| value.filter(|value| !value.trim().is_empty());
+    let value = non_blank(get(name));
+    let Some(path) = non_blank(get(&format!("{name}_FILE"))) else {
+        return Ok(value);
+    };
+    if value.is_some() {
+        return Err(format!("{name} and {name}_FILE are both set, set only one"));
+    }
+    let contents = fs::read_to_string(&path)
+        .map_err(|error| format!("{name}_FILE could not read {path} ({error})"))?;
+    let contents = contents
+        .strip_suffix('\n')
+        .map_or(contents.as_str(), |rest| {
+            rest.strip_suffix('\r').unwrap_or(rest)
+        });
+    Ok(non_blank(Some(contents.to_owned())))
+}
+
+pub fn resolve_env_files<I>(vars: I) -> Result<Vec<(String, String)>, String>
+where
+    I: IntoIterator<Item = (String, String)>,
+{
+    let vars: Vec<(String, String)> = vars.into_iter().collect();
+    let get = |key: &str| {
+        vars.iter()
+            .find_map(|(name, value)| (name == key).then(|| value.clone()))
+    };
+    let mut resolved = Vec::new();
+    for (key, _) in &vars {
+        let Some(name) = key
+            .strip_suffix("_FILE")
+            .filter(|name| name.starts_with("FLUXER_"))
+        else {
+            continue;
+        };
+        if let Some(value) = resolve_env_value(name, get)? {
+            resolved.push((name.to_owned(), value));
+        }
+    }
+    let rest: Vec<(String, String)> = vars
+        .iter()
+        .filter(|(key, _)| !resolved.iter().any(|(name, _)| name == key))
+        .cloned()
+        .collect();
+    resolved.extend(rest);
+    Ok(resolved)
 }
 
 pub fn read_env(name: &str, fallback: &str) -> String {
@@ -763,6 +815,106 @@ mod tests {
         assert_eq!(None, port);
     }
 
+    fn secret_file(dir: &tempfile::TempDir, name: &str, contents: &str) -> String {
+        let path = dir.path().join(name);
+        fs::write(&path, contents).expect("write secret file");
+        path.to_string_lossy().into_owned()
+    }
+
+    fn pairs_reader(pairs: Vec<(String, String)>) -> impl Fn(&str) -> Option<String> {
+        move |key| {
+            pairs
+                .iter()
+                .find_map(|(name, value)| (name == key).then(|| value.clone()))
+        }
+    }
+
+    #[test]
+    fn env_value_reads_name_file_when_name_is_blank() {
+        let dir = tempfile::tempdir().expect("tempdir");
+        let path = secret_file(&dir, "secret", "from-file\r\n");
+        let unset = pairs_reader(vec![("X_FILE".to_owned(), path.clone())]);
+        let blank = pairs_reader(vec![
+            ("X".to_owned(), " ".to_owned()),
+            ("X_FILE".to_owned(), path),
+        ]);
+        assert_eq!(
+            Ok(Some("from-file".to_owned())),
+            resolve_env_value("X", unset)
+        );
+        assert_eq!(
+            Ok(Some("from-file".to_owned())),
+            resolve_env_value("X", blank)
+        );
+    }
+
+    #[test]
+    fn env_value_trims_only_one_trailing_newline() {
+        let dir = tempfile::tempdir().expect("tempdir");
+        let pem = secret_file(&dir, "pem", "-----BEGIN-----\nabc\n-----END-----\n\n");
+        let empty = secret_file(&dir, "empty", "\n");
+        assert_eq!(
+            Ok(Some("-----BEGIN-----\nabc\n-----END-----\n".to_owned())),
+            resolve_env_value("X", pairs_reader(vec![("X_FILE".to_owned(), pem)]))
+        );
+        assert_eq!(
+            Ok(None),
+            resolve_env_value("X", pairs_reader(vec![("X_FILE".to_owned(), empty)]))
+        );
+    }
+
+    #[test]
+    fn env_value_rejects_name_and_name_file_together() {
+        let reader = pairs_reader(vec![
+            ("X".to_owned(), "direct".to_owned()),
+            ("X_FILE".to_owned(), "/run/secrets/x".to_owned()),
+        ]);
+        assert_eq!(
+            Err("X and X_FILE are both set, set only one".to_owned()),
+            resolve_env_value("X", reader)
+        );
+    }
+
+    #[test]
+    fn env_value_names_the_file_when_it_is_missing() {
+        let dir = tempfile::tempdir().expect("tempdir");
+        let path = dir.path().join("missing").to_string_lossy().into_owned();
+        let error = resolve_env_value("X", pairs_reader(vec![("X_FILE".to_owned(), path.clone())]))
+            .expect_err("missing file fails");
+        assert!(error.starts_with(&format!("X_FILE could not read {path} (")));
+    }
+
+    #[test]
+    fn resolve_env_files_replaces_fluxer_names_with_file_values() {
+        let dir = tempfile::tempdir().expect("tempdir");
+        let path = secret_file(&dir, "secret", "from-file\n");
+        let missing = dir.path().join("missing").to_string_lossy().into_owned();
+        let resolved = resolve_env_files(vec![
+            ("FLUXER_X".to_owned(), String::new()),
+            ("FLUXER_X_FILE".to_owned(), path),
+            ("FLUXER_Y".to_owned(), "plain".to_owned()),
+            ("FLUXER_Y_FILE".to_owned(), String::new()),
+            ("SSL_CERT_FILE".to_owned(), missing),
+        ])
+        .expect("resolves");
+        let values = |key: &str| {
+            resolved
+                .iter()
+                .filter(|(name, _)| name == key)
+                .map(|(_, value)| value.as_str())
+                .collect::<Vec<_>>()
+        };
+        assert_eq!(vec!["from-file"], values("FLUXER_X"));
+        assert_eq!(vec!["plain"], values("FLUXER_Y"));
+        assert!(
+            resolve_env_files(vec![
+                ("FLUXER_X".to_owned(), "direct".to_owned()),
+                ("FLUXER_X_FILE".to_owned(), "/run/secrets/x".to_owned()),
+            ])
+            .is_err()
+        );
+    }
+
     #[test]
     fn a_blank_value_reads_as_unset() {
         unsafe {

fluxer_docs/src/content/docs/operator/configuration.mdx

@@ -36,6 +36,34 @@ Precedence, highest first:
 
 Use `true` or `false` for booleans, decimal integers for integer settings, and the specified object or array for JSON settings. Defaults and accepted values are listed below. Every service built from this version of the stack files or later reads an empty or blank value the same as an unset one, so an empty value restores the default. Older images do not, which [Match the images to the stack files](/operator/upgrading/#match-the-images-to-the-stack-files) covers.
 
+### Reading a secret from a file
+
+Any setting a Fluxer service reads for its configuration can come from a file instead, such as a Docker secret. Set the variable's name with `_FILE` appended to the file's path, and leave the variable itself empty. The service reads the file at startup and drops one trailing newline. Startup fails, naming both variables, when the variable and its `_FILE` form are both set. It also fails, naming the path, when the file is missing, unreadable or not valid UTF-8. The name to use is the one the service sees, so `POSTGRES_PASSWORD` in `.env` becomes `FLUXER_POSTGRES_PASSWORD_FILE`.
+
+Add the secrets through a local Compose override. `FLUXER_SUDO_MODE_SECRET` is read by `api` and `worker` only:
+
+```yaml
+secrets:
+  sudo_mode_secret:
+    file: ./secrets/sudo_mode_secret
+
+services:
+  api:
+    secrets: [sudo_mode_secret]
+    environment:
+      FLUXER_SUDO_MODE_SECRET: ''
+      FLUXER_SUDO_MODE_SECRET_FILE: /run/secrets/sudo_mode_secret
+  worker:
+    secrets: [sudo_mode_secret]
+    environment:
+      FLUXER_SUDO_MODE_SECRET: ''
+      FLUXER_SUDO_MODE_SECRET_FILE: /run/secrets/sudo_mode_secret
+```
+
+Compose still checks the required names in `.env` before it applies the override, so keep a placeholder value there. Every Fluxer service gets the same shared settings, so a secret several services read, such as `FLUXER_POSTGRES_PASSWORD`, needs the same two entries on each of them: `api`, `worker`, `gateway`, `media-proxy`, `push`, `admin`, and `snowflakes`, `users`, `gifs`, `messages` and `unfurl` with their `-shard` services. Any service left out keeps the placeholder. The secret file must be readable by the user the container runs as.
+
+The bundled backing containers read their own settings. The `postgres` container takes `POSTGRES_PASSWORD: ''` and `POSTGRES_PASSWORD_FILE` in its own `environment`, and on a fresh volume it creates the database with that password. `seaweedfs-init`, `meilisearch` and `livekit` have no `_FILE` form, so a file-based S3, Meilisearch or LiveKit secret still needs its real value in `.env` for them. `FLUXER_ENV`, `LOG_LEVEL`, `FLUXER_DISABLE_RATE_LIMITS` and the `FLUXER_ERLANG_*` settings other than `FLUXER_ERLANG_COOKIE` are read directly, so set them as plain values.
+
 ## Core identity and public address
 
 `FLUXER_DOMAIN` is required. Everything else here is optional.

fluxer_gateway/scripts/docker_entrypoint.sh

@@ -70,6 +70,21 @@ else
 	rm -f "$node_name_file"
 fi
 
+case "${FLUXER_ERLANG_COOKIE_FILE:-}" in
+*[![:space:]]*)
+	case "${FLUXER_ERLANG_COOKIE:-}" in
+	*[![:space:]]*)
+		echo 'FLUXER_ERLANG_COOKIE and FLUXER_ERLANG_COOKIE_FILE are both set, set only one.' >&2
+		exit 1
+		;;
+	esac
+	if ! FLUXER_ERLANG_COOKIE="$(cat -- "$FLUXER_ERLANG_COOKIE_FILE")"; then
+		echo "FLUXER_ERLANG_COOKIE_FILE could not read $FLUXER_ERLANG_COOKIE_FILE." >&2
+		exit 1
+	fi
+	;;
+esac
+
 if [ -z "${FLUXER_ERLANG_COOKIE:-}" ]; then
 	echo 'FLUXER_ERLANG_COOKIE is required.' >&2
 	exit 1

fluxer_gateway/scripts/fluxer_gateway_release_wrapper.sh

@@ -18,4 +18,8 @@ if [ -r "$node_name_file" ]; then
 	FLUXER_ERLANG_NODE_NAME="$(cat "$node_name_file")"
 	export FLUXER_ERLANG_NODE_NAME
 fi
+if [ -r "${FLUXER_ERLANG_COOKIE_FILE:-}" ] && [ -z "$(printf '%s' "${FLUXER_ERLANG_COOKIE:-}" | tr -d '[:space:]')" ]; then
+	FLUXER_ERLANG_COOKIE="$(cat "$FLUXER_ERLANG_COOKIE_FILE")"
+	export FLUXER_ERLANG_COOKIE
+fi
 exec "$script_dir/fluxer_gateway.real" "$@"

fluxer_gateway/src/gateway/fluxer_gateway_config.erl

@@ -305,11 +305,44 @@ env_string(Name, Default) ->
 
 -spec env_value(string()) -> string() | undefined.
 env_value(Name) ->
+    Value = non_blank_env(Name),
+    FileName = Name ++ "_FILE",
+    case non_blank_env(FileName) of
+        undefined -> Value;
+        Path when Value =:= undefined -> read_env_file(FileName, Path);
+        _ -> erlang:error({ambiguous_env, Name, FileName})
+    end.
+
+-spec non_blank_env(string()) -> string() | undefined.
+non_blank_env(Name) ->
     case os:getenv(Name) of
         false -> undefined;
         Value -> non_blank(Value)
     end.
 
+-spec read_env_file(string(), string()) -> string() | undefined.
+read_env_file(FileName, Path) ->
+    case file:read_file(Path) of
+        {ok, Contents} -> env_file_value(FileName, Path, strip_newline(Contents));
+        {error, Reason} -> erlang:error({unreadable_env_file, FileName, Path, Reason})
+    end.
+
+-spec env_file_value(string(), string(), binary()) -> string() | undefined.
+env_file_value(FileName, Path, Contents) ->
+    case unicode:characters_to_list(Contents) of
+        Value when is_list(Value) -> non_blank(Value);
+        _ -> erlang:error({invalid_env_file, FileName, Path})
+    end.
+
+-spec strip_newline(binary()) -> binary().
+strip_newline(Contents) ->
+    Size = byte_size(Contents),
+    case Contents of
+        <<Rest:(Size - 2)/binary, "\r\n">> -> Rest;
+        <<Rest:(Size - 1)/binary, "\n">> -> Rest;
+        _ -> Contents
+    end.
+
 -spec non_blank(string()) -> string() | undefined.
 non_blank(Value) ->
     case string:trim(Value) of

fluxer_gateway/test/fluxer_gateway_config_tests.erl

@@ -272,6 +272,64 @@ public_endpoints_defaults_test() ->
     ?assertEqual(undefined, maps:get(media_proxy_endpoint, Config)),
     ?assertEqual(<<"http://localhost:8088">>, maps:get(static_cdn_endpoint, Config)).
 
+env_value_reads_name_file_test() ->
+    with_env_file(<<"from-file\r\n">>, fun(Path) ->
+        with_envs(
+            [{"FLUXER_GATEWAY_TEST_SECRET", ""}, {"FLUXER_GATEWAY_TEST_SECRET_FILE", Path}],
+            fun() ->
+                ?assertEqual(
+                    "from-file", fluxer_gateway_config:env_value("FLUXER_GATEWAY_TEST_SECRET")
+                )
+            end
+        )
+    end).
+
+env_value_trims_only_one_newline_test() ->
+    with_env_file(<<"line1\nline2\n\n">>, fun(Path) ->
+        with_env("FLUXER_GATEWAY_TEST_SECRET_FILE", Path, fun() ->
+            ?assertEqual(
+                "line1\nline2\n", fluxer_gateway_config:env_value("FLUXER_GATEWAY_TEST_SECRET")
+            )
+        end)
+    end).
+
+env_value_rejects_name_and_name_file_test() ->
+    with_envs(
+        [
+            {"FLUXER_GATEWAY_TEST_SECRET", "direct"},
+            {"FLUXER_GATEWAY_TEST_SECRET_FILE", "/run/secrets/x"}
+        ],
+        fun() ->
+            ?assertError(
+                {ambiguous_env, "FLUXER_GATEWAY_TEST_SECRET",
+                    "FLUXER_GATEWAY_TEST_SECRET_FILE"},
+                fluxer_gateway_config:env_value("FLUXER_GATEWAY_TEST_SECRET")
+            )
+        end
+    ).
+
+env_value_rejects_missing_name_file_test() ->
+    Path = "/nonexistent/fluxer-gateway-test-secret",
+    with_env("FLUXER_GATEWAY_TEST_SECRET_FILE", Path, fun() ->
+        ?assertError(
+            {unreadable_env_file, "FLUXER_GATEWAY_TEST_SECRET_FILE", Path, enoent},
+            fluxer_gateway_config:env_value("FLUXER_GATEWAY_TEST_SECRET")
+        )
+    end).
+
+with_env_file(Contents, Fun) ->
+    Path = filename:join(
+        filename:basedir(user_cache, "fluxer_gateway_tests"),
+        integer_to_list(erlang:unique_integer([positive]))
+    ),
+    ok = filelib:ensure_dir(Path),
+    ok = file:write_file(Path, Contents),
+    try
+        Fun(Path)
+    after
+        file:delete(Path)
+    end.
+
 with_envs([], Fun) ->
     Fun();
 with_envs([{Name, Value} | Rest], Fun) ->

fluxer_media_proxy/src/cli.rs

@@ -48,7 +48,9 @@ pub enum StorageBackendArg {
 }
 
 pub fn load_config(args: &Args) -> anyhow::Result<Config> {
-    load_config_from_iter(args, std::env::vars())
+    let vars =
+        fluxer_common::config::resolve_env_files(std::env::vars()).map_err(anyhow::Error::msg)?;
+    load_config_from_iter(args, vars)
 }
 
 pub fn load_config_from_iter<I, K, V>(args: &Args, vars: I) -> anyhow::Result<Config>

fluxer_media_proxy/src/config/mod.rs

@@ -13,7 +13,7 @@ use parse::{
     parse_mode_env, parse_policy_mode, parse_storage_backend, parse_u16, parse_u64, parse_usize,
     validate_read_endpoint,
 };
-use std::{env, path::PathBuf};
+use std::path::PathBuf;
 
 #[derive(Clone, Copy, Debug, Eq, PartialEq)]
 pub enum StorageBackend {
@@ -144,10 +144,6 @@ pub struct Config {
 }
 
 impl Config {
-    pub fn load_from_env() -> anyhow::Result<Self> {
-        Self::load_from_iter(env::vars())
-    }
-
     pub fn load_from_iter<I, K, V>(vars: I) -> anyhow::Result<Self>
     where
         I: IntoIterator<Item = (K, V)>,

fluxer_push/src/cli.rs

@@ -27,7 +27,9 @@ pub enum Command {
 }
 
 pub fn load_config(args: &Args) -> anyhow::Result<Config> {
-    load_config_from_iter(args, std::env::vars())
+    let vars =
+        fluxer_svc::config::resolve_env_files(std::env::vars()).map_err(anyhow::Error::msg)?;
+    load_config_from_iter(args, vars)
 }
 
 fn load_config_from_iter<I, K, V>(args: &Args, vars: I) -> anyhow::Result<Config>

fluxer_svc/src/config.rs

@@ -1,6 +1,7 @@
 // SPDX-License-Identifier: AGPL-3.0-or-later
 
 use std::env;
+use std::fs;
 use std::net::{IpAddr, SocketAddr};
 use std::time::Duration;
 
@@ -52,7 +53,7 @@ pub enum DatabaseBackend {
 
 impl ServiceConfig {
     pub fn from_env() -> anyhow::Result<Self> {
-        Self::from_env_reader(|name| env::var(name).ok())
+        Self::from_env_reader(env_var)
     }
 
     fn from_env_reader<F>(get: F) -> anyhow::Result<Self>
@@ -203,7 +204,63 @@ fn default_max_concurrent_requests(service_name: &str) -> usize {
 }
 
 pub fn optional_env(name: &str) -> Option<String> {
-    optional_from(&|key| env::var(key).ok(), name)
+    env_var(name)
+}
+
+fn env_var(name: &str) -> Option<String> {
+    resolve_env_value(name, |key| env::var(key).ok()).unwrap_or_else(|error| panic!("{error}"))
+}
+
+fn resolve_env_value<F>(name: &str, get: F) -> Result<Option<String>, String>
+where
+    F: Fn(&str) -> Option<String>,
+{
+    let non_blank = |value: Option<String>| value.filter(|value| !value.trim().is_empty());
+    let value = non_blank(get(name));
+    let Some(path) = non_blank(get(&format!("{name}_FILE"))) else {
+        return Ok(value);
+    };
+    if value.is_some() {
+        return Err(format!("{name} and {name}_FILE are both set, set only one"));
+    }
+    let contents = fs::read_to_string(&path)
+        .map_err(|error| format!("{name}_FILE could not read {path} ({error})"))?;
+    let contents = contents
+        .strip_suffix('\n')
+        .map_or(contents.as_str(), |rest| {
+            rest.strip_suffix('\r').unwrap_or(rest)
+        });
+    Ok(non_blank(Some(contents.to_owned())))
+}
+
+pub fn resolve_env_files<I>(vars: I) -> Result<Vec<(String, String)>, String>
+where
+    I: IntoIterator<Item = (String, String)>,
+{
+    let vars: Vec<(String, String)> = vars.into_iter().collect();
+    let get = |key: &str| {
+        vars.iter()
+            .find_map(|(name, value)| (name == key).then(|| value.clone()))
+    };
+    let mut resolved = Vec::new();
+    for (key, _) in &vars {
+        let Some(name) = key
+            .strip_suffix("_FILE")
+            .filter(|name| name.starts_with("FLUXER_"))
+        else {
+            continue;
+        };
+        if let Some(value) = resolve_env_value(name, get)? {
+            resolved.push((name.to_owned(), value));
+        }
+    }
+    let rest: Vec<(String, String)> = vars
+        .iter()
+        .filter(|(key, _)| !resolved.iter().any(|(name, _)| name == key))
+        .cloned()
+        .collect();
+    resolved.extend(rest);
+    Ok(resolved)
 }
 
 fn optional_from<F>(get: &F, name: &str) -> Option<String>
@@ -276,6 +333,46 @@ mod tests {
         .unwrap()
     }
 
+    #[test]
+    fn resolves_name_file_entries() {
+        let dir = env::temp_dir().join(format!("fluxer-svc-env-file-{}", std::process::id()));
+        fs::create_dir_all(&dir).unwrap();
+        let path = dir.join("token");
+        fs::write(&path, "from-file\r\n").unwrap();
+        let path = path.to_string_lossy().into_owned();
+        let missing = dir.join("missing").to_string_lossy().into_owned();
+        let resolved = resolve_env_files(vec![
+            ("FLUXER_NATS_AUTH_TOKEN".to_owned(), String::new()),
+            ("FLUXER_NATS_AUTH_TOKEN_FILE".to_owned(), path.clone()),
+        ])
+        .unwrap();
+        let both = resolve_env_files(vec![
+            ("FLUXER_X".to_owned(), "direct".to_owned()),
+            ("FLUXER_X_FILE".to_owned(), path),
+        ]);
+        let unreadable = resolve_env_files(vec![("FLUXER_X_FILE".to_owned(), missing.clone())]);
+        let unrelated = resolve_env_files(vec![("SSL_CERT_FILE".to_owned(), missing.clone())]);
+        fs::remove_dir_all(&dir).unwrap();
+        assert_eq!(
+            vec!["from-file"],
+            resolved
+                .iter()
+                .filter(|(name, _)| name == "FLUXER_NATS_AUTH_TOKEN")
+                .map(|(_, value)| value.as_str())
+                .collect::<Vec<_>>()
+        );
+        assert_eq!(
+            Err("FLUXER_X and FLUXER_X_FILE are both set, set only one".to_owned()),
+            both
+        );
+        assert!(
+            unreadable
+                .unwrap_err()
+                .starts_with(&format!("FLUXER_X_FILE could not read {missing} ("))
+        );
+        assert_eq!(Ok(vec![("SSL_CERT_FILE".to_owned(), missing)]), unrelated);
+    }
+
     #[test]
     fn extracts_statefulset_ordinal() {
         assert_eq!(shard_id_from_pod_name("my-service-0"), Some(0));

fluxer_unfurl/src/resolvers/klipy.rs

@@ -4,7 +4,6 @@ use super::{ResolveContext, Resolver, ResolverResult};
 use crate::http_fetch;
 use crate::media_proxy::{MediaMetadata, embed_media_flags};
 use crate::types::{EmbedMedia, EmbedProvider, MessageEmbed};
-use fluxer_svc::config::optional_env;
 use std::future::Future;
 use std::pin::Pin;
 use std::time::Duration;
@@ -49,7 +48,7 @@ impl Resolver for KlipyResolver {
         ctx: &'a ResolveContext<'_>,
     ) -> Pin<Box<dyn Future<Output = anyhow::Result<ResolverResult>> + Send + 'a>> {
         Box::pin(async move {
-            let Some(api_key) = ctx.klipy_api_key.clone().or_else(klipy_api_key) else {
+            let Some(api_key) = ctx.klipy_api_key.clone() else {
                 return Ok(ResolverResult { embeds: vec![] });
             };
             let formats = match resolve_media_via_api(ctx, &api_key).await {
@@ -136,10 +135,6 @@ fn klipy_resource(kind: &str) -> &'static str {
     }
 }
 
-fn klipy_api_key() -> Option<String> {
-    optional_env("FLUXER_KLIPY_API_KEY").or_else(|| optional_env("KLIPY_API_KEY"))
-}
-
 async fn resolve_media_via_api(
     ctx: &ResolveContext<'_>,
     api_key: &str,

fluxer_unfurl/src/resolvers/youtube.rs

@@ -5,7 +5,6 @@ use crate::http_fetch;
 use crate::media_proxy::embed_media_flags;
 use crate::text_limits;
 use crate::types::{EmbedAuthor, EmbedMedia, EmbedProvider, MessageEmbed};
-use fluxer_svc::config::optional_env;
 use serde::Deserialize;
 use std::future::Future;
 use std::pin::Pin;
@@ -61,7 +60,7 @@ impl Resolver for YouTubeResolver {
                 }
             };
 
-            let Some(api_key) = ctx.youtube_api_key.clone().or_else(youtube_api_key) else {
+            let Some(api_key) = ctx.youtube_api_key.clone() else {
                 tracing::debug!("No YouTube API key configured");
                 return Ok(ResolverResult { embeds: vec![] });
             };
@@ -251,10 +250,6 @@ struct YouTubeThumbnail {
     height: Option<u32>,
 }
 
-fn youtube_api_key() -> Option<String> {
-    optional_env("FLUXER_YOUTUBE_API_KEY").or_else(|| optional_env("YOUTUBE_API_KEY"))
-}
-
 fn build_youtube_api_url(video_id: &str, api_key: &str) -> anyhow::Result<Url> {
     let mut url = Url::parse(YOUTUBE_API_BASE)?;
     url.query_pairs_mut()

fluxer_unfurl/src/shard_impl.rs

@@ -24,6 +24,8 @@ pub struct UnfurlShard {
     resolvers: Vec<Box<dyn crate::resolvers::Resolver>>,
     media_proxy: MediaProxyClient,
     self_hosted: bool,
+    youtube_api_key: Option<String>,
+    klipy_api_key: Option<String>,
 }
 
 impl UnfurlShard {
@@ -78,6 +80,10 @@ impl UnfurlShard {
             resolvers,
             media_proxy,
             self_hosted,
+            youtube_api_key: optional_env("FLUXER_YOUTUBE_API_KEY")
+                .or_else(|| optional_env("YOUTUBE_API_KEY")),
+            klipy_api_key: optional_env("FLUXER_KLIPY_API_KEY")
+                .or_else(|| optional_env("KLIPY_API_KEY")),
         }
     }
 
@@ -100,6 +106,8 @@ impl UnfurlShard {
                 internal_http_client(),
             ),
             self_hosted: false,
+            youtube_api_key: None,
+            klipy_api_key: None,
         }
     }
 
@@ -125,8 +133,12 @@ impl UnfurlShard {
             nsfw_mode,
             media_proxy: &self.media_proxy,
             self_hosted: self.self_hosted,
-            youtube_api_key: youtube_api_key.map(str::to_owned),
+            youtube_api_key: youtube_api_key
-            klipy_api_key: klipy_api_key.map(str::to_owned),
+                .map(str::to_owned)
+                .or_else(|| self.youtube_api_key.clone()),
+            klipy_api_key: klipy_api_key
+                .map(str::to_owned)
+                .or_else(|| self.klipy_api_key.clone()),
         };
 
         if let Some(idx) = matched_resolver_idx {

fluxer_users/src/pseudonym.rs

@@ -1,5 +1,6 @@
 // SPDX-License-Identifier: AGPL-3.0-or-later
 
+use fluxer_svc::config::optional_env;
 use hmac::{KeyInit, Mac};
 use std::sync::{LazyLock, OnceLock};
 
@@ -46,8 +47,8 @@ pub fn configure(secret: Option<String>, environment: Option<&str>) -> anyhow::R
 
 pub fn configure_from_env() -> anyhow::Result<()> {
     configure(
-        std::env::var(SECRET_ENV).ok(),
+        optional_env(SECRET_ENV),
-        std::env::var("FLUXER_ENV").ok().as_deref(),
+        optional_env("FLUXER_ENV").as_deref(),
     )
 }
 

packages/config/src/__tests__/ConfigLoader.test.ts

@@ -1,6 +1,9 @@
 // SPDX-License-Identifier: AGPL-3.0-or-later
 
 import {generateKeyPairSync} from 'node:crypto';
+import {mkdtempSync, rmSync, writeFileSync} from 'node:fs';
+import {tmpdir} from 'node:os';
+import {join} from 'node:path';
 import {getConfig, loadConfig, resetConfig} from '@fluxer/config/src/ConfigLoader';
 import {afterEach, beforeEach, describe, expect, test, vi} from 'vitest';
 
@@ -80,6 +83,24 @@ describe('ConfigLoader', () => {
 		expect((await loadConfig()).domain.base_domain).toBe('localhost');
 	});
 
+	test('loadConfig reads secrets from NAME_FILE', async () => {
+		const dir = mkdtempSync(join(tmpdir(), 'fluxer-config-file-'));
+		try {
+			const path = join(dir, 'postgres_password');
+			writeFileSync(path, 'from-secret-file\n');
+			stubMinimalEnv({FLUXER_POSTGRES_PASSWORD: '', FLUXER_POSTGRES_PASSWORD_FILE: path});
+			const config = await loadConfig();
+			expect(config.database.postgres.password).toBe('from-secret-file');
+		} finally {
+			rmSync(dir, {recursive: true, force: true});
+		}
+	});
+
+	test('loadConfig rejects NAME and NAME_FILE together', async () => {
+		stubMinimalEnv({FLUXER_SUDO_MODE_SECRET_FILE: '/run/secrets/sudo'});
+		await expect(loadConfig()).rejects.toThrow('FLUXER_SUDO_MODE_SECRET and FLUXER_SUDO_MODE_SECRET_FILE are both set');
+	});
+
 	test('getConfig throws when config is not loaded', () => {
 		expect(() => getConfig()).toThrow('Config not loaded');
 	});

packages/config/src/__tests__/EnvironmentOverrides.test.ts

@@ -1,7 +1,14 @@
 // SPDX-License-Identifier: AGPL-3.0-or-later
 
-import {buildNamedFluxerEnvOverrides, setNestedValue} from '@fluxer/config/src/config_loader/EnvironmentOverrides';
+import {mkdtempSync, rmSync, writeFileSync} from 'node:fs';
-import {describe, expect, test} from 'vitest';
+import {tmpdir} from 'node:os';
+import {join} from 'node:path';
+import {
+	buildNamedFluxerEnvOverrides,
+	readEnvValue,
+	setNestedValue,
+} from '@fluxer/config/src/config_loader/EnvironmentOverrides';
+import {afterAll, describe, expect, test} from 'vitest';
 
 describe('setNestedValue', () => {
 	test('sets a top-level key', () => {
@@ -316,3 +323,69 @@ describe('buildNamedFluxerEnvOverrides', () => {
 		);
 	});
 });
+
+describe('readEnvValue with NAME_FILE', () => {
+	const dir = mkdtempSync(join(tmpdir(), 'fluxer-env-file-'));
+	afterAll(() => rmSync(dir, {recursive: true, force: true}));
+
+	function secretFile(name: string, contents: string): string {
+		const path = join(dir, name);
+		writeFileSync(path, contents);
+		return path;
+	}
+
+	test('reads the value from NAME_FILE when NAME is unset', () => {
+		const path = secretFile('plain', 'from-file\n');
+		expect(readEnvValue({FLUXER_SUDO_MODE_SECRET_FILE: path}, 'FLUXER_SUDO_MODE_SECRET')).toBe('from-file');
+	});
+
+	test('reads the value from NAME_FILE when NAME is blank', () => {
+		const path = secretFile('blank', 'from-file');
+		expect(
+			readEnvValue({FLUXER_SUDO_MODE_SECRET: ' ', FLUXER_SUDO_MODE_SECRET_FILE: path}, 'FLUXER_SUDO_MODE_SECRET'),
+		).toBe('from-file');
+	});
+
+	test('trims only one trailing newline', () => {
+		const crlf = secretFile('crlf', 'value\r\n');
+		const multi = secretFile('multi', '-----BEGIN-----\nabc\n-----END-----\n\n');
+		expect(readEnvValue({X_FILE: crlf}, 'X')).toBe('value');
+		expect(readEnvValue({X_FILE: multi}, 'X')).toBe('-----BEGIN-----\nabc\n-----END-----\n');
+	});
+
+	test('treats an empty file as unset', () => {
+		const path = secretFile('empty', '\n');
+		expect(readEnvValue({X_FILE: path}, 'X')).toBeUndefined();
+	});
+
+	test('keeps NAME when NAME_FILE is blank', () => {
+		expect(readEnvValue({X: 'direct', X_FILE: ''}, 'X')).toBe('direct');
+	});
+
+	test('rejects NAME and NAME_FILE together', () => {
+		const path = secretFile('both', 'from-file');
+		expect(() => readEnvValue({X: 'direct', X_FILE: path}, 'X')).toThrow('X and X_FILE are both set, set only one');
+	});
+
+	test('names NAME_FILE and the path when the file is missing', () => {
+		const path = join(dir, 'missing');
+		expect(() => readEnvValue({X_FILE: path}, 'X')).toThrow(`X_FILE could not read ${path} (ENOENT)`);
+	});
+
+	test('rejects a file that is not valid UTF-8', () => {
+		const path = join(dir, 'binary');
+		writeFileSync(path, Buffer.from([0xff, 0x61]));
+		expect(() => readEnvValue({X_FILE: path}, 'X')).toThrow(`X_FILE could not read ${path} (`);
+	});
+
+	test('feeds named overrides and aliases', () => {
+		const secret = secretFile('stripe', 'sk_test_file\n');
+		const nats = secretFile('nats', 'nats://nats:4222\n');
+		expect(
+			buildNamedFluxerEnvOverrides({FLUXER_STRIPE_SECRET_KEY_FILE: secret, FLUXER_NATS_CORE_URL_FILE: nats}),
+		).toMatchObject({
+			integrations: {stripe: {secret_key: 'sk_test_file'}},
+			services: {nats: {core_url: 'nats://nats:4222'}},
+		});
+	});
+});

packages/config/src/config_loader/EnvironmentOverrides.ts

@@ -1,5 +1,6 @@
 // SPDX-License-Identifier: AGPL-3.0-or-later
 
+import {readFileSync} from 'node:fs';
 import {type ConfigObject, isConfigObject} from '@fluxer/config/src/config_loader/ConfigObject';
 
 type ConfigPathKey = string | number;
@@ -420,11 +421,29 @@ const NAMED_FLUXER_ENV_ALIASES: Record<string, string | undefined> = {
 
 export const NAMED_FLUXER_ENV_NAMES = Object.keys(NAMED_FLUXER_ENV_OVERRIDES);
 
-export function readEnvValue(env: NodeJS.ProcessEnv, name: string): string | undefined {
+function nonBlank(value: string | undefined): string | undefined {
-	const value = env[name];
 	return value === undefined || value.trim().length === 0 ? undefined : value;
 }
 
+export function readEnvValue(env: NodeJS.ProcessEnv, name: string): string | undefined {
+	const value = nonBlank(env[name]);
+	const filePath = nonBlank(env[`${name}_FILE`]);
+	if (filePath === undefined) {
+		return value;
+	}
+	if (value !== undefined) {
+		throw new Error(`${name} and ${name}_FILE are both set, set only one`);
+	}
+	let contents: string;
+	try {
+		contents = new TextDecoder('utf-8', {fatal: true}).decode(readFileSync(filePath));
+	} catch (error) {
+		const reason = error instanceof Error && 'code' in error ? String(error.code) : 'unreadable';
+		throw new Error(`${name}_FILE could not read ${filePath} (${reason})`);
+	}
+	return nonBlank(contents.replace(/\r?\n$/, ''));
+}
+
 export function buildNamedFluxerEnvOverrides(env: NodeJS.ProcessEnv): ConfigObject {
 	const overrides: ConfigObject = {};
 	for (const [envKey, mapping] of Object.entries(NAMED_FLUXER_ENV_OVERRIDES)) {