fix(installer): pull images before the first start (#3248)

Co-authored-by: Hampus <[email protected]>

.github/SECURITY.md

@@ -1,6 +1,6 @@
 # Security policy
 
-Do not report a vulnerability in a pull request or on feedback.fluxer.com.
+Do not report a vulnerability in a pull request, on feedback.fluxer.com, in a Fluxer community, or in a direct message to staff.
 
 Submit a report through <https://fluxer.app/security> or email <security@fluxer.com>. Include the affected component, impact, reproduction steps, and supporting evidence. Remove unrelated personal data and secrets.
 

deploy/self-hosting/.env.example

@@ -263,7 +263,7 @@ FLUXER_VAPID_PRIVATE_KEY=CHANGE_ME
 # only when a browser must reach an origin the defaults do not cover. Separate
 # several with spaces or commas. The three values below are illustrations.
 #FLUXER_CSP_EXTRA_DEFAULT_SRC=
-#FLUXER_CSP_EXTRA_CONNECT_SRC=wss://livekit.example.com:7881
+#FLUXER_CSP_EXTRA_CONNECT_SRC=wss://livekit.example.com
 #FLUXER_CSP_EXTRA_IMG_SRC=https://cdn.example.com
 #FLUXER_CSP_EXTRA_MEDIA_SRC=
 #FLUXER_CSP_EXTRA_FONT_SRC=

fluxer_api/src/api/guild/tests/GuildTemplateImport.test.ts

@@ -6,7 +6,7 @@ import {type ApiTestHarness, createApiTestHarness} from '@app/api/test/ApiTestHa
 import {createBuilder} from '@app/api/test/TestRequestBuilder';
 import {ChannelTypes, Permissions} from '@fluxer/constants/src/ChannelConstants';
 import {SystemChannelFlags} from '@fluxer/constants/src/GuildConstants';
-import {VOICE_CHANNEL_USER_LIMIT_MAX} from '@fluxer/constants/src/LimitConstants';
+import {CHANNEL_TOPIC_MAX_LENGTH, VOICE_CHANNEL_USER_LIMIT_MAX} from '@fluxer/constants/src/LimitConstants';
 import type {GuildResponse} from '@fluxer/schema/src/domains/guild/GuildResponseSchemas';
 import {afterAll, beforeAll, beforeEach, describe, expect, test} from 'vitest';
 
@@ -215,7 +215,7 @@ describe('Guild Template Import', () => {
 		['a slowmode above the channel maximum', {rate_limit_per_user: 1_000_000_000}],
 		['a fractional position', {position: 0.5}],
 		['a negative position', {position: -3}],
-		['a topic above the channel maximum', {topic: 'x'.repeat(1025)}],
+		['a topic above the template maximum', {topic: 'x'.repeat(4097)}],
 		['a name above the channel maximum', {name: 'x'.repeat(101)}],
 		['a negative user limit', {type: ChannelTypes.GUILD_VOICE, user_limit: -1}],
 		['a voice connection limit above the maximum', {type: ChannelTypes.GUILD_VOICE, voice_connection_limit: 100_000}],
@@ -273,6 +273,26 @@ describe('Guild Template Import', () => {
 		expect(channels.find((channel) => channel.name === 'town-hall')?.user_limit).toBe(VOICE_CHANNEL_USER_LIMIT_MAX);
 		expect(channels.find((channel) => channel.name === 'lounge')?.voice_connection_limit).toBe(100);
 	});
+	test('accepts forum-length topics and shortens them to the channel topic limit', async () => {
+		const account = await createTestAccount(harness);
+		const longTopic = `${'a'.repeat(CHANNEL_TOPIC_MAX_LENGTH - 1)}\u{1F600}${'b'.repeat(300)}`;
+		const guild = await createBuilder<GuildResponse>(harness, account.token)
+			.post('/guilds')
+			.body({
+				name: 'Forum Guild',
+				template: buildMinimalTemplate({
+					channels: [
+						{id: 6001, type: ChannelTypes.GUILD_TEXT, name: 'general', position: 0, topic: longTopic},
+						{id: 6002, type: 15, name: 'projects', position: 1, topic: 'c'.repeat(1356)},
+					],
+				}),
+			})
+			.execute();
+		const channels = await getGuildChannels(harness, account.token, guild.id);
+		const general = channels.find((channel) => channel.name === 'general');
+		expect(general?.topic).toBe('a'.repeat(CHANNEL_TOPIC_MAX_LENGTH - 1));
+		expect(channels.find((channel) => channel.name === 'projects')).toBeUndefined();
+	});
 });
 
 const DEFAULT_EVERYONE_PERMISSIONS = Permissions.VIEW_CHANNEL.toString();

fluxer_api/src/api/openapi/openapi.json

@@ -33075,8 +33075,8 @@
 						"anyOf": [{"type": "string", "maxLength": 100}, {"type": "null"}]
 					},
 					"topic": {
-						"description": "The channel topic",
-						"anyOf": [{"type": "string", "maxLength": 1024}, {"type": "null"}]
+						"description": "The channel topic, shortened to the Fluxer channel topic limit",
+						"anyOf": [{"type": "string", "maxLength": 4096}, {"type": "null"}]
 					},
 					"position": {"description": "The position of the channel", "$ref": "#/components/schemas/Int32Type"},
 					"parent_id": {

fluxer_common/src/config.rs

@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: AGPL-3.0-or-later
 
-use std::{env, path::Path};
+use std::{env, fs, path::Path};
 
 #[derive(Clone, Debug, Eq, PartialEq)]
 pub enum GeoipSourceConfig {
@@ -124,7 +124,59 @@ fn percent_decode(value: &str) -> String {
 }
 
 pub fn env_value(name: &str) -> Option<String> {
-    env::var(name).ok().filter(|value| !value.trim().is_empty())
+    resolve_env_value(name, |key| env::var(key).ok()).unwrap_or_else(|error| panic!("{error}"))
+}
+
+pub fn resolve_env_value<F>(name: &str, get: F) -> Result<Option<String>, String>
+where
+    F: Fn(&str) -> Option<String>,
+{
+    let non_blank = |value: Option<String>| value.filter(|value| !value.trim().is_empty());
+    let value = non_blank(get(name));
+    let Some(path) = non_blank(get(&format!("{name}_FILE"))) else {
+        return Ok(value);
+    };
+    if value.is_some() {
+        return Err(format!("{name} and {name}_FILE are both set, set only one"));
+    }
+    let contents = fs::read_to_string(&path)
+        .map_err(|error| format!("{name}_FILE could not read {path} ({error})"))?;
+    let contents = contents
+        .strip_suffix('\n')
+        .map_or(contents.as_str(), |rest| {
+            rest.strip_suffix('\r').unwrap_or(rest)
+        });
+    Ok(non_blank(Some(contents.to_owned())))
+}
+
+pub fn resolve_env_files<I>(vars: I) -> Result<Vec<(String, String)>, String>
+where
+    I: IntoIterator<Item = (String, String)>,
+{
+    let vars: Vec<(String, String)> = vars.into_iter().collect();
+    let get = |key: &str| {
+        vars.iter()
+            .find_map(|(name, value)| (name == key).then(|| value.clone()))
+    };
+    let mut resolved = Vec::new();
+    for (key, _) in &vars {
+        let Some(name) = key
+            .strip_suffix("_FILE")
+            .filter(|name| name.starts_with("FLUXER_"))
+        else {
+            continue;
+        };
+        if let Some(value) = resolve_env_value(name, get)? {
+            resolved.push((name.to_owned(), value));
+        }
+    }
+    let rest: Vec<(String, String)> = vars
+        .iter()
+        .filter(|(key, _)| !resolved.iter().any(|(name, _)| name == key))
+        .cloned()
+        .collect();
+    resolved.extend(rest);
+    Ok(resolved)
 }
 
 pub fn read_env(name: &str, fallback: &str) -> String {
@@ -763,6 +815,106 @@ mod tests {
         assert_eq!(None, port);
     }
 
+    fn secret_file(dir: &tempfile::TempDir, name: &str, contents: &str) -> String {
+        let path = dir.path().join(name);
+        fs::write(&path, contents).expect("write secret file");
+        path.to_string_lossy().into_owned()
+    }
+
+    fn pairs_reader(pairs: Vec<(String, String)>) -> impl Fn(&str) -> Option<String> {
+        move |key| {
+            pairs
+                .iter()
+                .find_map(|(name, value)| (name == key).then(|| value.clone()))
+        }
+    }
+
+    #[test]
+    fn env_value_reads_name_file_when_name_is_blank() {
+        let dir = tempfile::tempdir().expect("tempdir");
+        let path = secret_file(&dir, "secret", "from-file\r\n");
+        let unset = pairs_reader(vec![("X_FILE".to_owned(), path.clone())]);
+        let blank = pairs_reader(vec![
+            ("X".to_owned(), " ".to_owned()),
+            ("X_FILE".to_owned(), path),
+        ]);
+        assert_eq!(
+            Ok(Some("from-file".to_owned())),
+            resolve_env_value("X", unset)
+        );
+        assert_eq!(
+            Ok(Some("from-file".to_owned())),
+            resolve_env_value("X", blank)
+        );
+    }
+
+    #[test]
+    fn env_value_trims_only_one_trailing_newline() {
+        let dir = tempfile::tempdir().expect("tempdir");
+        let pem = secret_file(&dir, "pem", "-----BEGIN-----\nabc\n-----END-----\n\n");
+        let empty = secret_file(&dir, "empty", "\n");
+        assert_eq!(
+            Ok(Some("-----BEGIN-----\nabc\n-----END-----\n".to_owned())),
+            resolve_env_value("X", pairs_reader(vec![("X_FILE".to_owned(), pem)]))
+        );
+        assert_eq!(
+            Ok(None),
+            resolve_env_value("X", pairs_reader(vec![("X_FILE".to_owned(), empty)]))
+        );
+    }
+
+    #[test]
+    fn env_value_rejects_name_and_name_file_together() {
+        let reader = pairs_reader(vec![
+            ("X".to_owned(), "direct".to_owned()),
+            ("X_FILE".to_owned(), "/run/secrets/x".to_owned()),
+        ]);
+        assert_eq!(
+            Err("X and X_FILE are both set, set only one".to_owned()),
+            resolve_env_value("X", reader)
+        );
+    }
+
+    #[test]
+    fn env_value_names_the_file_when_it_is_missing() {
+        let dir = tempfile::tempdir().expect("tempdir");
+        let path = dir.path().join("missing").to_string_lossy().into_owned();
+        let error = resolve_env_value("X", pairs_reader(vec![("X_FILE".to_owned(), path.clone())]))
+            .expect_err("missing file fails");
+        assert!(error.starts_with(&format!("X_FILE could not read {path} (")));
+    }
+
+    #[test]
+    fn resolve_env_files_replaces_fluxer_names_with_file_values() {
+        let dir = tempfile::tempdir().expect("tempdir");
+        let path = secret_file(&dir, "secret", "from-file\n");
+        let missing = dir.path().join("missing").to_string_lossy().into_owned();
+        let resolved = resolve_env_files(vec![
+            ("FLUXER_X".to_owned(), String::new()),
+            ("FLUXER_X_FILE".to_owned(), path),
+            ("FLUXER_Y".to_owned(), "plain".to_owned()),
+            ("FLUXER_Y_FILE".to_owned(), String::new()),
+            ("SSL_CERT_FILE".to_owned(), missing),
+        ])
+        .expect("resolves");
+        let values = |key: &str| {
+            resolved
+                .iter()
+                .filter(|(name, _)| name == key)
+                .map(|(_, value)| value.as_str())
+                .collect::<Vec<_>>()
+        };
+        assert_eq!(vec!["from-file"], values("FLUXER_X"));
+        assert_eq!(vec!["plain"], values("FLUXER_Y"));
+        assert!(
+            resolve_env_files(vec![
+                ("FLUXER_X".to_owned(), "direct".to_owned()),
+                ("FLUXER_X_FILE".to_owned(), "/run/secrets/x".to_owned()),
+            ])
+            .is_err()
+        );
+    }
+
     #[test]
     fn a_blank_value_reads_as_unset() {
         unsafe {

fluxer_docs/src/content/docs/operator/configuration.mdx

@@ -36,6 +36,34 @@ Precedence, highest first:
 
 Use `true` or `false` for booleans, decimal integers for integer settings, and the specified object or array for JSON settings. Defaults and accepted values are listed below. Every service built from this version of the stack files or later reads an empty or blank value the same as an unset one, so an empty value restores the default. Older images do not, which [Match the images to the stack files](/operator/upgrading/#match-the-images-to-the-stack-files) covers.
 
+### Reading a secret from a file
+
+Any setting a Fluxer service reads for its configuration can come from a file instead, such as a Docker secret. Set the variable's name with `_FILE` appended to the file's path, and leave the variable itself empty. The service reads the file at startup and drops one trailing newline. Startup fails, naming both variables, when the variable and its `_FILE` form are both set. It also fails, naming the path, when the file is missing, unreadable or not valid UTF-8. The name to use is the one the service sees, so `POSTGRES_PASSWORD` in `.env` becomes `FLUXER_POSTGRES_PASSWORD_FILE`.
+
+Add the secrets through a local Compose override. `FLUXER_SUDO_MODE_SECRET` is read by `api` and `worker` only:
+
+```yaml
+secrets:
+  sudo_mode_secret:
+    file: ./secrets/sudo_mode_secret
+
+services:
+  api:
+    secrets: [sudo_mode_secret]
+    environment:
+      FLUXER_SUDO_MODE_SECRET: ''
+      FLUXER_SUDO_MODE_SECRET_FILE: /run/secrets/sudo_mode_secret
+  worker:
+    secrets: [sudo_mode_secret]
+    environment:
+      FLUXER_SUDO_MODE_SECRET: ''
+      FLUXER_SUDO_MODE_SECRET_FILE: /run/secrets/sudo_mode_secret
+```
+
+Compose still checks the required names in `.env` before it applies the override, so keep a placeholder value there. Every Fluxer service gets the same shared settings, so a secret several services read, such as `FLUXER_POSTGRES_PASSWORD`, needs the same two entries on each of them: `api`, `worker`, `gateway`, `media-proxy`, `push`, `admin`, and `snowflakes`, `users`, `gifs`, `messages` and `unfurl` with their `-shard` services. Any service left out keeps the placeholder. The secret file must be readable by the user the container runs as.
+
+The bundled backing containers read their own settings. The `postgres` container takes `POSTGRES_PASSWORD: ''` and `POSTGRES_PASSWORD_FILE` in its own `environment`, and on a fresh volume it creates the database with that password. `seaweedfs-init`, `meilisearch` and `livekit` have no `_FILE` form, so a file-based S3, Meilisearch or LiveKit secret still needs its real value in `.env` for them. `FLUXER_ENV`, `LOG_LEVEL`, `FLUXER_DISABLE_RATE_LIMITS` and the `FLUXER_ERLANG_*` settings other than `FLUXER_ERLANG_COOKIE` are read directly, so set them as plain values.
+
 ## Core identity and public address
 
 `FLUXER_DOMAIN` is required. Everything else here is optional.
@@ -2137,13 +2165,15 @@ CORS origins are exactly the app endpoint, the `FLUXER_APP_ORIGIN_ALIASES` origi
 | --- | --- | --- |
 | postgres-data | Every account, message, and configuration row | Yes |
 | seaweedfs-data | Every uploaded file | Yes |
-| valkey-data | Deletion queues and shared cache | Yes |
+| valkey-data | Pending file deletions, other queues and shared cache | Yes |
 | nats-data | Pending and failed background jobs | Yes |
 | edge-data | Issued TLS certificates | Optional, a loss only costs a re-issue |
 | edge-config | The edge's own state | No |
-| meilisearch-data | The search index, rebuildable | No |
+| meilisearch-data | The search index, rebuildable from Postgres | Optional |
 
-Losing `valkey-data` can delay scheduled account and bulk-message deletions while their queues are rebuilt. Pending asset deletions and cache purges can be lost, so do not treat this volume as disposable cache.
+Losing `valkey-data` can delay scheduled account and bulk-message deletions while their queues are rebuilt. Pending asset deletions and cache purges exist only here and are lost with it, so do not treat this volume as disposable cache.
+
+Losing `meilisearch-data` leaves search empty until it is rebuilt. Nothing rebuilds it automatically, because Postgres still records every channel as indexed. Rebuild each index with [`POST /admin/search/indexes/{index_name}/refreshes`](/admin-api/search-indexes/). The `channel_messages` and `guild_members` indexes rebuild one community at a time and need a `guild_id` for each. A backup of the volume avoids those calls on a large instance.
 
 `nats-data` retains pending jobs in `JOBS` for up to 7 days and failed jobs in `JOBS_DLQ` for up to 30 days. A full jobs stream rejects new work. A full dead-letter stream drops its oldest entries, so investigate failures promptly. If dead-letter storage is unavailable, failed jobs remain in `JOBS` only until they expire. Losing this volume loses queued work, which is not automatically recovered from the database.
 

fluxer_docs/src/content/docs/operator/reverse-proxy.mdx

@@ -522,10 +522,13 @@ LiveKit signalling goes through `/livekit/*` like everything else. WebRTC media
 
 Both ports are published directly by the stack and must reach the host. A proxy or tunnel in front of `443` does nothing for them.
 
-Hosting LiveKit on a hostname other than `FLUXER_DOMAIN` means widening the Content-Security-Policy the web app runs under. The line goes in `.env`, beside `FLUXER_DOMAIN`:
+Signalling can use a hostname other than `FLUXER_DOMAIN`. Two lines go in `.env`, beside `FLUXER_DOMAIN`. The first points clients at the new hostname, and the second lets the web app's Content-Security-Policy connect to it:
 
 ```ini
-FLUXER_CSP_EXTRA_CONNECT_SRC=wss://livekit.example.com:7881
+FLUXER_LIVEKIT_URL=wss://livekit.example.com/livekit
+FLUXER_CSP_EXTRA_CONNECT_SRC=wss://livekit.example.com
 ```
 
-`app-proxy` builds the policy and reads its environment at container start, so apply the change with `docker compose up -d app-proxy`. `docker compose restart app-proxy` reuses the existing container with its old environment. [Content Security Policy](/operator/configuration/#content-security-policy) has the other variables.
+The policy entry is the origin alone, with no port or path. The hostname needs a valid TLS certificate and must reach the `/livekit` route on the edge with WebSocket upgrades. The media ports above never belong in the policy, because WebRTC media is not governed by `connect-src`. A separate hostname is rarely needed, since media goes to the address LiveKit advertises, not to a hostname.
+
+`api` writes `FLUXER_LIVEKIT_URL` into the default voice server at start, and `app-proxy` builds the policy at container start, so apply the change with `docker compose up -d api app-proxy`. `docker compose restart` reuses the existing containers with their old environment. [Content Security Policy](/operator/configuration/#content-security-policy) has the other variables.

fluxer_docs/src/installer/install.ps1

@@ -2267,10 +2267,14 @@ function Invoke-FluxerInstall {
 			exit 0
 		}
 
+		Write-FluxerLine 'Pulling images. The first start pulls eighteen of them, which takes several minutes.'
+		if ((Invoke-FluxerDocker @('compose', 'pull')) -ne 0) {
+			Stop-Fluxer 'docker compose pull failed. Nothing was started.' $FluxerExitDownload
+		}
 		if ((Invoke-FluxerDocker @('compose', 'up', '-d')) -ne 0) {
 			Stop-Fluxer 'docker compose up -d failed.' $FluxerExitUnhealthy
 		}
-		Wait-FluxerStack 'Waiting for the stack to report healthy. The first start pulls images and takes several minutes.'
+		Wait-FluxerStack 'Waiting for the stack to report healthy.'
 		$readyOrigin = Get-FluxerPublicOrigin $envPath
 		if ($readyOrigin.Length -eq 0) {
 			$readyOrigin = "https://$domainValue"

fluxer_docs/src/installer/install.sh

@@ -2245,11 +2245,15 @@ if [ "$opt_no_start" -eq 1 ]; then
 	exit 0
 fi
 
+fluxer_say 'Pulling images. The first start pulls eighteen of them, which takes several minutes.'
+if ! $fluxer_engine compose pull; then
+	fluxer_fail 4 "$fluxer_engine compose pull failed in $opt_dir. Nothing was started."
+fi
 fluxer_say 'Starting the stack.'
 if ! $fluxer_engine compose up -d; then
 	fluxer_fail 6 "$fluxer_engine compose up -d failed in $opt_dir. Read $fluxer_engine compose logs there."
 fi
-fluxer_say 'Waiting for every service to report ready. This takes several minutes on the first start, which pulls eighteen images.'
+fluxer_say 'Waiting for every service to report ready.'
 if ! fluxer_wait_ready; then
 	fluxer_fail 6 "The stack is not ready after $FLUXER_READY_TIMEOUT seconds. $(fluxer_not_ready_detail)
 Read $fluxer_engine compose logs in $opt_dir."

fluxer_gateway/scripts/docker_entrypoint.sh

@@ -70,6 +70,21 @@ else
 	rm -f "$node_name_file"
 fi
 
+case "${FLUXER_ERLANG_COOKIE_FILE:-}" in
+*[![:space:]]*)
+	case "${FLUXER_ERLANG_COOKIE:-}" in
+	*[![:space:]]*)
+		echo 'FLUXER_ERLANG_COOKIE and FLUXER_ERLANG_COOKIE_FILE are both set, set only one.' >&2
+		exit 1
+		;;
+	esac
+	if ! FLUXER_ERLANG_COOKIE="$(cat -- "$FLUXER_ERLANG_COOKIE_FILE")"; then
+		echo "FLUXER_ERLANG_COOKIE_FILE could not read $FLUXER_ERLANG_COOKIE_FILE." >&2
+		exit 1
+	fi
+	;;
+esac
+
 if [ -z "${FLUXER_ERLANG_COOKIE:-}" ]; then
 	echo 'FLUXER_ERLANG_COOKIE is required.' >&2
 	exit 1

fluxer_gateway/scripts/fluxer_gateway_release_wrapper.sh

@@ -18,4 +18,8 @@ if [ -r "$node_name_file" ]; then
 	FLUXER_ERLANG_NODE_NAME="$(cat "$node_name_file")"
 	export FLUXER_ERLANG_NODE_NAME
 fi
+if [ -r "${FLUXER_ERLANG_COOKIE_FILE:-}" ] && [ -z "$(printf '%s' "${FLUXER_ERLANG_COOKIE:-}" | tr -d '[:space:]')" ]; then
+	FLUXER_ERLANG_COOKIE="$(cat "$FLUXER_ERLANG_COOKIE_FILE")"
+	export FLUXER_ERLANG_COOKIE
+fi
 exec "$script_dir/fluxer_gateway.real" "$@"

fluxer_gateway/src/gateway/fluxer_gateway_config.erl

@@ -305,11 +305,44 @@ env_string(Name, Default) ->
 
 -spec env_value(string()) -> string() | undefined.
 env_value(Name) ->
+    Value = non_blank_env(Name),
+    FileName = Name ++ "_FILE",
+    case non_blank_env(FileName) of
+        undefined -> Value;
+        Path when Value =:= undefined -> read_env_file(FileName, Path);
+        _ -> erlang:error({ambiguous_env, Name, FileName})
+    end.
+
+-spec non_blank_env(string()) -> string() | undefined.
+non_blank_env(Name) ->
     case os:getenv(Name) of
         false -> undefined;
         Value -> non_blank(Value)
     end.
 
+-spec read_env_file(string(), string()) -> string() | undefined.
+read_env_file(FileName, Path) ->
+    case file:read_file(Path) of
+        {ok, Contents} -> env_file_value(FileName, Path, strip_newline(Contents));
+        {error, Reason} -> erlang:error({unreadable_env_file, FileName, Path, Reason})
+    end.
+
+-spec env_file_value(string(), string(), binary()) -> string() | undefined.
+env_file_value(FileName, Path, Contents) ->
+    case unicode:characters_to_list(Contents) of
+        Value when is_list(Value) -> non_blank(Value);
+        _ -> erlang:error({invalid_env_file, FileName, Path})
+    end.
+
+-spec strip_newline(binary()) -> binary().
+strip_newline(Contents) ->
+    Size = byte_size(Contents),
+    case Contents of
+        <<Rest:(Size - 2)/binary, "\r\n">> -> Rest;
+        <<Rest:(Size - 1)/binary, "\n">> -> Rest;
+        _ -> Contents
+    end.
+
 -spec non_blank(string()) -> string() | undefined.
 non_blank(Value) ->
     case string:trim(Value) of

fluxer_gateway/test/fluxer_gateway_config_tests.erl

@@ -272,6 +272,64 @@ public_endpoints_defaults_test() ->
     ?assertEqual(undefined, maps:get(media_proxy_endpoint, Config)),
     ?assertEqual(<<"http://localhost:8088">>, maps:get(static_cdn_endpoint, Config)).
 
+env_value_reads_name_file_test() ->
+    with_env_file(<<"from-file\r\n">>, fun(Path) ->
+        with_envs(
+            [{"FLUXER_GATEWAY_TEST_SECRET", ""}, {"FLUXER_GATEWAY_TEST_SECRET_FILE", Path}],
+            fun() ->
+                ?assertEqual(
+                    "from-file", fluxer_gateway_config:env_value("FLUXER_GATEWAY_TEST_SECRET")
+                )
+            end
+        )
+    end).
+
+env_value_trims_only_one_newline_test() ->
+    with_env_file(<<"line1\nline2\n\n">>, fun(Path) ->
+        with_env("FLUXER_GATEWAY_TEST_SECRET_FILE", Path, fun() ->
+            ?assertEqual(
+                "line1\nline2\n", fluxer_gateway_config:env_value("FLUXER_GATEWAY_TEST_SECRET")
+            )
+        end)
+    end).
+
+env_value_rejects_name_and_name_file_test() ->
+    with_envs(
+        [
+            {"FLUXER_GATEWAY_TEST_SECRET", "direct"},
+            {"FLUXER_GATEWAY_TEST_SECRET_FILE", "/run/secrets/x"}
+        ],
+        fun() ->
+            ?assertError(
+                {ambiguous_env, "FLUXER_GATEWAY_TEST_SECRET",
+                    "FLUXER_GATEWAY_TEST_SECRET_FILE"},
+                fluxer_gateway_config:env_value("FLUXER_GATEWAY_TEST_SECRET")
+            )
+        end
+    ).
+
+env_value_rejects_missing_name_file_test() ->
+    Path = "/nonexistent/fluxer-gateway-test-secret",
+    with_env("FLUXER_GATEWAY_TEST_SECRET_FILE", Path, fun() ->
+        ?assertError(
+            {unreadable_env_file, "FLUXER_GATEWAY_TEST_SECRET_FILE", Path, enoent},
+            fluxer_gateway_config:env_value("FLUXER_GATEWAY_TEST_SECRET")
+        )
+    end).
+
+with_env_file(Contents, Fun) ->
+    Path = filename:join(
+        filename:basedir(user_cache, "fluxer_gateway_tests"),
+        integer_to_list(erlang:unique_integer([positive]))
+    ),
+    ok = filelib:ensure_dir(Path),
+    ok = file:write_file(Path, Contents),
+    try
+        Fun(Path)
+    after
+        file:delete(Path)
+    end.
+
 with_envs([], Fun) ->
     Fun();
 with_envs([{Name, Value} | Rest], Fun) ->

fluxer_media_proxy/src/cli.rs

@@ -48,7 +48,9 @@ pub enum StorageBackendArg {
 }
 
 pub fn load_config(args: &Args) -> anyhow::Result<Config> {
-    load_config_from_iter(args, std::env::vars())
+    let vars =
+        fluxer_common::config::resolve_env_files(std::env::vars()).map_err(anyhow::Error::msg)?;
+    load_config_from_iter(args, vars)
 }
 
 pub fn load_config_from_iter<I, K, V>(args: &Args, vars: I) -> anyhow::Result<Config>

fluxer_media_proxy/src/config/mod.rs

@@ -13,7 +13,7 @@ use parse::{
     parse_mode_env, parse_policy_mode, parse_storage_backend, parse_u16, parse_u64, parse_usize,
     validate_read_endpoint,
 };
-use std::{env, path::PathBuf};
+use std::path::PathBuf;
 
 #[derive(Clone, Copy, Debug, Eq, PartialEq)]
 pub enum StorageBackend {
@@ -144,10 +144,6 @@ pub struct Config {
 }
 
 impl Config {
-    pub fn load_from_env() -> anyhow::Result<Self> {
-        Self::load_from_iter(env::vars())
-    }
-
     pub fn load_from_iter<I, K, V>(vars: I) -> anyhow::Result<Self>
     where
         I: IntoIterator<Item = (K, V)>,

fluxer_push/src/cli.rs

@@ -27,7 +27,9 @@ pub enum Command {
 }
 
 pub fn load_config(args: &Args) -> anyhow::Result<Config> {
-    load_config_from_iter(args, std::env::vars())
+    let vars =
+        fluxer_svc::config::resolve_env_files(std::env::vars()).map_err(anyhow::Error::msg)?;
+    load_config_from_iter(args, vars)
 }
 
 fn load_config_from_iter<I, K, V>(args: &Args, vars: I) -> anyhow::Result<Config>

fluxer_svc/src/config.rs

@@ -1,6 +1,7 @@
 // SPDX-License-Identifier: AGPL-3.0-or-later
 
 use std::env;
+use std::fs;
 use std::net::{IpAddr, SocketAddr};
 use std::time::Duration;
 
@@ -52,7 +53,7 @@ pub enum DatabaseBackend {
 
 impl ServiceConfig {
     pub fn from_env() -> anyhow::Result<Self> {
-        Self::from_env_reader(|name| env::var(name).ok())
+        Self::from_env_reader(env_var)
     }
 
     fn from_env_reader<F>(get: F) -> anyhow::Result<Self>
@@ -203,7 +204,63 @@ fn default_max_concurrent_requests(service_name: &str) -> usize {
 }
 
 pub fn optional_env(name: &str) -> Option<String> {
-    optional_from(&|key| env::var(key).ok(), name)
+    env_var(name)
+}
+
+fn env_var(name: &str) -> Option<String> {
+    resolve_env_value(name, |key| env::var(key).ok()).unwrap_or_else(|error| panic!("{error}"))
+}
+
+fn resolve_env_value<F>(name: &str, get: F) -> Result<Option<String>, String>
+where
+    F: Fn(&str) -> Option<String>,
+{
+    let non_blank = |value: Option<String>| value.filter(|value| !value.trim().is_empty());
+    let value = non_blank(get(name));
+    let Some(path) = non_blank(get(&format!("{name}_FILE"))) else {
+        return Ok(value);
+    };
+    if value.is_some() {
+        return Err(format!("{name} and {name}_FILE are both set, set only one"));
+    }
+    let contents = fs::read_to_string(&path)
+        .map_err(|error| format!("{name}_FILE could not read {path} ({error})"))?;
+    let contents = contents
+        .strip_suffix('\n')
+        .map_or(contents.as_str(), |rest| {
+            rest.strip_suffix('\r').unwrap_or(rest)
+        });
+    Ok(non_blank(Some(contents.to_owned())))
+}
+
+pub fn resolve_env_files<I>(vars: I) -> Result<Vec<(String, String)>, String>
+where
+    I: IntoIterator<Item = (String, String)>,
+{
+    let vars: Vec<(String, String)> = vars.into_iter().collect();
+    let get = |key: &str| {
+        vars.iter()
+            .find_map(|(name, value)| (name == key).then(|| value.clone()))
+    };
+    let mut resolved = Vec::new();
+    for (key, _) in &vars {
+        let Some(name) = key
+            .strip_suffix("_FILE")
+            .filter(|name| name.starts_with("FLUXER_"))
+        else {
+            continue;
+        };
+        if let Some(value) = resolve_env_value(name, get)? {
+            resolved.push((name.to_owned(), value));
+        }
+    }
+    let rest: Vec<(String, String)> = vars
+        .iter()
+        .filter(|(key, _)| !resolved.iter().any(|(name, _)| name == key))
+        .cloned()
+        .collect();
+    resolved.extend(rest);
+    Ok(resolved)
 }
 
 fn optional_from<F>(get: &F, name: &str) -> Option<String>
@@ -276,6 +333,46 @@ mod tests {
         .unwrap()
     }
 
+    #[test]
+    fn resolves_name_file_entries() {
+        let dir = env::temp_dir().join(format!("fluxer-svc-env-file-{}", std::process::id()));
+        fs::create_dir_all(&dir).unwrap();
+        let path = dir.join("token");
+        fs::write(&path, "from-file\r\n").unwrap();
+        let path = path.to_string_lossy().into_owned();
+        let missing = dir.join("missing").to_string_lossy().into_owned();
+        let resolved = resolve_env_files(vec![
+            ("FLUXER_NATS_AUTH_TOKEN".to_owned(), String::new()),
+            ("FLUXER_NATS_AUTH_TOKEN_FILE".to_owned(), path.clone()),
+        ])
+        .unwrap();
+        let both = resolve_env_files(vec![
+            ("FLUXER_X".to_owned(), "direct".to_owned()),
+            ("FLUXER_X_FILE".to_owned(), path),
+        ]);
+        let unreadable = resolve_env_files(vec![("FLUXER_X_FILE".to_owned(), missing.clone())]);
+        let unrelated = resolve_env_files(vec![("SSL_CERT_FILE".to_owned(), missing.clone())]);
+        fs::remove_dir_all(&dir).unwrap();
+        assert_eq!(
+            vec!["from-file"],
+            resolved
+                .iter()
+                .filter(|(name, _)| name == "FLUXER_NATS_AUTH_TOKEN")
+                .map(|(_, value)| value.as_str())
+                .collect::<Vec<_>>()
+        );
+        assert_eq!(
+            Err("FLUXER_X and FLUXER_X_FILE are both set, set only one".to_owned()),
+            both
+        );
+        assert!(
+            unreadable
+                .unwrap_err()
+                .starts_with(&format!("FLUXER_X_FILE could not read {missing} ("))
+        );
+        assert_eq!(Ok(vec![("SSL_CERT_FILE".to_owned(), missing)]), unrelated);
+    }
+
     #[test]
     fn extracts_statefulset_ordinal() {
         assert_eq!(shard_id_from_pod_name("my-service-0"), Some(0));

fluxer_unfurl/src/resolvers/klipy.rs

@@ -4,7 +4,6 @@ use super::{ResolveContext, Resolver, ResolverResult};
 use crate::http_fetch;
 use crate::media_proxy::{MediaMetadata, embed_media_flags};
 use crate::types::{EmbedMedia, EmbedProvider, MessageEmbed};
-use fluxer_svc::config::optional_env;
 use std::future::Future;
 use std::pin::Pin;
 use std::time::Duration;
@@ -49,7 +48,7 @@ impl Resolver for KlipyResolver {
         ctx: &'a ResolveContext<'_>,
     ) -> Pin<Box<dyn Future<Output = anyhow::Result<ResolverResult>> + Send + 'a>> {
         Box::pin(async move {
-            let Some(api_key) = ctx.klipy_api_key.clone().or_else(klipy_api_key) else {
+            let Some(api_key) = ctx.klipy_api_key.clone() else {
                 return Ok(ResolverResult { embeds: vec![] });
             };
             let formats = match resolve_media_via_api(ctx, &api_key).await {
@@ -136,10 +135,6 @@ fn klipy_resource(kind: &str) -> &'static str {
     }
 }
 
-fn klipy_api_key() -> Option<String> {
-    optional_env("FLUXER_KLIPY_API_KEY").or_else(|| optional_env("KLIPY_API_KEY"))
-}
-
 async fn resolve_media_via_api(
     ctx: &ResolveContext<'_>,
     api_key: &str,

fluxer_unfurl/src/resolvers/youtube.rs

@@ -5,7 +5,6 @@ use crate::http_fetch;
 use crate::media_proxy::embed_media_flags;
 use crate::text_limits;
 use crate::types::{EmbedAuthor, EmbedMedia, EmbedProvider, MessageEmbed};
-use fluxer_svc::config::optional_env;
 use serde::Deserialize;
 use std::future::Future;
 use std::pin::Pin;
@@ -61,7 +60,7 @@ impl Resolver for YouTubeResolver {
                 }
             };
 
-            let Some(api_key) = ctx.youtube_api_key.clone().or_else(youtube_api_key) else {
+            let Some(api_key) = ctx.youtube_api_key.clone() else {
                 tracing::debug!("No YouTube API key configured");
                 return Ok(ResolverResult { embeds: vec![] });
             };
@@ -251,10 +250,6 @@ struct YouTubeThumbnail {
     height: Option<u32>,
 }
 
-fn youtube_api_key() -> Option<String> {
-    optional_env("FLUXER_YOUTUBE_API_KEY").or_else(|| optional_env("YOUTUBE_API_KEY"))
-}
-
 fn build_youtube_api_url(video_id: &str, api_key: &str) -> anyhow::Result<Url> {
     let mut url = Url::parse(YOUTUBE_API_BASE)?;
     url.query_pairs_mut()

fluxer_unfurl/src/shard_impl.rs

@@ -24,6 +24,8 @@ pub struct UnfurlShard {
     resolvers: Vec<Box<dyn crate::resolvers::Resolver>>,
     media_proxy: MediaProxyClient,
     self_hosted: bool,
+    youtube_api_key: Option<String>,
+    klipy_api_key: Option<String>,
 }
 
 impl UnfurlShard {
@@ -78,6 +80,10 @@ impl UnfurlShard {
             resolvers,
             media_proxy,
             self_hosted,
+            youtube_api_key: optional_env("FLUXER_YOUTUBE_API_KEY")
+                .or_else(|| optional_env("YOUTUBE_API_KEY")),
+            klipy_api_key: optional_env("FLUXER_KLIPY_API_KEY")
+                .or_else(|| optional_env("KLIPY_API_KEY")),
         }
     }
 
@@ -100,6 +106,8 @@ impl UnfurlShard {
                 internal_http_client(),
             ),
             self_hosted: false,
+            youtube_api_key: None,
+            klipy_api_key: None,
         }
     }
 
@@ -125,8 +133,12 @@ impl UnfurlShard {
             nsfw_mode,
             media_proxy: &self.media_proxy,
             self_hosted: self.self_hosted,
-            youtube_api_key: youtube_api_key.map(str::to_owned),
-            klipy_api_key: klipy_api_key.map(str::to_owned),
+            youtube_api_key: youtube_api_key
+                .map(str::to_owned)
+                .or_else(|| self.youtube_api_key.clone()),
+            klipy_api_key: klipy_api_key
+                .map(str::to_owned)
+                .or_else(|| self.klipy_api_key.clone()),
         };
 
         if let Some(idx) = matched_resolver_idx {

fluxer_users/src/pseudonym.rs

@@ -1,5 +1,6 @@
 // SPDX-License-Identifier: AGPL-3.0-or-later
 
+use fluxer_svc::config::optional_env;
 use hmac::{KeyInit, Mac};
 use std::sync::{LazyLock, OnceLock};
 
@@ -46,8 +47,8 @@ pub fn configure(secret: Option<String>, environment: Option<&str>) -> anyhow::R
 
 pub fn configure_from_env() -> anyhow::Result<()> {
     configure(
-        std::env::var(SECRET_ENV).ok(),
-        std::env::var("FLUXER_ENV").ok().as_deref(),
+        optional_env(SECRET_ENV),
+        optional_env("FLUXER_ENV").as_deref(),
     )
 }
 

packages/config/src/__tests__/ConfigLoader.test.ts

@@ -1,6 +1,9 @@
 // SPDX-License-Identifier: AGPL-3.0-or-later
 
 import {generateKeyPairSync} from 'node:crypto';
+import {mkdtempSync, rmSync, writeFileSync} from 'node:fs';
+import {tmpdir} from 'node:os';
+import {join} from 'node:path';
 import {getConfig, loadConfig, resetConfig} from '@fluxer/config/src/ConfigLoader';
 import {afterEach, beforeEach, describe, expect, test, vi} from 'vitest';
 
@@ -80,6 +83,24 @@ describe('ConfigLoader', () => {
 		expect((await loadConfig()).domain.base_domain).toBe('localhost');
 	});
 
+	test('loadConfig reads secrets from NAME_FILE', async () => {
+		const dir = mkdtempSync(join(tmpdir(), 'fluxer-config-file-'));
+		try {
+			const path = join(dir, 'postgres_password');
+			writeFileSync(path, 'from-secret-file\n');
+			stubMinimalEnv({FLUXER_POSTGRES_PASSWORD: '', FLUXER_POSTGRES_PASSWORD_FILE: path});
+			const config = await loadConfig();
+			expect(config.database.postgres.password).toBe('from-secret-file');
+		} finally {
+			rmSync(dir, {recursive: true, force: true});
+		}
+	});
+
+	test('loadConfig rejects NAME and NAME_FILE together', async () => {
+		stubMinimalEnv({FLUXER_SUDO_MODE_SECRET_FILE: '/run/secrets/sudo'});
+		await expect(loadConfig()).rejects.toThrow('FLUXER_SUDO_MODE_SECRET and FLUXER_SUDO_MODE_SECRET_FILE are both set');
+	});
+
 	test('getConfig throws when config is not loaded', () => {
 		expect(() => getConfig()).toThrow('Config not loaded');
 	});

packages/config/src/__tests__/EnvironmentOverrides.test.ts

@@ -1,7 +1,14 @@
 // SPDX-License-Identifier: AGPL-3.0-or-later
 
-import {buildNamedFluxerEnvOverrides, setNestedValue} from '@fluxer/config/src/config_loader/EnvironmentOverrides';
-import {describe, expect, test} from 'vitest';
+import {mkdtempSync, rmSync, writeFileSync} from 'node:fs';
+import {tmpdir} from 'node:os';
+import {join} from 'node:path';
+import {
+	buildNamedFluxerEnvOverrides,
+	readEnvValue,
+	setNestedValue,
+} from '@fluxer/config/src/config_loader/EnvironmentOverrides';
+import {afterAll, describe, expect, test} from 'vitest';
 
 describe('setNestedValue', () => {
 	test('sets a top-level key', () => {
@@ -316,3 +323,69 @@ describe('buildNamedFluxerEnvOverrides', () => {
 		);
 	});
 });
+
+describe('readEnvValue with NAME_FILE', () => {
+	const dir = mkdtempSync(join(tmpdir(), 'fluxer-env-file-'));
+	afterAll(() => rmSync(dir, {recursive: true, force: true}));
+
+	function secretFile(name: string, contents: string): string {
+		const path = join(dir, name);
+		writeFileSync(path, contents);
+		return path;
+	}
+
+	test('reads the value from NAME_FILE when NAME is unset', () => {
+		const path = secretFile('plain', 'from-file\n');
+		expect(readEnvValue({FLUXER_SUDO_MODE_SECRET_FILE: path}, 'FLUXER_SUDO_MODE_SECRET')).toBe('from-file');
+	});
+
+	test('reads the value from NAME_FILE when NAME is blank', () => {
+		const path = secretFile('blank', 'from-file');
+		expect(
+			readEnvValue({FLUXER_SUDO_MODE_SECRET: ' ', FLUXER_SUDO_MODE_SECRET_FILE: path}, 'FLUXER_SUDO_MODE_SECRET'),
+		).toBe('from-file');
+	});
+
+	test('trims only one trailing newline', () => {
+		const crlf = secretFile('crlf', 'value\r\n');
+		const multi = secretFile('multi', '-----BEGIN-----\nabc\n-----END-----\n\n');
+		expect(readEnvValue({X_FILE: crlf}, 'X')).toBe('value');
+		expect(readEnvValue({X_FILE: multi}, 'X')).toBe('-----BEGIN-----\nabc\n-----END-----\n');
+	});
+
+	test('treats an empty file as unset', () => {
+		const path = secretFile('empty', '\n');
+		expect(readEnvValue({X_FILE: path}, 'X')).toBeUndefined();
+	});
+
+	test('keeps NAME when NAME_FILE is blank', () => {
+		expect(readEnvValue({X: 'direct', X_FILE: ''}, 'X')).toBe('direct');
+	});
+
+	test('rejects NAME and NAME_FILE together', () => {
+		const path = secretFile('both', 'from-file');
+		expect(() => readEnvValue({X: 'direct', X_FILE: path}, 'X')).toThrow('X and X_FILE are both set, set only one');
+	});
+
+	test('names NAME_FILE and the path when the file is missing', () => {
+		const path = join(dir, 'missing');
+		expect(() => readEnvValue({X_FILE: path}, 'X')).toThrow(`X_FILE could not read ${path} (ENOENT)`);
+	});
+
+	test('rejects a file that is not valid UTF-8', () => {
+		const path = join(dir, 'binary');
+		writeFileSync(path, Buffer.from([0xff, 0x61]));
+		expect(() => readEnvValue({X_FILE: path}, 'X')).toThrow(`X_FILE could not read ${path} (`);
+	});
+
+	test('feeds named overrides and aliases', () => {
+		const secret = secretFile('stripe', 'sk_test_file\n');
+		const nats = secretFile('nats', 'nats://nats:4222\n');
+		expect(
+			buildNamedFluxerEnvOverrides({FLUXER_STRIPE_SECRET_KEY_FILE: secret, FLUXER_NATS_CORE_URL_FILE: nats}),
+		).toMatchObject({
+			integrations: {stripe: {secret_key: 'sk_test_file'}},
+			services: {nats: {core_url: 'nats://nats:4222'}},
+		});
+	});
+});

packages/config/src/config_loader/EnvironmentOverrides.ts

@@ -1,5 +1,6 @@
 // SPDX-License-Identifier: AGPL-3.0-or-later
 
+import {readFileSync} from 'node:fs';
 import {type ConfigObject, isConfigObject} from '@fluxer/config/src/config_loader/ConfigObject';
 
 type ConfigPathKey = string | number;
@@ -420,11 +421,29 @@ const NAMED_FLUXER_ENV_ALIASES: Record<string, string | undefined> = {
 
 export const NAMED_FLUXER_ENV_NAMES = Object.keys(NAMED_FLUXER_ENV_OVERRIDES);
 
-export function readEnvValue(env: NodeJS.ProcessEnv, name: string): string | undefined {
-	const value = env[name];
+function nonBlank(value: string | undefined): string | undefined {
 	return value === undefined || value.trim().length === 0 ? undefined : value;
 }
 
+export function readEnvValue(env: NodeJS.ProcessEnv, name: string): string | undefined {
+	const value = nonBlank(env[name]);
+	const filePath = nonBlank(env[`${name}_FILE`]);
+	if (filePath === undefined) {
+		return value;
+	}
+	if (value !== undefined) {
+		throw new Error(`${name} and ${name}_FILE are both set, set only one`);
+	}
+	let contents: string;
+	try {
+		contents = new TextDecoder('utf-8', {fatal: true}).decode(readFileSync(filePath));
+	} catch (error) {
+		const reason = error instanceof Error && 'code' in error ? String(error.code) : 'unreadable';
+		throw new Error(`${name}_FILE could not read ${filePath} (${reason})`);
+	}
+	return nonBlank(contents.replace(/\r?\n$/, ''));
+}
+
 export function buildNamedFluxerEnvOverrides(env: NodeJS.ProcessEnv): ConfigObject {
 	const overrides: ConfigObject = {};
 	for (const [envKey, mapping] of Object.entries(NAMED_FLUXER_ENV_OVERRIDES)) {

packages/schema/src/domains/guild/GuildTemplateSchemas.ts

@@ -12,6 +12,14 @@ import {ColorType, createStringType, Int32Type} from '@fluxer/schema/src/primiti
 import {z} from 'zod';
 
 const TEMPLATE_NAME_MAX_LENGTH = 100;
+const TEMPLATE_TOPIC_MAX_LENGTH = 4096;
+
+function clipTopic(value: string): string {
+	if (value.length <= CHANNEL_TOPIC_MAX_LENGTH) return value;
+	const last = value.charCodeAt(CHANNEL_TOPIC_MAX_LENGTH - 1);
+	const end = last >= 0xd800 && last <= 0xdbff ? CHANNEL_TOPIC_MAX_LENGTH - 1 : CHANNEL_TOPIC_MAX_LENGTH;
+	return value.slice(0, end);
+}
 
 const TemplateEntityId = z
 	.union([
@@ -50,7 +58,12 @@ export const TemplateChannel = z.object({
 		.nullish()
 		.transform((value) => value ?? '')
 		.describe('The name of the channel'),
-	topic: z.string().max(CHANNEL_TOPIC_MAX_LENGTH).nullish().describe('The channel topic'),
+	topic: z
+		.string()
+		.max(TEMPLATE_TOPIC_MAX_LENGTH)
+		.transform(clipTopic)
+		.nullish()
+		.describe('The channel topic, shortened to the Fluxer channel topic limit'),
 	position: Int32Type.describe('The position of the channel'),
 	parent_id: TemplateEntityId.nullish().describe('The template-local ID of the parent category'),
 	bitrate: z.number().int().nonnegative().nullish().describe('The bitrate for voice channels'),