Compare commits

...
Author SHA1 Message Date
HampusandGitHub bbfe809bef fix(app): crop animated images on web with libwebp (#2947) 2026-09-24 22:45:53 +02:00
HampusandGitHub e0843ac4f5 fix(app): keep guild folder expansion state local (#2944) 2026-09-24 17:52:33 +02:00
HampusandGitHub 43741cdad8 fix(gateway): always trim the connect snapshot for guild connects (#2943) 2026-09-24 17:09:48 +02:00
HampusandGitHub b8e3807262 Revert "fix(push): deliver direct messages without holding them" (#2942) 2026-09-24 17:09:44 +02:00
HampusandGitHub 3304f01a84 chore(i18n): recompile uk error catalog (#2941) 2026-09-24 17:09:36 +02:00
fluxer-weblate[bot]andGitHub 2ba463235b chore(i18n): update translations from Weblate (#2909) 2026-09-24 16:25:26 +02:00
fluxer-weblate[bot]andGitHub 15136fed59 chore(i18n): update translations from Weblate (#2923) 2026-09-24 16:25:05 +02:00
HampusandGitHub 6013581dd9 fix(push): deliver direct messages without holding them (#2938) 2026-09-24 16:21:42 +02:00
HampusandGitHub 7a91f128e9 fix(app-proxy): drop link preview metadata on self-hosted (#2936) 2026-09-24 16:07:00 +02:00
HampusandGitHub 963ffc5550 feat(push): scope read clears to the enrolled cohort (#2935) 2026-09-24 15:15:45 +02:00
HampusandGitHub a90991612c fix(gateway): truncate reads on an expired outbox entry (#2934) 2026-09-24 15:04:24 +02:00
HampusandGitHub 50ad23b760 fix(api): run the notification extension on every iOS alert (#2933) 2026-09-24 15:04:01 +02:00
HampusandGitHub 425dab983b fix(push): restore iOS avatars and stop misrouting relay endpoints (#2932) 2026-09-24 15:03:32 +02:00
HampusandGitHub a0825e77c4 feat(voice): ship the screen share delivery rework to everyone (#2931) 2026-09-24 14:57:40 +02:00
HampusandGitHub 88038a1d5b fix(voice): stop direct input capturing microphones in stereo (#2929) 2026-09-24 14:51:05 +02:00
HampusandGitHub c2c0fdb445 fix(app): make corner volume control the focused stream (#2928) 2026-09-24 14:04:05 +02:00
HampusandGitHub dcd5f09d6a feat(api): add env toggles for automatic phone flagging (#2927) 2026-09-24 03:36:35 +02:00
HampusandGitHub 590b1f36fd docs(downloads): document the canary apt and dnf repositories (#2926) 2026-09-24 03:29:52 +02:00
HampusandGitHub 168ac727f1 fix(desktop): set the deb package synopsis (#2925) 2026-09-24 03:29:33 +02:00
HampusandGitHub deb86dd92e fix(admin): format users list search hint (#2924) 2026-09-24 02:12:37 +02:00
omsterandGitHub 7ccec4d3b8 feat(admin): hint text for * search in user page (#2922) 2026-09-24 01:56:17 +02:00
omsterandGitHub 2f38bcdf26 fix(admin): ordering fixes for admin user search and meilisearch (#2920) 2026-09-24 01:45:56 +02:00
HampusandGitHub f2785941aa fix(app): point self-hosted users at their instance admins (#2921) 2026-09-24 01:42:33 +02:00
HampusandGitHub ea9f83a443 fix(push): keep read-state clears alive as long as the alert (#2919) 2026-09-24 01:26:18 +02:00
HampusandGitHub bd6ca7290e fix(api): allow deleting messages without send permission (#2918) 2026-09-24 01:14:01 +02:00
HampusandGitHub b85e975fb5 feat(push): deliver our own relay endpoints in process (#2917) 2026-09-24 01:07:09 +02:00
HampusandGitHub b6e504f68c fix(push): keep device tokens out of logs (#2916) 2026-09-24 00:33:50 +02:00
HampusandGitHub 5fde6eb484 feat(push): ring Android calls and harden the relay (#2915) 2026-09-24 00:07:15 +02:00
HampusandGitHub b16989d567 feat(push): ring incoming calls on Apple PushKit devices (#2911) 2026-09-23 20:21:08 +02:00
HampusandGitHub c9754ac11a fix(api): exempt internal rpc from the client ip check (#2910) 2026-09-23 18:03:36 +02:00
fluxer-weblate[bot]andGitHub f34e4a5115 chore(i18n): update translations from Weblate (#2903) 2026-09-23 17:28:25 +02:00
fluxer-weblate[bot]andGitHub 44b3615298 chore(i18n): update translations from Weblate (#2904) 2026-09-23 17:27:59 +02:00
HampusandGitHub 211e98307d perf(push): cache endpoint guard dns verdicts (#2907) 2026-09-23 17:27:19 +02:00
HampusandGitHub 18c303abf6 feat(push): relay notifications as encrypted web push (#2906) 2026-09-23 14:04:55 +02:00
JiraliteandGitHub 7021a58090 fix: allow copying message snapshots (#2905) 2026-09-23 14:01:10 +02:00
WagnerandGitHub 320725a587 fix(desktop): capture full pipewire quantum on linux (#2481) 2026-09-22 21:37:20 +02:00
fluxer-weblate[bot]andGitHub 8450edc072 chore(i18n): update translations from Weblate (#2895) 2026-09-22 21:28:24 +02:00
omsterandGitHub a1e2bf2c8d feat(dev/linux): select the wayland backend when reachable in the native desktop app (#2899)
Signed-off-by: omstr <[email protected]>
2026-09-22 21:27:59 +02:00
HampusandGitHub 82b2f4ec5e fix(app): put jxl and other image attachments in the mosaic (#2902) 2026-09-22 21:18:39 +02:00
HampusandGitHub c92e5d03a7 fix(api): accept any image or video attachment as embed media (#2901) 2026-09-22 21:18:35 +02:00
HampusandGitHub 91340c5c84 fix(markdown): compile the parser wasm asynchronously (#2900) 2026-09-22 19:58:38 +02:00
HampusandGitHub 045dd5d027 test(api): make the harvest token tamper test deterministic (#2894) 2026-09-22 02:20:18 +02:00
HampusandGitHub a21b9c4659 docs(readme): clean up the download prose (#2893) 2026-09-22 02:08:42 +02:00
HampusandGitHub 4b1b869802 docs(readme): point Linux installs at Flathub (#2892) 2026-09-22 02:04:06 +02:00
HampusandGitHub 1ab7e7dfcc fix(api): unfurl links to a self-hosted instance's own domain (#2891) 2026-09-22 02:00:51 +02:00
HampusandGitHub 31c53d2dff fix(app): stop pending stickers from reloading the channel (#2890) 2026-09-22 02:00:26 +02:00
HampusandGitHub 412a1ae79d perf(api): stop ledgering session payment reconciliation (#2889) 2026-09-22 01:37:21 +02:00
HampusandGitHub 0b2306ec3d fix(api): honour default TTLs and expire stale job ledger rows (#2887) 2026-09-21 23:16:39 +02:00
HampusandGitHub 242ed3a934 fix(desktop): drop orphaned Squirrel uninstall entry (#2885) 2026-09-21 20:03:33 +02:00
HampusandGitHub 70e1ce682a feat(emoji): add Unicode 17 emoji and fix mixed skin tones (#2883) 2026-09-21 16:26:06 +02:00
HampusandGitHub 7601bf98ee fix(channel): sync a cleared group DM name without a reload (#2882) 2026-09-21 15:34:18 +02:00
c7ec2a0f58 chore(tooling): Ignore .vscode/ in .gitignore (#2868)
Co-authored-by: Hampus <[email protected]>
2026-09-21 13:29:33 +02:00
XeonandGitHub 6a5e0056a8 fix(flatpak): Add a release tag and make small corrections (#2872) 2026-09-21 13:28:36 +02:00
HampusandGitHub 78d105b46e fix(desktop): stop looping on an update that never installs (#2879) 2026-09-21 03:36:11 +02:00
HampusandGitHub c68d62b8a0 fix(voice): darken screen share source titles in light theme (#2878) 2026-09-21 01:20:54 +02:00
HampusandGitHub df58020f4c fix(api): keep premium paid for after a subscription cancels (#2875) 2026-09-20 23:50:49 +02:00
HampusandGitHub f052ce05aa fix(workspace): point the Erlang extension at the repo root (#2871) 2026-09-20 19:49:06 +02:00
HampusandGitHub eedfd9275f fix(api): only require permissions a channel overwrite grants (#2867) 2026-09-20 17:56:22 +02:00
HampusandGitHub 416af4bec4 fix(docs): correct the flatpak and dnf signing instructions (#2865) 2026-09-20 16:42:33 +02:00
HampusandGitHub 108d282ddd chore(deps): pin pnpm 11 so the lockfile parses for packagers (#2864) 2026-09-20 15:30:50 +02:00
HampusandGitHub a6103244b0 docs(readme): fix the license wording and shrink the preview (#2862) 2026-09-20 15:11:06 +02:00
HampusandGitHub 38935c83c5 docs(readme): document every download and install method (#2861) 2026-09-20 15:05:54 +02:00
430 changed files with 36678 additions and 14948 deletions
+1 -1
View File
@@ -7,7 +7,7 @@ ARG USER_UID=1000
ARG USER_GID=1000
ARG NODE_MAJOR=26
ARG ELP_VERSION=2026-08-10
ARG PNPM_VERSION=12.4.2
ARG PNPM_VERSION=11.27.0
ARG WASM_BINDGEN_VERSION=0.2.128
ENV DEBIAN_FRONTEND=noninteractive
+5 -1
View File
@@ -38,7 +38,11 @@
"customizations": {
"vscode": {
"settings": {
"editor.defaultFormatter": "biomejs.biome"
"editor.defaultFormatter": "biomejs.biome",
"erlang.includePaths": ["."],
"search.exclude": {
"**/_build/default/lib/fluxer_gateway": true
}
},
"extensions": [
"biomejs.biome",
+1
View File
@@ -32,6 +32,7 @@
/fluxer_docs/.astro/
/fluxer_app/.devserver-cache.json
/fluxer_app/pkgs/libfluxcore/
/fluxer_app/pkgs/libfluxwebp/
/fluxer_app/src/features/i18n/locales/*/messages.mjs
/fluxer_app/src/features/messaging/utils/markdown/parser/MarkdownParserWasmBytes.ts
/fluxer_app/src/features/theme/styles/generated/
+3
View File
@@ -28,6 +28,9 @@ f:media_proxy:
f:messages:
- changed-files:
- any-glob-to-any-file: fluxer_messages/**/*
f:push:
- changed-files:
- any-glob-to-any-file: fluxer_push/**/*
f:snowflakes:
- changed-files:
- any-glob-to-any-file: fluxer_snowflakes/**/*
+36
View File
@@ -0,0 +1,36 @@
# SPDX-License-Identifier: AGPL-3.0-or-later
name: build push
on:
workflow_dispatch:
inputs:
build-version:
description: "Explicit Fluxer CalVer build version (YYYY.MDD.MICRO, UTC HHMMSS without leading zeroes) to use instead of automatic UTC clock allocation"
type: string
required: false
default: ""
permissions:
actions: read
contents: write
packages: write
jobs:
approve:
name: approve build release
permissions: {}
runs-on: ubuntu-24.04
environment: builds
timeout-minutes: 5
steps:
- name: approved
run: echo "Build release approved."
image:
needs: approve
uses: ./.github/workflows/_build-image.yaml
secrets: inherit
with:
image: fluxer-push
dockerfile: fluxer_push/Dockerfile
build-version: ${{ inputs['build-version'] }}
+22
View File
@@ -123,12 +123,16 @@ jobs:
with:
path: |
fluxer_app/pkgs/libfluxcore
fluxer_app/pkgs/libfluxwebp
fluxer_app/src/features/messaging/utils/markdown/parser/MarkdownParserWasmBytes.ts
key: >-
app-wasm-${{ runner.os }}-1.98.1-${{ hashFiles('Cargo.lock', 'tools/ci/src/app_wasm.rs',
'tools/ci/templates/libfluxcore_wrapper.js', 'tools/ci/templates/libfluxcore_wrapper.d.ts',
'fluxer_app/rust/libfluxcore/Cargo.toml', 'fluxer_app/rust/libfluxcore/Cargo.lock',
'fluxer_app/rust/libfluxcore/.cargo/config.toml', 'fluxer_app/rust/libfluxcore/src/**',
'fluxer_app/rust/libfluxwebp/Cargo.toml', 'fluxer_app/rust/libfluxwebp/Cargo.lock',
'fluxer_app/rust/libfluxwebp/src/**', 'fluxer_app/rust/libfluxwebp/shim/**',
'fluxer_app/rust/libfluxwebp/simd/**',
'packages/markdown_parser/rust/Cargo.toml', 'packages/markdown_parser/rust/.cargo/config.toml',
'packages/markdown_parser/rust/src/**') }}
@@ -142,12 +146,16 @@ jobs:
with:
path: |
fluxer_app/pkgs/libfluxcore
fluxer_app/pkgs/libfluxwebp
fluxer_app/src/features/messaging/utils/markdown/parser/MarkdownParserWasmBytes.ts
key: >-
app-wasm-${{ runner.os }}-1.98.1-${{ hashFiles('Cargo.lock', 'tools/ci/src/app_wasm.rs',
'tools/ci/templates/libfluxcore_wrapper.js', 'tools/ci/templates/libfluxcore_wrapper.d.ts',
'fluxer_app/rust/libfluxcore/Cargo.toml', 'fluxer_app/rust/libfluxcore/Cargo.lock',
'fluxer_app/rust/libfluxcore/.cargo/config.toml', 'fluxer_app/rust/libfluxcore/src/**',
'fluxer_app/rust/libfluxwebp/Cargo.toml', 'fluxer_app/rust/libfluxwebp/Cargo.lock',
'fluxer_app/rust/libfluxwebp/src/**', 'fluxer_app/rust/libfluxwebp/shim/**',
'fluxer_app/rust/libfluxwebp/simd/**',
'packages/markdown_parser/rust/Cargo.toml', 'packages/markdown_parser/rust/.cargo/config.toml',
'packages/markdown_parser/rust/src/**') }}
@@ -190,6 +198,9 @@ jobs:
- name: Check Rust dependencies
run: cargo deny --locked check -D warnings
- name: Check libfluxwebp dependencies
run: cargo deny --manifest-path fluxer_app/rust/libfluxwebp/Cargo.toml --config deny.toml --locked check licenses bans sources
- name: Check desktop native dependencies
run: tools/ci/check-desktop-native-workspaces.sh dependencies
@@ -242,6 +253,9 @@ jobs:
- name: Check formatting
run: cargo fmt --all -- --check
- name: Check formatting (libfluxwebp)
run: cargo fmt --manifest-path fluxer_app/rust/libfluxwebp/Cargo.toml -- --check
- name: Check formatting (desktop native workspaces)
run: tools/ci/check-desktop-native-workspaces.sh fmt
@@ -398,12 +412,16 @@ jobs:
with:
path: |
fluxer_app/pkgs/libfluxcore
fluxer_app/pkgs/libfluxwebp
fluxer_app/src/features/messaging/utils/markdown/parser/MarkdownParserWasmBytes.ts
key: >-
app-wasm-${{ runner.os }}-1.98.1-${{ hashFiles('Cargo.lock', 'tools/ci/src/app_wasm.rs',
'tools/ci/templates/libfluxcore_wrapper.js', 'tools/ci/templates/libfluxcore_wrapper.d.ts',
'fluxer_app/rust/libfluxcore/Cargo.toml', 'fluxer_app/rust/libfluxcore/Cargo.lock',
'fluxer_app/rust/libfluxcore/.cargo/config.toml', 'fluxer_app/rust/libfluxcore/src/**',
'fluxer_app/rust/libfluxwebp/Cargo.toml', 'fluxer_app/rust/libfluxwebp/Cargo.lock',
'fluxer_app/rust/libfluxwebp/src/**', 'fluxer_app/rust/libfluxwebp/shim/**',
'fluxer_app/rust/libfluxwebp/simd/**',
'packages/markdown_parser/rust/Cargo.toml', 'packages/markdown_parser/rust/.cargo/config.toml',
'packages/markdown_parser/rust/src/**') }}
@@ -417,12 +435,16 @@ jobs:
with:
path: |
fluxer_app/pkgs/libfluxcore
fluxer_app/pkgs/libfluxwebp
fluxer_app/src/features/messaging/utils/markdown/parser/MarkdownParserWasmBytes.ts
key: >-
app-wasm-${{ runner.os }}-1.98.1-${{ hashFiles('Cargo.lock', 'tools/ci/src/app_wasm.rs',
'tools/ci/templates/libfluxcore_wrapper.js', 'tools/ci/templates/libfluxcore_wrapper.d.ts',
'fluxer_app/rust/libfluxcore/Cargo.toml', 'fluxer_app/rust/libfluxcore/Cargo.lock',
'fluxer_app/rust/libfluxcore/.cargo/config.toml', 'fluxer_app/rust/libfluxcore/src/**',
'fluxer_app/rust/libfluxwebp/Cargo.toml', 'fluxer_app/rust/libfluxwebp/Cargo.lock',
'fluxer_app/rust/libfluxwebp/src/**', 'fluxer_app/rust/libfluxwebp/shim/**',
'fluxer_app/rust/libfluxwebp/simd/**',
'packages/markdown_parser/rust/Cargo.toml', 'packages/markdown_parser/rust/.cargo/config.toml',
'packages/markdown_parser/rust/src/**') }}
+2
View File
@@ -10,6 +10,7 @@
/.direnv/
/.fluxer/
/.pnpm-store/
/.vscode/
**/*.css.d.ts
**/*.tsbuildinfo
@@ -25,6 +26,7 @@
/fluxer_app/.devserver-cache.json
/fluxer_app/pkgs/libfluxcore/
/fluxer_app/pkgs/libfluxwebp/
/fluxer_app/src/features/i18n/locales/*/messages.mjs
/fluxer_app/src/features/messaging/utils/markdown/parser/MarkdownParserWasmBytes.ts
/fluxer_app/src/features/theme/styles/generated/
Generated
+37
View File
@@ -1607,6 +1607,7 @@ dependencies = [
"ff",
"generic-array",
"group",
"hkdf",
"pem-rfc7468",
"pkcs8",
"rand_core 0.6.4",
@@ -1881,6 +1882,32 @@ dependencies = [
"url",
]
[[package]]
name = "fluxer-push"
version = "0.1.0"
dependencies = [
"anyhow",
"axum",
"base64 0.23.1",
"clap",
"fluxer-svc",
"futures",
"hmac 0.13.0",
"p256",
"percent-encoding",
"rand 0.10.2",
"reqwest",
"ring",
"serde",
"serde_json",
"sha2 0.11.0",
"thiserror",
"tokio",
"tracing",
"tracing-subscriber",
"url",
]
[[package]]
name = "fluxer-snowflakes"
version = "0.1.0"
@@ -2308,6 +2335,15 @@ version = "0.4.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70"
[[package]]
name = "hkdf"
version = "0.12.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7b5f8eb2ad728638ea2c7d47a21db23b7b58a72ed6a38256b8a1849f15fbbdf7"
dependencies = [
"hmac 0.12.1",
]
[[package]]
name = "hmac"
version = "0.12.1"
@@ -3893,6 +3929,7 @@ dependencies = [
"futures-channel",
"futures-core",
"futures-util",
"h2",
"http 1.5.0",
"http-body 1.1.0",
"http-body-util",
+1
View File
@@ -7,6 +7,7 @@ members = [
"fluxer_gifs",
"fluxer_svc",
"fluxer_messages",
"fluxer_push",
"fluxer_snowflakes",
"tools/ci",
"tools/dev",
+157 -3
View File
@@ -6,18 +6,172 @@
</p>
<p align="center">
<a href="https://fluxer.app/donate">
<img src="https://img.shields.io/badge/Donate-fluxer.app%2Fdonate-brightgreen" alt="Donate" /></a>
<a href="https://fluxer.app/download">
<img src="https://img.shields.io/badge/Download-fluxer.app-4641D9" alt="Download" /></a>
<a href="https://docs.fluxer.app">
<img src="https://img.shields.io/badge/Docs-docs.fluxer.app-blue" alt="Documentation" /></a>
<a href="https://fluxer.app/donate">
<img src="https://img.shields.io/badge/Donate-fluxer.app%2Fdonate-brightgreen" alt="Donate" /></a>
<a href="./LICENSE">
<img src="https://img.shields.io/badge/License-AGPLv3-purple" alt="AGPLv3 License" /></a>
</p>
<p align="center">
<a href="https://flathub.org/apps/app.fluxer.Fluxer">
<img src="https://dl.flathub.org/assets/badges/flathub-badge-en.svg" alt="Get it on Flathub" height="60" /></a>
</p>
# Fluxer
Fluxer is a free and open source instant messaging and VoIP chat app built for friends, groups, and communities.
<p align="center">
<img src="./fluxer_static/marketing/screenshots/desktop-readme-1920w.png" alt="Fluxer app showcase" width="900">
<img src="./fluxer_static/marketing/screenshots/desktop-readme-1920w.png" alt="Fluxer running side by side on a desktop monitor and a phone" width="640">
</p>
## Download
| Windows | macOS | Linux | Android | iOS |
| --- | --- | --- | --- | --- |
| [Installer (x64)][win-setup-x64] | [Disk image][mac-dmg] | [Flathub][flathub] | [APK][android-apk] | [TestFlight][ios-testflight] |
| [Installer (ARM64)][win-setup-arm64] | | [deb (x64)][linux-deb-x64] | [Obtainium][obtainium] | |
| [Portable (x64)][win-portable-x64] | | [deb (ARM64)][linux-deb-arm64] | | |
| [Portable (ARM64)][win-portable-arm64] | | [rpm (x64)][linux-rpm-x64] | | |
| | | [rpm (ARM64)][linux-rpm-arm64] | | |
| | | [AppImage (x64)][linux-appimage-x64] | | |
| | | [AppImage (ARM64)][linux-appimage-arm64] | | |
| | | [tar.gz (x64)][linux-targz-x64] | | |
| | | [tar.gz (ARM64)][linux-targz-arm64] | | |
The macOS disk image runs on both Apple silicon and Intel. Windows and Linux need the build matching your processor.
On Linux, prefer a repository over a single file so Fluxer updates with the rest of your system.
## Linux package repositories
The package is `fluxer` for stable and `fluxer-canary` for canary. apt and dnf subscribe to one channel per entry file. pacman and Flatpak serve both from one repository.
### Flatpak
Stable is on [Flathub][flathub], the easiest route on most desktops:
```sh
flatpak install flathub app.fluxer.Fluxer
```
Flathub has stable only. To use Fluxer's own repository, open [the stable][flatpak-ref] or [the canary][flatpak-canary-ref] reference file and your software manager takes over. Some desktops also accept `flatpak+https://pkgs.fluxer.com/flatpak/fluxer.flatpakref` in the address bar.
From a terminal:
```sh
flatpak install https://pkgs.fluxer.com/flatpak/fluxer.flatpakref
```
### Debian and Ubuntu
```sh
sudo install -d -m 0755 /etc/apt/keyrings
sudo curl -fsSL -o /etc/apt/keyrings/fluxer-archive-keyring.gpg https://pkgs.fluxer.com/keys/fluxer-archive-keyring.gpg
sudo curl -fsSL -o /etc/apt/sources.list.d/fluxer.sources https://pkgs.fluxer.com/deb/fluxer.sources
sudo apt update && sudo apt install fluxer
```
For canary, use the canary entry file and package.
```sh
sudo curl -fsSL -o /etc/apt/sources.list.d/fluxer-canary.sources https://pkgs.fluxer.com/deb/fluxer-canary.sources
sudo apt update && sudo apt install fluxer-canary
```
A `.deb` installed from a download only updates once its channel's entry is added.
### Fedora and RHEL
```sh
sudo curl -fsSL -o /etc/yum.repos.d/fluxer.repo https://pkgs.fluxer.com/rpm/fluxer.repo
sudo dnf install fluxer
```
For canary, use the canary entry file and package.
```sh
sudo curl -fsSL -o /etc/yum.repos.d/fluxer-canary.repo https://pkgs.fluxer.com/rpm/fluxer-canary.repo
sudo dnf install fluxer-canary
```
RHEL, Rocky, Alma and CentOS Stream need `sudo dnf install epel-release` first, because their base repositories lack `libXScrnSaver`. Fedora does not.
### Arch Linux
The repository is signed, so pacman needs the key once:
```sh
sudo pacman-key --init
curl -fsSL -o /tmp/fluxer-archive-keyring.asc https://pkgs.fluxer.com/keys/fluxer-archive-keyring.asc
sudo pacman-key --add /tmp/fluxer-archive-keyring.asc
sudo pacman-key --lsign-key 09D01339EE128925F75E675C855C5BDE34D205D2
```
`--lsign-key` is what makes pacman trust it. Then add the repository:
```sh
sudo tee -a /etc/pacman.conf >/dev/null <<'REPO'
[fluxer]
SigLevel = Required TrustedOnly
Server = https://pkgs.fluxer.com/arch/$repo/os/$arch
REPO
sudo pacman -Syu fluxer
```
Write `$repo` and `$arch` literally. Both are pacman variables, not shell ones, hence the quoted heredoc.
Full setup notes, including canary, are in the [Linux repositories documentation][docs-linux].
## Other ways to run it
- [Open Fluxer in a browser](https://web.fluxer.app), no install needed.
- [Host your own instance][docs-selfhost] from this repository.
## Documentation
- [Documentation home][docs]
- [Downloads][docs-downloads]
- [Self-hosting][docs-selfhost]
## License
The source is licensed under the [AGPL-3.0-or-later](./LICENSE) license.
Fluxer branding, icons, default avatars, badge artwork, screenshots and marketing
imagery are copyright Fluxer, all rights reserved, as set out in
[fluxer_static/LICENSE](./fluxer_static/LICENSE). Third-party material keeps its own
terms, listed in
[fluxer_static/THIRD_PARTY_LICENSES.md](./fluxer_static/THIRD_PARTY_LICENSES.md).
Public availability of this repository does not grant trademark, brand, or
endorsement rights.
[win-setup-x64]: https://pkgs.fluxer.com/desktop/stable/win32/x64/latest/setup
[win-setup-arm64]: https://pkgs.fluxer.com/desktop/stable/win32/arm64/latest/setup
[win-portable-x64]: https://pkgs.fluxer.com/desktop/stable/win32/x64/latest/portable
[win-portable-arm64]: https://pkgs.fluxer.com/desktop/stable/win32/arm64/latest/portable
[mac-dmg]: https://pkgs.fluxer.com/desktop/stable/darwin/arm64/latest/dmg
[linux-deb-x64]: https://pkgs.fluxer.com/desktop/stable/linux/x64/latest/deb
[linux-deb-arm64]: https://pkgs.fluxer.com/desktop/stable/linux/arm64/latest/deb
[linux-rpm-x64]: https://pkgs.fluxer.com/desktop/stable/linux/x64/latest/rpm
[linux-rpm-arm64]: https://pkgs.fluxer.com/desktop/stable/linux/arm64/latest/rpm
[linux-appimage-x64]: https://pkgs.fluxer.com/desktop/stable/linux/x64/latest/appimage
[linux-appimage-arm64]: https://pkgs.fluxer.com/desktop/stable/linux/arm64/latest/appimage
[linux-targz-x64]: https://pkgs.fluxer.com/desktop/stable/linux/x64/latest/tar_gz
[linux-targz-arm64]: https://pkgs.fluxer.com/desktop/stable/linux/arm64/latest/tar_gz
[flatpak-ref]: https://pkgs.fluxer.com/flatpak/fluxer.flatpakref
[flatpak-canary-ref]: https://pkgs.fluxer.com/flatpak/fluxer-canary.flatpakref
[flathub]: https://flathub.org/apps/app.fluxer.Fluxer
[android-apk]: https://github.com/fluxerapp/flutter_client/releases
[obtainium]: https://obtainium.imranr.dev/
[ios-testflight]: https://testflight.apple.com/join/PKZR6pK9
[docs]: https://docs.fluxer.app
[docs-downloads]: https://docs.fluxer.app/downloads/overview/
[docs-linux]: https://docs.fluxer.app/downloads/linux-repositories/
[docs-selfhost]: https://docs.fluxer.app/operator/get-started/
+8
View File
@@ -147,6 +147,12 @@ FLUXER_VAPID_PRIVATE_KEY=CHANGE_ME
#FLUXER_PASSKEY_ADDITIONAL_ALLOWED_ORIGINS=https://chat.example.com
#FLUXER_PASSKEY_ADDITIONAL_ALLOWED_ORIGINS=http://chat.example.com:19080
# Notification jobs the push container holds at once, 1 to 1000000.
#FLUXER_PUSH_SERVICE_QUEUE_CAPACITY=10000
# Provider requests the push container sends at once, 1 to 65536.
#FLUXER_PUSH_SERVICE_SEND_CONCURRENCY=256
# Optional media policies, both off by default. See the operator docs.
#
# CORS limits which web origins may read media. A request with no Origin is
@@ -245,6 +251,7 @@ FLUXER_DISCOVERY_ENABLED=true
#FLUXER_GATEWAY_MEMORY_LIMIT=1gb
#FLUXER_GATEWAY_MEMORY_RESERVATION=384mb
#FLUXER_MEDIA_PROXY_MEMORY_LIMIT=512mb
#FLUXER_PUSH_MEMORY_LIMIT=256mb
#FLUXER_STATIC_PROXY_MEMORY_LIMIT=256mb
#FLUXER_APP_PROXY_MEMORY_LIMIT=256mb
#FLUXER_SNOWFLAKES_MEMORY_LIMIT=128mb
@@ -281,6 +288,7 @@ FLUXER_DISCOVERY_ENABLED=true
#FLUXER_POSTGRES_WORK_MEM=8MB
#FLUXER_POSTGRES_MAINTENANCE_WORK_MEM=256MB
#FLUXER_POSTGRES_AUTOVACUUM_WORK_MEM=128MB
#FLUXER_POSTGRES_SHM_SIZE=1gb
# The bundled Valkey holds durable state as well as cache, so it runs with an
# append-only file and with noeviction, which fails an over-limit write instead
+25 -1
View File
@@ -188,7 +188,7 @@ services:
-c autovacuum_vacuum_cost_limit=2000
-c track_io_timing=on
-c shared_preload_libraries=pg_stat_statements
shm_size: 256mb
shm_size: ${FLUXER_POSTGRES_SHM_SIZE:-1gb}
environment:
POSTGRES_DB: fluxer
POSTGRES_USER: fluxer
@@ -482,6 +482,30 @@ services:
seaweedfs-init: {condition: service_completed_successfully}
nats: {condition: service_healthy}
push:
<<: *fluxer-service
image: ${FLUXER_REGISTRY:-ghcr.io/${FLUXER_REGISTRY_OWNER:-fluxerapp}}/fluxer-push:${FLUXER_IMAGE_TAG:-v1}
deploy:
resources:
limits:
memory: ${FLUXER_PUSH_MEMORY_LIMIT:-256mb}
environment:
<<: *fluxer-env
FLUXER_PUSH_SERVICE_HOST: 0.0.0.0
FLUXER_PUSH_SERVICE_PORT: "8126"
FLUXER_PUSH_SERVICE_QUEUE_CAPACITY: "${FLUXER_PUSH_SERVICE_QUEUE_CAPACITY:-}"
FLUXER_PUSH_SERVICE_SEND_CONCURRENCY: "${FLUXER_PUSH_SERVICE_SEND_CONCURRENCY:-}"
healthcheck:
test: ["CMD", "/usr/local/bin/fluxer-push", "healthcheck"]
interval: 10s
timeout: 5s
retries: 30
start_period: 60s
start_interval: 1s
depends_on:
nats: {condition: service_healthy}
api: {condition: service_healthy}
static-proxy:
<<: *fluxer-service
image: ${FLUXER_REGISTRY:-ghcr.io/${FLUXER_REGISTRY_OWNER:-fluxerapp}}/fluxer-static:${FLUXER_IMAGE_TAG:-v1}
+1 -1
View File
@@ -9,7 +9,7 @@ WORKDIR /usr/src/app
RUN apt-get update \
&& apt-get install -y --no-install-recommends ca-certificates nodejs npm pkg-config \
&& npm install -g pnpm@12.4.2 \
&& npm install -g pnpm@11.27.0 \
&& rm -rf /var/lib/apt/lists/*
RUN npm install --no-audit --no-fund @tailwindcss/[email protected] [email protected]
+14 -8
View File
@@ -10524,7 +10524,7 @@
},
"gateway_rollout": {"$ref": "#/components/schemas/GatewayRolloutConfigResponse"},
"voice_noise_suppression": {"$ref": "#/components/schemas/VoiceNoiseSuppressionConfigResponse"},
"screen_share_delivery": {"$ref": "#/components/schemas/ScreenShareDeliveryConfigResponse"},
"push_service_delivery": {"$ref": "#/components/schemas/PushServiceDeliveryConfigResponse"},
"experiment_delivery": {"$ref": "#/components/schemas/ExperimentDeliveryConfigResponse"},
"registration": {
"type": "object",
@@ -10952,7 +10952,7 @@
"sso",
"gateway_rollout",
"voice_noise_suppression",
"screen_share_delivery",
"push_service_delivery",
"experiment_delivery",
"registration",
"self_hosted",
@@ -11087,9 +11087,9 @@
"nullable": true,
"allOf": [{"$ref": "#/components/schemas/VoiceNoiseSuppressionConfigUpdateRequest"}]
},
"screen_share_delivery": {
"push_service_delivery": {
"nullable": true,
"allOf": [{"$ref": "#/components/schemas/ScreenShareDeliveryConfigUpdateRequest"}]
"allOf": [{"$ref": "#/components/schemas/PushServiceDeliveryConfigUpdateRequest"}]
},
"experiment_delivery": {
"nullable": true,
@@ -15184,12 +15184,12 @@
"poll_jitter_percent": {"type": "integer", "minimum": 0, "maximum": 50}
}
},
"ScreenShareDeliveryConfigUpdateRequest": {
"PushServiceDeliveryConfigUpdateRequest": {
"type": "object",
"properties": {
"enabled": {"type": "boolean"},
"rollout_basis_points": {"type": "integer", "minimum": 0, "maximum": 10000},
"rollout_salt": {"type": "string", "minLength": 1, "maxLength": 64},
"rollout_salt": {"type": "string", "minLength": 1, "maxLength": 64, "pattern": "^[\\x20-\\x7e]+$"},
"included_user_ids": {
"maxItems": 1000,
"type": "array",
@@ -15267,13 +15267,19 @@
"required": ["poll_interval_seconds", "poll_jitter_percent"],
"additionalProperties": false
},
"ScreenShareDeliveryConfigResponse": {
"PushServiceDeliveryConfigResponse": {
"type": "object",
"properties": {
"enabled": {"default": false, "type": "boolean"},
"config_version": {"default": 0, "type": "integer", "minimum": 0, "maximum": 9007199254740991},
"rollout_basis_points": {"default": 0, "type": "integer", "minimum": 0, "maximum": 10000},
"rollout_salt": {"default": "screen-share-delivery-v1", "type": "string", "minLength": 1, "maxLength": 64},
"rollout_salt": {
"default": "push-service-delivery-v1",
"type": "string",
"minLength": 1,
"maxLength": 64,
"pattern": "^[\\x20-\\x7e]+$"
},
"included_user_ids": {
"default": [],
"maxItems": 1000,
+7 -43
View File
@@ -23,7 +23,7 @@ pub struct InstanceConfigResponse {
#[serde(default)]
pub voice_noise_suppression: VoiceNoiseSuppressionConfigResponse,
#[serde(default)]
pub screen_share_delivery: ScreenShareDeliveryConfigResponse,
pub push_service_delivery: PushServiceDeliveryConfigResponse,
#[serde(default)]
pub experiment_delivery: ExperimentDeliveryConfigResponse,
}
@@ -449,7 +449,7 @@ impl VoiceE2eeScope {
}
pub const EXPERIMENT_MAX_TARGETED_USERS: usize = 1_000;
pub const SCREEN_SHARE_DELIVERY_DEFAULT_SALT: &str = "screen-share-delivery-v1";
pub const PUSH_SERVICE_DELIVERY_DEFAULT_SALT: &str = "push-service-delivery-v1";
pub const VOICE_NS_MAX_GUILD_OVERRIDES: usize = 200;
impl NoiseSuppressionBackend {
@@ -542,7 +542,7 @@ pub struct VoiceNoiseSuppressionConfigUpdateRequest {
#[derive(Clone, Debug, Deserialize, Serialize)]
#[serde(default)]
pub struct ScreenShareDeliveryConfigResponse {
pub struct PushServiceDeliveryConfigResponse {
pub enabled: bool,
pub config_version: u64,
pub rollout_basis_points: u32,
@@ -551,13 +551,13 @@ pub struct ScreenShareDeliveryConfigResponse {
pub excluded_user_ids: Vec<String>,
}
impl Default for ScreenShareDeliveryConfigResponse {
impl Default for PushServiceDeliveryConfigResponse {
fn default() -> Self {
Self {
enabled: false,
config_version: 0,
rollout_basis_points: 0,
rollout_salt: SCREEN_SHARE_DELIVERY_DEFAULT_SALT.to_owned(),
rollout_salt: PUSH_SERVICE_DELIVERY_DEFAULT_SALT.to_owned(),
included_user_ids: Vec::new(),
excluded_user_ids: Vec::new(),
}
@@ -565,7 +565,7 @@ impl Default for ScreenShareDeliveryConfigResponse {
}
#[derive(Clone, Debug, Default, Serialize)]
pub struct ScreenShareDeliveryConfigUpdateRequest {
pub struct PushServiceDeliveryConfigUpdateRequest {
#[serde(skip_serializing_if = "Option::is_none")]
pub enabled: Option<bool>,
#[serde(skip_serializing_if = "Option::is_none")]
@@ -694,7 +694,7 @@ pub struct InstanceConfigUpdateRequest {
#[serde(skip_serializing_if = "Option::is_none")]
pub voice_noise_suppression: Option<VoiceNoiseSuppressionConfigUpdateRequest>,
#[serde(skip_serializing_if = "Option::is_none")]
pub screen_share_delivery: Option<ScreenShareDeliveryConfigUpdateRequest>,
pub push_service_delivery: Option<PushServiceDeliveryConfigUpdateRequest>,
#[serde(skip_serializing_if = "Option::is_none")]
pub experiment_delivery: Option<ExperimentDeliveryConfigUpdateRequest>,
}
@@ -1033,30 +1033,18 @@ mod tests {
.expect("admin schema");
let noise = serde_json::from_value::<VoiceNoiseSuppressionConfigResponse>(json!({}))
.expect("default noise config");
let screen_share = serde_json::from_value::<ScreenShareDeliveryConfigResponse>(json!({}))
.expect("default screen share config");
let delivery = serde_json::from_value::<ExperimentDeliveryConfigResponse>(json!({}))
.expect("default delivery config");
let noise = serde_json::to_value(noise).expect("serializable noise config");
let screen_share =
serde_json::to_value(screen_share).expect("serializable screen share config");
let delivery = serde_json::to_value(delivery).expect("serializable delivery config");
let generated_noise: generated_types::VoiceNoiseSuppressionConfigResponse =
serde_json::from_value(noise.clone()).expect("generated noise config contract");
let generated_screen_share: generated_types::ScreenShareDeliveryConfigResponse =
serde_json::from_value(screen_share.clone())
.expect("generated screen share config contract");
let generated_delivery: generated_types::ExperimentDeliveryConfigResponse =
serde_json::from_value(delivery.clone()).expect("generated delivery config contract");
assert_eq!(
serde_json::to_value(generated_noise).expect("serializable generated noise config"),
noise
);
assert_eq!(
serde_json::to_value(generated_screen_share)
.expect("serializable generated screen share config"),
screen_share
);
assert_eq!(
serde_json::to_value(generated_delivery)
.expect("serializable generated delivery config"),
@@ -1064,7 +1052,6 @@ mod tests {
);
for (name, value) in [
("VoiceNoiseSuppressionConfigResponse", noise),
("ScreenShareDeliveryConfigResponse", screen_share),
("ExperimentDeliveryConfigResponse", delivery),
] {
for (field, value) in value.as_object().expect("config object") {
@@ -1076,29 +1063,6 @@ mod tests {
}
}
#[test]
fn screen_share_delivery_update_preserves_empty_lists_and_omitted_fields() {
let update = ScreenShareDeliveryConfigUpdateRequest {
included_user_ids: Some(Vec::new()),
excluded_user_ids: Some(Vec::new()),
..Default::default()
};
let value = serde_json::to_value(update).expect("serializable update");
serde_json::from_value::<generated_types::ScreenShareDeliveryConfigUpdateRequest>(
value.clone(),
)
.expect("generated update contract");
assert_eq!(
value,
json!({"included_user_ids": [], "excluded_user_ids": []})
);
assert_eq!(
serde_json::to_value(ScreenShareDeliveryConfigUpdateRequest::default())
.expect("serializable update"),
json!({})
);
}
#[test]
fn noise_suppression_update_preserves_empty_lists_and_omitted_fields() {
let update = VoiceNoiseSuppressionConfigUpdateRequest {
+1 -1
View File
@@ -80,7 +80,7 @@ async fn reports_list(
return reports_error_page(
config,
&auth.0,
"That page is out of range. The reports search returns at most the first 10000 reports, so narrow the filters and start again.",
"That page is out of range. The reports search returns at most the first 10000 reports. Narrow the filters and start again.",
);
}
let search_query = query.q.as_deref().and_then(clean_string);
+25 -87
View File
@@ -18,7 +18,7 @@ use crate::{
InstancePolicyUpdateRequest, InstanceRegistrationConfigUpdateRequest,
InstanceServicesUpdateRequest, InstanceYoutubeIntegrationUpdateRequest,
LimitConfigUpdateRequest, LimitRule, LimitRuleFilters, NoiseSuppressionBackend,
PremiumMode, RegistrationMode, ScreenShareDeliveryConfigUpdateRequest,
PremiumMode, PushServiceDeliveryConfigUpdateRequest, RegistrationMode,
SsoConfigUpdateRequest, VOICE_NS_MAX_GUILD_OVERRIDES, VoiceE2eeScope,
VoiceNoiseSuppressionConfigUpdateRequest, VoiceNoiseSuppressionGuildOverride,
},
@@ -207,7 +207,7 @@ pub async fn instance_config_post(
Ok(update) => instance_config_result(client.update_instance_config(&update).await),
Err(message) => FlashData::error(message),
},
"update_screen_share_delivery" => match build_screen_share_delivery_update(&form) {
"update_push_service_delivery" => match build_push_service_delivery_update(&form) {
Ok(update) => instance_config_result(client.update_instance_config(&update).await),
Err(message) => FlashData::error(message),
},
@@ -496,6 +496,21 @@ fn parse_experiment_rollout_salt(
Ok(Some(salt.to_owned()))
}
fn parse_push_service_delivery_rollout_salt(
form: &MultiValueForm,
key: &str,
) -> Result<Option<String>, String> {
let salt = parse_experiment_rollout_salt(form, key)?;
if let Some(value) = salt.as_deref()
&& !value
.bytes()
.all(|byte| byte.is_ascii_graphic() || byte == b' ')
{
return Err("Rollout salt must use printable ASCII".to_owned());
}
Ok(salt)
}
fn is_experiment_snowflake(value: &str) -> bool {
!value.is_empty()
&& value.len() <= EXPERIMENT_MAX_SNOWFLAKE_LENGTH
@@ -633,30 +648,30 @@ fn build_voice_noise_suppression_update(
})
}
fn build_screen_share_delivery_update(
fn build_push_service_delivery_update(
form: &MultiValueForm,
) -> Result<InstanceConfigUpdateRequest, String> {
Ok(InstanceConfigUpdateRequest {
screen_share_delivery: Some(ScreenShareDeliveryConfigUpdateRequest {
enabled: Some(form.bool_value("screen_share_delivery_enabled")),
push_service_delivery: Some(PushServiceDeliveryConfigUpdateRequest {
enabled: Some(form.bool_value("push_service_delivery_enabled")),
rollout_basis_points: parse_form_number(
form,
"screen_share_delivery_rollout_basis_points",
"push_service_delivery_rollout_basis_points",
"Rollout basis points",
0,
EXPERIMENT_ROLLOUT_BASIS_POINTS_MAX,
)?,
rollout_salt: parse_experiment_rollout_salt(
rollout_salt: parse_push_service_delivery_rollout_salt(
form,
"screen_share_delivery_rollout_salt",
"push_service_delivery_rollout_salt",
)?,
included_user_ids: Some(parse_experiment_user_ids(
form.first("screen_share_delivery_included_user_ids")
form.first("push_service_delivery_included_user_ids")
.unwrap_or_default(),
"Included user IDs",
)?),
excluded_user_ids: Some(parse_experiment_user_ids(
form.first("screen_share_delivery_excluded_user_ids")
form.first("push_service_delivery_excluded_user_ids")
.unwrap_or_default(),
"Excluded user IDs",
)?),
@@ -1586,83 +1601,6 @@ mod tests {
}
}
#[test]
fn build_screen_share_delivery_update_reads_the_rollout_fields() {
let form = MultiValueForm::parse(
b"screen_share_delivery_enabled=true&screen_share_delivery_rollout_basis_points=%20250%20&screen_share_delivery_rollout_salt=%20screen-share-delivery-v2%20&screen_share_delivery_included_user_ids=1500000000000000001%0A1500000000000000002&screen_share_delivery_excluded_user_ids=1500000000000000003%2C%201500000000000000004",
);
let update = build_screen_share_delivery_update(&form)
.expect("valid form")
.screen_share_delivery
.expect("screen share delivery update");
assert_eq!(update.enabled, Some(true));
assert_eq!(update.rollout_basis_points, Some(250));
assert_eq!(
update.rollout_salt,
Some("screen-share-delivery-v2".to_owned())
);
assert_eq!(
update.included_user_ids,
Some(vec![
"1500000000000000001".to_owned(),
"1500000000000000002".to_owned()
])
);
assert_eq!(
update.excluded_user_ids,
Some(vec![
"1500000000000000003".to_owned(),
"1500000000000000004".to_owned()
])
);
}
#[test]
fn build_screen_share_delivery_update_leaves_the_feature_inert_when_nothing_is_submitted() {
let form = MultiValueForm::parse(b"_csrf=token");
let request = build_screen_share_delivery_update(&form).expect("valid form");
assert_eq!(
serde_json::to_value(request).expect("serializable update"),
serde_json::json!({"screen_share_delivery": {
"enabled": false,
"included_user_ids": [],
"excluded_user_ids": [],
}})
);
}
#[test]
fn build_screen_share_delivery_update_rejects_invalid_rollout_fields() {
for (form, message) in [
(
"screen_share_delivery_rollout_basis_points=10001",
"Rollout basis points must be a whole number between 0 and 10000",
),
(
"screen_share_delivery_rollout_basis_points=abc",
"Rollout basis points must be a whole number between 0 and 10000",
),
(
"screen_share_delivery_rollout_salt=%20%20",
"Rollout salt must be between 1 and 64 characters",
),
(
"screen_share_delivery_included_user_ids=123%2Cinvalid",
"Included user IDs entry 2 must contain 1 to 20 decimal digits",
),
(
"screen_share_delivery_excluded_user_ids=123%2Cinvalid",
"Excluded user IDs entry 2 must contain 1 to 20 decimal digits",
),
] {
let form = MultiValueForm::parse(form.as_bytes());
assert_eq!(
build_screen_share_delivery_update(&form).expect_err("invalid rollout field"),
message
);
}
}
#[test]
fn build_experiment_delivery_update_leaves_both_fields_unchanged_when_absent() {
let form = MultiValueForm::parse(b"_csrf=token");
@@ -5,10 +5,9 @@ use crate::{
AppPublicConfigResponse, EXPERIMENT_MAX_TARGETED_USERS, ExperimentDeliveryConfigResponse,
GatewayRolloutConfigResponse, InstanceConfigResponse, InstanceIntegrationsResponse,
InstanceMediaResponse, InstancePolicyResponse, InstanceRegistrationResponse,
LimitConfigResponse, NoiseSuppressionBackend, PendingRegistrationResponse,
RegistrationUrlResponse, SCREEN_SHARE_DELIVERY_DEFAULT_SALT,
ScreenShareDeliveryConfigResponse, SsoConfigResponse, VOICE_NS_MAX_GUILD_OVERRIDES,
VoiceNoiseSuppressionConfigResponse,
LimitConfigResponse, NoiseSuppressionBackend, PUSH_SERVICE_DELIVERY_DEFAULT_SALT,
PendingRegistrationResponse, PushServiceDeliveryConfigResponse, RegistrationUrlResponse,
SsoConfigResponse, VOICE_NS_MAX_GUILD_OVERRIDES, VoiceNoiseSuppressionConfigResponse,
},
config::AdminConfig,
middleware::auth::AuthContext,
@@ -149,7 +148,7 @@ pub fn instance_config_page(
html! {
(gateway_rollout_section(base, csrf_token, &instance_config.gateway_rollout))
(voice_noise_suppression_section(base, csrf_token, &instance_config.voice_noise_suppression))
(screen_share_delivery_section(base, csrf_token, &instance_config.screen_share_delivery))
(push_service_delivery_section(base, csrf_token, &instance_config.push_service_delivery))
(experiment_delivery_section(base, csrf_token, &instance_config.experiment_delivery))
@if let Some(limit_config) = limit_config {
(limit_config_section(base, limit_config))
@@ -1112,7 +1111,7 @@ fn voice_noise_suppression_section(
p class="text-xs text-neutral-500" {
"One snowflake per line, or comma separated. These users are targeted \
regardless of the percentage above. IDs must contain 1 to 20 decimal \
digits. Invalid entries prevent the save; blank entries and duplicate \
digits. Invalid entries prevent the save. Blank entries and duplicate \
IDs are ignored."
}
}
@@ -1131,7 +1130,7 @@ fn voice_noise_suppression_section(
))
p class="text-xs text-neutral-500" {
"Same format. Exclusion wins over both the always-on list and the \
percentage, so this is the per-user kill switch."
percentage. This is the per-user kill switch."
}
}
@@ -1178,62 +1177,60 @@ fn voice_noise_suppression_section(
)
}
fn screen_share_delivery_section(
fn push_service_delivery_section(
base: &str,
csrf_token: &str,
screen_share_delivery: &ScreenShareDeliveryConfigResponse,
push_service_delivery: &PushServiceDeliveryConfigResponse,
) -> Markup {
let status = if screen_share_delivery.enabled {
let status = if push_service_delivery.enabled {
("Live", BadgeVariant::Success)
} else {
("Inert", BadgeVariant::Default)
};
let included_user_ids = screen_share_delivery.included_user_ids.join("\n");
let excluded_user_ids = screen_share_delivery.excluded_user_ids.join("\n");
let included_user_ids = push_service_delivery.included_user_ids.join("\n");
let excluded_user_ids = push_service_delivery.excluded_user_ids.join("\n");
section_card_with_description(
"Screen Share Delivery",
"Pick how many clients publish screen shares through the reworked delivery path. While \
the master switch below is off nothing on this form reaches any client: every user \
keeps the screen share pipeline they have today, whatever the rest of these fields say. \
A client that is already sharing keeps the path it started on until the share ends.",
"Push Service Delivery",
"Routes push notification delivery for the selected accounts through the push service. \
Accounts the rollout does not select keep the current path.",
html! {
form method="post" action={(base) "/instance-config?action=update_screen_share_delivery"} {
form method="post" action={(base) "/instance-config?action=update_push_service_delivery"} {
(csrf_input(csrf_token))
div class="space-y-6" {
div class="flex flex-wrap items-center gap-2" {
h3 class="text-sm font-semibold text-neutral-900" { "Master switch" }
(badge(status.0, status.1))
span class="text-xs text-neutral-500" {
"Config version " (screen_share_delivery.config_version)
"Config version " (push_service_delivery.config_version)
}
}
(checkbox(
"screen_share_delivery_enabled",
"push_service_delivery_enabled",
"true",
"Serve screen share delivery assignments to clients",
screen_share_delivery.enabled,
"Hand push notifications to the push service",
push_service_delivery.enabled,
true,
))
p class="text-xs text-neutral-500" {
"Off is the safe state. With this unchecked every client is told the \
feature is inert and keeps its current behavior, so the rollout and \
targeting fields below have no effect at all."
"Off is the safe state. With this unchecked every notification keeps the \
current delivery path, so the rollout and targeting fields below have no \
effect at all."
}
h3 class="text-sm font-semibold text-neutral-900" { "Rollout" }
(number_field(
"screen_share_delivery_rollout_basis_points",
"push_service_delivery_rollout_basis_points",
"Rollout (basis points)",
&screen_share_delivery.rollout_basis_points.to_string(),
&push_service_delivery.rollout_basis_points.to_string(),
Some(0), Some(10000), "1",
Some("Share of users bucketed into the canary, in basis points: 0 is nobody, 100 is 1%, 10000 is everybody."),
))
div class="flex flex-col gap-2" {
(text_input(
"screen_share_delivery_rollout_salt",
"push_service_delivery_rollout_salt",
"Rollout Salt",
&screen_share_delivery.rollout_salt,
SCREEN_SHARE_DELIVERY_DEFAULT_SALT,
&push_service_delivery.rollout_salt,
PUSH_SERVICE_DELIVERY_DEFAULT_SALT,
))
p class="text-xs text-neutral-500" {
"Seeds the bucketing hash. Changing it reshuffles which users fall \
@@ -1243,7 +1240,7 @@ fn screen_share_delivery_section(
}
div class="flex flex-col gap-2" {
(textarea_input(
"screen_share_delivery_included_user_ids",
"push_service_delivery_included_user_ids",
"Always-on User IDs",
"1500000000000000001\n1500000000000000002",
&included_user_ids,
@@ -1251,19 +1248,19 @@ fn screen_share_delivery_section(
false,
))
(entry_count_hint(
screen_share_delivery.included_user_ids.len(),
push_service_delivery.included_user_ids.len(),
EXPERIMENT_MAX_TARGETED_USERS,
))
p class="text-xs text-neutral-500" {
"One snowflake per line, or comma separated. These users are targeted \
regardless of the percentage above. IDs must contain 1 to 20 decimal \
digits. Invalid entries prevent the save; blank entries and duplicate \
digits. Invalid entries prevent the save. Blank entries and duplicate \
IDs are ignored."
}
}
div class="flex flex-col gap-2" {
(textarea_input(
"screen_share_delivery_excluded_user_ids",
"push_service_delivery_excluded_user_ids",
"Never-on User IDs",
"1500000000000000003\n1500000000000000004",
&excluded_user_ids,
@@ -1271,17 +1268,17 @@ fn screen_share_delivery_section(
false,
))
(entry_count_hint(
screen_share_delivery.excluded_user_ids.len(),
push_service_delivery.excluded_user_ids.len(),
EXPERIMENT_MAX_TARGETED_USERS,
))
p class="text-xs text-neutral-500" {
"Same format. Exclusion wins over both the always-on list and the \
percentage, so this is the per-user kill switch."
percentage. This is the per-user kill switch."
}
}
(form_actions(html! {
(submit_button("Save Screen Share Delivery Configuration"))
(submit_button("Save Push Service Delivery Configuration"))
}))
}
}
@@ -1932,25 +1929,6 @@ mod tests {
assert!(!markup.contains("at the cap"));
}
#[test]
fn screen_share_delivery_section_shows_list_counts_and_the_master_switch() {
let screen_share_delivery = ScreenShareDeliveryConfigResponse {
included_user_ids: vec!["1500000000000000001".to_owned()],
excluded_user_ids: vec![
"1500000000000000002".to_owned(),
"1500000000000000003".to_owned(),
],
..ScreenShareDeliveryConfigResponse::default()
};
let markup =
screen_share_delivery_section("/admin", "csrf", &screen_share_delivery).into_string();
assert!(markup.contains("action=update_screen_share_delivery"));
assert!(markup.contains("screen_share_delivery_enabled"));
assert!(markup.contains("1 of 1000 stored"));
assert!(markup.contains("2 of 1000 stored"));
assert!(!markup.contains("at the cap"));
}
#[test]
fn voice_noise_suppression_section_flags_a_list_at_its_cap() {
let voice_noise_suppression = VoiceNoiseSuppressionConfigResponse {
@@ -114,7 +114,10 @@ pub fn users_list_page(
let content = html! {
div class="space-y-6" {
(page_header("Users", None))
div class="rounded-lg bg-white transition-all border border-neutral-200 p-4" {
div class="rounded-lg bg-white transition-all border border-neutral-200 p-3" {
p class="mb-1 text-xs text-neutral-500" {
"For example, type " span class="font-mono" { "*" } " in to search for all users."
}
(search_form(base, params))
}
(results_markup)
+5 -15
View File
@@ -409,13 +409,12 @@ fn deserialize_instance_config_response_with_unknown_keys() {
"future_object_knob": {"nested": true},
"future_list_knob": ["a", "b"]
},
"screen_share_delivery": {
"push_service_delivery": {
"enabled": true,
"config_version": 2,
"rollout_basis_points": 2500,
"rollout_salt": "screen-share-delivery-v1",
"included_user_ids": ["1500000000000000001"],
"future_delivery_knob": 9,
"config_version": 3,
"rollout_basis_points": 5000,
"rollout_salt": "push-service-delivery-v1",
"included_user_ids": ["1500000000000000002"],
"excluded_user_ids": []
},
"experiment_delivery": {"poll_interval_seconds": 300, "poll_jitter_percent": 15},
@@ -547,14 +546,6 @@ fn deserialize_instance_config_response_with_unknown_keys() {
assert_eq!(resp.voice_noise_suppression.rollout_basis_points, 10000);
assert_eq!(*resp.voice_noise_suppression.rollout_salt, "voice-ns-v1");
assert_eq!(resp.voice_noise_suppression.enabled_backends.len(), 3);
assert!(resp.screen_share_delivery.enabled);
assert_eq!(resp.screen_share_delivery.config_version, 2);
assert_eq!(resp.screen_share_delivery.rollout_basis_points, 2500);
assert_eq!(
*resp.screen_share_delivery.rollout_salt,
"screen-share-delivery-v1"
);
assert_eq!(resp.screen_share_delivery.included_user_ids.len(), 1);
assert_eq!(resp.experiment_delivery.poll_interval_seconds, 300);
assert!(resp.policy.single_community_guild_id.is_none());
assert_eq!(resp.policy.services.gif_enabled, Some(true));
@@ -565,7 +556,6 @@ fn deserialize_instance_config_response_with_unknown_keys() {
.replace("\"future_rollout_knob\": 3,", "")
.replace("\"future_presentation_knob\": \"verbose\",", "")
.replace("\"future_knob\": 7,", "")
.replace("\"future_delivery_knob\": 9,", "")
.replace("\"future_object_knob\": {\"nested\": true},", "")
.replace("\"future_list_knob\": [\"a\", \"b\"],", "")
.replace(
+3 -9
View File
@@ -465,7 +465,6 @@ async fn mutating_admin_pages_render_usable_csrf_tokens() {
"/instance-config?action=update_gateway_rollout",
"/instance-config?action=update_sso",
"/instance-config?action=update_voice_noise_suppression",
"/instance-config?action=update_screen_share_delivery",
"/instance-config?action=update_experiment_delivery",
][..],
),
@@ -817,6 +816,9 @@ async fn spawn_mock_api() -> String {
async fn mock_api(method: Method, uri: Uri) -> Response {
let path = uri.path().to_owned();
if method == Method::PATCH && path == "/admin/instance/config" {
return json_response(instance_config());
}
match (method, path.as_str()) {
(Method::GET, "/admin/users/@me") => json_response(json!({ "user": admin_user() })),
(Method::GET, "/admin/api-keys") => json_response(json!([])),
@@ -1197,14 +1199,6 @@ fn instance_config() -> Value {
"guild_overrides": [],
"suppression_strength": 80
},
"screen_share_delivery": {
"enabled": false,
"config_version": 0,
"rollout_basis_points": 0,
"rollout_salt": "screen-share-delivery-v1",
"included_user_ids": [],
"excluded_user_ids": []
},
"experiment_delivery": {
"poll_interval_seconds": 300,
"poll_jitter_percent": 15
+2 -2
View File
@@ -5,7 +5,7 @@ FROM node:26-trixie-slim AS base
WORKDIR /usr/src/app
RUN npm install -g pnpm@12.4.2
RUN npm install -g pnpm@11.27.0
FROM base AS deploy
@@ -57,7 +57,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
libvips42t64 && \
rm -rf /var/lib/apt/lists/*
RUN npm install -g pnpm@12.4.2
RUN npm install -g pnpm@11.27.0
COPY --from=deploy /out ./
COPY --from=deploy /usr/src/app/fluxer_api/dist ./dist
+1 -1
View File
@@ -94,5 +94,5 @@
"typescript": "catalog:ts7",
"vitest": "catalog:"
},
"packageManager": "pnpm@12.4.2"
"packageManager": "pnpm@11.27.0"
}
+13 -6
View File
@@ -11,6 +11,8 @@ interface PostgresIpInfoOptions {
}
const VALUE_SEPARATOR = '\u001f';
export const IPINFO_CACHE_TTL_SECONDS = 14 * 24 * 60 * 60;
export const IPINFO_REQUEST_AUDIT_TTL_SECONDS = 90 * 24 * 60 * 60;
function getClient(options: PostgresIpInfoOptions): IPostgresClient | null {
return options.client ?? options.getClient?.() ?? null;
@@ -34,12 +36,9 @@ async function upsertKvRow(
partitionKey: string,
key: string,
row: Record<string, unknown>,
ttlSeconds?: number,
ttlSeconds: number,
): Promise<void> {
const expiresAt =
ttlSeconds != null && Number.isFinite(ttlSeconds) && ttlSeconds > 0
? new Date(Date.now() + ttlSeconds * 1000)
: null;
const expiresAt = new Date(Date.now() + ttlSeconds * 1000);
await client.query(
`INSERT INTO ${table(client)} (table_name, partition_key, row_key, row_data, expires_at, updated_at)
VALUES ($1, $2, $3, $4::jsonb, $5, now())
@@ -77,7 +76,14 @@ export function createPostgresIpInfoCache(options: PostgresIpInfoOptions): IpInf
try {
const client = getClient(options);
if (!client) return;
await upsertKvRow(client, 'ipinfo_cache', rowKey([key]), rowKey([key]), {cache_key: key, payload}, ttlSeconds);
await upsertKvRow(
client,
'ipinfo_cache',
rowKey([key]),
rowKey([key]),
{cache_key: key, payload},
ttlSeconds != null && Number.isFinite(ttlSeconds) && ttlSeconds > 0 ? ttlSeconds : IPINFO_CACHE_TTL_SECONDS,
);
} catch (error) {
options.onError?.(error, 'ipinfo_cache_set');
}
@@ -124,6 +130,7 @@ export function createPostgresIpInfoRequestAuditLogger(options: PostgresIpInfoOp
is_residential_proxy: event.isResidentialProxy,
metadata_json: serializeMetadata(event.metadata),
},
IPINFO_REQUEST_AUDIT_TTL_SECONDS,
);
} catch (error) {
options.onError?.(error, 'ipinfo_request_audit_record');
+5 -3
View File
@@ -1,7 +1,7 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import assert from 'node:assert/strict';
import type {Pool, PoolClient, QueryResult, QueryResultRow} from 'pg';
import type {Pool, PoolClient, PoolConfig, QueryResult, QueryResultRow} from 'pg';
import pg from 'pg';
const MAX_DIAGNOSTIC_FIELD_LENGTH = 128;
@@ -131,7 +131,7 @@ class PostgresClient implements IPostgresClient {
}
private async openPool(): Promise<void> {
const pool = new pg.Pool({
const poolConfig: PoolConfig & {scramMaxIterations: number} = {
connectionString: this.config.url || undefined,
host: this.config.url ? undefined : (this.config.host ?? '127.0.0.1'),
port: this.config.url ? undefined : (this.config.port ?? 5432),
@@ -140,7 +140,9 @@ class PostgresClient implements IPostgresClient {
password: this.config.url ? undefined : (this.config.password ?? 'fluxer'),
ssl: this.config.ssl ? {rejectUnauthorized: true, ca: normalizePem(this.config.sslCa)} : undefined,
max: this.config.maxConnections ?? 20,
});
scramMaxIterations: 0,
};
const pool = new pg.Pool(poolConfig);
this.observePoolConnections(pool);
try {
const client = await pool.connect();
+4 -2
View File
@@ -272,8 +272,6 @@ export function buildAPIConfigFromMaster(master: MasterConfig): APIConfig {
donationProxyKey,
},
hosts: {
invite: extractHostname(master.endpoints.invite),
gift: extractHostname(master.endpoints.gift),
marketing: extractHostname(master.endpoints.marketing),
unfurlIgnored: master.services.api.unfurl_ignored_hosts,
},
@@ -478,6 +476,10 @@ export function buildAPIConfigFromMaster(master: MasterConfig): APIConfig {
},
abusePolicy: {
inboundPhoneCountryCodes: master.instance.abuse_policy.inbound_phone_country_codes,
phoneFlagging: {
enabled: master.instance.abuse_policy.phone_flagging.enabled,
exemptCountryCodes: master.instance.abuse_policy.phone_flagging.exempt_country_codes,
},
phoneVerification: {
inboundRequiredPrefixes: master.instance.abuse_policy.phone_verification.inbound_required_prefixes,
},
+38
View File
@@ -341,6 +341,7 @@ import {
type UsersPendingDeletionRow,
} from '@app/api/database/types/UserTypes';
import {ATTACHMENT_DECAY_COLUMNS, type AttachmentDecayRow} from '@app/api/types/AttachmentDecayTypes';
import {seconds} from 'itty-time';
export const Users = defineTable<UserRow, 'user_id'>({
name: 'users',
@@ -499,16 +500,19 @@ export const GuildAuditLogs = defineTable<GuildAuditLogRow, 'guild_id' | 'log_id
name: 'guild_audit_logs_v2',
columns: GUILD_AUDIT_LOG_COLUMNS,
primaryKey: ['guild_id', 'log_id'],
defaultTtlSeconds: seconds('45 days'),
});
export const GuildAuditLogsByUser = defineTable<GuildAuditLogRow, 'guild_id' | 'user_id' | 'log_id'>({
name: 'guild_audit_logs_v2_by_user',
columns: GUILD_AUDIT_LOG_COLUMNS,
primaryKey: ['guild_id', 'user_id', 'log_id'],
defaultTtlSeconds: seconds('45 days'),
});
export const GuildAuditLogsByAction = defineTable<GuildAuditLogRow, 'guild_id' | 'action_type' | 'log_id'>({
name: 'guild_audit_logs_v2_by_action',
columns: GUILD_AUDIT_LOG_COLUMNS,
primaryKey: ['guild_id', 'action_type', 'log_id'],
defaultTtlSeconds: seconds('45 days'),
});
export const GuildAuditLogsByUserAction = defineTable<
GuildAuditLogRow,
@@ -517,6 +521,7 @@ export const GuildAuditLogsByUserAction = defineTable<
name: 'guild_audit_logs_v2_by_user_action',
columns: GUILD_AUDIT_LOG_COLUMNS,
primaryKey: ['guild_id', 'user_id', 'action_type', 'log_id'],
defaultTtlSeconds: seconds('45 days'),
});
export const GuildMembershipMetadata = defineTable<GuildMembershipMetadataRow, 'guild_id' | 'user_id'>({
name: 'guild_membership_metadata',
@@ -655,6 +660,7 @@ export const RecentMentions = defineTable<RecentMentionRow, 'user_id' | 'message
name: 'recent_mentions',
columns: RECENT_MENTION_COLUMNS,
primaryKey: ['user_id', 'message_id'],
defaultTtlSeconds: seconds('7 days'),
});
interface RecentMentionsByGuildRow {
@@ -678,6 +684,7 @@ export const RecentMentionsByGuild = defineTable<RecentMentionsByGuildRow, 'user
name: 'recent_mentions_by_guild',
columns: RECENT_MENTIONS_BY_GUILD_COLUMNS,
primaryKey: ['user_id', 'guild_id', 'message_id'],
defaultTtlSeconds: seconds('7 days'),
});
export const SavedMessages = defineTable<SavedMessageRow, 'user_id' | 'message_id'>({
name: 'saved_messages',
@@ -688,6 +695,7 @@ export const PushSubscriptions = defineTable<PushSubscriptionRow, 'user_id' | 's
name: 'push_subscriptions',
columns: PUSH_SUBSCRIPTION_COLUMNS,
primaryKey: ['user_id', 'subscription_id'],
defaultTtlSeconds: seconds('90 days'),
});
export const Payments = defineTable<PaymentRow, 'checkout_session_id'>({
name: 'payments',
@@ -854,11 +862,13 @@ export const EmailVerificationTokens = defineTable<EmailVerificationTokenRow, 't
name: 'email_verification_tokens',
columns: EMAIL_VERIFICATION_TOKEN_COLUMNS,
primaryKey: ['token_', 'user_id'],
defaultTtlSeconds: seconds('24 hours'),
});
export const PasswordResetTokens = defineTable<PasswordResetTokenRow, 'token_' | 'user_id'>({
name: 'password_reset_tokens',
columns: PASSWORD_RESET_TOKEN_COLUMNS,
primaryKey: ['token_', 'user_id'],
defaultTtlSeconds: seconds('24 hours'),
});
export const PasswordResetTokensByUserId = defineTable<
{
@@ -870,16 +880,19 @@ export const PasswordResetTokensByUserId = defineTable<
name: 'password_reset_tokens_by_user_id',
columns: ['user_id', 'token_'],
primaryKey: ['user_id', 'token_'],
defaultTtlSeconds: seconds('24 hours'),
});
export const EmailRevertTokens = defineTable<EmailRevertTokenRow, 'token_' | 'user_id'>({
name: 'email_revert_tokens',
columns: EMAIL_REVERT_TOKEN_COLUMNS,
primaryKey: ['token_', 'user_id'],
defaultTtlSeconds: seconds('48 hours'),
});
export const PhoneTokens = defineTable<PhoneTokenRow, 'token_'>({
name: 'phone_tokens',
columns: PHONE_TOKEN_COLUMNS,
primaryKey: ['token_'],
defaultTtlSeconds: seconds('30 days'),
});
export const AuthSessions = defineTable<AuthSessionRow, 'session_id_hash'>({
name: 'auth_sessions',
@@ -901,11 +914,13 @@ export const AuthSessionTombstones = defineTable<AuthSessionTombstoneRow, 'user_
name: 'auth_session_tombstones',
columns: AUTH_SESSION_TOMBSTONE_COLUMNS,
primaryKey: ['user_id', 'session_id_hash'],
defaultTtlSeconds: seconds('30 days'),
});
export const UserCountryHistory = defineTable<UserCountryHistoryRow, 'user_id' | 'country'>({
name: 'user_country_history',
columns: USER_COUNTRY_HISTORY_COLUMNS,
primaryKey: ['user_id', 'country'],
defaultTtlSeconds: seconds('365 days'),
});
export const MfaBackupCodes = defineTable<MfaBackupCodeRow, 'user_id' | 'code'>({
name: 'mfa_backup_codes',
@@ -932,6 +947,7 @@ export const IpAuthorizationTokens = defineTable<IpAuthorizationTokenRow, 'token
name: 'ip_authorization_tokens',
columns: IP_AUTHORIZATION_TOKEN_COLUMNS,
primaryKey: ['token_', 'user_id'],
defaultTtlSeconds: seconds('30 minutes'),
});
export const AuthorizedIps = defineTable<AuthorizedIpRow, 'user_id' | 'ip'>({
name: 'authorized_ips_v2',
@@ -1057,26 +1073,31 @@ export const OAuth2AuthorizationCodes = defineTable<OAuth2AuthorizationCodeRow,
name: 'oauth2_authorization_codes',
columns: OAUTH2_AUTHORIZATION_CODE_COLUMNS,
primaryKey: ['code'],
defaultTtlSeconds: seconds('10 minutes'),
});
export const OAuth2AccessTokens = defineTable<OAuth2AccessTokenRow, 'token_'>({
name: 'oauth2_access_tokens',
columns: OAUTH2_ACCESS_TOKEN_COLUMNS,
primaryKey: ['token_'],
defaultTtlSeconds: seconds('7 days'),
});
export const OAuth2AccessTokensByUser = defineTable<OAuth2AccessTokenByUserRow, 'user_id' | 'token_'>({
name: 'oauth2_access_tokens_by_user',
columns: OAUTH2_ACCESS_TOKENS_BY_USER_COLUMNS,
primaryKey: ['user_id', 'token_'],
defaultTtlSeconds: seconds('7 days'),
});
export const OAuth2RefreshTokens = defineTable<OAuth2RefreshTokenRow, 'token_'>({
name: 'oauth2_refresh_tokens',
columns: OAUTH2_REFRESH_TOKEN_COLUMNS,
primaryKey: ['token_'],
defaultTtlSeconds: seconds('30 days'),
});
export const OAuth2RefreshTokensByUser = defineTable<OAuth2RefreshTokenByUserRow, 'user_id' | 'token_'>({
name: 'oauth2_refresh_tokens_by_user',
columns: OAUTH2_REFRESH_TOKENS_BY_USER_COLUMNS,
primaryKey: ['user_id', 'token_'],
defaultTtlSeconds: seconds('30 days'),
});
interface WebhooksByChannelRow {
@@ -1117,12 +1138,14 @@ export const JobsById = defineTable<JobByIdRow, 'job_id'>({
name: 'jobs_by_id',
columns: JOB_BY_ID_COLUMNS,
primaryKey: ['job_id'],
defaultTtlSeconds: seconds('90 days'),
});
export const JobsByDayBucket = defineTable<JobByDayBucketRow, 'bucket_day' | 'created_at' | 'job_id'>({
name: 'jobs_by_day_bucket',
columns: JOB_BY_DAY_BUCKET_COLUMNS,
primaryKey: ['bucket_day', 'created_at', 'job_id'],
partitionKey: ['bucket_day'],
defaultTtlSeconds: seconds('90 days'),
});
export const JobsActive = defineTable<JobActiveRow, 'job_id'>({
name: 'jobs_active',
@@ -1133,11 +1156,13 @@ export const AttachmentUploadTracesByKey = defineTable<AttachmentUploadTraceByKe
name: 'attachment_upload_traces_by_key',
columns: ATTACHMENT_UPLOAD_TRACE_BY_KEY_COLUMNS,
primaryKey: ['upload_key'],
defaultTtlSeconds: seconds('30 days'),
});
export const AttachmentUploadTracesByAttachment = defineTable<AttachmentUploadTraceByAttachmentRow, 'attachment_id'>({
name: 'attachment_upload_traces_by_attachment',
columns: ATTACHMENT_UPLOAD_TRACE_BY_ATTACHMENT_COLUMNS,
primaryKey: ['attachment_id'],
defaultTtlSeconds: seconds('30 days'),
});
export const NcmecAttachmentSubmissions = defineTable<NcmecAttachmentSubmissionRow, 'attachment_id'>({
name: 'ncmec_attachment_submissions',
@@ -1154,6 +1179,7 @@ export const RegistrationEventsByIp = defineTable<RegistrationEventByIpRow, 'ip'
columns: REGISTRATION_EVENT_BY_IP_COLUMNS,
primaryKey: ['ip', 'created_at', 'user_id'],
partitionKey: ['ip'],
defaultTtlSeconds: seconds('30 days'),
});
export const RegistrationEventsBySubnet = defineTable<
RegistrationEventBySubnetRow,
@@ -1164,6 +1190,7 @@ export const RegistrationEventsBySubnet = defineTable<
columns: REGISTRATION_EVENT_BY_SUBNET_COLUMNS,
primaryKey: ['subnet', 'created_at', 'user_id'],
partitionKey: ['subnet'],
defaultTtlSeconds: seconds('30 days'),
});
export const RegistrationEventsByEmailDomain = defineTable<
RegistrationEventByEmailDomainRow,
@@ -1174,6 +1201,7 @@ export const RegistrationEventsByEmailDomain = defineTable<
columns: REGISTRATION_EVENT_BY_EMAIL_DOMAIN_COLUMNS,
primaryKey: ['email_domain', 'created_at', 'user_id'],
partitionKey: ['email_domain'],
defaultTtlSeconds: seconds('30 days'),
});
export const RegistrationEventsByPlusAddressBase = defineTable<
RegistrationEventByPlusAddressBaseRow,
@@ -1184,6 +1212,7 @@ export const RegistrationEventsByPlusAddressBase = defineTable<
columns: REGISTRATION_EVENT_BY_PLUS_ADDRESS_BASE_COLUMNS,
primaryKey: ['plus_address_base', 'created_at', 'user_id'],
partitionKey: ['plus_address_base'],
defaultTtlSeconds: seconds('30 days'),
});
export const LatestRiskContextByUser = defineTable<LatestRiskContextByUserRow, 'user_id'>({
name: 'latest_risk_context_by_user',
@@ -1194,6 +1223,7 @@ export const SuspiciousIps = defineTable<SuspiciousIpRow, 'ip'>({
name: 'suspicious_ips',
columns: SUSPICIOUS_IP_COLUMNS,
primaryKey: ['ip'],
defaultTtlSeconds: seconds('180 days'),
});
export const RiskOutcomesByIp = defineTable<RiskOutcomeByIpRow, 'ip' | 'created_at' | 'user_id' | 'outcome_code', 'ip'>(
{
@@ -1201,6 +1231,7 @@ export const RiskOutcomesByIp = defineTable<RiskOutcomeByIpRow, 'ip' | 'created_
columns: RISK_OUTCOME_BY_IP_COLUMNS,
primaryKey: ['ip', 'created_at', 'user_id', 'outcome_code'],
partitionKey: ['ip'],
defaultTtlSeconds: seconds('180 days'),
},
);
export const RiskOutcomesBySubnet = defineTable<
@@ -1212,6 +1243,7 @@ export const RiskOutcomesBySubnet = defineTable<
columns: RISK_OUTCOME_BY_SUBNET_COLUMNS,
primaryKey: ['subnet', 'created_at', 'user_id', 'outcome_code'],
partitionKey: ['subnet'],
defaultTtlSeconds: seconds('180 days'),
});
export const RiskOutcomesByEmailDomain = defineTable<
RiskOutcomeByEmailDomainRow,
@@ -1222,6 +1254,7 @@ export const RiskOutcomesByEmailDomain = defineTable<
columns: RISK_OUTCOME_BY_EMAIL_DOMAIN_COLUMNS,
primaryKey: ['email_domain', 'created_at', 'user_id', 'outcome_code'],
partitionKey: ['email_domain'],
defaultTtlSeconds: seconds('180 days'),
});
export const RiskOutcomesByAsn = defineTable<
RiskOutcomeByAsnRow,
@@ -1232,6 +1265,7 @@ export const RiskOutcomesByAsn = defineTable<
columns: RISK_OUTCOME_BY_ASN_COLUMNS,
primaryKey: ['asn', 'created_at', 'user_id', 'outcome_code'],
partitionKey: ['asn'],
defaultTtlSeconds: seconds('180 days'),
});
export const RiskAssessments = defineTable<RiskAssessmentRow, 'assessment_id'>({
name: 'risk_assessments',
@@ -1248,6 +1282,7 @@ export const InboundSmsChallenges = defineTable<InboundSmsChallengeRow, 'challen
name: 'inbound_sms_challenges',
columns: INBOUND_SMS_CHALLENGE_COLUMNS,
primaryKey: ['challenge_code'],
defaultTtlSeconds: seconds('15 minutes'),
});
export const InboundSmsChallengesByUser = defineTable<
InboundSmsChallengeByUserRow,
@@ -1258,16 +1293,19 @@ export const InboundSmsChallengesByUser = defineTable<
columns: INBOUND_SMS_CHALLENGE_BY_USER_COLUMNS,
primaryKey: ['user_id', 'created_at'],
partitionKey: ['user_id'],
defaultTtlSeconds: seconds('15 minutes'),
});
export const PhoneLookupCache = defineTable<PhoneLookupCacheRow, 'phone'>({
name: 'phone_lookup_cache',
columns: PHONE_LOOKUP_CACHE_COLUMNS,
primaryKey: ['phone'],
defaultTtlSeconds: seconds('7 days'),
});
export const PhoneVerificationAttempts = defineTable<PhoneVerificationAttemptRow, 'attempt_id'>({
name: 'phone_verification_attempts',
columns: PHONE_VERIFICATION_ATTEMPT_COLUMNS,
primaryKey: ['attempt_id'],
defaultTtlSeconds: seconds('90 days'),
});
export const BillingCustomers = defineTable<BillingCustomerRow, 'provider_id'>({
name: 'billing_customers',
+19 -1
View File
@@ -2,7 +2,15 @@
import type {AdminAuditLog, BannedIpEntry, BannedIpKind, IAdminRepository} from '@app/api/admin/IAdminRepository';
import {createUserID} from '@app/api/BrandedTypes';
import {deleteOneOrMany, fetchMany, fetchOne, upsertOne} from '@app/api/database/CassandraQueryExecution';
import {Config} from '@app/api/Config';
import {ContentBlocklistCategory} from '@app/api/constants/ContentModeration';
import {
deleteOneOrMany,
executeConditional,
fetchMany,
fetchOne,
upsertOne,
} from '@app/api/database/CassandraQueryExecution';
import type {
AdminAuditLogRow,
BannedAvatarHashRow,
@@ -282,6 +290,7 @@ export class AdminRepository implements IAdminRepository {
}
async isEmailDomainDisposable(domain: string): Promise<boolean> {
if (!Config.blocklistFeeds.enabled) return false;
const domainLower = domain.toLowerCase();
if (isAccountPolicyContactDomainReputationExempt(domainLower)) return false;
const result = await fetchOne<{
@@ -395,6 +404,15 @@ export class AdminRepository implements IAdminRepository {
await deleteOneOrMany(BannedFileShas.deleteByPk({sha256_hex: sha256Hex.toLowerCase()}));
}
async unbanFeedFileSha(sha256Hex: string): Promise<boolean> {
return executeConditional(
BannedFileShas.conditionalDeleteByPk(
{sha256_hex: sha256Hex.toLowerCase()},
{added_by: null, category: ContentBlocklistCategory.MALWARE_BAZAAR},
),
);
}
async loadAllBannedFileShas(): Promise<Array<BannedFileShaRow>> {
return fetchMany<BannedFileShaRow>(LOAD_ALL_BANNED_FILE_SHAS_QUERY.bind({}));
}
@@ -109,6 +109,8 @@ export abstract class IAdminRepository {
abstract unbanFileSha(sha256Hex: string): Promise<void>;
abstract unbanFeedFileSha(sha256Hex: string): Promise<boolean>;
abstract loadAllBannedFileShas(): Promise<Array<BannedFileShaRow>>;
abstract isAvatarHashBanned(hashShort: string): Promise<boolean>;
@@ -13,7 +13,11 @@ import {deriveSsoRedirectUri, normalizeAndValidateSsoConfig} from '@app/api/inst
import {requireAdminACL} from '@app/api/middleware/AdminMiddleware';
import {RateLimitMiddleware} from '@app/api/middleware/RateLimitMiddleware';
import {OpenAPI} from '@app/api/middleware/ResponseTypeMiddleware';
import {getGatewayRolloutConfigPublisher, getInstanceConfigRepository} from '@app/api/middleware/ServiceSingletons';
import {
getGatewayRolloutConfigPublisher,
getInstanceConfigRepository,
getPushServiceDeliveryConfigPublisher,
} from '@app/api/middleware/ServiceSingletons';
import {RateLimitConfigs} from '@app/api/RateLimitConfig';
import type {HonoApp, HonoEnv} from '@app/api/types/HonoEnv';
import {Validator} from '@app/api/Validator';
@@ -31,7 +35,7 @@ import {
RegistrationUrlIdParam,
} from '@fluxer/schema/src/domains/admin/AdminSchemas';
import {GatewayRolloutConfigSchema} from '@fluxer/schema/src/domains/admin/GatewayRolloutSchemas';
import {ScreenShareDeliveryConfigSchema} from '@fluxer/schema/src/domains/admin/ScreenShareDeliverySchemas';
import {PushServiceDeliveryConfigSchema} from '@fluxer/schema/src/domains/admin/PushServiceDeliverySchemas';
import {VoiceNoiseSuppressionConfigSchema} from '@fluxer/schema/src/domains/admin/VoiceNoiseSuppressionSchemas';
import {UserIdParam} from '@fluxer/schema/src/domains/common/CommonParamSchemas';
import {ExperimentDeliveryConfigSchema} from '@fluxer/schema/src/domains/experiment/ExperimentSchemas';
@@ -60,7 +64,7 @@ async function buildInstanceConfigResponse(): Promise<InstanceConfigResponse> {
ssoConfig,
gatewayRollout,
voiceNoiseSuppression,
screenShareDelivery,
pushServiceDelivery,
experimentDelivery,
registrationConfig,
registrationUrls,
@@ -69,7 +73,7 @@ async function buildInstanceConfigResponse(): Promise<InstanceConfigResponse> {
instanceConfigRepository.getSsoConfig(),
instanceConfigRepository.getGatewayRolloutConfig(),
instanceConfigRepository.getVoiceNoiseSuppressionConfig(),
instanceConfigRepository.getScreenShareDeliveryConfig(),
instanceConfigRepository.getPushServiceDeliveryConfig(),
instanceConfigRepository.getExperimentDeliveryConfig(),
instanceConfigRepository.getRegistrationConfig(),
instanceConfigRepository.getRegistrationUrlsForAdmin(),
@@ -101,7 +105,7 @@ async function buildInstanceConfigResponse(): Promise<InstanceConfigResponse> {
},
gateway_rollout: gatewayRollout,
voice_noise_suppression: voiceNoiseSuppression,
screen_share_delivery: screenShareDelivery,
push_service_delivery: pushServiceDelivery,
experiment_delivery: experimentDelivery,
registration: {
...registrationConfig,
@@ -247,43 +251,42 @@ export function InstanceConfigAdminController(app: HonoApp) {
const shouldGrantSetupCompleterAdmin =
appPublicBeforeUpdate !== null && completesInitialSetup(data, appPublicBeforeUpdate.setup.configured);
if (data.gateway_rollout) {
const currentRollout = await instanceConfigRepository.getGatewayRolloutConfig();
const merged = {...currentRollout, ...data.gateway_rollout};
const validated = GatewayRolloutConfigSchema.parse(merged);
await instanceConfigRepository.setGatewayRolloutConfig(validated);
await getGatewayRolloutConfigPublisher().publish(validated);
const patch = data.gateway_rollout;
const landed = await instanceConfigRepository.updateGatewayRolloutConfig((current) =>
GatewayRolloutConfigSchema.parse({...current, ...patch}),
);
await getGatewayRolloutConfigPublisher().publish(landed);
}
if (data.voice_noise_suppression) {
const patch = omitUndefinedFields(data.voice_noise_suppression);
if (Object.keys(patch).length > 0) {
const currentNoiseSuppression = await instanceConfigRepository.getVoiceNoiseSuppressionConfig();
const validated = VoiceNoiseSuppressionConfigSchema.parse({
...currentNoiseSuppression,
...patch,
config_version: currentNoiseSuppression.config_version + 1,
});
await instanceConfigRepository.setVoiceNoiseSuppressionConfig(validated);
await instanceConfigRepository.updateVoiceNoiseSuppressionConfig((current) =>
VoiceNoiseSuppressionConfigSchema.parse({
...current,
...patch,
config_version: current.config_version + 1,
}),
);
}
}
if (data.screen_share_delivery) {
const patch = omitUndefinedFields(data.screen_share_delivery);
if (data.push_service_delivery) {
const patch = omitUndefinedFields(data.push_service_delivery);
if (Object.keys(patch).length > 0) {
const currentScreenShareDelivery = await instanceConfigRepository.getScreenShareDeliveryConfig();
const validated = ScreenShareDeliveryConfigSchema.parse({
...currentScreenShareDelivery,
...patch,
config_version: currentScreenShareDelivery.config_version + 1,
});
await instanceConfigRepository.setScreenShareDeliveryConfig(validated);
const landed = await instanceConfigRepository.updatePushServiceDeliveryConfig((current) =>
PushServiceDeliveryConfigSchema.parse({
...current,
...patch,
config_version: current.config_version + 1,
}),
);
await getPushServiceDeliveryConfigPublisher().publish(landed);
}
}
if (data.experiment_delivery) {
const currentExperimentDelivery = await instanceConfigRepository.getExperimentDeliveryConfig();
const validated = ExperimentDeliveryConfigSchema.parse({
...currentExperimentDelivery,
...data.experiment_delivery,
});
await instanceConfigRepository.setExperimentDeliveryConfig(validated);
const patch = data.experiment_delivery;
await instanceConfigRepository.updateExperimentDeliveryConfig((current) =>
ExperimentDeliveryConfigSchema.parse({...current, ...patch}),
);
}
if (data.sso) {
const sso = data.sso;
@@ -308,21 +311,22 @@ export function InstanceConfigAdminController(app: HonoApp) {
const validated = await normalizeAndValidateSsoConfig(next, {
testModeEnabled: Config.dev.testModeEnabled,
});
const supplied = <T>(field: keyof typeof sso, value: T): T | undefined =>
readOptionalField(sso, field) === undefined ? undefined : value;
await instanceConfigRepository.setSsoConfig({
enabled: validated.enabled,
enforced: validated.enforced,
displayName: next.displayName,
issuer: validated.issuer,
authorizationUrl: validated.authorizationUrl,
tokenUrl: validated.tokenUrl,
userInfoUrl: validated.userInfoUrl,
jwksUrl: validated.jwksUrl,
clientId: validated.clientId,
enabled: supplied('enabled', validated.enabled),
enforced: supplied('enforced', validated.enforced),
displayName: supplied('display_name', next.displayName),
issuer: supplied('issuer', validated.issuer),
authorizationUrl: supplied('authorization_url', validated.authorizationUrl),
tokenUrl: supplied('token_url', validated.tokenUrl),
userInfoUrl: supplied('userinfo_url', validated.userInfoUrl),
jwksUrl: supplied('jwks_url', validated.jwksUrl),
clientId: supplied('client_id', validated.clientId),
clientSecret: readOptionalField(sso, 'client_secret'),
scope: next.scope,
allowedEmailDomains: validated.allowedEmailDomains,
autoProvision: next.autoProvision,
redirectUri: null,
scope: supplied('scope', next.scope),
allowedEmailDomains: supplied('allowed_domains', validated.allowedEmailDomains),
autoProvision: supplied('auto_provision', next.autoProvision),
});
}
if (data.registration) {
@@ -625,7 +629,6 @@ export function InstanceConfigAdminController(app: HonoApp) {
async (ctx) => {
const userId = ctx.req.valid('param').user_id.toString();
const decision = ctx.req.valid('json').status === 'approved' ? 'approve' : 'reject';
await instanceConfigRepository.getPendingRegistrations();
await updatePendingRegistrationUser(ctx, userId, decision);
await instanceConfigRepository.removePendingRegistration(userId);
return ctx.json(await buildInstanceConfigResponse());
@@ -638,27 +641,47 @@ async function applyInstancePolicyUpdate(
policy: NonNullable<InstanceConfigUpdateRequest['policy']>,
): Promise<void> {
const instanceConfigRepository = getInstanceConfigRepository();
const [current, appPublic] = await Promise.all([
instanceConfigRepository.getInstancePolicyConfig(),
instanceConfigRepository.getAppPublicConfig(),
]);
const appPublic = await instanceConfigRepository.getAppPublicConfig();
const adminUser =
policy.single_community_enabled === true
? await ctx.get('userRepository').findUnique(ctx.get('adminUserId'))
: null;
let enablesSingleCommunity = false;
await instanceConfigRepository.updateInstancePolicyConfig((current) => {
const planned = planInstancePolicyPatch(policy, current, {
setupConfigured: appPublic.setup.configured,
adminUserFound: adminUser !== null,
});
enablesSingleCommunity = planned.enablesSingleCommunity;
return planned.patch;
});
if (enablesSingleCommunity && adminUser) {
await ctx.get('singleCommunityService').ensureStockCommunity({
owner: adminUser,
name: policy.single_community_name?.trim() || appPublic.branding.product_name,
});
}
if (policy.premium_mode !== undefined) {
await ctx.get('limitConfigService').updatePolicyConfig({premium_mode: policy.premium_mode});
}
}
function planInstancePolicyPatch(
policy: NonNullable<InstanceConfigUpdateRequest['policy']>,
current: InstancePolicyConfig,
context: {setupConfigured: boolean; adminUserFound: boolean},
): {patch: Partial<InstancePolicyConfig>; enablesSingleCommunity: boolean} {
const patch: Partial<InstancePolicyConfig> = {};
let enablesSingleCommunity = false;
if (
policy.single_community_enabled !== undefined &&
policy.single_community_enabled !== current.single_community_enabled
) {
if (policy.single_community_enabled) {
if (appPublic.setup.configured && current.single_community_guild_id == null) {
if ((context.setupConfigured && current.single_community_guild_id == null) || !context.adminUserFound) {
throw new InstancePolicyTransitionNotAllowedError();
}
const adminUser = await ctx.get('userRepository').findUnique(ctx.get('adminUserId'));
if (!adminUser) {
throw new InstancePolicyTransitionNotAllowedError();
}
await ctx.get('singleCommunityService').ensureStockCommunity({
owner: adminUser,
name: policy.single_community_name?.trim() || appPublic.branding.product_name,
});
enablesSingleCommunity = true;
} else {
patch.single_community_enabled = false;
}
@@ -679,9 +702,6 @@ async function applyInstancePolicyUpdate(
patch.direct_messages_locked = true;
}
}
if (policy.premium_mode !== undefined) {
patch.premium_mode = policy.premium_mode;
}
if (policy.services) {
if (policy.services.gif_enabled !== undefined) {
patch.gif_enabled = policy.services.gif_enabled ?? null;
@@ -704,11 +724,7 @@ async function applyInstancePolicyUpdate(
patch.deferred_phone_gate_member_threshold = policy.deferred_phone_gate.member_threshold;
}
}
if (patch.premium_mode !== undefined) {
await ctx.get('limitConfigService').updatePolicyConfig(patch);
} else if (Object.keys(patch).length > 0) {
await instanceConfigRepository.setInstancePolicyConfig(patch);
}
return {patch, enablesSingleCommunity};
}
async function updatePendingRegistrationUser(
@@ -11,6 +11,7 @@ import {Logger} from '@app/api/Logger';
import {getGuildSearchService, getUserSearchService} from '@app/api/SearchFactory';
import {FeatureTemporarilyDisabledError} from '@fluxer/errors/src/domains/core/FeatureTemporarilyDisabledError';
import {InputValidationError} from '@fluxer/errors/src/domains/core/InputValidationError';
import type {UserSearchFilters} from '@fluxer/schema/src/contracts/search/SearchDocumentTypes';
import type {WorkerJobPayload} from '@pkgs/worker/src/contracts/WorkerTypes';
interface RefreshSearchIndexJobPayload extends WorkerJobPayload {
@@ -130,16 +131,28 @@ export class AdminSearchService {
throw new FeatureTemporarilyDisabledError();
}
const query = data.query?.trim() || '';
const isBrowseAll = query === '' || query === '*';
const searchFilters: UserSearchFilters = isBrowseAll
? {sortBy: 'createdAt', sortOrder: 'asc'}
: {sortBy: 'relevance'};
const directUserId = /^\d+$/.test(query) ? createUserID(BigInt(query)) : null;
const canResolveDirectUser = directUserId !== null && !isSyntheticUserId(directUserId) && data.offset === 0;
const [searchResult, directUser] = await Promise.all([
userSearchService.search(query, {}, {limit: data.limit, offset: data.offset}),
userSearchService.search(query, searchFilters, {limit: data.limit, offset: data.offset}),
canResolveDirectUser ? userRepository.findUnique(directUserId).catch(() => null) : Promise.resolve(null),
]);
const {hits, total} = searchResult;
const userIds = hits.map((hit) => createUserID(BigInt(hit.id)));
const users = await userRepository.listUsers(userIds);
const response = await Promise.all(users.map((user) => mapUserToAdminResponse(user, cacheService, acls)));
const usersById = new Map(users.map((user) => [user.id.toString(), user]));
const orderedUsers = [];
for (const userId of userIds) {
const user = usersById.get(userId.toString());
if (user) {
orderedUsers.push(user);
}
}
const response = await Promise.all(orderedUsers.map((user) => mapUserToAdminResponse(user, cacheService, acls)));
if (directUser && data.offset === 0) {
const directId = directUser.id.toString();
if (!response.some((u) => u.id === directId)) {
@@ -0,0 +1,107 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import type {AdminAuditLog} from '@app/api/admin/IAdminRepository';
import type {TestAccount} from '@app/api/auth/tests/AuthTestUtils';
import {createTestAccount, setUserACLs} from '@app/api/auth/tests/AuthTestUtils';
import {setCassandraQueryExecutorForTesting} from '@app/api/database/CassandraQueryExecution';
import {PushServiceDeliveryConfigPublisher} from '@app/api/instance/PushServiceDeliveryConfigPublisher';
import {InstanceConfigWriteRaceExecutor} from '@app/api/instance/tests/InstanceConfigWriteRaceExecutor';
import {getAdminRepository} from '@app/api/middleware/ServiceSingletons';
import type {ApiTestHarness} from '@app/api/test/ApiTestHarness';
import {createApiTestHarness} from '@app/api/test/ApiTestHarness';
import {InMemoryCassandraQueryExecutor} from '@app/api/test/InMemoryCassandraQueryExecutor';
import {HTTP_STATUS} from '@app/api/test/TestConstants';
import {createBuilder} from '@app/api/test/TestRequestBuilder';
import {AdminACLs} from '@fluxer/constants/src/AdminACLs';
import {APIErrorCodes} from '@fluxer/constants/src/ApiErrorCodes';
import type {InstanceConfigResponse} from '@fluxer/schema/src/domains/admin/AdminSchemas';
import {
DEFAULT_PUSH_SERVICE_DELIVERY_CONFIG,
type PushServiceDeliveryConfig,
} from '@fluxer/schema/src/domains/admin/PushServiceDeliverySchemas';
import {afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi} from 'vitest';
const PUSH_SERVICE_DELIVERY_CONFIG_KEY = 'push_service_delivery_config';
describe('instance config admin PATCH under concurrent writes', () => {
let harness: ApiTestHarness;
let executor: InstanceConfigWriteRaceExecutor;
beforeAll(async () => {
harness = await createApiTestHarness();
executor = new InstanceConfigWriteRaceExecutor(new InMemoryCassandraQueryExecutor());
setCassandraQueryExecutorForTesting(executor);
});
beforeEach(async () => {
await harness.reset();
});
afterEach(() => {
vi.restoreAllMocks();
});
afterAll(async () => {
await harness.shutdown();
});
const createAdmin = async (): Promise<TestAccount> =>
await setUserACLs(harness, await createTestAccount(harness), [
AdminACLs.AUTHENTICATE,
AdminACLs.INSTANCE_CONFIG_VIEW,
AdminACLs.INSTANCE_CONFIG_UPDATE,
]);
const patchConfig = (admin: TestAccount, body: Record<string, unknown>) =>
createBuilder<InstanceConfigResponse>(harness, admin.token).patch('/admin/instance/config').body(body);
const spyOnPushDeliveryPublishes = () =>
vi.spyOn(PushServiceDeliveryConfigPublisher.prototype, 'publish').mockResolvedValue(undefined);
async function readStoredPushServiceDelivery(): Promise<PushServiceDeliveryConfig> {
const raw = await executor.readDirectly(PUSH_SERVICE_DELIVERY_CONFIG_KEY);
if (raw === null) throw new Error('push service delivery config was never stored');
return JSON.parse(raw) as PushServiceDeliveryConfig;
}
async function listConfigUpdateAudits(): Promise<Array<AdminAuditLog>> {
const logs = await getAdminRepository().listAllAuditLogsPaginated(100000);
return logs.filter((log) => log.action === 'update_instance_config');
}
it('answers with a conflict and neither writes, publishes nor audits once every attempt has lost the race', async () => {
const publish = spyOnPushDeliveryPublishes();
const admin = await createAdmin();
await patchConfig(admin, {push_service_delivery: {enabled: true, rollout_basis_points: 1000}}).execute();
publish.mockClear();
const auditsBefore = await listConfigUpdateAudits();
executor.watch(PUSH_SERVICE_DELIVERY_CONFIG_KEY);
let competingWrites = 0;
executor.competeBeforeEachWrite(async () => {
competingWrites++;
await executor.writeDirectly(
PUSH_SERVICE_DELIVERY_CONFIG_KEY,
JSON.stringify({
...DEFAULT_PUSH_SERVICE_DELIVERY_CONFIG,
enabled: false,
rollout_basis_points: 1000,
config_version: 100 + competingWrites,
}),
);
});
await patchConfig(admin, {push_service_delivery: {rollout_basis_points: 5000}})
.expect(HTTP_STATUS.CONFLICT, APIErrorCodes.CONFLICT)
.execute();
expect(executor.events).not.toContain('write');
expect(await readStoredPushServiceDelivery()).toEqual({
...DEFAULT_PUSH_SERVICE_DELIVERY_CONFIG,
enabled: false,
rollout_basis_points: 1000,
config_version: 100 + competingWrites,
});
expect(publish).not.toHaveBeenCalled();
expect(await listConfigUpdateAudits()).toHaveLength(auditsBefore.length);
});
});
@@ -0,0 +1,94 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import type {TestAccount} from '@app/api/auth/tests/AuthTestUtils';
import {createTestAccount, setUserACLs} from '@app/api/auth/tests/AuthTestUtils';
import {setCassandraQueryExecutorForTesting} from '@app/api/database/CassandraQueryExecution';
import {InstanceConfigWriteRaceExecutor} from '@app/api/instance/tests/InstanceConfigWriteRaceExecutor';
import type {ApiTestHarness} from '@app/api/test/ApiTestHarness';
import {createApiTestHarness} from '@app/api/test/ApiTestHarness';
import {InMemoryCassandraQueryExecutor} from '@app/api/test/InMemoryCassandraQueryExecutor';
import {HTTP_STATUS} from '@app/api/test/TestConstants';
import {createBuilder} from '@app/api/test/TestRequestBuilder';
import {AdminACLs} from '@fluxer/constants/src/AdminACLs';
import {APIErrorCodes} from '@fluxer/constants/src/ApiErrorCodes';
import type {InstanceConfigResponse} from '@fluxer/schema/src/domains/admin/AdminSchemas';
import {afterAll, beforeAll, beforeEach, describe, expect, it} from 'vitest';
const INSTANCE_POLICY_CONFIG_KEY = 'instance_policy_config';
describe('instance config admin PATCH against state another node changed', () => {
let harness: ApiTestHarness;
let executor: InstanceConfigWriteRaceExecutor;
beforeAll(async () => {
harness = await createApiTestHarness();
executor = new InstanceConfigWriteRaceExecutor(new InMemoryCassandraQueryExecutor());
setCassandraQueryExecutorForTesting(executor);
});
beforeEach(async () => {
await harness.reset();
});
afterAll(async () => {
await harness.shutdown();
});
const createAdmin = async (): Promise<TestAccount> =>
await setUserACLs(harness, await createTestAccount(harness), [
AdminACLs.AUTHENTICATE,
AdminACLs.INSTANCE_CONFIG_VIEW,
AdminACLs.INSTANCE_CONFIG_UPDATE,
]);
const patchConfig = (admin: TestAccount, body: Record<string, unknown>) =>
createBuilder<InstanceConfigResponse>(harness, admin.token).patch('/admin/instance/config').body(body);
it('keeps an SSO field another node changed when a patch changes a different one', async () => {
const admin = await createAdmin();
await patchConfig(admin, {sso: {display_name: 'Before', client_id: 'client-before'}}).execute();
await executor.writeDirectly('sso_display_name', 'Changed on another node');
await patchConfig(admin, {sso: {client_id: 'client-after'}}).execute();
expect(await executor.readDirectly('sso_display_name')).toBe('Changed on another node');
expect(await executor.readDirectly('sso_client_id')).toBe('client-after');
});
it('refuses to disable direct messages when their lock lands between the read and the write', async () => {
const admin = await createAdmin();
await patchConfig(admin, {policy: {services: {gif_enabled: true}}}).execute();
executor.watch(INSTANCE_POLICY_CONFIG_KEY);
let competed = false;
executor.competeBeforeEachWrite(async () => {
if (competed) return;
competed = true;
await executor.writeDirectly(
INSTANCE_POLICY_CONFIG_KEY,
JSON.stringify({direct_messages_disabled: false, direct_messages_locked: true, gif_enabled: true}),
);
});
await patchConfig(admin, {policy: {direct_messages_disabled: true}})
.expect(HTTP_STATUS.BAD_REQUEST, APIErrorCodes.INSTANCE_POLICY_TRANSITION_NOT_ALLOWED)
.execute();
const stored = JSON.parse((await executor.readDirectly(INSTANCE_POLICY_CONFIG_KEY)) ?? 'null');
expect(stored).toMatchObject({direct_messages_disabled: false, direct_messages_locked: true, gif_enabled: true});
});
it('applies the DM rule and a premium mode change from one request', async () => {
const admin = await createAdmin();
await patchConfig(admin, {policy: {direct_messages_disabled: true}}).execute();
const updated = await patchConfig(admin, {
policy: {direct_messages_disabled: false, premium_mode: 'mirror'},
}).execute();
expect(updated.policy).toMatchObject({
direct_messages_disabled: false,
direct_messages_locked: true,
premium_mode: 'mirror',
});
});
});
@@ -0,0 +1,95 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import {configureMiddleware} from '@app/api/app/MiddlewarePipeline';
import {Config} from '@app/api/Config';
import {setInjectedWorkerService} from '@app/api/middleware/ServiceRegistry';
import {NoopLogger} from '@app/api/test/mocks/NoopLogger';
import {NoopWorkerService} from '@app/api/test/NoopWorkerService';
import type {HonoEnv} from '@app/api/types/HonoEnv';
import {AppErrorHandler, AppNotFoundHandler} from '@fluxer/errors/src/domains/core/ErrorHandlers';
import {Hono} from 'hono';
import {afterEach, beforeAll, beforeEach, describe, expect, it} from 'vitest';
const CLIENT_IP_HEADER_NAME = 'x-real-ip';
function createProductionApp(): Hono<HonoEnv> {
const routes = new Hono<HonoEnv>({strict: true});
configureMiddleware(routes, {
logger: new NoopLogger(),
nodeEnv: 'production',
corsOrigins: ['https://web.fluxer.app'],
trustClientIpHeader: true,
clientIpHeaderName: CLIENT_IP_HEADER_NAME,
maxInflightRequests: 100,
torExitBlockingEnabled: false,
});
routes.onError(AppErrorHandler);
routes.notFound(AppNotFoundHandler);
routes.post('/internal/rpc', (ctx) => ctx.json({ok: true}));
routes.get('/connections/bluesky/jwks.json', (ctx) => ctx.json({keys: []}));
routes.get('/users/@me', (ctx) => ctx.json({ok: true}));
const app = new Hono<HonoEnv>({strict: true});
app.route('/v1', routes);
app.route('/', routes);
app.onError(AppErrorHandler);
app.notFound(AppNotFoundHandler);
return app;
}
describe('client ip requirements across the production middleware pipeline', () => {
let previousTestModeEnabled: boolean;
let previousTrustClientIpHeader: boolean;
let previousClientIpHeader: string;
beforeAll(() => {
setInjectedWorkerService(new NoopWorkerService());
});
beforeEach(() => {
previousTestModeEnabled = Config.dev.testModeEnabled;
previousTrustClientIpHeader = Config.proxy.trust_client_ip_header;
previousClientIpHeader = Config.proxy.client_ip_header;
Config.dev.testModeEnabled = false;
Config.proxy.trust_client_ip_header = true;
Config.proxy.client_ip_header = CLIENT_IP_HEADER_NAME;
});
afterEach(() => {
Config.dev.testModeEnabled = previousTestModeEnabled;
Config.proxy.trust_client_ip_header = previousTrustClientIpHeader;
Config.proxy.client_ip_header = previousClientIpHeader;
});
it('serves the internal rpc route without a client ip header', async () => {
const app = createProductionApp();
const response = await app.request('http://api:8080/internal/rpc', {
method: 'POST',
headers: {'content-type': 'application/json'},
body: '{}',
});
expect(response.status).toBe(200);
});
it('serves the internal rpc route with a client ip header', async () => {
const app = createProductionApp();
const response = await app.request('http://api:8080/internal/rpc', {
method: 'POST',
headers: {'content-type': 'application/json', [CLIENT_IP_HEADER_NAME]: '203.0.113.10'},
body: '{}',
});
expect(response.status).toBe(200);
});
it('serves an exempt public route without a client ip header', async () => {
const app = createProductionApp();
const response = await app.request('http://api:8080/connections/bluesky/jwks.json');
expect(response.status).toBe(200);
});
it('still rejects a non exempt route without a client ip header', async () => {
const app = createProductionApp();
const response = await app.request('http://api:8080/users/@me');
expect(response.status).toBe(403);
expect(await response.json()).toMatchObject({code: 'FORBIDDEN'});
});
});
+72 -20
View File
@@ -7,12 +7,14 @@ import * as AuthUtility from '@app/api/auth/AuthUtility';
import type {IRegistrationRiskEvaluator} from '@app/api/auth/services/IRegistrationRiskEvaluator';
import {createEmailVerificationToken, createInviteCode, createUserID, type UserID} from '@app/api/BrandedTypes';
import type {APIConfig} from '@app/api/config/APIConfig';
import type {UserRow} from '@app/api/database/types/UserTypes';
import type {IDiscriminatorService} from '@app/api/infrastructure/DiscriminatorService';
import type {KVActivityTracker} from '@app/api/infrastructure/KVActivityTracker';
import {
type InstanceConfigRepository,
type InstanceRegistrationUrl,
REGISTRATION_PENDING_APPROVAL_TRAIT,
type RegistrationUrlClaim,
} from '@app/api/instance/InstanceConfigRepository';
import type {SingleCommunityService} from '@app/api/instance/SingleCommunityService';
import type {InviteService} from '@app/api/invite/InviteService';
@@ -21,7 +23,7 @@ import {profileSubstringBlocklistCache} from '@app/api/middleware/ProfileSubstri
import type {RequestCache} from '@app/api/middleware/RequestCacheMiddleware';
import type {User} from '@app/api/models/User';
import {UserSettings} from '@app/api/models/UserSettings';
import {countryRequiresInboundPhoneVerification} from '@app/api/risk/AbusePolicy';
import {countryRequiresInboundPhoneVerification, stripDisallowedPhoneFlags} from '@app/api/risk/AbusePolicy';
import {
type IAccountPolicyEvaluator,
isAssessmentThresholdAuditEvent,
@@ -135,9 +137,6 @@ export async function register(
}
const now = new Date();
const registrationAccess = await resolveRegistrationAccess(instanceConfigRepository, data.registration_url_code);
if (registrationAccess.pendingApproval) {
await instanceConfigRepository.getPendingRegistrations();
}
const clientIp = requireClientIp(request, {
trustClientIpHeader: config.proxy.trust_client_ip_header,
clientIpHeaderName: config.proxy.client_ip_header,
@@ -228,7 +227,7 @@ export async function register(
const userLocale = parseAcceptLanguage(acceptLanguage);
const passwordHash = data.password ? await AuthPassword.hashPassword(ctx, data.password) : null;
const flags = config.nodeEnv === 'development' ? UserFlags.STAFF : 0n;
let user = await users.create({
const userRow: UserRow = {
user_id: userId,
username,
discriminator,
@@ -287,7 +286,39 @@ export async function register(
mention_flags: null,
last_voice_activity_sharing_change_at: null,
version: 1,
});
};
const registrationUrlUse = await claimRegistrationUrlUse(
instanceConfigRepository,
registrationAccess.registrationUrl,
userId,
);
let user: User;
let createAttempted = false;
try {
if (registrationAccess.pendingApproval) {
await instanceConfigRepository.addPendingRegistration({
user_id: userId.toString(),
username: userRow.username,
discriminator: userRow.discriminator,
global_name: userRow.global_name,
email: rawEmail,
requested_at: now.toISOString(),
registration_url_id: registrationAccess.registrationUrl?.id ?? null,
client_ip: clientIp,
});
}
createAttempted = true;
user = await users.create(userRow);
} catch (error) {
if (!createAttempted) {
await withdrawSignupOfUncreatedAccount(instanceConfigRepository, {
userId,
registrationUrlUse,
pendingApproval: registrationAccess.pendingApproval,
});
}
throw error;
}
await users.upsertSettings(
UserSettings.getDefaultUserSettings({
userId,
@@ -331,7 +362,9 @@ export async function register(
action: riskResult.recommendedAction,
},
});
const combinedFlags = await deferPhoneFlagsUntilCommunityJoin(policyDecision.flagBits);
const combinedFlags = await deferPhoneFlagsUntilCommunityJoin(
await stripDisallowedPhoneFlags(policyDecision.flagBits, async () => countryCode),
);
const createdAt = new Date();
const riskContext = deriveLatestRiskContext({
userId: userId.toString(),
@@ -401,20 +434,7 @@ export async function register(
}
if (rawEmail && emailEnabled) await maybeSendVerificationEmail(ctx, {user, email: rawEmail});
await users.createAuthorizedIp(userId, clientIp);
if (registrationAccess.registrationUrl) {
await instanceConfigRepository.recordRegistrationUrlUse(registrationAccess.registrationUrl.id, user.id.toString());
}
if (registrationAccess.pendingApproval) {
await instanceConfigRepository.addPendingRegistration({
user_id: user.id.toString(),
username: user.username,
discriminator: user.discriminator,
global_name: user.globalName,
email: rawEmail,
requested_at: now.toISOString(),
registration_url_id: registrationAccess.registrationUrl?.id ?? null,
client_ip: clientIp,
});
return {
registration_pending_approval: true,
user_id: user.id.toString(),
@@ -469,6 +489,38 @@ function shouldAttemptBootstrapAdminGrant(
);
}
async function claimRegistrationUrlUse(
instanceConfigRepository: InstanceConfigRepository,
registrationUrl: InstanceRegistrationUrl | null,
userId: UserID,
): Promise<RegistrationUrlClaim | null> {
if (registrationUrl === null) return null;
const use = await instanceConfigRepository.claimRegistrationUrlUse(registrationUrl.id, userId.toString());
if (use === null) {
throw new RegistrationUrlInvalidError();
}
return use;
}
async function withdrawSignupOfUncreatedAccount(
instanceConfigRepository: InstanceConfigRepository,
signup: {userId: UserID; registrationUrlUse: RegistrationUrlClaim | null; pendingApproval: boolean},
): Promise<void> {
try {
if (signup.registrationUrlUse !== null) {
await instanceConfigRepository.releaseRegistrationUrlUse(signup.registrationUrlUse);
}
if (signup.pendingApproval) {
await instanceConfigRepository.removePendingRegistration(signup.userId.toString());
}
} catch (error) {
Logger.warn(
{userId: signup.userId.toString(), registrationUrlId: signup.registrationUrlUse?.registration_url_id, error},
'[AuthRegistration] Failed to withdraw the registration URL use or pending approval of an account that was never created',
);
}
}
async function resolveRegistrationAccess(
instanceConfigRepository: InstanceConfigRepository,
registrationUrlCode: string | null | undefined,
+24 -13
View File
@@ -382,21 +382,8 @@ export class SsoService {
throw new RegistrationClosedError();
}
const pendingApproval = registrationConfig.mode === 'approval';
if (pendingApproval) {
await this.instanceConfigRepository.getPendingRegistrations();
}
const user = await this.provisionUserFromClaims(claims, config, {pendingApproval});
if (pendingApproval) {
await this.instanceConfigRepository.addPendingRegistration({
user_id: user.id.toString(),
username: user.username,
discriminator: user.discriminator,
global_name: user.globalName,
email: user.email,
requested_at: new Date().toISOString(),
registration_url_id: null,
client_ip: null,
});
throw new RegistrationPendingApprovalError();
}
return user;
@@ -537,8 +524,22 @@ export class SsoService {
version: 1,
} as const;
await this.claimSsoIdentity(userId, claims.sub, config);
let createAttempted = false;
let userCreated = false;
try {
if (options?.pendingApproval) {
await this.instanceConfigRepository.addPendingRegistration({
user_id: userId.toString(),
username,
discriminator: discriminatorResult.discriminator,
global_name: globalName,
email: userRow.email,
requested_at: now.toISOString(),
registration_url_id: null,
client_ip: null,
});
}
createAttempted = true;
const user = await users.create(userRow);
userCreated = true;
await users.upsertSettings(
@@ -557,6 +558,16 @@ export class SsoService {
await this.ssoIdentityRepository.releaseIdentity(config.providerId, claims.sub).catch((releaseError) => {
getLogger().error({releaseError}, 'Failed to release SSO identity after user provisioning failed');
});
if (options?.pendingApproval && !createAttempted) {
await this.instanceConfigRepository
.removePendingRegistration(userId.toString())
.catch((removeError: unknown) => {
getLogger().error(
{userId: userId.toString(), removeError},
'Failed to withdraw the pending approval of an SSO user that was never created',
);
});
}
}
throw error;
}
@@ -9,6 +9,7 @@ import {
loginAccount,
registerUser,
} from '@app/api/auth/tests/AuthTestUtils';
import {Config} from '@app/api/Config';
import {setInjectedRegistrationRiskEvaluator} from '@app/api/middleware/ServiceMiddleware';
import {getInstanceConfigRepository} from '@app/api/middleware/ServiceSingletons';
import {
@@ -33,7 +34,7 @@ import {
SuspiciousActivityFlags,
} from '@fluxer/constants/src/UserConstants';
import type {GuildResponse} from '@fluxer/schema/src/domains/guild/GuildResponseSchemas';
import {afterAll, beforeAll, beforeEach, describe, expect, it, vi} from 'vitest';
import {afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi} from 'vitest';
function phoneRiskEvaluator(level: RiskLevelType, riskScore: number): IRegistrationRiskEvaluator {
return {
@@ -241,6 +242,59 @@ describe('Deferred phone verification gate', () => {
expect(flags & SuspiciousActivityFlags.REQUIRE_VERIFIED_PHONE).not.toBe(0);
});
describe('with phone flagging disabled', () => {
const originalPhoneFlagging = {...Config.abusePolicy.phoneFlagging};
afterEach(() => {
Config.abusePolicy.phoneFlagging = originalPhoneFlagging;
});
it('sets no phone requirement and no deferral at registration', async () => {
await getInstanceConfigRepository().setInstancePolicyConfig({deferred_phone_gate_enabled: true});
Config.abusePolicy.phoneFlagging = {enabled: false, exemptCountryCodes: []};
setInjectedRegistrationRiskEvaluator(phoneRiskEvaluator(RiskLevel.High, 70));
const registration = await registerUser(harness, {
email: createUniqueEmail('flagging-off'),
username: createUniqueUsername('flagging_off'),
global_name: 'Flagging Off',
password: 'StrongPassword!123',
date_of_birth: '2000-01-01',
consent: true,
});
const flags = await readFlags(registration.user_id);
expect(flags & SuspiciousActivityFlags.REQUIRE_VERIFIED_PHONE).toBe(0);
expect(flags & DEFERRED_PHONE_ON_COMMUNITY_JOIN).toBe(0);
});
it('keeps an existing deferral dormant on a qualifying join', async () => {
await getInstanceConfigRepository().setInstancePolicyConfig({
deferred_phone_gate_enabled: true,
deferred_phone_gate_member_threshold: 1,
deferred_phone_gate_window_hours: 24,
});
const {inviteCode} = await createGuildWithInvite(harness);
const filler = await createTestAccount(harness);
await createBuilder(harness, filler.token).post(`/invites/${inviteCode}`).expect(200).execute();
setInjectedRegistrationRiskEvaluator(phoneRiskEvaluator(RiskLevel.High, 70));
const registration = await registerUser(harness, {
email: createUniqueEmail('flagging-off-join'),
username: createUniqueUsername('flagging_off_join'),
global_name: 'Flagging Off Join',
password: 'StrongPassword!123',
date_of_birth: '2000-01-01',
consent: true,
});
setInjectedRegistrationRiskEvaluator(undefined);
expect((await readFlags(registration.user_id)) & DEFERRED_PHONE_ON_COMMUNITY_JOIN).not.toBe(0);
Config.abusePolicy.phoneFlagging = {enabled: false, exemptCountryCodes: []};
await createBuilder(harness, registration.token).post(`/invites/${inviteCode}`).expect(200).execute();
const flags = await readFlags(registration.user_id);
expect(flags & DEFERRED_PHONE_ON_COMMUNITY_JOIN).not.toBe(0);
expect(flags & PHONE_GATE_PROMOTED_FROM_DEFERRAL).toBe(0);
});
});
describe('phone gate escape', () => {
async function configurePhoneGate(
overrides: {
@@ -0,0 +1,436 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import {createHash} from 'node:crypto';
import {
createAuthHarness,
createTestAccount,
createUniqueEmail,
createUniqueUsername,
enableSso,
setUserACLs,
type TestAccount,
} from '@app/api/auth/tests/AuthTestUtils';
import {createUserID} from '@app/api/BrandedTypes';
import type {UserRow} from '@app/api/database/types/UserTypes';
import {
InstanceConfigRepository,
REGISTRATION_PENDING_APPROVAL_TRAIT,
} from '@app/api/instance/InstanceConfigRepository';
import {getInstanceConfigRepository} from '@app/api/middleware/ServiceSingletons';
import type {ApiTestHarness} from '@app/api/test/ApiTestHarness';
import {HTTP_STATUS} from '@app/api/test/TestConstants';
import {createBuilder, createBuilderWithoutAuth} from '@app/api/test/TestRequestBuilder';
import {UserRepository} from '@app/api/user/repositories/UserRepository';
import {AdminACLs} from '@fluxer/constants/src/AdminACLs';
import {APIErrorCodes} from '@fluxer/constants/src/ApiErrorCodes';
import type {InstanceConfigResponse} from '@fluxer/schema/src/domains/admin/AdminSchemas';
import {afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi} from 'vitest';
const REGISTRATION_URLS_KEY = 'registration_urls';
const REGISTRATION_PENDING_APPROVALS_KEY = 'registration_pending_approvals';
interface RegistrationResponse {
user_id?: string;
token?: string;
registration_pending_approval?: true;
code?: string;
}
function registrationBody(prefix: string, registrationUrlCode?: string): Record<string, unknown> {
return {
email: createUniqueEmail(prefix),
username: createUniqueUsername(prefix),
global_name: 'Signup Race',
password: 'a-strong-password',
date_of_birth: '2000-01-01',
consent: true,
...(registrationUrlCode === undefined ? {} : {registration_url_code: registrationUrlCode}),
};
}
describe('signups racing on registration URLs and pending approvals', () => {
let harness: ApiTestHarness;
let admin: TestAccount;
beforeAll(async () => {
harness = await createAuthHarness();
});
beforeEach(async () => {
await harness.reset();
admin = await setUserACLs(harness, await createTestAccount(harness), [
AdminACLs.AUTHENTICATE,
AdminACLs.INSTANCE_CONFIG_VIEW,
AdminACLs.INSTANCE_CONFIG_UPDATE,
]);
});
afterEach(() => {
vi.restoreAllMocks();
});
afterAll(async () => {
await harness?.shutdown();
});
const register = (prefix: string, registrationUrlCode?: string) =>
createBuilderWithoutAuth<RegistrationResponse>(harness)
.post('/auth/register')
.body(registrationBody(prefix, registrationUrlCode))
.executeRaw();
const readAdminConfig = (): Promise<InstanceConfigResponse> =>
createBuilder<InstanceConfigResponse>(harness, admin.token).get('/admin/instance/config').execute();
const completeSso = async (prefix: string) => {
const start = await createBuilderWithoutAuth<{state: string}>(harness)
.post('/auth/sso/start')
.body({redirect_to: '/me'})
.execute();
return createBuilderWithoutAuth(harness)
.post('/auth/sso/complete')
.body({code: createUniqueEmail(prefix), state: start.state})
.executeRaw();
};
const failCreateAfterTheUserRowIsWritten = () => {
const create = UserRepository.prototype.create;
vi.spyOn(UserRepository.prototype, 'create').mockImplementationOnce(async function (
this: UserRepository,
row: UserRow,
) {
await create.call(this, row);
throw new Error('the user indexes could not be written after the user row');
});
};
const failAfterThePendingApprovalIsStored = () => {
const addPendingRegistration = InstanceConfigRepository.prototype.addPendingRegistration;
vi.spyOn(InstanceConfigRepository.prototype, 'addPendingRegistration').mockImplementationOnce(async function (
this: InstanceConfigRepository,
entry: Parameters<InstanceConfigRepository['addPendingRegistration']>[0],
) {
await addPendingRegistration.call(this, entry);
throw new Error('the pending approval could not be published');
});
};
const expectOnePendingAccount = async () => {
const pending = (await readAdminConfig()).registration.pending_registrations;
expect(pending).toHaveLength(1);
const account = await new UserRepository().findUnique(createUserID(BigInt(pending[0]!.user_id)));
expect(account?.traits.has(REGISTRATION_PENDING_APPROVAL_TRAIT)).toBe(true);
};
it('never lets concurrent signups through a capped registration URL exceed max_uses', async () => {
const repository = getInstanceConfigRepository();
await repository.setRegistrationConfig({mode: 'closed', admin_registration_urls_enabled: true});
const {code, registrationUrl} = await repository.createRegistrationUrl({
label: 'Capped',
createdByUserId: '1',
expiresAt: null,
maxUses: 2,
approvalRequired: false,
});
const attempts = await Promise.all(Array.from({length: 6}, (_, index) => register(`capped${index}`, code)));
const admitted = attempts.filter((attempt) => attempt.response.status === HTTP_STATUS.OK);
const refused = attempts.filter((attempt) => attempt.response.status !== HTTP_STATUS.OK);
expect(admitted).toHaveLength(2);
for (const attempt of refused) {
expect(attempt.response.status).toBe(HTTP_STATUS.BAD_REQUEST);
expect(attempt.json.code).toBe(APIErrorCodes.REGISTRATION_URL_INVALID);
}
const stored = (await readAdminConfig()).registration.urls.find((url) => url.id === registrationUrl.id);
expect(stored?.use_count).toBe(2);
expect(admitted.map((attempt) => attempt.json.user_id)).toContain(stored?.last_used_by_user_id);
});
it('admits exactly max_uses when 120 signups race through a registration URL capped at 40', async () => {
const repository = getInstanceConfigRepository();
await repository.setRegistrationConfig({mode: 'closed', admin_registration_urls_enabled: true});
const {code, registrationUrl} = await repository.createRegistrationUrl({
label: 'Capped at 40',
createdByUserId: '1',
expiresAt: null,
maxUses: 40,
approvalRequired: false,
});
const registerUntilDecided = async (prefix: string) => {
for (let attempt = 0; attempt < 20; attempt += 1) {
const result = await register(`${prefix}r${attempt}`, code);
if (result.response.status !== HTTP_STATUS.SERVICE_UNAVAILABLE) return result;
}
throw new Error('a signup never reached a decision');
};
const attempts = await Promise.all(Array.from({length: 120}, (_, index) => registerUntilDecided(`surge${index}`)));
const admitted = attempts.filter((attempt) => attempt.response.status === HTTP_STATUS.OK);
expect(admitted).toHaveLength(40);
for (const attempt of attempts.filter((entry) => entry.response.status !== HTTP_STATUS.OK)) {
expect(attempt.response.status).toBe(HTTP_STATUS.BAD_REQUEST);
expect(attempt.json.code).toBe(APIErrorCodes.REGISTRATION_URL_INVALID);
}
const stored = (await readAdminConfig()).registration.urls.find((url) => url.id === registrationUrl.id);
expect(stored?.use_count).toBe(40);
});
it('counts every concurrent signup through an uncapped registration URL', async () => {
const repository = getInstanceConfigRepository();
await repository.setRegistrationConfig({mode: 'closed', admin_registration_urls_enabled: true});
const {code, registrationUrl} = await repository.createRegistrationUrl({
label: 'Uncapped',
createdByUserId: '1',
expiresAt: null,
maxUses: null,
approvalRequired: false,
});
const attempts = await Promise.all(Array.from({length: 5}, (_, index) => register(`uncapped${index}`, code)));
expect(attempts.map((attempt) => attempt.response.status)).toEqual(Array(5).fill(HTTP_STATUS.OK));
const stored = (await readAdminConfig()).registration.urls.find((url) => url.id === registrationUrl.id);
expect(stored?.use_count).toBe(5);
});
it('gives the seat and the pending entry back when the signup failed before the account was created', async () => {
const repository = getInstanceConfigRepository();
await repository.setRegistrationConfig({mode: 'closed', admin_registration_urls_enabled: true});
const {code, registrationUrl} = await repository.createRegistrationUrl({
label: 'Single use',
createdByUserId: '1',
expiresAt: null,
maxUses: 1,
approvalRequired: true,
});
failAfterThePendingApprovalIsStored();
const failed = await register('seatreleased', code);
expect(failed.response.status).toBe(HTTP_STATUS.INTERNAL_SERVER_ERROR);
const withdrawn = await readAdminConfig();
expect(withdrawn.registration.pending_registrations).toEqual([]);
expect(withdrawn.registration.urls.find((url) => url.id === registrationUrl.id)?.use_count).toBe(0);
const retried = await register('seatreleasedretry', code);
expect(retried.response.status).toBe(HTTP_STATUS.OK);
const stored = (await readAdminConfig()).registration.urls.find((url) => url.id === registrationUrl.id);
expect(stored).toMatchObject({use_count: 1, last_used_by_user_id: retried.json.user_id});
});
it('keeps the seat when the account create itself failed, because the row may still have landed', async () => {
const repository = getInstanceConfigRepository();
await repository.setRegistrationConfig({mode: 'closed', admin_registration_urls_enabled: true});
const {code, registrationUrl} = await repository.createRegistrationUrl({
label: 'Single use',
createdByUserId: '1',
expiresAt: null,
maxUses: 1,
approvalRequired: false,
});
vi.spyOn(UserRepository.prototype, 'create').mockRejectedValueOnce(new Error('the user row write failed'));
const failed = await register('seatkeptoncreate', code);
expect(failed.response.status).toBe(HTTP_STATUS.INTERNAL_SERVER_ERROR);
const second = await register('seatkeptcreate2', code);
expect(second.response.status).toBe(HTTP_STATUS.BAD_REQUEST);
expect(second.json.code).toBe(APIErrorCodes.REGISTRATION_URL_INVALID);
const stored = (await readAdminConfig()).registration.urls.find((url) => url.id === registrationUrl.id);
expect(stored?.use_count).toBe(1);
});
it('keeps the seat of an account whose row was written before its creation failed', async () => {
const repository = getInstanceConfigRepository();
await repository.setRegistrationConfig({mode: 'closed', admin_registration_urls_enabled: true});
const {code, registrationUrl} = await repository.createRegistrationUrl({
label: 'Single use',
createdByUserId: '1',
expiresAt: null,
maxUses: 1,
approvalRequired: false,
});
failCreateAfterTheUserRowIsWritten();
const failed = await register('seatkept', code);
expect(failed.response.status).toBe(HTTP_STATUS.INTERNAL_SERVER_ERROR);
const second = await register('seatkeptsecond', code);
expect(second.response.status).toBe(HTTP_STATUS.BAD_REQUEST);
expect(second.json.code).toBe(APIErrorCodes.REGISTRATION_URL_INVALID);
const stored = (await readAdminConfig()).registration.urls.find((url) => url.id === registrationUrl.id);
expect(stored?.use_count).toBe(1);
});
it('honours the use count and cap already stored on a registration URL', async () => {
const repository = getInstanceConfigRepository();
await repository.setRegistrationConfig({mode: 'closed', admin_registration_urls_enabled: true});
const id = 'b3c4f0b2-8a6e-4c41-9f55-3f0c2a7d1e90';
await repository.setConfig(
REGISTRATION_URLS_KEY,
JSON.stringify([
{
id,
label: 'Issued earlier',
code_hash: createHash('sha256').update(id).digest('hex'),
created_by_user_id: '1400000000000000001',
created_at: '2026-09-01T00:00:00.000Z',
expires_at: null,
max_uses: 2,
use_count: 1,
revoked_at: null,
approval_required: false,
last_used_at: '2026-09-02T00:00:00.000Z',
last_used_by_user_id: '1400000000000000002',
},
]),
);
const before = (await readAdminConfig()).registration.urls.find((url) => url.id === id);
expect(before).toMatchObject({use_count: 1, max_uses: 2, last_used_by_user_id: '1400000000000000002'});
const first = await register('storedinvite', id);
expect(first.response.status).toBe(HTTP_STATUS.OK);
const second = await register('storedinviteagain', id);
expect(second.response.status).toBe(HTTP_STATUS.BAD_REQUEST);
expect(second.json.code).toBe(APIErrorCodes.REGISTRATION_URL_INVALID);
const after = (await readAdminConfig()).registration.urls.find((url) => url.id === id);
expect(after).toMatchObject({use_count: 2, max_uses: 2, last_used_by_user_id: first.json.user_id});
});
it('keeps every pending approval when approval-mode signups race', async () => {
await getInstanceConfigRepository().setRegistrationConfig({mode: 'approval'});
const attempts = await Promise.all(Array.from({length: 5}, (_, index) => register(`pending${index}`)));
expect(attempts.map((attempt) => attempt.json.registration_pending_approval)).toEqual(Array(5).fill(true));
const pending = (await readAdminConfig()).registration.pending_registrations.map((entry) => entry.user_id);
expect(pending.toSorted()).toEqual(attempts.map((attempt) => attempt.json.user_id).toSorted());
});
it('lists an approval-mode account whose signup failed after the account was created', async () => {
await getInstanceConfigRepository().setRegistrationConfig({mode: 'approval'});
vi.spyOn(UserRepository.prototype, 'createAuthorizedIp').mockRejectedValueOnce(
new Error('the authorized IP write failed'),
);
const failed = await register('pendingstranded');
expect(failed.response.status).toBe(HTTP_STATUS.INTERNAL_SERVER_ERROR);
await expectOnePendingAccount();
});
it('lists an approval-mode account whose row was written before its creation failed', async () => {
await getInstanceConfigRepository().setRegistrationConfig({mode: 'approval'});
failCreateAfterTheUserRowIsWritten();
const failed = await register('pendingrowwritten');
expect(failed.response.status).toBe(HTTP_STATUS.INTERNAL_SERVER_ERROR);
await expectOnePendingAccount();
});
it('keeps the pending approval of an approval-mode signup whose account create failed', async () => {
await getInstanceConfigRepository().setRegistrationConfig({mode: 'approval'});
vi.spyOn(UserRepository.prototype, 'create').mockRejectedValueOnce(new Error('the user row write failed'));
const failed = await register('pendingkept');
expect(failed.response.status).toBe(HTTP_STATUS.INTERNAL_SERVER_ERROR);
expect((await readAdminConfig()).registration.pending_registrations).toHaveLength(1);
});
it('lists no pending approval for an approval-mode signup that failed before the account was created', async () => {
await getInstanceConfigRepository().setRegistrationConfig({mode: 'approval'});
failAfterThePendingApprovalIsStored();
const failed = await register('pendingnever');
expect(failed.response.status).toBe(HTTP_STATUS.INTERNAL_SERVER_ERROR);
expect((await readAdminConfig()).registration.pending_registrations).toEqual([]);
});
it('lists an SSO account provisioned in approval mode whose provisioning failed after the account was created', async () => {
await enableSso(harness, admin.token, {enforced: false});
await getInstanceConfigRepository().setRegistrationConfig({mode: 'approval'});
vi.spyOn(UserRepository.prototype, 'upsertSettings').mockRejectedValueOnce(new Error('the settings write failed'));
const failed = await completeSso('ssopendingstranded');
expect(failed.response.status).toBe(HTTP_STATUS.INTERNAL_SERVER_ERROR);
await expectOnePendingAccount();
});
it('lists an SSO account provisioned in approval mode whose row was written before its creation failed', async () => {
await enableSso(harness, admin.token, {enforced: false});
await getInstanceConfigRepository().setRegistrationConfig({mode: 'approval'});
failCreateAfterTheUserRowIsWritten();
const failed = await completeSso('ssopendingrowwritten');
expect(failed.response.status).toBe(HTTP_STATUS.INTERNAL_SERVER_ERROR);
await expectOnePendingAccount();
});
it('keeps the pending approval of an SSO signup in approval mode whose account create failed', async () => {
await enableSso(harness, admin.token, {enforced: false});
await getInstanceConfigRepository().setRegistrationConfig({mode: 'approval'});
vi.spyOn(UserRepository.prototype, 'create').mockRejectedValueOnce(new Error('the user row write failed'));
const failed = await completeSso('ssopendingkept');
expect(failed.response.status).toBe(HTTP_STATUS.INTERNAL_SERVER_ERROR);
expect((await readAdminConfig()).registration.pending_registrations).toHaveLength(1);
});
it('lists no pending approval for an SSO signup in approval mode that failed before the account was created', async () => {
await enableSso(harness, admin.token, {enforced: false});
await getInstanceConfigRepository().setRegistrationConfig({mode: 'approval'});
failAfterThePendingApprovalIsStored();
const failed = await completeSso('ssopendingnever');
expect(failed.response.status).toBe(HTTP_STATUS.INTERNAL_SERVER_ERROR);
expect((await readAdminConfig()).registration.pending_registrations).toEqual([]);
});
it('keeps a stored pending approval listed until an admin decides it', async () => {
const account = await createTestAccount(harness);
await getInstanceConfigRepository().setConfig(
REGISTRATION_PENDING_APPROVALS_KEY,
JSON.stringify([
{
user_id: account.userId,
username: 'stored_pending',
discriminator: 1,
global_name: null,
email: account.email,
requested_at: '2026-09-01T00:00:00.000Z',
registration_url_id: null,
client_ip: '127.0.0.1',
},
]),
);
await getInstanceConfigRepository().setRegistrationConfig({mode: 'approval'});
const fresh = await register('pendingafter');
const listed = (await readAdminConfig()).registration.pending_registrations.map((entry) => entry.user_id);
expect(listed.toSorted()).toEqual([account.userId, fresh.json.user_id].toSorted());
const decided = await createBuilder<InstanceConfigResponse>(harness, admin.token)
.patch(`/admin/instance/pending-registrations/${account.userId}`)
.body({status: 'approved'})
.expect(HTTP_STATUS.OK)
.execute();
expect(decided.registration.pending_registrations.map((entry) => entry.user_id)).toEqual([fresh.json.user_id]);
expect(
JSON.parse(
(await getInstanceConfigRepository().getConfig(REGISTRATION_PENDING_APPROVALS_KEY)) ?? 'null',
) as Array<{user_id: string}>,
).toEqual([expect.objectContaining({user_id: fresh.json.user_id})]);
});
});
+1 -1
View File
@@ -150,7 +150,7 @@ function serializeGroupDMChannel(channel: Channel): ChannelResponse {
return {
...serializeBaseChannelFields(channel),
...serializeMessageableFields(channel),
name: channel.name ?? undefined,
name: channel.name ?? null,
icon: channel.iconHash ?? null,
owner_id: channel.ownerId ? channel.ownerId.toString() : null,
nicks: nicknameMap.size > 0 ? nicks : undefined,
@@ -0,0 +1,210 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import {createChannelID, createUserID, type UserID} from '@app/api/BrandedTypes';
import type {IChannelRepository} from '@app/api/channel/IChannelRepository';
import {CallService} from '@app/api/channel/services/CallService';
import type {IGuildRepositoryAggregate} from '@app/api/guild/repositories/IGuildRepositoryAggregate';
import type {CallCaller, CallData, IGatewayService} from '@app/api/infrastructure/IGatewayService';
import type {ISnowflakeService} from '@app/api/infrastructure/ISnowflakeService';
import type {IVoiceRoomStore} from '@app/api/infrastructure/IVoiceRoomStore';
import type {UserCacheService} from '@app/api/infrastructure/UserCacheService';
import type {RequestCache} from '@app/api/middleware/RequestCacheMiddleware';
import type {Channel} from '@app/api/models/Channel';
import type {User} from '@app/api/models/User';
import type {ReadStateService} from '@app/api/read_state/ReadStateService';
import type {IUserRepository} from '@app/api/user/IUserRepository';
import {ChannelTypes} from '@fluxer/constants/src/ChannelConstants';
import type {UserPartialResponse} from '@fluxer/schema/src/domains/user/UserResponseSchemas';
import {beforeEach, describe, expect, it} from 'vitest';
const CALLER_ID = createUserID(1n);
const RECIPIENT_ID = createUserID(2n);
const CHANNEL_ID = createChannelID(12n);
interface CallerOverrides {
username?: string;
globalName?: string | null;
avatar?: string | null;
nickname?: string;
userRowMissing?: boolean;
}
interface Harness {
service: CallService;
created: Array<CallCaller | undefined>;
rung: Array<CallCaller | undefined>;
}
const EXISTING_CALL: CallData = {
channel_id: CHANNEL_ID.toString(),
message_id: '99',
region: 'automatic',
ringing: [],
recipients: [CALLER_ID.toString(), RECIPIENT_ID.toString()],
voice_states: [],
};
function harness(overrides: CallerOverrides, existingCall: CallData | null): Harness {
const username = overrides.username ?? 'elias';
const globalName = overrides.globalName === undefined ? 'Elias' : overrides.globalName;
const avatar = overrides.avatar === undefined ? 'a1b2c3d4' : overrides.avatar;
const nicknames = new Map<string, string>();
if (overrides.nickname !== undefined) {
nicknames.set(CALLER_ID.toString(), overrides.nickname);
}
const channel = {
id: CHANNEL_ID,
type: ChannelTypes.GROUP_DM,
recipientIds: new Set<UserID>([CALLER_ID, RECIPIENT_ID]),
nicknames,
} as unknown as Channel;
const created: Array<CallCaller | undefined> = [];
const rung: Array<CallCaller | undefined> = [];
const channelRepository = {
findUnique: async () => channel,
upsertMessage: async () => {},
getMessage: async () => null,
} as unknown as IChannelRepository;
const userRepository = {
findUnique: async () => (overrides.userRowMissing ? null : ({...callerUser(username, globalName, avatar)} as User)),
listUsers: async () => [],
findSettings: async () => null,
isDmChannelOpen: async () => true,
} as unknown as IUserRepository;
const gatewayService = {
getCall: async () => existingCall,
createCall: async (
_channelId: unknown,
_messageId: string,
_region: string,
_ringing: Array<string>,
_recipients: Array<string>,
caller?: CallCaller,
) => {
created.push(caller);
return EXISTING_CALL;
},
ringCallRecipients: async (_channelId: unknown, _recipients: Array<string>, caller?: CallCaller) => {
rung.push(caller);
return true;
},
} as unknown as IGatewayService;
const userCacheService = {
getUserPartialResponse: async (): Promise<UserPartialResponse> =>
({
id: CALLER_ID.toString(),
username,
discriminator: '0001',
global_name: globalName,
avatar,
avatar_color: null,
flags: 0,
}) as unknown as UserPartialResponse,
} as unknown as UserCacheService;
const snowflakeService = {
generateForChannel: async () => 7777n,
} as unknown as ISnowflakeService;
const readStateService = {
ackMessage: async () => {},
bulkIncrementMentionCounts: async () => {},
} as unknown as ReadStateService;
const service = new CallService(
channelRepository,
userRepository,
{} as unknown as IGuildRepositoryAggregate,
gatewayService,
userCacheService,
snowflakeService,
readStateService,
null,
{} as unknown as IVoiceRoomStore,
);
return {service, created, rung};
}
function callerUser(username: string, globalName: string | null, avatar: string | null): Partial<User> {
return {
id: CALLER_ID,
username,
globalName,
avatarHash: avatar,
isBot: false,
};
}
const requestCache = {
userPartials: new Map(),
} as unknown as RequestCache;
describe('CallService caller identity', () => {
let harnessState: Harness;
const createCall = (overrides: CallerOverrides = {}) => {
harnessState = harness(overrides, null);
return harnessState.service.createOrGetCall({
userId: CALLER_ID,
channelId: CHANNEL_ID,
ringing: [RECIPIENT_ID],
requestCache,
});
};
const ringExistingCall = (overrides: CallerOverrides = {}) => {
harnessState = harness(overrides, EXISTING_CALL);
return harnessState.service.ringCallRecipients({
userId: CALLER_ID,
channelId: CHANNEL_ID,
requestCache,
});
};
beforeEach(() => {
requestCache.userPartials.clear();
});
it('sends the caller id, display name and avatar hash to createCall', async () => {
await createCall();
expect(harnessState.created).toEqual([{id: '1', name: 'Elias', avatar: 'a1b2c3d4'}]);
});
it('prefers the group dm nickname over the global name on createCall', async () => {
await createCall({nickname: 'Eli'});
expect(harnessState.created[0]?.name).toBe('Eli');
});
it('falls back to the username when the caller has no nickname and no global name', async () => {
await createCall({globalName: null});
expect(harnessState.created[0]?.name).toBe('elias');
});
it('sends a null avatar when the caller has no custom avatar', async () => {
await createCall({avatar: null});
expect(harnessState.created[0]).toEqual({id: '1', name: 'Elias', avatar: null});
});
it('sends no caller at all when the caller user row is gone', async () => {
await createCall({userRowMissing: true});
expect(harnessState.created).toEqual([undefined]);
});
it('sends the caller id, display name and avatar hash to ringCallRecipients', async () => {
await ringExistingCall();
expect(harnessState.rung).toEqual([{id: '1', name: 'Elias', avatar: 'a1b2c3d4'}]);
});
it('prefers the group dm nickname over the global name on ringCallRecipients', async () => {
await ringExistingCall({nickname: 'Eli'});
expect(harnessState.rung[0]?.name).toBe('Eli');
});
it('falls back to the username on ringCallRecipients', async () => {
await ringExistingCall({globalName: null});
expect(harnessState.rung[0]?.name).toBe('elias');
});
it('resolves the caller on the ring branch and not on the create branch', async () => {
await ringExistingCall();
expect(harnessState.created).toEqual([]);
expect(harnessState.rung).toHaveLength(1);
});
});
@@ -12,6 +12,7 @@ import type {ISnowflakeService} from '@app/api/infrastructure/ISnowflakeService'
import type {IVoiceRoomStore} from '@app/api/infrastructure/IVoiceRoomStore';
import type {UserCacheService} from '@app/api/infrastructure/UserCacheService';
import type {RequestCache} from '@app/api/middleware/RequestCacheMiddleware';
import type {Channel} from '@app/api/models/Channel';
import type {ReadStateService} from '@app/api/read_state/ReadStateService';
import type {IUserRepository} from '@app/api/user/IUserRepository';
import type {VoiceAccessContext, VoiceAvailabilityService} from '@app/api/voice/VoiceAvailabilityService';
@@ -208,14 +209,26 @@ export class CallService {
has_reaction: false,
version: 1,
});
const author = await this.userRepository.findUnique(userId);
const call = await this.gatewayService.createCall(
channelId,
messageId.toString(),
selectedRegion,
ringing.map((id) => id.toString()),
allRecipients.map((id) => id.toString()),
author
? {
id: userId.toString(),
name: this.resolveCallerName({
channel,
userId,
globalName: author.globalName,
username: author.username,
}),
avatar: author.avatarHash,
}
: undefined,
);
const author = await this.userRepository.findUnique(userId);
await incrementDmMentionCounts({
readStateService: this.readStateService,
userRepository: this.userRepository,
@@ -390,13 +403,45 @@ export class CallService {
longitude,
});
} else {
const caller = await this.userCacheService.getUserPartialResponse(userId, requestCache);
await this.gatewayService.ringCallRecipients(
channelId,
recipientsToRing.map((id) => id.toString()),
{
id: userId.toString(),
name: this.resolveCallerName({
channel,
userId,
globalName: caller.global_name,
username: caller.username,
}),
avatar: caller.avatar,
},
);
}
}
private resolveCallerName({
channel,
userId,
globalName,
username,
}: {
channel: Channel;
userId: UserID;
globalName: string | null;
username: string;
}): string {
const nickname = channel.nicknames.get(userId.toString());
if (nickname) {
return nickname;
}
if (globalName) {
return globalName;
}
return username;
}
async stopRingingCallRecipients({
userId,
channelId,
@@ -24,6 +24,7 @@ import {deleteChannelMessageSearchDocuments} from '@app/api/search/MessageSearch
import type {IUserRepository} from '@app/api/user/IUserRepository';
import {serializeChannelForAudit} from '@app/api/utils/AuditSerializationUtils';
import {applyProtectedOverwriteBits} from '@app/api/utils/featureUtils';
import {overwriteGrantedBits} from '@app/api/utils/PermissionUtils';
import type {VoiceAvailabilityService} from '@app/api/voice/VoiceAvailabilityService';
import type {VoiceRegionAvailability} from '@app/api/voice/VoiceModel';
import type {IWebhookRepository} from '@app/api/webhook/IWebhookRepository';
@@ -208,25 +209,6 @@ export class ChannelOperationsService {
userId,
channelId: channel.id,
});
if (!isOwner) {
for (const overwrite of data.permission_overwrites ?? []) {
const allowPerms = (overwrite.allow ? BigInt(overwrite.allow) : 0n) & ALL_PERMISSIONS;
if ((allowPerms & ~channelPermissions) !== 0n) {
throw new MissingPermissionsError();
}
}
const nextDeny = new Map<RoleID | UserID, bigint>();
for (const overwrite of data.permission_overwrites ?? []) {
const targetKey = overwrite.type === 0 ? createRoleID(overwrite.id) : createUserID(overwrite.id);
nextDeny.set(targetKey, (overwrite.deny ? BigInt(overwrite.deny) : 0n) & ALL_PERMISSIONS);
}
for (const [targetId, existing] of previousPermissionOverwrites ?? []) {
const removedDeny = existing.deny & ~(nextDeny.get(targetId) ?? 0n);
if ((removedDeny & ~channelPermissions) !== 0n) {
throw new MissingPermissionsError();
}
}
}
permissionOverwrites = new Map();
for (const overwrite of data.permission_overwrites ?? []) {
const targetId = overwrite.type === 0 ? createRoleID(overwrite.id) : createUserID(overwrite.id);
@@ -251,6 +233,18 @@ export class ChannelOperationsService {
}),
);
}
if (!isOwner) {
const targetIds = new Set([...(previousPermissionOverwrites?.keys() ?? []), ...permissionOverwrites.keys()]);
for (const targetId of targetIds) {
const grantedBits = overwriteGrantedBits(
previousPermissionOverwrites?.get(targetId),
permissionOverwrites.get(targetId),
);
if ((grantedBits & ~channelPermissions) !== 0n) {
throw new MissingPermissionsError();
}
}
}
}
const requestedParentId =
data.parent_id !== undefined ? (data.parent_id ? createChannelID(data.parent_id) : null) : channel.parentId;
@@ -646,9 +640,8 @@ export class ChannelOperationsService {
const sanitizedAllow = protectedBits.allow;
const sanitizedDeny = protectedBits.deny;
const hasAdministrator = (userPermissions & Permissions.ADMINISTRATOR) !== 0n;
if (!hasAdministrator && (sanitizedAllow & ~userPermissions) !== 0n) throw new MissingPermissionsError();
const removedDeny = (existing?.deny ?? 0n) & ~sanitizedDeny;
if (!hasAdministrator && (removedDeny & ~userPermissions) !== 0n) throw new MissingPermissionsError();
const grantedBits = overwriteGrantedBits(existing, {allow: sanitizedAllow, deny: sanitizedDeny});
if (!hasAdministrator && (grantedBits & ~userPermissions) !== 0n) throw new MissingPermissionsError();
const previousPermissionOverwrites = channel.permissionOverwrites;
const nextOverwrite = new ChannelPermissionOverwrite({
type: params.overwrite.type,
@@ -3,7 +3,7 @@
import type {AttachmentID, ChannelID} from '@app/api/BrandedTypes';
import type {AttachmentRequestData} from '@app/api/channel/AttachmentDTOs';
import type {RichEmbedMediaWithMetadata} from '@app/api/channel/EmbedTypes';
import {makeAttachmentCdnUrl} from '@app/api/channel/services/message/MessageHelpers';
import {getContentType, makeAttachmentCdnUrl} from '@app/api/channel/services/message/MessageHelpers';
import {ValidationErrorCodes} from '@fluxer/constants/src/ValidationErrorCodes';
import {InputValidationError} from '@fluxer/errors/src/domains/core/InputValidationError';
import type {RichEmbedRequest} from '@fluxer/schema/src/domains/message/MessageRequestSchemas';
@@ -26,7 +26,10 @@ interface RichEmbedRequestWithMetadata extends Omit<RichEmbedRequest, 'image' |
thumbnail?: RichEmbedMediaWithMetadata | null;
}
const SUPPORTED_IMAGE_EXTENSIONS = new Set(['png', 'jpg', 'jpeg', 'webp', 'gif']);
function isEmbeddableMediaType(contentType: string): boolean {
const normalized = contentType.toLowerCase();
return normalized.startsWith('image/') || normalized.startsWith('video/');
}
export class MessageEmbedAttachmentResolver {
validateAttachmentReferences(params: {
@@ -69,8 +72,7 @@ export class MessageEmbedAttachmentResolver {
{filename},
);
}
const extension = filename.split('.').pop()?.toLowerCase();
if (!extension || !SUPPORTED_IMAGE_EXTENSIONS.has(extension)) {
if (!isEmbeddableMediaType(getContentType(filename))) {
throw InputValidationError.fromCode(
`embeds[${embedIndex}].${field}`,
ValidationErrorCodes.ATTACHMENT_MUST_BE_IMAGE,
@@ -137,8 +139,7 @@ export class MessageEmbedAttachmentResolver {
if (!attachmentData) {
throw InputValidationError.fromCode(field, ValidationErrorCodes.REFERENCED_ATTACHMENT_NOT_FOUND, {filename});
}
const extension = filename.split('.').pop()?.toLowerCase();
if (!extension || !SUPPORTED_IMAGE_EXTENSIONS.has(extension)) {
if (!isEmbeddableMediaType(attachmentData.metadata.content_type)) {
throw InputValidationError.fromCode(field, ValidationErrorCodes.ATTACHMENT_MUST_BE_IMAGE, {filename});
}
return attachmentData;
@@ -252,9 +252,7 @@ export class MessageValidationService {
const isAuthor = message.authorId === userId;
if (!guild) return isAuthor;
if (isAuthor) return true;
const canManageMessages =
(await hasPermission(Permissions.SEND_MESSAGES)) && (await hasPermission(Permissions.MANAGE_MESSAGES));
return canManageMessages;
return hasPermission(Permissions.MANAGE_MESSAGES);
}
private validateVoiceMessageConstraints(
@@ -294,6 +294,48 @@ describe('Channel Permission Overwrites', () => {
expect(overwrite?.allow).toBe(Permissions.VIEW_CHANNEL.toString());
expect(overwrite?.deny).toBe(Permissions.MANAGE_MESSAGES.toString());
});
test('should let an editor change an overwrite that already allows a permission they lack', async () => {
const {owner, members, guild, systemChannel} = await setupTestGuildWithMembers(harness, 1);
const manager = members[0];
const managerRole = await createRole(harness, owner.token, guild.id, {
name: 'Queue Manager',
permissions: Permissions.MANAGE_ROLES.toString(),
});
const botRole = await createRole(harness, owner.token, guild.id, {name: 'Bot'});
await addMemberRole(harness, owner.token, guild.id, manager.userId, managerRole.id);
await createPermissionOverwrite(harness, owner.token, systemChannel.id, botRole.id, {
type: 0,
allow: Permissions.PIN_MESSAGES.toString(),
deny: '0',
});
await createBuilder(harness, manager.token)
.put(`/channels/${systemChannel.id}/permissions/${botRole.id}`)
.body({
type: 0,
allow: (Permissions.PIN_MESSAGES | Permissions.SEND_MESSAGES).toString(),
deny: '0',
})
.expect(HTTP_STATUS.NO_CONTENT)
.execute();
const updated = await getChannel(harness, owner.token, systemChannel.id);
const botOverwrite = updated.permission_overwrites?.find((o) => o.id === botRole.id);
expect(botOverwrite?.allow).toBe((Permissions.PIN_MESSAGES | Permissions.SEND_MESSAGES).toString());
});
test('should reject an editor granting a permission they lack', async () => {
const {owner, members, guild, systemChannel} = await setupTestGuildWithMembers(harness, 1);
const manager = members[0];
const managerRole = await createRole(harness, owner.token, guild.id, {
name: 'Queue Manager',
permissions: Permissions.MANAGE_ROLES.toString(),
});
const botRole = await createRole(harness, owner.token, guild.id, {name: 'Bot'});
await addMemberRole(harness, owner.token, guild.id, manager.userId, managerRole.id);
await createBuilder(harness, manager.token)
.put(`/channels/${systemChannel.id}/permissions/${botRole.id}`)
.body({type: 0, allow: Permissions.PIN_MESSAGES.toString(), deny: '0'})
.expect(HTTP_STATUS.FORBIDDEN)
.execute();
});
test('should propagate category permission patches only to children that were synced when the category changed', async () => {
const {owner, guild} = await setupTestGuildWithMembers(harness, 0);
const targetRole = await createRole(harness, owner.token, guild.id, {name: 'Readers'});
@@ -512,6 +512,34 @@ describe('Embed Attachment URL Resolution', () => {
expect(json.embeds).toHaveLength(1);
expect(json.embeds![0].image?.url).not.toContain('attachment://');
});
it('should accept image and video attachments beyond the legacy image extensions', async () => {
const account = await createTestAccount(harness);
const guild = await createGuild(harness, account.token, 'Media Type Guild');
const channel = await createChannel(harness, account.token, guild.id, 'test-channel');
const channelId = guild.system_channel_id ?? channel.id;
const payload = {
content: 'Test with jxl and mp4 embed media',
attachments: [
{id: 0, filename: 'photo.jxl'},
{id: 1, filename: 'clip.mp4'},
],
embeds: [
{
title: 'Media Embed',
image: {url: 'attachment://clip.mp4'},
thumbnail: {url: 'attachment://photo.jxl'},
},
],
};
const {response, json} = await sendMessageWithAttachments(harness, account.token, channelId, payload, [
{index: 0, filename: 'photo.jxl', data: Buffer.from('jxl bytes')},
{index: 1, filename: 'clip.mp4', data: Buffer.from('mp4 bytes')},
]);
expect(response.status).toBe(200);
expect(json.embeds).toHaveLength(1);
expect(json.embeds![0].image?.url).not.toContain('attachment://');
expect(json.embeds![0].thumbnail?.url).not.toContain('attachment://');
});
});
describe('Multiple Embeds and Files', () => {
it('should handle multiple embeds with different URL types', async () => {
@@ -0,0 +1,62 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import {createTestAccount} from '@app/api/auth/tests/AuthTestUtils';
import {createFriendship, createGroupDmChannel, getChannel} from '@app/api/channel/tests/ChannelTestUtils';
import {ensureSessionStarted} from '@app/api/message/tests/MessageTestUtils';
import {type ApiTestHarness, createApiTestHarness} from '@app/api/test/ApiTestHarness';
import {NoopGatewayService} from '@app/api/test/NoopGatewayService';
import {HTTP_STATUS} from '@app/api/test/TestConstants';
import {createBuilder} from '@app/api/test/TestRequestBuilder';
import type {ChannelResponse} from '@fluxer/schema/src/domains/channel/ChannelSchemas';
import {afterAll, beforeAll, beforeEach, describe, expect, it, vi} from 'vitest';
describe('Group DM name clear', () => {
let harness: ApiTestHarness;
beforeAll(async () => {
harness = await createApiTestHarness();
});
beforeEach(async () => {
await harness.reset();
});
afterAll(async () => {
await harness?.shutdown();
});
it.each([
['an empty string', ''],
['null', null],
])('sends a null name to every recipient when cleared with %s', async (_label, clearedName) => {
const user1 = await createTestAccount(harness);
const user2 = await createTestAccount(harness);
const user3 = await createTestAccount(harness);
await ensureSessionStarted(harness, user1.token);
await ensureSessionStarted(harness, user2.token);
await ensureSessionStarted(harness, user3.token);
await createFriendship(harness, user1, user2);
await createFriendship(harness, user1, user3);
const groupDm = await createGroupDmChannel(harness, user1.token, [user2.userId, user3.userId]);
await createBuilder<ChannelResponse>(harness, user1.token)
.patch(`/channels/${groupDm.id}`)
.body({name: 'Weekend plans'})
.expect(HTTP_STATUS.OK)
.execute();
const dispatchSpy = vi.spyOn(NoopGatewayService.prototype, 'dispatchPresence');
try {
const cleared = await createBuilder<ChannelResponse>(harness, user1.token)
.patch(`/channels/${groupDm.id}`)
.body({name: clearedName})
.expect(HTTP_STATUS.OK)
.execute();
expect(cleared).toHaveProperty('name', null);
const channelUpdates = dispatchSpy.mock.calls.filter(([params]) => params.event === 'CHANNEL_UPDATE');
expect(channelUpdates.map(([params]) => params.userId.toString()).sort()).toEqual(
[user1.userId, user2.userId, user3.userId].sort(),
);
for (const [params] of channelUpdates) {
expect(params.data).toHaveProperty('name', null);
}
} finally {
dispatchSpy.mockRestore();
}
expect(await getChannel(harness, user2.token, groupDm.id)).toHaveProperty('name', null);
});
});
@@ -0,0 +1,61 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import type {TestAccount} from '@app/api/auth/tests/AuthTestUtils';
import {
createPermissionOverwrite,
sendChannelMessage,
setupTestGuildWithMembers,
} from '@app/api/channel/tests/ChannelTestUtils';
import {type ApiTestHarness, createApiTestHarness} from '@app/api/test/ApiTestHarness';
import {HTTP_STATUS} from '@app/api/test/TestConstants';
import {createBuilder} from '@app/api/test/TestRequestBuilder';
import {Permissions} from '@fluxer/constants/src/ChannelConstants';
import {afterAll, beforeAll, beforeEach, describe, it} from 'vitest';
describe('Message delete permissions', () => {
let harness: ApiTestHarness;
beforeAll(async () => {
harness = await createApiTestHarness();
});
beforeEach(async () => {
await harness.reset();
});
afterAll(async () => {
await harness?.shutdown();
});
it('lets a member with MANAGE_MESSAGES but without SEND_MESSAGES delete another member message', async () => {
const {owner, members, systemChannel} = await setupTestGuildWithMembers(harness, 2);
const [author, moderator] = members as [TestAccount, TestAccount];
const message = await sendChannelMessage(harness, author.token, systemChannel.id, 'delete me');
await createPermissionOverwrite(harness, owner.token, systemChannel.id, moderator.userId, {
type: 1,
allow: Permissions.MANAGE_MESSAGES.toString(),
deny: Permissions.SEND_MESSAGES.toString(),
});
await createBuilder(harness, moderator.token)
.delete(`/channels/${systemChannel.id}/messages/${message.id}`)
.expect(HTTP_STATUS.NO_CONTENT)
.execute();
await createBuilder(harness, author.token)
.get(`/channels/${systemChannel.id}/messages/${message.id}`)
.expect(HTTP_STATUS.NOT_FOUND)
.execute();
});
it('refuses a member without MANAGE_MESSAGES deleting another member message', async () => {
const {members, systemChannel} = await setupTestGuildWithMembers(harness, 2);
const [author, other] = members as [TestAccount, TestAccount];
const message = await sendChannelMessage(harness, author.token, systemChannel.id, 'keep me');
await createBuilder(harness, other.token)
.delete(`/channels/${systemChannel.id}/messages/${message.id}`)
.expect(HTTP_STATUS.FORBIDDEN, 'MISSING_PERMISSIONS')
.execute();
});
});
+4 -2
View File
@@ -143,8 +143,6 @@ export interface APIConfig {
donationProxyKey: string;
};
hosts: {
invite: string;
gift: string;
marketing: string;
unfurlIgnored: Array<string>;
};
@@ -338,6 +336,10 @@ export interface APIConfig {
};
abusePolicy: {
inboundPhoneCountryCodes: Array<string>;
phoneFlagging: {
enabled: boolean;
exemptCountryCodes: Array<string>;
};
phoneVerification: {
inboundRequiredPrefixes: Array<string>;
};
@@ -1,5 +1,7 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import type {BannedFileShaRow} from '@app/api/database/types/AdminArchiveTypes';
export const BANNED_URLS_REFRESH_CHANNEL = 'banned_urls_refresh';
export const BANNED_URL_DOMAINS_REFRESH_CHANNEL = 'banned_url_domains_refresh';
export const BANNED_FILE_SHAS_REFRESH_CHANNEL = 'banned_file_shas_refresh';
@@ -23,3 +25,7 @@ export const ContentBlocklistCategory = {
GIFCT: 'gifct',
STOP_NCII: 'stop_ncii',
} as const;
export function isBlocklistFeedFileSha(row: Pick<BannedFileShaRow, 'category' | 'added_by'>): boolean {
return row.added_by == null && row.category === ContentBlocklistCategory.MALWARE_BAZAAR;
}
@@ -0,0 +1,89 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import fs from 'node:fs';
import path from 'node:path';
import {fileURLToPath} from 'node:url';
import {DEFAULT_TTL_TABLES} from '@app/api/database/PostgresKvDefaultTtlExpiry';
import * as DonationTables from '@app/api/donation/DonationTables';
import * as Tables from '@app/api/Tables';
import {IPINFO_CACHE_TTL_SECONDS, IPINFO_REQUEST_AUDIT_TTL_SECONDS} from '@pkgs/geoip/src/PostgresIpInfoKv';
import {describe, expect, it} from 'vitest';
const THIS_DIR = path.dirname(fileURLToPath(import.meta.url));
const REPO_ROOT = path.resolve(THIS_DIR, '../../../..');
interface SchemaTable {
name: string;
options: string;
}
const SCHEMA = JSON.parse(fs.readFileSync(path.join(REPO_ROOT, 'tools/dev/cassandra_target_schema.json'), 'utf8')) as {
tables: Array<SchemaTable>;
};
const SCHEMA_DEFAULTS = new Map<string, number>(
SCHEMA.tables.flatMap((table): Array<[string, number]> => {
const match = /default_time_to_live = (\d+)/.exec(table.options);
return match ? [[table.name, Number(match[1])]] : [];
}),
);
const DSL_TABLES = [...Object.values(Tables), ...Object.values(DonationTables)];
const DSL_NAMES = new Set<string>(DSL_TABLES.map((table) => table.name));
const NON_DSL_DEFAULTS: Record<string, number | null> = {
ipinfo_cache: IPINFO_CACHE_TTL_SECONDS,
ipinfo_requests_by_hour: IPINFO_REQUEST_AUDIT_TTL_SECONDS,
billing_webhook_events: null,
forensic_identifier_by_key_day: null,
forensic_identifier_by_request: null,
forensic_request_meta_by_actor_day: null,
forensic_request_meta_by_id: null,
forensic_request_meta_by_route_day_shard: null,
forensic_resource_exposure_by_request: null,
forensic_resource_exposure_by_route_day_shard: null,
forensic_resource_exposure_by_subject_day: null,
};
const OWN_EXPIRY_PASS = new Set(['jobs_by_id', 'jobs_by_day_bucket']);
function schemaDefault(name: string): number {
return SCHEMA_DEFAULTS.get(name) ?? 0;
}
function byName(left: {name: string}, right: {name: string}): number {
return left.name.localeCompare(right.name);
}
describe('Cassandra default TTL parity', () => {
it('declares every Cassandra default TTL on the matching table', () => {
const mismatches = DSL_TABLES.flatMap((table) => {
const declared = table.defaultTtlSeconds ?? 0;
return declared === schemaDefault(table.name)
? []
: [{table: table.name, declared, schema: schemaDefault(table.name)}];
});
expect(mismatches).toEqual([]);
});
it('declares a writer or no writer for every other table with a default', () => {
const undeclared = [...SCHEMA_DEFAULTS]
.filter(([name, ttl]) => ttl > 0 && !DSL_NAMES.has(name) && !Object.hasOwn(NON_DSL_DEFAULTS, name))
.map(([name]) => name);
expect(undeclared).toEqual([]);
const stale = Object.keys(NON_DSL_DEFAULTS).filter((name) => schemaDefault(name) === 0 || DSL_NAMES.has(name));
expect(stale).toEqual([]);
const mismatched = Object.entries(NON_DSL_DEFAULTS)
.filter(([name, ttl]) => ttl !== null && ttl !== schemaDefault(name))
.map(([name]) => name);
expect(mismatched).toEqual([]);
});
it('the Postgres expiry pass covers every table with a default except the job ledger', () => {
const expected = [...SCHEMA_DEFAULTS]
.filter(([name, ttl]) => ttl > 0 && NON_DSL_DEFAULTS[name] !== null && !OWN_EXPIRY_PASS.has(name))
.map(([name, ttl]) => ({name, defaultTtlSeconds: ttl}))
.sort(byName);
expect([...DEFAULT_TTL_TABLES].sort(byName)).toEqual(expected);
});
});
@@ -12,6 +12,7 @@ interface TableMetadata {
columns: ReadonlyArray<string>;
primaryKey: ReadonlyArray<string>;
partitionKey: ReadonlyArray<string>;
defaultTtlSeconds?: number;
}
const kvMetaRegistry = new Map<string, KvQueryMeta<Record<string, unknown>>>();
@@ -24,6 +25,7 @@ export function registerTableSpec<Row extends object>(tableSpec: KvTableSpec<Row
columns: tableSpec.columns as ReadonlyArray<string>,
primaryKey: tableSpec.primaryKey as ReadonlyArray<string>,
partitionKey: tableSpec.partitionKey as ReadonlyArray<string>,
defaultTtlSeconds: tableSpec.defaultTtlSeconds,
};
tableRegistry.set(tableSpec.name, metadata);
}
@@ -1,5 +1,6 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import {getTableMetadata} from '@app/api/database/CassandraMetaRegistry';
import {defineTable} from '@app/api/database/CassandraTableDsl';
import {Db, type PreparedQuery} from '@app/api/database/CassandraTypes';
import {describe, expect, it} from 'vitest';
@@ -76,3 +77,41 @@ describe('CassandraTableDsl select templates', () => {
expect(longQuery.cql).not.toContain('LIMIT 20');
});
});
describe('CassandraTableDsl default TTL', () => {
it('keeps the CQL of a table with a default TTL free of USING TTL', () => {
const DefaultTtlRows = defineTable<TtlHelperTestRow, 'id'>({
name: 'default_ttl_dsl_rows',
columns: ['id', 'value'],
primaryKey: ['id'],
defaultTtlSeconds: 600,
});
expect(DefaultTtlRows.defaultTtlSeconds).toBe(600);
const queries = [
DefaultTtlRows.insert({id: 'insert', value: 'a'}),
DefaultTtlRows.upsertAll({id: 'upsert', value: 'b'}),
DefaultTtlRows.patchByPk({id: 'patch'}, {value: Db.set('c')}),
];
for (const query of queries) {
expect(query.cql).not.toContain('USING TTL');
expect(query.kvMeta?.table.defaultTtlSeconds).toBe(600);
}
expect(getTableMetadata('default_ttl_dsl_rows')?.defaultTtlSeconds).toBe(600);
expect(TtlHelperTestRows.defaultTtlSeconds).toBeUndefined();
expect(getTableMetadata('ttl_helper_test_rows')?.defaultTtlSeconds).toBeUndefined();
});
it('rejects a default TTL of zero, a fraction or past the maximum', () => {
for (const defaultTtlSeconds of [0, 1.5, 630_720_001]) {
expect(() =>
defineTable<TtlHelperTestRow, 'id'>({
name: 'default_ttl_dsl_rejected_rows',
columns: ['id', 'value'],
primaryKey: ['id'],
defaultTtlSeconds,
}),
).toThrow();
}
expect(getTableMetadata('default_ttl_dsl_rejected_rows')).toBeUndefined();
});
});
@@ -83,6 +83,7 @@ export function defineTable<Row extends object, PK extends ColumnName<Row>, Part
columns: ReadonlyArray<ColumnName<Row>>;
primaryKey: ReadonlyArray<PK>;
partitionKey?: ReadonlyArray<PartKey>;
defaultTtlSeconds?: number;
}): Table<Row, PK, PartKey> {
const columns = [...def.columns];
const pk = [...def.primaryKey];
@@ -91,11 +92,15 @@ export function defineTable<Row extends object, PK extends ColumnName<Row>, Part
for (const c of columns) assertCqlIdentifier(c as string);
for (const k of pk) assertCqlIdentifier(k as string);
for (const k of partitionKey) assertCqlIdentifier(k as string);
if (def.defaultTtlSeconds !== undefined && validateTtlSeconds(def.defaultTtlSeconds) === 0) {
throw new Error(`Table "${def.name}" needs a positive default TTL`);
}
const tableSpec: KvTableSpec<Row> = {
name: def.name,
columns,
primaryKey: pk as ReadonlyArray<ColumnName<Row>>,
partitionKey: partitionKey as ReadonlyArray<ColumnName<Row>>,
defaultTtlSeconds: def.defaultTtlSeconds,
};
registerTableSpec(tableSpec);
const nonPkColumns = columns.filter((c) => !pk.includes(c as PK)) as Array<Exclude<ColumnName<Row>, PK>>;
@@ -685,6 +690,7 @@ WHERE ${pk.map((k) => `${k} = :${k}`).join(' AND ')};
columns: def.columns,
primaryKey: def.primaryKey,
partitionKey: partitionKey,
defaultTtlSeconds: def.defaultTtlSeconds,
selectCql,
select,
updateAllCql() {
@@ -56,6 +56,7 @@ export interface KvTableSpec<Row extends object = Record<string, unknown>> {
columns: ReadonlyArray<ColumnName<Row>>;
primaryKey: ReadonlyArray<ColumnName<Row>>;
partitionKey: ReadonlyArray<ColumnName<Row>>;
defaultTtlSeconds?: number;
}
export interface KvColumnParam<Row extends object = Record<string, unknown>> {
@@ -190,6 +191,7 @@ export interface Table<Row extends object, PK extends ColumnName<Row>, PartKey e
columns: ReadonlyArray<ColumnName<Row>>;
primaryKey: ReadonlyArray<PK>;
partitionKey: ReadonlyArray<PartKey>;
defaultTtlSeconds: number | undefined;
selectCql(opts?: {
columns?: ReadonlyArray<ColumnName<Row>>;
where?: WhereExpr<Row> | ReadonlyArray<WhereExpr<Row>>;
@@ -0,0 +1,490 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import {spawnSync} from 'node:child_process';
import {createServer} from 'node:net';
import {defineTable} from '@app/api/database/CassandraTableDsl';
import {Db} from '@app/api/database/CassandraTypes';
import {
DEFAULT_TTL_EXPIRY_RESUME,
DEFAULT_TTL_TABLES,
expireLegacyDefaultTtlRows,
} from '@app/api/database/PostgresKvDefaultTtlExpiry';
import {
ensurePostgresKvSchema,
PostgresKvQueryExecutor,
pruneExpiredPostgresKvRows,
} from '@app/api/database/PostgresKvQueryExecutor';
import {startDockerContainer} from '@app/api/test/DockerTestContainer';
import {
getDefaultPostgresClient,
type IPostgresClient,
initPostgres,
shutdownPostgres,
} from '@pkgs/postgres/src/Client';
import {afterAll, beforeAll, beforeEach, describe, expect, it} from 'vitest';
const KV_TABLE = 'kv_default_ttl';
const CONTAINER = `fluxer-kvttl-${process.pid.toString(36)}-${Date.now().toString(36)}`;
const dockerAvailable = spawnSync('docker', ['version'], {stdio: 'ignore'}).status === 0;
const DEFAULT_TTL_SECONDS = 600;
interface ProbeRow {
id: string;
value: string | null;
note: string | null;
}
interface OwnedProbeRow {
owner: string;
id: string;
value: string | null;
}
const DefaultTtlProbe = defineTable<ProbeRow, 'id'>({
name: 'default_ttl_probe',
columns: ['id', 'value', 'note'],
primaryKey: ['id'],
defaultTtlSeconds: DEFAULT_TTL_SECONDS,
});
const DefaultTtlProbeRows = defineTable<OwnedProbeRow, 'owner' | 'id', 'owner'>({
name: 'default_ttl_probe_rows',
columns: ['owner', 'id', 'value'],
primaryKey: ['owner', 'id'],
partitionKey: ['owner'],
defaultTtlSeconds: DEFAULT_TTL_SECONDS,
});
const NoTtlProbe = defineTable<ProbeRow, 'id'>({
name: 'no_ttl_probe',
columns: ['id', 'value', 'note'],
primaryKey: ['id'],
});
async function sleep(ms: number): Promise<void> {
await new Promise((resolve) => setTimeout(resolve, ms));
}
async function freePort(): Promise<number> {
return new Promise((resolve, reject) => {
const server = createServer();
server.on('error', reject);
server.listen(0, '127.0.0.1', () => {
const address = server.address();
if (typeof address === 'string' || address === null) {
reject(new Error('no port'));
return;
}
const port = address.port;
server.close(() => resolve(port));
});
});
}
function expectExpiresIn(value: Date | number | null, ttlSeconds: number): void {
expect(value).toBeInstanceOf(Date);
const remainingSeconds = ((value as Date).getTime() - Date.now()) / 1000;
expect(remainingSeconds).toBeGreaterThan(ttlSeconds - 60);
expect(remainingSeconds).toBeLessThanOrEqual(ttlSeconds);
}
describe.skipIf(!dockerAvailable)('Postgres KV default TTL', () => {
let raw: IPostgresClient;
let executor: PostgresKvQueryExecutor;
async function stored(table: string, id: string): Promise<{expires_at: Date | number | null; row_data: object}> {
const result = await raw.query<{expires_at: Date | number | null; row_data: object}>(
`SELECT expires_at, row_data FROM ${KV_TABLE} WHERE table_name = $1 AND row_data ->> 'id' = $2`,
[table, id],
);
expect(result.rows).toHaveLength(1);
return result.rows[0]!;
}
async function expiresAt(table: string, id: string): Promise<Date | number | null> {
return (await stored(table, id)).expires_at;
}
async function neverExpires(table: string, id: string): Promise<boolean> {
const result = await raw.query<{forever: boolean}>(
`SELECT expires_at = 'infinity'::timestamptz AS forever FROM ${KV_TABLE} WHERE table_name = $1 AND row_data ->> 'id' = $2`,
[table, id],
);
return result.rows[0]?.forever === true;
}
async function setExpiry(table: string, id: string, expression: string): Promise<void> {
await raw.query(
`UPDATE ${KV_TABLE} SET expires_at = ${expression} WHERE table_name = $1 AND row_data ->> 'id' = $2`,
[table, id],
);
}
async function seed(table: string, key: string, age: string, expires: Date | string | null = null): Promise<string> {
const result = await raw.query<{updated_at: string}>(
`INSERT INTO ${KV_TABLE} (table_name, partition_key, row_key, row_data, expires_at, updated_at)
VALUES ($1, $2, $2, '{}'::jsonb, $3::timestamptz, now() - $4::interval)
RETURNING updated_at::text`,
[table, key, expires, age],
);
return result.rows[0]!.updated_at;
}
async function remaining(): Promise<Array<{table_name: string; row_key: string}>> {
const result = await raw.query<{table_name: string; row_key: string}>(
`SELECT table_name, row_key FROM ${KV_TABLE} WHERE table_name <> '__fluxer_schema_migrations' ORDER BY table_name, row_key`,
);
return result.rows;
}
async function ageMarker(): Promise<void> {
await raw.query(
`UPDATE ${KV_TABLE} SET row_data = jsonb_build_object('applied_at', now() - interval '2 days') WHERE table_name = '__fluxer_schema_migrations' AND row_key = 'default_ttl_expiry_v1'`,
);
}
async function resumePoint(): Promise<object | null> {
const result = await raw.query<{row_data: object}>(
`SELECT row_data FROM ${KV_TABLE} WHERE table_name = '__fluxer_schema_migrations' AND row_key = $1`,
[DEFAULT_TTL_EXPIRY_RESUME],
);
return result.rows[0]?.row_data ?? null;
}
async function markerCount(): Promise<number> {
const result = await raw.query<{n: number}>(
`SELECT count(*)::int AS n FROM ${KV_TABLE} WHERE table_name = '__fluxer_schema_migrations' AND row_key = 'default_ttl_expiry_v1'`,
);
return result.rows[0]!.n;
}
beforeAll(async () => {
const port = await freePort();
startDockerContainer([
'run',
'-d',
'--name',
CONTAINER,
'-e',
'POSTGRES_USER=fluxer',
'-e',
'POSTGRES_PASSWORD=fluxer',
'-e',
'POSTGRES_DB=fluxer',
'-p',
`127.0.0.1:${port}:5432`,
'postgres:16-alpine',
'-c',
'fsync=off',
]);
let ready = false;
for (let attempt = 0; attempt < 180 && !ready; attempt += 1) {
await sleep(500);
const probe = spawnSync('docker', ['exec', CONTAINER, 'pg_isready', '-U', 'fluxer', '-d', 'fluxer'], {
stdio: 'ignore',
});
if (probe.status !== 0) continue;
try {
await initPostgres({
url: `postgres://fluxer:[email protected]:${port}/fluxer`,
maxConnections: 4,
kvTable: KV_TABLE,
});
await getDefaultPostgresClient().query('SELECT 1');
ready = true;
} catch {
await shutdownPostgres().catch(() => {});
}
}
if (!ready) throw new Error('postgres never came up');
raw = getDefaultPostgresClient();
await ensurePostgresKvSchema(raw);
executor = new PostgresKvQueryExecutor(raw);
}, 900_000);
beforeEach(async () => {
await raw.query(`DELETE FROM ${KV_TABLE}`);
});
afterAll(async () => {
await shutdownPostgres().catch(() => {});
spawnSync('docker', ['rm', '-f', CONTAINER], {stdio: 'ignore'});
});
it('gives every full-row write without a TTL the table default', async () => {
await executor.executeQuery(DefaultTtlProbe.insert({id: 'insert', value: 'a', note: null}));
await executor.executeQuery(DefaultTtlProbe.upsertAll({id: 'upsert', value: 'b', note: 'n'}));
expect(
await executor.executeQuery(DefaultTtlProbe.insertIfNotExists({id: 'claimed', value: 'c', note: null})),
).toEqual([{'[applied]': true}]);
expect(
await executor.executeQuery(
DefaultTtlProbeRows.conditionalBatch([{action: 'insert', row: {owner: 'o', id: 'batched', value: 'd'}}]),
),
).toEqual([{'[applied]': true}]);
for (const id of ['insert', 'upsert', 'claimed']) {
expectExpiresIn(await expiresAt('default_ttl_probe', id), DEFAULT_TTL_SECONDS);
}
expectExpiresIn(await expiresAt('default_ttl_probe_rows', 'batched'), DEFAULT_TTL_SECONDS);
});
it('keeps an explicit TTL ahead of the default', async () => {
await executor.executeQuery(DefaultTtlProbe.insertWithTtl({id: 'short', value: 'a', note: null}, 60));
expectExpiresIn(await expiresAt('default_ttl_probe', 'short'), 60);
await executor.executeQuery(DefaultTtlProbe.insert({id: 'patched', value: 'a', note: null}));
await executor.executeQuery(DefaultTtlProbe.patchByPkWithTtl({id: 'patched'}, {value: Db.set('b')}, 60));
expectExpiresIn(await expiresAt('default_ttl_probe', 'patched'), 60);
});
it('keeps an explicit TTL of zero as no expiry', async () => {
await executor.executeQuery(DefaultTtlProbe.insertWithTtl({id: 'forever', value: 'a', note: null}, 0));
expect(await neverExpires('default_ttl_probe', 'forever')).toBe(true);
expect(
await executor.executeQuery(
DefaultTtlProbe.select({where: DefaultTtlProbe.where.eq('id')}).bind({id: 'forever'}),
),
).toEqual([{id: 'forever', value: 'a', note: null}]);
await executor.executeQuery(DefaultTtlProbe.patchByPk({id: 'forever'}, {note: Db.set('patched')}));
expect(await neverExpires('default_ttl_probe', 'forever')).toBe(true);
await pruneExpiredPostgresKvRows(raw);
expect(await neverExpires('default_ttl_probe', 'forever')).toBe(true);
});
it('raises a patched row to the default but never lowers it', async () => {
await executor.executeQuery(DefaultTtlProbe.insertWithTtl({id: 'longer', value: 'a', note: null}, 3600));
await executor.executeQuery(DefaultTtlProbe.patchByPk({id: 'longer'}, {note: Db.set('patched')}));
expectExpiresIn(await expiresAt('default_ttl_probe', 'longer'), 3600);
await executor.executeQuery(DefaultTtlProbe.insert({id: 'soon', value: 'a', note: null}));
await setExpiry('default_ttl_probe', 'soon', "now() + interval '5 seconds'");
await executor.executeQuery(DefaultTtlProbe.patchByPk({id: 'soon'}, {note: Db.set('patched')}));
expectExpiresIn(await expiresAt('default_ttl_probe', 'soon'), DEFAULT_TTL_SECONDS);
await executor.executeQuery(DefaultTtlProbe.patchByPk({id: 'missing'}, {note: Db.set('created')}));
expectExpiresIn(await expiresAt('default_ttl_probe', 'missing'), DEFAULT_TTL_SECONDS);
await executor.executeQuery(DefaultTtlProbe.insert({id: 'unset', value: 'a', note: null}));
await setExpiry('default_ttl_probe', 'unset', 'NULL');
await executor.executeQuery(DefaultTtlProbe.patchByPk({id: 'unset'}, {note: Db.set('patched')}));
expectExpiresIn(await expiresAt('default_ttl_probe', 'unset'), DEFAULT_TTL_SECONDS);
await executor.executeQuery(DefaultTtlProbe.insert({id: 'expired', value: 'a', note: null}));
await setExpiry('default_ttl_probe', 'expired', "now() - interval '1 second'");
await executor.executeQuery(DefaultTtlProbe.patchByPk({id: 'expired'}, {note: Db.set('patched')}));
const revived = await stored('default_ttl_probe', 'expired');
expect(revived.row_data).toEqual({id: 'expired', note: 'patched'});
expectExpiresIn(revived.expires_at, DEFAULT_TTL_SECONDS);
});
it('raises conditional patches the same way', async () => {
await executor.executeQuery(DefaultTtlProbe.insert({id: 'soon', value: 'a', note: null}));
await setExpiry('default_ttl_probe', 'soon', "now() + interval '5 seconds'");
expect(
await executor.executeQuery(
DefaultTtlProbe.conditionalPatchByPk({id: 'soon'}, {note: Db.set('patched')}, {value: 'a'}),
),
).toEqual([{'[applied]': true}]);
expectExpiresIn(await expiresAt('default_ttl_probe', 'soon'), DEFAULT_TTL_SECONDS);
await executor.executeQuery(DefaultTtlProbe.insertWithTtl({id: 'longer', value: 'a', note: null}, 3600));
expect(
await executor.executeQuery(
DefaultTtlProbe.conditionalPatchByPk({id: 'longer'}, {note: Db.set('patched')}, {value: 'a'}),
),
).toEqual([{'[applied]': true}]);
expectExpiresIn(await expiresAt('default_ttl_probe', 'longer'), 3600);
await executor.executeQuery(DefaultTtlProbeRows.insert({owner: 'o', id: 'existing', value: 'old'}));
await setExpiry('default_ttl_probe_rows', 'existing', 'NULL');
expect(
await executor.executeQuery(
DefaultTtlProbeRows.conditionalBatch([
{action: 'insert', row: {owner: 'o', id: 'added', value: 'new'}},
{
action: 'patch',
pk: {owner: 'o', id: 'existing'},
patch: {value: Db.set('updated')},
expected: {value: 'old'},
},
]),
),
).toEqual([{'[applied]': true}]);
expectExpiresIn(await expiresAt('default_ttl_probe_rows', 'added'), DEFAULT_TTL_SECONDS);
expectExpiresIn(await expiresAt('default_ttl_probe_rows', 'existing'), DEFAULT_TTL_SECONDS);
});
it('leaves tables without a default untouched', async () => {
await executor.executeQuery(NoTtlProbe.insert({id: 'plain', value: 'a', note: null}));
expect(await expiresAt('no_ttl_probe', 'plain')).toBeNull();
await executor.executeQuery(NoTtlProbe.patchByPk({id: 'plain'}, {note: Db.set('patched')}));
expect(await expiresAt('no_ttl_probe', 'plain')).toBeNull();
await executor.executeQuery(NoTtlProbe.insertWithTtl({id: 'zero', value: 'a', note: null}, 0));
expect(await expiresAt('no_ttl_probe', 'zero')).toBeNull();
});
it('gives rows an older image wrote the expiry of their last write and deletes the ones past it', async () => {
const mentionWrittenAt = await seed('recent_mentions', 'rm-day', '1 day');
await seed('recent_mentions', 'rm-week', '8 days');
await seed('attachment_upload_traces_by_key', 'at-31', '31 days');
await seed('attachment_upload_traces_by_key', 'at-29', '29 days');
await seed('phone_lookup_cache', 'pl-8', '8 days');
await seed('donor_magic_link_tokens', 'dm-hour', '1 hour');
await seed('ipinfo_requests_by_hour', 'ip-day', '1 day');
await seed('jobs_by_id', 'job', '100 days');
await seed('users', 'user', '100 days');
await seed('recent_mentions', 'rm-forever', '1 day', 'infinity');
await seed('recent_mentions', 'rm-hour', '30 days', new Date(Date.now() + 3_600_000));
expect(await expireLegacyDefaultTtlRows(raw, Date.now() + 60_000)).toEqual({
deleted: 4,
expiring: 3,
complete: true,
});
expect(await remaining()).toEqual([
{table_name: 'attachment_upload_traces_by_key', row_key: 'at-29'},
{table_name: 'ipinfo_requests_by_hour', row_key: 'ip-day'},
{table_name: 'jobs_by_id', row_key: 'job'},
{table_name: 'recent_mentions', row_key: 'rm-day'},
{table_name: 'recent_mentions', row_key: 'rm-forever'},
{table_name: 'recent_mentions', row_key: 'rm-hour'},
{table_name: 'users', row_key: 'user'},
]);
const exact = await raw.query<{row_key: string; exact: boolean; unchanged: boolean | null}>(
`SELECT row_key,
expires_at = updated_at + CASE table_name WHEN 'recent_mentions' THEN interval '7 days' WHEN 'attachment_upload_traces_by_key' THEN interval '30 days' ELSE interval '90 days' END AS exact,
CASE WHEN row_key = 'rm-day' THEN updated_at = $1::timestamptz END AS unchanged
FROM ${KV_TABLE}
WHERE row_key IN ('rm-day', 'at-29', 'ip-day')
ORDER BY row_key`,
[mentionWrittenAt],
);
expect(exact.rows).toEqual([
{row_key: 'at-29', exact: true, unchanged: null},
{row_key: 'ip-day', exact: true, unchanged: null},
{row_key: 'rm-day', exact: true, unchanged: true},
]);
const untouched = await raw.query<{row_key: string; state: string}>(
`SELECT row_key, CASE WHEN expires_at IS NULL THEN 'unset' WHEN expires_at = 'infinity' THEN 'forever' ELSE 'set' END AS state
FROM ${KV_TABLE}
WHERE row_key IN ('job', 'user', 'rm-forever', 'rm-hour')
ORDER BY row_key`,
);
expect(untouched.rows).toEqual([
{row_key: 'job', state: 'unset'},
{row_key: 'rm-forever', state: 'forever'},
{row_key: 'rm-hour', state: 'set'},
{row_key: 'user', state: 'unset'},
]);
expect(await markerCount()).toBe(0);
expect(await expireLegacyDefaultTtlRows(raw, Date.now() + 60_000)).toEqual({
deleted: 0,
expiring: 0,
complete: true,
});
expect(await markerCount()).toBe(1);
expect(await expireLegacyDefaultTtlRows(raw, Date.now() + 60_000)).toBeNull();
});
it('checks again a day after a clean pass', async () => {
expect(await expireLegacyDefaultTtlRows(raw, Date.now() + 60_000)).toEqual({
deleted: 0,
expiring: 0,
complete: true,
});
expect(await expireLegacyDefaultTtlRows(raw, Date.now() + 60_000)).toBeNull();
await seed('recent_mentions', 'rm-rolled-back', '1 day');
expect(await expireLegacyDefaultTtlRows(raw, Date.now() + 60_000)).toBeNull();
const before = await raw.query(`SELECT expires_at FROM ${KV_TABLE} WHERE row_key = 'rm-rolled-back'`);
expect(before.rows).toEqual([{expires_at: null}]);
await ageMarker();
expect(await expireLegacyDefaultTtlRows(raw, Date.now() + 60_000)).toEqual({
deleted: 0,
expiring: 1,
complete: true,
});
expect(await expireLegacyDefaultTtlRows(raw, Date.now() + 60_000)).toEqual({
deleted: 0,
expiring: 0,
complete: true,
});
expect(await expireLegacyDefaultTtlRows(raw, Date.now() + 60_000)).toBeNull();
});
it('pages through more rows than one page holds and stops at its deadline', async () => {
await raw.query(
`INSERT INTO ${KV_TABLE} (table_name, partition_key, row_key, row_data, updated_at)
SELECT 'recent_mentions', 'rm-' || lpad(g::text, 5, '0'), 'rm-' || lpad(g::text, 5, '0'), '{}'::jsonb, now() - interval '1 day'
FROM generate_series(1, 2300) g`,
);
expect(await expireLegacyDefaultTtlRows(raw, Date.now() - 1)).toEqual({
deleted: 0,
expiring: 0,
complete: false,
});
expect(await markerCount()).toBe(0);
expect(await expireLegacyDefaultTtlRows(raw, Date.now() + 60_000)).toEqual({
deleted: 0,
expiring: 2300,
complete: true,
});
const unset = await raw.query<{n: number}>(
`SELECT count(*)::int AS n FROM ${KV_TABLE} WHERE table_name = 'recent_mentions' AND expires_at IS NULL`,
);
expect(unset.rows[0]).toEqual({n: 0});
expect(await expireLegacyDefaultTtlRows(raw, Date.now() + 60_000)).toEqual({
deleted: 0,
expiring: 0,
complete: true,
});
expect(await expireLegacyDefaultTtlRows(raw, Date.now() + 60_000)).toBeNull();
});
it('saves where a run stopped and starts the next run there', async () => {
const first = DEFAULT_TTL_TABLES[0]!.name;
const last = DEFAULT_TTL_TABLES.at(-1)!.name;
await seed(first, 'a', '1 hour');
await seed(first, 'z', '1 hour');
await seed(last, 'k', '1 hour');
expect(await expireLegacyDefaultTtlRows(raw, Date.now() - 1)).toEqual({deleted: 0, expiring: 0, complete: false});
expect(await resumePoint()).toEqual({table: first, row_key: '', unset: 0});
await raw.query(
`UPDATE ${KV_TABLE} SET row_data = jsonb_build_object('table', $1::text, 'row_key', 'm', 'unset', 0) WHERE table_name = '__fluxer_schema_migrations' AND row_key = $2`,
[first, DEFAULT_TTL_EXPIRY_RESUME],
);
expect(await expireLegacyDefaultTtlRows(raw, Date.now() + 60_000)).toEqual({
deleted: 0,
expiring: 2,
complete: true,
});
const untouched = await raw.query<{expires_at: Date | null}>(
`SELECT expires_at FROM ${KV_TABLE} WHERE table_name = $1 AND row_key = 'a'`,
[first],
);
expect(untouched.rows).toEqual([{expires_at: null}]);
expect(await resumePoint()).toBeNull();
expect(await markerCount()).toBe(0);
expect(await expireLegacyDefaultTtlRows(raw, Date.now() + 60_000)).toEqual({
deleted: 0,
expiring: 1,
complete: true,
});
expect(await expireLegacyDefaultTtlRows(raw, Date.now() + 60_000)).toEqual({
deleted: 0,
expiring: 0,
complete: true,
});
expect(await markerCount()).toBe(1);
});
});
@@ -0,0 +1,142 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import {
POSTGRES_KV_MIGRATION_TABLE,
postgresKvPassIsFresh,
recordPostgresKvCleanPass,
} from '@app/api/database/PostgresKvQueryExecutor';
import * as DonationTables from '@app/api/donation/DonationTables';
import * as Tables from '@app/api/Tables';
import {IPINFO_CACHE_TTL_SECONDS, IPINFO_REQUEST_AUDIT_TTL_SECONDS} from '@pkgs/geoip/src/PostgresIpInfoKv';
import {type IPostgresClient, quoteIdentifier} from '@pkgs/postgres/src/Client';
import {ms} from 'itty-time';
const DEFAULT_TTL_EXPIRY_MARKER = 'default_ttl_expiry_v1';
export const DEFAULT_TTL_EXPIRY_RESUME = 'default_ttl_expiry_v1_resume';
const PAGE_SIZE = 2000;
const CLEAN_PASS_INTERVAL_MS = ms('1 day');
const OWN_EXPIRY_PASS = new Set<string>([Tables.JobsById.name, Tables.JobsByDayBucket.name]);
export const DEFAULT_TTL_TABLES: ReadonlyArray<{name: string; defaultTtlSeconds: number}> = [
...[...Object.values(Tables), ...Object.values(DonationTables)].flatMap((table) =>
table.defaultTtlSeconds === undefined || OWN_EXPIRY_PASS.has(table.name)
? []
: [{name: table.name, defaultTtlSeconds: table.defaultTtlSeconds}],
),
{name: 'ipinfo_cache', defaultTtlSeconds: IPINFO_CACHE_TTL_SECONDS},
{name: 'ipinfo_requests_by_hour', defaultTtlSeconds: IPINFO_REQUEST_AUDIT_TTL_SECONDS},
];
export interface LegacyDefaultTtlExpiryResult {
deleted: number;
expiring: number;
complete: boolean;
}
interface ResumePoint {
table: string;
rowKey: string;
unset: number;
}
async function readResumePoint(client: IPostgresClient, kvTable: string): Promise<ResumePoint | null> {
const result = await client.query<{row_data: Record<string, unknown>}>(
`SELECT row_data FROM ${kvTable} WHERE table_name = $1 AND row_key = $2`,
[POSTGRES_KV_MIGRATION_TABLE, DEFAULT_TTL_EXPIRY_RESUME],
);
const data = result.rows[0]?.row_data;
if (typeof data?.table !== 'string' || typeof data.row_key !== 'string' || typeof data.unset !== 'number') {
return null;
}
return {table: data.table, rowKey: data.row_key, unset: data.unset};
}
async function writeResumePoint(client: IPostgresClient, kvTable: string, point: ResumePoint | null): Promise<void> {
if (point === null) {
await client.query(`DELETE FROM ${kvTable} WHERE table_name = $1 AND row_key = $2`, [
POSTGRES_KV_MIGRATION_TABLE,
DEFAULT_TTL_EXPIRY_RESUME,
]);
return;
}
await client.query(
`INSERT INTO ${kvTable} (table_name, partition_key, row_key, row_data)
VALUES ($1, $2, $2, jsonb_build_object('table', $3::text, 'row_key', $4::text, 'unset', $5::bigint))
ON CONFLICT (table_name, row_key) DO UPDATE SET row_data = EXCLUDED.row_data, updated_at = now()`,
[POSTGRES_KV_MIGRATION_TABLE, DEFAULT_TTL_EXPIRY_RESUME, point.table, point.rowKey, point.unset],
);
}
function pageSql(table: string): string {
return `
WITH page AS (
SELECT kv.row_key, kv.expires_at IS NULL AS unset
FROM ${table} kv
WHERE kv.table_name = $1 AND kv.row_key > $2
ORDER BY kv.row_key
LIMIT $3
), removed AS (
DELETE FROM ${table} kv
USING page
WHERE kv.table_name = $1 AND kv.row_key = page.row_key AND kv.expires_at IS NULL
AND kv.updated_at + make_interval(secs => $4::double precision) <= now()
RETURNING 1
), expiring AS (
UPDATE ${table} kv
SET expires_at = kv.updated_at + make_interval(secs => $4::double precision)
FROM page
WHERE kv.table_name = $1 AND kv.row_key = page.row_key AND kv.expires_at IS NULL
AND kv.updated_at + make_interval(secs => $4::double precision) > now()
RETURNING 1
)
SELECT
(SELECT max(row_key) FROM page) AS last_row_key,
(SELECT count(*) FROM page WHERE unset) AS unset,
(SELECT count(*) FROM removed) AS deleted,
(SELECT count(*) FROM expiring) AS expiring`;
}
export async function expireLegacyDefaultTtlRows(
client: IPostgresClient,
deadlineMs: number,
): Promise<LegacyDefaultTtlExpiryResult | null> {
if (await postgresKvPassIsFresh(client, DEFAULT_TTL_EXPIRY_MARKER, CLEAN_PASS_INTERVAL_MS)) {
return null;
}
const kvTable = quoteIdentifier(client.kvTable());
const sql = pageSql(kvTable);
const resume = await readResumePoint(client, kvTable);
const resumeIndex = resume === null ? -1 : DEFAULT_TTL_TABLES.findIndex((target) => target.name === resume.table);
let unset = resumeIndex < 0 ? 0 : resume!.unset;
let deleted = 0;
let expiring = 0;
for (let index = Math.max(resumeIndex, 0); index < DEFAULT_TTL_TABLES.length; index += 1) {
const target = DEFAULT_TTL_TABLES[index]!;
let cursor = index === resumeIndex ? resume!.rowKey : '';
for (;;) {
if (Date.now() >= deadlineMs) {
await writeResumePoint(client, kvTable, {table: target.name, rowKey: cursor, unset});
return {deleted, expiring, complete: false};
}
const result = await client.query<{
last_row_key: string | null;
unset: string;
deleted: string;
expiring: string;
}>(sql, [target.name, cursor, PAGE_SIZE, target.defaultTtlSeconds]);
const page = result.rows[0];
if (!page || page.last_row_key === null) {
break;
}
unset += Number(page.unset);
deleted += Number(page.deleted);
expiring += Number(page.expiring);
cursor = page.last_row_key;
}
}
await writeResumePoint(client, kvTable, null);
if (unset === 0) {
await recordPostgresKvCleanPass(client, DEFAULT_TTL_EXPIRY_MARKER);
}
return {deleted, expiring, complete: true};
}
@@ -89,6 +89,28 @@ const NUMERIC_ROW_KEY_NUMBER_PATTERN = '^(-?[0-9]+(?:\\.[0-9]+)?(?:[eE][-+]?[0-9
const EXPIRED_STORED_ROW = 'kv.expires_at IS NOT NULL AND kv.expires_at <= now()';
const MERGED_ROW_DATA = `CASE WHEN ${EXPIRED_STORED_ROW} THEN EXCLUDED.row_data ELSE kv.row_data || EXCLUDED.row_data END`;
const KEPT_EXPIRES_AT = `CASE WHEN ${EXPIRED_STORED_ROW} THEN NULL ELSE kv.expires_at END`;
const NO_EXPIRY = 'infinity';
export async function postgresKvPassIsFresh(
client: IPostgresClient,
marker: string,
maxAgeMs: number,
): Promise<boolean> {
const result = await client.query(
`SELECT 1 FROM ${quoteIdentifier(client.kvTable())} WHERE table_name = $1 AND row_key = $2 AND (row_data ->> 'applied_at')::timestamptz > now() - make_interval(secs => $3::double precision)`,
[POSTGRES_KV_MIGRATION_TABLE, marker, maxAgeMs / 1000],
);
return result.rows.length > 0;
}
export async function recordPostgresKvCleanPass(client: IPostgresClient, marker: string): Promise<void> {
await client.query(
`INSERT INTO ${quoteIdentifier(client.kvTable())} (table_name, partition_key, row_key, row_data)
VALUES ($1, $2, $2, jsonb_build_object('applied_at', now()))
ON CONFLICT (table_name, row_key) DO UPDATE SET row_data = EXCLUDED.row_data, updated_at = now()`,
[POSTGRES_KV_MIGRATION_TABLE, marker],
);
}
function numericRowKeyExpr(column: string): string {
return `(COALESCE(substring(${column} from '${NUMERIC_ROW_KEY_BIGINT_PATTERN}'), substring(${column} from '${NUMERIC_ROW_KEY_NUMBER_PATTERN}'))::numeric)`;
@@ -333,20 +355,21 @@ function projectRow(row: Row, columns: ReadonlyArray<string> | undefined): Row {
return projected;
}
function rowComparator(meta: KvQueryMeta): (left: Row, right: Row) => number {
if (meta.orderBy) {
const column = meta.orderBy.col as string;
const direction = meta.orderBy.direction === 'DESC' ? -1 : 1;
return (left, right) => compareValues(left[column], right[column]) * direction;
function compareColumns(columns: ReadonlyArray<string>, left: Row, right: Row): number {
for (const column of columns) {
const cmp = compareValues(left[column], right[column]);
if (cmp !== 0) return cmp;
}
const columns = meta.table.primaryKey as ReadonlyArray<string>;
return (left, right) => {
for (const column of columns) {
const cmp = compareValues(left[column], right[column]);
if (cmp !== 0) return cmp;
}
return 0;
};
return 0;
}
function rowComparator(meta: KvQueryMeta): (left: Row, right: Row) => number {
const primaryKey = meta.table.primaryKey as ReadonlyArray<string>;
if (!meta.orderBy) return (left, right) => compareColumns(primaryKey, left, right);
const column = meta.orderBy.col as string;
const columns = [column, ...primaryKey.slice(primaryKey.indexOf(column) + 1)];
const direction = meta.orderBy.direction === 'DESC' ? -1 : 1;
return (left, right) => compareColumns(columns, left, right) * direction;
}
function sortRows(meta: KvQueryMeta, rows: Array<Row>): Array<Row> {
@@ -679,7 +702,7 @@ function logFullScan(meta: KvQueryMeta): void {
logWarn({table: meta.table.name, action: meta.action, where: shape.summary || 'none'}, 'Postgres KV full table scan');
}
function ttlExpiresAt(meta: KvQueryMeta, params: CassandraParams): Date | null | undefined {
function ttlExpiresAt(meta: KvQueryMeta, params: CassandraParams): Date | typeof NO_EXPIRY | null | undefined {
const ttlParam = meta.ttlParamName;
if (!ttlParam) return undefined;
const ttlRaw = params[ttlParam];
@@ -687,7 +710,13 @@ function ttlExpiresAt(meta: KvQueryMeta, params: CassandraParams): Date | null |
throw new Error(`TTL parameter ${ttlParam} must be a number`);
}
const ttlSeconds = validateTtlSeconds(ttlRaw);
return ttlSeconds === 0 ? null : new Date(Date.now() + ttlSeconds * 1000);
if (ttlSeconds === 0) return meta.table.defaultTtlSeconds === undefined ? null : NO_EXPIRY;
return new Date(Date.now() + ttlSeconds * 1000);
}
function defaultExpiresAt(meta: KvQueryMeta): Date | undefined {
const ttlSeconds = meta.table.defaultTtlSeconds;
return ttlSeconds === undefined ? undefined : new Date(Date.now() + ttlSeconds * 1000);
}
function encodePageState(pageState: PageState): string {
@@ -1191,7 +1220,8 @@ export class PostgresKvQueryExecutor {
'kv_del_expired',
);
}
const expiresAt = ttlExpiresAt(meta, params) ?? null;
const explicit = ttlExpiresAt(meta, params);
const expiresAt = explicit === undefined ? (defaultExpiresAt(meta) ?? null) : explicit;
const result = await db.query(
`INSERT INTO ${this.table} AS kv (table_name, partition_key, row_key, row_data, expires_at, updated_at)
VALUES ($1, $2, $3, $4::jsonb, $5, now())
@@ -1244,10 +1274,14 @@ WHERE NOT $6`,
}
bindings.push(JSON.stringify(encodeRow(paramsRow(params, meta.patchKeys))));
const assignments = [`row_data = kv.row_data || $${bindings.length}::jsonb`, 'updated_at = now()'];
const expiresAt = ttlExpiresAt(meta, params);
if (expiresAt !== undefined) {
bindings.push(expiresAt);
const explicit = ttlExpiresAt(meta, params);
const fallback = explicit === undefined ? defaultExpiresAt(meta) : undefined;
if (explicit !== undefined) {
bindings.push(explicit);
assignments.push(`expires_at = $${bindings.length}`);
} else if (fallback !== undefined) {
bindings.push(fallback);
assignments.push(`expires_at = GREATEST(kv.expires_at, $${bindings.length}::timestamptz)`);
}
sql = `UPDATE ${this.table} kv SET ${assignments.join(', ')} WHERE ${where}`;
}
@@ -1346,15 +1380,27 @@ WHERE NOT $6`,
for (const column of meta.patchKeys ?? []) {
incoming[column] = column in params ? params[column] : null;
}
const ttl = ttlExpiresAt(meta, params);
const expiresAtExpr = ttl === undefined ? KEPT_EXPIRES_AT : 'EXCLUDED.expires_at';
const explicit = ttlExpiresAt(meta, params);
const fallback = explicit === undefined ? defaultExpiresAt(meta) : undefined;
const [expiresAtExpr, statementName] =
explicit !== undefined
? ['EXCLUDED.expires_at', 'kv_patch_set_ttl']
: fallback !== undefined
? ['GREATEST(kv.expires_at, EXCLUDED.expires_at)', 'kv_patch_default_ttl']
: [KEPT_EXPIRES_AT, 'kv_patch_keep_ttl'];
await db.query(
`INSERT INTO ${this.table} AS kv (table_name, partition_key, row_key, row_data, expires_at, updated_at)
VALUES ($1, $2, $3, $4::jsonb, $5, now())
ON CONFLICT (table_name, row_key)
DO UPDATE SET partition_key = EXCLUDED.partition_key, row_data = ${MERGED_ROW_DATA}, expires_at = ${expiresAtExpr}, updated_at = now()`,
[meta.table.name, partitionKey(meta, incoming), key, JSON.stringify(encodeRow(incoming)), ttl ?? null],
ttl === undefined ? 'kv_patch_keep_ttl' : 'kv_patch_set_ttl',
[
meta.table.name,
partitionKey(meta, incoming),
key,
JSON.stringify(encodeRow(incoming)),
explicit ?? fallback ?? null,
],
statementName,
);
}
@@ -52,6 +52,8 @@ const Composite: KvTableSpec<Row> = {
partitionKey: ['owner_id'],
};
const Expiring: KvTableSpec<Row> = {...Composite, name: 'stmt_expiring', defaultTtlSeconds: 600};
const Bucketed: KvTableSpec<Row> = {
name: 'stmt_bucketed',
columns: ['bucket', 'item_id', 'payload'],
@@ -118,6 +120,7 @@ async function runShapes(): Promise<Array<Statement>> {
meta(Composite, 'patch', [eq('owner_id'), eq('item_id')], {patchKeys: ['payload'], ttlParamName: 'ttl_'}),
{...OWNER_ITEM, ttl_: 600} as CassandraParams,
],
[meta(Expiring, 'patch', [eq('owner_id'), eq('item_id')], {patchKeys: ['payload']}), OWNER_ITEM],
];
for (const [kvMeta, params] of cases) {
await executor.executeQuery({cql: `__stmt_${kvMeta.action}`, params, kvMeta: kvMeta as KvQueryMeta});
@@ -142,6 +145,7 @@ describe('PostgresKvQueryExecutor statement names', () => {
'kv_del_keys',
'kv_del_rowkeys',
'kv_get_row',
'kv_patch_default_ttl',
'kv_patch_keep_ttl',
'kv_patch_set_ttl',
'kv_sel_range',
@@ -231,6 +235,17 @@ async function exerciseKvShapes(executor: PostgresKvQueryExecutor): Promise<void
kvMeta: meta(Composite, 'select', [eq('owner_id'), eq('item_id')]) as KvQueryMeta,
});
expect(patched.map((row) => row.payload)).toEqual(['patched']);
await executor.executeQuery({
cql: '__stmt_patch_default_ttl',
params: {owner_id: 'o5', item_id: 'i5', payload: 'defaulted'} as CassandraParams,
kvMeta: meta(Expiring, 'patch', [eq('owner_id'), eq('item_id')], {patchKeys: ['payload']}) as KvQueryMeta,
});
const defaulted = await executor.executeQuery<Row>({
cql: '__stmt_point',
params: {owner_id: 'o5', item_id: 'i5'} as CassandraParams,
kvMeta: meta(Expiring, 'select', [eq('owner_id'), eq('item_id')]) as KvQueryMeta,
});
expect(defaulted.map((row) => row.payload)).toEqual(['defaulted']);
await executor.executeQuery({
cql: '__stmt_delete',
params: {owner_id: 'o0', item_id: 'i0'} as CassandraParams,
@@ -323,6 +338,7 @@ describe.skipIf(!dockerAvailable)('PostgresKvQueryExecutor statement names again
'kv_del_expired',
'kv_del_rowkeys',
'kv_get_row',
'kv_patch_default_ttl',
'kv_patch_keep_ttl',
'kv_patch_set_ttl',
'kv_sel_range',
@@ -0,0 +1,90 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import {spawnSync} from 'node:child_process';
import {createServer} from 'node:net';
import {startDockerContainer} from '@app/api/test/DockerTestContainer';
import {getDefaultPostgresClient, initPostgres, shutdownPostgres} from '@pkgs/postgres/src/Client';
import {afterAll, beforeAll, describe, expect, it} from 'vitest';
const CONTAINER = `fluxer-kvscram-${process.pid.toString(36)}-${Date.now().toString(36)}`;
const dockerAvailable = spawnSync('docker', ['version'], {stdio: 'ignore'}).status === 0;
const SCRAM_ITERATIONS = 200_000;
async function sleep(ms: number): Promise<void> {
await new Promise((resolve) => setTimeout(resolve, ms));
}
async function freePort(): Promise<number> {
return new Promise((resolve, reject) => {
const server = createServer();
server.on('error', reject);
server.listen(0, '127.0.0.1', () => {
const address = server.address();
if (typeof address === 'string' || address === null) {
reject(new Error('no port'));
return;
}
const port = address.port;
server.close(() => resolve(port));
});
});
}
describe.skipIf(!dockerAvailable)('postgres client against a server with raised SCRAM iterations', () => {
let port: number;
beforeAll(async () => {
port = await freePort();
startDockerContainer([
'run',
'-d',
'--name',
CONTAINER,
'-e',
'POSTGRES_USER=fluxer',
'-e',
'POSTGRES_PASSWORD=fluxer',
'-e',
'POSTGRES_DB=fluxer',
'-p',
`127.0.0.1:${port}:5432`,
'postgres:16-alpine',
'-c',
'fsync=off',
'-c',
`scram_iterations=${SCRAM_ITERATIONS}`,
]);
let ready = false;
for (let attempt = 0; attempt < 180 && !ready; attempt += 1) {
await sleep(500);
const probe = spawnSync(
'docker',
['exec', CONTAINER, 'psql', '-h', '127.0.0.1', '-U', 'fluxer', '-d', 'fluxer', '-Atc', 'SELECT 1'],
{stdio: 'ignore'},
);
ready = probe.status === 0;
}
if (!ready) throw new Error('postgres never came up');
const rehash = spawnSync(
'docker',
['exec', CONTAINER, 'psql', '-U', 'fluxer', '-d', 'fluxer', '-Atc', "ALTER ROLE fluxer PASSWORD 'fluxer'"],
{
stdio: 'ignore',
},
);
if (rehash.status !== 0) throw new Error('could not re-hash the role password');
}, 900_000);
afterAll(async () => {
await shutdownPostgres().catch(() => {});
spawnSync('docker', ['rm', '-f', CONTAINER], {stdio: 'ignore'});
});
it('connects when the role verifier uses more iterations than the driver default allows', async () => {
await initPostgres({url: `postgres://fluxer:[email protected]:${port}/fluxer`, maxConnections: 1});
const verifier = await getDefaultPostgresClient().query<{rolpassword: string}>(
"SELECT rolpassword FROM pg_authid WHERE rolname = 'fluxer'",
);
expect(verifier.rows[0]?.rolpassword.startsWith(`SCRAM-SHA-256$${SCRAM_ITERATIONS}:`)).toBe(true);
});
});
@@ -15,7 +15,12 @@ import type {GuildFolderIcon, MentionReplyPreference} from '@fluxer/constants/sr
import type {types} from 'cassandra-driver';
type Nullish<T> = T | null;
export type PushSubscriptionPlatform = 'web_push' | 'android_fcm' | 'ios_apns' | 'android_unified_push';
export type PushSubscriptionPlatform =
| 'web_push'
| 'android_fcm'
| 'ios_apns'
| 'ios_apns_voip'
| 'android_unified_push';
export interface UserRow {
user_id: UserID;
@@ -13,6 +13,7 @@ import {
type DonorMagicLinkTokenRow,
type DonorRow,
} from '@app/api/database/types/DonationTypes';
import {seconds} from 'itty-time';
export const Donors = defineTable<DonorRow, 'email'>({
name: 'donors',
@@ -43,9 +44,11 @@ export const DonorMagicLinkTokens = defineTable<DonorMagicLinkTokenRow, 'token_'
name: 'donor_magic_link_tokens',
columns: DONOR_MAGIC_LINK_TOKEN_COLUMNS,
primaryKey: ['token_'],
defaultTtlSeconds: seconds('15 minutes'),
});
export const DonorMagicLinkTokensByEmail = defineTable<DonorMagicLinkTokenByEmailRow, 'donor_email' | 'token_'>({
name: 'donor_magic_link_tokens_by_email',
columns: DONOR_MAGIC_LINK_TOKEN_BY_EMAIL_COLUMNS,
primaryKey: ['donor_email', 'token_'],
defaultTtlSeconds: seconds('15 minutes'),
});
@@ -8,7 +8,6 @@ import {RateLimitConfigs} from '@app/api/RateLimitConfig';
import type {HonoApp} from '@app/api/types/HonoEnv';
import {entityTagMatches} from '@app/api/utils/EntityTag';
import {Headers as HttpHeaders} from '@fluxer/constants/src/Headers';
import {resolveScreenShareDeliveryAssignment} from '@fluxer/schema/src/domains/admin/ScreenShareDeliverySchemas';
import {resolveVoiceNoiseSuppressionAssignment} from '@fluxer/schema/src/domains/admin/VoiceNoiseSuppressionSchemas';
import {ExperimentAssignmentsResponse} from '@fluxer/schema/src/domains/experiment/ExperimentSchemas';
@@ -29,10 +28,9 @@ export function ExperimentController(app: HonoApp) {
}),
async (ctx) => {
const instanceConfigRepository = ctx.get('instanceConfigRepository');
const [delivery, voiceConfig, screenShareConfig] = await Promise.all([
const [delivery, voiceConfig] = await Promise.all([
instanceConfigRepository.getExperimentDeliveryConfig(),
instanceConfigRepository.getVoiceNoiseSuppressionConfig(),
instanceConfigRepository.getScreenShareDeliveryConfig(),
]);
const userId = ctx.get('user').id.toString();
const body: ExperimentAssignmentsResponse = {
@@ -40,7 +38,6 @@ export function ExperimentController(app: HonoApp) {
poll_jitter_percent: delivery.poll_jitter_percent,
assignments: {
voice_noise_suppression: resolveVoiceNoiseSuppressionAssignment(voiceConfig, userId),
screen_share_delivery: resolveScreenShareDeliveryAssignment(screenShareConfig, userId),
},
};
const etag = `"${createHash('sha256').update(JSON.stringify(body)).digest('hex')}"`;
@@ -6,10 +6,6 @@ import {type ApiTestHarness, createApiTestHarness} from '@app/api/test/ApiTestHa
import {HTTP_STATUS} from '@app/api/test/TestConstants';
import {createBuilder, createBuilderWithoutAuth} from '@app/api/test/TestRequestBuilder';
import {AdminACLs} from '@fluxer/constants/src/AdminACLs';
import {
DEFAULT_SCREEN_SHARE_DELIVERY_CONFIG,
INERT_SCREEN_SHARE_DELIVERY_ASSIGNMENT,
} from '@fluxer/schema/src/domains/admin/ScreenShareDeliverySchemas';
import {
DEFAULT_VOICE_NOISE_SUPPRESSION_CONFIG,
INERT_VOICE_NOISE_SUPPRESSION_ASSIGNMENT,
@@ -19,7 +15,6 @@ import {
DEFAULT_EXPERIMENT_POLL_JITTER_PERCENT,
type ExperimentAssignmentsResponse,
type ExperimentDeliveryConfigResponse,
readScreenShareDeliveryAssignment,
readVoiceNoiseSuppressionAssignment,
} from '@fluxer/schema/src/domains/experiment/ExperimentSchemas';
import {afterAll, beforeAll, beforeEach, describe, expect, it} from 'vitest';
@@ -56,7 +51,6 @@ describe('GET /experiments', () => {
poll_jitter_percent: DEFAULT_EXPERIMENT_POLL_JITTER_PERCENT,
assignments: {
voice_noise_suppression: INERT_VOICE_NOISE_SUPPRESSION_ASSIGNMENT,
screen_share_delivery: INERT_SCREEN_SHARE_DELIVERY_ASSIGNMENT,
},
});
});
@@ -88,52 +82,6 @@ describe('GET /experiments', () => {
expect(readVoiceNoiseSuppressionAssignment(body).enabled).toBe(false);
});
it('populates the screen share assignment key even when the rollout is disabled', async () => {
const account = await createTestAccount(harness);
const body = await createBuilder<ExperimentAssignmentsResponse>(harness, account.token).get(ENDPOINT).execute();
expect(Object.hasOwn(body.assignments, 'screen_share_delivery')).toBe(true);
expect(readScreenShareDeliveryAssignment(body).enabled).toBe(false);
});
it('resolves the screen share caller through the allowlist', async () => {
const targeted = await createTestAccount(harness);
const untargeted = await createTestAccount(harness);
await getInstanceConfigRepository().setScreenShareDeliveryConfig({
...DEFAULT_SCREEN_SHARE_DELIVERY_CONFIG,
enabled: true,
config_version: 4,
rollout_basis_points: 0,
included_user_ids: [targeted.userId],
});
const targetedBody = await createBuilder<ExperimentAssignmentsResponse>(harness, targeted.token)
.get(ENDPOINT)
.execute();
expect(targetedBody.assignments.screen_share_delivery).toEqual({enabled: true});
const untargetedBody = await createBuilder<ExperimentAssignmentsResponse>(harness, untargeted.token)
.get(ENDPOINT)
.execute();
expect(untargetedBody.assignments.screen_share_delivery).toEqual({enabled: false});
});
it('keeps the screen share exclusion ahead of a full rollout', async () => {
const excluded = await createTestAccount(harness);
await getInstanceConfigRepository().setScreenShareDeliveryConfig({
...DEFAULT_SCREEN_SHARE_DELIVERY_CONFIG,
enabled: true,
rollout_basis_points: 10000,
included_user_ids: [excluded.userId],
excluded_user_ids: [excluded.userId],
});
const body = await createBuilder<ExperimentAssignmentsResponse>(harness, excluded.token).get(ENDPOINT).execute();
expect(body.assignments.screen_share_delivery).toEqual({enabled: false});
});
it('serves the delivery cadence from the delivery config and not from the voice config', async () => {
const account = await createTestAccount(harness);
await getInstanceConfigRepository().setExperimentDeliveryConfig({
@@ -248,30 +196,6 @@ describe('GET /experiments', () => {
});
});
it('serves a fresh body once the screen share config changes', async () => {
const account = await createTestAccount(harness);
const first = await createBuilder<ExperimentAssignmentsResponse>(harness, account.token)
.get(ENDPOINT)
.executeWithResponse();
const staleEtag = first.response.headers.get('etag') as string;
await getInstanceConfigRepository().setScreenShareDeliveryConfig({
...DEFAULT_SCREEN_SHARE_DELIVERY_CONFIG,
enabled: true,
config_version: 1,
rollout_basis_points: 10000,
});
const refreshed = await createBuilder<ExperimentAssignmentsResponse>(harness, account.token)
.get(ENDPOINT)
.header('If-None-Match', staleEtag)
.executeWithResponse();
expect(refreshed.response.status).toBe(HTTP_STATUS.OK);
expect(refreshed.response.headers.get('etag')).not.toBe(staleEtag);
expect(refreshed.json?.assignments.screen_share_delivery).toEqual({enabled: true});
});
it('serves a fresh body once the delivery config changes', async () => {
const account = await createTestAccount(harness);
@@ -328,44 +252,6 @@ describe('GET /experiments', () => {
});
});
it('bumps the screen share config version on every admin update without the client sending one', async () => {
const admin = await setUserACLs(harness, await createTestAccount(harness), [
AdminACLs.AUTHENTICATE,
AdminACLs.INSTANCE_CONFIG_VIEW,
AdminACLs.INSTANCE_CONFIG_UPDATE,
]);
const afterFirst = await createBuilder<{screen_share_delivery: {config_version: number; enabled: boolean}}>(
harness,
admin.token,
)
.patch('/admin/instance/config')
.body({screen_share_delivery: {enabled: true, rollout_basis_points: 10000}})
.execute();
expect(afterFirst.screen_share_delivery).toMatchObject({config_version: 1, enabled: true});
const afterSecond = await createBuilder<{screen_share_delivery: {config_version: number; enabled: boolean}}>(
harness,
admin.token,
)
.patch('/admin/instance/config')
.body({screen_share_delivery: {rollout_salt: 'screen-share-delivery-v2'}})
.execute();
expect(afterSecond.screen_share_delivery).toMatchObject({config_version: 2, enabled: true});
const afterEmpty = await createBuilder<{screen_share_delivery: {config_version: number; enabled: boolean}}>(
harness,
admin.token,
)
.patch('/admin/instance/config')
.body({screen_share_delivery: {}})
.execute();
expect(afterEmpty.screen_share_delivery).toMatchObject({config_version: 2, enabled: true});
const body = await createBuilder<ExperimentAssignmentsResponse>(harness, admin.token).get(ENDPOINT).execute();
expect(body.assignments.screen_share_delivery).toEqual({enabled: true});
});
it('leaves the config version alone for an admin update that sets no field', async () => {
const admin = await setUserACLs(harness, await createTestAccount(harness), [
AdminACLs.AUTHENTICATE,
@@ -15,6 +15,8 @@ import {AuditLogActionType} from '@fluxer/constants/src/AuditLogActionType';
import type {IWorkerService} from '@pkgs/worker/src/contracts/IWorkerService';
import {ms} from 'itty-time';
const MESSAGE_DELETE_BATCH_DELAY_MS = ms('30 seconds');
interface MessageDeleteBatchGroup {
logs: Array<GuildAuditLog>;
userId: UserID;
@@ -81,14 +83,15 @@ export class GuildAuditLogService {
}
async scheduleMessageDeleteBatchJob(guildId: GuildID): Promise<void> {
const runAt = new Date(Date.now() + ms('30 seconds'));
const batchWindow = Math.floor(Date.now() / MESSAGE_DELETE_BATCH_DELAY_MS);
await this.workerService.addJob(
'batchGuildAuditLogMessageDeletes',
{guildId: guildId.toString()},
{
jobKey: `batch-audit-log-message-deletes:${guildId}`,
runAt,
jobKey: `batch-audit-log-message-deletes:${guildId}:${batchWindow}`,
runAt: new Date((batchWindow + 2) * MESSAGE_DELETE_BATCH_DELAY_MS),
maxAttempts: 3,
skipLedger: true,
},
);
}
@@ -3,6 +3,7 @@
import {requireEmailVerified} from '@app/api/auth/EmailVerificationUtils';
import type {GuildID, InviteCode, RoleID, UserID} from '@app/api/BrandedTypes';
import {createChannelID, createRoleID} from '@app/api/BrandedTypes';
import {Config} from '@app/api/Config';
import type {ChannelService} from '@app/api/channel/services/ChannelService';
import {assertMutableUserId} from '@app/api/constants/Core';
import type {GuildMemberRow} from '@app/api/database/types/GuildTypes';
@@ -421,6 +422,13 @@ export class GuildMemberOperationsService {
memberCount: guild.memberCount,
accountAgeMs: Date.now() - snowflakeToDate(BigInt(user.id)).getTime(),
};
if (
!Config.abusePolicy.phoneFlagging.enabled &&
(getEffectiveSuspiciousFlags(user) & PHONE_REQUIREMENT_FLAGS) === 0
) {
Logger.info(logContext, 'deferred_phone_gate.skipped_phone_flagging_disabled');
return;
}
if (status !== 'ok') {
const undeferredFlags = getEffectiveSuspiciousFlags({
...user,
@@ -60,7 +60,7 @@ function createService(roleNames: Map<string, string> = new Map()) {
{addJob} as unknown as IWorkerService<WorkerTaskName>,
{dispatchGuild} as unknown as IGatewayService,
);
return {service, createAuditLog, batchDeleteAndCreateAuditLogs, getRole, dispatchGuild};
return {service, createAuditLog, batchDeleteAndCreateAuditLogs, getRole, dispatchGuild, addJob};
}
function overwrites(
@@ -376,3 +376,26 @@ describe('GuildAuditLogService.recordPermissionOverwriteDiff', () => {
expect(dispatchGuild).not.toHaveBeenCalled();
});
});
describe('GuildAuditLogService.scheduleMessageDeleteBatchJob', () => {
it('gives every delete in one 30 second window a single batch job that runs after the window closes, without a ledger row', async () => {
vi.useFakeTimers({toFake: ['Date']});
try {
const {service, addJob} = createService();
for (const at of ['2026-09-21T12:00:00.000Z', '2026-09-21T12:00:29.999Z', '2026-09-21T12:00:40.000Z']) {
vi.setSystemTime(new Date(at));
await service.scheduleMessageDeleteBatchJob(GUILD_ID);
}
const options = addJob.mock.calls.map((call) => call[2] as {jobKey: string; runAt: Date; skipLedger: boolean});
expect(options.every((option) => option.skipLedger)).toBe(true);
expect(options[0]!.jobKey).toBe(options[1]!.jobKey);
expect(options[2]!.jobKey).not.toBe(options[1]!.jobKey);
expect(options[0]!.runAt.getTime()).toBeGreaterThan(new Date('2026-09-21T12:00:29.999Z').getTime());
expect(options[1]!.runAt).toEqual(options[0]!.runAt);
expect(options[2]!.runAt).toEqual(new Date('2026-09-21T12:01:30.000Z'));
expect(options[2]!.runAt.getTime() - options[0]!.runAt.getTime()).toBe(30_000);
} finally {
vi.useRealTimers();
}
});
});
@@ -3,7 +3,7 @@
import {Config} from '@app/api/Config';
import {GatewayRpcMethodError, GatewayRpcMethodErrorCodes} from '@app/api/infrastructure/GatewayRpcError';
import type {IGatewayRpcTransport} from '@app/api/infrastructure/IGatewayRpcTransport';
import type {CallData} from '@app/api/infrastructure/IGatewayService';
import {type CallCaller, type CallData, callCallerRpcParams} from '@app/api/infrastructure/IGatewayService';
import {NatsGatewayRpcTransport} from '@app/api/infrastructure/NatsGatewayRpcTransport';
import {Logger} from '@app/api/Logger';
import {NatsConnectionManager} from '@pkgs/nats/src/NatsConnectionManager';
@@ -128,6 +128,7 @@ export class GatewayRpcClient {
region: string,
ringing: Array<string>,
recipients: Array<string>,
caller?: CallCaller,
): Promise<CallData> {
return this.call<CallData>('call.create', {
channel_id: channelId,
@@ -135,6 +136,7 @@ export class GatewayRpcClient {
region,
ringing,
recipients,
...callCallerRpcParams(caller),
});
}
@@ -142,8 +144,8 @@ export class GatewayRpcClient {
return this.call('call.update_region', {channel_id: channelId, region});
}
async ringCallRecipients(channelId: string, recipients: Array<string>): Promise<boolean> {
return this.call('call.ring', {channel_id: channelId, recipients});
async ringCallRecipients(channelId: string, recipients: Array<string>, caller?: CallCaller): Promise<boolean> {
return this.call('call.ring', {channel_id: channelId, recipients, ...callCallerRpcParams(caller)});
}
async stopRingingCallRecipients(channelId: string, recipients: Array<string>): Promise<boolean> {
@@ -6,16 +6,18 @@ import {SYSTEM_USER_ID} from '@app/api/constants/Core';
import type {GatewayDispatchEvent} from '@app/api/constants/Gateway';
import {GatewayRpcClient} from '@app/api/infrastructure/GatewayRpcClient';
import {GatewayRpcMethodError, GatewayRpcMethodErrorCodes} from '@app/api/infrastructure/GatewayRpcError';
import type {
CallData,
GatewayChannelMention,
GatewayGuildMemoryStats,
GatewayMentionSources,
GatewayMentionSourcesPage,
GatewayNodeStats,
GatewayVoiceStateCounts,
GatewayVoiceStateEntry,
GuildChannelAuthContext,
import {
type CallCaller,
type CallData,
callCallerRpcParams,
type GatewayChannelMention,
type GatewayGuildMemoryStats,
type GatewayMentionSources,
type GatewayMentionSourcesPage,
type GatewayNodeStats,
type GatewayVoiceStateCounts,
type GatewayVoiceStateEntry,
type GuildChannelAuthContext,
} from '@app/api/infrastructure/IGatewayService';
import {Logger} from '@app/api/Logger';
import {APIErrorCodes} from '@fluxer/constants/src/ApiErrorCodes';
@@ -1695,6 +1697,7 @@ export class GatewayService {
region: string,
ringing: Array<string>,
recipients: Array<string>,
caller?: CallCaller,
): Promise<CallData> {
return this.call<CallData>('call.create', {
channel_id: channelId.toString(),
@@ -1702,6 +1705,7 @@ export class GatewayService {
region,
ringing,
recipients,
...callCallerRpcParams(caller),
});
}
@@ -1709,8 +1713,12 @@ export class GatewayService {
return this.call<boolean>('call.update_region', {channel_id: channelId.toString(), region});
}
async ringCallRecipients(channelId: ChannelID, recipients: Array<string>): Promise<boolean> {
return this.call<boolean>('call.ring', {channel_id: channelId.toString(), recipients});
async ringCallRecipients(channelId: ChannelID, recipients: Array<string>, caller?: CallCaller): Promise<boolean> {
return this.call<boolean>('call.ring', {
channel_id: channelId.toString(),
recipients,
...callCallerRpcParams(caller),
});
}
async stopRingingCallRecipients(channelId: ChannelID, recipients: Array<string>): Promise<boolean> {
@@ -24,6 +24,19 @@ export interface CallData {
voice_states: Array<VoiceState>;
}
export interface CallCaller {
id: string;
name: string;
avatar: string | null;
}
export function callCallerRpcParams(caller: CallCaller | undefined): Record<string, unknown> {
if (!caller) {
return {};
}
return {caller_id: caller.id, caller_name: caller.name, caller_avatar: caller.avatar};
}
export interface GatewayGuildMemoryStatsEntry {
node_id: string;
guild_id: string | null;
@@ -381,11 +394,12 @@ export abstract class IGatewayService {
region: string,
ringing: Array<string>,
recipients: Array<string>,
caller?: CallCaller,
): Promise<CallData>;
abstract updateCallRegion(channelId: ChannelID, region: string | null): Promise<boolean>;
abstract ringCallRecipients(channelId: ChannelID, recipients: Array<string>): Promise<boolean>;
abstract ringCallRecipients(channelId: ChannelID, recipients: Array<string>, caller?: CallCaller): Promise<boolean>;
abstract stopRingingCallRecipients(channelId: ChannelID, recipients: Array<string>): Promise<boolean>;
@@ -84,6 +84,41 @@ describe('stripNonJpegImageMetadataForUpload', () => {
});
});
function riffChunk(type: string, data: Uint8Array): Uint8Array {
const out = new Uint8Array(8 + data.length + (data.length & 1));
out.set(textBytes(type), 0);
new DataView(out.buffer).setUint32(4, data.length, true);
out.set(data, 8);
return out;
}
function webp(chunks: ReadonlyArray<Uint8Array>): Uint8Array {
const body = concatBytes(chunks);
const header = concatBytes([textBytes('RIFF'), new Uint8Array(4), textBytes('WEBP')]);
new DataView(header.buffer).setUint32(4, 4 + body.length, true);
return concatBytes([header, body]);
}
describe('stripNonJpegImageMetadataForUpload for WebP', () => {
it('drops EXIF and XMP chunks without re-encoding frames', async () => {
const vp8x = new Uint8Array(10);
vp8x[0] = 0x02 | 0x08 | 0x04;
const anmf = riffChunk('ANMF', new Uint8Array([9, 8, 7]));
const input = webp([
riffChunk('VP8X', vp8x),
riffChunk('ANIM', new Uint8Array(6)),
anmf,
riffChunk('EXIF', textBytes('GPS=1,2')),
riffChunk('XMP ', textBytes('private metadata')),
]);
const stripped = await stripNonJpegImageMetadataForUpload(input, 'image/webp');
const expectedVp8x = new Uint8Array(10);
expectedVp8x[0] = 0x02;
expect(stripped.contentType).toBe('image/webp');
expect(stripped.body).toEqual(webp([riffChunk('VP8X', expectedVp8x), riffChunk('ANIM', new Uint8Array(6)), anmf]));
});
});
describe('buildProcessedMediaObject', () => {
it('leaves non-media objects for plain copy', async () => {
await expect(buildProcessedMediaObject(textBytes('plain text'), 'text/plain')).resolves.toBeNull();
@@ -162,6 +162,8 @@ export async function stripNonJpegImageMetadataForUpload(
contentType: normalizedContentType === 'image/apng' ? 'image/apng' : 'image/png',
};
}
const strippedWebp = isWebp(data) ? stripWebpMetadataChunks(data) : null;
if (strippedWebp) return {body: strippedWebp, contentType: 'image/webp'};
const image = sharp(data, {animated: true});
const metadata = await image.metadata();
switch (metadata.format) {
@@ -242,6 +244,50 @@ function stripPngMetadataChunks(data: Uint8Array): Uint8Array {
return output;
}
const WEBP_CHUNKS_TO_KEEP = new Set(['VP8 ', 'VP8L', 'VP8X', 'ALPH', 'ANIM', 'ANMF', 'ICCP']);
const WEBP_VP8X_EXIF_FLAG = 0x08;
const WEBP_VP8X_XMP_FLAG = 0x04;
function readFourCc(data: Uint8Array, offset: number): string {
return String.fromCharCode(data[offset]!, data[offset + 1]!, data[offset + 2]!, data[offset + 3]!);
}
function readU32LE(data: Uint8Array, offset: number): number {
return (data[offset]! | (data[offset + 1]! << 8) | (data[offset + 2]! << 16) | (data[offset + 3]! << 24)) >>> 0;
}
function isWebp(data: Uint8Array): boolean {
return data.length >= 12 && readFourCc(data, 0) === 'RIFF' && readFourCc(data, 8) === 'WEBP';
}
function stripWebpMetadataChunks(data: Uint8Array): Uint8Array | null {
const riffEnd = Math.min(data.length, 8 + readU32LE(data, 4));
const chunks: Array<Uint8Array> = [];
let offset = 12;
while (offset + 8 <= riffEnd) {
const length = readU32LE(data, offset + 4);
const chunkEnd = offset + 8 + length + (length & 1);
if (offset + 8 + length > riffEnd) return null;
const type = readFourCc(data, offset);
if (WEBP_CHUNKS_TO_KEEP.has(type)) {
const chunk = data.slice(offset, Math.min(chunkEnd, riffEnd));
if (type === 'VP8X' && length > 0) chunk[8] = (chunk[8] ?? 0) & ~(WEBP_VP8X_EXIF_FLAG | WEBP_VP8X_XMP_FLAG);
chunks.push(chunk);
}
offset = chunkEnd;
}
const bodyLength = chunks.reduce((sum, chunk) => sum + chunk.length, 0);
const output = new Uint8Array(12 + bodyLength);
output.set(data.subarray(0, 12));
new DataView(output.buffer).setUint32(4, 4 + bodyLength, true);
let cursor = 12;
for (const chunk of chunks) {
output.set(chunk, cursor);
cursor += chunk.length;
}
return output;
}
function imageExtensionForContentType(contentType: string): string {
if (contentType.includes('svg')) return 'svg';
if (contentType.includes('tiff')) return 'tiff';
@@ -0,0 +1,86 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import {createChannelID} from '@app/api/BrandedTypes';
import {GatewayRpcClient} from '@app/api/infrastructure/GatewayRpcClient';
import {GatewayService} from '@app/api/infrastructure/GatewayService';
import type {IGatewayRpcTransport} from '@app/api/infrastructure/IGatewayRpcTransport';
import {afterEach, describe, expect, it} from 'vitest';
const CHANNEL_ID = createChannelID(12n);
interface RecordedCall {
method: string;
params: Record<string, unknown>;
}
function recordingService(recorded: Array<RecordedCall>): GatewayService {
const transport: IGatewayRpcTransport = {
async call(method: string, params: Record<string, unknown>): Promise<unknown> {
recorded.push({method, params});
return null;
},
async destroy(): Promise<void> {},
};
GatewayRpcClient.createForTests(transport);
return new GatewayService();
}
describe('call rpc caller params', () => {
afterEach(async () => {
await GatewayRpcClient.resetForTests();
});
it('sends the caller to call.create as caller_id, caller_name and caller_avatar', async () => {
const recorded: Array<RecordedCall> = [];
const service = recordingService(recorded);
await service.createCall(CHANNEL_ID, '99', 'automatic', ['2'], ['1', '2'], {
id: '1',
name: 'Elias',
avatar: 'a1b2c3d4',
});
expect(recorded).toHaveLength(1);
expect(recorded[0].method).toBe('call.create');
expect(recorded[0].params.caller_id).toBe('1');
expect(recorded[0].params.caller_name).toBe('Elias');
expect(recorded[0].params.caller_avatar).toBe('a1b2c3d4');
});
it('sends the caller to call.ring as caller_id, caller_name and caller_avatar', async () => {
const recorded: Array<RecordedCall> = [];
const service = recordingService(recorded);
await service.ringCallRecipients(CHANNEL_ID, ['2'], {id: '1', name: 'Elias', avatar: 'a1b2c3d4'});
expect(recorded).toHaveLength(1);
expect(recorded[0].method).toBe('call.ring');
expect(recorded[0].params.caller_id).toBe('1');
expect(recorded[0].params.caller_name).toBe('Elias');
expect(recorded[0].params.caller_avatar).toBe('a1b2c3d4');
});
it('sends caller_avatar as null when the caller has no avatar', async () => {
const recorded: Array<RecordedCall> = [];
const service = recordingService(recorded);
await service.ringCallRecipients(CHANNEL_ID, ['2'], {id: '1', name: 'Elias', avatar: null});
expect(recorded[0].params.caller_avatar).toBeNull();
expect(Object.hasOwn(recorded[0].params, 'caller_avatar')).toBe(true);
});
it('omits every caller key from call.create when no caller was resolved', async () => {
const recorded: Array<RecordedCall> = [];
const service = recordingService(recorded);
await service.createCall(CHANNEL_ID, '99', 'automatic', ['2'], ['1', '2']);
expect(recorded[0].params).toEqual({
channel_id: '12',
message_id: '99',
region: 'automatic',
ringing: ['2'],
recipients: ['1', '2'],
});
});
it('omits every caller key from call.ring when no caller was resolved', async () => {
const recorded: Array<RecordedCall> = [];
const service = recordingService(recorded);
await service.ringCallRecipients(CHANNEL_ID, ['2']);
expect(recorded[0].params).toEqual({channel_id: '12', recipients: ['2']});
});
});
@@ -1,18 +1,23 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import {spawnSync} from 'node:child_process';
import {createHash} from 'node:crypto';
import {createServer} from 'node:net';
import type {CassandraQueryExecutorForTesting} from '@app/api/database/CassandraQueryExecution';
import {setCassandraQueryExecutorForTesting} from '@app/api/database/CassandraQueryExecution';
import type {PreparedQuery} from '@app/api/database/CassandraTypes';
import {ensurePostgresKvSchema, PostgresKvQueryExecutor} from '@app/api/database/PostgresKvQueryExecutor';
import {
INSTANCE_CONFIG_REFRESH_CHANNEL,
INSTANCE_CONFIG_WRITE_ATTEMPTS,
InstanceConfigRepository,
InstanceConfigWriteConflictError,
type InstanceRegistrationConfig,
} from '@app/api/instance/InstanceConfigRepository';
import {InstanceConfigWriteRaceExecutor} from '@app/api/instance/tests/InstanceConfigWriteRaceExecutor';
import {startDockerContainer} from '@app/api/test/DockerTestContainer';
import {InMemoryCassandraQueryExecutor} from '@app/api/test/InMemoryCassandraQueryExecutor';
import {MockKVProvider} from '@app/api/test/mocks/MockKVProvider';
import {
DEFAULT_SCREEN_SHARE_DELIVERY_CONFIG,
type ScreenShareDeliveryConfig,
} from '@fluxer/schema/src/domains/admin/ScreenShareDeliverySchemas';
import {
DEFAULT_VOICE_NOISE_SUPPRESSION_CONFIG,
type VoiceNoiseSuppressionConfig,
@@ -21,14 +26,25 @@ import {
DEFAULT_EXPERIMENT_DELIVERY_CONFIG,
type ExperimentDeliveryConfig,
} from '@fluxer/schema/src/domains/experiment/ExperimentSchemas';
import {afterEach, describe, expect, it, vi} from 'vitest';
import {
getDefaultPostgresClient,
type IPostgresClient,
initPostgres,
shutdownPostgres,
} from '@pkgs/postgres/src/Client';
import {afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi} from 'vitest';
const VOICE_NOISE_SUPPRESSION_CONFIG_KEY = 'voice_noise_suppression_config';
const SCREEN_SHARE_DELIVERY_CONFIG_KEY = 'screen_share_delivery_config';
const EXPERIMENT_DELIVERY_CONFIG_KEY = 'experiment_delivery_config';
const APP_PUBLIC_CONFIG_KEY = 'app_public_config';
const INSTANCE_POLICY_CONFIG_KEY = 'instance_policy_config';
const INSTANCE_INTEGRATIONS_CONFIG_KEY = 'instance_integrations_config';
const REGISTRATION_CONFIG_KEY = 'registration_config';
const REGISTRATION_URLS_KEY = 'registration_urls';
const REGISTRATION_PENDING_APPROVALS_KEY = 'registration_pending_approvals';
const POSTGRES_KV_TABLE = 'kv_instance_config_races';
const POSTGRES_CONTAINER = `fluxer-instance-config-races-${process.pid.toString(36)}-${Date.now().toString(36)}`;
const dockerAvailable = spawnSync('docker', ['version'], {stdio: 'ignore'}).status === 0;
class CountingInMemoryCassandraQueryExecutor extends InMemoryCassandraQueryExecutor {
instanceConfigSelects = 0;
@@ -335,70 +351,6 @@ describe('InstanceConfigRepository', () => {
});
});
it('returns the default screen share delivery config when the key is absent', async () => {
const executor = new CountingInMemoryCassandraQueryExecutor();
setCassandraQueryExecutorForTesting(executor);
const kvProvider = new MockKVProvider();
const repository = createRepository(kvProvider);
await expect(repository.getScreenShareDeliveryConfig()).resolves.toEqual(DEFAULT_SCREEN_SHARE_DELIVERY_CONFIG);
});
it.each([
{name: 'unparseable text', stored: 'not-json'},
{name: 'a json array', stored: '[]'},
{name: 'out-of-range values', stored: '{"rollout_basis_points":99999}'},
{name: 'a non-boolean enabled flag', stored: '{"enabled":"yes"}'},
])('falls back to the default screen share delivery config for $name', async ({stored}) => {
const executor = new CountingInMemoryCassandraQueryExecutor();
setCassandraQueryExecutorForTesting(executor);
const kvProvider = new MockKVProvider();
const repository = createRepository(kvProvider);
await repository.setConfig(SCREEN_SHARE_DELIVERY_CONFIG_KEY, stored);
await expect(repository.getScreenShareDeliveryConfig()).resolves.toEqual(DEFAULT_SCREEN_SHARE_DELIVERY_CONFIG);
});
it('round-trips a stored screen share delivery config', async () => {
const executor = new CountingInMemoryCassandraQueryExecutor();
setCassandraQueryExecutorForTesting(executor);
const kvProvider = new MockKVProvider();
const repository = createRepository(kvProvider);
const config: ScreenShareDeliveryConfig = {
...DEFAULT_SCREEN_SHARE_DELIVERY_CONFIG,
enabled: true,
config_version: 5,
rollout_basis_points: 2500,
rollout_salt: 'screen-share-delivery-v2',
included_user_ids: ['1400000000000000001'],
excluded_user_ids: ['1400000000000000002'],
};
await repository.setScreenShareDeliveryConfig(config);
await expect(repository.getScreenShareDeliveryConfig()).resolves.toEqual(config);
});
it('fills newly added screen share delivery fields from the schema defaults', async () => {
const executor = new CountingInMemoryCassandraQueryExecutor();
setCassandraQueryExecutorForTesting(executor);
const kvProvider = new MockKVProvider();
const repository = createRepository(kvProvider);
await repository.setConfig(
SCREEN_SHARE_DELIVERY_CONFIG_KEY,
JSON.stringify({enabled: true, config_version: 2, rollout_basis_points: 1000}),
);
await expect(repository.getScreenShareDeliveryConfig()).resolves.toEqual({
...DEFAULT_SCREEN_SHARE_DELIVERY_CONFIG,
enabled: true,
config_version: 2,
rollout_basis_points: 1000,
});
});
it('returns the default experiment delivery config when the key is absent', async () => {
const executor = new CountingInMemoryCassandraQueryExecutor();
setCassandraQueryExecutorForTesting(executor);
@@ -471,26 +423,6 @@ describe('InstanceConfigRepository', () => {
});
});
it('publishes a refresh so another repository observes the screen share delivery config', async () => {
const executor = new CountingInMemoryCassandraQueryExecutor();
setCassandraQueryExecutorForTesting(executor);
const kvProvider = new MockKVProvider();
const reader = createRepository(kvProvider);
const writer = createRepository(kvProvider);
await expect(reader.getScreenShareDeliveryConfig()).resolves.toEqual(DEFAULT_SCREEN_SHARE_DELIVERY_CONFIG);
await writer.setScreenShareDeliveryConfig({
...DEFAULT_SCREEN_SHARE_DELIVERY_CONFIG,
enabled: true,
config_version: 1,
});
await vi.waitFor(async () => {
expect(await reader.getScreenShareDeliveryConfig()).toMatchObject({enabled: true, config_version: 1});
});
});
it('uses the registration URL id as the admin-visible registration code', async () => {
const executor = new CountingInMemoryCassandraQueryExecutor();
setCassandraQueryExecutorForTesting(executor);
@@ -512,3 +444,466 @@ describe('InstanceConfigRepository', () => {
});
});
});
async function sleep(ms: number): Promise<void> {
await new Promise((resolve) => setTimeout(resolve, ms));
}
async function freePort(): Promise<number> {
return new Promise((resolve, reject) => {
const server = createServer();
server.on('error', reject);
server.listen(0, '127.0.0.1', () => {
const address = server.address();
if (typeof address === 'string' || address === null) {
reject(new Error('no port'));
return;
}
server.close(() => resolve(address.port));
});
});
}
function describeConcurrentInstanceConfigWrites(prepareBase: () => Promise<CassandraQueryExecutorForTesting>): void {
const pods: Array<InstanceConfigRepository> = [];
let executor: InstanceConfigWriteRaceExecutor;
beforeEach(async () => {
executor = new InstanceConfigWriteRaceExecutor(await prepareBase());
setCassandraQueryExecutorForTesting(executor);
});
afterEach(async () => {
await Promise.all(pods.map((pod) => pod.shutdown()));
pods.length = 0;
});
function createPod(): InstanceConfigRepository {
const pod = new InstanceConfigRepository(new MockKVProvider());
pods.push(pod);
return pod;
}
async function readStoredRegistrationConfig(): Promise<unknown> {
const raw = await executor.readDirectly(REGISTRATION_CONFIG_KEY);
return raw === null ? null : JSON.parse(raw);
}
it('applies two concurrent patches on top of each other instead of dropping one', async () => {
const first = createPod();
const second = createPod();
await first.setRegistrationConfig({mode: 'open', admin_registration_urls_enabled: true});
await second.getRegistrationConfig();
executor.watch(REGISTRATION_CONFIG_KEY);
executor.pauseWritesUntil(2);
await Promise.all([
first.setRegistrationConfig({mode: 'closed'}),
second.setRegistrationConfig({admin_registration_urls_enabled: false}),
]);
expect(executor.events.filter((event) => event === 'write rejected')).toHaveLength(1);
expect(executor.events.filter((event) => event === 'write')).toHaveLength(2);
expect(await readStoredRegistrationConfig()).toEqual({mode: 'closed', admin_registration_urls_enabled: false});
});
it('lets one of two concurrent first writes create the config and applies the other on top', async () => {
const first = createPod();
const second = createPod();
await first.getRegistrationConfig();
await second.getRegistrationConfig();
executor.watch(REGISTRATION_CONFIG_KEY);
executor.pauseWritesUntil(2);
await Promise.all([
first.setRegistrationConfig({mode: 'closed'}),
second.setRegistrationConfig({admin_registration_urls_enabled: false}),
]);
expect(executor.events.filter((event) => event === 'write rejected')).toHaveLength(1);
expect(executor.events.filter((event) => event === 'write')).toHaveLength(2);
expect(await readStoredRegistrationConfig()).toEqual({mode: 'closed', admin_registration_urls_enabled: false});
});
it('re-reads the database, not its stale cache, when a concurrent write lands between its read and its write', async () => {
const stale = createPod();
const other = createPod();
await stale.setRegistrationConfig({mode: 'open', admin_registration_urls_enabled: true});
await stale.getRegistrationConfig();
await other.setRegistrationConfig({mode: 'approval'});
expect(await stale.getRegistrationConfig()).toEqual({mode: 'open', admin_registration_urls_enabled: true});
executor.watch(REGISTRATION_CONFIG_KEY);
let competed = false;
executor.competeBeforeEachWrite(async () => {
if (competed) return;
competed = true;
await executor.writeDirectly(
REGISTRATION_CONFIG_KEY,
JSON.stringify({mode: 'approval', admin_registration_urls_enabled: false}),
);
});
await stale.setRegistrationConfig({mode: 'closed'});
expect(executor.events).toEqual(['read', 'write rejected', 'read', 'write']);
expect(await readStoredRegistrationConfig()).toEqual({mode: 'closed', admin_registration_urls_enabled: false});
});
it('fails loudly and writes nothing once every attempt has lost the race', async () => {
const pod = createPod();
await pod.setRegistrationConfig({mode: 'open', admin_registration_urls_enabled: true});
executor.watch(REGISTRATION_CONFIG_KEY);
let competingWrites = 0;
executor.competeBeforeEachWrite(async () => {
competingWrites++;
await executor.writeDirectly(
REGISTRATION_CONFIG_KEY,
JSON.stringify({mode: 'approval', admin_registration_urls_enabled: competingWrites % 2 === 0}),
);
});
const write = pod.setRegistrationConfig({mode: 'closed'});
await expect(write).rejects.toBeInstanceOf(InstanceConfigWriteConflictError);
await expect(write).rejects.toMatchObject({
status: 409,
code: 'CONFLICT',
message: expect.stringContaining(REGISTRATION_CONFIG_KEY),
});
expect(executor.events.filter((event) => event === 'write rejected')).toHaveLength(INSTANCE_CONFIG_WRITE_ATTEMPTS);
expect(executor.events).not.toContain('write');
expect(await readStoredRegistrationConfig()).toEqual({
mode: 'approval',
admin_registration_urls_enabled: INSTANCE_CONFIG_WRITE_ATTEMPTS % 2 === 0,
});
});
it('keeps a pending registration another pod added while this pod held a stale list', async () => {
const first = createPod();
const second = createPod();
await first.getPendingRegistrations();
await second.getPendingRegistrations();
await first.addPendingRegistration(pendingRegistration('1400000000000000011'));
await second.addPendingRegistration(pendingRegistration('1400000000000000012'));
const listed = await createPod().getPendingRegistrations();
expect(listed.map((entry) => entry.user_id)).toEqual(['1400000000000000011', '1400000000000000012']);
});
it('keeps a pending registration another pod stored and removes it once decided', async () => {
const pod = createPod();
await executor.writeDirectly(
REGISTRATION_PENDING_APPROVALS_KEY,
JSON.stringify([pendingRegistration('1400000000000000021')]),
);
await pod.addPendingRegistration(pendingRegistration('1400000000000000022'));
expect((await createPod().getPendingRegistrations()).map((entry) => entry.user_id)).toEqual([
'1400000000000000021',
'1400000000000000022',
]);
await pod.removePendingRegistration('1400000000000000021');
await pod.removePendingRegistration('1400000000000000022');
expect(await createPod().getPendingRegistrations()).toEqual([]);
expect(await executor.readDirectly(REGISTRATION_PENDING_APPROVALS_KEY)).toBe('[]');
});
it('keeps a registration URL another pod created while this pod held a stale list', async () => {
const first = createPod();
const second = createPod();
await first.getRegistrationUrls();
await second.getRegistrationUrls();
const created = [
await first.createRegistrationUrl(registrationUrlParams(null)),
await second.createRegistrationUrl(registrationUrlParams(null)),
];
const listed = await createPod().getRegistrationUrlsForAdmin();
expect(listed.map((url) => url.id).toSorted()).toEqual(created.map((entry) => entry.registrationUrl.id).toSorted());
});
it('refuses a registration URL another pod revoked while this pod held a stale list', async () => {
const admin = createPod();
const signup = createPod();
const {code, registrationUrl} = await admin.createRegistrationUrl(registrationUrlParams(null));
expect(await signup.resolveRegistrationUrlCode(code)).not.toBeNull();
await admin.revokeRegistrationUrl(registrationUrl.id);
await expect(signup.resolveRegistrationUrlCode(code)).resolves.toBeNull();
await expect(signup.claimRegistrationUrlUse(registrationUrl.id, '1400000000000000501')).resolves.toBeNull();
const [listed] = await createPod().getRegistrationUrlsForAdmin();
expect(listed?.use_count).toBe(0);
});
it('admits a registration URL another pod created while this pod held a stale list', async () => {
const admin = createPod();
const signup = createPod();
await signup.getRegistrationUrls();
const {code, registrationUrl} = await admin.createRegistrationUrl(registrationUrlParams(1));
await expect(signup.resolveRegistrationUrlCode(code)).resolves.toMatchObject({
id: registrationUrl.id,
approval_required: false,
});
await expect(signup.claimRegistrationUrlUse(registrationUrl.id, '1400000000000000601')).resolves.not.toBeNull();
const [listed] = await createPod().getRegistrationUrlsForAdmin();
expect(listed?.use_count).toBe(1);
});
it('never seats more signups than max_uses when pods claim the same registration URL at once', async () => {
const pods = [createPod(), createPod(), createPod()];
const {code} = await pods[0]!.createRegistrationUrl(registrationUrlParams(2));
const registrationUrl = await pods[0]!.resolveRegistrationUrlCode(code);
if (registrationUrl === null) throw new Error('registration URL did not resolve');
const claims = await Promise.all(
Array.from({length: 6}, (_, index) =>
pods[index % pods.length]!.claimRegistrationUrlUse(registrationUrl.id, `14000000000000001${index}0`),
),
);
expect(claims.filter((claim) => claim !== null)).toHaveLength(2);
const [listed] = await createPod().getRegistrationUrlsForAdmin();
expect(listed?.use_count).toBe(2);
await expect(createPod().resolveRegistrationUrlCode(code)).resolves.toBeNull();
});
it('refuses a claim whose retry finds the registration URL exhausted, rather than reporting the lost attempt', async () => {
const pod = createPod();
const {code, registrationUrl} = await pod.createRegistrationUrl(registrationUrlParams(1));
expect(await pod.resolveRegistrationUrlCode(code)).not.toBeNull();
executor.watch(REGISTRATION_URLS_KEY);
let competed = false;
executor.competeBeforeEachWrite(async () => {
if (competed) return;
competed = true;
const stored = JSON.parse((await executor.readDirectly(REGISTRATION_URLS_KEY)) ?? 'null') as Array<
Record<string, unknown>
>;
await executor.writeDirectly(
REGISTRATION_URLS_KEY,
JSON.stringify(
stored.map((entry) => ({
...entry,
use_count: 1,
last_used_at: '2026-09-20T00:00:00.000Z',
last_used_by_user_id: '1400000000000000901',
})),
),
);
});
await expect(pod.claimRegistrationUrlUse(registrationUrl.id, '1400000000000000902')).resolves.toBeNull();
expect(executor.events).toEqual(['read', 'write rejected', 'read']);
const [listed] = await createPod().getRegistrationUrlsForAdmin();
expect(listed).toMatchObject({use_count: 1, last_used_by_user_id: '1400000000000000901'});
});
it('counts concurrent uses of an uncapped registration URL without ever refusing one', async () => {
const pods = [createPod(), createPod()];
const {code} = await pods[0]!.createRegistrationUrl(registrationUrlParams(null));
const registrationUrl = await pods[0]!.resolveRegistrationUrlCode(code);
if (registrationUrl === null) throw new Error('registration URL did not resolve');
const claims = await Promise.all(
Array.from({length: 5}, (_, index) =>
pods[index % pods.length]!.claimRegistrationUrlUse(registrationUrl.id, `14000000000000002${index}0`),
),
);
expect(claims.every((claim) => claim !== null)).toBe(true);
const [listed] = await createPod().getRegistrationUrlsForAdmin();
expect(listed?.use_count).toBe(5);
});
it('frees a released seat for the next signup', async () => {
const pod = createPod();
const {code} = await pod.createRegistrationUrl(registrationUrlParams(1));
const registrationUrl = await pod.resolveRegistrationUrlCode(code);
if (registrationUrl === null) throw new Error('registration URL did not resolve');
const failedSignup = await pod.claimRegistrationUrlUse(registrationUrl.id, '1400000000000000301');
if (failedSignup === null) throw new Error('the first claim was refused');
await expect(pod.claimRegistrationUrlUse(registrationUrl.id, '1400000000000000302')).resolves.toBeNull();
await pod.releaseRegistrationUrlUse(failedSignup);
await expect(pod.claimRegistrationUrlUse(registrationUrl.id, '1400000000000000303')).resolves.not.toBeNull();
const [listed] = await createPod().getRegistrationUrlsForAdmin();
expect(listed).toMatchObject({use_count: 1, last_used_by_user_id: '1400000000000000303'});
});
it('enforces max_uses against the use count already stored in the blob', async () => {
const pod = createPod();
const id = 'b3c4f0b2-8a6e-4c41-9f55-3f0c2a7d1e91';
await executor.writeDirectly(
REGISTRATION_URLS_KEY,
JSON.stringify([
{
id,
label: 'Issued earlier',
code_hash: createHash('sha256').update(id).digest('hex'),
created_by_user_id: '1400000000000000001',
created_at: '2026-09-01T00:00:00.000Z',
expires_at: null,
max_uses: 3,
use_count: 2,
revoked_at: null,
approval_required: true,
last_used_at: '2026-09-02T00:00:00.000Z',
last_used_by_user_id: '1400000000000000002',
},
]),
);
expect(await pod.getRegistrationUrlsForAdmin()).toEqual([
expect.objectContaining({
id,
use_count: 2,
max_uses: 3,
approval_required: true,
last_used_at: '2026-09-02T00:00:00.000Z',
last_used_by_user_id: '1400000000000000002',
}),
]);
const registrationUrl = await pod.resolveRegistrationUrlCode(id);
if (registrationUrl === null) throw new Error('stored registration URL did not resolve');
expect(registrationUrl).toMatchObject({id, approval_required: true});
await expect(pod.claimRegistrationUrlUse(registrationUrl.id, '1400000000000000401')).resolves.not.toBeNull();
await expect(pod.claimRegistrationUrlUse(registrationUrl.id, '1400000000000000402')).resolves.toBeNull();
const [listed] = await createPod().getRegistrationUrlsForAdmin();
expect(listed).toMatchObject({use_count: 3, max_uses: 3, last_used_by_user_id: '1400000000000000401'});
await expect(pod.resolveRegistrationUrlCode(id)).resolves.toBeNull();
expect(JSON.parse((await executor.readDirectly(REGISTRATION_URLS_KEY)) ?? 'null')[0]).toMatchObject({
use_count: 3,
max_uses: 3,
});
});
it('keeps an SSO field another pod changed while this pod held a stale snapshot', async () => {
const first = createPod();
const second = createPod();
await first.getSsoConfig();
await second.getSsoConfig();
await first.setSsoConfig({displayName: 'Set by the first pod'});
await second.setSsoConfig({clientId: 'set-by-the-second-pod'});
expect(await createPod().getSsoConfig()).toMatchObject({
displayName: 'Set by the first pod',
clientId: 'set-by-the-second-pod',
});
});
it('leaves an SSO row alone when another pod wrote it between this pod reading and writing it', async () => {
const pod = createPod();
await pod.getSsoConfig();
executor.watch('sso_enforced');
let competed = false;
executor.competeBeforeEachWrite(async () => {
if (competed) return;
competed = true;
await executor.writeDirectly('sso_enforced', 'true');
});
await pod.setSsoConfig({displayName: 'Only the display name'});
expect(await executor.readDirectly('sso_enforced')).toBe('true');
expect(await executor.readDirectly('sso_display_name')).toBe('Only the display name');
});
}
function pendingRegistration(userId: string) {
return {
user_id: userId,
username: `pending_${userId.slice(-3)}`,
discriminator: 1,
global_name: null,
email: `${userId}@example.com`,
requested_at: `2026-09-01T00:00:${userId.slice(-2)}.000Z`,
registration_url_id: null,
client_ip: '127.0.0.1',
};
}
function registrationUrlParams(maxUses: number | null) {
return {
label: maxUses === null ? 'Uncapped' : `Capped at ${maxUses}`,
createdByUserId: '1400000000000000001',
expiresAt: null,
maxUses,
approvalRequired: false,
};
}
describe('InstanceConfigRepository concurrent writes', () => {
describe('in memory', () => {
describeConcurrentInstanceConfigWrites(async () => new InMemoryCassandraQueryExecutor());
});
describe.skipIf(!dockerAvailable)('on postgres', () => {
let client: IPostgresClient;
beforeAll(async () => {
const port = await freePort();
startDockerContainer([
'run',
'-d',
'--name',
POSTGRES_CONTAINER,
'-e',
'POSTGRES_USER=fluxer',
'-e',
'POSTGRES_PASSWORD=fluxer',
'-e',
'POSTGRES_DB=fluxer',
'-p',
`127.0.0.1:${port}:5432`,
'postgres:16-alpine',
'-c',
'fsync=off',
]);
let ready = false;
for (let attempt = 0; attempt < 180 && !ready; attempt += 1) {
await sleep(500);
const probe = spawnSync('docker', ['exec', POSTGRES_CONTAINER, 'pg_isready', '-U', 'fluxer', '-d', 'fluxer'], {
stdio: 'ignore',
});
if (probe.status !== 0) continue;
try {
await initPostgres({
url: `postgres://fluxer:[email protected]:${port}/fluxer`,
maxConnections: 4,
kvTable: POSTGRES_KV_TABLE,
});
await getDefaultPostgresClient().query('SELECT 1');
ready = true;
} catch {
await shutdownPostgres().catch(() => {});
}
}
if (!ready) throw new Error('postgres never came up');
client = getDefaultPostgresClient();
await ensurePostgresKvSchema(client);
}, 900_000);
afterAll(async () => {
setCassandraQueryExecutorForTesting(new InMemoryCassandraQueryExecutor());
await shutdownPostgres().catch(() => {});
spawnSync('docker', ['rm', '-f', POSTGRES_CONTAINER], {stdio: 'ignore'});
});
describeConcurrentInstanceConfigWrites(async () => {
await client.query(`DELETE FROM ${POSTGRES_KV_TABLE}`);
return new PostgresKvQueryExecutor(client);
});
});
});
@@ -3,7 +3,8 @@
import crypto from 'node:crypto';
import {Config} from '@app/api/Config';
import type {APIConfig, BlueskyOAuthConfig, BlueskyOAuthKeyConfig} from '@app/api/config/APIConfig';
import {fetchMany, fetchOne, upsertOne} from '@app/api/database/CassandraQueryExecution';
import {executeConditional, fetchMany, fetchOne, upsertOne} from '@app/api/database/CassandraQueryExecution';
import {Db, type PreparedQuery} from '@app/api/database/CassandraTypes';
import type {InstanceConfigurationRow} from '@app/api/database/types/InstanceConfigTypes';
import {
getDefaultDateOfBirthCollection,
@@ -17,6 +18,9 @@ import {resolveDeferredPhoneGateEnabled, setCachedDeferredPhoneGateEnabled} from
import {InstanceConfiguration} from '@app/api/Tables';
import {DEFAULT_DECAY_CONSTANTS, DEFAULT_RENEWAL_CONSTANTS} from '@app/api/utils/AttachmentDecay';
import {isJsonRecord} from '@app/api/utils/JsonBoundaryUtils';
import {APIErrorCodes} from '@fluxer/constants/src/ApiErrorCodes';
import {ConflictError} from '@fluxer/errors/src/domains/core/ConflictError';
import {ServiceUnavailableError} from '@fluxer/errors/src/domains/core/ServiceUnavailableError';
import type {LimitConfigSnapshot} from '@fluxer/limits/src/LimitTypes';
import {
InstanceConfigResponse,
@@ -29,9 +33,9 @@ import {
GatewayRolloutConfigSchema,
} from '@fluxer/schema/src/domains/admin/GatewayRolloutSchemas';
import {
type ScreenShareDeliveryConfig,
ScreenShareDeliveryConfigSchema,
} from '@fluxer/schema/src/domains/admin/ScreenShareDeliverySchemas';
type PushServiceDeliveryConfig,
PushServiceDeliveryConfigSchema,
} from '@fluxer/schema/src/domains/admin/PushServiceDeliverySchemas';
import {
type VoiceNoiseSuppressionConfig,
VoiceNoiseSuppressionConfigSchema,
@@ -58,7 +62,7 @@ import {z} from 'zod';
const GATEWAY_ROLLOUT_CONFIG_KEY = 'gateway_rollout_config';
const VOICE_NOISE_SUPPRESSION_CONFIG_KEY = 'voice_noise_suppression_config';
const SCREEN_SHARE_DELIVERY_CONFIG_KEY = 'screen_share_delivery_config';
const PUSH_SERVICE_DELIVERY_CONFIG_KEY = 'push_service_delivery_config';
const EXPERIMENT_DELIVERY_CONFIG_KEY = 'experiment_delivery_config';
const REGISTRATION_CONFIG_KEY = 'registration_config';
const REGISTRATION_URLS_KEY = 'registration_urls';
@@ -72,6 +76,22 @@ const INSTANCE_MEDIA_CONFIG_KEY = 'instance_media_config';
export const INSTANCE_CONFIG_REFRESH_CHANNEL = 'instance-config-refresh';
export const REGISTRATION_PENDING_APPROVAL_TRAIT = 'registration_pending_approval';
export const REGISTRATION_REJECTED_TRAIT = 'registration_rejected';
export const INSTANCE_CONFIG_WRITE_ATTEMPTS = 5;
export class InstanceConfigWriteConflictError extends ConflictError {
constructor(key: string) {
super({
code: APIErrorCodes.CONFLICT,
message: `Instance config "${key}" changed concurrently on all ${INSTANCE_CONFIG_WRITE_ATTEMPTS} write attempts. Nothing was written. Retry the change.`,
});
this.name = 'InstanceConfigWriteConflictError';
}
}
interface StoredValueUpdate<T> {
value: string | null;
result: T;
}
export type InstanceRegistrationConfig = InstanceRegistration;
@@ -277,6 +297,11 @@ export interface InstanceRegistrationUrl extends RegistrationUrlResponse {
type InstanceRegistrationUrlPublic = RegistrationUrlResponse;
type InstancePendingRegistration = PendingRegistrationResponse;
export interface RegistrationUrlClaim {
registration_url_id: string;
user_id: string;
}
const DEFAULT_REGISTRATION_CONFIG: InstanceRegistrationConfig = {
mode: 'open',
admin_registration_urls_enabled: true,
@@ -344,7 +369,7 @@ type StoredConfigSection =
| 'app public'
| 'gateway rollout'
| 'voice noise suppression'
| 'screen share delivery'
| 'push service delivery'
| 'experiment delivery'
| 'instance policy'
| 'integrations'
@@ -483,8 +508,8 @@ function parseStoredVoiceNoiseSuppressionConfig(raw: string | null): VoiceNoiseS
return parseStoredConfigOrDefault(VoiceNoiseSuppressionConfigSchema, raw, 'voice noise suppression');
}
function parseStoredScreenShareDeliveryConfig(raw: string | null): ScreenShareDeliveryConfig {
return parseStoredConfigOrDefault(ScreenShareDeliveryConfigSchema, raw, 'screen share delivery');
function parseStoredPushServiceDeliveryConfig(raw: string | null): PushServiceDeliveryConfig {
return parseStoredConfigOrDefault(PushServiceDeliveryConfigSchema, raw, 'push service delivery');
}
function parseStoredExperimentDeliveryConfig(raw: string | null): ExperimentDeliveryConfig {
@@ -910,6 +935,75 @@ function parseStoredSsoAllowedEmailDomains(raw: string | undefined, log = false)
return Array.from(domains).slice(0, MAX_SSO_ALLOWED_DOMAINS);
}
function readStoredSsoConfig(
configs: ReadonlyMap<string, string>,
options?: {includeSecret?: boolean},
): InstanceSsoConfig {
const flags = readStoredSsoFlags(configs);
const read = (key: string): string | null => {
const v = configs.get(key);
if (!v) return null;
const trimmed = v.trim();
return trimmed.length === 0 ? null : trimmed;
};
const allowedDomains = parseStoredSsoAllowedEmailDomains(configs.get('sso_allowed_domains'));
const clientSecret = read('sso_client_secret');
return {
...flags,
displayName: read('sso_display_name'),
issuer: read('sso_issuer'),
authorizationUrl: read('sso_authorization_url'),
tokenUrl: read('sso_token_url'),
userInfoUrl: read('sso_userinfo_url'),
jwksUrl: read('sso_jwks_url'),
clientId: read('sso_client_id'),
clientSecret: options?.includeSecret ? clientSecret : undefined,
clientSecretSet: Boolean(clientSecret),
scope: read('sso_scope'),
allowedEmailDomains: allowedDomains,
redirectUri: null,
};
}
interface SsoRowWrite {
key: string;
value: string | undefined;
unset: string;
}
function ssoRow<T>(key: string, value: T | undefined, current: T, format: (value: T) => string): SsoRowWrite {
return {key, value: value === undefined ? undefined : format(value), unset: format(current)};
}
function nextSsoRowValue(row: SsoRowWrite, raw: string | null): string | null {
const value = row.value ?? raw ?? row.unset;
return value === raw ? null : value;
}
function formatSsoBoolean(value: boolean): string {
return value ? 'true' : 'false';
}
function formatSsoString(value: string | null): string {
return value ?? '';
}
function formatSsoDomains(value: Array<string>): string {
return JSON.stringify(value);
}
function normalizeSsoAllowedEmailDomainsForWrite(domains: Array<string>, enabled: boolean): Array<string> {
try {
return normalizeSsoAllowedEmailDomains(domains);
} catch (error) {
if (enabled) {
throw error;
}
Logger.warn({error}, 'Clearing invalid SSO allowed domain config while SSO is disabled');
return [];
}
}
export class InstanceConfigRepository {
private readonly kvClient: IKVProvider | null;
private configCache: InstanceConfigCache;
@@ -993,6 +1087,57 @@ export class InstanceConfigRepository {
);
}
private async updateStoredConfig<T>(key: string, next: (raw: string | null) => T): Promise<T> {
const cache = this.configCache;
const {result} = await this.compareAndSetStoredValue(cache, key, (raw) => {
const config = next(raw);
return {value: JSON.stringify(config), result: config};
});
await this.publishRefresh(cache.sourceId);
return result;
}
private async compareAndSetStoredValue<T>(
cache: InstanceConfigCache,
key: string,
next: (raw: string | null) => StoredValueUpdate<T>,
): Promise<{result: T; written: boolean}> {
await cache.getSnapshot();
for (let attempt = 0; attempt < INSTANCE_CONFIG_WRITE_ATTEMPTS; attempt++) {
cache.assertActive();
const current = await this.fetchConfigForWrite(key);
cache.assertActive();
const {value, result} = next(current);
if (value === null) return {result, written: false};
if (await executeConditional(this.compareAndSetConfig(key, current, value))) {
cache.update(key, value);
return {result, written: true};
}
}
Logger.error(
{key, attempts: INSTANCE_CONFIG_WRITE_ATTEMPTS},
'Instance config write lost to a concurrent write on every attempt',
);
throw new InstanceConfigWriteConflictError(key);
}
private compareAndSetConfig(key: string, current: string | null, value: string): PreparedQuery {
const updatedAt = new Date();
if (current === null) {
return InstanceConfiguration.insertIfNotExists({key, value, updated_at: updatedAt});
}
return InstanceConfiguration.conditionalPatchByPk(
{key},
{value: Db.set(value), updated_at: Db.set(updatedAt)},
{value: current},
);
}
private async fetchConfigForWrite(key: string): Promise<string | null> {
const [row] = await fetchMany<InstanceConfigurationRow>(FETCH_CONFIG_QUERY, {key}, {consistency: 'serial'});
return row?.value ?? null;
}
private async fetchConfigFromDatabase(key: string): Promise<string | null> {
const row = await fetchOne<InstanceConfigurationRow>(FETCH_CONFIG_QUERY, {key});
return row?.value ?? null;
@@ -1014,7 +1159,7 @@ export class InstanceConfigRepository {
parseStoredGatewayRolloutConfig(snapshot.get(GATEWAY_ROLLOUT_CONFIG_KEY) ?? null),
);
parseStoredVoiceNoiseSuppressionConfig(snapshot.get(VOICE_NOISE_SUPPRESSION_CONFIG_KEY) ?? null);
parseStoredScreenShareDeliveryConfig(snapshot.get(SCREEN_SHARE_DELIVERY_CONFIG_KEY) ?? null);
parseStoredPushServiceDeliveryConfig(snapshot.get(PUSH_SERVICE_DELIVERY_CONFIG_KEY) ?? null);
parseStoredExperimentDeliveryConfig(snapshot.get(EXPERIMENT_DELIVERY_CONFIG_KEY) ?? null);
const policy = parseStoredInstancePolicyConfig(snapshot.get(INSTANCE_POLICY_CONFIG_KEY) ?? null);
checkStoredConfig('registration', () =>
@@ -1082,8 +1227,12 @@ export class InstanceConfigRepository {
return parseStoredGatewayRolloutConfig(raw);
}
async setGatewayRolloutConfig(config: GatewayRolloutConfig): Promise<void> {
await this.setConfig(GATEWAY_ROLLOUT_CONFIG_KEY, JSON.stringify(decodeGatewayRolloutConfig(config)));
updateGatewayRolloutConfig(
update: (current: GatewayRolloutConfig) => GatewayRolloutConfig,
): Promise<GatewayRolloutConfig> {
return this.updateStoredConfig(GATEWAY_ROLLOUT_CONFIG_KEY, (raw) =>
decodeGatewayRolloutConfig(update(parseStoredGatewayRolloutConfig(raw))),
);
}
async getVoiceNoiseSuppressionConfig(): Promise<VoiceNoiseSuppressionConfig> {
@@ -1092,18 +1241,36 @@ export class InstanceConfigRepository {
}
async setVoiceNoiseSuppressionConfig(config: VoiceNoiseSuppressionConfig): Promise<void> {
const validated = validateStoredConfig(VoiceNoiseSuppressionConfigSchema, config, 'voice noise suppression');
await this.setConfig(VOICE_NOISE_SUPPRESSION_CONFIG_KEY, JSON.stringify(validated));
await this.updateVoiceNoiseSuppressionConfig(() => config);
}
async getScreenShareDeliveryConfig(): Promise<ScreenShareDeliveryConfig> {
const raw = await this.getConfig(SCREEN_SHARE_DELIVERY_CONFIG_KEY);
return parseStoredScreenShareDeliveryConfig(raw);
updateVoiceNoiseSuppressionConfig(
update: (current: VoiceNoiseSuppressionConfig) => VoiceNoiseSuppressionConfig,
): Promise<VoiceNoiseSuppressionConfig> {
return this.updateStoredConfig(VOICE_NOISE_SUPPRESSION_CONFIG_KEY, (raw) =>
validateStoredConfig(
VoiceNoiseSuppressionConfigSchema,
update(parseStoredVoiceNoiseSuppressionConfig(raw)),
'voice noise suppression',
),
);
}
async setScreenShareDeliveryConfig(config: ScreenShareDeliveryConfig): Promise<void> {
const validated = validateStoredConfig(ScreenShareDeliveryConfigSchema, config, 'screen share delivery');
await this.setConfig(SCREEN_SHARE_DELIVERY_CONFIG_KEY, JSON.stringify(validated));
async getPushServiceDeliveryConfig(): Promise<PushServiceDeliveryConfig> {
const raw = await this.getConfig(PUSH_SERVICE_DELIVERY_CONFIG_KEY);
return parseStoredPushServiceDeliveryConfig(raw);
}
updatePushServiceDeliveryConfig(
update: (current: PushServiceDeliveryConfig) => PushServiceDeliveryConfig,
): Promise<PushServiceDeliveryConfig> {
return this.updateStoredConfig(PUSH_SERVICE_DELIVERY_CONFIG_KEY, (raw) =>
validateStoredConfig(
PushServiceDeliveryConfigSchema,
update(parseStoredPushServiceDeliveryConfig(raw)),
'push service delivery',
),
);
}
async getExperimentDeliveryConfig(): Promise<ExperimentDeliveryConfig> {
@@ -1112,8 +1279,19 @@ export class InstanceConfigRepository {
}
async setExperimentDeliveryConfig(config: ExperimentDeliveryConfig): Promise<void> {
const validated = validateStoredConfig(ExperimentDeliveryConfigSchema, config, 'experiment delivery');
await this.setConfig(EXPERIMENT_DELIVERY_CONFIG_KEY, JSON.stringify(validated));
await this.updateExperimentDeliveryConfig(() => config);
}
updateExperimentDeliveryConfig(
update: (current: ExperimentDeliveryConfig) => ExperimentDeliveryConfig,
): Promise<ExperimentDeliveryConfig> {
return this.updateStoredConfig(EXPERIMENT_DELIVERY_CONFIG_KEY, (raw) =>
validateStoredConfig(
ExperimentDeliveryConfigSchema,
update(parseStoredExperimentDeliveryConfig(raw)),
'experiment delivery',
),
);
}
async readLimitConfigInputs(): Promise<LimitConfigInputs> {
@@ -1149,26 +1327,27 @@ export class InstanceConfigRepository {
legal?: Partial<InstanceAppPublicConfig['legal']>;
registration?: Partial<InstanceAppPublicConfig['registration']>;
}): Promise<InstanceAppPublicConfig> {
const current = await this.getAppPublicConfig();
const next = decodeAppPublicConfig({
branding: {
...current.branding,
...(config.branding ?? {}),
},
setup: {
...current.setup,
...(config.setup ?? {}),
},
legal: {
...current.legal,
...(config.legal ?? {}),
},
registration: {
...current.registration,
...(config.registration ?? {}),
},
const next = await this.updateStoredConfig(APP_PUBLIC_CONFIG_KEY, (raw) => {
const current = parseStoredAppPublicConfig(raw);
return decodeAppPublicConfig({
branding: {
...current.branding,
...(config.branding ?? {}),
},
setup: {
...current.setup,
...(config.setup ?? {}),
},
legal: {
...current.legal,
...(config.legal ?? {}),
},
registration: {
...current.registration,
...(config.registration ?? {}),
},
});
});
await this.setConfig(APP_PUBLIC_CONFIG_KEY, JSON.stringify(next));
setCachedDateOfBirthCollection(next.registration.collect_date_of_birth);
return next;
}
@@ -1188,10 +1367,21 @@ export class InstanceConfigRepository {
return parseStoredInstancePolicyConfig(raw);
}
async setInstancePolicyConfig(config: Partial<InstancePolicyConfig>): Promise<InstancePolicyConfig> {
const current = await this.readStoredInstancePolicyConfig();
const next = decodeInstancePolicyConfig({...current, ...config});
await this.setConfig(INSTANCE_POLICY_CONFIG_KEY, JSON.stringify(next));
setInstancePolicyConfig(config: Partial<InstancePolicyConfig>): Promise<InstancePolicyConfig> {
return this.updateInstancePolicyConfig(() => config);
}
async updateInstancePolicyConfig(
plan: (current: InstancePolicyConfig) => Partial<InstancePolicyConfig>,
): Promise<InstancePolicyConfig> {
const cache = this.configCache;
const {result: next, written} = await this.compareAndSetStoredValue(cache, INSTANCE_POLICY_CONFIG_KEY, (raw) => {
const current = parseStoredInstancePolicyConfig(raw);
const patch = plan(current);
const config = decodeInstancePolicyConfig({...current, ...patch});
return {value: Object.keys(patch).length === 0 ? null : JSON.stringify(config), result: config};
});
if (written) await this.publishRefresh(cache.sourceId);
setCachedDeferredPhoneGateEnabled(resolveDeferredPhoneGateEnabled(next));
return next;
}
@@ -1201,37 +1391,37 @@ export class InstanceConfigRepository {
return parseStoredInstanceIntegrationsConfig(raw);
}
async setInstanceIntegrationsConfig(config: InstanceIntegrationsConfigPatch): Promise<InstanceIntegrationsConfig> {
const current = await this.getInstanceIntegrationsConfig();
const next = decodeInstanceIntegrationsConfig({
gif: {
...current.gif,
...(config.gif ?? {}),
},
youtube: {
...current.youtube,
...(config.youtube ?? {}),
},
captcha: {
...current.captcha,
...(config.captcha ?? {}),
},
email: {
...current.email,
...(config.email ?? {}),
smtp: {
...current.email.smtp,
...(config.email?.smtp ?? {}),
setInstanceIntegrationsConfig(config: InstanceIntegrationsConfigPatch): Promise<InstanceIntegrationsConfig> {
return this.updateStoredConfig(INSTANCE_INTEGRATIONS_CONFIG_KEY, (raw) => {
const current = parseStoredInstanceIntegrationsConfig(raw);
return decodeInstanceIntegrationsConfig({
gif: {
...current.gif,
...(config.gif ?? {}),
},
},
bluesky: {
...current.bluesky,
...(config.bluesky ?? {}),
keys: config.bluesky?.keys ?? current.bluesky.keys,
},
youtube: {
...current.youtube,
...(config.youtube ?? {}),
},
captcha: {
...current.captcha,
...(config.captcha ?? {}),
},
email: {
...current.email,
...(config.email ?? {}),
smtp: {
...current.email.smtp,
...(config.email?.smtp ?? {}),
},
},
bluesky: {
...current.bluesky,
...(config.bluesky ?? {}),
keys: config.bluesky?.keys ?? current.bluesky.keys,
},
});
});
await this.setConfig(INSTANCE_INTEGRATIONS_CONFIG_KEY, JSON.stringify(next));
return next;
}
async getInstanceMediaConfig(): Promise<InstanceMediaConfig> {
@@ -1239,16 +1429,16 @@ export class InstanceConfigRepository {
return parseStoredInstanceMediaConfig(raw);
}
async setInstanceMediaConfig(config: InstanceMediaConfigPatch): Promise<InstanceMediaConfig> {
const current = await this.getInstanceMediaConfig();
const next = decodeInstanceMediaConfig({
attachment_decay: {
...current.attachment_decay,
...(config.attachment_decay ?? {}),
},
setInstanceMediaConfig(config: InstanceMediaConfigPatch): Promise<InstanceMediaConfig> {
return this.updateStoredConfig(INSTANCE_MEDIA_CONFIG_KEY, (raw) => {
const current = parseStoredInstanceMediaConfig(raw);
return decodeInstanceMediaConfig({
attachment_decay: {
...current.attachment_decay,
...(config.attachment_decay ?? {}),
},
});
});
await this.setConfig(INSTANCE_MEDIA_CONFIG_KEY, JSON.stringify(next));
return next;
}
async getEffectiveAttachmentDecayConfig(): Promise<InstanceAttachmentDecayEffectiveConfig> {
@@ -1485,15 +1675,15 @@ export class InstanceConfigRepository {
return parseStoredRegistrationConfig(raw);
}
async setRegistrationConfig(config: Partial<InstanceRegistrationConfig>): Promise<InstanceRegistrationConfig> {
const current = await this.getRegistrationConfig();
const next = decodeRegistrationConfig({
mode: config.mode ?? current.mode,
admin_registration_urls_enabled:
config.admin_registration_urls_enabled ?? current.admin_registration_urls_enabled,
setRegistrationConfig(config: Partial<InstanceRegistrationConfig>): Promise<InstanceRegistrationConfig> {
return this.updateStoredConfig(REGISTRATION_CONFIG_KEY, (raw) => {
const current = parseStoredRegistrationConfig(raw);
return decodeRegistrationConfig({
mode: config.mode ?? current.mode,
admin_registration_urls_enabled:
config.admin_registration_urls_enabled ?? current.admin_registration_urls_enabled,
});
});
await this.setConfig(REGISTRATION_CONFIG_KEY, JSON.stringify(next));
return next;
}
async getRegistrationPublicConfig(): Promise<InstanceRegistrationConfig> {
@@ -1534,16 +1724,20 @@ export class InstanceConfigRepository {
last_used_at: null,
last_used_by_user_id: null,
};
const registrationUrls = await this.getRegistrationUrls();
await this.setRegistrationUrls([registrationUrl, ...registrationUrls]);
await this.updateStoredConfig(REGISTRATION_URLS_KEY, (raw) =>
validateStoredCollection(
StoredRegistrationUrlSchema,
[registrationUrl, ...parseStoredCollection(StoredRegistrationUrlSchema, raw, 'registration URLs')],
'registration URLs',
),
);
return {registrationUrl: redactRegistrationUrl(registrationUrl), code};
}
async revokeRegistrationUrl(id: string): Promise<void> {
const now = new Date().toISOString();
const registrationUrls = await this.getRegistrationUrls();
await this.setRegistrationUrls(
registrationUrls.map((registrationUrl) =>
await this.updateStoredConfig(REGISTRATION_URLS_KEY, (raw) =>
parseStoredCollection(StoredRegistrationUrlSchema, raw, 'registration URLs').map((registrationUrl) =>
registrationUrl.id === id && !registrationUrl.revoked_at
? {...registrationUrl, revoked_at: now}
: registrationUrl,
@@ -1556,9 +1750,8 @@ export class InstanceConfigRepository {
if (!normalizedCode) return null;
const hash = this.hashRegistrationUrlCode(normalizedCode);
const now = new Date();
const registrationUrls = await this.getRegistrationUrls();
return (
registrationUrls.find(
(await this.fetchRegistrationUrlDefinitions()).find(
(registrationUrl) =>
(registrationUrl.id === normalizedCode || registrationUrl.code_hash === hash) &&
isRegistrationUrlUsable(registrationUrl, now),
@@ -1566,21 +1759,68 @@ export class InstanceConfigRepository {
);
}
async recordRegistrationUrlUse(id: string, userId: string): Promise<void> {
const now = new Date().toISOString();
const registrationUrls = await this.getRegistrationUrls();
await this.setRegistrationUrls(
registrationUrls.map((registrationUrl) =>
registrationUrl.id === id
? {
...registrationUrl,
use_count: registrationUrl.use_count + 1,
last_used_at: now,
last_used_by_user_id: userId,
}
: registrationUrl,
),
);
async claimRegistrationUrlUse(registrationUrlId: string, userId: string): Promise<RegistrationUrlClaim | null> {
const cache = this.configCache;
let claimed: {result: RegistrationUrlClaim | null; written: boolean};
try {
claimed = await this.compareAndSetStoredValue<RegistrationUrlClaim | null>(
cache,
REGISTRATION_URLS_KEY,
(raw) => {
const registrationUrls = parseStoredCollection(StoredRegistrationUrlSchema, raw, 'registration URLs');
const now = new Date();
const claimable = registrationUrls.find(
(registrationUrl) =>
registrationUrl.id === registrationUrlId && isRegistrationUrlUsable(registrationUrl, now),
);
if (!claimable) return {value: null, result: null};
const next = registrationUrls.map((registrationUrl) =>
registrationUrl === claimable
? {
...registrationUrl,
use_count: registrationUrl.use_count + 1,
last_used_at: now.toISOString(),
last_used_by_user_id: userId,
}
: registrationUrl,
);
return {
value: JSON.stringify(validateStoredCollection(StoredRegistrationUrlSchema, next, 'registration URLs')),
result: {registration_url_id: registrationUrlId, user_id: userId},
};
},
);
} catch (error) {
if (error instanceof InstanceConfigWriteConflictError) throw new ServiceUnavailableError();
throw error;
}
if (claimed.written) await this.publishRefresh(cache.sourceId);
return claimed.result;
}
async releaseRegistrationUrlUse(claim: RegistrationUrlClaim): Promise<void> {
const cache = this.configCache;
try {
const {written} = await this.compareAndSetStoredValue<null>(cache, REGISTRATION_URLS_KEY, (raw) => {
const registrationUrls = parseStoredCollection(StoredRegistrationUrlSchema, raw, 'registration URLs');
const released = registrationUrls.find(
(registrationUrl) => registrationUrl.id === claim.registration_url_id && registrationUrl.use_count > 0,
);
if (!released) return {value: null, result: null};
const next = registrationUrls.map((registrationUrl) =>
registrationUrl === released
? {...registrationUrl, use_count: registrationUrl.use_count - 1}
: registrationUrl,
);
return {value: JSON.stringify(next), result: null};
});
if (written) await this.publishRefresh(cache.sourceId);
} catch (error) {
Logger.warn(
{registrationUrlId: claim.registration_url_id, userId: claim.user_id, error},
'Releasing a registration URL use failed',
);
}
}
async getPendingRegistrations(): Promise<Array<InstancePendingRegistration>> {
@@ -1591,104 +1831,82 @@ export class InstanceConfigRepository {
}
async addPendingRegistration(pendingRegistration: InstancePendingRegistration): Promise<void> {
const pendingRegistrations = await this.getPendingRegistrations();
const next = [
pendingRegistration,
...pendingRegistrations.filter((entry) => entry.user_id !== pendingRegistration.user_id),
];
await this.setPendingRegistrations(next);
await this.updateStoredConfig(REGISTRATION_PENDING_APPROVALS_KEY, (raw) =>
validateStoredCollection(
StoredPendingRegistrationSchema,
[
pendingRegistration,
...parseStoredCollection(StoredPendingRegistrationSchema, raw, 'pending registrations').filter(
(entry) => entry.user_id !== pendingRegistration.user_id,
),
],
'pending registrations',
),
);
}
async removePendingRegistration(userId: string): Promise<void> {
const pendingRegistrations = await this.getPendingRegistrations();
await this.setPendingRegistrations(pendingRegistrations.filter((entry) => entry.user_id !== userId));
await this.updateStoredConfig(REGISTRATION_PENDING_APPROVALS_KEY, (raw) =>
parseStoredCollection(StoredPendingRegistrationSchema, raw, 'pending registrations').filter(
(entry) => entry.user_id !== userId,
),
);
}
async getSsoConfig(options?: {includeSecret?: boolean}): Promise<InstanceSsoConfig> {
const configs = await this.getAllConfigs();
const flags = readStoredSsoFlags(configs);
const read = (key: string): string | null => {
const v = configs.get(key);
if (!v) return null;
const trimmed = v.trim();
return trimmed.length === 0 ? null : trimmed;
};
const allowedDomains = parseStoredSsoAllowedEmailDomains(configs.get('sso_allowed_domains'));
const clientSecret = read('sso_client_secret');
return {
...flags,
displayName: read('sso_display_name'),
issuer: read('sso_issuer'),
authorizationUrl: read('sso_authorization_url'),
tokenUrl: read('sso_token_url'),
userInfoUrl: read('sso_userinfo_url'),
jwksUrl: read('sso_jwks_url'),
clientId: read('sso_client_id'),
clientSecret: options?.includeSecret ? clientSecret : undefined,
clientSecretSet: Boolean(clientSecret),
scope: read('sso_scope'),
allowedEmailDomains: allowedDomains,
redirectUri: null,
};
return readStoredSsoConfig(await this.getAllConfigs(), options);
}
async setSsoConfig(config: Partial<InstanceSsoConfig>): Promise<InstanceSsoConfig> {
const current = await this.getSsoConfig({includeSecret: true});
const definedConfig = Object.fromEntries(
Object.entries(config).filter(([, value]) => value !== undefined),
) as Partial<InstanceSsoConfig>;
const next: InstanceSsoConfig = {
...current,
...definedConfig,
clientSecret: config.clientSecret !== undefined ? config.clientSecret : current.clientSecret,
};
if (config.enabled === true && config.enforced === undefined && !current.enabled) {
next.enforced = true;
}
let allowedEmailDomains: Array<string>;
try {
allowedEmailDomains = normalizeSsoAllowedEmailDomains(next.allowedEmailDomains);
} catch (error) {
if (next.enabled) {
throw error;
}
Logger.warn({error}, 'Clearing invalid SSO allowed domain config while SSO is disabled');
allowedEmailDomains = [];
}
const entries: Array<[string, string]> = [
['sso_enabled', next.enabled ? 'true' : 'false'],
['sso_enforced', next.enforced ? 'true' : 'false'],
['sso_display_name', next.displayName ?? ''],
['sso_issuer', next.issuer ?? ''],
['sso_authorization_url', next.authorizationUrl ?? ''],
['sso_token_url', next.tokenUrl ?? ''],
['sso_userinfo_url', next.userInfoUrl ?? ''],
['sso_jwks_url', next.jwksUrl ?? ''],
['sso_client_id', next.clientId ?? ''],
['sso_scope', next.scope ?? ''],
['sso_allowed_domains', JSON.stringify(allowedEmailDomains)],
['sso_auto_provision', next.autoProvision ? 'true' : 'false'],
['sso_redirect_uri', ''],
const configs = await this.getAllConfigs();
const current = readStoredSsoConfig(configs, {includeSecret: true});
const enabled = config.enabled ?? current.enabled;
const allowedEmailDomains =
config.allowedEmailDomains === undefined
? undefined
: normalizeSsoAllowedEmailDomainsForWrite(config.allowedEmailDomains, enabled);
const rows: Array<SsoRowWrite> = [
ssoRow('sso_enabled', config.enabled, current.enabled, formatSsoBoolean),
ssoRow('sso_enforced', config.enforced, current.enforced, formatSsoBoolean),
ssoRow('sso_display_name', config.displayName, current.displayName, formatSsoString),
ssoRow('sso_issuer', config.issuer, current.issuer, formatSsoString),
ssoRow('sso_authorization_url', config.authorizationUrl, current.authorizationUrl, formatSsoString),
ssoRow('sso_token_url', config.tokenUrl, current.tokenUrl, formatSsoString),
ssoRow('sso_userinfo_url', config.userInfoUrl, current.userInfoUrl, formatSsoString),
ssoRow('sso_jwks_url', config.jwksUrl, current.jwksUrl, formatSsoString),
ssoRow('sso_client_id', config.clientId, current.clientId, formatSsoString),
ssoRow('sso_scope', config.scope, current.scope, formatSsoString),
ssoRow('sso_allowed_domains', allowedEmailDomains, current.allowedEmailDomains, formatSsoDomains),
ssoRow('sso_auto_provision', config.autoProvision, current.autoProvision, formatSsoBoolean),
ssoRow('sso_redirect_uri', undefined, null, formatSsoString),
];
if (config.clientSecret !== undefined) {
entries.push(['sso_client_secret', config.clientSecret ?? '']);
rows.push(ssoRow('sso_client_secret', config.clientSecret, current.clientSecret ?? null, formatSsoString));
}
await this.setConfigs(entries);
const cache = this.configCache;
const results = await Promise.allSettled(
rows
.filter((row) => row.value !== undefined || !configs.has(row.key))
.map((row) =>
this.compareAndSetStoredValue(cache, row.key, (raw) => ({value: nextSsoRowValue(row, raw), result: null})),
),
);
const errors: Array<unknown> = results.flatMap((result) => (result.status === 'rejected' ? [result.reason] : []));
if (results.some((result) => result.status === 'fulfilled' && result.value.written)) {
try {
await this.publishRefresh(cache.sourceId);
} catch (error) {
errors.push(error);
}
}
if (errors.length === 1) throw errors[0];
if (errors.length > 1) throw new AggregateError(errors, 'Failed to write or publish the SSO config');
return this.getSsoConfig({includeSecret: true});
}
private async setRegistrationUrls(registrationUrls: Array<InstanceRegistrationUrl>): Promise<void> {
const validated = validateStoredCollection(StoredRegistrationUrlSchema, registrationUrls, 'registration URLs');
await this.setConfig(REGISTRATION_URLS_KEY, JSON.stringify(validated));
}
private async setPendingRegistrations(pendingRegistrations: Array<InstancePendingRegistration>): Promise<void> {
const validated = validateStoredCollection(
StoredPendingRegistrationSchema,
pendingRegistrations,
'pending registrations',
);
await this.setConfig(REGISTRATION_PENDING_APPROVALS_KEY, JSON.stringify(validated));
private async fetchRegistrationUrlDefinitions(): Promise<Array<InstanceRegistrationUrl>> {
const raw = await this.fetchConfigFromDatabase(REGISTRATION_URLS_KEY);
return parseStoredCollection(StoredRegistrationUrlSchema, raw, 'registration URLs');
}
private hashRegistrationUrlCode(code: string): string {
@@ -0,0 +1,30 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import type {PushServiceDeliveryConfig} from '@fluxer/schema/src/domains/admin/PushServiceDeliverySchemas';
import type {INatsConnectionManager} from '@pkgs/nats/src/INatsConnectionManager';
const textEncoder = new TextEncoder();
export const PUSH_SERVICE_DELIVERY_CONFIG_NATS_SUBJECT = 'config.push.delivery';
interface PushServiceDeliveryConfigNatsMessage {
type: 'push_service_delivery_config';
config: PushServiceDeliveryConfig;
}
export class PushServiceDeliveryConfigPublisher {
constructor(private readonly connectionManager: INatsConnectionManager) {}
async publish(config: PushServiceDeliveryConfig): Promise<void> {
if (this.connectionManager.isClosed()) {
await this.connectionManager.connect();
}
const connection = this.connectionManager.getConnection();
const message: PushServiceDeliveryConfigNatsMessage = {
type: 'push_service_delivery_config',
config,
};
connection.publish(PUSH_SERVICE_DELIVERY_CONFIG_NATS_SUBJECT, textEncoder.encode(JSON.stringify(message)));
await connection.flush();
}
}
@@ -0,0 +1,100 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import {getKvMeta} from '@app/api/database/CassandraMetaRegistry';
import type {CassandraQueryExecutorForTesting} from '@app/api/database/CassandraQueryExecution';
import type {CassandraParams, KvQueryMeta, PreparedQuery} from '@app/api/database/CassandraTypes';
import type {InstanceConfigurationRow} from '@app/api/database/types/InstanceConfigTypes';
import {InstanceConfiguration} from '@app/api/Tables';
type InstanceConfigWriteEvent = 'read' | 'write' | 'write rejected';
interface WriteGate {
size: number;
paused: Array<() => void>;
}
const FETCH_ROW_QUERY = InstanceConfiguration.selectCql({
where: InstanceConfiguration.where.eq('key'),
limit: 1,
});
export class InstanceConfigWriteRaceExecutor implements CassandraQueryExecutorForTesting {
readonly events: Array<InstanceConfigWriteEvent> = [];
private watchedKey: string | null = null;
private gate: WriteGate | null = null;
private beforeEachWrite: (() => Promise<void>) | null = null;
constructor(private readonly base: CassandraQueryExecutorForTesting) {}
watch(key: string): void {
this.watchedKey = key;
this.events.length = 0;
}
pauseWritesUntil(size: number): void {
this.gate = {size, paused: []};
}
competeBeforeEachWrite(write: () => Promise<void>): void {
this.beforeEachWrite = write;
}
async writeDirectly(key: string, value: string): Promise<void> {
await this.base.executeQuery(InstanceConfiguration.upsertAll({key, value, updated_at: new Date()}));
}
async readDirectly(key: string): Promise<string | null> {
const [row] = await this.base.executeQuery<InstanceConfigurationRow>({cql: FETCH_ROW_QUERY, params: {key}});
return row?.value ?? null;
}
async executeQuery<T = Record<string, unknown>, P extends CassandraParams = CassandraParams>(
query: PreparedQuery<P>,
): Promise<Array<T>> {
const meta = query.kvMeta ?? getKvMeta(query.cql);
if (this.watchedKey === null || !this.isWatched(meta, query.params)) {
return this.base.executeQuery<T, P>(query);
}
if (meta?.action === 'select') {
this.events.push('read');
return this.base.executeQuery<T, P>(query);
}
await this.passGate();
await this.beforeEachWrite?.();
const rows = await this.base.executeQuery<T, P>(query);
const applied = (rows[0] as {'[applied]'?: unknown} | undefined)?.['[applied]'];
this.events.push(applied === false ? 'write rejected' : 'write');
return rows;
}
executeBatch(queries: Array<{query: string; params: object; meta?: KvQueryMeta}>, atomic?: boolean): Promise<void> {
return this.base.executeBatch(queries, atomic);
}
reset(): void {
this.base.reset?.();
this.watchedKey = null;
this.gate = null;
this.beforeEachWrite = null;
this.events.length = 0;
}
async shutdown(): Promise<void> {
await this.base.shutdown?.();
}
private isWatched(meta: KvQueryMeta | null | undefined, params: CassandraParams): boolean {
return meta?.table.name === InstanceConfiguration.name && params.key === this.watchedKey;
}
private async passGate(): Promise<void> {
const gate = this.gate;
if (gate === null) return;
await new Promise<void>((release) => {
gate.paused.push(release);
if (gate.paused.length < gate.size) return;
this.gate = null;
for (const release of gate.paused) release();
});
}
}
@@ -32,10 +32,12 @@ export interface ListJobsResult {
}
export abstract class IJobLedgerRepository {
abstract createJob(input: CreateJobInput): Promise<void>;
abstract createJob(input: CreateJobInput): Promise<Date>;
abstract getJob(jobId: bigint): Promise<JobByIdRow | null>;
abstract discardJob(jobId: bigint, createdAt: Date): Promise<void>;
abstract markRunning(jobId: bigint, lane: string): Promise<void>;
abstract markSucceeded(jobId: bigint, result: Record<string, unknown> | null): Promise<void>;
@@ -1,9 +1,24 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import {setCassandraQueryExecutorForTesting} from '@app/api/database/CassandraQueryExecution';
import {
type CassandraQueryExecutorForTesting,
executeQuery,
fetchMany,
fetchOne,
setCassandraQueryExecutorForTesting,
} from '@app/api/database/CassandraQueryExecution';
import {Db, type PreparedQuery} from '@app/api/database/CassandraTypes';
import type {JobStatus} from '@app/api/database/types/JobLedgerTypes';
import {JobLedgerRepository} from '@app/api/jobs/JobLedgerRepository';
import {
EXPIRED_JOB_ERROR,
JOB_LEDGER_TTL_SECONDS,
JOB_STALE_AFTER_MS,
JobLedgerRepository,
} from '@app/api/jobs/JobLedgerRepository';
import {describeListJobsPaging} from '@app/api/jobs/ListJobsPagingSuite';
import {JobsActive, JobsByDayBucket, JobsById} from '@app/api/Tables';
import {InMemoryCassandraQueryExecutor} from '@app/api/test/InMemoryCassandraQueryExecutor';
import {createSnowflake} from '@fluxer/snowflake/src/Snowflake';
import {afterEach, beforeEach, describe, expect, it} from 'vitest';
let executor: InMemoryCassandraQueryExecutor;
@@ -22,6 +37,20 @@ async function createJob(repository: JobLedgerRepository, jobId: bigint, taskTyp
});
}
async function createJobAt(repository: JobLedgerRepository, jobId: bigint): Promise<Date> {
return repository.createJob({
jobId,
taskType: 'batchGuildAuditLogMessageDeletes',
payload: {},
requestedByUserId: null,
auditLogReason: null,
maxAttempts: 3,
runAt: null,
jetStreamLane: null,
jetStreamSeq: null,
});
}
async function listJobIdsByStatus(repository: JobLedgerRepository, status: JobStatus): Promise<Array<bigint>> {
const result = await repository.listJobs({limit: 50, cursor: null, filters: {status}, maxLookbackDays: 1});
return result.jobs.map((job) => job.job_id);
@@ -130,6 +159,197 @@ describe('JobLedgerRepository listJobs pagination', () => {
maxLookbackDays: 1,
});
expect(result.jobs.map((job) => job.job_id)).toEqual([1_000n, 1_001n, 1_002n, 1_003n, 1_004n]);
expect(result.jobs.map((job) => job.job_id)).toEqual([1_004n, 1_003n, 1_002n, 1_001n, 1_000n]);
});
});
describe('JobLedgerRepository listJobs on the in-memory executor', () => {
beforeEach(() => {
executor = new InMemoryCassandraQueryExecutor();
setCassandraQueryExecutorForTesting(executor);
});
afterEach(() => {
executor.reset();
setCassandraQueryExecutorForTesting(null);
});
describeListJobsPaging();
});
let staleSequence = 0;
function jobIdAgedDays(days: number): bigint {
staleSequence += 1;
return createSnowflake({timestamp: Date.now() - days * 86_400_000, sequence: staleSequence % 4096, workerId: 1});
}
async function createAgedJob(repository: JobLedgerRepository, days: number): Promise<bigint> {
const jobId = jobIdAgedDays(days);
await createJob(repository, jobId, 'syncUrlBlocklists');
return jobId;
}
async function activeJobIds(repository: JobLedgerRepository): Promise<Array<bigint>> {
return (await repository.listActiveJobs()).map((job) => job.job_id).sort((a, b) => (a < b ? -1 : 1));
}
function sweep(repository: JobLedgerRepository, maxCleared = 100) {
return repository.expireStaleActiveJobs({
staleBeforeMs: Date.now() - JOB_STALE_AFTER_MS,
pageSize: 100,
maxCleared,
});
}
describe('JobLedgerRepository expireStaleActiveJobs', () => {
beforeEach(() => {
executor = new InMemoryCassandraQueryExecutor();
setCassandraQueryExecutorForTesting(executor);
});
afterEach(() => {
executor.reset();
setCassandraQueryExecutorForTesting(null);
});
it('dead-letters queued and running jobs the jobs stream has outlived and keeps younger ones active', async () => {
const repository = new JobLedgerRepository();
const staleQueued = await createAgedJob(repository, 9);
const staleRunning = await createAgedJob(repository, 9);
await repository.markRunning(staleRunning, 'batch');
const weekOld = await createAgedJob(repository, 7);
const fresh = await createAgedJob(repository, 0);
expect(await sweep(repository)).toEqual({cleared: 2, expired: 2, complete: true});
for (const jobId of [staleQueued, staleRunning]) {
const job = await repository.getJob(jobId);
expect(job?.status).toBe('deadletter');
expect(job?.error_message).toBe(EXPIRED_JOB_ERROR);
expect(job?.completed_at).toBeInstanceOf(Date);
}
expect(await activeJobIds(repository)).toEqual([weekOld, fresh].sort((a, b) => (a < b ? -1 : 1)));
expect((await repository.getJob(weekOld))?.status).toBe('queued');
});
it('drops a stale active row without touching a finished or missing job', async () => {
const repository = new JobLedgerRepository();
const finished = await createAgedJob(repository, 9);
await repository.markSucceeded(finished, null);
const orphan = jobIdAgedDays(9);
for (const jobId of [finished, orphan]) {
await executeQuery(
JobsActive.patchByPkWithTtl({job_id: jobId}, {status: Db.set('running')}, JOB_LEDGER_TTL_SECONDS),
);
}
expect(await sweep(repository)).toEqual({cleared: 2, expired: 0, complete: true});
const job = await repository.getJob(finished);
expect(job?.status).toBe('succeeded');
expect(job?.error_message).toBeNull();
expect(await fetchOne(JobsById.select({where: JobsById.where.eq('job_id')}).bind({job_id: orphan}))).toBeNull();
expect(await fetchMany(JobsActive.select().bind({}))).toEqual([]);
});
it('stops at its per-run cap and picks up the rest on the next run', async () => {
const repository = new JobLedgerRepository();
for (let index = 0; index < 3; index += 1) {
await createAgedJob(repository, 9);
}
expect(await sweep(repository, 2)).toEqual({cleared: 2, expired: 2, complete: false});
expect(await activeJobIds(repository)).toHaveLength(1);
expect(await sweep(repository, 2)).toEqual({cleared: 1, expired: 1, complete: true});
expect(await activeJobIds(repository)).toEqual([]);
});
});
describe('JobLedgerRepository getJob', () => {
beforeEach(() => {
executor = new InMemoryCassandraQueryExecutor();
setCassandraQueryExecutorForTesting(executor);
});
afterEach(() => {
executor.reset();
setCassandraQueryExecutorForTesting(null);
});
it('hides a job row that lost its status, creation time or task type', async () => {
const repository = new JobLedgerRepository();
await createJob(repository, 9n, 'syncUrlBlocklists');
expect((await repository.getJob(9n))?.status).toBe('queued');
for (const column of ['status', 'created_at', 'task_type'] as const) {
await createJob(repository, 9n, 'syncUrlBlocklists');
await executeQuery(JobsById.patchByPk({job_id: 9n}, {[column]: Db.clear()}));
expect(await repository.getJob(9n)).toBeNull();
}
});
});
describe('JobLedgerRepository discardJob', () => {
let inner: InMemoryCassandraQueryExecutor;
let log: Array<string>;
let failDeleteOn: string | null;
beforeEach(() => {
inner = new InMemoryCassandraQueryExecutor();
log = [];
failDeleteOn = null;
const wrapper: CassandraQueryExecutorForTesting = {
async executeQuery<T>(query: PreparedQuery) {
const meta = query.kvMeta;
if (meta) log.push(`${meta.action} ${meta.table.name}`);
if (meta?.action === 'delete' && meta.table.name === failDeleteOn) {
throw new Error('write timeout');
}
return inner.executeQuery<T>(query);
},
executeBatch: (queries, atomic) => inner.executeBatch(queries, atomic),
};
setCassandraQueryExecutorForTesting(wrapper);
});
afterEach(() => {
inner.reset();
setCassandraQueryExecutorForTesting(null);
});
it('removes every ledger row of a duplicate with three deletes and no read', async () => {
const repository = new JobLedgerRepository();
const createdAt = await createJobAt(repository, 7n);
log.length = 0;
await repository.discardJob(7n, createdAt);
expect([...log].sort()).toEqual(['delete jobs_active', 'delete jobs_by_day_bucket', 'delete jobs_by_id']);
expect(await repository.getJob(7n)).toBeNull();
expect(await repository.listActiveJobs()).toEqual([]);
expect(
await fetchMany(
JobsByDayBucket.select({where: JobsByDayBucket.where.eq('bucket_day')}).bind({
bucket_day: createdAt.toISOString().slice(0, 10),
}),
),
).toEqual([]);
});
it('keeps deleting the other ledger rows when one delete fails', async () => {
const repository = new JobLedgerRepository();
const createdAt = await createJobAt(repository, 8n);
failDeleteOn = 'jobs_by_id';
await expect(repository.discardJob(8n, createdAt)).rejects.toThrow('write timeout');
expect(await repository.listActiveJobs()).toEqual([]);
expect(
await fetchMany(
JobsByDayBucket.select({where: JobsByDayBucket.where.eq('bucket_day')}).bind({
bucket_day: createdAt.toISOString().slice(0, 10),
}),
),
).toEqual([]);
});
});
+180 -66
View File
@@ -1,6 +1,14 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import {BatchBuilder, deleteOneOrMany, fetchMany, fetchOne, upsertOne} from '@app/api/database/CassandraQueryExecution';
import {
BatchBuilder,
deleteOneOrMany,
fetchMany,
fetchOne,
fetchPage,
type PagedQueryResult,
upsertOne,
} from '@app/api/database/CassandraQueryExecution';
import {Db} from '@app/api/database/CassandraTypes';
import type {JobActiveRow, JobByDayBucketRow, JobByIdRow, JobStatus} from '@app/api/database/types/JobLedgerTypes';
import {
@@ -11,6 +19,17 @@ import {
type ListJobsResult,
} from '@app/api/jobs/IJobLedgerRepository';
import {JobsActive, JobsByDayBucket, JobsById} from '@app/api/Tables';
import {awaitAll} from '@app/api/utils/ConcurrencyUtils';
import {JOBS_STREAM_MAX_AGE_MS} from '@app/api/worker/JetStreamWorkerQueue';
import {snowflakeToDate} from '@fluxer/snowflake/src/Snowflake';
import {ms, seconds} from 'itty-time';
export const JOB_LEDGER_TTL_SECONDS = seconds('90 days');
export const JOB_STALE_AFTER_MS = JOBS_STREAM_MAX_AGE_MS + ms('1 day');
export const EXPIRED_JOB_ERROR = 'Expired from the job queue';
const JOB_LEDGER_RETENTION_DAYS = JOB_LEDGER_TTL_SECONDS / seconds('1 day');
const NEWEST_FIRST = {col: 'created_at', direction: 'DESC'} as const;
const FETCH_JOB_BY_ID_QUERY = JobsById.select({
where: JobsById.where.eq('job_id'),
@@ -19,13 +38,58 @@ const FETCH_CANCEL_REQUESTED_QUERY = JobsById.select({
where: JobsById.where.eq('job_id'),
});
const ACTIVE_JOBS_QUERY = JobsActive.select();
const ACTIVE_JOB_IDS_QUERY = JobsActive.select({columns: ['job_id']});
const JOBS_AFTER_IN_TIE_QUERY = JobsByDayBucket.select({
where: [
JobsByDayBucket.where.eq('bucket_day'),
JobsByDayBucket.where.eq('created_at'),
JobsByDayBucket.where.lt('job_id'),
],
orderBy: NEWEST_FIRST,
});
type LedgerPosition = Pick<ListJobsCursor, 'createdAt' | 'jobId'>;
function bucketDayFor(d: Date): string {
return d.toISOString().slice(0, 10);
}
function previousBucketDay(day: string): string {
const date = new Date(`${day}T00:00:00Z`);
date.setUTCDate(date.getUTCDate() - 1);
return bucketDayFor(date);
}
function dayJobsQuery(olderThanPosition: boolean, limit: number | null) {
return JobsByDayBucket.select({
where: olderThanPosition
? [JobsByDayBucket.where.eq('bucket_day'), JobsByDayBucket.where.lt('created_at')]
: JobsByDayBucket.where.eq('bucket_day'),
orderBy: NEWEST_FIRST,
...(limit === null ? {} : {limit}),
});
}
async function fetchDayAfter(
bucketDay: string,
after: LedgerPosition | null,
limit: number | null,
): Promise<{rows: Array<JobByDayBucketRow>; exhausted: boolean}> {
if (after === null) {
const rows = await fetchMany<JobByDayBucketRow>(dayJobsQuery(false, limit).bind({bucket_day: bucketDay}));
return {rows, exhausted: limit === null || rows.length < limit};
}
const ties = await fetchMany<JobByDayBucketRow>(
JOBS_AFTER_IN_TIE_QUERY.bind({bucket_day: bucketDay, created_at: after.createdAt, job_id: after.jobId}),
);
const older = await fetchMany<JobByDayBucketRow>(
dayJobsQuery(true, limit).bind({bucket_day: bucketDay, created_at: after.createdAt}),
);
return {rows: [...ties, ...older], exhausted: limit === null || older.length < limit};
}
export class JobLedgerRepository extends IJobLedgerRepository {
async createJob(input: CreateJobInput): Promise<void> {
async createJob(input: CreateJobInput): Promise<Date> {
const now = new Date();
const status: JobStatus = 'queued';
const idRow: JobByIdRow = {
@@ -68,39 +132,62 @@ export class JobLedgerRepository extends IJobLedgerRepository {
started_at: null,
};
const batch = new BatchBuilder();
batch.addPrepared(JobsById.insert(idRow));
batch.addPrepared(JobsByDayBucket.insert(bucketRow));
batch.addPrepared(JobsActive.insert(activeRow));
batch.addPrepared(JobsById.insertWithTtl(idRow, JOB_LEDGER_TTL_SECONDS));
batch.addPrepared(JobsByDayBucket.insertWithTtl(bucketRow, JOB_LEDGER_TTL_SECONDS));
batch.addPrepared(JobsActive.insertWithTtl(activeRow, JOB_LEDGER_TTL_SECONDS));
await batch.executeChunked(10, false);
return now;
}
async getJob(jobId: bigint): Promise<JobByIdRow | null> {
return fetchOne<JobByIdRow>(FETCH_JOB_BY_ID_QUERY.bind({job_id: jobId}));
const row = await fetchOne<JobByIdRow>(FETCH_JOB_BY_ID_QUERY.bind({job_id: jobId}));
return row?.created_at && row.task_type && row.status ? row : null;
}
async discardJob(jobId: bigint, createdAt: Date): Promise<void> {
await awaitAll(
[
deleteOneOrMany(
JobsByDayBucket.deleteByPk({bucket_day: bucketDayFor(createdAt), created_at: createdAt, job_id: jobId}),
),
deleteOneOrMany(JobsById.deleteByPk({job_id: jobId})),
deleteOneOrMany(JobsActive.deleteByPk({job_id: jobId})),
],
'Ledger discard left rows behind',
);
}
async markRunning(jobId: bigint, lane: string): Promise<void> {
const startedAt = new Date();
const status: JobStatus = 'running';
await upsertOne(
JobsById.patchByPk(
JobsById.patchByPkWithTtl(
{job_id: jobId},
{status: Db.set(status), started_at: Db.set(startedAt), jet_stream_lane: Db.set(lane)},
JOB_LEDGER_TTL_SECONDS,
),
);
await upsertOne(
JobsActive.patchByPkWithTtl(
{job_id: jobId},
{status: Db.set(status), started_at: Db.set(startedAt)},
JOB_LEDGER_TTL_SECONDS,
),
);
await upsertOne(JobsActive.patchByPk({job_id: jobId}, {status: Db.set(status), started_at: Db.set(startedAt)}));
}
async markSucceeded(jobId: bigint, result: Record<string, unknown> | null): Promise<void> {
const completedAt = new Date();
const status: JobStatus = 'succeeded';
await upsertOne(
JobsById.patchByPk(
JobsById.patchByPkWithTtl(
{job_id: jobId},
{
status: Db.set(status),
completed_at: Db.set(completedAt),
result: result === null ? Db.clear() : Db.set(JSON.stringify(result)),
},
JOB_LEDGER_TTL_SECONDS,
),
);
await deleteOneOrMany(JobsActive.deleteByPk({job_id: jobId}));
@@ -109,7 +196,13 @@ export class JobLedgerRepository extends IJobLedgerRepository {
async markCancelled(jobId: bigint): Promise<void> {
const completedAt = new Date();
const status: JobStatus = 'cancelled';
await upsertOne(JobsById.patchByPk({job_id: jobId}, {status: Db.set(status), completed_at: Db.set(completedAt)}));
await upsertOne(
JobsById.patchByPkWithTtl(
{job_id: jobId},
{status: Db.set(status), completed_at: Db.set(completedAt)},
JOB_LEDGER_TTL_SECONDS,
),
);
await deleteOneOrMany(JobsActive.deleteByPk({job_id: jobId}));
}
@@ -117,9 +210,10 @@ export class JobLedgerRepository extends IJobLedgerRepository {
const completedAt = new Date();
const status: JobStatus = 'deadletter';
await upsertOne(
JobsById.patchByPk(
JobsById.patchByPkWithTtl(
{job_id: jobId},
{status: Db.set(status), completed_at: Db.set(completedAt), error_message: Db.set(errorMessage)},
JOB_LEDGER_TTL_SECONDS,
),
);
await deleteOneOrMany(JobsActive.deleteByPk({job_id: jobId}));
@@ -127,27 +221,30 @@ export class JobLedgerRepository extends IJobLedgerRepository {
async reportProgress(jobId: bigint, current: number, total: number | null, message: string | null): Promise<void> {
await upsertOne(
JobsById.patchByPk(
JobsById.patchByPkWithTtl(
{job_id: jobId},
{
progress_current: Db.set(BigInt(current)),
progress_total: total === null ? Db.clear() : Db.set(BigInt(total)),
progress_message: message === null ? Db.clear() : Db.set(message),
},
JOB_LEDGER_TTL_SECONDS,
),
);
}
async setContextLink(jobId: bigint, link: string): Promise<void> {
await upsertOne(JobsById.patchByPk({job_id: jobId}, {context_link: Db.set(link)}));
await upsertOne(JobsById.patchByPkWithTtl({job_id: jobId}, {context_link: Db.set(link)}, JOB_LEDGER_TTL_SECONDS));
}
async setJetStreamSeq(jobId: bigint, seq: string): Promise<void> {
await upsertOne(JobsById.patchByPk({job_id: jobId}, {jet_stream_seq: Db.set(seq)}));
await upsertOne(JobsById.patchByPkWithTtl({job_id: jobId}, {jet_stream_seq: Db.set(seq)}, JOB_LEDGER_TTL_SECONDS));
}
async requestCancel(jobId: bigint): Promise<void> {
await upsertOne(JobsById.patchByPk({job_id: jobId}, {cancel_requested: Db.set(true)}));
await upsertOne(
JobsById.patchByPkWithTtl({job_id: jobId}, {cancel_requested: Db.set(true)}, JOB_LEDGER_TTL_SECONDS),
);
}
async isCancelRequested(jobId: bigint): Promise<boolean> {
@@ -160,7 +257,9 @@ export class JobLedgerRepository extends IJobLedgerRepository {
async incrementAttempts(jobId: bigint): Promise<void> {
const row = await this.getJob(jobId);
if (!row) return;
await upsertOne(JobsById.patchByPk({job_id: jobId}, {attempts: Db.set(row.attempts + 1)}));
await upsertOne(
JobsById.patchByPkWithTtl({job_id: jobId}, {attempts: Db.set(row.attempts + 1)}, JOB_LEDGER_TTL_SECONDS),
);
}
async listJobs(opts: {
@@ -169,59 +268,74 @@ export class JobLedgerRepository extends IJobLedgerRepository {
filters: ListJobsFilters;
maxLookbackDays: number;
}): Promise<ListJobsResult> {
const {limit, cursor, filters, maxLookbackDays} = opts;
const startBucket = cursor ? new Date(`${cursor.bucketDay}T00:00:00Z`) : new Date();
const hasFilters = Boolean(
filters.status ||
filters.taskType ||
(filters.requestedByUserId !== undefined && filters.requestedByUserId !== null),
);
const collected: Array<JobByIdRow> = [];
let nextCursor: ListJobsCursor | null = null;
for (let dayOffset = 0; dayOffset <= maxLookbackDays && collected.length < limit; dayOffset++) {
const bucketDate = new Date(startBucket);
bucketDate.setUTCDate(bucketDate.getUTCDate() - dayOffset);
const bucketDay = bucketDayFor(bucketDate);
const remaining = limit - collected.length + 1;
const bucketLimit = hasFilters ? {} : {limit: remaining};
const useCursor = dayOffset === 0 && cursor !== null;
let bucketRows: Array<JobByDayBucketRow>;
if (useCursor && cursor) {
const query = JobsByDayBucket.select({
where: [JobsByDayBucket.where.eq('bucket_day'), JobsByDayBucket.where.lt('created_at')],
...bucketLimit,
});
bucketRows = await fetchMany<JobByDayBucketRow>(
query.bind({bucket_day: bucketDay, created_at: cursor.createdAt}),
);
} else {
const query = JobsByDayBucket.select({
where: JobsByDayBucket.where.eq('bucket_day'),
...bucketLimit,
});
bucketRows = await fetchMany<JobByDayBucketRow>(query.bind({bucket_day: bucketDay}));
const {limit, cursor, filters} = opts;
const now = Date.now();
const lookbackDays = Math.min(opts.maxLookbackDays, JOB_LEDGER_RETENTION_DAYS);
const oldestDay = bucketDayFor(new Date(now - lookbackDays * ms('1 day')));
const requestedBy = filters.requestedByUserId ?? null;
const wholeDays = Boolean(filters.status || filters.taskType || requestedBy !== null);
const jobs: Array<JobByIdRow> = [];
let lastRow: JobByDayBucketRow | null = null;
let day = bucketDayFor(new Date(Math.min(cursor ? cursor.createdAt.getTime() : now, now)));
let after: LedgerPosition | null = cursor;
while (jobs.length < limit && day >= oldestDay) {
const {rows, exhausted} = await fetchDayAfter(day, after, wholeDays ? null : limit - jobs.length);
for (const row of rows) {
after = {createdAt: row.created_at, jobId: row.job_id};
if (filters.taskType && row.task_type !== filters.taskType) continue;
if (requestedBy !== null && row.requested_by_user_id !== requestedBy) continue;
const job = await this.getJob(row.job_id);
if (!job || (filters.status && job.status !== filters.status)) continue;
jobs.push(job);
lastRow = row;
if (jobs.length === limit) break;
}
for (const r of bucketRows) {
if (filters.taskType && r.task_type !== filters.taskType) continue;
if (filters.requestedByUserId !== undefined && filters.requestedByUserId !== null) {
if (r.requested_by_user_id !== filters.requestedByUserId) continue;
}
const fullRow = await this.getJob(r.job_id);
if (!fullRow) continue;
if (filters.status && fullRow.status !== filters.status) continue;
if (collected.length >= limit) {
nextCursor = {bucketDay, createdAt: r.created_at, jobId: r.job_id};
break;
}
collected.push(fullRow);
if (exhausted) {
day = previousBucketDay(day);
after = null;
}
if (nextCursor) break;
}
if (nextCursor === null && collected.length >= limit) {
const last = collected[collected.length - 1];
nextCursor = {bucketDay: bucketDayFor(last.created_at), createdAt: last.created_at, jobId: last.job_id};
}
return {jobs: collected, nextCursor};
return {
jobs,
nextCursor:
lastRow && jobs.length === limit
? {bucketDay: lastRow.bucket_day, createdAt: lastRow.created_at, jobId: lastRow.job_id}
: null,
};
}
async expireStaleActiveJobs(opts: {
staleBeforeMs: number;
pageSize: number;
maxCleared: number;
}): Promise<{cleared: number; expired: number; complete: boolean}> {
let cleared = 0;
let expired = 0;
let pageState: string | null = null;
do {
const page: PagedQueryResult<Pick<JobActiveRow, 'job_id'>> = await fetchPage(
ACTIVE_JOB_IDS_QUERY.bind({}),
undefined,
{
pageSize: opts.pageSize,
pageState,
},
);
for (const {job_id: jobId} of page.rows) {
if (snowflakeToDate(jobId).getTime() >= opts.staleBeforeMs) continue;
if (cleared >= opts.maxCleared) return {cleared, expired, complete: false};
const job = await this.getJob(jobId);
if (job?.status === 'queued' || job?.status === 'running') {
await this.markDeadletter(jobId, EXPIRED_JOB_ERROR);
expired += 1;
} else {
await deleteOneOrMany(JobsActive.deleteByPk({job_id: jobId}));
}
cleared += 1;
}
pageState = page.pageState;
} while (pageState !== null);
return {cleared, expired, complete: true};
}
async listActiveJobs(): Promise<Array<JobByIdRow>> {
@@ -0,0 +1,228 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import {upsertOne} from '@app/api/database/CassandraQueryExecution';
import type {JobByIdRow, JobStatus} from '@app/api/database/types/JobLedgerTypes';
import type {ListJobsCursor, ListJobsFilters} from '@app/api/jobs/IJobLedgerRepository';
import {JobLedgerRepository} from '@app/api/jobs/JobLedgerRepository';
import {JobsByDayBucket, JobsById} from '@app/api/Tables';
import {afterEach, beforeEach, describe, expect, it, vi} from 'vitest';
const DAY_MS = 86_400_000;
const HOUR_MS = 3_600_000;
const TODAY_NOON = new Date(`${new Date().toISOString().slice(0, 10)}T12:00:00.000Z`);
const ADMIN_USER_ID = 4_242n;
interface ListedPage {
ids: Array<bigint>;
cursor: ListJobsCursor | null;
}
function at(daysAgo: number, hour: number): Date {
return new Date(TODAY_NOON.getTime() - daysAgo * DAY_MS + (hour - 12) * HOUR_MS);
}
async function seedJob(
jobId: bigint,
createdAt: Date,
opts: {taskType?: string; requestedBy?: bigint | null; status?: JobStatus; bucketOnly?: boolean} = {},
): Promise<void> {
const taskType = opts.taskType ?? 'A';
const requestedBy = opts.requestedBy ?? null;
await upsertOne(
JobsByDayBucket.insert({
bucket_day: createdAt.toISOString().slice(0, 10),
created_at: createdAt,
job_id: jobId,
task_type: taskType,
status: 'queued',
requested_by_user_id: requestedBy,
}),
);
if (opts.bucketOnly) return;
const row: JobByIdRow = {
job_id: jobId,
task_type: taskType,
status: opts.status ?? 'queued',
progress_current: null,
progress_total: null,
progress_message: null,
payload: '{}',
result: null,
error_message: null,
created_at: createdAt,
started_at: null,
completed_at: null,
requested_by_user_id: requestedBy,
audit_log_reason: null,
jet_stream_seq: null,
jet_stream_lane: 'batch',
attempts: 0,
max_attempts: 5,
run_at: null,
cancel_requested: false,
context_link: null,
};
await upsertOne(JobsById.insert(row));
}
async function listPages(opts: {
limit: number;
filters?: ListJobsFilters;
maxLookbackDays?: number;
cursor?: ListJobsCursor | null;
}): Promise<Array<ListedPage>> {
const repository = new JobLedgerRepository();
const pages: Array<ListedPage> = [];
let cursor = opts.cursor ?? null;
for (let page = 0; page < 50; page += 1) {
const result = await repository.listJobs({
limit: opts.limit,
cursor,
filters: opts.filters ?? {},
maxLookbackDays: opts.maxLookbackDays ?? 14,
});
pages.push({ids: result.jobs.map((job) => job.job_id), cursor: result.nextCursor});
if (result.nextCursor === null) return pages;
cursor = {
bucketDay: result.nextCursor.bucketDay,
createdAt: new Date(result.nextCursor.createdAt.toISOString()),
jobId: BigInt(result.nextCursor.jobId.toString()),
};
}
throw new Error('listJobs never stopped paging');
}
function expectPages(pages: Array<ListedPage>, limit: number, expected: Array<Array<bigint>>): void {
expect(pages.map((page) => page.ids)).toEqual(expected);
const ids = pages.flatMap((page) => page.ids);
expect(new Set(ids).size).toBe(ids.length);
for (const page of pages.slice(0, -1)) {
expect(page.ids).toHaveLength(limit);
expect(page.cursor?.jobId).toBe(page.ids.at(-1));
}
expect(pages.at(-1)?.cursor).toBeNull();
}
export function describeListJobsPaging(): void {
describe('listJobs paging', () => {
beforeEach(() => {
vi.useFakeTimers({toFake: ['Date']});
vi.setSystemTime(TODAY_NOON);
});
afterEach(() => {
vi.useRealTimers();
});
it('walks one day newest first through a tie group larger than the page', async () => {
await seedJob(20n, at(0, 11));
for (const jobId of [11n, 12n, 13n, 14n, 15n]) {
await seedJob(jobId, at(0, 10));
}
await seedJob(9n, at(0, 9));
await seedJob(5n, at(0, 8));
await seedJob(6n, at(0, 8));
const pages = await listPages({limit: 2});
expectPages(pages, 2, [[20n, 15n], [14n, 13n], [12n, 11n], [9n, 6n], [5n]]);
expect(pages[0]?.cursor).toEqual({
bucketDay: at(0, 10).toISOString().slice(0, 10),
createdAt: at(0, 10),
jobId: 15n,
});
});
it('crosses days and keeps the lookback window anchored on today', async () => {
await seedJob(41n, at(0, 11));
await seedJob(40n, at(0, 10));
await seedJob(32n, at(1, 11));
await seedJob(31n, at(1, 10));
await seedJob(30n, at(1, 9));
await seedJob(21n, at(2, 11));
await seedJob(20n, at(2, 10));
await seedJob(10n, at(3, 11));
expectPages(await listPages({limit: 2, maxLookbackDays: 2}), 2, [[41n, 40n], [32n, 31n], [30n, 21n], [20n]]);
expectPages(await listPages({limit: 2, maxLookbackDays: 3}), 2, [
[41n, 40n],
[32n, 31n],
[30n, 21n],
[20n, 10n],
[],
]);
const outside = await new JobLedgerRepository().listJobs({
limit: 2,
cursor: {bucketDay: at(3, 12).toISOString().slice(0, 10), createdAt: at(3, 12), jobId: 1n},
filters: {},
maxLookbackDays: 2,
});
expect(outside).toEqual({jobs: [], nextCursor: null});
});
it('starts a cursor dated in the future at today', async () => {
await seedJob(2n, at(0, 2));
await seedJob(1n, at(1, 2));
const farFuture = new Date('9999-12-31T00:00:00.000Z');
expectPages(await listPages({limit: 5, cursor: {bucketDay: '9999-12-31', createdAt: farFuture, jobId: 1n}}), 5, [
[2n, 1n],
]);
}, 2_000);
it('never lists a day past the 90-day retention', async () => {
await seedJob(890n, at(89, 11));
await seedJob(910n, at(91, 11));
expectPages(await listPages({limit: 10, maxLookbackDays: 120}), 10, [[890n]]);
});
it('fills a page past rows whose job record is missing instead of leaving the day', async () => {
for (let hour = 1; hour <= 8; hour += 1) {
await seedJob(BigInt(hour), at(0, hour), {bucketOnly: [3, 6, 7].includes(hour)});
}
expectPages(await listPages({limit: 3}), 3, [
[8n, 5n, 4n],
[2n, 1n],
]);
});
it('fills pages through task type, requester and status filters across days', async () => {
await seedJob(60n, at(0, 11));
await seedJob(59n, at(0, 10.5), {taskType: 'B'});
await seedJob(58n, at(0, 10));
await seedJob(57n, at(0, 9.5), {taskType: 'B', requestedBy: ADMIN_USER_ID});
await seedJob(56n, at(0, 9), {status: 'succeeded'});
await seedJob(55n, at(0, 8.5), {taskType: 'B'});
await seedJob(50n, at(1, 11));
await seedJob(49n, at(1, 10.5), {taskType: 'B', requestedBy: ADMIN_USER_ID});
await seedJob(48n, at(1, 10), {bucketOnly: true});
await seedJob(47n, at(1, 9.5));
expectPages(await listPages({limit: 2, filters: {taskType: 'A'}}), 2, [[60n, 58n], [56n, 50n], [47n]]);
expectPages(await listPages({limit: 1, filters: {status: 'succeeded'}}), 1, [[56n], []]);
expectPages(await listPages({limit: 5, filters: {requestedByUserId: ADMIN_USER_ID}}), 5, [[57n, 49n]]);
});
it('resumes after a cursor whose job was discarded between pages', async () => {
for (let hour = 1; hour <= 5; hour += 1) {
await seedJob(BigInt(hour), at(0, hour));
}
const repository = new JobLedgerRepository();
const first = await repository.listJobs({limit: 2, cursor: null, filters: {}, maxLookbackDays: 14});
expect(first.jobs.map((job) => job.job_id)).toEqual([5n, 4n]);
await repository.discardJob(4n, at(0, 4));
expectPages(await listPages({limit: 2, cursor: first.nextCursor}), 2, [[3n, 2n], [1n]]);
});
it('returns an empty page after a page that filled exactly', async () => {
await seedJob(1n, at(0, 1));
await seedJob(2n, at(0, 2));
expectPages(await listPages({limit: 2}), 2, [[2n, 1n], []]);
});
});
}
@@ -0,0 +1,600 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import {spawnSync} from 'node:child_process';
import {createServer} from 'node:net';
import {BatchBuilder, setCassandraQueryExecutorForTesting} from '@app/api/database/CassandraQueryExecution';
import {Db} from '@app/api/database/CassandraTypes';
import {ensurePostgresKvSchema, PostgresKvQueryExecutor} from '@app/api/database/PostgresKvQueryExecutor';
import type {JobActiveRow, JobByDayBucketRow, JobByIdRow, JobStatus} from '@app/api/database/types/JobLedgerTypes';
import {
EXPIRED_JOB_ERROR,
JOB_LEDGER_TTL_SECONDS,
JOB_STALE_AFTER_MS,
JobLedgerRepository,
} from '@app/api/jobs/JobLedgerRepository';
import {describeListJobsPaging} from '@app/api/jobs/ListJobsPagingSuite';
import {expireLegacyJobLedgerRows} from '@app/api/jobs/PostgresJobLedgerExpiry';
import {JobsActive, JobsByDayBucket, JobsById} from '@app/api/Tables';
import {startDockerContainer} from '@app/api/test/DockerTestContainer';
import {InMemoryCassandraQueryExecutor} from '@app/api/test/InMemoryCassandraQueryExecutor';
import {createSnowflake} from '@fluxer/snowflake/src/Snowflake';
import {
getDefaultPostgresClient,
type IPostgresClient,
initPostgres,
shutdownPostgres,
} from '@pkgs/postgres/src/Client';
import {afterAll, beforeAll, beforeEach, describe, expect, it} from 'vitest';
const KV_TABLE = 'kv_job_ledger_expiry';
const CONTAINER = `fluxer-kvjobs-${process.pid.toString(36)}-${Date.now().toString(36)}`;
const dockerAvailable = spawnSync('docker', ['version'], {stdio: 'ignore'}).status === 0;
const DAY_MS = 86_400_000;
const ADMIN_USER_ID = 1_234_567_890_123n;
async function sleep(ms: number): Promise<void> {
await new Promise((resolve) => setTimeout(resolve, ms));
}
async function freePort(): Promise<number> {
return new Promise((resolve, reject) => {
const server = createServer();
server.on('error', reject);
server.listen(0, '127.0.0.1', () => {
const address = server.address();
if (typeof address === 'string' || address === null) {
reject(new Error('no port'));
return;
}
const port = address.port;
server.close(() => resolve(port));
});
});
}
let sequence = 0;
function jobIdAgedDays(days: number): bigint {
sequence += 1;
return createSnowflake({timestamp: Date.now() - days * DAY_MS, sequence: sequence % 4096, workerId: 1});
}
interface LegacyJob {
jobId: bigint;
createdAt: Date;
}
async function seedLegacyJob(
executor: PostgresKvQueryExecutor,
opts: {ageDays: number; status: JobStatus; requestedBy: bigint | null; active: boolean},
): Promise<LegacyJob> {
const jobId = jobIdAgedDays(opts.ageDays);
const createdAt = new Date(Date.now() - opts.ageDays * DAY_MS);
const idRow: JobByIdRow = {
job_id: jobId,
task_type: opts.requestedBy === null ? 'flushUserActivityBuffer' : 'refreshSearchIndex',
status: opts.status,
progress_current: null,
progress_total: null,
progress_message: null,
payload: '{}',
result: null,
error_message: null,
created_at: createdAt,
started_at: null,
completed_at: opts.status === 'succeeded' ? createdAt : null,
requested_by_user_id: opts.requestedBy,
audit_log_reason: null,
jet_stream_seq: '1',
jet_stream_lane: 'batch',
attempts: 0,
max_attempts: 5,
run_at: null,
cancel_requested: false,
context_link: null,
};
const bucketRow: JobByDayBucketRow = {
bucket_day: createdAt.toISOString().slice(0, 10),
created_at: createdAt,
job_id: jobId,
task_type: idRow.task_type,
status: 'queued',
requested_by_user_id: opts.requestedBy,
};
await executor.executeQuery(JobsById.insert(idRow));
await executor.executeQuery(JobsByDayBucket.insert(bucketRow));
if (opts.active) {
const activeRow: JobActiveRow = {
job_id: jobId,
task_type: idRow.task_type,
status: opts.status,
requested_by_user_id: opts.requestedBy,
created_at: createdAt,
started_at: null,
};
await executor.executeQuery(JobsActive.insert(activeRow));
}
return {jobId, createdAt};
}
describe.skipIf(!dockerAvailable)('job ledger expiry against postgres', () => {
let raw: IPostgresClient;
let executor: PostgresKvQueryExecutor;
async function jobRows(
jobId: bigint,
): Promise<Array<{table_name: string; expires_at: Date | null; row_data: never}>> {
const result = await raw.query<{table_name: string; expires_at: Date | null; row_data: never}>(
`SELECT table_name, expires_at, row_data FROM ${KV_TABLE}
WHERE table_name IN ('jobs_by_id', 'jobs_active', 'jobs_by_day_bucket')
AND (row_key = $1 OR split_part(row_key, chr(31), 3) = $1)
ORDER BY table_name`,
[JSON.stringify({__fluxer_type: 'bigint', value: jobId.toString()})],
);
return result.rows;
}
async function waitForLockWait(): Promise<void> {
for (let attempt = 0; attempt < 400; attempt += 1) {
const waiting = await raw.query<{n: number}>(
`SELECT count(*)::int AS n FROM pg_stat_activity WHERE datname = current_database() AND wait_event_type = 'Lock'`,
);
if (waiting.rows[0]!.n > 0) return;
await sleep(25);
}
throw new Error('the pass never waited on the writer');
}
async function forgetLedgerExpiry(): Promise<void> {
await raw.query(
`UPDATE ${KV_TABLE} SET expires_at = NULL WHERE table_name IN ('jobs_by_id', 'jobs_by_day_bucket')`,
);
}
async function forgetExpiryOf(...jobIds: Array<bigint>): Promise<void> {
const keys = jobIds.map((jobId) => JSON.stringify({__fluxer_type: 'bigint', value: jobId.toString()}));
await raw.query(
`UPDATE ${KV_TABLE} SET expires_at = NULL
WHERE table_name IN ('jobs_by_id', 'jobs_by_day_bucket')
AND (row_key = ANY($1::text[]) OR split_part(row_key, chr(31), 3) = ANY($1::text[]))`,
[keys],
);
}
beforeAll(async () => {
const port = await freePort();
startDockerContainer([
'run',
'-d',
'--name',
CONTAINER,
'-e',
'POSTGRES_USER=fluxer',
'-e',
'POSTGRES_PASSWORD=fluxer',
'-e',
'POSTGRES_DB=fluxer',
'-p',
`127.0.0.1:${port}:5432`,
'postgres:16-alpine',
'-c',
'fsync=off',
]);
let ready = false;
for (let attempt = 0; attempt < 180 && !ready; attempt += 1) {
await sleep(500);
const probe = spawnSync('docker', ['exec', CONTAINER, 'pg_isready', '-U', 'fluxer', '-d', 'fluxer'], {
stdio: 'ignore',
});
if (probe.status !== 0) continue;
try {
await initPostgres({
url: `postgres://fluxer:[email protected]:${port}/fluxer`,
maxConnections: 4,
kvTable: KV_TABLE,
});
await getDefaultPostgresClient().query('SELECT 1');
ready = true;
} catch {
await shutdownPostgres().catch(() => {});
}
}
if (!ready) throw new Error('postgres never came up');
raw = getDefaultPostgresClient();
await ensurePostgresKvSchema(raw);
executor = new PostgresKvQueryExecutor(raw);
}, 900_000);
beforeEach(async () => {
await raw.query(`DELETE FROM ${KV_TABLE}`);
setCassandraQueryExecutorForTesting(executor);
});
afterAll(async () => {
setCassandraQueryExecutorForTesting(new InMemoryCassandraQueryExecutor());
await shutdownPostgres().catch(() => {});
spawnSync('docker', ['rm', '-f', CONTAINER], {stdio: 'ignore'});
});
it('writes every ledger row with an expiry and keeps it through the job lifecycle', async () => {
const repository = new JobLedgerRepository();
const jobId = jobIdAgedDays(0);
await repository.createJob({
jobId,
taskType: 'refreshSearchIndex',
payload: {},
requestedByUserId: ADMIN_USER_ID,
auditLogReason: null,
maxAttempts: 5,
runAt: null,
jetStreamLane: 'batch',
jetStreamSeq: null,
});
await repository.setJetStreamSeq(jobId, '7');
await repository.markRunning(jobId, 'batch');
await repository.reportProgress(jobId, 1, 2, 'half');
const running = await jobRows(jobId);
expect(running.map((row) => row.table_name)).toEqual(['jobs_active', 'jobs_by_day_bucket', 'jobs_by_id']);
for (const row of running) {
expect(row.expires_at).not.toBeNull();
const remainingSeconds = (row.expires_at!.getTime() - Date.now()) / 1000;
expect(remainingSeconds).toBeGreaterThan(JOB_LEDGER_TTL_SECONDS - 60);
expect(remainingSeconds).toBeLessThanOrEqual(JOB_LEDGER_TTL_SECONDS);
}
await repository.markSucceeded(jobId, {ok: true});
const done = await jobRows(jobId);
expect(done.map((row) => row.table_name)).toEqual(['jobs_by_day_bucket', 'jobs_by_id']);
expect(done.every((row) => row.expires_at !== null)).toBe(true);
expect((await repository.getJob(jobId))?.status).toBe('succeeded');
});
it('never leaves a row without an expiry when a patch lands on a job that is gone', async () => {
const repository = new JobLedgerRepository();
const patches: Array<(jobId: bigint) => Promise<void>> = [
(jobId) => repository.markRunning(jobId, 'batch'),
(jobId) => repository.markSucceeded(jobId, null),
(jobId) => repository.markCancelled(jobId),
(jobId) => repository.markDeadletter(jobId, 'boom'),
(jobId) => repository.reportProgress(jobId, 1, null, null),
(jobId) => repository.setContextLink(jobId, '/admin/jobs'),
(jobId) => repository.setJetStreamSeq(jobId, '1'),
(jobId) => repository.requestCancel(jobId),
];
for (const patch of patches) {
const jobId = jobIdAgedDays(0);
await patch(jobId);
const rows = await jobRows(jobId);
expect(rows.length).toBeGreaterThan(0);
expect(rows.every((row) => row.expires_at !== null)).toBe(true);
expect(await repository.getJob(jobId)).toBeNull();
}
expect(await repository.listActiveJobs()).toEqual([]);
});
it('discards every row of a job that never reached the stream', async () => {
const repository = new JobLedgerRepository();
const jobId = jobIdAgedDays(0);
const createdAt = await repository.createJob({
jobId,
taskType: 'batchGuildAuditLogMessageDeletes',
payload: {guildId: '1'},
requestedByUserId: null,
auditLogReason: null,
maxAttempts: 3,
runAt: new Date(Date.now() + 30_000),
jetStreamLane: 'batch',
jetStreamSeq: null,
});
expect(await jobRows(jobId)).toHaveLength(3);
await repository.discardJob(jobId, createdAt);
expect(await jobRows(jobId)).toEqual([]);
});
it('clears legacy rows by the same rules the expiry now enforces', async () => {
const repository = new JobLedgerRepository();
const cronDone = await seedLegacyJob(executor, {
ageDays: 20,
status: 'succeeded',
requestedBy: null,
active: false,
});
const cronStuck = await seedLegacyJob(executor, {ageDays: 20, status: 'queued', requestedBy: null, active: true});
const partialId = jobIdAgedDays(20);
await executor.executeQuery(
JobsById.patchByPk({job_id: partialId}, {status: Db.set('succeeded'), completed_at: Db.set(new Date())}),
);
const adminDone = await seedLegacyJob(executor, {
ageDays: 20,
status: 'succeeded',
requestedBy: ADMIN_USER_ID,
active: false,
});
const adminStuck = await seedLegacyJob(executor, {
ageDays: 20,
status: 'queued',
requestedBy: ADMIN_USER_ID,
active: true,
});
const adminRunning = await seedLegacyJob(executor, {
ageDays: 20,
status: 'running',
requestedBy: ADMIN_USER_ID,
active: true,
});
const adminAncient = await seedLegacyJob(executor, {
ageDays: 100,
status: 'succeeded',
requestedBy: ADMIN_USER_ID,
active: false,
});
const cronInFlight = await seedLegacyJob(executor, {
ageDays: 2,
status: 'queued',
requestedBy: null,
active: true,
});
await forgetLedgerExpiry();
const fresh = jobIdAgedDays(0);
await repository.createJob({
jobId: fresh,
taskType: 'syncUrlBlocklists',
payload: {},
requestedByUserId: null,
auditLogReason: null,
maxAttempts: 5,
runAt: null,
jetStreamLane: 'batch',
jetStreamSeq: null,
});
const freshBefore = await jobRows(fresh);
await raw.query(
`INSERT INTO ${KV_TABLE} (table_name, partition_key, row_key, row_data) VALUES ('users', 'u1', 'u1', '{}'::jsonb)`,
);
expect(await repository.getJob(partialId)).toBeNull();
const first = await expireLegacyJobLedgerRows(raw, Date.now() + 60_000);
expect(first).toEqual({deleted: 10, expiring: 6, complete: true});
expect(await jobRows(cronDone.jobId)).toEqual([]);
expect(await jobRows(cronStuck.jobId)).toEqual([]);
expect(await jobRows(partialId)).toEqual([]);
expect(await jobRows(adminAncient.jobId)).toEqual([]);
for (const kept of [adminDone, adminStuck, adminRunning]) {
const rows = await jobRows(kept.jobId);
expect(rows.map((row) => row.table_name)).toEqual(['jobs_by_day_bucket', 'jobs_by_id']);
for (const row of rows) {
const expected = kept.createdAt.getTime() + JOB_LEDGER_TTL_SECONDS * 1000;
expect(Math.abs(row.expires_at!.getTime() - expected)).toBeLessThan(2000);
}
}
expect((await repository.getJob(adminDone.jobId))?.status).toBe('succeeded');
for (const stuck of [adminStuck, adminRunning]) {
const expired = await repository.getJob(stuck.jobId);
expect(expired?.status).toBe('deadletter');
expect(expired?.error_message).toBe('Expired from the job queue');
expect(expired?.completed_at).toBeInstanceOf(Date);
}
const inFlight = await jobRows(cronInFlight.jobId);
expect(inFlight.map((row) => row.table_name)).toEqual(['jobs_active', 'jobs_by_day_bucket', 'jobs_by_id']);
expect(inFlight.every((row) => row.expires_at === null)).toBe(true);
expect((await repository.getJob(cronInFlight.jobId))?.status).toBe('queued');
expect(await jobRows(fresh)).toEqual(freshBefore);
const users = await raw.query(`SELECT expires_at FROM ${KV_TABLE} WHERE table_name = 'users'`);
expect(users.rows).toEqual([{expires_at: null}]);
const listed = await repository.listJobs({limit: 50, cursor: null, filters: {}, maxLookbackDays: 30});
expect(listed.jobs.map((job) => job.job_id).sort()).toEqual(
[adminDone.jobId, adminStuck.jobId, adminRunning.jobId, cronInFlight.jobId, fresh].sort(),
);
for (let pass = 0; pass < 2; pass += 1) {
expect(await expireLegacyJobLedgerRows(raw, Date.now() + 60_000)).toEqual({
deleted: 0,
expiring: 0,
complete: true,
});
}
});
it('runs again a day after a clean pass and clears rows an older image wrote in between', async () => {
expect(await expireLegacyJobLedgerRows(raw, Date.now() + 60_000)).toEqual({
deleted: 0,
expiring: 0,
complete: true,
});
const rolledBack = await seedLegacyJob(executor, {ageDays: 30, status: 'queued', requestedBy: null, active: true});
await forgetLedgerExpiry();
expect(await expireLegacyJobLedgerRows(raw, Date.now() + 60_000)).toBeNull();
await raw.query(
`UPDATE ${KV_TABLE} SET row_data = jsonb_build_object('applied_at', now() - interval '2 days') WHERE row_key = 'job_ledger_expiry_v1'`,
);
expect(await expireLegacyJobLedgerRows(raw, Date.now() + 60_000)).toEqual({
deleted: 3,
expiring: 0,
complete: true,
});
expect(await jobRows(rolledBack.jobId)).toEqual([]);
expect(await expireLegacyJobLedgerRows(raw, Date.now() + 60_000)).toEqual({
deleted: 0,
expiring: 0,
complete: true,
});
expect(await expireLegacyJobLedgerRows(raw, Date.now() + 60_000)).toBeNull();
});
it('expires stale active jobs page by page without fighting the legacy pass', async () => {
const repository = new JobLedgerRepository();
const legacyFirst = await seedLegacyJob(executor, {
ageDays: 20,
status: 'queued',
requestedBy: ADMIN_USER_ID,
active: true,
});
await forgetExpiryOf(legacyFirst.jobId);
await expireLegacyJobLedgerRows(raw, Date.now() + 60_000);
const legacyFirstRows = await jobRows(legacyFirst.jobId);
const stale: Array<bigint> = [];
for (let index = 0; index < 4; index += 1) {
const jobId = jobIdAgedDays(9);
await repository.createJob({
jobId,
taskType: 'syncUrlBlocklists',
payload: {},
requestedByUserId: null,
auditLogReason: null,
maxAttempts: 5,
runAt: null,
jetStreamLane: 'batch',
jetStreamSeq: null,
});
stale.push(jobId);
}
await repository.markRunning(stale[0]!, 'batch');
const fresh = jobIdAgedDays(0);
await repository.createJob({
jobId: fresh,
taskType: 'syncUrlBlocklists',
payload: {},
requestedByUserId: null,
auditLogReason: null,
maxAttempts: 5,
runAt: null,
jetStreamLane: 'batch',
jetStreamSeq: null,
});
const legacyStale = await seedLegacyJob(executor, {
ageDays: 20,
status: 'running',
requestedBy: ADMIN_USER_ID,
active: true,
});
const legacyYoung = await seedLegacyJob(executor, {ageDays: 2, status: 'queued', requestedBy: null, active: true});
await forgetExpiryOf(legacyStale.jobId, legacyYoung.jobId);
const orphan = jobIdAgedDays(9);
await executor.executeQuery(
JobsActive.patchByPkWithTtl({job_id: orphan}, {status: Db.set('running')}, JOB_LEDGER_TTL_SECONDS),
);
const sweep = () =>
repository.expireStaleActiveJobs({staleBeforeMs: Date.now() - JOB_STALE_AFTER_MS, pageSize: 2, maxCleared: 100});
expect(await sweep()).toEqual({cleared: 6, expired: 5, complete: true});
const active = (await repository.listActiveJobs()).map((job) => job.job_id).sort();
expect(active).toEqual([fresh, legacyYoung.jobId].sort());
for (const jobId of [...stale, legacyStale.jobId]) {
const job = await repository.getJob(jobId);
expect(job?.status).toBe('deadletter');
expect(job?.error_message).toBe(EXPIRED_JOB_ERROR);
const byId = (await jobRows(jobId)).find((row) => row.table_name === 'jobs_by_id');
expect(Math.abs(byId!.expires_at!.getTime() - (Date.now() + JOB_LEDGER_TTL_SECONDS * 1000))).toBeLessThan(60_000);
}
expect(await jobRows(orphan)).toEqual([]);
expect(await jobRows(legacyFirst.jobId)).toEqual(legacyFirstRows);
const legacyStaleById = (await jobRows(legacyStale.jobId)).find((row) => row.table_name === 'jobs_by_id');
await expireLegacyJobLedgerRows(raw, Date.now() + 60_000);
expect((await jobRows(legacyStale.jobId)).find((row) => row.table_name === 'jobs_by_id')).toEqual(legacyStaleById);
expect((await repository.listActiveJobs()).map((job) => job.job_id)).toContain(legacyYoung.jobId);
expect(await sweep()).toEqual({cleared: 0, expired: 0, complete: true});
});
it('leaves rows alone when a live writer gives them an expiry while the pass waits on them', async () => {
const cronDone = await seedLegacyJob(executor, {
ageDays: 20,
status: 'succeeded',
requestedBy: null,
active: false,
});
const adminDone = await seedLegacyJob(executor, {
ageDays: 20,
status: 'succeeded',
requestedBy: ADMIN_USER_ID,
active: false,
});
await forgetLedgerExpiry();
const liveKeys = [cronDone.jobId, adminDone.jobId].map((jobId) =>
JSON.stringify({__fluxer_type: 'bigint', value: jobId.toString()}),
);
let written!: () => void;
const writerHoldsRows = new Promise<void>((resolve) => {
written = resolve;
});
let release!: () => void;
const released = new Promise<void>((resolve) => {
release = resolve;
});
const writer = raw.transaction(async (db) => {
await db.query(
`UPDATE ${KV_TABLE} SET expires_at = now() + interval '1 hour', updated_at = now() WHERE table_name = 'jobs_by_id' AND row_key = ANY($1::text[])`,
[liveKeys],
);
written();
await released;
});
await writerHoldsRows;
const pass = expireLegacyJobLedgerRows(raw, Date.now() + 60_000);
await waitForLockWait();
release();
await writer;
expect(await pass).toEqual({deleted: 1, expiring: 1, complete: true});
for (const job of [cronDone, adminDone]) {
const byId = (await jobRows(job.jobId)).find((row) => row.table_name === 'jobs_by_id');
expect(byId).toBeDefined();
const remainingSeconds = (byId!.expires_at!.getTime() - Date.now()) / 1000;
expect(remainingSeconds).toBeGreaterThan(3000);
expect(remainingSeconds).toBeLessThanOrEqual(3660);
}
expect((await new JobLedgerRepository().getJob(adminDone.jobId))?.status).toBe('succeeded');
});
it('pages through more legacy rows than one page holds and stops at its deadline', async () => {
const batch = new BatchBuilder();
for (let index = 0; index < 2300; index += 1) {
const jobId = jobIdAgedDays(30);
const createdAt = new Date(Date.now() - 30 * DAY_MS);
batch.addPrepared(
JobsByDayBucket.insert({
bucket_day: createdAt.toISOString().slice(0, 10),
created_at: createdAt,
job_id: jobId,
task_type: 'flushUserActivityBuffer',
status: 'queued',
requested_by_user_id: null,
}),
);
}
await batch.executeChunked(500, false);
await forgetLedgerExpiry();
expect(await expireLegacyJobLedgerRows(raw, Date.now() - 1)).toEqual({
deleted: 0,
expiring: 0,
complete: false,
});
expect(await expireLegacyJobLedgerRows(raw, Date.now() + 60_000)).toEqual({
deleted: 2300,
expiring: 0,
complete: true,
});
const left = await raw.query(`SELECT count(*)::int AS n FROM ${KV_TABLE} WHERE table_name = 'jobs_by_day_bucket'`);
expect(left.rows[0]).toEqual({n: 0});
expect(await expireLegacyJobLedgerRows(raw, Date.now() + 60_000)).toEqual({
deleted: 0,
expiring: 0,
complete: true,
});
expect(await expireLegacyJobLedgerRows(raw, Date.now() + 60_000)).toBeNull();
});
describeListJobsPaging();
});
@@ -0,0 +1,149 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import {postgresKvPassIsFresh, recordPostgresKvCleanPass} from '@app/api/database/PostgresKvQueryExecutor';
import {EXPIRED_JOB_ERROR, JOB_LEDGER_TTL_SECONDS, JOB_STALE_AFTER_MS} from '@app/api/jobs/JobLedgerRepository';
import {FLUXER_EPOCH} from '@fluxer/constants/src/Core';
import {TIMESTAMP_SHIFT} from '@fluxer/snowflake/src/Snowflake';
import {type IPostgresClient, quoteIdentifier} from '@pkgs/postgres/src/Client';
import {ms} from 'itty-time';
const LEGACY_JOB_LEDGER_MARKER = 'job_ledger_expiry_v1';
const PAGE_SIZE = 2000;
const CLEAN_PASS_INTERVAL_MS = ms('1 day');
const BIGINT_KEY_PREFIX = '{"__fluxer_type":"bigint","value":"';
const BIGINT_KEY_SUFFIX = '"}';
function bigintKeyExpr(key: string): string {
const prefixLength = BIGINT_KEY_PREFIX.length;
const affixLength = prefixLength + BIGINT_KEY_SUFFIX.length;
return `(CASE WHEN left(${key}, ${prefixLength}) = '${BIGINT_KEY_PREFIX}' AND right(${key}, ${BIGINT_KEY_SUFFIX.length}) = '${BIGINT_KEY_SUFFIX}' THEN substr(${key}, ${prefixLength + 1}, length(${key}) - ${affixLength})::numeric END)`;
}
interface LedgerTable {
name: string;
jobIdExpr: string;
deleteStale: boolean;
markStaleDeadletter: boolean;
}
const LEDGER_TABLES: ReadonlyArray<LedgerTable> = [
{
name: 'jobs_active',
jobIdExpr: bigintKeyExpr('kv.row_key'),
deleteStale: true,
markStaleDeadletter: false,
},
{
name: 'jobs_by_id',
jobIdExpr: bigintKeyExpr('kv.row_key'),
deleteStale: false,
markStaleDeadletter: true,
},
{
name: 'jobs_by_day_bucket',
jobIdExpr: bigintKeyExpr('split_part(kv.row_key, chr(31), 3)'),
deleteStale: false,
markStaleDeadletter: false,
},
];
export interface LegacyJobLedgerExpiryResult {
deleted: number;
expiring: number;
complete: boolean;
}
function pageSql(table: string, target: LedgerTable): string {
const staleRemovable = target.deleteStale ? 'c.created_at < $7' : '(c.created_at < $7 AND c.system_job)';
const removable = `c.job_id IS NULL OR c.created_at < $6 OR ${staleRemovable}`;
const rowData = target.markStaleDeadletter
? `CASE WHEN c.unfinished THEN kv.row_data || jsonb_build_object('status', 'deadletter', 'error_message', $9::text, 'completed_at', jsonb_build_object('__fluxer_type', 'date', 'value', $10::text)) ELSE kv.row_data END`
: 'kv.row_data';
return `
WITH page AS (
SELECT kv.row_key, kv.row_data, ${target.jobIdExpr} AS job_id
FROM ${table} kv
WHERE kv.table_name = $1 AND kv.expires_at IS NULL AND kv.row_key > $2
ORDER BY kv.row_key
LIMIT $3
), classified AS (
SELECT
page.row_key,
page.job_id,
to_timestamp(((div(page.job_id, $4::numeric) + $5::numeric) / 1000)::double precision) AS created_at,
COALESCE(page.row_data -> 'requested_by_user_id', 'null'::jsonb) = 'null'::jsonb AS system_job,
COALESCE(page.row_data ->> 'status' IN ('queued', 'running'), false) AS unfinished
FROM page
), removed AS (
DELETE FROM ${table} kv
USING classified c
WHERE kv.table_name = $1 AND kv.row_key = c.row_key AND kv.expires_at IS NULL AND (${removable})
RETURNING kv.row_key
), expiring AS (
UPDATE ${table} kv
SET expires_at = c.created_at + make_interval(secs => $8::double precision), updated_at = now(), row_data = ${rowData}
FROM classified c
WHERE kv.table_name = $1 AND kv.row_key = c.row_key AND kv.expires_at IS NULL AND c.created_at < $7 AND NOT (${removable})
RETURNING kv.row_key
)
SELECT
(SELECT max(row_key) FROM page) AS last_row_key,
(SELECT count(*) FROM page) AS scanned,
(SELECT count(*) FROM removed) AS deleted,
(SELECT count(*) FROM expiring) AS expiring`;
}
export async function expireLegacyJobLedgerRows(
client: IPostgresClient,
deadlineMs: number,
): Promise<LegacyJobLedgerExpiryResult | null> {
const table = quoteIdentifier(client.kvTable());
if (await postgresKvPassIsFresh(client, LEGACY_JOB_LEDGER_MARKER, CLEAN_PASS_INTERVAL_MS)) {
return null;
}
const now = Date.now();
const retentionCutoff = new Date(now - JOB_LEDGER_TTL_SECONDS * 1000);
const staleCutoff = new Date(now - JOB_STALE_AFTER_MS);
const completedAt = new Date(now).toISOString();
let scanned = 0;
let deleted = 0;
let expiring = 0;
for (const target of LEDGER_TABLES) {
const sql = pageSql(table, target);
const deadletterValues = target.markStaleDeadletter ? [EXPIRED_JOB_ERROR, completedAt] : [];
let cursor = '';
for (;;) {
if (Date.now() >= deadlineMs) {
return {deleted, expiring, complete: false};
}
const result = await client.query<{
last_row_key: string | null;
scanned: string;
deleted: string;
expiring: string;
}>(sql, [
target.name,
cursor,
PAGE_SIZE,
(1n << TIMESTAMP_SHIFT).toString(),
FLUXER_EPOCH.toString(),
retentionCutoff,
staleCutoff,
JOB_LEDGER_TTL_SECONDS,
...deadletterValues,
]);
const page = result.rows[0];
if (!page || page.last_row_key === null) {
break;
}
scanned += Number(page.scanned);
deleted += Number(page.deleted);
expiring += Number(page.expiring);
cursor = page.last_row_key;
}
}
if (scanned === 0) {
await recordPostgresKvCleanPass(client, LEGACY_JOB_LEDGER_MARKER);
}
return {deleted, expiring, complete: true};
}
@@ -1,7 +1,8 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import {AdminRepository} from '@app/api/admin/AdminRepository';
import {BANNED_FILE_SHAS_REFRESH_CHANNEL} from '@app/api/constants/ContentModeration';
import {Config} from '@app/api/Config';
import {BANNED_FILE_SHAS_REFRESH_CHANNEL, isBlocklistFeedFileSha} from '@app/api/constants/ContentModeration';
import {Logger} from '@app/api/Logger';
import {RefreshSubscription} from '@app/api/utils/RefreshSubscription';
import type {IKVProvider} from '@pkgs/kv_client/src/IKVProvider';
@@ -38,8 +39,11 @@ class FileShaCache {
async refresh(): Promise<void> {
const rows = await this.adminRepository.loadAllBannedFileShas();
const next = new Set<string>();
const includeFeedRows = Config.blocklistFeeds.enabled;
for (const row of rows) {
if (row.sha256_hex) next.add(row.sha256_hex.toLowerCase());
if (!row.sha256_hex) continue;
if (!includeFeedRows && isBlocklistFeedFileSha(row)) continue;
next.add(row.sha256_hex.toLowerCase());
}
this.banned = next;
this.consecutiveFailures = 0;
@@ -15,6 +15,7 @@ interface RequireClientIpOptions {
const defaultExemptPaths: Array<string> = [
'/_health',
'/internal',
'/webhooks/livekit',
'/test',
'/connections/bluesky/client-metadata.json',
@@ -51,6 +51,7 @@ import {createUsersServiceClient} from '@app/api/infrastructure/UsersServiceClie
import {VirusScanService} from '@app/api/infrastructure/VirusScanService';
import {GatewayRolloutConfigPublisher} from '@app/api/instance/GatewayRolloutConfigPublisher';
import {InstanceConfigRepository} from '@app/api/instance/InstanceConfigRepository';
import {PushServiceDeliveryConfigPublisher} from '@app/api/instance/PushServiceDeliveryConfigPublisher';
import {InviteRepository} from '@app/api/invite/InviteRepository';
import {Logger} from '@app/api/Logger';
import {LimitConfigService} from '@app/api/limits/LimitConfigService';
@@ -155,6 +156,18 @@ export const getGatewayRolloutConfigPublisher = singleton(
}),
),
);
export const getPushServiceDeliveryConfigPublisher = singleton(
() =>
new PushServiceDeliveryConfigPublisher(
new NatsConnectionManager({
url: Config.nats.coreUrl,
token: Config.nats.authToken || undefined,
name: 'fluxer-api-push-service-delivery-config',
}),
),
);
export const getVisionarySlotRepository = singleton(() => new VisionarySlotRepository());
export const getCacheService: () => ICacheService = singleton(() => new KVCacheProvider({client: getKVClient()}));
export const getRateLimitService = singleton(() => new RateLimitService(getKVClient()));
@@ -1,6 +1,7 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import {AdminRepository} from '@app/api/admin/AdminRepository';
import {Config} from '@app/api/Config';
import {BANNED_URL_DOMAINS_REFRESH_CHANNEL, BANNED_URLS_REFRESH_CHANNEL} from '@app/api/constants/ContentModeration';
import type {IStorageService} from '@app/api/infrastructure/IStorageService';
import {Logger} from '@app/api/Logger';
@@ -68,7 +69,7 @@ class UrlBlocklistCache {
}
private async loadFeedUrls(): Promise<Set<string>> {
if (!this.storageService) return new Set();
if (!this.storageService || !Config.blocklistFeeds.enabled) return new Set();
const lines = await readLinesFromS3(this.storageService, RISK_S3_KEYS.feedUrls);
return new Set(lines);
}
@@ -5,7 +5,7 @@ import {Logger} from '@app/api/Logger';
import {hashAuthToken, recordAbuseSignal} from '@app/api/middleware/AbusiveIpAutoBanner';
import type {User} from '@app/api/models/User';
import type {HonoEnv} from '@app/api/types/HonoEnv';
import {requireRequestClientIp} from '@app/api/utils/RequestClientIp';
import {getRequestClientIp} from '@app/api/utils/RequestClientIp';
import {stripApiPrefix} from '@app/api/utils/RequestPathUtils';
import type {Context} from 'hono';
import {createMiddleware} from 'hono/factory';
@@ -60,7 +60,7 @@ function setUserInContext(ctx: Context<HonoEnv>, user: User, trackActivity: bool
ctx.set('user', user);
if (trackActivity) {
const now = new Date();
const ip = requireRequestClientIp(ctx);
const ip = getRequestClientIp(ctx);
const kvActivityTracker = ctx.get('kvActivityTracker');
const userActivityBuffer = ctx.get('userActivityBuffer');
userActivityBuffer.recordActivity(user.id, now, ip);
@@ -77,7 +77,7 @@ export const UserMiddleware = createMiddleware<HonoEnv>(async (ctx, next) => {
}
const rawAuthHeader = ctx.req.header('Authorization');
const parsed = parseAuthHeader(rawAuthHeader);
const resolvedClientIp = requireRequestClientIp(ctx);
const resolvedClientIp = getRequestClientIp(ctx);
ctx.set('oauthBearerToken', undefined);
ctx.set('oauthBearerApplicationId', undefined);
ctx.set('oauthBearerAllowed', false);
@@ -22,6 +22,7 @@ function createHarness(path = 'http://localhost/v1/messages'): Harness {
return ctx.text('ok');
});
app.get('/_health', (ctx) => ctx.text('OK'));
app.get('/internal/rpc', (ctx) => ctx.text('OK'));
app.onError(AppErrorHandler);
return {
request: async (headers) => app.request(path, {headers}),
@@ -80,6 +81,12 @@ describe('RequireClientIpMiddleware', () => {
expect(response.status).toBe(200);
});
it('leaves internal service to service calls alone', async () => {
const harness = createHarness('http://localhost/internal/rpc');
const response = await harness.request({});
expect(response.status).toBe(200);
});
it('passes every request through in test mode', async () => {
Config.dev.testModeEnabled = true;
const harness = createHarness();
+19 -18
View File
@@ -17030,7 +17030,7 @@
"content": {"application/json": {"schema": {"$ref": "#/components/schemas/Error"}}}
}
},
"description": "Registers a mobile push device token for APNs, Firebase Cloud Messaging, or UnifiedPush. UnifiedPush registrations include the endpoint URL plus Web Push encryption keys.",
"description": "Registers a mobile push device for APNs, Firebase Cloud Messaging, or UnifiedPush. A Web Push registration sends the endpoint URL with encryption_key and auth_secret. A raw registration sends the platform push token with no keys.",
"security": [{"sessionToken": []}],
"requestBody": {
"required": true,
@@ -22735,16 +22735,20 @@
"properties": {
"platform": {
"description": "The mobile push notification platform",
"x-enumNames": ["ANDROID_FCM", "IOS_APNS", "ANDROID_UNIFIED_PUSH"],
"x-enumNames": ["ANDROID_FCM", "IOS_APNS", "IOS_APNS_VOIP", "ANDROID_UNIFIED_PUSH"],
"x-enumDescriptions": [
"Firebase Cloud Messaging (Android)",
"Apple Push Notification Service (iOS)",
"Apple PushKit VoIP push, used only to ring an incoming call (iOS)",
"UnifiedPush (Android without Google services)"
],
"enum": ["android_fcm", "ios_apns", "android_unified_push"],
"enum": ["android_fcm", "ios_apns", "ios_apns_voip", "android_unified_push"],
"type": "string"
},
"token": {
"description": "The Web Push endpoint URL or raw platform push token used at registration",
"type": "string"
},
"token": {"description": "The platform-specific push notification token to unregister", "type": "string"},
"app_id": {
"description": "Client app channel or bundle mapping identifier, such as stable, beta, or canary",
"type": "string"
@@ -22799,16 +22803,20 @@
"properties": {
"platform": {
"description": "The mobile push notification platform",
"x-enumNames": ["ANDROID_FCM", "IOS_APNS", "ANDROID_UNIFIED_PUSH"],
"x-enumNames": ["ANDROID_FCM", "IOS_APNS", "IOS_APNS_VOIP", "ANDROID_UNIFIED_PUSH"],
"x-enumDescriptions": [
"Firebase Cloud Messaging (Android)",
"Apple Push Notification Service (iOS)",
"Apple PushKit VoIP push, used only to ring an incoming call (iOS)",
"UnifiedPush (Android without Google services)"
],
"enum": ["android_fcm", "ios_apns", "android_unified_push"],
"enum": ["android_fcm", "ios_apns", "ios_apns_voip", "android_unified_push"],
"type": "string"
},
"token": {
"description": "The Web Push endpoint URL when encryption keys are supplied, otherwise the raw platform push token",
"type": "string"
},
"token": {"description": "The platform-specific push notification token or endpoint URL", "type": "string"},
"user_agent": {"description": "The user agent string identifying the device", "type": "string"},
"app_id": {
"description": "Client app channel or bundle mapping identifier, such as stable, beta, or canary",
@@ -22825,11 +22833,11 @@
"type": "string"
},
"encryption_key": {
"description": "The P-256 ECDH public key for UnifiedPush encryption (base64url)",
"description": "The P-256 ECDH public key for Web Push encryption (base64url)",
"type": "string"
},
"auth_secret": {
"description": "The authentication secret for UnifiedPush encryption (base64url)",
"description": "The authentication secret for Web Push encryption (base64url)",
"type": "string"
}
},
@@ -24224,7 +24232,7 @@
"description": "The ID of the guild this channel belongs to",
"$ref": "#/components/schemas/SnowflakeStringType"
},
"name": {"description": "The name of the channel", "type": "string"},
"name": {"description": "The name of the channel", "type": ["string", "null"]},
"topic": {"description": "The topic of the channel", "type": ["string", "null"]},
"url": {
"description": "The URL associated with the channel",
@@ -27182,8 +27190,7 @@
"assignments": {
"type": "object",
"properties": {
"voice_noise_suppression": {"$ref": "#/components/schemas/VoiceNoiseSuppressionAssignmentResponse"},
"screen_share_delivery": {"$ref": "#/components/schemas/ScreenShareDeliveryAssignmentResponse"}
"voice_noise_suppression": {"$ref": "#/components/schemas/VoiceNoiseSuppressionAssignmentResponse"}
},
"additionalProperties": false
}
@@ -30687,12 +30694,6 @@
"additionalProperties": false
},
"DonationCurrency": {"type": "string", "enum": ["usd", "eur", "brl", "inr", "pln", "try", "sek", "dkk", "nok"]},
"ScreenShareDeliveryAssignmentResponse": {
"type": "object",
"properties": {"enabled": {"type": "boolean"}},
"required": ["enabled"],
"additionalProperties": false
},
"VoiceNoiseSuppressionAssignmentResponse": {
"type": "object",
"properties": {
+1 -3
View File
@@ -204,9 +204,7 @@ function buildApnsPayload(payload: Record<string, unknown>): Record<string, unkn
if (badge !== undefined) {
aps.badge = badge;
}
if (imageUrl) {
aps['mutable-content'] = 1;
}
aps['mutable-content'] = 1;
return {
...data,
title,
@@ -139,7 +139,7 @@ describe('ApnsPushService', () => {
notification: {title: 'Alice', body: 'Hello', icon: 'https://cdn.example/avatar.png'},
});
expect(payload.image_url).toBeUndefined();
expect(payload.aps).not.toHaveProperty('mutable-content');
expect(payload.aps).toHaveProperty('mutable-content', 1);
expect(payload.author_avatar_url).toBe('https://cdn.example/avatar.png');
});
it('imports the APNs signing key once per PEM and rejects a truncated one every time', async () => {
+42
View File
@@ -1,6 +1,29 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import {Config} from '@app/api/Config';
import {PHONE_REQUIREMENT_FLAGS, SuspiciousActivityFlags} from '@fluxer/constants/src/UserConstants';
const EMAIL_ONLY_EQUIVALENTS: ReadonlyArray<readonly [number, number]> = [
[SuspiciousActivityFlags.REQUIRE_VERIFIED_EMAIL_OR_VERIFIED_PHONE, SuspiciousActivityFlags.REQUIRE_VERIFIED_EMAIL],
[SuspiciousActivityFlags.REQUIRE_VERIFIED_EMAIL_OR_REVERIFIED_PHONE, SuspiciousActivityFlags.REQUIRE_VERIFIED_EMAIL],
[
SuspiciousActivityFlags.REQUIRE_REVERIFIED_EMAIL_OR_VERIFIED_PHONE,
SuspiciousActivityFlags.REQUIRE_REVERIFIED_EMAIL,
],
[
SuspiciousActivityFlags.REQUIRE_REVERIFIED_EMAIL_OR_REVERIFIED_PHONE,
SuspiciousActivityFlags.REQUIRE_REVERIFIED_EMAIL,
],
];
const PHONE_OFFERING_FLAGS = EMAIL_ONLY_EQUIVALENTS.reduce((mask, [either]) => mask | either, PHONE_REQUIREMENT_FLAGS);
function withoutPhoneOfferingFlags(flagBits: number): number {
return EMAIL_ONLY_EQUIVALENTS.reduce(
(next, [either, emailOnly]) => ((flagBits & either) !== 0 ? next | emailOnly : next),
flagBits & ~PHONE_OFFERING_FLAGS,
);
}
function normalizeCountryCode(countryCode: string | null | undefined): string | null {
const trimmed = countryCode?.trim();
@@ -17,6 +40,25 @@ export function countryRequiresInboundPhoneVerification(countryCode: string | nu
return configuredCountrySet(Config.abusePolicy.inboundPhoneCountryCodes).has(normalized);
}
export function phoneFlaggingAllowedForCountry(countryCode: string | null | undefined): boolean {
const {enabled, exemptCountryCodes} = Config.abusePolicy.phoneFlagging;
if (!enabled) return false;
const normalized = normalizeCountryCode(countryCode);
if (!normalized) return true;
return !configuredCountrySet(exemptCountryCodes).has(normalized);
}
export async function stripDisallowedPhoneFlags(
flagBits: number,
resolveCountryCode: () => Promise<string | null>,
): Promise<number> {
if ((flagBits & PHONE_OFFERING_FLAGS) === 0) return flagBits;
const {enabled, exemptCountryCodes} = Config.abusePolicy.phoneFlagging;
if (enabled && exemptCountryCodes.length === 0) return flagBits;
if (enabled && phoneFlaggingAllowedForCountry(await resolveCountryCode())) return flagBits;
return withoutPhoneOfferingFlags(flagBits);
}
export function phoneRequiresInboundVerification(
phone: string,
prefixes: ReadonlyArray<string> = Config.abusePolicy.phoneVerification.inboundRequiredPrefixes,
@@ -47,3 +47,7 @@ export async function readLinesFromS3(storage: IStorageService, key: string): Pr
return [];
}
}
export async function deleteRiskS3Object(storage: IStorageService, key: string): Promise<void> {
await storage.deleteObject(RISK_S3_BUCKET, key);
}
@@ -0,0 +1,86 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import {Config} from '@app/api/Config';
import {phoneFlaggingAllowedForCountry, stripDisallowedPhoneFlags} from '@app/api/risk/AbusePolicy';
import {SuspiciousActivityFlags} from '@fluxer/constants/src/UserConstants';
import {afterEach, beforeEach, describe, expect, it, vi} from 'vitest';
const PHONE_AND_EMAIL =
SuspiciousActivityFlags.REQUIRE_VERIFIED_EMAIL |
SuspiciousActivityFlags.REQUIRE_VERIFIED_PHONE |
SuspiciousActivityFlags.REQUIRE_INBOUND_PHONE_VERIFICATION;
describe('phone flagging policy', () => {
const original = {...Config.abusePolicy.phoneFlagging};
beforeEach(() => {
Config.abusePolicy.phoneFlagging = {enabled: true, exemptCountryCodes: []};
});
afterEach(() => {
Config.abusePolicy.phoneFlagging = original;
});
it('keeps phone flags by default without resolving the country', async () => {
const resolveCountryCode = vi.fn(async () => 'NG');
expect(await stripDisallowedPhoneFlags(PHONE_AND_EMAIL, resolveCountryCode)).toBe(PHONE_AND_EMAIL);
expect(resolveCountryCode).not.toHaveBeenCalled();
expect(phoneFlaggingAllowedForCountry('NG')).toBe(true);
});
it('strips only phone flags when disabled', async () => {
Config.abusePolicy.phoneFlagging = {enabled: false, exemptCountryCodes: []};
const resolveCountryCode = vi.fn(async () => 'NG');
expect(await stripDisallowedPhoneFlags(PHONE_AND_EMAIL, resolveCountryCode)).toBe(
SuspiciousActivityFlags.REQUIRE_VERIFIED_EMAIL,
);
expect(resolveCountryCode).not.toHaveBeenCalled();
expect(phoneFlaggingAllowedForCountry('NG')).toBe(false);
expect(phoneFlaggingAllowedForCountry(null)).toBe(false);
});
it('strips phone flags for exempt countries only', async () => {
Config.abusePolicy.phoneFlagging = {enabled: true, exemptCountryCodes: [' br', 'PT']};
expect(await stripDisallowedPhoneFlags(PHONE_AND_EMAIL, async () => 'BR')).toBe(
SuspiciousActivityFlags.REQUIRE_VERIFIED_EMAIL,
);
expect(await stripDisallowedPhoneFlags(PHONE_AND_EMAIL, async () => 'ng')).toBe(PHONE_AND_EMAIL);
expect(await stripDisallowedPhoneFlags(PHONE_AND_EMAIL, async () => null)).toBe(PHONE_AND_EMAIL);
expect(phoneFlaggingAllowedForCountry('pt')).toBe(false);
expect(phoneFlaggingAllowedForCountry('NG')).toBe(true);
});
it('replaces email or phone flags with their email only equivalent', async () => {
Config.abusePolicy.phoneFlagging = {enabled: false, exemptCountryCodes: []};
expect(
await stripDisallowedPhoneFlags(
SuspiciousActivityFlags.REQUIRE_VERIFIED_EMAIL_OR_VERIFIED_PHONE |
SuspiciousActivityFlags.REQUIRE_VERIFIED_EMAIL_OR_REVERIFIED_PHONE,
async () => null,
),
).toBe(SuspiciousActivityFlags.REQUIRE_VERIFIED_EMAIL);
expect(
await stripDisallowedPhoneFlags(
SuspiciousActivityFlags.REQUIRE_REVERIFIED_EMAIL_OR_VERIFIED_PHONE |
SuspiciousActivityFlags.REQUIRE_INBOUND_PHONE_VERIFICATION,
async () => null,
),
).toBe(SuspiciousActivityFlags.REQUIRE_REVERIFIED_EMAIL);
Config.abusePolicy.phoneFlagging = {enabled: true, exemptCountryCodes: ['BR']};
expect(
await stripDisallowedPhoneFlags(
SuspiciousActivityFlags.REQUIRE_REVERIFIED_EMAIL_OR_REVERIFIED_PHONE,
async () => 'BR',
),
).toBe(SuspiciousActivityFlags.REQUIRE_REVERIFIED_EMAIL);
});
it('skips the country lookup when no phone flags are present', async () => {
Config.abusePolicy.phoneFlagging = {enabled: true, exemptCountryCodes: ['BR']};
const resolveCountryCode = vi.fn(async () => 'BR');
expect(await stripDisallowedPhoneFlags(SuspiciousActivityFlags.REQUIRE_VERIFIED_EMAIL, resolveCountryCode)).toBe(
SuspiciousActivityFlags.REQUIRE_VERIFIED_EMAIL,
);
expect(resolveCountryCode).not.toHaveBeenCalled();
});
});
@@ -0,0 +1,75 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import type {IpInfoRequestAuditEvent} from '@pkgs/geoip/src/IpInfoService';
import {
createPostgresIpInfoCache,
createPostgresIpInfoRequestAuditLogger,
IPINFO_CACHE_TTL_SECONDS,
IPINFO_REQUEST_AUDIT_TTL_SECONDS,
} from '@pkgs/geoip/src/PostgresIpInfoKv';
import type {IPostgresClient} from '@pkgs/postgres/src/Client';
import {describe, expect, it} from 'vitest';
function recordingClient(writes: Array<Array<unknown>>): IPostgresClient {
return {
async query(_text: string, values?: Array<unknown>) {
writes.push(values ?? []);
return {rows: [], rowCount: 1};
},
kvTable() {
return 'kv';
},
} as never;
}
function expectExpiresIn(values: Array<unknown> | undefined, ttlSeconds: number): void {
const expiresAt = values?.[4];
expect(expiresAt).toBeInstanceOf(Date);
const remainingSeconds = ((expiresAt as Date).getTime() - Date.now()) / 1000;
expect(remainingSeconds).toBeGreaterThan(ttlSeconds - 10);
expect(remainingSeconds).toBeLessThanOrEqual(ttlSeconds);
}
const EVENT: IpInfoRequestAuditEvent = {
requestedAt: new Date('2026-09-21T12:00:00.000Z'),
ip: '192.0.2.1',
cacheKey: 'ip:192.0.2.1',
source: 'test',
reason: null,
outcome: 'http_success',
httpStatus: 200,
available: true,
riskNote: 'none',
latencyMs: 12,
requestUrl: 'https://ipinfo.test/192.0.2.1',
responseIp: '192.0.2.1',
countryCode: 'SE',
asnNumber: 64500,
isAnonymous: false,
isTor: false,
isVpn: false,
isProxy: false,
isResidentialProxy: false,
};
describe('Postgres ipinfo KV expiry', () => {
it('expires request audit rows after 90 days', async () => {
const writes: Array<Array<unknown>> = [];
await createPostgresIpInfoRequestAuditLogger({client: recordingClient(writes)}).record(EVENT);
expect(writes).toHaveLength(1);
expect(writes[0]?.[0]).toBe('ipinfo_requests_by_hour');
expectExpiresIn(writes[0], IPINFO_REQUEST_AUDIT_TTL_SECONDS);
});
it('falls back to the 14-day cache default', async () => {
const writes: Array<Array<unknown>> = [];
const cache = createPostgresIpInfoCache({client: recordingClient(writes)});
await cache.set('fallback', {ok: true});
await cache.set('zero', {ok: true}, 0);
await cache.set('short', {ok: true}, 60);
expect(writes.map((values) => values[0])).toEqual(['ipinfo_cache', 'ipinfo_cache', 'ipinfo_cache']);
expectExpiresIn(writes[0], IPINFO_CACHE_TTL_SECONDS);
expectExpiresIn(writes[1], IPINFO_CACHE_TTL_SECONDS);
expectExpiresIn(writes[2], 60);
});
});
+16 -1
View File
@@ -97,6 +97,7 @@ import {RateLimitError} from '@fluxer/errors/src/domains/core/RateLimitError';
import {UnauthorizedError} from '@fluxer/errors/src/domains/core/UnauthorizedError';
import {UnknownGuildError} from '@fluxer/errors/src/domains/guild/UnknownGuildError';
import {UnknownUserError} from '@fluxer/errors/src/domains/user/UnknownUserError';
import {pushServiceDeliveryEnrols} from '@fluxer/schema/src/domains/admin/PushServiceDeliverySchemas';
import type {ChannelResponse} from '@fluxer/schema/src/domains/channel/ChannelSchemas';
import type {VoiceStateResponse} from '@fluxer/schema/src/domains/gateway/GatewaySchemas';
import type {GuildMemberResponse} from '@fluxer/schema/src/domains/guild/GuildMemberSchemas';
@@ -430,6 +431,13 @@ export class RpcService {
}),
};
case 'send_apns_push': {
const deliveryConfig = await this.instanceConfigRepository.getPushServiceDeliveryConfig();
if (pushServiceDeliveryEnrols(deliveryConfig, request.user_id.toString())) {
Logger.warn(
{userId: request.user_id.toString(), configVersion: deliveryConfig.config_version},
'push service delivery path mismatch',
);
}
const result = await sendApnsPush({
userId: request.user_id.toString(),
subscriptionId: request.subscription_id,
@@ -635,6 +643,13 @@ export class RpcService {
data: {config: rolloutConfig},
};
}
case 'get_push_service_delivery_config': {
const config = await this.instanceConfigRepository.getPushServiceDeliveryConfig();
return {
type: 'get_push_service_delivery_config',
data: {config},
};
}
default: {
const exhaustiveCheck: never = request;
throw new Error(
@@ -864,7 +879,7 @@ export class RpcService {
if (!queueAllowed) {
return;
}
await this.workerService.addJob('reconcileUserPayments', {userId: userIdString});
await this.workerService.addJob('reconcileUserPayments', {userId: userIdString}, {skipLedger: true});
})
.catch((error) => {
Logger.warn(
@@ -10,7 +10,7 @@ import type {UserCacheService} from '@app/api/infrastructure/UserCacheService';
import {Logger} from '@app/api/Logger';
import type {RequestCache} from '@app/api/middleware/RequestCacheMiddleware';
import type {User} from '@app/api/models/User';
import {countryRequiresInboundPhoneVerification} from '@app/api/risk/AbusePolicy';
import {countryRequiresInboundPhoneVerification, phoneFlaggingAllowedForCountry} from '@app/api/risk/AbusePolicy';
import {
createRpcTimingNode,
RpcTimingRecorder,
@@ -311,6 +311,17 @@ export class RpcSessionStartService {
) {
return null;
}
if (
!timeRpcStepSync(timingSteps, 'check_phone_flagging_allowed', () =>
phoneFlaggingAllowedForCountry(geoipCountryIso),
)
) {
Logger.info(
{userId: user.id.toString(), countryIso: geoipCountryIso},
'Skipping configured-country inbound phone requirement: phone flagging disabled for this country',
);
return null;
}
if (
timeRpcStepSync(timingSteps, 'check_not_suspicious_flag', () => (user.flags & UserFlags.NOT_SUSPICIOUS) !== 0n)
) {
@@ -0,0 +1,37 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import {createTestUserWithPremium} from '@app/api/stripe/tests/StripeWebhookTestUtils';
import {type ApiTestHarness, createApiTestHarness} from '@app/api/test/ApiTestHarness';
import {NoopWorkerService} from '@app/api/test/NoopWorkerService';
import {HTTP_STATUS} from '@app/api/test/TestConstants';
import {createBuilder} from '@app/api/test/TestRequestBuilder';
import {UserPremiumTypes} from '@fluxer/constants/src/UserConstants';
import {afterEach, beforeEach, describe, expect, test, vi} from 'vitest';
describe('RpcService session payment reconciliation', () => {
let harness: ApiTestHarness;
beforeEach(async () => {
harness = await createApiTestHarness();
});
afterEach(async () => {
await harness?.shutdown();
});
test('queues payment reconciliation without a job record', async () => {
const account = await createTestUserWithPremium(harness, UserPremiumTypes.SUBSCRIPTION, {
stripeCustomerId: 'cus_rpc_session_reconcile',
});
const addJob = vi.spyOn(NoopWorkerService.prototype, 'addJob');
try {
await createBuilder(harness, '')
.post('/test/rpc-session-init')
.body({type: 'session', token: account.token, version: 1, ip: '127.0.0.1'})
.expect(HTTP_STATUS.OK)
.execute();
await vi.waitFor(() =>
expect(addJob).toHaveBeenCalledWith('reconcileUserPayments', {userId: account.userId}, {skipLedger: true}),
);
} finally {
addJob.mockRestore();
}
});
});
@@ -52,7 +52,8 @@ function snowflakeSeconds(snowflake: string): number {
function buildSort(sortBy: string, sortOrder: 'asc' | 'desc' | undefined): Array<string> | undefined {
if (sortBy === 'relevance') return undefined;
return [`${sortBy}:${sortOrder ?? 'desc'}`, 'id:desc'];
const direction = sortOrder ?? 'desc';
return [`${sortBy}:${direction}`, `id:${direction}`];
}
function buildTimestampSort(filters: MessageSearchFilters | AuditLogSearchFilters): Array<string> | undefined {

Some files were not shown because too many files have changed in this diff Show More