Compare commits

..
Author SHA1 Message Date
HampusandGitHub 1ab7e7dfcc fix(api): unfurl links to a self-hosted instance's own domain (#2891) 2026-09-22 02:00:51 +02:00
HampusandGitHub 31c53d2dff fix(app): stop pending stickers from reloading the channel (#2890) 2026-09-22 02:00:26 +02:00
HampusandGitHub 412a1ae79d perf(api): stop ledgering session payment reconciliation (#2889) 2026-09-22 01:37:21 +02:00
HampusandGitHub 0b2306ec3d fix(api): honour default TTLs and expire stale job ledger rows (#2887) 2026-09-21 23:16:39 +02:00
HampusandGitHub 242ed3a934 fix(desktop): drop orphaned Squirrel uninstall entry (#2885) 2026-09-21 20:03:33 +02:00
HampusandGitHub 70e1ce682a feat(emoji): add Unicode 17 emoji and fix mixed skin tones (#2883) 2026-09-21 16:26:06 +02:00
HampusandGitHub 7601bf98ee fix(channel): sync a cleared group DM name without a reload (#2882) 2026-09-21 15:34:18 +02:00
c7ec2a0f58 chore(tooling): Ignore .vscode/ in .gitignore (#2868)
Co-authored-by: Hampus <[email protected]>
2026-09-21 13:29:33 +02:00
XeonandGitHub 6a5e0056a8 fix(flatpak): Add a release tag and make small corrections (#2872) 2026-09-21 13:28:36 +02:00
HampusandGitHub 78d105b46e fix(desktop): stop looping on an update that never installs (#2879) 2026-09-21 03:36:11 +02:00
HampusandGitHub c68d62b8a0 fix(voice): darken screen share source titles in light theme (#2878) 2026-09-21 01:20:54 +02:00
HampusandGitHub df58020f4c fix(api): keep premium paid for after a subscription cancels (#2875) 2026-09-20 23:50:49 +02:00
HampusandGitHub f052ce05aa fix(workspace): point the Erlang extension at the repo root (#2871) 2026-09-20 19:49:06 +02:00
HampusandGitHub eedfd9275f fix(api): only require permissions a channel overwrite grants (#2867) 2026-09-20 17:56:22 +02:00
HampusandGitHub 416af4bec4 fix(docs): correct the flatpak and dnf signing instructions (#2865) 2026-09-20 16:42:33 +02:00
HampusandGitHub 108d282ddd chore(deps): pin pnpm 11 so the lockfile parses for packagers (#2864) 2026-09-20 15:30:50 +02:00
HampusandGitHub a6103244b0 docs(readme): fix the license wording and shrink the preview (#2862) 2026-09-20 15:11:06 +02:00
HampusandGitHub 38935c83c5 docs(readme): document every download and install method (#2861) 2026-09-20 15:05:54 +02:00
HampusandGitHub c157ab5752 feat(voice): rework screen share delivery behind an experiment (#2859) 2026-09-20 06:10:20 +02:00
HampusandGitHub 574a93257c docs(downloads): the pacman repository is signed (#2858) 2026-09-20 05:54:28 +02:00
HampusandGitHub ba7d8781cf feat(auth): make passkey two-factor authentication opt-in (#2857) 2026-09-20 05:06:50 +02:00
HampusandGitHub 3256af8d92 refactor(app-proxy): remove the stable time freeze (#2856) 2026-09-20 04:02:47 +02:00
HampusandGitHub 86043212f2 docs(downloads): one pacman repository holds both channels (#2855) 2026-09-20 02:50:08 +02:00
HampusandGitHub 5d85e88532 fix(search): suggest yourself in DM from: and mentions: filters (#2854) 2026-09-20 01:24:43 +02:00
HampusandGitHub e2abfd476a feat(api): redirect desktop downloads to pkgs (#2853) 2026-09-20 01:20:30 +02:00
HampusandGitHub 487febac8e fix(voice): make stereo microphones work in studio and custom (#2852) 2026-09-20 00:19:53 +02:00
HampusandGitHub a3454e8245 fix(installer): name the services that are not ready (#2851) 2026-09-19 23:34:08 +02:00
HampusandGitHub bf7567b768 fix(user): push guild member updates on profile field changes (#2850) 2026-09-19 23:30:25 +02:00
HampusandGitHub ac3450ab32 feat(ci): publish appimage zsync control files (#2849) 2026-09-19 22:22:54 +02:00
HampusandGitHub 5d034becb8 fix(installer): stop waiting for an absent bucket initialiser (#2848) 2026-09-19 22:14:13 +02:00
HampusandGitHub f9397d0db9 feat(ci): publish linux repositories from the desktop release (#2847) 2026-09-19 22:01:06 +02:00
HampusandGitHub 9005139dc8 fix(voice): stop stereo microphones publishing as mono (#2846) 2026-09-19 21:54:38 +02:00
HampusandGitHub d93604afa2 fix(voice): let screen shares use the hardware H.264 encoder (#2845) 2026-09-19 21:54:30 +02:00
HampusandGitHub c4f0b2ece0 feat(desktop): self-update appimages in place (#2843) 2026-09-19 19:06:53 +02:00
HampusandGitHub 98a42f612b fix(desktop): supersede the legacy linux packages on upgrade (#2842) 2026-09-19 18:50:37 +02:00
HampusandGitHub cc75e1318d fix(ci): raise the macos minimum to 13.0 (#2841) 2026-09-19 18:35:05 +02:00
HampusandGitHub 9027cbdf3e fix(voice): send screen shares at the quality the user picked (#2840) 2026-09-19 16:46:22 +02:00
HampusandGitHub 2119e10ed5 chore(static): update marketing screenshots and readme cover (#2839) 2026-09-19 16:44:32 +02:00
HampusandGitHub 87f3eb3c81 feat(desktop): add flatpak and arch packaging inputs (#2838) 2026-09-19 15:31:41 +02:00
HampusandGitHub f9bb8bd585 test(voice): remove the slow screen share delivery proof (#2836) 2026-09-19 02:33:32 +02:00
440 changed files with 35678 additions and 41010 deletions
+1 -1
View File
@@ -7,7 +7,7 @@ ARG USER_UID=1000
ARG USER_GID=1000
ARG NODE_MAJOR=26
ARG ELP_VERSION=2026-08-10
ARG PNPM_VERSION=12.4.2
ARG PNPM_VERSION=11.27.0
ARG WASM_BINDGEN_VERSION=0.2.128
ENV DEBIAN_FRONTEND=noninteractive
+5 -1
View File
@@ -38,7 +38,11 @@
"customizations": {
"vscode": {
"settings": {
"editor.defaultFormatter": "biomejs.biome"
"editor.defaultFormatter": "biomejs.biome",
"erlang.includePaths": ["."],
"search.exclude": {
"**/_build/default/lib/fluxer_gateway": true
}
},
"extensions": [
"biomejs.biome",
@@ -71,7 +71,6 @@ jobs:
BUILD_VERSION: ${{ needs.meta.outputs.build_version }}
PUBLIC_ASSET_BASE_URL: ""
BUNDLE_LOCAL_ASSETS: "true"
FLUXER_APP_PROXY_TIME_FREEZE_ENABLED: "false"
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1
env:
@@ -155,7 +154,6 @@ jobs:
BUILD_VERSION=${{ needs.meta.outputs.build_version }}
SOURCE_SHA=${{ github.sha }}
SOURCE_DATE=${{ steps.source.outputs.date }}
FLUXER_APP_PROXY_TIME_FREEZE_ENABLED=false
APP_ASSETS_REF=ghcr.io/${{ env.GHCR_OWNER }}/fluxer-app-proxy-self-hosted:${{ needs.meta.outputs.build_version }}-assets
APP_ASSETS_PLATFORM=linux/amd64
cache-from: type=registry,ref=ghcr.io/${{ env.GHCR_OWNER }}/fluxer-app-proxy-self-hosted:buildcache-${{ matrix.platform }}
+40 -81
View File
@@ -11,11 +11,6 @@ on:
- stable
- canary
default: stable
test_build:
description: Stash artifacts under desktop-test/ instead of desktop/ (API will not pick these up as a release).
required: false
default: false
type: boolean
build_version:
description: Explicit Fluxer CalVer build version (YYYY.MDD.MICRO, UTC HHMMSS without leading zeroes) to use instead of automatic UTC clock allocation.
required: false
@@ -32,13 +27,12 @@ permissions:
actions: read
concurrency:
group: desktop-${{ inputs.channel }}-${{ inputs.test_build && 'test' || 'release' }}
group: desktop-${{ inputs.channel }}
cancel-in-progress: true
env:
CHANNEL: ${{ inputs.channel }}
BUILD_CHANNEL: ${{ inputs.channel == 'canary' && 'canary' || 'stable' }}
TEST_BUILD: ${{ inputs.test_build && 'true' || 'false' }}
jobs:
meta:
@@ -53,8 +47,6 @@ jobs:
pub_date: ${{ steps.meta.outputs.pub_date }}
channel: ${{ steps.meta.outputs.channel }}
build_channel: ${{ steps.meta.outputs.build_channel }}
test_build: ${{ steps.meta.outputs.test_build }}
s3_prefix: ${{ steps.meta.outputs.s3_prefix }}
source_sha: ${{ steps.meta.outputs.source_sha }}
steps:
- name: Checkout source
@@ -85,7 +77,6 @@ jobs:
cargo run --locked --quiet --manifest-path tools/ci/Cargo.toml -- build-desktop
--step set_metadata
--channel "${{ inputs.channel }}"
--test-build "${{ inputs.test_build }}"
matrix:
name: Resolve build matrix
@@ -113,7 +104,7 @@ jobs:
--skip-targets "${{ inputs.skip_targets }}"
build:
name: Build ${{ matrix.platform }} (${{ matrix.arch }}, ${{ matrix.desktop_variant }})
name: Build ${{ matrix.platform }} (${{ matrix.arch }})
needs:
- meta
- matrix
@@ -137,15 +128,8 @@ jobs:
PUBLIC_BUILD_VERSION: ${{ needs.meta.outputs.version }}
PUB_DATE: ${{ needs.meta.outputs.pub_date }}
SOURCE_SHA: ${{ needs.meta.outputs.source_sha }}
S3_DESKTOP_PREFIX: ${{ needs.meta.outputs.s3_prefix }}
DESKTOP_HANDOFF_PREFIX: _handoff/desktop/${{ needs.meta.outputs.build_channel }}/${{ needs.meta.outputs.version }}/${{ needs.meta.outputs.source_sha }}
S3_ENDPOINT: ${{ vars.DOWNLOADS_S3_ENDPOINT }}
S3_BUCKET: ${{ vars.DOWNLOADS_S3_BUCKET }}
AWS_ACCESS_KEY_ID: ${{ secrets.DOWNLOADS_AWS_ACCESS_KEY_ID || secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.DOWNLOADS_AWS_SECRET_ACCESS_KEY || secrets.AWS_SECRET_ACCESS_KEY }}
DESKTOP_PLATFORM: ${{ matrix.platform }}
DESKTOP_ARCH: ${{ matrix.arch }}
DESKTOP_VARIANT: ${{ matrix.desktop_variant }}
PLATFORM: ${{ matrix.platform }}
ARCH: ${{ matrix.arch }}
ELECTRON_ARCH: ${{ matrix.electron_arch }}
@@ -477,6 +461,12 @@ jobs:
cargo run --locked --quiet --manifest-path ${{ github.workspace }}/_ci/tools/ci/Cargo.toml -- build-desktop
--step prepare_artifacts_unix
- name: Build AppImage update feed (Linux)
if: matrix.platform == 'linux'
run: >-
cargo run --locked --quiet --manifest-path ${{ github.workspace }}/_ci/tools/ci/Cargo.toml -- build-desktop
--step build_appimage_update_feed
- name: Normalize updater YAML (macOS)
if: matrix.platform == 'macos'
run: >-
@@ -495,13 +485,23 @@ jobs:
cargo run --locked --quiet --manifest-path ${{ github.workspace }}/_ci/tools/ci/Cargo.toml -- build-desktop
--step generate_checksums_windows
- name: Upload artifacts to S3 handoff
- name: Stage build artifacts
id: handoff
run: >-
cargo run --locked --quiet --manifest-path ${{ github.workspace }}/_ci/tools/ci/Cargo.toml -- build-desktop
--step upload_handoff
--step stage_handoff
- name: Upload build artifacts
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02
with:
name: ${{ steps.handoff.outputs.artifact_name }}
path: upload_staging
if-no-files-found: error
retention-days: 1
compression-level: 0
upload:
name: Upload to S3
name: Assemble desktop release assets
if: ${{ !cancelled() && needs.build.result == 'success' }}
needs:
- meta
@@ -520,17 +520,8 @@ jobs:
BUILD_VERSION: ${{ needs.meta.outputs.version }}
PUBLIC_BUILD_VERSION: ${{ needs.meta.outputs.version }}
PUB_DATE: ${{ needs.meta.outputs.pub_date }}
TEST_BUILD: ${{ needs.meta.outputs.test_build }}
SOURCE_SHA: ${{ needs.meta.outputs.source_sha }}
S3_DESKTOP_PREFIX: ${{ needs.meta.outputs.s3_prefix }}
DESKTOP_HANDOFF_PREFIX: _handoff/desktop/${{ needs.meta.outputs.build_channel }}/${{ needs.meta.outputs.version }}/${{ needs.meta.outputs.source_sha }}
DESKTOP_RELEASE_ASSETS_PREFIX: _handoff/desktop-release-assets/${{ needs.meta.outputs.build_channel }}/${{ needs.meta.outputs.version }}/${{ needs.meta.outputs.source_sha }}
DESKTOP_METADATA_PREFIX: _handoff/desktop-metadata/${{ needs.meta.outputs.build_channel }}/${{ needs.meta.outputs.version }}/${{ needs.meta.outputs.source_sha }}
S3_ENDPOINT: ${{ vars.DOWNLOADS_S3_ENDPOINT }}
S3_BUCKET: ${{ vars.DOWNLOADS_S3_BUCKET }}
PUBLIC_DL_BASE: https://api.fluxer.app/dl
AWS_ACCESS_KEY_ID: ${{ secrets.DOWNLOADS_AWS_ACCESS_KEY_ID || secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.DOWNLOADS_AWS_SECRET_ACCESS_KEY || secrets.AWS_SECRET_ACCESS_KEY }}
steps:
- name: Checkout source
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1
@@ -542,12 +533,13 @@ jobs:
with:
toolchain: "1.98.1"
- name: Download S3 handoff artifacts
run: >-
cargo run --locked --quiet --manifest-path tools/ci/Cargo.toml -- build-desktop
--step download_handoff
- name: Download build artifacts
uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093
with:
path: artifacts
pattern: fluxer-desktop-${{ needs.meta.outputs.build_channel }}-*
- name: Build S3 payload layout (+ manifest.json)
- name: Build payload layout (+ manifest.json)
env:
VERSION: ${{ needs.meta.outputs.version }}
PUB_DATE: ${{ needs.meta.outputs.pub_date }}
@@ -556,42 +548,27 @@ jobs:
--step build_payload
- name: Prepare GitHub release assets
if: needs.meta.outputs.test_build != 'true'
run: >-
cargo run --locked --quiet --manifest-path tools/ci/Cargo.toml -- build-desktop
--step prepare_release_assets
- name: Publish GitHub release descriptor
if: needs.meta.outputs.test_build != 'true'
run: >-
cargo run --locked --quiet --manifest-path tools/ci/Cargo.toml -- build-desktop
--step publish_release_descriptor
- name: Upload payload to S3
run: >-
cargo run --locked --quiet --manifest-path tools/ci/Cargo.toml -- build-desktop
--step upload_payload
- name: Upload GitHub release asset handoff
if: needs.meta.outputs.test_build != 'true'
run: >-
cargo run --locked --quiet --manifest-path tools/ci/Cargo.toml -- build-desktop
--step upload_release_assets
- name: Upload GitHub release assets
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02
with:
name: fluxer-desktop-release-assets
path: release_assets
if-no-files-found: error
retention-days: 1
compression-level: 0
- name: Build summary
run: >-
cargo run --locked --quiet --manifest-path tools/ci/Cargo.toml -- build-desktop
--step build_summary
- name: Cleanup S3 handoff
if: ${{ success() }}
run: >-
cargo run --locked --quiet --manifest-path tools/ci/Cargo.toml -- build-desktop
--step cleanup_handoff
publish_release:
name: Publish GitHub desktop release
if: ${{ !cancelled() && needs.upload.result == 'success' && needs.meta.outputs.test_build != 'true' }}
if: ${{ !cancelled() && needs.upload.result == 'success' }}
needs:
- meta
- upload
@@ -603,13 +580,6 @@ jobs:
env:
CHANNEL: ${{ needs.meta.outputs.build_channel }}
VERSION: ${{ needs.meta.outputs.version }}
S3_DESKTOP_PREFIX: ${{ needs.meta.outputs.s3_prefix }}
DESKTOP_RELEASE_ASSETS_PREFIX: _handoff/desktop-release-assets/${{ needs.meta.outputs.build_channel }}/${{ needs.meta.outputs.version }}/${{ needs.meta.outputs.source_sha }}
DESKTOP_METADATA_PREFIX: _handoff/desktop-metadata/${{ needs.meta.outputs.build_channel }}/${{ needs.meta.outputs.version }}/${{ needs.meta.outputs.source_sha }}
S3_ENDPOINT: ${{ vars.DOWNLOADS_S3_ENDPOINT }}
S3_BUCKET: ${{ vars.DOWNLOADS_S3_BUCKET }}
AWS_ACCESS_KEY_ID: ${{ secrets.DOWNLOADS_AWS_ACCESS_KEY_ID || secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.DOWNLOADS_AWS_SECRET_ACCESS_KEY || secrets.AWS_SECRET_ACCESS_KEY }}
steps:
- name: Checkout source
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1
@@ -622,9 +592,10 @@ jobs:
toolchain: "1.98.1"
- name: Download GitHub release assets
run: >-
cargo run --locked --quiet --manifest-path tools/ci/Cargo.toml -- build-desktop
--step download_release_assets
uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093
with:
name: fluxer-desktop-release-assets
path: release_assets
- name: Create token
id: create-token
@@ -656,15 +627,3 @@ jobs:
release_args+=(--prerelease)
fi
cargo run --locked --quiet --manifest-path tools/ci/Cargo.toml -- "${release_args[@]}"
- name: Publish GitHub release readiness marker
env:
SOURCE_SHA: ${{ needs.meta.outputs.source_sha }}
run: >-
cargo run --locked --quiet --manifest-path tools/ci/Cargo.toml -- build-desktop
--step publish_release_marker
- name: Publish payload metadata to S3
run: >-
cargo run --locked --quiet --manifest-path tools/ci/Cargo.toml -- build-desktop
--step publish_payload_metadata
+1
View File
@@ -10,6 +10,7 @@
/.direnv/
/.fluxer/
/.pnpm-store/
/.vscode/
**/*.css.d.ts
**/*.tsbuildinfo
Generated
-10
View File
@@ -1755,16 +1755,6 @@ dependencies = [
"zip",
]
[[package]]
name = "fluxer-content-update-frozen-snapshot"
version = "0.1.0"
dependencies = [
"anyhow",
"base64 0.23.1",
"sha2 0.11.0",
"tempfile",
]
[[package]]
name = "fluxer-dev"
version = "0.1.0"
-1
View File
@@ -9,7 +9,6 @@ members = [
"fluxer_messages",
"fluxer_snowflakes",
"tools/ci",
"tools/content/update-frozen-snapshot",
"tools/dev",
"tools/i18n_auto",
"fluxer_users",
+128 -3
View File
@@ -6,10 +6,12 @@
</p>
<p align="center">
<a href="https://fluxer.app/donate">
<img src="https://img.shields.io/badge/Donate-fluxer.app%2Fdonate-brightgreen" alt="Donate" /></a>
<a href="https://fluxer.app/download">
<img src="https://img.shields.io/badge/Download-fluxer.app-4641D9" alt="Download" /></a>
<a href="https://docs.fluxer.app">
<img src="https://img.shields.io/badge/Docs-docs.fluxer.app-blue" alt="Documentation" /></a>
<a href="https://fluxer.app/donate">
<img src="https://img.shields.io/badge/Donate-fluxer.app%2Fdonate-brightgreen" alt="Donate" /></a>
<a href="./LICENSE">
<img src="https://img.shields.io/badge/License-AGPLv3-purple" alt="AGPLv3 License" /></a>
</p>
@@ -19,5 +21,128 @@
Fluxer is a free and open source instant messaging and VoIP chat app built for friends, groups, and communities.
<p align="center">
<img src="./fluxer_static/marketing/screenshots/desktop-readme-1920w.png" alt="Fluxer app showcase" width="900">
<img src="./fluxer_static/marketing/screenshots/desktop-readme-1920w.png" alt="Fluxer running side by side on a desktop monitor and a phone" width="640">
</p>
## Download
| Windows | macOS | Linux | Android | iOS |
| --- | --- | --- | --- | --- |
| [Installer (x64)][win-setup-x64] | [Disk image][mac-dmg] | [Flatpak][flatpak-ref] | [APK][android-apk] | [TestFlight][ios-testflight] |
| [Installer (ARM64)][win-setup-arm64] | | [deb (x64)][linux-deb-x64] | [Obtainium][obtainium] | |
| [Portable (x64)][win-portable-x64] | | [deb (ARM64)][linux-deb-arm64] | | |
| [Portable (ARM64)][win-portable-arm64] | | [rpm (x64)][linux-rpm-x64] | | |
| | | [rpm (ARM64)][linux-rpm-arm64] | | |
| | | [AppImage (x64)][linux-appimage-x64] | | |
| | | [AppImage (ARM64)][linux-appimage-arm64] | | |
| | | [tar.gz (x64)][linux-targz-x64] | | |
| | | [tar.gz (ARM64)][linux-targz-arm64] | | |
The macOS disk image is universal and runs on both Apple silicon and Intel. Windows and Linux need the build that matches your processor.
On Linux, prefer a package repository over a file. Fluxer then updates with the rest of your system.
## Linux package repositories
All four repositories serve stable and canary. The package is `fluxer` for stable and `fluxer-canary` for canary.
### Flatpak
Opening [this reference file][flatpak-ref] hands the install to your desktop software manager. Some desktops also accept `flatpak+https://pkgs.fluxer.com/flatpak/fluxer.flatpakref` pasted into the address bar.
From a terminal:
```sh
flatpak install https://pkgs.fluxer.com/flatpak/fluxer.flatpakref
```
### Debian and Ubuntu
```sh
sudo install -d -m 0755 /etc/apt/keyrings
sudo curl -fsSL -o /etc/apt/keyrings/fluxer-archive-keyring.gpg https://pkgs.fluxer.com/keys/fluxer-archive-keyring.gpg
sudo curl -fsSL -o /etc/apt/sources.list.d/fluxer.sources https://pkgs.fluxer.com/deb/fluxer.sources
sudo apt update && sudo apt install fluxer
```
### Fedora and RHEL
```sh
sudo curl -fsSL -o /etc/yum.repos.d/fluxer.repo https://pkgs.fluxer.com/rpm/fluxer.repo
sudo dnf install fluxer
```
RHEL, Rocky, Alma and CentOS Stream need `sudo dnf install epel-release` first, because the base repositories do not ship `libXScrnSaver`. Fedora does not need this.
### Arch Linux
The repository is signed, so pacman needs the key in its own keyring once:
```sh
sudo pacman-key --init
curl -fsSL -o /tmp/fluxer-archive-keyring.asc https://pkgs.fluxer.com/keys/fluxer-archive-keyring.asc
sudo pacman-key --add /tmp/fluxer-archive-keyring.asc
sudo pacman-key --lsign-key 09D01339EE128925F75E675C855C5BDE34D205D2
```
`--lsign-key` is the step that makes pacman trust the key. Then add the repository:
```sh
sudo tee -a /etc/pacman.conf >/dev/null <<'REPO'
[fluxer]
SigLevel = Required TrustedOnly
Server = https://pkgs.fluxer.com/arch/$repo/os/$arch
REPO
sudo pacman -Syu fluxer
```
Write `$repo` and `$arch` literally. Both are pacman variables, not shell ones, which is why the heredoc above is quoted.
Full setup notes, including the canary channel, live in the [Linux repositories documentation][docs-linux].
## Other ways to run it
- [Open Fluxer in a browser](https://web.fluxer.app) with no install at all.
- [Host your own instance][docs-selfhost] from this repository.
## Documentation
- [Documentation home][docs]
- [Downloads][docs-downloads]
- [Self-hosting][docs-selfhost]
## License
The source is licensed under the [AGPL-3.0-or-later](./LICENSE) license.
Fluxer branding, icons, default avatars, badge artwork, screenshots and marketing
imagery are copyright Fluxer and all rights reserved, as set out in
[fluxer_static/LICENSE](./fluxer_static/LICENSE). Third-party material keeps its
own terms, listed in
[fluxer_static/THIRD_PARTY_LICENSES.md](./fluxer_static/THIRD_PARTY_LICENSES.md).
Public availability of this repository does not grant trademark, brand, or
endorsement rights.
[win-setup-x64]: https://pkgs.fluxer.com/desktop/stable/win32/x64/latest/setup
[win-setup-arm64]: https://pkgs.fluxer.com/desktop/stable/win32/arm64/latest/setup
[win-portable-x64]: https://pkgs.fluxer.com/desktop/stable/win32/x64/latest/portable
[win-portable-arm64]: https://pkgs.fluxer.com/desktop/stable/win32/arm64/latest/portable
[mac-dmg]: https://pkgs.fluxer.com/desktop/stable/darwin/arm64/latest/dmg
[linux-deb-x64]: https://pkgs.fluxer.com/desktop/stable/linux/x64/latest/deb
[linux-deb-arm64]: https://pkgs.fluxer.com/desktop/stable/linux/arm64/latest/deb
[linux-rpm-x64]: https://pkgs.fluxer.com/desktop/stable/linux/x64/latest/rpm
[linux-rpm-arm64]: https://pkgs.fluxer.com/desktop/stable/linux/arm64/latest/rpm
[linux-appimage-x64]: https://pkgs.fluxer.com/desktop/stable/linux/x64/latest/appimage
[linux-appimage-arm64]: https://pkgs.fluxer.com/desktop/stable/linux/arm64/latest/appimage
[linux-targz-x64]: https://pkgs.fluxer.com/desktop/stable/linux/x64/latest/tar_gz
[linux-targz-arm64]: https://pkgs.fluxer.com/desktop/stable/linux/arm64/latest/tar_gz
[flatpak-ref]: https://pkgs.fluxer.com/flatpak/fluxer.flatpakref
[android-apk]: https://github.com/fluxerapp/flutter_client/releases
[obtainium]: https://obtainium.imranr.dev/
[ios-testflight]: https://testflight.apple.com/join/PKZR6pK9
[docs]: https://docs.fluxer.app
[docs-downloads]: https://docs.fluxer.app/downloads/overview/
[docs-linux]: https://docs.fluxer.app/downloads/linux-repositories/
[docs-selfhost]: https://docs.fluxer.app/operator/get-started/
-1
View File
@@ -52,7 +52,6 @@ FLUXER_S3_SECRET_ACCESS_KEY=fluxer-secret
FLUXER_S3_FORCE_PATH_STYLE=true
FLUXER_S3_BUCKET_CDN=fluxer
FLUXER_S3_BUCKET_UPLOADS=fluxer-uploads
FLUXER_S3_BUCKET_DOWNLOADS=fluxer-downloads
FLUXER_S3_BUCKET_REPORTS=fluxer-reports
FLUXER_S3_BUCKET_HARVESTS=fluxer-harvests
FLUXER_S3_BUCKET_STATIC=fluxer-static
+1 -1
View File
@@ -82,7 +82,6 @@ MEILI_MASTER_KEY=CHANGE_ME
# exist already.
#FLUXER_S3_BUCKET_CDN=fluxer
#FLUXER_S3_BUCKET_UPLOADS=fluxer-uploads
#FLUXER_S3_BUCKET_DOWNLOADS=fluxer-downloads
#FLUXER_S3_BUCKET_REPORTS=fluxer-reports
#FLUXER_S3_BUCKET_HARVESTS=fluxer-harvests
@@ -282,6 +281,7 @@ FLUXER_DISCOVERY_ENABLED=true
#FLUXER_POSTGRES_WORK_MEM=8MB
#FLUXER_POSTGRES_MAINTENANCE_WORK_MEM=256MB
#FLUXER_POSTGRES_AUTOVACUUM_WORK_MEM=128MB
#FLUXER_POSTGRES_SHM_SIZE=1gb
# The bundled Valkey holds durable state as well as cache, so it runs with an
# append-only file and with noeviction, which fails an over-limit write instead
+2 -4
View File
@@ -46,7 +46,6 @@ x-fluxer-env: &fluxer-env
FLUXER_S3_FORCE_PATH_STYLE: "${FLUXER_S3_FORCE_PATH_STYLE:-true}"
FLUXER_S3_BUCKET_CDN: ${FLUXER_S3_BUCKET_CDN:-fluxer}
FLUXER_S3_BUCKET_UPLOADS: ${FLUXER_S3_BUCKET_UPLOADS:-fluxer-uploads}
FLUXER_S3_BUCKET_DOWNLOADS: ${FLUXER_S3_BUCKET_DOWNLOADS:-fluxer-downloads}
FLUXER_S3_BUCKET_REPORTS: ${FLUXER_S3_BUCKET_REPORTS:-fluxer-reports}
FLUXER_S3_BUCKET_HARVESTS: ${FLUXER_S3_BUCKET_HARVESTS:-fluxer-harvests}
AWS_ACCESS_KEY_ID: ${FLUXER_S3_ACCESS_KEY:?set FLUXER_S3_ACCESS_KEY in .env}
@@ -189,7 +188,7 @@ services:
-c autovacuum_vacuum_cost_limit=2000
-c track_io_timing=on
-c shared_preload_libraries=pg_stat_statements
shm_size: 256mb
shm_size: ${FLUXER_POSTGRES_SHM_SIZE:-1gb}
environment:
POSTGRES_DB: fluxer
POSTGRES_USER: fluxer
@@ -295,14 +294,13 @@ services:
FLUXER_S3_SECRET_KEY: ${FLUXER_S3_SECRET_KEY:?set FLUXER_S3_SECRET_KEY in .env}
FLUXER_S3_BUCKET_CDN: ${FLUXER_S3_BUCKET_CDN:-fluxer}
FLUXER_S3_BUCKET_UPLOADS: ${FLUXER_S3_BUCKET_UPLOADS:-fluxer-uploads}
FLUXER_S3_BUCKET_DOWNLOADS: ${FLUXER_S3_BUCKET_DOWNLOADS:-fluxer-downloads}
FLUXER_S3_BUCKET_REPORTS: ${FLUXER_S3_BUCKET_REPORTS:-fluxer-reports}
FLUXER_S3_BUCKET_HARVESTS: ${FLUXER_S3_BUCKET_HARVESTS:-fluxer-harvests}
entrypoint:
- /bin/sh
- -c
- >
buckets="$$FLUXER_S3_BUCKET_CDN $$FLUXER_S3_BUCKET_UPLOADS $$FLUXER_S3_BUCKET_DOWNLOADS $$FLUXER_S3_BUCKET_REPORTS $$FLUXER_S3_BUCKET_HARVESTS";
buckets="$$FLUXER_S3_BUCKET_CDN $$FLUXER_S3_BUCKET_UPLOADS $$FLUXER_S3_BUCKET_REPORTS $$FLUXER_S3_BUCKET_HARVESTS";
missing="$$buckets";
for attempt in $$(seq 1 60); do
if ! nc -z seaweedfs 9333 2>/dev/null; then
+1 -1
View File
@@ -9,7 +9,7 @@ WORKDIR /usr/src/app
RUN apt-get update \
&& apt-get install -y --no-install-recommends ca-certificates nodejs npm pkg-config \
&& npm install -g pnpm@12.4.2 \
&& npm install -g pnpm@11.27.0 \
&& rm -rf /var/lib/apt/lists/*
RUN npm install --no-audit --no-fund @tailwindcss/[email protected] [email protected]
-3
View File
@@ -15237,7 +15237,6 @@
"required": ["guild_id", "backend"]
}
},
"stereo_enabled": {"type": "boolean"},
"suppression_strength": {"type": "integer", "minimum": 0, "maximum": 100}
}
},
@@ -15342,7 +15341,6 @@
"additionalProperties": false
}
},
"stereo_enabled": {"default": false, "type": "boolean"},
"suppression_strength": {"default": 80, "type": "integer", "minimum": 0, "maximum": 100}
},
"required": [
@@ -15356,7 +15354,6 @@
"included_user_ids",
"excluded_user_ids",
"guild_overrides",
"stereo_enabled",
"suppression_strength"
],
"additionalProperties": false
+76 -43
View File
@@ -448,7 +448,8 @@ impl VoiceE2eeScope {
}
}
pub const VOICE_NS_MAX_TARGETED_USERS: usize = 1_000;
pub const EXPERIMENT_MAX_TARGETED_USERS: usize = 1_000;
pub const SCREEN_SHARE_DELIVERY_DEFAULT_SALT: &str = "screen-share-delivery-v1";
pub const VOICE_NS_MAX_GUILD_OVERRIDES: usize = 200;
impl NoiseSuppressionBackend {
@@ -494,7 +495,6 @@ pub struct VoiceNoiseSuppressionConfigResponse {
pub included_user_ids: Vec<String>,
pub excluded_user_ids: Vec<String>,
pub guild_overrides: Vec<VoiceNoiseSuppressionGuildOverride>,
pub stereo_enabled: bool,
pub suppression_strength: u32,
}
@@ -511,50 +511,11 @@ impl Default for VoiceNoiseSuppressionConfigResponse {
included_user_ids: Vec::new(),
excluded_user_ids: Vec::new(),
guild_overrides: Vec::new(),
stereo_enabled: false,
suppression_strength: 80,
}
}
}
#[derive(Clone, Debug, Deserialize, Serialize)]
#[serde(default)]
pub struct ScreenShareDeliveryConfigResponse {
pub enabled: bool,
pub config_version: u64,
pub rollout_basis_points: u32,
pub rollout_salt: String,
pub included_user_ids: Vec<String>,
pub excluded_user_ids: Vec<String>,
}
impl Default for ScreenShareDeliveryConfigResponse {
fn default() -> Self {
Self {
enabled: false,
config_version: 0,
rollout_basis_points: 0,
rollout_salt: "screen-share-delivery-v1".to_owned(),
included_user_ids: Vec::new(),
excluded_user_ids: Vec::new(),
}
}
}
#[derive(Clone, Debug, Default, Serialize)]
pub struct ScreenShareDeliveryConfigUpdateRequest {
#[serde(skip_serializing_if = "Option::is_none")]
pub enabled: Option<bool>,
#[serde(skip_serializing_if = "Option::is_none")]
pub rollout_basis_points: Option<u32>,
#[serde(skip_serializing_if = "Option::is_none")]
pub rollout_salt: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub included_user_ids: Option<Vec<String>>,
#[serde(skip_serializing_if = "Option::is_none")]
pub excluded_user_ids: Option<Vec<String>>,
}
#[derive(Clone, Debug, Default, Serialize)]
pub struct VoiceNoiseSuppressionConfigUpdateRequest {
#[serde(skip_serializing_if = "Option::is_none")]
@@ -576,11 +537,47 @@ pub struct VoiceNoiseSuppressionConfigUpdateRequest {
#[serde(skip_serializing_if = "Option::is_none")]
pub guild_overrides: Option<Vec<VoiceNoiseSuppressionGuildOverride>>,
#[serde(skip_serializing_if = "Option::is_none")]
pub stereo_enabled: Option<bool>,
#[serde(skip_serializing_if = "Option::is_none")]
pub suppression_strength: Option<u32>,
}
#[derive(Clone, Debug, Deserialize, Serialize)]
#[serde(default)]
pub struct ScreenShareDeliveryConfigResponse {
pub enabled: bool,
pub config_version: u64,
pub rollout_basis_points: u32,
pub rollout_salt: String,
pub included_user_ids: Vec<String>,
pub excluded_user_ids: Vec<String>,
}
impl Default for ScreenShareDeliveryConfigResponse {
fn default() -> Self {
Self {
enabled: false,
config_version: 0,
rollout_basis_points: 0,
rollout_salt: SCREEN_SHARE_DELIVERY_DEFAULT_SALT.to_owned(),
included_user_ids: Vec::new(),
excluded_user_ids: Vec::new(),
}
}
}
#[derive(Clone, Debug, Default, Serialize)]
pub struct ScreenShareDeliveryConfigUpdateRequest {
#[serde(skip_serializing_if = "Option::is_none")]
pub enabled: Option<bool>,
#[serde(skip_serializing_if = "Option::is_none")]
pub rollout_basis_points: Option<u32>,
#[serde(skip_serializing_if = "Option::is_none")]
pub rollout_salt: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub included_user_ids: Option<Vec<String>>,
#[serde(skip_serializing_if = "Option::is_none")]
pub excluded_user_ids: Option<Vec<String>>,
}
#[derive(Clone, Debug, Deserialize, Serialize)]
#[serde(default)]
pub struct ExperimentDeliveryConfigResponse {
@@ -1036,18 +1033,30 @@ mod tests {
.expect("admin schema");
let noise = serde_json::from_value::<VoiceNoiseSuppressionConfigResponse>(json!({}))
.expect("default noise config");
let screen_share = serde_json::from_value::<ScreenShareDeliveryConfigResponse>(json!({}))
.expect("default screen share config");
let delivery = serde_json::from_value::<ExperimentDeliveryConfigResponse>(json!({}))
.expect("default delivery config");
let noise = serde_json::to_value(noise).expect("serializable noise config");
let screen_share =
serde_json::to_value(screen_share).expect("serializable screen share config");
let delivery = serde_json::to_value(delivery).expect("serializable delivery config");
let generated_noise: generated_types::VoiceNoiseSuppressionConfigResponse =
serde_json::from_value(noise.clone()).expect("generated noise config contract");
let generated_screen_share: generated_types::ScreenShareDeliveryConfigResponse =
serde_json::from_value(screen_share.clone())
.expect("generated screen share config contract");
let generated_delivery: generated_types::ExperimentDeliveryConfigResponse =
serde_json::from_value(delivery.clone()).expect("generated delivery config contract");
assert_eq!(
serde_json::to_value(generated_noise).expect("serializable generated noise config"),
noise
);
assert_eq!(
serde_json::to_value(generated_screen_share)
.expect("serializable generated screen share config"),
screen_share
);
assert_eq!(
serde_json::to_value(generated_delivery)
.expect("serializable generated delivery config"),
@@ -1055,6 +1064,7 @@ mod tests {
);
for (name, value) in [
("VoiceNoiseSuppressionConfigResponse", noise),
("ScreenShareDeliveryConfigResponse", screen_share),
("ExperimentDeliveryConfigResponse", delivery),
] {
for (field, value) in value.as_object().expect("config object") {
@@ -1066,6 +1076,29 @@ mod tests {
}
}
#[test]
fn screen_share_delivery_update_preserves_empty_lists_and_omitted_fields() {
let update = ScreenShareDeliveryConfigUpdateRequest {
included_user_ids: Some(Vec::new()),
excluded_user_ids: Some(Vec::new()),
..Default::default()
};
let value = serde_json::to_value(update).expect("serializable update");
serde_json::from_value::<generated_types::ScreenShareDeliveryConfigUpdateRequest>(
value.clone(),
)
.expect("generated update contract");
assert_eq!(
value,
json!({"included_user_ids": [], "excluded_user_ids": []})
);
assert_eq!(
serde_json::to_value(ScreenShareDeliveryConfigUpdateRequest::default())
.expect("serializable update"),
json!({})
);
}
#[test]
fn noise_suppression_update_preserves_empty_lists_and_omitted_fields() {
let update = VoiceNoiseSuppressionConfigUpdateRequest {
+126 -52
View File
@@ -7,19 +7,19 @@ use crate::{
AppBrandingConfigUpdateRequest, AppLegalConfigUpdateRequest,
AppPublicConfigUpdateRequest, AppRegistrationConfigUpdateRequest,
AppSetupConfigUpdateRequest, CreateRegistrationUrlRequest,
DeferredPhoneGateUpdateRequest, ExperimentDeliveryConfigUpdateRequest,
GatewayRolloutConfigUpdateRequest, GatewayRolloutMode,
InstanceAttachmentDecayUpdateRequest, InstanceBlueskyIntegrationUpdateRequest,
InstanceBlueskyKeyIntegrationUpdateRequest, InstanceCaptchaIntegrationUpdateRequest,
InstanceConfigUpdateRequest, InstanceEmailIntegrationUpdateRequest,
InstanceEmailSmtpIntegrationUpdateRequest, InstanceEmailSmtpTestRequest,
InstanceGifIntegrationUpdateRequest, InstanceIntegrationsUpdateRequest,
InstanceMediaUpdateRequest, InstancePolicyUpdateRequest,
InstanceRegistrationConfigUpdateRequest, InstanceServicesUpdateRequest,
InstanceYoutubeIntegrationUpdateRequest, LimitConfigUpdateRequest, LimitRule,
LimitRuleFilters, NoiseSuppressionBackend, PremiumMode, RegistrationMode,
ScreenShareDeliveryConfigUpdateRequest, SsoConfigUpdateRequest,
VOICE_NS_MAX_GUILD_OVERRIDES, VOICE_NS_MAX_TARGETED_USERS, VoiceE2eeScope,
DeferredPhoneGateUpdateRequest, EXPERIMENT_MAX_TARGETED_USERS,
ExperimentDeliveryConfigUpdateRequest, GatewayRolloutConfigUpdateRequest,
GatewayRolloutMode, InstanceAttachmentDecayUpdateRequest,
InstanceBlueskyIntegrationUpdateRequest, InstanceBlueskyKeyIntegrationUpdateRequest,
InstanceCaptchaIntegrationUpdateRequest, InstanceConfigUpdateRequest,
InstanceEmailIntegrationUpdateRequest, InstanceEmailSmtpIntegrationUpdateRequest,
InstanceEmailSmtpTestRequest, InstanceGifIntegrationUpdateRequest,
InstanceIntegrationsUpdateRequest, InstanceMediaUpdateRequest,
InstancePolicyUpdateRequest, InstanceRegistrationConfigUpdateRequest,
InstanceServicesUpdateRequest, InstanceYoutubeIntegrationUpdateRequest,
LimitConfigUpdateRequest, LimitRule, LimitRuleFilters, NoiseSuppressionBackend,
PremiumMode, RegistrationMode, ScreenShareDeliveryConfigUpdateRequest,
SsoConfigUpdateRequest, VOICE_NS_MAX_GUILD_OVERRIDES, VoiceE2eeScope,
VoiceNoiseSuppressionConfigUpdateRequest, VoiceNoiseSuppressionGuildOverride,
},
},
@@ -518,9 +518,9 @@ fn parse_experiment_user_ids(value: &str, label: &str) -> Result<Vec<String>, St
if ids.iter().any(|existing| existing == candidate) {
continue;
}
if ids.len() == VOICE_NS_MAX_TARGETED_USERS {
if ids.len() == EXPERIMENT_MAX_TARGETED_USERS {
return Err(format!(
"{label} must contain at most {VOICE_NS_MAX_TARGETED_USERS} unique IDs"
"{label} must contain at most {EXPERIMENT_MAX_TARGETED_USERS} unique IDs"
));
}
ids.push(candidate.to_owned());
@@ -573,38 +573,6 @@ fn parse_voice_noise_suppression_guild_overrides(
Ok(overrides)
}
fn build_screen_share_delivery_update(
form: &MultiValueForm,
) -> Result<InstanceConfigUpdateRequest, String> {
Ok(InstanceConfigUpdateRequest {
screen_share_delivery: Some(ScreenShareDeliveryConfigUpdateRequest {
enabled: Some(form.bool_value("screen_share_delivery_enabled")),
rollout_basis_points: parse_form_number(
form,
"screen_share_delivery_rollout_basis_points",
"Rollout basis points",
0,
EXPERIMENT_ROLLOUT_BASIS_POINTS_MAX,
)?,
rollout_salt: parse_experiment_rollout_salt(
form,
"screen_share_delivery_rollout_salt",
)?,
included_user_ids: Some(parse_experiment_user_ids(
form.first("screen_share_delivery_included_user_ids")
.unwrap_or_default(),
"Included user IDs",
)?),
excluded_user_ids: Some(parse_experiment_user_ids(
form.first("screen_share_delivery_excluded_user_ids")
.unwrap_or_default(),
"Excluded user IDs",
)?),
}),
..Default::default()
})
}
fn build_voice_noise_suppression_update(
form: &MultiValueForm,
) -> Result<InstanceConfigUpdateRequest, String> {
@@ -653,7 +621,6 @@ fn build_voice_noise_suppression_update(
guild_overrides: Some(parse_voice_noise_suppression_guild_overrides(
form.first("voice_ns_guild_overrides").unwrap_or_default(),
)?),
stereo_enabled: Some(form.bool_value("voice_ns_stereo_enabled")),
suppression_strength: parse_form_number(
form,
"voice_ns_suppression_strength",
@@ -666,6 +633,38 @@ fn build_voice_noise_suppression_update(
})
}
fn build_screen_share_delivery_update(
form: &MultiValueForm,
) -> Result<InstanceConfigUpdateRequest, String> {
Ok(InstanceConfigUpdateRequest {
screen_share_delivery: Some(ScreenShareDeliveryConfigUpdateRequest {
enabled: Some(form.bool_value("screen_share_delivery_enabled")),
rollout_basis_points: parse_form_number(
form,
"screen_share_delivery_rollout_basis_points",
"Rollout basis points",
0,
EXPERIMENT_ROLLOUT_BASIS_POINTS_MAX,
)?,
rollout_salt: parse_experiment_rollout_salt(
form,
"screen_share_delivery_rollout_salt",
)?,
included_user_ids: Some(parse_experiment_user_ids(
form.first("screen_share_delivery_included_user_ids")
.unwrap_or_default(),
"Included user IDs",
)?),
excluded_user_ids: Some(parse_experiment_user_ids(
form.first("screen_share_delivery_excluded_user_ids")
.unwrap_or_default(),
"Excluded user IDs",
)?),
}),
..Default::default()
})
}
fn build_experiment_delivery_update(
form: &MultiValueForm,
) -> Result<InstanceConfigUpdateRequest, String> {
@@ -1283,7 +1282,6 @@ mod tests {
.expect("voice noise suppression update");
assert_eq!(update.enabled, Some(true));
assert_eq!(update.allow_user_override, Some(true));
assert_eq!(update.stereo_enabled, Some(false));
assert_eq!(
update.default_backend,
Some(NoiseSuppressionBackend::Rnnoise)
@@ -1309,7 +1307,6 @@ mod tests {
serde_json::json!({"voice_noise_suppression": {
"enabled": false,
"allow_user_override": false,
"stereo_enabled": false,
"enabled_backends": [],
"included_user_ids": [],
"excluded_user_ids": [],
@@ -1387,13 +1384,13 @@ mod tests {
#[test]
fn parse_experiment_user_ids_rejects_exceeding_the_cap() {
let value = (0..VOICE_NS_MAX_TARGETED_USERS)
let value = (0..EXPERIMENT_MAX_TARGETED_USERS)
.map(|index| index.to_string())
.collect::<Vec<_>>()
.join("\n");
let ids = parse_experiment_user_ids(&format!("{value}\n999"), "Included user IDs")
.expect("valid IDs at cap");
assert_eq!(ids.len(), VOICE_NS_MAX_TARGETED_USERS);
assert_eq!(ids.len(), EXPERIMENT_MAX_TARGETED_USERS);
assert_eq!(ids.last(), Some(&"999".to_owned()));
assert_eq!(
parse_experiment_user_ids(&format!("{value}\n1000"), "Included user IDs")
@@ -1589,6 +1586,83 @@ mod tests {
}
}
#[test]
fn build_screen_share_delivery_update_reads_the_rollout_fields() {
let form = MultiValueForm::parse(
b"screen_share_delivery_enabled=true&screen_share_delivery_rollout_basis_points=%20250%20&screen_share_delivery_rollout_salt=%20screen-share-delivery-v2%20&screen_share_delivery_included_user_ids=1500000000000000001%0A1500000000000000002&screen_share_delivery_excluded_user_ids=1500000000000000003%2C%201500000000000000004",
);
let update = build_screen_share_delivery_update(&form)
.expect("valid form")
.screen_share_delivery
.expect("screen share delivery update");
assert_eq!(update.enabled, Some(true));
assert_eq!(update.rollout_basis_points, Some(250));
assert_eq!(
update.rollout_salt,
Some("screen-share-delivery-v2".to_owned())
);
assert_eq!(
update.included_user_ids,
Some(vec![
"1500000000000000001".to_owned(),
"1500000000000000002".to_owned()
])
);
assert_eq!(
update.excluded_user_ids,
Some(vec![
"1500000000000000003".to_owned(),
"1500000000000000004".to_owned()
])
);
}
#[test]
fn build_screen_share_delivery_update_leaves_the_feature_inert_when_nothing_is_submitted() {
let form = MultiValueForm::parse(b"_csrf=token");
let request = build_screen_share_delivery_update(&form).expect("valid form");
assert_eq!(
serde_json::to_value(request).expect("serializable update"),
serde_json::json!({"screen_share_delivery": {
"enabled": false,
"included_user_ids": [],
"excluded_user_ids": [],
}})
);
}
#[test]
fn build_screen_share_delivery_update_rejects_invalid_rollout_fields() {
for (form, message) in [
(
"screen_share_delivery_rollout_basis_points=10001",
"Rollout basis points must be a whole number between 0 and 10000",
),
(
"screen_share_delivery_rollout_basis_points=abc",
"Rollout basis points must be a whole number between 0 and 10000",
),
(
"screen_share_delivery_rollout_salt=%20%20",
"Rollout salt must be between 1 and 64 characters",
),
(
"screen_share_delivery_included_user_ids=123%2Cinvalid",
"Included user IDs entry 2 must contain 1 to 20 decimal digits",
),
(
"screen_share_delivery_excluded_user_ids=123%2Cinvalid",
"Excluded user IDs entry 2 must contain 1 to 20 decimal digits",
),
] {
let form = MultiValueForm::parse(form.as_bytes());
assert_eq!(
build_screen_share_delivery_update(&form).expect_err("invalid rollout field"),
message
);
}
}
#[test]
fn build_experiment_delivery_update_leaves_both_fields_unchanged_when_absent() {
let form = MultiValueForm::parse(b"_csrf=token");
+5 -9
View File
@@ -73,15 +73,11 @@ pub async fn render(
.map(|r| r.sessions)
.map_err(|error| tracing::warn!(%error, user_id, "admin API request failed: list user sessions"))
.unwrap_or_default();
let webauthn_credentials = if u.authenticator_types.contains(&2) {
client
.list_webauthn_credentials(user_id)
.await
.map_err(|error| tracing::warn!(%error, user_id, "admin API request failed: list webauthn credentials"))
.unwrap_or_default()
} else {
Vec::new()
};
let webauthn_credentials = client
.list_webauthn_credentials(user_id)
.await
.map_err(|error| tracing::warn!(%error, user_id, "admin API request failed: list webauthn credentials"))
.unwrap_or_default();
Some(tabs::account::account_tab(
config,
&u,
@@ -2,12 +2,13 @@
use crate::{
api::types::{
AppPublicConfigResponse, ExperimentDeliveryConfigResponse, GatewayRolloutConfigResponse,
InstanceConfigResponse, InstanceIntegrationsResponse, InstanceMediaResponse,
InstancePolicyResponse, InstanceRegistrationResponse, LimitConfigResponse,
NoiseSuppressionBackend, PendingRegistrationResponse, RegistrationUrlResponse,
AppPublicConfigResponse, EXPERIMENT_MAX_TARGETED_USERS, ExperimentDeliveryConfigResponse,
GatewayRolloutConfigResponse, InstanceConfigResponse, InstanceIntegrationsResponse,
InstanceMediaResponse, InstancePolicyResponse, InstanceRegistrationResponse,
LimitConfigResponse, NoiseSuppressionBackend, PendingRegistrationResponse,
RegistrationUrlResponse, SCREEN_SHARE_DELIVERY_DEFAULT_SALT,
ScreenShareDeliveryConfigResponse, SsoConfigResponse, VOICE_NS_MAX_GUILD_OVERRIDES,
VOICE_NS_MAX_TARGETED_USERS, VoiceNoiseSuppressionConfigResponse,
VoiceNoiseSuppressionConfigResponse,
},
config::AdminConfig,
middleware::auth::AuthContext,
@@ -1106,7 +1107,7 @@ fn voice_noise_suppression_section(
))
(entry_count_hint(
voice_noise_suppression.included_user_ids.len(),
VOICE_NS_MAX_TARGETED_USERS,
EXPERIMENT_MAX_TARGETED_USERS,
))
p class="text-xs text-neutral-500" {
"One snowflake per line, or comma separated. These users are targeted \
@@ -1126,7 +1127,7 @@ fn voice_noise_suppression_section(
))
(entry_count_hint(
voice_noise_suppression.excluded_user_ids.len(),
VOICE_NS_MAX_TARGETED_USERS,
EXPERIMENT_MAX_TARGETED_USERS,
))
p class="text-xs text-neutral-500" {
"Same format. Exclusion wins over both the always-on list and the \
@@ -1158,17 +1159,6 @@ fn voice_noise_suppression_section(
}
h3 class="text-sm font-semibold text-neutral-900" { "Processing" }
(checkbox(
"voice_ns_stereo_enabled",
"true",
"Process stereo input instead of downmixing to mono",
voice_noise_suppression.stereo_enabled,
true,
))
p class="text-xs text-neutral-500" {
"Costs more CPU on the client. Leave off unless you are testing stereo \
capture."
}
div class="grid grid-cols-1 gap-4 sm:grid-cols-2" {
(number_field(
"voice_ns_suppression_strength",
@@ -1202,13 +1192,10 @@ fn screen_share_delivery_section(
let excluded_user_ids = screen_share_delivery.excluded_user_ids.join("\n");
section_card_with_description(
"Screen Share Delivery",
"Picks whether a targeted client adapts its own screen share after it starts. A targeted \
client measures what the encoder sends, steps the share down a resolution and frame rate \
ladder while it cannot keep up, probes back up when it can, remembers the settled rung \
per codec, and tells the sharer what is going out. A client that is not targeted \
publishes the same target and never changes it. Both arms still get the publish-time \
fixes and both arms still recover a stalled encoder. While the master switch below is \
off every client publishes and holds, whatever the rest of these fields say.",
"Pick how many clients publish screen shares through the reworked delivery path. While \
the master switch below is off nothing on this form reaches any client: every user \
keeps the screen share pipeline they have today, whatever the rest of these fields say. \
A client that is already sharing keeps the path it started on until the share ends.",
html! {
form method="post" action={(base) "/instance-config?action=update_screen_share_delivery"} {
(csrf_input(csrf_token))
@@ -1229,8 +1216,8 @@ fn screen_share_delivery_section(
))
p class="text-xs text-neutral-500" {
"Off is the safe state. With this unchecked every client is told the \
rollout is inert and holds its published target for the life of the \
share, so the rollout and targeting fields below have no effect at all."
feature is inert and keeps its current behavior, so the rollout and \
targeting fields below have no effect at all."
}
h3 class="text-sm font-semibold text-neutral-900" { "Rollout" }
@@ -1246,7 +1233,7 @@ fn screen_share_delivery_section(
"screen_share_delivery_rollout_salt",
"Rollout Salt",
&screen_share_delivery.rollout_salt,
"screen-share-delivery-v1",
SCREEN_SHARE_DELIVERY_DEFAULT_SALT,
))
p class="text-xs text-neutral-500" {
"Seeds the bucketing hash. Changing it reshuffles which users fall \
@@ -1263,6 +1250,10 @@ fn screen_share_delivery_section(
4,
false,
))
(entry_count_hint(
screen_share_delivery.included_user_ids.len(),
EXPERIMENT_MAX_TARGETED_USERS,
))
p class="text-xs text-neutral-500" {
"One snowflake per line, or comma separated. These users are targeted \
regardless of the percentage above. IDs must contain 1 to 20 decimal \
@@ -1279,6 +1270,10 @@ fn screen_share_delivery_section(
4,
false,
))
(entry_count_hint(
screen_share_delivery.excluded_user_ids.len(),
EXPERIMENT_MAX_TARGETED_USERS,
))
p class="text-xs text-neutral-500" {
"Same format. Exclusion wins over both the always-on list and the \
percentage, so this is the per-user kill switch."
@@ -1302,7 +1297,7 @@ fn experiment_delivery_section(
section_card_with_description(
"Experiment Delivery",
"How often every client revalidates its experiment assignments. This is instance-wide \
and covers every experiment, not just the one above. Raising the interval sheds \
and covers every experiment, not just the ones above. Raising the interval sheds \
request volume and makes a change take longer to reach a client. Raising the jitter \
spreads a fleet that has synchronised on one tick back out across the interval.",
html! {
@@ -1937,10 +1932,29 @@ mod tests {
assert!(!markup.contains("at the cap"));
}
#[test]
fn screen_share_delivery_section_shows_list_counts_and_the_master_switch() {
let screen_share_delivery = ScreenShareDeliveryConfigResponse {
included_user_ids: vec!["1500000000000000001".to_owned()],
excluded_user_ids: vec![
"1500000000000000002".to_owned(),
"1500000000000000003".to_owned(),
],
..ScreenShareDeliveryConfigResponse::default()
};
let markup =
screen_share_delivery_section("/admin", "csrf", &screen_share_delivery).into_string();
assert!(markup.contains("action=update_screen_share_delivery"));
assert!(markup.contains("screen_share_delivery_enabled"));
assert!(markup.contains("1 of 1000 stored"));
assert!(markup.contains("2 of 1000 stored"));
assert!(!markup.contains("at the cap"));
}
#[test]
fn voice_noise_suppression_section_flags_a_list_at_its_cap() {
let voice_noise_suppression = VoiceNoiseSuppressionConfigResponse {
included_user_ids: (0..VOICE_NS_MAX_TARGETED_USERS)
included_user_ids: (0..EXPERIMENT_MAX_TARGETED_USERS)
.map(|index| index.to_string())
.collect(),
..VoiceNoiseSuppressionConfigResponse::default()
+14 -5
View File
@@ -403,7 +403,6 @@ fn deserialize_instance_config_response_with_unknown_keys() {
"included_user_ids": [],
"excluded_user_ids": [],
"guild_overrides": [],
"stereo_enabled": false,
"suppression_strength": 80,
"future_presentation_knob": "verbose",
"future_knob": 7,
@@ -411,11 +410,12 @@ fn deserialize_instance_config_response_with_unknown_keys() {
"future_list_knob": ["a", "b"]
},
"screen_share_delivery": {
"enabled": false,
"config_version": 0,
"rollout_basis_points": 0,
"enabled": true,
"config_version": 2,
"rollout_basis_points": 2500,
"rollout_salt": "screen-share-delivery-v1",
"included_user_ids": [],
"included_user_ids": ["1500000000000000001"],
"future_delivery_knob": 9,
"excluded_user_ids": []
},
"experiment_delivery": {"poll_interval_seconds": 300, "poll_jitter_percent": 15},
@@ -547,6 +547,14 @@ fn deserialize_instance_config_response_with_unknown_keys() {
assert_eq!(resp.voice_noise_suppression.rollout_basis_points, 10000);
assert_eq!(*resp.voice_noise_suppression.rollout_salt, "voice-ns-v1");
assert_eq!(resp.voice_noise_suppression.enabled_backends.len(), 3);
assert!(resp.screen_share_delivery.enabled);
assert_eq!(resp.screen_share_delivery.config_version, 2);
assert_eq!(resp.screen_share_delivery.rollout_basis_points, 2500);
assert_eq!(
*resp.screen_share_delivery.rollout_salt,
"screen-share-delivery-v1"
);
assert_eq!(resp.screen_share_delivery.included_user_ids.len(), 1);
assert_eq!(resp.experiment_delivery.poll_interval_seconds, 300);
assert!(resp.policy.single_community_guild_id.is_none());
assert_eq!(resp.policy.services.gif_enabled, Some(true));
@@ -557,6 +565,7 @@ fn deserialize_instance_config_response_with_unknown_keys() {
.replace("\"future_rollout_knob\": 3,", "")
.replace("\"future_presentation_knob\": \"verbose\",", "")
.replace("\"future_knob\": 7,", "")
.replace("\"future_delivery_knob\": 9,", "")
.replace("\"future_object_knob\": {\"nested\": true},", "")
.replace("\"future_list_knob\": [\"a\", \"b\"],", "")
.replace(
+8 -1
View File
@@ -1195,9 +1195,16 @@ fn instance_config() -> Value {
"included_user_ids": [],
"excluded_user_ids": [],
"guild_overrides": [],
"stereo_enabled": false,
"suppression_strength": 80
},
"screen_share_delivery": {
"enabled": false,
"config_version": 0,
"rollout_basis_points": 0,
"rollout_salt": "screen-share-delivery-v1",
"included_user_ids": [],
"excluded_user_ids": []
},
"experiment_delivery": {
"poll_interval_seconds": 300,
"poll_jitter_percent": 15
@@ -95,39 +95,6 @@ async fn clearing_the_region_restriction_fields_reaches_the_api_as_empty_lists()
assert_eq!(body["allowed_guild_ids"], json!([]));
}
#[tokio::test]
async fn the_screen_share_delivery_form_keeps_the_included_and_excluded_lists_apart() {
let app = setup().await;
let csrf_token = csrf_token(&app).await;
let status = post_form(
&app,
"/instance-config?action=update_screen_share_delivery",
&format!(
"_csrf={csrf_token}&screen_share_delivery_enabled=true\
&screen_share_delivery_rollout_basis_points=2500\
&screen_share_delivery_rollout_salt=screen-share-delivery-v1\
&screen_share_delivery_included_user_ids=1500000000000000001\
&screen_share_delivery_excluded_user_ids=1500000000000000002"
),
)
.await;
assert_eq!(status, StatusCode::SEE_OTHER);
let body = captured_body(&app, "PATCH /admin/instance/config");
let delivery = &body["screen_share_delivery"];
assert_eq!(delivery["enabled"], json!(true));
assert_eq!(delivery["rollout_basis_points"], json!(2500));
assert_eq!(delivery["rollout_salt"], json!("screen-share-delivery-v1"));
assert_eq!(
delivery["included_user_ids"],
json!(["1500000000000000001"])
);
assert_eq!(
delivery["excluded_user_ids"],
json!(["1500000000000000002"])
);
}
struct TestApp {
router: Router,
session_cookie: String,
+2 -2
View File
@@ -5,7 +5,7 @@ FROM node:26-trixie-slim AS base
WORKDIR /usr/src/app
RUN npm install -g pnpm@12.4.2
RUN npm install -g pnpm@11.27.0
FROM base AS deploy
@@ -57,7 +57,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
libvips42t64 && \
rm -rf /var/lib/apt/lists/*
RUN npm install -g pnpm@12.4.2
RUN npm install -g pnpm@11.27.0
COPY --from=deploy /out ./
COPY --from=deploy /usr/src/app/fluxer_api/dist ./dist
+1 -1
View File
@@ -94,5 +94,5 @@
"typescript": "catalog:ts7",
"vitest": "catalog:"
},
"packageManager": "pnpm@12.4.2"
"packageManager": "pnpm@11.27.0"
}
+13 -6
View File
@@ -11,6 +11,8 @@ interface PostgresIpInfoOptions {
}
const VALUE_SEPARATOR = '\u001f';
export const IPINFO_CACHE_TTL_SECONDS = 14 * 24 * 60 * 60;
export const IPINFO_REQUEST_AUDIT_TTL_SECONDS = 90 * 24 * 60 * 60;
function getClient(options: PostgresIpInfoOptions): IPostgresClient | null {
return options.client ?? options.getClient?.() ?? null;
@@ -34,12 +36,9 @@ async function upsertKvRow(
partitionKey: string,
key: string,
row: Record<string, unknown>,
ttlSeconds?: number,
ttlSeconds: number,
): Promise<void> {
const expiresAt =
ttlSeconds != null && Number.isFinite(ttlSeconds) && ttlSeconds > 0
? new Date(Date.now() + ttlSeconds * 1000)
: null;
const expiresAt = new Date(Date.now() + ttlSeconds * 1000);
await client.query(
`INSERT INTO ${table(client)} (table_name, partition_key, row_key, row_data, expires_at, updated_at)
VALUES ($1, $2, $3, $4::jsonb, $5, now())
@@ -77,7 +76,14 @@ export function createPostgresIpInfoCache(options: PostgresIpInfoOptions): IpInf
try {
const client = getClient(options);
if (!client) return;
await upsertKvRow(client, 'ipinfo_cache', rowKey([key]), rowKey([key]), {cache_key: key, payload}, ttlSeconds);
await upsertKvRow(
client,
'ipinfo_cache',
rowKey([key]),
rowKey([key]),
{cache_key: key, payload},
ttlSeconds != null && Number.isFinite(ttlSeconds) && ttlSeconds > 0 ? ttlSeconds : IPINFO_CACHE_TTL_SECONDS,
);
} catch (error) {
options.onError?.(error, 'ipinfo_cache_set');
}
@@ -124,6 +130,7 @@ export function createPostgresIpInfoRequestAuditLogger(options: PostgresIpInfoOp
is_residential_proxy: event.isResidentialProxy,
metadata_json: serializeMetadata(event.metadata),
},
IPINFO_REQUEST_AUDIT_TTL_SECONDS,
);
} catch (error) {
options.onError?.(error, 'ipinfo_request_audit_record');
+5 -3
View File
@@ -1,7 +1,7 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import assert from 'node:assert/strict';
import type {Pool, PoolClient, QueryResult, QueryResultRow} from 'pg';
import type {Pool, PoolClient, PoolConfig, QueryResult, QueryResultRow} from 'pg';
import pg from 'pg';
const MAX_DIAGNOSTIC_FIELD_LENGTH = 128;
@@ -131,7 +131,7 @@ class PostgresClient implements IPostgresClient {
}
private async openPool(): Promise<void> {
const pool = new pg.Pool({
const poolConfig: PoolConfig & {scramMaxIterations: number} = {
connectionString: this.config.url || undefined,
host: this.config.url ? undefined : (this.config.host ?? '127.0.0.1'),
port: this.config.url ? undefined : (this.config.port ?? 5432),
@@ -140,7 +140,9 @@ class PostgresClient implements IPostgresClient {
password: this.config.url ? undefined : (this.config.password ?? 'fluxer'),
ssl: this.config.ssl ? {rejectUnauthorized: true, ca: normalizePem(this.config.sslCa)} : undefined,
max: this.config.maxConnections ?? 20,
});
scramMaxIterations: 0,
};
const pool = new pg.Pool(poolConfig);
this.observePoolConnections(pool);
try {
const client = await pool.connect();
-22
View File
@@ -3,7 +3,6 @@
import type {APIConfig, BlueskyOAuthConfig} from '@app/api/config/APIConfig';
import type {WorkerTaskName} from '@app/api/worker/WorkerLaneConfig';
import type {MasterConfig} from '@fluxer/config/src/MasterConfig';
import {resolveDownloadsProvider} from '@fluxer/config/src/S3DownloadsProvider';
import {parseIpAddress} from '@fluxer/ip_utils/src/IpAddress';
import {parseGeoipSourceConfig, resolveGeoipRuntimeSourceConfig} from '@pkgs/geoip/src/GeoipStartup';
@@ -92,18 +91,6 @@ function normalizeIpBanExemptIps(values: Array<string>): Array<string> {
return Array.from(normalized);
}
function normalizeCountryCodes(values: Array<string>, configName: string): ReadonlySet<string> {
const normalized = new Set<string>();
for (const value of values) {
const countryCode = value.trim().toUpperCase();
if (!/^[A-Z]{2}$/u.test(countryCode)) {
throw new Error(`${configName} contains an invalid ISO 3166-1 alpha-2 country code: ${value}`);
}
normalized.add(countryCode);
}
return normalized;
}
function mapPushProviderApps(
apps:
| Array<{
@@ -157,7 +144,6 @@ export function buildAPIConfigFromMaster(master: MasterConfig): APIConfig {
const s3Buckets = s3Config.buckets ?? {
cdn: '',
uploads: '',
downloads: '',
reports: '',
harvests: '',
};
@@ -174,10 +160,6 @@ export function buildAPIConfigFromMaster(master: MasterConfig): APIConfig {
requestTimeoutMs: master.services.api.request_timeout_ms,
maxInflightRequests: master.services.api.max_inflight_requests,
ipBanExemptIps: normalizeIpBanExemptIps(master.services.api.ip_ban_exempt_ips),
desktopGitHubRedirectCountries: normalizeCountryCodes(
master.services.api.desktop_github_redirect_countries,
'FLUXER_API_DESKTOP_GITHUB_REDIRECT_COUNTRIES',
),
cassandra: {
hosts: cassandraSource?.hosts.join(',') ?? '',
port: cassandraSource?.port ?? 9042,
@@ -290,8 +272,6 @@ export function buildAPIConfigFromMaster(master: MasterConfig): APIConfig {
donationProxyKey,
},
hosts: {
invite: extractHostname(master.endpoints.invite),
gift: extractHostname(master.endpoints.gift),
marketing: extractHostname(master.endpoints.marketing),
unfurlIgnored: master.services.api.unfurl_ignored_hosts,
},
@@ -304,7 +284,6 @@ export function buildAPIConfigFromMaster(master: MasterConfig): APIConfig {
cacheMinTtlSeconds: master.services.api.embeds.cache_min_ttl_seconds,
cacheRespectRemoteTtl: master.services.api.embeds.cache_respect_remote_ttl,
},
s3Downloads: resolveDownloadsProvider(master),
s3: {
endpoint: s3Config.endpoint,
presignedUrlBase: s3Config.presigned_url_base,
@@ -523,7 +502,6 @@ export function buildAPIConfigFromMaster(master: MasterConfig): APIConfig {
validateResponses: resolveValidateResponses(master),
},
presignedAttachmentUploadsEnabled: master.services.api.presigned_attachment_uploads_enabled ?? false,
presignedDownloadsEnabled: master.services.api.presigned_downloads_enabled ?? false,
presignedHarvestDownloadsEnabled: master.services.api.presigned_harvest_downloads_enabled ?? true,
attachmentDecayEnabled: master.attachment_decay_enabled,
deletionGracePeriodHours: master.dev.test_mode_enabled ? 0.01 : master.deletion_grace_period_hours,
+38
View File
@@ -341,6 +341,7 @@ import {
type UsersPendingDeletionRow,
} from '@app/api/database/types/UserTypes';
import {ATTACHMENT_DECAY_COLUMNS, type AttachmentDecayRow} from '@app/api/types/AttachmentDecayTypes';
import {seconds} from 'itty-time';
export const Users = defineTable<UserRow, 'user_id'>({
name: 'users',
@@ -499,16 +500,19 @@ export const GuildAuditLogs = defineTable<GuildAuditLogRow, 'guild_id' | 'log_id
name: 'guild_audit_logs_v2',
columns: GUILD_AUDIT_LOG_COLUMNS,
primaryKey: ['guild_id', 'log_id'],
defaultTtlSeconds: seconds('45 days'),
});
export const GuildAuditLogsByUser = defineTable<GuildAuditLogRow, 'guild_id' | 'user_id' | 'log_id'>({
name: 'guild_audit_logs_v2_by_user',
columns: GUILD_AUDIT_LOG_COLUMNS,
primaryKey: ['guild_id', 'user_id', 'log_id'],
defaultTtlSeconds: seconds('45 days'),
});
export const GuildAuditLogsByAction = defineTable<GuildAuditLogRow, 'guild_id' | 'action_type' | 'log_id'>({
name: 'guild_audit_logs_v2_by_action',
columns: GUILD_AUDIT_LOG_COLUMNS,
primaryKey: ['guild_id', 'action_type', 'log_id'],
defaultTtlSeconds: seconds('45 days'),
});
export const GuildAuditLogsByUserAction = defineTable<
GuildAuditLogRow,
@@ -517,6 +521,7 @@ export const GuildAuditLogsByUserAction = defineTable<
name: 'guild_audit_logs_v2_by_user_action',
columns: GUILD_AUDIT_LOG_COLUMNS,
primaryKey: ['guild_id', 'user_id', 'action_type', 'log_id'],
defaultTtlSeconds: seconds('45 days'),
});
export const GuildMembershipMetadata = defineTable<GuildMembershipMetadataRow, 'guild_id' | 'user_id'>({
name: 'guild_membership_metadata',
@@ -655,6 +660,7 @@ export const RecentMentions = defineTable<RecentMentionRow, 'user_id' | 'message
name: 'recent_mentions',
columns: RECENT_MENTION_COLUMNS,
primaryKey: ['user_id', 'message_id'],
defaultTtlSeconds: seconds('7 days'),
});
interface RecentMentionsByGuildRow {
@@ -678,6 +684,7 @@ export const RecentMentionsByGuild = defineTable<RecentMentionsByGuildRow, 'user
name: 'recent_mentions_by_guild',
columns: RECENT_MENTIONS_BY_GUILD_COLUMNS,
primaryKey: ['user_id', 'guild_id', 'message_id'],
defaultTtlSeconds: seconds('7 days'),
});
export const SavedMessages = defineTable<SavedMessageRow, 'user_id' | 'message_id'>({
name: 'saved_messages',
@@ -688,6 +695,7 @@ export const PushSubscriptions = defineTable<PushSubscriptionRow, 'user_id' | 's
name: 'push_subscriptions',
columns: PUSH_SUBSCRIPTION_COLUMNS,
primaryKey: ['user_id', 'subscription_id'],
defaultTtlSeconds: seconds('90 days'),
});
export const Payments = defineTable<PaymentRow, 'checkout_session_id'>({
name: 'payments',
@@ -854,11 +862,13 @@ export const EmailVerificationTokens = defineTable<EmailVerificationTokenRow, 't
name: 'email_verification_tokens',
columns: EMAIL_VERIFICATION_TOKEN_COLUMNS,
primaryKey: ['token_', 'user_id'],
defaultTtlSeconds: seconds('24 hours'),
});
export const PasswordResetTokens = defineTable<PasswordResetTokenRow, 'token_' | 'user_id'>({
name: 'password_reset_tokens',
columns: PASSWORD_RESET_TOKEN_COLUMNS,
primaryKey: ['token_', 'user_id'],
defaultTtlSeconds: seconds('24 hours'),
});
export const PasswordResetTokensByUserId = defineTable<
{
@@ -870,16 +880,19 @@ export const PasswordResetTokensByUserId = defineTable<
name: 'password_reset_tokens_by_user_id',
columns: ['user_id', 'token_'],
primaryKey: ['user_id', 'token_'],
defaultTtlSeconds: seconds('24 hours'),
});
export const EmailRevertTokens = defineTable<EmailRevertTokenRow, 'token_' | 'user_id'>({
name: 'email_revert_tokens',
columns: EMAIL_REVERT_TOKEN_COLUMNS,
primaryKey: ['token_', 'user_id'],
defaultTtlSeconds: seconds('48 hours'),
});
export const PhoneTokens = defineTable<PhoneTokenRow, 'token_'>({
name: 'phone_tokens',
columns: PHONE_TOKEN_COLUMNS,
primaryKey: ['token_'],
defaultTtlSeconds: seconds('30 days'),
});
export const AuthSessions = defineTable<AuthSessionRow, 'session_id_hash'>({
name: 'auth_sessions',
@@ -901,11 +914,13 @@ export const AuthSessionTombstones = defineTable<AuthSessionTombstoneRow, 'user_
name: 'auth_session_tombstones',
columns: AUTH_SESSION_TOMBSTONE_COLUMNS,
primaryKey: ['user_id', 'session_id_hash'],
defaultTtlSeconds: seconds('30 days'),
});
export const UserCountryHistory = defineTable<UserCountryHistoryRow, 'user_id' | 'country'>({
name: 'user_country_history',
columns: USER_COUNTRY_HISTORY_COLUMNS,
primaryKey: ['user_id', 'country'],
defaultTtlSeconds: seconds('365 days'),
});
export const MfaBackupCodes = defineTable<MfaBackupCodeRow, 'user_id' | 'code'>({
name: 'mfa_backup_codes',
@@ -932,6 +947,7 @@ export const IpAuthorizationTokens = defineTable<IpAuthorizationTokenRow, 'token
name: 'ip_authorization_tokens',
columns: IP_AUTHORIZATION_TOKEN_COLUMNS,
primaryKey: ['token_', 'user_id'],
defaultTtlSeconds: seconds('30 minutes'),
});
export const AuthorizedIps = defineTable<AuthorizedIpRow, 'user_id' | 'ip'>({
name: 'authorized_ips_v2',
@@ -1057,26 +1073,31 @@ export const OAuth2AuthorizationCodes = defineTable<OAuth2AuthorizationCodeRow,
name: 'oauth2_authorization_codes',
columns: OAUTH2_AUTHORIZATION_CODE_COLUMNS,
primaryKey: ['code'],
defaultTtlSeconds: seconds('10 minutes'),
});
export const OAuth2AccessTokens = defineTable<OAuth2AccessTokenRow, 'token_'>({
name: 'oauth2_access_tokens',
columns: OAUTH2_ACCESS_TOKEN_COLUMNS,
primaryKey: ['token_'],
defaultTtlSeconds: seconds('7 days'),
});
export const OAuth2AccessTokensByUser = defineTable<OAuth2AccessTokenByUserRow, 'user_id' | 'token_'>({
name: 'oauth2_access_tokens_by_user',
columns: OAUTH2_ACCESS_TOKENS_BY_USER_COLUMNS,
primaryKey: ['user_id', 'token_'],
defaultTtlSeconds: seconds('7 days'),
});
export const OAuth2RefreshTokens = defineTable<OAuth2RefreshTokenRow, 'token_'>({
name: 'oauth2_refresh_tokens',
columns: OAUTH2_REFRESH_TOKEN_COLUMNS,
primaryKey: ['token_'],
defaultTtlSeconds: seconds('30 days'),
});
export const OAuth2RefreshTokensByUser = defineTable<OAuth2RefreshTokenByUserRow, 'user_id' | 'token_'>({
name: 'oauth2_refresh_tokens_by_user',
columns: OAUTH2_REFRESH_TOKENS_BY_USER_COLUMNS,
primaryKey: ['user_id', 'token_'],
defaultTtlSeconds: seconds('30 days'),
});
interface WebhooksByChannelRow {
@@ -1117,12 +1138,14 @@ export const JobsById = defineTable<JobByIdRow, 'job_id'>({
name: 'jobs_by_id',
columns: JOB_BY_ID_COLUMNS,
primaryKey: ['job_id'],
defaultTtlSeconds: seconds('90 days'),
});
export const JobsByDayBucket = defineTable<JobByDayBucketRow, 'bucket_day' | 'created_at' | 'job_id'>({
name: 'jobs_by_day_bucket',
columns: JOB_BY_DAY_BUCKET_COLUMNS,
primaryKey: ['bucket_day', 'created_at', 'job_id'],
partitionKey: ['bucket_day'],
defaultTtlSeconds: seconds('90 days'),
});
export const JobsActive = defineTable<JobActiveRow, 'job_id'>({
name: 'jobs_active',
@@ -1133,11 +1156,13 @@ export const AttachmentUploadTracesByKey = defineTable<AttachmentUploadTraceByKe
name: 'attachment_upload_traces_by_key',
columns: ATTACHMENT_UPLOAD_TRACE_BY_KEY_COLUMNS,
primaryKey: ['upload_key'],
defaultTtlSeconds: seconds('30 days'),
});
export const AttachmentUploadTracesByAttachment = defineTable<AttachmentUploadTraceByAttachmentRow, 'attachment_id'>({
name: 'attachment_upload_traces_by_attachment',
columns: ATTACHMENT_UPLOAD_TRACE_BY_ATTACHMENT_COLUMNS,
primaryKey: ['attachment_id'],
defaultTtlSeconds: seconds('30 days'),
});
export const NcmecAttachmentSubmissions = defineTable<NcmecAttachmentSubmissionRow, 'attachment_id'>({
name: 'ncmec_attachment_submissions',
@@ -1154,6 +1179,7 @@ export const RegistrationEventsByIp = defineTable<RegistrationEventByIpRow, 'ip'
columns: REGISTRATION_EVENT_BY_IP_COLUMNS,
primaryKey: ['ip', 'created_at', 'user_id'],
partitionKey: ['ip'],
defaultTtlSeconds: seconds('30 days'),
});
export const RegistrationEventsBySubnet = defineTable<
RegistrationEventBySubnetRow,
@@ -1164,6 +1190,7 @@ export const RegistrationEventsBySubnet = defineTable<
columns: REGISTRATION_EVENT_BY_SUBNET_COLUMNS,
primaryKey: ['subnet', 'created_at', 'user_id'],
partitionKey: ['subnet'],
defaultTtlSeconds: seconds('30 days'),
});
export const RegistrationEventsByEmailDomain = defineTable<
RegistrationEventByEmailDomainRow,
@@ -1174,6 +1201,7 @@ export const RegistrationEventsByEmailDomain = defineTable<
columns: REGISTRATION_EVENT_BY_EMAIL_DOMAIN_COLUMNS,
primaryKey: ['email_domain', 'created_at', 'user_id'],
partitionKey: ['email_domain'],
defaultTtlSeconds: seconds('30 days'),
});
export const RegistrationEventsByPlusAddressBase = defineTable<
RegistrationEventByPlusAddressBaseRow,
@@ -1184,6 +1212,7 @@ export const RegistrationEventsByPlusAddressBase = defineTable<
columns: REGISTRATION_EVENT_BY_PLUS_ADDRESS_BASE_COLUMNS,
primaryKey: ['plus_address_base', 'created_at', 'user_id'],
partitionKey: ['plus_address_base'],
defaultTtlSeconds: seconds('30 days'),
});
export const LatestRiskContextByUser = defineTable<LatestRiskContextByUserRow, 'user_id'>({
name: 'latest_risk_context_by_user',
@@ -1194,6 +1223,7 @@ export const SuspiciousIps = defineTable<SuspiciousIpRow, 'ip'>({
name: 'suspicious_ips',
columns: SUSPICIOUS_IP_COLUMNS,
primaryKey: ['ip'],
defaultTtlSeconds: seconds('180 days'),
});
export const RiskOutcomesByIp = defineTable<RiskOutcomeByIpRow, 'ip' | 'created_at' | 'user_id' | 'outcome_code', 'ip'>(
{
@@ -1201,6 +1231,7 @@ export const RiskOutcomesByIp = defineTable<RiskOutcomeByIpRow, 'ip' | 'created_
columns: RISK_OUTCOME_BY_IP_COLUMNS,
primaryKey: ['ip', 'created_at', 'user_id', 'outcome_code'],
partitionKey: ['ip'],
defaultTtlSeconds: seconds('180 days'),
},
);
export const RiskOutcomesBySubnet = defineTable<
@@ -1212,6 +1243,7 @@ export const RiskOutcomesBySubnet = defineTable<
columns: RISK_OUTCOME_BY_SUBNET_COLUMNS,
primaryKey: ['subnet', 'created_at', 'user_id', 'outcome_code'],
partitionKey: ['subnet'],
defaultTtlSeconds: seconds('180 days'),
});
export const RiskOutcomesByEmailDomain = defineTable<
RiskOutcomeByEmailDomainRow,
@@ -1222,6 +1254,7 @@ export const RiskOutcomesByEmailDomain = defineTable<
columns: RISK_OUTCOME_BY_EMAIL_DOMAIN_COLUMNS,
primaryKey: ['email_domain', 'created_at', 'user_id', 'outcome_code'],
partitionKey: ['email_domain'],
defaultTtlSeconds: seconds('180 days'),
});
export const RiskOutcomesByAsn = defineTable<
RiskOutcomeByAsnRow,
@@ -1232,6 +1265,7 @@ export const RiskOutcomesByAsn = defineTable<
columns: RISK_OUTCOME_BY_ASN_COLUMNS,
primaryKey: ['asn', 'created_at', 'user_id', 'outcome_code'],
partitionKey: ['asn'],
defaultTtlSeconds: seconds('180 days'),
});
export const RiskAssessments = defineTable<RiskAssessmentRow, 'assessment_id'>({
name: 'risk_assessments',
@@ -1248,6 +1282,7 @@ export const InboundSmsChallenges = defineTable<InboundSmsChallengeRow, 'challen
name: 'inbound_sms_challenges',
columns: INBOUND_SMS_CHALLENGE_COLUMNS,
primaryKey: ['challenge_code'],
defaultTtlSeconds: seconds('15 minutes'),
});
export const InboundSmsChallengesByUser = defineTable<
InboundSmsChallengeByUserRow,
@@ -1258,16 +1293,19 @@ export const InboundSmsChallengesByUser = defineTable<
columns: INBOUND_SMS_CHALLENGE_BY_USER_COLUMNS,
primaryKey: ['user_id', 'created_at'],
partitionKey: ['user_id'],
defaultTtlSeconds: seconds('15 minutes'),
});
export const PhoneLookupCache = defineTable<PhoneLookupCacheRow, 'phone'>({
name: 'phone_lookup_cache',
columns: PHONE_LOOKUP_CACHE_COLUMNS,
primaryKey: ['phone'],
defaultTtlSeconds: seconds('7 days'),
});
export const PhoneVerificationAttempts = defineTable<PhoneVerificationAttemptRow, 'attempt_id'>({
name: 'phone_verification_attempts',
columns: PHONE_VERIFICATION_ATTEMPT_COLUMNS,
primaryKey: ['attempt_id'],
defaultTtlSeconds: seconds('90 days'),
});
export const BillingCustomers = defineTable<BillingCustomerRow, 'provider_id'>({
name: 'billing_customers',
+19 -1
View File
@@ -2,7 +2,15 @@
import type {AdminAuditLog, BannedIpEntry, BannedIpKind, IAdminRepository} from '@app/api/admin/IAdminRepository';
import {createUserID} from '@app/api/BrandedTypes';
import {deleteOneOrMany, fetchMany, fetchOne, upsertOne} from '@app/api/database/CassandraQueryExecution';
import {Config} from '@app/api/Config';
import {ContentBlocklistCategory} from '@app/api/constants/ContentModeration';
import {
deleteOneOrMany,
executeConditional,
fetchMany,
fetchOne,
upsertOne,
} from '@app/api/database/CassandraQueryExecution';
import type {
AdminAuditLogRow,
BannedAvatarHashRow,
@@ -282,6 +290,7 @@ export class AdminRepository implements IAdminRepository {
}
async isEmailDomainDisposable(domain: string): Promise<boolean> {
if (!Config.blocklistFeeds.enabled) return false;
const domainLower = domain.toLowerCase();
if (isAccountPolicyContactDomainReputationExempt(domainLower)) return false;
const result = await fetchOne<{
@@ -395,6 +404,15 @@ export class AdminRepository implements IAdminRepository {
await deleteOneOrMany(BannedFileShas.deleteByPk({sha256_hex: sha256Hex.toLowerCase()}));
}
async unbanFeedFileSha(sha256Hex: string): Promise<boolean> {
return executeConditional(
BannedFileShas.conditionalDeleteByPk(
{sha256_hex: sha256Hex.toLowerCase()},
{added_by: null, category: ContentBlocklistCategory.MALWARE_BAZAAR},
),
);
}
async loadAllBannedFileShas(): Promise<Array<BannedFileShaRow>> {
return fetchMany<BannedFileShaRow>(LOAD_ALL_BANNED_FILE_SHAS_QUERY.bind({}));
}
@@ -109,6 +109,8 @@ export abstract class IAdminRepository {
abstract unbanFileSha(sha256Hex: string): Promise<void>;
abstract unbanFeedFileSha(sha256Hex: string): Promise<boolean>;
abstract loadAllBannedFileShas(): Promise<Array<BannedFileShaRow>>;
abstract isAvatarHashBanned(hashShort: string): Promise<boolean>;
@@ -31,10 +31,10 @@ import {
RegistrationUrlIdParam,
} from '@fluxer/schema/src/domains/admin/AdminSchemas';
import {GatewayRolloutConfigSchema} from '@fluxer/schema/src/domains/admin/GatewayRolloutSchemas';
import {ScreenShareDeliveryConfigSchema} from '@fluxer/schema/src/domains/admin/ScreenShareDeliverySchemas';
import {VoiceNoiseSuppressionConfigSchema} from '@fluxer/schema/src/domains/admin/VoiceNoiseSuppressionSchemas';
import {UserIdParam} from '@fluxer/schema/src/domains/common/CommonParamSchemas';
import {ExperimentDeliveryConfigSchema} from '@fluxer/schema/src/domains/experiment/ExperimentSchemas';
import {ScreenShareDeliveryConfigSchema} from '@fluxer/schema/src/domains/experiment/ScreenShareDeliverySchemas';
import type {InstanceBranding} from '@fluxer/schema/src/domains/instance/InstanceSchemas';
import {SmtpEmailProvider} from '@pkgs/email/src/SmtpEmailProvider';
import type {Context} from 'hono';
@@ -1,14 +1,13 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import type {UserID} from '@app/api/BrandedTypes';
import {mapGuildMemberToResponse} from '@app/api/guild/GuildModel';
import type {IGuildRepositoryAggregate} from '@app/api/guild/repositories/IGuildRepositoryAggregate';
import type {IGatewayService} from '@app/api/infrastructure/IGatewayService';
import type {UserCacheService} from '@app/api/infrastructure/UserCacheService';
import {createRequestCache} from '@app/api/middleware/RequestCacheMiddleware';
import type {User} from '@app/api/models/User';
import type {IUserRepository} from '@app/api/user/IUserRepository';
import {BaseUserUpdatePropagator} from '@app/api/user/services/BaseUserUpdatePropagator';
import {propagatePartialUserChange} from '@app/api/user/services/PartialUserChangePropagation';
import {hasPartialUserFieldsChanged} from '@app/api/user/UserMappers';
interface AdminUserUpdatePropagatorDeps {
@@ -26,41 +25,11 @@ export class AdminUserUpdatePropagator extends BaseUserUpdatePropagator {
});
}
async propagateUserUpdate({
userId,
oldUser,
updatedUser,
}: {
userId: UserID;
oldUser: User;
updatedUser: User;
}): Promise<void> {
async propagateUserUpdate(params: {userId: UserID; oldUser: User; updatedUser: User}): Promise<void> {
const {oldUser, updatedUser} = params;
await this.dispatchUserUpdate(updatedUser);
if (hasPartialUserFieldsChanged(oldUser, updatedUser)) {
await this.updateUserCache(updatedUser);
await this.propagateToGuilds(userId);
await propagatePartialUserChange(this.deps, updatedUser);
}
}
private async propagateToGuilds(userId: UserID): Promise<void> {
const {userRepository, guildRepository, gatewayService, userCacheService} = this.deps;
const guildIds = await userRepository.getUserGuildIds(userId);
if (guildIds.length === 0) {
return;
}
const requestCache = createRequestCache();
for (const guildId of guildIds) {
const member = await guildRepository.getMember(guildId, userId);
if (!member) {
continue;
}
const memberResponse = await mapGuildMemberToResponse(member, userCacheService, requestCache);
await gatewayService.dispatchGuild({
guildId,
event: 'GUILD_MEMBER_UPDATE',
data: memberResponse,
});
}
requestCache.clear();
}
}
@@ -2,10 +2,10 @@
import {createTestAccount, createTotpSecret, generateTotpCode, setUserACLs} from '@app/api/auth/tests/AuthTestUtils';
import {
createRegistrationResponse,
createWebAuthnDevice,
registerWebAuthnCredential,
setWebAuthnTwoFactor,
type WebAuthnCredentialMetadata,
type WebAuthnRegistrationOptions,
} from '@app/api/auth/tests/WebAuthnTestUtils';
import {type ApiTestHarness, createApiTestHarness} from '@app/api/test/ApiTestHarness';
import {createBuilder} from '@app/api/test/TestRequestBuilder';
@@ -31,13 +31,15 @@ describe('Admin WebAuthn credential delete', () => {
afterAll(async () => {
await harness?.shutdown();
});
test('removes the WebAuthn authenticator type when admin deletes the last credential', async () => {
let admin = await createTestAccount(harness);
admin = await setUserACLs(harness, admin, [
async function createAdmin() {
const admin = await createTestAccount(harness);
return await setUserACLs(harness, admin, [
AdminACLs.AUTHENTICATE,
AdminACLs.USER_LOOKUP,
AdminACLs.USER_UPDATE_MFA,
]);
}
async function createPasskeyTarget(twoFactorEnabled: boolean) {
const target = await createTestAccount(harness);
const device = createWebAuthnDevice();
const secret = createTotpSecret();
@@ -45,28 +47,19 @@ describe('Admin WebAuthn credential delete', () => {
.post('/users/@me/mfa/totp/enable')
.body({secret, code: generateTotpCode(secret), password: target.password})
.execute();
const registrationOptions = await createBuilder<WebAuthnRegistrationOptions>(harness, target.token)
.post('/users/@me/mfa/webauthn/credentials/registration-options')
.body({mfa_method: 'totp', mfa_code: generateTotpCode(secret)})
.execute();
if (registrationOptions.rp.id) {
device.rpId = registrationOptions.rp.id;
}
await createBuilder(harness, target.token)
.post('/users/@me/mfa/webauthn/credentials')
.body({
response: createRegistrationResponse(device, registrationOptions, 'Admin Delete Test Passkey'),
challenge: registrationOptions.challenge,
name: 'Admin Delete Test Passkey',
await registerWebAuthnCredential(
harness,
target.token,
device,
() => ({mfa_method: 'totp', mfa_code: generateTotpCode(secret)}),
'Admin Delete Test Passkey',
);
if (twoFactorEnabled) {
await setWebAuthnTwoFactor(harness, target.token, true, {
mfa_method: 'totp',
mfa_code: generateTotpCode(secret),
})
.expect(204)
.execute();
const credentialsBeforeDelete = await createBuilder<Array<WebAuthnCredentialMetadata>>(harness, target.token)
.get('/users/@me/mfa/webauthn/credentials')
.execute();
expect(credentialsBeforeDelete).toHaveLength(1);
});
}
await createBuilder(harness, target.token)
.post('/users/@me/mfa/totp/disable')
.body({
@@ -76,6 +69,15 @@ describe('Admin WebAuthn credential delete', () => {
})
.expect(204)
.execute();
return target;
}
test('removes the WebAuthn authenticator type when admin deletes the last credential of a two-factor user', async () => {
const admin = await createAdmin();
const target = await createPasskeyTarget(true);
const credentialsBeforeDelete = await createBuilder<Array<WebAuthnCredentialMetadata>>(harness, target.token)
.get('/users/@me/mfa/webauthn/credentials')
.execute();
expect(credentialsBeforeDelete).toHaveLength(1);
const userBeforeDelete = await createBuilder<AdminLookupResponse>(harness, `${admin.token}`)
.get(`/admin/users/${target.userId}`)
.execute();
@@ -93,4 +95,28 @@ describe('Admin WebAuthn credential delete', () => {
.execute();
expect(userAfterDelete.users[0]?.authenticator_types).toEqual([]);
});
test('leaves the authenticator types empty throughout for a user who never turned passkey two-factor on', async () => {
const admin = await createAdmin();
const target = await createPasskeyTarget(false);
const credentialsBeforeDelete = await createBuilder<Array<WebAuthnCredentialMetadata>>(harness, target.token)
.get('/users/@me/mfa/webauthn/credentials')
.execute();
expect(credentialsBeforeDelete).toHaveLength(1);
const userBeforeDelete = await createBuilder<AdminLookupResponse>(harness, `${admin.token}`)
.get(`/admin/users/${target.userId}`)
.execute();
expect(userBeforeDelete.users[0]?.authenticator_types).toEqual([]);
await createBuilder(harness, `${admin.token}`)
.delete(`/admin/users/${target.userId}/webauthn-credentials/${credentialsBeforeDelete[0]!.id}`)
.expect(204)
.execute();
const credentialsAfterDelete = await createBuilder<Array<WebAuthnCredentialMetadata>>(harness, target.token)
.get('/users/@me/mfa/webauthn/credentials')
.execute();
expect(credentialsAfterDelete).toHaveLength(0);
const userAfterDelete = await createBuilder<AdminLookupResponse>(harness, `${admin.token}`)
.get(`/admin/users/${target.userId}`)
.execute();
expect(userAfterDelete.users[0]?.authenticator_types).toEqual([]);
});
});
+23 -10
View File
@@ -5,6 +5,7 @@ import * as AuthMfa from '@app/api/auth/AuthMfa';
import * as AuthPassword from '@app/api/auth/AuthPassword';
import * as AuthSession from '@app/api/auth/AuthSession';
import * as AuthUtility from '@app/api/auth/AuthUtility';
import {resolveWebAuthnSecondFactor} from '@app/api/auth/services/WebAuthnSecondFactor';
import {
createInviteCode,
createIpAuthorizationTicket,
@@ -30,6 +31,7 @@ import {ValidationErrorCodes} from '@fluxer/constants/src/ValidationErrorCodes';
import {IpAuthorizationRequiredError} from '@fluxer/errors/src/domains/auth/IpAuthorizationRequiredError';
import {IpAuthorizationResendCooldownError} from '@fluxer/errors/src/domains/auth/IpAuthorizationResendCooldownError';
import {IpAuthorizationResendLimitExceededError} from '@fluxer/errors/src/domains/auth/IpAuthorizationResendLimitExceededError';
import {MfaNotEnabledError} from '@fluxer/errors/src/domains/auth/MfaNotEnabledError';
import {RegistrationPendingApprovalError} from '@fluxer/errors/src/domains/auth/RegistrationPendingApprovalError';
import {RegistrationRejectedError} from '@fluxer/errors/src/domains/auth/RegistrationRejectedError';
import {InputValidationError} from '@fluxer/errors/src/domains/core/InputValidationError';
@@ -72,12 +74,13 @@ interface LoginTokenResult {
token: string;
}
interface LoginMfaResult {
export interface LoginMfaResult {
mfa: true;
ticket: string;
allowed_methods: Array<string>;
totp: boolean;
webauthn: boolean;
backup_codes: boolean;
}
type LoginResult = LoginTokenResult | LoginMfaResult;
@@ -323,7 +326,8 @@ export async function login(
}
}
if (hasMfa) {
return await createMfaTicketResponse(ctx, currentUser);
const webauthnIsSecondFactor = await resolveWebAuthnSecondFactor(ctx, currentUser);
return await createMfaTicketResponse(ctx, currentUser, webauthnIsSecondFactor);
}
if (data.invite_code && inviteService) {
try {
@@ -387,13 +391,14 @@ export async function loginMfaTotp(
throw new UnknownUserError();
}
AuthUtility.assertNonBotUser(ctx, user);
if (!user.totpSecret || !user.authenticatorTypes?.has(UserAuthenticatorTypes.TOTP)) {
const hasTotp = Boolean(user.totpSecret) && user.authenticatorTypes.has(UserAuthenticatorTypes.TOTP);
if (!hasTotp && !(await AuthMfa.hasUnconsumedBackupCodes(ctx, user.id))) {
throw InputValidationError.fromCode('code', ValidationErrorCodes.TOTP_NOT_ENABLED);
}
await consumeMfaAttempt(ctx, {userId: user.id.toString(), ticket, field: 'code'});
const isValid = await AuthMfa.verifyMfaCode(ctx, {
userId: user.id,
mfaSecret: user.totpSecret,
mfaSecret: hasTotp ? user.totpSecret : null,
code,
allowBackup: true,
});
@@ -424,6 +429,9 @@ export async function loginMfaWebAuthn(
throw new UnknownUserError();
}
AuthUtility.assertNonBotUser(ctx, user);
if (!(await resolveWebAuthnSecondFactor(ctx, user))) {
throw new MfaNotEnabledError();
}
await consumeMfaAttempt(ctx, {userId: user.id.toString(), ticket, field: 'ticket'});
await AuthMfa.verifyWebAuthnAuthentication(ctx, user.id, response, challenge, 'mfa', ticket);
await cache.delete(`mfa-ticket:${ticket}`);
@@ -436,21 +444,26 @@ export async function loginMfaWebAuthn(
return {user_id: user.id.toString(), token};
}
async function createMfaTicketResponse(ctx: ApiContext, user: User): Promise<LoginMfaResult> {
const {users, cache} = ctx.services;
export async function createMfaTicketResponse(
ctx: ApiContext,
user: User,
webauthnIsSecondFactor: boolean,
): Promise<LoginMfaResult> {
const {cache} = ctx.services;
const ticket = createMfaTicket(await AuthUtility.generateSecureToken(ctx));
await cache.set(`mfa-ticket:${ticket}`, user.id.toString(), seconds('5 minutes'));
const credentials = await users.listWebAuthnCredentials(user.id);
const hasWebauthn = credentials.length > 0;
const hasTotp = user.authenticatorTypes.has(UserAuthenticatorTypes.TOTP);
const hasBackupCodes = await AuthMfa.hasUnconsumedBackupCodes(ctx, user.id);
const allowedMethods: Array<string> = [];
if (hasTotp) allowedMethods.push('totp');
if (hasWebauthn) allowedMethods.push('webauthn');
if (webauthnIsSecondFactor) allowedMethods.push('webauthn');
if (hasBackupCodes) allowedMethods.push('backup_codes');
return {
mfa: true,
ticket,
allowed_methods: allowedMethods,
totp: hasTotp,
webauthn: hasWebauthn,
webauthn: webauthnIsSecondFactor,
backup_codes: hasBackupCodes,
};
}
+91 -39
View File
@@ -2,9 +2,11 @@
import {timingSafeEqual} from 'node:crypto';
import type {ApiContext} from '@app/api/ApiContext';
import {deriveSudoMethods, userHasMfa} from '@app/api/auth/services/SudoMethods';
import * as AuthUtility from '@app/api/auth/AuthUtility';
import {deriveSudoMethods, userHasMfa, userHasSudoCapability} from '@app/api/auth/services/SudoMethods';
import {createUserID, type UserID} from '@app/api/BrandedTypes';
import {Logger} from '@app/api/Logger';
import type {MfaBackupCode} from '@app/api/models/MfaBackupCode';
import type {User} from '@app/api/models/User';
import type {WebAuthnCredential} from '@app/api/models/WebAuthnCredential';
import {mapUserToPrivateResponse} from '@app/api/user/UserMappers';
@@ -48,7 +50,7 @@ interface SudoMfaVerificationResult {
interface VerifyMfaCodeParams {
userId: UserID;
mfaSecret: string;
mfaSecret: string | null;
code: string;
allowBackup?: boolean;
}
@@ -56,9 +58,15 @@ interface VerifyMfaCodeParams {
interface AvailableMfaMethods {
totp: boolean;
webauthn: boolean;
backup_codes: boolean;
has_mfa: boolean;
}
interface SetWebAuthnTwoFactorResult {
user: User;
backupCodes: Array<MfaBackupCode> | null;
}
function constantTimeEquals(a: string, b: string): boolean {
const bufferA = Buffer.from(a);
const bufferB = Buffer.from(b);
@@ -72,24 +80,31 @@ function normalizeBackupCode(code: string): string {
return code.toLowerCase().replace(/[^a-z0-9]/g, '');
}
export async function hasUnconsumedBackupCodes(ctx: ApiContext, userId: UserID): Promise<boolean> {
const backupCodes = await ctx.services.users.listMfaBackupCodes(userId);
return backupCodes.some((backupCode) => !backupCode.consumed);
}
export async function verifyMfaCode(ctx: ApiContext, params: VerifyMfaCodeParams): Promise<boolean> {
const {userId, mfaSecret, code, allowBackup = false} = params;
const {users, cache, config} = ctx.services;
try {
const totp = new TotpGenerator(mfaSecret);
const isValidTotp = await totp.validateTotp(code);
if (isValidTotp) {
if (config.dev.testModeEnabled) {
return true;
}
const reuseKey = `mfa-totp:${userId}:${code}`;
const lockToken = await cache.acquireLock(reuseKey, seconds('90 seconds'));
if (lockToken) {
return true;
if (mfaSecret !== null) {
try {
const totp = new TotpGenerator(mfaSecret);
const isValidTotp = await totp.validateTotp(code);
if (isValidTotp) {
if (config.dev.testModeEnabled) {
return true;
}
const reuseKey = `mfa-totp:${userId}:${code}`;
const lockToken = await cache.acquireLock(reuseKey, seconds('90 seconds'));
if (lockToken) {
return true;
}
}
} catch (error) {
Logger.error({userId, code: `${code.slice(0, 3)}***`, error}, 'Failed to validate TOTP code');
}
} catch (error) {
Logger.error({userId, code: `${code.slice(0, 3)}***`, error}, 'Failed to validate TOTP code');
}
if (allowBackup) {
const normalizedCode = normalizeBackupCode(code);
@@ -145,8 +160,7 @@ export async function verifyWebAuthnRegistration(
expectedChallenge: string,
name: string,
): Promise<void> {
const {users, gateway, botMfaMirror, config} = ctx.services;
const user = await users.findUniqueAssert(userId);
const {users, config} = ctx.services;
const existingCredentials = await users.listWebAuthnCredentials(userId);
await consumeWebAuthnChallenge(ctx, expectedChallenge, 'registration', {userId});
if (existingCredentials.length >= 10) {
@@ -214,13 +228,6 @@ export async function verifyWebAuthnRegistration(
name,
);
}
const authenticatorTypes = user.authenticatorTypes || new Set<number>();
if (!authenticatorTypes.has(UserAuthenticatorTypes.WEBAUTHN)) {
authenticatorTypes.add(UserAuthenticatorTypes.WEBAUTHN);
const updatedUser = await users.patchUpsert(userId, {authenticator_types: authenticatorTypes}, user.toRow());
await gateway.dispatchPresence({userId, event: 'USER_UPDATE', data: mapUserToPrivateResponse(updatedUser)});
await botMfaMirror.syncAuthenticatorTypesForOwner(updatedUser);
}
await dispatchWebAuthnCredentialsUpdate(ctx, userId);
}
@@ -234,15 +241,55 @@ export async function deleteWebAuthnCredential(ctx: ApiContext, userId: UserID,
const remainingCredentials = await users.listWebAuthnCredentials(userId);
if (remainingCredentials.length === 0) {
const user = await users.findUniqueAssert(userId);
const authenticatorTypes = user.authenticatorTypes || new Set<number>();
authenticatorTypes.delete(UserAuthenticatorTypes.WEBAUTHN);
const updatedUser = await users.patchUpsert(userId, {authenticator_types: authenticatorTypes}, user.toRow());
await gateway.dispatchPresence({userId, event: 'USER_UPDATE', data: mapUserToPrivateResponse(updatedUser)});
await botMfaMirror.syncAuthenticatorTypesForOwner(updatedUser);
if (user.authenticatorTypes.has(UserAuthenticatorTypes.WEBAUTHN)) {
const authenticatorTypes = new Set<number>(user.authenticatorTypes ?? []);
authenticatorTypes.delete(UserAuthenticatorTypes.WEBAUTHN);
const updatedUser = await users.patchUpsert(userId, {authenticator_types: authenticatorTypes}, user.toRow());
if (!userHasMfa(updatedUser)) {
await users.clearMfaBackupCodes(userId);
}
await gateway.dispatchPresence({userId, event: 'USER_UPDATE', data: mapUserToPrivateResponse(updatedUser)});
await botMfaMirror.syncAuthenticatorTypesForOwner(updatedUser);
}
}
await dispatchWebAuthnCredentialsUpdate(ctx, userId);
}
export async function setWebAuthnTwoFactor(
ctx: ApiContext,
userId: UserID,
enabled: boolean,
): Promise<SetWebAuthnTwoFactorResult> {
const {users, gateway, botMfaMirror} = ctx.services;
const user = await users.findUniqueAssert(userId);
const credentials = await users.listWebAuthnCredentials(userId);
if (enabled && credentials.length === 0) {
throw new NoPasskeysRegisteredError();
}
const authenticatorTypes = new Set<number>(user.authenticatorTypes ?? []);
if (authenticatorTypes.has(UserAuthenticatorTypes.WEBAUTHN) === enabled) {
return {user, backupCodes: null};
}
if (enabled) {
authenticatorTypes.add(UserAuthenticatorTypes.WEBAUTHN);
} else {
authenticatorTypes.delete(UserAuthenticatorTypes.WEBAUTHN);
}
const updatedUser = await users.patchUpsert(userId, {authenticator_types: authenticatorTypes}, user.toRow());
let backupCodes: Array<MfaBackupCode> | null = null;
if (enabled) {
const existingBackupCodes = await users.listMfaBackupCodes(userId);
if (existingBackupCodes.every((backupCode) => backupCode.consumed)) {
backupCodes = await users.createMfaBackupCodes(userId, AuthUtility.generateBackupCodes(ctx));
}
} else if (!userHasMfa(updatedUser)) {
await users.clearMfaBackupCodes(userId);
}
await gateway.dispatchPresence({userId, event: 'USER_UPDATE', data: mapUserToPrivateResponse(updatedUser)});
await botMfaMirror.syncAuthenticatorTypesForOwner(updatedUser);
return {user: updatedUser, backupCodes};
}
export async function renameWebAuthnCredential(
ctx: ApiContext,
userId: UserID,
@@ -430,16 +477,17 @@ export async function verifySudoMfa(
const {users} = ctx.services;
const {userId, method, code, webauthnResponse, webauthnChallenge} = params;
const user = await users.findUnique(userId);
const hasMfa =
(user?.authenticatorTypes?.has(UserAuthenticatorTypes.TOTP) ?? false) ||
(user?.authenticatorTypes?.has(UserAuthenticatorTypes.WEBAUTHN) ?? false);
if (!user || !hasMfa) {
if (!user) {
return {success: false, error: 'MFA not enabled'};
}
const credentials = await users.listWebAuthnCredentials(userId);
const hasPasskeyCredentials = credentials.length > 0;
if (!userHasSudoCapability(user, hasPasskeyCredentials)) {
return {success: false, error: 'MFA not enabled'};
}
switch (method) {
case 'totp': {
if (!code) return {success: false, error: 'TOTP code is required'};
if (!user.totpSecret) return {success: false, error: 'TOTP is not enabled'};
await consumeSudoMfaAttempt(ctx, userId);
const isValid = await verifyMfaCode(ctx, {userId, mfaSecret: user.totpSecret, code, allowBackup: true});
if (isValid) {
@@ -451,7 +499,7 @@ export async function verifySudoMfa(
if (!webauthnResponse || !webauthnChallenge) {
return {success: false, error: 'WebAuthn response and challenge are required'};
}
if (!user.authenticatorTypes?.has(UserAuthenticatorTypes.WEBAUTHN)) {
if (!hasPasskeyCredentials) {
return {success: false, error: 'WebAuthn is not enabled'};
}
try {
@@ -467,15 +515,19 @@ export async function verifySudoMfa(
}
export async function getAvailableMfaMethods(ctx: ApiContext, userId: UserID): Promise<AvailableMfaMethods> {
const user = await ctx.services.users.findUnique(userId);
const {users} = ctx.services;
const user = await users.findUnique(userId);
if (!user) {
return {totp: false, webauthn: false, has_mfa: false};
return {totp: false, webauthn: false, backup_codes: false, has_mfa: false};
}
const methods = deriveSudoMethods(user);
const credentials = await users.listWebAuthnCredentials(userId);
const hasPasskeyCredentials = credentials.length > 0;
const methods = deriveSudoMethods(user, hasPasskeyCredentials, await hasUnconsumedBackupCodes(ctx, userId));
return {
totp: methods.totp,
webauthn: methods.webauthn,
has_mfa: userHasMfa(user),
backup_codes: methods.backup_codes,
has_mfa: userHasSudoCapability(user, hasPasskeyCredentials),
};
}
+22 -50
View File
@@ -2,12 +2,14 @@
import crypto from 'node:crypto';
import type {ApiContext} from '@app/api/ApiContext';
import {createMfaTicketResponse, type LoginMfaResult} from '@app/api/auth/AuthLogin';
import * as AuthSession from '@app/api/auth/AuthSession';
import * as AuthUtility from '@app/api/auth/AuthUtility';
import {createMfaTicket, createPasswordResetToken} from '@app/api/BrandedTypes';
import {resolveWebAuthnSecondFactor} from '@app/api/auth/services/WebAuthnSecondFactor';
import {createPasswordResetToken} from '@app/api/BrandedTypes';
import {Config} from '@app/api/Config';
import type {UserRow} from '@app/api/database/types/UserTypes';
import {Logger} from '@app/api/Logger';
import type {User} from '@app/api/models/User';
import {EXTERNAL_RESPONSE_LIMITS} from '@app/api/utils/ExternalResponseLimits';
import * as FetchUtils from '@app/api/utils/FetchUtils';
import {hashPassword as hashPasswordUtil, verifyPassword as verifyPasswordUtil} from '@app/api/utils/PasswordUtils';
@@ -19,7 +21,7 @@ import {InputValidationError} from '@fluxer/errors/src/domains/core/InputValidat
import {requireClientIp} from '@fluxer/ip_utils/src/ClientIp';
import {getSameIpDecisionKey} from '@fluxer/ip_utils/src/IpAddress';
import type {ForgotPasswordRequest, ResetPasswordRequest} from '@fluxer/schema/src/domains/auth/AuthSchemas';
import {ms, seconds} from 'itty-time';
import {ms} from 'itty-time';
const PWNED_PASSWORDS_TIMEOUT_MS = ms('5 seconds');
const PWNED_PASSWORD_CACHE_MAX_PREFIXES = 128;
@@ -91,13 +93,7 @@ type ResetPasswordResult =
user_id: string;
token: string;
}
| {
mfa: true;
ticket: string;
allowed_methods: Array<string>;
totp: boolean;
webauthn: boolean;
};
| LoginMfaResult;
const pwnedPasswordCache = new PwnedPasswordCache(PWNED_PASSWORD_CACHE_MAX_PREFIXES, ms('1 hour'));
@@ -267,22 +263,26 @@ export async function resetPassword(
if (await isPasswordPwned(ctx, data.password)) {
throw InputValidationError.fromCode('password', ValidationErrorCodes.PASSWORD_IS_TOO_COMMON);
}
const webauthnIsSecondFactor = await resolveWebAuthnSecondFactor(ctx, user);
const hasMfa = user.authenticatorTypes.has(UserAuthenticatorTypes.TOTP) || webauthnIsSecondFactor;
const newPasswordHash = await hashPassword(ctx, data.password);
const updatedUser = await users.patchUpsert(
user.id,
{
password_hash: newPasswordHash,
password_last_changed_at: new Date(),
},
user.toRow(),
);
const updates: Partial<UserRow> = {
password_hash: newPasswordHash,
password_last_changed_at: new Date(),
};
if (webauthnIsSecondFactor && !user.authenticatorTypes.has(UserAuthenticatorTypes.WEBAUTHN)) {
const authenticatorTypes = new Set<number>(user.authenticatorTypes);
authenticatorTypes.add(UserAuthenticatorTypes.WEBAUTHN);
updates.authenticator_types = authenticatorTypes;
}
const updatedUser = await users.patchUpsert(user.id, updates, user.toRow());
if (updates.authenticator_types) {
await ctx.services.botMfaMirror.syncAuthenticatorTypesForOwner(updatedUser);
}
await AuthSession.terminateAllUserSessions(ctx, user.id);
await users.deletePasswordResetToken(data.token);
const hasMfa =
updatedUser.authenticatorTypes.has(UserAuthenticatorTypes.TOTP) ||
updatedUser.authenticatorTypes.has(UserAuthenticatorTypes.WEBAUTHN);
if (hasMfa) {
return await createMfaTicketResponse(ctx, updatedUser);
return await createMfaTicketResponse(ctx, updatedUser, webauthnIsSecondFactor);
}
const [token] = await AuthSession.createAuthSession(ctx, {
user: updatedUser,
@@ -290,31 +290,3 @@ export async function resetPassword(
});
return {user_id: updatedUser.id.toString(), token};
}
async function createMfaTicketResponse(
ctx: ApiContext,
user: User,
): Promise<{
mfa: true;
ticket: string;
allowed_methods: Array<string>;
totp: boolean;
webauthn: boolean;
}> {
const {users, cache} = ctx.services;
const ticket = createMfaTicket(await AuthUtility.generateSecureToken(ctx));
await cache.set(`mfa-ticket:${ticket}`, user.id.toString(), seconds('5 minutes'));
const credentials = await users.listWebAuthnCredentials(user.id);
const hasWebauthn = credentials.length > 0;
const hasTotp = user.authenticatorTypes.has(UserAuthenticatorTypes.TOTP);
const allowedMethods: Array<string> = [];
if (hasTotp) allowedMethods.push('totp');
if (hasWebauthn) allowedMethods.push('webauthn');
return {
mfa: true,
ticket: ticket,
allowed_methods: allowedMethods,
totp: hasTotp,
webauthn: hasWebauthn,
};
}
@@ -417,6 +417,7 @@ export class AuthRequestService {
...result,
totp: allowedMethods.has('totp'),
webauthn: allowedMethods.has('webauthn'),
backup_codes: allowedMethods.has('backup_codes'),
};
}
}
@@ -3,6 +3,15 @@
import {UserAuthenticatorTypes} from '@fluxer/constants/src/UserConstants';
import type {SudoModeMethods} from '@fluxer/errors/src/domains/auth/SudoModeRequiredError';
interface SudoMethodsUser {
totpSecret?: string | null;
authenticatorTypes?: Set<number> | null;
}
function hasTotpEnrolled(user: SudoMethodsUser): boolean {
return (user.totpSecret ?? null) !== null && (user.authenticatorTypes?.has(UserAuthenticatorTypes.TOTP) ?? false);
}
export function userHasMfa(user: {authenticatorTypes?: Set<number> | null}): boolean {
return (
(user.authenticatorTypes?.has(UserAuthenticatorTypes.TOTP) ?? false) ||
@@ -10,13 +19,18 @@ export function userHasMfa(user: {authenticatorTypes?: Set<number> | null}): boo
);
}
export function deriveSudoMethods(user: {
totpSecret?: string | null;
authenticatorTypes?: Set<number> | null;
}): SudoModeMethods {
const authenticatorTypes = user.authenticatorTypes ?? null;
export function userHasSudoCapability(user: SudoMethodsUser, hasPasskeyCredentials: boolean): boolean {
return hasTotpEnrolled(user) || hasPasskeyCredentials;
}
export function deriveSudoMethods(
user: SudoMethodsUser,
hasPasskeyCredentials: boolean,
hasBackupCodes: boolean,
): SudoModeMethods {
return {
totp: (user.totpSecret ?? null) !== null && (authenticatorTypes?.has(UserAuthenticatorTypes.TOTP) ?? false),
webauthn: authenticatorTypes?.has(UserAuthenticatorTypes.WEBAUTHN) ?? false,
totp: hasTotpEnrolled(user),
webauthn: hasPasskeyCredentials,
backup_codes: hasBackupCodes,
};
}
@@ -2,7 +2,7 @@
import * as AuthMfa from '@app/api/auth/AuthMfa';
import * as AuthPassword from '@app/api/auth/AuthPassword';
import {deriveSudoMethods, userHasMfa} from '@app/api/auth/services/SudoMethods';
import {deriveSudoMethods, userHasMfa, userHasSudoCapability} from '@app/api/auth/services/SudoMethods';
import {getSudoModeService} from '@app/api/auth/services/SudoModeService';
import {SUDO_MODE_HEADER} from '@app/api/middleware/SudoModeMiddleware';
import type {User} from '@app/api/models/User';
@@ -26,8 +26,9 @@ type SudoVerificationMethod = 'password' | 'mfa' | 'sudo_token';
export function hasNoVerifiableCredential(
user: {passwordHash: string | null; isBot: boolean},
hasMfa: boolean,
hasPasskeyCredentials: boolean,
): boolean {
if (user.isBot || hasMfa) {
if (user.isBot || hasMfa || hasPasskeyCredentials) {
return false;
}
return user.passwordHash === null;
@@ -52,18 +53,22 @@ async function verifySudoMode(
if (user.isBot) {
return {verified: true, method: 'sudo_token'};
}
const apiContext = ctx.get('apiContext');
const credentials = await apiContext.services.users.listWebAuthnCredentials(user.id);
const hasPasskeyCredentials = credentials.length > 0;
const hasMfa = userHasMfa(user);
const issueSudoToken = options.issueSudoToken ?? hasMfa;
if (hasMfa && ctx.get('sudoModeValid')) {
const hasSudoCapability = userHasSudoCapability(user, hasPasskeyCredentials);
const issueSudoToken = options.issueSudoToken ?? hasSudoCapability;
if (hasSudoCapability && ctx.get('sudoModeValid')) {
const sudoToken = ctx.get('sudoModeToken') ?? ctx.req.header(SUDO_MODE_HEADER) ?? undefined;
return {verified: true, method: 'sudo_token', sudoToken: issueSudoToken ? sudoToken : undefined};
}
const incomingToken = ctx.req.header(SUDO_MODE_HEADER);
if (!hasMfa && incomingToken && ctx.get('sudoModeValid')) {
if (!hasSudoCapability && incomingToken && ctx.get('sudoModeValid')) {
return {verified: true, method: 'sudo_token', sudoToken: issueSudoToken ? incomingToken : undefined};
}
if (hasMfa && body.mfa_method) {
const result = await AuthMfa.verifySudoMfa(ctx.get('apiContext'), {
if (hasSudoCapability && body.mfa_method) {
const result = await AuthMfa.verifySudoMfa(apiContext, {
userId: user.id,
method: body.mfa_method,
code: body.mfa_code,
@@ -77,14 +82,14 @@ async function verifySudoMode(
const sudoToken = issueSudoToken ? await sudoModeService.generateSudoToken(user.id) : undefined;
return {verified: true, sudoToken, method: 'mfa'};
}
if (hasNoVerifiableCredential(user, hasMfa)) {
if (hasNoVerifiableCredential(user, hasMfa, hasPasskeyCredentials)) {
return {verified: true, method: 'password'};
}
if (body.password && !hasMfa) {
if (!user.passwordHash) {
throw InputValidationError.fromCode('password', ValidationErrorCodes.PASSWORD_NOT_SET);
}
const passwordValid = await AuthPassword.verifyPassword(ctx.get('apiContext'), {
const passwordValid = await AuthPassword.verifyPassword(apiContext, {
password: body.password,
passwordHash: user.passwordHash,
});
@@ -93,7 +98,8 @@ async function verifySudoMode(
}
return {verified: true, method: 'password'};
}
throw new SudoModeRequiredError(hasMfa, deriveSudoMethods(user));
const hasBackupCodes = await AuthMfa.hasUnconsumedBackupCodes(apiContext, user.id);
throw new SudoModeRequiredError(hasSudoCapability, deriveSudoMethods(user, hasPasskeyCredentials, hasBackupCodes));
}
function setSudoTokenHeader(
@@ -0,0 +1,22 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import type {ApiContext} from '@app/api/ApiContext';
import type {User} from '@app/api/models/User';
import {UserAuthenticatorTypes} from '@fluxer/constants/src/UserConstants';
interface WebAuthnSecondFactorUser {
passwordHash: string | null;
authenticatorTypes?: Set<number> | null;
}
export function webAuthnIsSecondFactor(user: WebAuthnSecondFactorUser, hasPasskeyCredentials: boolean): boolean {
return (
(user.authenticatorTypes?.has(UserAuthenticatorTypes.WEBAUTHN) ?? false) ||
(user.passwordHash === null && hasPasskeyCredentials)
);
}
export async function resolveWebAuthnSecondFactor(ctx: ApiContext, user: User): Promise<boolean> {
const credentials = await ctx.services.users.listWebAuthnCredentials(user.id);
return webAuthnIsSecondFactor(user, credentials.length > 0);
}
+2 -12
View File
@@ -22,20 +22,10 @@ export interface LoginMfaResponse {
allowed_methods: Array<string>;
totp: boolean;
webauthn: boolean;
backup_codes: boolean;
}
type LoginResponse =
| {
user_id: string;
token: string;
}
| {
mfa: true;
ticket: string;
allowed_methods: Array<string>;
totp: boolean;
webauthn: boolean;
};
type LoginResponse = LoginSuccessResponse | LoginMfaResponse;
export interface UserMeResponse {
id: string;
@@ -627,6 +627,7 @@ describe('Email change flow', () => {
methods?: {
totp?: boolean;
webauthn?: boolean;
backup_codes?: boolean;
};
}>(harness, mfaAccount.token)
.patch('/users/@me')
@@ -634,13 +635,14 @@ describe('Email change flow', () => {
.expect(403, 'SUDO_MODE_REQUIRED')
.execute();
expect(noSudoResp.has_mfa).toBe(true);
expect(noSudoResp.methods).toEqual({totp: true, webauthn: false});
expect(noSudoResp.methods).toEqual({totp: true, webauthn: false, backup_codes: true});
const passwordOnlyResp = await createBuilder<{
code: string;
has_mfa?: boolean;
methods?: {
totp?: boolean;
webauthn?: boolean;
backup_codes?: boolean;
};
}>(harness, mfaAccount.token)
.patch('/users/@me')
@@ -648,7 +650,7 @@ describe('Email change flow', () => {
.expect(403, 'SUDO_MODE_REQUIRED')
.execute();
expect(passwordOnlyResp.has_mfa).toBe(true);
expect(passwordOnlyResp.methods).toEqual({totp: true, webauthn: false});
expect(passwordOnlyResp.methods).toEqual({totp: true, webauthn: false, backup_codes: true});
const updated = await createBuilder<UserPrivateResponse>(harness, mfaAccount.token)
.patch('/users/@me')
.body({
@@ -712,6 +714,7 @@ describe('Email change flow', () => {
methods?: {
totp?: boolean;
webauthn?: boolean;
backup_codes?: boolean;
};
}>(harness, mfaAccount.token)
.post('/users/@me/email-change/apply')
@@ -719,13 +722,14 @@ describe('Email change flow', () => {
.expect(403, 'SUDO_MODE_REQUIRED')
.execute();
expect(noSudoResp.has_mfa).toBe(true);
expect(noSudoResp.methods).toEqual({totp: true, webauthn: false});
expect(noSudoResp.methods).toEqual({totp: true, webauthn: false, backup_codes: true});
const passwordOnlyResp = await createBuilder<{
code: string;
has_mfa?: boolean;
methods?: {
totp?: boolean;
webauthn?: boolean;
backup_codes?: boolean;
};
}>(harness, mfaAccount.token)
.post('/users/@me/email-change/apply')
@@ -733,7 +737,7 @@ describe('Email change flow', () => {
.expect(403, 'SUDO_MODE_REQUIRED')
.execute();
expect(passwordOnlyResp.has_mfa).toBe(true);
expect(passwordOnlyResp.methods).toEqual({totp: true, webauthn: false});
expect(passwordOnlyResp.methods).toEqual({totp: true, webauthn: false, backup_codes: true});
const updated = await createBuilder<UserPrivateResponse>(harness, mfaAccount.token)
.post('/users/@me/email-change/apply')
.body({
@@ -776,6 +780,7 @@ describe('Email change flow', () => {
methods?: {
totp?: boolean;
webauthn?: boolean;
backup_codes?: boolean;
};
}>(harness, account.token)
.post('/users/@me/email-change/apply')
@@ -783,7 +788,7 @@ describe('Email change flow', () => {
.expect(403, 'SUDO_MODE_REQUIRED')
.execute();
expect(noSudoResp.has_mfa).toBe(false);
expect(noSudoResp.methods).toEqual({totp: false, webauthn: false});
expect(noSudoResp.methods).toEqual({totp: false, webauthn: false, backup_codes: false});
const updated = await createBuilder<UserPrivateResponse>(harness, account.token)
.post('/users/@me/email-change/apply')
.body({email_token: emailToken, password: account.password})
@@ -887,6 +892,7 @@ describe('Email change flow', () => {
methods?: {
totp?: boolean;
webauthn?: boolean;
backup_codes?: boolean;
};
}>(harness, mfaAccount.token)
.post('/users/@me/email-change/apply')
@@ -894,7 +900,7 @@ describe('Email change flow', () => {
.expect(403, 'SUDO_MODE_REQUIRED')
.execute();
expect(discovery.has_mfa).toBe(true);
expect(discovery.methods).toEqual({totp: true, webauthn: false});
expect(discovery.methods).toEqual({totp: true, webauthn: false, backup_codes: true});
const applied = await createBuilder<UserPrivateResponse>(harness, mfaAccount.token)
.post('/users/@me/email-change/apply')
.body({
@@ -10,6 +10,7 @@ import {
createAuthenticationResponse,
createRegistrationResponse,
createWebAuthnDevice,
setWebAuthnTwoFactor,
type WebAuthnAuthenticationOptions,
type WebAuthnDevice,
type WebAuthnRegistrationOptions,
@@ -37,6 +38,7 @@ interface LoginMfaResponse {
interface SudoMfaMethodsResponse {
totp: boolean;
webauthn: boolean;
backup_codes: boolean;
has_mfa: boolean;
}
@@ -46,6 +48,7 @@ interface SudoModeRequiredResponse {
methods?: {
totp?: boolean;
webauthn?: boolean;
backup_codes?: boolean;
};
}
@@ -73,6 +76,7 @@ async function loginWithTotp(harness: ApiTestHarness, account: TestAccount, secr
async function setupWebAuthnOnlyUser(
harness: ApiTestHarness,
account: TestAccount,
twoFactorEnabled: boolean,
): Promise<{
account: TestAccount;
device: WebAuthnDevice;
@@ -124,6 +128,12 @@ async function setupWebAuthnOnlyUser(
})
.expect(204)
.execute();
if (twoFactorEnabled) {
await setWebAuthnTwoFactor(harness, updatedAccount.token, true, {
mfa_method: 'totp',
mfa_code: backupCodes.backup_codes[5]!.code,
});
}
await createBuilder(harness, updatedAccount.token)
.post('/users/@me/mfa/totp/disable')
.body({
@@ -162,9 +172,9 @@ describe('MFA Consistency Tests', () => {
await harness?.shutdown();
});
describe('WebAuthn sudo verification flow', () => {
test('WebAuthn user can complete sudo verification with passkey', async () => {
test('WebAuthn user with two-factor on can complete sudo verification with passkey', async () => {
const account = await createTestAccount(harness);
const {account: webauthnAccount, device} = await setupWebAuthnOnlyUser(harness, account);
const {account: webauthnAccount, device} = await setupWebAuthnOnlyUser(harness, account, true);
const sudoOptions = await createBuilder<WebAuthnAuthenticationOptions>(harness, webauthnAccount.token)
.post('/users/@me/sudo/webauthn/authentication-options')
.body(null)
@@ -180,9 +190,27 @@ describe('MFA Consistency Tests', () => {
.expect(204)
.execute();
});
test('WebAuthn-only user cannot use password for sudo verification', async () => {
test('WebAuthn user with two-factor off can complete sudo verification with passkey', async () => {
const account = await createTestAccount(harness);
const {account: webauthnAccount} = await setupWebAuthnOnlyUser(harness, account);
const {account: webauthnAccount, device} = await setupWebAuthnOnlyUser(harness, account, false);
const sudoOptions = await createBuilder<WebAuthnAuthenticationOptions>(harness, webauthnAccount.token)
.post('/users/@me/sudo/webauthn/authentication-options')
.body(null)
.execute();
const sudoAssertion = createAuthenticationResponse(device, sudoOptions);
await createBuilder(harness, webauthnAccount.token)
.post('/users/@me/disable')
.body({
mfa_method: 'webauthn',
webauthn_response: sudoAssertion,
webauthn_challenge: sudoOptions.challenge,
})
.expect(204)
.execute();
});
test('WebAuthn-only user with two-factor on cannot use password for sudo verification', async () => {
const account = await createTestAccount(harness);
const {account: webauthnAccount} = await setupWebAuthnOnlyUser(harness, account, true);
const errorResp = await createBuilder<{
code: string;
}>(harness, webauthnAccount.token)
@@ -194,6 +222,17 @@ describe('MFA Consistency Tests', () => {
.execute();
expect(errorResp.code).toBe('SUDO_MODE_REQUIRED');
});
test('WebAuthn-only user with two-factor off can use password for sudo verification', async () => {
const account = await createTestAccount(harness);
const {account: webauthnAccount} = await setupWebAuthnOnlyUser(harness, account, false);
await createBuilder(harness, webauthnAccount.token)
.post('/users/@me/disable')
.body({
password: account.password,
})
.expect(204)
.execute();
});
});
describe('Password-only sudo flow for non-MFA users', () => {
test('Non-MFA user can use password for sudo verification', async () => {
@@ -323,14 +362,37 @@ describe('MFA Consistency Tests', () => {
const methods = await createBuilder<SudoMfaMethodsResponse>(harness, account.token)
.get('/users/@me/sudo/mfa-methods')
.execute();
expect(methods).toEqual({totp: false, webauthn: false, has_mfa: false});
expect(methods).toEqual({totp: false, webauthn: false, backup_codes: false, has_mfa: false});
const errorResp = await createBuilder<SudoModeRequiredResponse>(harness, account.token)
.post('/users/@me/disable')
.body({})
.expect(403, 'SUDO_MODE_REQUIRED')
.execute();
expect(errorResp.has_mfa).toBe(methods.has_mfa);
expect(errorResp.methods).toEqual({totp: methods.totp, webauthn: methods.webauthn});
expect(errorResp.methods).toEqual({
totp: methods.totp,
webauthn: methods.webauthn,
backup_codes: methods.backup_codes,
});
});
test('mfa-methods reports webauthn for a passkey user with two-factor off', async () => {
const account = await createTestAccount(harness);
const {account: webauthnAccount} = await setupWebAuthnOnlyUser(harness, account, false);
const methods = await createBuilder<SudoMfaMethodsResponse>(harness, webauthnAccount.token)
.get('/users/@me/sudo/mfa-methods')
.execute();
expect(methods).toEqual({totp: false, webauthn: true, backup_codes: false, has_mfa: true});
const errorResp = await createBuilder<SudoModeRequiredResponse>(harness, webauthnAccount.token)
.post('/users/@me/disable')
.body({})
.expect(403, 'SUDO_MODE_REQUIRED')
.execute();
expect(errorResp.has_mfa).toBe(methods.has_mfa);
expect(errorResp.methods).toEqual({
totp: methods.totp,
webauthn: methods.webauthn,
backup_codes: methods.backup_codes,
});
});
test('mfa-methods agrees with the SUDO_MODE_REQUIRED body for an enrolled TOTP user', async () => {
const account = await createTestAccount(harness);
@@ -347,14 +409,18 @@ describe('MFA Consistency Tests', () => {
const methods = await createBuilder<SudoMfaMethodsResponse>(harness, loggedIn.token)
.get('/users/@me/sudo/mfa-methods')
.execute();
expect(methods).toEqual({totp: true, webauthn: false, has_mfa: true});
expect(methods).toEqual({totp: true, webauthn: false, backup_codes: true, has_mfa: true});
const errorResp = await createBuilder<SudoModeRequiredResponse>(harness, loggedIn.token)
.post('/users/@me/disable')
.body({})
.expect(403, 'SUDO_MODE_REQUIRED')
.execute();
expect(errorResp.has_mfa).toBe(methods.has_mfa);
expect(errorResp.methods).toEqual({totp: methods.totp, webauthn: methods.webauthn});
expect(errorResp.methods).toEqual({
totp: methods.totp,
webauthn: methods.webauthn,
backup_codes: methods.backup_codes,
});
});
});
describe('MFA requirement propagates to sensitive operations', () => {
@@ -8,9 +8,9 @@ import {
seedMfaTicket,
} from '@app/api/auth/tests/AuthTestUtils';
import {
createRegistrationResponse,
createWebAuthnDevice,
type WebAuthnRegistrationOptions,
registerWebAuthnCredential,
setWebAuthnTwoFactor,
} from '@app/api/auth/tests/WebAuthnTestUtils';
import type {ApiTestHarness} from '@app/api/test/ApiTestHarness';
import {HTTP_STATUS} from '@app/api/test/TestConstants';
@@ -41,7 +41,7 @@ describe('Auth MFA TOTP without secret', () => {
.execute();
expect(login.code).toBe('INVALID_FORM_BODY');
});
it('rejects TOTP login when only WebAuthn is enabled', async () => {
it('rejects TOTP login when passkey two-factor is on and no TOTP secret remains', async () => {
const account = await createTestAccount(harness);
const device = createWebAuthnDevice();
const secret = createTotpSecret();
@@ -53,25 +53,14 @@ describe('Auth MFA TOTP without secret', () => {
.post('/users/@me/mfa/totp/enable')
.body({secret, code: generateTotpCode(secret), password: account.password})
.execute();
const regOptions = await createBuilder<WebAuthnRegistrationOptions>(harness, account.token)
.post('/users/@me/mfa/webauthn/credentials/registration-options')
.body({mfa_method: 'totp', mfa_code: generateTotpCode(secret)})
.execute();
if (regOptions.rp.id) {
device.rpId = regOptions.rp.id;
}
const registrationResponse = createRegistrationResponse(device, regOptions, 'Test Passkey');
await createBuilder(harness, account.token)
.post('/users/@me/mfa/webauthn/credentials')
.body({
response: registrationResponse,
challenge: regOptions.challenge,
name: 'Test Passkey',
mfa_method: 'totp',
mfa_code: generateTotpCode(secret),
})
.expect(204)
.execute();
await registerWebAuthnCredential(harness, account.token, device, () => ({
mfa_method: 'totp',
mfa_code: generateTotpCode(secret),
}));
await setWebAuthnTwoFactor(harness, account.token, true, {
mfa_method: 'totp',
mfa_code: generateTotpCode(secret),
});
await createBuilder(harness, account.token)
.post('/users/@me/mfa/totp/disable')
.body({
@@ -105,4 +94,39 @@ describe('Auth MFA TOTP without secret', () => {
.execute();
expect(bypassAttempt.code).toBe('INVALID_FORM_BODY');
});
it('issues a session token when passkey two-factor is off and no TOTP secret remains', async () => {
const account = await createTestAccount(harness);
const device = createWebAuthnDevice();
const secret = createTotpSecret();
const totpData = await createBuilder<{
backup_codes: Array<{
code: string;
}>;
}>(harness, account.token)
.post('/users/@me/mfa/totp/enable')
.body({secret, code: generateTotpCode(secret), password: account.password})
.execute();
await registerWebAuthnCredential(harness, account.token, device, () => ({
mfa_method: 'totp',
mfa_code: generateTotpCode(secret),
}));
await createBuilder(harness, account.token)
.post('/users/@me/mfa/totp/disable')
.body({
code: totpData.backup_codes[0]!.code,
mfa_method: 'totp',
mfa_code: generateTotpCode(secret),
})
.expect(204)
.execute();
const login = await createBuilderWithoutAuth<{
mfa?: true;
token: string;
}>(harness)
.post('/auth/login')
.body({email: account.email, password: account.password})
.execute();
expect(login.mfa).toBeUndefined();
expect(login.token).toBeTruthy();
});
});
@@ -4,13 +4,25 @@ import {
clearTestEmails,
createAuthHarness,
createTestAccount,
createUniqueEmail,
findLastTestEmail,
type LoginSuccessResponse,
listTestEmails,
type TestAccount,
type TestEmailRecord,
totpCodeNow,
unclaimAccount,
} from '@app/api/auth/tests/AuthTestUtils';
import {
createAuthenticationResponse,
createWebAuthnDevice,
registerWebAuthnCredential,
setWebAuthnTwoFactor,
type WebAuthnAuthenticationOptions,
} from '@app/api/auth/tests/WebAuthnTestUtils';
import type {ApiTestHarness} from '@app/api/test/ApiTestHarness';
import {createBuilder, createBuilderWithoutAuth} from '@app/api/test/TestRequestBuilder';
import {UserAuthenticatorTypes} from '@fluxer/constants/src/UserConstants';
import {afterAll, beforeAll, beforeEach, describe, expect, it} from 'vitest';
interface MfaRequiredResponse {
@@ -19,6 +31,7 @@ interface MfaRequiredResponse {
allowed_methods: Array<string>;
totp: boolean;
webauthn: boolean;
backup_codes: boolean;
}
async function waitForEmail(harness: ApiTestHarness, type: string, recipient: string): Promise<TestEmailRecord> {
@@ -34,6 +47,34 @@ async function waitForEmail(harness: ApiTestHarness, type: string, recipient: st
throw new Error(`Email not found: type=${type}, recipient=${recipient}`);
}
async function claimEmailWithoutPassword(harness: ApiTestHarness, account: TestAccount): Promise<TestAccount> {
await unclaimAccount(harness, account.userId);
const start = await createBuilder<{ticket: string; original_proof?: string}>(harness, account.token)
.post('/users/@me/email-change/start')
.body({})
.execute();
const email = createUniqueEmail('passwordless-reset');
await createBuilder(harness, account.token)
.post('/users/@me/email-change/request-new')
.body({ticket: start.ticket, new_email: email, original_proof: start.original_proof})
.execute();
const newEmail = await waitForEmail(harness, 'email_change_new', email);
const verify = await createBuilder<{email_token: string}>(harness, account.token)
.post('/users/@me/email-change/verify-new')
.body({ticket: start.ticket, code: newEmail.metadata['code'], original_proof: start.original_proof})
.execute();
await createBuilder(harness, account.token).patch('/users/@me').body({email_token: verify.email_token}).execute();
return {...account, email};
}
async function requestPasswordReset(harness: ApiTestHarness, email: string): Promise<string> {
await clearTestEmails(harness);
await createBuilderWithoutAuth(harness).post('/auth/forgot').body({email}).expect(204).execute();
const mail = await waitForEmail(harness, 'password_reset', email);
const token = mail.metadata['token'];
expect(token).toBeDefined();
return token!;
}
describe('Auth reset password requires MFA', () => {
let harness: ApiTestHarness;
beforeAll(async () => {
@@ -66,7 +107,8 @@ describe('Auth reset password requires MFA', () => {
expect(resetResp.ticket).toBeDefined();
expect(resetResp.totp).toBe(true);
expect(resetResp.webauthn).toBe(false);
expect(resetResp.allowed_methods).toEqual(['totp']);
expect(resetResp.backup_codes).toBe(true);
expect(resetResp.allowed_methods).toEqual(['totp', 'backup_codes']);
const mfaResp = await createBuilderWithoutAuth<{
token: string;
}>(harness)
@@ -86,4 +128,112 @@ describe('Auth reset password requires MFA', () => {
expect(login.totp).toBe(true);
expect(login.webauthn).toBe(false);
});
it('returns a session after password reset for a passkey user who left two-factor off', async () => {
const account = await createTestAccount(harness);
const device = createWebAuthnDevice();
await registerWebAuthnCredential(harness, account.token, device, () => ({password: account.password}));
await clearTestEmails(harness);
await createBuilderWithoutAuth(harness).post('/auth/forgot').body({email: account.email}).expect(204).execute();
const email = await waitForEmail(harness, 'password_reset', account.email);
const token = email.metadata['token'];
expect(token).toBeDefined();
const resetResp = await createBuilderWithoutAuth<LoginSuccessResponse | MfaRequiredResponse>(harness)
.post('/auth/reset')
.body({token, password: 'new-strong-password-123'})
.execute();
expect('mfa' in resetResp).toBe(false);
expect((resetResp as LoginSuccessResponse).token).toBeTruthy();
});
it('returns an MFA ticket after password reset for a passkey user who turned two-factor on', async () => {
const account = await createTestAccount(harness);
const device = createWebAuthnDevice();
await registerWebAuthnCredential(harness, account.token, device, () => ({password: account.password}));
await setWebAuthnTwoFactor(harness, account.token, true, {password: account.password});
await clearTestEmails(harness);
await createBuilderWithoutAuth(harness).post('/auth/forgot').body({email: account.email}).expect(204).execute();
const email = await waitForEmail(harness, 'password_reset', account.email);
const token = email.metadata['token'];
expect(token).toBeDefined();
const resetResp = await createBuilderWithoutAuth<MfaRequiredResponse>(harness)
.post('/auth/reset')
.body({token, password: 'new-strong-password-123'})
.execute();
expect(resetResp.mfa).toBe(true);
expect(resetResp.totp).toBe(false);
expect(resetResp.webauthn).toBe(true);
expect(resetResp.allowed_methods).toContain('webauthn');
const mfaOptions = await createBuilderWithoutAuth<WebAuthnAuthenticationOptions>(harness)
.post('/auth/login/mfa/webauthn/authentication-options')
.body({ticket: resetResp.ticket})
.execute();
if (mfaOptions.rpId) {
device.rpId = mfaOptions.rpId;
}
const mfaResp = await createBuilderWithoutAuth<{
token: string;
}>(harness)
.post('/auth/login/mfa/webauthn')
.body({
response: createAuthenticationResponse(device, mfaOptions),
challenge: mfaOptions.challenge,
ticket: resetResp.ticket,
})
.execute();
expect(mfaResp.token).toBeDefined();
});
it('returns an MFA ticket after password reset for an account with no password that holds a passkey', async () => {
const base = await createTestAccount(harness);
const account = await claimEmailWithoutPassword(harness, base);
const device = createWebAuthnDevice();
await registerWebAuthnCredential(harness, account.token, device, () => ({}));
const token = await requestPasswordReset(harness, account.email);
const resetResp = await createBuilderWithoutAuth<MfaRequiredResponse>(harness)
.post('/auth/reset')
.body({token, password: 'new-strong-password-123'})
.execute();
expect(resetResp.mfa).toBe(true);
expect(resetResp.totp).toBe(false);
expect(resetResp.webauthn).toBe(true);
expect(resetResp.allowed_methods).toContain('webauthn');
const mfaOptions = await createBuilderWithoutAuth<WebAuthnAuthenticationOptions>(harness)
.post('/auth/login/mfa/webauthn/authentication-options')
.body({ticket: resetResp.ticket})
.execute();
if (mfaOptions.rpId) {
device.rpId = mfaOptions.rpId;
}
const mfaResp = await createBuilderWithoutAuth<LoginSuccessResponse>(harness)
.post('/auth/login/mfa/webauthn')
.body({
response: createAuthenticationResponse(device, mfaOptions),
challenge: mfaOptions.challenge,
ticket: resetResp.ticket,
})
.execute();
expect(mfaResp.token).toBeTruthy();
const me = await createBuilder<{id: string; authenticator_types: Array<number>}>(harness, mfaResp.token)
.get('/users/@me')
.execute();
expect(me.id).toBe(account.userId);
expect(me.authenticator_types).toEqual([UserAuthenticatorTypes.WEBAUTHN]);
});
it('keeps demanding the passkey on a second password reset for an account that started with no password', async () => {
const base = await createTestAccount(harness);
const account = await claimEmailWithoutPassword(harness, base);
const device = createWebAuthnDevice();
await registerWebAuthnCredential(harness, account.token, device, () => ({}));
const firstToken = await requestPasswordReset(harness, account.email);
await createBuilderWithoutAuth<MfaRequiredResponse>(harness)
.post('/auth/reset')
.body({token: firstToken, password: 'new-strong-password-123'})
.execute();
const secondToken = await requestPasswordReset(harness, account.email);
const secondReset = await createBuilderWithoutAuth<MfaRequiredResponse>(harness)
.post('/auth/reset')
.body({token: secondToken, password: 'another-strong-password-456'})
.execute();
expect(secondReset.mfa).toBe(true);
expect(secondReset.webauthn).toBe(true);
expect(secondReset.allowed_methods).toContain('webauthn');
});
});
@@ -0,0 +1,71 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import {createAuthHarness, createTestAccount, unclaimAccount} from '@app/api/auth/tests/AuthTestUtils';
import {
createSudoWebAuthnBody,
createWebAuthnDevice,
registerWebAuthnCredential,
} from '@app/api/auth/tests/WebAuthnTestUtils';
import type {ApiTestHarness} from '@app/api/test/ApiTestHarness';
import {HTTP_STATUS} from '@app/api/test/TestConstants';
import {createBuilder} from '@app/api/test/TestRequestBuilder';
import {afterAll, beforeAll, beforeEach, describe, expect, it} from 'vitest';
interface SudoMfaMethodsResponse {
totp: boolean;
webauthn: boolean;
backup_codes: boolean;
has_mfa: boolean;
}
describe('Sudo mode for passwordless accounts holding a passkey', () => {
let harness: ApiTestHarness;
beforeAll(async () => {
harness = await createAuthHarness();
});
beforeEach(async () => {
await harness.reset();
});
afterAll(async () => {
await harness?.shutdown();
});
it('still waves through a passwordless account that holds no credential at all', async () => {
const account = await createTestAccount(harness);
await unclaimAccount(harness, account.userId);
await createBuilder(harness, account.token)
.post('/users/@me/disable')
.body({})
.expect(HTTP_STATUS.NO_CONTENT)
.execute();
});
it('challenges a passwordless account that holds a passkey it never made a second factor', async () => {
const account = await createTestAccount(harness);
const device = createWebAuthnDevice();
await registerWebAuthnCredential(harness, account.token, device, () => ({password: account.password}));
await unclaimAccount(harness, account.userId);
const methods = await createBuilder<SudoMfaMethodsResponse>(harness, account.token)
.get('/users/@me/sudo/mfa-methods')
.execute();
expect(methods).toEqual({totp: false, webauthn: true, backup_codes: false, has_mfa: true});
const errorResp = await createBuilder<{
code: string;
}>(harness, account.token)
.post('/users/@me/disable')
.body({})
.expect(HTTP_STATUS.FORBIDDEN, 'SUDO_MODE_REQUIRED')
.execute();
expect(errorResp.code).toBe('SUDO_MODE_REQUIRED');
});
it('lets the passwordless passkey holder clear the challenge with an assertion', async () => {
const account = await createTestAccount(harness);
const device = createWebAuthnDevice();
await registerWebAuthnCredential(harness, account.token, device, () => ({password: account.password}));
await unclaimAccount(harness, account.userId);
const sudoBody = await createSudoWebAuthnBody(harness, account.token, device);
await createBuilder(harness, account.token)
.post('/users/@me/disable')
.body(sudoBody)
.expect(HTTP_STATUS.NO_CONTENT)
.execute();
});
});
@@ -1,6 +1,6 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import {userHasMfa} from '@app/api/auth/services/SudoMethods';
import {userHasMfa, userHasSudoCapability} from '@app/api/auth/services/SudoMethods';
import {hasNoVerifiableCredential} from '@app/api/auth/services/SudoVerificationService';
import {createUserID} from '@app/api/BrandedTypes';
import {EMPTY_USER_ROW, type UserRow} from '@app/api/database/types/UserTypes';
@@ -25,17 +25,17 @@ describe('sudo verification credential capability', () => {
it('lets an SSO provisioned account without a password satisfy sudo mode', () => {
const user = createUser({password_hash: null, traits: new Set<string>(['sso'])});
expect(user.isUnclaimedAccount()).toBe(false);
expect(hasNoVerifiableCredential(user, userHasMfa(user))).toBe(true);
expect(hasNoVerifiableCredential(user, userHasMfa(user), false)).toBe(true);
});
it('still lets an unclaimed account satisfy sudo mode', () => {
const user = createUser({password_hash: null});
expect(hasNoVerifiableCredential(user, userHasMfa(user))).toBe(true);
expect(hasNoVerifiableCredential(user, userHasMfa(user), false)).toBe(true);
});
it('still requires a password from accounts that have one', () => {
const user = createUser({password_hash: 'hash', traits: new Set<string>(['sso'])});
expect(hasNoVerifiableCredential(user, userHasMfa(user))).toBe(false);
expect(hasNoVerifiableCredential(user, userHasMfa(user), false)).toBe(false);
});
it('still requires MFA from an SSO account that enrolled a second factor', () => {
@@ -45,11 +45,36 @@ describe('sudo verification credential capability', () => {
authenticator_types: new Set<number>([UserAuthenticatorTypes.TOTP]),
});
expect(userHasMfa(user)).toBe(true);
expect(hasNoVerifiableCredential(user, userHasMfa(user))).toBe(false);
expect(hasNoVerifiableCredential(user, userHasMfa(user), false)).toBe(false);
});
it('never applies to bots', () => {
const user = createUser({password_hash: null, bot: true});
expect(hasNoVerifiableCredential(user, userHasMfa(user))).toBe(false);
expect(hasNoVerifiableCredential(user, userHasMfa(user), false)).toBe(false);
});
it('still requires MFA from a passwordless account holding a passkey it never made a second factor', () => {
const user = createUser({password_hash: null, traits: new Set<string>(['sso'])});
expect(userHasMfa(user)).toBe(false);
expect(userHasSudoCapability(user, true)).toBe(true);
expect(hasNoVerifiableCredential(user, userHasMfa(user), true)).toBe(false);
});
it('still requires MFA from an unclaimed account holding a passkey', () => {
const user = createUser({password_hash: null});
expect(hasNoVerifiableCredential(user, userHasMfa(user), true)).toBe(false);
});
it('reports no sudo capability for an account with a TOTP secret that was never enrolled', () => {
const user = createUser({totp_secret: 'JBSWY3DPEHPK3PXP'});
expect(userHasSudoCapability(user, false)).toBe(false);
});
it('reports sudo capability from an enrolled TOTP secret without any passkey', () => {
const user = createUser({
totp_secret: 'JBSWY3DPEHPK3PXP',
authenticator_types: new Set<number>([UserAuthenticatorTypes.TOTP]),
});
expect(userHasSudoCapability(user, false)).toBe(true);
});
});
@@ -6,11 +6,13 @@ import {
createTotpSecret,
createWebAuthnDevice,
generateTotpCode,
registerWebAuthnCredential,
type WebAuthnCredentialMetadata,
type WebAuthnRegistrationOptions,
} from '@app/api/auth/tests/WebAuthnTestUtils';
import type {ApiTestHarness} from '@app/api/test/ApiTestHarness';
import {createBuilder} from '@app/api/test/TestRequestBuilder';
import {UserAuthenticatorTypes} from '@fluxer/constants/src/UserConstants';
import {afterAll, beforeAll, beforeEach, describe, expect, it} from 'vitest';
describe('WebAuthn credential registration', () => {
@@ -64,4 +66,23 @@ describe('WebAuthn credential registration', () => {
expect(credentials[0].name).toBe('Test Passkey');
expect(credentials[0].id).toBe(device.credentialId.toString('base64url'));
});
it('does not turn passkeys into a second factor when a credential is registered', async () => {
const account = await createTestAccount(harness);
const device = createWebAuthnDevice();
const secret = createTotpSecret();
await createBuilder(harness, account.token)
.post('/users/@me/mfa/totp/enable')
.body({secret, code: generateTotpCode(secret), password: account.password})
.execute();
await registerWebAuthnCredential(harness, account.token, device, () => ({
mfa_method: 'totp',
mfa_code: generateTotpCode(secret),
}));
const me = await createBuilder<{
authenticator_types: Array<number>;
}>(harness, account.token)
.get('/users/@me')
.execute();
expect(me.authenticator_types).toEqual([UserAuthenticatorTypes.TOTP]);
});
});
@@ -1,54 +1,65 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import {createTestAccount, createTotpSecret, generateTotpCode} from '@app/api/auth/tests/AuthTestUtils';
import {
createAuthenticationResponse,
createRegistrationResponse,
createTestAccount,
createTotpSecret,
generateTotpCode,
type LoginMfaResponse,
type LoginSuccessResponse,
type TestAccount,
} from '@app/api/auth/tests/AuthTestUtils';
import {
createSudoWebAuthnBody,
createWebAuthnDevice,
loginWithDiscoverablePasskey,
registerWebAuthnCredential,
setWebAuthnTwoFactor,
type WebAuthnAuthenticationOptions,
type WebAuthnDevice,
} from '@app/api/auth/tests/WebAuthnTestUtils';
import {type ApiTestHarness, createApiTestHarness} from '@app/api/test/ApiTestHarness';
import {createBuilder, createBuilderWithoutAuth} from '@app/api/test/TestRequestBuilder';
import {beforeEach, describe, expect, test} from 'vitest';
interface BackupCodesResponse {
backup_codes: Array<{
code: string;
}>;
}
interface LoginMfaResponse {
mfa: true;
ticket: string;
totp: boolean;
webauthn: boolean;
}
interface WebAuthnRegistrationOptions {
challenge: string;
rp: {
id: string;
name: string;
};
user: {
id: string;
name: string;
displayName: string;
};
authenticatorSelection?: {
residentKey?: string;
requireResidentKey?: boolean;
userVerification?: string;
};
}
interface WebAuthnAuthenticationOptions {
challenge: string;
rpId: string;
allowCredentials?: Array<{
id: string;
type: string;
}>;
userVerification: string;
async function setupPasskeyOnlyAccount(
harness: ApiTestHarness,
twoFactorEnabled: boolean,
): Promise<{
account: TestAccount;
device: WebAuthnDevice;
}> {
const account = await createTestAccount(harness);
const device = createWebAuthnDevice();
const secret = createTotpSecret();
await createBuilder(harness, account.token)
.post('/users/@me/mfa/totp/enable')
.body({
secret,
code: generateTotpCode(secret),
password: account.password,
})
.execute();
await registerWebAuthnCredential(harness, account.token, device, () => ({
mfa_method: 'totp',
mfa_code: generateTotpCode(secret),
}));
if (twoFactorEnabled) {
await setWebAuthnTwoFactor(harness, account.token, true, {
mfa_method: 'totp',
mfa_code: generateTotpCode(secret),
});
}
await createBuilder(harness, account.token)
.post('/users/@me/mfa/totp/disable')
.body({
code: generateTotpCode(secret),
mfa_method: 'totp',
mfa_code: generateTotpCode(secret),
})
.expect(204)
.execute();
const token = await loginWithDiscoverablePasskey(harness, device);
return {account: {...account, token}, device};
}
describe('WebAuthn MFA Consistency Tests', () => {
@@ -56,84 +67,8 @@ describe('WebAuthn MFA Consistency Tests', () => {
beforeEach(async () => {
harness = await createApiTestHarness();
});
test('WebAuthn-only user cannot use password for sudo - password rejected with 403', async () => {
const account = await createTestAccount(harness);
const device = createWebAuthnDevice();
const secret = createTotpSecret();
const backupCodes = await createBuilder<BackupCodesResponse>(harness, account.token)
.post('/users/@me/mfa/totp/enable')
.body({
secret,
code: generateTotpCode(secret),
password: account.password,
})
.execute();
const login = await createBuilderWithoutAuth<LoginMfaResponse>(harness)
.post('/auth/login')
.body({
email: account.email,
password: account.password,
})
.execute();
expect(login.mfa).toBe(true);
const mfaLogin = await createBuilderWithoutAuth<{
token: string;
}>(harness)
.post('/auth/login/mfa/totp')
.body({
code: backupCodes.backup_codes[0]!.code,
ticket: login.ticket,
})
.execute();
account.token = mfaLogin.token;
const registrationOptions = await createBuilder<WebAuthnRegistrationOptions>(harness, account.token)
.post('/users/@me/mfa/webauthn/credentials/registration-options')
.body({
mfa_method: 'totp',
mfa_code: backupCodes.backup_codes[1]!.code,
})
.execute();
expect(registrationOptions.authenticatorSelection).toMatchObject({
residentKey: 'preferred',
requireResidentKey: false,
userVerification: 'preferred',
});
const registrationResponse = createRegistrationResponse(device, registrationOptions, 'Test Passkey');
await createBuilder(harness, account.token)
.post('/users/@me/mfa/webauthn/credentials')
.body({
response: registrationResponse,
challenge: registrationOptions.challenge,
name: 'Test Passkey',
mfa_method: 'totp',
mfa_code: backupCodes.backup_codes[2]!.code,
})
.expect(204)
.execute();
await createBuilder(harness, account.token)
.post('/users/@me/mfa/totp/disable')
.body({
code: backupCodes.backup_codes[3]!.code,
mfa_method: 'totp',
mfa_code: backupCodes.backup_codes[4]!.code,
})
.expect(204)
.execute();
const discoverableOptions = await createBuilderWithoutAuth<WebAuthnAuthenticationOptions>(harness)
.post('/auth/webauthn/authentication-options')
.body(null)
.execute();
const discoverableAssertion = createAuthenticationResponse(device, discoverableOptions);
const passkeyLogin = await createBuilderWithoutAuth<{
token: string;
}>(harness)
.post('/auth/webauthn/authenticate')
.body({
response: discoverableAssertion,
challenge: discoverableOptions.challenge,
})
.execute();
account.token = passkeyLogin.token;
test('passkey user with two-factor on cannot use password for sudo - password rejected with 403', async () => {
const {account} = await setupPasskeyOnlyAccount(harness, true);
const {json: errorResp} = await createBuilder<{
code: string;
}>(harness, account.token)
@@ -145,18 +80,34 @@ describe('WebAuthn MFA Consistency Tests', () => {
.executeWithResponse();
expect(errorResp.code).toBe('SUDO_MODE_REQUIRED');
});
test('WebAuthn-only user can use WebAuthn for sudo verification', async () => {
const account = await createTestAccount(harness);
const device = createWebAuthnDevice();
const secret = createTotpSecret();
const backupCodes = await createBuilder<BackupCodesResponse>(harness, account.token)
.post('/users/@me/mfa/totp/enable')
test('passkey user with two-factor off can still use password for sudo', async () => {
const {account} = await setupPasskeyOnlyAccount(harness, false);
await createBuilder(harness, account.token)
.post('/users/@me/disable')
.body({
secret,
code: generateTotpCode(secret),
password: account.password,
})
.expect(204)
.execute();
});
test('passkey user with two-factor on can use WebAuthn for sudo verification', async () => {
const {account, device} = await setupPasskeyOnlyAccount(harness, true);
const sudoBody = await createSudoWebAuthnBody(harness, account.token, device);
const {response: disableResp} = await createBuilder(harness, account.token)
.post('/users/@me/disable')
.body(sudoBody)
.expect(204)
.executeWithResponse();
const sudoToken = disableResp.headers.get('x-sudo-mode-token');
expect(sudoToken).toBeNull();
});
test('passkey user with two-factor off can use WebAuthn for sudo verification', async () => {
const {account, device} = await setupPasskeyOnlyAccount(harness, false);
const sudoBody = await createSudoWebAuthnBody(harness, account.token, device);
await createBuilder(harness, account.token).post('/users/@me/disable').body(sudoBody).expect(204).execute();
});
test('passkey user with two-factor on requires MFA when logging in with password', async () => {
const {account} = await setupPasskeyOnlyAccount(harness, true);
const login = await createBuilderWithoutAuth<LoginMfaResponse>(harness)
.post('/auth/login')
.body({
@@ -165,144 +116,35 @@ describe('WebAuthn MFA Consistency Tests', () => {
})
.execute();
expect(login.mfa).toBe(true);
const mfaLogin = await createBuilderWithoutAuth<{
token: string;
}>(harness)
.post('/auth/login/mfa/totp')
expect(login.ticket).toBeTruthy();
expect(login.webauthn).toBe(true);
});
test('passkey user with two-factor off logs in with password and receives a session token', async () => {
const {account} = await setupPasskeyOnlyAccount(harness, false);
const login = await createBuilderWithoutAuth<LoginSuccessResponse | LoginMfaResponse>(harness)
.post('/auth/login')
.body({
code: backupCodes.backup_codes[0]!.code,
ticket: login.ticket,
email: account.email,
password: account.password,
})
.execute();
account.token = mfaLogin.token;
const registrationOptions = await createBuilder<WebAuthnRegistrationOptions>(harness, account.token)
.post('/users/@me/mfa/webauthn/credentials/registration-options')
.body({
mfa_method: 'totp',
mfa_code: backupCodes.backup_codes[1]!.code,
})
expect('mfa' in login).toBe(false);
expect((login as LoginSuccessResponse).token).toBeTruthy();
const userInfo = await createBuilder<{
id: string;
}>(harness, (login as LoginSuccessResponse).token)
.get('/users/@me')
.execute();
const registrationResponse = createRegistrationResponse(device, registrationOptions, 'Test Passkey');
await createBuilder(harness, account.token)
.post('/users/@me/mfa/webauthn/credentials')
.body({
response: registrationResponse,
challenge: registrationOptions.challenge,
name: 'Test Passkey',
mfa_method: 'totp',
mfa_code: backupCodes.backup_codes[2]!.code,
})
.expect(204)
.execute();
await createBuilder(harness, account.token)
.post('/users/@me/mfa/totp/disable')
.body({
code: backupCodes.backup_codes[3]!.code,
mfa_method: 'totp',
mfa_code: backupCodes.backup_codes[4]!.code,
})
.expect(204)
.execute();
const discoverableOptions = await createBuilderWithoutAuth<WebAuthnAuthenticationOptions>(harness)
.post('/auth/webauthn/authentication-options')
.body(null)
.execute();
const discoverableAssertion = createAuthenticationResponse(device, discoverableOptions);
const passkeyLogin = await createBuilderWithoutAuth<{
token: string;
}>(harness)
.post('/auth/webauthn/authenticate')
.body({
response: discoverableAssertion,
challenge: discoverableOptions.challenge,
})
.execute();
account.token = passkeyLogin.token;
expect(userInfo.id).toBe(account.userId);
});
test('sudo WebAuthn options stay available to a passkey user with two-factor off', async () => {
const {account, device} = await setupPasskeyOnlyAccount(harness, false);
const sudoOptions = await createBuilder<WebAuthnAuthenticationOptions>(harness, account.token)
.post('/users/@me/sudo/webauthn/authentication-options')
.body(null)
.execute();
expect(sudoOptions.userVerification).toBe('discouraged');
const sudoAssertion = createAuthenticationResponse(device, sudoOptions);
const {response: disableResp2} = await createBuilder(harness, account.token)
.post('/users/@me/disable')
.body({
mfa_method: 'webauthn',
webauthn_response: sudoAssertion,
webauthn_challenge: sudoOptions.challenge,
})
.expect(204)
.executeWithResponse();
const sudoToken = disableResp2.headers.get('x-sudo-mode-token');
expect(sudoToken).toBeNull();
});
test('WebAuthn-only user requires MFA when logging in with password', async () => {
const account = await createTestAccount(harness);
const device = createWebAuthnDevice();
const secret = createTotpSecret();
const backupCodes = await createBuilder<BackupCodesResponse>(harness, account.token)
.post('/users/@me/mfa/totp/enable')
.body({
secret,
code: generateTotpCode(secret),
password: account.password,
})
.execute();
const login = await createBuilderWithoutAuth<LoginMfaResponse>(harness)
.post('/auth/login')
.body({
email: account.email,
password: account.password,
})
.execute();
expect(login.mfa).toBe(true);
const mfaLogin = await createBuilderWithoutAuth<{
token: string;
}>(harness)
.post('/auth/login/mfa/totp')
.body({
code: backupCodes.backup_codes[0]!.code,
ticket: login.ticket,
})
.execute();
account.token = mfaLogin.token;
const registrationOptions = await createBuilder<WebAuthnRegistrationOptions>(harness, account.token)
.post('/users/@me/mfa/webauthn/credentials/registration-options')
.body({
mfa_method: 'totp',
mfa_code: backupCodes.backup_codes[1]!.code,
})
.execute();
const registrationResponse = createRegistrationResponse(device, registrationOptions, 'Test Passkey');
await createBuilder(harness, account.token)
.post('/users/@me/mfa/webauthn/credentials')
.body({
response: registrationResponse,
challenge: registrationOptions.challenge,
name: 'Test Passkey',
mfa_method: 'totp',
mfa_code: backupCodes.backup_codes[2]!.code,
})
.expect(204)
.execute();
await createBuilder(harness, account.token)
.post('/users/@me/mfa/totp/disable')
.body({
code: backupCodes.backup_codes[3]!.code,
mfa_method: 'totp',
mfa_code: backupCodes.backup_codes[4]!.code,
})
.expect(204)
.execute();
const login2 = await createBuilderWithoutAuth<LoginMfaResponse>(harness)
.post('/auth/login')
.body({
email: account.email,
password: account.password,
})
.execute();
expect(login2.mfa).toBe(true);
expect(login2.ticket).toBeTruthy();
expect(login2.webauthn).toBe(true);
expect(sudoOptions.allowCredentials?.length).toBeGreaterThan(0);
expect(device.credentialId.length).toBeGreaterThan(0);
});
});
@@ -4,16 +4,17 @@ import {
createAuthHarness,
createTestAccount,
type LoginMfaResponse,
type LoginSuccessResponse,
loginUser,
} from '@app/api/auth/tests/AuthTestUtils';
import {
createAuthenticationResponse,
createRegistrationResponse,
createTotpSecret,
createWebAuthnDevice,
generateTotpCode,
registerWebAuthnCredential,
setWebAuthnTwoFactor,
type WebAuthnAuthenticationOptions,
type WebAuthnRegistrationOptions,
} from '@app/api/auth/tests/WebAuthnTestUtils';
import type {ApiTestHarness} from '@app/api/test/ApiTestHarness';
import {createBuilder, createBuilderWithoutAuth} from '@app/api/test/TestRequestBuilder';
@@ -30,7 +31,7 @@ describe('WebAuthn MFA login', () => {
afterAll(async () => {
await harness?.shutdown();
});
it('validates the WebAuthn MFA login flow', async () => {
it('validates the WebAuthn MFA login flow when passkey two-factor is turned on', async () => {
const account = await createTestAccount(harness);
const device = createWebAuthnDevice();
const secret = createTotpSecret();
@@ -38,25 +39,17 @@ describe('WebAuthn MFA login', () => {
.post('/users/@me/mfa/totp/enable')
.body({secret, code: generateTotpCode(secret), password: account.password})
.execute();
const regOptions = await createBuilder<WebAuthnRegistrationOptions>(harness, account.token)
.post('/users/@me/mfa/webauthn/credentials/registration-options')
.body({mfa_method: 'totp', mfa_code: generateTotpCode(secret)})
.execute();
if (regOptions.rp.id) {
device.rpId = regOptions.rp.id;
}
const registrationResponse = createRegistrationResponse(device, regOptions, 'MFA Passkey');
await createBuilder(harness, account.token)
.post('/users/@me/mfa/webauthn/credentials')
.body({
response: registrationResponse,
challenge: regOptions.challenge,
name: 'MFA Passkey',
mfa_method: 'totp',
mfa_code: generateTotpCode(secret),
})
.expect(204)
.execute();
await registerWebAuthnCredential(
harness,
account.token,
device,
() => ({mfa_method: 'totp', mfa_code: generateTotpCode(secret)}),
'MFA Passkey',
);
await setWebAuthnTwoFactor(harness, account.token, true, {
mfa_method: 'totp',
mfa_code: generateTotpCode(secret),
});
const loginResp = await loginUser(harness, {email: account.email, password: account.password});
expect('mfa' in loginResp && loginResp.mfa).toBe(true);
const loginMfaResp = loginResp as LoginMfaResponse;
@@ -83,7 +76,7 @@ describe('WebAuthn MFA login', () => {
.body({
response: mfaAssertion,
challenge: mfaOptions.challenge,
ticket: (loginResp as LoginMfaResponse).ticket,
ticket: loginMfaResp.ticket,
})
.execute();
expect(webauthnMfaLogin.token).toBeTruthy();
@@ -94,4 +87,39 @@ describe('WebAuthn MFA login', () => {
.execute();
expect(userInfo.id).toBe(account.userId);
});
it('issues a session token instead of an MFA ticket when passkey two-factor is left off', async () => {
const account = await createTestAccount(harness);
const device = createWebAuthnDevice();
const secret = createTotpSecret();
await createBuilder(harness, account.token)
.post('/users/@me/mfa/totp/enable')
.body({secret, code: generateTotpCode(secret), password: account.password})
.execute();
await registerWebAuthnCredential(
harness,
account.token,
device,
() => ({mfa_method: 'totp', mfa_code: generateTotpCode(secret)}),
'MFA Passkey',
);
await createBuilder(harness, account.token)
.post('/users/@me/mfa/totp/disable')
.body({
code: generateTotpCode(secret),
mfa_method: 'totp',
mfa_code: generateTotpCode(secret),
})
.expect(204)
.execute();
const loginResp = await loginUser(harness, {email: account.email, password: account.password});
expect('mfa' in loginResp).toBe(false);
const loginSuccessResp = loginResp as LoginSuccessResponse;
expect(loginSuccessResp.token).toBeTruthy();
const userInfo = await createBuilder<{
id: string;
}>(harness, loginSuccessResp.token)
.get('/users/@me')
.execute();
expect(userInfo.id).toBe(account.userId);
});
});
@@ -0,0 +1,124 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import {
createAuthHarness,
createTestAccount,
createTotpSecret,
generateTotpCode,
type LoginMfaResponse,
type LoginSuccessResponse,
loginUser,
} from '@app/api/auth/tests/AuthTestUtils';
import {
createAuthenticationResponse,
createWebAuthnDevice,
loginWithDiscoverablePasskey,
registerWebAuthnCredential,
type WebAuthnAuthenticationOptions,
} from '@app/api/auth/tests/WebAuthnTestUtils';
import type {ApiTestHarness} from '@app/api/test/ApiTestHarness';
import {HTTP_STATUS} from '@app/api/test/TestConstants';
import {createBuilder, createBuilderWithoutAuth} from '@app/api/test/TestRequestBuilder';
import {UserAuthenticatorTypes} from '@fluxer/constants/src/UserConstants';
import {afterAll, beforeAll, beforeEach, describe, expect, it} from 'vitest';
describe('WebAuthn opt-in login', () => {
let harness: ApiTestHarness;
beforeAll(async () => {
harness = await createAuthHarness();
});
beforeEach(async () => {
await harness.reset();
});
afterAll(async () => {
await harness?.shutdown();
});
it('does not offer webauthn at login to a TOTP user whose passkey is opted out', async () => {
const account = await createTestAccount(harness);
const device = createWebAuthnDevice();
const secret = createTotpSecret();
await createBuilder(harness, account.token)
.post('/users/@me/mfa/totp/enable')
.body({secret, code: generateTotpCode(secret), password: account.password})
.execute();
await registerWebAuthnCredential(harness, account.token, device, () => ({
mfa_method: 'totp',
mfa_code: generateTotpCode(secret),
}));
const login = (await loginUser(harness, {
email: account.email,
password: account.password,
})) as LoginMfaResponse;
expect(login.mfa).toBe(true);
expect(login.totp).toBe(true);
expect(login.webauthn).toBe(false);
expect(login.allowed_methods).not.toContain('webauthn');
expect(login.allowed_methods).toContain('totp');
});
it('rejects the WebAuthn MFA login route for a user who never turned passkey two-factor on', async () => {
const account = await createTestAccount(harness);
const device = createWebAuthnDevice();
const secret = createTotpSecret();
await createBuilder(harness, account.token)
.post('/users/@me/mfa/totp/enable')
.body({secret, code: generateTotpCode(secret), password: account.password})
.execute();
await registerWebAuthnCredential(harness, account.token, device, () => ({
mfa_method: 'totp',
mfa_code: generateTotpCode(secret),
}));
const login = (await loginUser(harness, {
email: account.email,
password: account.password,
})) as LoginMfaResponse;
const mfaOptions = await createBuilderWithoutAuth<WebAuthnAuthenticationOptions>(harness)
.post('/auth/login/mfa/webauthn/authentication-options')
.body({ticket: login.ticket})
.execute();
if (mfaOptions.rpId) {
device.rpId = mfaOptions.rpId;
}
await createBuilderWithoutAuth(harness)
.post('/auth/login/mfa/webauthn')
.body({
response: createAuthenticationResponse(device, mfaOptions),
challenge: mfaOptions.challenge,
ticket: login.ticket,
})
.expect(HTTP_STATUS.BAD_REQUEST, 'TWO_FACTOR_REQUIRED')
.execute();
const totpLogin = await createBuilderWithoutAuth<{
token: string;
}>(harness)
.post('/auth/login/mfa/totp')
.body({ticket: login.ticket, code: generateTotpCode(secret)})
.execute();
expect(totpLogin.token).toBeTruthy();
});
it('completes the passwordless journey for an account that never enrolled TOTP or the toggle', async () => {
const account = await createTestAccount(harness);
const device = createWebAuthnDevice();
await registerWebAuthnCredential(harness, account.token, device, () => ({password: account.password}));
const me = await createBuilder<{
authenticator_types: Array<number>;
mfa_enabled: boolean;
}>(harness, account.token)
.get('/users/@me')
.execute();
expect(me.authenticator_types).toEqual([]);
expect(me.mfa_enabled).toBe(false);
const passwordLogin = await loginUser(harness, {email: account.email, password: account.password});
expect('mfa' in passwordLogin).toBe(false);
expect((passwordLogin as LoginSuccessResponse).token).toBeTruthy();
const passkeyToken = await loginWithDiscoverablePasskey(harness, device);
expect(passkeyToken).toBeTruthy();
const passkeyMe = await createBuilder<{
id: string;
authenticator_types: Array<number>;
}>(harness, passkeyToken)
.get('/users/@me')
.execute();
expect(passkeyMe.id).toBe(account.userId);
expect(passkeyMe.authenticator_types).not.toContain(UserAuthenticatorTypes.WEBAUTHN);
});
});
@@ -9,6 +9,8 @@ import {
generateKeyPairSync,
randomBytes,
} from 'node:crypto';
import type {ApiTestHarness} from '@app/api/test/ApiTestHarness';
import {createBuilder, createBuilderWithoutAuth} from '@app/api/test/TestRequestBuilder';
import {decode as base32Decode, encode as base32Encode} from 'hi-base32';
export interface WebAuthnDevice {
@@ -54,6 +56,22 @@ export interface WebAuthnCredentialMetadata {
name: string;
}
export type SudoVerificationBody = Record<string, unknown>;
export type SudoVerificationBodyFactory = () => SudoVerificationBody;
export interface WebAuthnTwoFactorResult {
user: {
id: string;
mfa_enabled: boolean;
authenticator_types: Array<number>;
};
backup_codes: Array<{
code: string;
consumed: boolean;
}> | null;
}
interface AuthenticatorAttestationResponse {
clientDataJSON: string;
attestationObject: string;
@@ -415,3 +433,80 @@ export function createAuthenticationResponseWithoutUV(
},
};
}
export async function registerWebAuthnCredential(
harness: ApiTestHarness,
token: string,
device: WebAuthnDevice,
createSudoBody: SudoVerificationBodyFactory,
name = 'Test Passkey',
): Promise<void> {
const options = await createBuilder<WebAuthnRegistrationOptions>(harness, token)
.post('/users/@me/mfa/webauthn/credentials/registration-options')
.body(createSudoBody())
.execute();
if (options.rp.id) {
device.rpId = options.rp.id;
}
await createBuilder(harness, token)
.post('/users/@me/mfa/webauthn/credentials')
.body({
response: createRegistrationResponse(device, options, name),
challenge: options.challenge,
name,
...createSudoBody(),
})
.expect(204)
.execute();
}
export async function createSudoWebAuthnBody(
harness: ApiTestHarness,
token: string,
device: WebAuthnDevice,
): Promise<SudoVerificationBody> {
const options = await createBuilder<WebAuthnAuthenticationOptions>(harness, token)
.post('/users/@me/sudo/webauthn/authentication-options')
.body(null)
.execute();
if (options.rpId) {
device.rpId = options.rpId;
}
return {
mfa_method: 'webauthn',
webauthn_response: createAuthenticationResponse(device, options),
webauthn_challenge: options.challenge,
};
}
export async function setWebAuthnTwoFactor(
harness: ApiTestHarness,
token: string,
enabled: boolean,
sudo: SudoVerificationBody,
): Promise<WebAuthnTwoFactorResult> {
return createBuilder<WebAuthnTwoFactorResult>(harness, token)
.put('/users/@me/mfa/webauthn/two-factor')
.body({enabled, ...sudo})
.execute();
}
export async function loginWithDiscoverablePasskey(harness: ApiTestHarness, device: WebAuthnDevice): Promise<string> {
const options = await createBuilderWithoutAuth<WebAuthnAuthenticationOptions>(harness)
.post('/auth/webauthn/authentication-options')
.body(null)
.execute();
if (options.rpId) {
device.rpId = options.rpId;
}
const login = await createBuilderWithoutAuth<{
token: string;
}>(harness)
.post('/auth/webauthn/authenticate')
.body({
response: createAuthenticationResponse(device, options),
challenge: options.challenge,
})
.execute();
return login.token;
}
@@ -0,0 +1,217 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import {
createAuthHarness,
createTestAccount,
createTotpSecret,
generateTotpCode,
type LoginMfaResponse,
loginUser,
} from '@app/api/auth/tests/AuthTestUtils';
import {
createSudoWebAuthnBody,
createWebAuthnDevice,
registerWebAuthnCredential,
type SudoVerificationBody,
setWebAuthnTwoFactor,
type WebAuthnCredentialMetadata,
} from '@app/api/auth/tests/WebAuthnTestUtils';
import type {ApiTestHarness} from '@app/api/test/ApiTestHarness';
import {createBuilder, createBuilderWithoutAuth} from '@app/api/test/TestRequestBuilder';
import {afterAll, beforeAll, beforeEach, describe, expect, it} from 'vitest';
interface BackupCode {
code: string;
consumed: boolean;
}
async function readBackupCodes(
harness: ApiTestHarness,
token: string,
sudo: SudoVerificationBody,
): Promise<Array<BackupCode>> {
const response = await createBuilder<{
backup_codes: Array<BackupCode>;
}>(harness, token)
.post('/users/@me/mfa/backup-codes')
.body({regenerate: false, ...sudo})
.execute();
return response.backup_codes;
}
describe('WebAuthn two-factor backup codes', () => {
let harness: ApiTestHarness;
beforeAll(async () => {
harness = await createAuthHarness();
});
beforeEach(async () => {
await harness.reset();
});
afterAll(async () => {
await harness?.shutdown();
});
it('mints backup codes when passkey two-factor is turned on for an account with no TOTP', async () => {
const account = await createTestAccount(harness);
const device = createWebAuthnDevice();
await registerWebAuthnCredential(harness, account.token, device, () => ({password: account.password}));
const enabled = await setWebAuthnTwoFactor(harness, account.token, true, {password: account.password});
expect(enabled.backup_codes).not.toBeNull();
expect(enabled.backup_codes!.length).toBeGreaterThan(0);
expect(enabled.backup_codes!.every((backupCode) => !backupCode.consumed)).toBe(true);
const sudoBody = await createSudoWebAuthnBody(harness, account.token, device);
const stored = await readBackupCodes(harness, account.token, sudoBody);
expect(stored.map((backupCode) => backupCode.code).sort()).toEqual(
enabled.backup_codes!.map((backupCode) => backupCode.code).sort(),
);
});
it('mints nothing when the account already holds backup codes from TOTP', async () => {
const account = await createTestAccount(harness);
const device = createWebAuthnDevice();
const secret = createTotpSecret();
await createBuilder(harness, account.token)
.post('/users/@me/mfa/totp/enable')
.body({secret, code: generateTotpCode(secret), password: account.password})
.execute();
await registerWebAuthnCredential(harness, account.token, device, () => ({
mfa_method: 'totp',
mfa_code: generateTotpCode(secret),
}));
const enabled = await setWebAuthnTwoFactor(harness, account.token, true, {
mfa_method: 'totp',
mfa_code: generateTotpCode(secret),
});
expect(enabled.backup_codes).toBeNull();
});
it('accepts a minted backup code at login when the passkey is unavailable', async () => {
const account = await createTestAccount(harness);
const device = createWebAuthnDevice();
await registerWebAuthnCredential(harness, account.token, device, () => ({password: account.password}));
const enabled = await setWebAuthnTwoFactor(harness, account.token, true, {password: account.password});
const login = (await loginUser(harness, {
email: account.email,
password: account.password,
})) as LoginMfaResponse;
expect(login.mfa).toBe(true);
expect(login.totp).toBe(false);
expect(login.webauthn).toBe(true);
expect(login.backup_codes).toBe(true);
expect(login.allowed_methods).toContain('backup_codes');
const backupLogin = await createBuilderWithoutAuth<{
token: string;
}>(harness)
.post('/auth/login/mfa/totp')
.body({ticket: login.ticket, code: enabled.backup_codes![0]!.code})
.execute();
expect(backupLogin.token).toBeTruthy();
const me = await createBuilder<{
id: string;
}>(harness, backupLogin.token)
.get('/users/@me')
.execute();
expect(me.id).toBe(account.userId);
});
it('keeps backup codes when TOTP is disabled while passkey two-factor stays on', async () => {
const account = await createTestAccount(harness);
const device = createWebAuthnDevice();
const secret = createTotpSecret();
await createBuilder(harness, account.token)
.post('/users/@me/mfa/totp/enable')
.body({secret, code: generateTotpCode(secret), password: account.password})
.execute();
await registerWebAuthnCredential(harness, account.token, device, () => ({
mfa_method: 'totp',
mfa_code: generateTotpCode(secret),
}));
await setWebAuthnTwoFactor(harness, account.token, true, {
mfa_method: 'totp',
mfa_code: generateTotpCode(secret),
});
await createBuilder(harness, account.token)
.post('/users/@me/mfa/totp/disable')
.body({
code: generateTotpCode(secret),
mfa_method: 'totp',
mfa_code: generateTotpCode(secret),
})
.expect(204)
.execute();
const sudoBody = await createSudoWebAuthnBody(harness, account.token, device);
const stored = await readBackupCodes(harness, account.token, sudoBody);
expect(stored.length).toBeGreaterThan(0);
});
it('keeps backup codes when passkey two-factor is turned off while TOTP stays on', async () => {
const account = await createTestAccount(harness);
const device = createWebAuthnDevice();
const secret = createTotpSecret();
await createBuilder(harness, account.token)
.post('/users/@me/mfa/totp/enable')
.body({secret, code: generateTotpCode(secret), password: account.password})
.execute();
await registerWebAuthnCredential(harness, account.token, device, () => ({
mfa_method: 'totp',
mfa_code: generateTotpCode(secret),
}));
await setWebAuthnTwoFactor(harness, account.token, true, {
mfa_method: 'totp',
mfa_code: generateTotpCode(secret),
});
await setWebAuthnTwoFactor(harness, account.token, false, {
mfa_method: 'totp',
mfa_code: generateTotpCode(secret),
});
const stored = await readBackupCodes(harness, account.token, {
mfa_method: 'totp',
mfa_code: generateTotpCode(secret),
});
expect(stored.length).toBeGreaterThan(0);
});
it('clears backup codes only once no second factor remains', async () => {
const account = await createTestAccount(harness);
const device = createWebAuthnDevice();
const secret = createTotpSecret();
await createBuilder(harness, account.token)
.post('/users/@me/mfa/totp/enable')
.body({secret, code: generateTotpCode(secret), password: account.password})
.execute();
await registerWebAuthnCredential(harness, account.token, device, () => ({
mfa_method: 'totp',
mfa_code: generateTotpCode(secret),
}));
await setWebAuthnTwoFactor(harness, account.token, true, {
mfa_method: 'totp',
mfa_code: generateTotpCode(secret),
});
await setWebAuthnTwoFactor(harness, account.token, false, {
mfa_method: 'totp',
mfa_code: generateTotpCode(secret),
});
await createBuilder(harness, account.token)
.post('/users/@me/mfa/totp/disable')
.body({
code: generateTotpCode(secret),
mfa_method: 'totp',
mfa_code: generateTotpCode(secret),
})
.expect(204)
.execute();
const stored = await readBackupCodes(harness, account.token, {password: account.password});
expect(stored).toHaveLength(0);
});
it('clears backup codes when the last passkey is deleted and nothing else remains', async () => {
const account = await createTestAccount(harness);
const device = createWebAuthnDevice();
await registerWebAuthnCredential(harness, account.token, device, () => ({password: account.password}));
await setWebAuthnTwoFactor(harness, account.token, true, {password: account.password});
const credentials = await createBuilder<Array<WebAuthnCredentialMetadata>>(harness, account.token)
.get('/users/@me/mfa/webauthn/credentials')
.execute();
const sudoBody = await createSudoWebAuthnBody(harness, account.token, device);
await createBuilder(harness, account.token)
.delete(`/users/@me/mfa/webauthn/credentials/${credentials[0]!.id}`)
.body(sudoBody)
.expect(204)
.execute();
const stored = await readBackupCodes(harness, account.token, {password: account.password});
expect(stored).toHaveLength(0);
});
});
@@ -0,0 +1,181 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import {createAuthHarness, createTestAccount, loginUser, type TestAccount} from '@app/api/auth/tests/AuthTestUtils';
import {
createWebAuthnDevice,
registerWebAuthnCredential,
setWebAuthnTwoFactor,
type WebAuthnDevice,
} from '@app/api/auth/tests/WebAuthnTestUtils';
import {Config} from '@app/api/Config';
import type {ApiTestHarness} from '@app/api/test/ApiTestHarness';
import {HTTP_STATUS} from '@app/api/test/TestConstants';
import {createBuilder} from '@app/api/test/TestRequestBuilder';
import {UserAuthenticatorTypes} from '@fluxer/constants/src/UserConstants';
import {ValidationErrorCodes} from '@fluxer/constants/src/ValidationErrorCodes';
import {afterAll, beforeAll, beforeEach, describe, expect, it} from 'vitest';
interface PrivateUserResponse {
id: string;
mfa_enabled: boolean;
authenticator_types: Array<number>;
}
interface SudoMfaMethodsResponse {
totp: boolean;
webauthn: boolean;
backup_codes: boolean;
has_mfa: boolean;
}
interface SudoModeRequiredResponse {
code: string;
has_mfa?: boolean;
methods?: {
totp?: boolean;
webauthn?: boolean;
backup_codes?: boolean;
};
}
interface ValidationErrorBody {
code: string;
errors: Array<{path: string; code: string}>;
}
interface BackupCode {
code: string;
consumed: boolean;
}
async function withTotpReplayProtection<T>(run: () => Promise<T>): Promise<T> {
const previous = Config.dev.testModeEnabled;
Config.dev.testModeEnabled = false;
try {
return await run();
} finally {
Config.dev.testModeEnabled = previous;
}
}
async function createPasskeyOnlyTwoFactorAccount(
harness: ApiTestHarness,
): Promise<{account: TestAccount; device: WebAuthnDevice; backupCodes: Array<string>}> {
const account = await createTestAccount(harness);
const device = createWebAuthnDevice();
await registerWebAuthnCredential(harness, account.token, device, () => ({password: account.password}));
const enabled = await setWebAuthnTwoFactor(harness, account.token, true, {password: account.password});
expect(enabled.user.authenticator_types).toEqual([UserAuthenticatorTypes.WEBAUTHN]);
expect(enabled.backup_codes).not.toBeNull();
return {account, device, backupCodes: enabled.backup_codes!.map((backupCode) => backupCode.code)};
}
async function fetchMe(harness: ApiTestHarness, token: string): Promise<PrivateUserResponse> {
return createBuilder<PrivateUserResponse>(harness, token).get('/users/@me').execute();
}
describe('Sudo mode recovery for passkey two-factor accounts without TOTP', () => {
let harness: ApiTestHarness;
beforeAll(async () => {
harness = await createAuthHarness();
});
beforeEach(async () => {
await harness.reset();
});
afterAll(async () => {
await harness?.shutdown();
});
it('turns passkey two-factor off with a backup code when the passkey cannot be used', async () => {
const {account, backupCodes} = await createPasskeyOnlyTwoFactorAccount(harness);
await createBuilder(harness, account.token)
.put('/users/@me/mfa/webauthn/two-factor')
.body({enabled: false, password: account.password})
.expect(HTTP_STATUS.FORBIDDEN, 'SUDO_MODE_REQUIRED')
.execute();
await withTotpReplayProtection(async () => {
const disabled = await createBuilder<{user: PrivateUserResponse}>(harness, account.token)
.put('/users/@me/mfa/webauthn/two-factor')
.body({enabled: false, mfa_method: 'totp', mfa_code: backupCodes[0]!})
.expect(HTTP_STATUS.OK)
.execute();
expect(disabled.user.authenticator_types).toEqual([]);
expect(disabled.user.mfa_enabled).toBe(false);
});
const me = await fetchMe(harness, account.token);
expect(me.authenticator_types).toEqual([]);
expect(me.mfa_enabled).toBe(false);
const login = await loginUser(harness, {email: account.email, password: account.password});
expect('mfa' in login).toBe(false);
});
it('advertises the backup code option in the sudo methods endpoint and the sudo mode challenge alike', async () => {
const {account} = await createPasskeyOnlyTwoFactorAccount(harness);
const methods = await createBuilder<SudoMfaMethodsResponse>(harness, account.token)
.get('/users/@me/sudo/mfa-methods')
.execute();
expect(methods).toEqual({totp: false, webauthn: true, backup_codes: true, has_mfa: true});
const challenge = await createBuilder<SudoModeRequiredResponse>(harness, account.token)
.put('/users/@me/mfa/webauthn/two-factor')
.body({enabled: false})
.expect(HTTP_STATUS.FORBIDDEN, 'SUDO_MODE_REQUIRED')
.execute();
expect(challenge.has_mfa).toBe(methods.has_mfa);
expect(challenge.methods).toEqual({
totp: methods.totp,
webauthn: methods.webauthn,
backup_codes: methods.backup_codes,
});
});
it('spends a backup code accepted as a sudo proof and refuses the same code afterwards', async () => {
const {account, backupCodes} = await createPasskeyOnlyTwoFactorAccount(harness);
const spent = backupCodes[0]!;
await withTotpReplayProtection(async () => {
const stored = await createBuilder<{backup_codes: Array<BackupCode>}>(harness, account.token)
.post('/users/@me/mfa/backup-codes')
.body({regenerate: false, mfa_method: 'totp', mfa_code: spent})
.expect(HTTP_STATUS.OK)
.execute();
expect(stored.backup_codes.find((backupCode) => backupCode.code === spent)?.consumed).toBe(true);
const error = await createBuilder<ValidationErrorBody>(harness, account.token)
.put('/users/@me/mfa/webauthn/two-factor')
.body({enabled: false, mfa_method: 'totp', mfa_code: spent})
.expect(HTTP_STATUS.BAD_REQUEST, 'INVALID_FORM_BODY')
.execute();
expect(error.errors[0]?.path).toBe('mfa_code');
expect(error.errors[0]?.code).toBe(ValidationErrorCodes.INVALID_MFA_CODE);
});
const me = await fetchMe(harness, account.token);
expect(me.authenticator_types).toEqual([UserAuthenticatorTypes.WEBAUTHN]);
});
it('rejects a backup code that was never minted and leaves passkey two-factor on', async () => {
const {account} = await createPasskeyOnlyTwoFactorAccount(harness);
await withTotpReplayProtection(async () => {
const error = await createBuilder<ValidationErrorBody>(harness, account.token)
.put('/users/@me/mfa/webauthn/two-factor')
.body({enabled: false, mfa_method: 'totp', mfa_code: 'aaaa-bbbb'})
.expect(HTTP_STATUS.BAD_REQUEST, 'INVALID_FORM_BODY')
.execute();
expect(error.errors[0]?.path).toBe('mfa_code');
expect(error.errors[0]?.code).toBe(ValidationErrorCodes.INVALID_MFA_CODE);
});
const me = await fetchMe(harness, account.token);
expect(me.authenticator_types).toEqual([UserAuthenticatorTypes.WEBAUTHN]);
});
it('rejects a code-entry sudo proof for a passkey account that holds neither TOTP nor backup codes', async () => {
const account = await createTestAccount(harness);
const device = createWebAuthnDevice();
await registerWebAuthnCredential(harness, account.token, device, () => ({password: account.password}));
const methods = await createBuilder<SudoMfaMethodsResponse>(harness, account.token)
.get('/users/@me/sudo/mfa-methods')
.execute();
expect(methods).toEqual({totp: false, webauthn: true, backup_codes: false, has_mfa: true});
await withTotpReplayProtection(async () => {
const error = await createBuilder<ValidationErrorBody>(harness, account.token)
.post('/users/@me/disable')
.body({mfa_method: 'totp', mfa_code: 'aaaa-bbbb'})
.expect(HTTP_STATUS.BAD_REQUEST, 'INVALID_FORM_BODY')
.execute();
expect(error.errors[0]?.path).toBe('mfa_code');
expect(error.errors[0]?.code).toBe(ValidationErrorCodes.INVALID_MFA_CODE);
});
});
});
@@ -0,0 +1,126 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import {createAuthHarness, createTestAccount} from '@app/api/auth/tests/AuthTestUtils';
import {
createSudoWebAuthnBody,
createWebAuthnDevice,
registerWebAuthnCredential,
setWebAuthnTwoFactor,
type WebAuthnCredentialMetadata,
type WebAuthnTwoFactorResult,
} from '@app/api/auth/tests/WebAuthnTestUtils';
import type {ApiTestHarness} from '@app/api/test/ApiTestHarness';
import {HTTP_STATUS} from '@app/api/test/TestConstants';
import {createBuilder} from '@app/api/test/TestRequestBuilder';
import {UserAuthenticatorTypes} from '@fluxer/constants/src/UserConstants';
import {afterAll, beforeAll, beforeEach, describe, expect, it} from 'vitest';
interface PrivateUserResponse {
id: string;
mfa_enabled: boolean;
authenticator_types: Array<number>;
}
describe('WebAuthn two-factor toggle', () => {
let harness: ApiTestHarness;
beforeAll(async () => {
harness = await createAuthHarness();
});
beforeEach(async () => {
await harness.reset();
});
afterAll(async () => {
await harness?.shutdown();
});
it('reports an empty authenticator types array for an account with no second factor', async () => {
const account = await createTestAccount(harness);
const me = await createBuilder<PrivateUserResponse>(harness, account.token).get('/users/@me').execute();
expect(me.authenticator_types).toEqual([]);
expect(me.mfa_enabled).toBe(false);
});
it('rejects enabling passkey two-factor when the account has no registered credential', async () => {
const account = await createTestAccount(harness);
await createBuilder(harness, account.token)
.put('/users/@me/mfa/webauthn/two-factor')
.body({enabled: true, password: account.password})
.expect(HTTP_STATUS.BAD_REQUEST, 'NO_PASSKEYS_REGISTERED')
.execute();
const me = await createBuilder<PrivateUserResponse>(harness, account.token).get('/users/@me').execute();
expect(me.authenticator_types).toEqual([]);
});
it('round trips enabling and disabling passkey two-factor', async () => {
const account = await createTestAccount(harness);
const device = createWebAuthnDevice();
await registerWebAuthnCredential(harness, account.token, device, () => ({password: account.password}));
const enabled = await setWebAuthnTwoFactor(harness, account.token, true, {password: account.password});
expect(enabled.user.authenticator_types).toEqual([UserAuthenticatorTypes.WEBAUTHN]);
expect(enabled.user.mfa_enabled).toBe(true);
const afterEnable = await createBuilder<PrivateUserResponse>(harness, account.token).get('/users/@me').execute();
expect(afterEnable.authenticator_types).toEqual([UserAuthenticatorTypes.WEBAUTHN]);
const sudoBody = await createSudoWebAuthnBody(harness, account.token, device);
const disabled = await setWebAuthnTwoFactor(harness, account.token, false, sudoBody);
expect(disabled.user.authenticator_types).toEqual([]);
expect(disabled.user.mfa_enabled).toBe(false);
const afterDisable = await createBuilder<PrivateUserResponse>(harness, account.token).get('/users/@me').execute();
expect(afterDisable.authenticator_types).toEqual([]);
});
it('refuses to disable passkey two-factor without a sudo proof', async () => {
const account = await createTestAccount(harness);
const device = createWebAuthnDevice();
await registerWebAuthnCredential(harness, account.token, device, () => ({password: account.password}));
await setWebAuthnTwoFactor(harness, account.token, true, {password: account.password});
await createBuilder(harness, account.token)
.put('/users/@me/mfa/webauthn/two-factor')
.body({enabled: false})
.expect(HTTP_STATUS.FORBIDDEN, 'SUDO_MODE_REQUIRED')
.execute();
await createBuilder(harness, account.token)
.put('/users/@me/mfa/webauthn/two-factor')
.body({enabled: false, password: account.password})
.expect(HTTP_STATUS.FORBIDDEN, 'SUDO_MODE_REQUIRED')
.execute();
const me = await createBuilder<PrivateUserResponse>(harness, account.token).get('/users/@me').execute();
expect(me.authenticator_types).toEqual([UserAuthenticatorTypes.WEBAUTHN]);
});
it('accepts a passkey assertion as the sudo proof for disabling passkey two-factor', async () => {
const account = await createTestAccount(harness);
const device = createWebAuthnDevice();
await registerWebAuthnCredential(harness, account.token, device, () => ({password: account.password}));
await setWebAuthnTwoFactor(harness, account.token, true, {password: account.password});
const sudoBody = await createSudoWebAuthnBody(harness, account.token, device);
const disabled = await createBuilder<WebAuthnTwoFactorResult>(harness, account.token)
.put('/users/@me/mfa/webauthn/two-factor')
.body({enabled: false, ...sudoBody})
.execute();
expect(disabled.user.authenticator_types).toEqual([]);
});
it('leaves the account untouched when the requested state already matches', async () => {
const account = await createTestAccount(harness);
const device = createWebAuthnDevice();
await registerWebAuthnCredential(harness, account.token, device, () => ({password: account.password}));
const firstEnable = await setWebAuthnTwoFactor(harness, account.token, true, {password: account.password});
expect(firstEnable.backup_codes).not.toBeNull();
const sudoBody = await createSudoWebAuthnBody(harness, account.token, device);
const secondEnable = await setWebAuthnTwoFactor(harness, account.token, true, sudoBody);
expect(secondEnable.backup_codes).toBeNull();
expect(secondEnable.user.authenticator_types).toEqual([UserAuthenticatorTypes.WEBAUTHN]);
});
it('drops the passkey second factor when the last credential is deleted', async () => {
const account = await createTestAccount(harness);
const device = createWebAuthnDevice();
await registerWebAuthnCredential(harness, account.token, device, () => ({password: account.password}));
await setWebAuthnTwoFactor(harness, account.token, true, {password: account.password});
const credentials = await createBuilder<Array<WebAuthnCredentialMetadata>>(harness, account.token)
.get('/users/@me/mfa/webauthn/credentials')
.execute();
const sudoBody = await createSudoWebAuthnBody(harness, account.token, device);
await createBuilder(harness, account.token)
.delete(`/users/@me/mfa/webauthn/credentials/${credentials[0]!.id}`)
.body(sudoBody)
.expect(204)
.execute();
const me = await createBuilder<PrivateUserResponse>(harness, account.token).get('/users/@me').execute();
expect(me.authenticator_types).toEqual([]);
expect(me.mfa_enabled).toBe(false);
});
});
+1 -1
View File
@@ -150,7 +150,7 @@ function serializeGroupDMChannel(channel: Channel): ChannelResponse {
return {
...serializeBaseChannelFields(channel),
...serializeMessageableFields(channel),
name: channel.name ?? undefined,
name: channel.name ?? null,
icon: channel.iconHash ?? null,
owner_id: channel.ownerId ? channel.ownerId.toString() : null,
nicks: nicknameMap.size > 0 ? nicks : undefined,
@@ -24,6 +24,7 @@ import {deleteChannelMessageSearchDocuments} from '@app/api/search/MessageSearch
import type {IUserRepository} from '@app/api/user/IUserRepository';
import {serializeChannelForAudit} from '@app/api/utils/AuditSerializationUtils';
import {applyProtectedOverwriteBits} from '@app/api/utils/featureUtils';
import {overwriteGrantedBits} from '@app/api/utils/PermissionUtils';
import type {VoiceAvailabilityService} from '@app/api/voice/VoiceAvailabilityService';
import type {VoiceRegionAvailability} from '@app/api/voice/VoiceModel';
import type {IWebhookRepository} from '@app/api/webhook/IWebhookRepository';
@@ -208,25 +209,6 @@ export class ChannelOperationsService {
userId,
channelId: channel.id,
});
if (!isOwner) {
for (const overwrite of data.permission_overwrites ?? []) {
const allowPerms = (overwrite.allow ? BigInt(overwrite.allow) : 0n) & ALL_PERMISSIONS;
if ((allowPerms & ~channelPermissions) !== 0n) {
throw new MissingPermissionsError();
}
}
const nextDeny = new Map<RoleID | UserID, bigint>();
for (const overwrite of data.permission_overwrites ?? []) {
const targetKey = overwrite.type === 0 ? createRoleID(overwrite.id) : createUserID(overwrite.id);
nextDeny.set(targetKey, (overwrite.deny ? BigInt(overwrite.deny) : 0n) & ALL_PERMISSIONS);
}
for (const [targetId, existing] of previousPermissionOverwrites ?? []) {
const removedDeny = existing.deny & ~(nextDeny.get(targetId) ?? 0n);
if ((removedDeny & ~channelPermissions) !== 0n) {
throw new MissingPermissionsError();
}
}
}
permissionOverwrites = new Map();
for (const overwrite of data.permission_overwrites ?? []) {
const targetId = overwrite.type === 0 ? createRoleID(overwrite.id) : createUserID(overwrite.id);
@@ -251,6 +233,18 @@ export class ChannelOperationsService {
}),
);
}
if (!isOwner) {
const targetIds = new Set([...(previousPermissionOverwrites?.keys() ?? []), ...permissionOverwrites.keys()]);
for (const targetId of targetIds) {
const grantedBits = overwriteGrantedBits(
previousPermissionOverwrites?.get(targetId),
permissionOverwrites.get(targetId),
);
if ((grantedBits & ~channelPermissions) !== 0n) {
throw new MissingPermissionsError();
}
}
}
}
const requestedParentId =
data.parent_id !== undefined ? (data.parent_id ? createChannelID(data.parent_id) : null) : channel.parentId;
@@ -646,9 +640,8 @@ export class ChannelOperationsService {
const sanitizedAllow = protectedBits.allow;
const sanitizedDeny = protectedBits.deny;
const hasAdministrator = (userPermissions & Permissions.ADMINISTRATOR) !== 0n;
if (!hasAdministrator && (sanitizedAllow & ~userPermissions) !== 0n) throw new MissingPermissionsError();
const removedDeny = (existing?.deny ?? 0n) & ~sanitizedDeny;
if (!hasAdministrator && (removedDeny & ~userPermissions) !== 0n) throw new MissingPermissionsError();
const grantedBits = overwriteGrantedBits(existing, {allow: sanitizedAllow, deny: sanitizedDeny});
if (!hasAdministrator && (grantedBits & ~userPermissions) !== 0n) throw new MissingPermissionsError();
const previousPermissionOverwrites = channel.permissionOverwrites;
const nextOverwrite = new ChannelPermissionOverwrite({
type: params.overwrite.type,
@@ -294,6 +294,48 @@ describe('Channel Permission Overwrites', () => {
expect(overwrite?.allow).toBe(Permissions.VIEW_CHANNEL.toString());
expect(overwrite?.deny).toBe(Permissions.MANAGE_MESSAGES.toString());
});
test('should let an editor change an overwrite that already allows a permission they lack', async () => {
const {owner, members, guild, systemChannel} = await setupTestGuildWithMembers(harness, 1);
const manager = members[0];
const managerRole = await createRole(harness, owner.token, guild.id, {
name: 'Queue Manager',
permissions: Permissions.MANAGE_ROLES.toString(),
});
const botRole = await createRole(harness, owner.token, guild.id, {name: 'Bot'});
await addMemberRole(harness, owner.token, guild.id, manager.userId, managerRole.id);
await createPermissionOverwrite(harness, owner.token, systemChannel.id, botRole.id, {
type: 0,
allow: Permissions.PIN_MESSAGES.toString(),
deny: '0',
});
await createBuilder(harness, manager.token)
.put(`/channels/${systemChannel.id}/permissions/${botRole.id}`)
.body({
type: 0,
allow: (Permissions.PIN_MESSAGES | Permissions.SEND_MESSAGES).toString(),
deny: '0',
})
.expect(HTTP_STATUS.NO_CONTENT)
.execute();
const updated = await getChannel(harness, owner.token, systemChannel.id);
const botOverwrite = updated.permission_overwrites?.find((o) => o.id === botRole.id);
expect(botOverwrite?.allow).toBe((Permissions.PIN_MESSAGES | Permissions.SEND_MESSAGES).toString());
});
test('should reject an editor granting a permission they lack', async () => {
const {owner, members, guild, systemChannel} = await setupTestGuildWithMembers(harness, 1);
const manager = members[0];
const managerRole = await createRole(harness, owner.token, guild.id, {
name: 'Queue Manager',
permissions: Permissions.MANAGE_ROLES.toString(),
});
const botRole = await createRole(harness, owner.token, guild.id, {name: 'Bot'});
await addMemberRole(harness, owner.token, guild.id, manager.userId, managerRole.id);
await createBuilder(harness, manager.token)
.put(`/channels/${systemChannel.id}/permissions/${botRole.id}`)
.body({type: 0, allow: Permissions.PIN_MESSAGES.toString(), deny: '0'})
.expect(HTTP_STATUS.FORBIDDEN)
.execute();
});
test('should propagate category permission patches only to children that were synced when the category changed', async () => {
const {owner, guild} = await setupTestGuildWithMembers(harness, 0);
const targetRole = await createRole(harness, owner.token, guild.id, {name: 'Readers'});
@@ -0,0 +1,62 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import {createTestAccount} from '@app/api/auth/tests/AuthTestUtils';
import {createFriendship, createGroupDmChannel, getChannel} from '@app/api/channel/tests/ChannelTestUtils';
import {ensureSessionStarted} from '@app/api/message/tests/MessageTestUtils';
import {type ApiTestHarness, createApiTestHarness} from '@app/api/test/ApiTestHarness';
import {NoopGatewayService} from '@app/api/test/NoopGatewayService';
import {HTTP_STATUS} from '@app/api/test/TestConstants';
import {createBuilder} from '@app/api/test/TestRequestBuilder';
import type {ChannelResponse} from '@fluxer/schema/src/domains/channel/ChannelSchemas';
import {afterAll, beforeAll, beforeEach, describe, expect, it, vi} from 'vitest';
describe('Group DM name clear', () => {
let harness: ApiTestHarness;
beforeAll(async () => {
harness = await createApiTestHarness();
});
beforeEach(async () => {
await harness.reset();
});
afterAll(async () => {
await harness?.shutdown();
});
it.each([
['an empty string', ''],
['null', null],
])('sends a null name to every recipient when cleared with %s', async (_label, clearedName) => {
const user1 = await createTestAccount(harness);
const user2 = await createTestAccount(harness);
const user3 = await createTestAccount(harness);
await ensureSessionStarted(harness, user1.token);
await ensureSessionStarted(harness, user2.token);
await ensureSessionStarted(harness, user3.token);
await createFriendship(harness, user1, user2);
await createFriendship(harness, user1, user3);
const groupDm = await createGroupDmChannel(harness, user1.token, [user2.userId, user3.userId]);
await createBuilder<ChannelResponse>(harness, user1.token)
.patch(`/channels/${groupDm.id}`)
.body({name: 'Weekend plans'})
.expect(HTTP_STATUS.OK)
.execute();
const dispatchSpy = vi.spyOn(NoopGatewayService.prototype, 'dispatchPresence');
try {
const cleared = await createBuilder<ChannelResponse>(harness, user1.token)
.patch(`/channels/${groupDm.id}`)
.body({name: clearedName})
.expect(HTTP_STATUS.OK)
.execute();
expect(cleared).toHaveProperty('name', null);
const channelUpdates = dispatchSpy.mock.calls.filter(([params]) => params.event === 'CHANNEL_UPDATE');
expect(channelUpdates.map(([params]) => params.userId.toString()).sort()).toEqual(
[user1.userId, user2.userId, user3.userId].sort(),
);
for (const [params] of channelUpdates) {
expect(params.data).toHaveProperty('name', null);
}
} finally {
dispatchSpy.mockRestore();
}
expect(await getChannel(harness, user2.token, groupDm.id)).toHaveProperty('name', null);
});
});
-7
View File
@@ -2,7 +2,6 @@
import type {WorkerTaskName} from '@app/api/worker/WorkerLaneConfig';
import type {CachePurgeAdapterName} from '@fluxer/config/src/MasterConfig';
import type {ResolvedDownloadsProvider} from '@fluxer/config/src/S3DownloadsProvider';
export type APIWorkerMode = 'all_lanes' | 'single_lane' | 'single_task';
export type APIWorkerLaneName = 'realtime' | 'unfurl' | 'lifecycle' | 'batch';
@@ -48,7 +47,6 @@ export interface APIConfig {
requestTimeoutMs: number;
maxInflightRequests: number;
ipBanExemptIps: Array<string>;
desktopGitHubRedirectCountries: ReadonlySet<string>;
cassandra: {
hosts: string;
port: number;
@@ -145,8 +143,6 @@ export interface APIConfig {
donationProxyKey: string;
};
hosts: {
invite: string;
gift: string;
marketing: string;
unfurlIgnored: Array<string>;
};
@@ -171,10 +167,8 @@ export interface APIConfig {
uploads: string;
reports: string;
harvests: string;
downloads: string;
};
};
s3Downloads: ResolvedDownloadsProvider;
email: {
enabled: boolean;
provider: 'smtp' | 'none';
@@ -368,7 +362,6 @@ export interface APIConfig {
validateResponses: boolean;
};
presignedAttachmentUploadsEnabled: boolean;
presignedDownloadsEnabled: boolean;
presignedHarvestDownloadsEnabled: boolean;
attachmentDecayEnabled: boolean;
deletionGracePeriodHours: number;
@@ -1,5 +1,7 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import type {BannedFileShaRow} from '@app/api/database/types/AdminArchiveTypes';
export const BANNED_URLS_REFRESH_CHANNEL = 'banned_urls_refresh';
export const BANNED_URL_DOMAINS_REFRESH_CHANNEL = 'banned_url_domains_refresh';
export const BANNED_FILE_SHAS_REFRESH_CHANNEL = 'banned_file_shas_refresh';
@@ -23,3 +25,7 @@ export const ContentBlocklistCategory = {
GIFCT: 'gifct',
STOP_NCII: 'stop_ncii',
} as const;
export function isBlocklistFeedFileSha(row: Pick<BannedFileShaRow, 'category' | 'added_by'>): boolean {
return row.added_by == null && row.category === ContentBlocklistCategory.MALWARE_BAZAAR;
}
@@ -0,0 +1,89 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import fs from 'node:fs';
import path from 'node:path';
import {fileURLToPath} from 'node:url';
import {DEFAULT_TTL_TABLES} from '@app/api/database/PostgresKvDefaultTtlExpiry';
import * as DonationTables from '@app/api/donation/DonationTables';
import * as Tables from '@app/api/Tables';
import {IPINFO_CACHE_TTL_SECONDS, IPINFO_REQUEST_AUDIT_TTL_SECONDS} from '@pkgs/geoip/src/PostgresIpInfoKv';
import {describe, expect, it} from 'vitest';
const THIS_DIR = path.dirname(fileURLToPath(import.meta.url));
const REPO_ROOT = path.resolve(THIS_DIR, '../../../..');
interface SchemaTable {
name: string;
options: string;
}
const SCHEMA = JSON.parse(fs.readFileSync(path.join(REPO_ROOT, 'tools/dev/cassandra_target_schema.json'), 'utf8')) as {
tables: Array<SchemaTable>;
};
const SCHEMA_DEFAULTS = new Map<string, number>(
SCHEMA.tables.flatMap((table): Array<[string, number]> => {
const match = /default_time_to_live = (\d+)/.exec(table.options);
return match ? [[table.name, Number(match[1])]] : [];
}),
);
const DSL_TABLES = [...Object.values(Tables), ...Object.values(DonationTables)];
const DSL_NAMES = new Set<string>(DSL_TABLES.map((table) => table.name));
const NON_DSL_DEFAULTS: Record<string, number | null> = {
ipinfo_cache: IPINFO_CACHE_TTL_SECONDS,
ipinfo_requests_by_hour: IPINFO_REQUEST_AUDIT_TTL_SECONDS,
billing_webhook_events: null,
forensic_identifier_by_key_day: null,
forensic_identifier_by_request: null,
forensic_request_meta_by_actor_day: null,
forensic_request_meta_by_id: null,
forensic_request_meta_by_route_day_shard: null,
forensic_resource_exposure_by_request: null,
forensic_resource_exposure_by_route_day_shard: null,
forensic_resource_exposure_by_subject_day: null,
};
const OWN_EXPIRY_PASS = new Set(['jobs_by_id', 'jobs_by_day_bucket']);
function schemaDefault(name: string): number {
return SCHEMA_DEFAULTS.get(name) ?? 0;
}
function byName(left: {name: string}, right: {name: string}): number {
return left.name.localeCompare(right.name);
}
describe('Cassandra default TTL parity', () => {
it('declares every Cassandra default TTL on the matching table', () => {
const mismatches = DSL_TABLES.flatMap((table) => {
const declared = table.defaultTtlSeconds ?? 0;
return declared === schemaDefault(table.name)
? []
: [{table: table.name, declared, schema: schemaDefault(table.name)}];
});
expect(mismatches).toEqual([]);
});
it('declares a writer or no writer for every other table with a default', () => {
const undeclared = [...SCHEMA_DEFAULTS]
.filter(([name, ttl]) => ttl > 0 && !DSL_NAMES.has(name) && !Object.hasOwn(NON_DSL_DEFAULTS, name))
.map(([name]) => name);
expect(undeclared).toEqual([]);
const stale = Object.keys(NON_DSL_DEFAULTS).filter((name) => schemaDefault(name) === 0 || DSL_NAMES.has(name));
expect(stale).toEqual([]);
const mismatched = Object.entries(NON_DSL_DEFAULTS)
.filter(([name, ttl]) => ttl !== null && ttl !== schemaDefault(name))
.map(([name]) => name);
expect(mismatched).toEqual([]);
});
it('the Postgres expiry pass covers every table with a default except the job ledger', () => {
const expected = [...SCHEMA_DEFAULTS]
.filter(([name, ttl]) => ttl > 0 && NON_DSL_DEFAULTS[name] !== null && !OWN_EXPIRY_PASS.has(name))
.map(([name, ttl]) => ({name, defaultTtlSeconds: ttl}))
.sort(byName);
expect([...DEFAULT_TTL_TABLES].sort(byName)).toEqual(expected);
});
});
@@ -12,6 +12,7 @@ interface TableMetadata {
columns: ReadonlyArray<string>;
primaryKey: ReadonlyArray<string>;
partitionKey: ReadonlyArray<string>;
defaultTtlSeconds?: number;
}
const kvMetaRegistry = new Map<string, KvQueryMeta<Record<string, unknown>>>();
@@ -24,6 +25,7 @@ export function registerTableSpec<Row extends object>(tableSpec: KvTableSpec<Row
columns: tableSpec.columns as ReadonlyArray<string>,
primaryKey: tableSpec.primaryKey as ReadonlyArray<string>,
partitionKey: tableSpec.partitionKey as ReadonlyArray<string>,
defaultTtlSeconds: tableSpec.defaultTtlSeconds,
};
tableRegistry.set(tableSpec.name, metadata);
}
@@ -1,5 +1,6 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import {getTableMetadata} from '@app/api/database/CassandraMetaRegistry';
import {defineTable} from '@app/api/database/CassandraTableDsl';
import {Db, type PreparedQuery} from '@app/api/database/CassandraTypes';
import {describe, expect, it} from 'vitest';
@@ -76,3 +77,41 @@ describe('CassandraTableDsl select templates', () => {
expect(longQuery.cql).not.toContain('LIMIT 20');
});
});
describe('CassandraTableDsl default TTL', () => {
it('keeps the CQL of a table with a default TTL free of USING TTL', () => {
const DefaultTtlRows = defineTable<TtlHelperTestRow, 'id'>({
name: 'default_ttl_dsl_rows',
columns: ['id', 'value'],
primaryKey: ['id'],
defaultTtlSeconds: 600,
});
expect(DefaultTtlRows.defaultTtlSeconds).toBe(600);
const queries = [
DefaultTtlRows.insert({id: 'insert', value: 'a'}),
DefaultTtlRows.upsertAll({id: 'upsert', value: 'b'}),
DefaultTtlRows.patchByPk({id: 'patch'}, {value: Db.set('c')}),
];
for (const query of queries) {
expect(query.cql).not.toContain('USING TTL');
expect(query.kvMeta?.table.defaultTtlSeconds).toBe(600);
}
expect(getTableMetadata('default_ttl_dsl_rows')?.defaultTtlSeconds).toBe(600);
expect(TtlHelperTestRows.defaultTtlSeconds).toBeUndefined();
expect(getTableMetadata('ttl_helper_test_rows')?.defaultTtlSeconds).toBeUndefined();
});
it('rejects a default TTL of zero, a fraction or past the maximum', () => {
for (const defaultTtlSeconds of [0, 1.5, 630_720_001]) {
expect(() =>
defineTable<TtlHelperTestRow, 'id'>({
name: 'default_ttl_dsl_rejected_rows',
columns: ['id', 'value'],
primaryKey: ['id'],
defaultTtlSeconds,
}),
).toThrow();
}
expect(getTableMetadata('default_ttl_dsl_rejected_rows')).toBeUndefined();
});
});
@@ -83,6 +83,7 @@ export function defineTable<Row extends object, PK extends ColumnName<Row>, Part
columns: ReadonlyArray<ColumnName<Row>>;
primaryKey: ReadonlyArray<PK>;
partitionKey?: ReadonlyArray<PartKey>;
defaultTtlSeconds?: number;
}): Table<Row, PK, PartKey> {
const columns = [...def.columns];
const pk = [...def.primaryKey];
@@ -91,11 +92,15 @@ export function defineTable<Row extends object, PK extends ColumnName<Row>, Part
for (const c of columns) assertCqlIdentifier(c as string);
for (const k of pk) assertCqlIdentifier(k as string);
for (const k of partitionKey) assertCqlIdentifier(k as string);
if (def.defaultTtlSeconds !== undefined && validateTtlSeconds(def.defaultTtlSeconds) === 0) {
throw new Error(`Table "${def.name}" needs a positive default TTL`);
}
const tableSpec: KvTableSpec<Row> = {
name: def.name,
columns,
primaryKey: pk as ReadonlyArray<ColumnName<Row>>,
partitionKey: partitionKey as ReadonlyArray<ColumnName<Row>>,
defaultTtlSeconds: def.defaultTtlSeconds,
};
registerTableSpec(tableSpec);
const nonPkColumns = columns.filter((c) => !pk.includes(c as PK)) as Array<Exclude<ColumnName<Row>, PK>>;
@@ -685,6 +690,7 @@ WHERE ${pk.map((k) => `${k} = :${k}`).join(' AND ')};
columns: def.columns,
primaryKey: def.primaryKey,
partitionKey: partitionKey,
defaultTtlSeconds: def.defaultTtlSeconds,
selectCql,
select,
updateAllCql() {
@@ -56,6 +56,7 @@ export interface KvTableSpec<Row extends object = Record<string, unknown>> {
columns: ReadonlyArray<ColumnName<Row>>;
primaryKey: ReadonlyArray<ColumnName<Row>>;
partitionKey: ReadonlyArray<ColumnName<Row>>;
defaultTtlSeconds?: number;
}
export interface KvColumnParam<Row extends object = Record<string, unknown>> {
@@ -190,6 +191,7 @@ export interface Table<Row extends object, PK extends ColumnName<Row>, PartKey e
columns: ReadonlyArray<ColumnName<Row>>;
primaryKey: ReadonlyArray<PK>;
partitionKey: ReadonlyArray<PartKey>;
defaultTtlSeconds: number | undefined;
selectCql(opts?: {
columns?: ReadonlyArray<ColumnName<Row>>;
where?: WhereExpr<Row> | ReadonlyArray<WhereExpr<Row>>;
@@ -0,0 +1,490 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import {spawnSync} from 'node:child_process';
import {createServer} from 'node:net';
import {defineTable} from '@app/api/database/CassandraTableDsl';
import {Db} from '@app/api/database/CassandraTypes';
import {
DEFAULT_TTL_EXPIRY_RESUME,
DEFAULT_TTL_TABLES,
expireLegacyDefaultTtlRows,
} from '@app/api/database/PostgresKvDefaultTtlExpiry';
import {
ensurePostgresKvSchema,
PostgresKvQueryExecutor,
pruneExpiredPostgresKvRows,
} from '@app/api/database/PostgresKvQueryExecutor';
import {startDockerContainer} from '@app/api/test/DockerTestContainer';
import {
getDefaultPostgresClient,
type IPostgresClient,
initPostgres,
shutdownPostgres,
} from '@pkgs/postgres/src/Client';
import {afterAll, beforeAll, beforeEach, describe, expect, it} from 'vitest';
const KV_TABLE = 'kv_default_ttl';
const CONTAINER = `fluxer-kvttl-${process.pid.toString(36)}-${Date.now().toString(36)}`;
const dockerAvailable = spawnSync('docker', ['version'], {stdio: 'ignore'}).status === 0;
const DEFAULT_TTL_SECONDS = 600;
interface ProbeRow {
id: string;
value: string | null;
note: string | null;
}
interface OwnedProbeRow {
owner: string;
id: string;
value: string | null;
}
const DefaultTtlProbe = defineTable<ProbeRow, 'id'>({
name: 'default_ttl_probe',
columns: ['id', 'value', 'note'],
primaryKey: ['id'],
defaultTtlSeconds: DEFAULT_TTL_SECONDS,
});
const DefaultTtlProbeRows = defineTable<OwnedProbeRow, 'owner' | 'id', 'owner'>({
name: 'default_ttl_probe_rows',
columns: ['owner', 'id', 'value'],
primaryKey: ['owner', 'id'],
partitionKey: ['owner'],
defaultTtlSeconds: DEFAULT_TTL_SECONDS,
});
const NoTtlProbe = defineTable<ProbeRow, 'id'>({
name: 'no_ttl_probe',
columns: ['id', 'value', 'note'],
primaryKey: ['id'],
});
async function sleep(ms: number): Promise<void> {
await new Promise((resolve) => setTimeout(resolve, ms));
}
async function freePort(): Promise<number> {
return new Promise((resolve, reject) => {
const server = createServer();
server.on('error', reject);
server.listen(0, '127.0.0.1', () => {
const address = server.address();
if (typeof address === 'string' || address === null) {
reject(new Error('no port'));
return;
}
const port = address.port;
server.close(() => resolve(port));
});
});
}
function expectExpiresIn(value: Date | number | null, ttlSeconds: number): void {
expect(value).toBeInstanceOf(Date);
const remainingSeconds = ((value as Date).getTime() - Date.now()) / 1000;
expect(remainingSeconds).toBeGreaterThan(ttlSeconds - 60);
expect(remainingSeconds).toBeLessThanOrEqual(ttlSeconds);
}
describe.skipIf(!dockerAvailable)('Postgres KV default TTL', () => {
let raw: IPostgresClient;
let executor: PostgresKvQueryExecutor;
async function stored(table: string, id: string): Promise<{expires_at: Date | number | null; row_data: object}> {
const result = await raw.query<{expires_at: Date | number | null; row_data: object}>(
`SELECT expires_at, row_data FROM ${KV_TABLE} WHERE table_name = $1 AND row_data ->> 'id' = $2`,
[table, id],
);
expect(result.rows).toHaveLength(1);
return result.rows[0]!;
}
async function expiresAt(table: string, id: string): Promise<Date | number | null> {
return (await stored(table, id)).expires_at;
}
async function neverExpires(table: string, id: string): Promise<boolean> {
const result = await raw.query<{forever: boolean}>(
`SELECT expires_at = 'infinity'::timestamptz AS forever FROM ${KV_TABLE} WHERE table_name = $1 AND row_data ->> 'id' = $2`,
[table, id],
);
return result.rows[0]?.forever === true;
}
async function setExpiry(table: string, id: string, expression: string): Promise<void> {
await raw.query(
`UPDATE ${KV_TABLE} SET expires_at = ${expression} WHERE table_name = $1 AND row_data ->> 'id' = $2`,
[table, id],
);
}
async function seed(table: string, key: string, age: string, expires: Date | string | null = null): Promise<string> {
const result = await raw.query<{updated_at: string}>(
`INSERT INTO ${KV_TABLE} (table_name, partition_key, row_key, row_data, expires_at, updated_at)
VALUES ($1, $2, $2, '{}'::jsonb, $3::timestamptz, now() - $4::interval)
RETURNING updated_at::text`,
[table, key, expires, age],
);
return result.rows[0]!.updated_at;
}
async function remaining(): Promise<Array<{table_name: string; row_key: string}>> {
const result = await raw.query<{table_name: string; row_key: string}>(
`SELECT table_name, row_key FROM ${KV_TABLE} WHERE table_name <> '__fluxer_schema_migrations' ORDER BY table_name, row_key`,
);
return result.rows;
}
async function ageMarker(): Promise<void> {
await raw.query(
`UPDATE ${KV_TABLE} SET row_data = jsonb_build_object('applied_at', now() - interval '2 days') WHERE table_name = '__fluxer_schema_migrations' AND row_key = 'default_ttl_expiry_v1'`,
);
}
async function resumePoint(): Promise<object | null> {
const result = await raw.query<{row_data: object}>(
`SELECT row_data FROM ${KV_TABLE} WHERE table_name = '__fluxer_schema_migrations' AND row_key = $1`,
[DEFAULT_TTL_EXPIRY_RESUME],
);
return result.rows[0]?.row_data ?? null;
}
async function markerCount(): Promise<number> {
const result = await raw.query<{n: number}>(
`SELECT count(*)::int AS n FROM ${KV_TABLE} WHERE table_name = '__fluxer_schema_migrations' AND row_key = 'default_ttl_expiry_v1'`,
);
return result.rows[0]!.n;
}
beforeAll(async () => {
const port = await freePort();
startDockerContainer([
'run',
'-d',
'--name',
CONTAINER,
'-e',
'POSTGRES_USER=fluxer',
'-e',
'POSTGRES_PASSWORD=fluxer',
'-e',
'POSTGRES_DB=fluxer',
'-p',
`127.0.0.1:${port}:5432`,
'postgres:16-alpine',
'-c',
'fsync=off',
]);
let ready = false;
for (let attempt = 0; attempt < 180 && !ready; attempt += 1) {
await sleep(500);
const probe = spawnSync('docker', ['exec', CONTAINER, 'pg_isready', '-U', 'fluxer', '-d', 'fluxer'], {
stdio: 'ignore',
});
if (probe.status !== 0) continue;
try {
await initPostgres({
url: `postgres://fluxer:[email protected]:${port}/fluxer`,
maxConnections: 4,
kvTable: KV_TABLE,
});
await getDefaultPostgresClient().query('SELECT 1');
ready = true;
} catch {
await shutdownPostgres().catch(() => {});
}
}
if (!ready) throw new Error('postgres never came up');
raw = getDefaultPostgresClient();
await ensurePostgresKvSchema(raw);
executor = new PostgresKvQueryExecutor(raw);
}, 900_000);
beforeEach(async () => {
await raw.query(`DELETE FROM ${KV_TABLE}`);
});
afterAll(async () => {
await shutdownPostgres().catch(() => {});
spawnSync('docker', ['rm', '-f', CONTAINER], {stdio: 'ignore'});
});
it('gives every full-row write without a TTL the table default', async () => {
await executor.executeQuery(DefaultTtlProbe.insert({id: 'insert', value: 'a', note: null}));
await executor.executeQuery(DefaultTtlProbe.upsertAll({id: 'upsert', value: 'b', note: 'n'}));
expect(
await executor.executeQuery(DefaultTtlProbe.insertIfNotExists({id: 'claimed', value: 'c', note: null})),
).toEqual([{'[applied]': true}]);
expect(
await executor.executeQuery(
DefaultTtlProbeRows.conditionalBatch([{action: 'insert', row: {owner: 'o', id: 'batched', value: 'd'}}]),
),
).toEqual([{'[applied]': true}]);
for (const id of ['insert', 'upsert', 'claimed']) {
expectExpiresIn(await expiresAt('default_ttl_probe', id), DEFAULT_TTL_SECONDS);
}
expectExpiresIn(await expiresAt('default_ttl_probe_rows', 'batched'), DEFAULT_TTL_SECONDS);
});
it('keeps an explicit TTL ahead of the default', async () => {
await executor.executeQuery(DefaultTtlProbe.insertWithTtl({id: 'short', value: 'a', note: null}, 60));
expectExpiresIn(await expiresAt('default_ttl_probe', 'short'), 60);
await executor.executeQuery(DefaultTtlProbe.insert({id: 'patched', value: 'a', note: null}));
await executor.executeQuery(DefaultTtlProbe.patchByPkWithTtl({id: 'patched'}, {value: Db.set('b')}, 60));
expectExpiresIn(await expiresAt('default_ttl_probe', 'patched'), 60);
});
it('keeps an explicit TTL of zero as no expiry', async () => {
await executor.executeQuery(DefaultTtlProbe.insertWithTtl({id: 'forever', value: 'a', note: null}, 0));
expect(await neverExpires('default_ttl_probe', 'forever')).toBe(true);
expect(
await executor.executeQuery(
DefaultTtlProbe.select({where: DefaultTtlProbe.where.eq('id')}).bind({id: 'forever'}),
),
).toEqual([{id: 'forever', value: 'a', note: null}]);
await executor.executeQuery(DefaultTtlProbe.patchByPk({id: 'forever'}, {note: Db.set('patched')}));
expect(await neverExpires('default_ttl_probe', 'forever')).toBe(true);
await pruneExpiredPostgresKvRows(raw);
expect(await neverExpires('default_ttl_probe', 'forever')).toBe(true);
});
it('raises a patched row to the default but never lowers it', async () => {
await executor.executeQuery(DefaultTtlProbe.insertWithTtl({id: 'longer', value: 'a', note: null}, 3600));
await executor.executeQuery(DefaultTtlProbe.patchByPk({id: 'longer'}, {note: Db.set('patched')}));
expectExpiresIn(await expiresAt('default_ttl_probe', 'longer'), 3600);
await executor.executeQuery(DefaultTtlProbe.insert({id: 'soon', value: 'a', note: null}));
await setExpiry('default_ttl_probe', 'soon', "now() + interval '5 seconds'");
await executor.executeQuery(DefaultTtlProbe.patchByPk({id: 'soon'}, {note: Db.set('patched')}));
expectExpiresIn(await expiresAt('default_ttl_probe', 'soon'), DEFAULT_TTL_SECONDS);
await executor.executeQuery(DefaultTtlProbe.patchByPk({id: 'missing'}, {note: Db.set('created')}));
expectExpiresIn(await expiresAt('default_ttl_probe', 'missing'), DEFAULT_TTL_SECONDS);
await executor.executeQuery(DefaultTtlProbe.insert({id: 'unset', value: 'a', note: null}));
await setExpiry('default_ttl_probe', 'unset', 'NULL');
await executor.executeQuery(DefaultTtlProbe.patchByPk({id: 'unset'}, {note: Db.set('patched')}));
expectExpiresIn(await expiresAt('default_ttl_probe', 'unset'), DEFAULT_TTL_SECONDS);
await executor.executeQuery(DefaultTtlProbe.insert({id: 'expired', value: 'a', note: null}));
await setExpiry('default_ttl_probe', 'expired', "now() - interval '1 second'");
await executor.executeQuery(DefaultTtlProbe.patchByPk({id: 'expired'}, {note: Db.set('patched')}));
const revived = await stored('default_ttl_probe', 'expired');
expect(revived.row_data).toEqual({id: 'expired', note: 'patched'});
expectExpiresIn(revived.expires_at, DEFAULT_TTL_SECONDS);
});
it('raises conditional patches the same way', async () => {
await executor.executeQuery(DefaultTtlProbe.insert({id: 'soon', value: 'a', note: null}));
await setExpiry('default_ttl_probe', 'soon', "now() + interval '5 seconds'");
expect(
await executor.executeQuery(
DefaultTtlProbe.conditionalPatchByPk({id: 'soon'}, {note: Db.set('patched')}, {value: 'a'}),
),
).toEqual([{'[applied]': true}]);
expectExpiresIn(await expiresAt('default_ttl_probe', 'soon'), DEFAULT_TTL_SECONDS);
await executor.executeQuery(DefaultTtlProbe.insertWithTtl({id: 'longer', value: 'a', note: null}, 3600));
expect(
await executor.executeQuery(
DefaultTtlProbe.conditionalPatchByPk({id: 'longer'}, {note: Db.set('patched')}, {value: 'a'}),
),
).toEqual([{'[applied]': true}]);
expectExpiresIn(await expiresAt('default_ttl_probe', 'longer'), 3600);
await executor.executeQuery(DefaultTtlProbeRows.insert({owner: 'o', id: 'existing', value: 'old'}));
await setExpiry('default_ttl_probe_rows', 'existing', 'NULL');
expect(
await executor.executeQuery(
DefaultTtlProbeRows.conditionalBatch([
{action: 'insert', row: {owner: 'o', id: 'added', value: 'new'}},
{
action: 'patch',
pk: {owner: 'o', id: 'existing'},
patch: {value: Db.set('updated')},
expected: {value: 'old'},
},
]),
),
).toEqual([{'[applied]': true}]);
expectExpiresIn(await expiresAt('default_ttl_probe_rows', 'added'), DEFAULT_TTL_SECONDS);
expectExpiresIn(await expiresAt('default_ttl_probe_rows', 'existing'), DEFAULT_TTL_SECONDS);
});
it('leaves tables without a default untouched', async () => {
await executor.executeQuery(NoTtlProbe.insert({id: 'plain', value: 'a', note: null}));
expect(await expiresAt('no_ttl_probe', 'plain')).toBeNull();
await executor.executeQuery(NoTtlProbe.patchByPk({id: 'plain'}, {note: Db.set('patched')}));
expect(await expiresAt('no_ttl_probe', 'plain')).toBeNull();
await executor.executeQuery(NoTtlProbe.insertWithTtl({id: 'zero', value: 'a', note: null}, 0));
expect(await expiresAt('no_ttl_probe', 'zero')).toBeNull();
});
it('gives rows an older image wrote the expiry of their last write and deletes the ones past it', async () => {
const mentionWrittenAt = await seed('recent_mentions', 'rm-day', '1 day');
await seed('recent_mentions', 'rm-week', '8 days');
await seed('attachment_upload_traces_by_key', 'at-31', '31 days');
await seed('attachment_upload_traces_by_key', 'at-29', '29 days');
await seed('phone_lookup_cache', 'pl-8', '8 days');
await seed('donor_magic_link_tokens', 'dm-hour', '1 hour');
await seed('ipinfo_requests_by_hour', 'ip-day', '1 day');
await seed('jobs_by_id', 'job', '100 days');
await seed('users', 'user', '100 days');
await seed('recent_mentions', 'rm-forever', '1 day', 'infinity');
await seed('recent_mentions', 'rm-hour', '30 days', new Date(Date.now() + 3_600_000));
expect(await expireLegacyDefaultTtlRows(raw, Date.now() + 60_000)).toEqual({
deleted: 4,
expiring: 3,
complete: true,
});
expect(await remaining()).toEqual([
{table_name: 'attachment_upload_traces_by_key', row_key: 'at-29'},
{table_name: 'ipinfo_requests_by_hour', row_key: 'ip-day'},
{table_name: 'jobs_by_id', row_key: 'job'},
{table_name: 'recent_mentions', row_key: 'rm-day'},
{table_name: 'recent_mentions', row_key: 'rm-forever'},
{table_name: 'recent_mentions', row_key: 'rm-hour'},
{table_name: 'users', row_key: 'user'},
]);
const exact = await raw.query<{row_key: string; exact: boolean; unchanged: boolean | null}>(
`SELECT row_key,
expires_at = updated_at + CASE table_name WHEN 'recent_mentions' THEN interval '7 days' WHEN 'attachment_upload_traces_by_key' THEN interval '30 days' ELSE interval '90 days' END AS exact,
CASE WHEN row_key = 'rm-day' THEN updated_at = $1::timestamptz END AS unchanged
FROM ${KV_TABLE}
WHERE row_key IN ('rm-day', 'at-29', 'ip-day')
ORDER BY row_key`,
[mentionWrittenAt],
);
expect(exact.rows).toEqual([
{row_key: 'at-29', exact: true, unchanged: null},
{row_key: 'ip-day', exact: true, unchanged: null},
{row_key: 'rm-day', exact: true, unchanged: true},
]);
const untouched = await raw.query<{row_key: string; state: string}>(
`SELECT row_key, CASE WHEN expires_at IS NULL THEN 'unset' WHEN expires_at = 'infinity' THEN 'forever' ELSE 'set' END AS state
FROM ${KV_TABLE}
WHERE row_key IN ('job', 'user', 'rm-forever', 'rm-hour')
ORDER BY row_key`,
);
expect(untouched.rows).toEqual([
{row_key: 'job', state: 'unset'},
{row_key: 'rm-forever', state: 'forever'},
{row_key: 'rm-hour', state: 'set'},
{row_key: 'user', state: 'unset'},
]);
expect(await markerCount()).toBe(0);
expect(await expireLegacyDefaultTtlRows(raw, Date.now() + 60_000)).toEqual({
deleted: 0,
expiring: 0,
complete: true,
});
expect(await markerCount()).toBe(1);
expect(await expireLegacyDefaultTtlRows(raw, Date.now() + 60_000)).toBeNull();
});
it('checks again a day after a clean pass', async () => {
expect(await expireLegacyDefaultTtlRows(raw, Date.now() + 60_000)).toEqual({
deleted: 0,
expiring: 0,
complete: true,
});
expect(await expireLegacyDefaultTtlRows(raw, Date.now() + 60_000)).toBeNull();
await seed('recent_mentions', 'rm-rolled-back', '1 day');
expect(await expireLegacyDefaultTtlRows(raw, Date.now() + 60_000)).toBeNull();
const before = await raw.query(`SELECT expires_at FROM ${KV_TABLE} WHERE row_key = 'rm-rolled-back'`);
expect(before.rows).toEqual([{expires_at: null}]);
await ageMarker();
expect(await expireLegacyDefaultTtlRows(raw, Date.now() + 60_000)).toEqual({
deleted: 0,
expiring: 1,
complete: true,
});
expect(await expireLegacyDefaultTtlRows(raw, Date.now() + 60_000)).toEqual({
deleted: 0,
expiring: 0,
complete: true,
});
expect(await expireLegacyDefaultTtlRows(raw, Date.now() + 60_000)).toBeNull();
});
it('pages through more rows than one page holds and stops at its deadline', async () => {
await raw.query(
`INSERT INTO ${KV_TABLE} (table_name, partition_key, row_key, row_data, updated_at)
SELECT 'recent_mentions', 'rm-' || lpad(g::text, 5, '0'), 'rm-' || lpad(g::text, 5, '0'), '{}'::jsonb, now() - interval '1 day'
FROM generate_series(1, 2300) g`,
);
expect(await expireLegacyDefaultTtlRows(raw, Date.now() - 1)).toEqual({
deleted: 0,
expiring: 0,
complete: false,
});
expect(await markerCount()).toBe(0);
expect(await expireLegacyDefaultTtlRows(raw, Date.now() + 60_000)).toEqual({
deleted: 0,
expiring: 2300,
complete: true,
});
const unset = await raw.query<{n: number}>(
`SELECT count(*)::int AS n FROM ${KV_TABLE} WHERE table_name = 'recent_mentions' AND expires_at IS NULL`,
);
expect(unset.rows[0]).toEqual({n: 0});
expect(await expireLegacyDefaultTtlRows(raw, Date.now() + 60_000)).toEqual({
deleted: 0,
expiring: 0,
complete: true,
});
expect(await expireLegacyDefaultTtlRows(raw, Date.now() + 60_000)).toBeNull();
});
it('saves where a run stopped and starts the next run there', async () => {
const first = DEFAULT_TTL_TABLES[0]!.name;
const last = DEFAULT_TTL_TABLES.at(-1)!.name;
await seed(first, 'a', '1 hour');
await seed(first, 'z', '1 hour');
await seed(last, 'k', '1 hour');
expect(await expireLegacyDefaultTtlRows(raw, Date.now() - 1)).toEqual({deleted: 0, expiring: 0, complete: false});
expect(await resumePoint()).toEqual({table: first, row_key: '', unset: 0});
await raw.query(
`UPDATE ${KV_TABLE} SET row_data = jsonb_build_object('table', $1::text, 'row_key', 'm', 'unset', 0) WHERE table_name = '__fluxer_schema_migrations' AND row_key = $2`,
[first, DEFAULT_TTL_EXPIRY_RESUME],
);
expect(await expireLegacyDefaultTtlRows(raw, Date.now() + 60_000)).toEqual({
deleted: 0,
expiring: 2,
complete: true,
});
const untouched = await raw.query<{expires_at: Date | null}>(
`SELECT expires_at FROM ${KV_TABLE} WHERE table_name = $1 AND row_key = 'a'`,
[first],
);
expect(untouched.rows).toEqual([{expires_at: null}]);
expect(await resumePoint()).toBeNull();
expect(await markerCount()).toBe(0);
expect(await expireLegacyDefaultTtlRows(raw, Date.now() + 60_000)).toEqual({
deleted: 0,
expiring: 1,
complete: true,
});
expect(await expireLegacyDefaultTtlRows(raw, Date.now() + 60_000)).toEqual({
deleted: 0,
expiring: 0,
complete: true,
});
expect(await markerCount()).toBe(1);
});
});
@@ -0,0 +1,142 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import {
POSTGRES_KV_MIGRATION_TABLE,
postgresKvPassIsFresh,
recordPostgresKvCleanPass,
} from '@app/api/database/PostgresKvQueryExecutor';
import * as DonationTables from '@app/api/donation/DonationTables';
import * as Tables from '@app/api/Tables';
import {IPINFO_CACHE_TTL_SECONDS, IPINFO_REQUEST_AUDIT_TTL_SECONDS} from '@pkgs/geoip/src/PostgresIpInfoKv';
import {type IPostgresClient, quoteIdentifier} from '@pkgs/postgres/src/Client';
import {ms} from 'itty-time';
const DEFAULT_TTL_EXPIRY_MARKER = 'default_ttl_expiry_v1';
export const DEFAULT_TTL_EXPIRY_RESUME = 'default_ttl_expiry_v1_resume';
const PAGE_SIZE = 2000;
const CLEAN_PASS_INTERVAL_MS = ms('1 day');
const OWN_EXPIRY_PASS = new Set<string>([Tables.JobsById.name, Tables.JobsByDayBucket.name]);
export const DEFAULT_TTL_TABLES: ReadonlyArray<{name: string; defaultTtlSeconds: number}> = [
...[...Object.values(Tables), ...Object.values(DonationTables)].flatMap((table) =>
table.defaultTtlSeconds === undefined || OWN_EXPIRY_PASS.has(table.name)
? []
: [{name: table.name, defaultTtlSeconds: table.defaultTtlSeconds}],
),
{name: 'ipinfo_cache', defaultTtlSeconds: IPINFO_CACHE_TTL_SECONDS},
{name: 'ipinfo_requests_by_hour', defaultTtlSeconds: IPINFO_REQUEST_AUDIT_TTL_SECONDS},
];
export interface LegacyDefaultTtlExpiryResult {
deleted: number;
expiring: number;
complete: boolean;
}
interface ResumePoint {
table: string;
rowKey: string;
unset: number;
}
async function readResumePoint(client: IPostgresClient, kvTable: string): Promise<ResumePoint | null> {
const result = await client.query<{row_data: Record<string, unknown>}>(
`SELECT row_data FROM ${kvTable} WHERE table_name = $1 AND row_key = $2`,
[POSTGRES_KV_MIGRATION_TABLE, DEFAULT_TTL_EXPIRY_RESUME],
);
const data = result.rows[0]?.row_data;
if (typeof data?.table !== 'string' || typeof data.row_key !== 'string' || typeof data.unset !== 'number') {
return null;
}
return {table: data.table, rowKey: data.row_key, unset: data.unset};
}
async function writeResumePoint(client: IPostgresClient, kvTable: string, point: ResumePoint | null): Promise<void> {
if (point === null) {
await client.query(`DELETE FROM ${kvTable} WHERE table_name = $1 AND row_key = $2`, [
POSTGRES_KV_MIGRATION_TABLE,
DEFAULT_TTL_EXPIRY_RESUME,
]);
return;
}
await client.query(
`INSERT INTO ${kvTable} (table_name, partition_key, row_key, row_data)
VALUES ($1, $2, $2, jsonb_build_object('table', $3::text, 'row_key', $4::text, 'unset', $5::bigint))
ON CONFLICT (table_name, row_key) DO UPDATE SET row_data = EXCLUDED.row_data, updated_at = now()`,
[POSTGRES_KV_MIGRATION_TABLE, DEFAULT_TTL_EXPIRY_RESUME, point.table, point.rowKey, point.unset],
);
}
function pageSql(table: string): string {
return `
WITH page AS (
SELECT kv.row_key, kv.expires_at IS NULL AS unset
FROM ${table} kv
WHERE kv.table_name = $1 AND kv.row_key > $2
ORDER BY kv.row_key
LIMIT $3
), removed AS (
DELETE FROM ${table} kv
USING page
WHERE kv.table_name = $1 AND kv.row_key = page.row_key AND kv.expires_at IS NULL
AND kv.updated_at + make_interval(secs => $4::double precision) <= now()
RETURNING 1
), expiring AS (
UPDATE ${table} kv
SET expires_at = kv.updated_at + make_interval(secs => $4::double precision)
FROM page
WHERE kv.table_name = $1 AND kv.row_key = page.row_key AND kv.expires_at IS NULL
AND kv.updated_at + make_interval(secs => $4::double precision) > now()
RETURNING 1
)
SELECT
(SELECT max(row_key) FROM page) AS last_row_key,
(SELECT count(*) FROM page WHERE unset) AS unset,
(SELECT count(*) FROM removed) AS deleted,
(SELECT count(*) FROM expiring) AS expiring`;
}
export async function expireLegacyDefaultTtlRows(
client: IPostgresClient,
deadlineMs: number,
): Promise<LegacyDefaultTtlExpiryResult | null> {
if (await postgresKvPassIsFresh(client, DEFAULT_TTL_EXPIRY_MARKER, CLEAN_PASS_INTERVAL_MS)) {
return null;
}
const kvTable = quoteIdentifier(client.kvTable());
const sql = pageSql(kvTable);
const resume = await readResumePoint(client, kvTable);
const resumeIndex = resume === null ? -1 : DEFAULT_TTL_TABLES.findIndex((target) => target.name === resume.table);
let unset = resumeIndex < 0 ? 0 : resume!.unset;
let deleted = 0;
let expiring = 0;
for (let index = Math.max(resumeIndex, 0); index < DEFAULT_TTL_TABLES.length; index += 1) {
const target = DEFAULT_TTL_TABLES[index]!;
let cursor = index === resumeIndex ? resume!.rowKey : '';
for (;;) {
if (Date.now() >= deadlineMs) {
await writeResumePoint(client, kvTable, {table: target.name, rowKey: cursor, unset});
return {deleted, expiring, complete: false};
}
const result = await client.query<{
last_row_key: string | null;
unset: string;
deleted: string;
expiring: string;
}>(sql, [target.name, cursor, PAGE_SIZE, target.defaultTtlSeconds]);
const page = result.rows[0];
if (!page || page.last_row_key === null) {
break;
}
unset += Number(page.unset);
deleted += Number(page.deleted);
expiring += Number(page.expiring);
cursor = page.last_row_key;
}
}
await writeResumePoint(client, kvTable, null);
if (unset === 0) {
await recordPostgresKvCleanPass(client, DEFAULT_TTL_EXPIRY_MARKER);
}
return {deleted, expiring, complete: true};
}
@@ -89,6 +89,28 @@ const NUMERIC_ROW_KEY_NUMBER_PATTERN = '^(-?[0-9]+(?:\\.[0-9]+)?(?:[eE][-+]?[0-9
const EXPIRED_STORED_ROW = 'kv.expires_at IS NOT NULL AND kv.expires_at <= now()';
const MERGED_ROW_DATA = `CASE WHEN ${EXPIRED_STORED_ROW} THEN EXCLUDED.row_data ELSE kv.row_data || EXCLUDED.row_data END`;
const KEPT_EXPIRES_AT = `CASE WHEN ${EXPIRED_STORED_ROW} THEN NULL ELSE kv.expires_at END`;
const NO_EXPIRY = 'infinity';
export async function postgresKvPassIsFresh(
client: IPostgresClient,
marker: string,
maxAgeMs: number,
): Promise<boolean> {
const result = await client.query(
`SELECT 1 FROM ${quoteIdentifier(client.kvTable())} WHERE table_name = $1 AND row_key = $2 AND (row_data ->> 'applied_at')::timestamptz > now() - make_interval(secs => $3::double precision)`,
[POSTGRES_KV_MIGRATION_TABLE, marker, maxAgeMs / 1000],
);
return result.rows.length > 0;
}
export async function recordPostgresKvCleanPass(client: IPostgresClient, marker: string): Promise<void> {
await client.query(
`INSERT INTO ${quoteIdentifier(client.kvTable())} (table_name, partition_key, row_key, row_data)
VALUES ($1, $2, $2, jsonb_build_object('applied_at', now()))
ON CONFLICT (table_name, row_key) DO UPDATE SET row_data = EXCLUDED.row_data, updated_at = now()`,
[POSTGRES_KV_MIGRATION_TABLE, marker],
);
}
function numericRowKeyExpr(column: string): string {
return `(COALESCE(substring(${column} from '${NUMERIC_ROW_KEY_BIGINT_PATTERN}'), substring(${column} from '${NUMERIC_ROW_KEY_NUMBER_PATTERN}'))::numeric)`;
@@ -333,20 +355,21 @@ function projectRow(row: Row, columns: ReadonlyArray<string> | undefined): Row {
return projected;
}
function rowComparator(meta: KvQueryMeta): (left: Row, right: Row) => number {
if (meta.orderBy) {
const column = meta.orderBy.col as string;
const direction = meta.orderBy.direction === 'DESC' ? -1 : 1;
return (left, right) => compareValues(left[column], right[column]) * direction;
function compareColumns(columns: ReadonlyArray<string>, left: Row, right: Row): number {
for (const column of columns) {
const cmp = compareValues(left[column], right[column]);
if (cmp !== 0) return cmp;
}
const columns = meta.table.primaryKey as ReadonlyArray<string>;
return (left, right) => {
for (const column of columns) {
const cmp = compareValues(left[column], right[column]);
if (cmp !== 0) return cmp;
}
return 0;
};
return 0;
}
function rowComparator(meta: KvQueryMeta): (left: Row, right: Row) => number {
const primaryKey = meta.table.primaryKey as ReadonlyArray<string>;
if (!meta.orderBy) return (left, right) => compareColumns(primaryKey, left, right);
const column = meta.orderBy.col as string;
const columns = [column, ...primaryKey.slice(primaryKey.indexOf(column) + 1)];
const direction = meta.orderBy.direction === 'DESC' ? -1 : 1;
return (left, right) => compareColumns(columns, left, right) * direction;
}
function sortRows(meta: KvQueryMeta, rows: Array<Row>): Array<Row> {
@@ -679,7 +702,7 @@ function logFullScan(meta: KvQueryMeta): void {
logWarn({table: meta.table.name, action: meta.action, where: shape.summary || 'none'}, 'Postgres KV full table scan');
}
function ttlExpiresAt(meta: KvQueryMeta, params: CassandraParams): Date | null | undefined {
function ttlExpiresAt(meta: KvQueryMeta, params: CassandraParams): Date | typeof NO_EXPIRY | null | undefined {
const ttlParam = meta.ttlParamName;
if (!ttlParam) return undefined;
const ttlRaw = params[ttlParam];
@@ -687,7 +710,13 @@ function ttlExpiresAt(meta: KvQueryMeta, params: CassandraParams): Date | null |
throw new Error(`TTL parameter ${ttlParam} must be a number`);
}
const ttlSeconds = validateTtlSeconds(ttlRaw);
return ttlSeconds === 0 ? null : new Date(Date.now() + ttlSeconds * 1000);
if (ttlSeconds === 0) return meta.table.defaultTtlSeconds === undefined ? null : NO_EXPIRY;
return new Date(Date.now() + ttlSeconds * 1000);
}
function defaultExpiresAt(meta: KvQueryMeta): Date | undefined {
const ttlSeconds = meta.table.defaultTtlSeconds;
return ttlSeconds === undefined ? undefined : new Date(Date.now() + ttlSeconds * 1000);
}
function encodePageState(pageState: PageState): string {
@@ -1191,7 +1220,8 @@ export class PostgresKvQueryExecutor {
'kv_del_expired',
);
}
const expiresAt = ttlExpiresAt(meta, params) ?? null;
const explicit = ttlExpiresAt(meta, params);
const expiresAt = explicit === undefined ? (defaultExpiresAt(meta) ?? null) : explicit;
const result = await db.query(
`INSERT INTO ${this.table} AS kv (table_name, partition_key, row_key, row_data, expires_at, updated_at)
VALUES ($1, $2, $3, $4::jsonb, $5, now())
@@ -1244,10 +1274,14 @@ WHERE NOT $6`,
}
bindings.push(JSON.stringify(encodeRow(paramsRow(params, meta.patchKeys))));
const assignments = [`row_data = kv.row_data || $${bindings.length}::jsonb`, 'updated_at = now()'];
const expiresAt = ttlExpiresAt(meta, params);
if (expiresAt !== undefined) {
bindings.push(expiresAt);
const explicit = ttlExpiresAt(meta, params);
const fallback = explicit === undefined ? defaultExpiresAt(meta) : undefined;
if (explicit !== undefined) {
bindings.push(explicit);
assignments.push(`expires_at = $${bindings.length}`);
} else if (fallback !== undefined) {
bindings.push(fallback);
assignments.push(`expires_at = GREATEST(kv.expires_at, $${bindings.length}::timestamptz)`);
}
sql = `UPDATE ${this.table} kv SET ${assignments.join(', ')} WHERE ${where}`;
}
@@ -1346,15 +1380,27 @@ WHERE NOT $6`,
for (const column of meta.patchKeys ?? []) {
incoming[column] = column in params ? params[column] : null;
}
const ttl = ttlExpiresAt(meta, params);
const expiresAtExpr = ttl === undefined ? KEPT_EXPIRES_AT : 'EXCLUDED.expires_at';
const explicit = ttlExpiresAt(meta, params);
const fallback = explicit === undefined ? defaultExpiresAt(meta) : undefined;
const [expiresAtExpr, statementName] =
explicit !== undefined
? ['EXCLUDED.expires_at', 'kv_patch_set_ttl']
: fallback !== undefined
? ['GREATEST(kv.expires_at, EXCLUDED.expires_at)', 'kv_patch_default_ttl']
: [KEPT_EXPIRES_AT, 'kv_patch_keep_ttl'];
await db.query(
`INSERT INTO ${this.table} AS kv (table_name, partition_key, row_key, row_data, expires_at, updated_at)
VALUES ($1, $2, $3, $4::jsonb, $5, now())
ON CONFLICT (table_name, row_key)
DO UPDATE SET partition_key = EXCLUDED.partition_key, row_data = ${MERGED_ROW_DATA}, expires_at = ${expiresAtExpr}, updated_at = now()`,
[meta.table.name, partitionKey(meta, incoming), key, JSON.stringify(encodeRow(incoming)), ttl ?? null],
ttl === undefined ? 'kv_patch_keep_ttl' : 'kv_patch_set_ttl',
[
meta.table.name,
partitionKey(meta, incoming),
key,
JSON.stringify(encodeRow(incoming)),
explicit ?? fallback ?? null,
],
statementName,
);
}
@@ -52,6 +52,8 @@ const Composite: KvTableSpec<Row> = {
partitionKey: ['owner_id'],
};
const Expiring: KvTableSpec<Row> = {...Composite, name: 'stmt_expiring', defaultTtlSeconds: 600};
const Bucketed: KvTableSpec<Row> = {
name: 'stmt_bucketed',
columns: ['bucket', 'item_id', 'payload'],
@@ -118,6 +120,7 @@ async function runShapes(): Promise<Array<Statement>> {
meta(Composite, 'patch', [eq('owner_id'), eq('item_id')], {patchKeys: ['payload'], ttlParamName: 'ttl_'}),
{...OWNER_ITEM, ttl_: 600} as CassandraParams,
],
[meta(Expiring, 'patch', [eq('owner_id'), eq('item_id')], {patchKeys: ['payload']}), OWNER_ITEM],
];
for (const [kvMeta, params] of cases) {
await executor.executeQuery({cql: `__stmt_${kvMeta.action}`, params, kvMeta: kvMeta as KvQueryMeta});
@@ -142,6 +145,7 @@ describe('PostgresKvQueryExecutor statement names', () => {
'kv_del_keys',
'kv_del_rowkeys',
'kv_get_row',
'kv_patch_default_ttl',
'kv_patch_keep_ttl',
'kv_patch_set_ttl',
'kv_sel_range',
@@ -231,6 +235,17 @@ async function exerciseKvShapes(executor: PostgresKvQueryExecutor): Promise<void
kvMeta: meta(Composite, 'select', [eq('owner_id'), eq('item_id')]) as KvQueryMeta,
});
expect(patched.map((row) => row.payload)).toEqual(['patched']);
await executor.executeQuery({
cql: '__stmt_patch_default_ttl',
params: {owner_id: 'o5', item_id: 'i5', payload: 'defaulted'} as CassandraParams,
kvMeta: meta(Expiring, 'patch', [eq('owner_id'), eq('item_id')], {patchKeys: ['payload']}) as KvQueryMeta,
});
const defaulted = await executor.executeQuery<Row>({
cql: '__stmt_point',
params: {owner_id: 'o5', item_id: 'i5'} as CassandraParams,
kvMeta: meta(Expiring, 'select', [eq('owner_id'), eq('item_id')]) as KvQueryMeta,
});
expect(defaulted.map((row) => row.payload)).toEqual(['defaulted']);
await executor.executeQuery({
cql: '__stmt_delete',
params: {owner_id: 'o0', item_id: 'i0'} as CassandraParams,
@@ -323,6 +338,7 @@ describe.skipIf(!dockerAvailable)('PostgresKvQueryExecutor statement names again
'kv_del_expired',
'kv_del_rowkeys',
'kv_get_row',
'kv_patch_default_ttl',
'kv_patch_keep_ttl',
'kv_patch_set_ttl',
'kv_sel_range',
@@ -0,0 +1,90 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import {spawnSync} from 'node:child_process';
import {createServer} from 'node:net';
import {startDockerContainer} from '@app/api/test/DockerTestContainer';
import {getDefaultPostgresClient, initPostgres, shutdownPostgres} from '@pkgs/postgres/src/Client';
import {afterAll, beforeAll, describe, expect, it} from 'vitest';
const CONTAINER = `fluxer-kvscram-${process.pid.toString(36)}-${Date.now().toString(36)}`;
const dockerAvailable = spawnSync('docker', ['version'], {stdio: 'ignore'}).status === 0;
const SCRAM_ITERATIONS = 200_000;
async function sleep(ms: number): Promise<void> {
await new Promise((resolve) => setTimeout(resolve, ms));
}
async function freePort(): Promise<number> {
return new Promise((resolve, reject) => {
const server = createServer();
server.on('error', reject);
server.listen(0, '127.0.0.1', () => {
const address = server.address();
if (typeof address === 'string' || address === null) {
reject(new Error('no port'));
return;
}
const port = address.port;
server.close(() => resolve(port));
});
});
}
describe.skipIf(!dockerAvailable)('postgres client against a server with raised SCRAM iterations', () => {
let port: number;
beforeAll(async () => {
port = await freePort();
startDockerContainer([
'run',
'-d',
'--name',
CONTAINER,
'-e',
'POSTGRES_USER=fluxer',
'-e',
'POSTGRES_PASSWORD=fluxer',
'-e',
'POSTGRES_DB=fluxer',
'-p',
`127.0.0.1:${port}:5432`,
'postgres:16-alpine',
'-c',
'fsync=off',
'-c',
`scram_iterations=${SCRAM_ITERATIONS}`,
]);
let ready = false;
for (let attempt = 0; attempt < 180 && !ready; attempt += 1) {
await sleep(500);
const probe = spawnSync(
'docker',
['exec', CONTAINER, 'psql', '-h', '127.0.0.1', '-U', 'fluxer', '-d', 'fluxer', '-Atc', 'SELECT 1'],
{stdio: 'ignore'},
);
ready = probe.status === 0;
}
if (!ready) throw new Error('postgres never came up');
const rehash = spawnSync(
'docker',
['exec', CONTAINER, 'psql', '-U', 'fluxer', '-d', 'fluxer', '-Atc', "ALTER ROLE fluxer PASSWORD 'fluxer'"],
{
stdio: 'ignore',
},
);
if (rehash.status !== 0) throw new Error('could not re-hash the role password');
}, 900_000);
afterAll(async () => {
await shutdownPostgres().catch(() => {});
spawnSync('docker', ['rm', '-f', CONTAINER], {stdio: 'ignore'});
});
it('connects when the role verifier uses more iterations than the driver default allows', async () => {
await initPostgres({url: `postgres://fluxer:[email protected]:${port}/fluxer`, maxConnections: 1});
const verifier = await getDefaultPostgresClient().query<{rolpassword: string}>(
"SELECT rolpassword FROM pg_authid WHERE rolname = 'fluxer'",
);
expect(verifier.rows[0]?.rolpassword.startsWith(`SCRAM-SHA-256$${SCRAM_ITERATIONS}:`)).toBe(true);
});
});
@@ -13,6 +13,7 @@ import {
type DonorMagicLinkTokenRow,
type DonorRow,
} from '@app/api/database/types/DonationTypes';
import {seconds} from 'itty-time';
export const Donors = defineTable<DonorRow, 'email'>({
name: 'donors',
@@ -43,9 +44,11 @@ export const DonorMagicLinkTokens = defineTable<DonorMagicLinkTokenRow, 'token_'
name: 'donor_magic_link_tokens',
columns: DONOR_MAGIC_LINK_TOKEN_COLUMNS,
primaryKey: ['token_'],
defaultTtlSeconds: seconds('15 minutes'),
});
export const DonorMagicLinkTokensByEmail = defineTable<DonorMagicLinkTokenByEmailRow, 'donor_email' | 'token_'>({
name: 'donor_magic_link_tokens_by_email',
columns: DONOR_MAGIC_LINK_TOKEN_BY_EMAIL_COLUMNS,
primaryKey: ['donor_email', 'token_'],
defaultTtlSeconds: seconds('15 minutes'),
});
@@ -1,201 +0,0 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import {isJsonRecord} from '@app/api/utils/JsonBoundaryUtils';
import type {DesktopArch, DesktopChannel, DesktopPlatform} from '@fluxer/schema/src/domains/download/DownloadSchemas';
const DESKTOP_BUCKET_PREFIX = 'desktop';
const MIN_RELEASE_ROUTE_COUNT = 28;
const MAX_RELEASE_ROUTE_COUNT = 128;
const MIN_RELEASE_ASSET_COUNT = 24;
interface DesktopReleaseAsset {
storage_key: string;
release_asset: string;
sha256: string;
size: number;
}
interface DesktopReleaseDescriptor {
schema_version: 1;
channel: DesktopChannel;
version: string;
release_tag: string;
source_sha: string;
assets: Array<DesktopReleaseAsset>;
}
interface DesktopReleaseReadiness {
schema_version: 1;
channel: DesktopChannel;
version: string;
release_tag: string;
source_sha: string;
descriptor_sha256: string;
}
interface DesktopArtifactScope {
channel: DesktopChannel;
plat: DesktopPlatform;
arch: DesktopArch;
}
export function parseDesktopArtifactScope(key: string): DesktopArtifactScope | null {
const segments = key.split('/');
if (segments.length !== 5 || segments[0] !== DESKTOP_BUCKET_PREFIX || segments[4].length === 0) {
return null;
}
const [, channel, plat, arch] = segments;
if (
(channel !== 'stable' && channel !== 'canary') ||
(plat !== 'win32' && plat !== 'darwin' && plat !== 'linux') ||
(arch !== 'x64' && arch !== 'arm64')
) {
return null;
}
return {channel, plat, arch};
}
function parseDesktopReleaseAsset(value: unknown): DesktopReleaseAsset | null {
if (
!isJsonRecord(value) ||
typeof value.storage_key !== 'string' ||
typeof value.release_asset !== 'string' ||
typeof value.sha256 !== 'string' ||
typeof value.size !== 'number'
) {
return null;
}
if (
!/^desktop\/(?:stable|canary)\/(?:win32|darwin|linux)\/(?:x64|arm64)\/[A-Za-z0-9._-]+$/u.test(value.storage_key) ||
!/^[A-Za-z0-9._-]+$/u.test(value.release_asset) ||
!/^[a-f0-9]{64}$/u.test(value.sha256) ||
!Number.isSafeInteger(value.size) ||
value.size <= 0
) {
return null;
}
return {
storage_key: value.storage_key,
release_asset: value.release_asset,
sha256: value.sha256,
size: value.size,
};
}
export function parseDesktopReleaseDescriptor(value: unknown): DesktopReleaseDescriptor | null {
if (
!isJsonRecord(value) ||
value.schema_version !== 1 ||
(value.channel !== 'stable' && value.channel !== 'canary') ||
typeof value.version !== 'string' ||
!/^\d+\.\d+\.\d+$/u.test(value.version) ||
typeof value.release_tag !== 'string' ||
typeof value.source_sha !== 'string' ||
!/^[a-f0-9]{40}$/u.test(value.source_sha) ||
!Array.isArray(value.assets) ||
value.assets.length < MIN_RELEASE_ROUTE_COUNT ||
value.assets.length > MAX_RELEASE_ROUTE_COUNT
) {
return null;
}
const expectedTag = `fluxer-desktop-${value.channel}@${value.version}`;
const expectedStoragePrefix = `desktop/${value.channel}/`;
const expectedReleasePrefix = `${value.channel === 'canary' ? 'Fluxer-Canary' : 'Fluxer'}-${value.version}-`;
const descriptorName = `${expectedReleasePrefix}release-manifest.json`;
if (value.release_tag !== expectedTag) {
return null;
}
const storageKeys = new Set<string>();
const routeCounts = new Map<string, number>();
const releaseAssets = new Map<string, {sha256: string; size: number}>();
const releaseAssetNames = new Map<string, string>([[descriptorName.toLowerCase(), descriptorName]]);
const assets: Array<DesktopReleaseAsset> = [];
for (const rawAsset of value.assets) {
const asset = parseDesktopReleaseAsset(rawAsset);
if (
!asset?.storage_key.startsWith(expectedStoragePrefix) ||
!asset.release_asset.startsWith(expectedReleasePrefix) ||
storageKeys.has(asset.storage_key)
) {
return null;
}
storageKeys.add(asset.storage_key);
const [, , platform, arch, filename] = asset.storage_key.split('/');
const platformToken = platform === 'win32' ? 'win' : platform === 'darwin' ? 'mac' : 'linux';
const releaseFilename =
platform === 'darwin' && filename.toLowerCase() === 'releases.json' ? 'releases.json' : filename;
const expectedReleaseAsset = filename.startsWith(expectedReleasePrefix)
? filename
: `${expectedReleasePrefix}${platformToken}-${arch}-${releaseFilename}`;
if (
asset.release_asset !== expectedReleaseAsset ||
asset.release_asset.toLowerCase() === descriptorName.toLowerCase()
) {
return null;
}
const caseFoldedReleaseAsset = asset.release_asset.toLowerCase();
const existingReleaseAssetName = releaseAssetNames.get(caseFoldedReleaseAsset);
if (existingReleaseAssetName && existingReleaseAssetName !== asset.release_asset) {
return null;
}
releaseAssetNames.set(caseFoldedReleaseAsset, asset.release_asset);
const scope = `${platform}/${arch}`;
routeCounts.set(scope, (routeCounts.get(scope) ?? 0) + 1);
const existing = releaseAssets.get(asset.release_asset);
if (existing && (existing.sha256 !== asset.sha256 || existing.size !== asset.size)) {
return null;
}
releaseAssets.set(asset.release_asset, {sha256: asset.sha256, size: asset.size});
assets.push(asset);
}
if (releaseAssets.size < MIN_RELEASE_ASSET_COUNT || releaseAssets.size > MAX_RELEASE_ROUTE_COUNT) {
return null;
}
const expectedRouteCounts = new Map([
['darwin/arm64', 4],
['darwin/x64', 4],
['linux/arm64', 4],
['linux/x64', 4],
['win32/arm64', 6],
['win32/x64', 6],
]);
if (
routeCounts.size !== expectedRouteCounts.size ||
Array.from(expectedRouteCounts).some(([scope, count]) => (routeCounts.get(scope) ?? 0) < count)
) {
return null;
}
return {
schema_version: 1,
channel: value.channel,
version: value.version,
release_tag: value.release_tag,
source_sha: value.source_sha,
assets,
};
}
export function parseDesktopReleaseReadiness(value: unknown): DesktopReleaseReadiness | null {
if (
!isJsonRecord(value) ||
value.schema_version !== 1 ||
(value.channel !== 'stable' && value.channel !== 'canary') ||
typeof value.version !== 'string' ||
!/^\d+\.\d+\.\d+$/u.test(value.version) ||
typeof value.release_tag !== 'string' ||
typeof value.source_sha !== 'string' ||
!/^[a-f0-9]{40}$/u.test(value.source_sha) ||
typeof value.descriptor_sha256 !== 'string' ||
!/^[a-f0-9]{64}$/u.test(value.descriptor_sha256)
) {
return null;
}
return {
schema_version: 1,
channel: value.channel,
version: value.version,
release_tag: value.release_tag,
source_sha: value.source_sha,
descriptor_sha256: value.descriptor_sha256,
};
}
+27 -322
View File
@@ -1,370 +1,75 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import {Readable} from 'node:stream';
import {Config} from '@app/api/Config';
import {resolveArtifactRoute} from '@app/api/download/DownloadRouting';
import type {DesktopChecksumFile, DownloadService, DownloadStreamResult} from '@app/api/download/DownloadService';
import {
DESKTOP_REDIRECT_PREFIX,
DOWNLOAD_PREFIX,
downloadCacheControlForKey,
UnsatisfiableRangeError,
} from '@app/api/download/DownloadService';
import {OpenAPI} from '@app/api/middleware/ResponseTypeMiddleware';
import {DESKTOP_REDIRECT_PREFIX, DOWNLOAD_PREFIX, resolveDownloadRedirect} from '@app/api/download/DownloadRedirects';
import type {HonoEnv} from '@app/api/types/HonoEnv';
import {Validator} from '@app/api/Validator';
import {
DesktopChecksumRedirectParam,
DesktopRedirectParam,
DesktopTestBuildQuery,
DesktopVersionedChecksumRedirectParam,
DesktopVersionedRedirectParam,
DesktopVersionedZsyncRedirectParam,
DesktopVersionsParam,
DesktopVersionsQuery,
DesktopVersionsResponse,
DownloadChecksumResponse,
DownloadFileResponse,
VersionInfoResponse,
DesktopZsyncRedirectParam,
} from '@fluxer/schema/src/domains/download/DownloadSchemas';
import type {Context, Hono} from 'hono';
function artifactFilename(key: string, filenameOverride?: string): string {
return filenameOverride ?? key.split('/').pop() ?? 'download';
}
function artifactRedirectResponse(location: string, cacheControl = 'no-store'): Response {
function redirectToPackageOrigin(ctx: Context<HonoEnv>): Response {
const redirect = resolveDownloadRedirect(ctx.req.path);
if (!redirect) {
return ctx.text('Not Found', 404);
}
return new Response(null, {
status: 302,
headers: new Headers({
Location: location,
'Cache-Control': cacheControl,
Location: redirect.location,
'Cache-Control': redirect.cacheControl,
'Accept-Ranges': 'bytes',
}),
});
}
function setCommonArtifactHeaders(
headers: Headers,
key: string,
cacheControl: string,
filenameOverride: string | undefined,
contentType: string | null | undefined,
contentDisposition: string | null | undefined,
etag: string | null | undefined,
lastModified: Date | null | undefined,
): void {
const filename = artifactFilename(key, filenameOverride);
headers.set('Content-Type', contentType ?? 'application/octet-stream');
headers.set('Content-Disposition', contentDisposition ?? `attachment; filename="${encodeURIComponent(filename)}"`);
headers.set('Accept-Ranges', 'bytes');
headers.set('Cache-Control', cacheControl);
if (etag) {
headers.set('ETag', etag);
}
if (lastModified) {
headers.set('Last-Modified', lastModified.toUTCString());
}
}
async function headArtifactResponse(
ctx: Context<HonoEnv>,
downloadService: DownloadService,
key: string,
cacheControl: string,
filenameOverride?: string,
): Promise<Response> {
const metadata = await downloadService.getDownloadMetadata({key});
if (!metadata) {
return ctx.text('Not Found', 404);
}
const headers = new Headers();
setCommonArtifactHeaders(
headers,
key,
cacheControl,
filenameOverride,
metadata.contentType,
undefined,
metadata.etag,
metadata.lastModified,
);
headers.set('Content-Length', String(metadata.contentLength));
return new Response(null, {status: 200, headers});
}
const PRESIGNED_DOWNLOAD_TTL_SECONDS = 900;
async function streamArtifactResponse(
ctx: Context<HonoEnv>,
downloadService: DownloadService,
key: string,
cacheControl: string,
filenameOverride?: string,
): Promise<Response> {
const route = await resolveArtifactRoute({request: ctx.req.raw, downloadService, key, cacheControl});
if (route.kind === 'redirect') {
return artifactRedirectResponse(route.location, route.cacheControl);
}
if (ctx.req.method === 'HEAD') {
return headArtifactResponse(ctx, downloadService, key, route.cacheControl, filenameOverride);
}
if (downloadService.isPresignedDownloadEnabled()) {
const location = await downloadService.getPresignedDownloadRedirect({
key,
filename: artifactFilename(key, filenameOverride),
expiresIn: PRESIGNED_DOWNLOAD_TTL_SECONDS,
});
if (!location) {
return ctx.text('Not Found', 404);
}
return artifactRedirectResponse(location);
}
const range = ctx.req.header('range') ?? undefined;
let result: DownloadStreamResult | null;
try {
result = await downloadService.streamDownload({key, range});
} catch (error) {
if (error instanceof UnsatisfiableRangeError) {
const headers = new Headers();
headers.set('Accept-Ranges', 'bytes');
headers.set('Content-Range', `bytes */${error.totalSize}`);
headers.set('Cache-Control', route.cacheControl);
return new Response(null, {status: 416, headers});
}
throw error;
}
if (!result) {
return ctx.text('Not Found', 404);
}
const headers = new Headers();
setCommonArtifactHeaders(
headers,
key,
route.cacheControl,
filenameOverride,
result.contentType,
result.contentDisposition,
result.etag,
result.lastModified,
);
headers.set('Content-Length', String(result.contentLength));
if (result.contentRange) {
headers.set('Content-Range', result.contentRange);
}
const body = Readable.toWeb(result.body) as ReadableStream;
return new Response(body, {status: result.contentRange ? 206 : 200, headers});
}
function checksumFileResponse(ctx: Context<HonoEnv>, checksum: DesktopChecksumFile, cacheControl: string): Response {
const headers = new Headers();
const body = ctx.req.method === 'HEAD' ? null : checksum.body;
headers.set('Content-Type', 'text/plain; charset=utf-8');
headers.set('Content-Disposition', `attachment; filename="${encodeURIComponent(`${checksum.filename}.sha256`)}"`);
headers.set('Cache-Control', cacheControl);
headers.set('Content-Length', String(new TextEncoder().encode(checksum.body).byteLength));
return new Response(body, {status: 200, headers});
}
export function DownloadController(routes: Hono<HonoEnv>): void {
routes.get(
`${DESKTOP_REDIRECT_PREFIX}/:channel/:plat/:arch/latest`,
Validator('param', DesktopVersionsParam),
Validator('query', DesktopTestBuildQuery),
OpenAPI({
operationId: 'get_latest_desktop_version',
summary: 'Get latest desktop version',
responseSchema: VersionInfoResponse,
statusCode: 200,
security: [],
tags: ['Downloads'],
description:
'Returns metadata for the latest desktop version including download URLs and SHA-256 checksums for all available formats. Pass ?test=1 to resolve against unreleased test builds.',
}),
async (ctx) => {
const {channel, plat, arch} = ctx.req.valid('param');
const {test} = ctx.req.valid('query');
const result = await ctx.get('downloadService').getLatestDesktopVersion({
channel,
plat,
arch,
baseUrl: Config.endpoints.apiClient,
test,
});
if (!result) {
return ctx.text('Not Found', 404);
}
return ctx.json(result, 200, {
'Cache-Control': 'public, max-age=300',
});
},
async (ctx) => redirectToPackageOrigin(ctx),
);
routes.on(
['GET', 'HEAD'],
`${DESKTOP_REDIRECT_PREFIX}/:channel/:plat/:arch/latest/:format{[a-z_]+\\.sha256}`,
Validator('param', DesktopChecksumRedirectParam),
Validator('query', DesktopTestBuildQuery),
OpenAPI({
operationId: 'download_latest_desktop_version_checksum',
summary: 'Download latest desktop version checksum',
responseSchema: DownloadChecksumResponse,
responseContentType: 'text/plain',
statusCode: 200,
security: [],
tags: ['Downloads'],
description:
'Returns a plain text SHA-256 checksum file for the latest available desktop application version. The format path segment must end in .sha256, for example appimage.sha256.',
}),
async (ctx) => {
const {channel, plat, arch, format} = ctx.req.valid('param');
const {test} = ctx.req.valid('query');
const checksum = await ctx
.get('downloadService')
.resolveLatestDesktopChecksumFile({channel, plat, arch, format, test});
if (!checksum) {
return ctx.text('Not Found', 404);
}
return checksumFileResponse(ctx, checksum, 'no-store');
},
async (ctx) => redirectToPackageOrigin(ctx),
);
routes.on(
['GET', 'HEAD'],
`${DESKTOP_REDIRECT_PREFIX}/:channel/:plat/:arch/latest/:format{[a-z_]+\\.zsync}`,
Validator('param', DesktopZsyncRedirectParam),
async (ctx) => redirectToPackageOrigin(ctx),
);
routes.on(
['GET', 'HEAD'],
`${DESKTOP_REDIRECT_PREFIX}/:channel/:plat/:arch/latest/:format`,
Validator('param', DesktopRedirectParam),
Validator('query', DesktopTestBuildQuery),
OpenAPI({
operationId: 'download_latest_desktop_version',
summary: 'Download latest desktop version',
responseSchema: DownloadFileResponse,
responseContentType: '*/*',
statusCode: [200, 206, 302],
bodylessStatusCodes: [302],
security: [],
tags: ['Downloads'],
description:
'Streams the latest available desktop application version for the specified platform and architecture. Pass ?test=1 to download an unreleased test build.',
}),
async (ctx) => {
const {channel, plat, arch, format} = ctx.req.valid('param');
const {test} = ctx.req.valid('query');
const downloadService = ctx.get('downloadService');
const key = await downloadService.resolveLatestDesktopKey({channel, plat, arch, format, test});
if (!key) {
return ctx.text('Not Found', 404);
}
return streamArtifactResponse(ctx, downloadService, key, 'no-store');
},
async (ctx) => redirectToPackageOrigin(ctx),
);
routes.get(
`${DESKTOP_REDIRECT_PREFIX}/:channel/:plat/:arch/versions`,
Validator('param', DesktopVersionsParam),
Validator('query', DesktopVersionsQuery),
OpenAPI({
operationId: 'list_desktop_versions',
summary: 'List desktop versions',
responseSchema: DesktopVersionsResponse,
statusCode: 200,
security: [],
tags: ['Downloads'],
description: 'Lists available desktop versions with pagination for the specified platform and architecture.',
}),
async (ctx) => {
const {channel, plat, arch} = ctx.req.valid('param');
const {limit, before, after, test} = ctx.req.valid('query');
const {versions, hasMore} = await ctx.get('downloadService').listDesktopVersions({
channel,
plat,
arch,
limit,
before,
after,
baseUrl: Config.endpoints.apiClient,
test,
});
return ctx.json({versions, has_more: hasMore}, 200, {
'Cache-Control': 'public, max-age=300',
});
},
routes.on(
['GET', 'HEAD'],
`${DESKTOP_REDIRECT_PREFIX}/:channel/:plat/:arch/:version/:format{[a-z_]+\\.zsync}`,
Validator('param', DesktopVersionedZsyncRedirectParam),
async (ctx) => redirectToPackageOrigin(ctx),
);
routes.on(
['GET', 'HEAD'],
`${DESKTOP_REDIRECT_PREFIX}/:channel/:plat/:arch/:version/:format{[a-z_]+\\.sha256}`,
Validator('param', DesktopVersionedChecksumRedirectParam),
Validator('query', DesktopTestBuildQuery),
OpenAPI({
operationId: 'download_desktop_version_checksum',
summary: 'Download desktop version checksum',
responseSchema: DownloadChecksumResponse,
responseContentType: 'text/plain',
statusCode: 200,
security: [],
tags: ['Downloads'],
description:
'Returns a plain text SHA-256 checksum file for a specific desktop application version. The format path segment must end in .sha256, for example appimage.sha256.',
}),
async (ctx) => {
const {channel, plat, arch, version, format} = ctx.req.valid('param');
const {test} = ctx.req.valid('query');
const checksum = await ctx
.get('downloadService')
.resolveVersionedDesktopChecksumFile({channel, plat, arch, version, format, test});
if (!checksum) {
return ctx.text('Not Found', 404);
}
return checksumFileResponse(ctx, checksum, downloadCacheControlForKey(checksum.key));
},
async (ctx) => redirectToPackageOrigin(ctx),
);
routes.on(
['GET', 'HEAD'],
`${DESKTOP_REDIRECT_PREFIX}/:channel/:plat/:arch/:version/:format`,
Validator('param', DesktopVersionedRedirectParam),
Validator('query', DesktopTestBuildQuery),
OpenAPI({
operationId: 'download_desktop_version',
summary: 'Download desktop version',
responseSchema: DownloadFileResponse,
responseContentType: '*/*',
statusCode: [200, 206, 302],
bodylessStatusCodes: [302],
security: [],
tags: ['Downloads'],
description:
'Streams a specific desktop application version for the given platform and architecture. Pass ?test=1 to download an unreleased test build.',
}),
async (ctx) => {
const {channel, plat, arch, version, format} = ctx.req.valid('param');
const {test} = ctx.req.valid('query');
const downloadService = ctx.get('downloadService');
const key = await downloadService.resolveVersionedDesktopKey({channel, plat, arch, version, format, test});
if (!key) {
return ctx.text('Not Found', 404);
}
return streamArtifactResponse(ctx, downloadService, key, downloadCacheControlForKey(key));
},
);
routes.on(
['GET', 'HEAD'],
`${DOWNLOAD_PREFIX}/*`,
Validator('query', DesktopTestBuildQuery),
OpenAPI({
operationId: 'download_file',
summary: 'Download file',
responseSchema: DownloadFileResponse,
responseContentType: '*/*',
statusCode: [200, 206, 302],
bodylessStatusCodes: [302],
security: [],
tags: ['Downloads'],
description:
'Streams the requested file from storage. Pass ?test=1 on a desktop/ path to resolve against the desktop-test/ bucket prefix instead.',
}),
async (ctx) => {
const {test} = ctx.req.valid('query');
const downloadService = ctx.get('downloadService');
const key = await downloadService.resolveDownloadKey({path: ctx.req.path, test});
if (!key) {
return ctx.text('Not Found', 404);
}
return streamArtifactResponse(ctx, downloadService, key, downloadCacheControlForKey(key));
},
async (ctx) => redirectToPackageOrigin(ctx),
);
routes.on(['GET', 'HEAD'], `${DOWNLOAD_PREFIX}/*`, async (ctx) => redirectToPackageOrigin(ctx));
}
@@ -0,0 +1,83 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import {posix} from 'node:path';
export const PKGS_BASE_URL = 'https://pkgs.fluxer.com';
export const DOWNLOAD_PREFIX = '/dl';
export const DESKTOP_REDIRECT_PREFIX = `${DOWNLOAD_PREFIX}/desktop`;
export const DESKTOP_COORDINATE_DOCUMENTS = new Map<string, string>([['latest', 'latest.json']]);
const DESKTOP_PATH_PREFIX = 'desktop/';
const PLATFORM_ARCH_PATH = /^(desktop\/(?:stable|canary)\/(?:win32|darwin|linux))-(x64|arm64)(\/.+)$/u;
const COORDINATE_DOCUMENT_PATH = /^(desktop\/(?:stable|canary)\/(?:win32|darwin|linux)\/(?:x64|arm64))\/([a-z]+)$/u;
const MUTABLE_REDIRECT_CACHE_CONTROL = 'no-store';
const VERSIONED_REDIRECT_CACHE_CONTROL = 'public, max-age=31536000';
const SCOPE_SEGMENT_INDEX = 4;
interface DownloadRedirect {
location: string;
cacheControl: string;
}
function isReleaseFeedFilename(filename: string): boolean {
return (
filename === 'manifest.json' ||
filename === 'latest.json' ||
filename === 'version.json' ||
filename.endsWith('.yml') ||
filename.endsWith('.yaml') ||
filename.startsWith('RELEASES') ||
(filename.startsWith('releases') && filename.endsWith('.json')) ||
(filename.startsWith('assets') && filename.endsWith('.json'))
);
}
function normalizePlatformArchPath(path: string): string {
const match = path.match(PLATFORM_ARCH_PATH);
return match ? `${match[1]}/${match[2]}${match[3]}` : path;
}
function resolveCoordinateDocument(path: string): string {
const match = path.match(COORDINATE_DOCUMENT_PATH);
const document = match ? DESKTOP_COORDINATE_DOCUMENTS.get(match[2]) : undefined;
return match && document ? `${match[1]}/${document}` : path;
}
export function resolveDownloadObjectPath(requestPath: string): string | null {
if (!requestPath.startsWith(DOWNLOAD_PREFIX)) {
return null;
}
const normalized = posix.normalize(requestPath.slice(DOWNLOAD_PREFIX.length).replace(/^\/+/u, ''));
if (normalized.length === 0 || normalized.startsWith('/') || normalized.startsWith('..')) {
return null;
}
for (const segment of normalized.split('/')) {
if (segment.length === 0 || segment === '.' || segment === '..' || segment.includes('\0')) {
return null;
}
}
const objectPath = resolveCoordinateDocument(normalizePlatformArchPath(normalized));
return objectPath.startsWith(DESKTOP_PATH_PREFIX) ? objectPath : null;
}
export function downloadRedirectCacheControl(objectPath: string): string {
const segments = objectPath.split('/');
if (segments[SCOPE_SEGMENT_INDEX] === 'latest') {
return MUTABLE_REDIRECT_CACHE_CONTROL;
}
return isReleaseFeedFilename(segments[segments.length - 1])
? MUTABLE_REDIRECT_CACHE_CONTROL
: VERSIONED_REDIRECT_CACHE_CONTROL;
}
export function resolveDownloadRedirect(requestPath: string): DownloadRedirect | null {
const objectPath = resolveDownloadObjectPath(requestPath);
if (!objectPath) {
return null;
}
return {
location: `${PKGS_BASE_URL}/${objectPath}`,
cacheControl: downloadRedirectCacheControl(objectPath),
};
}
@@ -1,53 +0,0 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import {Config} from '@app/api/Config';
import {parseDesktopArtifactScope} from '@app/api/download/DesktopReleaseContract';
import type {DownloadService, GitHubDesktopReleaseResolution} from '@app/api/download/DownloadService';
import {Logger} from '@app/api/Logger';
import {lookupGeoip} from '@app/api/utils/IpUtils';
const COUNTRY_DEPENDENT_CACHE_CONTROL = 'private, no-store';
type ArtifactRoute =
| {kind: 'storage'; cacheControl: string}
| {kind: 'redirect'; cacheControl: string; location: string};
export async function resolveArtifactRoute(params: {
request: Request;
downloadService: DownloadService;
key: string;
cacheControl: string;
}): Promise<ArtifactRoute> {
if (Config.instance.selfHosted) {
return {kind: 'storage', cacheControl: params.cacheControl};
}
if (Config.desktopGitHubRedirectCountries.size === 0) {
return {kind: 'storage', cacheControl: params.cacheControl};
}
if (!parseDesktopArtifactScope(params.key)) {
return {kind: 'storage', cacheControl: params.cacheControl};
}
const geoip = await lookupGeoip(params.request);
const countryCode = geoip.countryCode?.trim().toUpperCase();
if (!countryCode || !Config.desktopGitHubRedirectCountries.has(countryCode)) {
return {kind: 'storage', cacheControl: COUNTRY_DEPENDENT_CACHE_CONTROL};
}
let release: GitHubDesktopReleaseResolution;
try {
release = await params.downloadService.resolveGitHubDesktopRelease(params.key);
} catch (error) {
Logger.error({error, key: params.key}, 'Failed to resolve GitHub desktop download route');
return {kind: 'storage', cacheControl: COUNTRY_DEPENDENT_CACHE_CONTROL};
}
if (release.kind === 'not_current') {
return {kind: 'storage', cacheControl: COUNTRY_DEPENDENT_CACHE_CONTROL};
}
if (release.kind === 'ready') {
return {
kind: 'redirect',
cacheControl: COUNTRY_DEPENDENT_CACHE_CONTROL,
location: release.location,
};
}
return {kind: 'storage', cacheControl: COUNTRY_DEPENDENT_CACHE_CONTROL};
}
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,187 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import {DownloadController} from '@app/api/download/DownloadController';
import {PKGS_BASE_URL} from '@app/api/download/DownloadRedirects';
import type {HonoEnv} from '@app/api/types/HonoEnv';
import {Hono} from 'hono';
import {describe, expect, it} from 'vitest';
const COUNTRY_HEADERS = {'cf-ipcountry': 'BR', 'x-forwarded-for': '203.0.113.7'};
function createApp() {
const app = new Hono<HonoEnv>();
app.onError((_error, ctx) => ctx.text('Bad Request', 400));
DownloadController(app);
return app;
}
async function request(path: string, init?: RequestInit) {
const response = await createApp().request(path, init);
return {
status: response.status,
location: response.headers.get('Location'),
cacheControl: response.headers.get('Cache-Control'),
body: await response.text(),
};
}
describe('legacy desktop download routes', () => {
it('redirects the latest metadata route at the document the publisher writes', async () => {
const response = await request('/dl/desktop/stable/darwin/arm64/latest');
expect(response.status).toBe(302);
expect(response.location).toBe(`${PKGS_BASE_URL}/desktop/stable/darwin/arm64/latest.json`);
expect(response.cacheControl).toBe('no-store');
expect(response.body).toBe('');
});
it('redirects the latest artifact route', async () => {
const response = await request('/dl/desktop/stable/linux/x64/latest/appimage');
expect(response.status).toBe(302);
expect(response.location).toBe(`${PKGS_BASE_URL}/desktop/stable/linux/x64/latest/appimage`);
expect(response.cacheControl).toBe('no-store');
});
it('redirects the latest checksum route', async () => {
const response = await request('/dl/desktop/stable/linux/x64/latest/appimage.sha256');
expect(response.status).toBe(302);
expect(response.location).toBe(`${PKGS_BASE_URL}/desktop/stable/linux/x64/latest/appimage.sha256`);
expect(response.cacheControl).toBe('no-store');
});
it('stops mapping the retired version listing route, so it passes through to an origin path that serves nothing', async () => {
const response = await request('/dl/desktop/canary/linux/arm64/versions');
expect(response.status).toBe(302);
expect(response.location).toBe(`${PKGS_BASE_URL}/desktop/canary/linux/arm64/versions`);
});
it('redirects the latest appimage zsync sidecar rather than rejecting it', async () => {
const response = await request('/dl/desktop/canary/linux/x64/latest/appimage.zsync');
expect(response.status).toBe(302);
expect(response.location).toBe(`${PKGS_BASE_URL}/desktop/canary/linux/x64/latest/appimage.zsync`);
expect(response.cacheControl).toBe('no-store');
});
it('redirects the versioned appimage zsync sidecar and lets the redirect be cached', async () => {
const response = await request('/dl/desktop/canary/linux/x64/1.4.2/appimage.zsync');
expect(response.status).toBe(302);
expect(response.location).toBe(`${PKGS_BASE_URL}/desktop/canary/linux/x64/1.4.2/appimage.zsync`);
expect(response.cacheControl).toBe('public, max-age=31536000');
});
it('answers HEAD on the zsync sidecar the way it answers GET', async () => {
const response = await request('/dl/desktop/canary/linux/x64/latest/appimage.zsync', {method: 'HEAD'});
expect(response.status).toBe(302);
expect(response.location).toBe(`${PKGS_BASE_URL}/desktop/canary/linux/x64/latest/appimage.zsync`);
});
it('still rejects a zsync sidecar for a format that publishes none', async () => {
const response = await request('/dl/desktop/canary/linux/x64/latest/deb.zsync');
expect(response.status).toBe(400);
});
it('redirects the versioned artifact route and lets the redirect be cached', async () => {
const response = await request('/dl/desktop/stable/win32/x64/1.4.2/setup');
expect(response.status).toBe(302);
expect(response.location).toBe(`${PKGS_BASE_URL}/desktop/stable/win32/x64/1.4.2/setup`);
expect(response.cacheControl).toBe('public, max-age=31536000');
});
it('redirects the versioned checksum route', async () => {
const response = await request('/dl/desktop/stable/win32/x64/1.4.2/setup.sha256');
expect(response.status).toBe(302);
expect(response.location).toBe(`${PKGS_BASE_URL}/desktop/stable/win32/x64/1.4.2/setup.sha256`);
expect(response.cacheControl).toBe('public, max-age=31536000');
});
it('answers HEAD on the artifact routes the way it answers GET', async () => {
const response = await request('/dl/desktop/stable/win32/x64/1.4.2/setup', {method: 'HEAD'});
expect(response.status).toBe(302);
expect(response.location).toBe(`${PKGS_BASE_URL}/desktop/stable/win32/x64/1.4.2/setup`);
});
it('rejects a format outside the closed registry before it reaches the redirector', async () => {
const response = await request('/dl/desktop/stable/linux/x64/latest/msix');
expect(response.status).toBe(400);
});
});
describe('release feed routes', () => {
it.each([
'/dl/desktop/stable/darwin/arm64/RELEASES.json',
'/dl/desktop/stable/win32/x64/RELEASES',
'/dl/desktop/canary/win32/arm64/releases.canary.json',
'/dl/desktop/stable/win32/x64/releases.win.json',
'/dl/desktop/stable/linux/x64/manifest.json',
])('redirects %s without caching the redirect', async (path) => {
const response = await request(path);
expect(response.status).toBe(302);
expect(response.location).toBe(`${PKGS_BASE_URL}${path.slice('/dl'.length)}`);
expect(response.cacheControl).toBe('no-store');
});
it('redirects a nupkg named by a RELEASES body', async () => {
const response = await request('/dl/desktop/stable/win32/x64/fluxer_app-0.0.8-full.nupkg');
expect(response.status).toBe(302);
expect(response.location).toBe(`${PKGS_BASE_URL}/desktop/stable/win32/x64/fluxer_app-0.0.8-full.nupkg`);
expect(response.cacheControl).toBe('public, max-age=31536000');
});
});
describe('channel, platform and architecture targeting', () => {
it.each([
['stable', 'darwin', 'arm64'],
['stable', 'darwin', 'x64'],
['stable', 'win32', 'x64'],
['stable', 'win32', 'arm64'],
['stable', 'linux', 'x64'],
['stable', 'linux', 'arm64'],
['canary', 'darwin', 'arm64'],
['canary', 'win32', 'x64'],
['canary', 'linux', 'arm64'],
])('keeps %s/%s/%s in the redirect target', async (channel, plat, arch) => {
const response = await request(`/dl/desktop/${channel}/${plat}/${arch}/latest`);
expect(response.status).toBe(302);
expect(response.location).toBe(`${PKGS_BASE_URL}/desktop/${channel}/${plat}/${arch}/latest.json`);
});
});
describe('the geoip and github release route is gone', () => {
it('sends every country to the package origin with the same cache control', async () => {
const path = '/dl/desktop/stable/darwin/arm64/1.4.2/dmg';
const withCountry = await request(path, {headers: COUNTRY_HEADERS});
const withoutCountry = await request(path);
expect(withCountry).toEqual(withoutCountry);
expect(withCountry.location).toBe(`${PKGS_BASE_URL}/desktop/stable/darwin/arm64/1.4.2/dmg`);
expect(withCountry.cacheControl).not.toBe('private, no-store');
});
it('never points a download at github', async () => {
const response = await request('/dl/desktop/stable/darwin/arm64/1.4.2/zip', {headers: COUNTRY_HEADERS});
expect(response.location).not.toContain('github.com');
expect(response.location?.startsWith(`${PKGS_BASE_URL}/`)).toBe(true);
});
});
describe('paths the redirector refuses', () => {
it('answers 404 for a key outside the desktop prefix', async () => {
const response = await request('/dl/harvests/dump.zip');
expect(response.status).toBe(404);
expect(response.body).toBe('Not Found');
});
it('answers 404 for a traversal attempt', async () => {
const response = await request('/dl/desktop/../harvests/dump.zip');
expect(response.status).toBe(404);
});
it('answers 404 for the retired test build prefix', async () => {
const response = await request('/dl/desktop-test/canary/linux/x64/latest/appimage');
expect(response.status).toBe(404);
});
it('ignores a test query parameter rather than resolving another prefix', async () => {
const response = await request('/dl/desktop/canary/linux/x64/latest/appimage?test=1');
expect(response.status).toBe(302);
expect(response.location).toBe(`${PKGS_BASE_URL}/desktop/canary/linux/x64/latest/appimage`);
});
});
@@ -1,79 +0,0 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import {DownloadService} from '@app/api/download/DownloadService';
import type {IStorageService} from '@app/api/infrastructure/IStorageService';
import {describe, expect, it} from 'vitest';
const OBJECT_METADATA = {
contentLength: 285_567_850,
contentType: 'application/x-apple-diskimage',
etag: '"abc123"',
lastModified: new Date('2026-08-17T00:00:00Z'),
};
interface PresignCall {
bucket: string;
key: string;
expiresIn?: number;
responseContentType?: string;
responseContentDisposition?: string;
}
function createService(overrides: {metadata?: typeof OBJECT_METADATA | null} = {}) {
const presignCalls: Array<PresignCall> = [];
const storageService = {
getObjectMetadata: async () => (overrides.metadata === undefined ? OBJECT_METADATA : overrides.metadata),
getPresignedDownloadURL: async (params: PresignCall) => {
presignCalls.push(params);
return `https://storage.example.test/${params.key}?signed=1`;
},
} as unknown as IStorageService;
return {service: new DownloadService(storageService), presignCalls};
}
describe('presigned download redirects', () => {
it('signs the requested object and returns its URL', async () => {
const {service, presignCalls} = createService();
const url = await service.getPresignedDownloadRedirect({
key: 'desktop/canary/darwin/universal/Fluxer.dmg',
filename: 'Fluxer.dmg',
expiresIn: 900,
});
expect(url).toBe('https://storage.example.test/desktop/canary/darwin/universal/Fluxer.dmg?signed=1');
expect(presignCalls).toHaveLength(1);
expect(presignCalls[0]?.key).toBe('desktop/canary/darwin/universal/Fluxer.dmg');
expect(presignCalls[0]?.expiresIn).toBe(900);
});
it('preserves the download filename and content type through the redirect', async () => {
const {service, presignCalls} = createService();
await service.getPresignedDownloadRedirect({
key: 'desktop/canary/darwin/universal/Fluxer.dmg',
filename: 'Fluxer Canary.dmg',
expiresIn: 900,
});
expect(presignCalls[0]?.responseContentType).toBe('application/x-apple-diskimage');
expect(presignCalls[0]?.responseContentDisposition).toBe(
`attachment; filename="${encodeURIComponent('Fluxer Canary.dmg')}"`,
);
});
it('falls back to a binary content type when storage reports none', async () => {
const {service, presignCalls} = createService({
metadata: {...OBJECT_METADATA, contentType: null} as unknown as typeof OBJECT_METADATA,
});
await service.getPresignedDownloadRedirect({key: 'desktop/x.bin', filename: 'x.bin', expiresIn: 900});
expect(presignCalls[0]?.responseContentType).toBe('application/octet-stream');
});
it('returns null for a missing object so the caller can answer 404 without signing', async () => {
const {service, presignCalls} = createService({metadata: null});
const url = await service.getPresignedDownloadRedirect({
key: 'desktop/missing.dmg',
filename: 'missing.dmg',
expiresIn: 900,
});
expect(url).toBeNull();
expect(presignCalls).toHaveLength(0);
});
});
@@ -0,0 +1,135 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import {
DESKTOP_COORDINATE_DOCUMENTS,
downloadRedirectCacheControl,
PKGS_BASE_URL,
resolveDownloadObjectPath,
resolveDownloadRedirect,
} from '@app/api/download/DownloadRedirects';
import {describe, expect, it} from 'vitest';
const MUTABLE = 'no-store';
const IMMUTABLE = 'public, max-age=31536000';
describe('the coordinate document contract the publisher writes', () => {
it('names the exact files scripts/packages/stage-desktop.sh and retention.sh publish', () => {
expect(Object.fromEntries(DESKTOP_COORDINATE_DOCUMENTS)).toEqual({
latest: 'latest.json',
});
});
it('resolves the bare coordinate names to those files on every coordinate', () => {
for (const channel of ['stable', 'canary']) {
for (const plat of ['win32', 'darwin', 'linux']) {
for (const arch of ['x64', 'arm64']) {
expect(resolveDownloadObjectPath(`/dl/desktop/${channel}/${plat}/${arch}/latest`)).toBe(
`desktop/${channel}/${plat}/${arch}/latest.json`,
);
expect(resolveDownloadObjectPath(`/dl/desktop/${channel}/${plat}/${arch}/versions`)).toBe(
`desktop/${channel}/${plat}/${arch}/versions`,
);
}
}
}
});
it('leaves the latest directory alone when a format or a sidecar follows it', () => {
expect(resolveDownloadObjectPath('/dl/desktop/stable/linux/x64/latest/appimage')).toBe(
'desktop/stable/linux/x64/latest/appimage',
);
expect(resolveDownloadObjectPath('/dl/desktop/stable/linux/x64/latest/appimage.zsync')).toBe(
'desktop/stable/linux/x64/latest/appimage.zsync',
);
});
it('rewrites the legacy platform-arch form to the same documents', () => {
expect(resolveDownloadObjectPath('/dl/desktop/stable/linux-x64/latest')).toBe(
'desktop/stable/linux/x64/latest.json',
);
expect(resolveDownloadObjectPath('/dl/desktop/stable/linux-x64/versions')).toBe(
'desktop/stable/linux/x64/versions',
);
});
it('rewrites nothing else that sits at the coordinate root', () => {
expect(resolveDownloadObjectPath('/dl/desktop/stable/linux/x64/manifest.json')).toBe(
'desktop/stable/linux/x64/manifest.json',
);
expect(resolveDownloadObjectPath('/dl/desktop/stable/win32/x64/RELEASES')).toBe(
'desktop/stable/win32/x64/RELEASES',
);
expect(resolveDownloadObjectPath('/dl/desktop/stable/linux/x64/constructor')).toBe(
'desktop/stable/linux/x64/constructor',
);
});
});
describe('download object paths', () => {
it('strips the /dl prefix and keeps the rest of the path verbatim', () => {
expect(resolveDownloadObjectPath('/dl/desktop/stable/darwin/arm64/RELEASES.json')).toBe(
'desktop/stable/darwin/arm64/RELEASES.json',
);
expect(resolveDownloadObjectPath('/dl/desktop/stable/win32/x64/1.4.2/Fluxer-1.4.2-win-x64.exe')).toBe(
'desktop/stable/win32/x64/1.4.2/Fluxer-1.4.2-win-x64.exe',
);
});
it('normalises the legacy platform-arch segment to the published layout', () => {
expect(resolveDownloadObjectPath('/dl/desktop/stable/linux-x64/manifest.json')).toBe(
'desktop/stable/linux/x64/manifest.json',
);
});
it('refuses a key outside the desktop prefix', () => {
expect(resolveDownloadObjectPath('/dl/reports/secret.json')).toBeNull();
expect(resolveDownloadObjectPath('/dl/desktop-test/canary/linux/x64/latest/appimage')).toBeNull();
expect(resolveDownloadObjectPath('/dl/')).toBeNull();
expect(resolveDownloadObjectPath('/other/desktop/stable/linux/x64/latest')).toBeNull();
});
it('refuses a traversal attempt rather than pointing at another prefix', () => {
expect(resolveDownloadObjectPath('/dl/desktop/../harvests/dump.zip')).toBeNull();
expect(resolveDownloadObjectPath('/dl/../desktop/stable/linux/x64/latest')).toBeNull();
expect(resolveDownloadObjectPath('/dl/desktop/stable/linux/x64/lat\0est')).toBeNull();
});
});
describe('download redirect cache control', () => {
it('never caches a redirect to a mutable document', () => {
expect(downloadRedirectCacheControl('desktop/stable/darwin/arm64/latest.json')).toBe(MUTABLE);
expect(downloadRedirectCacheControl('desktop/stable/darwin/arm64/version.json')).toBe(MUTABLE);
expect(downloadRedirectCacheControl('desktop/stable/linux/x64/latest/appimage')).toBe(MUTABLE);
expect(downloadRedirectCacheControl('desktop/stable/linux/x64/latest/appimage.sha256')).toBe(MUTABLE);
expect(downloadRedirectCacheControl('desktop/stable/linux/x64/latest/appimage.zsync')).toBe(MUTABLE);
expect(downloadRedirectCacheControl('desktop/stable/darwin/arm64/RELEASES.json')).toBe(MUTABLE);
expect(downloadRedirectCacheControl('desktop/stable/win32/x64/RELEASES')).toBe(MUTABLE);
expect(downloadRedirectCacheControl('desktop/canary/win32/x64/releases.canary.json')).toBe(MUTABLE);
expect(downloadRedirectCacheControl('desktop/stable/linux/x64/manifest.json')).toBe(MUTABLE);
expect(downloadRedirectCacheControl('desktop/stable/linux/x64/latest-linux.yml')).toBe(MUTABLE);
});
it('caches a redirect to a version pinned artifact for a year', () => {
expect(downloadRedirectCacheControl('desktop/stable/darwin/arm64/1.4.2/dmg')).toBe(IMMUTABLE);
expect(downloadRedirectCacheControl('desktop/stable/darwin/arm64/1.4.2/dmg.sha256')).toBe(IMMUTABLE);
expect(downloadRedirectCacheControl('desktop/stable/linux/x64/1.4.2/appimage.zsync')).toBe(IMMUTABLE);
expect(downloadRedirectCacheControl('desktop/stable/win32/x64/fluxer_app-0.0.8-full.nupkg')).toBe(IMMUTABLE);
});
});
describe('download redirects', () => {
it('points every desktop path at the package origin', () => {
expect(resolveDownloadRedirect('/dl/desktop/stable/darwin/arm64/1.4.2/dmg')).toEqual({
location: `${PKGS_BASE_URL}/desktop/stable/darwin/arm64/1.4.2/dmg`,
cacheControl: IMMUTABLE,
});
expect(resolveDownloadRedirect('/dl/desktop/canary/linux/arm64/latest')).toEqual({
location: `${PKGS_BASE_URL}/desktop/canary/linux/arm64/latest.json`,
cacheControl: MUTABLE,
});
});
it('returns null for a path it refuses to map', () => {
expect(resolveDownloadRedirect('/dl/harvests/dump.zip')).toBeNull();
});
});
@@ -1,71 +0,0 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import {Readable} from 'node:stream';
import {DownloadService} from '@app/api/download/DownloadService';
import type {IStorageService} from '@app/api/infrastructure/IStorageService';
import {describe, expect, it} from 'vitest';
const PREFIX = 'desktop/stable/darwin/x64';
const MANIFEST_KEY = `${PREFIX}/manifest.json`;
const LISTED_FILENAME = 'Fluxer-1.2.3-mac-universal.dmg';
const MANIFEST_FILENAME = 'Fluxer-1.3.0-mac-universal.dmg';
const LATEST_PARAMS = {
channel: 'stable',
plat: 'darwin',
arch: 'x64',
format: 'dmg',
} as const;
function createService(overrides: {manifestBody?: string | null; objectKeys?: Array<string>} = {}) {
const objectKeys = overrides.objectKeys ?? [`${PREFIX}/${LISTED_FILENAME}`];
const storageService = {
streamObject: async (params: {key: string}) => {
if (params.key !== MANIFEST_KEY) {
return null;
}
const body = overrides.manifestBody;
if (body == null) {
return null;
}
const buffer = Buffer.from(body, 'utf8');
return {body: Readable.from([buffer]), contentLength: buffer.byteLength};
},
listObjects: async () => objectKeys.map((key) => ({key})),
getObjectMetadata: async (_bucket: string, key: string) =>
objectKeys.includes(key) ? {contentLength: 1, contentType: 'application/x-apple-diskimage'} : null,
} as unknown as IStorageService;
return new DownloadService(storageService);
}
describe('desktop manifest parsing', () => {
it('falls back to the object listing when the manifest is not valid JSON', async () => {
const service = createService({manifestBody: '{not json'});
await expect(service.resolveLatestDesktopKey({...LATEST_PARAMS})).resolves.toBe(`${PREFIX}/${LISTED_FILENAME}`);
});
it('returns null rather than throwing when the manifest is malformed and no artifact is listed', async () => {
const service = createService({manifestBody: '{not json', objectKeys: []});
await expect(service.resolveLatestDesktopKey({...LATEST_PARAMS})).resolves.toBeNull();
});
it('falls back to the object listing when the manifest parses to an array', async () => {
const service = createService({manifestBody: '[]'});
await expect(service.resolveLatestDesktopKey({...LATEST_PARAMS})).resolves.toBe(`${PREFIX}/${LISTED_FILENAME}`);
});
it('still resolves through a well-formed manifest', async () => {
const service = createService({
manifestBody: JSON.stringify({
channel: 'stable',
platform: 'darwin',
arch: 'x64',
version: '1.3.0',
pub_date: '2026-08-17T00:00:00Z',
files: {dmg: MANIFEST_FILENAME},
}),
objectKeys: [`${PREFIX}/${LISTED_FILENAME}`, `${PREFIX}/${MANIFEST_FILENAME}`],
});
await expect(service.resolveLatestDesktopKey({...LATEST_PARAMS})).resolves.toBe(`${PREFIX}/${MANIFEST_FILENAME}`);
});
});
@@ -1,340 +0,0 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import {createHash} from 'node:crypto';
import {Readable} from 'node:stream';
import {getConfig} from '@app/api/Config';
import {DownloadService} from '@app/api/download/DownloadService';
import type {IStorageService} from '@app/api/infrastructure/IStorageService';
import {S3ServiceException} from '@aws-sdk/client-s3';
import {describe, expect, it} from 'vitest';
const PREFIX = 'desktop/canary/linux/x64';
const TEST_PREFIX = 'desktop-test/canary/linux/x64';
const RELEASES_PREFIX = 'desktop/canary/github-releases';
const SOURCE_SHA = 'b'.repeat(40);
const V904 = '2026.904.135113';
const V908 = '2026.908.173325';
const V909 = '2026.909.202036';
const LATEST_PARAMS = {channel: 'canary', plat: 'linux', arch: 'x64'} as const;
const APPIMAGE_PARAMS = {...LATEST_PARAMS, format: 'appimage'} as const;
const RELEASE_ROUTES: ReadonlyArray<readonly [string, string, number]> = [
['darwin', 'arm64', 4],
['darwin', 'x64', 4],
['linux', 'arm64', 4],
['linux', 'x64', 4],
['win32', 'arm64', 6],
['win32', 'x64', 6],
];
type StoredObjects = Map<string, string>;
function sha256Hex(value: string): string {
return createHash('sha256').update(value).digest('hex');
}
function appImageFilename(version: string): string {
return `Fluxer-Canary-${version}-linux-x86_64.AppImage`;
}
function uploadBuild(objects: StoredObjects, version: string, options: {prefix?: string; checksum?: boolean} = {}) {
const prefix = options.prefix ?? PREFIX;
const filename = appImageFilename(version);
objects.set(`${prefix}/${filename}`, filename);
if (options.checksum !== false) {
objects.set(`${prefix}/${filename}.sha256`, `${sha256Hex(filename)} ${filename}`);
}
objects.set(
`${prefix}/manifest.json`,
JSON.stringify({
channel: 'canary',
platform: 'linux',
arch: 'x64',
version,
pub_date: '2026-09-08T18:06:00Z',
files: {appimage: {filename, sha256: sha256Hex(filename)}},
}),
);
}
function publishDescriptor(objects: StoredObjects, version: string, routes = RELEASE_ROUTES): string {
const assets = routes.flatMap(([plat, arch, count]) =>
Array.from({length: count}, (_, index) => {
const filename =
plat === 'linux' && arch === 'x64' && index === 0
? appImageFilename(version)
: `Fluxer-Canary-${version}-${plat}-${arch}-${index}.bin`;
return {
storage_key: `desktop/canary/${plat}/${arch}/${filename}`,
release_asset: filename,
sha256: sha256Hex(filename),
size: 1,
};
}),
);
const descriptor = JSON.stringify({
schema_version: 1,
channel: 'canary',
version,
release_tag: `fluxer-desktop-canary@${version}`,
source_sha: SOURCE_SHA,
assets,
});
objects.set(`${RELEASES_PREFIX}/${version}.json`, descriptor);
return descriptor;
}
function publishMarker(objects: StoredObjects, version: string, descriptor: string) {
objects.set(
`${RELEASES_PREFIX}/${version}.ready.json`,
JSON.stringify({
schema_version: 1,
channel: 'canary',
version,
release_tag: `fluxer-desktop-canary@${version}`,
source_sha: SOURCE_SHA,
descriptor_sha256: sha256Hex(descriptor),
}),
);
}
function releaseBuild(objects: StoredObjects, version: string) {
const descriptor = publishDescriptor(objects, version);
uploadBuild(objects, version);
publishMarker(objects, version, descriptor);
}
function incidentObjects(): StoredObjects {
const objects: StoredObjects = new Map();
releaseBuild(objects, V904);
publishDescriptor(objects, V908);
uploadBuild(objects, V908);
return objects;
}
function createService(objects: StoredObjects, onRead?: (key: string) => void) {
const reads: Array<string> = [];
const listings: Array<string> = [];
const storageService = {
streamObject: async (params: {key: string}) => {
reads.push(params.key);
onRead?.(params.key);
const body = objects.get(params.key);
if (body == null) {
return null;
}
const buffer = Buffer.from(body, 'utf8');
return {body: Readable.from([buffer]), contentLength: buffer.byteLength};
},
listObjects: async (params: {prefix: string}) => {
listings.push(params.prefix);
return Array.from(objects.keys())
.filter((key) => key.startsWith(params.prefix))
.sort()
.map((key) => ({key}));
},
getObjectMetadata: async (_bucket: string, key: string) =>
objects.has(key) ? {contentLength: 1, contentType: 'application/octet-stream'} : null,
} as unknown as IStorageService;
return {service: new DownloadService(storageService), reads, listings};
}
async function resolveLatest(service: DownloadService, test?: boolean) {
const metadata = await service.getLatestDesktopVersion({...LATEST_PARAMS, test});
const key = await service.resolveLatestDesktopKey({...APPIMAGE_PARAMS, test});
const checksum = await service.resolveLatestDesktopChecksumFile({...APPIMAGE_PARAMS, test});
return {version: metadata?.version, key, checksum: checksum?.body};
}
function latestOf(version: string, prefix = PREFIX) {
const filename = appImageFilename(version);
return {version, key: `${prefix}/${filename}`, checksum: `${sha256Hex(filename)} ${filename}\n`};
}
describe('desktop release readiness', () => {
it('offers a published manifest version after reading only its release state', async () => {
const objects: StoredObjects = new Map();
releaseBuild(objects, V904);
releaseBuild(objects, V909);
const {service, reads, listings} = createService(objects);
await expect(service.getLatestDesktopVersion({...LATEST_PARAMS})).resolves.toMatchObject({version: V909});
expect(reads).toEqual([
`${PREFIX}/manifest.json`,
`${RELEASES_PREFIX}/${V909}.json`,
`${RELEASES_PREFIX}/${V909}.ready.json`,
]);
expect(listings).toEqual([]);
await expect(resolveLatest(service)).resolves.toEqual(latestOf(V909));
});
it('falls back to the newest published version while the manifest version awaits its release', async () => {
const {service} = createService(incidentObjects());
await expect(resolveLatest(service)).resolves.toEqual(latestOf(V904));
});
it('reads each release state once and one checksum while the manifest version awaits its release', async () => {
const {service, reads, listings} = createService(incidentObjects());
await expect(service.getLatestDesktopVersion({...LATEST_PARAMS})).resolves.toMatchObject({version: V904});
expect(reads).toEqual([
`${PREFIX}/manifest.json`,
`${RELEASES_PREFIX}/${V908}.json`,
`${RELEASES_PREFIX}/${V908}.ready.json`,
`${RELEASES_PREFIX}/${V904}.json`,
`${RELEASES_PREFIX}/${V904}.ready.json`,
`${PREFIX}/${appImageFilename(V904)}.sha256`,
]);
expect(listings).toEqual([`${PREFIX}/`]);
});
it('offers a manifest version that has no release descriptor', async () => {
const objects: StoredObjects = new Map();
uploadBuild(objects, V904);
uploadBuild(objects, V908);
const {service} = createService(objects);
await expect(resolveLatest(service)).resolves.toEqual(latestOf(V908));
});
it('treats a readiness marker that does not match the stored descriptor as unpublished', async () => {
const objects: StoredObjects = new Map();
releaseBuild(objects, V904);
publishDescriptor(objects, V908);
uploadBuild(objects, V908);
publishMarker(objects, V908, 'another descriptor');
const {service} = createService(objects);
await expect(service.resolveGitHubDesktopRelease(`${PREFIX}/${appImageFilename(V908)}`)).resolves.toEqual({
kind: 'awaiting_release',
});
await expect(resolveLatest(service)).resolves.toEqual(latestOf(V904));
});
it('offers a version whose descriptor the parser rejects when its readiness marker matches', async () => {
const objects: StoredObjects = new Map();
releaseBuild(objects, V904);
const descriptor = publishDescriptor(
objects,
V908,
RELEASE_ROUTES.map(([plat, arch, count]) => [plat, arch, plat === 'linux' ? count - 1 : count] as const),
);
uploadBuild(objects, V908);
publishMarker(objects, V908, descriptor);
const {service} = createService(objects);
await expect(resolveLatest(service)).resolves.toEqual(latestOf(V908));
await expect(service.resolveGitHubDesktopRelease(`${PREFIX}/${appImageFilename(V908)}`)).rejects.toThrow(
'Invalid GitHub desktop release descriptor',
);
});
it.each([
['descriptor', `${RELEASES_PREFIX}/${V908}.json`],
['readiness marker', `${RELEASES_PREFIX}/${V908}.ready.json`],
])(
'offers the manifest version when reading its release %s fails with a storage error',
async (_name, failingKey) => {
const {service} = createService(incidentObjects(), (key) => {
if (key === failingKey) {
throw new S3ServiceException({
name: 'SlowDown',
$fault: 'server',
$metadata: {httpStatusCode: 503},
message: 'Please reduce your request rate.',
});
}
});
await expect(resolveLatest(service)).resolves.toEqual(latestOf(V908));
},
);
it('still resolves the unpublished version through versioned routes', async () => {
const objects: StoredObjects = new Map();
releaseBuild(objects, V904);
publishDescriptor(objects, V908);
uploadBuild(objects, V908, {checksum: false});
const {service} = createService(objects);
const params = {...APPIMAGE_PARAMS, version: V908};
const filename = appImageFilename(V908);
await expect(service.resolveVersionedDesktopKey(params)).resolves.toBe(`${PREFIX}/${filename}`);
await expect(service.resolveVersionedDesktopChecksumFile(params)).resolves.toMatchObject({
sha256: sha256Hex(filename),
});
});
it('keeps offering the manifest version on self-hosted instances', async () => {
const config = getConfig();
const originalSelfHosted = config.instance.selfHosted;
config.instance.selfHosted = true;
try {
const {service, reads} = createService(incidentObjects());
await expect(resolveLatest(service)).resolves.toEqual(latestOf(V908));
expect(reads.filter((key) => key.startsWith(RELEASES_PREFIX))).toEqual([]);
} finally {
config.instance.selfHosted = originalSelfHosted;
}
});
it('keeps offering the newest test build', async () => {
const objects: StoredObjects = new Map();
publishDescriptor(objects, V908);
uploadBuild(objects, V908, {prefix: TEST_PREFIX});
const {service, reads} = createService(objects);
await expect(resolveLatest(service, true)).resolves.toEqual(latestOf(V908, TEST_PREFIX));
expect(reads.filter((key) => key.startsWith(RELEASES_PREFIX))).toEqual([]);
});
it('offers 904 while 908 awaits its release, then 909 once its marker lands', async () => {
const objects = incidentObjects();
const {service} = createService(objects);
await expect(resolveLatest(service)).resolves.toEqual(latestOf(V904));
await expect(service.resolveGitHubDesktopRelease(`${PREFIX}/${appImageFilename(V908)}`)).resolves.toEqual({
kind: 'awaiting_release',
});
const descriptor = publishDescriptor(objects, V909);
uploadBuild(objects, V909);
await expect(resolveLatest(service)).resolves.toEqual(latestOf(V904));
publishMarker(objects, V909, descriptor);
await expect(resolveLatest(service)).resolves.toEqual(latestOf(V909));
await expect(service.resolveGitHubDesktopRelease(`${PREFIX}/${appImageFilename(V909)}`)).resolves.toEqual({
kind: 'ready',
location: `https://github.com/fluxerapp/fluxer/releases/download/${encodeURIComponent(`fluxer-desktop-canary@${V909}`)}/${appImageFilename(V909)}`,
});
});
it('offers the newest version when ten unpublished versions hide a published one', async () => {
const objects: StoredObjects = new Map();
releaseBuild(objects, V904);
const newest = '2026.908.170009';
for (let build = 0; build < 10; build++) {
const version = `2026.908.${170000 + build}`;
publishDescriptor(objects, version);
uploadBuild(objects, version);
}
const {service, reads} = createService(objects);
await expect(resolveLatest(service)).resolves.toEqual(latestOf(newest));
expect(reads).not.toContain(`${RELEASES_PREFIX}/${V904}.ready.json`);
});
it('pairs the latest checksum with the filename of the same version when a marker lands mid-request', async () => {
const objects = incidentObjects();
const descriptor = publishDescriptor(objects, V909);
uploadBuild(objects, V909);
let manifestReads = 0;
const {service} = createService(objects, (key) => {
if (key !== `${PREFIX}/manifest.json`) {
return;
}
manifestReads += 1;
if (manifestReads === 2) {
publishMarker(objects, V909, descriptor);
}
});
const checksum = await service.resolveLatestDesktopChecksumFile({...APPIMAGE_PARAMS});
expect(checksum?.body).toBe(latestOf(V904).checksum);
});
it('lists an unpublished version while latest skips it', async () => {
const {service} = createService(incidentObjects());
const listed = await service.listDesktopVersions({...LATEST_PARAMS, limit: 10});
expect(listed.versions.map((entry) => entry.version)).toEqual([V908, V904]);
await expect(resolveLatest(service)).resolves.toEqual(latestOf(V904));
});
});
@@ -1,143 +0,0 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import {Readable} from 'node:stream';
import {DownloadService} from '@app/api/download/DownloadService';
import type {IStorageService} from '@app/api/infrastructure/IStorageService';
import {describe, expect, it} from 'vitest';
const PREFIX = 'desktop/canary/linux/x64';
const BASE_URL = 'https://api.example.test';
const V1 = '2026.901.100000';
const V2 = '2026.902.100000';
const V3 = '2026.903.100000';
const V4 = '2026.904.100000';
const V5 = '2026.905.100000';
const LIST_PARAMS = {channel: 'canary', plat: 'linux', arch: 'x64', baseUrl: BASE_URL} as const;
type StoredObject = {body?: string; lastModified?: Date};
type StoredObjects = Map<string, StoredObject>;
function appImageFilename(version: string): string {
return `Fluxer-Canary-${version}-linux-x86_64.AppImage`;
}
function debFilename(version: string): string {
return `Fluxer-Canary-${version}-linux-amd64.deb`;
}
function addArtifact(objects: StoredObjects, filename: string, options: {sha256?: string; lastModified?: Date} = {}) {
objects.set(`${PREFIX}/${filename}`, {lastModified: options.lastModified});
if (options.sha256 !== undefined) {
objects.set(`${PREFIX}/${filename}.sha256`, {body: `${options.sha256} ${filename}\n`});
}
}
function createService(objects: StoredObjects) {
const reads: Array<string> = [];
const storageService = {
streamObject: async (params: {key: string}) => {
reads.push(params.key);
const object = objects.get(params.key);
if (object?.body == null) {
return null;
}
const buffer = Buffer.from(object.body, 'utf8');
return {body: Readable.from([buffer]), contentLength: buffer.byteLength};
},
listObjects: async (params: {prefix: string}) =>
Array.from(objects.entries())
.filter(([key]) => key.startsWith(params.prefix))
.sort(([left], [right]) => (left < right ? -1 : 1))
.map(([key, object]) => ({key, lastModified: object.lastModified})),
getObjectMetadata: async () => null,
} as unknown as IStorageService;
return {service: new DownloadService(storageService), reads};
}
function versionNumbers(versions: Array<{version: string}>): Array<string> {
return versions.map((entry) => entry.version);
}
describe('desktop version listing', () => {
it('lists versions newest first with the files of each version', async () => {
const objects: StoredObjects = new Map();
addArtifact(objects, appImageFilename(V1), {lastModified: new Date('2026-09-01T10:00:00Z')});
addArtifact(objects, appImageFilename(V3), {lastModified: new Date('2026-09-03T10:00:00Z')});
addArtifact(objects, debFilename(V3), {lastModified: new Date('2026-09-03T12:00:00Z')});
addArtifact(objects, appImageFilename(V5), {lastModified: new Date('2026-09-05T10:00:00Z')});
const {service} = createService(objects);
const listed = await service.listDesktopVersions({...LIST_PARAMS, limit: 10});
expect(versionNumbers(listed.versions)).toEqual([V5, V3, V1]);
expect(listed.hasMore).toBe(false);
expect(Object.keys(listed.versions[1].files).sort()).toEqual(['appimage', 'deb']);
expect(listed.versions[1].pub_date).toBe('2026-09-03T12:00:00.000Z');
expect(listed.versions[0].files.appimage.url).toBe(`${BASE_URL}/dl/desktop/canary/linux/x64/${V5}/appimage`);
});
it('excludes names that are not artefacts for the requested coordinate', async () => {
const objects: StoredObjects = new Map();
addArtifact(objects, appImageFilename(V3), {sha256: 'a'.repeat(64)});
objects.set(`${PREFIX}/nested/${appImageFilename(V5)}`, {});
objects.set(`${PREFIX}/manifest.json`, {body: '{}'});
objects.set(`${PREFIX}/RELEASES.json`, {body: '{}'});
objects.set(`${PREFIX}/releases.json`, {body: '{}'});
objects.set(`${PREFIX}/latest-linux.yml`, {body: 'version: 1'});
objects.set(`${PREFIX}/${appImageFilename(V4)}.blockmap`, {});
objects.set(`${PREFIX}/Fluxer-Canary-${V4}-linux-aarch64.AppImage`, {});
objects.set(`${PREFIX}/Fluxer-Canary-${V4}-mac-universal.dmg`, {});
const {service} = createService(objects);
const listed = await service.listDesktopVersions({...LIST_PARAMS, limit: 10});
expect(versionNumbers(listed.versions)).toEqual([V3]);
expect(Object.keys(listed.versions[0].files)).toEqual(['appimage']);
});
it('pages with limit, before and after and reports whether more remain', async () => {
const objects: StoredObjects = new Map();
for (const version of [V1, V2, V3, V4, V5]) {
addArtifact(objects, appImageFilename(version));
}
const {service} = createService(objects);
const firstPage = await service.listDesktopVersions({...LIST_PARAMS, limit: 2});
expect(versionNumbers(firstPage.versions)).toEqual([V5, V4]);
expect(firstPage.hasMore).toBe(true);
const olderPage = await service.listDesktopVersions({...LIST_PARAMS, limit: 2, before: V3});
expect(versionNumbers(olderPage.versions)).toEqual([V2, V1]);
expect(olderPage.hasMore).toBe(false);
const newerPage = await service.listDesktopVersions({...LIST_PARAMS, limit: 2, after: V3});
expect(versionNumbers(newerPage.versions)).toEqual([V5, V4]);
expect(newerPage.hasMore).toBe(false);
const between = await service.listDesktopVersions({...LIST_PARAMS, limit: 1, before: V5, after: V1});
expect(versionNumbers(between.versions)).toEqual([V4]);
expect(between.hasMore).toBe(true);
});
it('reports the sibling hash and treats a missing or malformed one as absent', async () => {
const hash = 'b'.repeat(64);
const objects: StoredObjects = new Map();
addArtifact(objects, appImageFilename(V3), {sha256: hash});
addArtifact(objects, appImageFilename(V2));
addArtifact(objects, appImageFilename(V1), {sha256: 'C'.repeat(64)});
const {service} = createService(objects);
const listed = await service.listDesktopVersions({...LIST_PARAMS, limit: 10});
expect(listed.versions[0].files.appimage).toEqual({
url: `${BASE_URL}/dl/desktop/canary/linux/x64/${V3}/appimage`,
sha256: hash,
checksum_url: `${BASE_URL}/dl/desktop/canary/linux/x64/${V3}/appimage.sha256`,
});
expect(listed.versions[1].files.appimage.sha256).toBeNull();
expect(listed.versions[1].files.appimage.checksum_url).toBeNull();
expect(listed.versions[2].files.appimage.sha256).toBeNull();
expect(listed.versions[2].files.appimage.checksum_url).toBeNull();
});
it('reads a checksum only for the versions it returns', async () => {
const objects: StoredObjects = new Map();
for (const version of [V1, V2, V3, V4, V5]) {
addArtifact(objects, appImageFilename(version), {sha256: 'd'.repeat(64)});
}
const {service, reads} = createService(objects);
await service.listDesktopVersions({...LIST_PARAMS, limit: 2});
expect(reads).toEqual([`${PREFIX}/${appImageFilename(V5)}.sha256`, `${PREFIX}/${appImageFilename(V4)}.sha256`]);
});
});
@@ -1,62 +0,0 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import {Readable} from 'node:stream';
import {Config} from '@app/api/Config';
import {DownloadService} from '@app/api/download/DownloadService';
import type {IStorageService} from '@app/api/infrastructure/IStorageService';
import {describe, expect, it} from 'vitest';
const PREFIX = 'desktop/stable/darwin/x64';
const MANIFEST_KEY = `${PREFIX}/manifest.json`;
const FILENAME = 'Fluxer-1.3.0-mac-universal.dmg';
const SHA256 = 'a'.repeat(64);
const LATEST_PARAMS = {
channel: 'stable',
plat: 'darwin',
arch: 'x64',
} as const;
const MANIFEST_BODY = JSON.stringify({
channel: 'stable',
platform: 'darwin',
arch: 'x64',
version: '1.3.0',
pub_date: '2026-08-17T00:00:00Z',
files: {dmg: {filename: FILENAME, sha256: SHA256}},
});
function createService() {
const objectKeys = [`${PREFIX}/${FILENAME}`];
const storageService = {
streamObject: async (params: {key: string}) => {
if (params.key !== MANIFEST_KEY) {
return null;
}
const buffer = Buffer.from(MANIFEST_BODY, 'utf8');
return {body: Readable.from([buffer]), contentLength: buffer.byteLength};
},
listObjects: async () => objectKeys.map((key) => ({key})),
getObjectMetadata: async (_bucket: string, key: string) =>
objectKeys.includes(key) ? {contentLength: 1, contentType: 'application/x-apple-diskimage'} : null,
} as unknown as IStorageService;
return new DownloadService(storageService);
}
describe('desktop download base url', () => {
it('builds artifact urls from the configured client API endpoint', async () => {
const version = await createService().getLatestDesktopVersion({...LATEST_PARAMS});
const base = Config.endpoints.apiClient.replace(/\/+$/u, '');
expect(base.length).toBeGreaterThan(0);
expect(version?.files.dmg?.url).toBe(`${base}/dl/desktop/stable/darwin/x64/1.3.0/dmg`);
expect(version?.files.dmg?.checksum_url).toBe(`${base}/dl/desktop/stable/darwin/x64/1.3.0/dmg.sha256`);
});
it('prefers an explicit base url and strips its trailing slashes', async () => {
const version = await createService().getLatestDesktopVersion({
...LATEST_PARAMS,
baseUrl: 'https://chat.example.com/api//',
});
expect(version?.files.dmg?.url).toBe('https://chat.example.com/api/dl/desktop/stable/darwin/x64/1.3.0/dmg');
});
});
@@ -8,9 +8,9 @@ import {RateLimitConfigs} from '@app/api/RateLimitConfig';
import type {HonoApp} from '@app/api/types/HonoEnv';
import {entityTagMatches} from '@app/api/utils/EntityTag';
import {Headers as HttpHeaders} from '@fluxer/constants/src/Headers';
import {resolveScreenShareDeliveryAssignment} from '@fluxer/schema/src/domains/admin/ScreenShareDeliverySchemas';
import {resolveVoiceNoiseSuppressionAssignment} from '@fluxer/schema/src/domains/admin/VoiceNoiseSuppressionSchemas';
import {ExperimentAssignmentsResponse} from '@fluxer/schema/src/domains/experiment/ExperimentSchemas';
import {resolveScreenShareDeliveryAssignment} from '@fluxer/schema/src/domains/experiment/ScreenShareDeliverySchemas';
export function ExperimentController(app: HonoApp) {
app.get(
@@ -29,7 +29,7 @@ export function ExperimentController(app: HonoApp) {
}),
async (ctx) => {
const instanceConfigRepository = ctx.get('instanceConfigRepository');
const [delivery, voiceConfig, screenShareDeliveryConfig] = await Promise.all([
const [delivery, voiceConfig, screenShareConfig] = await Promise.all([
instanceConfigRepository.getExperimentDeliveryConfig(),
instanceConfigRepository.getVoiceNoiseSuppressionConfig(),
instanceConfigRepository.getScreenShareDeliveryConfig(),
@@ -40,7 +40,7 @@ export function ExperimentController(app: HonoApp) {
poll_jitter_percent: delivery.poll_jitter_percent,
assignments: {
voice_noise_suppression: resolveVoiceNoiseSuppressionAssignment(voiceConfig, userId),
screen_share_delivery: resolveScreenShareDeliveryAssignment(screenShareDeliveryConfig, userId),
screen_share_delivery: resolveScreenShareDeliveryAssignment(screenShareConfig, userId),
},
};
const etag = `"${createHash('sha256').update(JSON.stringify(body)).digest('hex')}"`;
@@ -6,6 +6,10 @@ import {type ApiTestHarness, createApiTestHarness} from '@app/api/test/ApiTestHa
import {HTTP_STATUS} from '@app/api/test/TestConstants';
import {createBuilder, createBuilderWithoutAuth} from '@app/api/test/TestRequestBuilder';
import {AdminACLs} from '@fluxer/constants/src/AdminACLs';
import {
DEFAULT_SCREEN_SHARE_DELIVERY_CONFIG,
INERT_SCREEN_SHARE_DELIVERY_ASSIGNMENT,
} from '@fluxer/schema/src/domains/admin/ScreenShareDeliverySchemas';
import {
DEFAULT_VOICE_NOISE_SUPPRESSION_CONFIG,
INERT_VOICE_NOISE_SUPPRESSION_ASSIGNMENT,
@@ -15,12 +19,9 @@ import {
DEFAULT_EXPERIMENT_POLL_JITTER_PERCENT,
type ExperimentAssignmentsResponse,
type ExperimentDeliveryConfigResponse,
readScreenShareDeliveryAssignment,
readVoiceNoiseSuppressionAssignment,
} from '@fluxer/schema/src/domains/experiment/ExperimentSchemas';
import {
INERT_SCREEN_SHARE_DELIVERY_ASSIGNMENT,
ScreenShareDeliveryConfigSchema,
} from '@fluxer/schema/src/domains/experiment/ScreenShareDeliverySchemas';
import {afterAll, beforeAll, beforeEach, describe, expect, it} from 'vitest';
const NOT_MODIFIED = 304;
@@ -87,6 +88,52 @@ describe('GET /experiments', () => {
expect(readVoiceNoiseSuppressionAssignment(body).enabled).toBe(false);
});
it('populates the screen share assignment key even when the rollout is disabled', async () => {
const account = await createTestAccount(harness);
const body = await createBuilder<ExperimentAssignmentsResponse>(harness, account.token).get(ENDPOINT).execute();
expect(Object.hasOwn(body.assignments, 'screen_share_delivery')).toBe(true);
expect(readScreenShareDeliveryAssignment(body).enabled).toBe(false);
});
it('resolves the screen share caller through the allowlist', async () => {
const targeted = await createTestAccount(harness);
const untargeted = await createTestAccount(harness);
await getInstanceConfigRepository().setScreenShareDeliveryConfig({
...DEFAULT_SCREEN_SHARE_DELIVERY_CONFIG,
enabled: true,
config_version: 4,
rollout_basis_points: 0,
included_user_ids: [targeted.userId],
});
const targetedBody = await createBuilder<ExperimentAssignmentsResponse>(harness, targeted.token)
.get(ENDPOINT)
.execute();
expect(targetedBody.assignments.screen_share_delivery).toEqual({enabled: true});
const untargetedBody = await createBuilder<ExperimentAssignmentsResponse>(harness, untargeted.token)
.get(ENDPOINT)
.execute();
expect(untargetedBody.assignments.screen_share_delivery).toEqual({enabled: false});
});
it('keeps the screen share exclusion ahead of a full rollout', async () => {
const excluded = await createTestAccount(harness);
await getInstanceConfigRepository().setScreenShareDeliveryConfig({
...DEFAULT_SCREEN_SHARE_DELIVERY_CONFIG,
enabled: true,
rollout_basis_points: 10000,
included_user_ids: [excluded.userId],
excluded_user_ids: [excluded.userId],
});
const body = await createBuilder<ExperimentAssignmentsResponse>(harness, excluded.token).get(ENDPOINT).execute();
expect(body.assignments.screen_share_delivery).toEqual({enabled: false});
});
it('serves the delivery cadence from the delivery config and not from the voice config', async () => {
const account = await createTestAccount(harness);
await getInstanceConfigRepository().setExperimentDeliveryConfig({
@@ -144,92 +191,6 @@ describe('GET /experiments', () => {
});
});
it('resolves the screen share delivery caller through its own config', async () => {
const targeted = await createTestAccount(harness);
const untargeted = await createTestAccount(harness);
await getInstanceConfigRepository().setScreenShareDeliveryConfig(
ScreenShareDeliveryConfigSchema.parse({
enabled: true,
config_version: 21,
rollout_basis_points: 0,
included_user_ids: [targeted.userId],
}),
);
const targetedBody = await createBuilder<ExperimentAssignmentsResponse>(harness, targeted.token)
.get(ENDPOINT)
.execute();
expect(targetedBody.assignments.screen_share_delivery).toEqual({
enabled: true,
config_version: 21,
user_targeted: true,
source: 'user_rule',
});
const untargetedBody = await createBuilder<ExperimentAssignmentsResponse>(harness, untargeted.token)
.get(ENDPOINT)
.execute();
expect(untargetedBody.assignments.screen_share_delivery).toEqual({
enabled: true,
config_version: 21,
user_targeted: false,
source: null,
});
});
it('excludes a screen share delivery user the allowlist and the canary both hold', async () => {
const account = await createTestAccount(harness);
await getInstanceConfigRepository().setScreenShareDeliveryConfig(
ScreenShareDeliveryConfigSchema.parse({
enabled: true,
config_version: 3,
rollout_basis_points: 10000,
included_user_ids: [account.userId],
excluded_user_ids: [account.userId],
}),
);
const body = await createBuilder<ExperimentAssignmentsResponse>(harness, account.token).get(ENDPOINT).execute();
expect(body.assignments.screen_share_delivery).toEqual({
enabled: true,
config_version: 3,
user_targeted: false,
source: null,
});
});
it('bumps the screen share delivery config version on an admin update and serves it', async () => {
const admin = await setUserACLs(harness, await createTestAccount(harness), [
AdminACLs.AUTHENTICATE,
AdminACLs.INSTANCE_CONFIG_VIEW,
AdminACLs.INSTANCE_CONFIG_UPDATE,
]);
const patched = await createBuilder<{
screen_share_delivery: {config_version: number; enabled: boolean; rollout_basis_points: number};
voice_noise_suppression: {config_version: number};
}>(harness, admin.token)
.patch('/admin/instance/config')
.body({screen_share_delivery: {enabled: true, rollout_basis_points: 10000}})
.execute();
expect(patched.screen_share_delivery).toMatchObject({
config_version: 1,
enabled: true,
rollout_basis_points: 10000,
});
expect(patched.voice_noise_suppression.config_version).toBe(0);
const body = await createBuilder<ExperimentAssignmentsResponse>(harness, admin.token).get(ENDPOINT).execute();
expect(body.assignments.screen_share_delivery).toMatchObject({
enabled: true,
config_version: 1,
user_targeted: true,
source: 'canary',
});
expect(body.assignments.voice_noise_suppression).toEqual(INERT_VOICE_NOISE_SUPPRESSION_ASSIGNMENT);
});
it('revalidates with a strong etag and answers 304 when nothing changed', async () => {
const account = await createTestAccount(harness);
@@ -287,6 +248,30 @@ describe('GET /experiments', () => {
});
});
it('serves a fresh body once the screen share config changes', async () => {
const account = await createTestAccount(harness);
const first = await createBuilder<ExperimentAssignmentsResponse>(harness, account.token)
.get(ENDPOINT)
.executeWithResponse();
const staleEtag = first.response.headers.get('etag') as string;
await getInstanceConfigRepository().setScreenShareDeliveryConfig({
...DEFAULT_SCREEN_SHARE_DELIVERY_CONFIG,
enabled: true,
config_version: 1,
rollout_basis_points: 10000,
});
const refreshed = await createBuilder<ExperimentAssignmentsResponse>(harness, account.token)
.get(ENDPOINT)
.header('If-None-Match', staleEtag)
.executeWithResponse();
expect(refreshed.response.status).toBe(HTTP_STATUS.OK);
expect(refreshed.response.headers.get('etag')).not.toBe(staleEtag);
expect(refreshed.json?.assignments.screen_share_delivery).toEqual({enabled: true});
});
it('serves a fresh body once the delivery config changes', async () => {
const account = await createTestAccount(harness);
@@ -343,6 +328,44 @@ describe('GET /experiments', () => {
});
});
it('bumps the screen share config version on every admin update without the client sending one', async () => {
const admin = await setUserACLs(harness, await createTestAccount(harness), [
AdminACLs.AUTHENTICATE,
AdminACLs.INSTANCE_CONFIG_VIEW,
AdminACLs.INSTANCE_CONFIG_UPDATE,
]);
const afterFirst = await createBuilder<{screen_share_delivery: {config_version: number; enabled: boolean}}>(
harness,
admin.token,
)
.patch('/admin/instance/config')
.body({screen_share_delivery: {enabled: true, rollout_basis_points: 10000}})
.execute();
expect(afterFirst.screen_share_delivery).toMatchObject({config_version: 1, enabled: true});
const afterSecond = await createBuilder<{screen_share_delivery: {config_version: number; enabled: boolean}}>(
harness,
admin.token,
)
.patch('/admin/instance/config')
.body({screen_share_delivery: {rollout_salt: 'screen-share-delivery-v2'}})
.execute();
expect(afterSecond.screen_share_delivery).toMatchObject({config_version: 2, enabled: true});
const afterEmpty = await createBuilder<{screen_share_delivery: {config_version: number; enabled: boolean}}>(
harness,
admin.token,
)
.patch('/admin/instance/config')
.body({screen_share_delivery: {}})
.execute();
expect(afterEmpty.screen_share_delivery).toMatchObject({config_version: 2, enabled: true});
const body = await createBuilder<ExperimentAssignmentsResponse>(harness, admin.token).get(ENDPOINT).execute();
expect(body.assignments.screen_share_delivery).toEqual({enabled: true});
});
it('leaves the config version alone for an admin update that sets no field', async () => {
const admin = await setUserACLs(harness, await createTestAccount(harness), [
AdminACLs.AUTHENTICATE,
@@ -15,6 +15,8 @@ import {AuditLogActionType} from '@fluxer/constants/src/AuditLogActionType';
import type {IWorkerService} from '@pkgs/worker/src/contracts/IWorkerService';
import {ms} from 'itty-time';
const MESSAGE_DELETE_BATCH_DELAY_MS = ms('30 seconds');
interface MessageDeleteBatchGroup {
logs: Array<GuildAuditLog>;
userId: UserID;
@@ -81,14 +83,15 @@ export class GuildAuditLogService {
}
async scheduleMessageDeleteBatchJob(guildId: GuildID): Promise<void> {
const runAt = new Date(Date.now() + ms('30 seconds'));
const batchWindow = Math.floor(Date.now() / MESSAGE_DELETE_BATCH_DELAY_MS);
await this.workerService.addJob(
'batchGuildAuditLogMessageDeletes',
{guildId: guildId.toString()},
{
jobKey: `batch-audit-log-message-deletes:${guildId}`,
runAt,
jobKey: `batch-audit-log-message-deletes:${guildId}:${batchWindow}`,
runAt: new Date((batchWindow + 2) * MESSAGE_DELETE_BATCH_DELAY_MS),
maxAttempts: 3,
skipLedger: true,
},
);
}
@@ -60,7 +60,7 @@ function createService(roleNames: Map<string, string> = new Map()) {
{addJob} as unknown as IWorkerService<WorkerTaskName>,
{dispatchGuild} as unknown as IGatewayService,
);
return {service, createAuditLog, batchDeleteAndCreateAuditLogs, getRole, dispatchGuild};
return {service, createAuditLog, batchDeleteAndCreateAuditLogs, getRole, dispatchGuild, addJob};
}
function overwrites(
@@ -376,3 +376,26 @@ describe('GuildAuditLogService.recordPermissionOverwriteDiff', () => {
expect(dispatchGuild).not.toHaveBeenCalled();
});
});
describe('GuildAuditLogService.scheduleMessageDeleteBatchJob', () => {
it('gives every delete in one 30 second window a single batch job that runs after the window closes, without a ledger row', async () => {
vi.useFakeTimers({toFake: ['Date']});
try {
const {service, addJob} = createService();
for (const at of ['2026-09-21T12:00:00.000Z', '2026-09-21T12:00:29.999Z', '2026-09-21T12:00:40.000Z']) {
vi.setSystemTime(new Date(at));
await service.scheduleMessageDeleteBatchJob(GUILD_ID);
}
const options = addJob.mock.calls.map((call) => call[2] as {jobKey: string; runAt: Date; skipLedger: boolean});
expect(options.every((option) => option.skipLedger)).toBe(true);
expect(options[0]!.jobKey).toBe(options[1]!.jobKey);
expect(options[2]!.jobKey).not.toBe(options[1]!.jobKey);
expect(options[0]!.runAt.getTime()).toBeGreaterThan(new Date('2026-09-21T12:00:29.999Z').getTime());
expect(options[1]!.runAt).toEqual(options[0]!.runAt);
expect(options[2]!.runAt).toEqual(new Date('2026-09-21T12:01:30.000Z'));
expect(options[2]!.runAt.getTime() - options[0]!.runAt.getTime()).toBe(30_000);
} finally {
vi.useRealTimers();
}
});
});
@@ -35,7 +35,7 @@ import {
} from '@aws-sdk/client-s3';
import {Upload} from '@aws-sdk/lib-storage';
import {getSignedUrl} from '@aws-sdk/s3-request-presigner';
import type {S3ProviderSettings} from '@fluxer/config/src/S3DownloadsProvider';
import type {S3ProviderSettings} from '@fluxer/config/src/S3ProviderSettings';
import {isSupportedMediaContentType} from '@pkgs/mime_utils/src/ContentTypeUtils';
import {seconds} from 'itty-time';
import {temporaryFile} from 'tempy';
@@ -1,18 +0,0 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import {Config} from '@app/api/Config';
import {createDownloadsStorageService} from '@app/api/infrastructure/StorageServiceFactory';
import {describe, expect, it} from 'vitest';
describe('createDownloadsStorageService', () => {
it('returns null when no downloads override is configured', () => {
expect(Config.s3Downloads.isOverridden).toBe(false);
expect(createDownloadsStorageService()).toBeNull();
});
it('resolves the downloads provider to the shared provider by default', () => {
expect(Config.s3Downloads.settings.endpoint).toBe(Config.s3.endpoint);
expect(Config.s3Downloads.settings.region).toBe(Config.s3.region);
expect(Config.s3Downloads.settings.accessKeyId).toBe(Config.s3.accessKeyId);
});
});
@@ -25,13 +25,6 @@ export function createStorageService(): IStorageService {
return withChangeFeed(new StorageService());
}
export function createDownloadsStorageService(): IStorageService | null {
if (!Config.s3Downloads.isOverridden) {
return null;
}
return withChangeFeed(new StorageService(Config.s3Downloads.settings));
}
export async function shutdownStorageChangeFeed(): Promise<void> {
const feed = changeFeed;
changeFeed = null;
@@ -9,6 +9,10 @@ import {
} from '@app/api/instance/InstanceConfigRepository';
import {InMemoryCassandraQueryExecutor} from '@app/api/test/InMemoryCassandraQueryExecutor';
import {MockKVProvider} from '@app/api/test/mocks/MockKVProvider';
import {
DEFAULT_SCREEN_SHARE_DELIVERY_CONFIG,
type ScreenShareDeliveryConfig,
} from '@fluxer/schema/src/domains/admin/ScreenShareDeliverySchemas';
import {
DEFAULT_VOICE_NOISE_SUPPRESSION_CONFIG,
type VoiceNoiseSuppressionConfig,
@@ -17,16 +21,11 @@ import {
DEFAULT_EXPERIMENT_DELIVERY_CONFIG,
type ExperimentDeliveryConfig,
} from '@fluxer/schema/src/domains/experiment/ExperimentSchemas';
import {
type ScreenShareDeliveryConfig,
ScreenShareDeliveryConfigSchema,
} from '@fluxer/schema/src/domains/experiment/ScreenShareDeliverySchemas';
import {afterEach, describe, expect, it, vi} from 'vitest';
const VOICE_NOISE_SUPPRESSION_CONFIG_KEY = 'voice_noise_suppression_config';
const EXPERIMENT_DELIVERY_CONFIG_KEY = 'experiment_delivery_config';
const SCREEN_SHARE_DELIVERY_CONFIG_KEY = 'screen_share_delivery_config';
const DEFAULT_SCREEN_SHARE_DELIVERY_CONFIG: ScreenShareDeliveryConfig = ScreenShareDeliveryConfigSchema.parse({});
const EXPERIMENT_DELIVERY_CONFIG_KEY = 'experiment_delivery_config';
const APP_PUBLIC_CONFIG_KEY = 'app_public_config';
const INSTANCE_POLICY_CONFIG_KEY = 'instance_policy_config';
const INSTANCE_INTEGRATIONS_CONFIG_KEY = 'instance_integrations_config';
@@ -310,7 +309,6 @@ describe('InstanceConfigRepository', () => {
included_user_ids: ['1400000000000000001'],
excluded_user_ids: ['1400000000000000002'],
guild_overrides: [{guild_id: '2400000000000000001', backend: 'rnnoise'}],
stereo_enabled: true,
suppression_strength: 55,
};
await repository.setVoiceNoiseSuppressionConfig(config);
@@ -337,6 +335,70 @@ describe('InstanceConfigRepository', () => {
});
});
it('returns the default screen share delivery config when the key is absent', async () => {
const executor = new CountingInMemoryCassandraQueryExecutor();
setCassandraQueryExecutorForTesting(executor);
const kvProvider = new MockKVProvider();
const repository = createRepository(kvProvider);
await expect(repository.getScreenShareDeliveryConfig()).resolves.toEqual(DEFAULT_SCREEN_SHARE_DELIVERY_CONFIG);
});
it.each([
{name: 'unparseable text', stored: 'not-json'},
{name: 'a json array', stored: '[]'},
{name: 'out-of-range values', stored: '{"rollout_basis_points":99999}'},
{name: 'a non-boolean enabled flag', stored: '{"enabled":"yes"}'},
])('falls back to the default screen share delivery config for $name', async ({stored}) => {
const executor = new CountingInMemoryCassandraQueryExecutor();
setCassandraQueryExecutorForTesting(executor);
const kvProvider = new MockKVProvider();
const repository = createRepository(kvProvider);
await repository.setConfig(SCREEN_SHARE_DELIVERY_CONFIG_KEY, stored);
await expect(repository.getScreenShareDeliveryConfig()).resolves.toEqual(DEFAULT_SCREEN_SHARE_DELIVERY_CONFIG);
});
it('round-trips a stored screen share delivery config', async () => {
const executor = new CountingInMemoryCassandraQueryExecutor();
setCassandraQueryExecutorForTesting(executor);
const kvProvider = new MockKVProvider();
const repository = createRepository(kvProvider);
const config: ScreenShareDeliveryConfig = {
...DEFAULT_SCREEN_SHARE_DELIVERY_CONFIG,
enabled: true,
config_version: 5,
rollout_basis_points: 2500,
rollout_salt: 'screen-share-delivery-v2',
included_user_ids: ['1400000000000000001'],
excluded_user_ids: ['1400000000000000002'],
};
await repository.setScreenShareDeliveryConfig(config);
await expect(repository.getScreenShareDeliveryConfig()).resolves.toEqual(config);
});
it('fills newly added screen share delivery fields from the schema defaults', async () => {
const executor = new CountingInMemoryCassandraQueryExecutor();
setCassandraQueryExecutorForTesting(executor);
const kvProvider = new MockKVProvider();
const repository = createRepository(kvProvider);
await repository.setConfig(
SCREEN_SHARE_DELIVERY_CONFIG_KEY,
JSON.stringify({enabled: true, config_version: 2, rollout_basis_points: 1000}),
);
await expect(repository.getScreenShareDeliveryConfig()).resolves.toEqual({
...DEFAULT_SCREEN_SHARE_DELIVERY_CONFIG,
enabled: true,
config_version: 2,
rollout_basis_points: 1000,
});
});
it('returns the default experiment delivery config when the key is absent', async () => {
const executor = new CountingInMemoryCassandraQueryExecutor();
setCassandraQueryExecutorForTesting(executor);
@@ -389,73 +451,6 @@ describe('InstanceConfigRepository', () => {
});
});
it('returns the default screen share delivery config when the key is absent', async () => {
const executor = new CountingInMemoryCassandraQueryExecutor();
setCassandraQueryExecutorForTesting(executor);
const kvProvider = new MockKVProvider();
const repository = createRepository(kvProvider);
await expect(repository.getScreenShareDeliveryConfig()).resolves.toEqual(DEFAULT_SCREEN_SHARE_DELIVERY_CONFIG);
});
it('reads the screen share delivery config from its own key', async () => {
const executor = new CountingInMemoryCassandraQueryExecutor();
setCassandraQueryExecutorForTesting(executor);
const kvProvider = new MockKVProvider();
const repository = createRepository(kvProvider);
await repository.setConfig(
SCREEN_SHARE_DELIVERY_CONFIG_KEY,
JSON.stringify({enabled: true, config_version: 4, rollout_basis_points: 2500}),
);
await repository.setVoiceNoiseSuppressionConfig({
...DEFAULT_VOICE_NOISE_SUPPRESSION_CONFIG,
enabled: false,
config_version: 77,
});
await expect(repository.getScreenShareDeliveryConfig()).resolves.toEqual({
...DEFAULT_SCREEN_SHARE_DELIVERY_CONFIG,
enabled: true,
config_version: 4,
rollout_basis_points: 2500,
});
});
it('round-trips a stored screen share delivery config', async () => {
const executor = new CountingInMemoryCassandraQueryExecutor();
setCassandraQueryExecutorForTesting(executor);
const kvProvider = new MockKVProvider();
const repository = createRepository(kvProvider);
const config: ScreenShareDeliveryConfig = {
enabled: true,
config_version: 6,
rollout_basis_points: 1000,
rollout_salt: 'screen-share-delivery-v2',
included_user_ids: ['1400000000000000001'],
excluded_user_ids: ['1400000000000000002'],
};
await repository.setScreenShareDeliveryConfig(config);
await expect(repository.getScreenShareDeliveryConfig()).resolves.toEqual(config);
await expect(repository.getConfig(SCREEN_SHARE_DELIVERY_CONFIG_KEY)).resolves.toBe(JSON.stringify(config));
});
it('fills missing screen share delivery fields from the schema defaults', async () => {
const executor = new CountingInMemoryCassandraQueryExecutor();
setCassandraQueryExecutorForTesting(executor);
const kvProvider = new MockKVProvider();
const repository = createRepository(kvProvider);
await repository.setConfig(SCREEN_SHARE_DELIVERY_CONFIG_KEY, JSON.stringify({enabled: true}));
await expect(repository.getScreenShareDeliveryConfig()).resolves.toEqual({
...DEFAULT_SCREEN_SHARE_DELIVERY_CONFIG,
enabled: true,
});
});
it('publishes a refresh so another repository observes the voice noise suppression config', async () => {
const executor = new CountingInMemoryCassandraQueryExecutor();
setCassandraQueryExecutorForTesting(executor);
@@ -476,6 +471,26 @@ describe('InstanceConfigRepository', () => {
});
});
it('publishes a refresh so another repository observes the screen share delivery config', async () => {
const executor = new CountingInMemoryCassandraQueryExecutor();
setCassandraQueryExecutorForTesting(executor);
const kvProvider = new MockKVProvider();
const reader = createRepository(kvProvider);
const writer = createRepository(kvProvider);
await expect(reader.getScreenShareDeliveryConfig()).resolves.toEqual(DEFAULT_SCREEN_SHARE_DELIVERY_CONFIG);
await writer.setScreenShareDeliveryConfig({
...DEFAULT_SCREEN_SHARE_DELIVERY_CONFIG,
enabled: true,
config_version: 1,
});
await vi.waitFor(async () => {
expect(await reader.getScreenShareDeliveryConfig()).toMatchObject({enabled: true, config_version: 1});
});
});
it('uses the registration URL id as the admin-visible registration code', async () => {
const executor = new CountingInMemoryCassandraQueryExecutor();
setCassandraQueryExecutorForTesting(executor);
@@ -28,6 +28,10 @@ import {
type GatewayRolloutConfig,
GatewayRolloutConfigSchema,
} from '@fluxer/schema/src/domains/admin/GatewayRolloutSchemas';
import {
type ScreenShareDeliveryConfig,
ScreenShareDeliveryConfigSchema,
} from '@fluxer/schema/src/domains/admin/ScreenShareDeliverySchemas';
import {
type VoiceNoiseSuppressionConfig,
VoiceNoiseSuppressionConfigSchema,
@@ -36,10 +40,6 @@ import {
type ExperimentDeliveryConfig,
ExperimentDeliveryConfigSchema,
} from '@fluxer/schema/src/domains/experiment/ExperimentSchemas';
import {
type ScreenShareDeliveryConfig,
ScreenShareDeliveryConfigSchema,
} from '@fluxer/schema/src/domains/experiment/ScreenShareDeliverySchemas';
import {
type InstanceAppPublic,
InstanceAppPublicSchema,
@@ -32,10 +32,12 @@ export interface ListJobsResult {
}
export abstract class IJobLedgerRepository {
abstract createJob(input: CreateJobInput): Promise<void>;
abstract createJob(input: CreateJobInput): Promise<Date>;
abstract getJob(jobId: bigint): Promise<JobByIdRow | null>;
abstract discardJob(jobId: bigint, createdAt: Date): Promise<void>;
abstract markRunning(jobId: bigint, lane: string): Promise<void>;
abstract markSucceeded(jobId: bigint, result: Record<string, unknown> | null): Promise<void>;
@@ -1,9 +1,24 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import {setCassandraQueryExecutorForTesting} from '@app/api/database/CassandraQueryExecution';
import {
type CassandraQueryExecutorForTesting,
executeQuery,
fetchMany,
fetchOne,
setCassandraQueryExecutorForTesting,
} from '@app/api/database/CassandraQueryExecution';
import {Db, type PreparedQuery} from '@app/api/database/CassandraTypes';
import type {JobStatus} from '@app/api/database/types/JobLedgerTypes';
import {JobLedgerRepository} from '@app/api/jobs/JobLedgerRepository';
import {
EXPIRED_JOB_ERROR,
JOB_LEDGER_TTL_SECONDS,
JOB_STALE_AFTER_MS,
JobLedgerRepository,
} from '@app/api/jobs/JobLedgerRepository';
import {describeListJobsPaging} from '@app/api/jobs/ListJobsPagingSuite';
import {JobsActive, JobsByDayBucket, JobsById} from '@app/api/Tables';
import {InMemoryCassandraQueryExecutor} from '@app/api/test/InMemoryCassandraQueryExecutor';
import {createSnowflake} from '@fluxer/snowflake/src/Snowflake';
import {afterEach, beforeEach, describe, expect, it} from 'vitest';
let executor: InMemoryCassandraQueryExecutor;
@@ -22,6 +37,20 @@ async function createJob(repository: JobLedgerRepository, jobId: bigint, taskTyp
});
}
async function createJobAt(repository: JobLedgerRepository, jobId: bigint): Promise<Date> {
return repository.createJob({
jobId,
taskType: 'batchGuildAuditLogMessageDeletes',
payload: {},
requestedByUserId: null,
auditLogReason: null,
maxAttempts: 3,
runAt: null,
jetStreamLane: null,
jetStreamSeq: null,
});
}
async function listJobIdsByStatus(repository: JobLedgerRepository, status: JobStatus): Promise<Array<bigint>> {
const result = await repository.listJobs({limit: 50, cursor: null, filters: {status}, maxLookbackDays: 1});
return result.jobs.map((job) => job.job_id);
@@ -130,6 +159,197 @@ describe('JobLedgerRepository listJobs pagination', () => {
maxLookbackDays: 1,
});
expect(result.jobs.map((job) => job.job_id)).toEqual([1_000n, 1_001n, 1_002n, 1_003n, 1_004n]);
expect(result.jobs.map((job) => job.job_id)).toEqual([1_004n, 1_003n, 1_002n, 1_001n, 1_000n]);
});
});
describe('JobLedgerRepository listJobs on the in-memory executor', () => {
beforeEach(() => {
executor = new InMemoryCassandraQueryExecutor();
setCassandraQueryExecutorForTesting(executor);
});
afterEach(() => {
executor.reset();
setCassandraQueryExecutorForTesting(null);
});
describeListJobsPaging();
});
let staleSequence = 0;
function jobIdAgedDays(days: number): bigint {
staleSequence += 1;
return createSnowflake({timestamp: Date.now() - days * 86_400_000, sequence: staleSequence % 4096, workerId: 1});
}
async function createAgedJob(repository: JobLedgerRepository, days: number): Promise<bigint> {
const jobId = jobIdAgedDays(days);
await createJob(repository, jobId, 'syncUrlBlocklists');
return jobId;
}
async function activeJobIds(repository: JobLedgerRepository): Promise<Array<bigint>> {
return (await repository.listActiveJobs()).map((job) => job.job_id).sort((a, b) => (a < b ? -1 : 1));
}
function sweep(repository: JobLedgerRepository, maxCleared = 100) {
return repository.expireStaleActiveJobs({
staleBeforeMs: Date.now() - JOB_STALE_AFTER_MS,
pageSize: 100,
maxCleared,
});
}
describe('JobLedgerRepository expireStaleActiveJobs', () => {
beforeEach(() => {
executor = new InMemoryCassandraQueryExecutor();
setCassandraQueryExecutorForTesting(executor);
});
afterEach(() => {
executor.reset();
setCassandraQueryExecutorForTesting(null);
});
it('dead-letters queued and running jobs the jobs stream has outlived and keeps younger ones active', async () => {
const repository = new JobLedgerRepository();
const staleQueued = await createAgedJob(repository, 9);
const staleRunning = await createAgedJob(repository, 9);
await repository.markRunning(staleRunning, 'batch');
const weekOld = await createAgedJob(repository, 7);
const fresh = await createAgedJob(repository, 0);
expect(await sweep(repository)).toEqual({cleared: 2, expired: 2, complete: true});
for (const jobId of [staleQueued, staleRunning]) {
const job = await repository.getJob(jobId);
expect(job?.status).toBe('deadletter');
expect(job?.error_message).toBe(EXPIRED_JOB_ERROR);
expect(job?.completed_at).toBeInstanceOf(Date);
}
expect(await activeJobIds(repository)).toEqual([weekOld, fresh].sort((a, b) => (a < b ? -1 : 1)));
expect((await repository.getJob(weekOld))?.status).toBe('queued');
});
it('drops a stale active row without touching a finished or missing job', async () => {
const repository = new JobLedgerRepository();
const finished = await createAgedJob(repository, 9);
await repository.markSucceeded(finished, null);
const orphan = jobIdAgedDays(9);
for (const jobId of [finished, orphan]) {
await executeQuery(
JobsActive.patchByPkWithTtl({job_id: jobId}, {status: Db.set('running')}, JOB_LEDGER_TTL_SECONDS),
);
}
expect(await sweep(repository)).toEqual({cleared: 2, expired: 0, complete: true});
const job = await repository.getJob(finished);
expect(job?.status).toBe('succeeded');
expect(job?.error_message).toBeNull();
expect(await fetchOne(JobsById.select({where: JobsById.where.eq('job_id')}).bind({job_id: orphan}))).toBeNull();
expect(await fetchMany(JobsActive.select().bind({}))).toEqual([]);
});
it('stops at its per-run cap and picks up the rest on the next run', async () => {
const repository = new JobLedgerRepository();
for (let index = 0; index < 3; index += 1) {
await createAgedJob(repository, 9);
}
expect(await sweep(repository, 2)).toEqual({cleared: 2, expired: 2, complete: false});
expect(await activeJobIds(repository)).toHaveLength(1);
expect(await sweep(repository, 2)).toEqual({cleared: 1, expired: 1, complete: true});
expect(await activeJobIds(repository)).toEqual([]);
});
});
describe('JobLedgerRepository getJob', () => {
beforeEach(() => {
executor = new InMemoryCassandraQueryExecutor();
setCassandraQueryExecutorForTesting(executor);
});
afterEach(() => {
executor.reset();
setCassandraQueryExecutorForTesting(null);
});
it('hides a job row that lost its status, creation time or task type', async () => {
const repository = new JobLedgerRepository();
await createJob(repository, 9n, 'syncUrlBlocklists');
expect((await repository.getJob(9n))?.status).toBe('queued');
for (const column of ['status', 'created_at', 'task_type'] as const) {
await createJob(repository, 9n, 'syncUrlBlocklists');
await executeQuery(JobsById.patchByPk({job_id: 9n}, {[column]: Db.clear()}));
expect(await repository.getJob(9n)).toBeNull();
}
});
});
describe('JobLedgerRepository discardJob', () => {
let inner: InMemoryCassandraQueryExecutor;
let log: Array<string>;
let failDeleteOn: string | null;
beforeEach(() => {
inner = new InMemoryCassandraQueryExecutor();
log = [];
failDeleteOn = null;
const wrapper: CassandraQueryExecutorForTesting = {
async executeQuery<T>(query: PreparedQuery) {
const meta = query.kvMeta;
if (meta) log.push(`${meta.action} ${meta.table.name}`);
if (meta?.action === 'delete' && meta.table.name === failDeleteOn) {
throw new Error('write timeout');
}
return inner.executeQuery<T>(query);
},
executeBatch: (queries, atomic) => inner.executeBatch(queries, atomic),
};
setCassandraQueryExecutorForTesting(wrapper);
});
afterEach(() => {
inner.reset();
setCassandraQueryExecutorForTesting(null);
});
it('removes every ledger row of a duplicate with three deletes and no read', async () => {
const repository = new JobLedgerRepository();
const createdAt = await createJobAt(repository, 7n);
log.length = 0;
await repository.discardJob(7n, createdAt);
expect([...log].sort()).toEqual(['delete jobs_active', 'delete jobs_by_day_bucket', 'delete jobs_by_id']);
expect(await repository.getJob(7n)).toBeNull();
expect(await repository.listActiveJobs()).toEqual([]);
expect(
await fetchMany(
JobsByDayBucket.select({where: JobsByDayBucket.where.eq('bucket_day')}).bind({
bucket_day: createdAt.toISOString().slice(0, 10),
}),
),
).toEqual([]);
});
it('keeps deleting the other ledger rows when one delete fails', async () => {
const repository = new JobLedgerRepository();
const createdAt = await createJobAt(repository, 8n);
failDeleteOn = 'jobs_by_id';
await expect(repository.discardJob(8n, createdAt)).rejects.toThrow('write timeout');
expect(await repository.listActiveJobs()).toEqual([]);
expect(
await fetchMany(
JobsByDayBucket.select({where: JobsByDayBucket.where.eq('bucket_day')}).bind({
bucket_day: createdAt.toISOString().slice(0, 10),
}),
),
).toEqual([]);
});
});
+180 -66
View File
@@ -1,6 +1,14 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import {BatchBuilder, deleteOneOrMany, fetchMany, fetchOne, upsertOne} from '@app/api/database/CassandraQueryExecution';
import {
BatchBuilder,
deleteOneOrMany,
fetchMany,
fetchOne,
fetchPage,
type PagedQueryResult,
upsertOne,
} from '@app/api/database/CassandraQueryExecution';
import {Db} from '@app/api/database/CassandraTypes';
import type {JobActiveRow, JobByDayBucketRow, JobByIdRow, JobStatus} from '@app/api/database/types/JobLedgerTypes';
import {
@@ -11,6 +19,17 @@ import {
type ListJobsResult,
} from '@app/api/jobs/IJobLedgerRepository';
import {JobsActive, JobsByDayBucket, JobsById} from '@app/api/Tables';
import {awaitAll} from '@app/api/utils/ConcurrencyUtils';
import {JOBS_STREAM_MAX_AGE_MS} from '@app/api/worker/JetStreamWorkerQueue';
import {snowflakeToDate} from '@fluxer/snowflake/src/Snowflake';
import {ms, seconds} from 'itty-time';
export const JOB_LEDGER_TTL_SECONDS = seconds('90 days');
export const JOB_STALE_AFTER_MS = JOBS_STREAM_MAX_AGE_MS + ms('1 day');
export const EXPIRED_JOB_ERROR = 'Expired from the job queue';
const JOB_LEDGER_RETENTION_DAYS = JOB_LEDGER_TTL_SECONDS / seconds('1 day');
const NEWEST_FIRST = {col: 'created_at', direction: 'DESC'} as const;
const FETCH_JOB_BY_ID_QUERY = JobsById.select({
where: JobsById.where.eq('job_id'),
@@ -19,13 +38,58 @@ const FETCH_CANCEL_REQUESTED_QUERY = JobsById.select({
where: JobsById.where.eq('job_id'),
});
const ACTIVE_JOBS_QUERY = JobsActive.select();
const ACTIVE_JOB_IDS_QUERY = JobsActive.select({columns: ['job_id']});
const JOBS_AFTER_IN_TIE_QUERY = JobsByDayBucket.select({
where: [
JobsByDayBucket.where.eq('bucket_day'),
JobsByDayBucket.where.eq('created_at'),
JobsByDayBucket.where.lt('job_id'),
],
orderBy: NEWEST_FIRST,
});
type LedgerPosition = Pick<ListJobsCursor, 'createdAt' | 'jobId'>;
function bucketDayFor(d: Date): string {
return d.toISOString().slice(0, 10);
}
function previousBucketDay(day: string): string {
const date = new Date(`${day}T00:00:00Z`);
date.setUTCDate(date.getUTCDate() - 1);
return bucketDayFor(date);
}
function dayJobsQuery(olderThanPosition: boolean, limit: number | null) {
return JobsByDayBucket.select({
where: olderThanPosition
? [JobsByDayBucket.where.eq('bucket_day'), JobsByDayBucket.where.lt('created_at')]
: JobsByDayBucket.where.eq('bucket_day'),
orderBy: NEWEST_FIRST,
...(limit === null ? {} : {limit}),
});
}
async function fetchDayAfter(
bucketDay: string,
after: LedgerPosition | null,
limit: number | null,
): Promise<{rows: Array<JobByDayBucketRow>; exhausted: boolean}> {
if (after === null) {
const rows = await fetchMany<JobByDayBucketRow>(dayJobsQuery(false, limit).bind({bucket_day: bucketDay}));
return {rows, exhausted: limit === null || rows.length < limit};
}
const ties = await fetchMany<JobByDayBucketRow>(
JOBS_AFTER_IN_TIE_QUERY.bind({bucket_day: bucketDay, created_at: after.createdAt, job_id: after.jobId}),
);
const older = await fetchMany<JobByDayBucketRow>(
dayJobsQuery(true, limit).bind({bucket_day: bucketDay, created_at: after.createdAt}),
);
return {rows: [...ties, ...older], exhausted: limit === null || older.length < limit};
}
export class JobLedgerRepository extends IJobLedgerRepository {
async createJob(input: CreateJobInput): Promise<void> {
async createJob(input: CreateJobInput): Promise<Date> {
const now = new Date();
const status: JobStatus = 'queued';
const idRow: JobByIdRow = {
@@ -68,39 +132,62 @@ export class JobLedgerRepository extends IJobLedgerRepository {
started_at: null,
};
const batch = new BatchBuilder();
batch.addPrepared(JobsById.insert(idRow));
batch.addPrepared(JobsByDayBucket.insert(bucketRow));
batch.addPrepared(JobsActive.insert(activeRow));
batch.addPrepared(JobsById.insertWithTtl(idRow, JOB_LEDGER_TTL_SECONDS));
batch.addPrepared(JobsByDayBucket.insertWithTtl(bucketRow, JOB_LEDGER_TTL_SECONDS));
batch.addPrepared(JobsActive.insertWithTtl(activeRow, JOB_LEDGER_TTL_SECONDS));
await batch.executeChunked(10, false);
return now;
}
async getJob(jobId: bigint): Promise<JobByIdRow | null> {
return fetchOne<JobByIdRow>(FETCH_JOB_BY_ID_QUERY.bind({job_id: jobId}));
const row = await fetchOne<JobByIdRow>(FETCH_JOB_BY_ID_QUERY.bind({job_id: jobId}));
return row?.created_at && row.task_type && row.status ? row : null;
}
async discardJob(jobId: bigint, createdAt: Date): Promise<void> {
await awaitAll(
[
deleteOneOrMany(
JobsByDayBucket.deleteByPk({bucket_day: bucketDayFor(createdAt), created_at: createdAt, job_id: jobId}),
),
deleteOneOrMany(JobsById.deleteByPk({job_id: jobId})),
deleteOneOrMany(JobsActive.deleteByPk({job_id: jobId})),
],
'Ledger discard left rows behind',
);
}
async markRunning(jobId: bigint, lane: string): Promise<void> {
const startedAt = new Date();
const status: JobStatus = 'running';
await upsertOne(
JobsById.patchByPk(
JobsById.patchByPkWithTtl(
{job_id: jobId},
{status: Db.set(status), started_at: Db.set(startedAt), jet_stream_lane: Db.set(lane)},
JOB_LEDGER_TTL_SECONDS,
),
);
await upsertOne(
JobsActive.patchByPkWithTtl(
{job_id: jobId},
{status: Db.set(status), started_at: Db.set(startedAt)},
JOB_LEDGER_TTL_SECONDS,
),
);
await upsertOne(JobsActive.patchByPk({job_id: jobId}, {status: Db.set(status), started_at: Db.set(startedAt)}));
}
async markSucceeded(jobId: bigint, result: Record<string, unknown> | null): Promise<void> {
const completedAt = new Date();
const status: JobStatus = 'succeeded';
await upsertOne(
JobsById.patchByPk(
JobsById.patchByPkWithTtl(
{job_id: jobId},
{
status: Db.set(status),
completed_at: Db.set(completedAt),
result: result === null ? Db.clear() : Db.set(JSON.stringify(result)),
},
JOB_LEDGER_TTL_SECONDS,
),
);
await deleteOneOrMany(JobsActive.deleteByPk({job_id: jobId}));
@@ -109,7 +196,13 @@ export class JobLedgerRepository extends IJobLedgerRepository {
async markCancelled(jobId: bigint): Promise<void> {
const completedAt = new Date();
const status: JobStatus = 'cancelled';
await upsertOne(JobsById.patchByPk({job_id: jobId}, {status: Db.set(status), completed_at: Db.set(completedAt)}));
await upsertOne(
JobsById.patchByPkWithTtl(
{job_id: jobId},
{status: Db.set(status), completed_at: Db.set(completedAt)},
JOB_LEDGER_TTL_SECONDS,
),
);
await deleteOneOrMany(JobsActive.deleteByPk({job_id: jobId}));
}
@@ -117,9 +210,10 @@ export class JobLedgerRepository extends IJobLedgerRepository {
const completedAt = new Date();
const status: JobStatus = 'deadletter';
await upsertOne(
JobsById.patchByPk(
JobsById.patchByPkWithTtl(
{job_id: jobId},
{status: Db.set(status), completed_at: Db.set(completedAt), error_message: Db.set(errorMessage)},
JOB_LEDGER_TTL_SECONDS,
),
);
await deleteOneOrMany(JobsActive.deleteByPk({job_id: jobId}));
@@ -127,27 +221,30 @@ export class JobLedgerRepository extends IJobLedgerRepository {
async reportProgress(jobId: bigint, current: number, total: number | null, message: string | null): Promise<void> {
await upsertOne(
JobsById.patchByPk(
JobsById.patchByPkWithTtl(
{job_id: jobId},
{
progress_current: Db.set(BigInt(current)),
progress_total: total === null ? Db.clear() : Db.set(BigInt(total)),
progress_message: message === null ? Db.clear() : Db.set(message),
},
JOB_LEDGER_TTL_SECONDS,
),
);
}
async setContextLink(jobId: bigint, link: string): Promise<void> {
await upsertOne(JobsById.patchByPk({job_id: jobId}, {context_link: Db.set(link)}));
await upsertOne(JobsById.patchByPkWithTtl({job_id: jobId}, {context_link: Db.set(link)}, JOB_LEDGER_TTL_SECONDS));
}
async setJetStreamSeq(jobId: bigint, seq: string): Promise<void> {
await upsertOne(JobsById.patchByPk({job_id: jobId}, {jet_stream_seq: Db.set(seq)}));
await upsertOne(JobsById.patchByPkWithTtl({job_id: jobId}, {jet_stream_seq: Db.set(seq)}, JOB_LEDGER_TTL_SECONDS));
}
async requestCancel(jobId: bigint): Promise<void> {
await upsertOne(JobsById.patchByPk({job_id: jobId}, {cancel_requested: Db.set(true)}));
await upsertOne(
JobsById.patchByPkWithTtl({job_id: jobId}, {cancel_requested: Db.set(true)}, JOB_LEDGER_TTL_SECONDS),
);
}
async isCancelRequested(jobId: bigint): Promise<boolean> {
@@ -160,7 +257,9 @@ export class JobLedgerRepository extends IJobLedgerRepository {
async incrementAttempts(jobId: bigint): Promise<void> {
const row = await this.getJob(jobId);
if (!row) return;
await upsertOne(JobsById.patchByPk({job_id: jobId}, {attempts: Db.set(row.attempts + 1)}));
await upsertOne(
JobsById.patchByPkWithTtl({job_id: jobId}, {attempts: Db.set(row.attempts + 1)}, JOB_LEDGER_TTL_SECONDS),
);
}
async listJobs(opts: {
@@ -169,59 +268,74 @@ export class JobLedgerRepository extends IJobLedgerRepository {
filters: ListJobsFilters;
maxLookbackDays: number;
}): Promise<ListJobsResult> {
const {limit, cursor, filters, maxLookbackDays} = opts;
const startBucket = cursor ? new Date(`${cursor.bucketDay}T00:00:00Z`) : new Date();
const hasFilters = Boolean(
filters.status ||
filters.taskType ||
(filters.requestedByUserId !== undefined && filters.requestedByUserId !== null),
);
const collected: Array<JobByIdRow> = [];
let nextCursor: ListJobsCursor | null = null;
for (let dayOffset = 0; dayOffset <= maxLookbackDays && collected.length < limit; dayOffset++) {
const bucketDate = new Date(startBucket);
bucketDate.setUTCDate(bucketDate.getUTCDate() - dayOffset);
const bucketDay = bucketDayFor(bucketDate);
const remaining = limit - collected.length + 1;
const bucketLimit = hasFilters ? {} : {limit: remaining};
const useCursor = dayOffset === 0 && cursor !== null;
let bucketRows: Array<JobByDayBucketRow>;
if (useCursor && cursor) {
const query = JobsByDayBucket.select({
where: [JobsByDayBucket.where.eq('bucket_day'), JobsByDayBucket.where.lt('created_at')],
...bucketLimit,
});
bucketRows = await fetchMany<JobByDayBucketRow>(
query.bind({bucket_day: bucketDay, created_at: cursor.createdAt}),
);
} else {
const query = JobsByDayBucket.select({
where: JobsByDayBucket.where.eq('bucket_day'),
...bucketLimit,
});
bucketRows = await fetchMany<JobByDayBucketRow>(query.bind({bucket_day: bucketDay}));
const {limit, cursor, filters} = opts;
const now = Date.now();
const lookbackDays = Math.min(opts.maxLookbackDays, JOB_LEDGER_RETENTION_DAYS);
const oldestDay = bucketDayFor(new Date(now - lookbackDays * ms('1 day')));
const requestedBy = filters.requestedByUserId ?? null;
const wholeDays = Boolean(filters.status || filters.taskType || requestedBy !== null);
const jobs: Array<JobByIdRow> = [];
let lastRow: JobByDayBucketRow | null = null;
let day = bucketDayFor(new Date(Math.min(cursor ? cursor.createdAt.getTime() : now, now)));
let after: LedgerPosition | null = cursor;
while (jobs.length < limit && day >= oldestDay) {
const {rows, exhausted} = await fetchDayAfter(day, after, wholeDays ? null : limit - jobs.length);
for (const row of rows) {
after = {createdAt: row.created_at, jobId: row.job_id};
if (filters.taskType && row.task_type !== filters.taskType) continue;
if (requestedBy !== null && row.requested_by_user_id !== requestedBy) continue;
const job = await this.getJob(row.job_id);
if (!job || (filters.status && job.status !== filters.status)) continue;
jobs.push(job);
lastRow = row;
if (jobs.length === limit) break;
}
for (const r of bucketRows) {
if (filters.taskType && r.task_type !== filters.taskType) continue;
if (filters.requestedByUserId !== undefined && filters.requestedByUserId !== null) {
if (r.requested_by_user_id !== filters.requestedByUserId) continue;
}
const fullRow = await this.getJob(r.job_id);
if (!fullRow) continue;
if (filters.status && fullRow.status !== filters.status) continue;
if (collected.length >= limit) {
nextCursor = {bucketDay, createdAt: r.created_at, jobId: r.job_id};
break;
}
collected.push(fullRow);
if (exhausted) {
day = previousBucketDay(day);
after = null;
}
if (nextCursor) break;
}
if (nextCursor === null && collected.length >= limit) {
const last = collected[collected.length - 1];
nextCursor = {bucketDay: bucketDayFor(last.created_at), createdAt: last.created_at, jobId: last.job_id};
}
return {jobs: collected, nextCursor};
return {
jobs,
nextCursor:
lastRow && jobs.length === limit
? {bucketDay: lastRow.bucket_day, createdAt: lastRow.created_at, jobId: lastRow.job_id}
: null,
};
}
async expireStaleActiveJobs(opts: {
staleBeforeMs: number;
pageSize: number;
maxCleared: number;
}): Promise<{cleared: number; expired: number; complete: boolean}> {
let cleared = 0;
let expired = 0;
let pageState: string | null = null;
do {
const page: PagedQueryResult<Pick<JobActiveRow, 'job_id'>> = await fetchPage(
ACTIVE_JOB_IDS_QUERY.bind({}),
undefined,
{
pageSize: opts.pageSize,
pageState,
},
);
for (const {job_id: jobId} of page.rows) {
if (snowflakeToDate(jobId).getTime() >= opts.staleBeforeMs) continue;
if (cleared >= opts.maxCleared) return {cleared, expired, complete: false};
const job = await this.getJob(jobId);
if (job?.status === 'queued' || job?.status === 'running') {
await this.markDeadletter(jobId, EXPIRED_JOB_ERROR);
expired += 1;
} else {
await deleteOneOrMany(JobsActive.deleteByPk({job_id: jobId}));
}
cleared += 1;
}
pageState = page.pageState;
} while (pageState !== null);
return {cleared, expired, complete: true};
}
async listActiveJobs(): Promise<Array<JobByIdRow>> {
@@ -0,0 +1,228 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import {upsertOne} from '@app/api/database/CassandraQueryExecution';
import type {JobByIdRow, JobStatus} from '@app/api/database/types/JobLedgerTypes';
import type {ListJobsCursor, ListJobsFilters} from '@app/api/jobs/IJobLedgerRepository';
import {JobLedgerRepository} from '@app/api/jobs/JobLedgerRepository';
import {JobsByDayBucket, JobsById} from '@app/api/Tables';
import {afterEach, beforeEach, describe, expect, it, vi} from 'vitest';
const DAY_MS = 86_400_000;
const HOUR_MS = 3_600_000;
const TODAY_NOON = new Date(`${new Date().toISOString().slice(0, 10)}T12:00:00.000Z`);
const ADMIN_USER_ID = 4_242n;
interface ListedPage {
ids: Array<bigint>;
cursor: ListJobsCursor | null;
}
function at(daysAgo: number, hour: number): Date {
return new Date(TODAY_NOON.getTime() - daysAgo * DAY_MS + (hour - 12) * HOUR_MS);
}
async function seedJob(
jobId: bigint,
createdAt: Date,
opts: {taskType?: string; requestedBy?: bigint | null; status?: JobStatus; bucketOnly?: boolean} = {},
): Promise<void> {
const taskType = opts.taskType ?? 'A';
const requestedBy = opts.requestedBy ?? null;
await upsertOne(
JobsByDayBucket.insert({
bucket_day: createdAt.toISOString().slice(0, 10),
created_at: createdAt,
job_id: jobId,
task_type: taskType,
status: 'queued',
requested_by_user_id: requestedBy,
}),
);
if (opts.bucketOnly) return;
const row: JobByIdRow = {
job_id: jobId,
task_type: taskType,
status: opts.status ?? 'queued',
progress_current: null,
progress_total: null,
progress_message: null,
payload: '{}',
result: null,
error_message: null,
created_at: createdAt,
started_at: null,
completed_at: null,
requested_by_user_id: requestedBy,
audit_log_reason: null,
jet_stream_seq: null,
jet_stream_lane: 'batch',
attempts: 0,
max_attempts: 5,
run_at: null,
cancel_requested: false,
context_link: null,
};
await upsertOne(JobsById.insert(row));
}
async function listPages(opts: {
limit: number;
filters?: ListJobsFilters;
maxLookbackDays?: number;
cursor?: ListJobsCursor | null;
}): Promise<Array<ListedPage>> {
const repository = new JobLedgerRepository();
const pages: Array<ListedPage> = [];
let cursor = opts.cursor ?? null;
for (let page = 0; page < 50; page += 1) {
const result = await repository.listJobs({
limit: opts.limit,
cursor,
filters: opts.filters ?? {},
maxLookbackDays: opts.maxLookbackDays ?? 14,
});
pages.push({ids: result.jobs.map((job) => job.job_id), cursor: result.nextCursor});
if (result.nextCursor === null) return pages;
cursor = {
bucketDay: result.nextCursor.bucketDay,
createdAt: new Date(result.nextCursor.createdAt.toISOString()),
jobId: BigInt(result.nextCursor.jobId.toString()),
};
}
throw new Error('listJobs never stopped paging');
}
function expectPages(pages: Array<ListedPage>, limit: number, expected: Array<Array<bigint>>): void {
expect(pages.map((page) => page.ids)).toEqual(expected);
const ids = pages.flatMap((page) => page.ids);
expect(new Set(ids).size).toBe(ids.length);
for (const page of pages.slice(0, -1)) {
expect(page.ids).toHaveLength(limit);
expect(page.cursor?.jobId).toBe(page.ids.at(-1));
}
expect(pages.at(-1)?.cursor).toBeNull();
}
export function describeListJobsPaging(): void {
describe('listJobs paging', () => {
beforeEach(() => {
vi.useFakeTimers({toFake: ['Date']});
vi.setSystemTime(TODAY_NOON);
});
afterEach(() => {
vi.useRealTimers();
});
it('walks one day newest first through a tie group larger than the page', async () => {
await seedJob(20n, at(0, 11));
for (const jobId of [11n, 12n, 13n, 14n, 15n]) {
await seedJob(jobId, at(0, 10));
}
await seedJob(9n, at(0, 9));
await seedJob(5n, at(0, 8));
await seedJob(6n, at(0, 8));
const pages = await listPages({limit: 2});
expectPages(pages, 2, [[20n, 15n], [14n, 13n], [12n, 11n], [9n, 6n], [5n]]);
expect(pages[0]?.cursor).toEqual({
bucketDay: at(0, 10).toISOString().slice(0, 10),
createdAt: at(0, 10),
jobId: 15n,
});
});
it('crosses days and keeps the lookback window anchored on today', async () => {
await seedJob(41n, at(0, 11));
await seedJob(40n, at(0, 10));
await seedJob(32n, at(1, 11));
await seedJob(31n, at(1, 10));
await seedJob(30n, at(1, 9));
await seedJob(21n, at(2, 11));
await seedJob(20n, at(2, 10));
await seedJob(10n, at(3, 11));
expectPages(await listPages({limit: 2, maxLookbackDays: 2}), 2, [[41n, 40n], [32n, 31n], [30n, 21n], [20n]]);
expectPages(await listPages({limit: 2, maxLookbackDays: 3}), 2, [
[41n, 40n],
[32n, 31n],
[30n, 21n],
[20n, 10n],
[],
]);
const outside = await new JobLedgerRepository().listJobs({
limit: 2,
cursor: {bucketDay: at(3, 12).toISOString().slice(0, 10), createdAt: at(3, 12), jobId: 1n},
filters: {},
maxLookbackDays: 2,
});
expect(outside).toEqual({jobs: [], nextCursor: null});
});
it('starts a cursor dated in the future at today', async () => {
await seedJob(2n, at(0, 2));
await seedJob(1n, at(1, 2));
const farFuture = new Date('9999-12-31T00:00:00.000Z');
expectPages(await listPages({limit: 5, cursor: {bucketDay: '9999-12-31', createdAt: farFuture, jobId: 1n}}), 5, [
[2n, 1n],
]);
}, 2_000);
it('never lists a day past the 90-day retention', async () => {
await seedJob(890n, at(89, 11));
await seedJob(910n, at(91, 11));
expectPages(await listPages({limit: 10, maxLookbackDays: 120}), 10, [[890n]]);
});
it('fills a page past rows whose job record is missing instead of leaving the day', async () => {
for (let hour = 1; hour <= 8; hour += 1) {
await seedJob(BigInt(hour), at(0, hour), {bucketOnly: [3, 6, 7].includes(hour)});
}
expectPages(await listPages({limit: 3}), 3, [
[8n, 5n, 4n],
[2n, 1n],
]);
});
it('fills pages through task type, requester and status filters across days', async () => {
await seedJob(60n, at(0, 11));
await seedJob(59n, at(0, 10.5), {taskType: 'B'});
await seedJob(58n, at(0, 10));
await seedJob(57n, at(0, 9.5), {taskType: 'B', requestedBy: ADMIN_USER_ID});
await seedJob(56n, at(0, 9), {status: 'succeeded'});
await seedJob(55n, at(0, 8.5), {taskType: 'B'});
await seedJob(50n, at(1, 11));
await seedJob(49n, at(1, 10.5), {taskType: 'B', requestedBy: ADMIN_USER_ID});
await seedJob(48n, at(1, 10), {bucketOnly: true});
await seedJob(47n, at(1, 9.5));
expectPages(await listPages({limit: 2, filters: {taskType: 'A'}}), 2, [[60n, 58n], [56n, 50n], [47n]]);
expectPages(await listPages({limit: 1, filters: {status: 'succeeded'}}), 1, [[56n], []]);
expectPages(await listPages({limit: 5, filters: {requestedByUserId: ADMIN_USER_ID}}), 5, [[57n, 49n]]);
});
it('resumes after a cursor whose job was discarded between pages', async () => {
for (let hour = 1; hour <= 5; hour += 1) {
await seedJob(BigInt(hour), at(0, hour));
}
const repository = new JobLedgerRepository();
const first = await repository.listJobs({limit: 2, cursor: null, filters: {}, maxLookbackDays: 14});
expect(first.jobs.map((job) => job.job_id)).toEqual([5n, 4n]);
await repository.discardJob(4n, at(0, 4));
expectPages(await listPages({limit: 2, cursor: first.nextCursor}), 2, [[3n, 2n], [1n]]);
});
it('returns an empty page after a page that filled exactly', async () => {
await seedJob(1n, at(0, 1));
await seedJob(2n, at(0, 2));
expectPages(await listPages({limit: 2}), 2, [[2n, 1n], []]);
});
});
}
@@ -0,0 +1,600 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import {spawnSync} from 'node:child_process';
import {createServer} from 'node:net';
import {BatchBuilder, setCassandraQueryExecutorForTesting} from '@app/api/database/CassandraQueryExecution';
import {Db} from '@app/api/database/CassandraTypes';
import {ensurePostgresKvSchema, PostgresKvQueryExecutor} from '@app/api/database/PostgresKvQueryExecutor';
import type {JobActiveRow, JobByDayBucketRow, JobByIdRow, JobStatus} from '@app/api/database/types/JobLedgerTypes';
import {
EXPIRED_JOB_ERROR,
JOB_LEDGER_TTL_SECONDS,
JOB_STALE_AFTER_MS,
JobLedgerRepository,
} from '@app/api/jobs/JobLedgerRepository';
import {describeListJobsPaging} from '@app/api/jobs/ListJobsPagingSuite';
import {expireLegacyJobLedgerRows} from '@app/api/jobs/PostgresJobLedgerExpiry';
import {JobsActive, JobsByDayBucket, JobsById} from '@app/api/Tables';
import {startDockerContainer} from '@app/api/test/DockerTestContainer';
import {InMemoryCassandraQueryExecutor} from '@app/api/test/InMemoryCassandraQueryExecutor';
import {createSnowflake} from '@fluxer/snowflake/src/Snowflake';
import {
getDefaultPostgresClient,
type IPostgresClient,
initPostgres,
shutdownPostgres,
} from '@pkgs/postgres/src/Client';
import {afterAll, beforeAll, beforeEach, describe, expect, it} from 'vitest';
const KV_TABLE = 'kv_job_ledger_expiry';
const CONTAINER = `fluxer-kvjobs-${process.pid.toString(36)}-${Date.now().toString(36)}`;
const dockerAvailable = spawnSync('docker', ['version'], {stdio: 'ignore'}).status === 0;
const DAY_MS = 86_400_000;
const ADMIN_USER_ID = 1_234_567_890_123n;
async function sleep(ms: number): Promise<void> {
await new Promise((resolve) => setTimeout(resolve, ms));
}
async function freePort(): Promise<number> {
return new Promise((resolve, reject) => {
const server = createServer();
server.on('error', reject);
server.listen(0, '127.0.0.1', () => {
const address = server.address();
if (typeof address === 'string' || address === null) {
reject(new Error('no port'));
return;
}
const port = address.port;
server.close(() => resolve(port));
});
});
}
let sequence = 0;
function jobIdAgedDays(days: number): bigint {
sequence += 1;
return createSnowflake({timestamp: Date.now() - days * DAY_MS, sequence: sequence % 4096, workerId: 1});
}
interface LegacyJob {
jobId: bigint;
createdAt: Date;
}
async function seedLegacyJob(
executor: PostgresKvQueryExecutor,
opts: {ageDays: number; status: JobStatus; requestedBy: bigint | null; active: boolean},
): Promise<LegacyJob> {
const jobId = jobIdAgedDays(opts.ageDays);
const createdAt = new Date(Date.now() - opts.ageDays * DAY_MS);
const idRow: JobByIdRow = {
job_id: jobId,
task_type: opts.requestedBy === null ? 'flushUserActivityBuffer' : 'refreshSearchIndex',
status: opts.status,
progress_current: null,
progress_total: null,
progress_message: null,
payload: '{}',
result: null,
error_message: null,
created_at: createdAt,
started_at: null,
completed_at: opts.status === 'succeeded' ? createdAt : null,
requested_by_user_id: opts.requestedBy,
audit_log_reason: null,
jet_stream_seq: '1',
jet_stream_lane: 'batch',
attempts: 0,
max_attempts: 5,
run_at: null,
cancel_requested: false,
context_link: null,
};
const bucketRow: JobByDayBucketRow = {
bucket_day: createdAt.toISOString().slice(0, 10),
created_at: createdAt,
job_id: jobId,
task_type: idRow.task_type,
status: 'queued',
requested_by_user_id: opts.requestedBy,
};
await executor.executeQuery(JobsById.insert(idRow));
await executor.executeQuery(JobsByDayBucket.insert(bucketRow));
if (opts.active) {
const activeRow: JobActiveRow = {
job_id: jobId,
task_type: idRow.task_type,
status: opts.status,
requested_by_user_id: opts.requestedBy,
created_at: createdAt,
started_at: null,
};
await executor.executeQuery(JobsActive.insert(activeRow));
}
return {jobId, createdAt};
}
describe.skipIf(!dockerAvailable)('job ledger expiry against postgres', () => {
let raw: IPostgresClient;
let executor: PostgresKvQueryExecutor;
async function jobRows(
jobId: bigint,
): Promise<Array<{table_name: string; expires_at: Date | null; row_data: never}>> {
const result = await raw.query<{table_name: string; expires_at: Date | null; row_data: never}>(
`SELECT table_name, expires_at, row_data FROM ${KV_TABLE}
WHERE table_name IN ('jobs_by_id', 'jobs_active', 'jobs_by_day_bucket')
AND (row_key = $1 OR split_part(row_key, chr(31), 3) = $1)
ORDER BY table_name`,
[JSON.stringify({__fluxer_type: 'bigint', value: jobId.toString()})],
);
return result.rows;
}
async function waitForLockWait(): Promise<void> {
for (let attempt = 0; attempt < 400; attempt += 1) {
const waiting = await raw.query<{n: number}>(
`SELECT count(*)::int AS n FROM pg_stat_activity WHERE datname = current_database() AND wait_event_type = 'Lock'`,
);
if (waiting.rows[0]!.n > 0) return;
await sleep(25);
}
throw new Error('the pass never waited on the writer');
}
async function forgetLedgerExpiry(): Promise<void> {
await raw.query(
`UPDATE ${KV_TABLE} SET expires_at = NULL WHERE table_name IN ('jobs_by_id', 'jobs_by_day_bucket')`,
);
}
async function forgetExpiryOf(...jobIds: Array<bigint>): Promise<void> {
const keys = jobIds.map((jobId) => JSON.stringify({__fluxer_type: 'bigint', value: jobId.toString()}));
await raw.query(
`UPDATE ${KV_TABLE} SET expires_at = NULL
WHERE table_name IN ('jobs_by_id', 'jobs_by_day_bucket')
AND (row_key = ANY($1::text[]) OR split_part(row_key, chr(31), 3) = ANY($1::text[]))`,
[keys],
);
}
beforeAll(async () => {
const port = await freePort();
startDockerContainer([
'run',
'-d',
'--name',
CONTAINER,
'-e',
'POSTGRES_USER=fluxer',
'-e',
'POSTGRES_PASSWORD=fluxer',
'-e',
'POSTGRES_DB=fluxer',
'-p',
`127.0.0.1:${port}:5432`,
'postgres:16-alpine',
'-c',
'fsync=off',
]);
let ready = false;
for (let attempt = 0; attempt < 180 && !ready; attempt += 1) {
await sleep(500);
const probe = spawnSync('docker', ['exec', CONTAINER, 'pg_isready', '-U', 'fluxer', '-d', 'fluxer'], {
stdio: 'ignore',
});
if (probe.status !== 0) continue;
try {
await initPostgres({
url: `postgres://fluxer:[email protected]:${port}/fluxer`,
maxConnections: 4,
kvTable: KV_TABLE,
});
await getDefaultPostgresClient().query('SELECT 1');
ready = true;
} catch {
await shutdownPostgres().catch(() => {});
}
}
if (!ready) throw new Error('postgres never came up');
raw = getDefaultPostgresClient();
await ensurePostgresKvSchema(raw);
executor = new PostgresKvQueryExecutor(raw);
}, 900_000);
beforeEach(async () => {
await raw.query(`DELETE FROM ${KV_TABLE}`);
setCassandraQueryExecutorForTesting(executor);
});
afterAll(async () => {
setCassandraQueryExecutorForTesting(new InMemoryCassandraQueryExecutor());
await shutdownPostgres().catch(() => {});
spawnSync('docker', ['rm', '-f', CONTAINER], {stdio: 'ignore'});
});
it('writes every ledger row with an expiry and keeps it through the job lifecycle', async () => {
const repository = new JobLedgerRepository();
const jobId = jobIdAgedDays(0);
await repository.createJob({
jobId,
taskType: 'refreshSearchIndex',
payload: {},
requestedByUserId: ADMIN_USER_ID,
auditLogReason: null,
maxAttempts: 5,
runAt: null,
jetStreamLane: 'batch',
jetStreamSeq: null,
});
await repository.setJetStreamSeq(jobId, '7');
await repository.markRunning(jobId, 'batch');
await repository.reportProgress(jobId, 1, 2, 'half');
const running = await jobRows(jobId);
expect(running.map((row) => row.table_name)).toEqual(['jobs_active', 'jobs_by_day_bucket', 'jobs_by_id']);
for (const row of running) {
expect(row.expires_at).not.toBeNull();
const remainingSeconds = (row.expires_at!.getTime() - Date.now()) / 1000;
expect(remainingSeconds).toBeGreaterThan(JOB_LEDGER_TTL_SECONDS - 60);
expect(remainingSeconds).toBeLessThanOrEqual(JOB_LEDGER_TTL_SECONDS);
}
await repository.markSucceeded(jobId, {ok: true});
const done = await jobRows(jobId);
expect(done.map((row) => row.table_name)).toEqual(['jobs_by_day_bucket', 'jobs_by_id']);
expect(done.every((row) => row.expires_at !== null)).toBe(true);
expect((await repository.getJob(jobId))?.status).toBe('succeeded');
});
it('never leaves a row without an expiry when a patch lands on a job that is gone', async () => {
const repository = new JobLedgerRepository();
const patches: Array<(jobId: bigint) => Promise<void>> = [
(jobId) => repository.markRunning(jobId, 'batch'),
(jobId) => repository.markSucceeded(jobId, null),
(jobId) => repository.markCancelled(jobId),
(jobId) => repository.markDeadletter(jobId, 'boom'),
(jobId) => repository.reportProgress(jobId, 1, null, null),
(jobId) => repository.setContextLink(jobId, '/admin/jobs'),
(jobId) => repository.setJetStreamSeq(jobId, '1'),
(jobId) => repository.requestCancel(jobId),
];
for (const patch of patches) {
const jobId = jobIdAgedDays(0);
await patch(jobId);
const rows = await jobRows(jobId);
expect(rows.length).toBeGreaterThan(0);
expect(rows.every((row) => row.expires_at !== null)).toBe(true);
expect(await repository.getJob(jobId)).toBeNull();
}
expect(await repository.listActiveJobs()).toEqual([]);
});
it('discards every row of a job that never reached the stream', async () => {
const repository = new JobLedgerRepository();
const jobId = jobIdAgedDays(0);
const createdAt = await repository.createJob({
jobId,
taskType: 'batchGuildAuditLogMessageDeletes',
payload: {guildId: '1'},
requestedByUserId: null,
auditLogReason: null,
maxAttempts: 3,
runAt: new Date(Date.now() + 30_000),
jetStreamLane: 'batch',
jetStreamSeq: null,
});
expect(await jobRows(jobId)).toHaveLength(3);
await repository.discardJob(jobId, createdAt);
expect(await jobRows(jobId)).toEqual([]);
});
it('clears legacy rows by the same rules the expiry now enforces', async () => {
const repository = new JobLedgerRepository();
const cronDone = await seedLegacyJob(executor, {
ageDays: 20,
status: 'succeeded',
requestedBy: null,
active: false,
});
const cronStuck = await seedLegacyJob(executor, {ageDays: 20, status: 'queued', requestedBy: null, active: true});
const partialId = jobIdAgedDays(20);
await executor.executeQuery(
JobsById.patchByPk({job_id: partialId}, {status: Db.set('succeeded'), completed_at: Db.set(new Date())}),
);
const adminDone = await seedLegacyJob(executor, {
ageDays: 20,
status: 'succeeded',
requestedBy: ADMIN_USER_ID,
active: false,
});
const adminStuck = await seedLegacyJob(executor, {
ageDays: 20,
status: 'queued',
requestedBy: ADMIN_USER_ID,
active: true,
});
const adminRunning = await seedLegacyJob(executor, {
ageDays: 20,
status: 'running',
requestedBy: ADMIN_USER_ID,
active: true,
});
const adminAncient = await seedLegacyJob(executor, {
ageDays: 100,
status: 'succeeded',
requestedBy: ADMIN_USER_ID,
active: false,
});
const cronInFlight = await seedLegacyJob(executor, {
ageDays: 2,
status: 'queued',
requestedBy: null,
active: true,
});
await forgetLedgerExpiry();
const fresh = jobIdAgedDays(0);
await repository.createJob({
jobId: fresh,
taskType: 'syncUrlBlocklists',
payload: {},
requestedByUserId: null,
auditLogReason: null,
maxAttempts: 5,
runAt: null,
jetStreamLane: 'batch',
jetStreamSeq: null,
});
const freshBefore = await jobRows(fresh);
await raw.query(
`INSERT INTO ${KV_TABLE} (table_name, partition_key, row_key, row_data) VALUES ('users', 'u1', 'u1', '{}'::jsonb)`,
);
expect(await repository.getJob(partialId)).toBeNull();
const first = await expireLegacyJobLedgerRows(raw, Date.now() + 60_000);
expect(first).toEqual({deleted: 10, expiring: 6, complete: true});
expect(await jobRows(cronDone.jobId)).toEqual([]);
expect(await jobRows(cronStuck.jobId)).toEqual([]);
expect(await jobRows(partialId)).toEqual([]);
expect(await jobRows(adminAncient.jobId)).toEqual([]);
for (const kept of [adminDone, adminStuck, adminRunning]) {
const rows = await jobRows(kept.jobId);
expect(rows.map((row) => row.table_name)).toEqual(['jobs_by_day_bucket', 'jobs_by_id']);
for (const row of rows) {
const expected = kept.createdAt.getTime() + JOB_LEDGER_TTL_SECONDS * 1000;
expect(Math.abs(row.expires_at!.getTime() - expected)).toBeLessThan(2000);
}
}
expect((await repository.getJob(adminDone.jobId))?.status).toBe('succeeded');
for (const stuck of [adminStuck, adminRunning]) {
const expired = await repository.getJob(stuck.jobId);
expect(expired?.status).toBe('deadletter');
expect(expired?.error_message).toBe('Expired from the job queue');
expect(expired?.completed_at).toBeInstanceOf(Date);
}
const inFlight = await jobRows(cronInFlight.jobId);
expect(inFlight.map((row) => row.table_name)).toEqual(['jobs_active', 'jobs_by_day_bucket', 'jobs_by_id']);
expect(inFlight.every((row) => row.expires_at === null)).toBe(true);
expect((await repository.getJob(cronInFlight.jobId))?.status).toBe('queued');
expect(await jobRows(fresh)).toEqual(freshBefore);
const users = await raw.query(`SELECT expires_at FROM ${KV_TABLE} WHERE table_name = 'users'`);
expect(users.rows).toEqual([{expires_at: null}]);
const listed = await repository.listJobs({limit: 50, cursor: null, filters: {}, maxLookbackDays: 30});
expect(listed.jobs.map((job) => job.job_id).sort()).toEqual(
[adminDone.jobId, adminStuck.jobId, adminRunning.jobId, cronInFlight.jobId, fresh].sort(),
);
for (let pass = 0; pass < 2; pass += 1) {
expect(await expireLegacyJobLedgerRows(raw, Date.now() + 60_000)).toEqual({
deleted: 0,
expiring: 0,
complete: true,
});
}
});
it('runs again a day after a clean pass and clears rows an older image wrote in between', async () => {
expect(await expireLegacyJobLedgerRows(raw, Date.now() + 60_000)).toEqual({
deleted: 0,
expiring: 0,
complete: true,
});
const rolledBack = await seedLegacyJob(executor, {ageDays: 30, status: 'queued', requestedBy: null, active: true});
await forgetLedgerExpiry();
expect(await expireLegacyJobLedgerRows(raw, Date.now() + 60_000)).toBeNull();
await raw.query(
`UPDATE ${KV_TABLE} SET row_data = jsonb_build_object('applied_at', now() - interval '2 days') WHERE row_key = 'job_ledger_expiry_v1'`,
);
expect(await expireLegacyJobLedgerRows(raw, Date.now() + 60_000)).toEqual({
deleted: 3,
expiring: 0,
complete: true,
});
expect(await jobRows(rolledBack.jobId)).toEqual([]);
expect(await expireLegacyJobLedgerRows(raw, Date.now() + 60_000)).toEqual({
deleted: 0,
expiring: 0,
complete: true,
});
expect(await expireLegacyJobLedgerRows(raw, Date.now() + 60_000)).toBeNull();
});
it('expires stale active jobs page by page without fighting the legacy pass', async () => {
const repository = new JobLedgerRepository();
const legacyFirst = await seedLegacyJob(executor, {
ageDays: 20,
status: 'queued',
requestedBy: ADMIN_USER_ID,
active: true,
});
await forgetExpiryOf(legacyFirst.jobId);
await expireLegacyJobLedgerRows(raw, Date.now() + 60_000);
const legacyFirstRows = await jobRows(legacyFirst.jobId);
const stale: Array<bigint> = [];
for (let index = 0; index < 4; index += 1) {
const jobId = jobIdAgedDays(9);
await repository.createJob({
jobId,
taskType: 'syncUrlBlocklists',
payload: {},
requestedByUserId: null,
auditLogReason: null,
maxAttempts: 5,
runAt: null,
jetStreamLane: 'batch',
jetStreamSeq: null,
});
stale.push(jobId);
}
await repository.markRunning(stale[0]!, 'batch');
const fresh = jobIdAgedDays(0);
await repository.createJob({
jobId: fresh,
taskType: 'syncUrlBlocklists',
payload: {},
requestedByUserId: null,
auditLogReason: null,
maxAttempts: 5,
runAt: null,
jetStreamLane: 'batch',
jetStreamSeq: null,
});
const legacyStale = await seedLegacyJob(executor, {
ageDays: 20,
status: 'running',
requestedBy: ADMIN_USER_ID,
active: true,
});
const legacyYoung = await seedLegacyJob(executor, {ageDays: 2, status: 'queued', requestedBy: null, active: true});
await forgetExpiryOf(legacyStale.jobId, legacyYoung.jobId);
const orphan = jobIdAgedDays(9);
await executor.executeQuery(
JobsActive.patchByPkWithTtl({job_id: orphan}, {status: Db.set('running')}, JOB_LEDGER_TTL_SECONDS),
);
const sweep = () =>
repository.expireStaleActiveJobs({staleBeforeMs: Date.now() - JOB_STALE_AFTER_MS, pageSize: 2, maxCleared: 100});
expect(await sweep()).toEqual({cleared: 6, expired: 5, complete: true});
const active = (await repository.listActiveJobs()).map((job) => job.job_id).sort();
expect(active).toEqual([fresh, legacyYoung.jobId].sort());
for (const jobId of [...stale, legacyStale.jobId]) {
const job = await repository.getJob(jobId);
expect(job?.status).toBe('deadletter');
expect(job?.error_message).toBe(EXPIRED_JOB_ERROR);
const byId = (await jobRows(jobId)).find((row) => row.table_name === 'jobs_by_id');
expect(Math.abs(byId!.expires_at!.getTime() - (Date.now() + JOB_LEDGER_TTL_SECONDS * 1000))).toBeLessThan(60_000);
}
expect(await jobRows(orphan)).toEqual([]);
expect(await jobRows(legacyFirst.jobId)).toEqual(legacyFirstRows);
const legacyStaleById = (await jobRows(legacyStale.jobId)).find((row) => row.table_name === 'jobs_by_id');
await expireLegacyJobLedgerRows(raw, Date.now() + 60_000);
expect((await jobRows(legacyStale.jobId)).find((row) => row.table_name === 'jobs_by_id')).toEqual(legacyStaleById);
expect((await repository.listActiveJobs()).map((job) => job.job_id)).toContain(legacyYoung.jobId);
expect(await sweep()).toEqual({cleared: 0, expired: 0, complete: true});
});
it('leaves rows alone when a live writer gives them an expiry while the pass waits on them', async () => {
const cronDone = await seedLegacyJob(executor, {
ageDays: 20,
status: 'succeeded',
requestedBy: null,
active: false,
});
const adminDone = await seedLegacyJob(executor, {
ageDays: 20,
status: 'succeeded',
requestedBy: ADMIN_USER_ID,
active: false,
});
await forgetLedgerExpiry();
const liveKeys = [cronDone.jobId, adminDone.jobId].map((jobId) =>
JSON.stringify({__fluxer_type: 'bigint', value: jobId.toString()}),
);
let written!: () => void;
const writerHoldsRows = new Promise<void>((resolve) => {
written = resolve;
});
let release!: () => void;
const released = new Promise<void>((resolve) => {
release = resolve;
});
const writer = raw.transaction(async (db) => {
await db.query(
`UPDATE ${KV_TABLE} SET expires_at = now() + interval '1 hour', updated_at = now() WHERE table_name = 'jobs_by_id' AND row_key = ANY($1::text[])`,
[liveKeys],
);
written();
await released;
});
await writerHoldsRows;
const pass = expireLegacyJobLedgerRows(raw, Date.now() + 60_000);
await waitForLockWait();
release();
await writer;
expect(await pass).toEqual({deleted: 1, expiring: 1, complete: true});
for (const job of [cronDone, adminDone]) {
const byId = (await jobRows(job.jobId)).find((row) => row.table_name === 'jobs_by_id');
expect(byId).toBeDefined();
const remainingSeconds = (byId!.expires_at!.getTime() - Date.now()) / 1000;
expect(remainingSeconds).toBeGreaterThan(3000);
expect(remainingSeconds).toBeLessThanOrEqual(3660);
}
expect((await new JobLedgerRepository().getJob(adminDone.jobId))?.status).toBe('succeeded');
});
it('pages through more legacy rows than one page holds and stops at its deadline', async () => {
const batch = new BatchBuilder();
for (let index = 0; index < 2300; index += 1) {
const jobId = jobIdAgedDays(30);
const createdAt = new Date(Date.now() - 30 * DAY_MS);
batch.addPrepared(
JobsByDayBucket.insert({
bucket_day: createdAt.toISOString().slice(0, 10),
created_at: createdAt,
job_id: jobId,
task_type: 'flushUserActivityBuffer',
status: 'queued',
requested_by_user_id: null,
}),
);
}
await batch.executeChunked(500, false);
await forgetLedgerExpiry();
expect(await expireLegacyJobLedgerRows(raw, Date.now() - 1)).toEqual({
deleted: 0,
expiring: 0,
complete: false,
});
expect(await expireLegacyJobLedgerRows(raw, Date.now() + 60_000)).toEqual({
deleted: 2300,
expiring: 0,
complete: true,
});
const left = await raw.query(`SELECT count(*)::int AS n FROM ${KV_TABLE} WHERE table_name = 'jobs_by_day_bucket'`);
expect(left.rows[0]).toEqual({n: 0});
expect(await expireLegacyJobLedgerRows(raw, Date.now() + 60_000)).toEqual({
deleted: 0,
expiring: 0,
complete: true,
});
expect(await expireLegacyJobLedgerRows(raw, Date.now() + 60_000)).toBeNull();
});
describeListJobsPaging();
});
@@ -0,0 +1,149 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import {postgresKvPassIsFresh, recordPostgresKvCleanPass} from '@app/api/database/PostgresKvQueryExecutor';
import {EXPIRED_JOB_ERROR, JOB_LEDGER_TTL_SECONDS, JOB_STALE_AFTER_MS} from '@app/api/jobs/JobLedgerRepository';
import {FLUXER_EPOCH} from '@fluxer/constants/src/Core';
import {TIMESTAMP_SHIFT} from '@fluxer/snowflake/src/Snowflake';
import {type IPostgresClient, quoteIdentifier} from '@pkgs/postgres/src/Client';
import {ms} from 'itty-time';
const LEGACY_JOB_LEDGER_MARKER = 'job_ledger_expiry_v1';
const PAGE_SIZE = 2000;
const CLEAN_PASS_INTERVAL_MS = ms('1 day');
const BIGINT_KEY_PREFIX = '{"__fluxer_type":"bigint","value":"';
const BIGINT_KEY_SUFFIX = '"}';
function bigintKeyExpr(key: string): string {
const prefixLength = BIGINT_KEY_PREFIX.length;
const affixLength = prefixLength + BIGINT_KEY_SUFFIX.length;
return `(CASE WHEN left(${key}, ${prefixLength}) = '${BIGINT_KEY_PREFIX}' AND right(${key}, ${BIGINT_KEY_SUFFIX.length}) = '${BIGINT_KEY_SUFFIX}' THEN substr(${key}, ${prefixLength + 1}, length(${key}) - ${affixLength})::numeric END)`;
}
interface LedgerTable {
name: string;
jobIdExpr: string;
deleteStale: boolean;
markStaleDeadletter: boolean;
}
const LEDGER_TABLES: ReadonlyArray<LedgerTable> = [
{
name: 'jobs_active',
jobIdExpr: bigintKeyExpr('kv.row_key'),
deleteStale: true,
markStaleDeadletter: false,
},
{
name: 'jobs_by_id',
jobIdExpr: bigintKeyExpr('kv.row_key'),
deleteStale: false,
markStaleDeadletter: true,
},
{
name: 'jobs_by_day_bucket',
jobIdExpr: bigintKeyExpr('split_part(kv.row_key, chr(31), 3)'),
deleteStale: false,
markStaleDeadletter: false,
},
];
export interface LegacyJobLedgerExpiryResult {
deleted: number;
expiring: number;
complete: boolean;
}
function pageSql(table: string, target: LedgerTable): string {
const staleRemovable = target.deleteStale ? 'c.created_at < $7' : '(c.created_at < $7 AND c.system_job)';
const removable = `c.job_id IS NULL OR c.created_at < $6 OR ${staleRemovable}`;
const rowData = target.markStaleDeadletter
? `CASE WHEN c.unfinished THEN kv.row_data || jsonb_build_object('status', 'deadletter', 'error_message', $9::text, 'completed_at', jsonb_build_object('__fluxer_type', 'date', 'value', $10::text)) ELSE kv.row_data END`
: 'kv.row_data';
return `
WITH page AS (
SELECT kv.row_key, kv.row_data, ${target.jobIdExpr} AS job_id
FROM ${table} kv
WHERE kv.table_name = $1 AND kv.expires_at IS NULL AND kv.row_key > $2
ORDER BY kv.row_key
LIMIT $3
), classified AS (
SELECT
page.row_key,
page.job_id,
to_timestamp(((div(page.job_id, $4::numeric) + $5::numeric) / 1000)::double precision) AS created_at,
COALESCE(page.row_data -> 'requested_by_user_id', 'null'::jsonb) = 'null'::jsonb AS system_job,
COALESCE(page.row_data ->> 'status' IN ('queued', 'running'), false) AS unfinished
FROM page
), removed AS (
DELETE FROM ${table} kv
USING classified c
WHERE kv.table_name = $1 AND kv.row_key = c.row_key AND kv.expires_at IS NULL AND (${removable})
RETURNING kv.row_key
), expiring AS (
UPDATE ${table} kv
SET expires_at = c.created_at + make_interval(secs => $8::double precision), updated_at = now(), row_data = ${rowData}
FROM classified c
WHERE kv.table_name = $1 AND kv.row_key = c.row_key AND kv.expires_at IS NULL AND c.created_at < $7 AND NOT (${removable})
RETURNING kv.row_key
)
SELECT
(SELECT max(row_key) FROM page) AS last_row_key,
(SELECT count(*) FROM page) AS scanned,
(SELECT count(*) FROM removed) AS deleted,
(SELECT count(*) FROM expiring) AS expiring`;
}
export async function expireLegacyJobLedgerRows(
client: IPostgresClient,
deadlineMs: number,
): Promise<LegacyJobLedgerExpiryResult | null> {
const table = quoteIdentifier(client.kvTable());
if (await postgresKvPassIsFresh(client, LEGACY_JOB_LEDGER_MARKER, CLEAN_PASS_INTERVAL_MS)) {
return null;
}
const now = Date.now();
const retentionCutoff = new Date(now - JOB_LEDGER_TTL_SECONDS * 1000);
const staleCutoff = new Date(now - JOB_STALE_AFTER_MS);
const completedAt = new Date(now).toISOString();
let scanned = 0;
let deleted = 0;
let expiring = 0;
for (const target of LEDGER_TABLES) {
const sql = pageSql(table, target);
const deadletterValues = target.markStaleDeadletter ? [EXPIRED_JOB_ERROR, completedAt] : [];
let cursor = '';
for (;;) {
if (Date.now() >= deadlineMs) {
return {deleted, expiring, complete: false};
}
const result = await client.query<{
last_row_key: string | null;
scanned: string;
deleted: string;
expiring: string;
}>(sql, [
target.name,
cursor,
PAGE_SIZE,
(1n << TIMESTAMP_SHIFT).toString(),
FLUXER_EPOCH.toString(),
retentionCutoff,
staleCutoff,
JOB_LEDGER_TTL_SECONDS,
...deadletterValues,
]);
const page = result.rows[0];
if (!page || page.last_row_key === null) {
break;
}
scanned += Number(page.scanned);
deleted += Number(page.deleted);
expiring += Number(page.expiring);
cursor = page.last_row_key;
}
}
if (scanned === 0) {
await recordPostgresKvCleanPass(client, LEGACY_JOB_LEDGER_MARKER);
}
return {deleted, expiring, complete: true};
}
@@ -1,7 +1,8 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import {AdminRepository} from '@app/api/admin/AdminRepository';
import {BANNED_FILE_SHAS_REFRESH_CHANNEL} from '@app/api/constants/ContentModeration';
import {Config} from '@app/api/Config';
import {BANNED_FILE_SHAS_REFRESH_CHANNEL, isBlocklistFeedFileSha} from '@app/api/constants/ContentModeration';
import {Logger} from '@app/api/Logger';
import {RefreshSubscription} from '@app/api/utils/RefreshSubscription';
import type {IKVProvider} from '@pkgs/kv_client/src/IKVProvider';
@@ -38,8 +39,11 @@ class FileShaCache {
async refresh(): Promise<void> {
const rows = await this.adminRepository.loadAllBannedFileShas();
const next = new Set<string>();
const includeFeedRows = Config.blocklistFeeds.enabled;
for (const row of rows) {
if (row.sha256_hex) next.add(row.sha256_hex.toLowerCase());
if (!row.sha256_hex) continue;
if (!includeFeedRows && isBlocklistFeedFileSha(row)) continue;
next.add(row.sha256_hex.toLowerCase());
}
this.banned = next;
this.consecutiveFailures = 0;
@@ -77,7 +77,6 @@ import {
getContactChangeLogService,
getDiscriminatorService,
getDonationRepository,
getDownloadService,
getEmailChangeRepository,
getEmailDnsValidationService,
getEmailService,
@@ -535,10 +534,6 @@ class RequestServices implements RequestScopedServices {
return getContactChangeLogService();
}
get downloadService() {
return getDownloadService();
}
get emailService() {
return getEmailService();
}
@@ -19,7 +19,6 @@ import {createNcmecApiConfig, NcmecReporter} from '@app/api/csam/NcmecReporter';
import {NcmecRepository} from '@app/api/csam/NcmecRepository';
import {NcmecSubmissionService} from '@app/api/csam/NcmecSubmissionService';
import {DonationRepository} from '@app/api/donation/DonationRepository';
import {DownloadService} from '@app/api/download/DownloadService';
import {createEmailProvider} from '@app/api/email/EmailProviderFactory';
import {FavoriteMemeRepository} from '@app/api/favorite_meme/FavoriteMemeRepository';
import {GatewayRequestService} from '@app/api/gateway/GatewayRequestService';
@@ -46,7 +45,7 @@ import {KVActivityTracker} from '@app/api/infrastructure/KVActivityTracker';
import {KVBulkMessageDeletionQueueService} from '@app/api/infrastructure/KVBulkMessageDeletionQueueService';
import {NatsUnfurlerService} from '@app/api/infrastructure/NatsUnfurlerService';
import {PremiumStateReconciliationQueueService} from '@app/api/infrastructure/PremiumStateReconciliationQueueService';
import {createDownloadsStorageService, createStorageService} from '@app/api/infrastructure/StorageServiceFactory';
import {createStorageService} from '@app/api/infrastructure/StorageServiceFactory';
import {UserCacheService} from '@app/api/infrastructure/UserCacheService';
import {createUsersServiceClient} from '@app/api/infrastructure/UsersServiceClient';
import {VirusScanService} from '@app/api/infrastructure/VirusScanService';
@@ -217,10 +216,6 @@ export const getStorageService: () => IStorageService = (() => {
const fallback = singleton(() => createStorageService());
return () => _injectedStorageService ?? fallback();
})();
const getDownloadsStorageService: () => IStorageService = (() => {
const override = singleton(() => createDownloadsStorageService());
return () => override() ?? getStorageService();
})();
export const getErrorI18nService = singleton(() => new ErrorI18nService());
let limitConfigServiceInstance: LimitConfigService | null = null;
export const getLimitConfigService = singleton(
@@ -304,7 +299,6 @@ export function getKVAccountDeletionQueue(): KVAccountDeletionQueueService {
return accountDeletionQueue;
}
export const getDownloadService = singleton(() => new DownloadService(getDownloadsStorageService()));
export const getThemeService = singleton(() => new ThemeService(getStorageService()));
const getNcmecReporter = singleton(() => new NcmecReporter({config: createNcmecApiConfig(), fetch}));
const getNcmecRepository = singleton(() => new NcmecRepository());

Some files were not shown because too many files have changed in this diff Show More