mirror of
https://github.com/fluxerapp/fluxer
synced 2026-10-09 20:22:11 +09:00
Compare commits
24
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
1ab7e7dfcc | ||
|
|
31c53d2dff | ||
|
|
412a1ae79d | ||
|
|
0b2306ec3d | ||
|
|
242ed3a934 | ||
|
|
70e1ce682a | ||
|
|
7601bf98ee | ||
|
|
c7ec2a0f58 | ||
|
|
6a5e0056a8 | ||
|
|
78d105b46e | ||
|
|
c68d62b8a0 | ||
|
|
df58020f4c | ||
|
|
f052ce05aa | ||
|
|
eedfd9275f | ||
|
|
416af4bec4 | ||
|
|
108d282ddd | ||
|
|
a6103244b0 | ||
|
|
38935c83c5 | ||
|
|
c157ab5752 | ||
|
|
574a93257c | ||
|
|
ba7d8781cf | ||
|
|
3256af8d92 | ||
|
|
86043212f2 | ||
|
|
5d85e88532 |
@@ -7,7 +7,7 @@ ARG USER_UID=1000
|
||||
ARG USER_GID=1000
|
||||
ARG NODE_MAJOR=26
|
||||
ARG ELP_VERSION=2026-08-10
|
||||
ARG PNPM_VERSION=12.4.2
|
||||
ARG PNPM_VERSION=11.27.0
|
||||
ARG WASM_BINDGEN_VERSION=0.2.128
|
||||
|
||||
ENV DEBIAN_FRONTEND=noninteractive
|
||||
|
||||
@@ -38,7 +38,11 @@
|
||||
"customizations": {
|
||||
"vscode": {
|
||||
"settings": {
|
||||
"editor.defaultFormatter": "biomejs.biome"
|
||||
"editor.defaultFormatter": "biomejs.biome",
|
||||
"erlang.includePaths": ["."],
|
||||
"search.exclude": {
|
||||
"**/_build/default/lib/fluxer_gateway": true
|
||||
}
|
||||
},
|
||||
"extensions": [
|
||||
"biomejs.biome",
|
||||
|
||||
@@ -71,7 +71,6 @@ jobs:
|
||||
BUILD_VERSION: ${{ needs.meta.outputs.build_version }}
|
||||
PUBLIC_ASSET_BASE_URL: ""
|
||||
BUNDLE_LOCAL_ASSETS: "true"
|
||||
FLUXER_APP_PROXY_TIME_FREEZE_ENABLED: "false"
|
||||
steps:
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1
|
||||
env:
|
||||
@@ -155,7 +154,6 @@ jobs:
|
||||
BUILD_VERSION=${{ needs.meta.outputs.build_version }}
|
||||
SOURCE_SHA=${{ github.sha }}
|
||||
SOURCE_DATE=${{ steps.source.outputs.date }}
|
||||
FLUXER_APP_PROXY_TIME_FREEZE_ENABLED=false
|
||||
APP_ASSETS_REF=ghcr.io/${{ env.GHCR_OWNER }}/fluxer-app-proxy-self-hosted:${{ needs.meta.outputs.build_version }}-assets
|
||||
APP_ASSETS_PLATFORM=linux/amd64
|
||||
cache-from: type=registry,ref=ghcr.io/${{ env.GHCR_OWNER }}/fluxer-app-proxy-self-hosted:buildcache-${{ matrix.platform }}
|
||||
|
||||
@@ -10,6 +10,7 @@
|
||||
/.direnv/
|
||||
/.fluxer/
|
||||
/.pnpm-store/
|
||||
/.vscode/
|
||||
|
||||
**/*.css.d.ts
|
||||
**/*.tsbuildinfo
|
||||
|
||||
Generated
-10
@@ -1755,16 +1755,6 @@ dependencies = [
|
||||
"zip",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "fluxer-content-update-frozen-snapshot"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"base64 0.23.1",
|
||||
"sha2 0.11.0",
|
||||
"tempfile",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "fluxer-dev"
|
||||
version = "0.1.0"
|
||||
|
||||
@@ -9,7 +9,6 @@ members = [
|
||||
"fluxer_messages",
|
||||
"fluxer_snowflakes",
|
||||
"tools/ci",
|
||||
"tools/content/update-frozen-snapshot",
|
||||
"tools/dev",
|
||||
"tools/i18n_auto",
|
||||
"fluxer_users",
|
||||
|
||||
@@ -6,10 +6,12 @@
|
||||
</p>
|
||||
|
||||
<p align="center">
|
||||
<a href="https://fluxer.app/donate">
|
||||
<img src="https://img.shields.io/badge/Donate-fluxer.app%2Fdonate-brightgreen" alt="Donate" /></a>
|
||||
<a href="https://fluxer.app/download">
|
||||
<img src="https://img.shields.io/badge/Download-fluxer.app-4641D9" alt="Download" /></a>
|
||||
<a href="https://docs.fluxer.app">
|
||||
<img src="https://img.shields.io/badge/Docs-docs.fluxer.app-blue" alt="Documentation" /></a>
|
||||
<a href="https://fluxer.app/donate">
|
||||
<img src="https://img.shields.io/badge/Donate-fluxer.app%2Fdonate-brightgreen" alt="Donate" /></a>
|
||||
<a href="./LICENSE">
|
||||
<img src="https://img.shields.io/badge/License-AGPLv3-purple" alt="AGPLv3 License" /></a>
|
||||
</p>
|
||||
@@ -19,5 +21,128 @@
|
||||
Fluxer is a free and open source instant messaging and VoIP chat app built for friends, groups, and communities.
|
||||
|
||||
<p align="center">
|
||||
<img src="./fluxer_static/marketing/screenshots/desktop-readme-1920w.png" alt="Fluxer app showcase" width="900">
|
||||
<img src="./fluxer_static/marketing/screenshots/desktop-readme-1920w.png" alt="Fluxer running side by side on a desktop monitor and a phone" width="640">
|
||||
</p>
|
||||
|
||||
## Download
|
||||
|
||||
| Windows | macOS | Linux | Android | iOS |
|
||||
| --- | --- | --- | --- | --- |
|
||||
| [Installer (x64)][win-setup-x64] | [Disk image][mac-dmg] | [Flatpak][flatpak-ref] | [APK][android-apk] | [TestFlight][ios-testflight] |
|
||||
| [Installer (ARM64)][win-setup-arm64] | | [deb (x64)][linux-deb-x64] | [Obtainium][obtainium] | |
|
||||
| [Portable (x64)][win-portable-x64] | | [deb (ARM64)][linux-deb-arm64] | | |
|
||||
| [Portable (ARM64)][win-portable-arm64] | | [rpm (x64)][linux-rpm-x64] | | |
|
||||
| | | [rpm (ARM64)][linux-rpm-arm64] | | |
|
||||
| | | [AppImage (x64)][linux-appimage-x64] | | |
|
||||
| | | [AppImage (ARM64)][linux-appimage-arm64] | | |
|
||||
| | | [tar.gz (x64)][linux-targz-x64] | | |
|
||||
| | | [tar.gz (ARM64)][linux-targz-arm64] | | |
|
||||
|
||||
The macOS disk image is universal and runs on both Apple silicon and Intel. Windows and Linux need the build that matches your processor.
|
||||
|
||||
On Linux, prefer a package repository over a file. Fluxer then updates with the rest of your system.
|
||||
|
||||
## Linux package repositories
|
||||
|
||||
All four repositories serve stable and canary. The package is `fluxer` for stable and `fluxer-canary` for canary.
|
||||
|
||||
### Flatpak
|
||||
|
||||
Opening [this reference file][flatpak-ref] hands the install to your desktop software manager. Some desktops also accept `flatpak+https://pkgs.fluxer.com/flatpak/fluxer.flatpakref` pasted into the address bar.
|
||||
|
||||
From a terminal:
|
||||
|
||||
```sh
|
||||
flatpak install https://pkgs.fluxer.com/flatpak/fluxer.flatpakref
|
||||
```
|
||||
|
||||
### Debian and Ubuntu
|
||||
|
||||
```sh
|
||||
sudo install -d -m 0755 /etc/apt/keyrings
|
||||
sudo curl -fsSL -o /etc/apt/keyrings/fluxer-archive-keyring.gpg https://pkgs.fluxer.com/keys/fluxer-archive-keyring.gpg
|
||||
sudo curl -fsSL -o /etc/apt/sources.list.d/fluxer.sources https://pkgs.fluxer.com/deb/fluxer.sources
|
||||
sudo apt update && sudo apt install fluxer
|
||||
```
|
||||
|
||||
### Fedora and RHEL
|
||||
|
||||
```sh
|
||||
sudo curl -fsSL -o /etc/yum.repos.d/fluxer.repo https://pkgs.fluxer.com/rpm/fluxer.repo
|
||||
sudo dnf install fluxer
|
||||
```
|
||||
|
||||
RHEL, Rocky, Alma and CentOS Stream need `sudo dnf install epel-release` first, because the base repositories do not ship `libXScrnSaver`. Fedora does not need this.
|
||||
|
||||
### Arch Linux
|
||||
|
||||
The repository is signed, so pacman needs the key in its own keyring once:
|
||||
|
||||
```sh
|
||||
sudo pacman-key --init
|
||||
curl -fsSL -o /tmp/fluxer-archive-keyring.asc https://pkgs.fluxer.com/keys/fluxer-archive-keyring.asc
|
||||
sudo pacman-key --add /tmp/fluxer-archive-keyring.asc
|
||||
sudo pacman-key --lsign-key 09D01339EE128925F75E675C855C5BDE34D205D2
|
||||
```
|
||||
|
||||
`--lsign-key` is the step that makes pacman trust the key. Then add the repository:
|
||||
|
||||
```sh
|
||||
sudo tee -a /etc/pacman.conf >/dev/null <<'REPO'
|
||||
|
||||
[fluxer]
|
||||
SigLevel = Required TrustedOnly
|
||||
Server = https://pkgs.fluxer.com/arch/$repo/os/$arch
|
||||
REPO
|
||||
sudo pacman -Syu fluxer
|
||||
```
|
||||
|
||||
Write `$repo` and `$arch` literally. Both are pacman variables, not shell ones, which is why the heredoc above is quoted.
|
||||
|
||||
Full setup notes, including the canary channel, live in the [Linux repositories documentation][docs-linux].
|
||||
|
||||
## Other ways to run it
|
||||
|
||||
- [Open Fluxer in a browser](https://web.fluxer.app) with no install at all.
|
||||
- [Host your own instance][docs-selfhost] from this repository.
|
||||
|
||||
## Documentation
|
||||
|
||||
- [Documentation home][docs]
|
||||
- [Downloads][docs-downloads]
|
||||
- [Self-hosting][docs-selfhost]
|
||||
|
||||
## License
|
||||
|
||||
The source is licensed under the [AGPL-3.0-or-later](./LICENSE) license.
|
||||
|
||||
Fluxer branding, icons, default avatars, badge artwork, screenshots and marketing
|
||||
imagery are copyright Fluxer and all rights reserved, as set out in
|
||||
[fluxer_static/LICENSE](./fluxer_static/LICENSE). Third-party material keeps its
|
||||
own terms, listed in
|
||||
[fluxer_static/THIRD_PARTY_LICENSES.md](./fluxer_static/THIRD_PARTY_LICENSES.md).
|
||||
|
||||
Public availability of this repository does not grant trademark, brand, or
|
||||
endorsement rights.
|
||||
|
||||
[win-setup-x64]: https://pkgs.fluxer.com/desktop/stable/win32/x64/latest/setup
|
||||
[win-setup-arm64]: https://pkgs.fluxer.com/desktop/stable/win32/arm64/latest/setup
|
||||
[win-portable-x64]: https://pkgs.fluxer.com/desktop/stable/win32/x64/latest/portable
|
||||
[win-portable-arm64]: https://pkgs.fluxer.com/desktop/stable/win32/arm64/latest/portable
|
||||
[mac-dmg]: https://pkgs.fluxer.com/desktop/stable/darwin/arm64/latest/dmg
|
||||
[linux-deb-x64]: https://pkgs.fluxer.com/desktop/stable/linux/x64/latest/deb
|
||||
[linux-deb-arm64]: https://pkgs.fluxer.com/desktop/stable/linux/arm64/latest/deb
|
||||
[linux-rpm-x64]: https://pkgs.fluxer.com/desktop/stable/linux/x64/latest/rpm
|
||||
[linux-rpm-arm64]: https://pkgs.fluxer.com/desktop/stable/linux/arm64/latest/rpm
|
||||
[linux-appimage-x64]: https://pkgs.fluxer.com/desktop/stable/linux/x64/latest/appimage
|
||||
[linux-appimage-arm64]: https://pkgs.fluxer.com/desktop/stable/linux/arm64/latest/appimage
|
||||
[linux-targz-x64]: https://pkgs.fluxer.com/desktop/stable/linux/x64/latest/tar_gz
|
||||
[linux-targz-arm64]: https://pkgs.fluxer.com/desktop/stable/linux/arm64/latest/tar_gz
|
||||
[flatpak-ref]: https://pkgs.fluxer.com/flatpak/fluxer.flatpakref
|
||||
[android-apk]: https://github.com/fluxerapp/flutter_client/releases
|
||||
[obtainium]: https://obtainium.imranr.dev/
|
||||
[ios-testflight]: https://testflight.apple.com/join/PKZR6pK9
|
||||
[docs]: https://docs.fluxer.app
|
||||
[docs-downloads]: https://docs.fluxer.app/downloads/overview/
|
||||
[docs-linux]: https://docs.fluxer.app/downloads/linux-repositories/
|
||||
[docs-selfhost]: https://docs.fluxer.app/operator/get-started/
|
||||
|
||||
@@ -281,6 +281,7 @@ FLUXER_DISCOVERY_ENABLED=true
|
||||
#FLUXER_POSTGRES_WORK_MEM=8MB
|
||||
#FLUXER_POSTGRES_MAINTENANCE_WORK_MEM=256MB
|
||||
#FLUXER_POSTGRES_AUTOVACUUM_WORK_MEM=128MB
|
||||
#FLUXER_POSTGRES_SHM_SIZE=1gb
|
||||
|
||||
# The bundled Valkey holds durable state as well as cache, so it runs with an
|
||||
# append-only file and with noeviction, which fails an over-limit write instead
|
||||
|
||||
@@ -188,7 +188,7 @@ services:
|
||||
-c autovacuum_vacuum_cost_limit=2000
|
||||
-c track_io_timing=on
|
||||
-c shared_preload_libraries=pg_stat_statements
|
||||
shm_size: 256mb
|
||||
shm_size: ${FLUXER_POSTGRES_SHM_SIZE:-1gb}
|
||||
environment:
|
||||
POSTGRES_DB: fluxer
|
||||
POSTGRES_USER: fluxer
|
||||
|
||||
@@ -9,7 +9,7 @@ WORKDIR /usr/src/app
|
||||
|
||||
RUN apt-get update \
|
||||
&& apt-get install -y --no-install-recommends ca-certificates nodejs npm pkg-config \
|
||||
&& npm install -g pnpm@12.4.2 \
|
||||
&& npm install -g pnpm@11.27.0 \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
RUN npm install --no-audit --no-fund @tailwindcss/[email protected] [email protected]
|
||||
|
||||
@@ -10524,6 +10524,7 @@
|
||||
},
|
||||
"gateway_rollout": {"$ref": "#/components/schemas/GatewayRolloutConfigResponse"},
|
||||
"voice_noise_suppression": {"$ref": "#/components/schemas/VoiceNoiseSuppressionConfigResponse"},
|
||||
"screen_share_delivery": {"$ref": "#/components/schemas/ScreenShareDeliveryConfigResponse"},
|
||||
"experiment_delivery": {"$ref": "#/components/schemas/ExperimentDeliveryConfigResponse"},
|
||||
"registration": {
|
||||
"type": "object",
|
||||
@@ -10951,6 +10952,7 @@
|
||||
"sso",
|
||||
"gateway_rollout",
|
||||
"voice_noise_suppression",
|
||||
"screen_share_delivery",
|
||||
"experiment_delivery",
|
||||
"registration",
|
||||
"self_hosted",
|
||||
@@ -11085,6 +11087,10 @@
|
||||
"nullable": true,
|
||||
"allOf": [{"$ref": "#/components/schemas/VoiceNoiseSuppressionConfigUpdateRequest"}]
|
||||
},
|
||||
"screen_share_delivery": {
|
||||
"nullable": true,
|
||||
"allOf": [{"$ref": "#/components/schemas/ScreenShareDeliveryConfigUpdateRequest"}]
|
||||
},
|
||||
"experiment_delivery": {
|
||||
"nullable": true,
|
||||
"allOf": [{"$ref": "#/components/schemas/ExperimentDeliveryConfigUpdateRequest"}]
|
||||
@@ -15178,6 +15184,24 @@
|
||||
"poll_jitter_percent": {"type": "integer", "minimum": 0, "maximum": 50}
|
||||
}
|
||||
},
|
||||
"ScreenShareDeliveryConfigUpdateRequest": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"enabled": {"type": "boolean"},
|
||||
"rollout_basis_points": {"type": "integer", "minimum": 0, "maximum": 10000},
|
||||
"rollout_salt": {"type": "string", "minLength": 1, "maxLength": 64},
|
||||
"included_user_ids": {
|
||||
"maxItems": 1000,
|
||||
"type": "array",
|
||||
"items": {"type": "string", "pattern": "^\\d{1,20}$"}
|
||||
},
|
||||
"excluded_user_ids": {
|
||||
"maxItems": 1000,
|
||||
"type": "array",
|
||||
"items": {"type": "string", "pattern": "^\\d{1,20}$"}
|
||||
}
|
||||
}
|
||||
},
|
||||
"VoiceNoiseSuppressionConfigUpdateRequest": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
@@ -15243,6 +15267,36 @@
|
||||
"required": ["poll_interval_seconds", "poll_jitter_percent"],
|
||||
"additionalProperties": false
|
||||
},
|
||||
"ScreenShareDeliveryConfigResponse": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"enabled": {"default": false, "type": "boolean"},
|
||||
"config_version": {"default": 0, "type": "integer", "minimum": 0, "maximum": 9007199254740991},
|
||||
"rollout_basis_points": {"default": 0, "type": "integer", "minimum": 0, "maximum": 10000},
|
||||
"rollout_salt": {"default": "screen-share-delivery-v1", "type": "string", "minLength": 1, "maxLength": 64},
|
||||
"included_user_ids": {
|
||||
"default": [],
|
||||
"maxItems": 1000,
|
||||
"type": "array",
|
||||
"items": {"type": "string", "pattern": "^\\d{1,20}$"}
|
||||
},
|
||||
"excluded_user_ids": {
|
||||
"default": [],
|
||||
"maxItems": 1000,
|
||||
"type": "array",
|
||||
"items": {"type": "string", "pattern": "^\\d{1,20}$"}
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"enabled",
|
||||
"config_version",
|
||||
"rollout_basis_points",
|
||||
"rollout_salt",
|
||||
"included_user_ids",
|
||||
"excluded_user_ids"
|
||||
],
|
||||
"additionalProperties": false
|
||||
},
|
||||
"VoiceNoiseSuppressionConfigResponse": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
|
||||
@@ -23,6 +23,8 @@ pub struct InstanceConfigResponse {
|
||||
#[serde(default)]
|
||||
pub voice_noise_suppression: VoiceNoiseSuppressionConfigResponse,
|
||||
#[serde(default)]
|
||||
pub screen_share_delivery: ScreenShareDeliveryConfigResponse,
|
||||
#[serde(default)]
|
||||
pub experiment_delivery: ExperimentDeliveryConfigResponse,
|
||||
}
|
||||
|
||||
@@ -446,7 +448,8 @@ impl VoiceE2eeScope {
|
||||
}
|
||||
}
|
||||
|
||||
pub const VOICE_NS_MAX_TARGETED_USERS: usize = 1_000;
|
||||
pub const EXPERIMENT_MAX_TARGETED_USERS: usize = 1_000;
|
||||
pub const SCREEN_SHARE_DELIVERY_DEFAULT_SALT: &str = "screen-share-delivery-v1";
|
||||
pub const VOICE_NS_MAX_GUILD_OVERRIDES: usize = 200;
|
||||
|
||||
impl NoiseSuppressionBackend {
|
||||
@@ -537,6 +540,44 @@ pub struct VoiceNoiseSuppressionConfigUpdateRequest {
|
||||
pub suppression_strength: Option<u32>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Deserialize, Serialize)]
|
||||
#[serde(default)]
|
||||
pub struct ScreenShareDeliveryConfigResponse {
|
||||
pub enabled: bool,
|
||||
pub config_version: u64,
|
||||
pub rollout_basis_points: u32,
|
||||
pub rollout_salt: String,
|
||||
pub included_user_ids: Vec<String>,
|
||||
pub excluded_user_ids: Vec<String>,
|
||||
}
|
||||
|
||||
impl Default for ScreenShareDeliveryConfigResponse {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
enabled: false,
|
||||
config_version: 0,
|
||||
rollout_basis_points: 0,
|
||||
rollout_salt: SCREEN_SHARE_DELIVERY_DEFAULT_SALT.to_owned(),
|
||||
included_user_ids: Vec::new(),
|
||||
excluded_user_ids: Vec::new(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Default, Serialize)]
|
||||
pub struct ScreenShareDeliveryConfigUpdateRequest {
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub enabled: Option<bool>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub rollout_basis_points: Option<u32>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub rollout_salt: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub included_user_ids: Option<Vec<String>>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub excluded_user_ids: Option<Vec<String>>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Deserialize, Serialize)]
|
||||
#[serde(default)]
|
||||
pub struct ExperimentDeliveryConfigResponse {
|
||||
@@ -653,6 +694,8 @@ pub struct InstanceConfigUpdateRequest {
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub voice_noise_suppression: Option<VoiceNoiseSuppressionConfigUpdateRequest>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub screen_share_delivery: Option<ScreenShareDeliveryConfigUpdateRequest>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub experiment_delivery: Option<ExperimentDeliveryConfigUpdateRequest>,
|
||||
}
|
||||
|
||||
@@ -990,18 +1033,30 @@ mod tests {
|
||||
.expect("admin schema");
|
||||
let noise = serde_json::from_value::<VoiceNoiseSuppressionConfigResponse>(json!({}))
|
||||
.expect("default noise config");
|
||||
let screen_share = serde_json::from_value::<ScreenShareDeliveryConfigResponse>(json!({}))
|
||||
.expect("default screen share config");
|
||||
let delivery = serde_json::from_value::<ExperimentDeliveryConfigResponse>(json!({}))
|
||||
.expect("default delivery config");
|
||||
let noise = serde_json::to_value(noise).expect("serializable noise config");
|
||||
let screen_share =
|
||||
serde_json::to_value(screen_share).expect("serializable screen share config");
|
||||
let delivery = serde_json::to_value(delivery).expect("serializable delivery config");
|
||||
let generated_noise: generated_types::VoiceNoiseSuppressionConfigResponse =
|
||||
serde_json::from_value(noise.clone()).expect("generated noise config contract");
|
||||
let generated_screen_share: generated_types::ScreenShareDeliveryConfigResponse =
|
||||
serde_json::from_value(screen_share.clone())
|
||||
.expect("generated screen share config contract");
|
||||
let generated_delivery: generated_types::ExperimentDeliveryConfigResponse =
|
||||
serde_json::from_value(delivery.clone()).expect("generated delivery config contract");
|
||||
assert_eq!(
|
||||
serde_json::to_value(generated_noise).expect("serializable generated noise config"),
|
||||
noise
|
||||
);
|
||||
assert_eq!(
|
||||
serde_json::to_value(generated_screen_share)
|
||||
.expect("serializable generated screen share config"),
|
||||
screen_share
|
||||
);
|
||||
assert_eq!(
|
||||
serde_json::to_value(generated_delivery)
|
||||
.expect("serializable generated delivery config"),
|
||||
@@ -1009,6 +1064,7 @@ mod tests {
|
||||
);
|
||||
for (name, value) in [
|
||||
("VoiceNoiseSuppressionConfigResponse", noise),
|
||||
("ScreenShareDeliveryConfigResponse", screen_share),
|
||||
("ExperimentDeliveryConfigResponse", delivery),
|
||||
] {
|
||||
for (field, value) in value.as_object().expect("config object") {
|
||||
@@ -1020,6 +1076,29 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn screen_share_delivery_update_preserves_empty_lists_and_omitted_fields() {
|
||||
let update = ScreenShareDeliveryConfigUpdateRequest {
|
||||
included_user_ids: Some(Vec::new()),
|
||||
excluded_user_ids: Some(Vec::new()),
|
||||
..Default::default()
|
||||
};
|
||||
let value = serde_json::to_value(update).expect("serializable update");
|
||||
serde_json::from_value::<generated_types::ScreenShareDeliveryConfigUpdateRequest>(
|
||||
value.clone(),
|
||||
)
|
||||
.expect("generated update contract");
|
||||
assert_eq!(
|
||||
value,
|
||||
json!({"included_user_ids": [], "excluded_user_ids": []})
|
||||
);
|
||||
assert_eq!(
|
||||
serde_json::to_value(ScreenShareDeliveryConfigUpdateRequest::default())
|
||||
.expect("serializable update"),
|
||||
json!({})
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn noise_suppression_update_preserves_empty_lists_and_omitted_fields() {
|
||||
let update = VoiceNoiseSuppressionConfigUpdateRequest {
|
||||
|
||||
@@ -7,20 +7,20 @@ use crate::{
|
||||
AppBrandingConfigUpdateRequest, AppLegalConfigUpdateRequest,
|
||||
AppPublicConfigUpdateRequest, AppRegistrationConfigUpdateRequest,
|
||||
AppSetupConfigUpdateRequest, CreateRegistrationUrlRequest,
|
||||
DeferredPhoneGateUpdateRequest, ExperimentDeliveryConfigUpdateRequest,
|
||||
GatewayRolloutConfigUpdateRequest, GatewayRolloutMode,
|
||||
InstanceAttachmentDecayUpdateRequest, InstanceBlueskyIntegrationUpdateRequest,
|
||||
InstanceBlueskyKeyIntegrationUpdateRequest, InstanceCaptchaIntegrationUpdateRequest,
|
||||
InstanceConfigUpdateRequest, InstanceEmailIntegrationUpdateRequest,
|
||||
InstanceEmailSmtpIntegrationUpdateRequest, InstanceEmailSmtpTestRequest,
|
||||
InstanceGifIntegrationUpdateRequest, InstanceIntegrationsUpdateRequest,
|
||||
InstanceMediaUpdateRequest, InstancePolicyUpdateRequest,
|
||||
InstanceRegistrationConfigUpdateRequest, InstanceServicesUpdateRequest,
|
||||
InstanceYoutubeIntegrationUpdateRequest, LimitConfigUpdateRequest, LimitRule,
|
||||
LimitRuleFilters, NoiseSuppressionBackend, PremiumMode, RegistrationMode,
|
||||
SsoConfigUpdateRequest, VOICE_NS_MAX_GUILD_OVERRIDES, VOICE_NS_MAX_TARGETED_USERS,
|
||||
VoiceE2eeScope, VoiceNoiseSuppressionConfigUpdateRequest,
|
||||
VoiceNoiseSuppressionGuildOverride,
|
||||
DeferredPhoneGateUpdateRequest, EXPERIMENT_MAX_TARGETED_USERS,
|
||||
ExperimentDeliveryConfigUpdateRequest, GatewayRolloutConfigUpdateRequest,
|
||||
GatewayRolloutMode, InstanceAttachmentDecayUpdateRequest,
|
||||
InstanceBlueskyIntegrationUpdateRequest, InstanceBlueskyKeyIntegrationUpdateRequest,
|
||||
InstanceCaptchaIntegrationUpdateRequest, InstanceConfigUpdateRequest,
|
||||
InstanceEmailIntegrationUpdateRequest, InstanceEmailSmtpIntegrationUpdateRequest,
|
||||
InstanceEmailSmtpTestRequest, InstanceGifIntegrationUpdateRequest,
|
||||
InstanceIntegrationsUpdateRequest, InstanceMediaUpdateRequest,
|
||||
InstancePolicyUpdateRequest, InstanceRegistrationConfigUpdateRequest,
|
||||
InstanceServicesUpdateRequest, InstanceYoutubeIntegrationUpdateRequest,
|
||||
LimitConfigUpdateRequest, LimitRule, LimitRuleFilters, NoiseSuppressionBackend,
|
||||
PremiumMode, RegistrationMode, ScreenShareDeliveryConfigUpdateRequest,
|
||||
SsoConfigUpdateRequest, VOICE_NS_MAX_GUILD_OVERRIDES, VoiceE2eeScope,
|
||||
VoiceNoiseSuppressionConfigUpdateRequest, VoiceNoiseSuppressionGuildOverride,
|
||||
},
|
||||
},
|
||||
config::AdminConfig,
|
||||
@@ -207,6 +207,10 @@ pub async fn instance_config_post(
|
||||
Ok(update) => instance_config_result(client.update_instance_config(&update).await),
|
||||
Err(message) => FlashData::error(message),
|
||||
},
|
||||
"update_screen_share_delivery" => match build_screen_share_delivery_update(&form) {
|
||||
Ok(update) => instance_config_result(client.update_instance_config(&update).await),
|
||||
Err(message) => FlashData::error(message),
|
||||
},
|
||||
"update_experiment_delivery" => match build_experiment_delivery_update(&form) {
|
||||
Ok(update) => instance_config_result(client.update_instance_config(&update).await),
|
||||
Err(message) => FlashData::error(message),
|
||||
@@ -514,9 +518,9 @@ fn parse_experiment_user_ids(value: &str, label: &str) -> Result<Vec<String>, St
|
||||
if ids.iter().any(|existing| existing == candidate) {
|
||||
continue;
|
||||
}
|
||||
if ids.len() == VOICE_NS_MAX_TARGETED_USERS {
|
||||
if ids.len() == EXPERIMENT_MAX_TARGETED_USERS {
|
||||
return Err(format!(
|
||||
"{label} must contain at most {VOICE_NS_MAX_TARGETED_USERS} unique IDs"
|
||||
"{label} must contain at most {EXPERIMENT_MAX_TARGETED_USERS} unique IDs"
|
||||
));
|
||||
}
|
||||
ids.push(candidate.to_owned());
|
||||
@@ -629,6 +633,38 @@ fn build_voice_noise_suppression_update(
|
||||
})
|
||||
}
|
||||
|
||||
fn build_screen_share_delivery_update(
|
||||
form: &MultiValueForm,
|
||||
) -> Result<InstanceConfigUpdateRequest, String> {
|
||||
Ok(InstanceConfigUpdateRequest {
|
||||
screen_share_delivery: Some(ScreenShareDeliveryConfigUpdateRequest {
|
||||
enabled: Some(form.bool_value("screen_share_delivery_enabled")),
|
||||
rollout_basis_points: parse_form_number(
|
||||
form,
|
||||
"screen_share_delivery_rollout_basis_points",
|
||||
"Rollout basis points",
|
||||
0,
|
||||
EXPERIMENT_ROLLOUT_BASIS_POINTS_MAX,
|
||||
)?,
|
||||
rollout_salt: parse_experiment_rollout_salt(
|
||||
form,
|
||||
"screen_share_delivery_rollout_salt",
|
||||
)?,
|
||||
included_user_ids: Some(parse_experiment_user_ids(
|
||||
form.first("screen_share_delivery_included_user_ids")
|
||||
.unwrap_or_default(),
|
||||
"Included user IDs",
|
||||
)?),
|
||||
excluded_user_ids: Some(parse_experiment_user_ids(
|
||||
form.first("screen_share_delivery_excluded_user_ids")
|
||||
.unwrap_or_default(),
|
||||
"Excluded user IDs",
|
||||
)?),
|
||||
}),
|
||||
..Default::default()
|
||||
})
|
||||
}
|
||||
|
||||
fn build_experiment_delivery_update(
|
||||
form: &MultiValueForm,
|
||||
) -> Result<InstanceConfigUpdateRequest, String> {
|
||||
@@ -1348,13 +1384,13 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn parse_experiment_user_ids_rejects_exceeding_the_cap() {
|
||||
let value = (0..VOICE_NS_MAX_TARGETED_USERS)
|
||||
let value = (0..EXPERIMENT_MAX_TARGETED_USERS)
|
||||
.map(|index| index.to_string())
|
||||
.collect::<Vec<_>>()
|
||||
.join("\n");
|
||||
let ids = parse_experiment_user_ids(&format!("{value}\n999"), "Included user IDs")
|
||||
.expect("valid IDs at cap");
|
||||
assert_eq!(ids.len(), VOICE_NS_MAX_TARGETED_USERS);
|
||||
assert_eq!(ids.len(), EXPERIMENT_MAX_TARGETED_USERS);
|
||||
assert_eq!(ids.last(), Some(&"999".to_owned()));
|
||||
assert_eq!(
|
||||
parse_experiment_user_ids(&format!("{value}\n1000"), "Included user IDs")
|
||||
@@ -1550,6 +1586,83 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn build_screen_share_delivery_update_reads_the_rollout_fields() {
|
||||
let form = MultiValueForm::parse(
|
||||
b"screen_share_delivery_enabled=true&screen_share_delivery_rollout_basis_points=%20250%20&screen_share_delivery_rollout_salt=%20screen-share-delivery-v2%20&screen_share_delivery_included_user_ids=1500000000000000001%0A1500000000000000002&screen_share_delivery_excluded_user_ids=1500000000000000003%2C%201500000000000000004",
|
||||
);
|
||||
let update = build_screen_share_delivery_update(&form)
|
||||
.expect("valid form")
|
||||
.screen_share_delivery
|
||||
.expect("screen share delivery update");
|
||||
assert_eq!(update.enabled, Some(true));
|
||||
assert_eq!(update.rollout_basis_points, Some(250));
|
||||
assert_eq!(
|
||||
update.rollout_salt,
|
||||
Some("screen-share-delivery-v2".to_owned())
|
||||
);
|
||||
assert_eq!(
|
||||
update.included_user_ids,
|
||||
Some(vec![
|
||||
"1500000000000000001".to_owned(),
|
||||
"1500000000000000002".to_owned()
|
||||
])
|
||||
);
|
||||
assert_eq!(
|
||||
update.excluded_user_ids,
|
||||
Some(vec![
|
||||
"1500000000000000003".to_owned(),
|
||||
"1500000000000000004".to_owned()
|
||||
])
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn build_screen_share_delivery_update_leaves_the_feature_inert_when_nothing_is_submitted() {
|
||||
let form = MultiValueForm::parse(b"_csrf=token");
|
||||
let request = build_screen_share_delivery_update(&form).expect("valid form");
|
||||
assert_eq!(
|
||||
serde_json::to_value(request).expect("serializable update"),
|
||||
serde_json::json!({"screen_share_delivery": {
|
||||
"enabled": false,
|
||||
"included_user_ids": [],
|
||||
"excluded_user_ids": [],
|
||||
}})
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn build_screen_share_delivery_update_rejects_invalid_rollout_fields() {
|
||||
for (form, message) in [
|
||||
(
|
||||
"screen_share_delivery_rollout_basis_points=10001",
|
||||
"Rollout basis points must be a whole number between 0 and 10000",
|
||||
),
|
||||
(
|
||||
"screen_share_delivery_rollout_basis_points=abc",
|
||||
"Rollout basis points must be a whole number between 0 and 10000",
|
||||
),
|
||||
(
|
||||
"screen_share_delivery_rollout_salt=%20%20",
|
||||
"Rollout salt must be between 1 and 64 characters",
|
||||
),
|
||||
(
|
||||
"screen_share_delivery_included_user_ids=123%2Cinvalid",
|
||||
"Included user IDs entry 2 must contain 1 to 20 decimal digits",
|
||||
),
|
||||
(
|
||||
"screen_share_delivery_excluded_user_ids=123%2Cinvalid",
|
||||
"Excluded user IDs entry 2 must contain 1 to 20 decimal digits",
|
||||
),
|
||||
] {
|
||||
let form = MultiValueForm::parse(form.as_bytes());
|
||||
assert_eq!(
|
||||
build_screen_share_delivery_update(&form).expect_err("invalid rollout field"),
|
||||
message
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn build_experiment_delivery_update_leaves_both_fields_unchanged_when_absent() {
|
||||
let form = MultiValueForm::parse(b"_csrf=token");
|
||||
|
||||
@@ -73,15 +73,11 @@ pub async fn render(
|
||||
.map(|r| r.sessions)
|
||||
.map_err(|error| tracing::warn!(%error, user_id, "admin API request failed: list user sessions"))
|
||||
.unwrap_or_default();
|
||||
let webauthn_credentials = if u.authenticator_types.contains(&2) {
|
||||
client
|
||||
.list_webauthn_credentials(user_id)
|
||||
.await
|
||||
.map_err(|error| tracing::warn!(%error, user_id, "admin API request failed: list webauthn credentials"))
|
||||
.unwrap_or_default()
|
||||
} else {
|
||||
Vec::new()
|
||||
};
|
||||
let webauthn_credentials = client
|
||||
.list_webauthn_credentials(user_id)
|
||||
.await
|
||||
.map_err(|error| tracing::warn!(%error, user_id, "admin API request failed: list webauthn credentials"))
|
||||
.unwrap_or_default();
|
||||
Some(tabs::account::account_tab(
|
||||
config,
|
||||
&u,
|
||||
|
||||
@@ -2,11 +2,12 @@
|
||||
|
||||
use crate::{
|
||||
api::types::{
|
||||
AppPublicConfigResponse, ExperimentDeliveryConfigResponse, GatewayRolloutConfigResponse,
|
||||
InstanceConfigResponse, InstanceIntegrationsResponse, InstanceMediaResponse,
|
||||
InstancePolicyResponse, InstanceRegistrationResponse, LimitConfigResponse,
|
||||
NoiseSuppressionBackend, PendingRegistrationResponse, RegistrationUrlResponse,
|
||||
SsoConfigResponse, VOICE_NS_MAX_GUILD_OVERRIDES, VOICE_NS_MAX_TARGETED_USERS,
|
||||
AppPublicConfigResponse, EXPERIMENT_MAX_TARGETED_USERS, ExperimentDeliveryConfigResponse,
|
||||
GatewayRolloutConfigResponse, InstanceConfigResponse, InstanceIntegrationsResponse,
|
||||
InstanceMediaResponse, InstancePolicyResponse, InstanceRegistrationResponse,
|
||||
LimitConfigResponse, NoiseSuppressionBackend, PendingRegistrationResponse,
|
||||
RegistrationUrlResponse, SCREEN_SHARE_DELIVERY_DEFAULT_SALT,
|
||||
ScreenShareDeliveryConfigResponse, SsoConfigResponse, VOICE_NS_MAX_GUILD_OVERRIDES,
|
||||
VoiceNoiseSuppressionConfigResponse,
|
||||
},
|
||||
config::AdminConfig,
|
||||
@@ -148,6 +149,7 @@ pub fn instance_config_page(
|
||||
html! {
|
||||
(gateway_rollout_section(base, csrf_token, &instance_config.gateway_rollout))
|
||||
(voice_noise_suppression_section(base, csrf_token, &instance_config.voice_noise_suppression))
|
||||
(screen_share_delivery_section(base, csrf_token, &instance_config.screen_share_delivery))
|
||||
(experiment_delivery_section(base, csrf_token, &instance_config.experiment_delivery))
|
||||
@if let Some(limit_config) = limit_config {
|
||||
(limit_config_section(base, limit_config))
|
||||
@@ -1105,7 +1107,7 @@ fn voice_noise_suppression_section(
|
||||
))
|
||||
(entry_count_hint(
|
||||
voice_noise_suppression.included_user_ids.len(),
|
||||
VOICE_NS_MAX_TARGETED_USERS,
|
||||
EXPERIMENT_MAX_TARGETED_USERS,
|
||||
))
|
||||
p class="text-xs text-neutral-500" {
|
||||
"One snowflake per line, or comma separated. These users are targeted \
|
||||
@@ -1125,7 +1127,7 @@ fn voice_noise_suppression_section(
|
||||
))
|
||||
(entry_count_hint(
|
||||
voice_noise_suppression.excluded_user_ids.len(),
|
||||
VOICE_NS_MAX_TARGETED_USERS,
|
||||
EXPERIMENT_MAX_TARGETED_USERS,
|
||||
))
|
||||
p class="text-xs text-neutral-500" {
|
||||
"Same format. Exclusion wins over both the always-on list and the \
|
||||
@@ -1176,6 +1178,117 @@ fn voice_noise_suppression_section(
|
||||
)
|
||||
}
|
||||
|
||||
fn screen_share_delivery_section(
|
||||
base: &str,
|
||||
csrf_token: &str,
|
||||
screen_share_delivery: &ScreenShareDeliveryConfigResponse,
|
||||
) -> Markup {
|
||||
let status = if screen_share_delivery.enabled {
|
||||
("Live", BadgeVariant::Success)
|
||||
} else {
|
||||
("Inert", BadgeVariant::Default)
|
||||
};
|
||||
let included_user_ids = screen_share_delivery.included_user_ids.join("\n");
|
||||
let excluded_user_ids = screen_share_delivery.excluded_user_ids.join("\n");
|
||||
section_card_with_description(
|
||||
"Screen Share Delivery",
|
||||
"Pick how many clients publish screen shares through the reworked delivery path. While \
|
||||
the master switch below is off nothing on this form reaches any client: every user \
|
||||
keeps the screen share pipeline they have today, whatever the rest of these fields say. \
|
||||
A client that is already sharing keeps the path it started on until the share ends.",
|
||||
html! {
|
||||
form method="post" action={(base) "/instance-config?action=update_screen_share_delivery"} {
|
||||
(csrf_input(csrf_token))
|
||||
div class="space-y-6" {
|
||||
div class="flex flex-wrap items-center gap-2" {
|
||||
h3 class="text-sm font-semibold text-neutral-900" { "Master switch" }
|
||||
(badge(status.0, status.1))
|
||||
span class="text-xs text-neutral-500" {
|
||||
"Config version " (screen_share_delivery.config_version)
|
||||
}
|
||||
}
|
||||
(checkbox(
|
||||
"screen_share_delivery_enabled",
|
||||
"true",
|
||||
"Serve screen share delivery assignments to clients",
|
||||
screen_share_delivery.enabled,
|
||||
true,
|
||||
))
|
||||
p class="text-xs text-neutral-500" {
|
||||
"Off is the safe state. With this unchecked every client is told the \
|
||||
feature is inert and keeps its current behavior, so the rollout and \
|
||||
targeting fields below have no effect at all."
|
||||
}
|
||||
|
||||
h3 class="text-sm font-semibold text-neutral-900" { "Rollout" }
|
||||
(number_field(
|
||||
"screen_share_delivery_rollout_basis_points",
|
||||
"Rollout (basis points)",
|
||||
&screen_share_delivery.rollout_basis_points.to_string(),
|
||||
Some(0), Some(10000), "1",
|
||||
Some("Share of users bucketed into the canary, in basis points: 0 is nobody, 100 is 1%, 10000 is everybody."),
|
||||
))
|
||||
div class="flex flex-col gap-2" {
|
||||
(text_input(
|
||||
"screen_share_delivery_rollout_salt",
|
||||
"Rollout Salt",
|
||||
&screen_share_delivery.rollout_salt,
|
||||
SCREEN_SHARE_DELIVERY_DEFAULT_SALT,
|
||||
))
|
||||
p class="text-xs text-neutral-500" {
|
||||
"Seeds the bucketing hash. Changing it reshuffles which users fall \
|
||||
inside the percentage above. Leave it alone to keep the current \
|
||||
cohort stable."
|
||||
}
|
||||
}
|
||||
div class="flex flex-col gap-2" {
|
||||
(textarea_input(
|
||||
"screen_share_delivery_included_user_ids",
|
||||
"Always-on User IDs",
|
||||
"1500000000000000001\n1500000000000000002",
|
||||
&included_user_ids,
|
||||
4,
|
||||
false,
|
||||
))
|
||||
(entry_count_hint(
|
||||
screen_share_delivery.included_user_ids.len(),
|
||||
EXPERIMENT_MAX_TARGETED_USERS,
|
||||
))
|
||||
p class="text-xs text-neutral-500" {
|
||||
"One snowflake per line, or comma separated. These users are targeted \
|
||||
regardless of the percentage above. IDs must contain 1 to 20 decimal \
|
||||
digits. Invalid entries prevent the save; blank entries and duplicate \
|
||||
IDs are ignored."
|
||||
}
|
||||
}
|
||||
div class="flex flex-col gap-2" {
|
||||
(textarea_input(
|
||||
"screen_share_delivery_excluded_user_ids",
|
||||
"Never-on User IDs",
|
||||
"1500000000000000003\n1500000000000000004",
|
||||
&excluded_user_ids,
|
||||
4,
|
||||
false,
|
||||
))
|
||||
(entry_count_hint(
|
||||
screen_share_delivery.excluded_user_ids.len(),
|
||||
EXPERIMENT_MAX_TARGETED_USERS,
|
||||
))
|
||||
p class="text-xs text-neutral-500" {
|
||||
"Same format. Exclusion wins over both the always-on list and the \
|
||||
percentage, so this is the per-user kill switch."
|
||||
}
|
||||
}
|
||||
|
||||
(form_actions(html! {
|
||||
(submit_button("Save Screen Share Delivery Configuration"))
|
||||
}))
|
||||
}
|
||||
}
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
fn experiment_delivery_section(
|
||||
base: &str,
|
||||
csrf_token: &str,
|
||||
@@ -1184,7 +1297,7 @@ fn experiment_delivery_section(
|
||||
section_card_with_description(
|
||||
"Experiment Delivery",
|
||||
"How often every client revalidates its experiment assignments. This is instance-wide \
|
||||
and covers every experiment, not just the one above. Raising the interval sheds \
|
||||
and covers every experiment, not just the ones above. Raising the interval sheds \
|
||||
request volume and makes a change take longer to reach a client. Raising the jitter \
|
||||
spreads a fleet that has synchronised on one tick back out across the interval.",
|
||||
html! {
|
||||
@@ -1819,10 +1932,29 @@ mod tests {
|
||||
assert!(!markup.contains("at the cap"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn screen_share_delivery_section_shows_list_counts_and_the_master_switch() {
|
||||
let screen_share_delivery = ScreenShareDeliveryConfigResponse {
|
||||
included_user_ids: vec!["1500000000000000001".to_owned()],
|
||||
excluded_user_ids: vec![
|
||||
"1500000000000000002".to_owned(),
|
||||
"1500000000000000003".to_owned(),
|
||||
],
|
||||
..ScreenShareDeliveryConfigResponse::default()
|
||||
};
|
||||
let markup =
|
||||
screen_share_delivery_section("/admin", "csrf", &screen_share_delivery).into_string();
|
||||
assert!(markup.contains("action=update_screen_share_delivery"));
|
||||
assert!(markup.contains("screen_share_delivery_enabled"));
|
||||
assert!(markup.contains("1 of 1000 stored"));
|
||||
assert!(markup.contains("2 of 1000 stored"));
|
||||
assert!(!markup.contains("at the cap"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn voice_noise_suppression_section_flags_a_list_at_its_cap() {
|
||||
let voice_noise_suppression = VoiceNoiseSuppressionConfigResponse {
|
||||
included_user_ids: (0..VOICE_NS_MAX_TARGETED_USERS)
|
||||
included_user_ids: (0..EXPERIMENT_MAX_TARGETED_USERS)
|
||||
.map(|index| index.to_string())
|
||||
.collect(),
|
||||
..VoiceNoiseSuppressionConfigResponse::default()
|
||||
|
||||
@@ -409,6 +409,15 @@ fn deserialize_instance_config_response_with_unknown_keys() {
|
||||
"future_object_knob": {"nested": true},
|
||||
"future_list_knob": ["a", "b"]
|
||||
},
|
||||
"screen_share_delivery": {
|
||||
"enabled": true,
|
||||
"config_version": 2,
|
||||
"rollout_basis_points": 2500,
|
||||
"rollout_salt": "screen-share-delivery-v1",
|
||||
"included_user_ids": ["1500000000000000001"],
|
||||
"future_delivery_knob": 9,
|
||||
"excluded_user_ids": []
|
||||
},
|
||||
"experiment_delivery": {"poll_interval_seconds": 300, "poll_jitter_percent": 15},
|
||||
"registration": {
|
||||
"mode": "open",
|
||||
@@ -538,6 +547,14 @@ fn deserialize_instance_config_response_with_unknown_keys() {
|
||||
assert_eq!(resp.voice_noise_suppression.rollout_basis_points, 10000);
|
||||
assert_eq!(*resp.voice_noise_suppression.rollout_salt, "voice-ns-v1");
|
||||
assert_eq!(resp.voice_noise_suppression.enabled_backends.len(), 3);
|
||||
assert!(resp.screen_share_delivery.enabled);
|
||||
assert_eq!(resp.screen_share_delivery.config_version, 2);
|
||||
assert_eq!(resp.screen_share_delivery.rollout_basis_points, 2500);
|
||||
assert_eq!(
|
||||
*resp.screen_share_delivery.rollout_salt,
|
||||
"screen-share-delivery-v1"
|
||||
);
|
||||
assert_eq!(resp.screen_share_delivery.included_user_ids.len(), 1);
|
||||
assert_eq!(resp.experiment_delivery.poll_interval_seconds, 300);
|
||||
assert!(resp.policy.single_community_guild_id.is_none());
|
||||
assert_eq!(resp.policy.services.gif_enabled, Some(true));
|
||||
@@ -548,6 +565,7 @@ fn deserialize_instance_config_response_with_unknown_keys() {
|
||||
.replace("\"future_rollout_knob\": 3,", "")
|
||||
.replace("\"future_presentation_knob\": \"verbose\",", "")
|
||||
.replace("\"future_knob\": 7,", "")
|
||||
.replace("\"future_delivery_knob\": 9,", "")
|
||||
.replace("\"future_object_knob\": {\"nested\": true},", "")
|
||||
.replace("\"future_list_knob\": [\"a\", \"b\"],", "")
|
||||
.replace(
|
||||
|
||||
@@ -465,6 +465,7 @@ async fn mutating_admin_pages_render_usable_csrf_tokens() {
|
||||
"/instance-config?action=update_gateway_rollout",
|
||||
"/instance-config?action=update_sso",
|
||||
"/instance-config?action=update_voice_noise_suppression",
|
||||
"/instance-config?action=update_screen_share_delivery",
|
||||
"/instance-config?action=update_experiment_delivery",
|
||||
][..],
|
||||
),
|
||||
@@ -1196,6 +1197,14 @@ fn instance_config() -> Value {
|
||||
"guild_overrides": [],
|
||||
"suppression_strength": 80
|
||||
},
|
||||
"screen_share_delivery": {
|
||||
"enabled": false,
|
||||
"config_version": 0,
|
||||
"rollout_basis_points": 0,
|
||||
"rollout_salt": "screen-share-delivery-v1",
|
||||
"included_user_ids": [],
|
||||
"excluded_user_ids": []
|
||||
},
|
||||
"experiment_delivery": {
|
||||
"poll_interval_seconds": 300,
|
||||
"poll_jitter_percent": 15
|
||||
|
||||
@@ -5,7 +5,7 @@ FROM node:26-trixie-slim AS base
|
||||
|
||||
WORKDIR /usr/src/app
|
||||
|
||||
RUN npm install -g pnpm@12.4.2
|
||||
RUN npm install -g pnpm@11.27.0
|
||||
|
||||
FROM base AS deploy
|
||||
|
||||
@@ -57,7 +57,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
libvips42t64 && \
|
||||
rm -rf /var/lib/apt/lists/*
|
||||
|
||||
RUN npm install -g pnpm@12.4.2
|
||||
RUN npm install -g pnpm@11.27.0
|
||||
|
||||
COPY --from=deploy /out ./
|
||||
COPY --from=deploy /usr/src/app/fluxer_api/dist ./dist
|
||||
|
||||
@@ -94,5 +94,5 @@
|
||||
"typescript": "catalog:ts7",
|
||||
"vitest": "catalog:"
|
||||
},
|
||||
"packageManager": "pnpm@12.4.2"
|
||||
"packageManager": "pnpm@11.27.0"
|
||||
}
|
||||
|
||||
@@ -11,6 +11,8 @@ interface PostgresIpInfoOptions {
|
||||
}
|
||||
|
||||
const VALUE_SEPARATOR = '\u001f';
|
||||
export const IPINFO_CACHE_TTL_SECONDS = 14 * 24 * 60 * 60;
|
||||
export const IPINFO_REQUEST_AUDIT_TTL_SECONDS = 90 * 24 * 60 * 60;
|
||||
|
||||
function getClient(options: PostgresIpInfoOptions): IPostgresClient | null {
|
||||
return options.client ?? options.getClient?.() ?? null;
|
||||
@@ -34,12 +36,9 @@ async function upsertKvRow(
|
||||
partitionKey: string,
|
||||
key: string,
|
||||
row: Record<string, unknown>,
|
||||
ttlSeconds?: number,
|
||||
ttlSeconds: number,
|
||||
): Promise<void> {
|
||||
const expiresAt =
|
||||
ttlSeconds != null && Number.isFinite(ttlSeconds) && ttlSeconds > 0
|
||||
? new Date(Date.now() + ttlSeconds * 1000)
|
||||
: null;
|
||||
const expiresAt = new Date(Date.now() + ttlSeconds * 1000);
|
||||
await client.query(
|
||||
`INSERT INTO ${table(client)} (table_name, partition_key, row_key, row_data, expires_at, updated_at)
|
||||
VALUES ($1, $2, $3, $4::jsonb, $5, now())
|
||||
@@ -77,7 +76,14 @@ export function createPostgresIpInfoCache(options: PostgresIpInfoOptions): IpInf
|
||||
try {
|
||||
const client = getClient(options);
|
||||
if (!client) return;
|
||||
await upsertKvRow(client, 'ipinfo_cache', rowKey([key]), rowKey([key]), {cache_key: key, payload}, ttlSeconds);
|
||||
await upsertKvRow(
|
||||
client,
|
||||
'ipinfo_cache',
|
||||
rowKey([key]),
|
||||
rowKey([key]),
|
||||
{cache_key: key, payload},
|
||||
ttlSeconds != null && Number.isFinite(ttlSeconds) && ttlSeconds > 0 ? ttlSeconds : IPINFO_CACHE_TTL_SECONDS,
|
||||
);
|
||||
} catch (error) {
|
||||
options.onError?.(error, 'ipinfo_cache_set');
|
||||
}
|
||||
@@ -124,6 +130,7 @@ export function createPostgresIpInfoRequestAuditLogger(options: PostgresIpInfoOp
|
||||
is_residential_proxy: event.isResidentialProxy,
|
||||
metadata_json: serializeMetadata(event.metadata),
|
||||
},
|
||||
IPINFO_REQUEST_AUDIT_TTL_SECONDS,
|
||||
);
|
||||
} catch (error) {
|
||||
options.onError?.(error, 'ipinfo_request_audit_record');
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
import assert from 'node:assert/strict';
|
||||
import type {Pool, PoolClient, QueryResult, QueryResultRow} from 'pg';
|
||||
import type {Pool, PoolClient, PoolConfig, QueryResult, QueryResultRow} from 'pg';
|
||||
import pg from 'pg';
|
||||
|
||||
const MAX_DIAGNOSTIC_FIELD_LENGTH = 128;
|
||||
@@ -131,7 +131,7 @@ class PostgresClient implements IPostgresClient {
|
||||
}
|
||||
|
||||
private async openPool(): Promise<void> {
|
||||
const pool = new pg.Pool({
|
||||
const poolConfig: PoolConfig & {scramMaxIterations: number} = {
|
||||
connectionString: this.config.url || undefined,
|
||||
host: this.config.url ? undefined : (this.config.host ?? '127.0.0.1'),
|
||||
port: this.config.url ? undefined : (this.config.port ?? 5432),
|
||||
@@ -140,7 +140,9 @@ class PostgresClient implements IPostgresClient {
|
||||
password: this.config.url ? undefined : (this.config.password ?? 'fluxer'),
|
||||
ssl: this.config.ssl ? {rejectUnauthorized: true, ca: normalizePem(this.config.sslCa)} : undefined,
|
||||
max: this.config.maxConnections ?? 20,
|
||||
});
|
||||
scramMaxIterations: 0,
|
||||
};
|
||||
const pool = new pg.Pool(poolConfig);
|
||||
this.observePoolConnections(pool);
|
||||
try {
|
||||
const client = await pool.connect();
|
||||
|
||||
@@ -272,8 +272,6 @@ export function buildAPIConfigFromMaster(master: MasterConfig): APIConfig {
|
||||
donationProxyKey,
|
||||
},
|
||||
hosts: {
|
||||
invite: extractHostname(master.endpoints.invite),
|
||||
gift: extractHostname(master.endpoints.gift),
|
||||
marketing: extractHostname(master.endpoints.marketing),
|
||||
unfurlIgnored: master.services.api.unfurl_ignored_hosts,
|
||||
},
|
||||
|
||||
@@ -341,6 +341,7 @@ import {
|
||||
type UsersPendingDeletionRow,
|
||||
} from '@app/api/database/types/UserTypes';
|
||||
import {ATTACHMENT_DECAY_COLUMNS, type AttachmentDecayRow} from '@app/api/types/AttachmentDecayTypes';
|
||||
import {seconds} from 'itty-time';
|
||||
|
||||
export const Users = defineTable<UserRow, 'user_id'>({
|
||||
name: 'users',
|
||||
@@ -499,16 +500,19 @@ export const GuildAuditLogs = defineTable<GuildAuditLogRow, 'guild_id' | 'log_id
|
||||
name: 'guild_audit_logs_v2',
|
||||
columns: GUILD_AUDIT_LOG_COLUMNS,
|
||||
primaryKey: ['guild_id', 'log_id'],
|
||||
defaultTtlSeconds: seconds('45 days'),
|
||||
});
|
||||
export const GuildAuditLogsByUser = defineTable<GuildAuditLogRow, 'guild_id' | 'user_id' | 'log_id'>({
|
||||
name: 'guild_audit_logs_v2_by_user',
|
||||
columns: GUILD_AUDIT_LOG_COLUMNS,
|
||||
primaryKey: ['guild_id', 'user_id', 'log_id'],
|
||||
defaultTtlSeconds: seconds('45 days'),
|
||||
});
|
||||
export const GuildAuditLogsByAction = defineTable<GuildAuditLogRow, 'guild_id' | 'action_type' | 'log_id'>({
|
||||
name: 'guild_audit_logs_v2_by_action',
|
||||
columns: GUILD_AUDIT_LOG_COLUMNS,
|
||||
primaryKey: ['guild_id', 'action_type', 'log_id'],
|
||||
defaultTtlSeconds: seconds('45 days'),
|
||||
});
|
||||
export const GuildAuditLogsByUserAction = defineTable<
|
||||
GuildAuditLogRow,
|
||||
@@ -517,6 +521,7 @@ export const GuildAuditLogsByUserAction = defineTable<
|
||||
name: 'guild_audit_logs_v2_by_user_action',
|
||||
columns: GUILD_AUDIT_LOG_COLUMNS,
|
||||
primaryKey: ['guild_id', 'user_id', 'action_type', 'log_id'],
|
||||
defaultTtlSeconds: seconds('45 days'),
|
||||
});
|
||||
export const GuildMembershipMetadata = defineTable<GuildMembershipMetadataRow, 'guild_id' | 'user_id'>({
|
||||
name: 'guild_membership_metadata',
|
||||
@@ -655,6 +660,7 @@ export const RecentMentions = defineTable<RecentMentionRow, 'user_id' | 'message
|
||||
name: 'recent_mentions',
|
||||
columns: RECENT_MENTION_COLUMNS,
|
||||
primaryKey: ['user_id', 'message_id'],
|
||||
defaultTtlSeconds: seconds('7 days'),
|
||||
});
|
||||
|
||||
interface RecentMentionsByGuildRow {
|
||||
@@ -678,6 +684,7 @@ export const RecentMentionsByGuild = defineTable<RecentMentionsByGuildRow, 'user
|
||||
name: 'recent_mentions_by_guild',
|
||||
columns: RECENT_MENTIONS_BY_GUILD_COLUMNS,
|
||||
primaryKey: ['user_id', 'guild_id', 'message_id'],
|
||||
defaultTtlSeconds: seconds('7 days'),
|
||||
});
|
||||
export const SavedMessages = defineTable<SavedMessageRow, 'user_id' | 'message_id'>({
|
||||
name: 'saved_messages',
|
||||
@@ -688,6 +695,7 @@ export const PushSubscriptions = defineTable<PushSubscriptionRow, 'user_id' | 's
|
||||
name: 'push_subscriptions',
|
||||
columns: PUSH_SUBSCRIPTION_COLUMNS,
|
||||
primaryKey: ['user_id', 'subscription_id'],
|
||||
defaultTtlSeconds: seconds('90 days'),
|
||||
});
|
||||
export const Payments = defineTable<PaymentRow, 'checkout_session_id'>({
|
||||
name: 'payments',
|
||||
@@ -854,11 +862,13 @@ export const EmailVerificationTokens = defineTable<EmailVerificationTokenRow, 't
|
||||
name: 'email_verification_tokens',
|
||||
columns: EMAIL_VERIFICATION_TOKEN_COLUMNS,
|
||||
primaryKey: ['token_', 'user_id'],
|
||||
defaultTtlSeconds: seconds('24 hours'),
|
||||
});
|
||||
export const PasswordResetTokens = defineTable<PasswordResetTokenRow, 'token_' | 'user_id'>({
|
||||
name: 'password_reset_tokens',
|
||||
columns: PASSWORD_RESET_TOKEN_COLUMNS,
|
||||
primaryKey: ['token_', 'user_id'],
|
||||
defaultTtlSeconds: seconds('24 hours'),
|
||||
});
|
||||
export const PasswordResetTokensByUserId = defineTable<
|
||||
{
|
||||
@@ -870,16 +880,19 @@ export const PasswordResetTokensByUserId = defineTable<
|
||||
name: 'password_reset_tokens_by_user_id',
|
||||
columns: ['user_id', 'token_'],
|
||||
primaryKey: ['user_id', 'token_'],
|
||||
defaultTtlSeconds: seconds('24 hours'),
|
||||
});
|
||||
export const EmailRevertTokens = defineTable<EmailRevertTokenRow, 'token_' | 'user_id'>({
|
||||
name: 'email_revert_tokens',
|
||||
columns: EMAIL_REVERT_TOKEN_COLUMNS,
|
||||
primaryKey: ['token_', 'user_id'],
|
||||
defaultTtlSeconds: seconds('48 hours'),
|
||||
});
|
||||
export const PhoneTokens = defineTable<PhoneTokenRow, 'token_'>({
|
||||
name: 'phone_tokens',
|
||||
columns: PHONE_TOKEN_COLUMNS,
|
||||
primaryKey: ['token_'],
|
||||
defaultTtlSeconds: seconds('30 days'),
|
||||
});
|
||||
export const AuthSessions = defineTable<AuthSessionRow, 'session_id_hash'>({
|
||||
name: 'auth_sessions',
|
||||
@@ -901,11 +914,13 @@ export const AuthSessionTombstones = defineTable<AuthSessionTombstoneRow, 'user_
|
||||
name: 'auth_session_tombstones',
|
||||
columns: AUTH_SESSION_TOMBSTONE_COLUMNS,
|
||||
primaryKey: ['user_id', 'session_id_hash'],
|
||||
defaultTtlSeconds: seconds('30 days'),
|
||||
});
|
||||
export const UserCountryHistory = defineTable<UserCountryHistoryRow, 'user_id' | 'country'>({
|
||||
name: 'user_country_history',
|
||||
columns: USER_COUNTRY_HISTORY_COLUMNS,
|
||||
primaryKey: ['user_id', 'country'],
|
||||
defaultTtlSeconds: seconds('365 days'),
|
||||
});
|
||||
export const MfaBackupCodes = defineTable<MfaBackupCodeRow, 'user_id' | 'code'>({
|
||||
name: 'mfa_backup_codes',
|
||||
@@ -932,6 +947,7 @@ export const IpAuthorizationTokens = defineTable<IpAuthorizationTokenRow, 'token
|
||||
name: 'ip_authorization_tokens',
|
||||
columns: IP_AUTHORIZATION_TOKEN_COLUMNS,
|
||||
primaryKey: ['token_', 'user_id'],
|
||||
defaultTtlSeconds: seconds('30 minutes'),
|
||||
});
|
||||
export const AuthorizedIps = defineTable<AuthorizedIpRow, 'user_id' | 'ip'>({
|
||||
name: 'authorized_ips_v2',
|
||||
@@ -1057,26 +1073,31 @@ export const OAuth2AuthorizationCodes = defineTable<OAuth2AuthorizationCodeRow,
|
||||
name: 'oauth2_authorization_codes',
|
||||
columns: OAUTH2_AUTHORIZATION_CODE_COLUMNS,
|
||||
primaryKey: ['code'],
|
||||
defaultTtlSeconds: seconds('10 minutes'),
|
||||
});
|
||||
export const OAuth2AccessTokens = defineTable<OAuth2AccessTokenRow, 'token_'>({
|
||||
name: 'oauth2_access_tokens',
|
||||
columns: OAUTH2_ACCESS_TOKEN_COLUMNS,
|
||||
primaryKey: ['token_'],
|
||||
defaultTtlSeconds: seconds('7 days'),
|
||||
});
|
||||
export const OAuth2AccessTokensByUser = defineTable<OAuth2AccessTokenByUserRow, 'user_id' | 'token_'>({
|
||||
name: 'oauth2_access_tokens_by_user',
|
||||
columns: OAUTH2_ACCESS_TOKENS_BY_USER_COLUMNS,
|
||||
primaryKey: ['user_id', 'token_'],
|
||||
defaultTtlSeconds: seconds('7 days'),
|
||||
});
|
||||
export const OAuth2RefreshTokens = defineTable<OAuth2RefreshTokenRow, 'token_'>({
|
||||
name: 'oauth2_refresh_tokens',
|
||||
columns: OAUTH2_REFRESH_TOKEN_COLUMNS,
|
||||
primaryKey: ['token_'],
|
||||
defaultTtlSeconds: seconds('30 days'),
|
||||
});
|
||||
export const OAuth2RefreshTokensByUser = defineTable<OAuth2RefreshTokenByUserRow, 'user_id' | 'token_'>({
|
||||
name: 'oauth2_refresh_tokens_by_user',
|
||||
columns: OAUTH2_REFRESH_TOKENS_BY_USER_COLUMNS,
|
||||
primaryKey: ['user_id', 'token_'],
|
||||
defaultTtlSeconds: seconds('30 days'),
|
||||
});
|
||||
|
||||
interface WebhooksByChannelRow {
|
||||
@@ -1117,12 +1138,14 @@ export const JobsById = defineTable<JobByIdRow, 'job_id'>({
|
||||
name: 'jobs_by_id',
|
||||
columns: JOB_BY_ID_COLUMNS,
|
||||
primaryKey: ['job_id'],
|
||||
defaultTtlSeconds: seconds('90 days'),
|
||||
});
|
||||
export const JobsByDayBucket = defineTable<JobByDayBucketRow, 'bucket_day' | 'created_at' | 'job_id'>({
|
||||
name: 'jobs_by_day_bucket',
|
||||
columns: JOB_BY_DAY_BUCKET_COLUMNS,
|
||||
primaryKey: ['bucket_day', 'created_at', 'job_id'],
|
||||
partitionKey: ['bucket_day'],
|
||||
defaultTtlSeconds: seconds('90 days'),
|
||||
});
|
||||
export const JobsActive = defineTable<JobActiveRow, 'job_id'>({
|
||||
name: 'jobs_active',
|
||||
@@ -1133,11 +1156,13 @@ export const AttachmentUploadTracesByKey = defineTable<AttachmentUploadTraceByKe
|
||||
name: 'attachment_upload_traces_by_key',
|
||||
columns: ATTACHMENT_UPLOAD_TRACE_BY_KEY_COLUMNS,
|
||||
primaryKey: ['upload_key'],
|
||||
defaultTtlSeconds: seconds('30 days'),
|
||||
});
|
||||
export const AttachmentUploadTracesByAttachment = defineTable<AttachmentUploadTraceByAttachmentRow, 'attachment_id'>({
|
||||
name: 'attachment_upload_traces_by_attachment',
|
||||
columns: ATTACHMENT_UPLOAD_TRACE_BY_ATTACHMENT_COLUMNS,
|
||||
primaryKey: ['attachment_id'],
|
||||
defaultTtlSeconds: seconds('30 days'),
|
||||
});
|
||||
export const NcmecAttachmentSubmissions = defineTable<NcmecAttachmentSubmissionRow, 'attachment_id'>({
|
||||
name: 'ncmec_attachment_submissions',
|
||||
@@ -1154,6 +1179,7 @@ export const RegistrationEventsByIp = defineTable<RegistrationEventByIpRow, 'ip'
|
||||
columns: REGISTRATION_EVENT_BY_IP_COLUMNS,
|
||||
primaryKey: ['ip', 'created_at', 'user_id'],
|
||||
partitionKey: ['ip'],
|
||||
defaultTtlSeconds: seconds('30 days'),
|
||||
});
|
||||
export const RegistrationEventsBySubnet = defineTable<
|
||||
RegistrationEventBySubnetRow,
|
||||
@@ -1164,6 +1190,7 @@ export const RegistrationEventsBySubnet = defineTable<
|
||||
columns: REGISTRATION_EVENT_BY_SUBNET_COLUMNS,
|
||||
primaryKey: ['subnet', 'created_at', 'user_id'],
|
||||
partitionKey: ['subnet'],
|
||||
defaultTtlSeconds: seconds('30 days'),
|
||||
});
|
||||
export const RegistrationEventsByEmailDomain = defineTable<
|
||||
RegistrationEventByEmailDomainRow,
|
||||
@@ -1174,6 +1201,7 @@ export const RegistrationEventsByEmailDomain = defineTable<
|
||||
columns: REGISTRATION_EVENT_BY_EMAIL_DOMAIN_COLUMNS,
|
||||
primaryKey: ['email_domain', 'created_at', 'user_id'],
|
||||
partitionKey: ['email_domain'],
|
||||
defaultTtlSeconds: seconds('30 days'),
|
||||
});
|
||||
export const RegistrationEventsByPlusAddressBase = defineTable<
|
||||
RegistrationEventByPlusAddressBaseRow,
|
||||
@@ -1184,6 +1212,7 @@ export const RegistrationEventsByPlusAddressBase = defineTable<
|
||||
columns: REGISTRATION_EVENT_BY_PLUS_ADDRESS_BASE_COLUMNS,
|
||||
primaryKey: ['plus_address_base', 'created_at', 'user_id'],
|
||||
partitionKey: ['plus_address_base'],
|
||||
defaultTtlSeconds: seconds('30 days'),
|
||||
});
|
||||
export const LatestRiskContextByUser = defineTable<LatestRiskContextByUserRow, 'user_id'>({
|
||||
name: 'latest_risk_context_by_user',
|
||||
@@ -1194,6 +1223,7 @@ export const SuspiciousIps = defineTable<SuspiciousIpRow, 'ip'>({
|
||||
name: 'suspicious_ips',
|
||||
columns: SUSPICIOUS_IP_COLUMNS,
|
||||
primaryKey: ['ip'],
|
||||
defaultTtlSeconds: seconds('180 days'),
|
||||
});
|
||||
export const RiskOutcomesByIp = defineTable<RiskOutcomeByIpRow, 'ip' | 'created_at' | 'user_id' | 'outcome_code', 'ip'>(
|
||||
{
|
||||
@@ -1201,6 +1231,7 @@ export const RiskOutcomesByIp = defineTable<RiskOutcomeByIpRow, 'ip' | 'created_
|
||||
columns: RISK_OUTCOME_BY_IP_COLUMNS,
|
||||
primaryKey: ['ip', 'created_at', 'user_id', 'outcome_code'],
|
||||
partitionKey: ['ip'],
|
||||
defaultTtlSeconds: seconds('180 days'),
|
||||
},
|
||||
);
|
||||
export const RiskOutcomesBySubnet = defineTable<
|
||||
@@ -1212,6 +1243,7 @@ export const RiskOutcomesBySubnet = defineTable<
|
||||
columns: RISK_OUTCOME_BY_SUBNET_COLUMNS,
|
||||
primaryKey: ['subnet', 'created_at', 'user_id', 'outcome_code'],
|
||||
partitionKey: ['subnet'],
|
||||
defaultTtlSeconds: seconds('180 days'),
|
||||
});
|
||||
export const RiskOutcomesByEmailDomain = defineTable<
|
||||
RiskOutcomeByEmailDomainRow,
|
||||
@@ -1222,6 +1254,7 @@ export const RiskOutcomesByEmailDomain = defineTable<
|
||||
columns: RISK_OUTCOME_BY_EMAIL_DOMAIN_COLUMNS,
|
||||
primaryKey: ['email_domain', 'created_at', 'user_id', 'outcome_code'],
|
||||
partitionKey: ['email_domain'],
|
||||
defaultTtlSeconds: seconds('180 days'),
|
||||
});
|
||||
export const RiskOutcomesByAsn = defineTable<
|
||||
RiskOutcomeByAsnRow,
|
||||
@@ -1232,6 +1265,7 @@ export const RiskOutcomesByAsn = defineTable<
|
||||
columns: RISK_OUTCOME_BY_ASN_COLUMNS,
|
||||
primaryKey: ['asn', 'created_at', 'user_id', 'outcome_code'],
|
||||
partitionKey: ['asn'],
|
||||
defaultTtlSeconds: seconds('180 days'),
|
||||
});
|
||||
export const RiskAssessments = defineTable<RiskAssessmentRow, 'assessment_id'>({
|
||||
name: 'risk_assessments',
|
||||
@@ -1248,6 +1282,7 @@ export const InboundSmsChallenges = defineTable<InboundSmsChallengeRow, 'challen
|
||||
name: 'inbound_sms_challenges',
|
||||
columns: INBOUND_SMS_CHALLENGE_COLUMNS,
|
||||
primaryKey: ['challenge_code'],
|
||||
defaultTtlSeconds: seconds('15 minutes'),
|
||||
});
|
||||
export const InboundSmsChallengesByUser = defineTable<
|
||||
InboundSmsChallengeByUserRow,
|
||||
@@ -1258,16 +1293,19 @@ export const InboundSmsChallengesByUser = defineTable<
|
||||
columns: INBOUND_SMS_CHALLENGE_BY_USER_COLUMNS,
|
||||
primaryKey: ['user_id', 'created_at'],
|
||||
partitionKey: ['user_id'],
|
||||
defaultTtlSeconds: seconds('15 minutes'),
|
||||
});
|
||||
export const PhoneLookupCache = defineTable<PhoneLookupCacheRow, 'phone'>({
|
||||
name: 'phone_lookup_cache',
|
||||
columns: PHONE_LOOKUP_CACHE_COLUMNS,
|
||||
primaryKey: ['phone'],
|
||||
defaultTtlSeconds: seconds('7 days'),
|
||||
});
|
||||
export const PhoneVerificationAttempts = defineTable<PhoneVerificationAttemptRow, 'attempt_id'>({
|
||||
name: 'phone_verification_attempts',
|
||||
columns: PHONE_VERIFICATION_ATTEMPT_COLUMNS,
|
||||
primaryKey: ['attempt_id'],
|
||||
defaultTtlSeconds: seconds('90 days'),
|
||||
});
|
||||
export const BillingCustomers = defineTable<BillingCustomerRow, 'provider_id'>({
|
||||
name: 'billing_customers',
|
||||
|
||||
@@ -2,7 +2,15 @@
|
||||
|
||||
import type {AdminAuditLog, BannedIpEntry, BannedIpKind, IAdminRepository} from '@app/api/admin/IAdminRepository';
|
||||
import {createUserID} from '@app/api/BrandedTypes';
|
||||
import {deleteOneOrMany, fetchMany, fetchOne, upsertOne} from '@app/api/database/CassandraQueryExecution';
|
||||
import {Config} from '@app/api/Config';
|
||||
import {ContentBlocklistCategory} from '@app/api/constants/ContentModeration';
|
||||
import {
|
||||
deleteOneOrMany,
|
||||
executeConditional,
|
||||
fetchMany,
|
||||
fetchOne,
|
||||
upsertOne,
|
||||
} from '@app/api/database/CassandraQueryExecution';
|
||||
import type {
|
||||
AdminAuditLogRow,
|
||||
BannedAvatarHashRow,
|
||||
@@ -282,6 +290,7 @@ export class AdminRepository implements IAdminRepository {
|
||||
}
|
||||
|
||||
async isEmailDomainDisposable(domain: string): Promise<boolean> {
|
||||
if (!Config.blocklistFeeds.enabled) return false;
|
||||
const domainLower = domain.toLowerCase();
|
||||
if (isAccountPolicyContactDomainReputationExempt(domainLower)) return false;
|
||||
const result = await fetchOne<{
|
||||
@@ -395,6 +404,15 @@ export class AdminRepository implements IAdminRepository {
|
||||
await deleteOneOrMany(BannedFileShas.deleteByPk({sha256_hex: sha256Hex.toLowerCase()}));
|
||||
}
|
||||
|
||||
async unbanFeedFileSha(sha256Hex: string): Promise<boolean> {
|
||||
return executeConditional(
|
||||
BannedFileShas.conditionalDeleteByPk(
|
||||
{sha256_hex: sha256Hex.toLowerCase()},
|
||||
{added_by: null, category: ContentBlocklistCategory.MALWARE_BAZAAR},
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
async loadAllBannedFileShas(): Promise<Array<BannedFileShaRow>> {
|
||||
return fetchMany<BannedFileShaRow>(LOAD_ALL_BANNED_FILE_SHAS_QUERY.bind({}));
|
||||
}
|
||||
|
||||
@@ -109,6 +109,8 @@ export abstract class IAdminRepository {
|
||||
|
||||
abstract unbanFileSha(sha256Hex: string): Promise<void>;
|
||||
|
||||
abstract unbanFeedFileSha(sha256Hex: string): Promise<boolean>;
|
||||
|
||||
abstract loadAllBannedFileShas(): Promise<Array<BannedFileShaRow>>;
|
||||
|
||||
abstract isAvatarHashBanned(hashShort: string): Promise<boolean>;
|
||||
|
||||
@@ -31,6 +31,7 @@ import {
|
||||
RegistrationUrlIdParam,
|
||||
} from '@fluxer/schema/src/domains/admin/AdminSchemas';
|
||||
import {GatewayRolloutConfigSchema} from '@fluxer/schema/src/domains/admin/GatewayRolloutSchemas';
|
||||
import {ScreenShareDeliveryConfigSchema} from '@fluxer/schema/src/domains/admin/ScreenShareDeliverySchemas';
|
||||
import {VoiceNoiseSuppressionConfigSchema} from '@fluxer/schema/src/domains/admin/VoiceNoiseSuppressionSchemas';
|
||||
import {UserIdParam} from '@fluxer/schema/src/domains/common/CommonParamSchemas';
|
||||
import {ExperimentDeliveryConfigSchema} from '@fluxer/schema/src/domains/experiment/ExperimentSchemas';
|
||||
@@ -59,6 +60,7 @@ async function buildInstanceConfigResponse(): Promise<InstanceConfigResponse> {
|
||||
ssoConfig,
|
||||
gatewayRollout,
|
||||
voiceNoiseSuppression,
|
||||
screenShareDelivery,
|
||||
experimentDelivery,
|
||||
registrationConfig,
|
||||
registrationUrls,
|
||||
@@ -67,6 +69,7 @@ async function buildInstanceConfigResponse(): Promise<InstanceConfigResponse> {
|
||||
instanceConfigRepository.getSsoConfig(),
|
||||
instanceConfigRepository.getGatewayRolloutConfig(),
|
||||
instanceConfigRepository.getVoiceNoiseSuppressionConfig(),
|
||||
instanceConfigRepository.getScreenShareDeliveryConfig(),
|
||||
instanceConfigRepository.getExperimentDeliveryConfig(),
|
||||
instanceConfigRepository.getRegistrationConfig(),
|
||||
instanceConfigRepository.getRegistrationUrlsForAdmin(),
|
||||
@@ -98,6 +101,7 @@ async function buildInstanceConfigResponse(): Promise<InstanceConfigResponse> {
|
||||
},
|
||||
gateway_rollout: gatewayRollout,
|
||||
voice_noise_suppression: voiceNoiseSuppression,
|
||||
screen_share_delivery: screenShareDelivery,
|
||||
experiment_delivery: experimentDelivery,
|
||||
registration: {
|
||||
...registrationConfig,
|
||||
@@ -261,6 +265,18 @@ export function InstanceConfigAdminController(app: HonoApp) {
|
||||
await instanceConfigRepository.setVoiceNoiseSuppressionConfig(validated);
|
||||
}
|
||||
}
|
||||
if (data.screen_share_delivery) {
|
||||
const patch = omitUndefinedFields(data.screen_share_delivery);
|
||||
if (Object.keys(patch).length > 0) {
|
||||
const currentScreenShareDelivery = await instanceConfigRepository.getScreenShareDeliveryConfig();
|
||||
const validated = ScreenShareDeliveryConfigSchema.parse({
|
||||
...currentScreenShareDelivery,
|
||||
...patch,
|
||||
config_version: currentScreenShareDelivery.config_version + 1,
|
||||
});
|
||||
await instanceConfigRepository.setScreenShareDeliveryConfig(validated);
|
||||
}
|
||||
}
|
||||
if (data.experiment_delivery) {
|
||||
const currentExperimentDelivery = await instanceConfigRepository.getExperimentDeliveryConfig();
|
||||
const validated = ExperimentDeliveryConfigSchema.parse({
|
||||
|
||||
@@ -2,10 +2,10 @@
|
||||
|
||||
import {createTestAccount, createTotpSecret, generateTotpCode, setUserACLs} from '@app/api/auth/tests/AuthTestUtils';
|
||||
import {
|
||||
createRegistrationResponse,
|
||||
createWebAuthnDevice,
|
||||
registerWebAuthnCredential,
|
||||
setWebAuthnTwoFactor,
|
||||
type WebAuthnCredentialMetadata,
|
||||
type WebAuthnRegistrationOptions,
|
||||
} from '@app/api/auth/tests/WebAuthnTestUtils';
|
||||
import {type ApiTestHarness, createApiTestHarness} from '@app/api/test/ApiTestHarness';
|
||||
import {createBuilder} from '@app/api/test/TestRequestBuilder';
|
||||
@@ -31,13 +31,15 @@ describe('Admin WebAuthn credential delete', () => {
|
||||
afterAll(async () => {
|
||||
await harness?.shutdown();
|
||||
});
|
||||
test('removes the WebAuthn authenticator type when admin deletes the last credential', async () => {
|
||||
let admin = await createTestAccount(harness);
|
||||
admin = await setUserACLs(harness, admin, [
|
||||
async function createAdmin() {
|
||||
const admin = await createTestAccount(harness);
|
||||
return await setUserACLs(harness, admin, [
|
||||
AdminACLs.AUTHENTICATE,
|
||||
AdminACLs.USER_LOOKUP,
|
||||
AdminACLs.USER_UPDATE_MFA,
|
||||
]);
|
||||
}
|
||||
async function createPasskeyTarget(twoFactorEnabled: boolean) {
|
||||
const target = await createTestAccount(harness);
|
||||
const device = createWebAuthnDevice();
|
||||
const secret = createTotpSecret();
|
||||
@@ -45,28 +47,19 @@ describe('Admin WebAuthn credential delete', () => {
|
||||
.post('/users/@me/mfa/totp/enable')
|
||||
.body({secret, code: generateTotpCode(secret), password: target.password})
|
||||
.execute();
|
||||
const registrationOptions = await createBuilder<WebAuthnRegistrationOptions>(harness, target.token)
|
||||
.post('/users/@me/mfa/webauthn/credentials/registration-options')
|
||||
.body({mfa_method: 'totp', mfa_code: generateTotpCode(secret)})
|
||||
.execute();
|
||||
if (registrationOptions.rp.id) {
|
||||
device.rpId = registrationOptions.rp.id;
|
||||
}
|
||||
await createBuilder(harness, target.token)
|
||||
.post('/users/@me/mfa/webauthn/credentials')
|
||||
.body({
|
||||
response: createRegistrationResponse(device, registrationOptions, 'Admin Delete Test Passkey'),
|
||||
challenge: registrationOptions.challenge,
|
||||
name: 'Admin Delete Test Passkey',
|
||||
await registerWebAuthnCredential(
|
||||
harness,
|
||||
target.token,
|
||||
device,
|
||||
() => ({mfa_method: 'totp', mfa_code: generateTotpCode(secret)}),
|
||||
'Admin Delete Test Passkey',
|
||||
);
|
||||
if (twoFactorEnabled) {
|
||||
await setWebAuthnTwoFactor(harness, target.token, true, {
|
||||
mfa_method: 'totp',
|
||||
mfa_code: generateTotpCode(secret),
|
||||
})
|
||||
.expect(204)
|
||||
.execute();
|
||||
const credentialsBeforeDelete = await createBuilder<Array<WebAuthnCredentialMetadata>>(harness, target.token)
|
||||
.get('/users/@me/mfa/webauthn/credentials')
|
||||
.execute();
|
||||
expect(credentialsBeforeDelete).toHaveLength(1);
|
||||
});
|
||||
}
|
||||
await createBuilder(harness, target.token)
|
||||
.post('/users/@me/mfa/totp/disable')
|
||||
.body({
|
||||
@@ -76,6 +69,15 @@ describe('Admin WebAuthn credential delete', () => {
|
||||
})
|
||||
.expect(204)
|
||||
.execute();
|
||||
return target;
|
||||
}
|
||||
test('removes the WebAuthn authenticator type when admin deletes the last credential of a two-factor user', async () => {
|
||||
const admin = await createAdmin();
|
||||
const target = await createPasskeyTarget(true);
|
||||
const credentialsBeforeDelete = await createBuilder<Array<WebAuthnCredentialMetadata>>(harness, target.token)
|
||||
.get('/users/@me/mfa/webauthn/credentials')
|
||||
.execute();
|
||||
expect(credentialsBeforeDelete).toHaveLength(1);
|
||||
const userBeforeDelete = await createBuilder<AdminLookupResponse>(harness, `${admin.token}`)
|
||||
.get(`/admin/users/${target.userId}`)
|
||||
.execute();
|
||||
@@ -93,4 +95,28 @@ describe('Admin WebAuthn credential delete', () => {
|
||||
.execute();
|
||||
expect(userAfterDelete.users[0]?.authenticator_types).toEqual([]);
|
||||
});
|
||||
test('leaves the authenticator types empty throughout for a user who never turned passkey two-factor on', async () => {
|
||||
const admin = await createAdmin();
|
||||
const target = await createPasskeyTarget(false);
|
||||
const credentialsBeforeDelete = await createBuilder<Array<WebAuthnCredentialMetadata>>(harness, target.token)
|
||||
.get('/users/@me/mfa/webauthn/credentials')
|
||||
.execute();
|
||||
expect(credentialsBeforeDelete).toHaveLength(1);
|
||||
const userBeforeDelete = await createBuilder<AdminLookupResponse>(harness, `${admin.token}`)
|
||||
.get(`/admin/users/${target.userId}`)
|
||||
.execute();
|
||||
expect(userBeforeDelete.users[0]?.authenticator_types).toEqual([]);
|
||||
await createBuilder(harness, `${admin.token}`)
|
||||
.delete(`/admin/users/${target.userId}/webauthn-credentials/${credentialsBeforeDelete[0]!.id}`)
|
||||
.expect(204)
|
||||
.execute();
|
||||
const credentialsAfterDelete = await createBuilder<Array<WebAuthnCredentialMetadata>>(harness, target.token)
|
||||
.get('/users/@me/mfa/webauthn/credentials')
|
||||
.execute();
|
||||
expect(credentialsAfterDelete).toHaveLength(0);
|
||||
const userAfterDelete = await createBuilder<AdminLookupResponse>(harness, `${admin.token}`)
|
||||
.get(`/admin/users/${target.userId}`)
|
||||
.execute();
|
||||
expect(userAfterDelete.users[0]?.authenticator_types).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -5,6 +5,7 @@ import * as AuthMfa from '@app/api/auth/AuthMfa';
|
||||
import * as AuthPassword from '@app/api/auth/AuthPassword';
|
||||
import * as AuthSession from '@app/api/auth/AuthSession';
|
||||
import * as AuthUtility from '@app/api/auth/AuthUtility';
|
||||
import {resolveWebAuthnSecondFactor} from '@app/api/auth/services/WebAuthnSecondFactor';
|
||||
import {
|
||||
createInviteCode,
|
||||
createIpAuthorizationTicket,
|
||||
@@ -30,6 +31,7 @@ import {ValidationErrorCodes} from '@fluxer/constants/src/ValidationErrorCodes';
|
||||
import {IpAuthorizationRequiredError} from '@fluxer/errors/src/domains/auth/IpAuthorizationRequiredError';
|
||||
import {IpAuthorizationResendCooldownError} from '@fluxer/errors/src/domains/auth/IpAuthorizationResendCooldownError';
|
||||
import {IpAuthorizationResendLimitExceededError} from '@fluxer/errors/src/domains/auth/IpAuthorizationResendLimitExceededError';
|
||||
import {MfaNotEnabledError} from '@fluxer/errors/src/domains/auth/MfaNotEnabledError';
|
||||
import {RegistrationPendingApprovalError} from '@fluxer/errors/src/domains/auth/RegistrationPendingApprovalError';
|
||||
import {RegistrationRejectedError} from '@fluxer/errors/src/domains/auth/RegistrationRejectedError';
|
||||
import {InputValidationError} from '@fluxer/errors/src/domains/core/InputValidationError';
|
||||
@@ -72,12 +74,13 @@ interface LoginTokenResult {
|
||||
token: string;
|
||||
}
|
||||
|
||||
interface LoginMfaResult {
|
||||
export interface LoginMfaResult {
|
||||
mfa: true;
|
||||
ticket: string;
|
||||
allowed_methods: Array<string>;
|
||||
totp: boolean;
|
||||
webauthn: boolean;
|
||||
backup_codes: boolean;
|
||||
}
|
||||
|
||||
type LoginResult = LoginTokenResult | LoginMfaResult;
|
||||
@@ -323,7 +326,8 @@ export async function login(
|
||||
}
|
||||
}
|
||||
if (hasMfa) {
|
||||
return await createMfaTicketResponse(ctx, currentUser);
|
||||
const webauthnIsSecondFactor = await resolveWebAuthnSecondFactor(ctx, currentUser);
|
||||
return await createMfaTicketResponse(ctx, currentUser, webauthnIsSecondFactor);
|
||||
}
|
||||
if (data.invite_code && inviteService) {
|
||||
try {
|
||||
@@ -387,13 +391,14 @@ export async function loginMfaTotp(
|
||||
throw new UnknownUserError();
|
||||
}
|
||||
AuthUtility.assertNonBotUser(ctx, user);
|
||||
if (!user.totpSecret || !user.authenticatorTypes?.has(UserAuthenticatorTypes.TOTP)) {
|
||||
const hasTotp = Boolean(user.totpSecret) && user.authenticatorTypes.has(UserAuthenticatorTypes.TOTP);
|
||||
if (!hasTotp && !(await AuthMfa.hasUnconsumedBackupCodes(ctx, user.id))) {
|
||||
throw InputValidationError.fromCode('code', ValidationErrorCodes.TOTP_NOT_ENABLED);
|
||||
}
|
||||
await consumeMfaAttempt(ctx, {userId: user.id.toString(), ticket, field: 'code'});
|
||||
const isValid = await AuthMfa.verifyMfaCode(ctx, {
|
||||
userId: user.id,
|
||||
mfaSecret: user.totpSecret,
|
||||
mfaSecret: hasTotp ? user.totpSecret : null,
|
||||
code,
|
||||
allowBackup: true,
|
||||
});
|
||||
@@ -424,6 +429,9 @@ export async function loginMfaWebAuthn(
|
||||
throw new UnknownUserError();
|
||||
}
|
||||
AuthUtility.assertNonBotUser(ctx, user);
|
||||
if (!(await resolveWebAuthnSecondFactor(ctx, user))) {
|
||||
throw new MfaNotEnabledError();
|
||||
}
|
||||
await consumeMfaAttempt(ctx, {userId: user.id.toString(), ticket, field: 'ticket'});
|
||||
await AuthMfa.verifyWebAuthnAuthentication(ctx, user.id, response, challenge, 'mfa', ticket);
|
||||
await cache.delete(`mfa-ticket:${ticket}`);
|
||||
@@ -436,21 +444,26 @@ export async function loginMfaWebAuthn(
|
||||
return {user_id: user.id.toString(), token};
|
||||
}
|
||||
|
||||
async function createMfaTicketResponse(ctx: ApiContext, user: User): Promise<LoginMfaResult> {
|
||||
const {users, cache} = ctx.services;
|
||||
export async function createMfaTicketResponse(
|
||||
ctx: ApiContext,
|
||||
user: User,
|
||||
webauthnIsSecondFactor: boolean,
|
||||
): Promise<LoginMfaResult> {
|
||||
const {cache} = ctx.services;
|
||||
const ticket = createMfaTicket(await AuthUtility.generateSecureToken(ctx));
|
||||
await cache.set(`mfa-ticket:${ticket}`, user.id.toString(), seconds('5 minutes'));
|
||||
const credentials = await users.listWebAuthnCredentials(user.id);
|
||||
const hasWebauthn = credentials.length > 0;
|
||||
const hasTotp = user.authenticatorTypes.has(UserAuthenticatorTypes.TOTP);
|
||||
const hasBackupCodes = await AuthMfa.hasUnconsumedBackupCodes(ctx, user.id);
|
||||
const allowedMethods: Array<string> = [];
|
||||
if (hasTotp) allowedMethods.push('totp');
|
||||
if (hasWebauthn) allowedMethods.push('webauthn');
|
||||
if (webauthnIsSecondFactor) allowedMethods.push('webauthn');
|
||||
if (hasBackupCodes) allowedMethods.push('backup_codes');
|
||||
return {
|
||||
mfa: true,
|
||||
ticket,
|
||||
allowed_methods: allowedMethods,
|
||||
totp: hasTotp,
|
||||
webauthn: hasWebauthn,
|
||||
webauthn: webauthnIsSecondFactor,
|
||||
backup_codes: hasBackupCodes,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -2,9 +2,11 @@
|
||||
|
||||
import {timingSafeEqual} from 'node:crypto';
|
||||
import type {ApiContext} from '@app/api/ApiContext';
|
||||
import {deriveSudoMethods, userHasMfa} from '@app/api/auth/services/SudoMethods';
|
||||
import * as AuthUtility from '@app/api/auth/AuthUtility';
|
||||
import {deriveSudoMethods, userHasMfa, userHasSudoCapability} from '@app/api/auth/services/SudoMethods';
|
||||
import {createUserID, type UserID} from '@app/api/BrandedTypes';
|
||||
import {Logger} from '@app/api/Logger';
|
||||
import type {MfaBackupCode} from '@app/api/models/MfaBackupCode';
|
||||
import type {User} from '@app/api/models/User';
|
||||
import type {WebAuthnCredential} from '@app/api/models/WebAuthnCredential';
|
||||
import {mapUserToPrivateResponse} from '@app/api/user/UserMappers';
|
||||
@@ -48,7 +50,7 @@ interface SudoMfaVerificationResult {
|
||||
|
||||
interface VerifyMfaCodeParams {
|
||||
userId: UserID;
|
||||
mfaSecret: string;
|
||||
mfaSecret: string | null;
|
||||
code: string;
|
||||
allowBackup?: boolean;
|
||||
}
|
||||
@@ -56,9 +58,15 @@ interface VerifyMfaCodeParams {
|
||||
interface AvailableMfaMethods {
|
||||
totp: boolean;
|
||||
webauthn: boolean;
|
||||
backup_codes: boolean;
|
||||
has_mfa: boolean;
|
||||
}
|
||||
|
||||
interface SetWebAuthnTwoFactorResult {
|
||||
user: User;
|
||||
backupCodes: Array<MfaBackupCode> | null;
|
||||
}
|
||||
|
||||
function constantTimeEquals(a: string, b: string): boolean {
|
||||
const bufferA = Buffer.from(a);
|
||||
const bufferB = Buffer.from(b);
|
||||
@@ -72,24 +80,31 @@ function normalizeBackupCode(code: string): string {
|
||||
return code.toLowerCase().replace(/[^a-z0-9]/g, '');
|
||||
}
|
||||
|
||||
export async function hasUnconsumedBackupCodes(ctx: ApiContext, userId: UserID): Promise<boolean> {
|
||||
const backupCodes = await ctx.services.users.listMfaBackupCodes(userId);
|
||||
return backupCodes.some((backupCode) => !backupCode.consumed);
|
||||
}
|
||||
|
||||
export async function verifyMfaCode(ctx: ApiContext, params: VerifyMfaCodeParams): Promise<boolean> {
|
||||
const {userId, mfaSecret, code, allowBackup = false} = params;
|
||||
const {users, cache, config} = ctx.services;
|
||||
try {
|
||||
const totp = new TotpGenerator(mfaSecret);
|
||||
const isValidTotp = await totp.validateTotp(code);
|
||||
if (isValidTotp) {
|
||||
if (config.dev.testModeEnabled) {
|
||||
return true;
|
||||
}
|
||||
const reuseKey = `mfa-totp:${userId}:${code}`;
|
||||
const lockToken = await cache.acquireLock(reuseKey, seconds('90 seconds'));
|
||||
if (lockToken) {
|
||||
return true;
|
||||
if (mfaSecret !== null) {
|
||||
try {
|
||||
const totp = new TotpGenerator(mfaSecret);
|
||||
const isValidTotp = await totp.validateTotp(code);
|
||||
if (isValidTotp) {
|
||||
if (config.dev.testModeEnabled) {
|
||||
return true;
|
||||
}
|
||||
const reuseKey = `mfa-totp:${userId}:${code}`;
|
||||
const lockToken = await cache.acquireLock(reuseKey, seconds('90 seconds'));
|
||||
if (lockToken) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
Logger.error({userId, code: `${code.slice(0, 3)}***`, error}, 'Failed to validate TOTP code');
|
||||
}
|
||||
} catch (error) {
|
||||
Logger.error({userId, code: `${code.slice(0, 3)}***`, error}, 'Failed to validate TOTP code');
|
||||
}
|
||||
if (allowBackup) {
|
||||
const normalizedCode = normalizeBackupCode(code);
|
||||
@@ -145,8 +160,7 @@ export async function verifyWebAuthnRegistration(
|
||||
expectedChallenge: string,
|
||||
name: string,
|
||||
): Promise<void> {
|
||||
const {users, gateway, botMfaMirror, config} = ctx.services;
|
||||
const user = await users.findUniqueAssert(userId);
|
||||
const {users, config} = ctx.services;
|
||||
const existingCredentials = await users.listWebAuthnCredentials(userId);
|
||||
await consumeWebAuthnChallenge(ctx, expectedChallenge, 'registration', {userId});
|
||||
if (existingCredentials.length >= 10) {
|
||||
@@ -214,13 +228,6 @@ export async function verifyWebAuthnRegistration(
|
||||
name,
|
||||
);
|
||||
}
|
||||
const authenticatorTypes = user.authenticatorTypes || new Set<number>();
|
||||
if (!authenticatorTypes.has(UserAuthenticatorTypes.WEBAUTHN)) {
|
||||
authenticatorTypes.add(UserAuthenticatorTypes.WEBAUTHN);
|
||||
const updatedUser = await users.patchUpsert(userId, {authenticator_types: authenticatorTypes}, user.toRow());
|
||||
await gateway.dispatchPresence({userId, event: 'USER_UPDATE', data: mapUserToPrivateResponse(updatedUser)});
|
||||
await botMfaMirror.syncAuthenticatorTypesForOwner(updatedUser);
|
||||
}
|
||||
await dispatchWebAuthnCredentialsUpdate(ctx, userId);
|
||||
}
|
||||
|
||||
@@ -234,15 +241,55 @@ export async function deleteWebAuthnCredential(ctx: ApiContext, userId: UserID,
|
||||
const remainingCredentials = await users.listWebAuthnCredentials(userId);
|
||||
if (remainingCredentials.length === 0) {
|
||||
const user = await users.findUniqueAssert(userId);
|
||||
const authenticatorTypes = user.authenticatorTypes || new Set<number>();
|
||||
authenticatorTypes.delete(UserAuthenticatorTypes.WEBAUTHN);
|
||||
const updatedUser = await users.patchUpsert(userId, {authenticator_types: authenticatorTypes}, user.toRow());
|
||||
await gateway.dispatchPresence({userId, event: 'USER_UPDATE', data: mapUserToPrivateResponse(updatedUser)});
|
||||
await botMfaMirror.syncAuthenticatorTypesForOwner(updatedUser);
|
||||
if (user.authenticatorTypes.has(UserAuthenticatorTypes.WEBAUTHN)) {
|
||||
const authenticatorTypes = new Set<number>(user.authenticatorTypes ?? []);
|
||||
authenticatorTypes.delete(UserAuthenticatorTypes.WEBAUTHN);
|
||||
const updatedUser = await users.patchUpsert(userId, {authenticator_types: authenticatorTypes}, user.toRow());
|
||||
if (!userHasMfa(updatedUser)) {
|
||||
await users.clearMfaBackupCodes(userId);
|
||||
}
|
||||
await gateway.dispatchPresence({userId, event: 'USER_UPDATE', data: mapUserToPrivateResponse(updatedUser)});
|
||||
await botMfaMirror.syncAuthenticatorTypesForOwner(updatedUser);
|
||||
}
|
||||
}
|
||||
await dispatchWebAuthnCredentialsUpdate(ctx, userId);
|
||||
}
|
||||
|
||||
export async function setWebAuthnTwoFactor(
|
||||
ctx: ApiContext,
|
||||
userId: UserID,
|
||||
enabled: boolean,
|
||||
): Promise<SetWebAuthnTwoFactorResult> {
|
||||
const {users, gateway, botMfaMirror} = ctx.services;
|
||||
const user = await users.findUniqueAssert(userId);
|
||||
const credentials = await users.listWebAuthnCredentials(userId);
|
||||
if (enabled && credentials.length === 0) {
|
||||
throw new NoPasskeysRegisteredError();
|
||||
}
|
||||
const authenticatorTypes = new Set<number>(user.authenticatorTypes ?? []);
|
||||
if (authenticatorTypes.has(UserAuthenticatorTypes.WEBAUTHN) === enabled) {
|
||||
return {user, backupCodes: null};
|
||||
}
|
||||
if (enabled) {
|
||||
authenticatorTypes.add(UserAuthenticatorTypes.WEBAUTHN);
|
||||
} else {
|
||||
authenticatorTypes.delete(UserAuthenticatorTypes.WEBAUTHN);
|
||||
}
|
||||
const updatedUser = await users.patchUpsert(userId, {authenticator_types: authenticatorTypes}, user.toRow());
|
||||
let backupCodes: Array<MfaBackupCode> | null = null;
|
||||
if (enabled) {
|
||||
const existingBackupCodes = await users.listMfaBackupCodes(userId);
|
||||
if (existingBackupCodes.every((backupCode) => backupCode.consumed)) {
|
||||
backupCodes = await users.createMfaBackupCodes(userId, AuthUtility.generateBackupCodes(ctx));
|
||||
}
|
||||
} else if (!userHasMfa(updatedUser)) {
|
||||
await users.clearMfaBackupCodes(userId);
|
||||
}
|
||||
await gateway.dispatchPresence({userId, event: 'USER_UPDATE', data: mapUserToPrivateResponse(updatedUser)});
|
||||
await botMfaMirror.syncAuthenticatorTypesForOwner(updatedUser);
|
||||
return {user: updatedUser, backupCodes};
|
||||
}
|
||||
|
||||
export async function renameWebAuthnCredential(
|
||||
ctx: ApiContext,
|
||||
userId: UserID,
|
||||
@@ -430,16 +477,17 @@ export async function verifySudoMfa(
|
||||
const {users} = ctx.services;
|
||||
const {userId, method, code, webauthnResponse, webauthnChallenge} = params;
|
||||
const user = await users.findUnique(userId);
|
||||
const hasMfa =
|
||||
(user?.authenticatorTypes?.has(UserAuthenticatorTypes.TOTP) ?? false) ||
|
||||
(user?.authenticatorTypes?.has(UserAuthenticatorTypes.WEBAUTHN) ?? false);
|
||||
if (!user || !hasMfa) {
|
||||
if (!user) {
|
||||
return {success: false, error: 'MFA not enabled'};
|
||||
}
|
||||
const credentials = await users.listWebAuthnCredentials(userId);
|
||||
const hasPasskeyCredentials = credentials.length > 0;
|
||||
if (!userHasSudoCapability(user, hasPasskeyCredentials)) {
|
||||
return {success: false, error: 'MFA not enabled'};
|
||||
}
|
||||
switch (method) {
|
||||
case 'totp': {
|
||||
if (!code) return {success: false, error: 'TOTP code is required'};
|
||||
if (!user.totpSecret) return {success: false, error: 'TOTP is not enabled'};
|
||||
await consumeSudoMfaAttempt(ctx, userId);
|
||||
const isValid = await verifyMfaCode(ctx, {userId, mfaSecret: user.totpSecret, code, allowBackup: true});
|
||||
if (isValid) {
|
||||
@@ -451,7 +499,7 @@ export async function verifySudoMfa(
|
||||
if (!webauthnResponse || !webauthnChallenge) {
|
||||
return {success: false, error: 'WebAuthn response and challenge are required'};
|
||||
}
|
||||
if (!user.authenticatorTypes?.has(UserAuthenticatorTypes.WEBAUTHN)) {
|
||||
if (!hasPasskeyCredentials) {
|
||||
return {success: false, error: 'WebAuthn is not enabled'};
|
||||
}
|
||||
try {
|
||||
@@ -467,15 +515,19 @@ export async function verifySudoMfa(
|
||||
}
|
||||
|
||||
export async function getAvailableMfaMethods(ctx: ApiContext, userId: UserID): Promise<AvailableMfaMethods> {
|
||||
const user = await ctx.services.users.findUnique(userId);
|
||||
const {users} = ctx.services;
|
||||
const user = await users.findUnique(userId);
|
||||
if (!user) {
|
||||
return {totp: false, webauthn: false, has_mfa: false};
|
||||
return {totp: false, webauthn: false, backup_codes: false, has_mfa: false};
|
||||
}
|
||||
const methods = deriveSudoMethods(user);
|
||||
const credentials = await users.listWebAuthnCredentials(userId);
|
||||
const hasPasskeyCredentials = credentials.length > 0;
|
||||
const methods = deriveSudoMethods(user, hasPasskeyCredentials, await hasUnconsumedBackupCodes(ctx, userId));
|
||||
return {
|
||||
totp: methods.totp,
|
||||
webauthn: methods.webauthn,
|
||||
has_mfa: userHasMfa(user),
|
||||
backup_codes: methods.backup_codes,
|
||||
has_mfa: userHasSudoCapability(user, hasPasskeyCredentials),
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -2,12 +2,14 @@
|
||||
|
||||
import crypto from 'node:crypto';
|
||||
import type {ApiContext} from '@app/api/ApiContext';
|
||||
import {createMfaTicketResponse, type LoginMfaResult} from '@app/api/auth/AuthLogin';
|
||||
import * as AuthSession from '@app/api/auth/AuthSession';
|
||||
import * as AuthUtility from '@app/api/auth/AuthUtility';
|
||||
import {createMfaTicket, createPasswordResetToken} from '@app/api/BrandedTypes';
|
||||
import {resolveWebAuthnSecondFactor} from '@app/api/auth/services/WebAuthnSecondFactor';
|
||||
import {createPasswordResetToken} from '@app/api/BrandedTypes';
|
||||
import {Config} from '@app/api/Config';
|
||||
import type {UserRow} from '@app/api/database/types/UserTypes';
|
||||
import {Logger} from '@app/api/Logger';
|
||||
import type {User} from '@app/api/models/User';
|
||||
import {EXTERNAL_RESPONSE_LIMITS} from '@app/api/utils/ExternalResponseLimits';
|
||||
import * as FetchUtils from '@app/api/utils/FetchUtils';
|
||||
import {hashPassword as hashPasswordUtil, verifyPassword as verifyPasswordUtil} from '@app/api/utils/PasswordUtils';
|
||||
@@ -19,7 +21,7 @@ import {InputValidationError} from '@fluxer/errors/src/domains/core/InputValidat
|
||||
import {requireClientIp} from '@fluxer/ip_utils/src/ClientIp';
|
||||
import {getSameIpDecisionKey} from '@fluxer/ip_utils/src/IpAddress';
|
||||
import type {ForgotPasswordRequest, ResetPasswordRequest} from '@fluxer/schema/src/domains/auth/AuthSchemas';
|
||||
import {ms, seconds} from 'itty-time';
|
||||
import {ms} from 'itty-time';
|
||||
|
||||
const PWNED_PASSWORDS_TIMEOUT_MS = ms('5 seconds');
|
||||
const PWNED_PASSWORD_CACHE_MAX_PREFIXES = 128;
|
||||
@@ -91,13 +93,7 @@ type ResetPasswordResult =
|
||||
user_id: string;
|
||||
token: string;
|
||||
}
|
||||
| {
|
||||
mfa: true;
|
||||
ticket: string;
|
||||
allowed_methods: Array<string>;
|
||||
totp: boolean;
|
||||
webauthn: boolean;
|
||||
};
|
||||
| LoginMfaResult;
|
||||
|
||||
const pwnedPasswordCache = new PwnedPasswordCache(PWNED_PASSWORD_CACHE_MAX_PREFIXES, ms('1 hour'));
|
||||
|
||||
@@ -267,22 +263,26 @@ export async function resetPassword(
|
||||
if (await isPasswordPwned(ctx, data.password)) {
|
||||
throw InputValidationError.fromCode('password', ValidationErrorCodes.PASSWORD_IS_TOO_COMMON);
|
||||
}
|
||||
const webauthnIsSecondFactor = await resolveWebAuthnSecondFactor(ctx, user);
|
||||
const hasMfa = user.authenticatorTypes.has(UserAuthenticatorTypes.TOTP) || webauthnIsSecondFactor;
|
||||
const newPasswordHash = await hashPassword(ctx, data.password);
|
||||
const updatedUser = await users.patchUpsert(
|
||||
user.id,
|
||||
{
|
||||
password_hash: newPasswordHash,
|
||||
password_last_changed_at: new Date(),
|
||||
},
|
||||
user.toRow(),
|
||||
);
|
||||
const updates: Partial<UserRow> = {
|
||||
password_hash: newPasswordHash,
|
||||
password_last_changed_at: new Date(),
|
||||
};
|
||||
if (webauthnIsSecondFactor && !user.authenticatorTypes.has(UserAuthenticatorTypes.WEBAUTHN)) {
|
||||
const authenticatorTypes = new Set<number>(user.authenticatorTypes);
|
||||
authenticatorTypes.add(UserAuthenticatorTypes.WEBAUTHN);
|
||||
updates.authenticator_types = authenticatorTypes;
|
||||
}
|
||||
const updatedUser = await users.patchUpsert(user.id, updates, user.toRow());
|
||||
if (updates.authenticator_types) {
|
||||
await ctx.services.botMfaMirror.syncAuthenticatorTypesForOwner(updatedUser);
|
||||
}
|
||||
await AuthSession.terminateAllUserSessions(ctx, user.id);
|
||||
await users.deletePasswordResetToken(data.token);
|
||||
const hasMfa =
|
||||
updatedUser.authenticatorTypes.has(UserAuthenticatorTypes.TOTP) ||
|
||||
updatedUser.authenticatorTypes.has(UserAuthenticatorTypes.WEBAUTHN);
|
||||
if (hasMfa) {
|
||||
return await createMfaTicketResponse(ctx, updatedUser);
|
||||
return await createMfaTicketResponse(ctx, updatedUser, webauthnIsSecondFactor);
|
||||
}
|
||||
const [token] = await AuthSession.createAuthSession(ctx, {
|
||||
user: updatedUser,
|
||||
@@ -290,31 +290,3 @@ export async function resetPassword(
|
||||
});
|
||||
return {user_id: updatedUser.id.toString(), token};
|
||||
}
|
||||
|
||||
async function createMfaTicketResponse(
|
||||
ctx: ApiContext,
|
||||
user: User,
|
||||
): Promise<{
|
||||
mfa: true;
|
||||
ticket: string;
|
||||
allowed_methods: Array<string>;
|
||||
totp: boolean;
|
||||
webauthn: boolean;
|
||||
}> {
|
||||
const {users, cache} = ctx.services;
|
||||
const ticket = createMfaTicket(await AuthUtility.generateSecureToken(ctx));
|
||||
await cache.set(`mfa-ticket:${ticket}`, user.id.toString(), seconds('5 minutes'));
|
||||
const credentials = await users.listWebAuthnCredentials(user.id);
|
||||
const hasWebauthn = credentials.length > 0;
|
||||
const hasTotp = user.authenticatorTypes.has(UserAuthenticatorTypes.TOTP);
|
||||
const allowedMethods: Array<string> = [];
|
||||
if (hasTotp) allowedMethods.push('totp');
|
||||
if (hasWebauthn) allowedMethods.push('webauthn');
|
||||
return {
|
||||
mfa: true,
|
||||
ticket: ticket,
|
||||
allowed_methods: allowedMethods,
|
||||
totp: hasTotp,
|
||||
webauthn: hasWebauthn,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -417,6 +417,7 @@ export class AuthRequestService {
|
||||
...result,
|
||||
totp: allowedMethods.has('totp'),
|
||||
webauthn: allowedMethods.has('webauthn'),
|
||||
backup_codes: allowedMethods.has('backup_codes'),
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3,6 +3,15 @@
|
||||
import {UserAuthenticatorTypes} from '@fluxer/constants/src/UserConstants';
|
||||
import type {SudoModeMethods} from '@fluxer/errors/src/domains/auth/SudoModeRequiredError';
|
||||
|
||||
interface SudoMethodsUser {
|
||||
totpSecret?: string | null;
|
||||
authenticatorTypes?: Set<number> | null;
|
||||
}
|
||||
|
||||
function hasTotpEnrolled(user: SudoMethodsUser): boolean {
|
||||
return (user.totpSecret ?? null) !== null && (user.authenticatorTypes?.has(UserAuthenticatorTypes.TOTP) ?? false);
|
||||
}
|
||||
|
||||
export function userHasMfa(user: {authenticatorTypes?: Set<number> | null}): boolean {
|
||||
return (
|
||||
(user.authenticatorTypes?.has(UserAuthenticatorTypes.TOTP) ?? false) ||
|
||||
@@ -10,13 +19,18 @@ export function userHasMfa(user: {authenticatorTypes?: Set<number> | null}): boo
|
||||
);
|
||||
}
|
||||
|
||||
export function deriveSudoMethods(user: {
|
||||
totpSecret?: string | null;
|
||||
authenticatorTypes?: Set<number> | null;
|
||||
}): SudoModeMethods {
|
||||
const authenticatorTypes = user.authenticatorTypes ?? null;
|
||||
export function userHasSudoCapability(user: SudoMethodsUser, hasPasskeyCredentials: boolean): boolean {
|
||||
return hasTotpEnrolled(user) || hasPasskeyCredentials;
|
||||
}
|
||||
|
||||
export function deriveSudoMethods(
|
||||
user: SudoMethodsUser,
|
||||
hasPasskeyCredentials: boolean,
|
||||
hasBackupCodes: boolean,
|
||||
): SudoModeMethods {
|
||||
return {
|
||||
totp: (user.totpSecret ?? null) !== null && (authenticatorTypes?.has(UserAuthenticatorTypes.TOTP) ?? false),
|
||||
webauthn: authenticatorTypes?.has(UserAuthenticatorTypes.WEBAUTHN) ?? false,
|
||||
totp: hasTotpEnrolled(user),
|
||||
webauthn: hasPasskeyCredentials,
|
||||
backup_codes: hasBackupCodes,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
|
||||
import * as AuthMfa from '@app/api/auth/AuthMfa';
|
||||
import * as AuthPassword from '@app/api/auth/AuthPassword';
|
||||
import {deriveSudoMethods, userHasMfa} from '@app/api/auth/services/SudoMethods';
|
||||
import {deriveSudoMethods, userHasMfa, userHasSudoCapability} from '@app/api/auth/services/SudoMethods';
|
||||
import {getSudoModeService} from '@app/api/auth/services/SudoModeService';
|
||||
import {SUDO_MODE_HEADER} from '@app/api/middleware/SudoModeMiddleware';
|
||||
import type {User} from '@app/api/models/User';
|
||||
@@ -26,8 +26,9 @@ type SudoVerificationMethod = 'password' | 'mfa' | 'sudo_token';
|
||||
export function hasNoVerifiableCredential(
|
||||
user: {passwordHash: string | null; isBot: boolean},
|
||||
hasMfa: boolean,
|
||||
hasPasskeyCredentials: boolean,
|
||||
): boolean {
|
||||
if (user.isBot || hasMfa) {
|
||||
if (user.isBot || hasMfa || hasPasskeyCredentials) {
|
||||
return false;
|
||||
}
|
||||
return user.passwordHash === null;
|
||||
@@ -52,18 +53,22 @@ async function verifySudoMode(
|
||||
if (user.isBot) {
|
||||
return {verified: true, method: 'sudo_token'};
|
||||
}
|
||||
const apiContext = ctx.get('apiContext');
|
||||
const credentials = await apiContext.services.users.listWebAuthnCredentials(user.id);
|
||||
const hasPasskeyCredentials = credentials.length > 0;
|
||||
const hasMfa = userHasMfa(user);
|
||||
const issueSudoToken = options.issueSudoToken ?? hasMfa;
|
||||
if (hasMfa && ctx.get('sudoModeValid')) {
|
||||
const hasSudoCapability = userHasSudoCapability(user, hasPasskeyCredentials);
|
||||
const issueSudoToken = options.issueSudoToken ?? hasSudoCapability;
|
||||
if (hasSudoCapability && ctx.get('sudoModeValid')) {
|
||||
const sudoToken = ctx.get('sudoModeToken') ?? ctx.req.header(SUDO_MODE_HEADER) ?? undefined;
|
||||
return {verified: true, method: 'sudo_token', sudoToken: issueSudoToken ? sudoToken : undefined};
|
||||
}
|
||||
const incomingToken = ctx.req.header(SUDO_MODE_HEADER);
|
||||
if (!hasMfa && incomingToken && ctx.get('sudoModeValid')) {
|
||||
if (!hasSudoCapability && incomingToken && ctx.get('sudoModeValid')) {
|
||||
return {verified: true, method: 'sudo_token', sudoToken: issueSudoToken ? incomingToken : undefined};
|
||||
}
|
||||
if (hasMfa && body.mfa_method) {
|
||||
const result = await AuthMfa.verifySudoMfa(ctx.get('apiContext'), {
|
||||
if (hasSudoCapability && body.mfa_method) {
|
||||
const result = await AuthMfa.verifySudoMfa(apiContext, {
|
||||
userId: user.id,
|
||||
method: body.mfa_method,
|
||||
code: body.mfa_code,
|
||||
@@ -77,14 +82,14 @@ async function verifySudoMode(
|
||||
const sudoToken = issueSudoToken ? await sudoModeService.generateSudoToken(user.id) : undefined;
|
||||
return {verified: true, sudoToken, method: 'mfa'};
|
||||
}
|
||||
if (hasNoVerifiableCredential(user, hasMfa)) {
|
||||
if (hasNoVerifiableCredential(user, hasMfa, hasPasskeyCredentials)) {
|
||||
return {verified: true, method: 'password'};
|
||||
}
|
||||
if (body.password && !hasMfa) {
|
||||
if (!user.passwordHash) {
|
||||
throw InputValidationError.fromCode('password', ValidationErrorCodes.PASSWORD_NOT_SET);
|
||||
}
|
||||
const passwordValid = await AuthPassword.verifyPassword(ctx.get('apiContext'), {
|
||||
const passwordValid = await AuthPassword.verifyPassword(apiContext, {
|
||||
password: body.password,
|
||||
passwordHash: user.passwordHash,
|
||||
});
|
||||
@@ -93,7 +98,8 @@ async function verifySudoMode(
|
||||
}
|
||||
return {verified: true, method: 'password'};
|
||||
}
|
||||
throw new SudoModeRequiredError(hasMfa, deriveSudoMethods(user));
|
||||
const hasBackupCodes = await AuthMfa.hasUnconsumedBackupCodes(apiContext, user.id);
|
||||
throw new SudoModeRequiredError(hasSudoCapability, deriveSudoMethods(user, hasPasskeyCredentials, hasBackupCodes));
|
||||
}
|
||||
|
||||
function setSudoTokenHeader(
|
||||
|
||||
@@ -0,0 +1,22 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
import type {ApiContext} from '@app/api/ApiContext';
|
||||
import type {User} from '@app/api/models/User';
|
||||
import {UserAuthenticatorTypes} from '@fluxer/constants/src/UserConstants';
|
||||
|
||||
interface WebAuthnSecondFactorUser {
|
||||
passwordHash: string | null;
|
||||
authenticatorTypes?: Set<number> | null;
|
||||
}
|
||||
|
||||
export function webAuthnIsSecondFactor(user: WebAuthnSecondFactorUser, hasPasskeyCredentials: boolean): boolean {
|
||||
return (
|
||||
(user.authenticatorTypes?.has(UserAuthenticatorTypes.WEBAUTHN) ?? false) ||
|
||||
(user.passwordHash === null && hasPasskeyCredentials)
|
||||
);
|
||||
}
|
||||
|
||||
export async function resolveWebAuthnSecondFactor(ctx: ApiContext, user: User): Promise<boolean> {
|
||||
const credentials = await ctx.services.users.listWebAuthnCredentials(user.id);
|
||||
return webAuthnIsSecondFactor(user, credentials.length > 0);
|
||||
}
|
||||
@@ -22,20 +22,10 @@ export interface LoginMfaResponse {
|
||||
allowed_methods: Array<string>;
|
||||
totp: boolean;
|
||||
webauthn: boolean;
|
||||
backup_codes: boolean;
|
||||
}
|
||||
|
||||
type LoginResponse =
|
||||
| {
|
||||
user_id: string;
|
||||
token: string;
|
||||
}
|
||||
| {
|
||||
mfa: true;
|
||||
ticket: string;
|
||||
allowed_methods: Array<string>;
|
||||
totp: boolean;
|
||||
webauthn: boolean;
|
||||
};
|
||||
type LoginResponse = LoginSuccessResponse | LoginMfaResponse;
|
||||
|
||||
export interface UserMeResponse {
|
||||
id: string;
|
||||
|
||||
@@ -627,6 +627,7 @@ describe('Email change flow', () => {
|
||||
methods?: {
|
||||
totp?: boolean;
|
||||
webauthn?: boolean;
|
||||
backup_codes?: boolean;
|
||||
};
|
||||
}>(harness, mfaAccount.token)
|
||||
.patch('/users/@me')
|
||||
@@ -634,13 +635,14 @@ describe('Email change flow', () => {
|
||||
.expect(403, 'SUDO_MODE_REQUIRED')
|
||||
.execute();
|
||||
expect(noSudoResp.has_mfa).toBe(true);
|
||||
expect(noSudoResp.methods).toEqual({totp: true, webauthn: false});
|
||||
expect(noSudoResp.methods).toEqual({totp: true, webauthn: false, backup_codes: true});
|
||||
const passwordOnlyResp = await createBuilder<{
|
||||
code: string;
|
||||
has_mfa?: boolean;
|
||||
methods?: {
|
||||
totp?: boolean;
|
||||
webauthn?: boolean;
|
||||
backup_codes?: boolean;
|
||||
};
|
||||
}>(harness, mfaAccount.token)
|
||||
.patch('/users/@me')
|
||||
@@ -648,7 +650,7 @@ describe('Email change flow', () => {
|
||||
.expect(403, 'SUDO_MODE_REQUIRED')
|
||||
.execute();
|
||||
expect(passwordOnlyResp.has_mfa).toBe(true);
|
||||
expect(passwordOnlyResp.methods).toEqual({totp: true, webauthn: false});
|
||||
expect(passwordOnlyResp.methods).toEqual({totp: true, webauthn: false, backup_codes: true});
|
||||
const updated = await createBuilder<UserPrivateResponse>(harness, mfaAccount.token)
|
||||
.patch('/users/@me')
|
||||
.body({
|
||||
@@ -712,6 +714,7 @@ describe('Email change flow', () => {
|
||||
methods?: {
|
||||
totp?: boolean;
|
||||
webauthn?: boolean;
|
||||
backup_codes?: boolean;
|
||||
};
|
||||
}>(harness, mfaAccount.token)
|
||||
.post('/users/@me/email-change/apply')
|
||||
@@ -719,13 +722,14 @@ describe('Email change flow', () => {
|
||||
.expect(403, 'SUDO_MODE_REQUIRED')
|
||||
.execute();
|
||||
expect(noSudoResp.has_mfa).toBe(true);
|
||||
expect(noSudoResp.methods).toEqual({totp: true, webauthn: false});
|
||||
expect(noSudoResp.methods).toEqual({totp: true, webauthn: false, backup_codes: true});
|
||||
const passwordOnlyResp = await createBuilder<{
|
||||
code: string;
|
||||
has_mfa?: boolean;
|
||||
methods?: {
|
||||
totp?: boolean;
|
||||
webauthn?: boolean;
|
||||
backup_codes?: boolean;
|
||||
};
|
||||
}>(harness, mfaAccount.token)
|
||||
.post('/users/@me/email-change/apply')
|
||||
@@ -733,7 +737,7 @@ describe('Email change flow', () => {
|
||||
.expect(403, 'SUDO_MODE_REQUIRED')
|
||||
.execute();
|
||||
expect(passwordOnlyResp.has_mfa).toBe(true);
|
||||
expect(passwordOnlyResp.methods).toEqual({totp: true, webauthn: false});
|
||||
expect(passwordOnlyResp.methods).toEqual({totp: true, webauthn: false, backup_codes: true});
|
||||
const updated = await createBuilder<UserPrivateResponse>(harness, mfaAccount.token)
|
||||
.post('/users/@me/email-change/apply')
|
||||
.body({
|
||||
@@ -776,6 +780,7 @@ describe('Email change flow', () => {
|
||||
methods?: {
|
||||
totp?: boolean;
|
||||
webauthn?: boolean;
|
||||
backup_codes?: boolean;
|
||||
};
|
||||
}>(harness, account.token)
|
||||
.post('/users/@me/email-change/apply')
|
||||
@@ -783,7 +788,7 @@ describe('Email change flow', () => {
|
||||
.expect(403, 'SUDO_MODE_REQUIRED')
|
||||
.execute();
|
||||
expect(noSudoResp.has_mfa).toBe(false);
|
||||
expect(noSudoResp.methods).toEqual({totp: false, webauthn: false});
|
||||
expect(noSudoResp.methods).toEqual({totp: false, webauthn: false, backup_codes: false});
|
||||
const updated = await createBuilder<UserPrivateResponse>(harness, account.token)
|
||||
.post('/users/@me/email-change/apply')
|
||||
.body({email_token: emailToken, password: account.password})
|
||||
@@ -887,6 +892,7 @@ describe('Email change flow', () => {
|
||||
methods?: {
|
||||
totp?: boolean;
|
||||
webauthn?: boolean;
|
||||
backup_codes?: boolean;
|
||||
};
|
||||
}>(harness, mfaAccount.token)
|
||||
.post('/users/@me/email-change/apply')
|
||||
@@ -894,7 +900,7 @@ describe('Email change flow', () => {
|
||||
.expect(403, 'SUDO_MODE_REQUIRED')
|
||||
.execute();
|
||||
expect(discovery.has_mfa).toBe(true);
|
||||
expect(discovery.methods).toEqual({totp: true, webauthn: false});
|
||||
expect(discovery.methods).toEqual({totp: true, webauthn: false, backup_codes: true});
|
||||
const applied = await createBuilder<UserPrivateResponse>(harness, mfaAccount.token)
|
||||
.post('/users/@me/email-change/apply')
|
||||
.body({
|
||||
|
||||
@@ -10,6 +10,7 @@ import {
|
||||
createAuthenticationResponse,
|
||||
createRegistrationResponse,
|
||||
createWebAuthnDevice,
|
||||
setWebAuthnTwoFactor,
|
||||
type WebAuthnAuthenticationOptions,
|
||||
type WebAuthnDevice,
|
||||
type WebAuthnRegistrationOptions,
|
||||
@@ -37,6 +38,7 @@ interface LoginMfaResponse {
|
||||
interface SudoMfaMethodsResponse {
|
||||
totp: boolean;
|
||||
webauthn: boolean;
|
||||
backup_codes: boolean;
|
||||
has_mfa: boolean;
|
||||
}
|
||||
|
||||
@@ -46,6 +48,7 @@ interface SudoModeRequiredResponse {
|
||||
methods?: {
|
||||
totp?: boolean;
|
||||
webauthn?: boolean;
|
||||
backup_codes?: boolean;
|
||||
};
|
||||
}
|
||||
|
||||
@@ -73,6 +76,7 @@ async function loginWithTotp(harness: ApiTestHarness, account: TestAccount, secr
|
||||
async function setupWebAuthnOnlyUser(
|
||||
harness: ApiTestHarness,
|
||||
account: TestAccount,
|
||||
twoFactorEnabled: boolean,
|
||||
): Promise<{
|
||||
account: TestAccount;
|
||||
device: WebAuthnDevice;
|
||||
@@ -124,6 +128,12 @@ async function setupWebAuthnOnlyUser(
|
||||
})
|
||||
.expect(204)
|
||||
.execute();
|
||||
if (twoFactorEnabled) {
|
||||
await setWebAuthnTwoFactor(harness, updatedAccount.token, true, {
|
||||
mfa_method: 'totp',
|
||||
mfa_code: backupCodes.backup_codes[5]!.code,
|
||||
});
|
||||
}
|
||||
await createBuilder(harness, updatedAccount.token)
|
||||
.post('/users/@me/mfa/totp/disable')
|
||||
.body({
|
||||
@@ -162,9 +172,9 @@ describe('MFA Consistency Tests', () => {
|
||||
await harness?.shutdown();
|
||||
});
|
||||
describe('WebAuthn sudo verification flow', () => {
|
||||
test('WebAuthn user can complete sudo verification with passkey', async () => {
|
||||
test('WebAuthn user with two-factor on can complete sudo verification with passkey', async () => {
|
||||
const account = await createTestAccount(harness);
|
||||
const {account: webauthnAccount, device} = await setupWebAuthnOnlyUser(harness, account);
|
||||
const {account: webauthnAccount, device} = await setupWebAuthnOnlyUser(harness, account, true);
|
||||
const sudoOptions = await createBuilder<WebAuthnAuthenticationOptions>(harness, webauthnAccount.token)
|
||||
.post('/users/@me/sudo/webauthn/authentication-options')
|
||||
.body(null)
|
||||
@@ -180,9 +190,27 @@ describe('MFA Consistency Tests', () => {
|
||||
.expect(204)
|
||||
.execute();
|
||||
});
|
||||
test('WebAuthn-only user cannot use password for sudo verification', async () => {
|
||||
test('WebAuthn user with two-factor off can complete sudo verification with passkey', async () => {
|
||||
const account = await createTestAccount(harness);
|
||||
const {account: webauthnAccount} = await setupWebAuthnOnlyUser(harness, account);
|
||||
const {account: webauthnAccount, device} = await setupWebAuthnOnlyUser(harness, account, false);
|
||||
const sudoOptions = await createBuilder<WebAuthnAuthenticationOptions>(harness, webauthnAccount.token)
|
||||
.post('/users/@me/sudo/webauthn/authentication-options')
|
||||
.body(null)
|
||||
.execute();
|
||||
const sudoAssertion = createAuthenticationResponse(device, sudoOptions);
|
||||
await createBuilder(harness, webauthnAccount.token)
|
||||
.post('/users/@me/disable')
|
||||
.body({
|
||||
mfa_method: 'webauthn',
|
||||
webauthn_response: sudoAssertion,
|
||||
webauthn_challenge: sudoOptions.challenge,
|
||||
})
|
||||
.expect(204)
|
||||
.execute();
|
||||
});
|
||||
test('WebAuthn-only user with two-factor on cannot use password for sudo verification', async () => {
|
||||
const account = await createTestAccount(harness);
|
||||
const {account: webauthnAccount} = await setupWebAuthnOnlyUser(harness, account, true);
|
||||
const errorResp = await createBuilder<{
|
||||
code: string;
|
||||
}>(harness, webauthnAccount.token)
|
||||
@@ -194,6 +222,17 @@ describe('MFA Consistency Tests', () => {
|
||||
.execute();
|
||||
expect(errorResp.code).toBe('SUDO_MODE_REQUIRED');
|
||||
});
|
||||
test('WebAuthn-only user with two-factor off can use password for sudo verification', async () => {
|
||||
const account = await createTestAccount(harness);
|
||||
const {account: webauthnAccount} = await setupWebAuthnOnlyUser(harness, account, false);
|
||||
await createBuilder(harness, webauthnAccount.token)
|
||||
.post('/users/@me/disable')
|
||||
.body({
|
||||
password: account.password,
|
||||
})
|
||||
.expect(204)
|
||||
.execute();
|
||||
});
|
||||
});
|
||||
describe('Password-only sudo flow for non-MFA users', () => {
|
||||
test('Non-MFA user can use password for sudo verification', async () => {
|
||||
@@ -323,14 +362,37 @@ describe('MFA Consistency Tests', () => {
|
||||
const methods = await createBuilder<SudoMfaMethodsResponse>(harness, account.token)
|
||||
.get('/users/@me/sudo/mfa-methods')
|
||||
.execute();
|
||||
expect(methods).toEqual({totp: false, webauthn: false, has_mfa: false});
|
||||
expect(methods).toEqual({totp: false, webauthn: false, backup_codes: false, has_mfa: false});
|
||||
const errorResp = await createBuilder<SudoModeRequiredResponse>(harness, account.token)
|
||||
.post('/users/@me/disable')
|
||||
.body({})
|
||||
.expect(403, 'SUDO_MODE_REQUIRED')
|
||||
.execute();
|
||||
expect(errorResp.has_mfa).toBe(methods.has_mfa);
|
||||
expect(errorResp.methods).toEqual({totp: methods.totp, webauthn: methods.webauthn});
|
||||
expect(errorResp.methods).toEqual({
|
||||
totp: methods.totp,
|
||||
webauthn: methods.webauthn,
|
||||
backup_codes: methods.backup_codes,
|
||||
});
|
||||
});
|
||||
test('mfa-methods reports webauthn for a passkey user with two-factor off', async () => {
|
||||
const account = await createTestAccount(harness);
|
||||
const {account: webauthnAccount} = await setupWebAuthnOnlyUser(harness, account, false);
|
||||
const methods = await createBuilder<SudoMfaMethodsResponse>(harness, webauthnAccount.token)
|
||||
.get('/users/@me/sudo/mfa-methods')
|
||||
.execute();
|
||||
expect(methods).toEqual({totp: false, webauthn: true, backup_codes: false, has_mfa: true});
|
||||
const errorResp = await createBuilder<SudoModeRequiredResponse>(harness, webauthnAccount.token)
|
||||
.post('/users/@me/disable')
|
||||
.body({})
|
||||
.expect(403, 'SUDO_MODE_REQUIRED')
|
||||
.execute();
|
||||
expect(errorResp.has_mfa).toBe(methods.has_mfa);
|
||||
expect(errorResp.methods).toEqual({
|
||||
totp: methods.totp,
|
||||
webauthn: methods.webauthn,
|
||||
backup_codes: methods.backup_codes,
|
||||
});
|
||||
});
|
||||
test('mfa-methods agrees with the SUDO_MODE_REQUIRED body for an enrolled TOTP user', async () => {
|
||||
const account = await createTestAccount(harness);
|
||||
@@ -347,14 +409,18 @@ describe('MFA Consistency Tests', () => {
|
||||
const methods = await createBuilder<SudoMfaMethodsResponse>(harness, loggedIn.token)
|
||||
.get('/users/@me/sudo/mfa-methods')
|
||||
.execute();
|
||||
expect(methods).toEqual({totp: true, webauthn: false, has_mfa: true});
|
||||
expect(methods).toEqual({totp: true, webauthn: false, backup_codes: true, has_mfa: true});
|
||||
const errorResp = await createBuilder<SudoModeRequiredResponse>(harness, loggedIn.token)
|
||||
.post('/users/@me/disable')
|
||||
.body({})
|
||||
.expect(403, 'SUDO_MODE_REQUIRED')
|
||||
.execute();
|
||||
expect(errorResp.has_mfa).toBe(methods.has_mfa);
|
||||
expect(errorResp.methods).toEqual({totp: methods.totp, webauthn: methods.webauthn});
|
||||
expect(errorResp.methods).toEqual({
|
||||
totp: methods.totp,
|
||||
webauthn: methods.webauthn,
|
||||
backup_codes: methods.backup_codes,
|
||||
});
|
||||
});
|
||||
});
|
||||
describe('MFA requirement propagates to sensitive operations', () => {
|
||||
|
||||
@@ -8,9 +8,9 @@ import {
|
||||
seedMfaTicket,
|
||||
} from '@app/api/auth/tests/AuthTestUtils';
|
||||
import {
|
||||
createRegistrationResponse,
|
||||
createWebAuthnDevice,
|
||||
type WebAuthnRegistrationOptions,
|
||||
registerWebAuthnCredential,
|
||||
setWebAuthnTwoFactor,
|
||||
} from '@app/api/auth/tests/WebAuthnTestUtils';
|
||||
import type {ApiTestHarness} from '@app/api/test/ApiTestHarness';
|
||||
import {HTTP_STATUS} from '@app/api/test/TestConstants';
|
||||
@@ -41,7 +41,7 @@ describe('Auth MFA TOTP without secret', () => {
|
||||
.execute();
|
||||
expect(login.code).toBe('INVALID_FORM_BODY');
|
||||
});
|
||||
it('rejects TOTP login when only WebAuthn is enabled', async () => {
|
||||
it('rejects TOTP login when passkey two-factor is on and no TOTP secret remains', async () => {
|
||||
const account = await createTestAccount(harness);
|
||||
const device = createWebAuthnDevice();
|
||||
const secret = createTotpSecret();
|
||||
@@ -53,25 +53,14 @@ describe('Auth MFA TOTP without secret', () => {
|
||||
.post('/users/@me/mfa/totp/enable')
|
||||
.body({secret, code: generateTotpCode(secret), password: account.password})
|
||||
.execute();
|
||||
const regOptions = await createBuilder<WebAuthnRegistrationOptions>(harness, account.token)
|
||||
.post('/users/@me/mfa/webauthn/credentials/registration-options')
|
||||
.body({mfa_method: 'totp', mfa_code: generateTotpCode(secret)})
|
||||
.execute();
|
||||
if (regOptions.rp.id) {
|
||||
device.rpId = regOptions.rp.id;
|
||||
}
|
||||
const registrationResponse = createRegistrationResponse(device, regOptions, 'Test Passkey');
|
||||
await createBuilder(harness, account.token)
|
||||
.post('/users/@me/mfa/webauthn/credentials')
|
||||
.body({
|
||||
response: registrationResponse,
|
||||
challenge: regOptions.challenge,
|
||||
name: 'Test Passkey',
|
||||
mfa_method: 'totp',
|
||||
mfa_code: generateTotpCode(secret),
|
||||
})
|
||||
.expect(204)
|
||||
.execute();
|
||||
await registerWebAuthnCredential(harness, account.token, device, () => ({
|
||||
mfa_method: 'totp',
|
||||
mfa_code: generateTotpCode(secret),
|
||||
}));
|
||||
await setWebAuthnTwoFactor(harness, account.token, true, {
|
||||
mfa_method: 'totp',
|
||||
mfa_code: generateTotpCode(secret),
|
||||
});
|
||||
await createBuilder(harness, account.token)
|
||||
.post('/users/@me/mfa/totp/disable')
|
||||
.body({
|
||||
@@ -105,4 +94,39 @@ describe('Auth MFA TOTP without secret', () => {
|
||||
.execute();
|
||||
expect(bypassAttempt.code).toBe('INVALID_FORM_BODY');
|
||||
});
|
||||
it('issues a session token when passkey two-factor is off and no TOTP secret remains', async () => {
|
||||
const account = await createTestAccount(harness);
|
||||
const device = createWebAuthnDevice();
|
||||
const secret = createTotpSecret();
|
||||
const totpData = await createBuilder<{
|
||||
backup_codes: Array<{
|
||||
code: string;
|
||||
}>;
|
||||
}>(harness, account.token)
|
||||
.post('/users/@me/mfa/totp/enable')
|
||||
.body({secret, code: generateTotpCode(secret), password: account.password})
|
||||
.execute();
|
||||
await registerWebAuthnCredential(harness, account.token, device, () => ({
|
||||
mfa_method: 'totp',
|
||||
mfa_code: generateTotpCode(secret),
|
||||
}));
|
||||
await createBuilder(harness, account.token)
|
||||
.post('/users/@me/mfa/totp/disable')
|
||||
.body({
|
||||
code: totpData.backup_codes[0]!.code,
|
||||
mfa_method: 'totp',
|
||||
mfa_code: generateTotpCode(secret),
|
||||
})
|
||||
.expect(204)
|
||||
.execute();
|
||||
const login = await createBuilderWithoutAuth<{
|
||||
mfa?: true;
|
||||
token: string;
|
||||
}>(harness)
|
||||
.post('/auth/login')
|
||||
.body({email: account.email, password: account.password})
|
||||
.execute();
|
||||
expect(login.mfa).toBeUndefined();
|
||||
expect(login.token).toBeTruthy();
|
||||
});
|
||||
});
|
||||
|
||||
@@ -4,13 +4,25 @@ import {
|
||||
clearTestEmails,
|
||||
createAuthHarness,
|
||||
createTestAccount,
|
||||
createUniqueEmail,
|
||||
findLastTestEmail,
|
||||
type LoginSuccessResponse,
|
||||
listTestEmails,
|
||||
type TestAccount,
|
||||
type TestEmailRecord,
|
||||
totpCodeNow,
|
||||
unclaimAccount,
|
||||
} from '@app/api/auth/tests/AuthTestUtils';
|
||||
import {
|
||||
createAuthenticationResponse,
|
||||
createWebAuthnDevice,
|
||||
registerWebAuthnCredential,
|
||||
setWebAuthnTwoFactor,
|
||||
type WebAuthnAuthenticationOptions,
|
||||
} from '@app/api/auth/tests/WebAuthnTestUtils';
|
||||
import type {ApiTestHarness} from '@app/api/test/ApiTestHarness';
|
||||
import {createBuilder, createBuilderWithoutAuth} from '@app/api/test/TestRequestBuilder';
|
||||
import {UserAuthenticatorTypes} from '@fluxer/constants/src/UserConstants';
|
||||
import {afterAll, beforeAll, beforeEach, describe, expect, it} from 'vitest';
|
||||
|
||||
interface MfaRequiredResponse {
|
||||
@@ -19,6 +31,7 @@ interface MfaRequiredResponse {
|
||||
allowed_methods: Array<string>;
|
||||
totp: boolean;
|
||||
webauthn: boolean;
|
||||
backup_codes: boolean;
|
||||
}
|
||||
|
||||
async function waitForEmail(harness: ApiTestHarness, type: string, recipient: string): Promise<TestEmailRecord> {
|
||||
@@ -34,6 +47,34 @@ async function waitForEmail(harness: ApiTestHarness, type: string, recipient: st
|
||||
throw new Error(`Email not found: type=${type}, recipient=${recipient}`);
|
||||
}
|
||||
|
||||
async function claimEmailWithoutPassword(harness: ApiTestHarness, account: TestAccount): Promise<TestAccount> {
|
||||
await unclaimAccount(harness, account.userId);
|
||||
const start = await createBuilder<{ticket: string; original_proof?: string}>(harness, account.token)
|
||||
.post('/users/@me/email-change/start')
|
||||
.body({})
|
||||
.execute();
|
||||
const email = createUniqueEmail('passwordless-reset');
|
||||
await createBuilder(harness, account.token)
|
||||
.post('/users/@me/email-change/request-new')
|
||||
.body({ticket: start.ticket, new_email: email, original_proof: start.original_proof})
|
||||
.execute();
|
||||
const newEmail = await waitForEmail(harness, 'email_change_new', email);
|
||||
const verify = await createBuilder<{email_token: string}>(harness, account.token)
|
||||
.post('/users/@me/email-change/verify-new')
|
||||
.body({ticket: start.ticket, code: newEmail.metadata['code'], original_proof: start.original_proof})
|
||||
.execute();
|
||||
await createBuilder(harness, account.token).patch('/users/@me').body({email_token: verify.email_token}).execute();
|
||||
return {...account, email};
|
||||
}
|
||||
|
||||
async function requestPasswordReset(harness: ApiTestHarness, email: string): Promise<string> {
|
||||
await clearTestEmails(harness);
|
||||
await createBuilderWithoutAuth(harness).post('/auth/forgot').body({email}).expect(204).execute();
|
||||
const mail = await waitForEmail(harness, 'password_reset', email);
|
||||
const token = mail.metadata['token'];
|
||||
expect(token).toBeDefined();
|
||||
return token!;
|
||||
}
|
||||
describe('Auth reset password requires MFA', () => {
|
||||
let harness: ApiTestHarness;
|
||||
beforeAll(async () => {
|
||||
@@ -66,7 +107,8 @@ describe('Auth reset password requires MFA', () => {
|
||||
expect(resetResp.ticket).toBeDefined();
|
||||
expect(resetResp.totp).toBe(true);
|
||||
expect(resetResp.webauthn).toBe(false);
|
||||
expect(resetResp.allowed_methods).toEqual(['totp']);
|
||||
expect(resetResp.backup_codes).toBe(true);
|
||||
expect(resetResp.allowed_methods).toEqual(['totp', 'backup_codes']);
|
||||
const mfaResp = await createBuilderWithoutAuth<{
|
||||
token: string;
|
||||
}>(harness)
|
||||
@@ -86,4 +128,112 @@ describe('Auth reset password requires MFA', () => {
|
||||
expect(login.totp).toBe(true);
|
||||
expect(login.webauthn).toBe(false);
|
||||
});
|
||||
it('returns a session after password reset for a passkey user who left two-factor off', async () => {
|
||||
const account = await createTestAccount(harness);
|
||||
const device = createWebAuthnDevice();
|
||||
await registerWebAuthnCredential(harness, account.token, device, () => ({password: account.password}));
|
||||
await clearTestEmails(harness);
|
||||
await createBuilderWithoutAuth(harness).post('/auth/forgot').body({email: account.email}).expect(204).execute();
|
||||
const email = await waitForEmail(harness, 'password_reset', account.email);
|
||||
const token = email.metadata['token'];
|
||||
expect(token).toBeDefined();
|
||||
const resetResp = await createBuilderWithoutAuth<LoginSuccessResponse | MfaRequiredResponse>(harness)
|
||||
.post('/auth/reset')
|
||||
.body({token, password: 'new-strong-password-123'})
|
||||
.execute();
|
||||
expect('mfa' in resetResp).toBe(false);
|
||||
expect((resetResp as LoginSuccessResponse).token).toBeTruthy();
|
||||
});
|
||||
it('returns an MFA ticket after password reset for a passkey user who turned two-factor on', async () => {
|
||||
const account = await createTestAccount(harness);
|
||||
const device = createWebAuthnDevice();
|
||||
await registerWebAuthnCredential(harness, account.token, device, () => ({password: account.password}));
|
||||
await setWebAuthnTwoFactor(harness, account.token, true, {password: account.password});
|
||||
await clearTestEmails(harness);
|
||||
await createBuilderWithoutAuth(harness).post('/auth/forgot').body({email: account.email}).expect(204).execute();
|
||||
const email = await waitForEmail(harness, 'password_reset', account.email);
|
||||
const token = email.metadata['token'];
|
||||
expect(token).toBeDefined();
|
||||
const resetResp = await createBuilderWithoutAuth<MfaRequiredResponse>(harness)
|
||||
.post('/auth/reset')
|
||||
.body({token, password: 'new-strong-password-123'})
|
||||
.execute();
|
||||
expect(resetResp.mfa).toBe(true);
|
||||
expect(resetResp.totp).toBe(false);
|
||||
expect(resetResp.webauthn).toBe(true);
|
||||
expect(resetResp.allowed_methods).toContain('webauthn');
|
||||
const mfaOptions = await createBuilderWithoutAuth<WebAuthnAuthenticationOptions>(harness)
|
||||
.post('/auth/login/mfa/webauthn/authentication-options')
|
||||
.body({ticket: resetResp.ticket})
|
||||
.execute();
|
||||
if (mfaOptions.rpId) {
|
||||
device.rpId = mfaOptions.rpId;
|
||||
}
|
||||
const mfaResp = await createBuilderWithoutAuth<{
|
||||
token: string;
|
||||
}>(harness)
|
||||
.post('/auth/login/mfa/webauthn')
|
||||
.body({
|
||||
response: createAuthenticationResponse(device, mfaOptions),
|
||||
challenge: mfaOptions.challenge,
|
||||
ticket: resetResp.ticket,
|
||||
})
|
||||
.execute();
|
||||
expect(mfaResp.token).toBeDefined();
|
||||
});
|
||||
it('returns an MFA ticket after password reset for an account with no password that holds a passkey', async () => {
|
||||
const base = await createTestAccount(harness);
|
||||
const account = await claimEmailWithoutPassword(harness, base);
|
||||
const device = createWebAuthnDevice();
|
||||
await registerWebAuthnCredential(harness, account.token, device, () => ({}));
|
||||
const token = await requestPasswordReset(harness, account.email);
|
||||
const resetResp = await createBuilderWithoutAuth<MfaRequiredResponse>(harness)
|
||||
.post('/auth/reset')
|
||||
.body({token, password: 'new-strong-password-123'})
|
||||
.execute();
|
||||
expect(resetResp.mfa).toBe(true);
|
||||
expect(resetResp.totp).toBe(false);
|
||||
expect(resetResp.webauthn).toBe(true);
|
||||
expect(resetResp.allowed_methods).toContain('webauthn');
|
||||
const mfaOptions = await createBuilderWithoutAuth<WebAuthnAuthenticationOptions>(harness)
|
||||
.post('/auth/login/mfa/webauthn/authentication-options')
|
||||
.body({ticket: resetResp.ticket})
|
||||
.execute();
|
||||
if (mfaOptions.rpId) {
|
||||
device.rpId = mfaOptions.rpId;
|
||||
}
|
||||
const mfaResp = await createBuilderWithoutAuth<LoginSuccessResponse>(harness)
|
||||
.post('/auth/login/mfa/webauthn')
|
||||
.body({
|
||||
response: createAuthenticationResponse(device, mfaOptions),
|
||||
challenge: mfaOptions.challenge,
|
||||
ticket: resetResp.ticket,
|
||||
})
|
||||
.execute();
|
||||
expect(mfaResp.token).toBeTruthy();
|
||||
const me = await createBuilder<{id: string; authenticator_types: Array<number>}>(harness, mfaResp.token)
|
||||
.get('/users/@me')
|
||||
.execute();
|
||||
expect(me.id).toBe(account.userId);
|
||||
expect(me.authenticator_types).toEqual([UserAuthenticatorTypes.WEBAUTHN]);
|
||||
});
|
||||
it('keeps demanding the passkey on a second password reset for an account that started with no password', async () => {
|
||||
const base = await createTestAccount(harness);
|
||||
const account = await claimEmailWithoutPassword(harness, base);
|
||||
const device = createWebAuthnDevice();
|
||||
await registerWebAuthnCredential(harness, account.token, device, () => ({}));
|
||||
const firstToken = await requestPasswordReset(harness, account.email);
|
||||
await createBuilderWithoutAuth<MfaRequiredResponse>(harness)
|
||||
.post('/auth/reset')
|
||||
.body({token: firstToken, password: 'new-strong-password-123'})
|
||||
.execute();
|
||||
const secondToken = await requestPasswordReset(harness, account.email);
|
||||
const secondReset = await createBuilderWithoutAuth<MfaRequiredResponse>(harness)
|
||||
.post('/auth/reset')
|
||||
.body({token: secondToken, password: 'another-strong-password-456'})
|
||||
.execute();
|
||||
expect(secondReset.mfa).toBe(true);
|
||||
expect(secondReset.webauthn).toBe(true);
|
||||
expect(secondReset.allowed_methods).toContain('webauthn');
|
||||
});
|
||||
});
|
||||
|
||||
@@ -0,0 +1,71 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
import {createAuthHarness, createTestAccount, unclaimAccount} from '@app/api/auth/tests/AuthTestUtils';
|
||||
import {
|
||||
createSudoWebAuthnBody,
|
||||
createWebAuthnDevice,
|
||||
registerWebAuthnCredential,
|
||||
} from '@app/api/auth/tests/WebAuthnTestUtils';
|
||||
import type {ApiTestHarness} from '@app/api/test/ApiTestHarness';
|
||||
import {HTTP_STATUS} from '@app/api/test/TestConstants';
|
||||
import {createBuilder} from '@app/api/test/TestRequestBuilder';
|
||||
import {afterAll, beforeAll, beforeEach, describe, expect, it} from 'vitest';
|
||||
|
||||
interface SudoMfaMethodsResponse {
|
||||
totp: boolean;
|
||||
webauthn: boolean;
|
||||
backup_codes: boolean;
|
||||
has_mfa: boolean;
|
||||
}
|
||||
|
||||
describe('Sudo mode for passwordless accounts holding a passkey', () => {
|
||||
let harness: ApiTestHarness;
|
||||
beforeAll(async () => {
|
||||
harness = await createAuthHarness();
|
||||
});
|
||||
beforeEach(async () => {
|
||||
await harness.reset();
|
||||
});
|
||||
afterAll(async () => {
|
||||
await harness?.shutdown();
|
||||
});
|
||||
it('still waves through a passwordless account that holds no credential at all', async () => {
|
||||
const account = await createTestAccount(harness);
|
||||
await unclaimAccount(harness, account.userId);
|
||||
await createBuilder(harness, account.token)
|
||||
.post('/users/@me/disable')
|
||||
.body({})
|
||||
.expect(HTTP_STATUS.NO_CONTENT)
|
||||
.execute();
|
||||
});
|
||||
it('challenges a passwordless account that holds a passkey it never made a second factor', async () => {
|
||||
const account = await createTestAccount(harness);
|
||||
const device = createWebAuthnDevice();
|
||||
await registerWebAuthnCredential(harness, account.token, device, () => ({password: account.password}));
|
||||
await unclaimAccount(harness, account.userId);
|
||||
const methods = await createBuilder<SudoMfaMethodsResponse>(harness, account.token)
|
||||
.get('/users/@me/sudo/mfa-methods')
|
||||
.execute();
|
||||
expect(methods).toEqual({totp: false, webauthn: true, backup_codes: false, has_mfa: true});
|
||||
const errorResp = await createBuilder<{
|
||||
code: string;
|
||||
}>(harness, account.token)
|
||||
.post('/users/@me/disable')
|
||||
.body({})
|
||||
.expect(HTTP_STATUS.FORBIDDEN, 'SUDO_MODE_REQUIRED')
|
||||
.execute();
|
||||
expect(errorResp.code).toBe('SUDO_MODE_REQUIRED');
|
||||
});
|
||||
it('lets the passwordless passkey holder clear the challenge with an assertion', async () => {
|
||||
const account = await createTestAccount(harness);
|
||||
const device = createWebAuthnDevice();
|
||||
await registerWebAuthnCredential(harness, account.token, device, () => ({password: account.password}));
|
||||
await unclaimAccount(harness, account.userId);
|
||||
const sudoBody = await createSudoWebAuthnBody(harness, account.token, device);
|
||||
await createBuilder(harness, account.token)
|
||||
.post('/users/@me/disable')
|
||||
.body(sudoBody)
|
||||
.expect(HTTP_STATUS.NO_CONTENT)
|
||||
.execute();
|
||||
});
|
||||
});
|
||||
@@ -1,6 +1,6 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
import {userHasMfa} from '@app/api/auth/services/SudoMethods';
|
||||
import {userHasMfa, userHasSudoCapability} from '@app/api/auth/services/SudoMethods';
|
||||
import {hasNoVerifiableCredential} from '@app/api/auth/services/SudoVerificationService';
|
||||
import {createUserID} from '@app/api/BrandedTypes';
|
||||
import {EMPTY_USER_ROW, type UserRow} from '@app/api/database/types/UserTypes';
|
||||
@@ -25,17 +25,17 @@ describe('sudo verification credential capability', () => {
|
||||
it('lets an SSO provisioned account without a password satisfy sudo mode', () => {
|
||||
const user = createUser({password_hash: null, traits: new Set<string>(['sso'])});
|
||||
expect(user.isUnclaimedAccount()).toBe(false);
|
||||
expect(hasNoVerifiableCredential(user, userHasMfa(user))).toBe(true);
|
||||
expect(hasNoVerifiableCredential(user, userHasMfa(user), false)).toBe(true);
|
||||
});
|
||||
|
||||
it('still lets an unclaimed account satisfy sudo mode', () => {
|
||||
const user = createUser({password_hash: null});
|
||||
expect(hasNoVerifiableCredential(user, userHasMfa(user))).toBe(true);
|
||||
expect(hasNoVerifiableCredential(user, userHasMfa(user), false)).toBe(true);
|
||||
});
|
||||
|
||||
it('still requires a password from accounts that have one', () => {
|
||||
const user = createUser({password_hash: 'hash', traits: new Set<string>(['sso'])});
|
||||
expect(hasNoVerifiableCredential(user, userHasMfa(user))).toBe(false);
|
||||
expect(hasNoVerifiableCredential(user, userHasMfa(user), false)).toBe(false);
|
||||
});
|
||||
|
||||
it('still requires MFA from an SSO account that enrolled a second factor', () => {
|
||||
@@ -45,11 +45,36 @@ describe('sudo verification credential capability', () => {
|
||||
authenticator_types: new Set<number>([UserAuthenticatorTypes.TOTP]),
|
||||
});
|
||||
expect(userHasMfa(user)).toBe(true);
|
||||
expect(hasNoVerifiableCredential(user, userHasMfa(user))).toBe(false);
|
||||
expect(hasNoVerifiableCredential(user, userHasMfa(user), false)).toBe(false);
|
||||
});
|
||||
|
||||
it('never applies to bots', () => {
|
||||
const user = createUser({password_hash: null, bot: true});
|
||||
expect(hasNoVerifiableCredential(user, userHasMfa(user))).toBe(false);
|
||||
expect(hasNoVerifiableCredential(user, userHasMfa(user), false)).toBe(false);
|
||||
});
|
||||
|
||||
it('still requires MFA from a passwordless account holding a passkey it never made a second factor', () => {
|
||||
const user = createUser({password_hash: null, traits: new Set<string>(['sso'])});
|
||||
expect(userHasMfa(user)).toBe(false);
|
||||
expect(userHasSudoCapability(user, true)).toBe(true);
|
||||
expect(hasNoVerifiableCredential(user, userHasMfa(user), true)).toBe(false);
|
||||
});
|
||||
|
||||
it('still requires MFA from an unclaimed account holding a passkey', () => {
|
||||
const user = createUser({password_hash: null});
|
||||
expect(hasNoVerifiableCredential(user, userHasMfa(user), true)).toBe(false);
|
||||
});
|
||||
|
||||
it('reports no sudo capability for an account with a TOTP secret that was never enrolled', () => {
|
||||
const user = createUser({totp_secret: 'JBSWY3DPEHPK3PXP'});
|
||||
expect(userHasSudoCapability(user, false)).toBe(false);
|
||||
});
|
||||
|
||||
it('reports sudo capability from an enrolled TOTP secret without any passkey', () => {
|
||||
const user = createUser({
|
||||
totp_secret: 'JBSWY3DPEHPK3PXP',
|
||||
authenticator_types: new Set<number>([UserAuthenticatorTypes.TOTP]),
|
||||
});
|
||||
expect(userHasSudoCapability(user, false)).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -6,11 +6,13 @@ import {
|
||||
createTotpSecret,
|
||||
createWebAuthnDevice,
|
||||
generateTotpCode,
|
||||
registerWebAuthnCredential,
|
||||
type WebAuthnCredentialMetadata,
|
||||
type WebAuthnRegistrationOptions,
|
||||
} from '@app/api/auth/tests/WebAuthnTestUtils';
|
||||
import type {ApiTestHarness} from '@app/api/test/ApiTestHarness';
|
||||
import {createBuilder} from '@app/api/test/TestRequestBuilder';
|
||||
import {UserAuthenticatorTypes} from '@fluxer/constants/src/UserConstants';
|
||||
import {afterAll, beforeAll, beforeEach, describe, expect, it} from 'vitest';
|
||||
|
||||
describe('WebAuthn credential registration', () => {
|
||||
@@ -64,4 +66,23 @@ describe('WebAuthn credential registration', () => {
|
||||
expect(credentials[0].name).toBe('Test Passkey');
|
||||
expect(credentials[0].id).toBe(device.credentialId.toString('base64url'));
|
||||
});
|
||||
it('does not turn passkeys into a second factor when a credential is registered', async () => {
|
||||
const account = await createTestAccount(harness);
|
||||
const device = createWebAuthnDevice();
|
||||
const secret = createTotpSecret();
|
||||
await createBuilder(harness, account.token)
|
||||
.post('/users/@me/mfa/totp/enable')
|
||||
.body({secret, code: generateTotpCode(secret), password: account.password})
|
||||
.execute();
|
||||
await registerWebAuthnCredential(harness, account.token, device, () => ({
|
||||
mfa_method: 'totp',
|
||||
mfa_code: generateTotpCode(secret),
|
||||
}));
|
||||
const me = await createBuilder<{
|
||||
authenticator_types: Array<number>;
|
||||
}>(harness, account.token)
|
||||
.get('/users/@me')
|
||||
.execute();
|
||||
expect(me.authenticator_types).toEqual([UserAuthenticatorTypes.TOTP]);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1,54 +1,65 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
import {createTestAccount, createTotpSecret, generateTotpCode} from '@app/api/auth/tests/AuthTestUtils';
|
||||
import {
|
||||
createAuthenticationResponse,
|
||||
createRegistrationResponse,
|
||||
createTestAccount,
|
||||
createTotpSecret,
|
||||
generateTotpCode,
|
||||
type LoginMfaResponse,
|
||||
type LoginSuccessResponse,
|
||||
type TestAccount,
|
||||
} from '@app/api/auth/tests/AuthTestUtils';
|
||||
import {
|
||||
createSudoWebAuthnBody,
|
||||
createWebAuthnDevice,
|
||||
loginWithDiscoverablePasskey,
|
||||
registerWebAuthnCredential,
|
||||
setWebAuthnTwoFactor,
|
||||
type WebAuthnAuthenticationOptions,
|
||||
type WebAuthnDevice,
|
||||
} from '@app/api/auth/tests/WebAuthnTestUtils';
|
||||
import {type ApiTestHarness, createApiTestHarness} from '@app/api/test/ApiTestHarness';
|
||||
import {createBuilder, createBuilderWithoutAuth} from '@app/api/test/TestRequestBuilder';
|
||||
import {beforeEach, describe, expect, test} from 'vitest';
|
||||
|
||||
interface BackupCodesResponse {
|
||||
backup_codes: Array<{
|
||||
code: string;
|
||||
}>;
|
||||
}
|
||||
|
||||
interface LoginMfaResponse {
|
||||
mfa: true;
|
||||
ticket: string;
|
||||
totp: boolean;
|
||||
webauthn: boolean;
|
||||
}
|
||||
|
||||
interface WebAuthnRegistrationOptions {
|
||||
challenge: string;
|
||||
rp: {
|
||||
id: string;
|
||||
name: string;
|
||||
};
|
||||
user: {
|
||||
id: string;
|
||||
name: string;
|
||||
displayName: string;
|
||||
};
|
||||
authenticatorSelection?: {
|
||||
residentKey?: string;
|
||||
requireResidentKey?: boolean;
|
||||
userVerification?: string;
|
||||
};
|
||||
}
|
||||
|
||||
interface WebAuthnAuthenticationOptions {
|
||||
challenge: string;
|
||||
rpId: string;
|
||||
allowCredentials?: Array<{
|
||||
id: string;
|
||||
type: string;
|
||||
}>;
|
||||
userVerification: string;
|
||||
async function setupPasskeyOnlyAccount(
|
||||
harness: ApiTestHarness,
|
||||
twoFactorEnabled: boolean,
|
||||
): Promise<{
|
||||
account: TestAccount;
|
||||
device: WebAuthnDevice;
|
||||
}> {
|
||||
const account = await createTestAccount(harness);
|
||||
const device = createWebAuthnDevice();
|
||||
const secret = createTotpSecret();
|
||||
await createBuilder(harness, account.token)
|
||||
.post('/users/@me/mfa/totp/enable')
|
||||
.body({
|
||||
secret,
|
||||
code: generateTotpCode(secret),
|
||||
password: account.password,
|
||||
})
|
||||
.execute();
|
||||
await registerWebAuthnCredential(harness, account.token, device, () => ({
|
||||
mfa_method: 'totp',
|
||||
mfa_code: generateTotpCode(secret),
|
||||
}));
|
||||
if (twoFactorEnabled) {
|
||||
await setWebAuthnTwoFactor(harness, account.token, true, {
|
||||
mfa_method: 'totp',
|
||||
mfa_code: generateTotpCode(secret),
|
||||
});
|
||||
}
|
||||
await createBuilder(harness, account.token)
|
||||
.post('/users/@me/mfa/totp/disable')
|
||||
.body({
|
||||
code: generateTotpCode(secret),
|
||||
mfa_method: 'totp',
|
||||
mfa_code: generateTotpCode(secret),
|
||||
})
|
||||
.expect(204)
|
||||
.execute();
|
||||
const token = await loginWithDiscoverablePasskey(harness, device);
|
||||
return {account: {...account, token}, device};
|
||||
}
|
||||
|
||||
describe('WebAuthn MFA Consistency Tests', () => {
|
||||
@@ -56,84 +67,8 @@ describe('WebAuthn MFA Consistency Tests', () => {
|
||||
beforeEach(async () => {
|
||||
harness = await createApiTestHarness();
|
||||
});
|
||||
test('WebAuthn-only user cannot use password for sudo - password rejected with 403', async () => {
|
||||
const account = await createTestAccount(harness);
|
||||
const device = createWebAuthnDevice();
|
||||
const secret = createTotpSecret();
|
||||
const backupCodes = await createBuilder<BackupCodesResponse>(harness, account.token)
|
||||
.post('/users/@me/mfa/totp/enable')
|
||||
.body({
|
||||
secret,
|
||||
code: generateTotpCode(secret),
|
||||
password: account.password,
|
||||
})
|
||||
.execute();
|
||||
const login = await createBuilderWithoutAuth<LoginMfaResponse>(harness)
|
||||
.post('/auth/login')
|
||||
.body({
|
||||
email: account.email,
|
||||
password: account.password,
|
||||
})
|
||||
.execute();
|
||||
expect(login.mfa).toBe(true);
|
||||
const mfaLogin = await createBuilderWithoutAuth<{
|
||||
token: string;
|
||||
}>(harness)
|
||||
.post('/auth/login/mfa/totp')
|
||||
.body({
|
||||
code: backupCodes.backup_codes[0]!.code,
|
||||
ticket: login.ticket,
|
||||
})
|
||||
.execute();
|
||||
account.token = mfaLogin.token;
|
||||
const registrationOptions = await createBuilder<WebAuthnRegistrationOptions>(harness, account.token)
|
||||
.post('/users/@me/mfa/webauthn/credentials/registration-options')
|
||||
.body({
|
||||
mfa_method: 'totp',
|
||||
mfa_code: backupCodes.backup_codes[1]!.code,
|
||||
})
|
||||
.execute();
|
||||
expect(registrationOptions.authenticatorSelection).toMatchObject({
|
||||
residentKey: 'preferred',
|
||||
requireResidentKey: false,
|
||||
userVerification: 'preferred',
|
||||
});
|
||||
const registrationResponse = createRegistrationResponse(device, registrationOptions, 'Test Passkey');
|
||||
await createBuilder(harness, account.token)
|
||||
.post('/users/@me/mfa/webauthn/credentials')
|
||||
.body({
|
||||
response: registrationResponse,
|
||||
challenge: registrationOptions.challenge,
|
||||
name: 'Test Passkey',
|
||||
mfa_method: 'totp',
|
||||
mfa_code: backupCodes.backup_codes[2]!.code,
|
||||
})
|
||||
.expect(204)
|
||||
.execute();
|
||||
await createBuilder(harness, account.token)
|
||||
.post('/users/@me/mfa/totp/disable')
|
||||
.body({
|
||||
code: backupCodes.backup_codes[3]!.code,
|
||||
mfa_method: 'totp',
|
||||
mfa_code: backupCodes.backup_codes[4]!.code,
|
||||
})
|
||||
.expect(204)
|
||||
.execute();
|
||||
const discoverableOptions = await createBuilderWithoutAuth<WebAuthnAuthenticationOptions>(harness)
|
||||
.post('/auth/webauthn/authentication-options')
|
||||
.body(null)
|
||||
.execute();
|
||||
const discoverableAssertion = createAuthenticationResponse(device, discoverableOptions);
|
||||
const passkeyLogin = await createBuilderWithoutAuth<{
|
||||
token: string;
|
||||
}>(harness)
|
||||
.post('/auth/webauthn/authenticate')
|
||||
.body({
|
||||
response: discoverableAssertion,
|
||||
challenge: discoverableOptions.challenge,
|
||||
})
|
||||
.execute();
|
||||
account.token = passkeyLogin.token;
|
||||
test('passkey user with two-factor on cannot use password for sudo - password rejected with 403', async () => {
|
||||
const {account} = await setupPasskeyOnlyAccount(harness, true);
|
||||
const {json: errorResp} = await createBuilder<{
|
||||
code: string;
|
||||
}>(harness, account.token)
|
||||
@@ -145,18 +80,34 @@ describe('WebAuthn MFA Consistency Tests', () => {
|
||||
.executeWithResponse();
|
||||
expect(errorResp.code).toBe('SUDO_MODE_REQUIRED');
|
||||
});
|
||||
test('WebAuthn-only user can use WebAuthn for sudo verification', async () => {
|
||||
const account = await createTestAccount(harness);
|
||||
const device = createWebAuthnDevice();
|
||||
const secret = createTotpSecret();
|
||||
const backupCodes = await createBuilder<BackupCodesResponse>(harness, account.token)
|
||||
.post('/users/@me/mfa/totp/enable')
|
||||
test('passkey user with two-factor off can still use password for sudo', async () => {
|
||||
const {account} = await setupPasskeyOnlyAccount(harness, false);
|
||||
await createBuilder(harness, account.token)
|
||||
.post('/users/@me/disable')
|
||||
.body({
|
||||
secret,
|
||||
code: generateTotpCode(secret),
|
||||
password: account.password,
|
||||
})
|
||||
.expect(204)
|
||||
.execute();
|
||||
});
|
||||
test('passkey user with two-factor on can use WebAuthn for sudo verification', async () => {
|
||||
const {account, device} = await setupPasskeyOnlyAccount(harness, true);
|
||||
const sudoBody = await createSudoWebAuthnBody(harness, account.token, device);
|
||||
const {response: disableResp} = await createBuilder(harness, account.token)
|
||||
.post('/users/@me/disable')
|
||||
.body(sudoBody)
|
||||
.expect(204)
|
||||
.executeWithResponse();
|
||||
const sudoToken = disableResp.headers.get('x-sudo-mode-token');
|
||||
expect(sudoToken).toBeNull();
|
||||
});
|
||||
test('passkey user with two-factor off can use WebAuthn for sudo verification', async () => {
|
||||
const {account, device} = await setupPasskeyOnlyAccount(harness, false);
|
||||
const sudoBody = await createSudoWebAuthnBody(harness, account.token, device);
|
||||
await createBuilder(harness, account.token).post('/users/@me/disable').body(sudoBody).expect(204).execute();
|
||||
});
|
||||
test('passkey user with two-factor on requires MFA when logging in with password', async () => {
|
||||
const {account} = await setupPasskeyOnlyAccount(harness, true);
|
||||
const login = await createBuilderWithoutAuth<LoginMfaResponse>(harness)
|
||||
.post('/auth/login')
|
||||
.body({
|
||||
@@ -165,144 +116,35 @@ describe('WebAuthn MFA Consistency Tests', () => {
|
||||
})
|
||||
.execute();
|
||||
expect(login.mfa).toBe(true);
|
||||
const mfaLogin = await createBuilderWithoutAuth<{
|
||||
token: string;
|
||||
}>(harness)
|
||||
.post('/auth/login/mfa/totp')
|
||||
expect(login.ticket).toBeTruthy();
|
||||
expect(login.webauthn).toBe(true);
|
||||
});
|
||||
test('passkey user with two-factor off logs in with password and receives a session token', async () => {
|
||||
const {account} = await setupPasskeyOnlyAccount(harness, false);
|
||||
const login = await createBuilderWithoutAuth<LoginSuccessResponse | LoginMfaResponse>(harness)
|
||||
.post('/auth/login')
|
||||
.body({
|
||||
code: backupCodes.backup_codes[0]!.code,
|
||||
ticket: login.ticket,
|
||||
email: account.email,
|
||||
password: account.password,
|
||||
})
|
||||
.execute();
|
||||
account.token = mfaLogin.token;
|
||||
const registrationOptions = await createBuilder<WebAuthnRegistrationOptions>(harness, account.token)
|
||||
.post('/users/@me/mfa/webauthn/credentials/registration-options')
|
||||
.body({
|
||||
mfa_method: 'totp',
|
||||
mfa_code: backupCodes.backup_codes[1]!.code,
|
||||
})
|
||||
expect('mfa' in login).toBe(false);
|
||||
expect((login as LoginSuccessResponse).token).toBeTruthy();
|
||||
const userInfo = await createBuilder<{
|
||||
id: string;
|
||||
}>(harness, (login as LoginSuccessResponse).token)
|
||||
.get('/users/@me')
|
||||
.execute();
|
||||
const registrationResponse = createRegistrationResponse(device, registrationOptions, 'Test Passkey');
|
||||
await createBuilder(harness, account.token)
|
||||
.post('/users/@me/mfa/webauthn/credentials')
|
||||
.body({
|
||||
response: registrationResponse,
|
||||
challenge: registrationOptions.challenge,
|
||||
name: 'Test Passkey',
|
||||
mfa_method: 'totp',
|
||||
mfa_code: backupCodes.backup_codes[2]!.code,
|
||||
})
|
||||
.expect(204)
|
||||
.execute();
|
||||
await createBuilder(harness, account.token)
|
||||
.post('/users/@me/mfa/totp/disable')
|
||||
.body({
|
||||
code: backupCodes.backup_codes[3]!.code,
|
||||
mfa_method: 'totp',
|
||||
mfa_code: backupCodes.backup_codes[4]!.code,
|
||||
})
|
||||
.expect(204)
|
||||
.execute();
|
||||
const discoverableOptions = await createBuilderWithoutAuth<WebAuthnAuthenticationOptions>(harness)
|
||||
.post('/auth/webauthn/authentication-options')
|
||||
.body(null)
|
||||
.execute();
|
||||
const discoverableAssertion = createAuthenticationResponse(device, discoverableOptions);
|
||||
const passkeyLogin = await createBuilderWithoutAuth<{
|
||||
token: string;
|
||||
}>(harness)
|
||||
.post('/auth/webauthn/authenticate')
|
||||
.body({
|
||||
response: discoverableAssertion,
|
||||
challenge: discoverableOptions.challenge,
|
||||
})
|
||||
.execute();
|
||||
account.token = passkeyLogin.token;
|
||||
expect(userInfo.id).toBe(account.userId);
|
||||
});
|
||||
test('sudo WebAuthn options stay available to a passkey user with two-factor off', async () => {
|
||||
const {account, device} = await setupPasskeyOnlyAccount(harness, false);
|
||||
const sudoOptions = await createBuilder<WebAuthnAuthenticationOptions>(harness, account.token)
|
||||
.post('/users/@me/sudo/webauthn/authentication-options')
|
||||
.body(null)
|
||||
.execute();
|
||||
expect(sudoOptions.userVerification).toBe('discouraged');
|
||||
const sudoAssertion = createAuthenticationResponse(device, sudoOptions);
|
||||
const {response: disableResp2} = await createBuilder(harness, account.token)
|
||||
.post('/users/@me/disable')
|
||||
.body({
|
||||
mfa_method: 'webauthn',
|
||||
webauthn_response: sudoAssertion,
|
||||
webauthn_challenge: sudoOptions.challenge,
|
||||
})
|
||||
.expect(204)
|
||||
.executeWithResponse();
|
||||
const sudoToken = disableResp2.headers.get('x-sudo-mode-token');
|
||||
expect(sudoToken).toBeNull();
|
||||
});
|
||||
test('WebAuthn-only user requires MFA when logging in with password', async () => {
|
||||
const account = await createTestAccount(harness);
|
||||
const device = createWebAuthnDevice();
|
||||
const secret = createTotpSecret();
|
||||
const backupCodes = await createBuilder<BackupCodesResponse>(harness, account.token)
|
||||
.post('/users/@me/mfa/totp/enable')
|
||||
.body({
|
||||
secret,
|
||||
code: generateTotpCode(secret),
|
||||
password: account.password,
|
||||
})
|
||||
.execute();
|
||||
const login = await createBuilderWithoutAuth<LoginMfaResponse>(harness)
|
||||
.post('/auth/login')
|
||||
.body({
|
||||
email: account.email,
|
||||
password: account.password,
|
||||
})
|
||||
.execute();
|
||||
expect(login.mfa).toBe(true);
|
||||
const mfaLogin = await createBuilderWithoutAuth<{
|
||||
token: string;
|
||||
}>(harness)
|
||||
.post('/auth/login/mfa/totp')
|
||||
.body({
|
||||
code: backupCodes.backup_codes[0]!.code,
|
||||
ticket: login.ticket,
|
||||
})
|
||||
.execute();
|
||||
account.token = mfaLogin.token;
|
||||
const registrationOptions = await createBuilder<WebAuthnRegistrationOptions>(harness, account.token)
|
||||
.post('/users/@me/mfa/webauthn/credentials/registration-options')
|
||||
.body({
|
||||
mfa_method: 'totp',
|
||||
mfa_code: backupCodes.backup_codes[1]!.code,
|
||||
})
|
||||
.execute();
|
||||
const registrationResponse = createRegistrationResponse(device, registrationOptions, 'Test Passkey');
|
||||
await createBuilder(harness, account.token)
|
||||
.post('/users/@me/mfa/webauthn/credentials')
|
||||
.body({
|
||||
response: registrationResponse,
|
||||
challenge: registrationOptions.challenge,
|
||||
name: 'Test Passkey',
|
||||
mfa_method: 'totp',
|
||||
mfa_code: backupCodes.backup_codes[2]!.code,
|
||||
})
|
||||
.expect(204)
|
||||
.execute();
|
||||
await createBuilder(harness, account.token)
|
||||
.post('/users/@me/mfa/totp/disable')
|
||||
.body({
|
||||
code: backupCodes.backup_codes[3]!.code,
|
||||
mfa_method: 'totp',
|
||||
mfa_code: backupCodes.backup_codes[4]!.code,
|
||||
})
|
||||
.expect(204)
|
||||
.execute();
|
||||
const login2 = await createBuilderWithoutAuth<LoginMfaResponse>(harness)
|
||||
.post('/auth/login')
|
||||
.body({
|
||||
email: account.email,
|
||||
password: account.password,
|
||||
})
|
||||
.execute();
|
||||
expect(login2.mfa).toBe(true);
|
||||
expect(login2.ticket).toBeTruthy();
|
||||
expect(login2.webauthn).toBe(true);
|
||||
expect(sudoOptions.allowCredentials?.length).toBeGreaterThan(0);
|
||||
expect(device.credentialId.length).toBeGreaterThan(0);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -4,16 +4,17 @@ import {
|
||||
createAuthHarness,
|
||||
createTestAccount,
|
||||
type LoginMfaResponse,
|
||||
type LoginSuccessResponse,
|
||||
loginUser,
|
||||
} from '@app/api/auth/tests/AuthTestUtils';
|
||||
import {
|
||||
createAuthenticationResponse,
|
||||
createRegistrationResponse,
|
||||
createTotpSecret,
|
||||
createWebAuthnDevice,
|
||||
generateTotpCode,
|
||||
registerWebAuthnCredential,
|
||||
setWebAuthnTwoFactor,
|
||||
type WebAuthnAuthenticationOptions,
|
||||
type WebAuthnRegistrationOptions,
|
||||
} from '@app/api/auth/tests/WebAuthnTestUtils';
|
||||
import type {ApiTestHarness} from '@app/api/test/ApiTestHarness';
|
||||
import {createBuilder, createBuilderWithoutAuth} from '@app/api/test/TestRequestBuilder';
|
||||
@@ -30,7 +31,7 @@ describe('WebAuthn MFA login', () => {
|
||||
afterAll(async () => {
|
||||
await harness?.shutdown();
|
||||
});
|
||||
it('validates the WebAuthn MFA login flow', async () => {
|
||||
it('validates the WebAuthn MFA login flow when passkey two-factor is turned on', async () => {
|
||||
const account = await createTestAccount(harness);
|
||||
const device = createWebAuthnDevice();
|
||||
const secret = createTotpSecret();
|
||||
@@ -38,25 +39,17 @@ describe('WebAuthn MFA login', () => {
|
||||
.post('/users/@me/mfa/totp/enable')
|
||||
.body({secret, code: generateTotpCode(secret), password: account.password})
|
||||
.execute();
|
||||
const regOptions = await createBuilder<WebAuthnRegistrationOptions>(harness, account.token)
|
||||
.post('/users/@me/mfa/webauthn/credentials/registration-options')
|
||||
.body({mfa_method: 'totp', mfa_code: generateTotpCode(secret)})
|
||||
.execute();
|
||||
if (regOptions.rp.id) {
|
||||
device.rpId = regOptions.rp.id;
|
||||
}
|
||||
const registrationResponse = createRegistrationResponse(device, regOptions, 'MFA Passkey');
|
||||
await createBuilder(harness, account.token)
|
||||
.post('/users/@me/mfa/webauthn/credentials')
|
||||
.body({
|
||||
response: registrationResponse,
|
||||
challenge: regOptions.challenge,
|
||||
name: 'MFA Passkey',
|
||||
mfa_method: 'totp',
|
||||
mfa_code: generateTotpCode(secret),
|
||||
})
|
||||
.expect(204)
|
||||
.execute();
|
||||
await registerWebAuthnCredential(
|
||||
harness,
|
||||
account.token,
|
||||
device,
|
||||
() => ({mfa_method: 'totp', mfa_code: generateTotpCode(secret)}),
|
||||
'MFA Passkey',
|
||||
);
|
||||
await setWebAuthnTwoFactor(harness, account.token, true, {
|
||||
mfa_method: 'totp',
|
||||
mfa_code: generateTotpCode(secret),
|
||||
});
|
||||
const loginResp = await loginUser(harness, {email: account.email, password: account.password});
|
||||
expect('mfa' in loginResp && loginResp.mfa).toBe(true);
|
||||
const loginMfaResp = loginResp as LoginMfaResponse;
|
||||
@@ -83,7 +76,7 @@ describe('WebAuthn MFA login', () => {
|
||||
.body({
|
||||
response: mfaAssertion,
|
||||
challenge: mfaOptions.challenge,
|
||||
ticket: (loginResp as LoginMfaResponse).ticket,
|
||||
ticket: loginMfaResp.ticket,
|
||||
})
|
||||
.execute();
|
||||
expect(webauthnMfaLogin.token).toBeTruthy();
|
||||
@@ -94,4 +87,39 @@ describe('WebAuthn MFA login', () => {
|
||||
.execute();
|
||||
expect(userInfo.id).toBe(account.userId);
|
||||
});
|
||||
it('issues a session token instead of an MFA ticket when passkey two-factor is left off', async () => {
|
||||
const account = await createTestAccount(harness);
|
||||
const device = createWebAuthnDevice();
|
||||
const secret = createTotpSecret();
|
||||
await createBuilder(harness, account.token)
|
||||
.post('/users/@me/mfa/totp/enable')
|
||||
.body({secret, code: generateTotpCode(secret), password: account.password})
|
||||
.execute();
|
||||
await registerWebAuthnCredential(
|
||||
harness,
|
||||
account.token,
|
||||
device,
|
||||
() => ({mfa_method: 'totp', mfa_code: generateTotpCode(secret)}),
|
||||
'MFA Passkey',
|
||||
);
|
||||
await createBuilder(harness, account.token)
|
||||
.post('/users/@me/mfa/totp/disable')
|
||||
.body({
|
||||
code: generateTotpCode(secret),
|
||||
mfa_method: 'totp',
|
||||
mfa_code: generateTotpCode(secret),
|
||||
})
|
||||
.expect(204)
|
||||
.execute();
|
||||
const loginResp = await loginUser(harness, {email: account.email, password: account.password});
|
||||
expect('mfa' in loginResp).toBe(false);
|
||||
const loginSuccessResp = loginResp as LoginSuccessResponse;
|
||||
expect(loginSuccessResp.token).toBeTruthy();
|
||||
const userInfo = await createBuilder<{
|
||||
id: string;
|
||||
}>(harness, loginSuccessResp.token)
|
||||
.get('/users/@me')
|
||||
.execute();
|
||||
expect(userInfo.id).toBe(account.userId);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -0,0 +1,124 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
import {
|
||||
createAuthHarness,
|
||||
createTestAccount,
|
||||
createTotpSecret,
|
||||
generateTotpCode,
|
||||
type LoginMfaResponse,
|
||||
type LoginSuccessResponse,
|
||||
loginUser,
|
||||
} from '@app/api/auth/tests/AuthTestUtils';
|
||||
import {
|
||||
createAuthenticationResponse,
|
||||
createWebAuthnDevice,
|
||||
loginWithDiscoverablePasskey,
|
||||
registerWebAuthnCredential,
|
||||
type WebAuthnAuthenticationOptions,
|
||||
} from '@app/api/auth/tests/WebAuthnTestUtils';
|
||||
import type {ApiTestHarness} from '@app/api/test/ApiTestHarness';
|
||||
import {HTTP_STATUS} from '@app/api/test/TestConstants';
|
||||
import {createBuilder, createBuilderWithoutAuth} from '@app/api/test/TestRequestBuilder';
|
||||
import {UserAuthenticatorTypes} from '@fluxer/constants/src/UserConstants';
|
||||
import {afterAll, beforeAll, beforeEach, describe, expect, it} from 'vitest';
|
||||
|
||||
describe('WebAuthn opt-in login', () => {
|
||||
let harness: ApiTestHarness;
|
||||
beforeAll(async () => {
|
||||
harness = await createAuthHarness();
|
||||
});
|
||||
beforeEach(async () => {
|
||||
await harness.reset();
|
||||
});
|
||||
afterAll(async () => {
|
||||
await harness?.shutdown();
|
||||
});
|
||||
it('does not offer webauthn at login to a TOTP user whose passkey is opted out', async () => {
|
||||
const account = await createTestAccount(harness);
|
||||
const device = createWebAuthnDevice();
|
||||
const secret = createTotpSecret();
|
||||
await createBuilder(harness, account.token)
|
||||
.post('/users/@me/mfa/totp/enable')
|
||||
.body({secret, code: generateTotpCode(secret), password: account.password})
|
||||
.execute();
|
||||
await registerWebAuthnCredential(harness, account.token, device, () => ({
|
||||
mfa_method: 'totp',
|
||||
mfa_code: generateTotpCode(secret),
|
||||
}));
|
||||
const login = (await loginUser(harness, {
|
||||
email: account.email,
|
||||
password: account.password,
|
||||
})) as LoginMfaResponse;
|
||||
expect(login.mfa).toBe(true);
|
||||
expect(login.totp).toBe(true);
|
||||
expect(login.webauthn).toBe(false);
|
||||
expect(login.allowed_methods).not.toContain('webauthn');
|
||||
expect(login.allowed_methods).toContain('totp');
|
||||
});
|
||||
it('rejects the WebAuthn MFA login route for a user who never turned passkey two-factor on', async () => {
|
||||
const account = await createTestAccount(harness);
|
||||
const device = createWebAuthnDevice();
|
||||
const secret = createTotpSecret();
|
||||
await createBuilder(harness, account.token)
|
||||
.post('/users/@me/mfa/totp/enable')
|
||||
.body({secret, code: generateTotpCode(secret), password: account.password})
|
||||
.execute();
|
||||
await registerWebAuthnCredential(harness, account.token, device, () => ({
|
||||
mfa_method: 'totp',
|
||||
mfa_code: generateTotpCode(secret),
|
||||
}));
|
||||
const login = (await loginUser(harness, {
|
||||
email: account.email,
|
||||
password: account.password,
|
||||
})) as LoginMfaResponse;
|
||||
const mfaOptions = await createBuilderWithoutAuth<WebAuthnAuthenticationOptions>(harness)
|
||||
.post('/auth/login/mfa/webauthn/authentication-options')
|
||||
.body({ticket: login.ticket})
|
||||
.execute();
|
||||
if (mfaOptions.rpId) {
|
||||
device.rpId = mfaOptions.rpId;
|
||||
}
|
||||
await createBuilderWithoutAuth(harness)
|
||||
.post('/auth/login/mfa/webauthn')
|
||||
.body({
|
||||
response: createAuthenticationResponse(device, mfaOptions),
|
||||
challenge: mfaOptions.challenge,
|
||||
ticket: login.ticket,
|
||||
})
|
||||
.expect(HTTP_STATUS.BAD_REQUEST, 'TWO_FACTOR_REQUIRED')
|
||||
.execute();
|
||||
const totpLogin = await createBuilderWithoutAuth<{
|
||||
token: string;
|
||||
}>(harness)
|
||||
.post('/auth/login/mfa/totp')
|
||||
.body({ticket: login.ticket, code: generateTotpCode(secret)})
|
||||
.execute();
|
||||
expect(totpLogin.token).toBeTruthy();
|
||||
});
|
||||
it('completes the passwordless journey for an account that never enrolled TOTP or the toggle', async () => {
|
||||
const account = await createTestAccount(harness);
|
||||
const device = createWebAuthnDevice();
|
||||
await registerWebAuthnCredential(harness, account.token, device, () => ({password: account.password}));
|
||||
const me = await createBuilder<{
|
||||
authenticator_types: Array<number>;
|
||||
mfa_enabled: boolean;
|
||||
}>(harness, account.token)
|
||||
.get('/users/@me')
|
||||
.execute();
|
||||
expect(me.authenticator_types).toEqual([]);
|
||||
expect(me.mfa_enabled).toBe(false);
|
||||
const passwordLogin = await loginUser(harness, {email: account.email, password: account.password});
|
||||
expect('mfa' in passwordLogin).toBe(false);
|
||||
expect((passwordLogin as LoginSuccessResponse).token).toBeTruthy();
|
||||
const passkeyToken = await loginWithDiscoverablePasskey(harness, device);
|
||||
expect(passkeyToken).toBeTruthy();
|
||||
const passkeyMe = await createBuilder<{
|
||||
id: string;
|
||||
authenticator_types: Array<number>;
|
||||
}>(harness, passkeyToken)
|
||||
.get('/users/@me')
|
||||
.execute();
|
||||
expect(passkeyMe.id).toBe(account.userId);
|
||||
expect(passkeyMe.authenticator_types).not.toContain(UserAuthenticatorTypes.WEBAUTHN);
|
||||
});
|
||||
});
|
||||
@@ -9,6 +9,8 @@ import {
|
||||
generateKeyPairSync,
|
||||
randomBytes,
|
||||
} from 'node:crypto';
|
||||
import type {ApiTestHarness} from '@app/api/test/ApiTestHarness';
|
||||
import {createBuilder, createBuilderWithoutAuth} from '@app/api/test/TestRequestBuilder';
|
||||
import {decode as base32Decode, encode as base32Encode} from 'hi-base32';
|
||||
|
||||
export interface WebAuthnDevice {
|
||||
@@ -54,6 +56,22 @@ export interface WebAuthnCredentialMetadata {
|
||||
name: string;
|
||||
}
|
||||
|
||||
export type SudoVerificationBody = Record<string, unknown>;
|
||||
|
||||
export type SudoVerificationBodyFactory = () => SudoVerificationBody;
|
||||
|
||||
export interface WebAuthnTwoFactorResult {
|
||||
user: {
|
||||
id: string;
|
||||
mfa_enabled: boolean;
|
||||
authenticator_types: Array<number>;
|
||||
};
|
||||
backup_codes: Array<{
|
||||
code: string;
|
||||
consumed: boolean;
|
||||
}> | null;
|
||||
}
|
||||
|
||||
interface AuthenticatorAttestationResponse {
|
||||
clientDataJSON: string;
|
||||
attestationObject: string;
|
||||
@@ -415,3 +433,80 @@ export function createAuthenticationResponseWithoutUV(
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export async function registerWebAuthnCredential(
|
||||
harness: ApiTestHarness,
|
||||
token: string,
|
||||
device: WebAuthnDevice,
|
||||
createSudoBody: SudoVerificationBodyFactory,
|
||||
name = 'Test Passkey',
|
||||
): Promise<void> {
|
||||
const options = await createBuilder<WebAuthnRegistrationOptions>(harness, token)
|
||||
.post('/users/@me/mfa/webauthn/credentials/registration-options')
|
||||
.body(createSudoBody())
|
||||
.execute();
|
||||
if (options.rp.id) {
|
||||
device.rpId = options.rp.id;
|
||||
}
|
||||
await createBuilder(harness, token)
|
||||
.post('/users/@me/mfa/webauthn/credentials')
|
||||
.body({
|
||||
response: createRegistrationResponse(device, options, name),
|
||||
challenge: options.challenge,
|
||||
name,
|
||||
...createSudoBody(),
|
||||
})
|
||||
.expect(204)
|
||||
.execute();
|
||||
}
|
||||
|
||||
export async function createSudoWebAuthnBody(
|
||||
harness: ApiTestHarness,
|
||||
token: string,
|
||||
device: WebAuthnDevice,
|
||||
): Promise<SudoVerificationBody> {
|
||||
const options = await createBuilder<WebAuthnAuthenticationOptions>(harness, token)
|
||||
.post('/users/@me/sudo/webauthn/authentication-options')
|
||||
.body(null)
|
||||
.execute();
|
||||
if (options.rpId) {
|
||||
device.rpId = options.rpId;
|
||||
}
|
||||
return {
|
||||
mfa_method: 'webauthn',
|
||||
webauthn_response: createAuthenticationResponse(device, options),
|
||||
webauthn_challenge: options.challenge,
|
||||
};
|
||||
}
|
||||
|
||||
export async function setWebAuthnTwoFactor(
|
||||
harness: ApiTestHarness,
|
||||
token: string,
|
||||
enabled: boolean,
|
||||
sudo: SudoVerificationBody,
|
||||
): Promise<WebAuthnTwoFactorResult> {
|
||||
return createBuilder<WebAuthnTwoFactorResult>(harness, token)
|
||||
.put('/users/@me/mfa/webauthn/two-factor')
|
||||
.body({enabled, ...sudo})
|
||||
.execute();
|
||||
}
|
||||
|
||||
export async function loginWithDiscoverablePasskey(harness: ApiTestHarness, device: WebAuthnDevice): Promise<string> {
|
||||
const options = await createBuilderWithoutAuth<WebAuthnAuthenticationOptions>(harness)
|
||||
.post('/auth/webauthn/authentication-options')
|
||||
.body(null)
|
||||
.execute();
|
||||
if (options.rpId) {
|
||||
device.rpId = options.rpId;
|
||||
}
|
||||
const login = await createBuilderWithoutAuth<{
|
||||
token: string;
|
||||
}>(harness)
|
||||
.post('/auth/webauthn/authenticate')
|
||||
.body({
|
||||
response: createAuthenticationResponse(device, options),
|
||||
challenge: options.challenge,
|
||||
})
|
||||
.execute();
|
||||
return login.token;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,217 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
import {
|
||||
createAuthHarness,
|
||||
createTestAccount,
|
||||
createTotpSecret,
|
||||
generateTotpCode,
|
||||
type LoginMfaResponse,
|
||||
loginUser,
|
||||
} from '@app/api/auth/tests/AuthTestUtils';
|
||||
import {
|
||||
createSudoWebAuthnBody,
|
||||
createWebAuthnDevice,
|
||||
registerWebAuthnCredential,
|
||||
type SudoVerificationBody,
|
||||
setWebAuthnTwoFactor,
|
||||
type WebAuthnCredentialMetadata,
|
||||
} from '@app/api/auth/tests/WebAuthnTestUtils';
|
||||
import type {ApiTestHarness} from '@app/api/test/ApiTestHarness';
|
||||
import {createBuilder, createBuilderWithoutAuth} from '@app/api/test/TestRequestBuilder';
|
||||
import {afterAll, beforeAll, beforeEach, describe, expect, it} from 'vitest';
|
||||
|
||||
interface BackupCode {
|
||||
code: string;
|
||||
consumed: boolean;
|
||||
}
|
||||
|
||||
async function readBackupCodes(
|
||||
harness: ApiTestHarness,
|
||||
token: string,
|
||||
sudo: SudoVerificationBody,
|
||||
): Promise<Array<BackupCode>> {
|
||||
const response = await createBuilder<{
|
||||
backup_codes: Array<BackupCode>;
|
||||
}>(harness, token)
|
||||
.post('/users/@me/mfa/backup-codes')
|
||||
.body({regenerate: false, ...sudo})
|
||||
.execute();
|
||||
return response.backup_codes;
|
||||
}
|
||||
|
||||
describe('WebAuthn two-factor backup codes', () => {
|
||||
let harness: ApiTestHarness;
|
||||
beforeAll(async () => {
|
||||
harness = await createAuthHarness();
|
||||
});
|
||||
beforeEach(async () => {
|
||||
await harness.reset();
|
||||
});
|
||||
afterAll(async () => {
|
||||
await harness?.shutdown();
|
||||
});
|
||||
it('mints backup codes when passkey two-factor is turned on for an account with no TOTP', async () => {
|
||||
const account = await createTestAccount(harness);
|
||||
const device = createWebAuthnDevice();
|
||||
await registerWebAuthnCredential(harness, account.token, device, () => ({password: account.password}));
|
||||
const enabled = await setWebAuthnTwoFactor(harness, account.token, true, {password: account.password});
|
||||
expect(enabled.backup_codes).not.toBeNull();
|
||||
expect(enabled.backup_codes!.length).toBeGreaterThan(0);
|
||||
expect(enabled.backup_codes!.every((backupCode) => !backupCode.consumed)).toBe(true);
|
||||
const sudoBody = await createSudoWebAuthnBody(harness, account.token, device);
|
||||
const stored = await readBackupCodes(harness, account.token, sudoBody);
|
||||
expect(stored.map((backupCode) => backupCode.code).sort()).toEqual(
|
||||
enabled.backup_codes!.map((backupCode) => backupCode.code).sort(),
|
||||
);
|
||||
});
|
||||
it('mints nothing when the account already holds backup codes from TOTP', async () => {
|
||||
const account = await createTestAccount(harness);
|
||||
const device = createWebAuthnDevice();
|
||||
const secret = createTotpSecret();
|
||||
await createBuilder(harness, account.token)
|
||||
.post('/users/@me/mfa/totp/enable')
|
||||
.body({secret, code: generateTotpCode(secret), password: account.password})
|
||||
.execute();
|
||||
await registerWebAuthnCredential(harness, account.token, device, () => ({
|
||||
mfa_method: 'totp',
|
||||
mfa_code: generateTotpCode(secret),
|
||||
}));
|
||||
const enabled = await setWebAuthnTwoFactor(harness, account.token, true, {
|
||||
mfa_method: 'totp',
|
||||
mfa_code: generateTotpCode(secret),
|
||||
});
|
||||
expect(enabled.backup_codes).toBeNull();
|
||||
});
|
||||
it('accepts a minted backup code at login when the passkey is unavailable', async () => {
|
||||
const account = await createTestAccount(harness);
|
||||
const device = createWebAuthnDevice();
|
||||
await registerWebAuthnCredential(harness, account.token, device, () => ({password: account.password}));
|
||||
const enabled = await setWebAuthnTwoFactor(harness, account.token, true, {password: account.password});
|
||||
const login = (await loginUser(harness, {
|
||||
email: account.email,
|
||||
password: account.password,
|
||||
})) as LoginMfaResponse;
|
||||
expect(login.mfa).toBe(true);
|
||||
expect(login.totp).toBe(false);
|
||||
expect(login.webauthn).toBe(true);
|
||||
expect(login.backup_codes).toBe(true);
|
||||
expect(login.allowed_methods).toContain('backup_codes');
|
||||
const backupLogin = await createBuilderWithoutAuth<{
|
||||
token: string;
|
||||
}>(harness)
|
||||
.post('/auth/login/mfa/totp')
|
||||
.body({ticket: login.ticket, code: enabled.backup_codes![0]!.code})
|
||||
.execute();
|
||||
expect(backupLogin.token).toBeTruthy();
|
||||
const me = await createBuilder<{
|
||||
id: string;
|
||||
}>(harness, backupLogin.token)
|
||||
.get('/users/@me')
|
||||
.execute();
|
||||
expect(me.id).toBe(account.userId);
|
||||
});
|
||||
it('keeps backup codes when TOTP is disabled while passkey two-factor stays on', async () => {
|
||||
const account = await createTestAccount(harness);
|
||||
const device = createWebAuthnDevice();
|
||||
const secret = createTotpSecret();
|
||||
await createBuilder(harness, account.token)
|
||||
.post('/users/@me/mfa/totp/enable')
|
||||
.body({secret, code: generateTotpCode(secret), password: account.password})
|
||||
.execute();
|
||||
await registerWebAuthnCredential(harness, account.token, device, () => ({
|
||||
mfa_method: 'totp',
|
||||
mfa_code: generateTotpCode(secret),
|
||||
}));
|
||||
await setWebAuthnTwoFactor(harness, account.token, true, {
|
||||
mfa_method: 'totp',
|
||||
mfa_code: generateTotpCode(secret),
|
||||
});
|
||||
await createBuilder(harness, account.token)
|
||||
.post('/users/@me/mfa/totp/disable')
|
||||
.body({
|
||||
code: generateTotpCode(secret),
|
||||
mfa_method: 'totp',
|
||||
mfa_code: generateTotpCode(secret),
|
||||
})
|
||||
.expect(204)
|
||||
.execute();
|
||||
const sudoBody = await createSudoWebAuthnBody(harness, account.token, device);
|
||||
const stored = await readBackupCodes(harness, account.token, sudoBody);
|
||||
expect(stored.length).toBeGreaterThan(0);
|
||||
});
|
||||
it('keeps backup codes when passkey two-factor is turned off while TOTP stays on', async () => {
|
||||
const account = await createTestAccount(harness);
|
||||
const device = createWebAuthnDevice();
|
||||
const secret = createTotpSecret();
|
||||
await createBuilder(harness, account.token)
|
||||
.post('/users/@me/mfa/totp/enable')
|
||||
.body({secret, code: generateTotpCode(secret), password: account.password})
|
||||
.execute();
|
||||
await registerWebAuthnCredential(harness, account.token, device, () => ({
|
||||
mfa_method: 'totp',
|
||||
mfa_code: generateTotpCode(secret),
|
||||
}));
|
||||
await setWebAuthnTwoFactor(harness, account.token, true, {
|
||||
mfa_method: 'totp',
|
||||
mfa_code: generateTotpCode(secret),
|
||||
});
|
||||
await setWebAuthnTwoFactor(harness, account.token, false, {
|
||||
mfa_method: 'totp',
|
||||
mfa_code: generateTotpCode(secret),
|
||||
});
|
||||
const stored = await readBackupCodes(harness, account.token, {
|
||||
mfa_method: 'totp',
|
||||
mfa_code: generateTotpCode(secret),
|
||||
});
|
||||
expect(stored.length).toBeGreaterThan(0);
|
||||
});
|
||||
it('clears backup codes only once no second factor remains', async () => {
|
||||
const account = await createTestAccount(harness);
|
||||
const device = createWebAuthnDevice();
|
||||
const secret = createTotpSecret();
|
||||
await createBuilder(harness, account.token)
|
||||
.post('/users/@me/mfa/totp/enable')
|
||||
.body({secret, code: generateTotpCode(secret), password: account.password})
|
||||
.execute();
|
||||
await registerWebAuthnCredential(harness, account.token, device, () => ({
|
||||
mfa_method: 'totp',
|
||||
mfa_code: generateTotpCode(secret),
|
||||
}));
|
||||
await setWebAuthnTwoFactor(harness, account.token, true, {
|
||||
mfa_method: 'totp',
|
||||
mfa_code: generateTotpCode(secret),
|
||||
});
|
||||
await setWebAuthnTwoFactor(harness, account.token, false, {
|
||||
mfa_method: 'totp',
|
||||
mfa_code: generateTotpCode(secret),
|
||||
});
|
||||
await createBuilder(harness, account.token)
|
||||
.post('/users/@me/mfa/totp/disable')
|
||||
.body({
|
||||
code: generateTotpCode(secret),
|
||||
mfa_method: 'totp',
|
||||
mfa_code: generateTotpCode(secret),
|
||||
})
|
||||
.expect(204)
|
||||
.execute();
|
||||
const stored = await readBackupCodes(harness, account.token, {password: account.password});
|
||||
expect(stored).toHaveLength(0);
|
||||
});
|
||||
it('clears backup codes when the last passkey is deleted and nothing else remains', async () => {
|
||||
const account = await createTestAccount(harness);
|
||||
const device = createWebAuthnDevice();
|
||||
await registerWebAuthnCredential(harness, account.token, device, () => ({password: account.password}));
|
||||
await setWebAuthnTwoFactor(harness, account.token, true, {password: account.password});
|
||||
const credentials = await createBuilder<Array<WebAuthnCredentialMetadata>>(harness, account.token)
|
||||
.get('/users/@me/mfa/webauthn/credentials')
|
||||
.execute();
|
||||
const sudoBody = await createSudoWebAuthnBody(harness, account.token, device);
|
||||
await createBuilder(harness, account.token)
|
||||
.delete(`/users/@me/mfa/webauthn/credentials/${credentials[0]!.id}`)
|
||||
.body(sudoBody)
|
||||
.expect(204)
|
||||
.execute();
|
||||
const stored = await readBackupCodes(harness, account.token, {password: account.password});
|
||||
expect(stored).toHaveLength(0);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,181 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
import {createAuthHarness, createTestAccount, loginUser, type TestAccount} from '@app/api/auth/tests/AuthTestUtils';
|
||||
import {
|
||||
createWebAuthnDevice,
|
||||
registerWebAuthnCredential,
|
||||
setWebAuthnTwoFactor,
|
||||
type WebAuthnDevice,
|
||||
} from '@app/api/auth/tests/WebAuthnTestUtils';
|
||||
import {Config} from '@app/api/Config';
|
||||
import type {ApiTestHarness} from '@app/api/test/ApiTestHarness';
|
||||
import {HTTP_STATUS} from '@app/api/test/TestConstants';
|
||||
import {createBuilder} from '@app/api/test/TestRequestBuilder';
|
||||
import {UserAuthenticatorTypes} from '@fluxer/constants/src/UserConstants';
|
||||
import {ValidationErrorCodes} from '@fluxer/constants/src/ValidationErrorCodes';
|
||||
import {afterAll, beforeAll, beforeEach, describe, expect, it} from 'vitest';
|
||||
|
||||
interface PrivateUserResponse {
|
||||
id: string;
|
||||
mfa_enabled: boolean;
|
||||
authenticator_types: Array<number>;
|
||||
}
|
||||
|
||||
interface SudoMfaMethodsResponse {
|
||||
totp: boolean;
|
||||
webauthn: boolean;
|
||||
backup_codes: boolean;
|
||||
has_mfa: boolean;
|
||||
}
|
||||
|
||||
interface SudoModeRequiredResponse {
|
||||
code: string;
|
||||
has_mfa?: boolean;
|
||||
methods?: {
|
||||
totp?: boolean;
|
||||
webauthn?: boolean;
|
||||
backup_codes?: boolean;
|
||||
};
|
||||
}
|
||||
|
||||
interface ValidationErrorBody {
|
||||
code: string;
|
||||
errors: Array<{path: string; code: string}>;
|
||||
}
|
||||
|
||||
interface BackupCode {
|
||||
code: string;
|
||||
consumed: boolean;
|
||||
}
|
||||
|
||||
async function withTotpReplayProtection<T>(run: () => Promise<T>): Promise<T> {
|
||||
const previous = Config.dev.testModeEnabled;
|
||||
Config.dev.testModeEnabled = false;
|
||||
try {
|
||||
return await run();
|
||||
} finally {
|
||||
Config.dev.testModeEnabled = previous;
|
||||
}
|
||||
}
|
||||
|
||||
async function createPasskeyOnlyTwoFactorAccount(
|
||||
harness: ApiTestHarness,
|
||||
): Promise<{account: TestAccount; device: WebAuthnDevice; backupCodes: Array<string>}> {
|
||||
const account = await createTestAccount(harness);
|
||||
const device = createWebAuthnDevice();
|
||||
await registerWebAuthnCredential(harness, account.token, device, () => ({password: account.password}));
|
||||
const enabled = await setWebAuthnTwoFactor(harness, account.token, true, {password: account.password});
|
||||
expect(enabled.user.authenticator_types).toEqual([UserAuthenticatorTypes.WEBAUTHN]);
|
||||
expect(enabled.backup_codes).not.toBeNull();
|
||||
return {account, device, backupCodes: enabled.backup_codes!.map((backupCode) => backupCode.code)};
|
||||
}
|
||||
|
||||
async function fetchMe(harness: ApiTestHarness, token: string): Promise<PrivateUserResponse> {
|
||||
return createBuilder<PrivateUserResponse>(harness, token).get('/users/@me').execute();
|
||||
}
|
||||
|
||||
describe('Sudo mode recovery for passkey two-factor accounts without TOTP', () => {
|
||||
let harness: ApiTestHarness;
|
||||
beforeAll(async () => {
|
||||
harness = await createAuthHarness();
|
||||
});
|
||||
beforeEach(async () => {
|
||||
await harness.reset();
|
||||
});
|
||||
afterAll(async () => {
|
||||
await harness?.shutdown();
|
||||
});
|
||||
it('turns passkey two-factor off with a backup code when the passkey cannot be used', async () => {
|
||||
const {account, backupCodes} = await createPasskeyOnlyTwoFactorAccount(harness);
|
||||
await createBuilder(harness, account.token)
|
||||
.put('/users/@me/mfa/webauthn/two-factor')
|
||||
.body({enabled: false, password: account.password})
|
||||
.expect(HTTP_STATUS.FORBIDDEN, 'SUDO_MODE_REQUIRED')
|
||||
.execute();
|
||||
await withTotpReplayProtection(async () => {
|
||||
const disabled = await createBuilder<{user: PrivateUserResponse}>(harness, account.token)
|
||||
.put('/users/@me/mfa/webauthn/two-factor')
|
||||
.body({enabled: false, mfa_method: 'totp', mfa_code: backupCodes[0]!})
|
||||
.expect(HTTP_STATUS.OK)
|
||||
.execute();
|
||||
expect(disabled.user.authenticator_types).toEqual([]);
|
||||
expect(disabled.user.mfa_enabled).toBe(false);
|
||||
});
|
||||
const me = await fetchMe(harness, account.token);
|
||||
expect(me.authenticator_types).toEqual([]);
|
||||
expect(me.mfa_enabled).toBe(false);
|
||||
const login = await loginUser(harness, {email: account.email, password: account.password});
|
||||
expect('mfa' in login).toBe(false);
|
||||
});
|
||||
it('advertises the backup code option in the sudo methods endpoint and the sudo mode challenge alike', async () => {
|
||||
const {account} = await createPasskeyOnlyTwoFactorAccount(harness);
|
||||
const methods = await createBuilder<SudoMfaMethodsResponse>(harness, account.token)
|
||||
.get('/users/@me/sudo/mfa-methods')
|
||||
.execute();
|
||||
expect(methods).toEqual({totp: false, webauthn: true, backup_codes: true, has_mfa: true});
|
||||
const challenge = await createBuilder<SudoModeRequiredResponse>(harness, account.token)
|
||||
.put('/users/@me/mfa/webauthn/two-factor')
|
||||
.body({enabled: false})
|
||||
.expect(HTTP_STATUS.FORBIDDEN, 'SUDO_MODE_REQUIRED')
|
||||
.execute();
|
||||
expect(challenge.has_mfa).toBe(methods.has_mfa);
|
||||
expect(challenge.methods).toEqual({
|
||||
totp: methods.totp,
|
||||
webauthn: methods.webauthn,
|
||||
backup_codes: methods.backup_codes,
|
||||
});
|
||||
});
|
||||
it('spends a backup code accepted as a sudo proof and refuses the same code afterwards', async () => {
|
||||
const {account, backupCodes} = await createPasskeyOnlyTwoFactorAccount(harness);
|
||||
const spent = backupCodes[0]!;
|
||||
await withTotpReplayProtection(async () => {
|
||||
const stored = await createBuilder<{backup_codes: Array<BackupCode>}>(harness, account.token)
|
||||
.post('/users/@me/mfa/backup-codes')
|
||||
.body({regenerate: false, mfa_method: 'totp', mfa_code: spent})
|
||||
.expect(HTTP_STATUS.OK)
|
||||
.execute();
|
||||
expect(stored.backup_codes.find((backupCode) => backupCode.code === spent)?.consumed).toBe(true);
|
||||
const error = await createBuilder<ValidationErrorBody>(harness, account.token)
|
||||
.put('/users/@me/mfa/webauthn/two-factor')
|
||||
.body({enabled: false, mfa_method: 'totp', mfa_code: spent})
|
||||
.expect(HTTP_STATUS.BAD_REQUEST, 'INVALID_FORM_BODY')
|
||||
.execute();
|
||||
expect(error.errors[0]?.path).toBe('mfa_code');
|
||||
expect(error.errors[0]?.code).toBe(ValidationErrorCodes.INVALID_MFA_CODE);
|
||||
});
|
||||
const me = await fetchMe(harness, account.token);
|
||||
expect(me.authenticator_types).toEqual([UserAuthenticatorTypes.WEBAUTHN]);
|
||||
});
|
||||
it('rejects a backup code that was never minted and leaves passkey two-factor on', async () => {
|
||||
const {account} = await createPasskeyOnlyTwoFactorAccount(harness);
|
||||
await withTotpReplayProtection(async () => {
|
||||
const error = await createBuilder<ValidationErrorBody>(harness, account.token)
|
||||
.put('/users/@me/mfa/webauthn/two-factor')
|
||||
.body({enabled: false, mfa_method: 'totp', mfa_code: 'aaaa-bbbb'})
|
||||
.expect(HTTP_STATUS.BAD_REQUEST, 'INVALID_FORM_BODY')
|
||||
.execute();
|
||||
expect(error.errors[0]?.path).toBe('mfa_code');
|
||||
expect(error.errors[0]?.code).toBe(ValidationErrorCodes.INVALID_MFA_CODE);
|
||||
});
|
||||
const me = await fetchMe(harness, account.token);
|
||||
expect(me.authenticator_types).toEqual([UserAuthenticatorTypes.WEBAUTHN]);
|
||||
});
|
||||
it('rejects a code-entry sudo proof for a passkey account that holds neither TOTP nor backup codes', async () => {
|
||||
const account = await createTestAccount(harness);
|
||||
const device = createWebAuthnDevice();
|
||||
await registerWebAuthnCredential(harness, account.token, device, () => ({password: account.password}));
|
||||
const methods = await createBuilder<SudoMfaMethodsResponse>(harness, account.token)
|
||||
.get('/users/@me/sudo/mfa-methods')
|
||||
.execute();
|
||||
expect(methods).toEqual({totp: false, webauthn: true, backup_codes: false, has_mfa: true});
|
||||
await withTotpReplayProtection(async () => {
|
||||
const error = await createBuilder<ValidationErrorBody>(harness, account.token)
|
||||
.post('/users/@me/disable')
|
||||
.body({mfa_method: 'totp', mfa_code: 'aaaa-bbbb'})
|
||||
.expect(HTTP_STATUS.BAD_REQUEST, 'INVALID_FORM_BODY')
|
||||
.execute();
|
||||
expect(error.errors[0]?.path).toBe('mfa_code');
|
||||
expect(error.errors[0]?.code).toBe(ValidationErrorCodes.INVALID_MFA_CODE);
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,126 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
import {createAuthHarness, createTestAccount} from '@app/api/auth/tests/AuthTestUtils';
|
||||
import {
|
||||
createSudoWebAuthnBody,
|
||||
createWebAuthnDevice,
|
||||
registerWebAuthnCredential,
|
||||
setWebAuthnTwoFactor,
|
||||
type WebAuthnCredentialMetadata,
|
||||
type WebAuthnTwoFactorResult,
|
||||
} from '@app/api/auth/tests/WebAuthnTestUtils';
|
||||
import type {ApiTestHarness} from '@app/api/test/ApiTestHarness';
|
||||
import {HTTP_STATUS} from '@app/api/test/TestConstants';
|
||||
import {createBuilder} from '@app/api/test/TestRequestBuilder';
|
||||
import {UserAuthenticatorTypes} from '@fluxer/constants/src/UserConstants';
|
||||
import {afterAll, beforeAll, beforeEach, describe, expect, it} from 'vitest';
|
||||
|
||||
interface PrivateUserResponse {
|
||||
id: string;
|
||||
mfa_enabled: boolean;
|
||||
authenticator_types: Array<number>;
|
||||
}
|
||||
|
||||
describe('WebAuthn two-factor toggle', () => {
|
||||
let harness: ApiTestHarness;
|
||||
beforeAll(async () => {
|
||||
harness = await createAuthHarness();
|
||||
});
|
||||
beforeEach(async () => {
|
||||
await harness.reset();
|
||||
});
|
||||
afterAll(async () => {
|
||||
await harness?.shutdown();
|
||||
});
|
||||
it('reports an empty authenticator types array for an account with no second factor', async () => {
|
||||
const account = await createTestAccount(harness);
|
||||
const me = await createBuilder<PrivateUserResponse>(harness, account.token).get('/users/@me').execute();
|
||||
expect(me.authenticator_types).toEqual([]);
|
||||
expect(me.mfa_enabled).toBe(false);
|
||||
});
|
||||
it('rejects enabling passkey two-factor when the account has no registered credential', async () => {
|
||||
const account = await createTestAccount(harness);
|
||||
await createBuilder(harness, account.token)
|
||||
.put('/users/@me/mfa/webauthn/two-factor')
|
||||
.body({enabled: true, password: account.password})
|
||||
.expect(HTTP_STATUS.BAD_REQUEST, 'NO_PASSKEYS_REGISTERED')
|
||||
.execute();
|
||||
const me = await createBuilder<PrivateUserResponse>(harness, account.token).get('/users/@me').execute();
|
||||
expect(me.authenticator_types).toEqual([]);
|
||||
});
|
||||
it('round trips enabling and disabling passkey two-factor', async () => {
|
||||
const account = await createTestAccount(harness);
|
||||
const device = createWebAuthnDevice();
|
||||
await registerWebAuthnCredential(harness, account.token, device, () => ({password: account.password}));
|
||||
const enabled = await setWebAuthnTwoFactor(harness, account.token, true, {password: account.password});
|
||||
expect(enabled.user.authenticator_types).toEqual([UserAuthenticatorTypes.WEBAUTHN]);
|
||||
expect(enabled.user.mfa_enabled).toBe(true);
|
||||
const afterEnable = await createBuilder<PrivateUserResponse>(harness, account.token).get('/users/@me').execute();
|
||||
expect(afterEnable.authenticator_types).toEqual([UserAuthenticatorTypes.WEBAUTHN]);
|
||||
const sudoBody = await createSudoWebAuthnBody(harness, account.token, device);
|
||||
const disabled = await setWebAuthnTwoFactor(harness, account.token, false, sudoBody);
|
||||
expect(disabled.user.authenticator_types).toEqual([]);
|
||||
expect(disabled.user.mfa_enabled).toBe(false);
|
||||
const afterDisable = await createBuilder<PrivateUserResponse>(harness, account.token).get('/users/@me').execute();
|
||||
expect(afterDisable.authenticator_types).toEqual([]);
|
||||
});
|
||||
it('refuses to disable passkey two-factor without a sudo proof', async () => {
|
||||
const account = await createTestAccount(harness);
|
||||
const device = createWebAuthnDevice();
|
||||
await registerWebAuthnCredential(harness, account.token, device, () => ({password: account.password}));
|
||||
await setWebAuthnTwoFactor(harness, account.token, true, {password: account.password});
|
||||
await createBuilder(harness, account.token)
|
||||
.put('/users/@me/mfa/webauthn/two-factor')
|
||||
.body({enabled: false})
|
||||
.expect(HTTP_STATUS.FORBIDDEN, 'SUDO_MODE_REQUIRED')
|
||||
.execute();
|
||||
await createBuilder(harness, account.token)
|
||||
.put('/users/@me/mfa/webauthn/two-factor')
|
||||
.body({enabled: false, password: account.password})
|
||||
.expect(HTTP_STATUS.FORBIDDEN, 'SUDO_MODE_REQUIRED')
|
||||
.execute();
|
||||
const me = await createBuilder<PrivateUserResponse>(harness, account.token).get('/users/@me').execute();
|
||||
expect(me.authenticator_types).toEqual([UserAuthenticatorTypes.WEBAUTHN]);
|
||||
});
|
||||
it('accepts a passkey assertion as the sudo proof for disabling passkey two-factor', async () => {
|
||||
const account = await createTestAccount(harness);
|
||||
const device = createWebAuthnDevice();
|
||||
await registerWebAuthnCredential(harness, account.token, device, () => ({password: account.password}));
|
||||
await setWebAuthnTwoFactor(harness, account.token, true, {password: account.password});
|
||||
const sudoBody = await createSudoWebAuthnBody(harness, account.token, device);
|
||||
const disabled = await createBuilder<WebAuthnTwoFactorResult>(harness, account.token)
|
||||
.put('/users/@me/mfa/webauthn/two-factor')
|
||||
.body({enabled: false, ...sudoBody})
|
||||
.execute();
|
||||
expect(disabled.user.authenticator_types).toEqual([]);
|
||||
});
|
||||
it('leaves the account untouched when the requested state already matches', async () => {
|
||||
const account = await createTestAccount(harness);
|
||||
const device = createWebAuthnDevice();
|
||||
await registerWebAuthnCredential(harness, account.token, device, () => ({password: account.password}));
|
||||
const firstEnable = await setWebAuthnTwoFactor(harness, account.token, true, {password: account.password});
|
||||
expect(firstEnable.backup_codes).not.toBeNull();
|
||||
const sudoBody = await createSudoWebAuthnBody(harness, account.token, device);
|
||||
const secondEnable = await setWebAuthnTwoFactor(harness, account.token, true, sudoBody);
|
||||
expect(secondEnable.backup_codes).toBeNull();
|
||||
expect(secondEnable.user.authenticator_types).toEqual([UserAuthenticatorTypes.WEBAUTHN]);
|
||||
});
|
||||
it('drops the passkey second factor when the last credential is deleted', async () => {
|
||||
const account = await createTestAccount(harness);
|
||||
const device = createWebAuthnDevice();
|
||||
await registerWebAuthnCredential(harness, account.token, device, () => ({password: account.password}));
|
||||
await setWebAuthnTwoFactor(harness, account.token, true, {password: account.password});
|
||||
const credentials = await createBuilder<Array<WebAuthnCredentialMetadata>>(harness, account.token)
|
||||
.get('/users/@me/mfa/webauthn/credentials')
|
||||
.execute();
|
||||
const sudoBody = await createSudoWebAuthnBody(harness, account.token, device);
|
||||
await createBuilder(harness, account.token)
|
||||
.delete(`/users/@me/mfa/webauthn/credentials/${credentials[0]!.id}`)
|
||||
.body(sudoBody)
|
||||
.expect(204)
|
||||
.execute();
|
||||
const me = await createBuilder<PrivateUserResponse>(harness, account.token).get('/users/@me').execute();
|
||||
expect(me.authenticator_types).toEqual([]);
|
||||
expect(me.mfa_enabled).toBe(false);
|
||||
});
|
||||
});
|
||||
@@ -150,7 +150,7 @@ function serializeGroupDMChannel(channel: Channel): ChannelResponse {
|
||||
return {
|
||||
...serializeBaseChannelFields(channel),
|
||||
...serializeMessageableFields(channel),
|
||||
name: channel.name ?? undefined,
|
||||
name: channel.name ?? null,
|
||||
icon: channel.iconHash ?? null,
|
||||
owner_id: channel.ownerId ? channel.ownerId.toString() : null,
|
||||
nicks: nicknameMap.size > 0 ? nicks : undefined,
|
||||
|
||||
@@ -24,6 +24,7 @@ import {deleteChannelMessageSearchDocuments} from '@app/api/search/MessageSearch
|
||||
import type {IUserRepository} from '@app/api/user/IUserRepository';
|
||||
import {serializeChannelForAudit} from '@app/api/utils/AuditSerializationUtils';
|
||||
import {applyProtectedOverwriteBits} from '@app/api/utils/featureUtils';
|
||||
import {overwriteGrantedBits} from '@app/api/utils/PermissionUtils';
|
||||
import type {VoiceAvailabilityService} from '@app/api/voice/VoiceAvailabilityService';
|
||||
import type {VoiceRegionAvailability} from '@app/api/voice/VoiceModel';
|
||||
import type {IWebhookRepository} from '@app/api/webhook/IWebhookRepository';
|
||||
@@ -208,25 +209,6 @@ export class ChannelOperationsService {
|
||||
userId,
|
||||
channelId: channel.id,
|
||||
});
|
||||
if (!isOwner) {
|
||||
for (const overwrite of data.permission_overwrites ?? []) {
|
||||
const allowPerms = (overwrite.allow ? BigInt(overwrite.allow) : 0n) & ALL_PERMISSIONS;
|
||||
if ((allowPerms & ~channelPermissions) !== 0n) {
|
||||
throw new MissingPermissionsError();
|
||||
}
|
||||
}
|
||||
const nextDeny = new Map<RoleID | UserID, bigint>();
|
||||
for (const overwrite of data.permission_overwrites ?? []) {
|
||||
const targetKey = overwrite.type === 0 ? createRoleID(overwrite.id) : createUserID(overwrite.id);
|
||||
nextDeny.set(targetKey, (overwrite.deny ? BigInt(overwrite.deny) : 0n) & ALL_PERMISSIONS);
|
||||
}
|
||||
for (const [targetId, existing] of previousPermissionOverwrites ?? []) {
|
||||
const removedDeny = existing.deny & ~(nextDeny.get(targetId) ?? 0n);
|
||||
if ((removedDeny & ~channelPermissions) !== 0n) {
|
||||
throw new MissingPermissionsError();
|
||||
}
|
||||
}
|
||||
}
|
||||
permissionOverwrites = new Map();
|
||||
for (const overwrite of data.permission_overwrites ?? []) {
|
||||
const targetId = overwrite.type === 0 ? createRoleID(overwrite.id) : createUserID(overwrite.id);
|
||||
@@ -251,6 +233,18 @@ export class ChannelOperationsService {
|
||||
}),
|
||||
);
|
||||
}
|
||||
if (!isOwner) {
|
||||
const targetIds = new Set([...(previousPermissionOverwrites?.keys() ?? []), ...permissionOverwrites.keys()]);
|
||||
for (const targetId of targetIds) {
|
||||
const grantedBits = overwriteGrantedBits(
|
||||
previousPermissionOverwrites?.get(targetId),
|
||||
permissionOverwrites.get(targetId),
|
||||
);
|
||||
if ((grantedBits & ~channelPermissions) !== 0n) {
|
||||
throw new MissingPermissionsError();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
const requestedParentId =
|
||||
data.parent_id !== undefined ? (data.parent_id ? createChannelID(data.parent_id) : null) : channel.parentId;
|
||||
@@ -646,9 +640,8 @@ export class ChannelOperationsService {
|
||||
const sanitizedAllow = protectedBits.allow;
|
||||
const sanitizedDeny = protectedBits.deny;
|
||||
const hasAdministrator = (userPermissions & Permissions.ADMINISTRATOR) !== 0n;
|
||||
if (!hasAdministrator && (sanitizedAllow & ~userPermissions) !== 0n) throw new MissingPermissionsError();
|
||||
const removedDeny = (existing?.deny ?? 0n) & ~sanitizedDeny;
|
||||
if (!hasAdministrator && (removedDeny & ~userPermissions) !== 0n) throw new MissingPermissionsError();
|
||||
const grantedBits = overwriteGrantedBits(existing, {allow: sanitizedAllow, deny: sanitizedDeny});
|
||||
if (!hasAdministrator && (grantedBits & ~userPermissions) !== 0n) throw new MissingPermissionsError();
|
||||
const previousPermissionOverwrites = channel.permissionOverwrites;
|
||||
const nextOverwrite = new ChannelPermissionOverwrite({
|
||||
type: params.overwrite.type,
|
||||
|
||||
@@ -294,6 +294,48 @@ describe('Channel Permission Overwrites', () => {
|
||||
expect(overwrite?.allow).toBe(Permissions.VIEW_CHANNEL.toString());
|
||||
expect(overwrite?.deny).toBe(Permissions.MANAGE_MESSAGES.toString());
|
||||
});
|
||||
test('should let an editor change an overwrite that already allows a permission they lack', async () => {
|
||||
const {owner, members, guild, systemChannel} = await setupTestGuildWithMembers(harness, 1);
|
||||
const manager = members[0];
|
||||
const managerRole = await createRole(harness, owner.token, guild.id, {
|
||||
name: 'Queue Manager',
|
||||
permissions: Permissions.MANAGE_ROLES.toString(),
|
||||
});
|
||||
const botRole = await createRole(harness, owner.token, guild.id, {name: 'Bot'});
|
||||
await addMemberRole(harness, owner.token, guild.id, manager.userId, managerRole.id);
|
||||
await createPermissionOverwrite(harness, owner.token, systemChannel.id, botRole.id, {
|
||||
type: 0,
|
||||
allow: Permissions.PIN_MESSAGES.toString(),
|
||||
deny: '0',
|
||||
});
|
||||
await createBuilder(harness, manager.token)
|
||||
.put(`/channels/${systemChannel.id}/permissions/${botRole.id}`)
|
||||
.body({
|
||||
type: 0,
|
||||
allow: (Permissions.PIN_MESSAGES | Permissions.SEND_MESSAGES).toString(),
|
||||
deny: '0',
|
||||
})
|
||||
.expect(HTTP_STATUS.NO_CONTENT)
|
||||
.execute();
|
||||
const updated = await getChannel(harness, owner.token, systemChannel.id);
|
||||
const botOverwrite = updated.permission_overwrites?.find((o) => o.id === botRole.id);
|
||||
expect(botOverwrite?.allow).toBe((Permissions.PIN_MESSAGES | Permissions.SEND_MESSAGES).toString());
|
||||
});
|
||||
test('should reject an editor granting a permission they lack', async () => {
|
||||
const {owner, members, guild, systemChannel} = await setupTestGuildWithMembers(harness, 1);
|
||||
const manager = members[0];
|
||||
const managerRole = await createRole(harness, owner.token, guild.id, {
|
||||
name: 'Queue Manager',
|
||||
permissions: Permissions.MANAGE_ROLES.toString(),
|
||||
});
|
||||
const botRole = await createRole(harness, owner.token, guild.id, {name: 'Bot'});
|
||||
await addMemberRole(harness, owner.token, guild.id, manager.userId, managerRole.id);
|
||||
await createBuilder(harness, manager.token)
|
||||
.put(`/channels/${systemChannel.id}/permissions/${botRole.id}`)
|
||||
.body({type: 0, allow: Permissions.PIN_MESSAGES.toString(), deny: '0'})
|
||||
.expect(HTTP_STATUS.FORBIDDEN)
|
||||
.execute();
|
||||
});
|
||||
test('should propagate category permission patches only to children that were synced when the category changed', async () => {
|
||||
const {owner, guild} = await setupTestGuildWithMembers(harness, 0);
|
||||
const targetRole = await createRole(harness, owner.token, guild.id, {name: 'Readers'});
|
||||
|
||||
@@ -0,0 +1,62 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
import {createTestAccount} from '@app/api/auth/tests/AuthTestUtils';
|
||||
import {createFriendship, createGroupDmChannel, getChannel} from '@app/api/channel/tests/ChannelTestUtils';
|
||||
import {ensureSessionStarted} from '@app/api/message/tests/MessageTestUtils';
|
||||
import {type ApiTestHarness, createApiTestHarness} from '@app/api/test/ApiTestHarness';
|
||||
import {NoopGatewayService} from '@app/api/test/NoopGatewayService';
|
||||
import {HTTP_STATUS} from '@app/api/test/TestConstants';
|
||||
import {createBuilder} from '@app/api/test/TestRequestBuilder';
|
||||
import type {ChannelResponse} from '@fluxer/schema/src/domains/channel/ChannelSchemas';
|
||||
import {afterAll, beforeAll, beforeEach, describe, expect, it, vi} from 'vitest';
|
||||
|
||||
describe('Group DM name clear', () => {
|
||||
let harness: ApiTestHarness;
|
||||
beforeAll(async () => {
|
||||
harness = await createApiTestHarness();
|
||||
});
|
||||
beforeEach(async () => {
|
||||
await harness.reset();
|
||||
});
|
||||
afterAll(async () => {
|
||||
await harness?.shutdown();
|
||||
});
|
||||
it.each([
|
||||
['an empty string', ''],
|
||||
['null', null],
|
||||
])('sends a null name to every recipient when cleared with %s', async (_label, clearedName) => {
|
||||
const user1 = await createTestAccount(harness);
|
||||
const user2 = await createTestAccount(harness);
|
||||
const user3 = await createTestAccount(harness);
|
||||
await ensureSessionStarted(harness, user1.token);
|
||||
await ensureSessionStarted(harness, user2.token);
|
||||
await ensureSessionStarted(harness, user3.token);
|
||||
await createFriendship(harness, user1, user2);
|
||||
await createFriendship(harness, user1, user3);
|
||||
const groupDm = await createGroupDmChannel(harness, user1.token, [user2.userId, user3.userId]);
|
||||
await createBuilder<ChannelResponse>(harness, user1.token)
|
||||
.patch(`/channels/${groupDm.id}`)
|
||||
.body({name: 'Weekend plans'})
|
||||
.expect(HTTP_STATUS.OK)
|
||||
.execute();
|
||||
const dispatchSpy = vi.spyOn(NoopGatewayService.prototype, 'dispatchPresence');
|
||||
try {
|
||||
const cleared = await createBuilder<ChannelResponse>(harness, user1.token)
|
||||
.patch(`/channels/${groupDm.id}`)
|
||||
.body({name: clearedName})
|
||||
.expect(HTTP_STATUS.OK)
|
||||
.execute();
|
||||
expect(cleared).toHaveProperty('name', null);
|
||||
const channelUpdates = dispatchSpy.mock.calls.filter(([params]) => params.event === 'CHANNEL_UPDATE');
|
||||
expect(channelUpdates.map(([params]) => params.userId.toString()).sort()).toEqual(
|
||||
[user1.userId, user2.userId, user3.userId].sort(),
|
||||
);
|
||||
for (const [params] of channelUpdates) {
|
||||
expect(params.data).toHaveProperty('name', null);
|
||||
}
|
||||
} finally {
|
||||
dispatchSpy.mockRestore();
|
||||
}
|
||||
expect(await getChannel(harness, user2.token, groupDm.id)).toHaveProperty('name', null);
|
||||
});
|
||||
});
|
||||
@@ -143,8 +143,6 @@ export interface APIConfig {
|
||||
donationProxyKey: string;
|
||||
};
|
||||
hosts: {
|
||||
invite: string;
|
||||
gift: string;
|
||||
marketing: string;
|
||||
unfurlIgnored: Array<string>;
|
||||
};
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
import type {BannedFileShaRow} from '@app/api/database/types/AdminArchiveTypes';
|
||||
|
||||
export const BANNED_URLS_REFRESH_CHANNEL = 'banned_urls_refresh';
|
||||
export const BANNED_URL_DOMAINS_REFRESH_CHANNEL = 'banned_url_domains_refresh';
|
||||
export const BANNED_FILE_SHAS_REFRESH_CHANNEL = 'banned_file_shas_refresh';
|
||||
@@ -23,3 +25,7 @@ export const ContentBlocklistCategory = {
|
||||
GIFCT: 'gifct',
|
||||
STOP_NCII: 'stop_ncii',
|
||||
} as const;
|
||||
|
||||
export function isBlocklistFeedFileSha(row: Pick<BannedFileShaRow, 'category' | 'added_by'>): boolean {
|
||||
return row.added_by == null && row.category === ContentBlocklistCategory.MALWARE_BAZAAR;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,89 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import {fileURLToPath} from 'node:url';
|
||||
import {DEFAULT_TTL_TABLES} from '@app/api/database/PostgresKvDefaultTtlExpiry';
|
||||
import * as DonationTables from '@app/api/donation/DonationTables';
|
||||
import * as Tables from '@app/api/Tables';
|
||||
import {IPINFO_CACHE_TTL_SECONDS, IPINFO_REQUEST_AUDIT_TTL_SECONDS} from '@pkgs/geoip/src/PostgresIpInfoKv';
|
||||
import {describe, expect, it} from 'vitest';
|
||||
|
||||
const THIS_DIR = path.dirname(fileURLToPath(import.meta.url));
|
||||
const REPO_ROOT = path.resolve(THIS_DIR, '../../../..');
|
||||
|
||||
interface SchemaTable {
|
||||
name: string;
|
||||
options: string;
|
||||
}
|
||||
|
||||
const SCHEMA = JSON.parse(fs.readFileSync(path.join(REPO_ROOT, 'tools/dev/cassandra_target_schema.json'), 'utf8')) as {
|
||||
tables: Array<SchemaTable>;
|
||||
};
|
||||
|
||||
const SCHEMA_DEFAULTS = new Map<string, number>(
|
||||
SCHEMA.tables.flatMap((table): Array<[string, number]> => {
|
||||
const match = /default_time_to_live = (\d+)/.exec(table.options);
|
||||
return match ? [[table.name, Number(match[1])]] : [];
|
||||
}),
|
||||
);
|
||||
|
||||
const DSL_TABLES = [...Object.values(Tables), ...Object.values(DonationTables)];
|
||||
const DSL_NAMES = new Set<string>(DSL_TABLES.map((table) => table.name));
|
||||
|
||||
const NON_DSL_DEFAULTS: Record<string, number | null> = {
|
||||
ipinfo_cache: IPINFO_CACHE_TTL_SECONDS,
|
||||
ipinfo_requests_by_hour: IPINFO_REQUEST_AUDIT_TTL_SECONDS,
|
||||
billing_webhook_events: null,
|
||||
forensic_identifier_by_key_day: null,
|
||||
forensic_identifier_by_request: null,
|
||||
forensic_request_meta_by_actor_day: null,
|
||||
forensic_request_meta_by_id: null,
|
||||
forensic_request_meta_by_route_day_shard: null,
|
||||
forensic_resource_exposure_by_request: null,
|
||||
forensic_resource_exposure_by_route_day_shard: null,
|
||||
forensic_resource_exposure_by_subject_day: null,
|
||||
};
|
||||
|
||||
const OWN_EXPIRY_PASS = new Set(['jobs_by_id', 'jobs_by_day_bucket']);
|
||||
|
||||
function schemaDefault(name: string): number {
|
||||
return SCHEMA_DEFAULTS.get(name) ?? 0;
|
||||
}
|
||||
|
||||
function byName(left: {name: string}, right: {name: string}): number {
|
||||
return left.name.localeCompare(right.name);
|
||||
}
|
||||
|
||||
describe('Cassandra default TTL parity', () => {
|
||||
it('declares every Cassandra default TTL on the matching table', () => {
|
||||
const mismatches = DSL_TABLES.flatMap((table) => {
|
||||
const declared = table.defaultTtlSeconds ?? 0;
|
||||
return declared === schemaDefault(table.name)
|
||||
? []
|
||||
: [{table: table.name, declared, schema: schemaDefault(table.name)}];
|
||||
});
|
||||
expect(mismatches).toEqual([]);
|
||||
});
|
||||
|
||||
it('declares a writer or no writer for every other table with a default', () => {
|
||||
const undeclared = [...SCHEMA_DEFAULTS]
|
||||
.filter(([name, ttl]) => ttl > 0 && !DSL_NAMES.has(name) && !Object.hasOwn(NON_DSL_DEFAULTS, name))
|
||||
.map(([name]) => name);
|
||||
expect(undeclared).toEqual([]);
|
||||
const stale = Object.keys(NON_DSL_DEFAULTS).filter((name) => schemaDefault(name) === 0 || DSL_NAMES.has(name));
|
||||
expect(stale).toEqual([]);
|
||||
const mismatched = Object.entries(NON_DSL_DEFAULTS)
|
||||
.filter(([name, ttl]) => ttl !== null && ttl !== schemaDefault(name))
|
||||
.map(([name]) => name);
|
||||
expect(mismatched).toEqual([]);
|
||||
});
|
||||
|
||||
it('the Postgres expiry pass covers every table with a default except the job ledger', () => {
|
||||
const expected = [...SCHEMA_DEFAULTS]
|
||||
.filter(([name, ttl]) => ttl > 0 && NON_DSL_DEFAULTS[name] !== null && !OWN_EXPIRY_PASS.has(name))
|
||||
.map(([name, ttl]) => ({name, defaultTtlSeconds: ttl}))
|
||||
.sort(byName);
|
||||
expect([...DEFAULT_TTL_TABLES].sort(byName)).toEqual(expected);
|
||||
});
|
||||
});
|
||||
@@ -12,6 +12,7 @@ interface TableMetadata {
|
||||
columns: ReadonlyArray<string>;
|
||||
primaryKey: ReadonlyArray<string>;
|
||||
partitionKey: ReadonlyArray<string>;
|
||||
defaultTtlSeconds?: number;
|
||||
}
|
||||
|
||||
const kvMetaRegistry = new Map<string, KvQueryMeta<Record<string, unknown>>>();
|
||||
@@ -24,6 +25,7 @@ export function registerTableSpec<Row extends object>(tableSpec: KvTableSpec<Row
|
||||
columns: tableSpec.columns as ReadonlyArray<string>,
|
||||
primaryKey: tableSpec.primaryKey as ReadonlyArray<string>,
|
||||
partitionKey: tableSpec.partitionKey as ReadonlyArray<string>,
|
||||
defaultTtlSeconds: tableSpec.defaultTtlSeconds,
|
||||
};
|
||||
tableRegistry.set(tableSpec.name, metadata);
|
||||
}
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
import {getTableMetadata} from '@app/api/database/CassandraMetaRegistry';
|
||||
import {defineTable} from '@app/api/database/CassandraTableDsl';
|
||||
import {Db, type PreparedQuery} from '@app/api/database/CassandraTypes';
|
||||
import {describe, expect, it} from 'vitest';
|
||||
@@ -76,3 +77,41 @@ describe('CassandraTableDsl select templates', () => {
|
||||
expect(longQuery.cql).not.toContain('LIMIT 20');
|
||||
});
|
||||
});
|
||||
|
||||
describe('CassandraTableDsl default TTL', () => {
|
||||
it('keeps the CQL of a table with a default TTL free of USING TTL', () => {
|
||||
const DefaultTtlRows = defineTable<TtlHelperTestRow, 'id'>({
|
||||
name: 'default_ttl_dsl_rows',
|
||||
columns: ['id', 'value'],
|
||||
primaryKey: ['id'],
|
||||
defaultTtlSeconds: 600,
|
||||
});
|
||||
expect(DefaultTtlRows.defaultTtlSeconds).toBe(600);
|
||||
const queries = [
|
||||
DefaultTtlRows.insert({id: 'insert', value: 'a'}),
|
||||
DefaultTtlRows.upsertAll({id: 'upsert', value: 'b'}),
|
||||
DefaultTtlRows.patchByPk({id: 'patch'}, {value: Db.set('c')}),
|
||||
];
|
||||
for (const query of queries) {
|
||||
expect(query.cql).not.toContain('USING TTL');
|
||||
expect(query.kvMeta?.table.defaultTtlSeconds).toBe(600);
|
||||
}
|
||||
expect(getTableMetadata('default_ttl_dsl_rows')?.defaultTtlSeconds).toBe(600);
|
||||
expect(TtlHelperTestRows.defaultTtlSeconds).toBeUndefined();
|
||||
expect(getTableMetadata('ttl_helper_test_rows')?.defaultTtlSeconds).toBeUndefined();
|
||||
});
|
||||
|
||||
it('rejects a default TTL of zero, a fraction or past the maximum', () => {
|
||||
for (const defaultTtlSeconds of [0, 1.5, 630_720_001]) {
|
||||
expect(() =>
|
||||
defineTable<TtlHelperTestRow, 'id'>({
|
||||
name: 'default_ttl_dsl_rejected_rows',
|
||||
columns: ['id', 'value'],
|
||||
primaryKey: ['id'],
|
||||
defaultTtlSeconds,
|
||||
}),
|
||||
).toThrow();
|
||||
}
|
||||
expect(getTableMetadata('default_ttl_dsl_rejected_rows')).toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
@@ -83,6 +83,7 @@ export function defineTable<Row extends object, PK extends ColumnName<Row>, Part
|
||||
columns: ReadonlyArray<ColumnName<Row>>;
|
||||
primaryKey: ReadonlyArray<PK>;
|
||||
partitionKey?: ReadonlyArray<PartKey>;
|
||||
defaultTtlSeconds?: number;
|
||||
}): Table<Row, PK, PartKey> {
|
||||
const columns = [...def.columns];
|
||||
const pk = [...def.primaryKey];
|
||||
@@ -91,11 +92,15 @@ export function defineTable<Row extends object, PK extends ColumnName<Row>, Part
|
||||
for (const c of columns) assertCqlIdentifier(c as string);
|
||||
for (const k of pk) assertCqlIdentifier(k as string);
|
||||
for (const k of partitionKey) assertCqlIdentifier(k as string);
|
||||
if (def.defaultTtlSeconds !== undefined && validateTtlSeconds(def.defaultTtlSeconds) === 0) {
|
||||
throw new Error(`Table "${def.name}" needs a positive default TTL`);
|
||||
}
|
||||
const tableSpec: KvTableSpec<Row> = {
|
||||
name: def.name,
|
||||
columns,
|
||||
primaryKey: pk as ReadonlyArray<ColumnName<Row>>,
|
||||
partitionKey: partitionKey as ReadonlyArray<ColumnName<Row>>,
|
||||
defaultTtlSeconds: def.defaultTtlSeconds,
|
||||
};
|
||||
registerTableSpec(tableSpec);
|
||||
const nonPkColumns = columns.filter((c) => !pk.includes(c as PK)) as Array<Exclude<ColumnName<Row>, PK>>;
|
||||
@@ -685,6 +690,7 @@ WHERE ${pk.map((k) => `${k} = :${k}`).join(' AND ')};
|
||||
columns: def.columns,
|
||||
primaryKey: def.primaryKey,
|
||||
partitionKey: partitionKey,
|
||||
defaultTtlSeconds: def.defaultTtlSeconds,
|
||||
selectCql,
|
||||
select,
|
||||
updateAllCql() {
|
||||
|
||||
@@ -56,6 +56,7 @@ export interface KvTableSpec<Row extends object = Record<string, unknown>> {
|
||||
columns: ReadonlyArray<ColumnName<Row>>;
|
||||
primaryKey: ReadonlyArray<ColumnName<Row>>;
|
||||
partitionKey: ReadonlyArray<ColumnName<Row>>;
|
||||
defaultTtlSeconds?: number;
|
||||
}
|
||||
|
||||
export interface KvColumnParam<Row extends object = Record<string, unknown>> {
|
||||
@@ -190,6 +191,7 @@ export interface Table<Row extends object, PK extends ColumnName<Row>, PartKey e
|
||||
columns: ReadonlyArray<ColumnName<Row>>;
|
||||
primaryKey: ReadonlyArray<PK>;
|
||||
partitionKey: ReadonlyArray<PartKey>;
|
||||
defaultTtlSeconds: number | undefined;
|
||||
selectCql(opts?: {
|
||||
columns?: ReadonlyArray<ColumnName<Row>>;
|
||||
where?: WhereExpr<Row> | ReadonlyArray<WhereExpr<Row>>;
|
||||
|
||||
@@ -0,0 +1,490 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
import {spawnSync} from 'node:child_process';
|
||||
import {createServer} from 'node:net';
|
||||
import {defineTable} from '@app/api/database/CassandraTableDsl';
|
||||
import {Db} from '@app/api/database/CassandraTypes';
|
||||
import {
|
||||
DEFAULT_TTL_EXPIRY_RESUME,
|
||||
DEFAULT_TTL_TABLES,
|
||||
expireLegacyDefaultTtlRows,
|
||||
} from '@app/api/database/PostgresKvDefaultTtlExpiry';
|
||||
import {
|
||||
ensurePostgresKvSchema,
|
||||
PostgresKvQueryExecutor,
|
||||
pruneExpiredPostgresKvRows,
|
||||
} from '@app/api/database/PostgresKvQueryExecutor';
|
||||
import {startDockerContainer} from '@app/api/test/DockerTestContainer';
|
||||
import {
|
||||
getDefaultPostgresClient,
|
||||
type IPostgresClient,
|
||||
initPostgres,
|
||||
shutdownPostgres,
|
||||
} from '@pkgs/postgres/src/Client';
|
||||
import {afterAll, beforeAll, beforeEach, describe, expect, it} from 'vitest';
|
||||
|
||||
const KV_TABLE = 'kv_default_ttl';
|
||||
const CONTAINER = `fluxer-kvttl-${process.pid.toString(36)}-${Date.now().toString(36)}`;
|
||||
const dockerAvailable = spawnSync('docker', ['version'], {stdio: 'ignore'}).status === 0;
|
||||
const DEFAULT_TTL_SECONDS = 600;
|
||||
|
||||
interface ProbeRow {
|
||||
id: string;
|
||||
value: string | null;
|
||||
note: string | null;
|
||||
}
|
||||
|
||||
interface OwnedProbeRow {
|
||||
owner: string;
|
||||
id: string;
|
||||
value: string | null;
|
||||
}
|
||||
|
||||
const DefaultTtlProbe = defineTable<ProbeRow, 'id'>({
|
||||
name: 'default_ttl_probe',
|
||||
columns: ['id', 'value', 'note'],
|
||||
primaryKey: ['id'],
|
||||
defaultTtlSeconds: DEFAULT_TTL_SECONDS,
|
||||
});
|
||||
|
||||
const DefaultTtlProbeRows = defineTable<OwnedProbeRow, 'owner' | 'id', 'owner'>({
|
||||
name: 'default_ttl_probe_rows',
|
||||
columns: ['owner', 'id', 'value'],
|
||||
primaryKey: ['owner', 'id'],
|
||||
partitionKey: ['owner'],
|
||||
defaultTtlSeconds: DEFAULT_TTL_SECONDS,
|
||||
});
|
||||
|
||||
const NoTtlProbe = defineTable<ProbeRow, 'id'>({
|
||||
name: 'no_ttl_probe',
|
||||
columns: ['id', 'value', 'note'],
|
||||
primaryKey: ['id'],
|
||||
});
|
||||
|
||||
async function sleep(ms: number): Promise<void> {
|
||||
await new Promise((resolve) => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
async function freePort(): Promise<number> {
|
||||
return new Promise((resolve, reject) => {
|
||||
const server = createServer();
|
||||
server.on('error', reject);
|
||||
server.listen(0, '127.0.0.1', () => {
|
||||
const address = server.address();
|
||||
if (typeof address === 'string' || address === null) {
|
||||
reject(new Error('no port'));
|
||||
return;
|
||||
}
|
||||
const port = address.port;
|
||||
server.close(() => resolve(port));
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
function expectExpiresIn(value: Date | number | null, ttlSeconds: number): void {
|
||||
expect(value).toBeInstanceOf(Date);
|
||||
const remainingSeconds = ((value as Date).getTime() - Date.now()) / 1000;
|
||||
expect(remainingSeconds).toBeGreaterThan(ttlSeconds - 60);
|
||||
expect(remainingSeconds).toBeLessThanOrEqual(ttlSeconds);
|
||||
}
|
||||
|
||||
describe.skipIf(!dockerAvailable)('Postgres KV default TTL', () => {
|
||||
let raw: IPostgresClient;
|
||||
let executor: PostgresKvQueryExecutor;
|
||||
|
||||
async function stored(table: string, id: string): Promise<{expires_at: Date | number | null; row_data: object}> {
|
||||
const result = await raw.query<{expires_at: Date | number | null; row_data: object}>(
|
||||
`SELECT expires_at, row_data FROM ${KV_TABLE} WHERE table_name = $1 AND row_data ->> 'id' = $2`,
|
||||
[table, id],
|
||||
);
|
||||
expect(result.rows).toHaveLength(1);
|
||||
return result.rows[0]!;
|
||||
}
|
||||
|
||||
async function expiresAt(table: string, id: string): Promise<Date | number | null> {
|
||||
return (await stored(table, id)).expires_at;
|
||||
}
|
||||
|
||||
async function neverExpires(table: string, id: string): Promise<boolean> {
|
||||
const result = await raw.query<{forever: boolean}>(
|
||||
`SELECT expires_at = 'infinity'::timestamptz AS forever FROM ${KV_TABLE} WHERE table_name = $1 AND row_data ->> 'id' = $2`,
|
||||
[table, id],
|
||||
);
|
||||
return result.rows[0]?.forever === true;
|
||||
}
|
||||
|
||||
async function setExpiry(table: string, id: string, expression: string): Promise<void> {
|
||||
await raw.query(
|
||||
`UPDATE ${KV_TABLE} SET expires_at = ${expression} WHERE table_name = $1 AND row_data ->> 'id' = $2`,
|
||||
[table, id],
|
||||
);
|
||||
}
|
||||
|
||||
async function seed(table: string, key: string, age: string, expires: Date | string | null = null): Promise<string> {
|
||||
const result = await raw.query<{updated_at: string}>(
|
||||
`INSERT INTO ${KV_TABLE} (table_name, partition_key, row_key, row_data, expires_at, updated_at)
|
||||
VALUES ($1, $2, $2, '{}'::jsonb, $3::timestamptz, now() - $4::interval)
|
||||
RETURNING updated_at::text`,
|
||||
[table, key, expires, age],
|
||||
);
|
||||
return result.rows[0]!.updated_at;
|
||||
}
|
||||
|
||||
async function remaining(): Promise<Array<{table_name: string; row_key: string}>> {
|
||||
const result = await raw.query<{table_name: string; row_key: string}>(
|
||||
`SELECT table_name, row_key FROM ${KV_TABLE} WHERE table_name <> '__fluxer_schema_migrations' ORDER BY table_name, row_key`,
|
||||
);
|
||||
return result.rows;
|
||||
}
|
||||
|
||||
async function ageMarker(): Promise<void> {
|
||||
await raw.query(
|
||||
`UPDATE ${KV_TABLE} SET row_data = jsonb_build_object('applied_at', now() - interval '2 days') WHERE table_name = '__fluxer_schema_migrations' AND row_key = 'default_ttl_expiry_v1'`,
|
||||
);
|
||||
}
|
||||
|
||||
async function resumePoint(): Promise<object | null> {
|
||||
const result = await raw.query<{row_data: object}>(
|
||||
`SELECT row_data FROM ${KV_TABLE} WHERE table_name = '__fluxer_schema_migrations' AND row_key = $1`,
|
||||
[DEFAULT_TTL_EXPIRY_RESUME],
|
||||
);
|
||||
return result.rows[0]?.row_data ?? null;
|
||||
}
|
||||
|
||||
async function markerCount(): Promise<number> {
|
||||
const result = await raw.query<{n: number}>(
|
||||
`SELECT count(*)::int AS n FROM ${KV_TABLE} WHERE table_name = '__fluxer_schema_migrations' AND row_key = 'default_ttl_expiry_v1'`,
|
||||
);
|
||||
return result.rows[0]!.n;
|
||||
}
|
||||
|
||||
beforeAll(async () => {
|
||||
const port = await freePort();
|
||||
startDockerContainer([
|
||||
'run',
|
||||
'-d',
|
||||
'--name',
|
||||
CONTAINER,
|
||||
'-e',
|
||||
'POSTGRES_USER=fluxer',
|
||||
'-e',
|
||||
'POSTGRES_PASSWORD=fluxer',
|
||||
'-e',
|
||||
'POSTGRES_DB=fluxer',
|
||||
'-p',
|
||||
`127.0.0.1:${port}:5432`,
|
||||
'postgres:16-alpine',
|
||||
'-c',
|
||||
'fsync=off',
|
||||
]);
|
||||
let ready = false;
|
||||
for (let attempt = 0; attempt < 180 && !ready; attempt += 1) {
|
||||
await sleep(500);
|
||||
const probe = spawnSync('docker', ['exec', CONTAINER, 'pg_isready', '-U', 'fluxer', '-d', 'fluxer'], {
|
||||
stdio: 'ignore',
|
||||
});
|
||||
if (probe.status !== 0) continue;
|
||||
try {
|
||||
await initPostgres({
|
||||
url: `postgres://fluxer:[email protected]:${port}/fluxer`,
|
||||
maxConnections: 4,
|
||||
kvTable: KV_TABLE,
|
||||
});
|
||||
await getDefaultPostgresClient().query('SELECT 1');
|
||||
ready = true;
|
||||
} catch {
|
||||
await shutdownPostgres().catch(() => {});
|
||||
}
|
||||
}
|
||||
if (!ready) throw new Error('postgres never came up');
|
||||
raw = getDefaultPostgresClient();
|
||||
await ensurePostgresKvSchema(raw);
|
||||
executor = new PostgresKvQueryExecutor(raw);
|
||||
}, 900_000);
|
||||
|
||||
beforeEach(async () => {
|
||||
await raw.query(`DELETE FROM ${KV_TABLE}`);
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
await shutdownPostgres().catch(() => {});
|
||||
spawnSync('docker', ['rm', '-f', CONTAINER], {stdio: 'ignore'});
|
||||
});
|
||||
|
||||
it('gives every full-row write without a TTL the table default', async () => {
|
||||
await executor.executeQuery(DefaultTtlProbe.insert({id: 'insert', value: 'a', note: null}));
|
||||
await executor.executeQuery(DefaultTtlProbe.upsertAll({id: 'upsert', value: 'b', note: 'n'}));
|
||||
expect(
|
||||
await executor.executeQuery(DefaultTtlProbe.insertIfNotExists({id: 'claimed', value: 'c', note: null})),
|
||||
).toEqual([{'[applied]': true}]);
|
||||
expect(
|
||||
await executor.executeQuery(
|
||||
DefaultTtlProbeRows.conditionalBatch([{action: 'insert', row: {owner: 'o', id: 'batched', value: 'd'}}]),
|
||||
),
|
||||
).toEqual([{'[applied]': true}]);
|
||||
|
||||
for (const id of ['insert', 'upsert', 'claimed']) {
|
||||
expectExpiresIn(await expiresAt('default_ttl_probe', id), DEFAULT_TTL_SECONDS);
|
||||
}
|
||||
expectExpiresIn(await expiresAt('default_ttl_probe_rows', 'batched'), DEFAULT_TTL_SECONDS);
|
||||
});
|
||||
|
||||
it('keeps an explicit TTL ahead of the default', async () => {
|
||||
await executor.executeQuery(DefaultTtlProbe.insertWithTtl({id: 'short', value: 'a', note: null}, 60));
|
||||
expectExpiresIn(await expiresAt('default_ttl_probe', 'short'), 60);
|
||||
|
||||
await executor.executeQuery(DefaultTtlProbe.insert({id: 'patched', value: 'a', note: null}));
|
||||
await executor.executeQuery(DefaultTtlProbe.patchByPkWithTtl({id: 'patched'}, {value: Db.set('b')}, 60));
|
||||
expectExpiresIn(await expiresAt('default_ttl_probe', 'patched'), 60);
|
||||
});
|
||||
|
||||
it('keeps an explicit TTL of zero as no expiry', async () => {
|
||||
await executor.executeQuery(DefaultTtlProbe.insertWithTtl({id: 'forever', value: 'a', note: null}, 0));
|
||||
expect(await neverExpires('default_ttl_probe', 'forever')).toBe(true);
|
||||
expect(
|
||||
await executor.executeQuery(
|
||||
DefaultTtlProbe.select({where: DefaultTtlProbe.where.eq('id')}).bind({id: 'forever'}),
|
||||
),
|
||||
).toEqual([{id: 'forever', value: 'a', note: null}]);
|
||||
|
||||
await executor.executeQuery(DefaultTtlProbe.patchByPk({id: 'forever'}, {note: Db.set('patched')}));
|
||||
expect(await neverExpires('default_ttl_probe', 'forever')).toBe(true);
|
||||
|
||||
await pruneExpiredPostgresKvRows(raw);
|
||||
expect(await neverExpires('default_ttl_probe', 'forever')).toBe(true);
|
||||
});
|
||||
|
||||
it('raises a patched row to the default but never lowers it', async () => {
|
||||
await executor.executeQuery(DefaultTtlProbe.insertWithTtl({id: 'longer', value: 'a', note: null}, 3600));
|
||||
await executor.executeQuery(DefaultTtlProbe.patchByPk({id: 'longer'}, {note: Db.set('patched')}));
|
||||
expectExpiresIn(await expiresAt('default_ttl_probe', 'longer'), 3600);
|
||||
|
||||
await executor.executeQuery(DefaultTtlProbe.insert({id: 'soon', value: 'a', note: null}));
|
||||
await setExpiry('default_ttl_probe', 'soon', "now() + interval '5 seconds'");
|
||||
await executor.executeQuery(DefaultTtlProbe.patchByPk({id: 'soon'}, {note: Db.set('patched')}));
|
||||
expectExpiresIn(await expiresAt('default_ttl_probe', 'soon'), DEFAULT_TTL_SECONDS);
|
||||
|
||||
await executor.executeQuery(DefaultTtlProbe.patchByPk({id: 'missing'}, {note: Db.set('created')}));
|
||||
expectExpiresIn(await expiresAt('default_ttl_probe', 'missing'), DEFAULT_TTL_SECONDS);
|
||||
|
||||
await executor.executeQuery(DefaultTtlProbe.insert({id: 'unset', value: 'a', note: null}));
|
||||
await setExpiry('default_ttl_probe', 'unset', 'NULL');
|
||||
await executor.executeQuery(DefaultTtlProbe.patchByPk({id: 'unset'}, {note: Db.set('patched')}));
|
||||
expectExpiresIn(await expiresAt('default_ttl_probe', 'unset'), DEFAULT_TTL_SECONDS);
|
||||
|
||||
await executor.executeQuery(DefaultTtlProbe.insert({id: 'expired', value: 'a', note: null}));
|
||||
await setExpiry('default_ttl_probe', 'expired', "now() - interval '1 second'");
|
||||
await executor.executeQuery(DefaultTtlProbe.patchByPk({id: 'expired'}, {note: Db.set('patched')}));
|
||||
const revived = await stored('default_ttl_probe', 'expired');
|
||||
expect(revived.row_data).toEqual({id: 'expired', note: 'patched'});
|
||||
expectExpiresIn(revived.expires_at, DEFAULT_TTL_SECONDS);
|
||||
});
|
||||
|
||||
it('raises conditional patches the same way', async () => {
|
||||
await executor.executeQuery(DefaultTtlProbe.insert({id: 'soon', value: 'a', note: null}));
|
||||
await setExpiry('default_ttl_probe', 'soon', "now() + interval '5 seconds'");
|
||||
expect(
|
||||
await executor.executeQuery(
|
||||
DefaultTtlProbe.conditionalPatchByPk({id: 'soon'}, {note: Db.set('patched')}, {value: 'a'}),
|
||||
),
|
||||
).toEqual([{'[applied]': true}]);
|
||||
expectExpiresIn(await expiresAt('default_ttl_probe', 'soon'), DEFAULT_TTL_SECONDS);
|
||||
|
||||
await executor.executeQuery(DefaultTtlProbe.insertWithTtl({id: 'longer', value: 'a', note: null}, 3600));
|
||||
expect(
|
||||
await executor.executeQuery(
|
||||
DefaultTtlProbe.conditionalPatchByPk({id: 'longer'}, {note: Db.set('patched')}, {value: 'a'}),
|
||||
),
|
||||
).toEqual([{'[applied]': true}]);
|
||||
expectExpiresIn(await expiresAt('default_ttl_probe', 'longer'), 3600);
|
||||
|
||||
await executor.executeQuery(DefaultTtlProbeRows.insert({owner: 'o', id: 'existing', value: 'old'}));
|
||||
await setExpiry('default_ttl_probe_rows', 'existing', 'NULL');
|
||||
expect(
|
||||
await executor.executeQuery(
|
||||
DefaultTtlProbeRows.conditionalBatch([
|
||||
{action: 'insert', row: {owner: 'o', id: 'added', value: 'new'}},
|
||||
{
|
||||
action: 'patch',
|
||||
pk: {owner: 'o', id: 'existing'},
|
||||
patch: {value: Db.set('updated')},
|
||||
expected: {value: 'old'},
|
||||
},
|
||||
]),
|
||||
),
|
||||
).toEqual([{'[applied]': true}]);
|
||||
expectExpiresIn(await expiresAt('default_ttl_probe_rows', 'added'), DEFAULT_TTL_SECONDS);
|
||||
expectExpiresIn(await expiresAt('default_ttl_probe_rows', 'existing'), DEFAULT_TTL_SECONDS);
|
||||
});
|
||||
|
||||
it('leaves tables without a default untouched', async () => {
|
||||
await executor.executeQuery(NoTtlProbe.insert({id: 'plain', value: 'a', note: null}));
|
||||
expect(await expiresAt('no_ttl_probe', 'plain')).toBeNull();
|
||||
await executor.executeQuery(NoTtlProbe.patchByPk({id: 'plain'}, {note: Db.set('patched')}));
|
||||
expect(await expiresAt('no_ttl_probe', 'plain')).toBeNull();
|
||||
await executor.executeQuery(NoTtlProbe.insertWithTtl({id: 'zero', value: 'a', note: null}, 0));
|
||||
expect(await expiresAt('no_ttl_probe', 'zero')).toBeNull();
|
||||
});
|
||||
|
||||
it('gives rows an older image wrote the expiry of their last write and deletes the ones past it', async () => {
|
||||
const mentionWrittenAt = await seed('recent_mentions', 'rm-day', '1 day');
|
||||
await seed('recent_mentions', 'rm-week', '8 days');
|
||||
await seed('attachment_upload_traces_by_key', 'at-31', '31 days');
|
||||
await seed('attachment_upload_traces_by_key', 'at-29', '29 days');
|
||||
await seed('phone_lookup_cache', 'pl-8', '8 days');
|
||||
await seed('donor_magic_link_tokens', 'dm-hour', '1 hour');
|
||||
await seed('ipinfo_requests_by_hour', 'ip-day', '1 day');
|
||||
await seed('jobs_by_id', 'job', '100 days');
|
||||
await seed('users', 'user', '100 days');
|
||||
await seed('recent_mentions', 'rm-forever', '1 day', 'infinity');
|
||||
await seed('recent_mentions', 'rm-hour', '30 days', new Date(Date.now() + 3_600_000));
|
||||
|
||||
expect(await expireLegacyDefaultTtlRows(raw, Date.now() + 60_000)).toEqual({
|
||||
deleted: 4,
|
||||
expiring: 3,
|
||||
complete: true,
|
||||
});
|
||||
expect(await remaining()).toEqual([
|
||||
{table_name: 'attachment_upload_traces_by_key', row_key: 'at-29'},
|
||||
{table_name: 'ipinfo_requests_by_hour', row_key: 'ip-day'},
|
||||
{table_name: 'jobs_by_id', row_key: 'job'},
|
||||
{table_name: 'recent_mentions', row_key: 'rm-day'},
|
||||
{table_name: 'recent_mentions', row_key: 'rm-forever'},
|
||||
{table_name: 'recent_mentions', row_key: 'rm-hour'},
|
||||
{table_name: 'users', row_key: 'user'},
|
||||
]);
|
||||
|
||||
const exact = await raw.query<{row_key: string; exact: boolean; unchanged: boolean | null}>(
|
||||
`SELECT row_key,
|
||||
expires_at = updated_at + CASE table_name WHEN 'recent_mentions' THEN interval '7 days' WHEN 'attachment_upload_traces_by_key' THEN interval '30 days' ELSE interval '90 days' END AS exact,
|
||||
CASE WHEN row_key = 'rm-day' THEN updated_at = $1::timestamptz END AS unchanged
|
||||
FROM ${KV_TABLE}
|
||||
WHERE row_key IN ('rm-day', 'at-29', 'ip-day')
|
||||
ORDER BY row_key`,
|
||||
[mentionWrittenAt],
|
||||
);
|
||||
expect(exact.rows).toEqual([
|
||||
{row_key: 'at-29', exact: true, unchanged: null},
|
||||
{row_key: 'ip-day', exact: true, unchanged: null},
|
||||
{row_key: 'rm-day', exact: true, unchanged: true},
|
||||
]);
|
||||
const untouched = await raw.query<{row_key: string; state: string}>(
|
||||
`SELECT row_key, CASE WHEN expires_at IS NULL THEN 'unset' WHEN expires_at = 'infinity' THEN 'forever' ELSE 'set' END AS state
|
||||
FROM ${KV_TABLE}
|
||||
WHERE row_key IN ('job', 'user', 'rm-forever', 'rm-hour')
|
||||
ORDER BY row_key`,
|
||||
);
|
||||
expect(untouched.rows).toEqual([
|
||||
{row_key: 'job', state: 'unset'},
|
||||
{row_key: 'rm-forever', state: 'forever'},
|
||||
{row_key: 'rm-hour', state: 'set'},
|
||||
{row_key: 'user', state: 'unset'},
|
||||
]);
|
||||
|
||||
expect(await markerCount()).toBe(0);
|
||||
expect(await expireLegacyDefaultTtlRows(raw, Date.now() + 60_000)).toEqual({
|
||||
deleted: 0,
|
||||
expiring: 0,
|
||||
complete: true,
|
||||
});
|
||||
expect(await markerCount()).toBe(1);
|
||||
expect(await expireLegacyDefaultTtlRows(raw, Date.now() + 60_000)).toBeNull();
|
||||
});
|
||||
|
||||
it('checks again a day after a clean pass', async () => {
|
||||
expect(await expireLegacyDefaultTtlRows(raw, Date.now() + 60_000)).toEqual({
|
||||
deleted: 0,
|
||||
expiring: 0,
|
||||
complete: true,
|
||||
});
|
||||
expect(await expireLegacyDefaultTtlRows(raw, Date.now() + 60_000)).toBeNull();
|
||||
|
||||
await seed('recent_mentions', 'rm-rolled-back', '1 day');
|
||||
expect(await expireLegacyDefaultTtlRows(raw, Date.now() + 60_000)).toBeNull();
|
||||
const before = await raw.query(`SELECT expires_at FROM ${KV_TABLE} WHERE row_key = 'rm-rolled-back'`);
|
||||
expect(before.rows).toEqual([{expires_at: null}]);
|
||||
|
||||
await ageMarker();
|
||||
expect(await expireLegacyDefaultTtlRows(raw, Date.now() + 60_000)).toEqual({
|
||||
deleted: 0,
|
||||
expiring: 1,
|
||||
complete: true,
|
||||
});
|
||||
expect(await expireLegacyDefaultTtlRows(raw, Date.now() + 60_000)).toEqual({
|
||||
deleted: 0,
|
||||
expiring: 0,
|
||||
complete: true,
|
||||
});
|
||||
expect(await expireLegacyDefaultTtlRows(raw, Date.now() + 60_000)).toBeNull();
|
||||
});
|
||||
|
||||
it('pages through more rows than one page holds and stops at its deadline', async () => {
|
||||
await raw.query(
|
||||
`INSERT INTO ${KV_TABLE} (table_name, partition_key, row_key, row_data, updated_at)
|
||||
SELECT 'recent_mentions', 'rm-' || lpad(g::text, 5, '0'), 'rm-' || lpad(g::text, 5, '0'), '{}'::jsonb, now() - interval '1 day'
|
||||
FROM generate_series(1, 2300) g`,
|
||||
);
|
||||
|
||||
expect(await expireLegacyDefaultTtlRows(raw, Date.now() - 1)).toEqual({
|
||||
deleted: 0,
|
||||
expiring: 0,
|
||||
complete: false,
|
||||
});
|
||||
expect(await markerCount()).toBe(0);
|
||||
expect(await expireLegacyDefaultTtlRows(raw, Date.now() + 60_000)).toEqual({
|
||||
deleted: 0,
|
||||
expiring: 2300,
|
||||
complete: true,
|
||||
});
|
||||
const unset = await raw.query<{n: number}>(
|
||||
`SELECT count(*)::int AS n FROM ${KV_TABLE} WHERE table_name = 'recent_mentions' AND expires_at IS NULL`,
|
||||
);
|
||||
expect(unset.rows[0]).toEqual({n: 0});
|
||||
expect(await expireLegacyDefaultTtlRows(raw, Date.now() + 60_000)).toEqual({
|
||||
deleted: 0,
|
||||
expiring: 0,
|
||||
complete: true,
|
||||
});
|
||||
expect(await expireLegacyDefaultTtlRows(raw, Date.now() + 60_000)).toBeNull();
|
||||
});
|
||||
|
||||
it('saves where a run stopped and starts the next run there', async () => {
|
||||
const first = DEFAULT_TTL_TABLES[0]!.name;
|
||||
const last = DEFAULT_TTL_TABLES.at(-1)!.name;
|
||||
await seed(first, 'a', '1 hour');
|
||||
await seed(first, 'z', '1 hour');
|
||||
await seed(last, 'k', '1 hour');
|
||||
|
||||
expect(await expireLegacyDefaultTtlRows(raw, Date.now() - 1)).toEqual({deleted: 0, expiring: 0, complete: false});
|
||||
expect(await resumePoint()).toEqual({table: first, row_key: '', unset: 0});
|
||||
|
||||
await raw.query(
|
||||
`UPDATE ${KV_TABLE} SET row_data = jsonb_build_object('table', $1::text, 'row_key', 'm', 'unset', 0) WHERE table_name = '__fluxer_schema_migrations' AND row_key = $2`,
|
||||
[first, DEFAULT_TTL_EXPIRY_RESUME],
|
||||
);
|
||||
expect(await expireLegacyDefaultTtlRows(raw, Date.now() + 60_000)).toEqual({
|
||||
deleted: 0,
|
||||
expiring: 2,
|
||||
complete: true,
|
||||
});
|
||||
const untouched = await raw.query<{expires_at: Date | null}>(
|
||||
`SELECT expires_at FROM ${KV_TABLE} WHERE table_name = $1 AND row_key = 'a'`,
|
||||
[first],
|
||||
);
|
||||
expect(untouched.rows).toEqual([{expires_at: null}]);
|
||||
expect(await resumePoint()).toBeNull();
|
||||
expect(await markerCount()).toBe(0);
|
||||
|
||||
expect(await expireLegacyDefaultTtlRows(raw, Date.now() + 60_000)).toEqual({
|
||||
deleted: 0,
|
||||
expiring: 1,
|
||||
complete: true,
|
||||
});
|
||||
expect(await expireLegacyDefaultTtlRows(raw, Date.now() + 60_000)).toEqual({
|
||||
deleted: 0,
|
||||
expiring: 0,
|
||||
complete: true,
|
||||
});
|
||||
expect(await markerCount()).toBe(1);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,142 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
import {
|
||||
POSTGRES_KV_MIGRATION_TABLE,
|
||||
postgresKvPassIsFresh,
|
||||
recordPostgresKvCleanPass,
|
||||
} from '@app/api/database/PostgresKvQueryExecutor';
|
||||
import * as DonationTables from '@app/api/donation/DonationTables';
|
||||
import * as Tables from '@app/api/Tables';
|
||||
import {IPINFO_CACHE_TTL_SECONDS, IPINFO_REQUEST_AUDIT_TTL_SECONDS} from '@pkgs/geoip/src/PostgresIpInfoKv';
|
||||
import {type IPostgresClient, quoteIdentifier} from '@pkgs/postgres/src/Client';
|
||||
import {ms} from 'itty-time';
|
||||
|
||||
const DEFAULT_TTL_EXPIRY_MARKER = 'default_ttl_expiry_v1';
|
||||
export const DEFAULT_TTL_EXPIRY_RESUME = 'default_ttl_expiry_v1_resume';
|
||||
const PAGE_SIZE = 2000;
|
||||
const CLEAN_PASS_INTERVAL_MS = ms('1 day');
|
||||
const OWN_EXPIRY_PASS = new Set<string>([Tables.JobsById.name, Tables.JobsByDayBucket.name]);
|
||||
|
||||
export const DEFAULT_TTL_TABLES: ReadonlyArray<{name: string; defaultTtlSeconds: number}> = [
|
||||
...[...Object.values(Tables), ...Object.values(DonationTables)].flatMap((table) =>
|
||||
table.defaultTtlSeconds === undefined || OWN_EXPIRY_PASS.has(table.name)
|
||||
? []
|
||||
: [{name: table.name, defaultTtlSeconds: table.defaultTtlSeconds}],
|
||||
),
|
||||
{name: 'ipinfo_cache', defaultTtlSeconds: IPINFO_CACHE_TTL_SECONDS},
|
||||
{name: 'ipinfo_requests_by_hour', defaultTtlSeconds: IPINFO_REQUEST_AUDIT_TTL_SECONDS},
|
||||
];
|
||||
|
||||
export interface LegacyDefaultTtlExpiryResult {
|
||||
deleted: number;
|
||||
expiring: number;
|
||||
complete: boolean;
|
||||
}
|
||||
|
||||
interface ResumePoint {
|
||||
table: string;
|
||||
rowKey: string;
|
||||
unset: number;
|
||||
}
|
||||
|
||||
async function readResumePoint(client: IPostgresClient, kvTable: string): Promise<ResumePoint | null> {
|
||||
const result = await client.query<{row_data: Record<string, unknown>}>(
|
||||
`SELECT row_data FROM ${kvTable} WHERE table_name = $1 AND row_key = $2`,
|
||||
[POSTGRES_KV_MIGRATION_TABLE, DEFAULT_TTL_EXPIRY_RESUME],
|
||||
);
|
||||
const data = result.rows[0]?.row_data;
|
||||
if (typeof data?.table !== 'string' || typeof data.row_key !== 'string' || typeof data.unset !== 'number') {
|
||||
return null;
|
||||
}
|
||||
return {table: data.table, rowKey: data.row_key, unset: data.unset};
|
||||
}
|
||||
|
||||
async function writeResumePoint(client: IPostgresClient, kvTable: string, point: ResumePoint | null): Promise<void> {
|
||||
if (point === null) {
|
||||
await client.query(`DELETE FROM ${kvTable} WHERE table_name = $1 AND row_key = $2`, [
|
||||
POSTGRES_KV_MIGRATION_TABLE,
|
||||
DEFAULT_TTL_EXPIRY_RESUME,
|
||||
]);
|
||||
return;
|
||||
}
|
||||
await client.query(
|
||||
`INSERT INTO ${kvTable} (table_name, partition_key, row_key, row_data)
|
||||
VALUES ($1, $2, $2, jsonb_build_object('table', $3::text, 'row_key', $4::text, 'unset', $5::bigint))
|
||||
ON CONFLICT (table_name, row_key) DO UPDATE SET row_data = EXCLUDED.row_data, updated_at = now()`,
|
||||
[POSTGRES_KV_MIGRATION_TABLE, DEFAULT_TTL_EXPIRY_RESUME, point.table, point.rowKey, point.unset],
|
||||
);
|
||||
}
|
||||
|
||||
function pageSql(table: string): string {
|
||||
return `
|
||||
WITH page AS (
|
||||
SELECT kv.row_key, kv.expires_at IS NULL AS unset
|
||||
FROM ${table} kv
|
||||
WHERE kv.table_name = $1 AND kv.row_key > $2
|
||||
ORDER BY kv.row_key
|
||||
LIMIT $3
|
||||
), removed AS (
|
||||
DELETE FROM ${table} kv
|
||||
USING page
|
||||
WHERE kv.table_name = $1 AND kv.row_key = page.row_key AND kv.expires_at IS NULL
|
||||
AND kv.updated_at + make_interval(secs => $4::double precision) <= now()
|
||||
RETURNING 1
|
||||
), expiring AS (
|
||||
UPDATE ${table} kv
|
||||
SET expires_at = kv.updated_at + make_interval(secs => $4::double precision)
|
||||
FROM page
|
||||
WHERE kv.table_name = $1 AND kv.row_key = page.row_key AND kv.expires_at IS NULL
|
||||
AND kv.updated_at + make_interval(secs => $4::double precision) > now()
|
||||
RETURNING 1
|
||||
)
|
||||
SELECT
|
||||
(SELECT max(row_key) FROM page) AS last_row_key,
|
||||
(SELECT count(*) FROM page WHERE unset) AS unset,
|
||||
(SELECT count(*) FROM removed) AS deleted,
|
||||
(SELECT count(*) FROM expiring) AS expiring`;
|
||||
}
|
||||
|
||||
export async function expireLegacyDefaultTtlRows(
|
||||
client: IPostgresClient,
|
||||
deadlineMs: number,
|
||||
): Promise<LegacyDefaultTtlExpiryResult | null> {
|
||||
if (await postgresKvPassIsFresh(client, DEFAULT_TTL_EXPIRY_MARKER, CLEAN_PASS_INTERVAL_MS)) {
|
||||
return null;
|
||||
}
|
||||
const kvTable = quoteIdentifier(client.kvTable());
|
||||
const sql = pageSql(kvTable);
|
||||
const resume = await readResumePoint(client, kvTable);
|
||||
const resumeIndex = resume === null ? -1 : DEFAULT_TTL_TABLES.findIndex((target) => target.name === resume.table);
|
||||
let unset = resumeIndex < 0 ? 0 : resume!.unset;
|
||||
let deleted = 0;
|
||||
let expiring = 0;
|
||||
for (let index = Math.max(resumeIndex, 0); index < DEFAULT_TTL_TABLES.length; index += 1) {
|
||||
const target = DEFAULT_TTL_TABLES[index]!;
|
||||
let cursor = index === resumeIndex ? resume!.rowKey : '';
|
||||
for (;;) {
|
||||
if (Date.now() >= deadlineMs) {
|
||||
await writeResumePoint(client, kvTable, {table: target.name, rowKey: cursor, unset});
|
||||
return {deleted, expiring, complete: false};
|
||||
}
|
||||
const result = await client.query<{
|
||||
last_row_key: string | null;
|
||||
unset: string;
|
||||
deleted: string;
|
||||
expiring: string;
|
||||
}>(sql, [target.name, cursor, PAGE_SIZE, target.defaultTtlSeconds]);
|
||||
const page = result.rows[0];
|
||||
if (!page || page.last_row_key === null) {
|
||||
break;
|
||||
}
|
||||
unset += Number(page.unset);
|
||||
deleted += Number(page.deleted);
|
||||
expiring += Number(page.expiring);
|
||||
cursor = page.last_row_key;
|
||||
}
|
||||
}
|
||||
await writeResumePoint(client, kvTable, null);
|
||||
if (unset === 0) {
|
||||
await recordPostgresKvCleanPass(client, DEFAULT_TTL_EXPIRY_MARKER);
|
||||
}
|
||||
return {deleted, expiring, complete: true};
|
||||
}
|
||||
@@ -89,6 +89,28 @@ const NUMERIC_ROW_KEY_NUMBER_PATTERN = '^(-?[0-9]+(?:\\.[0-9]+)?(?:[eE][-+]?[0-9
|
||||
const EXPIRED_STORED_ROW = 'kv.expires_at IS NOT NULL AND kv.expires_at <= now()';
|
||||
const MERGED_ROW_DATA = `CASE WHEN ${EXPIRED_STORED_ROW} THEN EXCLUDED.row_data ELSE kv.row_data || EXCLUDED.row_data END`;
|
||||
const KEPT_EXPIRES_AT = `CASE WHEN ${EXPIRED_STORED_ROW} THEN NULL ELSE kv.expires_at END`;
|
||||
const NO_EXPIRY = 'infinity';
|
||||
|
||||
export async function postgresKvPassIsFresh(
|
||||
client: IPostgresClient,
|
||||
marker: string,
|
||||
maxAgeMs: number,
|
||||
): Promise<boolean> {
|
||||
const result = await client.query(
|
||||
`SELECT 1 FROM ${quoteIdentifier(client.kvTable())} WHERE table_name = $1 AND row_key = $2 AND (row_data ->> 'applied_at')::timestamptz > now() - make_interval(secs => $3::double precision)`,
|
||||
[POSTGRES_KV_MIGRATION_TABLE, marker, maxAgeMs / 1000],
|
||||
);
|
||||
return result.rows.length > 0;
|
||||
}
|
||||
|
||||
export async function recordPostgresKvCleanPass(client: IPostgresClient, marker: string): Promise<void> {
|
||||
await client.query(
|
||||
`INSERT INTO ${quoteIdentifier(client.kvTable())} (table_name, partition_key, row_key, row_data)
|
||||
VALUES ($1, $2, $2, jsonb_build_object('applied_at', now()))
|
||||
ON CONFLICT (table_name, row_key) DO UPDATE SET row_data = EXCLUDED.row_data, updated_at = now()`,
|
||||
[POSTGRES_KV_MIGRATION_TABLE, marker],
|
||||
);
|
||||
}
|
||||
|
||||
function numericRowKeyExpr(column: string): string {
|
||||
return `(COALESCE(substring(${column} from '${NUMERIC_ROW_KEY_BIGINT_PATTERN}'), substring(${column} from '${NUMERIC_ROW_KEY_NUMBER_PATTERN}'))::numeric)`;
|
||||
@@ -333,20 +355,21 @@ function projectRow(row: Row, columns: ReadonlyArray<string> | undefined): Row {
|
||||
return projected;
|
||||
}
|
||||
|
||||
function rowComparator(meta: KvQueryMeta): (left: Row, right: Row) => number {
|
||||
if (meta.orderBy) {
|
||||
const column = meta.orderBy.col as string;
|
||||
const direction = meta.orderBy.direction === 'DESC' ? -1 : 1;
|
||||
return (left, right) => compareValues(left[column], right[column]) * direction;
|
||||
function compareColumns(columns: ReadonlyArray<string>, left: Row, right: Row): number {
|
||||
for (const column of columns) {
|
||||
const cmp = compareValues(left[column], right[column]);
|
||||
if (cmp !== 0) return cmp;
|
||||
}
|
||||
const columns = meta.table.primaryKey as ReadonlyArray<string>;
|
||||
return (left, right) => {
|
||||
for (const column of columns) {
|
||||
const cmp = compareValues(left[column], right[column]);
|
||||
if (cmp !== 0) return cmp;
|
||||
}
|
||||
return 0;
|
||||
};
|
||||
return 0;
|
||||
}
|
||||
|
||||
function rowComparator(meta: KvQueryMeta): (left: Row, right: Row) => number {
|
||||
const primaryKey = meta.table.primaryKey as ReadonlyArray<string>;
|
||||
if (!meta.orderBy) return (left, right) => compareColumns(primaryKey, left, right);
|
||||
const column = meta.orderBy.col as string;
|
||||
const columns = [column, ...primaryKey.slice(primaryKey.indexOf(column) + 1)];
|
||||
const direction = meta.orderBy.direction === 'DESC' ? -1 : 1;
|
||||
return (left, right) => compareColumns(columns, left, right) * direction;
|
||||
}
|
||||
|
||||
function sortRows(meta: KvQueryMeta, rows: Array<Row>): Array<Row> {
|
||||
@@ -679,7 +702,7 @@ function logFullScan(meta: KvQueryMeta): void {
|
||||
logWarn({table: meta.table.name, action: meta.action, where: shape.summary || 'none'}, 'Postgres KV full table scan');
|
||||
}
|
||||
|
||||
function ttlExpiresAt(meta: KvQueryMeta, params: CassandraParams): Date | null | undefined {
|
||||
function ttlExpiresAt(meta: KvQueryMeta, params: CassandraParams): Date | typeof NO_EXPIRY | null | undefined {
|
||||
const ttlParam = meta.ttlParamName;
|
||||
if (!ttlParam) return undefined;
|
||||
const ttlRaw = params[ttlParam];
|
||||
@@ -687,7 +710,13 @@ function ttlExpiresAt(meta: KvQueryMeta, params: CassandraParams): Date | null |
|
||||
throw new Error(`TTL parameter ${ttlParam} must be a number`);
|
||||
}
|
||||
const ttlSeconds = validateTtlSeconds(ttlRaw);
|
||||
return ttlSeconds === 0 ? null : new Date(Date.now() + ttlSeconds * 1000);
|
||||
if (ttlSeconds === 0) return meta.table.defaultTtlSeconds === undefined ? null : NO_EXPIRY;
|
||||
return new Date(Date.now() + ttlSeconds * 1000);
|
||||
}
|
||||
|
||||
function defaultExpiresAt(meta: KvQueryMeta): Date | undefined {
|
||||
const ttlSeconds = meta.table.defaultTtlSeconds;
|
||||
return ttlSeconds === undefined ? undefined : new Date(Date.now() + ttlSeconds * 1000);
|
||||
}
|
||||
|
||||
function encodePageState(pageState: PageState): string {
|
||||
@@ -1191,7 +1220,8 @@ export class PostgresKvQueryExecutor {
|
||||
'kv_del_expired',
|
||||
);
|
||||
}
|
||||
const expiresAt = ttlExpiresAt(meta, params) ?? null;
|
||||
const explicit = ttlExpiresAt(meta, params);
|
||||
const expiresAt = explicit === undefined ? (defaultExpiresAt(meta) ?? null) : explicit;
|
||||
const result = await db.query(
|
||||
`INSERT INTO ${this.table} AS kv (table_name, partition_key, row_key, row_data, expires_at, updated_at)
|
||||
VALUES ($1, $2, $3, $4::jsonb, $5, now())
|
||||
@@ -1244,10 +1274,14 @@ WHERE NOT $6`,
|
||||
}
|
||||
bindings.push(JSON.stringify(encodeRow(paramsRow(params, meta.patchKeys))));
|
||||
const assignments = [`row_data = kv.row_data || $${bindings.length}::jsonb`, 'updated_at = now()'];
|
||||
const expiresAt = ttlExpiresAt(meta, params);
|
||||
if (expiresAt !== undefined) {
|
||||
bindings.push(expiresAt);
|
||||
const explicit = ttlExpiresAt(meta, params);
|
||||
const fallback = explicit === undefined ? defaultExpiresAt(meta) : undefined;
|
||||
if (explicit !== undefined) {
|
||||
bindings.push(explicit);
|
||||
assignments.push(`expires_at = $${bindings.length}`);
|
||||
} else if (fallback !== undefined) {
|
||||
bindings.push(fallback);
|
||||
assignments.push(`expires_at = GREATEST(kv.expires_at, $${bindings.length}::timestamptz)`);
|
||||
}
|
||||
sql = `UPDATE ${this.table} kv SET ${assignments.join(', ')} WHERE ${where}`;
|
||||
}
|
||||
@@ -1346,15 +1380,27 @@ WHERE NOT $6`,
|
||||
for (const column of meta.patchKeys ?? []) {
|
||||
incoming[column] = column in params ? params[column] : null;
|
||||
}
|
||||
const ttl = ttlExpiresAt(meta, params);
|
||||
const expiresAtExpr = ttl === undefined ? KEPT_EXPIRES_AT : 'EXCLUDED.expires_at';
|
||||
const explicit = ttlExpiresAt(meta, params);
|
||||
const fallback = explicit === undefined ? defaultExpiresAt(meta) : undefined;
|
||||
const [expiresAtExpr, statementName] =
|
||||
explicit !== undefined
|
||||
? ['EXCLUDED.expires_at', 'kv_patch_set_ttl']
|
||||
: fallback !== undefined
|
||||
? ['GREATEST(kv.expires_at, EXCLUDED.expires_at)', 'kv_patch_default_ttl']
|
||||
: [KEPT_EXPIRES_AT, 'kv_patch_keep_ttl'];
|
||||
await db.query(
|
||||
`INSERT INTO ${this.table} AS kv (table_name, partition_key, row_key, row_data, expires_at, updated_at)
|
||||
VALUES ($1, $2, $3, $4::jsonb, $5, now())
|
||||
ON CONFLICT (table_name, row_key)
|
||||
DO UPDATE SET partition_key = EXCLUDED.partition_key, row_data = ${MERGED_ROW_DATA}, expires_at = ${expiresAtExpr}, updated_at = now()`,
|
||||
[meta.table.name, partitionKey(meta, incoming), key, JSON.stringify(encodeRow(incoming)), ttl ?? null],
|
||||
ttl === undefined ? 'kv_patch_keep_ttl' : 'kv_patch_set_ttl',
|
||||
[
|
||||
meta.table.name,
|
||||
partitionKey(meta, incoming),
|
||||
key,
|
||||
JSON.stringify(encodeRow(incoming)),
|
||||
explicit ?? fallback ?? null,
|
||||
],
|
||||
statementName,
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
@@ -52,6 +52,8 @@ const Composite: KvTableSpec<Row> = {
|
||||
partitionKey: ['owner_id'],
|
||||
};
|
||||
|
||||
const Expiring: KvTableSpec<Row> = {...Composite, name: 'stmt_expiring', defaultTtlSeconds: 600};
|
||||
|
||||
const Bucketed: KvTableSpec<Row> = {
|
||||
name: 'stmt_bucketed',
|
||||
columns: ['bucket', 'item_id', 'payload'],
|
||||
@@ -118,6 +120,7 @@ async function runShapes(): Promise<Array<Statement>> {
|
||||
meta(Composite, 'patch', [eq('owner_id'), eq('item_id')], {patchKeys: ['payload'], ttlParamName: 'ttl_'}),
|
||||
{...OWNER_ITEM, ttl_: 600} as CassandraParams,
|
||||
],
|
||||
[meta(Expiring, 'patch', [eq('owner_id'), eq('item_id')], {patchKeys: ['payload']}), OWNER_ITEM],
|
||||
];
|
||||
for (const [kvMeta, params] of cases) {
|
||||
await executor.executeQuery({cql: `__stmt_${kvMeta.action}`, params, kvMeta: kvMeta as KvQueryMeta});
|
||||
@@ -142,6 +145,7 @@ describe('PostgresKvQueryExecutor statement names', () => {
|
||||
'kv_del_keys',
|
||||
'kv_del_rowkeys',
|
||||
'kv_get_row',
|
||||
'kv_patch_default_ttl',
|
||||
'kv_patch_keep_ttl',
|
||||
'kv_patch_set_ttl',
|
||||
'kv_sel_range',
|
||||
@@ -231,6 +235,17 @@ async function exerciseKvShapes(executor: PostgresKvQueryExecutor): Promise<void
|
||||
kvMeta: meta(Composite, 'select', [eq('owner_id'), eq('item_id')]) as KvQueryMeta,
|
||||
});
|
||||
expect(patched.map((row) => row.payload)).toEqual(['patched']);
|
||||
await executor.executeQuery({
|
||||
cql: '__stmt_patch_default_ttl',
|
||||
params: {owner_id: 'o5', item_id: 'i5', payload: 'defaulted'} as CassandraParams,
|
||||
kvMeta: meta(Expiring, 'patch', [eq('owner_id'), eq('item_id')], {patchKeys: ['payload']}) as KvQueryMeta,
|
||||
});
|
||||
const defaulted = await executor.executeQuery<Row>({
|
||||
cql: '__stmt_point',
|
||||
params: {owner_id: 'o5', item_id: 'i5'} as CassandraParams,
|
||||
kvMeta: meta(Expiring, 'select', [eq('owner_id'), eq('item_id')]) as KvQueryMeta,
|
||||
});
|
||||
expect(defaulted.map((row) => row.payload)).toEqual(['defaulted']);
|
||||
await executor.executeQuery({
|
||||
cql: '__stmt_delete',
|
||||
params: {owner_id: 'o0', item_id: 'i0'} as CassandraParams,
|
||||
@@ -323,6 +338,7 @@ describe.skipIf(!dockerAvailable)('PostgresKvQueryExecutor statement names again
|
||||
'kv_del_expired',
|
||||
'kv_del_rowkeys',
|
||||
'kv_get_row',
|
||||
'kv_patch_default_ttl',
|
||||
'kv_patch_keep_ttl',
|
||||
'kv_patch_set_ttl',
|
||||
'kv_sel_range',
|
||||
|
||||
@@ -0,0 +1,90 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
import {spawnSync} from 'node:child_process';
|
||||
import {createServer} from 'node:net';
|
||||
import {startDockerContainer} from '@app/api/test/DockerTestContainer';
|
||||
import {getDefaultPostgresClient, initPostgres, shutdownPostgres} from '@pkgs/postgres/src/Client';
|
||||
import {afterAll, beforeAll, describe, expect, it} from 'vitest';
|
||||
|
||||
const CONTAINER = `fluxer-kvscram-${process.pid.toString(36)}-${Date.now().toString(36)}`;
|
||||
const dockerAvailable = spawnSync('docker', ['version'], {stdio: 'ignore'}).status === 0;
|
||||
const SCRAM_ITERATIONS = 200_000;
|
||||
|
||||
async function sleep(ms: number): Promise<void> {
|
||||
await new Promise((resolve) => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
async function freePort(): Promise<number> {
|
||||
return new Promise((resolve, reject) => {
|
||||
const server = createServer();
|
||||
server.on('error', reject);
|
||||
server.listen(0, '127.0.0.1', () => {
|
||||
const address = server.address();
|
||||
if (typeof address === 'string' || address === null) {
|
||||
reject(new Error('no port'));
|
||||
return;
|
||||
}
|
||||
const port = address.port;
|
||||
server.close(() => resolve(port));
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
describe.skipIf(!dockerAvailable)('postgres client against a server with raised SCRAM iterations', () => {
|
||||
let port: number;
|
||||
|
||||
beforeAll(async () => {
|
||||
port = await freePort();
|
||||
startDockerContainer([
|
||||
'run',
|
||||
'-d',
|
||||
'--name',
|
||||
CONTAINER,
|
||||
'-e',
|
||||
'POSTGRES_USER=fluxer',
|
||||
'-e',
|
||||
'POSTGRES_PASSWORD=fluxer',
|
||||
'-e',
|
||||
'POSTGRES_DB=fluxer',
|
||||
'-p',
|
||||
`127.0.0.1:${port}:5432`,
|
||||
'postgres:16-alpine',
|
||||
'-c',
|
||||
'fsync=off',
|
||||
'-c',
|
||||
`scram_iterations=${SCRAM_ITERATIONS}`,
|
||||
]);
|
||||
let ready = false;
|
||||
for (let attempt = 0; attempt < 180 && !ready; attempt += 1) {
|
||||
await sleep(500);
|
||||
const probe = spawnSync(
|
||||
'docker',
|
||||
['exec', CONTAINER, 'psql', '-h', '127.0.0.1', '-U', 'fluxer', '-d', 'fluxer', '-Atc', 'SELECT 1'],
|
||||
{stdio: 'ignore'},
|
||||
);
|
||||
ready = probe.status === 0;
|
||||
}
|
||||
if (!ready) throw new Error('postgres never came up');
|
||||
const rehash = spawnSync(
|
||||
'docker',
|
||||
['exec', CONTAINER, 'psql', '-U', 'fluxer', '-d', 'fluxer', '-Atc', "ALTER ROLE fluxer PASSWORD 'fluxer'"],
|
||||
{
|
||||
stdio: 'ignore',
|
||||
},
|
||||
);
|
||||
if (rehash.status !== 0) throw new Error('could not re-hash the role password');
|
||||
}, 900_000);
|
||||
|
||||
afterAll(async () => {
|
||||
await shutdownPostgres().catch(() => {});
|
||||
spawnSync('docker', ['rm', '-f', CONTAINER], {stdio: 'ignore'});
|
||||
});
|
||||
|
||||
it('connects when the role verifier uses more iterations than the driver default allows', async () => {
|
||||
await initPostgres({url: `postgres://fluxer:[email protected]:${port}/fluxer`, maxConnections: 1});
|
||||
const verifier = await getDefaultPostgresClient().query<{rolpassword: string}>(
|
||||
"SELECT rolpassword FROM pg_authid WHERE rolname = 'fluxer'",
|
||||
);
|
||||
expect(verifier.rows[0]?.rolpassword.startsWith(`SCRAM-SHA-256$${SCRAM_ITERATIONS}:`)).toBe(true);
|
||||
});
|
||||
});
|
||||
@@ -13,6 +13,7 @@ import {
|
||||
type DonorMagicLinkTokenRow,
|
||||
type DonorRow,
|
||||
} from '@app/api/database/types/DonationTypes';
|
||||
import {seconds} from 'itty-time';
|
||||
|
||||
export const Donors = defineTable<DonorRow, 'email'>({
|
||||
name: 'donors',
|
||||
@@ -43,9 +44,11 @@ export const DonorMagicLinkTokens = defineTable<DonorMagicLinkTokenRow, 'token_'
|
||||
name: 'donor_magic_link_tokens',
|
||||
columns: DONOR_MAGIC_LINK_TOKEN_COLUMNS,
|
||||
primaryKey: ['token_'],
|
||||
defaultTtlSeconds: seconds('15 minutes'),
|
||||
});
|
||||
export const DonorMagicLinkTokensByEmail = defineTable<DonorMagicLinkTokenByEmailRow, 'donor_email' | 'token_'>({
|
||||
name: 'donor_magic_link_tokens_by_email',
|
||||
columns: DONOR_MAGIC_LINK_TOKEN_BY_EMAIL_COLUMNS,
|
||||
primaryKey: ['donor_email', 'token_'],
|
||||
defaultTtlSeconds: seconds('15 minutes'),
|
||||
});
|
||||
|
||||
@@ -8,6 +8,7 @@ import {RateLimitConfigs} from '@app/api/RateLimitConfig';
|
||||
import type {HonoApp} from '@app/api/types/HonoEnv';
|
||||
import {entityTagMatches} from '@app/api/utils/EntityTag';
|
||||
import {Headers as HttpHeaders} from '@fluxer/constants/src/Headers';
|
||||
import {resolveScreenShareDeliveryAssignment} from '@fluxer/schema/src/domains/admin/ScreenShareDeliverySchemas';
|
||||
import {resolveVoiceNoiseSuppressionAssignment} from '@fluxer/schema/src/domains/admin/VoiceNoiseSuppressionSchemas';
|
||||
import {ExperimentAssignmentsResponse} from '@fluxer/schema/src/domains/experiment/ExperimentSchemas';
|
||||
|
||||
@@ -28,9 +29,10 @@ export function ExperimentController(app: HonoApp) {
|
||||
}),
|
||||
async (ctx) => {
|
||||
const instanceConfigRepository = ctx.get('instanceConfigRepository');
|
||||
const [delivery, voiceConfig] = await Promise.all([
|
||||
const [delivery, voiceConfig, screenShareConfig] = await Promise.all([
|
||||
instanceConfigRepository.getExperimentDeliveryConfig(),
|
||||
instanceConfigRepository.getVoiceNoiseSuppressionConfig(),
|
||||
instanceConfigRepository.getScreenShareDeliveryConfig(),
|
||||
]);
|
||||
const userId = ctx.get('user').id.toString();
|
||||
const body: ExperimentAssignmentsResponse = {
|
||||
@@ -38,6 +40,7 @@ export function ExperimentController(app: HonoApp) {
|
||||
poll_jitter_percent: delivery.poll_jitter_percent,
|
||||
assignments: {
|
||||
voice_noise_suppression: resolveVoiceNoiseSuppressionAssignment(voiceConfig, userId),
|
||||
screen_share_delivery: resolveScreenShareDeliveryAssignment(screenShareConfig, userId),
|
||||
},
|
||||
};
|
||||
const etag = `"${createHash('sha256').update(JSON.stringify(body)).digest('hex')}"`;
|
||||
|
||||
@@ -6,6 +6,10 @@ import {type ApiTestHarness, createApiTestHarness} from '@app/api/test/ApiTestHa
|
||||
import {HTTP_STATUS} from '@app/api/test/TestConstants';
|
||||
import {createBuilder, createBuilderWithoutAuth} from '@app/api/test/TestRequestBuilder';
|
||||
import {AdminACLs} from '@fluxer/constants/src/AdminACLs';
|
||||
import {
|
||||
DEFAULT_SCREEN_SHARE_DELIVERY_CONFIG,
|
||||
INERT_SCREEN_SHARE_DELIVERY_ASSIGNMENT,
|
||||
} from '@fluxer/schema/src/domains/admin/ScreenShareDeliverySchemas';
|
||||
import {
|
||||
DEFAULT_VOICE_NOISE_SUPPRESSION_CONFIG,
|
||||
INERT_VOICE_NOISE_SUPPRESSION_ASSIGNMENT,
|
||||
@@ -15,6 +19,7 @@ import {
|
||||
DEFAULT_EXPERIMENT_POLL_JITTER_PERCENT,
|
||||
type ExperimentAssignmentsResponse,
|
||||
type ExperimentDeliveryConfigResponse,
|
||||
readScreenShareDeliveryAssignment,
|
||||
readVoiceNoiseSuppressionAssignment,
|
||||
} from '@fluxer/schema/src/domains/experiment/ExperimentSchemas';
|
||||
import {afterAll, beforeAll, beforeEach, describe, expect, it} from 'vitest';
|
||||
@@ -51,6 +56,7 @@ describe('GET /experiments', () => {
|
||||
poll_jitter_percent: DEFAULT_EXPERIMENT_POLL_JITTER_PERCENT,
|
||||
assignments: {
|
||||
voice_noise_suppression: INERT_VOICE_NOISE_SUPPRESSION_ASSIGNMENT,
|
||||
screen_share_delivery: INERT_SCREEN_SHARE_DELIVERY_ASSIGNMENT,
|
||||
},
|
||||
});
|
||||
});
|
||||
@@ -82,6 +88,52 @@ describe('GET /experiments', () => {
|
||||
expect(readVoiceNoiseSuppressionAssignment(body).enabled).toBe(false);
|
||||
});
|
||||
|
||||
it('populates the screen share assignment key even when the rollout is disabled', async () => {
|
||||
const account = await createTestAccount(harness);
|
||||
|
||||
const body = await createBuilder<ExperimentAssignmentsResponse>(harness, account.token).get(ENDPOINT).execute();
|
||||
|
||||
expect(Object.hasOwn(body.assignments, 'screen_share_delivery')).toBe(true);
|
||||
expect(readScreenShareDeliveryAssignment(body).enabled).toBe(false);
|
||||
});
|
||||
|
||||
it('resolves the screen share caller through the allowlist', async () => {
|
||||
const targeted = await createTestAccount(harness);
|
||||
const untargeted = await createTestAccount(harness);
|
||||
await getInstanceConfigRepository().setScreenShareDeliveryConfig({
|
||||
...DEFAULT_SCREEN_SHARE_DELIVERY_CONFIG,
|
||||
enabled: true,
|
||||
config_version: 4,
|
||||
rollout_basis_points: 0,
|
||||
included_user_ids: [targeted.userId],
|
||||
});
|
||||
|
||||
const targetedBody = await createBuilder<ExperimentAssignmentsResponse>(harness, targeted.token)
|
||||
.get(ENDPOINT)
|
||||
.execute();
|
||||
expect(targetedBody.assignments.screen_share_delivery).toEqual({enabled: true});
|
||||
|
||||
const untargetedBody = await createBuilder<ExperimentAssignmentsResponse>(harness, untargeted.token)
|
||||
.get(ENDPOINT)
|
||||
.execute();
|
||||
expect(untargetedBody.assignments.screen_share_delivery).toEqual({enabled: false});
|
||||
});
|
||||
|
||||
it('keeps the screen share exclusion ahead of a full rollout', async () => {
|
||||
const excluded = await createTestAccount(harness);
|
||||
await getInstanceConfigRepository().setScreenShareDeliveryConfig({
|
||||
...DEFAULT_SCREEN_SHARE_DELIVERY_CONFIG,
|
||||
enabled: true,
|
||||
rollout_basis_points: 10000,
|
||||
included_user_ids: [excluded.userId],
|
||||
excluded_user_ids: [excluded.userId],
|
||||
});
|
||||
|
||||
const body = await createBuilder<ExperimentAssignmentsResponse>(harness, excluded.token).get(ENDPOINT).execute();
|
||||
|
||||
expect(body.assignments.screen_share_delivery).toEqual({enabled: false});
|
||||
});
|
||||
|
||||
it('serves the delivery cadence from the delivery config and not from the voice config', async () => {
|
||||
const account = await createTestAccount(harness);
|
||||
await getInstanceConfigRepository().setExperimentDeliveryConfig({
|
||||
@@ -196,6 +248,30 @@ describe('GET /experiments', () => {
|
||||
});
|
||||
});
|
||||
|
||||
it('serves a fresh body once the screen share config changes', async () => {
|
||||
const account = await createTestAccount(harness);
|
||||
|
||||
const first = await createBuilder<ExperimentAssignmentsResponse>(harness, account.token)
|
||||
.get(ENDPOINT)
|
||||
.executeWithResponse();
|
||||
const staleEtag = first.response.headers.get('etag') as string;
|
||||
|
||||
await getInstanceConfigRepository().setScreenShareDeliveryConfig({
|
||||
...DEFAULT_SCREEN_SHARE_DELIVERY_CONFIG,
|
||||
enabled: true,
|
||||
config_version: 1,
|
||||
rollout_basis_points: 10000,
|
||||
});
|
||||
|
||||
const refreshed = await createBuilder<ExperimentAssignmentsResponse>(harness, account.token)
|
||||
.get(ENDPOINT)
|
||||
.header('If-None-Match', staleEtag)
|
||||
.executeWithResponse();
|
||||
expect(refreshed.response.status).toBe(HTTP_STATUS.OK);
|
||||
expect(refreshed.response.headers.get('etag')).not.toBe(staleEtag);
|
||||
expect(refreshed.json?.assignments.screen_share_delivery).toEqual({enabled: true});
|
||||
});
|
||||
|
||||
it('serves a fresh body once the delivery config changes', async () => {
|
||||
const account = await createTestAccount(harness);
|
||||
|
||||
@@ -252,6 +328,44 @@ describe('GET /experiments', () => {
|
||||
});
|
||||
});
|
||||
|
||||
it('bumps the screen share config version on every admin update without the client sending one', async () => {
|
||||
const admin = await setUserACLs(harness, await createTestAccount(harness), [
|
||||
AdminACLs.AUTHENTICATE,
|
||||
AdminACLs.INSTANCE_CONFIG_VIEW,
|
||||
AdminACLs.INSTANCE_CONFIG_UPDATE,
|
||||
]);
|
||||
|
||||
const afterFirst = await createBuilder<{screen_share_delivery: {config_version: number; enabled: boolean}}>(
|
||||
harness,
|
||||
admin.token,
|
||||
)
|
||||
.patch('/admin/instance/config')
|
||||
.body({screen_share_delivery: {enabled: true, rollout_basis_points: 10000}})
|
||||
.execute();
|
||||
expect(afterFirst.screen_share_delivery).toMatchObject({config_version: 1, enabled: true});
|
||||
|
||||
const afterSecond = await createBuilder<{screen_share_delivery: {config_version: number; enabled: boolean}}>(
|
||||
harness,
|
||||
admin.token,
|
||||
)
|
||||
.patch('/admin/instance/config')
|
||||
.body({screen_share_delivery: {rollout_salt: 'screen-share-delivery-v2'}})
|
||||
.execute();
|
||||
expect(afterSecond.screen_share_delivery).toMatchObject({config_version: 2, enabled: true});
|
||||
|
||||
const afterEmpty = await createBuilder<{screen_share_delivery: {config_version: number; enabled: boolean}}>(
|
||||
harness,
|
||||
admin.token,
|
||||
)
|
||||
.patch('/admin/instance/config')
|
||||
.body({screen_share_delivery: {}})
|
||||
.execute();
|
||||
expect(afterEmpty.screen_share_delivery).toMatchObject({config_version: 2, enabled: true});
|
||||
|
||||
const body = await createBuilder<ExperimentAssignmentsResponse>(harness, admin.token).get(ENDPOINT).execute();
|
||||
expect(body.assignments.screen_share_delivery).toEqual({enabled: true});
|
||||
});
|
||||
|
||||
it('leaves the config version alone for an admin update that sets no field', async () => {
|
||||
const admin = await setUserACLs(harness, await createTestAccount(harness), [
|
||||
AdminACLs.AUTHENTICATE,
|
||||
|
||||
@@ -15,6 +15,8 @@ import {AuditLogActionType} from '@fluxer/constants/src/AuditLogActionType';
|
||||
import type {IWorkerService} from '@pkgs/worker/src/contracts/IWorkerService';
|
||||
import {ms} from 'itty-time';
|
||||
|
||||
const MESSAGE_DELETE_BATCH_DELAY_MS = ms('30 seconds');
|
||||
|
||||
interface MessageDeleteBatchGroup {
|
||||
logs: Array<GuildAuditLog>;
|
||||
userId: UserID;
|
||||
@@ -81,14 +83,15 @@ export class GuildAuditLogService {
|
||||
}
|
||||
|
||||
async scheduleMessageDeleteBatchJob(guildId: GuildID): Promise<void> {
|
||||
const runAt = new Date(Date.now() + ms('30 seconds'));
|
||||
const batchWindow = Math.floor(Date.now() / MESSAGE_DELETE_BATCH_DELAY_MS);
|
||||
await this.workerService.addJob(
|
||||
'batchGuildAuditLogMessageDeletes',
|
||||
{guildId: guildId.toString()},
|
||||
{
|
||||
jobKey: `batch-audit-log-message-deletes:${guildId}`,
|
||||
runAt,
|
||||
jobKey: `batch-audit-log-message-deletes:${guildId}:${batchWindow}`,
|
||||
runAt: new Date((batchWindow + 2) * MESSAGE_DELETE_BATCH_DELAY_MS),
|
||||
maxAttempts: 3,
|
||||
skipLedger: true,
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
@@ -60,7 +60,7 @@ function createService(roleNames: Map<string, string> = new Map()) {
|
||||
{addJob} as unknown as IWorkerService<WorkerTaskName>,
|
||||
{dispatchGuild} as unknown as IGatewayService,
|
||||
);
|
||||
return {service, createAuditLog, batchDeleteAndCreateAuditLogs, getRole, dispatchGuild};
|
||||
return {service, createAuditLog, batchDeleteAndCreateAuditLogs, getRole, dispatchGuild, addJob};
|
||||
}
|
||||
|
||||
function overwrites(
|
||||
@@ -376,3 +376,26 @@ describe('GuildAuditLogService.recordPermissionOverwriteDiff', () => {
|
||||
expect(dispatchGuild).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
describe('GuildAuditLogService.scheduleMessageDeleteBatchJob', () => {
|
||||
it('gives every delete in one 30 second window a single batch job that runs after the window closes, without a ledger row', async () => {
|
||||
vi.useFakeTimers({toFake: ['Date']});
|
||||
try {
|
||||
const {service, addJob} = createService();
|
||||
for (const at of ['2026-09-21T12:00:00.000Z', '2026-09-21T12:00:29.999Z', '2026-09-21T12:00:40.000Z']) {
|
||||
vi.setSystemTime(new Date(at));
|
||||
await service.scheduleMessageDeleteBatchJob(GUILD_ID);
|
||||
}
|
||||
const options = addJob.mock.calls.map((call) => call[2] as {jobKey: string; runAt: Date; skipLedger: boolean});
|
||||
expect(options.every((option) => option.skipLedger)).toBe(true);
|
||||
expect(options[0]!.jobKey).toBe(options[1]!.jobKey);
|
||||
expect(options[2]!.jobKey).not.toBe(options[1]!.jobKey);
|
||||
expect(options[0]!.runAt.getTime()).toBeGreaterThan(new Date('2026-09-21T12:00:29.999Z').getTime());
|
||||
expect(options[1]!.runAt).toEqual(options[0]!.runAt);
|
||||
expect(options[2]!.runAt).toEqual(new Date('2026-09-21T12:01:30.000Z'));
|
||||
expect(options[2]!.runAt.getTime() - options[0]!.runAt.getTime()).toBe(30_000);
|
||||
} finally {
|
||||
vi.useRealTimers();
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
@@ -9,6 +9,10 @@ import {
|
||||
} from '@app/api/instance/InstanceConfigRepository';
|
||||
import {InMemoryCassandraQueryExecutor} from '@app/api/test/InMemoryCassandraQueryExecutor';
|
||||
import {MockKVProvider} from '@app/api/test/mocks/MockKVProvider';
|
||||
import {
|
||||
DEFAULT_SCREEN_SHARE_DELIVERY_CONFIG,
|
||||
type ScreenShareDeliveryConfig,
|
||||
} from '@fluxer/schema/src/domains/admin/ScreenShareDeliverySchemas';
|
||||
import {
|
||||
DEFAULT_VOICE_NOISE_SUPPRESSION_CONFIG,
|
||||
type VoiceNoiseSuppressionConfig,
|
||||
@@ -20,6 +24,7 @@ import {
|
||||
import {afterEach, describe, expect, it, vi} from 'vitest';
|
||||
|
||||
const VOICE_NOISE_SUPPRESSION_CONFIG_KEY = 'voice_noise_suppression_config';
|
||||
const SCREEN_SHARE_DELIVERY_CONFIG_KEY = 'screen_share_delivery_config';
|
||||
const EXPERIMENT_DELIVERY_CONFIG_KEY = 'experiment_delivery_config';
|
||||
const APP_PUBLIC_CONFIG_KEY = 'app_public_config';
|
||||
const INSTANCE_POLICY_CONFIG_KEY = 'instance_policy_config';
|
||||
@@ -330,6 +335,70 @@ describe('InstanceConfigRepository', () => {
|
||||
});
|
||||
});
|
||||
|
||||
it('returns the default screen share delivery config when the key is absent', async () => {
|
||||
const executor = new CountingInMemoryCassandraQueryExecutor();
|
||||
setCassandraQueryExecutorForTesting(executor);
|
||||
const kvProvider = new MockKVProvider();
|
||||
const repository = createRepository(kvProvider);
|
||||
|
||||
await expect(repository.getScreenShareDeliveryConfig()).resolves.toEqual(DEFAULT_SCREEN_SHARE_DELIVERY_CONFIG);
|
||||
});
|
||||
|
||||
it.each([
|
||||
{name: 'unparseable text', stored: 'not-json'},
|
||||
{name: 'a json array', stored: '[]'},
|
||||
{name: 'out-of-range values', stored: '{"rollout_basis_points":99999}'},
|
||||
{name: 'a non-boolean enabled flag', stored: '{"enabled":"yes"}'},
|
||||
])('falls back to the default screen share delivery config for $name', async ({stored}) => {
|
||||
const executor = new CountingInMemoryCassandraQueryExecutor();
|
||||
setCassandraQueryExecutorForTesting(executor);
|
||||
const kvProvider = new MockKVProvider();
|
||||
const repository = createRepository(kvProvider);
|
||||
|
||||
await repository.setConfig(SCREEN_SHARE_DELIVERY_CONFIG_KEY, stored);
|
||||
|
||||
await expect(repository.getScreenShareDeliveryConfig()).resolves.toEqual(DEFAULT_SCREEN_SHARE_DELIVERY_CONFIG);
|
||||
});
|
||||
|
||||
it('round-trips a stored screen share delivery config', async () => {
|
||||
const executor = new CountingInMemoryCassandraQueryExecutor();
|
||||
setCassandraQueryExecutorForTesting(executor);
|
||||
const kvProvider = new MockKVProvider();
|
||||
const repository = createRepository(kvProvider);
|
||||
|
||||
const config: ScreenShareDeliveryConfig = {
|
||||
...DEFAULT_SCREEN_SHARE_DELIVERY_CONFIG,
|
||||
enabled: true,
|
||||
config_version: 5,
|
||||
rollout_basis_points: 2500,
|
||||
rollout_salt: 'screen-share-delivery-v2',
|
||||
included_user_ids: ['1400000000000000001'],
|
||||
excluded_user_ids: ['1400000000000000002'],
|
||||
};
|
||||
await repository.setScreenShareDeliveryConfig(config);
|
||||
|
||||
await expect(repository.getScreenShareDeliveryConfig()).resolves.toEqual(config);
|
||||
});
|
||||
|
||||
it('fills newly added screen share delivery fields from the schema defaults', async () => {
|
||||
const executor = new CountingInMemoryCassandraQueryExecutor();
|
||||
setCassandraQueryExecutorForTesting(executor);
|
||||
const kvProvider = new MockKVProvider();
|
||||
const repository = createRepository(kvProvider);
|
||||
|
||||
await repository.setConfig(
|
||||
SCREEN_SHARE_DELIVERY_CONFIG_KEY,
|
||||
JSON.stringify({enabled: true, config_version: 2, rollout_basis_points: 1000}),
|
||||
);
|
||||
|
||||
await expect(repository.getScreenShareDeliveryConfig()).resolves.toEqual({
|
||||
...DEFAULT_SCREEN_SHARE_DELIVERY_CONFIG,
|
||||
enabled: true,
|
||||
config_version: 2,
|
||||
rollout_basis_points: 1000,
|
||||
});
|
||||
});
|
||||
|
||||
it('returns the default experiment delivery config when the key is absent', async () => {
|
||||
const executor = new CountingInMemoryCassandraQueryExecutor();
|
||||
setCassandraQueryExecutorForTesting(executor);
|
||||
@@ -402,6 +471,26 @@ describe('InstanceConfigRepository', () => {
|
||||
});
|
||||
});
|
||||
|
||||
it('publishes a refresh so another repository observes the screen share delivery config', async () => {
|
||||
const executor = new CountingInMemoryCassandraQueryExecutor();
|
||||
setCassandraQueryExecutorForTesting(executor);
|
||||
const kvProvider = new MockKVProvider();
|
||||
const reader = createRepository(kvProvider);
|
||||
const writer = createRepository(kvProvider);
|
||||
|
||||
await expect(reader.getScreenShareDeliveryConfig()).resolves.toEqual(DEFAULT_SCREEN_SHARE_DELIVERY_CONFIG);
|
||||
|
||||
await writer.setScreenShareDeliveryConfig({
|
||||
...DEFAULT_SCREEN_SHARE_DELIVERY_CONFIG,
|
||||
enabled: true,
|
||||
config_version: 1,
|
||||
});
|
||||
|
||||
await vi.waitFor(async () => {
|
||||
expect(await reader.getScreenShareDeliveryConfig()).toMatchObject({enabled: true, config_version: 1});
|
||||
});
|
||||
});
|
||||
|
||||
it('uses the registration URL id as the admin-visible registration code', async () => {
|
||||
const executor = new CountingInMemoryCassandraQueryExecutor();
|
||||
setCassandraQueryExecutorForTesting(executor);
|
||||
|
||||
@@ -28,6 +28,10 @@ import {
|
||||
type GatewayRolloutConfig,
|
||||
GatewayRolloutConfigSchema,
|
||||
} from '@fluxer/schema/src/domains/admin/GatewayRolloutSchemas';
|
||||
import {
|
||||
type ScreenShareDeliveryConfig,
|
||||
ScreenShareDeliveryConfigSchema,
|
||||
} from '@fluxer/schema/src/domains/admin/ScreenShareDeliverySchemas';
|
||||
import {
|
||||
type VoiceNoiseSuppressionConfig,
|
||||
VoiceNoiseSuppressionConfigSchema,
|
||||
@@ -54,6 +58,7 @@ import {z} from 'zod';
|
||||
|
||||
const GATEWAY_ROLLOUT_CONFIG_KEY = 'gateway_rollout_config';
|
||||
const VOICE_NOISE_SUPPRESSION_CONFIG_KEY = 'voice_noise_suppression_config';
|
||||
const SCREEN_SHARE_DELIVERY_CONFIG_KEY = 'screen_share_delivery_config';
|
||||
const EXPERIMENT_DELIVERY_CONFIG_KEY = 'experiment_delivery_config';
|
||||
const REGISTRATION_CONFIG_KEY = 'registration_config';
|
||||
const REGISTRATION_URLS_KEY = 'registration_urls';
|
||||
@@ -339,6 +344,7 @@ type StoredConfigSection =
|
||||
| 'app public'
|
||||
| 'gateway rollout'
|
||||
| 'voice noise suppression'
|
||||
| 'screen share delivery'
|
||||
| 'experiment delivery'
|
||||
| 'instance policy'
|
||||
| 'integrations'
|
||||
@@ -477,6 +483,10 @@ function parseStoredVoiceNoiseSuppressionConfig(raw: string | null): VoiceNoiseS
|
||||
return parseStoredConfigOrDefault(VoiceNoiseSuppressionConfigSchema, raw, 'voice noise suppression');
|
||||
}
|
||||
|
||||
function parseStoredScreenShareDeliveryConfig(raw: string | null): ScreenShareDeliveryConfig {
|
||||
return parseStoredConfigOrDefault(ScreenShareDeliveryConfigSchema, raw, 'screen share delivery');
|
||||
}
|
||||
|
||||
function parseStoredExperimentDeliveryConfig(raw: string | null): ExperimentDeliveryConfig {
|
||||
return parseStoredConfigOrDefault(ExperimentDeliveryConfigSchema, raw, 'experiment delivery');
|
||||
}
|
||||
@@ -1004,6 +1014,7 @@ export class InstanceConfigRepository {
|
||||
parseStoredGatewayRolloutConfig(snapshot.get(GATEWAY_ROLLOUT_CONFIG_KEY) ?? null),
|
||||
);
|
||||
parseStoredVoiceNoiseSuppressionConfig(snapshot.get(VOICE_NOISE_SUPPRESSION_CONFIG_KEY) ?? null);
|
||||
parseStoredScreenShareDeliveryConfig(snapshot.get(SCREEN_SHARE_DELIVERY_CONFIG_KEY) ?? null);
|
||||
parseStoredExperimentDeliveryConfig(snapshot.get(EXPERIMENT_DELIVERY_CONFIG_KEY) ?? null);
|
||||
const policy = parseStoredInstancePolicyConfig(snapshot.get(INSTANCE_POLICY_CONFIG_KEY) ?? null);
|
||||
checkStoredConfig('registration', () =>
|
||||
@@ -1085,6 +1096,16 @@ export class InstanceConfigRepository {
|
||||
await this.setConfig(VOICE_NOISE_SUPPRESSION_CONFIG_KEY, JSON.stringify(validated));
|
||||
}
|
||||
|
||||
async getScreenShareDeliveryConfig(): Promise<ScreenShareDeliveryConfig> {
|
||||
const raw = await this.getConfig(SCREEN_SHARE_DELIVERY_CONFIG_KEY);
|
||||
return parseStoredScreenShareDeliveryConfig(raw);
|
||||
}
|
||||
|
||||
async setScreenShareDeliveryConfig(config: ScreenShareDeliveryConfig): Promise<void> {
|
||||
const validated = validateStoredConfig(ScreenShareDeliveryConfigSchema, config, 'screen share delivery');
|
||||
await this.setConfig(SCREEN_SHARE_DELIVERY_CONFIG_KEY, JSON.stringify(validated));
|
||||
}
|
||||
|
||||
async getExperimentDeliveryConfig(): Promise<ExperimentDeliveryConfig> {
|
||||
const raw = await this.getConfig(EXPERIMENT_DELIVERY_CONFIG_KEY);
|
||||
return parseStoredExperimentDeliveryConfig(raw);
|
||||
|
||||
@@ -32,10 +32,12 @@ export interface ListJobsResult {
|
||||
}
|
||||
|
||||
export abstract class IJobLedgerRepository {
|
||||
abstract createJob(input: CreateJobInput): Promise<void>;
|
||||
abstract createJob(input: CreateJobInput): Promise<Date>;
|
||||
|
||||
abstract getJob(jobId: bigint): Promise<JobByIdRow | null>;
|
||||
|
||||
abstract discardJob(jobId: bigint, createdAt: Date): Promise<void>;
|
||||
|
||||
abstract markRunning(jobId: bigint, lane: string): Promise<void>;
|
||||
|
||||
abstract markSucceeded(jobId: bigint, result: Record<string, unknown> | null): Promise<void>;
|
||||
|
||||
@@ -1,9 +1,24 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
import {setCassandraQueryExecutorForTesting} from '@app/api/database/CassandraQueryExecution';
|
||||
import {
|
||||
type CassandraQueryExecutorForTesting,
|
||||
executeQuery,
|
||||
fetchMany,
|
||||
fetchOne,
|
||||
setCassandraQueryExecutorForTesting,
|
||||
} from '@app/api/database/CassandraQueryExecution';
|
||||
import {Db, type PreparedQuery} from '@app/api/database/CassandraTypes';
|
||||
import type {JobStatus} from '@app/api/database/types/JobLedgerTypes';
|
||||
import {JobLedgerRepository} from '@app/api/jobs/JobLedgerRepository';
|
||||
import {
|
||||
EXPIRED_JOB_ERROR,
|
||||
JOB_LEDGER_TTL_SECONDS,
|
||||
JOB_STALE_AFTER_MS,
|
||||
JobLedgerRepository,
|
||||
} from '@app/api/jobs/JobLedgerRepository';
|
||||
import {describeListJobsPaging} from '@app/api/jobs/ListJobsPagingSuite';
|
||||
import {JobsActive, JobsByDayBucket, JobsById} from '@app/api/Tables';
|
||||
import {InMemoryCassandraQueryExecutor} from '@app/api/test/InMemoryCassandraQueryExecutor';
|
||||
import {createSnowflake} from '@fluxer/snowflake/src/Snowflake';
|
||||
import {afterEach, beforeEach, describe, expect, it} from 'vitest';
|
||||
|
||||
let executor: InMemoryCassandraQueryExecutor;
|
||||
@@ -22,6 +37,20 @@ async function createJob(repository: JobLedgerRepository, jobId: bigint, taskTyp
|
||||
});
|
||||
}
|
||||
|
||||
async function createJobAt(repository: JobLedgerRepository, jobId: bigint): Promise<Date> {
|
||||
return repository.createJob({
|
||||
jobId,
|
||||
taskType: 'batchGuildAuditLogMessageDeletes',
|
||||
payload: {},
|
||||
requestedByUserId: null,
|
||||
auditLogReason: null,
|
||||
maxAttempts: 3,
|
||||
runAt: null,
|
||||
jetStreamLane: null,
|
||||
jetStreamSeq: null,
|
||||
});
|
||||
}
|
||||
|
||||
async function listJobIdsByStatus(repository: JobLedgerRepository, status: JobStatus): Promise<Array<bigint>> {
|
||||
const result = await repository.listJobs({limit: 50, cursor: null, filters: {status}, maxLookbackDays: 1});
|
||||
return result.jobs.map((job) => job.job_id);
|
||||
@@ -130,6 +159,197 @@ describe('JobLedgerRepository listJobs pagination', () => {
|
||||
maxLookbackDays: 1,
|
||||
});
|
||||
|
||||
expect(result.jobs.map((job) => job.job_id)).toEqual([1_000n, 1_001n, 1_002n, 1_003n, 1_004n]);
|
||||
expect(result.jobs.map((job) => job.job_id)).toEqual([1_004n, 1_003n, 1_002n, 1_001n, 1_000n]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('JobLedgerRepository listJobs on the in-memory executor', () => {
|
||||
beforeEach(() => {
|
||||
executor = new InMemoryCassandraQueryExecutor();
|
||||
setCassandraQueryExecutorForTesting(executor);
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
executor.reset();
|
||||
setCassandraQueryExecutorForTesting(null);
|
||||
});
|
||||
|
||||
describeListJobsPaging();
|
||||
});
|
||||
|
||||
let staleSequence = 0;
|
||||
|
||||
function jobIdAgedDays(days: number): bigint {
|
||||
staleSequence += 1;
|
||||
return createSnowflake({timestamp: Date.now() - days * 86_400_000, sequence: staleSequence % 4096, workerId: 1});
|
||||
}
|
||||
|
||||
async function createAgedJob(repository: JobLedgerRepository, days: number): Promise<bigint> {
|
||||
const jobId = jobIdAgedDays(days);
|
||||
await createJob(repository, jobId, 'syncUrlBlocklists');
|
||||
return jobId;
|
||||
}
|
||||
|
||||
async function activeJobIds(repository: JobLedgerRepository): Promise<Array<bigint>> {
|
||||
return (await repository.listActiveJobs()).map((job) => job.job_id).sort((a, b) => (a < b ? -1 : 1));
|
||||
}
|
||||
|
||||
function sweep(repository: JobLedgerRepository, maxCleared = 100) {
|
||||
return repository.expireStaleActiveJobs({
|
||||
staleBeforeMs: Date.now() - JOB_STALE_AFTER_MS,
|
||||
pageSize: 100,
|
||||
maxCleared,
|
||||
});
|
||||
}
|
||||
|
||||
describe('JobLedgerRepository expireStaleActiveJobs', () => {
|
||||
beforeEach(() => {
|
||||
executor = new InMemoryCassandraQueryExecutor();
|
||||
setCassandraQueryExecutorForTesting(executor);
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
executor.reset();
|
||||
setCassandraQueryExecutorForTesting(null);
|
||||
});
|
||||
|
||||
it('dead-letters queued and running jobs the jobs stream has outlived and keeps younger ones active', async () => {
|
||||
const repository = new JobLedgerRepository();
|
||||
const staleQueued = await createAgedJob(repository, 9);
|
||||
const staleRunning = await createAgedJob(repository, 9);
|
||||
await repository.markRunning(staleRunning, 'batch');
|
||||
const weekOld = await createAgedJob(repository, 7);
|
||||
const fresh = await createAgedJob(repository, 0);
|
||||
|
||||
expect(await sweep(repository)).toEqual({cleared: 2, expired: 2, complete: true});
|
||||
|
||||
for (const jobId of [staleQueued, staleRunning]) {
|
||||
const job = await repository.getJob(jobId);
|
||||
expect(job?.status).toBe('deadletter');
|
||||
expect(job?.error_message).toBe(EXPIRED_JOB_ERROR);
|
||||
expect(job?.completed_at).toBeInstanceOf(Date);
|
||||
}
|
||||
expect(await activeJobIds(repository)).toEqual([weekOld, fresh].sort((a, b) => (a < b ? -1 : 1)));
|
||||
expect((await repository.getJob(weekOld))?.status).toBe('queued');
|
||||
});
|
||||
|
||||
it('drops a stale active row without touching a finished or missing job', async () => {
|
||||
const repository = new JobLedgerRepository();
|
||||
const finished = await createAgedJob(repository, 9);
|
||||
await repository.markSucceeded(finished, null);
|
||||
const orphan = jobIdAgedDays(9);
|
||||
for (const jobId of [finished, orphan]) {
|
||||
await executeQuery(
|
||||
JobsActive.patchByPkWithTtl({job_id: jobId}, {status: Db.set('running')}, JOB_LEDGER_TTL_SECONDS),
|
||||
);
|
||||
}
|
||||
|
||||
expect(await sweep(repository)).toEqual({cleared: 2, expired: 0, complete: true});
|
||||
|
||||
const job = await repository.getJob(finished);
|
||||
expect(job?.status).toBe('succeeded');
|
||||
expect(job?.error_message).toBeNull();
|
||||
expect(await fetchOne(JobsById.select({where: JobsById.where.eq('job_id')}).bind({job_id: orphan}))).toBeNull();
|
||||
expect(await fetchMany(JobsActive.select().bind({}))).toEqual([]);
|
||||
});
|
||||
|
||||
it('stops at its per-run cap and picks up the rest on the next run', async () => {
|
||||
const repository = new JobLedgerRepository();
|
||||
for (let index = 0; index < 3; index += 1) {
|
||||
await createAgedJob(repository, 9);
|
||||
}
|
||||
|
||||
expect(await sweep(repository, 2)).toEqual({cleared: 2, expired: 2, complete: false});
|
||||
expect(await activeJobIds(repository)).toHaveLength(1);
|
||||
expect(await sweep(repository, 2)).toEqual({cleared: 1, expired: 1, complete: true});
|
||||
expect(await activeJobIds(repository)).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('JobLedgerRepository getJob', () => {
|
||||
beforeEach(() => {
|
||||
executor = new InMemoryCassandraQueryExecutor();
|
||||
setCassandraQueryExecutorForTesting(executor);
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
executor.reset();
|
||||
setCassandraQueryExecutorForTesting(null);
|
||||
});
|
||||
|
||||
it('hides a job row that lost its status, creation time or task type', async () => {
|
||||
const repository = new JobLedgerRepository();
|
||||
await createJob(repository, 9n, 'syncUrlBlocklists');
|
||||
expect((await repository.getJob(9n))?.status).toBe('queued');
|
||||
for (const column of ['status', 'created_at', 'task_type'] as const) {
|
||||
await createJob(repository, 9n, 'syncUrlBlocklists');
|
||||
await executeQuery(JobsById.patchByPk({job_id: 9n}, {[column]: Db.clear()}));
|
||||
expect(await repository.getJob(9n)).toBeNull();
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe('JobLedgerRepository discardJob', () => {
|
||||
let inner: InMemoryCassandraQueryExecutor;
|
||||
let log: Array<string>;
|
||||
let failDeleteOn: string | null;
|
||||
|
||||
beforeEach(() => {
|
||||
inner = new InMemoryCassandraQueryExecutor();
|
||||
log = [];
|
||||
failDeleteOn = null;
|
||||
const wrapper: CassandraQueryExecutorForTesting = {
|
||||
async executeQuery<T>(query: PreparedQuery) {
|
||||
const meta = query.kvMeta;
|
||||
if (meta) log.push(`${meta.action} ${meta.table.name}`);
|
||||
if (meta?.action === 'delete' && meta.table.name === failDeleteOn) {
|
||||
throw new Error('write timeout');
|
||||
}
|
||||
return inner.executeQuery<T>(query);
|
||||
},
|
||||
executeBatch: (queries, atomic) => inner.executeBatch(queries, atomic),
|
||||
};
|
||||
setCassandraQueryExecutorForTesting(wrapper);
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
inner.reset();
|
||||
setCassandraQueryExecutorForTesting(null);
|
||||
});
|
||||
|
||||
it('removes every ledger row of a duplicate with three deletes and no read', async () => {
|
||||
const repository = new JobLedgerRepository();
|
||||
const createdAt = await createJobAt(repository, 7n);
|
||||
log.length = 0;
|
||||
|
||||
await repository.discardJob(7n, createdAt);
|
||||
|
||||
expect([...log].sort()).toEqual(['delete jobs_active', 'delete jobs_by_day_bucket', 'delete jobs_by_id']);
|
||||
expect(await repository.getJob(7n)).toBeNull();
|
||||
expect(await repository.listActiveJobs()).toEqual([]);
|
||||
expect(
|
||||
await fetchMany(
|
||||
JobsByDayBucket.select({where: JobsByDayBucket.where.eq('bucket_day')}).bind({
|
||||
bucket_day: createdAt.toISOString().slice(0, 10),
|
||||
}),
|
||||
),
|
||||
).toEqual([]);
|
||||
});
|
||||
|
||||
it('keeps deleting the other ledger rows when one delete fails', async () => {
|
||||
const repository = new JobLedgerRepository();
|
||||
const createdAt = await createJobAt(repository, 8n);
|
||||
failDeleteOn = 'jobs_by_id';
|
||||
|
||||
await expect(repository.discardJob(8n, createdAt)).rejects.toThrow('write timeout');
|
||||
|
||||
expect(await repository.listActiveJobs()).toEqual([]);
|
||||
expect(
|
||||
await fetchMany(
|
||||
JobsByDayBucket.select({where: JobsByDayBucket.where.eq('bucket_day')}).bind({
|
||||
bucket_day: createdAt.toISOString().slice(0, 10),
|
||||
}),
|
||||
),
|
||||
).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1,6 +1,14 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
import {BatchBuilder, deleteOneOrMany, fetchMany, fetchOne, upsertOne} from '@app/api/database/CassandraQueryExecution';
|
||||
import {
|
||||
BatchBuilder,
|
||||
deleteOneOrMany,
|
||||
fetchMany,
|
||||
fetchOne,
|
||||
fetchPage,
|
||||
type PagedQueryResult,
|
||||
upsertOne,
|
||||
} from '@app/api/database/CassandraQueryExecution';
|
||||
import {Db} from '@app/api/database/CassandraTypes';
|
||||
import type {JobActiveRow, JobByDayBucketRow, JobByIdRow, JobStatus} from '@app/api/database/types/JobLedgerTypes';
|
||||
import {
|
||||
@@ -11,6 +19,17 @@ import {
|
||||
type ListJobsResult,
|
||||
} from '@app/api/jobs/IJobLedgerRepository';
|
||||
import {JobsActive, JobsByDayBucket, JobsById} from '@app/api/Tables';
|
||||
import {awaitAll} from '@app/api/utils/ConcurrencyUtils';
|
||||
import {JOBS_STREAM_MAX_AGE_MS} from '@app/api/worker/JetStreamWorkerQueue';
|
||||
import {snowflakeToDate} from '@fluxer/snowflake/src/Snowflake';
|
||||
import {ms, seconds} from 'itty-time';
|
||||
|
||||
export const JOB_LEDGER_TTL_SECONDS = seconds('90 days');
|
||||
export const JOB_STALE_AFTER_MS = JOBS_STREAM_MAX_AGE_MS + ms('1 day');
|
||||
export const EXPIRED_JOB_ERROR = 'Expired from the job queue';
|
||||
|
||||
const JOB_LEDGER_RETENTION_DAYS = JOB_LEDGER_TTL_SECONDS / seconds('1 day');
|
||||
const NEWEST_FIRST = {col: 'created_at', direction: 'DESC'} as const;
|
||||
|
||||
const FETCH_JOB_BY_ID_QUERY = JobsById.select({
|
||||
where: JobsById.where.eq('job_id'),
|
||||
@@ -19,13 +38,58 @@ const FETCH_CANCEL_REQUESTED_QUERY = JobsById.select({
|
||||
where: JobsById.where.eq('job_id'),
|
||||
});
|
||||
const ACTIVE_JOBS_QUERY = JobsActive.select();
|
||||
const ACTIVE_JOB_IDS_QUERY = JobsActive.select({columns: ['job_id']});
|
||||
const JOBS_AFTER_IN_TIE_QUERY = JobsByDayBucket.select({
|
||||
where: [
|
||||
JobsByDayBucket.where.eq('bucket_day'),
|
||||
JobsByDayBucket.where.eq('created_at'),
|
||||
JobsByDayBucket.where.lt('job_id'),
|
||||
],
|
||||
orderBy: NEWEST_FIRST,
|
||||
});
|
||||
|
||||
type LedgerPosition = Pick<ListJobsCursor, 'createdAt' | 'jobId'>;
|
||||
|
||||
function bucketDayFor(d: Date): string {
|
||||
return d.toISOString().slice(0, 10);
|
||||
}
|
||||
|
||||
function previousBucketDay(day: string): string {
|
||||
const date = new Date(`${day}T00:00:00Z`);
|
||||
date.setUTCDate(date.getUTCDate() - 1);
|
||||
return bucketDayFor(date);
|
||||
}
|
||||
|
||||
function dayJobsQuery(olderThanPosition: boolean, limit: number | null) {
|
||||
return JobsByDayBucket.select({
|
||||
where: olderThanPosition
|
||||
? [JobsByDayBucket.where.eq('bucket_day'), JobsByDayBucket.where.lt('created_at')]
|
||||
: JobsByDayBucket.where.eq('bucket_day'),
|
||||
orderBy: NEWEST_FIRST,
|
||||
...(limit === null ? {} : {limit}),
|
||||
});
|
||||
}
|
||||
|
||||
async function fetchDayAfter(
|
||||
bucketDay: string,
|
||||
after: LedgerPosition | null,
|
||||
limit: number | null,
|
||||
): Promise<{rows: Array<JobByDayBucketRow>; exhausted: boolean}> {
|
||||
if (after === null) {
|
||||
const rows = await fetchMany<JobByDayBucketRow>(dayJobsQuery(false, limit).bind({bucket_day: bucketDay}));
|
||||
return {rows, exhausted: limit === null || rows.length < limit};
|
||||
}
|
||||
const ties = await fetchMany<JobByDayBucketRow>(
|
||||
JOBS_AFTER_IN_TIE_QUERY.bind({bucket_day: bucketDay, created_at: after.createdAt, job_id: after.jobId}),
|
||||
);
|
||||
const older = await fetchMany<JobByDayBucketRow>(
|
||||
dayJobsQuery(true, limit).bind({bucket_day: bucketDay, created_at: after.createdAt}),
|
||||
);
|
||||
return {rows: [...ties, ...older], exhausted: limit === null || older.length < limit};
|
||||
}
|
||||
|
||||
export class JobLedgerRepository extends IJobLedgerRepository {
|
||||
async createJob(input: CreateJobInput): Promise<void> {
|
||||
async createJob(input: CreateJobInput): Promise<Date> {
|
||||
const now = new Date();
|
||||
const status: JobStatus = 'queued';
|
||||
const idRow: JobByIdRow = {
|
||||
@@ -68,39 +132,62 @@ export class JobLedgerRepository extends IJobLedgerRepository {
|
||||
started_at: null,
|
||||
};
|
||||
const batch = new BatchBuilder();
|
||||
batch.addPrepared(JobsById.insert(idRow));
|
||||
batch.addPrepared(JobsByDayBucket.insert(bucketRow));
|
||||
batch.addPrepared(JobsActive.insert(activeRow));
|
||||
batch.addPrepared(JobsById.insertWithTtl(idRow, JOB_LEDGER_TTL_SECONDS));
|
||||
batch.addPrepared(JobsByDayBucket.insertWithTtl(bucketRow, JOB_LEDGER_TTL_SECONDS));
|
||||
batch.addPrepared(JobsActive.insertWithTtl(activeRow, JOB_LEDGER_TTL_SECONDS));
|
||||
await batch.executeChunked(10, false);
|
||||
return now;
|
||||
}
|
||||
|
||||
async getJob(jobId: bigint): Promise<JobByIdRow | null> {
|
||||
return fetchOne<JobByIdRow>(FETCH_JOB_BY_ID_QUERY.bind({job_id: jobId}));
|
||||
const row = await fetchOne<JobByIdRow>(FETCH_JOB_BY_ID_QUERY.bind({job_id: jobId}));
|
||||
return row?.created_at && row.task_type && row.status ? row : null;
|
||||
}
|
||||
|
||||
async discardJob(jobId: bigint, createdAt: Date): Promise<void> {
|
||||
await awaitAll(
|
||||
[
|
||||
deleteOneOrMany(
|
||||
JobsByDayBucket.deleteByPk({bucket_day: bucketDayFor(createdAt), created_at: createdAt, job_id: jobId}),
|
||||
),
|
||||
deleteOneOrMany(JobsById.deleteByPk({job_id: jobId})),
|
||||
deleteOneOrMany(JobsActive.deleteByPk({job_id: jobId})),
|
||||
],
|
||||
'Ledger discard left rows behind',
|
||||
);
|
||||
}
|
||||
|
||||
async markRunning(jobId: bigint, lane: string): Promise<void> {
|
||||
const startedAt = new Date();
|
||||
const status: JobStatus = 'running';
|
||||
await upsertOne(
|
||||
JobsById.patchByPk(
|
||||
JobsById.patchByPkWithTtl(
|
||||
{job_id: jobId},
|
||||
{status: Db.set(status), started_at: Db.set(startedAt), jet_stream_lane: Db.set(lane)},
|
||||
JOB_LEDGER_TTL_SECONDS,
|
||||
),
|
||||
);
|
||||
await upsertOne(
|
||||
JobsActive.patchByPkWithTtl(
|
||||
{job_id: jobId},
|
||||
{status: Db.set(status), started_at: Db.set(startedAt)},
|
||||
JOB_LEDGER_TTL_SECONDS,
|
||||
),
|
||||
);
|
||||
await upsertOne(JobsActive.patchByPk({job_id: jobId}, {status: Db.set(status), started_at: Db.set(startedAt)}));
|
||||
}
|
||||
|
||||
async markSucceeded(jobId: bigint, result: Record<string, unknown> | null): Promise<void> {
|
||||
const completedAt = new Date();
|
||||
const status: JobStatus = 'succeeded';
|
||||
await upsertOne(
|
||||
JobsById.patchByPk(
|
||||
JobsById.patchByPkWithTtl(
|
||||
{job_id: jobId},
|
||||
{
|
||||
status: Db.set(status),
|
||||
completed_at: Db.set(completedAt),
|
||||
result: result === null ? Db.clear() : Db.set(JSON.stringify(result)),
|
||||
},
|
||||
JOB_LEDGER_TTL_SECONDS,
|
||||
),
|
||||
);
|
||||
await deleteOneOrMany(JobsActive.deleteByPk({job_id: jobId}));
|
||||
@@ -109,7 +196,13 @@ export class JobLedgerRepository extends IJobLedgerRepository {
|
||||
async markCancelled(jobId: bigint): Promise<void> {
|
||||
const completedAt = new Date();
|
||||
const status: JobStatus = 'cancelled';
|
||||
await upsertOne(JobsById.patchByPk({job_id: jobId}, {status: Db.set(status), completed_at: Db.set(completedAt)}));
|
||||
await upsertOne(
|
||||
JobsById.patchByPkWithTtl(
|
||||
{job_id: jobId},
|
||||
{status: Db.set(status), completed_at: Db.set(completedAt)},
|
||||
JOB_LEDGER_TTL_SECONDS,
|
||||
),
|
||||
);
|
||||
await deleteOneOrMany(JobsActive.deleteByPk({job_id: jobId}));
|
||||
}
|
||||
|
||||
@@ -117,9 +210,10 @@ export class JobLedgerRepository extends IJobLedgerRepository {
|
||||
const completedAt = new Date();
|
||||
const status: JobStatus = 'deadletter';
|
||||
await upsertOne(
|
||||
JobsById.patchByPk(
|
||||
JobsById.patchByPkWithTtl(
|
||||
{job_id: jobId},
|
||||
{status: Db.set(status), completed_at: Db.set(completedAt), error_message: Db.set(errorMessage)},
|
||||
JOB_LEDGER_TTL_SECONDS,
|
||||
),
|
||||
);
|
||||
await deleteOneOrMany(JobsActive.deleteByPk({job_id: jobId}));
|
||||
@@ -127,27 +221,30 @@ export class JobLedgerRepository extends IJobLedgerRepository {
|
||||
|
||||
async reportProgress(jobId: bigint, current: number, total: number | null, message: string | null): Promise<void> {
|
||||
await upsertOne(
|
||||
JobsById.patchByPk(
|
||||
JobsById.patchByPkWithTtl(
|
||||
{job_id: jobId},
|
||||
{
|
||||
progress_current: Db.set(BigInt(current)),
|
||||
progress_total: total === null ? Db.clear() : Db.set(BigInt(total)),
|
||||
progress_message: message === null ? Db.clear() : Db.set(message),
|
||||
},
|
||||
JOB_LEDGER_TTL_SECONDS,
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
async setContextLink(jobId: bigint, link: string): Promise<void> {
|
||||
await upsertOne(JobsById.patchByPk({job_id: jobId}, {context_link: Db.set(link)}));
|
||||
await upsertOne(JobsById.patchByPkWithTtl({job_id: jobId}, {context_link: Db.set(link)}, JOB_LEDGER_TTL_SECONDS));
|
||||
}
|
||||
|
||||
async setJetStreamSeq(jobId: bigint, seq: string): Promise<void> {
|
||||
await upsertOne(JobsById.patchByPk({job_id: jobId}, {jet_stream_seq: Db.set(seq)}));
|
||||
await upsertOne(JobsById.patchByPkWithTtl({job_id: jobId}, {jet_stream_seq: Db.set(seq)}, JOB_LEDGER_TTL_SECONDS));
|
||||
}
|
||||
|
||||
async requestCancel(jobId: bigint): Promise<void> {
|
||||
await upsertOne(JobsById.patchByPk({job_id: jobId}, {cancel_requested: Db.set(true)}));
|
||||
await upsertOne(
|
||||
JobsById.patchByPkWithTtl({job_id: jobId}, {cancel_requested: Db.set(true)}, JOB_LEDGER_TTL_SECONDS),
|
||||
);
|
||||
}
|
||||
|
||||
async isCancelRequested(jobId: bigint): Promise<boolean> {
|
||||
@@ -160,7 +257,9 @@ export class JobLedgerRepository extends IJobLedgerRepository {
|
||||
async incrementAttempts(jobId: bigint): Promise<void> {
|
||||
const row = await this.getJob(jobId);
|
||||
if (!row) return;
|
||||
await upsertOne(JobsById.patchByPk({job_id: jobId}, {attempts: Db.set(row.attempts + 1)}));
|
||||
await upsertOne(
|
||||
JobsById.patchByPkWithTtl({job_id: jobId}, {attempts: Db.set(row.attempts + 1)}, JOB_LEDGER_TTL_SECONDS),
|
||||
);
|
||||
}
|
||||
|
||||
async listJobs(opts: {
|
||||
@@ -169,59 +268,74 @@ export class JobLedgerRepository extends IJobLedgerRepository {
|
||||
filters: ListJobsFilters;
|
||||
maxLookbackDays: number;
|
||||
}): Promise<ListJobsResult> {
|
||||
const {limit, cursor, filters, maxLookbackDays} = opts;
|
||||
const startBucket = cursor ? new Date(`${cursor.bucketDay}T00:00:00Z`) : new Date();
|
||||
const hasFilters = Boolean(
|
||||
filters.status ||
|
||||
filters.taskType ||
|
||||
(filters.requestedByUserId !== undefined && filters.requestedByUserId !== null),
|
||||
);
|
||||
const collected: Array<JobByIdRow> = [];
|
||||
let nextCursor: ListJobsCursor | null = null;
|
||||
for (let dayOffset = 0; dayOffset <= maxLookbackDays && collected.length < limit; dayOffset++) {
|
||||
const bucketDate = new Date(startBucket);
|
||||
bucketDate.setUTCDate(bucketDate.getUTCDate() - dayOffset);
|
||||
const bucketDay = bucketDayFor(bucketDate);
|
||||
const remaining = limit - collected.length + 1;
|
||||
const bucketLimit = hasFilters ? {} : {limit: remaining};
|
||||
const useCursor = dayOffset === 0 && cursor !== null;
|
||||
let bucketRows: Array<JobByDayBucketRow>;
|
||||
if (useCursor && cursor) {
|
||||
const query = JobsByDayBucket.select({
|
||||
where: [JobsByDayBucket.where.eq('bucket_day'), JobsByDayBucket.where.lt('created_at')],
|
||||
...bucketLimit,
|
||||
});
|
||||
bucketRows = await fetchMany<JobByDayBucketRow>(
|
||||
query.bind({bucket_day: bucketDay, created_at: cursor.createdAt}),
|
||||
);
|
||||
} else {
|
||||
const query = JobsByDayBucket.select({
|
||||
where: JobsByDayBucket.where.eq('bucket_day'),
|
||||
...bucketLimit,
|
||||
});
|
||||
bucketRows = await fetchMany<JobByDayBucketRow>(query.bind({bucket_day: bucketDay}));
|
||||
const {limit, cursor, filters} = opts;
|
||||
const now = Date.now();
|
||||
const lookbackDays = Math.min(opts.maxLookbackDays, JOB_LEDGER_RETENTION_DAYS);
|
||||
const oldestDay = bucketDayFor(new Date(now - lookbackDays * ms('1 day')));
|
||||
const requestedBy = filters.requestedByUserId ?? null;
|
||||
const wholeDays = Boolean(filters.status || filters.taskType || requestedBy !== null);
|
||||
const jobs: Array<JobByIdRow> = [];
|
||||
let lastRow: JobByDayBucketRow | null = null;
|
||||
let day = bucketDayFor(new Date(Math.min(cursor ? cursor.createdAt.getTime() : now, now)));
|
||||
let after: LedgerPosition | null = cursor;
|
||||
while (jobs.length < limit && day >= oldestDay) {
|
||||
const {rows, exhausted} = await fetchDayAfter(day, after, wholeDays ? null : limit - jobs.length);
|
||||
for (const row of rows) {
|
||||
after = {createdAt: row.created_at, jobId: row.job_id};
|
||||
if (filters.taskType && row.task_type !== filters.taskType) continue;
|
||||
if (requestedBy !== null && row.requested_by_user_id !== requestedBy) continue;
|
||||
const job = await this.getJob(row.job_id);
|
||||
if (!job || (filters.status && job.status !== filters.status)) continue;
|
||||
jobs.push(job);
|
||||
lastRow = row;
|
||||
if (jobs.length === limit) break;
|
||||
}
|
||||
for (const r of bucketRows) {
|
||||
if (filters.taskType && r.task_type !== filters.taskType) continue;
|
||||
if (filters.requestedByUserId !== undefined && filters.requestedByUserId !== null) {
|
||||
if (r.requested_by_user_id !== filters.requestedByUserId) continue;
|
||||
}
|
||||
const fullRow = await this.getJob(r.job_id);
|
||||
if (!fullRow) continue;
|
||||
if (filters.status && fullRow.status !== filters.status) continue;
|
||||
if (collected.length >= limit) {
|
||||
nextCursor = {bucketDay, createdAt: r.created_at, jobId: r.job_id};
|
||||
break;
|
||||
}
|
||||
collected.push(fullRow);
|
||||
if (exhausted) {
|
||||
day = previousBucketDay(day);
|
||||
after = null;
|
||||
}
|
||||
if (nextCursor) break;
|
||||
}
|
||||
if (nextCursor === null && collected.length >= limit) {
|
||||
const last = collected[collected.length - 1];
|
||||
nextCursor = {bucketDay: bucketDayFor(last.created_at), createdAt: last.created_at, jobId: last.job_id};
|
||||
}
|
||||
return {jobs: collected, nextCursor};
|
||||
return {
|
||||
jobs,
|
||||
nextCursor:
|
||||
lastRow && jobs.length === limit
|
||||
? {bucketDay: lastRow.bucket_day, createdAt: lastRow.created_at, jobId: lastRow.job_id}
|
||||
: null,
|
||||
};
|
||||
}
|
||||
|
||||
async expireStaleActiveJobs(opts: {
|
||||
staleBeforeMs: number;
|
||||
pageSize: number;
|
||||
maxCleared: number;
|
||||
}): Promise<{cleared: number; expired: number; complete: boolean}> {
|
||||
let cleared = 0;
|
||||
let expired = 0;
|
||||
let pageState: string | null = null;
|
||||
do {
|
||||
const page: PagedQueryResult<Pick<JobActiveRow, 'job_id'>> = await fetchPage(
|
||||
ACTIVE_JOB_IDS_QUERY.bind({}),
|
||||
undefined,
|
||||
{
|
||||
pageSize: opts.pageSize,
|
||||
pageState,
|
||||
},
|
||||
);
|
||||
for (const {job_id: jobId} of page.rows) {
|
||||
if (snowflakeToDate(jobId).getTime() >= opts.staleBeforeMs) continue;
|
||||
if (cleared >= opts.maxCleared) return {cleared, expired, complete: false};
|
||||
const job = await this.getJob(jobId);
|
||||
if (job?.status === 'queued' || job?.status === 'running') {
|
||||
await this.markDeadletter(jobId, EXPIRED_JOB_ERROR);
|
||||
expired += 1;
|
||||
} else {
|
||||
await deleteOneOrMany(JobsActive.deleteByPk({job_id: jobId}));
|
||||
}
|
||||
cleared += 1;
|
||||
}
|
||||
pageState = page.pageState;
|
||||
} while (pageState !== null);
|
||||
return {cleared, expired, complete: true};
|
||||
}
|
||||
|
||||
async listActiveJobs(): Promise<Array<JobByIdRow>> {
|
||||
|
||||
@@ -0,0 +1,228 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
import {upsertOne} from '@app/api/database/CassandraQueryExecution';
|
||||
import type {JobByIdRow, JobStatus} from '@app/api/database/types/JobLedgerTypes';
|
||||
import type {ListJobsCursor, ListJobsFilters} from '@app/api/jobs/IJobLedgerRepository';
|
||||
import {JobLedgerRepository} from '@app/api/jobs/JobLedgerRepository';
|
||||
import {JobsByDayBucket, JobsById} from '@app/api/Tables';
|
||||
import {afterEach, beforeEach, describe, expect, it, vi} from 'vitest';
|
||||
|
||||
const DAY_MS = 86_400_000;
|
||||
const HOUR_MS = 3_600_000;
|
||||
const TODAY_NOON = new Date(`${new Date().toISOString().slice(0, 10)}T12:00:00.000Z`);
|
||||
const ADMIN_USER_ID = 4_242n;
|
||||
|
||||
interface ListedPage {
|
||||
ids: Array<bigint>;
|
||||
cursor: ListJobsCursor | null;
|
||||
}
|
||||
|
||||
function at(daysAgo: number, hour: number): Date {
|
||||
return new Date(TODAY_NOON.getTime() - daysAgo * DAY_MS + (hour - 12) * HOUR_MS);
|
||||
}
|
||||
|
||||
async function seedJob(
|
||||
jobId: bigint,
|
||||
createdAt: Date,
|
||||
opts: {taskType?: string; requestedBy?: bigint | null; status?: JobStatus; bucketOnly?: boolean} = {},
|
||||
): Promise<void> {
|
||||
const taskType = opts.taskType ?? 'A';
|
||||
const requestedBy = opts.requestedBy ?? null;
|
||||
await upsertOne(
|
||||
JobsByDayBucket.insert({
|
||||
bucket_day: createdAt.toISOString().slice(0, 10),
|
||||
created_at: createdAt,
|
||||
job_id: jobId,
|
||||
task_type: taskType,
|
||||
status: 'queued',
|
||||
requested_by_user_id: requestedBy,
|
||||
}),
|
||||
);
|
||||
if (opts.bucketOnly) return;
|
||||
const row: JobByIdRow = {
|
||||
job_id: jobId,
|
||||
task_type: taskType,
|
||||
status: opts.status ?? 'queued',
|
||||
progress_current: null,
|
||||
progress_total: null,
|
||||
progress_message: null,
|
||||
payload: '{}',
|
||||
result: null,
|
||||
error_message: null,
|
||||
created_at: createdAt,
|
||||
started_at: null,
|
||||
completed_at: null,
|
||||
requested_by_user_id: requestedBy,
|
||||
audit_log_reason: null,
|
||||
jet_stream_seq: null,
|
||||
jet_stream_lane: 'batch',
|
||||
attempts: 0,
|
||||
max_attempts: 5,
|
||||
run_at: null,
|
||||
cancel_requested: false,
|
||||
context_link: null,
|
||||
};
|
||||
await upsertOne(JobsById.insert(row));
|
||||
}
|
||||
|
||||
async function listPages(opts: {
|
||||
limit: number;
|
||||
filters?: ListJobsFilters;
|
||||
maxLookbackDays?: number;
|
||||
cursor?: ListJobsCursor | null;
|
||||
}): Promise<Array<ListedPage>> {
|
||||
const repository = new JobLedgerRepository();
|
||||
const pages: Array<ListedPage> = [];
|
||||
let cursor = opts.cursor ?? null;
|
||||
for (let page = 0; page < 50; page += 1) {
|
||||
const result = await repository.listJobs({
|
||||
limit: opts.limit,
|
||||
cursor,
|
||||
filters: opts.filters ?? {},
|
||||
maxLookbackDays: opts.maxLookbackDays ?? 14,
|
||||
});
|
||||
pages.push({ids: result.jobs.map((job) => job.job_id), cursor: result.nextCursor});
|
||||
if (result.nextCursor === null) return pages;
|
||||
cursor = {
|
||||
bucketDay: result.nextCursor.bucketDay,
|
||||
createdAt: new Date(result.nextCursor.createdAt.toISOString()),
|
||||
jobId: BigInt(result.nextCursor.jobId.toString()),
|
||||
};
|
||||
}
|
||||
throw new Error('listJobs never stopped paging');
|
||||
}
|
||||
|
||||
function expectPages(pages: Array<ListedPage>, limit: number, expected: Array<Array<bigint>>): void {
|
||||
expect(pages.map((page) => page.ids)).toEqual(expected);
|
||||
const ids = pages.flatMap((page) => page.ids);
|
||||
expect(new Set(ids).size).toBe(ids.length);
|
||||
for (const page of pages.slice(0, -1)) {
|
||||
expect(page.ids).toHaveLength(limit);
|
||||
expect(page.cursor?.jobId).toBe(page.ids.at(-1));
|
||||
}
|
||||
expect(pages.at(-1)?.cursor).toBeNull();
|
||||
}
|
||||
|
||||
export function describeListJobsPaging(): void {
|
||||
describe('listJobs paging', () => {
|
||||
beforeEach(() => {
|
||||
vi.useFakeTimers({toFake: ['Date']});
|
||||
vi.setSystemTime(TODAY_NOON);
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
vi.useRealTimers();
|
||||
});
|
||||
|
||||
it('walks one day newest first through a tie group larger than the page', async () => {
|
||||
await seedJob(20n, at(0, 11));
|
||||
for (const jobId of [11n, 12n, 13n, 14n, 15n]) {
|
||||
await seedJob(jobId, at(0, 10));
|
||||
}
|
||||
await seedJob(9n, at(0, 9));
|
||||
await seedJob(5n, at(0, 8));
|
||||
await seedJob(6n, at(0, 8));
|
||||
|
||||
const pages = await listPages({limit: 2});
|
||||
|
||||
expectPages(pages, 2, [[20n, 15n], [14n, 13n], [12n, 11n], [9n, 6n], [5n]]);
|
||||
expect(pages[0]?.cursor).toEqual({
|
||||
bucketDay: at(0, 10).toISOString().slice(0, 10),
|
||||
createdAt: at(0, 10),
|
||||
jobId: 15n,
|
||||
});
|
||||
});
|
||||
|
||||
it('crosses days and keeps the lookback window anchored on today', async () => {
|
||||
await seedJob(41n, at(0, 11));
|
||||
await seedJob(40n, at(0, 10));
|
||||
await seedJob(32n, at(1, 11));
|
||||
await seedJob(31n, at(1, 10));
|
||||
await seedJob(30n, at(1, 9));
|
||||
await seedJob(21n, at(2, 11));
|
||||
await seedJob(20n, at(2, 10));
|
||||
await seedJob(10n, at(3, 11));
|
||||
|
||||
expectPages(await listPages({limit: 2, maxLookbackDays: 2}), 2, [[41n, 40n], [32n, 31n], [30n, 21n], [20n]]);
|
||||
expectPages(await listPages({limit: 2, maxLookbackDays: 3}), 2, [
|
||||
[41n, 40n],
|
||||
[32n, 31n],
|
||||
[30n, 21n],
|
||||
[20n, 10n],
|
||||
[],
|
||||
]);
|
||||
const outside = await new JobLedgerRepository().listJobs({
|
||||
limit: 2,
|
||||
cursor: {bucketDay: at(3, 12).toISOString().slice(0, 10), createdAt: at(3, 12), jobId: 1n},
|
||||
filters: {},
|
||||
maxLookbackDays: 2,
|
||||
});
|
||||
expect(outside).toEqual({jobs: [], nextCursor: null});
|
||||
});
|
||||
|
||||
it('starts a cursor dated in the future at today', async () => {
|
||||
await seedJob(2n, at(0, 2));
|
||||
await seedJob(1n, at(1, 2));
|
||||
const farFuture = new Date('9999-12-31T00:00:00.000Z');
|
||||
|
||||
expectPages(await listPages({limit: 5, cursor: {bucketDay: '9999-12-31', createdAt: farFuture, jobId: 1n}}), 5, [
|
||||
[2n, 1n],
|
||||
]);
|
||||
}, 2_000);
|
||||
|
||||
it('never lists a day past the 90-day retention', async () => {
|
||||
await seedJob(890n, at(89, 11));
|
||||
await seedJob(910n, at(91, 11));
|
||||
|
||||
expectPages(await listPages({limit: 10, maxLookbackDays: 120}), 10, [[890n]]);
|
||||
});
|
||||
|
||||
it('fills a page past rows whose job record is missing instead of leaving the day', async () => {
|
||||
for (let hour = 1; hour <= 8; hour += 1) {
|
||||
await seedJob(BigInt(hour), at(0, hour), {bucketOnly: [3, 6, 7].includes(hour)});
|
||||
}
|
||||
|
||||
expectPages(await listPages({limit: 3}), 3, [
|
||||
[8n, 5n, 4n],
|
||||
[2n, 1n],
|
||||
]);
|
||||
});
|
||||
|
||||
it('fills pages through task type, requester and status filters across days', async () => {
|
||||
await seedJob(60n, at(0, 11));
|
||||
await seedJob(59n, at(0, 10.5), {taskType: 'B'});
|
||||
await seedJob(58n, at(0, 10));
|
||||
await seedJob(57n, at(0, 9.5), {taskType: 'B', requestedBy: ADMIN_USER_ID});
|
||||
await seedJob(56n, at(0, 9), {status: 'succeeded'});
|
||||
await seedJob(55n, at(0, 8.5), {taskType: 'B'});
|
||||
await seedJob(50n, at(1, 11));
|
||||
await seedJob(49n, at(1, 10.5), {taskType: 'B', requestedBy: ADMIN_USER_ID});
|
||||
await seedJob(48n, at(1, 10), {bucketOnly: true});
|
||||
await seedJob(47n, at(1, 9.5));
|
||||
|
||||
expectPages(await listPages({limit: 2, filters: {taskType: 'A'}}), 2, [[60n, 58n], [56n, 50n], [47n]]);
|
||||
expectPages(await listPages({limit: 1, filters: {status: 'succeeded'}}), 1, [[56n], []]);
|
||||
expectPages(await listPages({limit: 5, filters: {requestedByUserId: ADMIN_USER_ID}}), 5, [[57n, 49n]]);
|
||||
});
|
||||
|
||||
it('resumes after a cursor whose job was discarded between pages', async () => {
|
||||
for (let hour = 1; hour <= 5; hour += 1) {
|
||||
await seedJob(BigInt(hour), at(0, hour));
|
||||
}
|
||||
const repository = new JobLedgerRepository();
|
||||
const first = await repository.listJobs({limit: 2, cursor: null, filters: {}, maxLookbackDays: 14});
|
||||
expect(first.jobs.map((job) => job.job_id)).toEqual([5n, 4n]);
|
||||
|
||||
await repository.discardJob(4n, at(0, 4));
|
||||
|
||||
expectPages(await listPages({limit: 2, cursor: first.nextCursor}), 2, [[3n, 2n], [1n]]);
|
||||
});
|
||||
|
||||
it('returns an empty page after a page that filled exactly', async () => {
|
||||
await seedJob(1n, at(0, 1));
|
||||
await seedJob(2n, at(0, 2));
|
||||
|
||||
expectPages(await listPages({limit: 2}), 2, [[2n, 1n], []]);
|
||||
});
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,600 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
import {spawnSync} from 'node:child_process';
|
||||
import {createServer} from 'node:net';
|
||||
import {BatchBuilder, setCassandraQueryExecutorForTesting} from '@app/api/database/CassandraQueryExecution';
|
||||
import {Db} from '@app/api/database/CassandraTypes';
|
||||
import {ensurePostgresKvSchema, PostgresKvQueryExecutor} from '@app/api/database/PostgresKvQueryExecutor';
|
||||
import type {JobActiveRow, JobByDayBucketRow, JobByIdRow, JobStatus} from '@app/api/database/types/JobLedgerTypes';
|
||||
import {
|
||||
EXPIRED_JOB_ERROR,
|
||||
JOB_LEDGER_TTL_SECONDS,
|
||||
JOB_STALE_AFTER_MS,
|
||||
JobLedgerRepository,
|
||||
} from '@app/api/jobs/JobLedgerRepository';
|
||||
import {describeListJobsPaging} from '@app/api/jobs/ListJobsPagingSuite';
|
||||
import {expireLegacyJobLedgerRows} from '@app/api/jobs/PostgresJobLedgerExpiry';
|
||||
import {JobsActive, JobsByDayBucket, JobsById} from '@app/api/Tables';
|
||||
import {startDockerContainer} from '@app/api/test/DockerTestContainer';
|
||||
import {InMemoryCassandraQueryExecutor} from '@app/api/test/InMemoryCassandraQueryExecutor';
|
||||
import {createSnowflake} from '@fluxer/snowflake/src/Snowflake';
|
||||
import {
|
||||
getDefaultPostgresClient,
|
||||
type IPostgresClient,
|
||||
initPostgres,
|
||||
shutdownPostgres,
|
||||
} from '@pkgs/postgres/src/Client';
|
||||
import {afterAll, beforeAll, beforeEach, describe, expect, it} from 'vitest';
|
||||
|
||||
const KV_TABLE = 'kv_job_ledger_expiry';
|
||||
const CONTAINER = `fluxer-kvjobs-${process.pid.toString(36)}-${Date.now().toString(36)}`;
|
||||
const dockerAvailable = spawnSync('docker', ['version'], {stdio: 'ignore'}).status === 0;
|
||||
const DAY_MS = 86_400_000;
|
||||
const ADMIN_USER_ID = 1_234_567_890_123n;
|
||||
|
||||
async function sleep(ms: number): Promise<void> {
|
||||
await new Promise((resolve) => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
async function freePort(): Promise<number> {
|
||||
return new Promise((resolve, reject) => {
|
||||
const server = createServer();
|
||||
server.on('error', reject);
|
||||
server.listen(0, '127.0.0.1', () => {
|
||||
const address = server.address();
|
||||
if (typeof address === 'string' || address === null) {
|
||||
reject(new Error('no port'));
|
||||
return;
|
||||
}
|
||||
const port = address.port;
|
||||
server.close(() => resolve(port));
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
let sequence = 0;
|
||||
|
||||
function jobIdAgedDays(days: number): bigint {
|
||||
sequence += 1;
|
||||
return createSnowflake({timestamp: Date.now() - days * DAY_MS, sequence: sequence % 4096, workerId: 1});
|
||||
}
|
||||
|
||||
interface LegacyJob {
|
||||
jobId: bigint;
|
||||
createdAt: Date;
|
||||
}
|
||||
|
||||
async function seedLegacyJob(
|
||||
executor: PostgresKvQueryExecutor,
|
||||
opts: {ageDays: number; status: JobStatus; requestedBy: bigint | null; active: boolean},
|
||||
): Promise<LegacyJob> {
|
||||
const jobId = jobIdAgedDays(opts.ageDays);
|
||||
const createdAt = new Date(Date.now() - opts.ageDays * DAY_MS);
|
||||
const idRow: JobByIdRow = {
|
||||
job_id: jobId,
|
||||
task_type: opts.requestedBy === null ? 'flushUserActivityBuffer' : 'refreshSearchIndex',
|
||||
status: opts.status,
|
||||
progress_current: null,
|
||||
progress_total: null,
|
||||
progress_message: null,
|
||||
payload: '{}',
|
||||
result: null,
|
||||
error_message: null,
|
||||
created_at: createdAt,
|
||||
started_at: null,
|
||||
completed_at: opts.status === 'succeeded' ? createdAt : null,
|
||||
requested_by_user_id: opts.requestedBy,
|
||||
audit_log_reason: null,
|
||||
jet_stream_seq: '1',
|
||||
jet_stream_lane: 'batch',
|
||||
attempts: 0,
|
||||
max_attempts: 5,
|
||||
run_at: null,
|
||||
cancel_requested: false,
|
||||
context_link: null,
|
||||
};
|
||||
const bucketRow: JobByDayBucketRow = {
|
||||
bucket_day: createdAt.toISOString().slice(0, 10),
|
||||
created_at: createdAt,
|
||||
job_id: jobId,
|
||||
task_type: idRow.task_type,
|
||||
status: 'queued',
|
||||
requested_by_user_id: opts.requestedBy,
|
||||
};
|
||||
await executor.executeQuery(JobsById.insert(idRow));
|
||||
await executor.executeQuery(JobsByDayBucket.insert(bucketRow));
|
||||
if (opts.active) {
|
||||
const activeRow: JobActiveRow = {
|
||||
job_id: jobId,
|
||||
task_type: idRow.task_type,
|
||||
status: opts.status,
|
||||
requested_by_user_id: opts.requestedBy,
|
||||
created_at: createdAt,
|
||||
started_at: null,
|
||||
};
|
||||
await executor.executeQuery(JobsActive.insert(activeRow));
|
||||
}
|
||||
return {jobId, createdAt};
|
||||
}
|
||||
|
||||
describe.skipIf(!dockerAvailable)('job ledger expiry against postgres', () => {
|
||||
let raw: IPostgresClient;
|
||||
let executor: PostgresKvQueryExecutor;
|
||||
|
||||
async function jobRows(
|
||||
jobId: bigint,
|
||||
): Promise<Array<{table_name: string; expires_at: Date | null; row_data: never}>> {
|
||||
const result = await raw.query<{table_name: string; expires_at: Date | null; row_data: never}>(
|
||||
`SELECT table_name, expires_at, row_data FROM ${KV_TABLE}
|
||||
WHERE table_name IN ('jobs_by_id', 'jobs_active', 'jobs_by_day_bucket')
|
||||
AND (row_key = $1 OR split_part(row_key, chr(31), 3) = $1)
|
||||
ORDER BY table_name`,
|
||||
[JSON.stringify({__fluxer_type: 'bigint', value: jobId.toString()})],
|
||||
);
|
||||
return result.rows;
|
||||
}
|
||||
|
||||
async function waitForLockWait(): Promise<void> {
|
||||
for (let attempt = 0; attempt < 400; attempt += 1) {
|
||||
const waiting = await raw.query<{n: number}>(
|
||||
`SELECT count(*)::int AS n FROM pg_stat_activity WHERE datname = current_database() AND wait_event_type = 'Lock'`,
|
||||
);
|
||||
if (waiting.rows[0]!.n > 0) return;
|
||||
await sleep(25);
|
||||
}
|
||||
throw new Error('the pass never waited on the writer');
|
||||
}
|
||||
|
||||
async function forgetLedgerExpiry(): Promise<void> {
|
||||
await raw.query(
|
||||
`UPDATE ${KV_TABLE} SET expires_at = NULL WHERE table_name IN ('jobs_by_id', 'jobs_by_day_bucket')`,
|
||||
);
|
||||
}
|
||||
|
||||
async function forgetExpiryOf(...jobIds: Array<bigint>): Promise<void> {
|
||||
const keys = jobIds.map((jobId) => JSON.stringify({__fluxer_type: 'bigint', value: jobId.toString()}));
|
||||
await raw.query(
|
||||
`UPDATE ${KV_TABLE} SET expires_at = NULL
|
||||
WHERE table_name IN ('jobs_by_id', 'jobs_by_day_bucket')
|
||||
AND (row_key = ANY($1::text[]) OR split_part(row_key, chr(31), 3) = ANY($1::text[]))`,
|
||||
[keys],
|
||||
);
|
||||
}
|
||||
|
||||
beforeAll(async () => {
|
||||
const port = await freePort();
|
||||
startDockerContainer([
|
||||
'run',
|
||||
'-d',
|
||||
'--name',
|
||||
CONTAINER,
|
||||
'-e',
|
||||
'POSTGRES_USER=fluxer',
|
||||
'-e',
|
||||
'POSTGRES_PASSWORD=fluxer',
|
||||
'-e',
|
||||
'POSTGRES_DB=fluxer',
|
||||
'-p',
|
||||
`127.0.0.1:${port}:5432`,
|
||||
'postgres:16-alpine',
|
||||
'-c',
|
||||
'fsync=off',
|
||||
]);
|
||||
let ready = false;
|
||||
for (let attempt = 0; attempt < 180 && !ready; attempt += 1) {
|
||||
await sleep(500);
|
||||
const probe = spawnSync('docker', ['exec', CONTAINER, 'pg_isready', '-U', 'fluxer', '-d', 'fluxer'], {
|
||||
stdio: 'ignore',
|
||||
});
|
||||
if (probe.status !== 0) continue;
|
||||
try {
|
||||
await initPostgres({
|
||||
url: `postgres://fluxer:[email protected]:${port}/fluxer`,
|
||||
maxConnections: 4,
|
||||
kvTable: KV_TABLE,
|
||||
});
|
||||
await getDefaultPostgresClient().query('SELECT 1');
|
||||
ready = true;
|
||||
} catch {
|
||||
await shutdownPostgres().catch(() => {});
|
||||
}
|
||||
}
|
||||
if (!ready) throw new Error('postgres never came up');
|
||||
raw = getDefaultPostgresClient();
|
||||
await ensurePostgresKvSchema(raw);
|
||||
executor = new PostgresKvQueryExecutor(raw);
|
||||
}, 900_000);
|
||||
|
||||
beforeEach(async () => {
|
||||
await raw.query(`DELETE FROM ${KV_TABLE}`);
|
||||
setCassandraQueryExecutorForTesting(executor);
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
setCassandraQueryExecutorForTesting(new InMemoryCassandraQueryExecutor());
|
||||
await shutdownPostgres().catch(() => {});
|
||||
spawnSync('docker', ['rm', '-f', CONTAINER], {stdio: 'ignore'});
|
||||
});
|
||||
|
||||
it('writes every ledger row with an expiry and keeps it through the job lifecycle', async () => {
|
||||
const repository = new JobLedgerRepository();
|
||||
const jobId = jobIdAgedDays(0);
|
||||
await repository.createJob({
|
||||
jobId,
|
||||
taskType: 'refreshSearchIndex',
|
||||
payload: {},
|
||||
requestedByUserId: ADMIN_USER_ID,
|
||||
auditLogReason: null,
|
||||
maxAttempts: 5,
|
||||
runAt: null,
|
||||
jetStreamLane: 'batch',
|
||||
jetStreamSeq: null,
|
||||
});
|
||||
await repository.setJetStreamSeq(jobId, '7');
|
||||
await repository.markRunning(jobId, 'batch');
|
||||
await repository.reportProgress(jobId, 1, 2, 'half');
|
||||
|
||||
const running = await jobRows(jobId);
|
||||
expect(running.map((row) => row.table_name)).toEqual(['jobs_active', 'jobs_by_day_bucket', 'jobs_by_id']);
|
||||
for (const row of running) {
|
||||
expect(row.expires_at).not.toBeNull();
|
||||
const remainingSeconds = (row.expires_at!.getTime() - Date.now()) / 1000;
|
||||
expect(remainingSeconds).toBeGreaterThan(JOB_LEDGER_TTL_SECONDS - 60);
|
||||
expect(remainingSeconds).toBeLessThanOrEqual(JOB_LEDGER_TTL_SECONDS);
|
||||
}
|
||||
|
||||
await repository.markSucceeded(jobId, {ok: true});
|
||||
const done = await jobRows(jobId);
|
||||
expect(done.map((row) => row.table_name)).toEqual(['jobs_by_day_bucket', 'jobs_by_id']);
|
||||
expect(done.every((row) => row.expires_at !== null)).toBe(true);
|
||||
expect((await repository.getJob(jobId))?.status).toBe('succeeded');
|
||||
});
|
||||
|
||||
it('never leaves a row without an expiry when a patch lands on a job that is gone', async () => {
|
||||
const repository = new JobLedgerRepository();
|
||||
const patches: Array<(jobId: bigint) => Promise<void>> = [
|
||||
(jobId) => repository.markRunning(jobId, 'batch'),
|
||||
(jobId) => repository.markSucceeded(jobId, null),
|
||||
(jobId) => repository.markCancelled(jobId),
|
||||
(jobId) => repository.markDeadletter(jobId, 'boom'),
|
||||
(jobId) => repository.reportProgress(jobId, 1, null, null),
|
||||
(jobId) => repository.setContextLink(jobId, '/admin/jobs'),
|
||||
(jobId) => repository.setJetStreamSeq(jobId, '1'),
|
||||
(jobId) => repository.requestCancel(jobId),
|
||||
];
|
||||
for (const patch of patches) {
|
||||
const jobId = jobIdAgedDays(0);
|
||||
await patch(jobId);
|
||||
const rows = await jobRows(jobId);
|
||||
expect(rows.length).toBeGreaterThan(0);
|
||||
expect(rows.every((row) => row.expires_at !== null)).toBe(true);
|
||||
expect(await repository.getJob(jobId)).toBeNull();
|
||||
}
|
||||
expect(await repository.listActiveJobs()).toEqual([]);
|
||||
});
|
||||
|
||||
it('discards every row of a job that never reached the stream', async () => {
|
||||
const repository = new JobLedgerRepository();
|
||||
const jobId = jobIdAgedDays(0);
|
||||
const createdAt = await repository.createJob({
|
||||
jobId,
|
||||
taskType: 'batchGuildAuditLogMessageDeletes',
|
||||
payload: {guildId: '1'},
|
||||
requestedByUserId: null,
|
||||
auditLogReason: null,
|
||||
maxAttempts: 3,
|
||||
runAt: new Date(Date.now() + 30_000),
|
||||
jetStreamLane: 'batch',
|
||||
jetStreamSeq: null,
|
||||
});
|
||||
expect(await jobRows(jobId)).toHaveLength(3);
|
||||
|
||||
await repository.discardJob(jobId, createdAt);
|
||||
|
||||
expect(await jobRows(jobId)).toEqual([]);
|
||||
});
|
||||
|
||||
it('clears legacy rows by the same rules the expiry now enforces', async () => {
|
||||
const repository = new JobLedgerRepository();
|
||||
const cronDone = await seedLegacyJob(executor, {
|
||||
ageDays: 20,
|
||||
status: 'succeeded',
|
||||
requestedBy: null,
|
||||
active: false,
|
||||
});
|
||||
const cronStuck = await seedLegacyJob(executor, {ageDays: 20, status: 'queued', requestedBy: null, active: true});
|
||||
const partialId = jobIdAgedDays(20);
|
||||
await executor.executeQuery(
|
||||
JobsById.patchByPk({job_id: partialId}, {status: Db.set('succeeded'), completed_at: Db.set(new Date())}),
|
||||
);
|
||||
const adminDone = await seedLegacyJob(executor, {
|
||||
ageDays: 20,
|
||||
status: 'succeeded',
|
||||
requestedBy: ADMIN_USER_ID,
|
||||
active: false,
|
||||
});
|
||||
const adminStuck = await seedLegacyJob(executor, {
|
||||
ageDays: 20,
|
||||
status: 'queued',
|
||||
requestedBy: ADMIN_USER_ID,
|
||||
active: true,
|
||||
});
|
||||
const adminRunning = await seedLegacyJob(executor, {
|
||||
ageDays: 20,
|
||||
status: 'running',
|
||||
requestedBy: ADMIN_USER_ID,
|
||||
active: true,
|
||||
});
|
||||
const adminAncient = await seedLegacyJob(executor, {
|
||||
ageDays: 100,
|
||||
status: 'succeeded',
|
||||
requestedBy: ADMIN_USER_ID,
|
||||
active: false,
|
||||
});
|
||||
const cronInFlight = await seedLegacyJob(executor, {
|
||||
ageDays: 2,
|
||||
status: 'queued',
|
||||
requestedBy: null,
|
||||
active: true,
|
||||
});
|
||||
await forgetLedgerExpiry();
|
||||
const fresh = jobIdAgedDays(0);
|
||||
await repository.createJob({
|
||||
jobId: fresh,
|
||||
taskType: 'syncUrlBlocklists',
|
||||
payload: {},
|
||||
requestedByUserId: null,
|
||||
auditLogReason: null,
|
||||
maxAttempts: 5,
|
||||
runAt: null,
|
||||
jetStreamLane: 'batch',
|
||||
jetStreamSeq: null,
|
||||
});
|
||||
const freshBefore = await jobRows(fresh);
|
||||
await raw.query(
|
||||
`INSERT INTO ${KV_TABLE} (table_name, partition_key, row_key, row_data) VALUES ('users', 'u1', 'u1', '{}'::jsonb)`,
|
||||
);
|
||||
|
||||
expect(await repository.getJob(partialId)).toBeNull();
|
||||
|
||||
const first = await expireLegacyJobLedgerRows(raw, Date.now() + 60_000);
|
||||
expect(first).toEqual({deleted: 10, expiring: 6, complete: true});
|
||||
|
||||
expect(await jobRows(cronDone.jobId)).toEqual([]);
|
||||
expect(await jobRows(cronStuck.jobId)).toEqual([]);
|
||||
expect(await jobRows(partialId)).toEqual([]);
|
||||
expect(await jobRows(adminAncient.jobId)).toEqual([]);
|
||||
|
||||
for (const kept of [adminDone, adminStuck, adminRunning]) {
|
||||
const rows = await jobRows(kept.jobId);
|
||||
expect(rows.map((row) => row.table_name)).toEqual(['jobs_by_day_bucket', 'jobs_by_id']);
|
||||
for (const row of rows) {
|
||||
const expected = kept.createdAt.getTime() + JOB_LEDGER_TTL_SECONDS * 1000;
|
||||
expect(Math.abs(row.expires_at!.getTime() - expected)).toBeLessThan(2000);
|
||||
}
|
||||
}
|
||||
expect((await repository.getJob(adminDone.jobId))?.status).toBe('succeeded');
|
||||
for (const stuck of [adminStuck, adminRunning]) {
|
||||
const expired = await repository.getJob(stuck.jobId);
|
||||
expect(expired?.status).toBe('deadletter');
|
||||
expect(expired?.error_message).toBe('Expired from the job queue');
|
||||
expect(expired?.completed_at).toBeInstanceOf(Date);
|
||||
}
|
||||
|
||||
const inFlight = await jobRows(cronInFlight.jobId);
|
||||
expect(inFlight.map((row) => row.table_name)).toEqual(['jobs_active', 'jobs_by_day_bucket', 'jobs_by_id']);
|
||||
expect(inFlight.every((row) => row.expires_at === null)).toBe(true);
|
||||
expect((await repository.getJob(cronInFlight.jobId))?.status).toBe('queued');
|
||||
|
||||
expect(await jobRows(fresh)).toEqual(freshBefore);
|
||||
const users = await raw.query(`SELECT expires_at FROM ${KV_TABLE} WHERE table_name = 'users'`);
|
||||
expect(users.rows).toEqual([{expires_at: null}]);
|
||||
|
||||
const listed = await repository.listJobs({limit: 50, cursor: null, filters: {}, maxLookbackDays: 30});
|
||||
expect(listed.jobs.map((job) => job.job_id).sort()).toEqual(
|
||||
[adminDone.jobId, adminStuck.jobId, adminRunning.jobId, cronInFlight.jobId, fresh].sort(),
|
||||
);
|
||||
|
||||
for (let pass = 0; pass < 2; pass += 1) {
|
||||
expect(await expireLegacyJobLedgerRows(raw, Date.now() + 60_000)).toEqual({
|
||||
deleted: 0,
|
||||
expiring: 0,
|
||||
complete: true,
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
it('runs again a day after a clean pass and clears rows an older image wrote in between', async () => {
|
||||
expect(await expireLegacyJobLedgerRows(raw, Date.now() + 60_000)).toEqual({
|
||||
deleted: 0,
|
||||
expiring: 0,
|
||||
complete: true,
|
||||
});
|
||||
const rolledBack = await seedLegacyJob(executor, {ageDays: 30, status: 'queued', requestedBy: null, active: true});
|
||||
await forgetLedgerExpiry();
|
||||
expect(await expireLegacyJobLedgerRows(raw, Date.now() + 60_000)).toBeNull();
|
||||
|
||||
await raw.query(
|
||||
`UPDATE ${KV_TABLE} SET row_data = jsonb_build_object('applied_at', now() - interval '2 days') WHERE row_key = 'job_ledger_expiry_v1'`,
|
||||
);
|
||||
expect(await expireLegacyJobLedgerRows(raw, Date.now() + 60_000)).toEqual({
|
||||
deleted: 3,
|
||||
expiring: 0,
|
||||
complete: true,
|
||||
});
|
||||
expect(await jobRows(rolledBack.jobId)).toEqual([]);
|
||||
expect(await expireLegacyJobLedgerRows(raw, Date.now() + 60_000)).toEqual({
|
||||
deleted: 0,
|
||||
expiring: 0,
|
||||
complete: true,
|
||||
});
|
||||
expect(await expireLegacyJobLedgerRows(raw, Date.now() + 60_000)).toBeNull();
|
||||
});
|
||||
|
||||
it('expires stale active jobs page by page without fighting the legacy pass', async () => {
|
||||
const repository = new JobLedgerRepository();
|
||||
const legacyFirst = await seedLegacyJob(executor, {
|
||||
ageDays: 20,
|
||||
status: 'queued',
|
||||
requestedBy: ADMIN_USER_ID,
|
||||
active: true,
|
||||
});
|
||||
await forgetExpiryOf(legacyFirst.jobId);
|
||||
await expireLegacyJobLedgerRows(raw, Date.now() + 60_000);
|
||||
const legacyFirstRows = await jobRows(legacyFirst.jobId);
|
||||
|
||||
const stale: Array<bigint> = [];
|
||||
for (let index = 0; index < 4; index += 1) {
|
||||
const jobId = jobIdAgedDays(9);
|
||||
await repository.createJob({
|
||||
jobId,
|
||||
taskType: 'syncUrlBlocklists',
|
||||
payload: {},
|
||||
requestedByUserId: null,
|
||||
auditLogReason: null,
|
||||
maxAttempts: 5,
|
||||
runAt: null,
|
||||
jetStreamLane: 'batch',
|
||||
jetStreamSeq: null,
|
||||
});
|
||||
stale.push(jobId);
|
||||
}
|
||||
await repository.markRunning(stale[0]!, 'batch');
|
||||
const fresh = jobIdAgedDays(0);
|
||||
await repository.createJob({
|
||||
jobId: fresh,
|
||||
taskType: 'syncUrlBlocklists',
|
||||
payload: {},
|
||||
requestedByUserId: null,
|
||||
auditLogReason: null,
|
||||
maxAttempts: 5,
|
||||
runAt: null,
|
||||
jetStreamLane: 'batch',
|
||||
jetStreamSeq: null,
|
||||
});
|
||||
const legacyStale = await seedLegacyJob(executor, {
|
||||
ageDays: 20,
|
||||
status: 'running',
|
||||
requestedBy: ADMIN_USER_ID,
|
||||
active: true,
|
||||
});
|
||||
const legacyYoung = await seedLegacyJob(executor, {ageDays: 2, status: 'queued', requestedBy: null, active: true});
|
||||
await forgetExpiryOf(legacyStale.jobId, legacyYoung.jobId);
|
||||
const orphan = jobIdAgedDays(9);
|
||||
await executor.executeQuery(
|
||||
JobsActive.patchByPkWithTtl({job_id: orphan}, {status: Db.set('running')}, JOB_LEDGER_TTL_SECONDS),
|
||||
);
|
||||
|
||||
const sweep = () =>
|
||||
repository.expireStaleActiveJobs({staleBeforeMs: Date.now() - JOB_STALE_AFTER_MS, pageSize: 2, maxCleared: 100});
|
||||
expect(await sweep()).toEqual({cleared: 6, expired: 5, complete: true});
|
||||
|
||||
const active = (await repository.listActiveJobs()).map((job) => job.job_id).sort();
|
||||
expect(active).toEqual([fresh, legacyYoung.jobId].sort());
|
||||
for (const jobId of [...stale, legacyStale.jobId]) {
|
||||
const job = await repository.getJob(jobId);
|
||||
expect(job?.status).toBe('deadletter');
|
||||
expect(job?.error_message).toBe(EXPIRED_JOB_ERROR);
|
||||
const byId = (await jobRows(jobId)).find((row) => row.table_name === 'jobs_by_id');
|
||||
expect(Math.abs(byId!.expires_at!.getTime() - (Date.now() + JOB_LEDGER_TTL_SECONDS * 1000))).toBeLessThan(60_000);
|
||||
}
|
||||
expect(await jobRows(orphan)).toEqual([]);
|
||||
expect(await jobRows(legacyFirst.jobId)).toEqual(legacyFirstRows);
|
||||
|
||||
const legacyStaleById = (await jobRows(legacyStale.jobId)).find((row) => row.table_name === 'jobs_by_id');
|
||||
await expireLegacyJobLedgerRows(raw, Date.now() + 60_000);
|
||||
expect((await jobRows(legacyStale.jobId)).find((row) => row.table_name === 'jobs_by_id')).toEqual(legacyStaleById);
|
||||
expect((await repository.listActiveJobs()).map((job) => job.job_id)).toContain(legacyYoung.jobId);
|
||||
expect(await sweep()).toEqual({cleared: 0, expired: 0, complete: true});
|
||||
});
|
||||
|
||||
it('leaves rows alone when a live writer gives them an expiry while the pass waits on them', async () => {
|
||||
const cronDone = await seedLegacyJob(executor, {
|
||||
ageDays: 20,
|
||||
status: 'succeeded',
|
||||
requestedBy: null,
|
||||
active: false,
|
||||
});
|
||||
const adminDone = await seedLegacyJob(executor, {
|
||||
ageDays: 20,
|
||||
status: 'succeeded',
|
||||
requestedBy: ADMIN_USER_ID,
|
||||
active: false,
|
||||
});
|
||||
await forgetLedgerExpiry();
|
||||
const liveKeys = [cronDone.jobId, adminDone.jobId].map((jobId) =>
|
||||
JSON.stringify({__fluxer_type: 'bigint', value: jobId.toString()}),
|
||||
);
|
||||
let written!: () => void;
|
||||
const writerHoldsRows = new Promise<void>((resolve) => {
|
||||
written = resolve;
|
||||
});
|
||||
let release!: () => void;
|
||||
const released = new Promise<void>((resolve) => {
|
||||
release = resolve;
|
||||
});
|
||||
const writer = raw.transaction(async (db) => {
|
||||
await db.query(
|
||||
`UPDATE ${KV_TABLE} SET expires_at = now() + interval '1 hour', updated_at = now() WHERE table_name = 'jobs_by_id' AND row_key = ANY($1::text[])`,
|
||||
[liveKeys],
|
||||
);
|
||||
written();
|
||||
await released;
|
||||
});
|
||||
await writerHoldsRows;
|
||||
const pass = expireLegacyJobLedgerRows(raw, Date.now() + 60_000);
|
||||
await waitForLockWait();
|
||||
release();
|
||||
await writer;
|
||||
|
||||
expect(await pass).toEqual({deleted: 1, expiring: 1, complete: true});
|
||||
for (const job of [cronDone, adminDone]) {
|
||||
const byId = (await jobRows(job.jobId)).find((row) => row.table_name === 'jobs_by_id');
|
||||
expect(byId).toBeDefined();
|
||||
const remainingSeconds = (byId!.expires_at!.getTime() - Date.now()) / 1000;
|
||||
expect(remainingSeconds).toBeGreaterThan(3000);
|
||||
expect(remainingSeconds).toBeLessThanOrEqual(3660);
|
||||
}
|
||||
expect((await new JobLedgerRepository().getJob(adminDone.jobId))?.status).toBe('succeeded');
|
||||
});
|
||||
|
||||
it('pages through more legacy rows than one page holds and stops at its deadline', async () => {
|
||||
const batch = new BatchBuilder();
|
||||
for (let index = 0; index < 2300; index += 1) {
|
||||
const jobId = jobIdAgedDays(30);
|
||||
const createdAt = new Date(Date.now() - 30 * DAY_MS);
|
||||
batch.addPrepared(
|
||||
JobsByDayBucket.insert({
|
||||
bucket_day: createdAt.toISOString().slice(0, 10),
|
||||
created_at: createdAt,
|
||||
job_id: jobId,
|
||||
task_type: 'flushUserActivityBuffer',
|
||||
status: 'queued',
|
||||
requested_by_user_id: null,
|
||||
}),
|
||||
);
|
||||
}
|
||||
await batch.executeChunked(500, false);
|
||||
await forgetLedgerExpiry();
|
||||
|
||||
expect(await expireLegacyJobLedgerRows(raw, Date.now() - 1)).toEqual({
|
||||
deleted: 0,
|
||||
expiring: 0,
|
||||
complete: false,
|
||||
});
|
||||
expect(await expireLegacyJobLedgerRows(raw, Date.now() + 60_000)).toEqual({
|
||||
deleted: 2300,
|
||||
expiring: 0,
|
||||
complete: true,
|
||||
});
|
||||
const left = await raw.query(`SELECT count(*)::int AS n FROM ${KV_TABLE} WHERE table_name = 'jobs_by_day_bucket'`);
|
||||
expect(left.rows[0]).toEqual({n: 0});
|
||||
expect(await expireLegacyJobLedgerRows(raw, Date.now() + 60_000)).toEqual({
|
||||
deleted: 0,
|
||||
expiring: 0,
|
||||
complete: true,
|
||||
});
|
||||
expect(await expireLegacyJobLedgerRows(raw, Date.now() + 60_000)).toBeNull();
|
||||
});
|
||||
describeListJobsPaging();
|
||||
});
|
||||
@@ -0,0 +1,149 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
import {postgresKvPassIsFresh, recordPostgresKvCleanPass} from '@app/api/database/PostgresKvQueryExecutor';
|
||||
import {EXPIRED_JOB_ERROR, JOB_LEDGER_TTL_SECONDS, JOB_STALE_AFTER_MS} from '@app/api/jobs/JobLedgerRepository';
|
||||
import {FLUXER_EPOCH} from '@fluxer/constants/src/Core';
|
||||
import {TIMESTAMP_SHIFT} from '@fluxer/snowflake/src/Snowflake';
|
||||
import {type IPostgresClient, quoteIdentifier} from '@pkgs/postgres/src/Client';
|
||||
import {ms} from 'itty-time';
|
||||
|
||||
const LEGACY_JOB_LEDGER_MARKER = 'job_ledger_expiry_v1';
|
||||
const PAGE_SIZE = 2000;
|
||||
const CLEAN_PASS_INTERVAL_MS = ms('1 day');
|
||||
const BIGINT_KEY_PREFIX = '{"__fluxer_type":"bigint","value":"';
|
||||
const BIGINT_KEY_SUFFIX = '"}';
|
||||
|
||||
function bigintKeyExpr(key: string): string {
|
||||
const prefixLength = BIGINT_KEY_PREFIX.length;
|
||||
const affixLength = prefixLength + BIGINT_KEY_SUFFIX.length;
|
||||
return `(CASE WHEN left(${key}, ${prefixLength}) = '${BIGINT_KEY_PREFIX}' AND right(${key}, ${BIGINT_KEY_SUFFIX.length}) = '${BIGINT_KEY_SUFFIX}' THEN substr(${key}, ${prefixLength + 1}, length(${key}) - ${affixLength})::numeric END)`;
|
||||
}
|
||||
|
||||
interface LedgerTable {
|
||||
name: string;
|
||||
jobIdExpr: string;
|
||||
deleteStale: boolean;
|
||||
markStaleDeadletter: boolean;
|
||||
}
|
||||
|
||||
const LEDGER_TABLES: ReadonlyArray<LedgerTable> = [
|
||||
{
|
||||
name: 'jobs_active',
|
||||
jobIdExpr: bigintKeyExpr('kv.row_key'),
|
||||
deleteStale: true,
|
||||
markStaleDeadletter: false,
|
||||
},
|
||||
{
|
||||
name: 'jobs_by_id',
|
||||
jobIdExpr: bigintKeyExpr('kv.row_key'),
|
||||
deleteStale: false,
|
||||
markStaleDeadletter: true,
|
||||
},
|
||||
{
|
||||
name: 'jobs_by_day_bucket',
|
||||
jobIdExpr: bigintKeyExpr('split_part(kv.row_key, chr(31), 3)'),
|
||||
deleteStale: false,
|
||||
markStaleDeadletter: false,
|
||||
},
|
||||
];
|
||||
|
||||
export interface LegacyJobLedgerExpiryResult {
|
||||
deleted: number;
|
||||
expiring: number;
|
||||
complete: boolean;
|
||||
}
|
||||
|
||||
function pageSql(table: string, target: LedgerTable): string {
|
||||
const staleRemovable = target.deleteStale ? 'c.created_at < $7' : '(c.created_at < $7 AND c.system_job)';
|
||||
const removable = `c.job_id IS NULL OR c.created_at < $6 OR ${staleRemovable}`;
|
||||
const rowData = target.markStaleDeadletter
|
||||
? `CASE WHEN c.unfinished THEN kv.row_data || jsonb_build_object('status', 'deadletter', 'error_message', $9::text, 'completed_at', jsonb_build_object('__fluxer_type', 'date', 'value', $10::text)) ELSE kv.row_data END`
|
||||
: 'kv.row_data';
|
||||
return `
|
||||
WITH page AS (
|
||||
SELECT kv.row_key, kv.row_data, ${target.jobIdExpr} AS job_id
|
||||
FROM ${table} kv
|
||||
WHERE kv.table_name = $1 AND kv.expires_at IS NULL AND kv.row_key > $2
|
||||
ORDER BY kv.row_key
|
||||
LIMIT $3
|
||||
), classified AS (
|
||||
SELECT
|
||||
page.row_key,
|
||||
page.job_id,
|
||||
to_timestamp(((div(page.job_id, $4::numeric) + $5::numeric) / 1000)::double precision) AS created_at,
|
||||
COALESCE(page.row_data -> 'requested_by_user_id', 'null'::jsonb) = 'null'::jsonb AS system_job,
|
||||
COALESCE(page.row_data ->> 'status' IN ('queued', 'running'), false) AS unfinished
|
||||
FROM page
|
||||
), removed AS (
|
||||
DELETE FROM ${table} kv
|
||||
USING classified c
|
||||
WHERE kv.table_name = $1 AND kv.row_key = c.row_key AND kv.expires_at IS NULL AND (${removable})
|
||||
RETURNING kv.row_key
|
||||
), expiring AS (
|
||||
UPDATE ${table} kv
|
||||
SET expires_at = c.created_at + make_interval(secs => $8::double precision), updated_at = now(), row_data = ${rowData}
|
||||
FROM classified c
|
||||
WHERE kv.table_name = $1 AND kv.row_key = c.row_key AND kv.expires_at IS NULL AND c.created_at < $7 AND NOT (${removable})
|
||||
RETURNING kv.row_key
|
||||
)
|
||||
SELECT
|
||||
(SELECT max(row_key) FROM page) AS last_row_key,
|
||||
(SELECT count(*) FROM page) AS scanned,
|
||||
(SELECT count(*) FROM removed) AS deleted,
|
||||
(SELECT count(*) FROM expiring) AS expiring`;
|
||||
}
|
||||
|
||||
export async function expireLegacyJobLedgerRows(
|
||||
client: IPostgresClient,
|
||||
deadlineMs: number,
|
||||
): Promise<LegacyJobLedgerExpiryResult | null> {
|
||||
const table = quoteIdentifier(client.kvTable());
|
||||
if (await postgresKvPassIsFresh(client, LEGACY_JOB_LEDGER_MARKER, CLEAN_PASS_INTERVAL_MS)) {
|
||||
return null;
|
||||
}
|
||||
const now = Date.now();
|
||||
const retentionCutoff = new Date(now - JOB_LEDGER_TTL_SECONDS * 1000);
|
||||
const staleCutoff = new Date(now - JOB_STALE_AFTER_MS);
|
||||
const completedAt = new Date(now).toISOString();
|
||||
let scanned = 0;
|
||||
let deleted = 0;
|
||||
let expiring = 0;
|
||||
for (const target of LEDGER_TABLES) {
|
||||
const sql = pageSql(table, target);
|
||||
const deadletterValues = target.markStaleDeadletter ? [EXPIRED_JOB_ERROR, completedAt] : [];
|
||||
let cursor = '';
|
||||
for (;;) {
|
||||
if (Date.now() >= deadlineMs) {
|
||||
return {deleted, expiring, complete: false};
|
||||
}
|
||||
const result = await client.query<{
|
||||
last_row_key: string | null;
|
||||
scanned: string;
|
||||
deleted: string;
|
||||
expiring: string;
|
||||
}>(sql, [
|
||||
target.name,
|
||||
cursor,
|
||||
PAGE_SIZE,
|
||||
(1n << TIMESTAMP_SHIFT).toString(),
|
||||
FLUXER_EPOCH.toString(),
|
||||
retentionCutoff,
|
||||
staleCutoff,
|
||||
JOB_LEDGER_TTL_SECONDS,
|
||||
...deadletterValues,
|
||||
]);
|
||||
const page = result.rows[0];
|
||||
if (!page || page.last_row_key === null) {
|
||||
break;
|
||||
}
|
||||
scanned += Number(page.scanned);
|
||||
deleted += Number(page.deleted);
|
||||
expiring += Number(page.expiring);
|
||||
cursor = page.last_row_key;
|
||||
}
|
||||
}
|
||||
if (scanned === 0) {
|
||||
await recordPostgresKvCleanPass(client, LEGACY_JOB_LEDGER_MARKER);
|
||||
}
|
||||
return {deleted, expiring, complete: true};
|
||||
}
|
||||
@@ -1,7 +1,8 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
import {AdminRepository} from '@app/api/admin/AdminRepository';
|
||||
import {BANNED_FILE_SHAS_REFRESH_CHANNEL} from '@app/api/constants/ContentModeration';
|
||||
import {Config} from '@app/api/Config';
|
||||
import {BANNED_FILE_SHAS_REFRESH_CHANNEL, isBlocklistFeedFileSha} from '@app/api/constants/ContentModeration';
|
||||
import {Logger} from '@app/api/Logger';
|
||||
import {RefreshSubscription} from '@app/api/utils/RefreshSubscription';
|
||||
import type {IKVProvider} from '@pkgs/kv_client/src/IKVProvider';
|
||||
@@ -38,8 +39,11 @@ class FileShaCache {
|
||||
async refresh(): Promise<void> {
|
||||
const rows = await this.adminRepository.loadAllBannedFileShas();
|
||||
const next = new Set<string>();
|
||||
const includeFeedRows = Config.blocklistFeeds.enabled;
|
||||
for (const row of rows) {
|
||||
if (row.sha256_hex) next.add(row.sha256_hex.toLowerCase());
|
||||
if (!row.sha256_hex) continue;
|
||||
if (!includeFeedRows && isBlocklistFeedFileSha(row)) continue;
|
||||
next.add(row.sha256_hex.toLowerCase());
|
||||
}
|
||||
this.banned = next;
|
||||
this.consecutiveFailures = 0;
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
import {AdminRepository} from '@app/api/admin/AdminRepository';
|
||||
import {Config} from '@app/api/Config';
|
||||
import {BANNED_URL_DOMAINS_REFRESH_CHANNEL, BANNED_URLS_REFRESH_CHANNEL} from '@app/api/constants/ContentModeration';
|
||||
import type {IStorageService} from '@app/api/infrastructure/IStorageService';
|
||||
import {Logger} from '@app/api/Logger';
|
||||
@@ -68,7 +69,7 @@ class UrlBlocklistCache {
|
||||
}
|
||||
|
||||
private async loadFeedUrls(): Promise<Set<string>> {
|
||||
if (!this.storageService) return new Set();
|
||||
if (!this.storageService || !Config.blocklistFeeds.enabled) return new Set();
|
||||
const lines = await readLinesFromS3(this.storageService, RISK_S3_KEYS.feedUrls);
|
||||
return new Set(lines);
|
||||
}
|
||||
|
||||
@@ -16924,6 +16924,63 @@
|
||||
}
|
||||
}
|
||||
},
|
||||
"/users/@me/mfa/webauthn/two-factor": {
|
||||
"put": {
|
||||
"operationId": "set_webauthn_two_factor",
|
||||
"summary": "Set WebAuthn two-factor authentication",
|
||||
"tags": ["Users"],
|
||||
"responses": {
|
||||
"200": {
|
||||
"description": "Success",
|
||||
"content": {"application/json": {"schema": {"$ref": "#/components/schemas/WebAuthnTwoFactorResponse"}}}
|
||||
},
|
||||
"400": {
|
||||
"description": "Bad Request - The request was malformed or contained invalid data",
|
||||
"content": {"application/json": {"schema": {"$ref": "#/components/schemas/Error"}}}
|
||||
},
|
||||
"401": {
|
||||
"description": "Unauthorized - Authentication is required or the token is invalid",
|
||||
"content": {"application/json": {"schema": {"$ref": "#/components/schemas/Error"}}}
|
||||
},
|
||||
"403": {
|
||||
"description": "Forbidden - You do not have permission to perform this action",
|
||||
"content": {"application/json": {"schema": {"$ref": "#/components/schemas/Error"}}}
|
||||
},
|
||||
"429": {
|
||||
"description": "Too Many Requests - You are being rate limited",
|
||||
"content": {"application/json": {"schema": {"$ref": "#/components/schemas/ThrottledError"}}},
|
||||
"headers": {
|
||||
"Retry-After": {
|
||||
"description": "Number of seconds to wait before retrying (only on 429)",
|
||||
"schema": {"type": "integer"}
|
||||
},
|
||||
"X-RateLimit-Limit": {
|
||||
"description": "The number of requests that can be made in the current window",
|
||||
"schema": {"type": "integer"}
|
||||
},
|
||||
"X-RateLimit-Remaining": {
|
||||
"description": "The number of remaining requests that can be made",
|
||||
"schema": {"type": "integer"}
|
||||
},
|
||||
"X-RateLimit-Reset": {
|
||||
"description": "Unix timestamp when the rate limit resets",
|
||||
"schema": {"type": "integer"}
|
||||
}
|
||||
}
|
||||
},
|
||||
"500": {
|
||||
"description": "Internal Server Error - An unexpected error occurred",
|
||||
"content": {"application/json": {"schema": {"$ref": "#/components/schemas/Error"}}}
|
||||
}
|
||||
},
|
||||
"description": "Choose whether registered passkeys are required as a second factor when signing in with email and password. Enabling requires at least one registered credential and mints backup codes when the account has none. Requires sudo mode verification.",
|
||||
"security": [{"sessionToken": []}],
|
||||
"requestBody": {
|
||||
"required": true,
|
||||
"content": {"application/json": {"schema": {"$ref": "#/components/schemas/WebAuthnTwoFactorRequest"}}}
|
||||
}
|
||||
}
|
||||
},
|
||||
"/users/@me/mobile-devices": {
|
||||
"post": {
|
||||
"operationId": "register_mobile_push_device",
|
||||
@@ -21879,10 +21936,17 @@
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"totp": {"type": "boolean", "description": "Whether TOTP is enabled"},
|
||||
"webauthn": {"type": "boolean", "description": "Whether WebAuthn is enabled"},
|
||||
"has_mfa": {"type": "boolean", "description": "Whether any MFA method is enabled"}
|
||||
"webauthn": {
|
||||
"type": "boolean",
|
||||
"description": "Whether the account has at least one registered WebAuthn credential"
|
||||
},
|
||||
"backup_codes": {
|
||||
"type": "boolean",
|
||||
"description": "Whether the account has at least one unconsumed backup code"
|
||||
},
|
||||
"has_mfa": {"type": "boolean", "description": "Whether the account can satisfy a sudo mode challenge"}
|
||||
},
|
||||
"required": ["totp", "webauthn", "has_mfa"],
|
||||
"required": ["totp", "webauthn", "backup_codes", "has_mfa"],
|
||||
"additionalProperties": false
|
||||
},
|
||||
"VoiceActivitySharingUpdateRequest": {
|
||||
@@ -22782,6 +22846,62 @@
|
||||
"required": ["device_id"],
|
||||
"additionalProperties": false
|
||||
},
|
||||
"WebAuthnTwoFactorRequest": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"enabled": {
|
||||
"type": "boolean",
|
||||
"description": "Whether registered passkeys count as a second factor when logging in"
|
||||
},
|
||||
"password": {
|
||||
"description": "Account password for sudo verification",
|
||||
"$ref": "#/components/schemas/PasswordType"
|
||||
},
|
||||
"mfa_method": {
|
||||
"description": "MFA method to use for verification",
|
||||
"x-enumNames": ["TOTP", "WebAuthn"],
|
||||
"x-enumDescriptions": [
|
||||
"Time-based one-time password authentication via authenticator app",
|
||||
"Security key or biometric authentication"
|
||||
],
|
||||
"enum": ["totp", "webauthn"],
|
||||
"type": "string"
|
||||
},
|
||||
"mfa_code": {"description": "MFA verification code from an authenticator app", "type": "string"},
|
||||
"webauthn_response": {
|
||||
"description": "WebAuthn authentication response",
|
||||
"$ref": "#/components/schemas/WebAuthnAuthenticationResponse"
|
||||
},
|
||||
"webauthn_challenge": {"description": "WebAuthn challenge string", "type": "string"}
|
||||
},
|
||||
"required": ["enabled"]
|
||||
},
|
||||
"WebAuthnTwoFactorResponse": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"user": {"description": "The updated account", "$ref": "#/components/schemas/UserPrivateResponse"},
|
||||
"backup_codes": {
|
||||
"anyOf": [
|
||||
{
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"code": {"type": "string", "description": "The backup code"},
|
||||
"consumed": {"type": "boolean", "description": "Whether the code has been used"}
|
||||
},
|
||||
"required": ["code", "consumed"],
|
||||
"additionalProperties": false
|
||||
}
|
||||
},
|
||||
{"type": "null"}
|
||||
],
|
||||
"description": "Backup codes minted by this call, or null when none were minted"
|
||||
}
|
||||
},
|
||||
"required": ["user", "backup_codes"],
|
||||
"additionalProperties": false
|
||||
},
|
||||
"SudoVerificationSchema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
@@ -24104,7 +24224,7 @@
|
||||
"description": "The ID of the guild this channel belongs to",
|
||||
"$ref": "#/components/schemas/SnowflakeStringType"
|
||||
},
|
||||
"name": {"description": "The name of the channel", "type": "string"},
|
||||
"name": {"description": "The name of the channel", "type": ["string", "null"]},
|
||||
"topic": {"description": "The topic of the channel", "type": ["string", "null"]},
|
||||
"url": {
|
||||
"description": "The URL associated with the channel",
|
||||
@@ -27062,7 +27182,8 @@
|
||||
"assignments": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"voice_noise_suppression": {"$ref": "#/components/schemas/VoiceNoiseSuppressionAssignmentResponse"}
|
||||
"voice_noise_suppression": {"$ref": "#/components/schemas/VoiceNoiseSuppressionAssignmentResponse"},
|
||||
"screen_share_delivery": {"$ref": "#/components/schemas/ScreenShareDeliveryAssignmentResponse"}
|
||||
},
|
||||
"additionalProperties": false
|
||||
}
|
||||
@@ -28201,9 +28322,13 @@
|
||||
"description": "List of allowed MFA methods"
|
||||
},
|
||||
"totp": {"type": "boolean", "description": "Whether TOTP authenticator MFA is available"},
|
||||
"webauthn": {"type": "boolean", "description": "Whether WebAuthn security key MFA is available"}
|
||||
"webauthn": {"type": "boolean", "description": "Whether WebAuthn security key MFA is available"},
|
||||
"backup_codes": {
|
||||
"type": "boolean",
|
||||
"description": "Whether the account has at least one unconsumed backup code"
|
||||
}
|
||||
},
|
||||
"required": ["mfa", "ticket", "allowed_methods", "totp", "webauthn"],
|
||||
"required": ["mfa", "ticket", "allowed_methods", "totp", "webauthn", "backup_codes"],
|
||||
"additionalProperties": false
|
||||
}
|
||||
]
|
||||
@@ -28259,9 +28384,13 @@
|
||||
"description": "List of allowed MFA methods"
|
||||
},
|
||||
"totp": {"type": "boolean", "description": "Whether TOTP authenticator MFA is available"},
|
||||
"webauthn": {"type": "boolean", "description": "Whether WebAuthn security key MFA is available"}
|
||||
"webauthn": {"type": "boolean", "description": "Whether WebAuthn security key MFA is available"},
|
||||
"backup_codes": {
|
||||
"type": "boolean",
|
||||
"description": "Whether the account has at least one unconsumed backup code"
|
||||
}
|
||||
},
|
||||
"required": ["mfa", "ticket", "allowed_methods", "totp", "webauthn"],
|
||||
"required": ["mfa", "ticket", "allowed_methods", "totp", "webauthn", "backup_codes"],
|
||||
"additionalProperties": false
|
||||
},
|
||||
{"$ref": "#/components/schemas/AuthRegistrationPendingApprovalResponse"}
|
||||
@@ -30558,6 +30687,12 @@
|
||||
"additionalProperties": false
|
||||
},
|
||||
"DonationCurrency": {"type": "string", "enum": ["usd", "eur", "brl", "inr", "pln", "try", "sek", "dkk", "nok"]},
|
||||
"ScreenShareDeliveryAssignmentResponse": {
|
||||
"type": "object",
|
||||
"properties": {"enabled": {"type": "boolean"}},
|
||||
"required": ["enabled"],
|
||||
"additionalProperties": false
|
||||
},
|
||||
"VoiceNoiseSuppressionAssignmentResponse": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
@@ -33378,6 +33513,14 @@
|
||||
"required": ["id", "rawId", "type", "clientExtensionResults", "response"],
|
||||
"additionalProperties": {}
|
||||
},
|
||||
"UserAuthenticatorTypes": {
|
||||
"description": "Authenticator type",
|
||||
"type": "integer",
|
||||
"enum": [0, 2],
|
||||
"format": "int32",
|
||||
"x-enumNames": ["TOTP", "WEBAUTHN"],
|
||||
"x-enumDescriptions": ["Time-based one-time password authenticator", "WebAuthn authenticator"]
|
||||
},
|
||||
"HexString32Type": {"type": "string", "pattern": "^[a-f0-9]{32}$"},
|
||||
"PhoneNumberType": {"type": "string"},
|
||||
"RelationshipTypesInput": {
|
||||
@@ -33762,14 +33905,6 @@
|
||||
"enum": ["online", "dnd", "idle", "invisible"],
|
||||
"type": "string"
|
||||
},
|
||||
"UserAuthenticatorTypes": {
|
||||
"description": "Authenticator type",
|
||||
"type": "integer",
|
||||
"enum": [0, 2],
|
||||
"format": "int32",
|
||||
"x-enumNames": ["TOTP", "WEBAUTHN"],
|
||||
"x-enumDescriptions": ["Time-based one-time password authenticator", "WebAuthn authenticator"]
|
||||
},
|
||||
"HexString16Type": {"type": "string", "pattern": "^[a-f0-9]{16}$"},
|
||||
"Int64Type": {
|
||||
"anyOf": [{"type": "string"}, {"type": "integer", "minimum": -9007199254740991, "maximum": 9007199254740991}],
|
||||
|
||||
@@ -100,6 +100,10 @@ export const AuthRateLimitConfigs = {
|
||||
bucket: 'mfa:webauthn:delete',
|
||||
config: {limit: 10, windowMs: ms('1 minute')},
|
||||
} as RouteRateLimitConfig,
|
||||
MFA_WEBAUTHN_TWO_FACTOR: {
|
||||
bucket: 'mfa:webauthn:two_factor',
|
||||
config: {limit: 10, windowMs: ms('1 minute')},
|
||||
} as RouteRateLimitConfig,
|
||||
PHONE_SEND_VERIFICATION: {
|
||||
bucket: 'phone:send_verification',
|
||||
config: {limit: 5, windowMs: ms('1 minute')},
|
||||
|
||||
@@ -47,3 +47,7 @@ export async function readLinesFromS3(storage: IStorageService, key: string): Pr
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
export async function deleteRiskS3Object(storage: IStorageService, key: string): Promise<void> {
|
||||
await storage.deleteObject(RISK_S3_BUCKET, key);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,75 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
import type {IpInfoRequestAuditEvent} from '@pkgs/geoip/src/IpInfoService';
|
||||
import {
|
||||
createPostgresIpInfoCache,
|
||||
createPostgresIpInfoRequestAuditLogger,
|
||||
IPINFO_CACHE_TTL_SECONDS,
|
||||
IPINFO_REQUEST_AUDIT_TTL_SECONDS,
|
||||
} from '@pkgs/geoip/src/PostgresIpInfoKv';
|
||||
import type {IPostgresClient} from '@pkgs/postgres/src/Client';
|
||||
import {describe, expect, it} from 'vitest';
|
||||
|
||||
function recordingClient(writes: Array<Array<unknown>>): IPostgresClient {
|
||||
return {
|
||||
async query(_text: string, values?: Array<unknown>) {
|
||||
writes.push(values ?? []);
|
||||
return {rows: [], rowCount: 1};
|
||||
},
|
||||
kvTable() {
|
||||
return 'kv';
|
||||
},
|
||||
} as never;
|
||||
}
|
||||
|
||||
function expectExpiresIn(values: Array<unknown> | undefined, ttlSeconds: number): void {
|
||||
const expiresAt = values?.[4];
|
||||
expect(expiresAt).toBeInstanceOf(Date);
|
||||
const remainingSeconds = ((expiresAt as Date).getTime() - Date.now()) / 1000;
|
||||
expect(remainingSeconds).toBeGreaterThan(ttlSeconds - 10);
|
||||
expect(remainingSeconds).toBeLessThanOrEqual(ttlSeconds);
|
||||
}
|
||||
|
||||
const EVENT: IpInfoRequestAuditEvent = {
|
||||
requestedAt: new Date('2026-09-21T12:00:00.000Z'),
|
||||
ip: '192.0.2.1',
|
||||
cacheKey: 'ip:192.0.2.1',
|
||||
source: 'test',
|
||||
reason: null,
|
||||
outcome: 'http_success',
|
||||
httpStatus: 200,
|
||||
available: true,
|
||||
riskNote: 'none',
|
||||
latencyMs: 12,
|
||||
requestUrl: 'https://ipinfo.test/192.0.2.1',
|
||||
responseIp: '192.0.2.1',
|
||||
countryCode: 'SE',
|
||||
asnNumber: 64500,
|
||||
isAnonymous: false,
|
||||
isTor: false,
|
||||
isVpn: false,
|
||||
isProxy: false,
|
||||
isResidentialProxy: false,
|
||||
};
|
||||
|
||||
describe('Postgres ipinfo KV expiry', () => {
|
||||
it('expires request audit rows after 90 days', async () => {
|
||||
const writes: Array<Array<unknown>> = [];
|
||||
await createPostgresIpInfoRequestAuditLogger({client: recordingClient(writes)}).record(EVENT);
|
||||
expect(writes).toHaveLength(1);
|
||||
expect(writes[0]?.[0]).toBe('ipinfo_requests_by_hour');
|
||||
expectExpiresIn(writes[0], IPINFO_REQUEST_AUDIT_TTL_SECONDS);
|
||||
});
|
||||
|
||||
it('falls back to the 14-day cache default', async () => {
|
||||
const writes: Array<Array<unknown>> = [];
|
||||
const cache = createPostgresIpInfoCache({client: recordingClient(writes)});
|
||||
await cache.set('fallback', {ok: true});
|
||||
await cache.set('zero', {ok: true}, 0);
|
||||
await cache.set('short', {ok: true}, 60);
|
||||
expect(writes.map((values) => values[0])).toEqual(['ipinfo_cache', 'ipinfo_cache', 'ipinfo_cache']);
|
||||
expectExpiresIn(writes[0], IPINFO_CACHE_TTL_SECONDS);
|
||||
expectExpiresIn(writes[1], IPINFO_CACHE_TTL_SECONDS);
|
||||
expectExpiresIn(writes[2], 60);
|
||||
});
|
||||
});
|
||||
@@ -864,7 +864,7 @@ export class RpcService {
|
||||
if (!queueAllowed) {
|
||||
return;
|
||||
}
|
||||
await this.workerService.addJob('reconcileUserPayments', {userId: userIdString});
|
||||
await this.workerService.addJob('reconcileUserPayments', {userId: userIdString}, {skipLedger: true});
|
||||
})
|
||||
.catch((error) => {
|
||||
Logger.warn(
|
||||
|
||||
@@ -0,0 +1,37 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
import {createTestUserWithPremium} from '@app/api/stripe/tests/StripeWebhookTestUtils';
|
||||
import {type ApiTestHarness, createApiTestHarness} from '@app/api/test/ApiTestHarness';
|
||||
import {NoopWorkerService} from '@app/api/test/NoopWorkerService';
|
||||
import {HTTP_STATUS} from '@app/api/test/TestConstants';
|
||||
import {createBuilder} from '@app/api/test/TestRequestBuilder';
|
||||
import {UserPremiumTypes} from '@fluxer/constants/src/UserConstants';
|
||||
import {afterEach, beforeEach, describe, expect, test, vi} from 'vitest';
|
||||
|
||||
describe('RpcService session payment reconciliation', () => {
|
||||
let harness: ApiTestHarness;
|
||||
beforeEach(async () => {
|
||||
harness = await createApiTestHarness();
|
||||
});
|
||||
afterEach(async () => {
|
||||
await harness?.shutdown();
|
||||
});
|
||||
test('queues payment reconciliation without a job record', async () => {
|
||||
const account = await createTestUserWithPremium(harness, UserPremiumTypes.SUBSCRIPTION, {
|
||||
stripeCustomerId: 'cus_rpc_session_reconcile',
|
||||
});
|
||||
const addJob = vi.spyOn(NoopWorkerService.prototype, 'addJob');
|
||||
try {
|
||||
await createBuilder(harness, '')
|
||||
.post('/test/rpc-session-init')
|
||||
.body({type: 'session', token: account.token, version: 1, ip: '127.0.0.1'})
|
||||
.expect(HTTP_STATUS.OK)
|
||||
.execute();
|
||||
await vi.waitFor(() =>
|
||||
expect(addJob).toHaveBeenCalledWith('reconcileUserPayments', {userId: account.userId}, {skipLedger: true}),
|
||||
);
|
||||
} finally {
|
||||
addJob.mockRestore();
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -352,7 +352,16 @@ export class InMemoryCassandraQueryExecutor implements CassandraQueryExecutorFor
|
||||
let rows = [...this.table(meta).values()].filter((row) => matchesWhere(row, meta.where, params));
|
||||
if (meta.orderBy) {
|
||||
const direction = meta.orderBy.direction === 'DESC' ? -1 : 1;
|
||||
rows = rows.sort((a, b) => compareValues(a[meta.orderBy!.col], b[meta.orderBy!.col]) * direction);
|
||||
const column = meta.orderBy.col as string;
|
||||
const primaryKey = meta.table.primaryKey as ReadonlyArray<string>;
|
||||
const columns = [column, ...primaryKey.slice(primaryKey.indexOf(column) + 1)];
|
||||
rows = rows.sort((a, b) => {
|
||||
for (const c of columns) {
|
||||
const cmp = compareValues(a[c], b[c]);
|
||||
if (cmp !== 0) return cmp * direction;
|
||||
}
|
||||
return 0;
|
||||
});
|
||||
}
|
||||
if (typeof meta.limit === 'number') {
|
||||
rows = rows.slice(0, meta.limit);
|
||||
|
||||
@@ -154,7 +154,7 @@ export function mapUserToPrivateResponse(user: User): UserPrivateResponse {
|
||||
banner: stripBannerForUser(user),
|
||||
banner_color: user.bannerColor,
|
||||
mfa_enabled: authenticatorTypes.length > 0,
|
||||
authenticator_types: authenticatorTypes.length > 0 ? authenticatorTypes : undefined,
|
||||
authenticator_types: authenticatorTypes,
|
||||
verified: user.emailVerified,
|
||||
premium_type: isActuallyPremium ? (user.premiumType ?? UserPremiumTypes.NONE) : UserPremiumTypes.NONE,
|
||||
premium_since: isActuallyPremium ? (user.premiumSince?.toISOString() ?? null) : null,
|
||||
|
||||
@@ -31,6 +31,8 @@ import {
|
||||
WebAuthnCredentialListResponse,
|
||||
WebAuthnCredentialUpdateRequest,
|
||||
WebAuthnRegisterRequest,
|
||||
WebAuthnTwoFactorRequest,
|
||||
WebAuthnTwoFactorResponse,
|
||||
} from '@fluxer/schema/src/domains/auth/AuthSchemas';
|
||||
import {CredentialIdParam} from '@fluxer/schema/src/domains/common/CommonParamSchemas';
|
||||
import {EmptyBodyRequest} from '@fluxer/schema/src/domains/user/UserRequestSchemas';
|
||||
@@ -431,6 +433,30 @@ export function UserAuthController(app: HonoApp) {
|
||||
return ctx.body(null, 204);
|
||||
},
|
||||
);
|
||||
app.put(
|
||||
'/users/@me/mfa/webauthn/two-factor',
|
||||
RateLimitMiddleware(RateLimitConfigs.MFA_WEBAUTHN_TWO_FACTOR),
|
||||
LoginRequired,
|
||||
DefaultUserOnly,
|
||||
SudoModeMiddleware,
|
||||
Validator('json', WebAuthnTwoFactorRequest),
|
||||
OpenAPI({
|
||||
operationId: 'set_webauthn_two_factor',
|
||||
summary: 'Set WebAuthn two-factor authentication',
|
||||
responseSchema: WebAuthnTwoFactorResponse,
|
||||
statusCode: 200,
|
||||
security: ['bearerToken', 'sessionToken'],
|
||||
tags: ['Users'],
|
||||
description:
|
||||
'Choose whether registered passkeys are required as a second factor when signing in with email and password. Enabling requires at least one registered credential and mints backup codes when the account has none. Requires sudo mode verification.',
|
||||
}),
|
||||
async (ctx) => {
|
||||
const user = ctx.get('user');
|
||||
const body = ctx.req.valid('json');
|
||||
await requireSudoMode(ctx, user, body);
|
||||
return ctx.json(await ctx.get('userAuthRequestService').setWebAuthnTwoFactor({user, data: body}));
|
||||
},
|
||||
);
|
||||
app.get(
|
||||
'/users/@me/sudo/mfa-methods',
|
||||
RateLimitMiddleware(RateLimitConfigs.SUDO_MFA_METHODS),
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
import {randomUUID, timingSafeEqual} from 'node:crypto';
|
||||
import type {ApiContext} from '@app/api/ApiContext';
|
||||
import {requireEmailVerified} from '@app/api/auth/EmailVerificationUtils';
|
||||
import {userHasMfa} from '@app/api/auth/services/SudoMethods';
|
||||
import type {MfaBackupCode} from '@app/api/models/MfaBackupCode';
|
||||
import type {User} from '@app/api/models/User';
|
||||
import {regenerateMfaBackupCodes} from '@app/api/user/services/UserAuth';
|
||||
@@ -11,7 +12,6 @@ import {
|
||||
checkChangeRateLimit,
|
||||
generateChangeVerificationCode,
|
||||
} from '@app/api/user/services/UserChangeChallengeUtils';
|
||||
import {UserAuthenticatorTypes} from '@fluxer/constants/src/UserConstants';
|
||||
import {ValidationErrorCodes} from '@fluxer/constants/src/ValidationErrorCodes';
|
||||
import {MfaNotEnabledError} from '@fluxer/errors/src/domains/auth/MfaNotEnabledError';
|
||||
import {InputValidationError} from '@fluxer/errors/src/domains/core/InputValidationError';
|
||||
@@ -65,7 +65,7 @@ export class MfaBackupCodesChallengeService {
|
||||
if (!user.email) {
|
||||
throw InputValidationError.fromCode('email', ValidationErrorCodes.USER_DOES_NOT_HAVE_AN_EMAIL_ADDRESS);
|
||||
}
|
||||
if (!user.totpSecret || !user.authenticatorTypes.has(UserAuthenticatorTypes.TOTP)) {
|
||||
if (!userHasMfa(user)) {
|
||||
throw new MfaNotEnabledError();
|
||||
}
|
||||
await checkChangeRateLimit(rateLimit, {
|
||||
@@ -148,7 +148,7 @@ export class MfaBackupCodesChallengeService {
|
||||
maxAttempts: 5,
|
||||
windowMs: ms('15 minutes'),
|
||||
});
|
||||
if (!user.totpSecret || !user.authenticatorTypes.has(UserAuthenticatorTypes.TOTP)) {
|
||||
if (!userHasMfa(user)) {
|
||||
throw new MfaNotEnabledError();
|
||||
}
|
||||
if (!state.verification_proof) {
|
||||
|
||||
@@ -1,9 +1,10 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
import type {ApiContext} from '@app/api/ApiContext';
|
||||
import * as AuthMfa from '@app/api/auth/AuthMfa';
|
||||
import * as AuthPassword from '@app/api/auth/AuthPassword';
|
||||
import * as AuthSession from '@app/api/auth/AuthSession';
|
||||
import {deriveSudoMethods, userHasMfa} from '@app/api/auth/services/SudoMethods';
|
||||
import {deriveSudoMethods, userHasSudoCapability} from '@app/api/auth/services/SudoMethods';
|
||||
import type {SudoVerificationResult} from '@app/api/auth/services/SudoVerificationService';
|
||||
import {Config} from '@app/api/Config';
|
||||
import type {UserRow} from '@app/api/database/types/UserTypes';
|
||||
@@ -64,7 +65,6 @@ export class UserAccountSecurityService {
|
||||
const isUnclaimedAccount = user.isUnclaimedAccount();
|
||||
const identityVerifiedViaSudo = sudoContext?.method === 'mfa' || sudoContext?.method === 'sudo_token';
|
||||
const identityVerifiedViaPassword = sudoContext?.method === 'password';
|
||||
const hasMfa = userHasMfa(user);
|
||||
const rawEmail = data.email?.trim();
|
||||
const normalizedEmail = rawEmail?.toLowerCase();
|
||||
const hasPasswordRequiredChanges =
|
||||
@@ -74,7 +74,7 @@ export class UserAccountSecurityService {
|
||||
data.new_password !== undefined;
|
||||
const requiresVerification = hasPasswordRequiredChanges && !isUnclaimedAccount;
|
||||
if (requiresVerification && !identityVerifiedViaSudo && !identityVerifiedViaPassword) {
|
||||
throw new SudoModeRequiredError(hasMfa, deriveSudoMethods(user));
|
||||
throw await this.createSudoModeRequiredError(user);
|
||||
}
|
||||
if (isUnclaimedAccount && data.new_password) {
|
||||
updates.password_hash = await this.hashNewPassword(data.new_password);
|
||||
@@ -85,7 +85,7 @@ export class UserAccountSecurityService {
|
||||
throw InputValidationError.fromCode('password', ValidationErrorCodes.PASSWORD_NOT_SET);
|
||||
}
|
||||
if (!identityVerifiedViaSudo && !identityVerifiedViaPassword) {
|
||||
throw new SudoModeRequiredError(hasMfa, deriveSudoMethods(user));
|
||||
throw await this.createSudoModeRequiredError(user);
|
||||
}
|
||||
updates.password_hash = await this.hashNewPassword(data.new_password);
|
||||
updates.password_last_changed_at = new Date();
|
||||
@@ -164,6 +164,16 @@ export class UserAccountSecurityService {
|
||||
});
|
||||
}
|
||||
|
||||
private async createSudoModeRequiredError(user: User): Promise<SudoModeRequiredError> {
|
||||
const credentials = await this.deps.apiContext.services.users.listWebAuthnCredentials(user.id);
|
||||
const hasPasskeyCredentials = credentials.length > 0;
|
||||
const hasBackupCodes = await AuthMfa.hasUnconsumedBackupCodes(this.deps.apiContext, user.id);
|
||||
return new SudoModeRequiredError(
|
||||
userHasSudoCapability(user, hasPasskeyCredentials),
|
||||
deriveSudoMethods(user, hasPasskeyCredentials, hasBackupCodes),
|
||||
);
|
||||
}
|
||||
|
||||
private async hashNewPassword(newPassword: string): Promise<string> {
|
||||
if (await AuthPassword.isPasswordPwned(this.deps.apiContext, newPassword)) {
|
||||
throw InputValidationError.fromCode('new_password', ValidationErrorCodes.PASSWORD_IS_TOO_COMMON);
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
import type {ApiContext} from '@app/api/ApiContext';
|
||||
import * as AuthMfa from '@app/api/auth/AuthMfa';
|
||||
import * as AuthUtility from '@app/api/auth/AuthUtility';
|
||||
import {deriveSudoMethods, userHasMfa} from '@app/api/auth/services/SudoMethods';
|
||||
import {deriveSudoMethods, userHasMfa, userHasSudoCapability} from '@app/api/auth/services/SudoMethods';
|
||||
import type {SudoVerificationResult} from '@app/api/auth/services/SudoVerificationService';
|
||||
import type {MfaBackupCode} from '@app/api/models/MfaBackupCode';
|
||||
import type {User} from '@app/api/models/User';
|
||||
@@ -36,12 +36,23 @@ interface GetMfaBackupCodesParams {
|
||||
sudoContext: SudoVerificationResult;
|
||||
}
|
||||
|
||||
function assertSudoVerifiedForMfa(user: User, sudoContext: SudoVerificationResult): void {
|
||||
async function assertSudoVerifiedForMfa(
|
||||
ctx: ApiContext,
|
||||
user: User,
|
||||
sudoContext: SudoVerificationResult,
|
||||
): Promise<void> {
|
||||
const identityVerifiedViaSudo = sudoContext.method === 'mfa' || sudoContext.method === 'sudo_token';
|
||||
const identityVerifiedViaPassword = sudoContext.method === 'password';
|
||||
if (!identityVerifiedViaSudo && !identityVerifiedViaPassword) {
|
||||
throw new SudoModeRequiredError(userHasMfa(user), deriveSudoMethods(user));
|
||||
if (identityVerifiedViaSudo || identityVerifiedViaPassword) {
|
||||
return;
|
||||
}
|
||||
const credentials = await ctx.services.users.listWebAuthnCredentials(user.id);
|
||||
const hasPasskeyCredentials = credentials.length > 0;
|
||||
const hasBackupCodes = await AuthMfa.hasUnconsumedBackupCodes(ctx, user.id);
|
||||
throw new SudoModeRequiredError(
|
||||
userHasSudoCapability(user, hasPasskeyCredentials),
|
||||
deriveSudoMethods(user, hasPasskeyCredentials, hasBackupCodes),
|
||||
);
|
||||
}
|
||||
|
||||
export async function enableMfaTotp(
|
||||
@@ -49,7 +60,7 @@ export async function enableMfaTotp(
|
||||
{user, secret, code, sudoContext}: EnableMfaTotpParams,
|
||||
): Promise<Array<MfaBackupCode>> {
|
||||
const {users, botMfaMirror} = ctx.services;
|
||||
assertSudoVerifiedForMfa(user, sudoContext);
|
||||
await assertSudoVerifiedForMfa(ctx, user, sudoContext);
|
||||
if (user.totpSecret) throw new MfaNotDisabledError();
|
||||
const userId = user.id;
|
||||
if (!(await AuthMfa.verifyMfaCode(ctx, {userId: user.id, mfaSecret: secret, code}))) {
|
||||
@@ -75,7 +86,7 @@ export async function enableMfaTotp(
|
||||
export async function disableMfaTotp(ctx: ApiContext, {user, code, sudoContext}: DisableMfaTotpParams): Promise<void> {
|
||||
const {users, botMfaMirror} = ctx.services;
|
||||
if (!user.totpSecret) throw new MfaNotEnabledError();
|
||||
assertSudoVerifiedForMfa(user, sudoContext);
|
||||
await assertSudoVerifiedForMfa(ctx, user, sudoContext);
|
||||
if (
|
||||
sudoContext.method !== 'mfa' &&
|
||||
!(await AuthMfa.verifyMfaCode(ctx, {
|
||||
@@ -102,7 +113,9 @@ export async function disableMfaTotp(ctx: ApiContext, {user, code, sudoContext}:
|
||||
},
|
||||
user.toRow(),
|
||||
);
|
||||
await users.clearMfaBackupCodes(userId);
|
||||
if (!userHasMfa(updatedUser)) {
|
||||
await users.clearMfaBackupCodes(userId);
|
||||
}
|
||||
await dispatchUserUpdate(ctx, updatedUser);
|
||||
await botMfaMirror.syncAuthenticatorTypesForOwner(updatedUser);
|
||||
}
|
||||
@@ -112,7 +125,7 @@ export async function getMfaBackupCodes(
|
||||
{user, regenerate, sudoContext}: GetMfaBackupCodesParams,
|
||||
): Promise<Array<MfaBackupCode>> {
|
||||
const {users} = ctx.services;
|
||||
assertSudoVerifiedForMfa(user, sudoContext);
|
||||
await assertSudoVerifiedForMfa(ctx, user, sudoContext);
|
||||
if (regenerate) {
|
||||
return regenerateMfaBackupCodes(ctx, user);
|
||||
}
|
||||
|
||||
@@ -9,6 +9,7 @@ import type {IGuildRepositoryAggregate} from '@app/api/guild/repositories/IGuild
|
||||
import type {User} from '@app/api/models/User';
|
||||
import type {IUserRepository} from '@app/api/user/IUserRepository';
|
||||
import * as UserAuth from '@app/api/user/services/UserAuth';
|
||||
import {mapUserToPrivateResponse} from '@app/api/user/UserMappers';
|
||||
import {GuildVerificationLevel} from '@fluxer/constants/src/GuildConstants';
|
||||
import {UserAuthenticatorTypes} from '@fluxer/constants/src/UserConstants';
|
||||
import {PhoneAddNotEligibleError} from '@fluxer/errors/src/domains/auth/PhoneAddNotEligibleError';
|
||||
@@ -26,6 +27,8 @@ import type {
|
||||
WebAuthnCredentialListResponse,
|
||||
WebAuthnCredentialUpdateRequest,
|
||||
WebAuthnRegisterRequest,
|
||||
WebAuthnTwoFactorRequest,
|
||||
WebAuthnTwoFactorResponse,
|
||||
} from '@fluxer/schema/src/domains/auth/AuthSchemas';
|
||||
|
||||
interface UserAuthWithSudoRequest<T> {
|
||||
@@ -194,6 +197,22 @@ export class UserAuthRequestService {
|
||||
await AuthMfa.deleteWebAuthnCredential(this.apiContext, user.id, credentialId);
|
||||
}
|
||||
|
||||
async setWebAuthnTwoFactor({
|
||||
user,
|
||||
data,
|
||||
}: UserAuthRequest<WebAuthnTwoFactorRequest>): Promise<WebAuthnTwoFactorResponse> {
|
||||
if (data.enabled) {
|
||||
requireEmailVerified(user, 'mfa');
|
||||
}
|
||||
const result = await AuthMfa.setWebAuthnTwoFactor(this.apiContext, user.id, data.enabled);
|
||||
return {
|
||||
user: mapUserToPrivateResponse(result.user),
|
||||
backup_codes: result.backupCodes
|
||||
? result.backupCodes.map((backupCode) => ({code: backupCode.code, consumed: backupCode.consumed}))
|
||||
: null,
|
||||
};
|
||||
}
|
||||
|
||||
async listSudoMfaMethods(user: User): Promise<SudoMfaMethodsResponse> {
|
||||
return AuthMfa.getAvailableMfaMethods(this.apiContext, user.id);
|
||||
}
|
||||
|
||||
@@ -5,13 +5,18 @@ import * as RegexUtils from '@app/api/utils/RegexUtils';
|
||||
|
||||
let _invitePattern: RegExp | null = null;
|
||||
|
||||
function getInviteEndpointBase(): string {
|
||||
const url = new URL(Config.endpoints.invite);
|
||||
return `${url.hostname}${url.pathname.replace(/\/+$/, '')}`;
|
||||
}
|
||||
|
||||
function getInvitePattern(): RegExp {
|
||||
if (!_invitePattern) {
|
||||
_invitePattern = new RegExp(
|
||||
[
|
||||
'(?:https?:\\/\\/)?',
|
||||
'(?:',
|
||||
`${RegexUtils.escapeRegex(Config.hosts.invite)}(?:\\/#)?\\/(?!invite\\/)([a-zA-Z0-9\\-]{2,32})(?![a-zA-Z0-9\\-])`,
|
||||
`${RegexUtils.escapeRegex(getInviteEndpointBase())}(?:\\/#)?\\/(?!invite\\/)([a-zA-Z0-9\\-]{2,32})(?![a-zA-Z0-9\\-])`,
|
||||
'|',
|
||||
`${RegexUtils.escapeRegex(new URL(Config.endpoints.webApp).hostname)}(?:\\/#)?\\/invite\\/([a-zA-Z0-9\\-]{2,32})(?![a-zA-Z0-9\\-])`,
|
||||
')',
|
||||
|
||||
@@ -0,0 +1,44 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
import {overwriteGrantedBits} from '@app/api/utils/PermissionUtils';
|
||||
import {Permissions} from '@fluxer/constants/src/ChannelConstants';
|
||||
import {describe, expect, it} from 'vitest';
|
||||
|
||||
const BOT_OVERWRITE = {
|
||||
allow: Permissions.ADD_REACTIONS | Permissions.SEND_MESSAGES | Permissions.MANAGE_MESSAGES | Permissions.PIN_MESSAGES,
|
||||
deny: 0n,
|
||||
};
|
||||
|
||||
describe('overwriteGrantedBits', () => {
|
||||
it('grants nothing when an overwrite is resubmitted unmodified', () => {
|
||||
expect(overwriteGrantedBits(BOT_OVERWRITE, {...BOT_OVERWRITE})).toBe(0n);
|
||||
});
|
||||
|
||||
it('ignores allow bits that were already set when another bit is flipped', () => {
|
||||
const before = {allow: Permissions.PIN_MESSAGES, deny: 0n};
|
||||
const after = {allow: Permissions.PIN_MESSAGES | Permissions.SEND_MESSAGES, deny: 0n};
|
||||
expect(overwriteGrantedBits(before, after)).toBe(Permissions.SEND_MESSAGES);
|
||||
});
|
||||
|
||||
it('grants nothing when an allow bit is withdrawn', () => {
|
||||
const before = {allow: Permissions.SEND_MESSAGES | Permissions.MANAGE_MESSAGES, deny: 0n};
|
||||
const after = {allow: Permissions.SEND_MESSAGES, deny: 0n};
|
||||
expect(overwriteGrantedBits(before, after)).toBe(0n);
|
||||
});
|
||||
|
||||
it('grants nothing when a deny is added for a permission the editor lacks', () => {
|
||||
const after = {allow: Permissions.VIEW_CHANNEL, deny: Permissions.MANAGE_MESSAGES};
|
||||
expect(overwriteGrantedBits(null, after)).toBe(Permissions.VIEW_CHANNEL);
|
||||
});
|
||||
|
||||
it('grants the bits lifted out of deny', () => {
|
||||
const before = {allow: 0n, deny: Permissions.ADD_REACTIONS | Permissions.SEND_MESSAGES};
|
||||
const after = {allow: 0n, deny: Permissions.ADD_REACTIONS};
|
||||
expect(overwriteGrantedBits(before, after)).toBe(Permissions.SEND_MESSAGES);
|
||||
});
|
||||
|
||||
it('treats a removed overwrite as granting everything it denied', () => {
|
||||
const before = {allow: Permissions.SEND_MESSAGES, deny: Permissions.ADD_REACTIONS};
|
||||
expect(overwriteGrantedBits(before, undefined)).toBe(Permissions.ADD_REACTIONS);
|
||||
});
|
||||
});
|
||||
@@ -51,3 +51,14 @@ export async function hasPermission(
|
||||
): Promise<boolean> {
|
||||
return await gatewayService.checkPermission(params);
|
||||
}
|
||||
|
||||
export function overwriteGrantedBits(
|
||||
before: {allow: bigint; deny: bigint} | null | undefined,
|
||||
after: {allow: bigint; deny: bigint} | null | undefined,
|
||||
): bigint {
|
||||
const beforeAllow = before?.allow ?? 0n;
|
||||
const beforeDeny = before?.deny ?? 0n;
|
||||
const afterAllow = after?.allow ?? 0n;
|
||||
const afterDeny = after?.deny ?? 0n;
|
||||
return (afterAllow & ~beforeAllow) | (beforeDeny & ~afterDeny);
|
||||
}
|
||||
|
||||
@@ -8,24 +8,17 @@ import * as InviteUtils from '@app/api/utils/InviteUtils';
|
||||
import {URL_REGEX} from '@fluxer/constants/src/Core';
|
||||
import * as idna from 'idna-uts46-hx';
|
||||
|
||||
const MARKETING_PATH_PREFIXES = ['/channels/', '/theme/'];
|
||||
const CLIENT_ROUTE_PATH_PREFIXES = ['/channels/', '/theme/'];
|
||||
|
||||
interface ExcludedLinkBase {
|
||||
hostname: string;
|
||||
pathPrefix: string;
|
||||
}
|
||||
|
||||
function normalizeHostname(hostname: string | undefined) {
|
||||
return hostname?.trim().toLowerCase() || '';
|
||||
}
|
||||
|
||||
let _marketingHostname: string | null = null;
|
||||
|
||||
function getMarketingHostname() {
|
||||
if (!_marketingHostname) {
|
||||
_marketingHostname = normalizeHostname(Config.hosts.marketing);
|
||||
}
|
||||
return _marketingHostname;
|
||||
}
|
||||
|
||||
const isMarketingPath = (hostname: string, pathname: string) =>
|
||||
hostname === getMarketingHostname() && MARKETING_PATH_PREFIXES.some((prefix) => pathname.startsWith(prefix));
|
||||
|
||||
function getWebAppHostname() {
|
||||
try {
|
||||
return new URL(Config.endpoints.webApp).hostname;
|
||||
@@ -34,23 +27,32 @@ function getWebAppHostname() {
|
||||
}
|
||||
}
|
||||
|
||||
let _excludedHostnames: Set<string> | null = null;
|
||||
|
||||
function getExcludedHostnames(): Set<string> {
|
||||
if (!_excludedHostnames) {
|
||||
_excludedHostnames = new Set<string>();
|
||||
const addHostname = (hostname: string | undefined) => {
|
||||
const normalized = normalizeHostname(hostname);
|
||||
if (normalized) {
|
||||
_excludedHostnames!.add(normalized);
|
||||
}
|
||||
};
|
||||
addHostname(Config.hosts.invite);
|
||||
addHostname(Config.hosts.gift);
|
||||
Config.hosts.unfurlIgnored.forEach(addHostname);
|
||||
addHostname(getWebAppHostname());
|
||||
function endpointLinkBase(endpoint: string): ExcludedLinkBase | null {
|
||||
try {
|
||||
const url = new URL(endpoint);
|
||||
return {hostname: normalizeHostname(url.hostname), pathPrefix: `${url.pathname.replace(/\/+$/, '')}/`};
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
return _excludedHostnames;
|
||||
}
|
||||
|
||||
let _excludedLinkBases: Array<ExcludedLinkBase> | null = null;
|
||||
|
||||
function getExcludedLinkBases(): Array<ExcludedLinkBase> {
|
||||
if (!_excludedLinkBases) {
|
||||
const bases: Array<ExcludedLinkBase | null> = [
|
||||
...Config.hosts.unfurlIgnored.map((hostname) => ({hostname: normalizeHostname(hostname), pathPrefix: '/'})),
|
||||
endpointLinkBase(Config.endpoints.invite),
|
||||
endpointLinkBase(Config.endpoints.gift),
|
||||
];
|
||||
for (const hostname of [getWebAppHostname(), Config.hosts.marketing]) {
|
||||
for (const pathPrefix of CLIENT_ROUTE_PATH_PREFIXES) {
|
||||
bases.push({hostname: normalizeHostname(hostname), pathPrefix});
|
||||
}
|
||||
}
|
||||
_excludedLinkBases = bases.filter((base): base is ExcludedLinkBase => base !== null && base.hostname !== '');
|
||||
}
|
||||
return _excludedLinkBases;
|
||||
}
|
||||
|
||||
function idnaEncodeURL(url: string) {
|
||||
@@ -80,8 +82,9 @@ function isFluxerAppExcludedURL(url: string) {
|
||||
try {
|
||||
const parsedUrl = new URL(url);
|
||||
const hostname = normalizeHostname(parsedUrl.hostname);
|
||||
const isMarketingPathMatch = isMarketingPath(hostname, parsedUrl.pathname);
|
||||
return isMarketingPathMatch || getExcludedHostnames().has(hostname);
|
||||
return getExcludedLinkBases().some(
|
||||
(base) => base.hostname === hostname && parsedUrl.pathname.startsWith(base.pathPrefix),
|
||||
);
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,52 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
import {Logger} from '@app/api/Logger';
|
||||
import type {WorkerTaskName} from '@app/api/worker/WorkerLaneConfig';
|
||||
import {WorkerQueueOverflowError} from '@app/api/worker/WorkerQueueOverflowError';
|
||||
import type {WorkerService} from '@app/api/worker/WorkerService';
|
||||
import type {IKVProvider} from '@pkgs/kv_client/src/IKVProvider';
|
||||
import {ms} from 'itty-time';
|
||||
|
||||
const INITIAL_SYNC_KEY = 'sync:email_domains:initialized';
|
||||
const PURGE_KEY = 'sync:blocklist_feeds:purged';
|
||||
const CLAIM_TTL_SECONDS = ms('6 hours') / 1000;
|
||||
const FEED_TASKS = [
|
||||
'syncDisposableEmailDomains',
|
||||
'syncUrlBlocklists',
|
||||
'syncFileShaBlocklists',
|
||||
] as const satisfies ReadonlyArray<WorkerTaskName>;
|
||||
|
||||
export async function queueBlocklistFeedStartupJobs(
|
||||
kvClient: Pick<IKVProvider, 'setnx' | 'del'>,
|
||||
workerService: Pick<WorkerService, 'addJob'>,
|
||||
enabled: boolean,
|
||||
): Promise<void> {
|
||||
if (enabled) {
|
||||
if (await kvClient.setnx(INITIAL_SYNC_KEY, '1', CLAIM_TTL_SECONDS)) {
|
||||
Logger.info('Triggering initial disposable email domain sync');
|
||||
await queueJobs(workerService, ['syncDisposableEmailDomains']);
|
||||
}
|
||||
return;
|
||||
}
|
||||
const wasEnabled = (await kvClient.del(INITIAL_SYNC_KEY)) > 0;
|
||||
const claimed = await kvClient.setnx(PURGE_KEY, '1', CLAIM_TTL_SECONDS);
|
||||
if (!wasEnabled && !claimed) return;
|
||||
Logger.info('Removing blocklist feed data, blocklist feeds are disabled');
|
||||
await queueJobs(workerService, FEED_TASKS);
|
||||
}
|
||||
|
||||
async function queueJobs(
|
||||
workerService: Pick<WorkerService, 'addJob'>,
|
||||
tasks: ReadonlyArray<WorkerTaskName>,
|
||||
): Promise<void> {
|
||||
for (const task of tasks) {
|
||||
try {
|
||||
await workerService.addJob(task, {});
|
||||
} catch (error) {
|
||||
if (!(error instanceof WorkerQueueOverflowError)) {
|
||||
throw error;
|
||||
}
|
||||
Logger.warn({task}, 'Dropped blocklist feed job, jobs stream is at its limit');
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -17,13 +17,13 @@ import {
|
||||
StorageType,
|
||||
type StreamConfig,
|
||||
} from '@nats-io/jetstream';
|
||||
import {nanos} from '@nats-io/transport-node';
|
||||
import {millis, nanos} from '@nats-io/transport-node';
|
||||
import type {JetStreamConnectionManager} from '@pkgs/nats/src/JetStreamConnectionManager';
|
||||
import type {WorkerJobPayload} from '@pkgs/worker/src/contracts/WorkerTypes';
|
||||
|
||||
const STREAM_NAME = 'JOBS';
|
||||
const SUBJECT_PREFIX = 'jobs.';
|
||||
const MAX_AGE_MS = 7 * 24 * 60 * 60 * 1000;
|
||||
export const JOBS_STREAM_MAX_AGE_MS = 7 * 24 * 60 * 60 * 1000;
|
||||
const LEGACY_CONSUMER_NAME = 'workers';
|
||||
const DLQ_STREAM_NAME = 'JOBS_DLQ';
|
||||
const DLQ_SUBJECT_PREFIX = 'dlq.';
|
||||
@@ -59,7 +59,7 @@ const JOBS_STREAM: WorkerStreamDefinition = {
|
||||
name: STREAM_NAME,
|
||||
subject: `${SUBJECT_PREFIX}>`,
|
||||
retention: RetentionPolicy.Workqueue,
|
||||
maxAgeMs: MAX_AGE_MS,
|
||||
maxAgeMs: JOBS_STREAM_MAX_AGE_MS,
|
||||
minBytes: STREAM_MIN_BYTES,
|
||||
maxMessages: STREAM_MAX_MSGS,
|
||||
maxMessagesPerSubject: STREAM_MAX_MSGS_PER_SUBJECT,
|
||||
@@ -106,6 +106,7 @@ export class JetStreamWorkerQueue {
|
||||
private consumersReady = false;
|
||||
private streamSetup: Promise<void> | null = null;
|
||||
private dlqStreamSetup: Promise<void> | null = null;
|
||||
private jobsStreamMaxAgeMs = JOBS_STREAM_MAX_AGE_MS;
|
||||
|
||||
constructor(connectionManager: JetStreamConnectionManager) {
|
||||
this.connectionManager = connectionManager;
|
||||
@@ -125,11 +126,16 @@ export class JetStreamWorkerQueue {
|
||||
if (existingConfig === null) {
|
||||
await this.addStream(jsm);
|
||||
} else {
|
||||
this.jobsStreamMaxAgeMs = millis(existingConfig.max_age);
|
||||
await this.applyStreamLimits(jsm, existingConfig);
|
||||
}
|
||||
this.streamReady = true;
|
||||
}
|
||||
|
||||
getJobsStreamMaxAgeMs(): number {
|
||||
return this.jobsStreamMaxAgeMs;
|
||||
}
|
||||
|
||||
private async oversizedSubjects(jsm: JetStreamManager): Promise<Array<[string, number]> | null> {
|
||||
try {
|
||||
const info = await jsm.streams.info(STREAM_NAME, {subjects_filter: JOBS_STREAM.subject});
|
||||
@@ -462,7 +468,7 @@ export class JetStreamWorkerQueue {
|
||||
priority?: number;
|
||||
jobKey?: string;
|
||||
},
|
||||
): Promise<string> {
|
||||
): Promise<{seq: string; duplicate: boolean}> {
|
||||
const js = this.connectionManager.getJetStreamClient();
|
||||
const subject = `${SUBJECT_PREFIX}${taskType}`;
|
||||
const body = JSON.stringify({
|
||||
@@ -477,8 +483,7 @@ export class JetStreamWorkerQueue {
|
||||
const ack = await js.publish(subject, body, {
|
||||
msgID,
|
||||
});
|
||||
const jobId = `${ack.seq}`;
|
||||
return jobId;
|
||||
return {seq: `${ack.seq}`, duplicate: ack.duplicate === true};
|
||||
} catch (error) {
|
||||
const rejection = describeStreamRejection(error);
|
||||
if (rejection === null) {
|
||||
|
||||
@@ -67,6 +67,7 @@ const LANE_CONFIG = {
|
||||
consumerName: 'workers_batch',
|
||||
tasks: [
|
||||
'expireAttachments',
|
||||
'expireStaleJobs',
|
||||
'indexChannelMessages',
|
||||
'indexGuildMembers',
|
||||
'processAssetDeletionQueue',
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user