Compare commits

..
Author SHA1 Message Date
HampusandGitHub 20cdfd3009 fix(api): stop exporting unused worker heartbeat symbols (#2334) 2026-09-01 21:03:26 +02:00
HampusandGitHub 9f739427c4 fix(desktop): update rtrb past the double free advisory (#2336) 2026-09-01 21:03:19 +02:00
HampusandGitHub 0201cafd7e fix(admin): mark the crate unpublished so cargo deny passes (#2335) 2026-09-01 21:03:12 +02:00
HampusandGitHub 37f57bb29f fix(desktop): restore offline Flatpak builds (#2332) 2026-09-01 20:51:18 +02:00
HampusandGitHub ebd723679b docs(operator): refresh the bundle pin and tunnel setup (#2333) 2026-09-01 20:51:00 +02:00
HampusandGitHub 961fa1f007 fix(self-hosting): correct compose probes and origins (#2330) 2026-09-01 20:47:20 +02:00
HampusandGitHub 7900a4da0c feat(ci): pin releases to an immutable image set (#2327) 2026-09-01 20:47:20 +02:00
HampusandGitHub 8a24730884 fix(kv): align rust and typescript schema migration (#2328) 2026-09-01 20:47:19 +02:00
HampusandGitHub 1688e7dc50 fix(api): survive transient database errors in the worker (#2326) 2026-09-01 20:47:19 +02:00
HampusandGitHub aa267b54ec fix(api): dead-letter retired worker task types (#2323) 2026-09-01 20:47:19 +02:00
HampusandGitHub bc40073a02 fix(config): carry the public port into derived endpoints (#2329) 2026-09-01 20:47:18 +02:00
HampusandGitHub a93f9dd0af fix(app-proxy): separate readiness from liveness (#2322) 2026-09-01 20:47:18 +02:00
HampusandGitHub 53a9fdc4b6 fix(app-proxy): share one asset tree across architectures (#2325) 2026-09-01 20:47:17 +02:00
HampusandGitHub cef600277c fix(media-proxy): probe health with the binary not /dev/tcp (#2324) 2026-09-01 20:47:17 +02:00
HampusandGitHub bdac438329 fix(docker): emit consistent OCI metadata on every image (#2321) 2026-09-01 20:47:16 +02:00
HampusandGitHub 2d77f36a0b fix(ci): generate locale and channel files before typecheck (#2320) 2026-09-01 20:47:16 +02:00
HampusandGitHub 90aa810ce4 fix(gateway): share the relay dispatch bound across producers (#2315) 2026-09-01 04:34:41 +02:00
HampusandGitHub cf3af50464 fix(kv): bound multi key fan out by pipelining per hash slot (#2313) 2026-09-01 02:59:00 +02:00
HampusandGitHub 3b5b20c139 fix(gateway): bound relay dispatch without per-event probes (#2312) 2026-09-01 02:56:21 +02:00
HampusandGitHub 24cd163acd fix(kv): restore keyset paging for numeric key scans (#2314) 2026-09-01 02:54:10 +02:00
HampusandGitHub 49f76e5b40 fix(api): abort startup on unverifiable deletion queue state (#2311) 2026-09-01 02:45:29 +02:00
HampusandGitHub 871788f0a9 fix(gateway): reclaim ip connection counts from dead sockets (#2308) 2026-09-01 01:39:25 +02:00
HampusandGitHub 24138b70f1 fix(kv): stop multi-key commands spanning cluster slots (#2310) 2026-09-01 01:39:16 +02:00
HampusandGitHub da3332e711 fix(gateway): bound relay worker mailboxes without reordering (#2309) 2026-09-01 01:38:36 +02:00
HampusandGitHub 06e5cf2032 perf(gateway): evict presence tombstones in insertion order (#2307) 2026-09-01 01:34:57 +02:00
HampusandGitHub d4b1923c23 fix(gateway): stop presence evictions suppressing repair (#2305) 2026-09-01 01:33:13 +02:00
HampusandGitHub 42df4f6731 fix(kv): drop the row_key order probe that cliffed paged scans (#2306) 2026-09-01 01:32:33 +02:00
HampusandGitHub f1e6e94041 fix(cache): stop a timed out produce pinning its tracking entry (#2304) 2026-09-01 01:28:12 +02:00
HampusandGitHub 0cd12b2f32 test(api): build deletion queue users from the real row type (#2303) 2026-09-01 00:57:41 +02:00
HampusandGitHub 5da4d24d38 fix(kv): page scans by keyset so deletes cannot skip rows (#2302) 2026-09-01 00:29:24 +02:00
HampusandGitHub 7806d2ac02 fix(gateway): stop stale guild connect timers aborting connects (#2301) 2026-09-01 00:23:42 +02:00
HampusandGitHub 2c4d182d1f fix(gateway): keep dispatch ordered under relay backpressure (#2300) 2026-09-01 00:17:12 +02:00
HampusandGitHub dcd5f88d65 fix(gateway): stop rate limit tables dying with their creator (#2299) 2026-09-01 00:16:26 +02:00
HampusandGitHub 662f4ac93b fix(worker): stop skipped accounts starving the deletion queue (#2294) 2026-09-01 00:16:08 +02:00
HampusandGitHub a2480c6a02 fix(cache): time out a getOrSet produce that never settles (#2297) 2026-09-01 00:15:44 +02:00
HampusandGitHub c49460a44f fix(gateway): stop anti-entropy resurrecting deleted presence (#2298) 2026-09-01 00:14:32 +02:00
HampusandGitHub 6786dfe7e3 fix(gateway): stop dropping newly requested lazy ranges (#2293) 2026-09-01 00:14:24 +02:00
HampusandGitHub 7d710d881a fix(worker): lease premium reconciliation queue entries (#2296) 2026-09-01 00:14:08 +02:00
HampusandGitHub 2ea2e79f6f fix(voice): stop occupancy writes spanning kv cluster slots (#2295) 2026-09-01 00:11:29 +02:00
HampusandGitHub cd42dd8ca7 fix(api): rebuild the deletion queue under its lock (#2292) 2026-09-01 00:06:33 +02:00
HampusandGitHub f2eddeae4d fix(gateway): stop rate limit sweepers outliving their table (#2291) 2026-08-31 23:25:00 +02:00
HampusandGitHub 8e1a8fc7e3 fix(gateway): sweep stale shared ip and user rate buckets (#2290) 2026-08-31 22:53:37 +02:00
HampusandGitHub 87c08b051f fix(voice): finish the reconciliation sweep before stopping (#2289) 2026-08-31 22:38:04 +02:00
HampusandGitHub 9d95a80857 fix(worker): renew the deletion queue lock during a rebuild (#2287) 2026-08-31 22:38:00 +02:00
HampusandGitHub 9371b6d5de fix(worker): count each channel once in a bulk reindex (#2286) 2026-08-31 22:37:56 +02:00
HampusandGitHub bdcf4b25c0 chore(expressions): remove the pack residue cleanup tool (#2288) 2026-08-31 22:31:54 +02:00
HampusandGitHub 3dc344be65 fix(worker): keep attachment decay state on a stale expiry row (#2285) 2026-08-31 22:26:21 +02:00
HampusandGitHub 17ed0f70aa fix(gateway): release the user session count on a handoff fence (#2284) 2026-08-31 22:25:14 +02:00
HampusandGitHub be3e12e60d fix(gateway): clamp a heartbeat ack to the session sequence (#2283) 2026-08-31 22:23:43 +02:00
HampusandGitHub 4261cc2ea5 fix(worker): resubscribe when the job stream ends unexpectedly (#2282) 2026-08-31 22:22:14 +02:00
HampusandGitHub 88dbc27019 fix(gateway): group debounced reactions by their own message (#2281) 2026-08-31 22:21:14 +02:00
HampusandGitHub f38fc80c31 fix(gateway): clear the presence pid cache on a presence down (#2279) 2026-08-31 22:19:25 +02:00
HampusandGitHub 803fdaf443 fix(worker): stop replaying requeued asset deletions in a run (#2280) 2026-08-31 22:19:22 +02:00
HampusandGitHub 55d85db401 fix(gateway): drop the channel engine on an empty range list (#2278) 2026-08-31 22:17:19 +02:00
HampusandGitHub 7ce3d71c44 fix(gateway): stop a non-map opcode payload crashing the socket (#2277) 2026-08-31 22:14:26 +02:00
HampusandGitHub 04e150e4bf fix(gateway): keep the replay buffer across a session transfer (#2276) 2026-08-31 22:10:44 +02:00
HampusandGitHub 0f6b118921 fix(worker): catch up cron jobs missed by a delayed tick (#2275) 2026-08-31 22:06:15 +02:00
HampusandGitHub 44277e6aa2 fix(worker): drain in-flight jobs before the runner stops (#2274) 2026-08-31 22:06:12 +02:00
HampusandGitHub bb7e8cc6f1 fix(gateway): flush buffered presences in arrival order (#2273) 2026-08-31 22:04:49 +02:00
HampusandGitHub 32a64fb097 fix(cache): refcount produce tracking so deletes are not lost (#2272) 2026-08-31 22:00:48 +02:00
HampusandGitHub c4594397e7 fix(desktop): accept array-form AppRun sandbox fallback (#2271) 2026-08-31 21:42:03 +02:00
HampusandGitHub 6a188a4cdf fix(api): enforce guild bans when approving registrations (#2270) 2026-08-31 20:19:18 +02:00
HampusandGitHub 0ca0defd24 fix(desktop): keep notification sounds during fullscreen apps (#2269) 2026-08-31 19:51:42 +02:00
HampusandGitHub b0b84f9c98 fix(media-proxy): cap external streams with no declared length (#2267) 2026-08-31 19:23:48 +02:00
HampusandGitHub ef8d1225b5 refactor(api): split webhook attachment schemas (#2268) 2026-08-31 19:13:51 +02:00
HampusandGitHub 240b7e4388 feat(expressions): add an expression pack residue cleanup tool (#2265) 2026-08-31 19:06:31 +02:00
HampusandGitHub bd205d2250 fix(app-proxy): honour the shared Postgres settings (#2262) 2026-08-31 19:00:43 +02:00
174 changed files with 12999 additions and 1357 deletions
+1
View File
@@ -70,6 +70,7 @@ stage "app: typecheck" pnpm --filter fluxer_app typecheck
stage "app: unit tests" pnpm --filter fluxer_app exec vitest run
if [ "$QUICK" -eq 0 ]; then
stage "desktop: typecheck" pnpm --filter fluxer_desktop typecheck
stage "app: production build" pnpm --filter fluxer_app build
fi
+14 -14
View File
@@ -104,6 +104,9 @@ jobs:
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0
env:
GIT_CONFIG_GLOBAL: ${{ runner.temp }}/gitconfig
- name: resolve source date
id: source
run: echo "date=$(TZ=UTC git log -1 --no-show-signature --pretty=%cd --date=format-local:%Y-%m-%dT%H:%M:%SZ)" >> "$GITHUB_OUTPUT"
- uses: docker/setup-buildx-action@d7f5e7f509e45cec5c76c4d5afdd7de93d0b3df5
- uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee
with:
@@ -115,11 +118,13 @@ jobs:
context: ${{ inputs.context }}
file: ${{ inputs.dockerfile }}
push: true
provenance: false
provenance: mode=min
platforms: linux/${{ matrix.platform }}
tags: ghcr.io/${{ env.GHCR_OWNER }}/${{ inputs.image }}:${{ needs.meta.outputs.build_version }}-${{ matrix.platform }}
build-args: |
BUILD_VERSION=${{ needs.meta.outputs.build_version }}
SOURCE_SHA=${{ github.sha }}
SOURCE_DATE=${{ steps.source.outputs.date }}
${{ inputs.extra-build-args }}
cache-from: type=registry,ref=ghcr.io/${{ env.GHCR_OWNER }}/${{ inputs.image }}:buildcache-${{ matrix.platform }}
cache-to: type=registry,ref=ghcr.io/${{ env.GHCR_OWNER }}/${{ inputs.image }}:buildcache-${{ matrix.platform }},mode=max,image-manifest=true,oci-mediatypes=true,ignore-error=true
@@ -185,17 +190,12 @@ jobs:
- name: Advance moving image tags
env:
IMAGE: ghcr.io/${{ env.GHCR_OWNER }}/${{ inputs.image }}
VERSION: ${{ needs.meta.outputs.build_version }}
MOVING_TAGS: ${{ inputs.moving-tags }}
run: |
set -euo pipefail
tag_args=()
IFS=',' read -ra moving <<< "${MOVING_TAGS}"
for raw in "${moving[@]}"; do
tag="$(echo "$raw" | xargs)"
[ -n "$tag" ] && tag_args+=( "-t" "${IMAGE}:${tag}" )
done
if (( ${#tag_args[@]} > 0 )); then
docker buildx imagetools create "${tag_args[@]}" "${IMAGE}:${VERSION}"
fi
VERSION: ${{ needs.meta.outputs.build_version }}
run: >-
tools/ci/run.sh image-set
promote
--component "${{ inputs.image }}"
--build-version "${VERSION}"
--registry "ghcr.io/${{ env.GHCR_OWNER }}"
--moving-tags "${MOVING_TAGS}"
@@ -15,6 +15,13 @@ permissions:
contents: write
packages: write
concurrency:
group: publish-fluxer-app-proxy-self-hosted
cancel-in-progress: false
env:
GHCR_OWNER: ${{ github.repository_owner }}
jobs:
approve:
name: approve build release
@@ -26,13 +33,209 @@ jobs:
- name: approved
run: echo "Build release approved."
build:
meta:
name: resolve metadata
needs: approve
uses: ./.github/workflows/_build-image.yaml
secrets: inherit
with:
image: fluxer-app-proxy-self-hosted
dockerfile: fluxer_app_proxy/Dockerfile
build-version: ${{ inputs['build-version'] }}
extra-build-args: |
FLUXER_APP_PROXY_TIME_FREEZE_ENABLED=false
runs-on: ubuntu-24.04
timeout-minutes: 5
permissions:
contents: read
outputs:
build_version: ${{ steps.vars.outputs.build_version }}
steps:
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0
env:
GIT_CONFIG_GLOBAL: ${{ runner.temp }}/gitconfig
- name: Set up Rust toolchain (CI helpers)
uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9
with:
toolchain: "1.93.0"
- name: set variables
id: vars
run: >-
tools/ci/run.sh build-app-proxy
--step set_metadata
--build-version "${{ inputs['build-version'] }}"
dist:
name: build the canonical asset tree
needs: meta
runs-on: ubuntu-24.04
timeout-minutes: 60
permissions:
actions: read
contents: read
packages: write
env:
IMAGE_REPO: ghcr.io/${{ github.repository_owner }}/fluxer-app-proxy-self-hosted
BUILD_VERSION: ${{ needs.meta.outputs.build_version }}
PUBLIC_ASSET_BASE_URL: ""
BUNDLE_LOCAL_ASSETS: "true"
FLUXER_APP_PROXY_TIME_FREEZE_ENABLED: "false"
steps:
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0
env:
GIT_CONFIG_GLOBAL: ${{ runner.temp }}/gitconfig
- name: Set up Rust toolchain (CI helpers)
uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9
with:
toolchain: "1.93.0"
- name: prepare docker config
run: >-
tools/ci/run.sh build-app-proxy
--step prepare_docker_config
- uses: docker/setup-buildx-action@d7f5e7f509e45cec5c76c4d5afdd7de93d0b3df5
- name: configure ghcr auth
env:
GHCR_USERNAME: ${{ github.actor }}
GHCR_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: >-
tools/ci/run.sh build-app-proxy
--step configure_ghcr_auth
- name: build the dist once and publish it as the canonical asset image
env:
CACHE_FROM: type=registry,ref=ghcr.io/${{ env.GHCR_OWNER }}/fluxer-app-proxy-self-hosted:buildcache-dist
CACHE_TO: type=registry,ref=ghcr.io/${{ env.GHCR_OWNER }}/fluxer-app-proxy-self-hosted:buildcache-dist,mode=max,image-manifest=true,oci-mediatypes=true,ignore-error=true
DOCKER_BUILD_SUMMARY: false
DOCKER_BUILD_RECORD_UPLOAD: false
run: >-
tools/ci/run.sh build-app-proxy
--step build_dist
- name: generate asset manifest
run: >-
tools/ci/run.sh build-app-proxy
--step generate_asset_manifest
- name: verify every manifest asset ships in the image
run: >-
tools/ci/run.sh build-app-proxy
--step verify_published_assets
build:
name: build ${{ matrix.platform }}
needs: [meta, dist]
runs-on: ${{ matrix.runner }}
timeout-minutes: 75
permissions:
actions: read
contents: read
packages: write
strategy:
fail-fast: false
matrix:
include:
- platform: amd64
runner: ubuntu-24.04
- platform: arm64
runner: ubuntu-24.04-arm
steps:
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0
env:
GIT_CONFIG_GLOBAL: ${{ runner.temp }}/gitconfig
- name: resolve source date
id: source
run: echo "date=$(TZ=UTC git log -1 --no-show-signature --pretty=%cd --date=format-local:%Y-%m-%dT%H:%M:%SZ)" >> "$GITHUB_OUTPUT"
- uses: docker/setup-buildx-action@d7f5e7f509e45cec5c76c4d5afdd7de93d0b3df5
- uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- uses: docker/build-push-action@f9f3042f7e2789586610d6e8b85c8f03e5195baf
with:
context: .
file: fluxer_app_proxy/Dockerfile
push: true
provenance: false
platforms: linux/${{ matrix.platform }}
tags: ghcr.io/${{ env.GHCR_OWNER }}/fluxer-app-proxy-self-hosted:${{ needs.meta.outputs.build_version }}-${{ matrix.platform }}
build-args: |
BUILD_VERSION=${{ needs.meta.outputs.build_version }}
SOURCE_SHA=${{ github.sha }}
SOURCE_DATE=${{ steps.source.outputs.date }}
FLUXER_APP_PROXY_TIME_FREEZE_ENABLED=false
APP_ASSETS_REF=ghcr.io/${{ env.GHCR_OWNER }}/fluxer-app-proxy-self-hosted:${{ needs.meta.outputs.build_version }}-assets
APP_ASSETS_PLATFORM=linux/amd64
cache-from: type=registry,ref=ghcr.io/${{ env.GHCR_OWNER }}/fluxer-app-proxy-self-hosted:buildcache-${{ matrix.platform }}
cache-to: type=registry,ref=ghcr.io/${{ env.GHCR_OWNER }}/fluxer-app-proxy-self-hosted:buildcache-${{ matrix.platform }},mode=max,image-manifest=true,oci-mediatypes=true,ignore-error=true
env:
DOCKER_BUILD_SUMMARY: false
DOCKER_BUILD_RECORD_UPLOAD: false
merge:
name: merge multi-arch manifest
needs: [meta, build]
runs-on: ubuntu-24.04
timeout-minutes: 20
permissions:
contents: write
packages: write
steps:
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0
env:
GIT_CONFIG_GLOBAL: ${{ runner.temp }}/gitconfig
- name: Set up Rust toolchain (CI helpers)
uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9
with:
toolchain: "1.93.0"
- uses: docker/setup-buildx-action@d7f5e7f509e45cec5c76c4d5afdd7de93d0b3df5
- uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: verify cross-architecture asset parity
env:
APP_PROXY_ASSETS_REF: ghcr.io/${{ env.GHCR_OWNER }}/fluxer-app-proxy-self-hosted:${{ needs.meta.outputs.build_version }}-assets
APP_PROXY_AMD64_REF: ghcr.io/${{ env.GHCR_OWNER }}/fluxer-app-proxy-self-hosted:${{ needs.meta.outputs.build_version }}-amd64
APP_PROXY_ARM64_REF: ghcr.io/${{ env.GHCR_OWNER }}/fluxer-app-proxy-self-hosted:${{ needs.meta.outputs.build_version }}-arm64
run: >-
tools/ci/run.sh build-app-proxy
--step verify_asset_parity
- name: create and push multi-arch manifest
env:
IMAGE: ghcr.io/${{ env.GHCR_OWNER }}/fluxer-app-proxy-self-hosted
VERSION: ${{ needs.meta.outputs.build_version }}
run: |
set -euo pipefail
docker buildx imagetools create -t "${IMAGE}:${VERSION}" \
"${IMAGE}:${VERSION}-amd64" \
"${IMAGE}:${VERSION}-arm64"
docker buildx imagetools inspect "${IMAGE}:${VERSION}"
- name: Create token
id: create-token
uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1
with:
client-id: ${{ vars.FLUXER_CI_APP_ID }}
private-key: ${{ secrets.FLUXER_CI_APP_KEY }}
owner: fluxerapp
repositories: fluxer
permission-contents: write
- name: Publish GitHub release
env:
GH_TOKEN: ${{ steps.create-token.outputs.token }}
SOURCE_SHA: ${{ github.sha }}
VERSION: ${{ needs.meta.outputs.build_version }}
RELEASE_BASELINE_SHA: ${{ vars.RELEASE_BASELINE_SHA }}
run: >-
tools/ci/run.sh release
publish
--component fluxer-app-proxy-self-hosted
--build-version "${VERSION}"
--source-sha "${SOURCE_SHA}"
--previous-sha "${RELEASE_BASELINE_SHA}"
- name: Advance moving image tags
env:
VERSION: ${{ needs.meta.outputs.build_version }}
run: >-
tools/ci/run.sh image-set
promote
--component fluxer-app-proxy-self-hosted
--build-version "${VERSION}"
--registry "ghcr.io/${{ env.GHCR_OWNER }}"
--moving-tags v1,latest
+85 -16
View File
@@ -57,11 +57,11 @@ jobs:
--step set_metadata
--build-version "${{ inputs['build-version'] }}"
build:
name: build app-proxy (amd64)
dist:
name: build and publish the canonical asset tree
needs: meta
runs-on: ubuntu-24.04
timeout-minutes: 45
timeout-minutes: 60
permissions:
actions: read
contents: read
@@ -87,17 +87,17 @@ jobs:
tools/ci/run.sh build-app-proxy
--step configure_ghcr_auth
- name: build and push image + extract assets
- name: build the dist once and publish it as the canonical asset image
env:
BUILD_VERSION: ${{ needs.meta.outputs.build_version }}
PUBLIC_ASSET_BASE_URL: https://fluxerstatic.com
CACHE_FROM: type=registry,ref=ghcr.io/${{ env.GHCR_OWNER }}/fluxer-app-proxy:buildcache-amd64
CACHE_TO: type=registry,ref=ghcr.io/${{ env.GHCR_OWNER }}/fluxer-app-proxy:buildcache-amd64,mode=max,image-manifest=true,oci-mediatypes=true,ignore-error=true
CACHE_FROM: type=registry,ref=ghcr.io/${{ env.GHCR_OWNER }}/fluxer-app-proxy:buildcache-dist
CACHE_TO: type=registry,ref=ghcr.io/${{ env.GHCR_OWNER }}/fluxer-app-proxy:buildcache-dist,mode=max,image-manifest=true,oci-mediatypes=true,ignore-error=true
DOCKER_BUILD_SUMMARY: false
DOCKER_BUILD_RECORD_UPLOAD: false
run: >-
tools/ci/run.sh build-app-proxy
--step build_and_extract
--step build_dist
- name: generate asset manifest
run: >-
@@ -114,9 +114,63 @@ jobs:
tools/ci/run.sh build-app-proxy
--step upload_assets
- name: verify every uploaded asset is readable
env:
PUBLIC_ASSET_BASE_URL: https://fluxerstatic.com
run: >-
tools/ci/run.sh build-app-proxy
--step verify_published_assets
build:
name: build app-proxy (amd64)
needs: [meta, dist]
runs-on: ubuntu-24.04
timeout-minutes: 45
permissions:
actions: read
contents: read
packages: write
steps:
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0
env:
GIT_CONFIG_GLOBAL: ${{ runner.temp }}/gitconfig
- name: Set up Rust toolchain (CI helpers)
uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9
with:
toolchain: "1.93.0"
- name: resolve source date
id: source
run: echo "date=$(TZ=UTC git log -1 --no-show-signature --pretty=%cd --date=format-local:%Y-%m-%dT%H:%M:%SZ)" >> "$GITHUB_OUTPUT"
- name: prepare docker config
run: >-
tools/ci/run.sh build-app-proxy
--step prepare_docker_config
- uses: docker/setup-buildx-action@d7f5e7f509e45cec5c76c4d5afdd7de93d0b3df5
- name: configure ghcr auth
env:
GHCR_USERNAME: ${{ github.actor }}
GHCR_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: >-
tools/ci/run.sh build-app-proxy
--step configure_ghcr_auth
- name: build and push image
env:
BUILD_VERSION: ${{ needs.meta.outputs.build_version }}
SOURCE_SHA: ${{ github.sha }}
SOURCE_DATE: ${{ steps.source.outputs.date }}
PUBLIC_ASSET_BASE_URL: https://fluxerstatic.com
CACHE_FROM: type=registry,ref=ghcr.io/${{ env.GHCR_OWNER }}/fluxer-app-proxy:buildcache-amd64
CACHE_TO: type=registry,ref=ghcr.io/${{ env.GHCR_OWNER }}/fluxer-app-proxy:buildcache-amd64,mode=max,image-manifest=true,oci-mediatypes=true,ignore-error=true
DOCKER_BUILD_SUMMARY: false
DOCKER_BUILD_RECORD_UPLOAD: false
run: >-
tools/ci/run.sh build-app-proxy
--step build_image
build-arm64:
name: build app-proxy (arm64)
needs: meta
needs: [meta, dist]
runs-on: ubuntu-24.04-arm
timeout-minutes: 60
permissions:
@@ -127,6 +181,9 @@ jobs:
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0
env:
GIT_CONFIG_GLOBAL: ${{ runner.temp }}/gitconfig
- name: resolve source date
id: source
run: echo "date=$(TZ=UTC git log -1 --no-show-signature --pretty=%cd --date=format-local:%Y-%m-%dT%H:%M:%SZ)" >> "$GITHUB_OUTPUT"
- uses: docker/setup-buildx-action@d7f5e7f509e45cec5c76c4d5afdd7de93d0b3df5
- uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee
with:
@@ -143,8 +200,10 @@ jobs:
tags: ghcr.io/${{ env.GHCR_OWNER }}/fluxer-app-proxy:${{ needs.meta.outputs.build_version }}-arm64
build-args: |
BUILD_VERSION=${{ needs.meta.outputs.build_version }}
PUBLIC_ASSET_BASE_URL=https://fluxerstatic.com
BUNDLE_LOCAL_ASSETS=false
SOURCE_SHA=${{ github.sha }}
SOURCE_DATE=${{ steps.source.outputs.date }}
APP_ASSETS_REF=ghcr.io/${{ env.GHCR_OWNER }}/fluxer-app-proxy:${{ needs.meta.outputs.build_version }}-assets
APP_ASSETS_PLATFORM=linux/amd64
cache-from: type=registry,ref=ghcr.io/${{ env.GHCR_OWNER }}/fluxer-app-proxy:buildcache-arm64
cache-to: type=registry,ref=ghcr.io/${{ env.GHCR_OWNER }}/fluxer-app-proxy:buildcache-arm64,mode=max,image-manifest=true,oci-mediatypes=true,ignore-error=true
env:
@@ -155,7 +214,7 @@ jobs:
name: merge multi-arch manifest
needs: [meta, build, build-arm64]
runs-on: ubuntu-24.04
timeout-minutes: 10
timeout-minutes: 20
permissions:
contents: write
packages: write
@@ -173,6 +232,15 @@ jobs:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: verify cross-architecture asset parity
env:
APP_PROXY_ASSETS_REF: ghcr.io/${{ env.GHCR_OWNER }}/fluxer-app-proxy:${{ needs.meta.outputs.build_version }}-assets
APP_PROXY_AMD64_REF: ghcr.io/${{ env.GHCR_OWNER }}/fluxer-app-proxy:${{ needs.meta.outputs.build_version }}
APP_PROXY_ARM64_REF: ghcr.io/${{ env.GHCR_OWNER }}/fluxer-app-proxy:${{ needs.meta.outputs.build_version }}-arm64
run: >-
tools/ci/run.sh build-app-proxy
--step verify_asset_parity
- name: fuse amd64 + arm64 into a multi-arch manifest
env:
IMAGE: ghcr.io/${{ env.GHCR_OWNER }}/fluxer-app-proxy
@@ -212,10 +280,11 @@ jobs:
- name: Advance moving image tags
env:
IMAGE: ghcr.io/${{ env.GHCR_OWNER }}/fluxer-app-proxy
VERSION: ${{ needs.meta.outputs.build_version }}
run: >-
docker buildx imagetools create
-t "${IMAGE}:v1"
-t "${IMAGE}:latest"
"${IMAGE}:${VERSION}"
tools/ci/run.sh image-set
promote
--component fluxer-app-proxy
--build-version "${VERSION}"
--registry "ghcr.io/${{ env.GHCR_OWNER }}"
--moving-tags v1,latest
+142
View File
@@ -0,0 +1,142 @@
# SPDX-License-Identifier: AGPL-3.0-or-later
name: release image set
on:
workflow_dispatch:
inputs:
build-version:
description: "Explicit Fluxer CalVer build version (YYYY.MDD.MICRO, UTC HHMMSS without leading zeroes) to use instead of automatic UTC clock allocation"
type: string
required: false
default: ""
from-tag:
description: "Image tag every component is read from (v1 snapshots today's moving tags, a CalVer pins a coordinated build)"
type: string
required: false
default: "v1"
component-versions:
description: "Per-component overrides, one <image>=<version> entry per line (for example fluxer-api=2026.830.191141)"
type: string
required: false
default: ""
permissions:
actions: read
contents: write
packages: read
concurrency:
group: release-image-set
cancel-in-progress: false
defaults:
run:
shell: bash
env:
GHCR_OWNER: ${{ github.repository_owner }}
jobs:
approve:
name: approve image set release
permissions: {}
runs-on: ubuntu-24.04
environment: builds
timeout-minutes: 5
steps:
- name: approved
run: echo "Image set release approved."
manifest:
name: resolve and publish the image set
needs: approve
runs-on: ubuntu-24.04
timeout-minutes: 20
permissions:
contents: write
packages: read
steps:
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0
env:
GIT_CONFIG_GLOBAL: ${{ runner.temp }}/gitconfig
- name: Set up Rust toolchain (CI helpers)
uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9
with:
toolchain: "1.93.0"
- uses: docker/setup-buildx-action@d7f5e7f509e45cec5c76c4d5afdd7de93d0b3df5
- uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ github.token }}
- name: Create token
id: create-token
uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1
with:
client-id: ${{ vars.FLUXER_CI_APP_ID }}
private-key: ${{ secrets.FLUXER_CI_APP_KEY }}
owner: fluxerapp
repositories: fluxer
permission-contents: write
permission-packages: read
- name: set variables
id: vars
env:
GH_TOKEN: ${{ steps.create-token.outputs.token }}
FLUXER_BUILD_VERSION: ${{ inputs['build-version'] }}
run: >-
tools/ci/run.sh resolve-calver
--github-output
- name: resolve release image set
id: resolve
env:
GH_TOKEN: ${{ steps.create-token.outputs.token }}
VERSION: ${{ steps.vars.outputs.build_version }}
FROM_TAG: ${{ inputs['from-tag'] }}
COMPONENT_VERSIONS: ${{ inputs['component-versions'] }}
run: |
set -euo pipefail
args=(
image-set resolve
--version "${VERSION}"
--registry "ghcr.io/${GHCR_OWNER}"
--from-tag "${FROM_TAG}"
--out-dir release-out
--github-output
)
while IFS= read -r entry; do
entry="$(echo "$entry" | xargs)"
if [ -n "$entry" ]; then
args+=( --component-version "$entry" )
fi
done <<< "${COMPONENT_VERSIONS}"
tools/ci/run.sh "${args[@]}"
- name: verify release image set
env:
VERSION: ${{ steps.vars.outputs.build_version }}
run: >-
tools/ci/run.sh image-set verify
--manifest "release-out/fluxer-release-${VERSION}.json"
- name: Publish GitHub release
env:
GH_TOKEN: ${{ steps.create-token.outputs.token }}
VERSION: ${{ steps.vars.outputs.build_version }}
BUNDLE_COMMIT: ${{ steps.resolve.outputs.bundle_commit }}
run: |
set -euo pipefail
if [ -z "${BUNDLE_COMMIT}" ]; then
echo "image-set resolve reported no bundle commit" >&2
exit 1
fi
gh release create "fluxer-release@${VERSION}" \
--repo fluxerapp/fluxer \
--target "${BUNDLE_COMMIT}" \
--title "fluxer-release ${VERSION}" \
--latest=true \
--notes "Immutable image set for ${VERSION}. Every image in the set contains ${BUNDLE_COMMIT}, the commit this tag points at, so the bundle here is never newer than the images. Pin with: docker compose -f docker-compose.yml -f fluxer-release-${VERSION}.yml up -d" \
"release-out/fluxer-release-${VERSION}.json" \
"release-out/fluxer-release-${VERSION}.yml"
Generated
+1
View File
@@ -1958,6 +1958,7 @@ dependencies = [
"base64",
"chrono",
"cookie",
"fluxer_common",
"hmac 0.13.0",
"maud",
"openapiv3",
+48
View File
@@ -1,8 +1,41 @@
FLUXER_DOMAIN=chat.example.com
FLUXER_PUBLIC_SCHEME=https
FLUXER_PUBLIC_PORT=443
FLUXER_PUBLIC_ORIGIN=${FLUXER_PUBLIC_SCHEME}://${FLUXER_DOMAIN}
FLUXER_CADDY_SITE_ADDRESS=chat.example.com
# FLUXER_PUBLIC_ORIGIN is the origin browsers see. It must carry the port
# whenever FLUXER_PUBLIC_PORT is not the default for its scheme, because an
# origin written with a default port never matches a browser Origin header.
# Serving on any other port means setting all three, plus the published port
# below, and pointing FLUXER_CADDY_SITE_ADDRESS at the same scheme and host.
# Compose expands this file from top to bottom, so FLUXER_PUBLIC_ORIGIN has to
# stay below the two values it reads. Above them it silently expands to a bare
# host with a trailing colon.
#FLUXER_PUBLIC_SCHEME=http
#FLUXER_PUBLIC_PORT=19080
#FLUXER_PUBLIC_ORIGIN=${FLUXER_PUBLIC_SCHEME}://${FLUXER_DOMAIN}:${FLUXER_PUBLIC_PORT}
#FLUXER_HTTP_PORT=19080
# Ports Caddy publishes on the host. Caddy still listens on 80 and 443 inside
# the container, so change only these when something else already owns the
# standard ports or another proxy sits in front. Both take an optional bind
# address in front of the port, and 127.0.0.1 keeps the publish off every
# public interface. FLUXER_HTTPS_PORT moves the TCP and the UDP publish
# together, because HTTP/3 needs both on the same port.
#FLUXER_HTTP_PORT=80
#FLUXER_HTTPS_PORT=443
#FLUXER_HTTP_PORT=127.0.0.1:80
#FLUXER_HTTPS_PORT=127.0.0.1:443
# A tunnel or another proxy in front of the stack needs no HTTPS publish at all.
# tunnel.compose.yml ships beside this file and replaces Caddy's published ports
# with a single loopback HTTP publish, so nothing binds 443. FLUXER_HTTP_PORT
# still moves that one publish. Set the line below and plain docker compose
# commands pick the file up, or add it to your own -f flags if you pass any. The
# file uses the !override tag, which needs Compose 2.24.4 or newer.
#COMPOSE_FILE=docker-compose.yml:tunnel.compose.yml
FLUXER_REGISTRY_OWNER=fluxerapp
FLUXER_REGISTRY=ghcr.io/${FLUXER_REGISTRY_OWNER}
FLUXER_IMAGE_TAG=v1
@@ -30,6 +63,7 @@ [email protected]
#FLUXER_PASSKEY_RP_ID=chat.example.com
#FLUXER_PASSKEY_RP_NAME=Fluxer
#FLUXER_PASSKEY_ADDITIONAL_ALLOWED_ORIGINS=https://chat.example.com
#FLUXER_PASSKEY_ADDITIONAL_ALLOWED_ORIGINS=http://chat.example.com:19080
# Extra Content-Security-Policy sources, appended to the built-in ones. Set these
# only when a browser must reach an origin the defaults do not cover, such as a
@@ -49,6 +83,20 @@ [email protected]
LIVEKIT_API_KEY=fluxer
LIVEKIT_API_SECRET=CHANGE_ME
# Ports LiveKit publishes on the host for voice and video media. They take the
# same optional bind address as the Caddy ports above. This media does not pass
# through Caddy or through a tunnel, so it needs these ports reachable from
# clients. LiveKit advertises the port numbers from livekit.yaml, so publishing
# them on different host ports means changing that file too.
#FLUXER_LIVEKIT_TCP_PORT=7881
#FLUXER_LIVEKIT_UDP_PORT=7882
# The voice server URL clients connect to. It defaults to FLUXER_PUBLIC_ORIGIN
# plus /livekit, which the bundled Caddy proxies to the LiveKit container. Set
# it only when LiveKit lives on its own host, and add that origin to
# FLUXER_CSP_EXTRA_CONNECT_SRC when you do.
#FLUXER_LIVEKIT_URL=wss://voice.example.com
FLUXER_KLIPY_API_KEY=
FLUXER_EMAIL_ENABLED=false
+125 -56
View File
@@ -1,6 +1,17 @@
name: fluxer
x-fluxer-postgres-env: &fluxer-postgres-env
FLUXER_DATABASE_BACKEND: postgres
FLUXER_POSTGRES_HOST: postgres
FLUXER_POSTGRES_PORT: "5432"
FLUXER_POSTGRES_DATABASE: fluxer
FLUXER_POSTGRES_USERNAME: fluxer
FLUXER_POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?set POSTGRES_PASSWORD in .env}
FLUXER_POSTGRES_SSL: "false"
FLUXER_POSTGRES_PREPARED_STATEMENTS: ${FLUXER_POSTGRES_PREPARED_STATEMENTS:-true}
x-fluxer-env: &fluxer-env
<<: *fluxer-postgres-env
FLUXER_ENV: production
NODE_ENV: production
FLUXER_SELF_HOSTED: "true"
@@ -12,15 +23,6 @@ x-fluxer-env: &fluxer-env
FLUXER_API_HEADERS_TIMEOUT_MS: ${FLUXER_API_HEADERS_TIMEOUT_MS:-30000}
FLUXER_API_REQUEST_TIMEOUT_MS: ${FLUXER_API_REQUEST_TIMEOUT_MS:-120000}
FLUXER_DATABASE_BACKEND: postgres
FLUXER_POSTGRES_HOST: postgres
FLUXER_POSTGRES_PORT: "5432"
FLUXER_POSTGRES_DATABASE: fluxer
FLUXER_POSTGRES_USERNAME: fluxer
FLUXER_POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?set POSTGRES_PASSWORD in .env}
FLUXER_POSTGRES_SSL: "false"
FLUXER_POSTGRES_PREPARED_STATEMENTS: ${FLUXER_POSTGRES_PREPARED_STATEMENTS:-true}
FLUXER_KV_URL: redis://valkey:6379/0
FLUXER_NATS_URL: nats://nats:4222
FLUXER_NATS_JETSTREAM_URL: nats://nats:4222
@@ -53,6 +55,7 @@ x-fluxer-env: &fluxer-env
FLUXER_LIVEKIT_INTERNAL_URL: http://livekit:7880
FLUXER_LIVEKIT_WEBHOOK_URL: http://api:8080/webhooks/livekit
FLUXER_LIVEKIT_DEFAULT_REGION: '{"id":"default","name":"Default","emoji":"🌍","latitude":0,"longitude":0}'
FLUXER_LIVEKIT_URL: ${FLUXER_LIVEKIT_URL:-${FLUXER_PUBLIC_ORIGIN:-${FLUXER_PUBLIC_SCHEME:-https}://${FLUXER_DOMAIN}}/livekit}
FLUXER_KLIPY_API_KEY: ${FLUXER_KLIPY_API_KEY:-}
@@ -83,7 +86,7 @@ x-fluxer-env: &fluxer-env
FLUXER_VAPID_EMAIL: ${FLUXER_VAPID_EMAIL:-admin@${FLUXER_DOMAIN}}
FLUXER_PASSKEY_RP_ID: ${FLUXER_PASSKEY_RP_ID:-${FLUXER_DOMAIN}}
FLUXER_PASSKEY_RP_NAME: ${FLUXER_PASSKEY_RP_NAME:-Fluxer}
FLUXER_PASSKEY_ADDITIONAL_ALLOWED_ORIGINS: ${FLUXER_PASSKEY_ADDITIONAL_ALLOWED_ORIGINS:-${FLUXER_PUBLIC_SCHEME:-https}://${FLUXER_DOMAIN}}
FLUXER_PASSKEY_ADDITIONAL_ALLOWED_ORIGINS: ${FLUXER_PASSKEY_ADDITIONAL_ALLOWED_ORIGINS:-${FLUXER_PUBLIC_ORIGIN:-${FLUXER_PUBLIC_SCHEME:-https}://${FLUXER_DOMAIN}}}
FLUXER_GATEWAY_RPC_AUTH_TOKEN: ${FLUXER_GATEWAY_RPC_AUTH_TOKEN:?set FLUXER_GATEWAY_RPC_AUTH_TOKEN in .env}
FLUXER_MEDIA_PROXY_SECRET_KEY: ${FLUXER_MEDIA_PROXY_SECRET_KEY:?set FLUXER_MEDIA_PROXY_SECRET_KEY in .env}
FLUXER_MEDIA_PROXY_UPLOAD_RELAY_SECRET_BASE64: ${FLUXER_MEDIA_PROXY_UPLOAD_RELAY_SECRET_BASE64:?set FLUXER_MEDIA_PROXY_UPLOAD_RELAY_SECRET_BASE64 in .env}
@@ -93,15 +96,23 @@ x-fluxer-env: &fluxer-env
FLUXER_INTERNAL_API_ENDPOINT: http://api:8080
FLUXER_INTERNAL_GATEWAY_ENDPOINT: http://gateway:8080
FLUXER_INTERNAL_MEDIA_PROXY_ENDPOINT: http://media-proxy:8080
FLUXER_MARKETING_ENDPOINT: ${FLUXER_PUBLIC_SCHEME:-https}://${FLUXER_DOMAIN}
FLUXER_MARKETING_ENDPOINT: ${FLUXER_PUBLIC_ORIGIN:-${FLUXER_PUBLIC_SCHEME:-https}://${FLUXER_DOMAIN}}
FLUXER_MEDIA_PROXY_ENDPOINT: http://media-proxy:8080
FLUXER_MEDIA_ENDPOINT: ${FLUXER_PUBLIC_SCHEME:-https}://${FLUXER_DOMAIN}/media
FLUXER_MEDIA_PROXY_UPLOAD_RELAY_ENDPOINT: ${FLUXER_PUBLIC_SCHEME:-https}://${FLUXER_DOMAIN}/media
FLUXER_MEDIA_ENDPOINT: ${FLUXER_PUBLIC_ORIGIN:-${FLUXER_PUBLIC_SCHEME:-https}://${FLUXER_DOMAIN}}/media
FLUXER_MEDIA_PROXY_UPLOAD_RELAY_ENDPOINT: ${FLUXER_PUBLIC_ORIGIN:-${FLUXER_PUBLIC_SCHEME:-https}://${FLUXER_DOMAIN}}/media
x-fluxer-service: &fluxer-service
restart: unless-stopped
networks: [fluxer]
x-fluxer-svc-healthcheck: &fluxer-svc-healthcheck
test: ["CMD", "bash", "-c", "exec 3<>/dev/tcp/127.0.0.1/8090 && printf 'GET /_health HTTP/1.0\\r\\n\\r\\n' >&3 && head -n 1 <&3 | grep -q ' 200 '"]
interval: 10s
timeout: 5s
retries: 30
start_period: 60s
start_interval: 1s
services:
caddy:
image: caddy:2.10-alpine
@@ -112,15 +123,20 @@ services:
restart: unless-stopped
networks: [fluxer]
ports:
- "80:80"
- "443:443"
- "443:443/udp"
- "${FLUXER_HTTP_PORT:-80}:80"
- "${FLUXER_HTTPS_PORT:-443}:443"
- "${FLUXER_HTTPS_PORT:-443}:443/udp"
environment:
FLUXER_CADDY_SITE_ADDRESS: ${FLUXER_CADDY_SITE_ADDRESS:?set FLUXER_CADDY_SITE_ADDRESS in .env}
volumes:
- ./Caddyfile:/etc/caddy/Caddyfile:ro
- caddy-data:/data
- caddy-config:/config
healthcheck:
test: ["CMD", "wget", "-q", "-O", "/dev/null", "http://127.0.0.1:2019/config/"]
interval: 10s
timeout: 5s
retries: 10
depends_on:
api: {condition: service_started}
gateway: {condition: service_healthy}
@@ -205,6 +221,11 @@ services:
command: ["-js", "-sd", "/data", "-m", "8222"]
volumes:
- nats-data:/data
healthcheck:
test: ["CMD", "wget", "-q", "-O", "/dev/null", "http://127.0.0.1:8222/healthz"]
interval: 10s
timeout: 5s
retries: 10
meilisearch:
image: getmeili/meilisearch:v1.12
@@ -221,6 +242,11 @@ services:
MEILI_MASTER_KEY: ${MEILI_MASTER_KEY:?set MEILI_MASTER_KEY in .env}
volumes:
- meilisearch-data:/meili_data
healthcheck:
test: ["CMD", "wget", "-q", "-O", "/dev/null", "http://127.0.0.1:7700/health"]
interval: 10s
timeout: 5s
retries: 10
seaweedfs:
image: chrislusf/seaweedfs:4.34
@@ -233,6 +259,11 @@ services:
command: ["server", "-s3", "-dir=/data"]
volumes:
- seaweedfs-data:/data
healthcheck:
test: ["CMD", "wget", "-q", "-O", "/dev/null", "http://127.0.0.1:8333/"]
interval: 10s
timeout: 5s
retries: 20
seaweedfs-init:
image: chrislusf/seaweedfs:4.34
@@ -241,7 +272,8 @@ services:
limits:
memory: ${FLUXER_SEAWEEDFS_INIT_MEMORY_LIMIT:-128mb}
networks: [fluxer]
depends_on: [seaweedfs]
depends_on:
seaweedfs: {condition: service_healthy}
restart: "no"
entrypoint:
- /bin/sh
@@ -287,6 +319,11 @@ services:
ports:
- "${FLUXER_LIVEKIT_TCP_PORT:-7881}:7881"
- "${FLUXER_LIVEKIT_UDP_PORT:-7882}:7882/udp"
healthcheck:
test: ["CMD", "wget", "-q", "-O", "/dev/null", "http://127.0.0.1:7880/"]
interval: 10s
timeout: 5s
retries: 10
api:
<<: *fluxer-service
@@ -313,17 +350,17 @@ services:
depends_on:
postgres: {condition: service_healthy}
valkey: {condition: service_healthy}
nats: {condition: service_started}
meilisearch: {condition: service_started}
nats: {condition: service_healthy}
meilisearch: {condition: service_healthy}
seaweedfs-init: {condition: service_completed_successfully}
gifs: {condition: service_started}
gifs-shard: {condition: service_started}
snowflakes: {condition: service_started}
snowflakes-shard: {condition: service_started}
messages: {condition: service_started}
messages-shard: {condition: service_started}
users: {condition: service_started}
users-shard: {condition: service_started}
gifs: {condition: service_healthy}
gifs-shard: {condition: service_healthy}
snowflakes: {condition: service_healthy}
snowflakes-shard: {condition: service_healthy}
messages: {condition: service_healthy}
messages-shard: {condition: service_healthy}
users: {condition: service_healthy}
users-shard: {condition: service_healthy}
worker:
<<: *fluxer-service
@@ -343,14 +380,21 @@ services:
FLUXER_API_WORKER_ENABLE_CRON_SCHEDULER: "true"
FLUXER_API_WORKER_ENABLE_VOICE_RECONCILIATION: "true"
FLUXER_POSTGRES_MAX_CONNECTIONS: "25"
healthcheck:
test: ["CMD", "node", "-e", "const age=Date.now()-require('node:fs').statSync('/tmp/fluxer-worker-heartbeat').mtimeMs;if(age>30000){console.error('worker heartbeat is '+Math.round(age)+'ms old');process.exit(1)}"]
interval: 10s
timeout: 5s
retries: 3
start_period: 90s
start_interval: 1s
depends_on:
postgres: {condition: service_healthy}
valkey: {condition: service_healthy}
nats: {condition: service_started}
nats: {condition: service_healthy}
seaweedfs-init: {condition: service_completed_successfully}
snowflakes-shard: {condition: service_started}
messages-shard: {condition: service_started}
users-shard: {condition: service_started}
snowflakes-shard: {condition: service_healthy}
messages-shard: {condition: service_healthy}
users-shard: {condition: service_healthy}
gateway:
<<: *fluxer-service
@@ -364,8 +408,8 @@ services:
environment:
<<: *fluxer-env
FLUXER_GATEWAY_PORT: "8080"
FLUXER_GATEWAY_MEDIA_PROXY_ENDPOINT: ${FLUXER_PUBLIC_SCHEME:-https}://${FLUXER_DOMAIN}/media
FLUXER_GATEWAY_STATIC_CDN_ENDPOINT: ${FLUXER_PUBLIC_SCHEME:-https}://${FLUXER_DOMAIN}
FLUXER_GATEWAY_MEDIA_PROXY_ENDPOINT: ${FLUXER_PUBLIC_ORIGIN:-${FLUXER_PUBLIC_SCHEME:-https}://${FLUXER_DOMAIN}}/media
FLUXER_GATEWAY_STATIC_CDN_ENDPOINT: ${FLUXER_PUBLIC_ORIGIN:-${FLUXER_PUBLIC_SCHEME:-https}://${FLUXER_DOMAIN}}
FLUXER_GATEWAY_LOGGER_LEVEL: info
healthcheck:
test: ["CMD", "curl", "-fsS", "-o", "/dev/null", "http://127.0.0.1:8080/_health/ready"]
@@ -374,7 +418,7 @@ services:
retries: 30
start_period: 90s
depends_on:
nats: {condition: service_started}
nats: {condition: service_healthy}
valkey: {condition: service_healthy}
media-proxy:
@@ -390,11 +434,9 @@ services:
FLUXER_MEDIA_PROXY_PORT: "8080"
FLUXER_MEDIA_PROXY_MODE: upload
FLUXER_MEDIA_PROXY_STORAGE_BACKEND: s3
healthcheck:
disable: true
depends_on:
seaweedfs-init: {condition: service_completed_successfully}
nats: {condition: service_started}
nats: {condition: service_healthy}
static-proxy:
<<: *fluxer-service
@@ -403,6 +445,11 @@ services:
resources:
limits:
memory: ${FLUXER_STATIC_PROXY_MEMORY_LIMIT:-256mb}
healthcheck:
test: ["CMD", "wget", "-q", "-O", "/dev/null", "http://127.0.0.1:8080/avatars/0.png"]
interval: 10s
timeout: 5s
retries: 10
app-proxy:
<<: *fluxer-service
@@ -412,11 +459,12 @@ services:
limits:
memory: ${FLUXER_APP_PROXY_MEMORY_LIMIT:-256mb}
environment:
<<: *fluxer-postgres-env
FLUXER_APP_PROXY_HOST: 0.0.0.0
FLUXER_APP_PROXY_PORT: "8080"
DISCOVERY_UPSTREAM_URL: http://caddy:8088/api/.well-known/fluxer
PUBLIC_BOOTSTRAP_API_ENDPOINT: /api
PUBLIC_BOOTSTRAP_API_PUBLIC_ENDPOINT: ${FLUXER_PUBLIC_SCHEME:-https}://${FLUXER_DOMAIN}/api
PUBLIC_BOOTSTRAP_API_PUBLIC_ENDPOINT: ${FLUXER_PUBLIC_ORIGIN:-${FLUXER_PUBLIC_SCHEME:-https}://${FLUXER_DOMAIN}}/api
FLUXER_CSP_EXTRA_DEFAULT_SRC: ${FLUXER_CSP_EXTRA_DEFAULT_SRC:-}
FLUXER_CSP_EXTRA_CONNECT_SRC: ${FLUXER_CSP_EXTRA_CONNECT_SRC:-}
FLUXER_CSP_EXTRA_IMG_SRC: ${FLUXER_CSP_EXTRA_IMG_SRC:-}
@@ -431,7 +479,7 @@ services:
FLUXER_POSTGRES_MAX_CONNECTIONS: "5"
depends_on:
api: {condition: service_healthy}
caddy: {condition: service_started}
caddy: {condition: service_healthy}
postgres: {condition: service_healthy}
snowflakes:
@@ -445,8 +493,9 @@ services:
<<: *fluxer-env
FLUXER_SVC_NAME: snowflakes
FLUXER_SVC_MODE: router
healthcheck: *fluxer-svc-healthcheck
depends_on:
nats: {condition: service_started}
nats: {condition: service_healthy}
snowflakes-shard:
<<: *fluxer-service
@@ -460,8 +509,9 @@ services:
FLUXER_SVC_NAME: snowflakes
FLUXER_SVC_MODE: shard
FLUXER_SVC_SHARD_ID: "0"
healthcheck: *fluxer-svc-healthcheck
depends_on:
nats: {condition: service_started}
nats: {condition: service_healthy}
users:
<<: *fluxer-service
@@ -474,8 +524,9 @@ services:
<<: *fluxer-env
FLUXER_SVC_MODE: router
FLUXER_SVC_MAX_CONCURRENT_REQUESTS: "${FLUXER_SVC_MAX_CONCURRENT_REQUESTS:-20}"
healthcheck: *fluxer-svc-healthcheck
depends_on:
nats: {condition: service_started}
nats: {condition: service_healthy}
users-shard:
<<: *fluxer-service
@@ -490,8 +541,9 @@ services:
FLUXER_SVC_SHARD_ID: "0"
FLUXER_POSTGRES_MAX_CONNECTIONS: "20"
FLUXER_SVC_MAX_CONCURRENT_REQUESTS: "20"
healthcheck: *fluxer-svc-healthcheck
depends_on:
nats: {condition: service_started}
nats: {condition: service_healthy}
postgres: {condition: service_healthy}
gifs:
@@ -505,9 +557,10 @@ services:
<<: *fluxer-env
FLUXER_SVC_NAME: gifs
FLUXER_SVC_MODE: router
FLUXER_MEDIA_PROXY_PUBLIC_ENDPOINT: ${FLUXER_PUBLIC_SCHEME:-https}://${FLUXER_DOMAIN}/media
FLUXER_MEDIA_PROXY_PUBLIC_ENDPOINT: ${FLUXER_PUBLIC_ORIGIN:-${FLUXER_PUBLIC_SCHEME:-https}://${FLUXER_DOMAIN}}/media
healthcheck: *fluxer-svc-healthcheck
depends_on:
nats: {condition: service_started}
nats: {condition: service_healthy}
gifs-shard:
<<: *fluxer-service
@@ -521,9 +574,10 @@ services:
FLUXER_SVC_NAME: gifs
FLUXER_SVC_MODE: shard
FLUXER_SVC_SHARD_ID: "0"
FLUXER_MEDIA_PROXY_PUBLIC_ENDPOINT: ${FLUXER_PUBLIC_SCHEME:-https}://${FLUXER_DOMAIN}/media
FLUXER_MEDIA_PROXY_PUBLIC_ENDPOINT: ${FLUXER_PUBLIC_ORIGIN:-${FLUXER_PUBLIC_SCHEME:-https}://${FLUXER_DOMAIN}}/media
healthcheck: *fluxer-svc-healthcheck
depends_on:
nats: {condition: service_started}
nats: {condition: service_healthy}
messages:
<<: *fluxer-service
@@ -537,8 +591,9 @@ services:
FLUXER_SVC_NAME: messages
FLUXER_SVC_MODE: router
FLUXER_SVC_MAX_CONCURRENT_REQUESTS: "${FLUXER_SVC_MAX_CONCURRENT_REQUESTS:-20}"
healthcheck: *fluxer-svc-healthcheck
depends_on:
nats: {condition: service_started}
nats: {condition: service_healthy}
messages-shard:
<<: *fluxer-service
@@ -554,8 +609,9 @@ services:
FLUXER_SVC_SHARD_ID: "0"
FLUXER_POSTGRES_MAX_CONNECTIONS: "20"
FLUXER_SVC_MAX_CONCURRENT_REQUESTS: "20"
healthcheck: *fluxer-svc-healthcheck
depends_on:
nats: {condition: service_started}
nats: {condition: service_healthy}
postgres: {condition: service_healthy}
unfurl:
@@ -568,8 +624,11 @@ services:
environment:
<<: *fluxer-env
FLUXER_SVC_MODE: router
FLUXER_MEDIA_PROXY_PUBLIC_ENDPOINT: ${FLUXER_PUBLIC_ORIGIN:-${FLUXER_PUBLIC_SCHEME:-https}://${FLUXER_DOMAIN}}/media
FLUXER_STATIC_CDN_ENDPOINT: ${FLUXER_PUBLIC_ORIGIN:-${FLUXER_PUBLIC_SCHEME:-https}://${FLUXER_DOMAIN}}
healthcheck: *fluxer-svc-healthcheck
depends_on:
nats: {condition: service_started}
nats: {condition: service_healthy}
unfurl-shard:
<<: *fluxer-service
@@ -582,8 +641,11 @@ services:
<<: *fluxer-env
FLUXER_SVC_MODE: shard
FLUXER_SVC_SHARD_ID: "0"
FLUXER_MEDIA_PROXY_PUBLIC_ENDPOINT: ${FLUXER_PUBLIC_ORIGIN:-${FLUXER_PUBLIC_SCHEME:-https}://${FLUXER_DOMAIN}}/media
FLUXER_STATIC_CDN_ENDPOINT: ${FLUXER_PUBLIC_ORIGIN:-${FLUXER_PUBLIC_SCHEME:-https}://${FLUXER_DOMAIN}}
healthcheck: *fluxer-svc-healthcheck
depends_on:
nats: {condition: service_started}
nats: {condition: service_healthy}
admin:
<<: *fluxer-service
@@ -598,11 +660,18 @@ services:
FLUXER_ADMIN_PORT: "8080"
FLUXER_ADMIN_BASE_PATH: /admin
FLUXER_API_ENDPOINT: http://api:8080
FLUXER_ADMIN_ENDPOINT: ${FLUXER_PUBLIC_SCHEME:-https}://${FLUXER_DOMAIN}/admin
FLUXER_APP_ENDPOINT: ${FLUXER_PUBLIC_SCHEME:-https}://${FLUXER_DOMAIN}
FLUXER_MEDIA_ENDPOINT: ${FLUXER_PUBLIC_SCHEME:-https}://${FLUXER_DOMAIN}/media
FLUXER_STATIC_CDN_ENDPOINT: ${FLUXER_PUBLIC_SCHEME:-https}://${FLUXER_DOMAIN}
FLUXER_ADMIN_OAUTH_REDIRECT_URI: ${FLUXER_PUBLIC_SCHEME:-https}://${FLUXER_DOMAIN}/admin/oauth2_callback
FLUXER_ADMIN_ENDPOINT: ${FLUXER_PUBLIC_ORIGIN:-${FLUXER_PUBLIC_SCHEME:-https}://${FLUXER_DOMAIN}}/admin
FLUXER_APP_ENDPOINT: ${FLUXER_PUBLIC_ORIGIN:-${FLUXER_PUBLIC_SCHEME:-https}://${FLUXER_DOMAIN}}
FLUXER_MEDIA_ENDPOINT: ${FLUXER_PUBLIC_ORIGIN:-${FLUXER_PUBLIC_SCHEME:-https}://${FLUXER_DOMAIN}}/media
FLUXER_STATIC_CDN_ENDPOINT: ${FLUXER_PUBLIC_ORIGIN:-${FLUXER_PUBLIC_SCHEME:-https}://${FLUXER_DOMAIN}}
FLUXER_ADMIN_OAUTH_REDIRECT_URI: ${FLUXER_PUBLIC_ORIGIN:-${FLUXER_PUBLIC_SCHEME:-https}://${FLUXER_DOMAIN}}/admin/oauth2_callback
healthcheck:
test: ["CMD", "bash", "-c", "exec 3<>/dev/tcp/127.0.0.1/8080 && printf 'GET /_health HTTP/1.0\\r\\n\\r\\n' >&3 && head -n 1 <&3 | grep -q ' 200 '"]
interval: 10s
timeout: 5s
retries: 30
start_period: 60s
start_interval: 1s
depends_on:
api: {condition: service_healthy}
+4
View File
@@ -0,0 +1,4 @@
services:
caddy:
ports: !override
- "${FLUXER_HTTP_PORT:-127.0.0.1:80}:80"
+2
View File
@@ -3,6 +3,7 @@ name = "fluxer_admin"
version = "0.1.0"
edition.workspace = true
license.workspace = true
publish = false
build = "build.rs"
[dependencies]
@@ -11,6 +12,7 @@ axum = { version = "0.8.9", features = ["macros"] }
base64 = "0.22.1"
chrono = { version = "0.4", default-features = false, features = ["serde"] }
cookie = "0.18.1"
fluxer_common = { path = "../fluxer_common" }
hmac = "0.13.0"
maud = { version = "0.27.0", features = ["axum"] }
rand = "0.10"
+16 -1
View File
@@ -24,6 +24,7 @@ RUN TAILWIND_OXIDE_VERSION="4.2.1" \
COPY Cargo.lock Cargo.lock
COPY fluxer_admin fluxer_admin
COPY fluxer_common fluxer_common
COPY packages/fonts/manifest.json packages/fonts/manifest.json
COPY packages/fonts/NOTICE.md packages/fonts/NOTICE.md
COPY packages/fonts/LICENSE-IBM-PLEX.txt packages/fonts/LICENSE-IBM-PLEX.txt
@@ -31,7 +32,7 @@ COPY packages/fonts/files/FluxerSans packages/fonts/files/FluxerSans
COPY packages/fonts/files/FluxerMono packages/fonts/files/FluxerMono
RUN printf '%s\n' \
'[workspace]' \
'members = ["fluxer_admin"]' \
'members = ["fluxer_admin", "fluxer_common"]' \
'resolver = "2"' \
'' \
'[workspace.package]' \
@@ -59,6 +60,20 @@ RUN test "$(ls target/release/build/fluxer_admin-*/out/static/fonts/*.woff2 | wc
FROM debian:bookworm-slim AS runtime
ARG BUILD_VERSION=""
ARG SOURCE_SHA=""
ARG SOURCE_DATE=""
LABEL org.opencontainers.image.title="fluxer-admin"
LABEL org.opencontainers.image.description="Fluxer admin console"
LABEL org.opencontainers.image.licenses="AGPL-3.0-or-later"
LABEL org.opencontainers.image.vendor="Fluxer"
LABEL org.opencontainers.image.url="https://fluxer.app"
LABEL org.opencontainers.image.documentation="https://docs.fluxer.app"
LABEL org.opencontainers.image.source="https://github.com/fluxerapp/fluxer"
LABEL org.opencontainers.image.version="${BUILD_VERSION}"
LABEL org.opencontainers.image.revision="${SOURCE_SHA}"
LABEL org.opencontainers.image.created="${SOURCE_DATE}"
LABEL app.fluxer.build-version="${BUILD_VERSION}"
WORKDIR /usr/local/bin
+118 -20
View File
@@ -1,5 +1,6 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
use fluxer_common::config::normalize_public_endpoint_from_env;
use std::env;
const DEFAULT_ADMIN_OAUTH_CLIENT_ID: &str = "1234567890123456789";
@@ -42,14 +43,14 @@ pub enum RuntimeEnv {
impl AdminConfig {
pub fn from_env() -> Self {
let base_path = normalize_base_path(&read_env("FLUXER_ADMIN_BASE_PATH", ""));
let admin_endpoint = trim_trailing_slash(&read_env(
let admin_endpoint = normalize_public_endpoint_from_env(&trim_trailing_slash(&read_env(
"FLUXER_ADMIN_ENDPOINT",
"https://admin.fluxer.app",
));
let oauth_redirect_uri = read_env_preferred(
)));
let oauth_redirect_uri = normalize_public_endpoint_from_env(&read_env_preferred(
&["FLUXER_ADMIN_OAUTH_REDIRECT_URI"],
&format!("{admin_endpoint}/oauth2_callback"),
);
));
Self {
env: RuntimeEnv::from_env_value(&read_env("FLUXER_ENV", "development")),
@@ -63,17 +64,19 @@ impl AdminConfig {
"FLUXER_API_ENDPOINT",
"https://api.fluxer.app",
)),
media_endpoint: trim_trailing_slash(&read_env(
media_endpoint: normalize_public_endpoint_from_env(&trim_trailing_slash(&read_env(
"FLUXER_MEDIA_ENDPOINT",
"https://media.fluxer.app",
))),
static_cdn_endpoint: normalize_public_endpoint_from_env(&trim_trailing_slash(
&read_env("FLUXER_STATIC_CDN_ENDPOINT", ""),
)),
static_cdn_endpoint: trim_trailing_slash(&read_env("FLUXER_STATIC_CDN_ENDPOINT", "")),
admin_endpoint,
web_app_endpoint: trim_trailing_slash(&read_env(
web_app_endpoint: normalize_public_endpoint_from_env(&trim_trailing_slash(&read_env(
"FLUXER_APP_ENDPOINT",
"https://app.fluxer.app",
)),
))),
kv_url: read_env("FLUXER_KV_URL", ""),
oauth_client_id: read_env(
"FLUXER_ADMIN_OAUTH_CLIENT_ID",
@@ -166,6 +169,39 @@ pub(crate) fn read_bool_env(names: &[&str], fallback: bool) -> bool {
#[cfg(test)]
mod tests {
use super::*;
use std::sync::Mutex;
static ENV_LOCK: Mutex<()> = Mutex::new(());
const MANAGED_ENV: [&str; 11] = [
"FLUXER_ENV",
"FLUXER_ADMIN_HOST",
"FLUXER_ADMIN_PORT",
"FLUXER_ADMIN_ENDPOINT",
"FLUXER_ADMIN_OAUTH_CLIENT_ID",
"FLUXER_ADMIN_OAUTH_REDIRECT_URI",
"FLUXER_MASTER_CONFIG",
"FLUXER_APP_ENDPOINT",
"FLUXER_MEDIA_ENDPOINT",
"FLUXER_STATIC_CDN_ENDPOINT",
"FLUXER_BASE_DOMAIN",
];
fn config_from_env(vars: &[(&str, &str)]) -> AdminConfig {
let _guard = ENV_LOCK.lock().unwrap();
for name in MANAGED_ENV {
unsafe { env::remove_var(name) };
}
unsafe { env::remove_var("FLUXER_PUBLIC_PORT") };
for (name, value) in vars {
unsafe { env::set_var(name, value) };
}
let config = AdminConfig::from_env();
for (name, _) in vars {
unsafe { env::remove_var(name) };
}
config
}
#[test]
fn normalize_base_path_strips_trailing_slashes() {
@@ -287,18 +323,7 @@ mod tests {
#[test]
fn from_env_uses_defaults() {
for var in &[
"FLUXER_ENV",
"FLUXER_ADMIN_HOST",
"FLUXER_ADMIN_PORT",
"FLUXER_ADMIN_ENDPOINT",
"FLUXER_ADMIN_OAUTH_CLIENT_ID",
"FLUXER_ADMIN_OAUTH_REDIRECT_URI",
"FLUXER_MASTER_CONFIG",
] {
unsafe { env::remove_var(var) };
}
let config = AdminConfig::from_env();
let config = config_from_env(&[]);
assert_eq!(config.env, RuntimeEnv::Development);
assert_eq!(config.host, "0.0.0.0");
assert_eq!(config.port, 3020);
@@ -308,4 +333,77 @@ mod tests {
"https://admin.fluxer.app/oauth2_callback"
);
}
#[test]
fn a_non_default_public_port_reaches_the_public_endpoints() {
let config = config_from_env(&[
("FLUXER_BASE_DOMAIN", "fluxer.example"),
("FLUXER_PUBLIC_PORT", "19080"),
("FLUXER_ADMIN_ENDPOINT", "http://fluxer.example/admin"),
("FLUXER_APP_ENDPOINT", "http://fluxer.example:19080"),
("FLUXER_MEDIA_ENDPOINT", "http://fluxer.example/media"),
("FLUXER_STATIC_CDN_ENDPOINT", "https://cdn.example.net"),
(
"FLUXER_ADMIN_OAUTH_REDIRECT_URI",
"http://fluxer.example/admin/oauth2_callback",
),
]);
assert_eq!(config.admin_endpoint, "http://fluxer.example:19080/admin");
assert_eq!(config.media_endpoint, "http://fluxer.example:19080/media");
assert_eq!(config.web_app_endpoint, "http://fluxer.example:19080");
assert_eq!(config.static_cdn_endpoint, "https://cdn.example.net");
assert_eq!(
config.oauth_redirect_uri,
format!("{}/oauth2_callback", config.admin_endpoint)
);
}
#[test]
fn a_default_public_port_leaves_the_public_endpoints_alone() {
let config = config_from_env(&[
("FLUXER_BASE_DOMAIN", "fluxer.example"),
("FLUXER_PUBLIC_PORT", "443"),
("FLUXER_ADMIN_ENDPOINT", "https://fluxer.example/admin"),
("FLUXER_APP_ENDPOINT", "https://fluxer.example"),
("FLUXER_MEDIA_ENDPOINT", "https://fluxer.example/media"),
("FLUXER_STATIC_CDN_ENDPOINT", "https://fluxer.example"),
(
"FLUXER_ADMIN_OAUTH_REDIRECT_URI",
"https://fluxer.example/admin/oauth2_callback",
),
]);
assert_eq!(config.admin_endpoint, "https://fluxer.example/admin");
assert_eq!(config.media_endpoint, "https://fluxer.example/media");
assert_eq!(config.web_app_endpoint, "https://fluxer.example");
assert_eq!(config.static_cdn_endpoint, "https://fluxer.example");
assert_eq!(
config.oauth_redirect_uri,
"https://fluxer.example/admin/oauth2_callback"
);
}
#[test]
fn the_oauth_redirect_uri_matches_the_api_derived_admin_endpoint() {
let config = config_from_env(&[
("FLUXER_BASE_DOMAIN", "fluxer.example"),
("FLUXER_PUBLIC_PORT", "19080"),
("FLUXER_ADMIN_ENDPOINT", "http://fluxer.example/admin"),
(
"FLUXER_ADMIN_OAUTH_REDIRECT_URI",
"http://fluxer.example/admin/oauth2_callback",
),
]);
let api_admin_endpoint = fluxer_common::config::normalize_public_endpoint(
"http://fluxer.example/admin",
"fluxer.example",
Some(19080),
);
assert_eq!(
config.oauth_redirect_uri,
format!("{api_admin_endpoint}/oauth2_callback")
);
}
}
+14
View File
@@ -29,6 +29,20 @@ RUN pnpm deploy --legacy --filter=fluxer_api --prod --config.allowUnusedPatches=
FROM node:24-bookworm-slim
ARG BUILD_VERSION
ARG SOURCE_SHA
ARG SOURCE_DATE
LABEL org.opencontainers.image.title="fluxer-api"
LABEL org.opencontainers.image.description="Fluxer HTTP API and background workers"
LABEL org.opencontainers.image.licenses="AGPL-3.0-or-later"
LABEL org.opencontainers.image.vendor="Fluxer"
LABEL org.opencontainers.image.url="https://fluxer.app"
LABEL org.opencontainers.image.documentation="https://docs.fluxer.app"
LABEL org.opencontainers.image.source="https://github.com/fluxerapp/fluxer"
LABEL org.opencontainers.image.version="${BUILD_VERSION}"
LABEL org.opencontainers.image.revision="${SOURCE_SHA}"
LABEL org.opencontainers.image.created="${SOURCE_DATE}"
LABEL app.fluxer.build-version="${BUILD_VERSION}"
WORKDIR /usr/src/app/fluxer_api
+77 -16
View File
@@ -2,6 +2,8 @@
const CACHE_INFLIGHT_MAX_ENTRIES = 10000;
const CACHE_INFLIGHT_JOIN_RETRIES = 1;
const CACHE_PRODUCE_TIMEOUT_MS = 15000;
const CACHE_PRODUCE_TIMEOUT_MESSAGE = 'Cache produce timed out';
interface CacheMSetEntry<T> {
key: string;
@@ -9,6 +11,15 @@ interface CacheMSetEntry<T> {
ttlSeconds?: number;
}
interface CacheProduceTracking {
generation: number;
produces: number;
}
interface CacheProduceAbandonment {
abandoned: boolean;
}
export type CacheLookupResult<T> = {hit: true; value: T} | {hit: false};
type CacheTtlSeconds<T> = number | ((value: T) => number);
@@ -17,7 +28,7 @@ type CacheJoinResult<T> = {joined: true; value: T} | {joined: false; error: unkn
export abstract class ICacheService {
private readonly inflightValues = new Map<string, Promise<unknown>>();
private readonly produceInvalidations = new Map<string, number>();
private readonly produceInvalidations = new Map<string, CacheProduceTracking>();
abstract getEntry<T>(key: string): Promise<CacheLookupResult<T>>;
@@ -26,9 +37,9 @@ export abstract class ICacheService {
protected abstract deleteEntry(key: string): Promise<void>;
async delete(key: string): Promise<void> {
const pending = this.produceInvalidations.get(key);
if (pending !== undefined) {
this.produceInvalidations.set(key, pending + 1);
const tracked = this.produceInvalidations.get(key);
if (tracked) {
tracked.generation += 1;
}
await this.deleteEntry(key);
}
@@ -70,7 +81,12 @@ export abstract class ICacheService {
return entry.hit ? entry.value : null;
}
async getOrSet<T>(key: string, valueFactory: () => Promise<T>, ttlSeconds?: CacheTtlSeconds<T>): Promise<T> {
async getOrSet<T>(
key: string,
valueFactory: () => Promise<T>,
ttlSeconds?: CacheTtlSeconds<T>,
produceTimeoutMs: number = CACHE_PRODUCE_TIMEOUT_MS,
): Promise<T> {
let generation = this.trackProduce(key);
try {
for (let attempt = 0; ; attempt++) {
@@ -80,7 +96,7 @@ export abstract class ICacheService {
}
const inflight = this.inflightValues.get(key);
if (!inflight) {
return await this.produceSingleFlight(key, valueFactory, ttlSeconds, generation);
return await this.produceSingleFlight(key, valueFactory, ttlSeconds, generation, produceTimeoutMs);
}
const joined = await this.joinInflight<T>(inflight);
if (joined.joined) {
@@ -89,21 +105,34 @@ export abstract class ICacheService {
if (attempt >= CACHE_INFLIGHT_JOIN_RETRIES) {
throw joined.error;
}
generation = this.trackProduce(key);
generation = this.currentGeneration(key);
}
} finally {
this.releaseProduce(key, generation);
this.releaseProduce(key);
}
}
private trackProduce(key: string): number {
const generation = this.produceInvalidations.get(key) ?? 0;
this.produceInvalidations.set(key, generation);
return generation;
const tracked = this.produceInvalidations.get(key);
if (tracked) {
tracked.produces += 1;
return tracked.generation;
}
this.produceInvalidations.set(key, {generation: 0, produces: 1});
return 0;
}
private releaseProduce(key: string, generation: number): void {
if ((this.produceInvalidations.get(key) ?? 0) === generation) {
private currentGeneration(key: string): number {
return this.produceInvalidations.get(key)?.generation ?? 0;
}
private releaseProduce(key: string): void {
const tracked = this.produceInvalidations.get(key);
if (!tracked) {
return;
}
tracked.produces -= 1;
if (tracked.produces <= 0) {
this.produceInvalidations.delete(key);
}
}
@@ -121,25 +150,57 @@ export abstract class ICacheService {
valueFactory: () => Promise<T>,
ttlSeconds: CacheTtlSeconds<T> | undefined,
generation: number,
produceTimeoutMs: number,
): Promise<T> {
const abandonment: CacheProduceAbandonment = {abandoned: false};
const produced = this.boundProduce(
this.produceAndStore(key, valueFactory, ttlSeconds, generation, abandonment),
abandonment,
produceTimeoutMs,
);
if (this.inflightValues.size >= CACHE_INFLIGHT_MAX_ENTRIES) {
return await this.produceAndStore(key, valueFactory, ttlSeconds, generation);
return await produced;
}
const pending = this.produceAndStore(key, valueFactory, ttlSeconds, generation).finally(() => {
const pending = produced.finally(() => {
this.inflightValues.delete(key);
});
this.inflightValues.set(key, pending);
return await pending;
}
private boundProduce<T>(
produced: Promise<T>,
abandonment: CacheProduceAbandonment,
produceTimeoutMs: number,
): Promise<T> {
return new Promise<T>((resolve, reject) => {
const timer = setTimeout(() => {
abandonment.abandoned = true;
reject(new Error(CACHE_PRODUCE_TIMEOUT_MESSAGE));
}, produceTimeoutMs);
timer.unref?.();
produced.then(
(value) => {
clearTimeout(timer);
resolve(value);
},
(error: unknown) => {
clearTimeout(timer);
reject(error);
},
);
});
}
private async produceAndStore<T>(
key: string,
valueFactory: () => Promise<T>,
ttlSeconds: CacheTtlSeconds<T> | undefined,
generation: number,
abandonment: CacheProduceAbandonment,
): Promise<T> {
const value = await valueFactory();
if ((this.produceInvalidations.get(key) ?? 0) === generation) {
if (!abandonment.abandoned && this.currentGeneration(key) === generation) {
await this.set(key, value, typeof ttlSeconds === 'function' ? ttlSeconds(value) : ttlSeconds);
}
return value;
@@ -0,0 +1,57 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import {KVCacheProvider} from '@pkgs/cache/src/providers/KVCacheProvider';
import type {IKVPipeline, IKVProvider} from '@pkgs/kv_client/src/IKVProvider';
import {computeHashSlot} from '@pkgs/kv_client/src/KVHashSlots';
import {describe, expect, it} from 'vitest';
function createRecordingProvider(): {
client: IKVProvider;
commands: Array<Array<string>>;
} {
const commands: Array<Array<string>> = [];
const client = {
set: async (key: string) => {
commands.push([key]);
return 'OK';
},
setex: async (key: string) => {
commands.push([key]);
},
isClustered: () => true,
pipeline: () => {
const keys: Array<string> = [];
commands.push(keys);
const batch = {
set: (key: string) => {
keys.push(key);
return batch;
},
setex: (key: string) => {
keys.push(key);
return batch;
},
exec: async () => [],
} as unknown as IKVPipeline;
return batch;
},
} as unknown as IKVProvider;
return {client, commands};
}
describe('KVCacheProvider cluster hash slots', () => {
it('keeps a multi entry write off batched commands that span hash slots', async () => {
const {client, commands} = createRecordingProvider();
const provider = new KVCacheProvider({client});
expect(computeHashSlot('cache:alpha')).not.toBe(computeHashSlot('cache:beta'));
await provider.mset([
{key: 'cache:alpha', value: 1, ttlSeconds: 60},
{key: 'cache:beta', value: 2},
]);
expect(commands.flat().sort()).toEqual(['cache:alpha', 'cache:beta']);
expect(commands.filter((keys) => new Set(keys.map(computeHashSlot)).size > 1)).toEqual([]);
});
});
+107
View File
@@ -0,0 +1,107 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import {KVCacheProvider} from '@pkgs/cache/src/providers/KVCacheProvider';
import type {IKVPipeline, IKVProvider} from '@pkgs/kv_client/src/IKVProvider';
import {computeHashSlot} from '@pkgs/kv_client/src/KVHashSlots';
import {describe, expect, it} from 'vitest';
const MAX_CONCURRENT_ROUND_TRIPS = 16;
interface RecordingProvider {
client: IKVProvider;
batches: Array<Array<string>>;
peakInFlight: number;
}
function createRecordingProvider(clustered: boolean): RecordingProvider {
const recorder: RecordingProvider = {
client: {} as IKVProvider,
batches: [],
peakInFlight: 0,
};
let inFlight = 0;
const trackRoundTrip = async (keys: Array<string>): Promise<void> => {
recorder.batches.push(keys);
inFlight += 1;
recorder.peakInFlight = Math.max(recorder.peakInFlight, inFlight);
await new Promise((resolve) => setTimeout(resolve, 0));
inFlight -= 1;
};
recorder.client = {
isClustered: () => clustered,
set: async (key: string) => {
await trackRoundTrip([key]);
return 'OK';
},
setex: async (key: string) => {
await trackRoundTrip([key]);
},
pipeline: () => {
const keys: Array<string> = [];
const batch = {
set: (key: string) => {
keys.push(key);
return batch;
},
setex: (key: string) => {
keys.push(key);
return batch;
},
exec: async () => {
await trackRoundTrip(keys);
return [];
},
} as unknown as IKVPipeline;
return batch;
},
} as unknown as IKVProvider;
return recorder;
}
function createEntries(count: number): Array<{key: string; value: number; ttlSeconds: number}> {
return Array.from({length: count}, (_unused, index) => ({
key: `cache:entry:${index}`,
value: index,
ttlSeconds: 60,
}));
}
describe('KVCacheProvider multi entry write fan out', () => {
it('writes every entry in one round trip outside cluster mode', async () => {
const recorder = createRecordingProvider(false);
const provider = new KVCacheProvider({client: recorder.client});
await provider.mset(createEntries(1000));
expect(recorder.batches.map((keys) => keys.length)).toEqual([1000]);
expect(recorder.peakInFlight).toBe(1);
});
it('surfaces a failed command inside a batched write', async () => {
const client = {
isClustered: () => false,
pipeline: () => {
const batch = {
set: () => batch,
setex: () => batch,
exec: async () => [[new Error('write rejected'), null]],
} as unknown as IKVPipeline;
return batch;
},
} as unknown as IKVProvider;
const provider = new KVCacheProvider({client});
await expect(provider.mset(createEntries(2))).rejects.toThrow('write rejected');
});
it('bounds concurrent round trips when entries span hash slots', async () => {
const recorder = createRecordingProvider(true);
const provider = new KVCacheProvider({client: recorder.client});
await provider.mset(createEntries(1000));
expect(recorder.peakInFlight).toBeLessThanOrEqual(MAX_CONCURRENT_ROUND_TRIPS);
expect(recorder.batches.filter((keys) => new Set(keys.map(computeHashSlot)).size > 1)).toEqual([]);
expect(recorder.batches.flat().length).toBe(1000);
});
});
@@ -1,7 +1,11 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import {InMemoryProvider} from '@pkgs/cache/src/providers/InMemoryProvider';
import {describe, expect, it} from 'vitest';
import {describe, expect, it, vi} from 'vitest';
const INFLIGHT_OVERFLOW_ENTRIES = 10000;
const PRODUCE_TIMEOUT_MS = 50;
const PRODUCE_TIMEOUT_MESSAGE = 'Cache produce timed out';
function deferred<T>(): {promise: Promise<T>; resolve: (value: T) => void; reject: (error: Error) => void} {
let resolve!: (value: T) => void;
@@ -17,6 +21,20 @@ function flush(): Promise<void> {
return new Promise((resolve) => setTimeout(resolve, 0));
}
function settleWithin<T>(pending: Promise<T>, ms: number): Promise<T | 'pinned' | 'rejected'> {
return Promise.race([
pending.then(
(value) => value,
() => 'rejected' as const,
),
new Promise<'pinned'>((resolve) => setTimeout(() => resolve('pinned'), ms)),
]);
}
function trackedProduceKeys(cache: InMemoryProvider): Array<string> {
return [...(cache as unknown as {produceInvalidations: Map<string, unknown>}).produceInvalidations.keys()];
}
describe('cache invalidation during an in-flight produce', () => {
it('does not resurrect a value deleted while the factory was running', async () => {
const cache = new InMemoryProvider();
@@ -77,6 +95,192 @@ describe('cache invalidation during an in-flight produce', () => {
expect(await cache.get('session')).toBe('retried-session');
});
it('stores the value a retried produce built after the first produce was invalidated', async () => {
const cache = new InMemoryProvider();
const gates: Array<ReturnType<typeof deferred<string>>> = [];
const factory = async () => {
const gate = deferred<string>();
gates.push(gate);
return await gate.promise;
};
const producer = cache.getOrSet('session', factory, 30);
const joiner = cache.getOrSet('session', factory, 30);
await flush();
await cache.delete('session');
gates[0].reject(new Error('produce failed'));
await expect(producer).rejects.toThrow('produce failed');
await flush();
expect(gates).toHaveLength(2);
gates[1].resolve('retried-session');
await expect(joiner).resolves.toBe('retried-session');
expect(await cache.get('session')).toBe('retried-session');
});
it('does not resurrect a value deleted after a concurrent caller released its produce', async () => {
const cache = new InMemoryProvider();
const gate = deferred<string>();
const pending = cache.getOrSet('session', async () => await gate.promise, 30);
await flush();
await cache.set('session', 'served-from-cache', 30);
await expect(cache.getOrSet('session', async () => 'unused', 30)).resolves.toBe('served-from-cache');
await cache.delete('session');
gate.resolve('stale-produce');
await expect(pending).resolves.toBe('stale-produce');
expect(await cache.get('session')).toBeNull();
});
it('does not resurrect a value deleted while a second overflow produce was running', async () => {
const cache = new InMemoryProvider();
const fillers: Array<ReturnType<typeof deferred<string>>> = [];
const filling: Array<Promise<string>> = [];
for (let index = 0; index < INFLIGHT_OVERFLOW_ENTRIES; index++) {
const gate = deferred<string>();
fillers.push(gate);
filling.push(cache.getOrSet(`filler:${index}`, async () => await gate.promise, 30));
}
await flush();
const first = deferred<string>();
const second = deferred<string>();
const firstProduce = cache.getOrSet('session', async () => await first.promise, 30);
const secondProduce = cache.getOrSet('session', async () => await second.promise, 30);
await flush();
first.resolve('first-produce');
await expect(firstProduce).resolves.toBe('first-produce');
await cache.delete('session');
second.resolve('second-produce');
await expect(secondProduce).resolves.toBe('second-produce');
expect(await cache.get('session')).toBeNull();
for (const gate of fillers) {
gate.resolve('filler');
}
await Promise.all(filling);
});
it('drops produce tracking once the last produce for a key settles', async () => {
const cache = new InMemoryProvider();
for (let index = 0; index < 50; index++) {
const gate = deferred<string>();
const pending = cache.getOrSet(`session:${index}`, async () => await gate.promise, 30);
await cache.delete(`session:${index}`);
gate.resolve('value');
await pending;
}
const shared = deferred<string>();
const producer = cache.getOrSet('shared', async () => await shared.promise, 30);
const joiner = cache.getOrSet('shared', async () => 'unused', 30);
await flush();
await cache.delete('shared');
shared.resolve('shared-value');
await Promise.all([producer, joiner]);
const failing = cache.getOrSet(
'failing',
async () => {
throw new Error('produce failed');
},
30,
);
await expect(failing).rejects.toThrow('produce failed');
expect(trackedProduceKeys(cache)).toEqual([]);
});
it('does not pin a key forever when the factory never settles', async () => {
const cache = new InMemoryProvider();
const stuck = deferred<string>();
const pinned = cache.getOrSet('session', async () => await stuck.promise, 30, PRODUCE_TIMEOUT_MS);
await expect(settleWithin(pinned, 500)).resolves.toBe('rejected');
await expect(pinned).rejects.toThrow(PRODUCE_TIMEOUT_MESSAGE);
const recovered = cache.getOrSet('session', async () => 'recovered', 30, PRODUCE_TIMEOUT_MS);
await expect(settleWithin(recovered, 500)).resolves.toBe('recovered');
stuck.resolve('never-settled');
await flush();
expect(await cache.get('session')).toBe('recovered');
});
it('does not store a value produced by a factory that settled after the timeout', async () => {
const cache = new InMemoryProvider();
const stuck = deferred<string>();
const pending = cache.getOrSet('session', async () => await stuck.promise, 30, PRODUCE_TIMEOUT_MS);
await expect(pending).rejects.toThrow(PRODUCE_TIMEOUT_MESSAGE);
stuck.resolve('late-produce');
await flush();
expect(await cache.get('session')).toBeNull();
expect(trackedProduceKeys(cache)).toEqual([]);
});
it('retries once for the joiners when the producer times out', async () => {
const cache = new InMemoryProvider();
const gates: Array<ReturnType<typeof deferred<string>>> = [];
const factory = async () => {
const gate = deferred<string>();
gates.push(gate);
return await gate.promise;
};
const producer = cache.getOrSet('session', factory, 30, PRODUCE_TIMEOUT_MS);
const joiner = cache.getOrSet('session', factory, 30, PRODUCE_TIMEOUT_MS);
await expect(producer).rejects.toThrow(PRODUCE_TIMEOUT_MESSAGE);
await flush();
expect(gates).toHaveLength(2);
gates[1].resolve('retried-session');
await expect(joiner).resolves.toBe('retried-session');
expect(await cache.get('session')).toBe('retried-session');
gates[0].resolve('abandoned-produce');
await flush();
expect(await cache.get('session')).toBe('retried-session');
});
it('releases produce tracking when the factory never settles', async () => {
const cache = new InMemoryProvider();
const stuck = deferred<string>();
const pending = cache.getOrSet('session', async () => await stuck.promise, 30, PRODUCE_TIMEOUT_MS);
await expect(pending).rejects.toThrow(PRODUCE_TIMEOUT_MESSAGE);
await flush();
expect(trackedProduceKeys(cache)).toEqual([]);
});
it('stores a sibling produce that succeeded after an overflow produce timed out', async () => {
const cache = new InMemoryProvider();
const fillers: Array<ReturnType<typeof deferred<string>>> = [];
const filling: Array<Promise<string>> = [];
for (let index = 0; index < INFLIGHT_OVERFLOW_ENTRIES; index++) {
const gate = deferred<string>();
fillers.push(gate);
filling.push(cache.getOrSet(`filler:${index}`, async () => await gate.promise, 30));
}
await flush();
const stuck = deferred<string>();
const sibling = deferred<string>();
const abandoned = cache.getOrSet('session', async () => await stuck.promise, 30, PRODUCE_TIMEOUT_MS);
const succeeding = cache.getOrSet('session', async () => await sibling.promise, 30, PRODUCE_TIMEOUT_MS * 100);
await expect(abandoned).rejects.toThrow(PRODUCE_TIMEOUT_MESSAGE);
sibling.resolve('sibling-produce');
await expect(succeeding).resolves.toBe('sibling-produce');
expect(await cache.get('session')).toBe('sibling-produce');
for (const gate of fillers) {
gate.resolve('filler');
}
await Promise.all(filling);
});
it('does not hold the event loop open while a produce is in flight', async () => {
const cache = new InMemoryProvider();
const stuck = deferred<string>();
const timers: Array<NodeJS.Timeout> = [];
const scheduled = globalThis.setTimeout;
const spy = vi.spyOn(globalThis, 'setTimeout').mockImplementation(((handler: () => void, ms?: number) => {
const timer = scheduled(handler, ms);
if (ms === PRODUCE_TIMEOUT_MS) {
timers.push(timer);
}
return timer;
}) as typeof globalThis.setTimeout);
const pending = cache.getOrSet('session', async () => await stuck.promise, 30, PRODUCE_TIMEOUT_MS);
await flush();
spy.mockRestore();
expect(timers).toHaveLength(1);
expect(timers[0].hasRef()).toBe(false);
await expect(pending).rejects.toThrow(PRODUCE_TIMEOUT_MESSAGE);
});
it('keeps a later produce cacheable after an earlier one was invalidated', async () => {
const cache = new InMemoryProvider();
const first = deferred<string>();
+21 -30
View File
@@ -11,6 +11,7 @@ import type {CacheLogger, CacheTelemetry} from '@pkgs/cache/src/CacheProviderTyp
import {parseCachedValue, safeJsonParse, serializeValue} from '@pkgs/cache/src/CacheSerialization';
import {type CacheLookupResult, ICacheService} from '@pkgs/cache/src/ICacheService';
import type {IKVProvider} from '@pkgs/kv_client/src/IKVProvider';
import {runSlotBatches, splitIntoSlotBatches} from '@pkgs/kv_client/src/KVHashSlots';
interface KVCacheProviderConfig {
client: IKVProvider;
@@ -137,37 +138,27 @@ export class KVCacheProvider extends ICacheService {
}>,
): Promise<void> {
if (entries.length === 0) return;
const withoutTtl: Array<{
key: string;
value: T;
}> = [];
const withTtl: Array<{
key: string;
value: T;
ttlSeconds: number;
}> = [];
for (const entry of entries) {
if (entry.ttlSeconds) {
withTtl.push({
key: entry.key,
value: entry.value,
ttlSeconds: entry.ttlSeconds,
});
} else {
withoutTtl.push({
key: entry.key,
value: entry.value,
});
const serialized = entries.map((entry) => ({
key: entry.key,
value: serializeValue(entry.value),
ttlSeconds: entry.ttlSeconds,
}));
const batches = splitIntoSlotBatches(serialized, (entry) => entry.key, this.client.isClustered());
await runSlotBatches(batches, async (batch) => {
const pipeline = this.client.pipeline();
for (const entry of batch) {
if (entry.ttlSeconds) {
pipeline.setex(entry.key, entry.ttlSeconds, entry.value);
} else {
pipeline.set(entry.key, entry.value);
}
}
}
const pipeline = this.client.pipeline();
for (const entry of withoutTtl) {
pipeline.set(entry.key, serializeValue(entry.value));
}
for (const entry of withTtl) {
pipeline.setex(entry.key, entry.ttlSeconds, serializeValue(entry.value));
}
await pipeline.exec();
for (const [error] of await pipeline.exec()) {
if (error) {
throw error;
}
}
});
}
async deletePattern(pattern: string): Promise<number> {
@@ -88,6 +88,7 @@ export interface IKVProvider {
refillIntervalMs: number,
): Promise<number>;
scheduleBulkDeletion(queueKey: string, secondaryKey: string, score: number, value: string): Promise<void>;
claimBulkDeletion(queueKey: string, member: string, maxScore: number, leaseScore: number): Promise<boolean>;
removeBulkDeletion(queueKey: string, secondaryKey: string, member?: string): Promise<boolean>;
scan(pattern: string, count: number): Promise<Array<string>>;
dequeuePurgeBatch(
@@ -103,5 +104,6 @@ export interface IKVProvider {
}>;
pipeline(): IKVPipeline;
multi(): IKVPipeline;
isClustered(): boolean;
health(): Promise<boolean>;
}
+62 -4
View File
@@ -16,6 +16,7 @@ import {
parseRangeByScoreArguments,
parseSetArguments,
} from '@pkgs/kv_client/src/KVCommandArguments';
import {runSlotBatches, splitIntoSlotBatches} from '@pkgs/kv_client/src/KVHashSlots';
import {KVPipeline} from '@pkgs/kv_client/src/KVPipeline';
import {KVSubscription} from '@pkgs/kv_client/src/KVSubscription';
import Redis, {Cluster} from 'ioredis';
@@ -223,6 +224,17 @@ tokens = tokens - #urls
redis.call('SET', bucketKey, cjson.encode({tokens = tokens, lastRefill = lastRefill}), 'EX', 3600)
return cjson.encode({urls = urls, tokens = #urls})
`;
const CLAIM_BULK_DELETION_SCRIPT = `
local score = redis.call('ZSCORE', KEYS[1], ARGV[1])
if not score then
return 0
end
if tonumber(score) > tonumber(ARGV[2]) then
return 0
end
redis.call('ZADD', KEYS[1], ARGV[3], ARGV[1])
return 1
`;
const REMOVE_BULK_DELETION_SCRIPT = `
local member = ARGV[1]
if member ~= '' and redis.call('ZREM', KEYS[1], member) == 1 then
@@ -344,7 +356,23 @@ export class KVClient implements IKVProvider {
}
async mget(...keys: Array<string>): Promise<Array<string | null>> {
return await this.execute('mget', async () => this.client.mget(...keys));
if (keys.length === 0) {
return [];
}
return await this.execute('mget', async () => {
const values = new Array<string | null>(keys.length).fill(null);
const batches = this.splitBySlot(
keys.map((key, index) => ({key, index})),
(entry) => entry.key,
);
await runSlotBatches(batches, async (batch) => {
const batchValues = await this.client.mget(...batch.map((entry) => entry.key));
for (const [position, entry] of batch.entries()) {
values[entry.index] = batchValues[position] ?? null;
}
});
return values;
});
}
async mset(...args: Array<string>): Promise<void> {
@@ -352,9 +380,11 @@ export class KVClient implements IKVProvider {
if (entries.length === 0) {
return;
}
const pairs = entries.flatMap((entry) => [entry.key, entry.value]);
await this.execute('mset', async () => {
await this.client.mset(...pairs);
const batches = this.splitBySlot(entries, (entry) => entry.key);
await runSlotBatches(batches, async (batch) => {
await this.client.mset(...batch.flatMap((entry) => [entry.key, entry.value]));
});
});
}
@@ -362,7 +392,14 @@ export class KVClient implements IKVProvider {
if (keys.length === 0) {
return 0;
}
return await this.execute('del', async () => this.client.del(...keys));
return await this.execute('del', async () => {
const deleted: Array<number> = [];
const batches = this.splitBySlot(keys, (key) => key);
await runSlotBatches(batches, async (batch) => {
deleted.push(await this.client.del(...batch));
});
return deleted.reduce((total, count) => total + count, 0);
});
}
async exists(key: string): Promise<number> {
@@ -613,6 +650,19 @@ export class KVClient implements IKVProvider {
);
}
async claimBulkDeletion(queueKey: string, member: string, maxScore: number, leaseScore: number): Promise<boolean> {
const result = await this.executeScript(
'claimBulkDeletion',
CLAIM_BULK_DELETION_SCRIPT,
1,
queueKey,
member,
maxScore,
leaseScore,
);
return Number(result) === 1;
}
async removeBulkDeletion(queueKey: string, secondaryKey: string, member = ''): Promise<boolean> {
const result = await this.executeScript(
'removeBulkDeletion',
@@ -678,6 +728,14 @@ export class KVClient implements IKVProvider {
});
}
isClustered(): boolean {
return this.config.mode === 'cluster';
}
private splitBySlot<T>(items: ReadonlyArray<T>, keyOf: (item: T) => string): Array<Array<T>> {
return splitIntoSlotBatches(items, keyOf, this.isClustered());
}
pipeline(): IKVPipeline {
return new KVPipeline({
createCommander: () => this.client.pipeline(),
@@ -0,0 +1,68 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
const HASH_SLOT_COUNT = 16384;
const MAX_CONCURRENT_SLOT_BATCHES = 16;
function extractHashTag(key: string): string {
const start = key.indexOf('{');
if (start === -1) {
return key;
}
const end = key.indexOf('}', start + 1);
if (end > start + 1) {
return key.slice(start + 1, end);
}
return key;
}
export function computeHashSlot(key: string): number {
const hashed = extractHashTag(key);
let crc = 0;
for (let index = 0; index < hashed.length; index += 1) {
crc ^= (hashed.charCodeAt(index) & 0xff) << 8;
for (let bit = 0; bit < 8; bit += 1) {
crc = (crc & 0x8000) === 0 ? (crc << 1) & 0xffff : ((crc << 1) ^ 0x1021) & 0xffff;
}
}
return crc % HASH_SLOT_COUNT;
}
export function splitIntoSlotBatches<T>(
items: ReadonlyArray<T>,
keyOf: (item: T) => string,
clustered: boolean,
): Array<Array<T>> {
if (items.length === 0) {
return [];
}
if (!clustered) {
return [[...items]];
}
const batches = new Map<number, Array<T>>();
for (const item of items) {
const slot = computeHashSlot(keyOf(item));
const batch = batches.get(slot);
if (batch) {
batch.push(item);
} else {
batches.set(slot, [item]);
}
}
return [...batches.values()];
}
export async function runSlotBatches<T>(batches: ReadonlyArray<T>, run: (batch: T) => Promise<void>): Promise<void> {
if (batches.length <= MAX_CONCURRENT_SLOT_BATCHES) {
await Promise.all(batches.map(async (batch) => await run(batch)));
return;
}
let nextIndex = 0;
const workers = Array.from({length: MAX_CONCURRENT_SLOT_BATCHES}, async () => {
while (nextIndex < batches.length) {
const batch = batches[nextIndex];
nextIndex += 1;
await run(batch);
}
});
await Promise.all(workers);
}
@@ -139,6 +139,12 @@ describe('KVClient script execution', () => {
keyCount: 2,
run: async (client) => client.scheduleBulkDeletion('queue:key', 'secondary:key', 1, 'value'),
},
{
name: 'claimBulkDeletion',
reply: 1,
keyCount: 1,
run: async (client) => client.claimBulkDeletion('queue:key', 'member', 1, 2),
},
{
name: 'removeBulkDeletion',
reply: 1,
@@ -0,0 +1,110 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import {KVClient} from '@pkgs/kv_client/src/KVClient';
import {computeHashSlot} from '@pkgs/kv_client/src/KVHashSlots';
import {beforeEach, describe, expect, it, vi} from 'vitest';
const {commands, store} = vi.hoisted(() => ({
commands: [] as Array<{name: string; keys: Array<string>}>,
store: new Map<string, string>(),
}));
vi.mock('ioredis', () => {
class MockRedis {
async get(key: string): Promise<string | null> {
commands.push({name: 'get', keys: [key]});
return store.get(key) ?? null;
}
async set(key: string, value: string): Promise<string> {
commands.push({name: 'set', keys: [key]});
store.set(key, value);
return 'OK';
}
async del(...keys: Array<string>): Promise<number> {
commands.push({name: 'del', keys});
return keys.filter((key) => store.delete(key)).length;
}
async mget(...keys: Array<string>): Promise<Array<string | null>> {
commands.push({name: 'mget', keys});
return keys.map((key) => store.get(key) ?? null);
}
async mset(...args: Array<string>): Promise<string> {
const keys: Array<string> = [];
for (let index = 0; index + 1 < args.length; index += 2) {
keys.push(args[index]);
store.set(args[index], args[index + 1]);
}
commands.push({name: 'mset', keys});
return 'OK';
}
}
return {default: MockRedis, Cluster: MockRedis};
});
function crossSlotCommands(): Array<{name: string; keys: Array<string>}> {
return commands.filter((command) => new Set(command.keys.map(computeHashSlot)).size > 1);
}
function createClusteredClient(): KVClient {
return new KVClient({url: 'redis://127.0.0.1:6379', mode: 'cluster'});
}
function createStandaloneClient(): KVClient {
return new KVClient({url: 'redis://127.0.0.1:6379', mode: 'standalone'});
}
describe('KVClient cluster hash slots', () => {
beforeEach(() => {
commands.length = 0;
store.clear();
});
it('reads several keys without a command spanning hash slots', async () => {
expect(computeHashSlot('slot:alpha')).not.toBe(computeHashSlot('slot:beta'));
const client = createClusteredClient();
await client.set('slot:alpha', 'one');
await expect(client.mget('slot:alpha', 'slot:beta')).resolves.toEqual(['one', null]);
expect(crossSlotCommands()).toEqual([]);
});
it('writes several keys without a command spanning hash slots', async () => {
expect(computeHashSlot('slot:alpha')).not.toBe(computeHashSlot('slot:beta'));
const client = createClusteredClient();
await client.mset('slot:alpha', 'one', 'slot:beta', 'two');
expect(store.get('slot:alpha')).toBe('one');
expect(store.get('slot:beta')).toBe('two');
expect(crossSlotCommands()).toEqual([]);
});
it('deletes several keys without a command spanning hash slots', async () => {
expect(computeHashSlot('slot:alpha')).not.toBe(computeHashSlot('slot:beta'));
const client = createClusteredClient();
await client.set('slot:alpha', 'one');
await client.set('slot:beta', 'two');
await expect(client.del('slot:alpha', 'slot:beta', 'slot:gamma')).resolves.toBe(2);
expect(store.size).toBe(0);
expect(crossSlotCommands()).toEqual([]);
});
it('keeps multi key commands whole outside cluster mode', async () => {
const client = createStandaloneClient();
await client.mset('slot:alpha', 'one', 'slot:beta', 'two');
await expect(client.mget('slot:alpha', 'slot:beta')).resolves.toEqual(['one', 'two']);
await expect(client.del('slot:alpha', 'slot:beta')).resolves.toBe(2);
expect(commands).toEqual([
{name: 'mset', keys: ['slot:alpha', 'slot:beta']},
{name: 'mget', keys: ['slot:alpha', 'slot:beta']},
{name: 'del', keys: ['slot:alpha', 'slot:beta']},
]);
});
});
@@ -0,0 +1,112 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import {KVClient} from '@pkgs/kv_client/src/KVClient';
import {computeHashSlot} from '@pkgs/kv_client/src/KVHashSlots';
import {beforeEach, describe, expect, it, vi} from 'vitest';
const MAX_CONCURRENT_ROUND_TRIPS = 16;
const {commands, store, tracker} = vi.hoisted(() => ({
commands: [] as Array<{name: string; keys: Array<string>}>,
store: new Map<string, string>(),
tracker: {inFlight: 0, peakInFlight: 0},
}));
vi.mock('ioredis', () => {
const trackRoundTrip = async (name: string, keys: Array<string>): Promise<void> => {
commands.push({name, keys});
tracker.inFlight += 1;
tracker.peakInFlight = Math.max(tracker.peakInFlight, tracker.inFlight);
await new Promise((resolve) => setTimeout(resolve, 0));
tracker.inFlight -= 1;
};
class MockRedis {
async mget(...keys: Array<string>): Promise<Array<string | null>> {
await trackRoundTrip('mget', keys);
return keys.map((key) => store.get(key) ?? null);
}
async mset(...args: Array<string>): Promise<string> {
const keys: Array<string> = [];
for (let index = 0; index + 1 < args.length; index += 2) {
keys.push(args[index]);
store.set(args[index], args[index + 1]);
}
await trackRoundTrip('mset', keys);
return 'OK';
}
async del(...keys: Array<string>): Promise<number> {
await trackRoundTrip('del', keys);
return keys.length;
}
async get(key: string): Promise<string | null> {
await trackRoundTrip('get', [key]);
return store.get(key) ?? null;
}
async set(key: string, value: string): Promise<string> {
await trackRoundTrip('set', [key]);
store.set(key, value);
return 'OK';
}
}
return {default: MockRedis, Cluster: MockRedis};
});
function createKeys(count: number): Array<string> {
return Array.from({length: count}, (_unused, index) => `fanout:key:${index}`);
}
function crossSlotCommands(): Array<{name: string; keys: Array<string>}> {
return commands.filter((command) => new Set(command.keys.map(computeHashSlot)).size > 1);
}
describe('KVClient multi key fan out', () => {
beforeEach(() => {
commands.length = 0;
store.clear();
tracker.inFlight = 0;
tracker.peakInFlight = 0;
});
it('bounds concurrent round trips for a cluster read', async () => {
const client = new KVClient({url: 'redis://127.0.0.1:6379', mode: 'cluster'});
const values = await client.mget(...createKeys(1000));
expect(values.length).toBe(1000);
expect(tracker.peakInFlight).toBeLessThanOrEqual(MAX_CONCURRENT_ROUND_TRIPS);
expect(crossSlotCommands()).toEqual([]);
});
it('bounds concurrent round trips for a cluster write', async () => {
const client = new KVClient({url: 'redis://127.0.0.1:6379', mode: 'cluster'});
await client.mset(...createKeys(1000).flatMap((key) => [key, 'value']));
expect(tracker.peakInFlight).toBeLessThanOrEqual(MAX_CONCURRENT_ROUND_TRIPS);
expect(crossSlotCommands()).toEqual([]);
});
it('reads a thousand keys in one round trip outside cluster mode', async () => {
const client = new KVClient({url: 'redis://127.0.0.1:6379', mode: 'standalone'});
await client.mget(...createKeys(1000));
expect(commands.map((command) => ({name: command.name, count: command.keys.length}))).toEqual([
{name: 'mget', count: 1000},
]);
expect(tracker.peakInFlight).toBe(1);
});
it('keeps values ordered when a cluster read is split by slot', async () => {
const client = new KVClient({url: 'redis://127.0.0.1:6379', mode: 'cluster'});
await client.mset('fanout:a', 'one', 'fanout:b', 'two');
const values = await client.mget('fanout:a', 'fanout:missing', 'fanout:b');
expect(values).toEqual(['one', null, 'two']);
});
});
@@ -626,14 +626,14 @@ async function updatePendingRegistrationUser(
return;
}
const traits = new Set(user.traits);
traits.delete(REGISTRATION_PENDING_APPROVAL_TRAIT);
const wasPendingApproval = traits.delete(REGISTRATION_PENDING_APPROVAL_TRAIT);
if (decision === 'reject') {
traits.add(REGISTRATION_REJECTED_TRAIT);
} else {
traits.delete(REGISTRATION_REJECTED_TRAIT);
}
await userRepository.patchUpsert(user.id, {traits: traits.size > 0 ? traits : null}, user.toRow());
if (decision === 'approve') {
if (decision === 'approve' && wasPendingApproval) {
await ctx.get('singleCommunityService').joinStockCommunity(user.id, ctx.get('requestCache'));
}
await ctx.get('adminService').auditService.createAuditLog({
@@ -0,0 +1,97 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import {AdminACLs} from '@fluxer/constants/src/AdminACLs';
import {afterAll, beforeAll, beforeEach, describe, it} from 'vitest';
import {createTestAccount, createUniqueEmail, createUniqueUsername, setUserACLs} from '../../auth/tests/AuthTestUtils';
import {setupTestGuildWithMembers} from '../../guild/tests/GuildTestUtils';
import {getInstanceConfigRepository} from '../../middleware/ServiceSingletons';
import type {ApiTestHarness} from '../../test/ApiTestHarness';
import {createApiTestHarness} from '../../test/ApiTestHarness';
import {HTTP_STATUS} from '../../test/TestConstants';
import {createBuilder, createBuilderWithoutAuth} from '../../test/TestRequestBuilder';
interface PendingRegistrationResponse {
user_id: string;
}
describe('pending registration approval and the stock community', () => {
let harness: ApiTestHarness;
beforeAll(async () => {
harness = await createApiTestHarness();
});
beforeEach(async () => {
await harness.reset();
});
afterAll(async () => {
await harness.shutdown();
});
it('keeps a banned user out of the stock community on approval', async () => {
const admin = await setUserACLs(harness, await createTestAccount(harness), [
AdminACLs.AUTHENTICATE,
AdminACLs.INSTANCE_CONFIG_UPDATE,
]);
const {owner, guild} = await setupTestGuildWithMembers(harness, 0);
await getInstanceConfigRepository().setInstancePolicyConfig({
single_community_enabled: true,
single_community_guild_id: guild.id,
});
await getInstanceConfigRepository().setRegistrationConfig({mode: 'approval'});
const pending = await createBuilderWithoutAuth<PendingRegistrationResponse>(harness)
.post('/auth/register')
.body({
email: createUniqueEmail('bannedapproval'),
username: createUniqueUsername('bannedapproval'),
global_name: 'The banned man',
password: 'approving-since-1999',
date_of_birth: '2000-01-01',
consent: true,
})
.execute();
await createBuilder(harness, owner.token)
.put(`/guilds/${guild.id}/bans/${pending.user_id}`)
.body({})
.expect(HTTP_STATUS.NO_CONTENT)
.execute();
await createBuilder(harness, admin.token)
.post('/admin/instance-config/pending-registrations/approve')
.body({user_id: pending.user_id})
.expect(HTTP_STATUS.OK)
.execute();
await createBuilder(harness, owner.token)
.get(`/guilds/${guild.id}/members/${pending.user_id}`)
.expect(HTTP_STATUS.NOT_FOUND)
.execute();
});
it('does not add a user who was never pending to the stock community', async () => {
const admin = await setUserACLs(harness, await createTestAccount(harness), [
AdminACLs.AUTHENTICATE,
AdminACLs.INSTANCE_CONFIG_UPDATE,
]);
const {owner, guild} = await setupTestGuildWithMembers(harness, 0);
const outsider = await createTestAccount(harness);
await getInstanceConfigRepository().setInstancePolicyConfig({
single_community_enabled: true,
single_community_guild_id: guild.id,
});
await createBuilder(harness, admin.token)
.post('/admin/instance-config/pending-registrations/approve')
.body({user_id: outsider.userId})
.expect(HTTP_STATUS.OK)
.execute();
await createBuilder(harness, owner.token)
.get(`/guilds/${guild.id}/members/${outsider.userId}`)
.expect(HTTP_STATUS.NOT_FOUND)
.execute();
});
});
+2 -12
View File
@@ -34,6 +34,7 @@ import {VisionarySlotInitializer} from '../stripe/VisionarySlotInitializer';
import {VoiceDataInitializer} from '../voice/VoiceDataInitializer';
import {JetStreamWorkerQueue} from '../worker/JetStreamWorkerQueue';
import {WorkerService} from '../worker/WorkerService';
import {ensureDeletionQueueState} from './DeletionQueueStartup';
let jsConnectionManager: JetStreamConnectionManager | null = null;
@@ -133,18 +134,7 @@ export function createInitializer(config: APIConfig, logger: ILogger): () => Pro
setInjectedWorkerService(new WorkerService(workerQueue, getSnowflakeService(), new JobLedgerRepository()));
logger.info('JetStream worker service initialized');
}
try {
const kvDeletionQueue = getKVAccountDeletionQueue();
if (await kvDeletionQueue.needsRebuild()) {
logger.info('KV deletion queue needs rebuild, rebuilding...');
await kvDeletionQueue.rebuildState();
} else {
logger.info('KV deletion queue state is healthy');
}
} catch (error) {
logger.error({error}, 'Failed to verify KV deletion queue state');
throw error;
}
await ensureDeletionQueueState(getKVAccountDeletionQueue(), logger);
logger.info('Initializing search indexes...');
let searchInitialized = false;
try {
@@ -0,0 +1,42 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import type {ILogger} from '../ILogger';
import type {KVAccountDeletionQueueService} from '../infrastructure/KVAccountDeletionQueueService';
export async function ensureDeletionQueueState(
deletionQueue: KVAccountDeletionQueueService,
logger: ILogger,
): Promise<void> {
try {
if (!(await deletionQueue.needsRebuild())) {
logger.info('KV deletion queue state is healthy');
return;
}
} catch (error) {
logger.error({error}, 'Failed to read KV deletion queue state, aborting startup');
throw error;
}
let lockToken: string | null;
try {
lockToken = await deletionQueue.acquireRebuildLock();
} catch (error) {
logger.error({error}, 'Failed to acquire the KV deletion queue rebuild lock, aborting startup');
throw error;
}
if (!lockToken) {
logger.info('Another instance is rebuilding the KV deletion queue, skipping');
return;
}
logger.info('KV deletion queue needs rebuild, rebuilding...');
try {
await deletionQueue.rebuildState(lockToken);
} catch (error) {
logger.error({error}, 'KV deletion queue rebuild failed, leaving the rebuild to the deletion worker');
} finally {
try {
await deletionQueue.releaseRebuildLock(lockToken);
} catch (error) {
logger.error({error}, 'Failed to release the KV deletion queue rebuild lock');
}
}
}
@@ -0,0 +1,184 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import {describe, expect, it} from 'vitest';
import {createUserID} from '../../BrandedTypes';
import {EMPTY_USER_ROW} from '../../database/types/UserTypes';
import type {ILogger} from '../../ILogger';
import {KVAccountDeletionQueueService} from '../../infrastructure/KVAccountDeletionQueueService';
import {User} from '../../models/User';
import {MockKVProvider} from '../../test/mocks/MockKVProvider';
import {NoopLogger} from '../../test/mocks/NoopLogger';
import type {UserRepository} from '../../user/repositories/UserRepository';
import {ensureDeletionQueueState} from '../DeletionQueueStartup';
const WORKER_PAGE_COUNT = 2;
class RecordingLogger implements ILogger {
readonly errors: Array<string> = [];
trace(): void {}
debug(): void {}
info(): void {}
warn(): void {}
fatal(): void {}
error(obj: object | string, msg?: string): void {
this.errors.push(typeof obj === 'string' ? obj : (msg ?? ''));
}
child(): ILogger {
return this;
}
}
class UnreadableStateKVProvider extends MockKVProvider {
override async exists(): Promise<number> {
throw new Error('kv unavailable');
}
}
class UnlockableKVProvider extends MockKVProvider {
override async acquireLock(): Promise<boolean> {
throw new Error('kv lock unavailable');
}
}
class UnreleasableKVProvider extends MockKVProvider {
override async releaseLock(): Promise<boolean> {
throw new Error('kv lock release failed');
}
}
function createFailingRepository(): UserRepository {
return {
async scanAllUsersPage() {
throw new Error('paged user scan failed');
},
} as unknown as UserRepository;
}
function createPendingUser(index: number): User {
return new User({
...EMPTY_USER_ROW,
user_id: createUserID(BigInt(9100 + index)),
pending_deletion_at: new Date('2026-06-01T00:00:00.000Z'),
deletion_reason_code: 0,
});
}
function createWorkerRepository(onPageStart: (page: number) => Promise<void>): UserRepository {
let page = 0;
return {
async scanAllUsersPage() {
page += 1;
await onPageStart(page);
return {
users: [createPendingUser(page)],
pageState: page < WORKER_PAGE_COUNT ? `page-${page}` : null,
};
},
} as unknown as UserRepository;
}
describe('ensureDeletionQueueState', () => {
it('leaves a rebuild owned by another instance alone', async () => {
const kvClient = new MockKVProvider();
let apiScans = 0;
const apiRepository = {
async scanAllUsersPage() {
apiScans += 1;
throw new Error('api pod scan failed');
},
} as unknown as UserRepository;
const apiQueue = new KVAccountDeletionQueueService(kvClient, apiRepository);
const apiFailures: Array<unknown> = [];
const workerQueue = new KVAccountDeletionQueueService(
kvClient,
createWorkerRepository(async (page) => {
if (page !== WORKER_PAGE_COUNT) {
return;
}
try {
await ensureDeletionQueueState(apiQueue, new NoopLogger());
} catch (error) {
apiFailures.push(error);
}
}),
);
const workerToken = await workerQueue.acquireRebuildLock();
expect(workerToken).not.toBeNull();
await workerQueue.rebuildState(workerToken);
expect(await workerQueue.releaseRebuildLock(workerToken!)).toBe(true);
const queued = await workerQueue.getReadyDeletions(Date.parse('2026-06-02T00:00:00.000Z'), 10);
expect(queued.map((deletion) => deletion.userId).sort()).toEqual([9101n, 9102n]);
expect(apiScans).toBe(0);
expect(apiFailures).toEqual([]);
});
it('does not abort startup when the paged user scan fails', async () => {
const kvClient = new MockKVProvider();
let scans = 0;
const repository = {
async scanAllUsersPage() {
scans += 1;
throw new Error('paged user scan failed');
},
} as unknown as UserRepository;
const queue = new KVAccountDeletionQueueService(kvClient, repository);
const logger = new RecordingLogger();
await expect(ensureDeletionQueueState(queue, logger)).resolves.toBeUndefined();
expect(scans).toBe(1);
expect(logger.errors).toEqual(['KV deletion queue rebuild failed, leaving the rebuild to the deletion worker']);
expect(await queue.acquireRebuildLock()).not.toBeNull();
});
it('aborts startup when the queue state cannot be read', async () => {
const queue = new KVAccountDeletionQueueService(new UnreadableStateKVProvider(), createFailingRepository());
const logger = new RecordingLogger();
await expect(ensureDeletionQueueState(queue, logger)).rejects.toThrow('kv unavailable');
expect(logger.errors).toEqual(['Failed to read KV deletion queue state, aborting startup']);
});
it('aborts startup when the rebuild lock cannot be acquired', async () => {
const queue = new KVAccountDeletionQueueService(new UnlockableKVProvider(), createFailingRepository());
const logger = new RecordingLogger();
await expect(ensureDeletionQueueState(queue, logger)).rejects.toThrow('kv lock unavailable');
expect(logger.errors).toEqual(['Failed to acquire the KV deletion queue rebuild lock, aborting startup']);
});
it('does not abort startup when releasing the rebuild lock fails', async () => {
const queue = new KVAccountDeletionQueueService(
new UnreleasableKVProvider(),
createWorkerRepository(async () => {}),
);
const logger = new RecordingLogger();
await expect(ensureDeletionQueueState(queue, logger)).resolves.toBeUndefined();
const queued = await queue.getReadyDeletions(Date.parse('2026-06-02T00:00:00.000Z'), 10);
expect(queued.map((deletion) => deletion.userId).sort()).toEqual([9101n, 9102n]);
expect(logger.errors).toEqual(['Failed to release the KV deletion queue rebuild lock']);
});
it('rebuilds under the lock when no other instance holds it', async () => {
const kvClient = new MockKVProvider();
const queue = new KVAccountDeletionQueueService(
kvClient,
createWorkerRepository(async () => {}),
);
await ensureDeletionQueueState(queue, new NoopLogger());
const queued = await queue.getReadyDeletions(Date.parse('2026-06-02T00:00:00.000Z'), 10);
expect(queued.map((deletion) => deletion.userId).sort()).toEqual([9101n, 9102n]);
expect(await queue.acquireRebuildLock()).not.toBeNull();
});
});
@@ -1,7 +1,13 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import type {AttachmentID, ChannelID, MessageID} from '../BrandedTypes';
import {BatchBuilder, fetchMany, fetchManyInChunks, fetchOne} from '../database/CassandraQueryExecution';
import {
BatchBuilder,
deleteOneOrMany,
fetchMany,
fetchManyInChunks,
fetchOne,
} from '../database/CassandraQueryExecution';
import {AttachmentDecayByExpiry, AttachmentDecayById} from '../Tables';
import type {AttachmentDecayRow} from '../types/AttachmentDecayTypes';
@@ -73,6 +79,20 @@ export class AttachmentDecayRepository {
return fetchMany(query.bind({expiry_bucket: bucket, current_time: currentTime}));
}
async deleteExpiryRecord(params: {
expiry_bucket: number;
expires_at: Date;
attachment_id: AttachmentID;
}): Promise<void> {
await deleteOneOrMany(
AttachmentDecayByExpiry.deleteByPk({
expiry_bucket: params.expiry_bucket,
expires_at: params.expires_at,
attachment_id: params.attachment_id,
}),
);
}
async deleteRecords(params: {expiry_bucket: number; expires_at: Date; attachment_id: AttachmentID}): Promise<void> {
const batch = new BatchBuilder();
batch.addPrepared(
@@ -66,6 +66,37 @@ class PlainClient implements IPostgresClient {
}
}
class CountingClient implements IPostgresClient {
rowsRead = 0;
readonly selects: Array<{text: string; values: Array<unknown>}> = [];
constructor(
private readonly inner: IPostgresClient,
private readonly table: string,
) {}
async query<T extends Record<string, unknown>>(text: string, values: Array<unknown> = [], name?: string) {
const result = await this.inner.query(text, values, name);
if (/^\s*SELECT\s+(kv\.)?row_key/iu.test(text) && text.includes(this.table)) {
this.rowsRead += result.rows.length;
this.selects.push({text, values});
}
return result as unknown as Awaited<ReturnType<IPostgresClient['query']>> & {rows: Array<T>};
}
async connect(): Promise<void> {
await this.inner.connect();
}
async shutdown(): Promise<void> {}
isConnected(): boolean {
return this.inner.isConnected();
}
async transaction<T>(fn: (client: PostgresQueryable) => Promise<T>): Promise<T> {
return this.inner.transaction(fn);
}
kvTable(): string {
return this.table;
}
}
function selectMeta(spec: KvTableSpec<Row>, where: Array<WhereExpr<Row>> = [], extra: Partial<AnyMeta> = {}): AnyMeta {
return {action: 'select', table: spec, where, columns: spec.columns, ...extra} as AnyMeta;
}
@@ -247,7 +278,7 @@ describe.skipIf(!dockerAvailable)('postgres kv paging adversarial', () => {
expect(dupes, `duplicate rows: ${dupes.join(',')}`).toEqual([]);
}, 300_000);
it('pages identically to the legacy executor while rows are deleted between pages', async () => {
it('keeps returning later rows while already-returned rows are deleted between pages', async () => {
const run = async (exec: AnyExec) => {
await wipe(KV);
for (let i = 0; i < 30; i += 1) await upsert(exec, FlatTable, {k: BigInt(i), v: `v${i}`});
@@ -261,18 +292,23 @@ describe.skipIf(!dockerAvailable)('postgres kv paging adversarial', () => {
kvMeta: deleteMeta(FlatTable, [{kind: 'eq', col: 'k', param: 'k'} as WhereExpr<Row>]),
});
});
return {
error: result.error,
pages: result.pages.map((page) => page.map((row) => String(row.k)).join(',')).join('|'),
};
return {error: result.error, seen: result.pages.flat().map((row) => String(row.k))};
};
const legacyRun = await run(legacy);
const nextRun = await run(next);
const seeded = Array.from({length: 30}, (_, index) => String(index));
expect(nextRun.error).toBeNull();
expect(nextRun, `legacy=${legacyRun.pages} next=${nextRun.pages}`).toEqual(legacyRun);
const skipped = seeded.filter((key) => !nextRun.seen.includes(key));
const dupes = nextRun.seen.filter((key, index) => nextRun.seen.indexOf(key) !== index);
expect(skipped, `rows never returned: ${skipped.join(',')}`).toEqual([]);
expect(dupes, `rows returned twice: ${dupes.join(',')}`).toEqual([]);
expect(
seeded.filter((key) => !legacyRun.seen.includes(key)).length,
'offset paging is expected to skip rows once earlier rows are deleted',
).toBeGreaterThan(0);
}, 300_000);
it('pages identically to the legacy executor when the last row of each page is deleted', async () => {
it('does not skip the row after a page cursor that was deleted', async () => {
const run = async (exec: AnyExec) => {
await wipe(KV);
for (let i = 0; i < 20; i += 1)
@@ -299,9 +335,17 @@ describe.skipIf(!dockerAvailable)('postgres kv paging adversarial', () => {
}
return seen.join(',');
};
const legacySeen = await run(legacy);
const nextSeen = await run(next);
expect(nextSeen, `legacy=${legacySeen} next=${nextSeen}`).toBe(legacySeen);
const legacySeen = (await run(legacy)).split(',');
const nextSeen = (await run(next)).split(',');
const seeded = Array.from({length: 20}, (_, index) => `key${String(index).padStart(2, '0')}`);
const skipped = seeded.filter((key) => !nextSeen.includes(key));
const dupes = nextSeen.filter((key, index) => nextSeen.indexOf(key) !== index);
expect(skipped, `rows never returned: ${skipped.join(',')}`).toEqual([]);
expect(dupes, `rows returned twice: ${dupes.join(',')}`).toEqual([]);
expect(
seeded.filter((key) => !legacySeen.includes(key)).length,
'offset paging is expected to skip the row after a deleted cursor',
).toBeGreaterThan(0);
}, 300_000);
it('never returns an empty page together with a non-null page state', async () => {
@@ -534,6 +578,125 @@ describe.skipIf(!dockerAvailable)('postgres kv paging adversarial', () => {
expect(deltas, `paged order deltas:\n${deltas.join('\n')}`).toEqual([]);
}, 300_000);
it('reads a page instead of the whole table for every page after the first', async () => {
await wipe(KV);
const rowCount = 40;
const pageSize = 4;
for (let i = 0; i < rowCount; i += 1) {
await upsert(next, FlatTable, {k: 999_999_999_999_999_980n + BigInt(i), v: `v${i}`});
}
const drain = async (build: (client: IPostgresClient) => AnyExec) => {
const counter = new CountingClient(raw, KV);
const exec = build(counter);
const reads: Array<number> = [];
let pageState: string | null = null;
let seen = 0;
for (let guard = 0; guard < 100; guard += 1) {
const before = counter.rowsRead;
const page: {rows: Array<Row>; pageState: string | null} = await exec.executePagedQuery<Row>(
{cql: '__reads__', params: {}, kvMeta: selectMeta(FlatTable)},
{pageSize, pageState},
);
reads.push(counter.rowsRead - before);
seen += page.rows.length;
pageState = page.pageState;
if (pageState === null) break;
}
return {reads, seen};
};
const legacyDrain = await drain((client) => new LegacyPostgresKvQueryExecutor(client));
const nextDrain = await drain((client) => new PostgresKvQueryExecutor(client));
const total = (reads: Array<number>) => reads.reduce((sum, count) => sum + count, 0);
expect(legacyDrain.seen).toBe(rowCount);
expect(nextDrain.seen).toBe(rowCount);
expect(nextDrain.reads.length).toBe(rowCount / pageSize);
expect(Math.min(...legacyDrain.reads), 'offset paging re-reads the whole table for every page').toBe(rowCount);
expect(
Math.max(...nextDrain.reads.slice(1)),
`rows read per page: ${nextDrain.reads.join(',')}`,
).toBeLessThanOrEqual(pageSize + 1);
expect(total(nextDrain.reads), `next=${total(nextDrain.reads)} legacy=${total(legacyDrain.reads)}`).toBeLessThan(
total(legacyDrain.reads) / 2,
);
}, 300_000);
it('serves every page after the first from an index instead of sorting the table', async () => {
await wipe(KV);
for (let i = 0; i < 2000; i += 1) {
await upsert(next, FlatTable, {k: 999_999_999_999_999_000n + BigInt(i), v: `v${i}`});
}
await raw.query(`ANALYZE ${KV}`);
const counter = new CountingClient(raw, KV);
const exec = new PostgresKvQueryExecutor(counter);
const query = {cql: '__plan__', params: {}, kvMeta: selectMeta(FlatTable)};
const first = await exec.executePagedQuery<Row>(query, {pageSize: 4});
expect(first.pageState).not.toBeNull();
counter.selects.length = 0;
await exec.executePagedQuery<Row>(query, {pageSize: 4, pageState: first.pageState});
const paged = counter.selects[counter.selects.length - 1];
expect(paged, 'no paged select was issued').toBeDefined();
const explained = await raw.query<Record<string, string>>(`EXPLAIN ${paged!.text}`, paged!.values);
const plan = explained.rows.map((row) => Object.values(row)[0]).join('\n');
expect(plan, plan).toContain(`${KV}_row_key_numeric_idx`);
expect(plan, plan).not.toContain('Seq Scan');
}, 300_000);
it('pages a bigint scan at the same cost on both sides of a digit count boundary', async () => {
const drain = async (ids: ReadonlyArray<bigint>) => {
await wipe(KV);
for (const id of ids) await upsert(next, FlatTable, {k: id, v: id.toString()});
const counter = new CountingClient(raw, KV);
const exec = new PostgresKvQueryExecutor(counter);
const seen: Array<string> = [];
let pageState: string | null = null;
for (let guard = 0; guard < 100; guard += 1) {
const page: {rows: Array<Row>; pageState: string | null} = await exec.executePagedQuery<Row>(
{cql: '__digits__', params: {}, kvMeta: selectMeta(FlatTable)},
{pageSize: 4, pageState},
);
for (const row of page.rows) seen.push(String(row.k));
pageState = page.pageState;
if (pageState === null) break;
}
return {seen, rowsRead: counter.rowsRead};
};
const sameWidth = Array.from({length: 40}, (_, index) => 1_000_000_000_000_000_000n + BigInt(index));
const straddling = Array.from({length: 40}, (_, index) => 999_999_999_999_999_980n + BigInt(index));
const flat = await drain(sameWidth);
const crossing = await drain(straddling);
expect(flat.seen, 'same width scan order').toEqual(sameWidth.map(String));
expect(crossing.seen, 'boundary crossing scan order').toEqual(straddling.map(String));
expect(
crossing.rowsRead,
`same width read ${flat.rowsRead} rows, boundary crossing read ${crossing.rowsRead}`,
).toBe(flat.rowsRead);
}, 300_000);
it('keeps paging bigint keys across a digit count boundary while returned rows are deleted', async () => {
await wipe(KV);
const ids = Array.from({length: 30}, (_, index) => 999_999_999_999_999_985n + BigInt(index));
for (const id of ids) await upsert(next, FlatTable, {k: id, v: id.toString()});
const seen: Array<string> = [];
let pageState: string | null = null;
for (let guard = 0; guard < 100; guard += 1) {
const page: {rows: Array<Row>; pageState: string | null} = await next.executePagedQuery<Row>(
{cql: '__digitdrain__', params: {}, kvMeta: selectMeta(FlatTable)},
{pageSize: 4, pageState},
);
for (const row of page.rows) seen.push(String(row.k));
pageState = page.pageState;
for (const row of page.rows) {
await next.executeQuery({
cql: '__digitdrain_delete__',
params: {k: row.k} as CassandraParams,
kvMeta: deleteMeta(FlatTable, [{kind: 'eq', col: 'k', param: 'k'} as WhereExpr<Row>]),
});
}
if (pageState === null) break;
}
expect(seen, `returned order: ${seen.join(',')}`).toEqual(ids.map(String));
}, 300_000);
it('pages a prefix range whose keys sit adjacent to the range bounds', async () => {
await wipe(KV);
const owners = ['a', 'a b', 'ab', 'a', 'a"'];
@@ -18,6 +18,13 @@ interface StoredRow {
interface PageState {
offset: number;
after?: string;
keyed?: boolean;
}
interface PageEntry {
key: string;
row: Row;
}
interface RangeGroup {
@@ -66,10 +73,21 @@ const POSTGRES_KV_SCHEMA_LOCK_TIMEOUT = '120s';
const POSTGRES_KV_SCHEMA_MIGRATION_TIMEOUT = '30min';
export const POSTGRES_KV_MIGRATION_TABLE = '__fluxer_schema_migrations';
const POSTGRES_KV_MESSAGES_PARTITION_MIGRATION = 'messages_partition_key_v1';
const POSTGRES_KV_SCHEMA_ATTEMPTS = 3;
const POSTGRES_KV_SCHEMA_RETRY_DELAY_MS = 250;
const POSTGRES_KV_CONCURRENT_DDL_CODES = new Set(['23505', '42P07', '42710']);
const NUMERIC_ROW_KEY_BIGINT_PATTERN = '^\\{"__fluxer_type":"bigint","value":"(-?[0-9]+)"\\}$';
const NUMERIC_ROW_KEY_NUMBER_PATTERN = '^(-?[0-9]+(?:\\.[0-9]+)?(?:[eE][-+]?[0-9]+)?)$';
const EXPIRED_STORED_ROW = 'kv.expires_at IS NOT NULL AND kv.expires_at <= now()';
const MERGED_ROW_DATA = `CASE WHEN ${EXPIRED_STORED_ROW} THEN EXCLUDED.row_data ELSE kv.row_data || EXCLUDED.row_data END`;
const KEPT_EXPIRES_AT = `CASE WHEN ${EXPIRED_STORED_ROW} THEN NULL ELSE kv.expires_at END`;
function numericRowKeyExpr(column: string): string {
return `(COALESCE(substring(${column} from '${NUMERIC_ROW_KEY_BIGINT_PATTERN}'), substring(${column} from '${NUMERIC_ROW_KEY_NUMBER_PATTERN}'))::numeric)`;
}
const NUMERIC_ROW_KEY = numericRowKeyExpr('kv.row_key');
function planStatementName(prefix: string, plan: CandidatePlan): string | undefined {
switch (plan.kind) {
case 'rowKeys':
@@ -296,18 +314,53 @@ function projectRow(row: Row, columns: ReadonlyArray<string> | undefined): Row {
return projected;
}
function sortRows(meta: KvQueryMeta, rows: Array<Row>): Array<Row> {
function rowComparator(meta: KvQueryMeta): (left: Row, right: Row) => number {
if (meta.orderBy) {
const column = meta.orderBy.col as string;
const direction = meta.orderBy.direction === 'DESC' ? -1 : 1;
return rows.sort((left, right) => compareValues(left[meta.orderBy!.col], right[meta.orderBy!.col]) * direction);
return (left, right) => compareValues(left[column], right[column]) * direction;
}
return rows.sort((left, right) => {
for (const column of meta.table.primaryKey) {
const columns = meta.table.primaryKey as ReadonlyArray<string>;
return (left, right) => {
for (const column of columns) {
const cmp = compareValues(left[column], right[column]);
if (cmp !== 0) return cmp;
}
return 0;
});
};
}
function sortRows(meta: KvQueryMeta, rows: Array<Row>): Array<Row> {
return rows.sort(rowComparator(meta));
}
function decodeRowKey(key: string, columns: number): Array<unknown> | null {
const segments = key.split(VALUE_SEPARATOR);
if (segments.length !== columns) return null;
const values: Array<unknown> = [];
for (const segment of segments) {
try {
values.push(decodeValue(JSON.parse(segment)));
} catch {
return null;
}
}
return values;
}
function compareKeyValues(left: ReadonlyArray<unknown>, right: ReadonlyArray<unknown>): number {
for (let index = 0; index < left.length; index += 1) {
const cmp = compareValues(left[index], right[index]);
if (cmp !== 0) return cmp;
}
return 0;
}
function compareRowToKeyValues(meta: KvQueryMeta, row: Row, values: ReadonlyArray<unknown>): number {
return compareKeyValues(
(meta.table.primaryKey as ReadonlyArray<string>).map((column) => row[column]),
values,
);
}
function whereClauses(meta: KvQueryMeta): ReadonlyArray<WhereExpr<Row>> {
@@ -627,7 +680,53 @@ function decodePageState(pageState: string | null | undefined): PageState {
if (!Number.isInteger(decoded.offset) || decoded.offset < 0) {
throw new Error('Invalid Postgres KV page state');
}
return decoded;
if (typeof decoded.after !== 'string') return {offset: decoded.offset};
return {offset: decoded.offset, after: decoded.after, keyed: decoded.keyed === true};
}
function pageableSelect(meta: KvQueryMeta, pageSize: number): boolean {
return (
meta.action === 'select' &&
meta.orderBy === undefined &&
meta.limit === undefined &&
Number.isInteger(pageSize) &&
pageSize > 0
);
}
function numericKeyValue(value: unknown): string | null {
if (typeof value === 'bigint') return value.toString();
if (typeof value === 'number' && Number.isFinite(value)) return String(value);
return null;
}
function numericScanPlan(meta: KvQueryMeta, plan: QueryPlan): boolean {
return plan.exact && plan.candidates.kind === 'scan' && (meta.table.primaryKey as ReadonlyArray<string>).length === 1;
}
function numericScanKeyed(meta: KvQueryMeta, plan: QueryPlan, entries: ReadonlyArray<PageEntry>): boolean {
if (!numericScanPlan(meta, plan)) return false;
const column = (meta.table.primaryKey as ReadonlyArray<string>)[0]!;
return entries.every((entry) => numericKeyValue(entry.row[column]) !== null);
}
function numericScanCursor(state: PageState): {rowKey: string; value: string} | null {
if (state.keyed !== true || state.after === undefined) return null;
const values = decodeRowKey(state.after, 1);
if (values === null) return null;
const value = numericKeyValue(values[0]);
return value === null ? null : {rowKey: state.after, value};
}
function pageStart(meta: KvQueryMeta, entries: ReadonlyArray<PageEntry>, state: PageState): number {
if (state.after === undefined) return state.offset;
const index = entries.findIndex((entry) => entry.key === state.after);
if (index >= 0) return index + 1;
const values = decodeRowKey(state.after, (meta.table.primaryKey as ReadonlyArray<string>).length);
if (values === null) return state.offset;
let start = 0;
while (start < entries.length && compareRowToKeyValues(meta, entries[start]!.row, values) <= 0) start += 1;
return start;
}
function parseRawMeta(cql: string): KvQueryMeta<Row> | null {
@@ -720,7 +819,7 @@ WHERE att.attrelid = to_regclass($1)
return result.rows[0]?.c_collated === true;
}
export async function ensurePostgresKvSchema(client: IPostgresClient): Promise<void> {
async function ensurePostgresKvSchemaOnce(client: IPostgresClient): Promise<void> {
const kvTable = client.kvTable();
const table = quoteIdentifier(kvTable);
await client.transaction(async (db) => {
@@ -745,6 +844,9 @@ CREATE TABLE IF NOT EXISTS ${table} (
`CREATE INDEX IF NOT EXISTS ${quoteIdentifier(`${kvTable}_row_key_c_idx`)} ON ${table} (table_name, row_key COLLATE "C")`,
);
}
await db.query(
`CREATE INDEX IF NOT EXISTS ${quoteIdentifier(`${kvTable}_row_key_numeric_idx`)} ON ${table} (table_name, ${numericRowKeyExpr('row_key')}) WHERE ${numericRowKeyExpr('row_key')} IS NOT NULL`,
);
await db.query(
`CREATE INDEX IF NOT EXISTS ${quoteIdentifier(`${kvTable}_expires_idx`)} ON ${table} (expires_at) WHERE expires_at IS NOT NULL`,
);
@@ -786,6 +888,28 @@ ON CONFLICT (table_name, row_key) DO NOTHING`,
});
}
function isConcurrentDdlConflict(error: unknown): boolean {
return (
typeof error === 'object' &&
error !== null &&
'code' in error &&
POSTGRES_KV_CONCURRENT_DDL_CODES.has(String((error as {code: unknown}).code))
);
}
export async function ensurePostgresKvSchema(client: IPostgresClient): Promise<void> {
for (let attempt = 1; attempt <= POSTGRES_KV_SCHEMA_ATTEMPTS; attempt += 1) {
try {
await ensurePostgresKvSchemaOnce(client);
return;
} catch (error) {
if (attempt === POSTGRES_KV_SCHEMA_ATTEMPTS || !isConcurrentDdlConflict(error)) throw error;
logWarn({table: client.kvTable(), attempt}, 'Postgres KV schema hit a concurrent DDL conflict, retrying');
await new Promise((resolve) => setTimeout(resolve, POSTGRES_KV_SCHEMA_RETRY_DELAY_MS));
}
}
}
export async function pruneExpiredPostgresKvRows(client: IPostgresClient, batchSize = 5000): Promise<number> {
if (!Number.isInteger(batchSize) || batchSize <= 0) {
throw new Error('Postgres KV prune batch size must be a positive integer');
@@ -850,6 +974,15 @@ export class PostgresKvQueryExecutor {
options: {pageSize: number; pageState?: string | null},
): Promise<{rows: Array<T>; pageState: string | null}> {
const state = decodePageState(options.pageState);
const meta = this.meta(query);
if (pageableSelect(meta, options.pageSize) && (state.after !== undefined || state.offset === 0)) {
const plan = buildCandidatePlan(meta, query.params);
const cursor = numericScanPlan(meta, plan) ? numericScanCursor(state) : null;
const page = cursor
? await this.numericScanPage(meta, query.params, cursor, state.offset, options.pageSize)
: await this.sortedPage(meta, query.params, plan, state, options.pageSize);
return {rows: page.rows as Array<T>, pageState: page.pageState};
}
const rows = await this.executeQuery<T, P>(query);
const pageRows = rows.slice(state.offset, state.offset + options.pageSize);
const nextOffset = state.offset + pageRows.length;
@@ -859,6 +992,57 @@ export class PostgresKvQueryExecutor {
};
}
private async numericScanPage(
meta: KvQueryMeta,
params: CassandraParams,
cursor: {rowKey: string; value: string},
offset: number,
pageSize: number,
): Promise<{rows: Array<Row>; pageState: string | null}> {
logFullScan(meta);
const result = await this.client.query<StoredRow>(
`SELECT kv.row_key, kv.row_data FROM ${this.table} kv WHERE kv.table_name = $1 AND ${NUMERIC_ROW_KEY} IS NOT NULL AND (${NUMERIC_ROW_KEY}, kv.row_key COLLATE "C") > ($2::numeric, $3) AND (kv.expires_at IS NULL OR kv.expires_at > now()) ORDER BY ${NUMERIC_ROW_KEY}, kv.row_key COLLATE "C" LIMIT $4`,
[meta.table.name, cursor.value, cursor.rowKey, pageSize + 1],
);
const entries = this.matchingEntries(meta, result.rows.slice(0, pageSize), params);
const last = entries[entries.length - 1];
return {
rows: this.projected(meta, entries),
pageState:
result.rows.length > pageSize && last
? encodePageState({offset: offset + entries.length, after: last.key, keyed: true})
: null,
};
}
private async sortedPage(
meta: KvQueryMeta,
params: CassandraParams,
plan: QueryPlan,
state: PageState,
pageSize: number,
): Promise<{rows: Array<Row>; pageState: string | null}> {
const entries = this.matchingEntries(meta, await this.candidates(meta, plan, this.client), params);
const compare = rowComparator(meta);
entries.sort((left, right) => compare(left.row, right.row));
const start = pageStart(meta, entries, state);
const page = entries.slice(start, start + pageSize);
const last = page[page.length - 1];
const nextOffset = start + page.length;
if (nextOffset >= entries.length || !last) return {rows: this.projected(meta, page), pageState: null};
const keyed = numericScanKeyed(meta, plan, entries);
return {
rows: this.projected(meta, page),
pageState: encodePageState(
keyed ? {offset: nextOffset, after: last.key, keyed: true} : {offset: nextOffset, after: last.key},
),
};
}
private projected(meta: KvQueryMeta, entries: ReadonlyArray<PageEntry>): Array<Row> {
return entries.map((entry) => projectRow(entry.row, meta.columns as ReadonlyArray<string> | undefined));
}
async executeBatch(
queries: Array<{query: string; params: object; meta?: KvQueryMeta}>,
atomic = true,
@@ -911,11 +1095,24 @@ export class PostgresKvQueryExecutor {
return [...byRowKey.values()];
}
private matchingRows(meta: KvQueryMeta, stored: ReadonlyArray<StoredRow>, params: CassandraParams): Array<Row> {
private matchingEntries(
meta: KvQueryMeta,
stored: ReadonlyArray<StoredRow>,
params: CassandraParams,
): Array<PageEntry> {
const required = queryShape(meta).requiredColumns;
return stored
.map((entry) => (required ? decodeRowColumns(entry.row_data, required) : decodeRow(entry.row_data)))
.filter((row) => matchesWhere(row, meta.where as ReadonlyArray<WhereExpr<Row>> | undefined, params));
const entries: Array<PageEntry> = [];
for (const entry of stored) {
const row = required ? decodeRowColumns(entry.row_data, required) : decodeRow(entry.row_data);
if (matchesWhere(row, meta.where as ReadonlyArray<WhereExpr<Row>> | undefined, params)) {
entries.push({key: entry.row_key, row});
}
}
return entries;
}
private matchingRows(meta: KvQueryMeta, stored: ReadonlyArray<StoredRow>, params: CassandraParams): Array<Row> {
return this.matchingEntries(meta, stored, params).map((entry) => entry.row);
}
private async select(
@@ -0,0 +1,72 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import fs from 'node:fs';
import path from 'node:path';
import {fileURLToPath} from 'node:url';
import {describe, expect, it} from 'vitest';
const THIS_DIR = path.dirname(fileURLToPath(import.meta.url));
const REPO_ROOT = path.resolve(THIS_DIR, '../../../..');
const TYPESCRIPT_SOURCE = fs.readFileSync(path.join(THIS_DIR, 'PostgresKvQueryExecutor.ts'), 'utf8');
const RUST_SOURCE = fs.readFileSync(path.join(REPO_ROOT, 'fluxer_svc/src/postgres.rs'), 'utf8');
const RUST_INDEX_SUFFIXES = [
'expires_idx',
'message_reactions_message_idx',
'messages_message_idx',
'partition_idx',
'partition_row_idx',
'row_key_c_idx',
];
const SHARED_LITERALS = ['120s', '30min', '__fluxer_schema_migrations', 'messages_partition_key_v1'];
const BACKFILL_FRAGMENTS = [
"split_part(row_key, chr(31), 3) <> ''",
'split_part(row_key, chr(31), 1) || chr(31) || split_part(row_key, chr(31), 2)',
];
function indexSuffixes(source: string, pattern: RegExp): Array<string> {
return [...new Set([...source.matchAll(pattern)].map((match) => match[1]!))].sort();
}
function lockNamespace(source: string, pattern: RegExp): number {
const match = pattern.exec(source);
expect(match, 'missing Postgres KV schema lock namespace').not.toBeNull();
return Number.parseInt(match![1]!.replaceAll('_', ''), 16);
}
describe('Postgres KV schema parity between the API and fluxer_svc', () => {
it('creates the same indexes apart from the API-only numeric index', () => {
const typescript = indexSuffixes(TYPESCRIPT_SOURCE, /\$\{kvTable\}_([a-z_]+)/g);
const rust = indexSuffixes(RUST_SOURCE, /\{kv_table\}_([a-z_]+)/g);
expect(rust).toEqual(RUST_INDEX_SUFFIXES);
expect(typescript.filter((suffix) => !rust.includes(suffix))).toEqual(['row_key_numeric_idx']);
expect(rust.filter((suffix) => !typescript.includes(suffix))).toEqual([]);
});
it('takes the same advisory lock namespace', () => {
expect(lockNamespace(RUST_SOURCE, /POSTGRES_KV_SCHEMA_LOCK_NAMESPACE: i32 = (0x[0-9a-fA-F_]+)/)).toBe(
lockNamespace(TYPESCRIPT_SOURCE, /POSTGRES_KV_SCHEMA_LOCK_NAMESPACE = (0x[0-9a-fA-F_]+)/),
);
});
it('shares the schema timeouts and the migration marker identity', () => {
for (const literal of SHARED_LITERALS) {
expect(TYPESCRIPT_SOURCE, literal).toContain(literal);
expect(RUST_SOURCE, literal).toContain(literal);
}
});
it('never disables the statement timeout in fluxer_svc', () => {
expect(RUST_SOURCE).not.toContain("set_config('statement_timeout', '0'");
});
it('backfills message partition keys with byte-identical SQL', () => {
for (const fragment of BACKFILL_FRAGMENTS) {
expect(TYPESCRIPT_SOURCE, fragment).toContain(fragment);
expect(RUST_SOURCE, fragment).toContain(fragment);
}
});
});
@@ -306,9 +306,13 @@ suite('postgres kv upgrade safety', () => {
WHERE tablename = '${OLD}' AND indexname <> '${OLD}_row_key_c_idx'
EXCEPT SELECT replace(indexdef, '${NEW}', 'KV') FROM pg_indexes WHERE tablename = '${NEW}')
UNION ALL
(SELECT replace(indexdef, '${NEW}', 'KV') FROM pg_indexes WHERE tablename = '${NEW}'
(SELECT replace(indexdef, '${NEW}', 'KV') FROM pg_indexes
WHERE tablename = '${NEW}' AND indexname <> '${NEW}_row_key_numeric_idx'
EXCEPT SELECT replace(indexdef, '${OLD}', 'KV') FROM pg_indexes WHERE tablename = '${OLD}')
) d`);
const added = await raw.query<{indexname: string}>(`
SELECT indexname FROM pg_indexes WHERE tablename = '${NEW}'
EXCEPT SELECT replace(indexname, '${OLD}', '${NEW}') FROM pg_indexes WHERE tablename = '${OLD}'`);
const collations = await raw.query<{tablename: string; attname: string; collname: string}>(`
SELECT cls.relname AS tablename, att.attname, col.collname
FROM pg_attribute att
@@ -322,6 +326,7 @@ suite('postgres kv upgrade safety', () => {
);
expect(Number(diff.rows[0]!.n)).toBe(0);
expect(Number(schemaDiff.rows[0]!.n)).toBe(0);
expect(added.rows.map((r) => r.indexname)).toEqual([`${NEW}_row_key_numeric_idx`]);
expect(collations.rows.filter((r) => r.tablename === OLD).map((r) => r.collname)).toEqual(['default', 'default']);
expect(collations.rows.filter((r) => r.tablename === NEW).map((r) => r.collname)).toEqual(['C', 'C']);
expect(cIndexes.rows.map((r) => r.tablename)).toEqual([OLD]);
@@ -362,6 +367,37 @@ suite('postgres kv upgrade safety', () => {
]);
}, 120_000);
it('survives a peer that creates the table without the schema lock', async () => {
const RACE = `${KV}_race`;
await raw.query(`DROP TABLE IF EXISTS ${RACE}`);
let release = () => {};
const gate = new Promise<void>((resolve) => {
release = resolve;
});
const peer = raw.transaction(async (db) => {
await db.query(`
CREATE TABLE IF NOT EXISTS ${RACE} (
table_name text NOT NULL,
partition_key text COLLATE "C" NOT NULL,
row_key text COLLATE "C" NOT NULL,
row_data jsonb NOT NULL,
expires_at timestamptz,
updated_at timestamptz NOT NULL DEFAULT now(),
PRIMARY KEY (table_name, row_key)
)`);
await gate;
});
const booting = ensurePostgresKvSchema(new TableClient(raw, RACE)).then(
() => 'ok',
(error: Error) => `failed: ${error.message}`,
);
await sleep(500);
release();
await peer;
expect(await booting).toBe('ok');
await raw.query(`DROP TABLE IF EXISTS ${RACE}`);
}, 120_000);
it('backfills the messages partition key once and never scans for it again', async () => {
const BACKFILL = 'kv_backfill';
const SEP = String.fromCharCode(31);
@@ -0,0 +1,99 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
const TRANSIENT_POSTGRES_SQLSTATES: ReadonlySet<string> = new Set([
'08000',
'08001',
'08003',
'08004',
'08006',
'08007',
'57P01',
'57P02',
'57P03',
]);
const TRANSIENT_SOCKET_CODES: ReadonlySet<string> = new Set([
'EAI_AGAIN',
'ECONNABORTED',
'ECONNREFUSED',
'ECONNRESET',
'EHOSTUNREACH',
'ENETDOWN',
'ENETUNREACH',
'ENOTFOUND',
'EPIPE',
'ETIMEDOUT',
]);
const TRANSIENT_DRIVER_MESSAGES: ReadonlySet<string> = new Set([
'Client has encountered a connection error and is not queryable',
'Client was closed and is not queryable',
'Connection terminated',
'Connection terminated due to connection timeout',
'Connection terminated unexpectedly',
'timeout exceeded when trying to connect',
]);
const MAX_ERROR_CHAIN_DEPTH = 8;
type ErrorNode = Record<string, unknown>;
function readString(value: unknown): string | null {
return typeof value === 'string' ? value : null;
}
function collectErrorChain(error: unknown): Array<ErrorNode> {
const nodes: Array<ErrorNode> = [];
const queue: Array<unknown> = [error];
const seen = new Set<unknown>();
while (queue.length > 0 && nodes.length < MAX_ERROR_CHAIN_DEPTH) {
const current = queue.shift();
if (typeof current !== 'object' || current === null || seen.has(current)) {
continue;
}
seen.add(current);
const node = current as ErrorNode;
nodes.push(node);
if ('cause' in node) {
queue.push(node['cause']);
}
const aggregated = node['errors'];
if (Array.isArray(aggregated)) {
for (const nested of aggregated) {
queue.push(nested);
}
}
}
return nodes;
}
function carriesPostgresClient(node: ErrorNode): boolean {
const client = node['client'];
return typeof client === 'object' && client !== null;
}
function hasTransientSqlState(node: ErrorNode): boolean {
const code = readString(node['code']);
return code !== null && TRANSIENT_POSTGRES_SQLSTATES.has(code);
}
function hasTransientSocketCode(node: ErrorNode): boolean {
const code = readString(node['code']);
return code !== null && TRANSIENT_SOCKET_CODES.has(code);
}
function hasTransientDriverMessage(node: ErrorNode): boolean {
const message = readString(node['message']);
return message !== null && TRANSIENT_DRIVER_MESSAGES.has(message);
}
export function isTransientDatabaseError(error: unknown): boolean {
const nodes = collectErrorChain(error);
if (nodes.some(hasTransientSqlState)) {
return true;
}
if (nodes.some(carriesPostgresClient) && nodes.some(hasTransientSocketCode)) {
return true;
}
return nodes.some(hasTransientDriverMessage);
}
@@ -6,7 +6,7 @@ import {ms, seconds} from 'itty-time';
import type {UserID} from '../BrandedTypes';
import {Logger} from '../Logger';
import type {UserRepository} from '../user/repositories/UserRepository';
import {resolvePendingDeletionReasonCode} from '../user/services/PendingDeletionCoordinator';
import {isPendingDeletionBlocked, resolvePendingDeletionReasonCode} from '../user/services/PendingDeletionCoordinator';
interface QueuedDeletion {
userId: bigint;
@@ -59,7 +59,7 @@ export class KVAccountDeletionQueueService {
}
}
async rebuildState(): Promise<void> {
async rebuildState(lockToken: string | null = null): Promise<void> {
Logger.info('Starting deletion queue rebuild from primary database');
try {
await this.kvClient.del(QUEUE_KEY);
@@ -76,7 +76,7 @@ export class KVAccountDeletionQueueService {
}
let batchQueued = 0;
for (const user of users) {
if (user.pendingDeletionAt) {
if (user.pendingDeletionAt && !isPendingDeletionBlocked(user)) {
const queueItem: QueuedDeletion = {
userId: user.id,
deletionReasonCode: resolvePendingDeletionReasonCode(user, 0),
@@ -91,6 +91,9 @@ export class KVAccountDeletionQueueService {
totalQueued += batchQueued;
totalProcessed += users.length;
pageState = page.pageState;
if (lockToken !== null) {
await this.renewRebuildLock(lockToken);
}
if (totalProcessed % 10000 === 0) {
Logger.debug({totalProcessed, totalQueued}, 'Deletion queue rebuild progress');
}
@@ -174,6 +177,17 @@ export class KVAccountDeletionQueueService {
}
}
private async renewRebuildLock(token: string): Promise<void> {
try {
const renewed = await this.kvClient.extendLock(REBUILD_LOCK_KEY, token, REBUILD_LOCK_TTL);
if (!renewed) {
Logger.warn({token}, 'Deletion queue rebuild lock was no longer held on renewal');
}
} catch (error) {
Logger.error({error, token}, 'Failed to renew deletion queue rebuild lock');
}
}
async releaseRebuildLock(token: string): Promise<boolean> {
try {
const released = await this.kvClient.releaseLock(REBUILD_LOCK_KEY, token);
@@ -1,6 +1,7 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import type {IKVProvider} from '@pkgs/kv_client/src/IKVProvider';
import {runSlotBatches, splitIntoSlotBatches} from '@pkgs/kv_client/src/KVHashSlots';
import {seconds} from 'itty-time';
import type {UserID} from '../BrandedTypes';
import {Logger} from '../Logger';
@@ -74,6 +75,21 @@ export class KVActivityTracker {
return age > STATE_VERSION_TTL_SECONDS;
}
private async writeActivityBatch(entries: ReadonlyArray<{key: string; value: string}>): Promise<void> {
const batches = splitIntoSlotBatches(entries, (entry) => entry.key, this.kvClient.isClustered());
await runSlotBatches(batches, async (batch) => {
const pipeline = this.kvClient.pipeline();
for (const entry of batch) {
pipeline.setex(entry.key, TTL_SECONDS, entry.value);
}
for (const [error] of await pipeline.exec()) {
if (error) {
throw error;
}
}
});
}
async rebuildActivities(): Promise<void> {
Logger.info('Starting activity tracker rebuild from Cassandra');
const userRepository = new UserRepository();
@@ -81,8 +97,7 @@ export class KVActivityTracker {
const kvBatchSize = 1000;
let processedCount = 0;
let usersWithActivity = 0;
let pipeline = this.kvClient.pipeline();
let pipelineCount = 0;
let batch: Array<{key: string; value: string}> = [];
let pageState: string | null = null;
let iterationCount = 0;
while (!this.isShuttingDown) {
@@ -93,15 +108,11 @@ export class KVActivityTracker {
}
for (const user of users) {
if (user.lastActiveAt) {
const key = this.getActivityKey(user.id);
const value = user.lastActiveAt.getTime().toString();
pipeline.setex(key, TTL_SECONDS, value);
pipelineCount++;
batch.push({key: this.getActivityKey(user.id), value: user.lastActiveAt.getTime().toString()});
usersWithActivity++;
if (pipelineCount >= kvBatchSize) {
await pipeline.exec();
pipeline = this.kvClient.pipeline();
pipelineCount = 0;
if (batch.length >= kvBatchSize) {
await this.writeActivityBatch(batch);
batch = [];
}
}
processedCount++;
@@ -122,8 +133,8 @@ export class KVActivityTracker {
Logger.warn({processedCount, usersWithActivity}, 'Activity tracker rebuild interrupted by shutdown');
return;
}
if (pipelineCount > 0) {
await pipeline.exec();
if (batch.length > 0) {
await this.writeActivityBatch(batch);
}
await this.kvClient.setex(STATE_VERSION_KEY, STATE_VERSION_TTL_SECONDS, Date.now().toString());
Logger.info({processedCount, usersWithActivity}, 'Activity tracker rebuild completed');
@@ -29,6 +29,16 @@ export class PremiumStateReconciliationQueueService {
}
}
async claimUser(userId: UserID, nowMs: number, leaseUntilMs: number): Promise<boolean> {
try {
const value = this.serializeQueueValue(userId);
return await this.kvClient.claimBulkDeletion(QUEUE_KEY, value, nowMs, leaseUntilMs);
} catch (error) {
Logger.error({error, userId: userId.toString()}, 'Failed to claim user for premium state reconciliation');
throw error;
}
}
async removeUser(userId: UserID): Promise<boolean> {
try {
const secondaryKey = this.getSecondaryKey(userId);
@@ -106,7 +106,7 @@ export class VoiceRoomStore {
const member = this.buildOccupancyMember(guildId, channelId);
const regionKey = `${VOICE_OCCUPANCY_REGION_KEY_PREFIX}:${regionId}`;
const serverKey = `${VOICE_OCCUPANCY_SERVER_KEY_PREFIX}:${regionId}:${serverId}`;
await this.kvClient.multi().sadd(regionKey, member).sadd(serverKey, member).exec();
await Promise.all([this.kvClient.sadd(regionKey, member), this.kvClient.sadd(serverKey, member)]);
}
private async removeOccupancy(
@@ -118,7 +118,7 @@ export class VoiceRoomStore {
const member = this.buildOccupancyMember(guildId, channelId);
const regionKey = `${VOICE_OCCUPANCY_REGION_KEY_PREFIX}:${regionId}`;
const serverKey = `${VOICE_OCCUPANCY_SERVER_KEY_PREFIX}:${regionId}:${serverId}`;
await this.kvClient.multi().srem(regionKey, member).srem(serverKey, member).exec();
await Promise.all([this.kvClient.srem(regionKey, member), this.kvClient.srem(serverKey, member)]);
}
private buildOccupancyMember(guildId: bigint | undefined, channelId: bigint): string {
@@ -0,0 +1,47 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import {UserFlags} from '@fluxer/constants/src/UserConstants';
import {describe, expect, it} from 'vitest';
import {createUserID} from '../../BrandedTypes';
import {EMPTY_USER_ROW, type UserRow} from '../../database/types/UserTypes';
import {User} from '../../models/User';
import {MockKVProvider} from '../../test/mocks/MockKVProvider';
import type {UserRepository} from '../../user/repositories/UserRepository';
import {KVAccountDeletionQueueService} from '../KVAccountDeletionQueueService';
function createUser(id: bigint, overrides: Partial<Pick<UserRow, 'bot' | 'flags'>> = {}): User {
return new User({
...EMPTY_USER_ROW,
user_id: createUserID(id),
pending_deletion_at: new Date('2026-06-01T00:00:00.000Z'),
deletion_reason_code: 0,
bot: overrides.bot ?? false,
flags: overrides.flags ?? 0n,
});
}
function createUserRepository(users: Array<User>): UserRepository {
let served = false;
return {
async scanAllUsersPage() {
if (served) {
return {users: [], pageState: null};
}
served = true;
return {users, pageState: null};
},
} as unknown as UserRepository;
}
describe('KVAccountDeletionQueueService rebuild', () => {
it('does not requeue accounts the deletion worker refuses to process', async () => {
const kvClient = new MockKVProvider();
const users = [createUser(1n, {bot: true}), createUser(2n, {flags: UserFlags.APP_STORE_REVIEWER}), createUser(3n)];
const service = new KVAccountDeletionQueueService(kvClient, createUserRepository(users));
await service.rebuildState();
expect(await service.getQueueSize()).toBe(1);
expect(await service.getReadyDeletions(Date.now(), 100)).toEqual([{userId: 3n, deletionReasonCode: 0}]);
});
});
@@ -0,0 +1,58 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import {ms} from 'itty-time';
import {afterEach, describe, expect, it, vi} from 'vitest';
import {createUserID} from '../../BrandedTypes';
import {EMPTY_USER_ROW} from '../../database/types/UserTypes';
import {User} from '../../models/User';
import {MockKVProvider} from '../../test/mocks/MockKVProvider';
import type {UserRepository} from '../../user/repositories/UserRepository';
import {KVAccountDeletionQueueService} from '../KVAccountDeletionQueueService';
const PAGE_DURATION_MS = ms('3 minutes');
const PAGE_COUNT = 3;
function createPendingUser(index: number): User {
return new User({
...EMPTY_USER_ROW,
user_id: createUserID(BigInt(7000 + index)),
pending_deletion_at: new Date('2026-06-01T00:00:00.000Z'),
deletion_reason_code: 0,
bot: false,
flags: 0n,
});
}
function createSlowUserRepository(): UserRepository {
let page = 0;
return {
async scanAllUsersPage() {
vi.advanceTimersByTime(PAGE_DURATION_MS);
page += 1;
return {
users: [createPendingUser(page)],
pageState: page < PAGE_COUNT ? `page-${page}` : null,
};
},
} as unknown as UserRepository;
}
describe('KVAccountDeletionQueueService rebuild lock', () => {
afterEach(() => {
vi.useRealTimers();
});
it('keeps holding the rebuild lock across a scan longer than the lock ttl', async () => {
vi.useFakeTimers();
vi.setSystemTime(new Date('2026-06-01T00:00:00.000Z'));
const kvClient = new MockKVProvider();
const service = new KVAccountDeletionQueueService(kvClient, createSlowUserRepository());
const token = await service.acquireRebuildLock();
expect(token).not.toBeNull();
await service.rebuildState(token);
expect(await service.acquireRebuildLock()).toBeNull();
expect(await service.releaseRebuildLock(token!)).toBe(true);
});
});
@@ -0,0 +1,34 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import {computeHashSlot} from '@pkgs/kv_client/src/KVHashSlots';
import {afterEach, describe, expect, it, vi} from 'vitest';
import {createUserID} from '../../BrandedTypes';
import type {User} from '../../models/User';
import {BatchRecordingKVProvider} from '../../test/mocks/BatchRecordingKVProvider';
import {UserRepository} from '../../user/repositories/UserRepository';
import {KVActivityTracker} from '../KVActivityTracker';
function createUser(id: bigint, lastActiveAt: Date): User {
return {id: createUserID(id), lastActiveAt} as unknown as User;
}
describe('KVActivityTracker cluster hash slots', () => {
afterEach(() => {
vi.restoreAllMocks();
});
it('keeps the rebuild writes off batched commands that span hash slots', async () => {
const kvClient = new BatchRecordingKVProvider();
const lastActiveAt = new Date('2026-06-01T00:00:00.000Z');
const users = [createUser(1234n, lastActiveAt), createUser(5678n, lastActiveAt)];
vi.spyOn(UserRepository.prototype, 'scanAllUsersPage').mockResolvedValue({users, pageState: null});
expect(computeHashSlot('user_activity:1234')).not.toBe(computeHashSlot('user_activity:5678'));
await new KVActivityTracker(kvClient).rebuildActivities();
expect(await kvClient.get('user_activity:1234')).toBe(lastActiveAt.getTime().toString());
expect(await kvClient.get('user_activity:5678')).toBe(lastActiveAt.getTime().toString());
expect(kvClient.crossSlotBatches()).toEqual([]);
});
});
@@ -0,0 +1,107 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import type {IKVPipeline} from '@pkgs/kv_client/src/IKVProvider';
import {afterEach, describe, expect, it, vi} from 'vitest';
import {createUserID} from '../../BrandedTypes';
import type {User} from '../../models/User';
import {BatchRecordingKVProvider} from '../../test/mocks/BatchRecordingKVProvider';
import {UserRepository} from '../../user/repositories/UserRepository';
import {KVActivityTracker} from '../KVActivityTracker';
const REBUILD_KV_BATCH_SIZE = 1000;
const MAX_CONCURRENT_ROUND_TRIPS = 16;
class FanoutRecordingKVProvider extends BatchRecordingKVProvider {
inFlight = 0;
peakInFlight = 0;
failingKey: string | null = null;
private insidePipeline = 0;
override pipeline(): IKVPipeline {
const inner = super.pipeline();
return {
...inner,
exec: async () =>
await this.trackRoundTrip(async () => {
this.insidePipeline += 1;
try {
return await inner.exec();
} finally {
this.insidePipeline -= 1;
}
}),
};
}
override async setex(key: string, ttlSeconds: number, value: string): Promise<void> {
if (key === this.failingKey) {
throw new Error('write rejected');
}
if (this.insidePipeline > 0) {
await super.setex(key, ttlSeconds, value);
return;
}
await this.trackRoundTrip(async () => {
await super.setex(key, ttlSeconds, value);
});
}
private async trackRoundTrip<T>(run: () => Promise<T>): Promise<T> {
this.inFlight += 1;
this.peakInFlight = Math.max(this.peakInFlight, this.inFlight);
try {
await new Promise((resolve) => setTimeout(resolve, 0));
return await run();
} finally {
this.inFlight -= 1;
}
}
}
function mockUserPage(count: number): Array<User> {
const lastActiveAt = new Date('2026-06-01T00:00:00.000Z');
const users = Array.from(
{length: count},
(_unused, index) => ({id: createUserID(BigInt(index + 1)), lastActiveAt}) as unknown as User,
);
vi.spyOn(UserRepository.prototype, 'scanAllUsersPage').mockResolvedValue({users, pageState: null});
return users;
}
describe('KVActivityTracker rebuild fan out', () => {
afterEach(() => {
vi.restoreAllMocks();
});
it('writes a full rebuild batch in one round trip outside cluster mode', async () => {
const kvClient = new FanoutRecordingKVProvider();
kvClient.clustered = false;
mockUserPage(REBUILD_KV_BATCH_SIZE);
await new KVActivityTracker(kvClient).rebuildActivities();
expect(kvClient.batches.map((batch) => batch.keys.length)).toEqual([REBUILD_KV_BATCH_SIZE]);
expect(kvClient.peakInFlight).toBe(1);
});
it('bounds concurrent round trips when a rebuild batch spans hash slots', async () => {
const kvClient = new FanoutRecordingKVProvider();
const users = mockUserPage(REBUILD_KV_BATCH_SIZE);
await new KVActivityTracker(kvClient).rebuildActivities();
expect(kvClient.peakInFlight).toBeLessThanOrEqual(MAX_CONCURRENT_ROUND_TRIPS);
expect(kvClient.crossSlotBatches()).toEqual([]);
expect(kvClient.batches.flatMap((batch) => batch.keys).length).toBe(REBUILD_KV_BATCH_SIZE);
expect(await kvClient.get(`user_activity:${users[0].id}`)).toBe(users[0].lastActiveAt?.getTime().toString());
});
it('fails the rebuild when a batched write fails', async () => {
const kvClient = new FanoutRecordingKVProvider();
const users = mockUserPage(REBUILD_KV_BATCH_SIZE);
kvClient.failingKey = `user_activity:${users[0].id}`;
await expect(new KVActivityTracker(kvClient).rebuildActivities()).rejects.toThrow('write rejected');
});
});
@@ -0,0 +1,32 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import {computeHashSlot} from '@pkgs/kv_client/src/KVHashSlots';
import {describe, expect, it} from 'vitest';
import {createChannelID, createGuildID} from '../../BrandedTypes';
import {BatchRecordingKVProvider} from '../../test/mocks/BatchRecordingKVProvider';
import {VOICE_OCCUPANCY_REGION_KEY_PREFIX, VOICE_OCCUPANCY_SERVER_KEY_PREFIX} from '../../voice/VoiceConstants';
import {VoiceRoomStore} from '../VoiceRoomStore';
describe('VoiceRoomStore cluster hash slots', () => {
it('keeps occupancy writes off batched commands that span hash slots', async () => {
const kvClient = new BatchRecordingKVProvider();
const store = new VoiceRoomStore(kvClient);
const guildId = createGuildID(1234n);
const channelId = createChannelID(5678n);
const regionKey = `${VOICE_OCCUPANCY_REGION_KEY_PREFIX}:us-east`;
const serverKey = `${VOICE_OCCUPANCY_SERVER_KEY_PREFIX}:us-east:voice-1`;
const member = 'guild:1234:channel:5678';
expect(computeHashSlot(regionKey)).not.toBe(computeHashSlot(serverKey));
await store.pinRoomServer(guildId, channelId, 'us-east', 'voice-1', 'wss://voice-1.example');
expect(await kvClient.smembers(regionKey)).toEqual([member]);
expect(await kvClient.smembers(serverKey)).toEqual([member]);
await store.deleteRoomServer(guildId, channelId);
expect(await kvClient.smembers(regionKey)).toEqual([]);
expect(await kvClient.smembers(serverKey)).toEqual([]);
expect(kvClient.crossSlotBatches()).toEqual([]);
});
});
@@ -39,7 +39,6 @@ export class SingleCommunityService {
userId,
guildId,
skipGuildLimitCheck: true,
skipBanCheck: true,
joinSourceType: JoinSourceTypes.ADMIN_FORCE_ADD,
requestCache,
});
+72 -44
View File
@@ -22220,7 +22220,7 @@
"required": true,
"content": {
"application/json": {"schema": {"$ref": "#/components/schemas/WebhookMessageRequest"}},
"multipart/form-data": {"schema": {"$ref": "#/components/schemas/WebhookMessageRequest"}}
"multipart/form-data": {"schema": {"$ref": "#/components/schemas/WebhookMultipartMessageRequest"}}
}
}
}
@@ -34715,51 +34715,51 @@
"items": {"$ref": "#/components/schemas/RichEmbedRequest"},
"description": "Array of embed objects to include in the message"
},
"message_reference": {
"anyOf": [{"$ref": "#/components/schemas/MessageReferenceRequest"}, {"type": "null"}],
"description": "Reference to another message (for replies or forwards)"
},
"allowed_mentions": {
"anyOf": [{"$ref": "#/components/schemas/AllowedMentionsRequest"}, {"type": "null"}],
"description": "Controls which mentions trigger notifications"
},
"flags": {"$ref": "#/components/schemas/MessageFlags"},
"nonce": {"$ref": "#/components/schemas/MessageNonceRequest"},
"favorite_meme_id": {
"anyOf": [{"$ref": "#/components/schemas/SnowflakeType"}, {"type": "null"}],
"description": "ID of a favorite meme to attach"
},
"sticker_ids": {
"anyOf": [
{"type": "array", "items": {"$ref": "#/components/schemas/SnowflakeType"}, "maxItems": 3},
{"type": "null"}
],
"description": "Array of sticker IDs to include (max 3)"
},
"tts": {"type": "boolean", "description": "Whether this is a text-to-speech message"},
"username": {
"anyOf": [{"type": "string"}, {"type": "null"}],
"description": "Override the default username of the webhook for this message"
},
"avatar_url": {
"anyOf": [{"type": "string"}, {"type": "null"}],
"description": "Override the default avatar URL of the webhook for this message"
},
"attachments": {
"type": "array",
"items": {
"oneOf": [
{"$ref": "#/components/schemas/ClientUploadedAttachmentRequest"},
{
"type": "object",
"properties": {
"id": {
"oneOf": [
{"$ref": "#/components/schemas/SnowflakeType"},
{"$ref": "#/components/schemas/Int32Type"}
],
"description": "Attachment ID for referencing uploaded files"
},
"filename": {"type": "string", "description": "Name of the file (1-1024 characters)"},
"description": {
"type": "string",
"description": "Description for the attachment (max 4096 characters)"
},
"content_type": {"type": "string", "description": "MIME type of the file"},
"upload_filename": {"not": {}},
"file_size": {"not": {}},
"size": {"$ref": "#/components/schemas/NonNegativeSafeIntegerType"},
"url": {"type": "string", "description": "URL of the attachment"},
"proxy_url": {"type": "string", "description": "Proxied URL of the attachment"},
"height": {
"type": "integer",
"format": "int53",
"description": "Height of the image/video in pixels"
},
"width": {
"type": "integer",
"format": "int53",
"description": "Width of the image/video in pixels"
},
"ephemeral": {"type": "boolean", "description": "Whether this attachment is ephemeral"},
"duration": {"type": "number", "description": "Duration of audio file in seconds"},
"waveform": {"type": "string", "description": "Base64-encoded bytearray of audio waveform"},
"flags": {"$ref": "#/components/schemas/MessageAttachmentFlags"}
}
}
]
},
"description": "Array of attachment objects"
"items": {"$ref": "#/components/schemas/ClientUploadedAttachmentRequest"},
"description": "Array of attachments uploaded through the presigned upload endpoint"
}
}
},
"WebhookMultipartMessageRequest": {
"type": "object",
"properties": {
"content": {"anyOf": [{"$ref": "#/components/schemas/MessageContentRequest"}, {"type": "null"}]},
"embeds": {
"type": "array",
"items": {"$ref": "#/components/schemas/RichEmbedRequest"},
"description": "Array of embed objects to include in the message"
},
"message_reference": {
"anyOf": [{"$ref": "#/components/schemas/MessageReferenceRequest"}, {"type": "null"}],
@@ -34790,6 +34790,34 @@
"avatar_url": {
"anyOf": [{"type": "string"}, {"type": "null"}],
"description": "Override the default avatar URL of the webhook for this message"
},
"attachments": {
"type": "array",
"items": {
"type": "object",
"properties": {
"id": {
"oneOf": [{"$ref": "#/components/schemas/SnowflakeType"}, {"$ref": "#/components/schemas/Int32Type"}],
"description": "Attachment ID for referencing uploaded files"
},
"filename": {"type": "string", "description": "Name of the file (1-1024 characters)"},
"description": {
"type": "string",
"description": "Description for the attachment (max 4096 characters)"
},
"content_type": {"type": "string", "description": "MIME type of the file"},
"size": {"$ref": "#/components/schemas/NonNegativeSafeIntegerType"},
"url": {"type": "string", "description": "URL of the attachment"},
"proxy_url": {"type": "string", "description": "Proxied URL of the attachment"},
"height": {"type": "integer", "format": "int53", "description": "Height of the image/video in pixels"},
"width": {"type": "integer", "format": "int53", "description": "Width of the image/video in pixels"},
"ephemeral": {"type": "boolean", "description": "Whether this attachment is ephemeral"},
"duration": {"type": "number", "description": "Duration of audio file in seconds"},
"waveform": {"type": "string", "description": "Base64-encoded bytearray of audio waveform"},
"flags": {"$ref": "#/components/schemas/MessageAttachmentFlags"}
}
},
"description": "Array of multipart attachment metadata objects"
}
}
},
@@ -1055,6 +1055,7 @@ export class StripeCheckoutService {
private static readonly LOCALIZED_CARD_PREAPPROVAL_CONTINUE_LOCK_TTL_SECONDS = seconds('30 seconds');
private static readonly LOCALIZED_CARD_PREAPPROVAL_TTL_SECONDS = seconds('1 day');
private static readonly PRICE_CACHE_TTL_SECONDS = seconds('1 hour');
private static readonly PRICE_CACHE_PRODUCE_TIMEOUT_MS = 90000;
private resolveConfiguredPriceIds(countryCode?: string, pricingMode: PricingMode = 'localized'): ResolvedPriceIds {
const recurringCurrencyPreferences =
@@ -1247,6 +1248,7 @@ export class StripeCheckoutService {
};
},
StripeCheckoutService.PRICE_CACHE_TTL_SECONDS,
StripeCheckoutService.PRICE_CACHE_PRODUCE_TIMEOUT_MS,
);
} catch (error: unknown) {
Logger.warn({error, priceId}, 'Failed to retrieve Stripe price summary');
@@ -790,6 +790,7 @@ export class StripeSubscriptionService {
cacheKey,
async () => this.loadCurrentSubscriptionPrice(user.stripeSubscriptionId!),
StripeSubscriptionService.CURRENT_PRICE_CACHE_TTL_SECONDS,
StripeSubscriptionService.PRICE_CACHE_PRODUCE_TIMEOUT_MS,
);
} catch (error) {
Logger.warn(
@@ -843,6 +844,7 @@ export class StripeSubscriptionService {
return price.unit_amount ?? null;
},
StripeSubscriptionService.LIST_PRICE_CACHE_TTL_SECONDS,
StripeSubscriptionService.PRICE_CACHE_PRODUCE_TIMEOUT_MS,
);
} catch (error) {
Logger.warn({error, priceId}, 'Failed to retrieve Stripe list price amount');
@@ -1023,6 +1025,7 @@ export class StripeSubscriptionService {
private static readonly CURRENT_PRICE_CACHE_TTL_SECONDS = seconds('5 minutes');
private static readonly LIST_PRICE_CACHE_TTL_SECONDS = seconds('1 hour');
private static readonly PRICE_CACHE_PRODUCE_TIMEOUT_MS = 90000;
private static readonly USER_TRIAL_LOCK_TTL_SECONDS = seconds('30 seconds');
private static readonly USER_TRIAL_LOCK_MAX_WAIT_MS = 15000;
private static readonly USER_TRIAL_LOCK_RETRY_DELAY_MS = 100;
@@ -0,0 +1,94 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import type {IKVPipeline} from '@pkgs/kv_client/src/IKVProvider';
import {computeHashSlot} from '@pkgs/kv_client/src/KVHashSlots';
import {MockKVProvider} from './MockKVProvider';
type BatchMode = 'multi' | 'pipeline';
interface RecordedBatch {
mode: BatchMode;
keys: Array<string>;
}
export class BatchRecordingKVProvider extends MockKVProvider {
readonly batches: Array<RecordedBatch> = [];
override pipeline(): IKVPipeline {
return this.recordBatch('pipeline', super.pipeline());
}
override multi(): IKVPipeline {
return this.recordBatch('multi', super.multi());
}
crossSlotBatches(): Array<RecordedBatch> {
return this.batches.filter((batch) => new Set(batch.keys.map(computeHashSlot)).size > 1);
}
private recordBatch(mode: BatchMode, inner: IKVPipeline): IKVPipeline {
const batch: RecordedBatch = {mode, keys: []};
this.batches.push(batch);
const recorded: IKVPipeline = {
get: (key) => {
batch.keys.push(key);
inner.get(key);
return recorded;
},
set: (key, value) => {
batch.keys.push(key);
inner.set(key, value);
return recorded;
},
setex: (key, ttlSeconds, value) => {
batch.keys.push(key);
inner.setex(key, ttlSeconds, value);
return recorded;
},
del: (key) => {
batch.keys.push(key);
inner.del(key);
return recorded;
},
expire: (key, ttlSeconds) => {
batch.keys.push(key);
inner.expire(key, ttlSeconds);
return recorded;
},
sadd: (key, ...members) => {
batch.keys.push(key);
inner.sadd(key, ...members);
return recorded;
},
srem: (key, ...members) => {
batch.keys.push(key);
inner.srem(key, ...members);
return recorded;
},
zadd: (key, score, value) => {
batch.keys.push(key);
inner.zadd(key, score, value);
return recorded;
},
zrem: (key, ...members) => {
batch.keys.push(key);
inner.zrem(key, ...members);
return recorded;
},
hgetall: (key) => {
batch.keys.push(key);
inner.hgetall(key);
return recorded;
},
mset: (...args) => {
for (let index = 0; index + 1 < args.length; index += 2) {
batch.keys.push(args[index]);
}
inner.mset(...args);
return recorded;
},
exec: async () => await inner.exec(),
};
return recorded;
}
}
@@ -111,6 +111,7 @@ export class MockKVProvider implements IKVProvider {
key: string;
values: Array<string>;
}> = [];
clustered = true;
private subscription: MockKVSubscription;
private readonly stringStore = new Map<string, string>();
private readonly setStore = new Map<string, Set<string>>();
@@ -155,6 +156,7 @@ export class MockKVProvider implements IKVProvider {
readonly checkLeakyBucketLimitSpy = vi.fn();
readonly tryConsumeTokensSpy = vi.fn();
readonly scheduleBulkDeletionSpy = vi.fn();
readonly claimBulkDeletionSpy = vi.fn();
readonly removeBulkDeletionSpy = vi.fn();
readonly scanSpy = vi.fn();
readonly dequeuePurgeBatchSpy = vi.fn();
@@ -667,6 +669,17 @@ export class MockKVProvider implements IKVProvider {
this.expiries.delete(secondaryKey);
}
async claimBulkDeletion(queueKey: string, member: string, maxScore: number, leaseScore: number): Promise<boolean> {
this.claimBulkDeletionSpy(queueKey, member, maxScore, leaseScore);
this.evictIfExpired(queueKey);
const score = this.zsetStore.get(queueKey)?.get(member);
if (score === undefined || score > maxScore) {
return false;
}
await this.zadd(queueKey, leaseScore, member);
return true;
}
async removeBulkDeletion(queueKey: string, secondaryKey: string, member = ''): Promise<boolean> {
this.removeBulkDeletionSpy(queueKey, secondaryKey, member);
this.evictIfExpired(secondaryKey);
@@ -770,6 +783,10 @@ export class MockKVProvider implements IKVProvider {
return this.createPipeline();
}
isClustered(): boolean {
return this.clustered;
}
async health(): Promise<boolean> {
this.healthSpy();
return true;
@@ -1113,6 +1130,7 @@ export class MockKVProvider implements IKVProvider {
this.checkLeakyBucketLimitSpy.mockClear();
this.tryConsumeTokensSpy.mockClear();
this.scheduleBulkDeletionSpy.mockClear();
this.claimBulkDeletionSpy.mockClear();
this.removeBulkDeletionSpy.mockClear();
this.scanSpy.mockClear();
this.dequeuePurgeBatchSpy.mockClear();
@@ -35,6 +35,11 @@ interface PendingDeletionReasonUserLike {
flags: bigint;
}
interface PendingDeletionEligibilityUserLike {
isBot: boolean;
flags: bigint;
}
export async function reschedulePendingDeletion({
userId,
currentPendingDeletionAt,
@@ -83,3 +88,10 @@ export function resolvePendingDeletionReasonCode(
}
return 0;
}
export function isPendingDeletionBlocked(user: PendingDeletionEligibilityUserLike): boolean {
if (user.isBot) {
return true;
}
return (user.flags & UserFlags.APP_STORE_REVIEWER) !== 0n;
}
@@ -4,6 +4,15 @@ import {Config} from '../Config';
import {Logger} from '../Logger';
import {VoiceRepository} from './VoiceRepository';
export function resolveLivekitEndpoint(configuredUrl: string | undefined, apiPublicUrl: string): string {
if (configuredUrl) {
return configuredUrl;
}
const apiPublic = new URL(apiPublicUrl);
const protocol = apiPublic.protocol === 'https:' ? 'wss' : 'ws';
return `${protocol}://${apiPublic.host}/livekit`;
}
export class VoiceDataInitializer {
async initialize(): Promise<void> {
if (!Config.voice.enabled || !Config.voice.defaultRegion) {
@@ -19,7 +28,7 @@ export class VoiceDataInitializer {
try {
const repository = new VoiceRepository();
const serverId = `${defaultRegion.id}-server-1`;
const livekitEndpoint = this.resolveLivekitEndpoint();
const livekitEndpoint = resolveLivekitEndpoint(Config.voice.url, Config.endpoints.apiPublic);
const existingRegions = await repository.listRegions();
if (existingRegions.length === 0) {
Logger.info('[VoiceDataInitializer] Creating default voice region from config...');
@@ -91,12 +100,4 @@ export class VoiceDataInitializer {
Logger.error({error}, '[VoiceDataInitializer] Failed to initialise config-managed voice topology');
}
}
private resolveLivekitEndpoint(): string {
if (Config.voice.url) {
return Config.voice.url;
}
const protocol = new URL(Config.endpoints.apiPublic).protocol.slice(0, -1) === 'https' ? 'wss' : 'ws';
return `${protocol}://${new URL(Config.endpoints.apiPublic).hostname}/livekit`;
}
}
@@ -131,6 +131,8 @@ export class VoiceReconciliationWorker {
private intervalHandle: NodeJS.Timeout | null = null;
private reconciling = false;
private reconciliationLockLost = false;
private activeReconciliation: Promise<void> | null = null;
private stopping = false;
constructor(options: VoiceReconciliationWorkerOptions) {
this.gatewayService = options.gatewayService;
@@ -161,18 +163,24 @@ export class VoiceReconciliationWorker {
},
'Starting VoiceReconciliationWorker',
);
this.stopping = false;
void this.runReconciliation();
this.intervalHandle = setInterval(() => {
void this.runReconciliation();
}, this.intervalMs);
}
stop(): void {
async stop(): Promise<void> {
this.stopping = true;
if (this.intervalHandle) {
clearInterval(this.intervalHandle);
this.intervalHandle = null;
this.logger.info('Stopped VoiceReconciliationWorker');
}
const activeReconciliation = this.activeReconciliation;
if (activeReconciliation !== null) {
await activeReconciliation;
}
this.logger.info('Stopped VoiceReconciliationWorker');
}
async reconcile(): Promise<void> {
@@ -203,6 +211,10 @@ export class VoiceReconciliationWorker {
let totalErrors = 0;
let totalTransientSkips = 0;
for (const room of discovery.rooms) {
if (this.stopping) {
this.logger.info('Stopping reconciliation sweep because the worker is shutting down');
break;
}
if (this.reconciliationLockLost) {
this.logger.warn('Stopping reconciliation sweep because the cluster lock was lost');
break;
@@ -261,6 +273,17 @@ export class VoiceReconciliationWorker {
return;
}
this.reconciling = true;
const sweep = this.runReconciliationSweep();
this.activeReconciliation = sweep;
try {
await sweep;
} finally {
this.activeReconciliation = null;
this.reconciling = false;
}
}
private async runReconciliationSweep(): Promise<void> {
let lockToken: string | null = null;
let lockRenewalHandle: NodeJS.Timeout | null = null;
try {
@@ -287,7 +310,6 @@ export class VoiceReconciliationWorker {
await this.releaseReconciliationLock(lockToken);
}
this.reconciliationLockLost = false;
this.reconciling = false;
}
}
@@ -0,0 +1,48 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import {describe, expect, it} from 'vitest';
import {resolveLivekitEndpoint} from '../VoiceDataInitializer';
describe('resolveLivekitEndpoint', () => {
it('keeps the public port when the instance does not serve on the default port', () => {
expect(resolveLivekitEndpoint(undefined, 'http://inplace.localhost:19480/api')).toBe(
'ws://inplace.localhost:19480/livekit',
);
expect(resolveLivekitEndpoint('', 'https://fluxer.example.com:8443/api')).toBe(
'wss://fluxer.example.com:8443/livekit',
);
});
it('omits the port when the public endpoint uses the default port for its scheme', () => {
expect(resolveLivekitEndpoint(undefined, 'https://fluxer.example.com/api')).toBe(
'wss://fluxer.example.com/livekit',
);
expect(resolveLivekitEndpoint(undefined, 'https://fluxer.example.com:443/api')).toBe(
'wss://fluxer.example.com/livekit',
);
expect(resolveLivekitEndpoint(undefined, 'http://fluxer.example.com:80/api')).toBe(
'ws://fluxer.example.com/livekit',
);
});
it('maps https to wss and http to ws', () => {
expect(resolveLivekitEndpoint(undefined, 'https://fluxer.example.com/api')).toBe(
'wss://fluxer.example.com/livekit',
);
expect(resolveLivekitEndpoint(undefined, 'http://fluxer.example.com/api')).toBe('ws://fluxer.example.com/livekit');
});
it('brackets ipv6 literals so the derived endpoint stays parseable', () => {
expect(resolveLivekitEndpoint(undefined, 'http://[::1]:19480/api')).toBe('ws://[::1]:19480/livekit');
expect(resolveLivekitEndpoint(undefined, 'https://[2001:db8::1]/api')).toBe('wss://[2001:db8::1]/livekit');
const derived = new URL(resolveLivekitEndpoint(undefined, 'http://[::1]:19480/api'));
expect(derived.host).toBe('[::1]:19480');
expect(derived.pathname).toBe('/livekit');
});
it('returns the configured voice url unchanged when one is set', () => {
expect(resolveLivekitEndpoint('wss://voice.example.com', 'http://inplace.localhost:19480/api')).toBe(
'wss://voice.example.com',
);
});
});
@@ -0,0 +1,64 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import type {IKVProvider} from '@pkgs/kv_client/src/IKVProvider';
import {describe, expect, it, vi} from 'vitest';
import type {ILogger} from '../../ILogger';
import type {IGatewayService} from '../../infrastructure/IGatewayService';
import type {ILiveKitService} from '../../infrastructure/ILiveKitService';
import type {IVoiceRoomStore} from '../../infrastructure/IVoiceRoomStore';
import {VoiceReconciliationWorker} from '../VoiceReconciliationWorker';
function createLogger(): ILogger {
const logger = {
trace: vi.fn(),
debug: vi.fn(),
info: vi.fn(),
warn: vi.fn(),
error: vi.fn(),
child: () => logger,
};
return logger as unknown as ILogger;
}
function createHarness() {
const releaseLock = vi.fn().mockResolvedValue(true);
const kvClient = {
acquireLock: vi.fn().mockResolvedValue(true),
extendLock: vi.fn().mockResolvedValue(true),
releaseLock,
setnx: vi.fn().mockResolvedValue(true),
get: vi.fn().mockResolvedValue(null),
setex: vi.fn().mockResolvedValue(undefined),
} as unknown as IKVProvider;
let finishDiscovery: () => void = () => {};
const discovery = new Promise<{rooms: []}>((resolve) => {
finishDiscovery = () => resolve({rooms: []});
});
const getActiveVoiceRooms = vi.fn().mockReturnValue(discovery);
const worker = new VoiceReconciliationWorker({
gatewayService: {getActiveVoiceRooms} as unknown as IGatewayService,
liveKitService: {
listActiveRooms: async () => ({rooms: [], errors: [], completed: true, searchedServers: 0}),
} as unknown as ILiveKitService,
voiceRoomStore: {listPinnedRooms: async () => []} as unknown as IVoiceRoomStore,
kvClient,
logger: createLogger(),
intervalMs: 60000,
staggerDelayMs: 0,
});
return {worker, releaseLock, getActiveVoiceRooms, finishDiscovery: () => finishDiscovery()};
}
describe('VoiceReconciliationWorker stop', () => {
it('releases the reconciliation lock before stop resolves', async () => {
const {worker, releaseLock, getActiveVoiceRooms, finishDiscovery} = createHarness();
worker.start();
await vi.waitFor(() => expect(getActiveVoiceRooms).toHaveBeenCalled());
setTimeout(finishDiscovery, 0);
await worker.stop();
expect(releaseLock).toHaveBeenCalledTimes(1);
});
});
@@ -19,6 +19,7 @@ import {
WebhookExecuteQueryRequest,
WebhookMessageEditRequest,
WebhookMessageRequest,
WebhookMultipartMessageRequest,
WebhookTokenUpdateRequest,
WebhookUpdateRequest,
} from '@fluxer/schema/src/domains/webhook/WebhookRequestSchemas';
@@ -47,6 +48,14 @@ function validateWebhookMessagePayload(data: unknown): WebhookMessageRequest {
return validationResult.data;
}
function validateWebhookMultipartMessagePayload(data: unknown): WebhookMultipartMessageRequest {
const validationResult = WebhookMultipartMessageRequest.safeParse(normalizeMessageRequestPayload(data));
if (!validationResult.success) {
throw InputValidationError.fromCode('message_data', ValidationErrorCodes.INVALID_MESSAGE_DATA);
}
return validationResult.data;
}
async function parseWebhookJsonMessageData(ctx: Context<HonoEnv>): Promise<WebhookExecuteMessageData> {
let data: unknown;
try {
@@ -89,10 +98,11 @@ async function parseWebhookMultipartMessageData(
if (!parsedPayload) {
throw InputValidationError.fromCode('message_data', ValidationErrorCodes.INVALID_MESSAGE_DATA);
}
const webhookData = validateWebhookMessagePayload(parsedPayload);
const webhookData = validateWebhookMultipartMessagePayload(parsedPayload);
return {
...webhookData,
...messageData,
attachments: messageData.attachments,
username: webhookData.username,
avatar_url: webhookData.avatar_url,
};
@@ -333,7 +343,7 @@ export function WebhookController(app: HonoApp) {
'Executes the webhook by sending a message to its configured channel. If the wait query parameter is true, returns the created message object; otherwise returns a 204 status with no content.',
responseSchema: MessageResponseSchema,
requestSchema: WebhookMessageRequest,
requestFormSchema: WebhookMessageRequest,
requestFormSchema: WebhookMultipartMessageRequest,
statusCode: 200,
tags: ['Webhooks'],
}),
+53 -16
View File
@@ -3,9 +3,12 @@
import type {LoggerInterface} from '@fluxer/logger/src/LoggerInterface';
import type {IKVProvider} from '@pkgs/kv_client/src/IKVProvider';
import type {WorkerJobPayload} from '@pkgs/worker/src/contracts/WorkerTypes';
import {WORKER_CRON_STALE_AFTER_MS, type WorkerHeartbeat, type WorkerHeartbeatSignal} from './WorkerHeartbeat';
import type {WorkerTaskName} from './WorkerLaneConfig';
import type {WorkerService} from './WorkerService';
const MAX_CATCHUP_SECONDS = 60;
interface CronDefinition {
id: string;
taskType: WorkerTaskName;
@@ -77,17 +80,35 @@ function matchesCronExpression(expression: string, date: Date): boolean {
);
}
function findLatestDueSecond(expression: string, fromSeconds: number, toSeconds: number): number | null {
for (let second = toSeconds; second >= fromSeconds; second--) {
if (matchesCronExpression(expression, new Date(second * 1000))) {
return second;
}
}
return null;
}
export class CronScheduler {
private readonly workerService: WorkerService;
private readonly logger: LoggerInterface;
private readonly kvClient: IKVProvider | null;
private readonly heartbeat: WorkerHeartbeat | null;
private readonly definitions: Map<string, CronDefinition> = new Map();
private intervalId: NodeJS.Timeout | null = null;
private lastTickSecond: number | null = null;
private heartbeatSignal: WorkerHeartbeatSignal | null = null;
constructor(workerService: WorkerService, logger: LoggerInterface, kvClient: IKVProvider | null = null) {
constructor(
workerService: WorkerService,
logger: LoggerInterface,
kvClient: IKVProvider | null = null,
heartbeat: WorkerHeartbeat | null = null,
) {
this.workerService = workerService;
this.logger = logger;
this.kvClient = kvClient;
this.heartbeat = heartbeat;
}
upsert(
@@ -111,6 +132,7 @@ export class CronScheduler {
if (this.intervalId !== null) {
return;
}
this.heartbeatSignal = this.heartbeat?.register('cron', WORKER_CRON_STALE_AFTER_MS) ?? null;
this.intervalId = setInterval(() => {
this.tick().catch((error) => {
this.logger.error({err: error}, 'Cron scheduler tick failed');
@@ -124,28 +146,43 @@ export class CronScheduler {
clearInterval(this.intervalId);
this.intervalId = null;
}
this.heartbeatSignal?.release();
this.heartbeatSignal = null;
this.lastTickSecond = null;
}
private async tick(): Promise<void> {
const now = new Date();
const nowSeconds = Math.floor(now.getTime() / 1000);
try {
await this.runDueDefinitions();
} finally {
this.heartbeatSignal?.report();
}
}
private async runDueDefinitions(): Promise<void> {
const nowSeconds = Math.floor(Date.now() / 1000);
const previousTickSecond = this.lastTickSecond;
this.lastTickSecond = nowSeconds;
const fromSeconds =
previousTickSecond === null || previousTickSecond >= nowSeconds
? nowSeconds
: Math.max(previousTickSecond + 1, nowSeconds - MAX_CATCHUP_SECONDS);
for (const def of this.definitions.values()) {
if (def.lastFired === nowSeconds) {
const dueSecond = findLatestDueSecond(def.cronExpression, fromSeconds, nowSeconds);
if (dueSecond === null || def.lastFired === dueSecond) {
continue;
}
if (matchesCronExpression(def.cronExpression, now)) {
def.lastFired = nowSeconds;
try {
const jobKey = `cron:${def.id}:${nowSeconds}`;
const acquired = await this.acquireEnqueueLease(jobKey);
if (!acquired) {
continue;
}
await this.workerService.addJob(def.taskType, def.payload, {jobKey, skipLedger: !def.ledger});
this.logger.debug({cronId: def.id, taskType: def.taskType}, 'Cron job fired');
} catch (error) {
this.logger.error({err: error, cronId: def.id, taskType: def.taskType}, 'Failed to enqueue cron job');
def.lastFired = dueSecond;
try {
const jobKey = `cron:${def.id}:${dueSecond}`;
const acquired = await this.acquireEnqueueLease(jobKey);
if (!acquired) {
continue;
}
await this.workerService.addJob(def.taskType, def.payload, {jobKey, skipLedger: !def.ledger});
this.logger.debug({cronId: def.id, taskType: def.taskType}, 'Cron job fired');
} catch (error) {
this.logger.error({err: error, cronId: def.id, taskType: def.taskType}, 'Failed to enqueue cron job');
}
}
}
@@ -131,7 +131,7 @@ export class JetStreamWorkerQueue {
}
const jsm = await this.connectionManager.getJetStreamManager();
for (const lane of lanes) {
const filterSubjects = lane.taskTypes.map((t) => `${SUBJECT_PREFIX}${t}`);
const filterSubjects = [...lane.taskTypes, ...lane.retiredTaskTypes].map((t) => `${SUBJECT_PREFIX}${t}`);
const config = {
durable_name: lane.consumerName,
ack_policy: AckPolicy.Explicit,
@@ -352,7 +352,7 @@ export async function initializeWorkerDependencies(snowflakeService: ISnowflakeS
export async function shutdownWorkerDependencies(deps: WorkerDependencies): Promise<void> {
Logger.info('Shutting down worker dependencies...');
if (deps.voiceReconciliationWorker !== null) {
deps.voiceReconciliationWorker.stop();
await deps.voiceReconciliationWorker.stop();
}
Logger.info('Worker dependencies shut down successfully');
}
@@ -0,0 +1,134 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import {writeFileSync} from 'node:fs';
import type {ILogger} from '../ILogger';
const WORKER_HEARTBEAT_PATH = '/tmp/fluxer-worker-heartbeat';
export const WORKER_HEARTBEAT_WRITE_INTERVAL_MS = 5000;
export const WORKER_LANE_HEARTBEAT_INTERVAL_MS = 5000;
export const WORKER_LANE_STALE_AFTER_MS = 30000;
export const WORKER_CRON_STALE_AFTER_MS = 90000;
export interface WorkerHeartbeatSignal {
report(): void;
release(): void;
}
interface WorkerHeartbeatOptions {
logger: Pick<ILogger, 'info' | 'error'>;
path?: string;
intervalMs?: number;
now?: () => number;
write?: (path: string, contents: string) => void;
}
interface WorkerHeartbeatComponent {
staleAfterMs: number;
lastReportedAt: number;
}
export class WorkerHeartbeat {
private readonly logger: Pick<ILogger, 'info' | 'error'>;
private readonly path: string;
private readonly intervalMs: number;
private readonly now: () => number;
private readonly write: (path: string, contents: string) => void;
private readonly components = new Map<string, WorkerHeartbeatComponent>();
private intervalId: NodeJS.Timeout | null = null;
private stalled = false;
constructor(options: WorkerHeartbeatOptions) {
this.logger = options.logger;
this.path = options.path ?? WORKER_HEARTBEAT_PATH;
this.intervalMs = options.intervalMs ?? WORKER_HEARTBEAT_WRITE_INTERVAL_MS;
this.now = options.now ?? Date.now;
this.write = options.write ?? ((path, contents) => writeFileSync(path, contents));
}
getPath(): string {
return this.path;
}
register(name: string, staleAfterMs: number): WorkerHeartbeatSignal {
const component: WorkerHeartbeatComponent = {staleAfterMs, lastReportedAt: this.now()};
this.components.set(name, component);
return {
report: () => {
component.lastReportedAt = this.now();
},
release: () => {
if (this.components.get(name) === component) {
this.components.delete(name);
}
},
};
}
stalledComponents(): Array<string> {
const at = this.now();
const stalled: Array<string> = [];
for (const [name, component] of this.components) {
if (at - component.lastReportedAt > component.staleAfterMs) {
stalled.push(name);
}
}
return stalled;
}
writeOnce(): boolean {
const stalled = this.stalledComponents();
if (stalled.length > 0) {
if (!this.stalled) {
this.stalled = true;
this.logger.error(
{components: stalled, path: this.path},
'Worker heartbeat stalled, the container will report unhealthy',
);
}
return false;
}
try {
this.write(this.path, this.snapshot());
} catch (error) {
this.logger.error({err: error, path: this.path}, 'Failed to write the worker heartbeat file');
return false;
}
if (this.stalled) {
this.stalled = false;
this.logger.info({path: this.path}, 'Worker heartbeat recovered');
}
return true;
}
start(): void {
if (this.intervalId !== null) {
return;
}
this.writeOnce();
this.intervalId = setInterval(() => {
this.writeOnce();
}, this.intervalMs);
this.logger.info(
{path: this.path, intervalMs: this.intervalMs, components: [...this.components.keys()]},
'Worker heartbeat started',
);
}
stop(): void {
if (this.intervalId !== null) {
clearInterval(this.intervalId);
this.intervalId = null;
}
}
private snapshot(): string {
const at = this.now();
return JSON.stringify({
at: new Date(at).toISOString(),
components: [...this.components].map(([name, component]) => ({
name,
ageMs: at - component.lastReportedAt,
})),
});
}
}
@@ -5,6 +5,7 @@ import type {APIWorkerLaneName, APIWorkerMode} from '../config/APIConfig';
interface LaneSettings {
readonly consumerName: string;
readonly tasks: ReadonlyArray<string>;
readonly retiredTasks: ReadonlyArray<string>;
readonly concurrency: number;
readonly maxAckPending: number;
readonly ackWaitMs: number;
@@ -15,6 +16,7 @@ const LANE_CONFIG = {
realtime: {
consumerName: 'workers_realtime',
tasks: ['handleMentions', 'handleMentionChunk'] as const,
retiredTasks: [],
concurrency: 10,
maxAckPending: 50,
ackWaitMs: 15000,
@@ -23,6 +25,7 @@ const LANE_CONFIG = {
unfurl: {
consumerName: 'workers_unfurl',
tasks: ['extractEmbeds'] as const,
retiredTasks: [],
concurrency: 20,
maxAckPending: 200,
ackWaitMs: 30000,
@@ -54,6 +57,7 @@ const LANE_CONFIG = {
'bulkAddGuildMembers',
'bulkBanFileShas',
] as const,
retiredTasks: ['sendScheduledMessage'],
concurrency: 8,
maxAckPending: 50,
ackWaitMs: 60000,
@@ -79,6 +83,7 @@ const LANE_CONFIG = {
'syncFileShaBlocklists',
'flushUserActivityBuffer',
] as const,
retiredTasks: [],
concurrency: 12,
maxAckPending: 100,
ackWaitMs: 120000,
@@ -92,6 +97,7 @@ interface WorkerLaneDefinition {
name: APIWorkerLaneName;
consumerName: string;
taskTypes: ReadonlyArray<WorkerTaskName>;
retiredTaskTypes: ReadonlyArray<string>;
concurrency: number;
maxAckPending: number;
ackWaitMs: number;
@@ -111,6 +117,7 @@ function makeLane(name: APIWorkerLaneName): WorkerLaneDefinition {
name,
consumerName: config.consumerName,
taskTypes: config.tasks,
retiredTaskTypes: config.retiredTasks,
concurrency: config.concurrency,
maxAckPending: config.maxAckPending,
ackWaitMs: config.ackWaitMs,
@@ -160,6 +167,7 @@ function resolveSingleTaskLane(taskName: WorkerTaskName | undefined): WorkerLane
name: parentLane.name,
consumerName: `worker_${taskName}`,
taskTypes: [taskName],
retiredTaskTypes: [],
concurrency: parentLane.concurrency,
maxAckPending: parentLane.maxAckPending,
ackWaitMs: parentLane.ackWaitMs,
@@ -231,6 +239,12 @@ function validateLaneCompleteness(registeredTasks: Record<string, unknown>): voi
if (missingFromRegistry.length > 0) {
errors.push(`Lane tasks not found in registry: ${missingFromRegistry.join(', ')}`);
}
const retiredCollisions = WORKER_LANES.flatMap<string>((lane) => [...lane.retiredTaskTypes]).filter((task) =>
registeredTaskNames.has(task),
);
if (retiredCollisions.length > 0) {
errors.push(`Retired tasks registered again: ${retiredCollisions.join(', ')}`);
}
if (errors.length > 0) {
throw new Error(`Worker lane configuration mismatch:\n${errors.join('\n')}`);
}
+19 -11
View File
@@ -23,12 +23,14 @@ import {CronScheduler} from './CronScheduler';
import {JetStreamWorkerQueue} from './JetStreamWorkerQueue';
import {clearWorkerDependencies, setWorkerDependencies} from './WorkerContext';
import {initializeWorkerDependencies, shutdownWorkerDependencies, type WorkerDependencies} from './WorkerDependencies';
import {WorkerHeartbeat} from './WorkerHeartbeat';
import {
resolveCronSchedulerEnabled,
resolveWorkerLanes,
validateLaneCompleteness,
type WorkerLaneDefinition,
} from './WorkerLaneConfig';
import {createWorkerProcessErrorHandler} from './WorkerProcessErrorHandler';
import {WorkerQueueOverflowError} from './WorkerQueueOverflowError';
import {WorkerRunner} from './WorkerRunner';
import {WorkerService} from './WorkerService';
@@ -88,6 +90,7 @@ export async function startWorkerMain(): Promise<void> {
let snowflakeService: ISnowflakeService | null = null;
let dependencies: WorkerDependencies | null = null;
let cron: CronScheduler | null = null;
const heartbeat = new WorkerHeartbeat({logger: Logger});
const runners: Array<WorkerRunner> = [];
let searchInitialized = false;
let shuttingDown = false;
@@ -106,6 +109,7 @@ export async function startWorkerMain(): Promise<void> {
}
shuttingDown = true;
Logger.info('Shutting down worker backend...');
await cleanupStep('heartbeat', () => heartbeat.stop());
await cleanupStep('cron', () => cron?.stop());
await cleanupStep('runners', async () => {
await Promise.all(runners.map((runner) => runner.stop()));
@@ -228,7 +232,7 @@ export async function startWorkerMain(): Promise<void> {
}
}
}
cron = new CronScheduler(workerService, Logger, dependencies.kvClient);
cron = new CronScheduler(workerService, Logger, dependencies.kvClient, heartbeat);
registerCronJobs(cron);
for (const lane of activeWorkerLanes) {
const laneTasks: Record<string, WorkerTaskHandler> = {};
@@ -240,6 +244,7 @@ export async function startWorkerMain(): Promise<void> {
}
const runner = new WorkerRunner({
tasks: laneTasks,
retiredTaskTypes: lane.retiredTaskTypes,
queue,
consumerName: lane.consumerName,
laneName: lane.name,
@@ -247,6 +252,7 @@ export async function startWorkerMain(): Promise<void> {
concurrency: lane.concurrency,
maxDeliver: lane.maxDeliver,
ackWaitMs: lane.ackWaitMs,
heartbeat,
});
runners.push(runner);
}
@@ -277,18 +283,20 @@ export async function startWorkerMain(): Promise<void> {
{lanes: activeWorkerLanes.map((l) => `${l.name}(${l.concurrency})`).join(', ')},
'Worker runners started',
);
heartbeat.start();
setupGracefulShutdown(shutdown, {logger: Logger, timeoutMs: 30000});
process.on('uncaughtException', async (error) => {
Logger.error({err: error}, 'Uncaught Exception');
setTimeout(() => process.exit(1), ms('5 seconds')).unref();
await shutdown();
process.exit(1);
const handleProcessError = createWorkerProcessErrorHandler({
logger: Logger,
shutdown,
exit: (code) => {
process.exit(code);
},
});
process.on('unhandledRejection', async (reason: unknown) => {
Logger.error({err: reason}, 'Unhandled Rejection at Promise');
setTimeout(() => process.exit(1), ms('5 seconds')).unref();
await shutdown();
process.exit(1);
process.on('uncaughtException', (error) => {
void handleProcessError('uncaughtException', error);
});
process.on('unhandledRejection', (reason: unknown) => {
void handleProcessError('unhandledRejection', reason);
});
} catch (error: unknown) {
Logger.error({err: error}, 'Failed to start worker backend');
@@ -0,0 +1,44 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import {ms} from 'itty-time';
import {isTransientDatabaseError} from '../database/TransientDatabaseError';
import type {ILogger} from '../ILogger';
export type WorkerProcessErrorSource = 'uncaughtException' | 'unhandledRejection';
type WorkerProcessErrorHandler = (source: WorkerProcessErrorSource, error: unknown) => Promise<void>;
interface WorkerProcessErrorHandlerOptions {
logger: Pick<ILogger, 'error' | 'warn'>;
shutdown: () => Promise<void>;
exit: (code: number) => void;
forceExitDelayMs?: number;
}
const FATAL_MESSAGE: Record<WorkerProcessErrorSource, string> = {
uncaughtException: 'Uncaught Exception',
unhandledRejection: 'Unhandled Rejection at Promise',
};
export function createWorkerProcessErrorHandler(options: WorkerProcessErrorHandlerOptions): WorkerProcessErrorHandler {
const forceExitDelayMs = options.forceExitDelayMs ?? ms('5 seconds');
return async (source, error) => {
if (isTransientDatabaseError(error)) {
options.logger.warn(
{err: error, source},
'Transient database connection error reached the worker process, keeping the worker running',
);
return;
}
options.logger.error({err: error, source}, FATAL_MESSAGE[source]);
const forceExit = setTimeout(() => options.exit(1), forceExitDelayMs);
forceExit.unref();
try {
await options.shutdown();
} catch (shutdownError) {
options.logger.error({err: shutdownError, source}, 'Worker shutdown failed while handling a fatal error');
}
clearTimeout(forceExit);
options.exit(1);
};
}
+136 -13
View File
@@ -8,9 +8,17 @@ import type {IJobLedgerRepository} from '../jobs/IJobLedgerRepository';
import {Logger} from '../Logger';
import {getWorkerService} from '../middleware/ServiceRegistry';
import {isJsonRecord, parseJsonRecord} from '../utils/JsonBoundaryUtils';
import {
WORKER_LANE_HEARTBEAT_INTERVAL_MS,
WORKER_LANE_STALE_AFTER_MS,
type WorkerHeartbeat,
type WorkerHeartbeatSignal,
} from './WorkerHeartbeat';
const MAX_DLQ_PUBLISH_ATTEMPTS = 3;
const MIN_ACK_HEARTBEAT_MS = 1000;
const RESUBSCRIBE_DELAY_MS = 5000;
const RETIRED_TASK_REASON = 'task type retired';
interface WorkerRunnerJetStreamClient {
consumers: {
@@ -43,6 +51,7 @@ interface WorkerRunnerQueue {
interface WorkerRunnerOptions {
tasks: Record<string, WorkerTaskHandler>;
retiredTaskTypes?: ReadonlyArray<string>;
queue: WorkerRunnerQueue;
consumerName: string;
laneName: string;
@@ -51,10 +60,12 @@ interface WorkerRunnerOptions {
concurrency?: number;
maxDeliver?: number;
ackWaitMs?: number;
heartbeat?: WorkerHeartbeat;
}
export class WorkerRunner {
private readonly tasks: Record<string, WorkerTaskHandler>;
private readonly retiredTaskTypes: Set<string>;
private readonly queue: WorkerRunnerQueue;
private readonly consumerName: string;
private readonly laneName: string;
@@ -64,12 +75,17 @@ export class WorkerRunner {
private readonly ackWaitMs: number;
private readonly workerService: IWorkerService;
private readonly ledger: IJobLedgerRepository;
private readonly heartbeat: WorkerHeartbeat | null;
private heartbeatSignal: WorkerHeartbeatSignal | null = null;
private heartbeatTimer: ReturnType<typeof setInterval> | null = null;
private running = false;
private consumerMessages: ConsumerMessages | null = null;
private processingLoop: Promise<void> | null = null;
private readonly inFlightJobs = new Set<Promise<void>>();
constructor(options: WorkerRunnerOptions) {
this.tasks = options.tasks;
this.retiredTaskTypes = new Set(options.retiredTaskTypes ?? []);
this.queue = options.queue;
this.consumerName = options.consumerName;
this.laneName = options.laneName;
@@ -79,6 +95,7 @@ export class WorkerRunner {
this.ackWaitMs = options.ackWaitMs ?? 60000;
this.workerService = getWorkerService();
this.ledger = options.ledger;
this.heartbeat = options.heartbeat ?? null;
}
async start(): Promise<void> {
@@ -88,15 +105,10 @@ export class WorkerRunner {
}
this.running = true;
Logger.info({workerId: this.workerId, lane: this.laneName, concurrency: this.concurrency}, 'Worker starting');
const js = this.queue.getConnectionManager().getJetStreamClient();
const consumer = await js.consumers.get(this.queue.getStreamName(), this.consumerName);
const prefetch = Math.max(this.concurrency * 2, 16);
this.consumerMessages = await consumer.consume({
max_messages: prefetch,
idle_heartbeat: 5000,
});
this.processMessages().catch((error) => {
Logger.error({workerId: this.workerId, err: error}, 'Worker message processing failed unexpectedly');
this.startHeartbeat();
this.consumerMessages = await this.openConsumerMessages();
this.processingLoop = this.consumeUntilStopped(this.consumerMessages).finally(() => {
this.stopHeartbeatTicker();
});
}
@@ -109,14 +121,81 @@ export class WorkerRunner {
await this.consumerMessages.close();
this.consumerMessages = null;
}
if (this.processingLoop !== null) {
await this.processingLoop;
this.processingLoop = null;
}
this.stopHeartbeatTicker();
this.heartbeatSignal?.release();
this.heartbeatSignal = null;
Logger.info({workerId: this.workerId}, 'Worker stopped');
}
private async processMessages(): Promise<void> {
if (this.consumerMessages === null) {
private startHeartbeat(): void {
if (this.heartbeat === null) {
return;
}
for await (const msg of this.consumerMessages) {
this.heartbeatSignal = this.heartbeat.register(`lane:${this.laneName}`, WORKER_LANE_STALE_AFTER_MS);
this.heartbeatTimer = setInterval(() => {
this.heartbeatSignal?.report();
}, WORKER_LANE_HEARTBEAT_INTERVAL_MS);
}
private stopHeartbeatTicker(): void {
if (this.heartbeatTimer !== null) {
clearInterval(this.heartbeatTimer);
this.heartbeatTimer = null;
}
}
private async openConsumerMessages(): Promise<ConsumerMessages> {
const js = this.queue.getConnectionManager().getJetStreamClient();
const consumer = await js.consumers.get(this.queue.getStreamName(), this.consumerName);
const prefetch = Math.max(this.concurrency * 2, 16);
return await consumer.consume({
max_messages: prefetch,
idle_heartbeat: 5000,
});
}
private async consumeUntilStopped(initialMessages: ConsumerMessages): Promise<void> {
let messages: ConsumerMessages | null = initialMessages;
while (this.running) {
if (messages === null) {
await new Promise((resolve) => setTimeout(resolve, RESUBSCRIBE_DELAY_MS));
if (!this.running) {
break;
}
try {
messages = await this.openConsumerMessages();
this.consumerMessages = messages;
} catch (error) {
Logger.error(
{workerId: this.workerId, lane: this.laneName, err: error},
'Failed to resubscribe the worker consumer',
);
}
continue;
}
try {
await this.processMessages(messages);
} catch (error) {
Logger.error({workerId: this.workerId, err: error}, 'Worker message processing failed unexpectedly');
}
messages = null;
this.consumerMessages = null;
if (this.running) {
Logger.error(
{workerId: this.workerId, lane: this.laneName},
'Worker message stream ended while running, resubscribing',
);
}
}
Logger.info({workerId: this.workerId}, 'Worker message iterator ended');
}
private async processMessages(consumerMessages: ConsumerMessages): Promise<void> {
for await (const msg of consumerMessages) {
if (!this.running) {
break;
}
@@ -154,10 +233,13 @@ export class WorkerRunner {
this.inFlightJobs.add(jobPromise);
}
await Promise.allSettled(this.inFlightJobs);
Logger.info({workerId: this.workerId}, 'Worker message iterator ended');
}
protected async processJob(taskType: string, msg: JsMsg): Promise<boolean> {
if (this.retiredTaskTypes.has(taskType)) {
await this.retireJob(taskType, msg);
return false;
}
const task = this.tasks[taskType];
if (!task) {
Logger.error({taskType, seq: msg.seq}, 'Unknown task type, terminating message');
@@ -324,6 +406,47 @@ export class WorkerRunner {
}
}
private async retireJob(taskType: string, msg: JsMsg): Promise<void> {
const decoded = parseJsonRecord(new TextDecoder().decode(msg.data));
const jobPayload = decoded && isJsonRecord(decoded.payload) ? decoded.payload : {};
const runAt = decoded && typeof decoded.run_at === 'string' ? decoded.run_at : undefined;
let ledgerJobId: bigint | null = null;
const embedded = jobPayload['__jobId'];
if (typeof embedded === 'string') {
try {
ledgerJobId = BigInt(embedded);
} catch {
ledgerJobId = null;
}
delete jobPayload['__jobId'];
}
Logger.warn(
{taskType, seq: msg.seq, jobId: ledgerJobId?.toString()},
'Retired task type from an older release, moving to dead-letter queue',
);
try {
await this.queue.publishToDlq(taskType, jobPayload, {
originalSeq: msg.seq,
errorMessage: RETIRED_TASK_REASON,
deliveryCount: msg.info.deliveryCount,
lane: this.laneName,
runAt,
});
} catch (error) {
Logger.error({taskType, seq: msg.seq, err: error}, 'Failed to dead-letter a retired job');
msg.nak(5000);
return;
}
if (ledgerJobId !== null) {
try {
await this.ledger.markDeadletter(ledgerJobId, RETIRED_TASK_REASON);
} catch (err) {
Logger.warn({err, jobId: ledgerJobId.toString()}, 'Ledger markDeadletter failed');
}
}
msg.term(RETIRED_TASK_REASON);
}
private startAckHeartbeat(taskType: string, msg: JsMsg): ReturnType<typeof setInterval> {
const heartbeat = setInterval(
() => {
@@ -29,7 +29,7 @@ export async function processExpiredAttachments(now = new Date()): Promise<void>
for (const row of expired) {
const metadata = await repo.fetchById(row.attachment_id);
if (!metadata) {
await repo.deleteRecords({
await repo.deleteExpiryRecord({
expiry_bucket: row.expiry_bucket,
expires_at: row.expires_at,
attachment_id: row.attachment_id,
@@ -38,7 +38,7 @@ export async function processExpiredAttachments(now = new Date()): Promise<void>
continue;
}
if (metadata.expires_at > row.expires_at) {
await repo.deleteRecords({
await repo.deleteExpiryRecord({
expiry_bucket: row.expiry_bucket,
expires_at: row.expires_at,
attachment_id: row.attachment_id,
@@ -1,6 +1,7 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import type {WorkerTaskHandler} from '@pkgs/worker/src/contracts/WorkerTask';
import {seconds} from 'itty-time';
import {z} from 'zod';
import type {MessageID, UserID} from '../../BrandedTypes';
import {createChannelID} from '../../BrandedTypes';
@@ -15,6 +16,7 @@ const PayloadSchema = z.object({
channelCount: z.number().optional(),
});
const BULK_BATCH_SIZE = 5000;
const COMPLETION_KEY_TTL_SECONDS = seconds('1 day');
function getMessageIndexServices(): Array<IMessageSearchService> {
const services: Array<IMessageSearchService> = [];
const defaultService = getMessageSearchService();
@@ -64,7 +66,9 @@ const indexChannelMessages: WorkerTaskHandler = async (payload) => {
}
}
if (validated.completionKey && validated.channelCount) {
const completed = await kvClient.incr(validated.completionKey);
await kvClient.sadd(validated.completionKey, validated.channelId);
await kvClient.expire(validated.completionKey, COMPLETION_KEY_TTL_SECONDS);
const completed = await kvClient.scard(validated.completionKey);
if (completed >= validated.channelCount) {
try {
await Promise.all(searchServices.map((s) => s.refreshIndex()));
@@ -32,13 +32,14 @@ const processAssetDeletionQueue: WorkerTaskHandler = async (_payload, _helpers)
return;
}
Logger.info({queueSize}, 'Starting asset deletion queue processing');
const maxItems = Math.min(MAX_ITEMS_PER_RUN, queueSize);
let totalProcessed = 0;
let totalDeleted = 0;
let totalSkipped = 0;
let totalFailed = 0;
let totalCdnPurged = 0;
while (totalProcessed < MAX_ITEMS_PER_RUN) {
const batch = await assetDeletionQueue.getBatch(BATCH_SIZE);
while (totalProcessed < maxItems) {
const batch = await assetDeletionQueue.getBatch(Math.min(BATCH_SIZE, maxItems - totalProcessed));
if (batch.length === 0) {
break;
}
@@ -26,6 +26,7 @@ interface ReconcileResult {
const MAX_USERS_PER_RUN = 250;
const RETRY_DELAY_MS = 5 * 60 * 1000;
const CLAIM_LEASE_MS = 10 * 60 * 1000;
function getStripeSubscriptionCustomerId(subscription: Stripe.Subscription): string | null {
if (!subscription.customer) {
@@ -305,14 +306,18 @@ const processPremiumStateReconciliationQueue: WorkerTaskHandler = async (_payloa
let strippedNoSubscriptionCount = 0;
let failedCount = 0;
let requeuedCount = 0;
let claimedElsewhereCount = 0;
for (const userId of readyUserIds) {
const claimedAt = Date.now();
let claimed = false;
try {
await premiumStateReconciliationQueueService.removeUser(userId);
claimed = await premiumStateReconciliationQueueService.claimUser(userId, claimedAt, claimedAt + CLAIM_LEASE_MS);
} catch (error) {
Logger.warn(
{error, userId: userId.toString()},
'Failed to remove user from premium reconciliation queue before processing',
);
Logger.warn({error, userId: userId.toString()}, 'Failed to claim premium reconciliation queue entry');
}
if (!claimed) {
claimedElsewhereCount += 1;
continue;
}
try {
const result = await reconcileUserPremiumStateFromStripe({
@@ -344,6 +349,7 @@ const processPremiumStateReconciliationQueue: WorkerTaskHandler = async (_payloa
} else {
skippedCount += 1;
}
await premiumStateReconciliationQueueService.removeUser(userId);
} catch (error) {
failedCount += 1;
Logger.error(
@@ -371,6 +377,7 @@ const processPremiumStateReconciliationQueue: WorkerTaskHandler = async (_payloa
strippedNoSubscriptionCount,
failedCount,
requeuedCount,
claimedElsewhereCount,
},
'Finished processing premium reconciliation queue',
);
@@ -1,10 +1,12 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import {UserFlags} from '@fluxer/constants/src/UserConstants';
import type {WorkerTaskHandler} from '@pkgs/worker/src/contracts/WorkerTask';
import {createUserID} from '../../BrandedTypes';
import {Logger} from '../../Logger';
import {resolvePendingDeletionReasonCode} from '../../user/services/PendingDeletionCoordinator';
import {
isPendingDeletionBlocked,
resolvePendingDeletionReasonCode,
} from '../../user/services/PendingDeletionCoordinator';
import {getWorkerDependencies} from '../WorkerContext';
const userProcessPendingDeletions: WorkerTaskHandler = async (_payload, helpers) => {
@@ -18,7 +20,7 @@ const userProcessPendingDeletions: WorkerTaskHandler = async (_payload, helpers)
const lockToken = await deletionQueueService.acquireRebuildLock();
if (lockToken) {
try {
await deletionQueueService.rebuildState();
await deletionQueueService.rebuildState(lockToken);
await deletionQueueService.releaseRebuildLock(lockToken);
} catch (error) {
await deletionQueueService.releaseRebuildLock(lockToken);
@@ -42,12 +44,9 @@ const userProcessPendingDeletions: WorkerTaskHandler = async (_payload, helpers)
await deletionQueueService.removeFromQueue(userId);
continue;
}
if (user.isBot) {
Logger.info({userId}, 'User is a bot, skipping deletion');
continue;
}
if (user.flags & UserFlags.APP_STORE_REVIEWER) {
Logger.info({userId}, 'User is an app store reviewer, skipping deletion');
if (isPendingDeletionBlocked(user)) {
Logger.info({userId}, 'User is not eligible for automated deletion, removing from KV');
await deletionQueueService.removeFromQueue(userId);
continue;
}
const deletionReasonCode = resolvePendingDeletionReasonCode(user, deletion.deletionReasonCode);
@@ -0,0 +1,94 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import type {LoggerInterface} from '@fluxer/logger/src/LoggerInterface';
import type {IKVProvider} from '@pkgs/kv_client/src/IKVProvider';
import {afterEach, describe, expect, it, vi} from 'vitest';
import {CronScheduler} from '../CronScheduler';
import type {WorkerService} from '../WorkerService';
function createLogger(): LoggerInterface {
const logger = {
trace: vi.fn(),
debug: vi.fn(),
info: vi.fn(),
warn: vi.fn(),
error: vi.fn(),
child: () => logger,
};
return logger as unknown as LoggerInterface;
}
function createScheduler(): {
scheduler: CronScheduler;
addJob: ReturnType<typeof vi.fn>;
setnx: ReturnType<typeof vi.fn>;
} {
const addJob = vi.fn().mockResolvedValue(1n);
const setnx = vi.fn().mockResolvedValue(true);
const workerService = {addJob} as unknown as WorkerService;
const kvClient = {setnx} as unknown as IKVProvider;
return {scheduler: new CronScheduler(workerService, createLogger(), kvClient), addJob, setnx};
}
describe('CronScheduler', () => {
afterEach(() => {
vi.useRealTimers();
});
it('fires a schedule whose second was skipped by a stalled tick', async () => {
vi.useFakeTimers();
vi.setSystemTime(new Date('2026-01-01T00:00:00.000Z'));
const {scheduler, addJob, setnx} = createScheduler();
scheduler.upsert('expireAttachments', 'expireAttachments', {}, '5 * * * * *', {ledger: false});
scheduler.start();
await vi.advanceTimersByTimeAsync(3000);
expect(addJob).not.toHaveBeenCalled();
vi.setSystemTime(new Date('2026-01-01T00:00:10.400Z'));
await vi.advanceTimersByTimeAsync(2000);
scheduler.stop();
const skippedSecond = Math.floor(Date.parse('2026-01-01T00:00:05.000Z') / 1000);
expect(addJob).toHaveBeenCalledTimes(1);
expect(addJob).toHaveBeenCalledWith(
'expireAttachments',
{},
{
jobKey: `cron:expireAttachments:${skippedSecond}`,
skipLedger: true,
},
);
expect(setnx).toHaveBeenCalledTimes(1);
});
it('fires a skipped schedule once instead of once per skipped second', async () => {
vi.useFakeTimers();
vi.setSystemTime(new Date('2026-01-01T00:00:00.000Z'));
const {scheduler, addJob} = createScheduler();
scheduler.upsert('flushUserActivityBuffer', 'flushUserActivityBuffer', {}, '*/2 * * * * *', {ledger: false});
scheduler.start();
await vi.advanceTimersByTimeAsync(1000);
addJob.mockClear();
vi.setSystemTime(new Date('2026-01-01T00:00:20.400Z'));
await vi.advanceTimersByTimeAsync(1000);
scheduler.stop();
expect(addJob).toHaveBeenCalledTimes(1);
});
it('does not replay schedules from before the scheduler started', async () => {
vi.useFakeTimers();
vi.setSystemTime(new Date('2026-01-01T00:00:30.000Z'));
const {scheduler, addJob} = createScheduler();
scheduler.upsert('expireAttachments', 'expireAttachments', {}, '5 * * * * *', {ledger: false});
scheduler.start();
await vi.advanceTimersByTimeAsync(2000);
scheduler.stop();
expect(addJob).not.toHaveBeenCalled();
});
});
@@ -0,0 +1,91 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import {ms} from 'itty-time';
import {afterEach, describe, expect, it} from 'vitest';
import {AttachmentDecayRepository} from '../../attachment/AttachmentDecayRepository';
import {createAttachmentID, createChannelID, createMessageID} from '../../BrandedTypes';
import type {IAssetDeletionQueue, QueuedAssetDeletion} from '../../infrastructure/IAssetDeletionQueue';
import type {InstanceConfigRepository} from '../../instance/InstanceConfigRepository';
import {getExpiryBucket} from '../../utils/AttachmentDecay';
import {processExpiredAttachments} from '../tasks/ExpireAttachments';
import {clearWorkerDependencies, setWorkerDependenciesForTest} from '../WorkerContext';
const ATTACHMENT_ID = createAttachmentID(9001n);
const CHANNEL_ID = createChannelID(9002n);
const MESSAGE_ID = createMessageID(9003n);
const FILENAME = 'decaying.png';
const UPLOADED_AT = new Date('2026-01-01T00:00:00.000Z');
const FIRST_EXPIRY = new Date('2026-01-31T00:00:00.000Z');
const EXTENDED_EXPIRY = new Date('2026-02-25T00:00:00.000Z');
function createQueue(): {queue: IAssetDeletionQueue; queued: Array<QueuedAssetDeletion>} {
const queued: Array<QueuedAssetDeletion> = [];
const queue = {
async queueDeletion(item: Omit<QueuedAssetDeletion, 'queuedAt' | 'retryCount'>) {
queued.push({...item});
},
} as unknown as IAssetDeletionQueue;
return {queue, queued};
}
function installDependencies(queue: IAssetDeletionQueue): void {
setWorkerDependenciesForTest({
assetDeletionQueue: queue,
instanceConfigRepository: {
async getEffectiveAttachmentDecayConfig() {
return {enabled: true};
},
} as unknown as InstanceConfigRepository,
});
}
async function writeDecayRecord(repository: AttachmentDecayRepository, expiresAt: Date): Promise<void> {
await repository.upsert({
attachment_id: ATTACHMENT_ID,
channel_id: CHANNEL_ID,
message_id: MESSAGE_ID,
filename: FILENAME,
size_bytes: 1024n,
uploaded_at: UPLOADED_AT,
expires_at: expiresAt,
last_accessed_at: UPLOADED_AT,
cost: 1,
lifetime_days: 30,
status: null,
expiry_bucket: getExpiryBucket(expiresAt),
});
}
describe('processExpiredAttachments', () => {
afterEach(() => {
clearWorkerDependencies();
});
it('keeps the decay record when it clears a superseded expiry row', async () => {
const repository = new AttachmentDecayRepository();
const {queue} = createQueue();
installDependencies(queue);
await writeDecayRecord(repository, FIRST_EXPIRY);
await writeDecayRecord(repository, EXTENDED_EXPIRY);
await processExpiredAttachments(new Date(FIRST_EXPIRY.getTime() + ms('1 day')));
const record = await repository.fetchById(ATTACHMENT_ID);
expect(record?.expires_at.toISOString()).toBe(EXTENDED_EXPIRY.toISOString());
expect(await repository.fetchExpiredByBucket(getExpiryBucket(FIRST_EXPIRY), EXTENDED_EXPIRY, 10)).toHaveLength(0);
});
it('still queues the asset once the extended expiry passes', async () => {
const repository = new AttachmentDecayRepository();
const {queue, queued} = createQueue();
installDependencies(queue);
await writeDecayRecord(repository, FIRST_EXPIRY);
await writeDecayRecord(repository, EXTENDED_EXPIRY);
await processExpiredAttachments(new Date(FIRST_EXPIRY.getTime() + ms('1 day')));
await processExpiredAttachments(new Date(EXTENDED_EXPIRY.getTime() + ms('1 day')));
expect(queued.map((item) => item.s3Key)).toEqual([`attachments/${CHANNEL_ID}/${ATTACHMENT_ID}/${FILENAME}`]);
expect(await repository.fetchById(ATTACHMENT_ID)).toBeNull();
});
});
@@ -0,0 +1,67 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import type {WorkerTaskHelpers} from '@pkgs/worker/src/contracts/WorkerTask';
import {afterEach, describe, expect, it, vi} from 'vitest';
import type {ChannelRepository} from '../../channel/ChannelRepository';
import {setInjectedSearchProvider} from '../../SearchFactory';
import type {IMessageSearchService} from '../../search/IMessageSearchService';
import type {ISearchProvider} from '../../search/ISearchProvider';
import {MockKVProvider} from '../../test/mocks/MockKVProvider';
import {NoopLogger} from '../../test/mocks/NoopLogger';
import type {UserRepository} from '../../user/repositories/UserRepository';
import indexChannelMessages from '../tasks/IndexChannelMessages';
import {clearWorkerDependencies, setWorkerDependenciesForTest} from '../WorkerContext';
const COMPLETION_KEY = 'bulk_reindex:7000:remaining';
const HELPERS = {logger: new NoopLogger()} as unknown as WorkerTaskHelpers;
function createHarness(): {kvClient: MockKVProvider; refreshIndex: ReturnType<typeof vi.fn>} {
const kvClient = new MockKVProvider();
const refreshIndex = vi.fn().mockResolvedValue(undefined);
const messageSearchService = {
bulkIndexMessages: async () => {},
refreshIndex,
} as unknown as IMessageSearchService;
setInjectedSearchProvider({
getMessageSearchService: () => messageSearchService,
} as unknown as ISearchProvider);
setWorkerDependenciesForTest({
kvClient,
channelRepository: {
listMessages: async () => [],
findUnique: async () => null,
} as unknown as ChannelRepository,
userRepository: {listUsers: async () => []} as unknown as UserRepository,
});
return {kvClient, refreshIndex};
}
describe('indexChannelMessages', () => {
afterEach(() => {
clearWorkerDependencies();
setInjectedSearchProvider(undefined);
});
it('counts a redelivered channel once towards bulk reindex completion', async () => {
const {kvClient, refreshIndex} = createHarness();
const payload = {channelId: '5001', completionKey: COMPLETION_KEY, channelCount: 2};
await indexChannelMessages(payload, HELPERS);
await indexChannelMessages(payload, HELPERS);
expect(refreshIndex).not.toHaveBeenCalled();
await indexChannelMessages({...payload, channelId: '5002'}, HELPERS);
expect(refreshIndex).toHaveBeenCalledTimes(1);
expect(await kvClient.exists(COMPLETION_KEY)).toBe(0);
});
it('expires the completion key when the reindex never finishes', async () => {
const {kvClient} = createHarness();
await indexChannelMessages({channelId: '5001', completionKey: COMPLETION_KEY, channelCount: 2}, HELPERS);
expect(await kvClient.ttl(COMPLETION_KEY)).toBeGreaterThan(0);
});
});
@@ -4,6 +4,7 @@ import type {JetStreamConnectionManager} from '@pkgs/nats/src/JetStreamConnectio
import {DiscardPolicy, NatsError, RetentionPolicy, StorageType, type StreamConfig} from 'nats';
import {describe, expect, it} from 'vitest';
import {JetStreamWorkerQueue} from '../JetStreamWorkerQueue';
import {WORKER_LANES} from '../WorkerLaneConfig';
import {WorkerQueueOverflowError} from '../WorkerQueueOverflowError';
const EXPECTED_LIMITS = {
@@ -26,6 +27,11 @@ const LEGACY_CONFIG = {
discard_new_per_subject: false,
} as unknown as StreamConfig;
interface ConsumerAddConfig {
durable_name: string;
filter_subjects: Array<string>;
}
function streamLimitError(description: string): NatsError {
const error = new NatsError('503', '503');
error.api_error = {code: 503, err_code: 10077, description};
@@ -50,12 +56,26 @@ function createQueue(params: {
existing?: StreamConfig | null;
updateError?: Error;
publish?: (subject: string) => {seq: number};
}): {queue: JetStreamWorkerQueue; added: Array<Partial<StreamConfig>>; updated: Array<Partial<StreamConfig>>} {
}): {
queue: JetStreamWorkerQueue;
added: Array<Partial<StreamConfig>>;
updated: Array<Partial<StreamConfig>>;
consumerAdds: Array<ConsumerAddConfig>;
} {
const added: Array<Partial<StreamConfig>> = [];
const updated: Array<Partial<StreamConfig>> = [];
const consumerAdds: Array<ConsumerAddConfig> = [];
const connectionManager = {
getJetStreamManager: () =>
Promise.resolve({
consumers: {
add: (_stream: string, config: ConsumerAddConfig) => {
consumerAdds.push(config);
return Promise.resolve({});
},
delete: () => Promise.resolve(true),
info: () => Promise.reject(new Error('consumer not found')),
},
streams: {
info: () => {
if (!params.existing) {
@@ -83,7 +103,7 @@ function createQueue(params: {
},
}),
} as unknown as JetStreamConnectionManager;
return {queue: new JetStreamWorkerQueue(connectionManager), added, updated};
return {queue: new JetStreamWorkerQueue(connectionManager), added, updated, consumerAdds};
}
describe('jobs stream limits', () => {
@@ -143,3 +163,27 @@ describe('jobs stream enqueue shedding', () => {
await expect(queue.enqueue('extractEmbeds', {})).rejects.toBe(failure);
});
});
describe('lane consumer filters', () => {
it('keeps consuming retired subjects so legacy jobs are drained instead of orphaned', async () => {
const {queue, consumerAdds} = createQueue({existing: LEGACY_CONFIG});
await queue.ensureConsumers(WORKER_LANES);
const lifecycle = consumerAdds.find((config) => config.durable_name === 'workers_lifecycle');
expect(lifecycle?.filter_subjects).toContain('jobs.sendSystemDm');
expect(lifecycle?.filter_subjects).toContain('jobs.sendScheduledMessage');
});
it('leaves lanes without retired tasks filtering only their own subjects', async () => {
const {queue, consumerAdds} = createQueue({existing: LEGACY_CONFIG});
await queue.ensureConsumers(WORKER_LANES);
const unfurl = consumerAdds.find((config) => config.durable_name === 'workers_unfurl');
expect(unfurl?.filter_subjects).toEqual(['jobs.extractEmbeds']);
});
it('never claims the same subject from two lane consumers', async () => {
const {queue, consumerAdds} = createQueue({existing: LEGACY_CONFIG});
await queue.ensureConsumers(WORKER_LANES);
const allSubjects = consumerAdds.flatMap((config) => config.filter_subjects);
expect(new Set(allSubjects).size).toBe(allSubjects.length);
});
});
@@ -0,0 +1,77 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import type {WorkerTaskHelpers} from '@pkgs/worker/src/contracts/WorkerTask';
import {afterEach, describe, expect, it, vi} from 'vitest';
import {AssetDeletionQueue} from '../../infrastructure/AssetDeletionQueue';
import {NoopPurgeQueue} from '../../infrastructure/BunnyPurgeQueue';
import type {IStorageService} from '../../infrastructure/IStorageService';
import {MockKVProvider} from '../../test/mocks/MockKVProvider';
import {NoopLogger} from '../../test/mocks/NoopLogger';
import processAssetDeletionQueue from '../tasks/ProcessAssetDeletionQueue';
import {clearWorkerDependencies, setWorkerDependenciesForTest} from '../WorkerContext';
const HELPERS = {logger: new NoopLogger()} as unknown as WorkerTaskHelpers;
function createHarness(deleteObject: IStorageService['deleteObject']) {
const kvClient = new MockKVProvider();
const assetDeletionQueue = new AssetDeletionQueue(kvClient);
setWorkerDependenciesForTest({
assetDeletionQueue,
purgeQueue: new NoopPurgeQueue(),
storageService: {deleteObject} as unknown as IStorageService,
userRepository: {findUnique: async () => null} as never,
guildRepository: {findUnique: async () => null, getMember: async () => null} as never,
});
return {assetDeletionQueue, kvClient};
}
async function queueAssets(assetDeletionQueue: AssetDeletionQueue, count: number): Promise<void> {
for (let index = 0; index < count; index++) {
await assetDeletionQueue.queueDeletion({
s3Key: `attachments/100/200/file-${index}.png`,
cdnUrl: null,
reason: 'test',
});
}
}
describe('processAssetDeletionQueue', () => {
afterEach(() => {
clearWorkerDependencies();
});
it('attempts each queued asset once per run when storage is failing', async () => {
const deleteObject = vi.fn().mockRejectedValue(new Error('s3 unavailable'));
const {assetDeletionQueue} = createHarness(deleteObject);
await queueAssets(assetDeletionQueue, 3);
await expect(processAssetDeletionQueue({}, HELPERS)).rejects.toThrow(/3 failures/);
expect(deleteObject).toHaveBeenCalledTimes(3);
expect(await assetDeletionQueue.getQueueSize()).toBe(3);
const remaining = await assetDeletionQueue.getBatch(10);
expect(remaining.map((item) => item.retryCount)).toEqual([1, 1, 1]);
});
it('attempts each queued asset once per run across batch boundaries', async () => {
const deleteObject = vi.fn().mockRejectedValue(new Error('s3 unavailable'));
const {assetDeletionQueue} = createHarness(deleteObject);
await queueAssets(assetDeletionQueue, 60);
await expect(processAssetDeletionQueue({}, HELPERS)).rejects.toThrow(/60 failures/);
expect(deleteObject).toHaveBeenCalledTimes(60);
expect(await assetDeletionQueue.getQueueSize()).toBe(60);
});
it('drains the queue when storage succeeds', async () => {
const deleteObject = vi.fn().mockResolvedValue(undefined);
const {assetDeletionQueue} = createHarness(deleteObject);
await queueAssets(assetDeletionQueue, 60);
await processAssetDeletionQueue({}, HELPERS);
expect(deleteObject).toHaveBeenCalledTimes(60);
expect(await assetDeletionQueue.getQueueSize()).toBe(0);
});
});
@@ -0,0 +1,154 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import type {WorkerTaskHelpers} from '@pkgs/worker/src/contracts/WorkerTask';
import type Stripe from 'stripe';
import {afterEach, describe, expect, test} from 'vitest';
import {createUserID} from '../../BrandedTypes';
import {PremiumStateReconciliationQueueService} from '../../infrastructure/PremiumStateReconciliationQueueService';
import {MockKVProvider} from '../../test/mocks/MockKVProvider';
import {NoopLogger} from '../../test/mocks/NoopLogger';
import type {UserRepository} from '../../user/repositories/UserRepository';
import processPremiumStateReconciliationQueue from '../tasks/ProcessPremiumStateReconciliationQueue';
import {clearWorkerDependencies, setWorkerDependenciesForTest} from '../WorkerContext';
const USER_ID = createUserID(834271905123471361n);
const ONE_HOUR_MS = 60 * 60 * 1000;
function createHelpers(): WorkerTaskHelpers {
return {
logger: new NoopLogger(),
jobId: 1n,
addJob: async () => 0n,
reportProgress: async () => {},
shouldCancel: async () => false,
setContextLink: async () => {},
};
}
function createQueueService(): PremiumStateReconciliationQueueService {
return new PremiumStateReconciliationQueueService(new MockKVProvider());
}
describe('processPremiumStateReconciliationQueue', () => {
afterEach(() => {
clearWorkerDependencies();
});
test('keeps the user in the queue when the worker dies mid-reconciliation', async () => {
const queueService = createQueueService();
await queueService.enqueueUser(USER_ID, new Date(Date.now() - 1000));
let signalReconcileStarted: () => void = () => {};
const reconcileStarted = new Promise<void>((resolve) => {
signalReconcileStarted = resolve;
});
const userRepository = {
findUnique: async () => {
signalReconcileStarted();
return await new Promise<never>(() => {});
},
} as unknown as UserRepository;
setWorkerDependenciesForTest({
premiumStateReconciliationQueueService: queueService,
stripe: {} as Stripe,
userRepository,
});
void processPremiumStateReconciliationQueue({}, createHelpers());
await reconcileStarted;
expect(await queueService.getQueueSize()).toBe(1);
expect(await queueService.getReadyUserIds(Date.now(), 10)).toEqual([]);
expect(await queueService.getReadyUserIds(Date.now() + ONE_HOUR_MS, 10)).toEqual([USER_ID]);
});
test('removes the user from the queue once reconciliation commits', async () => {
const queueService = createQueueService();
await queueService.enqueueUser(USER_ID, new Date(Date.now() - 1000));
const userRepository = {
findUnique: async () => null,
} as unknown as UserRepository;
setWorkerDependenciesForTest({
premiumStateReconciliationQueueService: queueService,
stripe: {} as Stripe,
userRepository,
});
await processPremiumStateReconciliationQueue({}, createHelpers());
expect(await queueService.getQueueSize()).toBe(0);
expect(await queueService.getReadyUserIds(Date.now() + ONE_HOUR_MS, 10)).toEqual([]);
});
test('rejects a second claim until the lease expires', async () => {
const queueService = createQueueService();
await queueService.enqueueUser(USER_ID, new Date(Date.now() - 1000));
const now = Date.now();
expect(await queueService.claimUser(USER_ID, now, now + ONE_HOUR_MS)).toBe(true);
expect(await queueService.claimUser(USER_ID, now, now + ONE_HOUR_MS)).toBe(false);
expect(await queueService.claimUser(USER_ID, now + ONE_HOUR_MS, now + 2 * ONE_HOUR_MS)).toBe(true);
});
test('reconciles the user once when two runs claim the same entry', async () => {
const queueService = createQueueService();
await queueService.enqueueUser(USER_ID, new Date(Date.now() - 1000));
let signalFirstClaimEntered: () => void = () => {};
const firstClaimEntered = new Promise<void>((resolve) => {
signalFirstClaimEntered = resolve;
});
let releaseFirstClaim: () => void = () => {};
const firstClaimReleased = new Promise<void>((resolve) => {
releaseFirstClaim = resolve;
});
const claimUser = queueService.claimUser.bind(queueService);
let firstClaim = true;
queueService.claimUser = async (userId, nowMs, leaseUntilMs) => {
if (firstClaim) {
firstClaim = false;
signalFirstClaimEntered();
await firstClaimReleased;
}
return await claimUser(userId, nowMs, leaseUntilMs);
};
let signalReconcileStarted: () => void = () => {};
const reconcileStarted = new Promise<void>((resolve) => {
signalReconcileStarted = resolve;
});
let releaseReconcile: () => void = () => {};
const reconcileReleased = new Promise<void>((resolve) => {
releaseReconcile = resolve;
});
let findUniqueCalls = 0;
const userRepository = {
findUnique: async () => {
findUniqueCalls += 1;
signalReconcileStarted();
await reconcileReleased;
return null;
},
} as unknown as UserRepository;
setWorkerDependenciesForTest({
premiumStateReconciliationQueueService: queueService,
stripe: {} as Stripe,
userRepository,
});
const firstRun = processPremiumStateReconciliationQueue({}, createHelpers());
await firstClaimEntered;
const secondRun = processPremiumStateReconciliationQueue({}, createHelpers());
await reconcileStarted;
releaseFirstClaim();
releaseReconcile();
await Promise.all([firstRun, secondRun]);
expect(findUniqueCalls).toBe(1);
expect(await queueService.getQueueSize()).toBe(0);
});
});
@@ -0,0 +1,118 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import {UserFlags} from '@fluxer/constants/src/UserConstants';
import type {IWorkerService} from '@pkgs/worker/src/contracts/IWorkerService';
import type {WorkerTaskHelpers} from '@pkgs/worker/src/contracts/WorkerTask';
import {afterEach, describe, expect, test} from 'vitest';
import {createUserID, type UserID} from '../../BrandedTypes';
import {EMPTY_USER_ROW, type UserRow} from '../../database/types/UserTypes';
import {KVAccountDeletionQueueService} from '../../infrastructure/KVAccountDeletionQueueService';
import {User} from '../../models/User';
import {MockKVProvider} from '../../test/mocks/MockKVProvider';
import {NoopLogger} from '../../test/mocks/NoopLogger';
import type {UserRepository} from '../../user/repositories/UserRepository';
import userProcessPendingDeletions from '../tasks/UserProcessPendingDeletions';
import {clearWorkerDependencies, setWorkerDependenciesForTest} from '../WorkerContext';
function createFakeUser(
userId: UserID,
pendingDeletionAt: Date,
options: Partial<Pick<UserRow, 'bot' | 'flags'>> = {},
): User {
return new User({
...EMPTY_USER_ROW,
user_id: userId,
pending_deletion_at: pendingDeletionAt,
deletion_reason_code: 1,
bot: options.bot ?? false,
flags: options.flags ?? 0n,
});
}
async function createHarness(users: Array<User>) {
const kvClient = new MockKVProvider();
const removedPendingDeletions: Array<string> = [];
const scheduledJobs: Array<string> = [];
const usersById = new Map<string, User>();
for (const user of users) {
usersById.set(user.id.toString(), user);
}
const userRepository = {
async findUnique(userId: UserID): Promise<User | null> {
return usersById.get(userId.toString()) ?? null;
},
async removePendingDeletion(userId: UserID): Promise<void> {
removedPendingDeletions.push(userId.toString());
},
} as unknown as UserRepository;
const workerService = {
async addJob(name: string, payload: {userId: string}): Promise<bigint> {
scheduledJobs.push(`${name}:${payload.userId}`);
return 0n;
},
} as unknown as IWorkerService;
const deletionQueueService = new KVAccountDeletionQueueService(kvClient, userRepository);
await kvClient.set('deletion_queue:state_version', Date.now().toString());
for (const user of users) {
if (user.pendingDeletionAt) {
await deletionQueueService.scheduleDeletion(user.id, user.pendingDeletionAt, 1);
}
}
setWorkerDependenciesForTest({userRepository, workerService, deletionQueueService});
return {deletionQueueService, scheduledJobs, removedPendingDeletions};
}
function createHelpers(): WorkerTaskHelpers {
return {
logger: new NoopLogger(),
jobId: 1n,
addJob: async () => 0n,
reportProgress: async () => {},
shouldCancel: async () => false,
setContextLink: async () => {},
};
}
describe('userProcessPendingDeletions', () => {
afterEach(() => {
clearWorkerDependencies();
});
test('drains skipped accounts so a genuine deletion behind them is not starved', async () => {
const skippedAt = new Date(Date.now() - 10_000_000);
const genuineAt = new Date(Date.now() - 1_000);
const users: Array<User> = [];
for (let i = 0; i < 1000; i++) {
const userId = createUserID(BigInt(100_000 + i));
users.push(createFakeUser(userId, skippedAt, i % 2 === 0 ? {bot: true} : {flags: UserFlags.APP_STORE_REVIEWER}));
}
const genuineUserId = createUserID(999_999n);
users.push(createFakeUser(genuineUserId, genuineAt));
const harness = await createHarness(users);
await userProcessPendingDeletions({}, createHelpers());
const queueSizeAfterFirstPass = await harness.deletionQueueService.getQueueSize();
await userProcessPendingDeletions({}, createHelpers());
expect(harness.scheduledJobs).toEqual([`userProcessPendingDeletion:${genuineUserId.toString()}`]);
expect(harness.removedPendingDeletions).toEqual([genuineUserId.toString()]);
expect(queueSizeAfterFirstPass).toBe(1);
expect(await harness.deletionQueueService.getQueueSize()).toBe(0);
});
test('keeps skipped accounts out of the queue while the skip condition holds', async () => {
const pendingAt = new Date(Date.now() - 10_000);
const botId = createUserID(1n);
const reviewerId = createUserID(2n);
const harness = await createHarness([
createFakeUser(botId, pendingAt, {bot: true}),
createFakeUser(reviewerId, pendingAt, {flags: UserFlags.APP_STORE_REVIEWER}),
]);
await userProcessPendingDeletions({}, createHelpers());
expect(await harness.deletionQueueService.getQueueSize()).toBe(0);
expect(harness.scheduledJobs).toEqual([]);
expect(harness.removedPendingDeletions).toEqual([]);
});
});
@@ -0,0 +1,246 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import type {LoggerInterface} from '@fluxer/logger/src/LoggerInterface';
import type {IKVProvider} from '@pkgs/kv_client/src/IKVProvider';
import type {ConsumerMessages, JsMsg} from 'nats';
import {afterEach, beforeAll, describe, expect, it, vi} from 'vitest';
import type {IJobLedgerRepository} from '../../jobs/IJobLedgerRepository';
import {setInjectedWorkerService} from '../../middleware/ServiceRegistry';
import {NoopWorkerService} from '../../test/NoopWorkerService';
import {CronScheduler} from '../CronScheduler';
import {
WORKER_CRON_STALE_AFTER_MS,
WORKER_HEARTBEAT_WRITE_INTERVAL_MS,
WORKER_LANE_STALE_AFTER_MS,
WorkerHeartbeat,
} from '../WorkerHeartbeat';
import {WorkerRunner} from '../WorkerRunner';
import type {WorkerService} from '../WorkerService';
const HEARTBEAT_PATH = '/tmp/fluxer-worker-heartbeat-test';
const TASK_TYPE = 'processInactivityDeletions';
function createHeartbeat(): {
heartbeat: WorkerHeartbeat;
write: ReturnType<typeof vi.fn>;
logger: {info: ReturnType<typeof vi.fn>; error: ReturnType<typeof vi.fn>};
} {
const logger = {info: vi.fn(), error: vi.fn()};
const write = vi.fn();
const heartbeat = new WorkerHeartbeat({logger, path: HEARTBEAT_PATH, write});
return {heartbeat, write, logger};
}
class FakeConsumerMessages {
private notify: (() => void) | null = null;
private closed = false;
async close(): Promise<void> {
this.closed = true;
const notify = this.notify;
this.notify = null;
notify?.();
}
async *[Symbol.asyncIterator](): AsyncGenerator<JsMsg> {
while (!this.closed) {
await new Promise<void>((resolve) => {
this.notify = resolve;
});
}
}
}
function createRunner(messages: FakeConsumerMessages, heartbeat: WorkerHeartbeat): WorkerRunner {
return new WorkerRunner({
tasks: {[TASK_TYPE]: async () => {}},
queue: {
getConnectionManager: () => ({
getJetStreamClient: () => ({
consumers: {
get: async () => ({
consume: async () => messages as unknown as ConsumerMessages,
}),
},
}),
}),
getStreamName: () => 'JOBS',
publishToDlq: vi.fn(),
},
consumerName: 'workers_batch',
laneName: 'batch',
ledger: {} as IJobLedgerRepository,
concurrency: 12,
maxDeliver: 25,
ackWaitMs: 120000,
heartbeat,
});
}
function createCronLogger(): LoggerInterface {
const logger = {
trace: vi.fn(),
debug: vi.fn(),
info: vi.fn(),
warn: vi.fn(),
error: vi.fn(),
child: () => logger,
};
return logger as unknown as LoggerInterface;
}
function createScheduler(heartbeat: WorkerHeartbeat): CronScheduler {
const workerService = {addJob: vi.fn().mockResolvedValue(1n)} as unknown as WorkerService;
const kvClient = {setnx: vi.fn().mockResolvedValue(true)} as unknown as IKVProvider;
return new CronScheduler(workerService, createCronLogger(), kvClient, heartbeat);
}
describe('Worker heartbeat', () => {
beforeAll(() => {
setInjectedWorkerService(new NoopWorkerService());
});
afterEach(() => {
vi.useRealTimers();
});
it('keeps rewriting the file while every component reports', async () => {
vi.useFakeTimers();
const {heartbeat, write} = createHeartbeat();
const lane = heartbeat.register('lane:realtime', WORKER_LANE_STALE_AFTER_MS);
heartbeat.start();
expect(write).toHaveBeenCalledTimes(1);
write.mockClear();
for (let elapsed = 0; elapsed < WORKER_LANE_STALE_AFTER_MS * 3; elapsed += WORKER_HEARTBEAT_WRITE_INTERVAL_MS) {
lane.report();
await vi.advanceTimersByTimeAsync(WORKER_HEARTBEAT_WRITE_INTERVAL_MS);
}
heartbeat.stop();
expect(write).toHaveBeenCalledTimes((WORKER_LANE_STALE_AFTER_MS * 3) / WORKER_HEARTBEAT_WRITE_INTERVAL_MS);
expect(write.mock.calls[0]?.[0]).toBe(HEARTBEAT_PATH);
expect(heartbeat.stalledComponents()).toEqual([]);
});
it('stops rewriting the file once a component goes stale', async () => {
vi.useFakeTimers();
const {heartbeat, write, logger} = createHeartbeat();
heartbeat.register('lane:realtime', WORKER_LANE_STALE_AFTER_MS);
heartbeat.start();
await vi.advanceTimersByTimeAsync(WORKER_LANE_STALE_AFTER_MS);
const writesBeforeStall = write.mock.calls.length;
await vi.advanceTimersByTimeAsync(WORKER_LANE_STALE_AFTER_MS * 3);
heartbeat.stop();
expect(write).toHaveBeenCalledTimes(writesBeforeStall);
expect(heartbeat.stalledComponents()).toEqual(['lane:realtime']);
expect(logger.error).toHaveBeenCalledTimes(1);
expect(logger.error.mock.calls[0]?.[1]).toBe('Worker heartbeat stalled, the container will report unhealthy');
});
it('resumes rewriting the file when a stalled component reports again', async () => {
vi.useFakeTimers();
const {heartbeat, write, logger} = createHeartbeat();
const lane = heartbeat.register('lane:realtime', WORKER_LANE_STALE_AFTER_MS);
heartbeat.start();
await vi.advanceTimersByTimeAsync(WORKER_LANE_STALE_AFTER_MS * 3);
const writesBeforeRecovery = write.mock.calls.length;
lane.report();
await vi.advanceTimersByTimeAsync(WORKER_HEARTBEAT_WRITE_INTERVAL_MS);
heartbeat.stop();
expect(write).toHaveBeenCalledTimes(writesBeforeRecovery + 1);
expect(heartbeat.stalledComponents()).toEqual([]);
expect(logger.info).toHaveBeenCalledWith({path: HEARTBEAT_PATH}, 'Worker heartbeat recovered');
});
it('ignores a released component', async () => {
vi.useFakeTimers();
const {heartbeat, write} = createHeartbeat();
const lane = heartbeat.register('lane:realtime', WORKER_LANE_STALE_AFTER_MS);
heartbeat.start();
lane.release();
write.mockClear();
await vi.advanceTimersByTimeAsync(WORKER_LANE_STALE_AFTER_MS * 3);
heartbeat.stop();
expect(write).toHaveBeenCalledTimes((WORKER_LANE_STALE_AFTER_MS * 3) / WORKER_HEARTBEAT_WRITE_INTERVAL_MS);
expect(heartbeat.stalledComponents()).toEqual([]);
});
it('reports a lane for as long as the runner is running and releases it on stop', async () => {
vi.useFakeTimers();
const {heartbeat, write} = createHeartbeat();
const messages = new FakeConsumerMessages();
const runner = createRunner(messages, heartbeat);
heartbeat.start();
await runner.start();
write.mockClear();
await vi.advanceTimersByTimeAsync(WORKER_LANE_STALE_AFTER_MS * 3);
expect(heartbeat.stalledComponents()).toEqual([]);
expect(write).toHaveBeenCalledTimes((WORKER_LANE_STALE_AFTER_MS * 3) / WORKER_HEARTBEAT_WRITE_INTERVAL_MS);
await runner.stop();
await vi.advanceTimersByTimeAsync(WORKER_LANE_STALE_AFTER_MS * 3);
heartbeat.stop();
expect(heartbeat.stalledComponents()).toEqual([]);
});
it('stalls when a running lane stops ticking, which is what a frozen worker looks like', async () => {
vi.useFakeTimers();
const {heartbeat, write} = createHeartbeat();
const messages = new FakeConsumerMessages();
const runner = createRunner(messages, heartbeat);
heartbeat.start();
await runner.start();
vi.clearAllTimers();
write.mockClear();
await vi.advanceTimersByTimeAsync(WORKER_LANE_STALE_AFTER_MS * 3);
expect(write).not.toHaveBeenCalled();
expect(heartbeat.stalledComponents()).toEqual(['lane:batch']);
expect(heartbeat.writeOnce()).toBe(false);
await runner.stop();
heartbeat.stop();
});
it('reports the cron scheduler on every tick and releases it on stop', async () => {
vi.useFakeTimers();
vi.setSystemTime(new Date('2026-01-01T00:00:00.000Z'));
const {heartbeat} = createHeartbeat();
const scheduler = createScheduler(heartbeat);
scheduler.upsert('flushUserActivityBuffer', 'flushUserActivityBuffer', {}, '*/10 * * * * *', {ledger: false});
scheduler.start();
await vi.advanceTimersByTimeAsync(WORKER_CRON_STALE_AFTER_MS * 2);
expect(heartbeat.stalledComponents()).toEqual([]);
scheduler.stop();
await vi.advanceTimersByTimeAsync(WORKER_CRON_STALE_AFTER_MS * 2);
expect(heartbeat.stalledComponents()).toEqual([]);
});
it('stalls when the cron scheduler stops ticking while it is still registered', async () => {
vi.useFakeTimers();
vi.setSystemTime(new Date('2026-01-01T00:00:00.000Z'));
const {heartbeat} = createHeartbeat();
const scheduler = createScheduler(heartbeat);
scheduler.upsert('flushUserActivityBuffer', 'flushUserActivityBuffer', {}, '*/10 * * * * *', {ledger: false});
scheduler.start();
await vi.advanceTimersByTimeAsync(1000);
vi.clearAllTimers();
await vi.advanceTimersByTimeAsync(WORKER_CRON_STALE_AFTER_MS * 2);
expect(heartbeat.stalledComponents()).toEqual(['cron']);
});
});
@@ -1,7 +1,12 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import {describe, expect, it} from 'vitest';
import {resolveCronSchedulerEnabled, resolveWorkerLanes, WORKER_LANES} from '../WorkerLaneConfig';
import {
resolveCronSchedulerEnabled,
resolveWorkerLanes,
validateLaneCompleteness,
WORKER_LANES,
} from '../WorkerLaneConfig';
describe('WorkerLaneConfig', () => {
it('returns all lanes in all_lanes mode', () => {
@@ -56,6 +61,33 @@ describe('WorkerLaneConfig', () => {
expect(embedLane[0]!.name).toBe('unfurl');
expect(embedLane[0]!.taskTypes).toEqual(['extractEmbeds']);
});
it('keeps the retired scheduled message subject on the lifecycle lane only', () => {
const lanes = resolveWorkerLanes({
mode: 'all_lanes',
laneConcurrencyOverrides: {},
});
const lifecycleLane = lanes.find((lane) => lane.name === 'lifecycle');
expect(lifecycleLane?.retiredTaskTypes).toEqual(['sendScheduledMessage']);
expect(lifecycleLane?.taskTypes).not.toContain('sendScheduledMessage');
for (const lane of lanes.filter((lane) => lane.name !== 'lifecycle')) {
expect(lane.retiredTaskTypes).toEqual([]);
}
});
it('never claims a retired subject from a single_task lane', () => {
const lanes = resolveWorkerLanes({
mode: 'single_task',
taskName: 'processStripeWebhook',
laneConcurrencyOverrides: {},
});
expect(lanes[0]!.retiredTaskTypes).toEqual([]);
});
it('rejects a registry that brings a retired task name back', () => {
const registry = Object.fromEntries(WORKER_LANES.flatMap((lane) => lane.taskTypes).map((task) => [task, () => {}]));
expect(() => validateLaneCompleteness(registry)).not.toThrow();
expect(() => validateLaneCompleteness({...registry, sendScheduledMessage: () => {}})).toThrow(
/Retired tasks registered again: sendScheduledMessage/,
);
});
it('throws when single_lane mode has no lane configured', () => {
expect(() =>
resolveWorkerLanes({
@@ -0,0 +1,174 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import {describe, expect, it, vi} from 'vitest';
import {createWorkerProcessErrorHandler, type WorkerProcessErrorSource} from '../WorkerProcessErrorHandler';
function createHarness(overrides: {shutdown?: () => Promise<void>; forceExitDelayMs?: number} = {}) {
const logger = {error: vi.fn(), warn: vi.fn()};
const exit = vi.fn();
const shutdown = vi.fn(overrides.shutdown ?? (async () => {}));
const handle = createWorkerProcessErrorHandler({
logger,
shutdown,
exit,
forceExitDelayMs: overrides.forceExitDelayMs ?? 5,
});
return {logger, exit, shutdown, handle};
}
function pooledClientError(fields: Record<string, unknown>): Error {
const error = new Error(String(fields['message'] ?? 'pooled client error'));
Object.assign(error, {client: {}}, fields);
return error;
}
function adminShutdownError(): Error {
return pooledClientError({
message: 'terminating connection due to administrator command',
code: '57P01',
severity: 'FATAL',
routine: 'ProcessInterrupts',
length: 116,
name: 'error',
});
}
describe('Worker process error handler', () => {
it('keeps the worker running when Postgres terminates a pooled connection', async () => {
const {logger, exit, shutdown, handle} = createHarness();
await handle('uncaughtException', adminShutdownError());
expect(shutdown).not.toHaveBeenCalled();
expect(exit).not.toHaveBeenCalled();
expect(logger.error).not.toHaveBeenCalled();
expect(logger.warn).toHaveBeenCalledTimes(1);
const [context, message] = logger.warn.mock.calls[0]!;
expect(message).toBe('Transient database connection error reached the worker process, keeping the worker running');
expect(context).toMatchObject({source: 'uncaughtException'});
});
it.each([
['57P01 admin_shutdown', '57P01'],
['57P02 crash_shutdown', '57P02'],
['57P03 cannot_connect_now', '57P03'],
['08006 connection_failure', '08006'],
['08003 connection_does_not_exist', '08003'],
])('survives %s', async (_label, code) => {
const {exit, shutdown, handle} = createHarness();
await handle('uncaughtException', pooledClientError({message: 'connection lost', code}));
expect(shutdown).not.toHaveBeenCalled();
expect(exit).not.toHaveBeenCalled();
});
it.each([
['uncaughtException' as WorkerProcessErrorSource],
['unhandledRejection' as WorkerProcessErrorSource],
])('survives a transient error arriving as %s', async (source) => {
const {exit, shutdown, handle} = createHarness();
await handle(source, adminShutdownError());
expect(shutdown).not.toHaveBeenCalled();
expect(exit).not.toHaveBeenCalled();
});
it('survives a socket error raised by a pooled Postgres client', async () => {
const {exit, shutdown, handle} = createHarness();
await handle('uncaughtException', pooledClientError({message: 'read ECONNRESET', code: 'ECONNRESET'}));
expect(shutdown).not.toHaveBeenCalled();
expect(exit).not.toHaveBeenCalled();
});
it('survives a transient error wrapped in a cause chain', async () => {
const {exit, shutdown, handle} = createHarness();
const wrapped = new Error('Connection terminated due to connection timeout', {cause: adminShutdownError()});
await handle('uncaughtException', wrapped);
expect(shutdown).not.toHaveBeenCalled();
expect(exit).not.toHaveBeenCalled();
});
it('survives the pg driver telling us the client is no longer queryable', async () => {
const {exit, shutdown, handle} = createHarness();
await handle('uncaughtException', new Error('Client has encountered a connection error and is not queryable'));
expect(shutdown).not.toHaveBeenCalled();
expect(exit).not.toHaveBeenCalled();
});
it('still tears the worker down on a programming error', async () => {
const {logger, exit, shutdown, handle} = createHarness();
const bug = new TypeError('cannot read properties of undefined');
await handle('uncaughtException', bug);
expect(logger.warn).not.toHaveBeenCalled();
expect(logger.error).toHaveBeenCalledWith({err: bug, source: 'uncaughtException'}, 'Uncaught Exception');
expect(shutdown).toHaveBeenCalledTimes(1);
expect(exit).toHaveBeenCalledWith(1);
});
it('still tears the worker down on a Postgres error that is not connection level', async () => {
const {exit, shutdown, handle} = createHarness();
await handle(
'uncaughtException',
pooledClientError({message: 'duplicate key value violates unique constraint', code: '23505'}),
);
expect(shutdown).toHaveBeenCalledTimes(1);
expect(exit).toHaveBeenCalledWith(1);
});
it('does not treat a socket error from a non-Postgres source as transient', async () => {
const {exit, shutdown, handle} = createHarness();
const socketError = Object.assign(new Error('read ECONNRESET'), {code: 'ECONNRESET'});
await handle('uncaughtException', socketError);
expect(shutdown).toHaveBeenCalledTimes(1);
expect(exit).toHaveBeenCalledWith(1);
});
it('labels an unhandled rejection distinctly when it is fatal', async () => {
const {logger, exit, shutdown, handle} = createHarness();
const bug = new Error('boom');
await handle('unhandledRejection', bug);
expect(logger.error).toHaveBeenCalledWith(
{err: bug, source: 'unhandledRejection'},
'Unhandled Rejection at Promise',
);
expect(shutdown).toHaveBeenCalledTimes(1);
expect(exit).toHaveBeenCalledWith(1);
});
it('force exits when shutdown hangs on a fatal error', async () => {
const {exit, handle} = createHarness({shutdown: () => new Promise<void>(() => {}), forceExitDelayMs: 5});
void handle('uncaughtException', new Error('boom'));
await vi.waitFor(() => expect(exit).toHaveBeenCalledWith(1));
});
it('exits even when shutdown itself throws', async () => {
const {logger, exit, handle} = createHarness({
shutdown: async () => {
throw new Error('shutdown failed');
},
});
await handle('uncaughtException', new Error('boom'));
expect(exit).toHaveBeenCalledWith(1);
expect(logger.error).toHaveBeenCalledTimes(2);
});
});
@@ -0,0 +1,132 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import type {JsMsg} from 'nats';
import {afterEach, beforeAll, describe, expect, it, vi} from 'vitest';
import type {IJobLedgerRepository} from '../../jobs/IJobLedgerRepository';
import {setInjectedWorkerService} from '../../middleware/ServiceRegistry';
import {NoopWorkerService} from '../../test/NoopWorkerService';
import {WorkerRunner} from '../WorkerRunner';
const RETIRED_TASK_TYPE = 'sendScheduledMessage';
const RETIRED_REASON = 'task type retired';
const LEDGER_JOB_ID = '1509197195776110592';
const queueStub = {
getConnectionManager: () => {
throw new Error('WorkerRunner tests never consume messages');
},
getStreamName: () => 'JOBS',
publishToDlq: vi.fn(),
};
const ledgerStub = {
markDeadletter: vi.fn(),
};
class TestWorkerRunner extends WorkerRunner {
async runJob(taskType: string, msg: JsMsg): Promise<boolean> {
return await this.processJob(taskType, msg);
}
}
function createRunner(): TestWorkerRunner {
return new TestWorkerRunner({
tasks: {},
retiredTaskTypes: [RETIRED_TASK_TYPE],
queue: queueStub,
consumerName: 'workers_lifecycle',
laneName: 'lifecycle',
ledger: ledgerStub as unknown as IJobLedgerRepository,
concurrency: 8,
maxDeliver: 25,
ackWaitMs: 60000,
});
}
function createJobMessage(taskType: string, payload: Record<string, unknown>) {
const envelope = {
payload,
run_at: new Date(Date.now() + 30 * 24 * 60 * 60 * 1000).toISOString(),
max_attempts: 5,
priority: 0,
created_at: new Date().toISOString(),
};
return {
seq: 7,
subject: `jobs.${taskType}`,
redelivered: false,
data: new TextEncoder().encode(JSON.stringify(envelope)),
info: {deliveryCount: 1},
ack: vi.fn(),
nak: vi.fn(),
term: vi.fn(),
working: vi.fn(),
};
}
describe('Retired worker task types', () => {
beforeAll(() => {
setInjectedWorkerService(new NoopWorkerService());
});
afterEach(() => {
queueStub.publishToDlq.mockReset();
ledgerStub.markDeadletter.mockReset();
});
it('dead-letters a legacy job and closes its ledger row instead of redelivering it', async () => {
const runner = createRunner();
const msg = createJobMessage(RETIRED_TASK_TYPE, {
userId: '1',
scheduledMessageId: '2',
__jobId: LEDGER_JOB_ID,
});
await expect(runner.runJob(RETIRED_TASK_TYPE, msg as unknown as JsMsg)).resolves.toBe(false);
expect(queueStub.publishToDlq).toHaveBeenCalledTimes(1);
expect(queueStub.publishToDlq).toHaveBeenCalledWith(
RETIRED_TASK_TYPE,
{userId: '1', scheduledMessageId: '2'},
expect.objectContaining({errorMessage: RETIRED_REASON, lane: 'lifecycle', originalSeq: 7}),
);
expect(ledgerStub.markDeadletter).toHaveBeenCalledWith(BigInt(LEDGER_JOB_ID), RETIRED_REASON);
expect(msg.term).toHaveBeenCalledWith(RETIRED_REASON);
expect(msg.nak).not.toHaveBeenCalled();
expect(msg.ack).not.toHaveBeenCalled();
});
it('dead-letters a legacy job that carries no ledger id', async () => {
const runner = createRunner();
const msg = createJobMessage(RETIRED_TASK_TYPE, {userId: '1', scheduledMessageId: '2'});
await expect(runner.runJob(RETIRED_TASK_TYPE, msg as unknown as JsMsg)).resolves.toBe(false);
expect(queueStub.publishToDlq).toHaveBeenCalledTimes(1);
expect(ledgerStub.markDeadletter).not.toHaveBeenCalled();
expect(msg.term).toHaveBeenCalledWith(RETIRED_REASON);
expect(msg.nak).not.toHaveBeenCalled();
});
it('redelivers a retired job when the dead-letter publish fails', async () => {
const runner = createRunner();
queueStub.publishToDlq.mockRejectedValueOnce(new Error('no responders'));
const msg = createJobMessage(RETIRED_TASK_TYPE, {__jobId: LEDGER_JOB_ID});
await expect(runner.runJob(RETIRED_TASK_TYPE, msg as unknown as JsMsg)).resolves.toBe(false);
expect(ledgerStub.markDeadletter).not.toHaveBeenCalled();
expect(msg.term).not.toHaveBeenCalled();
expect(msg.nak).toHaveBeenCalledTimes(1);
});
it('still terminates a task type that was never registered or retired', async () => {
const runner = createRunner();
const msg = createJobMessage('neverShippedTask', {});
await expect(runner.runJob('neverShippedTask', msg as unknown as JsMsg)).resolves.toBe(false);
expect(queueStub.publishToDlq).not.toHaveBeenCalled();
expect(msg.term).toHaveBeenCalledWith(expect.stringMatching(/unknown task type/));
});
});
@@ -0,0 +1,175 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import type {ConsumerMessages, JsMsg} from 'nats';
import {afterEach, beforeAll, describe, expect, it, vi} from 'vitest';
import type {IJobLedgerRepository} from '../../jobs/IJobLedgerRepository';
import {setInjectedWorkerService} from '../../middleware/ServiceRegistry';
import {NoopWorkerService} from '../../test/NoopWorkerService';
import {WorkerRunner} from '../WorkerRunner';
const TASK_TYPE = 'processInactivityDeletions';
const RESUBSCRIBE_WINDOW_MS = 30000;
class FakeConsumerMessages {
private readonly pending: Array<JsMsg> = [];
private notify: (() => void) | null = null;
private closed = false;
private failure: Error | null = null;
push(msg: JsMsg): void {
this.pending.push(msg);
this.wake();
}
abort(error: Error): void {
this.failure = error;
this.wake();
}
end(): void {
this.closed = true;
this.wake();
}
async close(): Promise<void> {
this.end();
}
async *[Symbol.asyncIterator](): AsyncGenerator<JsMsg> {
while (true) {
while (this.pending.length > 0) {
yield this.pending.shift()!;
}
if (this.failure !== null) {
throw this.failure;
}
if (this.closed) {
return;
}
await new Promise<void>((resolve) => {
this.notify = resolve;
});
}
}
private wake(): void {
const notify = this.notify;
this.notify = null;
notify?.();
}
}
function createRunner(streams: Array<FakeConsumerMessages>): {runner: WorkerRunner; consumeCount: () => number} {
let consumed = 0;
const queue = {
getConnectionManager: () => ({
getJetStreamClient: () => ({
consumers: {
get: async () => ({
consume: async () => {
const stream = streams[consumed];
consumed += 1;
if (!stream) {
throw new Error('no more streams');
}
return stream as unknown as ConsumerMessages;
},
}),
},
}),
}),
getStreamName: () => 'JOBS',
publishToDlq: vi.fn(),
};
const runner = new WorkerRunner({
tasks: {[TASK_TYPE]: async () => {}},
queue,
consumerName: 'workers_batch',
laneName: 'batch',
ledger: {} as IJobLedgerRepository,
concurrency: 12,
maxDeliver: 25,
ackWaitMs: 120000,
});
return {runner, consumeCount: () => consumed};
}
function createJobMessage() {
const envelope = {
payload: {},
max_attempts: 5,
priority: 0,
created_at: new Date().toISOString(),
};
return {
seq: 1,
subject: `jobs.${TASK_TYPE}`,
redelivered: false,
data: new TextEncoder().encode(JSON.stringify(envelope)),
info: {deliveryCount: 1},
ack: vi.fn(),
nak: vi.fn(),
term: vi.fn(),
working: vi.fn(),
};
}
describe('Worker runner resubscribe', () => {
beforeAll(() => {
setInjectedWorkerService(new NoopWorkerService());
});
afterEach(() => {
vi.useRealTimers();
});
it('resubscribes after the message stream fails', async () => {
vi.useFakeTimers();
const streams = [new FakeConsumerMessages(), new FakeConsumerMessages()];
const {runner, consumeCount} = createRunner(streams);
await runner.start();
expect(consumeCount()).toBe(1);
streams[0]!.abort(new Error('consumer deleted'));
await vi.advanceTimersByTimeAsync(RESUBSCRIBE_WINDOW_MS);
expect(consumeCount()).toBe(2);
const msg = createJobMessage();
streams[1]!.push(msg as unknown as JsMsg);
await vi.advanceTimersByTimeAsync(0);
expect(msg.ack).toHaveBeenCalledTimes(1);
await runner.stop();
});
it('resubscribes after the message stream ends without a stop', async () => {
vi.useFakeTimers();
const streams = [new FakeConsumerMessages(), new FakeConsumerMessages()];
const {runner, consumeCount} = createRunner(streams);
await runner.start();
streams[0]!.end();
await vi.advanceTimersByTimeAsync(RESUBSCRIBE_WINDOW_MS);
expect(consumeCount()).toBe(2);
const msg = createJobMessage();
streams[1]!.push(msg as unknown as JsMsg);
await vi.advanceTimersByTimeAsync(0);
expect(msg.ack).toHaveBeenCalledTimes(1);
await runner.stop();
});
it('does not resubscribe after the runner is stopped', async () => {
vi.useFakeTimers();
const streams = [new FakeConsumerMessages(), new FakeConsumerMessages()];
const {runner, consumeCount} = createRunner(streams);
await runner.start();
await runner.stop();
await vi.advanceTimersByTimeAsync(RESUBSCRIBE_WINDOW_MS);
expect(consumeCount()).toBe(1);
});
});
@@ -0,0 +1,127 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import type {ConsumerMessages, JsMsg} from 'nats';
import {beforeAll, describe, expect, it, vi} from 'vitest';
import type {IJobLedgerRepository} from '../../jobs/IJobLedgerRepository';
import {setInjectedWorkerService} from '../../middleware/ServiceRegistry';
import {NoopWorkerService} from '../../test/NoopWorkerService';
import {WorkerRunner} from '../WorkerRunner';
const TASK_TYPE = 'processInactivityDeletions';
class FakeConsumerMessages {
private readonly pending: Array<JsMsg> = [];
private notify: (() => void) | null = null;
private closed = false;
push(msg: JsMsg): void {
this.pending.push(msg);
this.wake();
}
async close(): Promise<void> {
this.closed = true;
this.wake();
}
async *[Symbol.asyncIterator](): AsyncGenerator<JsMsg> {
while (true) {
while (this.pending.length > 0) {
yield this.pending.shift()!;
}
if (this.closed) {
return;
}
await new Promise<void>((resolve) => {
this.notify = resolve;
});
}
}
private wake(): void {
const notify = this.notify;
this.notify = null;
notify?.();
}
}
function createQueueStub(messages: FakeConsumerMessages) {
return {
getConnectionManager: () => ({
getJetStreamClient: () => ({
consumers: {
get: async () => ({
consume: async () => messages as unknown as ConsumerMessages,
}),
},
}),
}),
getStreamName: () => 'JOBS',
publishToDlq: vi.fn(),
};
}
function createRunner(task: () => Promise<void>, messages: FakeConsumerMessages): WorkerRunner {
return new WorkerRunner({
tasks: {[TASK_TYPE]: task},
queue: createQueueStub(messages),
consumerName: 'workers_batch',
laneName: 'batch',
ledger: {} as IJobLedgerRepository,
concurrency: 12,
maxDeliver: 25,
ackWaitMs: 120000,
});
}
function createJobMessage() {
const envelope = {
payload: {},
max_attempts: 5,
priority: 0,
created_at: new Date().toISOString(),
};
return {
seq: 1,
subject: `jobs.${TASK_TYPE}`,
redelivered: false,
data: new TextEncoder().encode(JSON.stringify(envelope)),
info: {deliveryCount: 1},
ack: vi.fn(),
nak: vi.fn(),
term: vi.fn(),
working: vi.fn(),
};
}
describe('Worker runner shutdown', () => {
beforeAll(() => {
setInjectedWorkerService(new NoopWorkerService());
});
it('drains in-flight jobs before stop resolves', async () => {
let started = false;
let finished = false;
let release: () => void = () => {};
const pending = new Promise<void>((resolve) => {
release = resolve;
});
const messages = new FakeConsumerMessages();
const runner = createRunner(async () => {
started = true;
await pending;
finished = true;
}, messages);
const msg = createJobMessage();
await runner.start();
messages.push(msg as unknown as JsMsg);
await vi.waitFor(() => expect(started).toBe(true));
setTimeout(release, 0);
await runner.stop();
expect(finished).toBe(true);
expect(msg.ack).toHaveBeenCalledTimes(1);
});
});
+1 -1
View File
@@ -22,7 +22,7 @@
"bench:gif-picker": "vitest bench --run src/features/channel/components/pickers/gif/GifPickerStateMachine.bench.ts src/features/channel/components/pickers/gif/GifPickerGridData.bench.ts src/features/channel/components/pickers/gif/GifPickerLoadingSkeletonGridLayout.bench.ts src/features/channel/components/pickers/shared/TileFlowSolver.bench.ts",
"bench:messages": "vitest bench --run src/features/messaging/utils/MessageGroupingUtils.bench.ts src/features/channel/components/ChannelMessageStreamUtils.bench.ts src/features/messaging/components/markdown/MarkdownRendering.bench.ts src/features/channel/components/MessageAttachmentUtils.bench.ts src/features/messaging/utils/MessagePaginationUtils.bench.ts src/features/platform/utils/ScrollPosition.bench.ts src/features/messaging/state/ReactionStateMachine.bench.ts",
"test:watch": "vitest",
"typecheck": "pnpm wasm:codegen && pnpm generate:colors && pnpm generate:message-layout && pnpm generate:theme-variables && pnpm generate:masks && pnpm generate:css-types && tsgo --noEmit",
"typecheck": "pnpm wasm:codegen && pnpm generate:colors && pnpm generate:message-layout && pnpm generate:theme-variables && pnpm generate:masks && pnpm generate:css-types && pnpm lingui:compile && tsgo --noEmit",
"typecheck:only": "tsgo --noEmit",
"check:message-layout": "pnpm tsx scripts/GenerateMessageLayoutCss.ts --check",
"generate:colors": "pnpm tsx scripts/GenerateColorSystem.ts",
+31 -12
View File
@@ -4,6 +4,8 @@ ARG BUILD_VERSION=""
ARG PUBLIC_ASSET_BASE_URL=""
ARG FLUXER_APP_PROXY_TIME_FREEZE_ENABLED="true"
ARG BUNDLE_LOCAL_ASSETS="true"
ARG APP_ASSETS_REF=app-assets
ARG APP_ASSETS_PLATFORM=$BUILDPLATFORM
# ---------- Stage 1: Build the SPA (fluxer_app) ----------
FROM node:24-bookworm-slim AS app-build
@@ -79,13 +81,19 @@ RUN --mount=type=cache,target=/usr/local/cargo/registry \
--mount=type=cache,target=/usr/local/cargo/git \
cd fluxer_app && pnpm build
# Extract the full SPA asset tree for local serving by the app proxy.
# ---------- Stage 2: The one canonical asset tree every architecture serves ----------
# Stage for CI to extract the full dist (docker bake app-dist target)
FROM alpine:3.21 AS app-dist
COPY --from=app-build /usr/src/app/fluxer_app/dist /dist
FROM alpine:3.21 AS app-assets
ARG BUNDLE_LOCAL_ASSETS=true
ARG PUBLIC_ASSET_BASE_URL=""
COPY --from=app-build /usr/src/app/fluxer_app/dist /assets
COPY fluxer_app_proxy/scripts/precompress_assets.sh /usr/local/bin/precompress_assets.sh
RUN if [ "${BUNDLE_LOCAL_ASSETS}" != "true" ]; then \
RUN apk add --no-cache brotli
RUN --mount=type=bind,from=app-dist,source=/dist,target=/dist \
cp -a /dist /assets \
&& if [ "${BUNDLE_LOCAL_ASSETS}" != "true" ]; then \
case "${PUBLIC_ASSET_BASE_URL}" in \
http://* | https://*) ;; \
*) \
@@ -95,15 +103,12 @@ RUN if [ "${BUNDLE_LOCAL_ASSETS}" != "true" ]; then \
;; \
esac; \
find /assets/assets -type f ! -name 'fonts-*.txt' -delete; \
fi
RUN apk add --no-cache brotli \
fi \
&& /usr/local/bin/precompress_assets.sh /assets
# Stage for CI to extract the full dist (docker bake app-dist target)
FROM alpine:3.21 AS app-dist
COPY --from=app-build /usr/src/app/fluxer_app/dist /dist
FROM --platform=${APP_ASSETS_PLATFORM} ${APP_ASSETS_REF} AS app-static
# ---------- Stage 2: Build the Rust proxy binary ----------
# ---------- Stage 3: Build the Rust proxy binary ----------
FROM rust:1-bookworm AS rust-builder
ARG FLUXER_APP_PROXY_TIME_FREEZE_ENABLED="true"
@@ -141,10 +146,24 @@ RUN if [ "${FLUXER_APP_PROXY_TIME_FREEZE_ENABLED}" = "false" ]; then \
fi \
&& cp target/release/fluxer_app_proxy /usr/local/bin/fluxer-app-proxy
# ---------- Stage 3: Runtime ----------
# ---------- Stage 4: Runtime ----------
FROM debian:bookworm-slim
ARG BUILD_VERSION=""
ARG SOURCE_SHA=""
ARG SOURCE_DATE=""
LABEL org.opencontainers.image.title="fluxer-app-proxy"
LABEL org.opencontainers.image.description="Fluxer web app and edge proxy"
LABEL org.opencontainers.image.licenses="AGPL-3.0-or-later"
LABEL org.opencontainers.image.vendor="Fluxer"
LABEL org.opencontainers.image.url="https://fluxer.app"
LABEL org.opencontainers.image.documentation="https://docs.fluxer.app"
LABEL org.opencontainers.image.source="https://github.com/fluxerapp/fluxer"
LABEL org.opencontainers.image.version="${BUILD_VERSION}"
LABEL org.opencontainers.image.revision="${SOURCE_SHA}"
LABEL org.opencontainers.image.created="${SOURCE_DATE}"
LABEL app.fluxer.build-version="${BUILD_VERSION}"
WORKDIR /srv/app
@@ -153,7 +172,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
&& rm -rf /var/lib/apt/lists/*
COPY --from=rust-builder /usr/local/bin/fluxer-app-proxy /usr/local/bin/fluxer-app-proxy
COPY --from=app-assets /assets ./static/
COPY --from=app-static /assets ./static/
ENV BUILD_VERSION="${BUILD_VERSION}"
ENV FLUXER_APP_PROXY_HOST="0.0.0.0"
@@ -164,6 +183,6 @@ USER 65532:65532
EXPOSE 8080
HEALTHCHECK --interval=30s --timeout=5s --start-period=5s --retries=3 \
CMD curl -f http://localhost:8080/_health || exit 1
CMD curl -f http://127.0.0.1:${FLUXER_APP_PROXY_PORT:-8080}/_ready || exit 1
CMD ["/usr/local/bin/fluxer-app-proxy"]
+15
View File
@@ -6,6 +6,10 @@ variable "BUNDLE_LOCAL_ASSETS" { default = "true" }
variable "IMAGE_REPO" { default = "" }
variable "CACHE_FROM" { default = "" }
variable "CACHE_TO" { default = "" }
variable "APP_ASSETS_REF" { default = "app-assets" }
variable "APP_ASSETS_PLATFORM" { default = "linux/amd64" }
variable "SOURCE_SHA" { default = "" }
variable "SOURCE_DATE" { default = "" }
group "default" {
targets = ["app-proxy", "app-dist"]
@@ -24,6 +28,10 @@ target "app-proxy" {
PUBLIC_ASSET_BASE_URL = PUBLIC_ASSET_BASE_URL
FLUXER_APP_PROXY_TIME_FREEZE_ENABLED = FLUXER_APP_PROXY_TIME_FREEZE_ENABLED
BUNDLE_LOCAL_ASSETS = BUNDLE_LOCAL_ASSETS
APP_ASSETS_REF = APP_ASSETS_REF
APP_ASSETS_PLATFORM = APP_ASSETS_PLATFORM
SOURCE_SHA = SOURCE_SHA
SOURCE_DATE = SOURCE_DATE
}
}
@@ -33,3 +41,10 @@ target "app-dist" {
tags = []
output = ["type=local,dest=app-dist-output"]
}
target "app-assets-image" {
inherits = ["app-proxy"]
target = "app-assets"
tags = IMAGE_REPO != "" ? ["${IMAGE_REPO}:${BUILD_VERSION}-assets"] : []
output = ["type=registry"]
}
+154 -2
View File
@@ -2,6 +2,7 @@
use crate::config::AppProxyConfig;
use crate::discovery_cache::DiscoveryResponse;
use reqwest::Url;
use serde::Serialize;
#[derive(Serialize)]
@@ -43,11 +44,14 @@ pub fn build_bootstrap_script(
geoip: &serde_json::Value,
nonce: &str,
) -> String {
let api_public_endpoint =
api_public_endpoint(config.bootstrap_api_public_endpoint.as_deref(), discovery);
let payload = BootstrapPayload {
config: BootstrapConfig {
release_channel: config.release_channel.as_str(),
bootstrap_api_endpoint: &config.bootstrap_api_endpoint,
bootstrap_api_public_endpoint: config.bootstrap_api_public_endpoint.as_deref(),
bootstrap_api_public_endpoint: api_public_endpoint,
},
instance: &discovery.data,
geoip,
@@ -56,7 +60,7 @@ pub fn build_bootstrap_script(
let legacy = LegacyConfig {
release_channel: config.release_channel.as_str(),
bootstrap_api_endpoint: &config.bootstrap_api_endpoint,
bootstrap_api_public_endpoint: config.bootstrap_api_public_endpoint.as_deref(),
bootstrap_api_public_endpoint: api_public_endpoint,
};
let bootstrap_json = escape_json_for_script(&serde_json::to_string(&payload).unwrap());
@@ -67,6 +71,50 @@ pub fn build_bootstrap_script(
)
}
fn api_public_endpoint<'a>(
configured: Option<&'a str>,
discovery: &'a DiscoveryResponse,
) -> Option<&'a str> {
let configured = configured?;
let Some(discovered) = discovered_api_public(discovery) else {
return Some(configured);
};
if has_explicit_port(configured) || !has_explicit_port(discovered) {
return Some(configured);
}
let (Ok(configured_url), Ok(discovered_url)) = (Url::parse(configured), Url::parse(discovered))
else {
return Some(configured);
};
if configured_url.scheme() != discovered_url.scheme()
|| configured_url.host_str() != discovered_url.host_str()
|| configured_url.path() != discovered_url.path()
{
return Some(configured);
}
Some(discovered)
}
fn discovered_api_public(discovery: &DiscoveryResponse) -> Option<&str> {
discovery
.data
.get("endpoints")
.and_then(|endpoints| endpoints.get("api_public"))
.and_then(serde_json::Value::as_str)
.map(str::trim)
.filter(|value| !value.is_empty())
}
fn has_explicit_port(url: &str) -> bool {
let Some(scheme_end) = url.find("://") else {
return false;
};
let rest = &url[scheme_end + 3..];
let authority_end = rest.find(['/', '\\', '?', '#']).unwrap_or(rest.len());
let host = rest[..authority_end].rsplit('@').next().unwrap_or_default();
host[host.rfind(']').map_or(0, |index| index + 1)..].contains(':')
}
const MEDIA_PRECONNECT_TAG: &str = r#"<link rel="preconnect" href="{{MEDIA_ENDPOINT}}">"#;
const STATIC_PRECONNECT_TAGS: [&str; 2] = [
r#"<link rel="preconnect" href="{{STATIC_CDN_ENDPOINT}}">"#,
@@ -411,4 +459,108 @@ mod tests {
let json = serde_json::to_string(&payload).unwrap();
assert!(!json.contains("bootstrapApiPublicEndpoint"));
}
fn discovery_offering(api_public: &str) -> DiscoveryResponse {
DiscoveryResponse {
data: serde_json::json!({"endpoints": {"api_public": api_public}}),
}
}
#[test]
fn the_discovered_port_repairs_a_portless_configured_endpoint() {
let discovery = discovery_offering("https://chat.example.test:8443/api");
assert_eq!(
api_public_endpoint(Some("https://chat.example.test/api"), &discovery),
Some("https://chat.example.test:8443/api")
);
}
#[test]
fn a_configured_endpoint_that_already_carries_a_port_is_left_alone() {
let discovery = discovery_offering("https://chat.example.test:8443/api");
assert_eq!(
api_public_endpoint(Some("https://chat.example.test:9443/api"), &discovery),
Some("https://chat.example.test:9443/api")
);
}
#[test]
fn a_discovered_endpoint_on_another_host_is_ignored() {
let discovery = discovery_offering("https://api.example.test:8443/api");
assert_eq!(
api_public_endpoint(Some("https://chat.example.test/api"), &discovery),
Some("https://chat.example.test/api")
);
}
#[test]
fn a_discovered_endpoint_on_another_path_is_ignored() {
let discovery = discovery_offering("https://chat.example.test:8443/v9/api");
assert_eq!(
api_public_endpoint(Some("https://chat.example.test/api"), &discovery),
Some("https://chat.example.test/api")
);
}
#[test]
fn a_discovered_endpoint_on_another_scheme_is_ignored() {
let discovery = discovery_offering("http://chat.example.test:8443/api");
assert_eq!(
api_public_endpoint(Some("https://chat.example.test/api"), &discovery),
Some("https://chat.example.test/api")
);
}
#[test]
fn a_discovery_document_without_a_snapshot_leaves_the_configured_endpoint_alone() {
let discovery = DiscoveryResponse {
data: serde_json::json!({}),
};
assert_eq!(
api_public_endpoint(Some("https://chat.example.test/api"), &discovery),
Some("https://chat.example.test/api")
);
}
#[test]
fn a_discovery_document_without_a_public_api_endpoint_changes_nothing() {
let discovery = DiscoveryResponse {
data: serde_json::json!({"endpoints": {"api_public": " "}}),
};
assert_eq!(
api_public_endpoint(Some("https://chat.example.test/api"), &discovery),
Some("https://chat.example.test/api")
);
}
#[test]
fn a_portless_discovered_endpoint_leaves_a_default_install_alone() {
let discovery = discovery_offering("https://chat.example.test/api");
assert_eq!(
api_public_endpoint(Some("https://chat.example.test/api"), &discovery),
Some("https://chat.example.test/api")
);
}
#[test]
fn an_unconfigured_public_endpoint_is_never_invented_from_discovery() {
let discovery = discovery_offering("https://chat.example.test:8443/api");
assert_eq!(api_public_endpoint(None, &discovery), None);
}
#[test]
fn the_boot_script_hands_the_repaired_endpoint_to_both_globals() {
let discovery = discovery_offering("https://chat.example.test:8443/api");
let mut config = AppProxyConfig::from_env();
config.bootstrap_api_public_endpoint = Some("https://chat.example.test/api".to_owned());
let script =
build_bootstrap_script(&config, &discovery, &serde_json::json!({}), "scriptnonce");
assert!(
script.contains(r#""bootstrapApiPublicEndpoint":"https://chat.example.test:8443/api""#)
);
assert!(script.contains(
r#""PUBLIC_BOOTSTRAP_API_PUBLIC_ENDPOINT":"https://chat.example.test:8443/api""#
));
assert!(!script.contains(r#""https://chat.example.test/api""#));
}
}
+156 -7
View File
@@ -200,9 +200,7 @@ impl AppProxyConfig {
env!("CARGO_PKG_VERSION"),
),
bootstrap_api_endpoint: cfg::read_env("PUBLIC_BOOTSTRAP_API_ENDPOINT", "/api"),
bootstrap_api_public_endpoint: cfg::non_empty_env(
"PUBLIC_BOOTSTRAP_API_PUBLIC_ENDPOINT",
),
bootstrap_api_public_endpoint: resolve_bootstrap_api_public_endpoint_from_env(),
csp: CspConfig::from_env(),
geoip_source,
geoip_s3_config,
@@ -251,10 +249,7 @@ impl AppProxyConfig {
postgres_ssl_ca: cfg::non_empty_env("FLUXER_POSTGRES_SSL_CA"),
postgres_max_connections,
postgres_kv_table: cfg::read_env("FLUXER_POSTGRES_KV_TABLE", "fluxer_kv"),
postgres_prepared_statements: cfg::read_bool_env(
&["FLUXER_POSTGRES_PREPARED_STATEMENTS"],
true,
),
postgres_prepared_statements: resolve_postgres_prepared_statements_from_env(),
}
}
}
@@ -274,6 +269,31 @@ fn resolve_time_freeze_enabled_from_env() -> bool {
resolve_time_freeze_enabled(|name| env::var(name).ok())
}
fn resolve_postgres_prepared_statements_from_env() -> bool {
resolve_postgres_prepared_statements(|name| env::var(name).ok())
}
fn resolve_bootstrap_api_public_endpoint_from_env() -> Option<String> {
resolve_bootstrap_api_public_endpoint(|name| env::var(name).ok())
}
fn resolve_bootstrap_api_public_endpoint<F>(mut read_var: F) -> Option<String>
where
F: FnMut(&str) -> Option<String>,
{
let endpoint = read_var("PUBLIC_BOOTSTRAP_API_PUBLIC_ENDPOINT")
.map(|value| value.trim().to_owned())
.filter(|value| !value.is_empty())?;
let base_domain = read_var("FLUXER_BASE_DOMAIN").unwrap_or_default();
let public_port = read_var("FLUXER_PUBLIC_PORT").and_then(|port| port.trim().parse().ok());
Some(cfg::normalize_public_endpoint(
&endpoint,
&base_domain,
public_port,
))
}
fn resolve_time_freeze_enabled<F>(mut read_var: F) -> bool
where
F: FnMut(&str) -> Option<String>,
@@ -292,6 +312,31 @@ fn parse_boolish(value: &str) -> bool {
)
}
fn resolve_postgres_prepared_statements<F>(mut read_var: F) -> bool
where
F: FnMut(&str) -> Option<String>,
{
let Some(value) = read_var("FLUXER_POSTGRES_PREPARED_STATEMENTS")
.map(|value| value.trim().to_ascii_lowercase())
.filter(|value| !value.is_empty())
else {
return true;
};
match value.as_str() {
"1" | "true" | "yes" | "y" | "on" => true,
"0" | "false" | "no" | "n" | "off" => false,
other => {
tracing::warn!(
env = "FLUXER_POSTGRES_PREPARED_STATEMENTS",
value = other,
"invalid value; falling back to default"
);
true
}
}
}
fn resolve_discovery_upstream_url<F>(mut read_var: F) -> String
where
F: FnMut(&str) -> Option<String>,
@@ -342,6 +387,73 @@ mod tests {
resolve_time_freeze_enabled(|name| env.get(name).map(|value| value.to_string()))
}
fn resolve_prepared_statements_from_pairs(pairs: &[(&str, &str)]) -> bool {
let env: HashMap<&str, &str> = pairs.iter().copied().collect();
resolve_postgres_prepared_statements(|name| env.get(name).map(|value| value.to_string()))
}
fn resolve_bootstrap_endpoint_from_pairs(pairs: &[(&str, &str)]) -> Option<String> {
let env: HashMap<&str, &str> = pairs.iter().copied().collect();
resolve_bootstrap_api_public_endpoint(|name| env.get(name).map(|value| value.to_string()))
}
#[test]
fn a_non_default_public_port_reaches_the_boot_html_api_endpoint() {
assert_eq!(
resolve_bootstrap_endpoint_from_pairs(&[
(
"PUBLIC_BOOTSTRAP_API_PUBLIC_ENDPOINT",
"http://fluxer.example/api",
),
("FLUXER_BASE_DOMAIN", "fluxer.example"),
("FLUXER_PUBLIC_PORT", "19080"),
]),
Some("http://fluxer.example:19080/api".to_owned())
);
}
#[test]
fn a_default_public_port_leaves_the_boot_html_api_endpoint_alone() {
assert_eq!(
resolve_bootstrap_endpoint_from_pairs(&[
(
"PUBLIC_BOOTSTRAP_API_PUBLIC_ENDPOINT",
"https://fluxer.example/api",
),
("FLUXER_BASE_DOMAIN", "fluxer.example"),
("FLUXER_PUBLIC_PORT", "443"),
]),
Some("https://fluxer.example/api".to_owned())
);
}
#[test]
fn the_boot_html_api_endpoint_keeps_a_port_it_already_carries() {
assert_eq!(
resolve_bootstrap_endpoint_from_pairs(&[
(
"PUBLIC_BOOTSTRAP_API_PUBLIC_ENDPOINT",
"http://fluxer.example:19080/api",
),
("FLUXER_BASE_DOMAIN", "fluxer.example"),
("FLUXER_PUBLIC_PORT", "19080"),
]),
Some("http://fluxer.example:19080/api".to_owned())
);
}
#[test]
fn the_boot_html_api_endpoint_is_untouched_without_a_base_domain_and_port() {
assert_eq!(
resolve_bootstrap_endpoint_from_pairs(&[(
"PUBLIC_BOOTSTRAP_API_PUBLIC_ENDPOINT",
"http://fluxer.example/api",
)]),
Some("http://fluxer.example/api".to_owned())
);
assert_eq!(resolve_bootstrap_endpoint_from_pairs(&[]), None);
}
#[test]
fn csp_config_default_has_no_extra_sources() {
let c = CspConfig::default();
@@ -429,6 +541,43 @@ mod tests {
);
}
#[test]
fn a_set_but_empty_prepared_statements_value_keeps_the_shared_default() {
assert!(resolve_prepared_statements_from_pairs(&[]));
assert!(
resolve_prepared_statements_from_pairs(&[("FLUXER_POSTGRES_PREPARED_STATEMENTS", "")]),
"an empty value disabled named statements here while every other service kept them"
);
assert!(resolve_prepared_statements_from_pairs(&[(
"FLUXER_POSTGRES_PREPARED_STATEMENTS",
" ",
)]));
}
#[test]
fn an_explicit_prepared_statements_value_is_honoured() {
assert!(!resolve_prepared_statements_from_pairs(&[(
"FLUXER_POSTGRES_PREPARED_STATEMENTS",
"false",
)]));
assert!(!resolve_prepared_statements_from_pairs(&[(
"FLUXER_POSTGRES_PREPARED_STATEMENTS",
"OFF",
)]));
assert!(resolve_prepared_statements_from_pairs(&[(
"FLUXER_POSTGRES_PREPARED_STATEMENTS",
"yes",
)]));
}
#[test]
fn a_non_boolean_prepared_statements_value_keeps_the_shared_default() {
assert!(resolve_prepared_statements_from_pairs(&[(
"FLUXER_POSTGRES_PREPARED_STATEMENTS",
"maybe",
)]));
}
#[test]
fn time_freeze_enabled_by_default_for_hosted_runtime() {
assert!(resolve_time_freeze_from_pairs(&[]));
+13
View File
@@ -89,6 +89,10 @@ impl DiscoveryCache {
self.cached.read().await.clone()
}
pub async fn has_snapshot(&self) -> bool {
self.cached.read().await.is_some()
}
pub fn start_background_refresh(
self: &Arc<Self>,
client: reqwest::Client,
@@ -148,6 +152,15 @@ mod tests {
assert!(cache.get().await.is_none());
}
#[tokio::test]
async fn a_seeded_cache_reports_a_snapshot_without_cloning_it() {
let cache = DiscoveryCache::new();
assert!(!cache.has_snapshot().await);
*cache.cached.write().await =
Some(serde_json::from_str(r#"{"api_code_version":"v1"}"#).unwrap());
assert!(cache.has_snapshot().await);
}
#[tokio::test]
async fn cached_discovery_never_touches_the_network() {
let cache = DiscoveryCache::new();
+70 -2
View File
@@ -14,15 +14,17 @@ use scylla::client::session::Session;
use scylla::statement::prepared::PreparedStatement;
use serde::Deserialize;
use std::collections::HashSet;
#[cfg(feature = "scylla")]
use std::sync::Arc;
use std::sync::{Arc, OnceLock};
use std::time::Duration;
use tokio::task::JoinHandle;
const INVITE_TYPE_GUILD: i32 = 0;
const INVITE_TYPE_GROUP_DM: i32 = 1;
const CHANNEL_TYPE_GROUP_DM: i32 = 3;
const MEDIA_SIZE_DEFAULT: i32 = 160;
const DEFAULT_AVATAR_COUNT: i64 = 6;
const CONNECT_RETRY_BASE: Duration = Duration::from_secs(5);
const CONNECT_RETRY_MAX: Duration = Duration::from_secs(60);
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct InvitePageMeta {
@@ -302,6 +304,42 @@ impl InviteMetaResolver {
}
}
pub fn start_background_connect(
slot: &Arc<OnceLock<InviteMetaResolver>>,
config: Arc<AppProxyConfig>,
) -> JoinHandle<()> {
let slot = Arc::clone(slot);
tokio::spawn(async move {
let mut failures: u32 = 0;
loop {
match InviteMetaResolver::connect(&config).await {
Ok(resolver) => {
let _ = slot.set(resolver);
tracing::info!("invite metadata resolver connected");
return;
}
Err(err) => {
failures = failures.saturating_add(1);
let backoff = connect_backoff(failures);
tracing::warn!(
%err,
failures,
backoff_ms = backoff.as_millis() as u64,
"invite metadata resolver connect failed; retrying"
);
tokio::time::sleep(backoff).await;
}
}
}
})
}
fn connect_backoff(failures: u32) -> Duration {
CONNECT_RETRY_BASE
.saturating_mul(1u32 << failures.min(5))
.min(CONNECT_RETRY_MAX)
}
fn invite_meta_cache(config: &AppProxyConfig) -> Cache<String, Option<InvitePageMeta>> {
Cache::builder()
.max_capacity(config.invite_meta_cache_max_entries)
@@ -759,6 +797,36 @@ fn escape_html_attr(value: &str) -> String {
mod tests {
use super::*;
#[test]
fn the_connect_backoff_grows_and_stops_at_the_ceiling() {
assert_eq!(connect_backoff(1), Duration::from_secs(10));
assert_eq!(connect_backoff(2), Duration::from_secs(20));
assert_eq!(connect_backoff(3), Duration::from_secs(40));
assert_eq!(connect_backoff(4), CONNECT_RETRY_MAX);
assert_eq!(connect_backoff(64), CONNECT_RETRY_MAX);
}
#[tokio::test]
async fn a_refused_database_never_fills_the_slot_and_never_gives_up() {
let mut config = AppProxyConfig::from_env();
config.database_backend = DatabaseBackend::Postgres;
config.postgres_url = None;
config.postgres_host = "127.0.0.1".to_owned();
config.postgres_port = 1;
config.postgres_ssl = false;
let slot = Arc::new(OnceLock::new());
let handle = start_background_connect(&slot, Arc::new(config));
tokio::time::sleep(Duration::from_millis(200)).await;
assert!(slot.get().is_none());
assert!(
!handle.is_finished(),
"a failed connect ended the retry loop and disabled invite metadata for the process"
);
handle.abort();
}
fn endpoints() -> InviteMetaEndpoints {
InviteMetaEndpoints {
media_endpoint: Some("https://media.example.test/media/".to_owned()),
+11 -16
View File
@@ -4,12 +4,11 @@ use anyhow::Context;
use fluxer_app_proxy::{
config::AppProxyConfig,
discovery_cache::DiscoveryCache,
geoip,
invite_meta::InviteMetaResolver,
geoip, invite_meta,
routes::build_router,
state::{AppState, build_http_client},
};
use std::sync::Arc;
use std::sync::{Arc, OnceLock};
use tokio::{net::TcpListener, runtime::Builder};
use tracing_subscriber::{layer::SubscriberExt, util::SubscriberInitExt};
@@ -21,7 +20,7 @@ fn main() -> anyhow::Result<()> {
.with(tracing_subscriber::fmt::layer())
.init();
let config = AppProxyConfig::from_env();
let config = Arc::new(AppProxyConfig::from_env());
let addr = format!("{}:{}", config.host, config.port);
let geoip = Arc::new(geoip::resolver_from_app_config(&config));
@@ -49,17 +48,10 @@ fn main() -> anyhow::Result<()> {
config.discovery_refresh_interval_ms,
);
let invite_meta = if config.invite_meta_enabled {
match InviteMetaResolver::connect(&config).await {
Ok(resolver) => Some(Arc::new(resolver)),
Err(err) => {
tracing::warn!(%err, "invite metadata resolver disabled; failed to connect to database");
None
}
}
} else {
None
};
let invite_meta = Arc::new(OnceLock::new());
let invite_meta_connect = config
.invite_meta_enabled
.then(|| invite_meta::start_background_connect(&invite_meta, Arc::clone(&config)));
let index_html = if config.index_upstream_url.is_none() {
let index_path = std::path::Path::new(&config.static_dir).join("index.html");
@@ -75,7 +67,7 @@ fn main() -> anyhow::Result<()> {
};
let state = AppState {
config: Arc::new(config),
config,
http_client,
discovery_cache,
geoip,
@@ -95,6 +87,9 @@ fn main() -> anyhow::Result<()> {
.context("app proxy server exited unexpectedly")?;
cancel.abort();
if let Some(handle) = invite_meta_connect {
handle.abort();
}
Ok(())
})
}
+2 -2
View File
@@ -379,8 +379,8 @@ mod tests {
use axum::http::header::HeaderName;
use fluxer_common::config::GeoipSourceConfig;
use fluxer_common::geoip::{GeoipConfig, GeoipResolver};
use std::sync::Arc;
use std::sync::atomic::{AtomicU64, Ordering};
use std::sync::{Arc, OnceLock};
use tower::ServiceExt;
async fn spawn_upstream(status: StatusCode, cache_control: &'static str) -> String {
@@ -427,7 +427,7 @@ mod tests {
trust_client_ip_header: false,
client_ip_header_name: "x-forwarded-for".to_owned(),
})),
invite_meta: None,
invite_meta: Arc::new(OnceLock::new()),
index_html: None,
}
}
+192
View File
@@ -1,5 +1,197 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
use crate::state::AppState;
use axum::{
extract::State,
http::StatusCode,
response::{IntoResponse, Response},
};
pub async fn health() -> &'static str {
"OK"
}
pub async fn ready(State(state): State<AppState>) -> Response {
readiness_report(
state.discovery_cache.has_snapshot().await,
state.config.invite_meta_enabled,
state.invite_meta.get().is_some(),
)
.into_response()
}
fn readiness_report(
discovery_cached: bool,
invite_meta_configured: bool,
invite_meta_connected: bool,
) -> (StatusCode, String) {
let mut degraded = Vec::new();
if invite_meta_configured && !invite_meta_connected {
degraded.push("invite_meta");
}
let suffix = if degraded.is_empty() {
String::new()
} else {
format!(" (degraded: {})", degraded.join(", "))
};
if discovery_cached {
(StatusCode::OK, format!("OK{suffix}"))
} else {
(
StatusCode::SERVICE_UNAVAILABLE,
format!("NOT READY: discovery{suffix}"),
)
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::config::AppProxyConfig;
use crate::discovery_cache::DiscoveryCache;
use crate::state::build_http_client;
use axum::Router;
use axum::body::Body;
use axum::http::{HeaderValue, Request as HttpRequest, header};
use fluxer_common::config::GeoipSourceConfig;
use fluxer_common::geoip::{GeoipConfig, GeoipResolver};
use std::sync::{Arc, OnceLock};
use tower::ServiceExt;
const DISCOVERY_BODY: &str = r#"{"api_code_version":"proxy-test"}"#;
async fn spawn_discovery_origin() -> String {
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let addr = listener.local_addr().unwrap();
let router = Router::new().fallback(|| async {
let mut response = Response::new(Body::from(DISCOVERY_BODY));
response.headers_mut().insert(
header::CONTENT_TYPE,
HeaderValue::from_static("application/json"),
);
response
});
tokio::spawn(async move {
axum::serve(listener, router).await.unwrap();
});
format!("http://{addr}/")
}
async fn probe_state(invite_meta_enabled: bool) -> AppState {
let mut config = AppProxyConfig::from_env();
config.invite_meta_enabled = invite_meta_enabled;
config.discovery_upstream_url = spawn_discovery_origin().await;
AppState {
config: Arc::new(config),
http_client: build_http_client().unwrap(),
discovery_cache: Arc::new(DiscoveryCache::new()),
geoip: Arc::new(GeoipResolver::from_config(&GeoipConfig {
geoip_source: GeoipSourceConfig::Filesystem {
maxmind_db_path: None,
},
geoip_s3_config: None,
trust_client_ip_header: false,
client_ip_header_name: "x-forwarded-for".to_owned(),
})),
invite_meta: Arc::new(OnceLock::new()),
index_html: None,
}
}
async fn warm_discovery(state: &AppState) {
state
.discovery_cache
.refresh(&state.http_client, &state.config.discovery_upstream_url)
.await
.unwrap();
}
async fn probe(state: AppState, path: &str) -> (StatusCode, String) {
let response = crate::routes::build_router(state)
.oneshot(
HttpRequest::builder()
.uri(path)
.body(Body::empty())
.unwrap(),
)
.await
.unwrap();
let status = response.status();
let body = axum::body::to_bytes(response.into_body(), usize::MAX)
.await
.unwrap();
(status, String::from_utf8(body.to_vec()).unwrap())
}
#[tokio::test]
async fn liveness_stays_constant_while_the_proxy_cannot_serve() {
let state = probe_state(false).await;
assert_eq!(
probe(state, "/_health").await,
(StatusCode::OK, "OK".to_owned())
);
}
#[tokio::test]
async fn readiness_fails_while_the_discovery_cache_is_empty() {
let state = probe_state(false).await;
let (status, body) = probe(state, "/_ready").await;
assert_eq!(status, StatusCode::SERVICE_UNAVAILABLE);
assert!(body.contains("discovery"), "{body}");
}
#[tokio::test]
async fn readiness_passes_once_a_discovery_snapshot_is_cached() {
let state = probe_state(false).await;
warm_discovery(&state).await;
assert_eq!(
probe(state, "/_ready").await,
(StatusCode::OK, "OK".to_owned())
);
}
#[tokio::test]
async fn readiness_survives_a_configured_invite_resolver_that_never_connects() {
let state = probe_state(true).await;
warm_discovery(&state).await;
let (status, body) = probe(state, "/_ready").await;
assert_eq!(status, StatusCode::OK);
assert!(body.contains("invite_meta"), "{body}");
}
#[tokio::test]
async fn an_empty_discovery_cache_fails_readiness_even_while_invite_metadata_is_degraded() {
let state = probe_state(true).await;
let (status, body) = probe(state, "/_ready").await;
assert_eq!(status, StatusCode::SERVICE_UNAVAILABLE);
assert!(body.contains("discovery"), "{body}");
assert!(body.contains("invite_meta"), "{body}");
}
#[test]
fn invite_metadata_is_reported_as_degraded_instead_of_gating_readiness() {
assert_eq!(readiness_report(true, false, false).0, StatusCode::OK);
assert_eq!(
readiness_report(true, true, true),
(StatusCode::OK, "OK".to_owned())
);
assert_eq!(
readiness_report(true, true, false),
(StatusCode::OK, "OK (degraded: invite_meta)".to_owned())
);
assert_eq!(
readiness_report(false, false, false),
(
StatusCode::SERVICE_UNAVAILABLE,
"NOT READY: discovery".to_owned()
)
);
assert_eq!(
readiness_report(false, true, false),
(
StatusCode::SERVICE_UNAVAILABLE,
"NOT READY: discovery (degraded: invite_meta)".to_owned()
)
);
}
}
+1
View File
@@ -34,6 +34,7 @@ const PERMISSIONS_POLICY_VALUE: &str = "accelerometer=(), camera=(self), ch-dpr=
pub fn build_router(state: AppState) -> Router {
Router::new()
.route("/_health", get(health::health))
.route("/_ready", get(health::ready))
.route(
"/.well-known/apple-app-site-association",
get(apple_association::apple_app_site_association),
+3 -3
View File
@@ -222,7 +222,7 @@ async fn resolve_invite_meta(
runtime_csp_sources: &RuntimeCspSources,
) -> Option<InvitePageMeta> {
let code = invite_code_from_path(request_path)?;
let resolver = state.invite_meta.as_ref()?;
let resolver = state.invite_meta.get()?;
let endpoints = InviteMetaEndpoints {
media_endpoint: runtime_csp_sources.media_endpoint.clone(),
static_cdn_endpoint: runtime_csp_sources.static_cdn_endpoint.clone(),
@@ -539,7 +539,7 @@ mod tests {
use axum::body::Body;
use fluxer_common::config::GeoipSourceConfig;
use fluxer_common::geoip::{GeoipConfig, GeoipResolver};
use std::sync::Arc;
use std::sync::{Arc, OnceLock};
#[test]
fn dev_asset_cache_buster_rewrites_script_and_link_assets() {
@@ -861,7 +861,7 @@ mod tests {
trust_client_ip_header: false,
client_ip_header_name: "x-forwarded-for".to_owned(),
})),
invite_meta: None,
invite_meta: Arc::new(OnceLock::new()),
index_html: cached_shell.map(Arc::from),
}
}
+2 -2
View File
@@ -4,7 +4,7 @@ use crate::config::AppProxyConfig;
use crate::discovery_cache::DiscoveryCache;
use crate::invite_meta::InviteMetaResolver;
use fluxer_common::geoip::GeoipResolver;
use std::sync::Arc;
use std::sync::{Arc, OnceLock};
use std::time::Duration;
#[derive(Clone)]
@@ -13,7 +13,7 @@ pub struct AppState {
pub http_client: reqwest::Client,
pub discovery_cache: Arc<DiscoveryCache>,
pub geoip: Arc<GeoipResolver>,
pub invite_meta: Option<Arc<InviteMetaResolver>>,
pub invite_meta: Arc<OnceLock<InviteMetaResolver>>,
pub index_html: Option<Arc<str>>,
}
+298
View File
@@ -196,6 +196,62 @@ pub fn trim_trailing_slash(value: &str) -> String {
value.trim_end_matches('/').to_owned()
}
fn is_default_port(scheme: &str, port: u16) -> bool {
matches!(
(scheme, port),
("http", 80) | ("https", 443) | ("ws", 80) | ("wss", 443)
)
}
fn strip_trailing_dot(host: &str) -> &str {
host.strip_suffix('.').unwrap_or(host)
}
pub fn normalize_public_endpoint(url: &str, base_domain: &str, public_port: Option<u16>) -> String {
let domain = base_domain.trim().to_lowercase();
let domain = strip_trailing_dot(&domain);
let Some(port) = public_port.filter(|port| *port != 0) else {
return url.to_owned();
};
if domain.is_empty() {
return url.to_owned();
}
let Ok(parsed) = reqwest::Url::parse(url) else {
return url.to_owned();
};
let host = parsed.host_str().unwrap_or_default().to_lowercase();
if strip_trailing_dot(&host) != domain {
return url.to_owned();
}
if is_default_port(parsed.scheme(), port) {
return url.to_owned();
}
let Some(scheme_end) = url.find("://") else {
return url.to_owned();
};
let authority_start = scheme_end + 3;
if url[authority_start..].starts_with('/') {
return url.to_owned();
}
let authority_end = url[authority_start..]
.find(['/', '\\', '?', '#'])
.map_or(url.len(), |index| authority_start + index);
let authority = &url[authority_start..authority_end];
let host = authority.rsplit('@').next().unwrap_or_default();
if host[host.rfind(']').map_or(0, |index| index + 1)..].contains(':') {
return url.to_owned();
}
format!("{}:{port}{}", &url[..authority_end], &url[authority_end..])
}
pub fn normalize_public_endpoint_from_env(url: &str) -> String {
normalize_public_endpoint(
url,
&read_env("FLUXER_BASE_DOMAIN", ""),
non_empty_env("FLUXER_PUBLIC_PORT").and_then(|port| port.parse().ok()),
)
}
#[cfg(test)]
mod tests {
use super::*;
@@ -284,6 +340,248 @@ mod tests {
assert_eq!(normalize_base_path("/"), "");
}
#[test]
fn default_https_install_is_untouched() {
for url in [
"https://fluxer.example",
"https://fluxer.example/media",
"https://fluxer.example/admin/oauth2_callback",
"wss://fluxer.example/gateway",
] {
assert_eq!(
url,
normalize_public_endpoint(url, "fluxer.example", Some(443))
);
}
}
#[test]
fn default_http_install_is_untouched() {
for url in [
"http://fluxer.example",
"http://fluxer.example/media",
"ws://fluxer.example/gateway",
] {
assert_eq!(
url,
normalize_public_endpoint(url, "fluxer.example", Some(80))
);
}
}
#[test]
fn inserts_a_non_default_port_for_the_base_domain() {
assert_eq!(
"http://fluxer.example:19080/media",
normalize_public_endpoint("http://fluxer.example/media", "fluxer.example", Some(19080))
);
assert_eq!(
"http://fluxer.example:19080",
normalize_public_endpoint("http://fluxer.example", "fluxer.example", Some(19080))
);
assert_eq!(
"https://fluxer.example:8443/admin/oauth2_callback",
normalize_public_endpoint(
"https://fluxer.example/admin/oauth2_callback",
"fluxer.example",
Some(8443)
)
);
}
#[test]
fn a_default_port_for_the_urls_own_scheme_is_never_inserted() {
assert_eq!(
"ws://fluxer.example/gateway",
normalize_public_endpoint("ws://fluxer.example/gateway", "fluxer.example", Some(80))
);
assert_eq!(
"wss://fluxer.example/gateway",
normalize_public_endpoint("wss://fluxer.example/gateway", "fluxer.example", Some(443))
);
assert_eq!(
"http://fluxer.example:443/media",
normalize_public_endpoint("http://fluxer.example/media", "fluxer.example", Some(443))
);
assert_eq!(
"https://fluxer.example:80/media",
normalize_public_endpoint("https://fluxer.example/media", "fluxer.example", Some(80))
);
}
#[test]
fn another_host_is_never_touched() {
for url in [
"https://cdn.example.net/assets",
"https://media.example.net",
"http://api:8080",
"http://media-proxy:8080",
] {
assert_eq!(
url,
normalize_public_endpoint(url, "fluxer.example", Some(19080))
);
}
assert_eq!(
"https://sub.fluxer.example/media",
normalize_public_endpoint(
"https://sub.fluxer.example/media",
"fluxer.example",
Some(19080)
)
);
}
#[test]
fn an_explicit_port_is_never_touched() {
for url in [
"http://fluxer.example:19080/media",
"http://fluxer.example:8080/media",
"https://fluxer.example:443/media",
"http://fluxer.example:80/media",
"http://user:[email protected]:19080/media",
] {
assert_eq!(
url,
normalize_public_endpoint(url, "fluxer.example", Some(19080))
);
}
}
#[test]
fn is_idempotent() {
let once =
normalize_public_endpoint("http://fluxer.example/media", "fluxer.example", Some(19080));
let twice = normalize_public_endpoint(&once, "fluxer.example", Some(19080));
assert_eq!("http://fluxer.example:19080/media", once);
assert_eq!(once, twice);
}
#[test]
fn an_unset_port_leaves_every_url_alone() {
assert_eq!(
"http://fluxer.example/media",
normalize_public_endpoint("http://fluxer.example/media", "fluxer.example", None)
);
assert_eq!(
"http://fluxer.example/media",
normalize_public_endpoint("http://fluxer.example/media", "fluxer.example", Some(0))
);
}
#[test]
fn an_empty_base_domain_leaves_every_url_alone() {
assert_eq!(
"http://fluxer.example/media",
normalize_public_endpoint("http://fluxer.example/media", "", Some(19080))
);
assert_eq!(
"http://fluxer.example/media",
normalize_public_endpoint("http://fluxer.example/media", " ", Some(19080))
);
}
#[test]
fn a_url_that_does_not_parse_is_returned_unchanged() {
for url in ["", "/api", "fluxer.example/media", "not a url", "://x"] {
assert_eq!(
url,
normalize_public_endpoint(url, "fluxer.example", Some(19080))
);
}
}
#[test]
fn matches_the_host_case_insensitively_and_ignores_a_trailing_dot() {
assert_eq!(
"http://FLUXER.example:19080/Media",
normalize_public_endpoint("http://FLUXER.example/Media", "Fluxer.Example", Some(19080))
);
assert_eq!(
"http://fluxer.example.:19080/media",
normalize_public_endpoint(
"http://fluxer.example./media",
"fluxer.example",
Some(19080)
)
);
assert_eq!(
"http://fluxer.example:19080/media",
normalize_public_endpoint(
"http://fluxer.example/media",
"fluxer.example.",
Some(19080)
)
);
}
#[test]
fn preserves_path_query_fragment_and_trailing_slash() {
assert_eq!(
"http://fluxer.example:19080/",
normalize_public_endpoint("http://fluxer.example/", "fluxer.example", Some(19080))
);
assert_eq!(
"http://fluxer.example:19080?a=1",
normalize_public_endpoint("http://fluxer.example?a=1", "fluxer.example", Some(19080))
);
assert_eq!(
"http://fluxer.example:19080#top",
normalize_public_endpoint("http://fluxer.example#top", "fluxer.example", Some(19080))
);
assert_eq!(
"http://fluxer.example:19080/media/x.png?v=1#frag",
normalize_public_endpoint(
"http://fluxer.example/media/x.png?v=1#frag",
"fluxer.example",
Some(19080)
)
);
}
#[test]
fn keeps_credentials_and_ipv6_literals_intact() {
assert_eq!(
"http://user:[email protected]:19080/media",
normalize_public_endpoint(
"http://user:[email protected]/media",
"fluxer.example",
Some(19080)
)
);
assert_eq!(
"http://[::1]:19080/media",
normalize_public_endpoint("http://[::1]/media", "[::1]", Some(19080))
);
assert_eq!(
"http://[::1]:8080/media",
normalize_public_endpoint("http://[::1]:8080/media", "[::1]", Some(19080))
);
}
#[test]
fn matches_the_typescript_normalizer_on_the_shared_vectors() {
let raw = include_str!("testdata/public_endpoint_vectors.json");
let vectors: serde_json::Value = serde_json::from_str(raw).expect("vectors parse as json");
let vectors = vectors.as_array().expect("vectors are an array");
assert!(!vectors.is_empty());
for vector in vectors {
let url = vector["url"].as_str().expect("vector carries a url");
let base_domain = vector["base_domain"]
.as_str()
.expect("vector carries a base domain");
let public_port = vector["public_port"].as_u64().map(|port| port as u16);
let expected = vector["normalized"]
.as_str()
.expect("vector carries a normalized url");
assert_eq!(
expected,
normalize_public_endpoint(url, base_domain, public_port),
"vector {url} @ {base_domain} port {public_port:?}"
);
}
}
#[test]
fn trim_trailing_slash_works() {
assert_eq!(
+440
View File
@@ -0,0 +1,440 @@
[
{
"url": "https://fluxer.example",
"base_domain": "fluxer.example",
"public_port": 443,
"normalized": "https://fluxer.example"
},
{
"url": "https://fluxer.example/media",
"base_domain": "fluxer.example",
"public_port": 443,
"normalized": "https://fluxer.example/media"
},
{
"url": "https://fluxer.example/admin",
"base_domain": "fluxer.example",
"public_port": 443,
"normalized": "https://fluxer.example/admin"
},
{
"url": "https://fluxer.example/admin/oauth2_callback",
"base_domain": "fluxer.example",
"public_port": 443,
"normalized": "https://fluxer.example/admin/oauth2_callback"
},
{
"url": "wss://fluxer.example/gateway",
"base_domain": "fluxer.example",
"public_port": 443,
"normalized": "wss://fluxer.example/gateway"
},
{
"url": "http://fluxer.example",
"base_domain": "fluxer.example",
"public_port": 80,
"normalized": "http://fluxer.example"
},
{
"url": "http://fluxer.example/media",
"base_domain": "fluxer.example",
"public_port": 80,
"normalized": "http://fluxer.example/media"
},
{
"url": "ws://fluxer.example/gateway",
"base_domain": "fluxer.example",
"public_port": 80,
"normalized": "ws://fluxer.example/gateway"
},
{
"url": "http://fluxer.example",
"base_domain": "fluxer.example",
"public_port": 19080,
"normalized": "http://fluxer.example:19080"
},
{
"url": "http://fluxer.example/",
"base_domain": "fluxer.example",
"public_port": 19080,
"normalized": "http://fluxer.example:19080/"
},
{
"url": "http://fluxer.example/media",
"base_domain": "fluxer.example",
"public_port": 19080,
"normalized": "http://fluxer.example:19080/media"
},
{
"url": "http://fluxer.example/api",
"base_domain": "fluxer.example",
"public_port": 19080,
"normalized": "http://fluxer.example:19080/api"
},
{
"url": "http://fluxer.example/admin",
"base_domain": "fluxer.example",
"public_port": 19080,
"normalized": "http://fluxer.example:19080/admin"
},
{
"url": "http://fluxer.example/admin/oauth2_callback",
"base_domain": "fluxer.example",
"public_port": 19080,
"normalized": "http://fluxer.example:19080/admin/oauth2_callback"
},
{
"url": "ws://fluxer.example/gateway",
"base_domain": "fluxer.example",
"public_port": 19080,
"normalized": "ws://fluxer.example:19080/gateway"
},
{
"url": "https://fluxer.example/admin/oauth2_callback",
"base_domain": "fluxer.example",
"public_port": 8443,
"normalized": "https://fluxer.example:8443/admin/oauth2_callback"
},
{
"url": "wss://fluxer.example/gateway",
"base_domain": "fluxer.example",
"public_port": 8443,
"normalized": "wss://fluxer.example:8443/gateway"
},
{
"url": "https://fluxer.example",
"base_domain": "fluxer.example",
"public_port": 8443,
"normalized": "https://fluxer.example:8443"
},
{
"url": "http://fluxer.example/media",
"base_domain": "fluxer.example",
"public_port": 443,
"normalized": "http://fluxer.example:443/media"
},
{
"url": "https://fluxer.example/media",
"base_domain": "fluxer.example",
"public_port": 80,
"normalized": "https://fluxer.example:80/media"
},
{
"url": "ws://fluxer.example/gateway",
"base_domain": "fluxer.example",
"public_port": 443,
"normalized": "ws://fluxer.example:443/gateway"
},
{
"url": "wss://fluxer.example/gateway",
"base_domain": "fluxer.example",
"public_port": 80,
"normalized": "wss://fluxer.example:80/gateway"
},
{
"url": "http://fluxer.example/media",
"base_domain": "fluxer.example",
"public_port": null,
"normalized": "http://fluxer.example/media"
},
{
"url": "http://fluxer.example/media",
"base_domain": "fluxer.example",
"public_port": 0,
"normalized": "http://fluxer.example/media"
},
{
"url": "http://fluxer.example/media",
"base_domain": "",
"public_port": 19080,
"normalized": "http://fluxer.example/media"
},
{
"url": "http://fluxer.example/media",
"base_domain": " ",
"public_port": 19080,
"normalized": "http://fluxer.example/media"
},
{
"url": "http://fluxer.example:19080/media",
"base_domain": "fluxer.example",
"public_port": 19080,
"normalized": "http://fluxer.example:19080/media"
},
{
"url": "http://fluxer.example:8080/media",
"base_domain": "fluxer.example",
"public_port": 19080,
"normalized": "http://fluxer.example:8080/media"
},
{
"url": "https://fluxer.example:443/media",
"base_domain": "fluxer.example",
"public_port": 19080,
"normalized": "https://fluxer.example:443/media"
},
{
"url": "http://fluxer.example:80/media",
"base_domain": "fluxer.example",
"public_port": 19080,
"normalized": "http://fluxer.example:80/media"
},
{
"url": "https://fluxer.example:8443/admin/oauth2_callback",
"base_domain": "fluxer.example",
"public_port": 8443,
"normalized": "https://fluxer.example:8443/admin/oauth2_callback"
},
{
"url": "ws://fluxer.example:19080/gateway",
"base_domain": "fluxer.example",
"public_port": 19080,
"normalized": "ws://fluxer.example:19080/gateway"
},
{
"url": "http://fluxer.example:8080?a=1",
"base_domain": "fluxer.example",
"public_port": 19080,
"normalized": "http://fluxer.example:8080?a=1"
},
{
"url": "https://cdn.example.net/assets",
"base_domain": "fluxer.example",
"public_port": 19080,
"normalized": "https://cdn.example.net/assets"
},
{
"url": "https://media.example.net",
"base_domain": "fluxer.example",
"public_port": 19080,
"normalized": "https://media.example.net"
},
{
"url": "https://sub.fluxer.example/media",
"base_domain": "fluxer.example",
"public_port": 19080,
"normalized": "https://sub.fluxer.example/media"
},
{
"url": "https://fluxer.example/media",
"base_domain": "example",
"public_port": 19080,
"normalized": "https://fluxer.example/media"
},
{
"url": "https://fluxer.example.evil.net/media",
"base_domain": "fluxer.example",
"public_port": 19080,
"normalized": "https://fluxer.example.evil.net/media"
},
{
"url": "http://api:8080",
"base_domain": "fluxer.example",
"public_port": 19080,
"normalized": "http://api:8080"
},
{
"url": "http://media-proxy:8080",
"base_domain": "fluxer.example",
"public_port": 19080,
"normalized": "http://media-proxy:8080"
},
{
"url": "http://FLUXER.example/Media",
"base_domain": "Fluxer.Example",
"public_port": 19080,
"normalized": "http://FLUXER.example:19080/Media"
},
{
"url": "HTTPS://fluxer.example/media",
"base_domain": "fluxer.example",
"public_port": 8443,
"normalized": "HTTPS://fluxer.example:8443/media"
},
{
"url": "http://fluxer.example./media",
"base_domain": "fluxer.example",
"public_port": 19080,
"normalized": "http://fluxer.example.:19080/media"
},
{
"url": "http://fluxer.example/media",
"base_domain": "fluxer.example.",
"public_port": 19080,
"normalized": "http://fluxer.example:19080/media"
},
{
"url": "http://fluxer.example./media",
"base_domain": "fluxer.example.",
"public_port": 19080,
"normalized": "http://fluxer.example.:19080/media"
},
{
"url": "http://fluxer.example?a=1",
"base_domain": "fluxer.example",
"public_port": 19080,
"normalized": "http://fluxer.example:19080?a=1"
},
{
"url": "http://fluxer.example#top",
"base_domain": "fluxer.example",
"public_port": 19080,
"normalized": "http://fluxer.example:19080#top"
},
{
"url": "http://fluxer.example/media/x.png?v=1#frag",
"base_domain": "fluxer.example",
"public_port": 19080,
"normalized": "http://fluxer.example:19080/media/x.png?v=1#frag"
},
{
"url": "http://fluxer.example/media/",
"base_domain": "fluxer.example",
"public_port": 19080,
"normalized": "http://fluxer.example:19080/media/"
},
{
"url": "http://fluxer.example/a%20b",
"base_domain": "fluxer.example",
"public_port": 19080,
"normalized": "http://fluxer.example:19080/a%20b"
},
{
"url": "http://fluxer.example#/media",
"base_domain": "fluxer.example",
"public_port": 19080,
"normalized": "http://fluxer.example:19080#/media"
},
{
"url": "http://user:[email protected]/media",
"base_domain": "fluxer.example",
"public_port": 19080,
"normalized": "http://user:[email protected]:19080/media"
},
{
"url": "http://user:[email protected]:19080/media",
"base_domain": "fluxer.example",
"public_port": 19080,
"normalized": "http://user:[email protected]:19080/media"
},
{
"url": "http://user:p@[email protected]/media",
"base_domain": "fluxer.example",
"public_port": 19080,
"normalized": "http://user:p@[email protected]:19080/media"
},
{
"url": "http://[::1]/media",
"base_domain": "[::1]",
"public_port": 19080,
"normalized": "http://[::1]:19080/media"
},
{
"url": "http://[::1]:8080/media",
"base_domain": "[::1]",
"public_port": 19080,
"normalized": "http://[::1]:8080/media"
},
{
"url": "http://127.0.0.1/media",
"base_domain": "127.0.0.1",
"public_port": 19080,
"normalized": "http://127.0.0.1:19080/media"
},
{
"url": "http://[2001:db8::1]/media",
"base_domain": "[2001:db8::1]",
"public_port": 19080,
"normalized": "http://[2001:db8::1]:19080/media"
},
{
"url": "http://[2001:DB8::1]/media",
"base_domain": "[2001:db8::1]",
"public_port": 19080,
"normalized": "http://[2001:DB8::1]:19080/media"
},
{
"url": "",
"base_domain": "fluxer.example",
"public_port": 19080,
"normalized": ""
},
{
"url": "/api",
"base_domain": "fluxer.example",
"public_port": 19080,
"normalized": "/api"
},
{
"url": "fluxer.example/media",
"base_domain": "fluxer.example",
"public_port": 19080,
"normalized": "fluxer.example/media"
},
{
"url": "not a url",
"base_domain": "fluxer.example",
"public_port": 19080,
"normalized": "not a url"
},
{
"url": "://x",
"base_domain": "fluxer.example",
"public_port": 19080,
"normalized": "://x"
},
{
"url": "file:///x",
"base_domain": "fluxer.example",
"public_port": 19080,
"normalized": "file:///x"
},
{
"url": "mailto:[email protected]",
"base_domain": "fluxer.example",
"public_port": 19080,
"normalized": "mailto:[email protected]"
},
{
"url": "http://fluxer.example\\evil",
"base_domain": "fluxer.example",
"public_port": 19080,
"normalized": "http://fluxer.example:19080\\evil"
},
{
"url": "https://fluxer.example:/media",
"base_domain": "fluxer.example",
"public_port": 8443,
"normalized": "https://fluxer.example:/media"
},
{
"url": "//fluxer.example/media",
"base_domain": "fluxer.example",
"public_port": 19080,
"normalized": "//fluxer.example/media"
},
{
"url": "https:fluxer.example/media",
"base_domain": "fluxer.example",
"public_port": 8443,
"normalized": "https:fluxer.example/media"
},
{
"url": "https:/fluxer.example/media",
"base_domain": "fluxer.example",
"public_port": 8443,
"normalized": "https:/fluxer.example/media"
},
{
"url": "https:////fluxer.example/media",
"base_domain": "fluxer.example",
"public_port": 8443,
"normalized": "https:////fluxer.example/media"
},
{
"url": "android:apk-key-hash:9NrCFDLVR8_ObZC-EWzaRA",
"base_domain": "fluxer.example",
"public_port": 8443,
"normalized": "android:apk-key-hash:9NrCFDLVR8_ObZC-EWzaRA"
}
]
+71 -65
View File
@@ -741,7 +741,7 @@ async function verifyPackagedNativeArtifacts(context) {
[
`Missing unpacked native runtime artifact(s) after packaging for ${platform}/${arch}:`,
...missing.map((entry) => ` - ${entry}`),
'Check electron-builder asarUnpack patterns and native package artifact sync.',
'Check electron-builder asar.unpack patterns and native package artifact sync.',
].join('\n'),
);
}
@@ -1203,7 +1203,8 @@ async function inspectAppImageLauncher(artifactPath) {
}
const appRunUsesNamespaceProbe = /unshare\s+(?:-Ur|--user)\s+true/.test(appRun);
if (!appRunUsesNamespaceProbe || !appRun.includes('NO_SANDBOX=--no-sandbox')) {
const appRunFallsBackToNoSandbox = /NO_SANDBOX=\(?--no-sandbox\)?/.test(appRun);
if (!appRunUsesNamespaceProbe || !appRunFallsBackToNoSandbox) {
violations.push('AppRun does not use the expected user-namespace probe before falling back to --no-sandbox');
}
@@ -1315,63 +1316,65 @@ module.exports = {
],
asar: {
smartUnpack: false,
unpack: [
'**/*.node',
'node_modules/@fluxer/win-process-loopback/*.node',
...winGameCaptureTargetArchs.map(
(arch) => `node_modules/@fluxer/win-game-capture/win-game-capture.win32-${arch}-msvc.node`,
),
'node_modules/@fluxer/win-clipboard/*.node',
'node_modules/@fluxer/win-shell/*.node',
'node_modules/@fluxer/win-toast/*.node',
'node_modules/@fluxer/linux-audio-capture/*.node',
'node_modules/@fluxer/linux-portals/*.node',
'node_modules/@fluxer/linux-screen-capture/*.node',
'node_modules/@fluxer/linux-screen-capture/obs-vkcapture/**/*',
'node_modules/@fluxer/linux-notifications/*.node',
'node_modules/@fluxer/linux-evdev/*.node',
'node_modules/@fluxer/system-hunspell/*.node',
'node_modules/@fluxer/macos-input-hook/*.node',
'node_modules/@fluxer/mac-app-audio/*.node',
'node_modules/@fluxer/mac-screen-capture/*.node',
'node_modules/@fluxer/mac-clipboard/*.node',
'node_modules/@fluxer/mac-sysctl/*.node',
'node_modules/@fluxer/mac-tcc/*.node',
'node_modules/@fluxer/windows-input-hook/*.node',
'node_modules/@fluxer/linux-input-hook/*.node',
'node_modules/@fluxer/platform-info/*.node',
'node_modules/@fluxer/webauthn/*.node',
'node_modules/@fluxer/webauthn/*.so*',
'node_modules/.pnpm/@fluxer+win-process-loopback@*/node_modules/@fluxer/win-process-loopback/*.node',
...winGameCaptureTargetArchs.map(
(arch) =>
`node_modules/.pnpm/@fluxer+win-game-capture@*/node_modules/@fluxer/win-game-capture/win-game-capture.win32-${arch}-msvc.node`,
),
'node_modules/.pnpm/@fluxer+win-clipboard@*/node_modules/@fluxer/win-clipboard/*.node',
'node_modules/.pnpm/@fluxer+win-shell@*/node_modules/@fluxer/win-shell/*.node',
'node_modules/.pnpm/@fluxer+win-toast@*/node_modules/@fluxer/win-toast/*.node',
'node_modules/.pnpm/@fluxer+windows-input-hook@*/node_modules/@fluxer/windows-input-hook/*.node',
'node_modules/.pnpm/@fluxer+linux-audio-capture@*/node_modules/@fluxer/linux-audio-capture/*.node',
'node_modules/.pnpm/@fluxer+linux-portals@*/node_modules/@fluxer/linux-portals/*.node',
'node_modules/.pnpm/@fluxer+linux-screen-capture@*/node_modules/@fluxer/linux-screen-capture/*.node',
'node_modules/.pnpm/@fluxer+linux-screen-capture@*/node_modules/@fluxer/linux-screen-capture/obs-vkcapture/**/*',
'node_modules/.pnpm/@fluxer+linux-notifications@*/node_modules/@fluxer/linux-notifications/*.node',
'node_modules/.pnpm/@fluxer+linux-evdev@*/node_modules/@fluxer/linux-evdev/*.node',
'node_modules/.pnpm/@fluxer+linux-input-hook@*/node_modules/@fluxer/linux-input-hook/*.node',
'node_modules/.pnpm/@fluxer+system-hunspell@*/node_modules/@fluxer/system-hunspell/*.node',
'node_modules/.pnpm/@fluxer+macos-input-hook@*/node_modules/@fluxer/macos-input-hook/*.node',
'node_modules/.pnpm/@fluxer+mac-app-audio@*/node_modules/@fluxer/mac-app-audio/*.node',
'node_modules/.pnpm/@fluxer+mac-screen-capture@*/node_modules/@fluxer/mac-screen-capture/*.node',
'node_modules/.pnpm/@fluxer+mac-clipboard@*/node_modules/@fluxer/mac-clipboard/*.node',
'node_modules/.pnpm/@fluxer+mac-sysctl@*/node_modules/@fluxer/mac-sysctl/*.node',
'node_modules/.pnpm/@fluxer+mac-tcc@*/node_modules/@fluxer/mac-tcc/*.node',
'node_modules/.pnpm/@fluxer+platform-info@*/node_modules/@fluxer/platform-info/*.node',
'node_modules/.pnpm/@fluxer+webauthn@*/node_modules/@fluxer/webauthn/*.node',
'node_modules/.pnpm/@fluxer+webauthn@*/node_modules/@fluxer/webauthn/*.so*',
],
},
asarUnpack: [
'**/*.node',
'node_modules/@fluxer/win-process-loopback/*.node',
...winGameCaptureTargetArchs.map(
(arch) => `node_modules/@fluxer/win-game-capture/win-game-capture.win32-${arch}-msvc.node`,
),
'node_modules/@fluxer/win-clipboard/*.node',
'node_modules/@fluxer/win-shell/*.node',
'node_modules/@fluxer/win-toast/*.node',
'node_modules/@fluxer/linux-audio-capture/*.node',
'node_modules/@fluxer/linux-portals/*.node',
'node_modules/@fluxer/linux-screen-capture/*.node',
'node_modules/@fluxer/linux-screen-capture/obs-vkcapture/**/*',
'node_modules/@fluxer/linux-notifications/*.node',
'node_modules/@fluxer/linux-evdev/*.node',
'node_modules/@fluxer/system-hunspell/*.node',
'node_modules/@fluxer/macos-input-hook/*.node',
'node_modules/@fluxer/mac-app-audio/*.node',
'node_modules/@fluxer/mac-screen-capture/*.node',
'node_modules/@fluxer/mac-clipboard/*.node',
'node_modules/@fluxer/mac-sysctl/*.node',
'node_modules/@fluxer/mac-tcc/*.node',
'node_modules/@fluxer/windows-input-hook/*.node',
'node_modules/@fluxer/linux-input-hook/*.node',
'node_modules/@fluxer/platform-info/*.node',
'node_modules/@fluxer/webauthn/*.node',
'node_modules/@fluxer/webauthn/*.so*',
'node_modules/.pnpm/@fluxer+win-process-loopback@*/node_modules/@fluxer/win-process-loopback/*.node',
...winGameCaptureTargetArchs.map(
(arch) =>
`node_modules/.pnpm/@fluxer+win-game-capture@*/node_modules/@fluxer/win-game-capture/win-game-capture.win32-${arch}-msvc.node`,
),
'node_modules/.pnpm/@fluxer+win-clipboard@*/node_modules/@fluxer/win-clipboard/*.node',
'node_modules/.pnpm/@fluxer+win-shell@*/node_modules/@fluxer/win-shell/*.node',
'node_modules/.pnpm/@fluxer+win-toast@*/node_modules/@fluxer/win-toast/*.node',
'node_modules/.pnpm/@fluxer+windows-input-hook@*/node_modules/@fluxer/windows-input-hook/*.node',
'node_modules/.pnpm/@fluxer+linux-audio-capture@*/node_modules/@fluxer/linux-audio-capture/*.node',
'node_modules/.pnpm/@fluxer+linux-portals@*/node_modules/@fluxer/linux-portals/*.node',
'node_modules/.pnpm/@fluxer+linux-screen-capture@*/node_modules/@fluxer/linux-screen-capture/*.node',
'node_modules/.pnpm/@fluxer+linux-screen-capture@*/node_modules/@fluxer/linux-screen-capture/obs-vkcapture/**/*',
'node_modules/.pnpm/@fluxer+linux-notifications@*/node_modules/@fluxer/linux-notifications/*.node',
'node_modules/.pnpm/@fluxer+linux-evdev@*/node_modules/@fluxer/linux-evdev/*.node',
'node_modules/.pnpm/@fluxer+linux-input-hook@*/node_modules/@fluxer/linux-input-hook/*.node',
'node_modules/.pnpm/@fluxer+system-hunspell@*/node_modules/@fluxer/system-hunspell/*.node',
'node_modules/.pnpm/@fluxer+macos-input-hook@*/node_modules/@fluxer/macos-input-hook/*.node',
'node_modules/.pnpm/@fluxer+mac-app-audio@*/node_modules/@fluxer/mac-app-audio/*.node',
'node_modules/.pnpm/@fluxer+mac-screen-capture@*/node_modules/@fluxer/mac-screen-capture/*.node',
'node_modules/.pnpm/@fluxer+mac-clipboard@*/node_modules/@fluxer/mac-clipboard/*.node',
'node_modules/.pnpm/@fluxer+mac-sysctl@*/node_modules/@fluxer/mac-sysctl/*.node',
'node_modules/.pnpm/@fluxer+mac-tcc@*/node_modules/@fluxer/mac-tcc/*.node',
'node_modules/.pnpm/@fluxer+platform-info@*/node_modules/@fluxer/platform-info/*.node',
'node_modules/.pnpm/@fluxer+webauthn@*/node_modules/@fluxer/webauthn/*.node',
'node_modules/.pnpm/@fluxer+webauthn@*/node_modules/@fluxer/webauthn/*.so*',
],
compression: 'normal',
npmRebuild: false,
nativeModules: {
npmRebuild: false,
},
protocols: [
{
name: appId,
@@ -1387,18 +1390,21 @@ module.exports = {
},
mac: {
category: 'public.app-category.social-networking',
x64ArchFiles: '**/@fluxer/**/*.node',
universal: {
x64ArchFiles: '**/@fluxer/**/*.node',
},
minimumSystemVersion: macOSMinimumSystemVersion,
icon: `build_resources/${iconDir}/_compiled/AppIcon.icns`,
darkModeSupport: true,
hardenedRuntime: true,
gatekeeperAssess: false,
notarize: true,
provisioningProfile,
entitlements: isCanary
? 'build_resources/entitlements.mac.canary.plist'
: 'build_resources/entitlements.mac.stable.plist',
entitlementsInherit: 'build_resources/entitlements.mac.inherit.plist',
sign: {
hardenedRuntime: true,
provisioningProfile,
entitlements: isCanary
? 'build_resources/entitlements.mac.canary.plist'
: 'build_resources/entitlements.mac.stable.plist',
entitlementsInherit: 'build_resources/entitlements.mac.inherit.plist',
},
target: [
{
target: 'dmg',
+2 -2
View File
@@ -420,9 +420,9 @@ checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a"
[[package]]
name = "rtrb"
version = "0.3.4"
version = "0.3.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4ade083ccbb4bf536df69d1f6432cc23deb7acccff86b183f3923a6fd56a1153"
checksum = "fae8ee26b0371a29a77d2b2d6b3ae13aa81def6f9bf1b1b92a32d279a5e709b7"
[[package]]
name = "rustversion"
+2 -2
View File
@@ -865,9 +865,9 @@ checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a"
[[package]]
name = "rtrb"
version = "0.3.4"
version = "0.3.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4ade083ccbb4bf536df69d1f6432cc23deb7acccff86b183f3923a6fd56a1153"
checksum = "fae8ee26b0371a29a77d2b2d6b3ae13aa81def6f9bf1b1b92a32d279a5e709b7"
[[package]]
name = "rustc-hash"

Some files were not shown because too many files have changed in this diff Show More