feat(push): relay notifications as encrypted web push (#2906)

fluxer_docs/astro.config.ts

@@ -200,6 +200,7 @@ export default defineConfig({
 						'http-api/users/relationships',
 						'http-api/users/notes',
 						'http-api/users/private-channels',
+						'http-api/users/push-notifications',
 						'http-api/users/content',
 						'http-api/users/gifts',
 						'http-api/users/data-harvest',

fluxer_docs/scripts/VerifyDocsCoverage.ts

@@ -102,8 +102,6 @@ const MAIN_SPEC_EXEMPT = new Map<string, {file: string; anchor: string; reason:
 
 const DELIBERATELY_UNDOCUMENTED = new Map([
 	['GET /users/@me/mobile-devices', 'mobile notifications, backported separately'],
-	['POST /users/@me/mobile-devices', 'mobile notifications, backported separately'],
-	['POST /users/@me/mobile-devices/unregister', 'mobile notifications, backported separately'],
 	['DELETE /users/@me/mobile-devices/{}', 'mobile notifications, backported separately'],
 	['GET /users/@me/push/subscriptions', 'push API, backported separately'],
 	['POST /users/@me/push/subscribe', 'push API, backported separately'],

fluxer_docs/scripts/VerifyDocsStyle.ts

@@ -34,7 +34,6 @@ const FORBIDDEN_MARKETING_WORDS = [
 ];
 
 const FORBIDDEN_TOPICS = [
-	{pattern: /mobile[- ]device/iu, reason: 'mobile notifications API does not exist in the live era'},
 	{pattern: /push subscription/iu, reason: 'push API does not exist in the live era'},
 	{pattern: /\/push\/events/u, reason: 'push events API does not exist in the live era'},
 	{pattern: /voice[- ]public[- ]key/iu, reason: 'voice connection API does not exist in the live era'},

fluxer_docs/src/content/docs/admin-api/instance.mdx

@@ -36,6 +36,7 @@ Missing settings use the defaults documented below. Invalid stored configuration
 | gateway_rollout | [Gateway rollout configuration](#gateway-rollout-configuration-object) object | Gateway admission and dispatch tuning |
 | voice_noise_suppression | [voice noise suppression configuration](#voice-noise-suppression-configuration-object) object | Client-side noise suppression rollout |
 | screen_share_delivery | [screen share delivery configuration](#screen-share-delivery-configuration-object) object | Reworked screen share delivery rollout |
+| push_service_delivery | [push service delivery configuration](#push-service-delivery-configuration-object) object | Push service delivery rollout |
 | experiment_delivery | [experiment delivery configuration](#experiment-delivery-configuration-object) object | Cadence every client polls the experiments route on |
 | registration | [registration configuration](#registration-configuration-object) object | Registration policy, issued URLs, and pending registrations |
 | self_hosted | boolean | Whether the deployment runs in self-hosted mode |
@@ -142,6 +143,23 @@ Every field is present on read. An absent document or missing field uses the def
 
 How often a client revalidates this rollout is not set here. It is set once for every experiment in the [experiment delivery configuration](#experiment-delivery-configuration-object) below.
 
+## Push service delivery configuration object
+
+The instance rollout of push service delivery.
+
+### Structure
+
+| Field | Type | Description |
+| --- | --- | --- |
+| enabled | boolean | Whether the rollout runs at all (default false) |
+| config_version | integer | Revision counter, raised by Fluxer and never accepted from a request |
+| rollout_basis_points | integer | Share of accounts the rollout selects, in basis points (0-10000, default 0) |
+| rollout_salt | string | Salt of the sampling hash (1-64 printable ASCII characters, default `push-service-delivery-v1`) |
+| included_user_ids | array[snowflake] | Accounts the rollout always selects, up to 1000 entries (default empty) |
+| excluded_user_ids | array[snowflake] | Accounts the rollout never selects, up to 1000 entries (default empty) |
+
+Every field is present on read. An absent document or missing field uses the defaults above.
+
 ## Experiment delivery configuration object
 
 How often a client polls [Get experiment assignments](/http-api/experiments/#get-experiment-assignments), and how widely those polls are spread. The setting is instance-wide and applies to every experiment at once, so adding an experiment adds no second cadence to tune.
@@ -551,6 +569,7 @@ The body has one optional object for each section. Fluxer leaves an absent secti
 | gateway_rollout? | object | Any subset of the [Gateway rollout configuration](#gateway-rollout-configuration-object) fields, each bound as documented there |
 | voice_noise_suppression? | object | Any subset of the [noise suppression](#voice-noise-suppression-configuration-object) fields |
 | screen_share_delivery? | object | Any subset of the [screen share delivery](#screen-share-delivery-configuration-object) fields |
+| push_service_delivery? | object | Any subset of the [push service delivery](#push-service-delivery-configuration-object) fields |
 | experiment_delivery? | object | Any subset of the [experiment delivery](#experiment-delivery-configuration-object) fields |
 | registration? | object | `mode` and `admin_registration_urls_enabled` |
 | app_public?<sup>2</sup> | object | `branding`, `setup`, `legal`, and `registration` sub-objects, each merged field by field |
@@ -566,6 +585,8 @@ The body has one optional object for each section. Fluxer leaves an absent secti
 
 `screen_share_delivery` works the same way, over the [screen share delivery configuration](#screen-share-delivery-configuration-object) fields and its own `config_version`.
 
+`push_service_delivery` works the same way, over the [push service delivery configuration](#push-service-delivery-configuration-object) fields and its own `config_version`.
+
 `experiment_delivery` takes both [experiment delivery configuration](#experiment-delivery-configuration-object) fields, each bound as documented there. It is a section of its own, so a write to it changes no `config_version` and changes no assignment, only the cadence on which clients ask for one.
 
 <sup>3</sup> A secret such as `klipy_api_key`, `api_key`, `hcaptcha_secret_key`, `turnstile_secret_key`, or the SMTP `password` is written when supplied and left alone when absent. `integrations.bluesky.keys` is the only way to write the Bluesky signing keys counted as `bluesky.key_count`. It takes up to 8 entries of `kid` (1-255 characters) and nullable `private_key` (up to 10000 characters), and replaces the stored key set outright
@@ -602,7 +623,7 @@ Fluxer skips URL validation while the merged configuration leaves single sign-on
 | 400 | [error response](/admin-api/#error-response) | A policy transition is refused, returned as `INSTANCE_POLICY_TRANSITION_NOT_ALLOWED` |
 
 :::caution[Sections are applied one after another]
-The order is `gateway_rollout`, `voice_noise_suppression`, `screen_share_delivery`, `experiment_delivery`, `sso`, `registration`, `app_public` branding, legal, and registration fields, `integrations`, `media`, `policy`, and finally `app_public.setup`. A failure part way through leaves the earlier sections written.
+The order is `gateway_rollout`, `voice_noise_suppression`, `screen_share_delivery`, `push_service_delivery`, `experiment_delivery`, `sso`, `registration`, `app_public` branding, legal, and registration fields, `integrations`, `media`, `policy`, and finally `app_public.setup`. A failure part way through leaves the earlier sections written.
 :::
 
 ### Side effects

fluxer_docs/src/content/docs/http-api/users/push-notifications.mdx

@@ -0,0 +1,152 @@
+---
+# SPDX-License-Identifier: AGPL-3.0-or-later
+title: Push notifications
+description: Registering a device for push delivery and decrypting what arrives.
+---
+
+import RouteHeader from '@/components/RouteHeader.astro';
+
+Fluxer delivers a notification to a registered device as [RFC 8291](https://datatracker.ietf.org/doc/html/rfc8291) Web Push. The client generates a P-256 key pair and an auth secret, registers the public half of the pair, and decrypts each delivery with the private half.
+
+Every route here requires a user session. A bot or OAuth2 bearer credential is refused with 403 `ACCESS_DENIED`. An account with an outstanding required action is refused with 403 `ACCOUNT_SUSPICIOUS_ACTIVITY`.
+
+## Registration shapes
+
+A registration takes one of two shapes on every platform.
+
+| Shape | What `token` holds | Keys |
+| --- | --- | --- |
+| Web Push | A publicly routable endpoint URL | Both `encryption_key` and `auth_secret` |
+| Legacy | A raw vendor device token | Neither key is sent |
+
+Fluxer reads the shape from the body rather than from `platform`. A `token` that parses as a URL without both keys is refused, and so is a pair of keys sent with a raw vendor token.
+
+`platform` names the transport the device was reached on.
+
+| Value | Meaning |
+| --- | --- |
+| `android_fcm` | Firebase Cloud Messaging |
+| `ios_apns` | Apple Push Notification service |
+| `android_unified_push` | UnifiedPush, on an Android build without Google services |
+
+`android_unified_push` is always a Web Push registration. Sending it with no keys is refused.
+
+## Device registration object
+
+The identifier Fluxer assigns to one registration.
+
+### Structure
+
+| Field | Type | Description |
+| --- | --- | --- |
+| device_id | string | The registration identifier, 32 lowercase hexadecimal characters |
+
+Fluxer derives the identifier from `platform`, `app_id`, `provider_environment`, and `token`. The same four values always produce the same identifier, and registering them twice replaces the stored entry.
+
+## Register mobile push device
+
+<RouteHeader method="POST" path="/v1/users/@me/mobile-devices" />
+
+Stores a push registration for the current account and returns its [device registration](#device-registration-object) object.
+
+### JSON body
+
+| Field | Type | Description |
+| --- | --- | --- |
+| platform | string | The [platform value](#registration-shapes) the device was reached on |
+| token<sup>1</sup> | string | The endpoint URL, or the raw vendor token on a legacy registration |
+| encryption_key?<sup>2</sup> | string | The base64url P-256 public key (1-1024 characters) |
+| auth_secret?<sup>2</sup> | string | The base64url auth secret (1-1024 characters) |
+| app_id? | string | The client build, such as `stable`, `beta`, or `canary` (default `stable`) |
+| provider_environment? | string | `production` or `development` |
+| user_agent? | string | A user agent string describing the device (1-1024 characters) |
+
+<sup>1</sup> 1 to 4096 characters. A value that parses as a URL is a Web Push registration and needs both keys, and a value that does not must be sent with neither
+
+<sup>2</sup> Sent together or not at all. Sending one alone is refused at the missing field
+
+An `ios_apns` registration with no `provider_environment` is stored as `production`. Every other platform stores no environment.
+
+Register an `https` endpoint. A Web Push registration whose `token` is not a valid URL is refused at `token`, and one whose host is a private or reserved address is refused with `URL_NOT_PUBLICLY_ROUTABLE`.
+
+### Response
+
+| Status | Body | Condition |
+| --- | --- | --- |
+| 200 | [device registration](#device-registration-object) object | The registration was stored |
+| 400 | [error response](/http-api/#error-response) | The body matches neither shape and the request returns `INVALID_FORM_BODY` |
+
+### Rate limit
+
+20 requests per minute for each authenticated user, on the `user:push:subscribe` bucket.
+
+## Unregister mobile push device
+
+<RouteHeader method="POST" path="/v1/users/@me/mobile-devices/unregister" />
+
+Removes the registration named by the values the client already holds. Returns 200 whether or not a registration was there.
+
+The four values below identify the registration the same way [Register mobile push device](#register-mobile-push-device) does. A value that differs from the one sent at registration names a different registration and removes nothing.
+
+### JSON body
+
+| Field | Type | Description |
+| --- | --- | --- |
+| platform | string | The [platform value](#registration-shapes) sent at registration |
+| token | string | The endpoint URL or raw vendor token sent at registration (1-4096 characters) |
+| app_id? | string | The client build sent at registration (default `stable`) |
+| provider_environment? | string | The environment sent at registration |
+
+### Response body
+
+| Field | Type | Description |
+| --- | --- | --- |
+| success | boolean | Whether the removal ran, always true |
+
+### Response
+
+| Status | Body | Condition |
+| --- | --- | --- |
+| 200 | response body | The registration was removed, or there was none |
+| 400 | [error response](/http-api/#error-response) | `platform` or `token` is missing or malformed and the request returns `INVALID_FORM_BODY` |
+
+### Rate limit
+
+40 requests per minute for each authenticated user, on the `user:push:unsubscribe` bucket.
+
+## Handling an incoming push
+
+Fluxer posts one encrypted record to the registered endpoint for each notification. The push service that owns the endpoint hands that record to the client.
+
+| Header | Value |
+| --- | --- |
+| Content-Encoding | Always `aes128gcm` |
+| Content-Type | Always `application/octet-stream` |
+| TTL | `86400` on a notification and `3600` on a clear |
+| Urgency | `high` on a notification and `low` on a clear |
+| Authorization | A VAPID token and the instance public key |
+
+The body is one `aes128gcm` record encrypted to the `encryption_key` and `auth_secret` the client registered. The client decrypts it locally with the private half of its key pair and its auth secret. Fluxer holds no key that opens the record after it is sealed.
+
+A record is 2816 bytes and its plaintext is at most 2713 bytes of JSON. A notification too large for that is shrunk before it is encrypted, one step at a time, until it fits.
+
+| Step | Effect |
+| --- | --- |
+| First | Media fields are dropped |
+| Second | Icon fields are dropped |
+| Third | The body text is shortened |
+| Last | Only a minimal payload is left |
+
+A client has to tolerate a missing field.
+
+Two kinds of payload arrive. A notification payload describes something to show. A clear payload sets `type` to `notification_clear` and `action` to `clear_channel`, and asks the client to dismiss what it already showed for one channel.
+
+An endpoint that answers 404 or 410 removes the registration. Fluxer retries a transient failure and keeps the registration.
+
+### When decryption fails
+
+A record that does not decrypt cannot be recovered. Discard it and show nothing.
+
+Decryption fails when the registered keys no longer match the pair the client holds. Regenerating the key pair without registering again does that. Fluxer sees none of it. The push service already answered 2xx and the registration stays live.
+
+The client is the only party that can repair it. Unregister the stale entry, then register again with the current public key and auth secret.

fluxer_docs/src/content/docs/operator/configuration.mdx

@@ -813,11 +813,11 @@ The Gateway reads the same names. A malformed pair, or a private key that does n
 
 ## Mobile push
 
-Both `api` and `gateway` read the APNs and FCM names. None appear in `.env.example` or in `docker-compose.yml`, so configuring mobile push means editing the Compose file. All are optional.
+`api`, `gateway`, and `push` read the APNs and FCM names. None appear in `.env.example` or in `docker-compose.yml`, so configuring mobile push means editing the Compose file. All are optional.
 
 #### `FLUXER_PUSH_APNS_ENABLED`
 
-Default `false`. The APNs switch. Must be set on both `api` and `gateway`.
+Default `false`. The APNs switch. Must be set on `api`, `gateway`, and `push`.
 
 #### `FLUXER_PUSH_APNS_TEAM_ID`
 
@@ -845,7 +845,7 @@ Default `[]`. Per-app APNs configuration. JSON array. An entry with no `app_id`
 
 #### `FLUXER_PUSH_FCM_ENABLED`
 
-Default `false`. The FCM switch. Must be set on both `api` and `gateway`.
+Default `false`. The FCM switch. Must be set on `api`, `gateway`, and `push`.
 
 #### `FLUXER_PUSH_FCM_PROJECT_ID`
 
@@ -875,6 +875,34 @@ Default `https://oauth2.googleapis.com/token`. The OAuth token endpoint. Change
 
 Default `[]`. Per-app FCM configuration. JSON array, under the same `app_id` rule as APNs.
 
+## Push service settings
+
+`push` is the push notification service in the bundled stack. It reads the VAPID pair from [Web push](#web-push), the APNs and FCM names from [Mobile push](#mobile-push), and `FLUXER_SVC_NATS_URL` with `FLUXER_NATS_AUTH_TOKEN` from [Message bus and internal services](#message-bus-and-internal-services). It refuses to start without `FLUXER_INTERNAL_API_ENDPOINT` and `FLUXER_GATEWAY_RPC_AUTH_TOKEN`. Compose supplies both. The names below belong to `push`. All are optional.
+
+#### `FLUXER_PUSH_SERVICE_HOST`
+
+Default `0.0.0.0`. The bind address. It must parse as an IP address. Also settable with `--bind-host`. Compose sets `0.0.0.0`.
+
+#### `FLUXER_PUSH_SERVICE_PORT`
+
+Default `8126`. The listen port for the health and metrics endpoints. Also settable with `--port`. Compose sets `8126`.
+
+#### `FLUXER_PUSH_SERVICE_QUEUE_CAPACITY`
+
+Default `10000`. Notification jobs `push` holds at once. Accepts 1 to 1000000. Compose passes it through from `.env`. An empty value keeps the default.
+
+#### `FLUXER_PUSH_SERVICE_SEND_CONCURRENCY`
+
+Default `256`. Provider requests in flight at once, across every job. Accepts 1 to 65536. Compose passes it through from `.env`. An empty value keeps the default.
+
+#### `FLUXER_PUSH_SERVICE_APNS_BASE_URL`
+
+No default. Replaces the APNs host in both environments. Set it only for a test double.
+
+#### `FLUXER_PUSH_SERVICE_FCM_BASE_URL`
+
+Default `https://fcm.googleapis.com`. The FCM host. Set it only for a proxy or a test double.
+
 ## Payments
 
 Stripe billing, which the shipped stack keeps off. All are optional.
@@ -1077,7 +1105,7 @@ Defaults to `debug` in development, `info` otherwise. The Node log level. Read b
 
 #### `RUST_LOG`
 
-Default `info`. The Rust log filter. Read by `media-proxy`, `app-proxy`, `admin`, and the internal services. Not in `.env.example` or the Compose file.
+Default `info`. The Rust log filter. Read by `media-proxy`, `app-proxy`, `admin`, `push`, and the internal services. Not in `.env.example` or the Compose file.
 
 #### `FLUXER_GATEWAY_LOGGER_LEVEL`
 
@@ -1111,7 +1139,7 @@ Default `development`. The runtime mode. `development`, `production`, or `test`.
 
 Default `false`. The self-host switch. Compose sets `true`. It relaxes the production Postgres SSL requirement, seeds the limit tier, gates registration, billing and discovery controllers, and turns blocklist feeds off.
 
-`/_metrics` on `api`, `media-proxy`, and `gateway`, plus the Gateway's `/_health/ready`, `/_health/drain`, and `/_health/undrain`, are gated to loopback peers, so no proxy reaches them. The probes that work from outside are `/api/_health`, `/gateway/_health`, `/media/_health`, and the edge's own `/_health`.
+`/_metrics` on `api`, `media-proxy`, `gateway`, and `push`, plus the Gateway's `/_health/ready`, `/_health/drain`, and `/_health/undrain`, are gated to loopback peers, so no proxy reaches them. The probes that work from outside are `/api/_health`, `/gateway/_health`, `/media/_health`, and the edge's own `/_health`.
 
 ## Feature flags and development switches
 
@@ -1663,7 +1691,7 @@ See [Bluesky connections](#bluesky-connections) for configuration.
 
 #### `FLUXER_PUSH_APNS_` and `FLUXER_PUSH_FCM_`
 
-Mobile push cannot be configured at all from the example.
+Mobile push cannot be configured at all from the example. `api`, `gateway`, and `push` all read these names. An override has to reach all three.
 
 #### `RUST_LOG`, `LOG_LEVEL` and `LOGGER_LEVEL`
 
@@ -1742,6 +1770,7 @@ The stack runs its containers on one Docker bridge network, which is private to
 | worker | fluxer-api | Background lanes and the cron scheduler |
 | gateway | fluxer-gateway | The Gateway WebSocket |
 | media-proxy | fluxer-media-proxy | Uploads, transforms, and media delivery |
+| push | fluxer-push | Push notification delivery |
 | admin | fluxer-admin | The admin dashboard |
 | snowflakes, snowflakes-shard | fluxer-snowflakes | Identifier allocation |
 | users, users-shard | fluxer-users | User reads and writes |
@@ -1758,7 +1787,7 @@ The stack runs its containers on one Docker bridge network, which is private to
 
 The edge and LiveKit are the only services that publish ports. The edge publishes 80/tcp, 443/tcp, 443/udp, or one plain-HTTP port under the overlay. LiveKit publishes 7881/tcp and 7882/udp. Everything else is reachable only over the bridge network.
 
-`api` is the one service an operator configures directly, through the shared environment block. `worker`, `gateway`, `app-proxy`, and `media-proxy` are touched rarely, `worker` for lane concurrency, `app-proxy` for CSP extras, and `media-proxy` for transform limits. The edge takes only the `FLUXER_EDGE_` variables, `postgres` only the password, `meilisearch` only the master key, `valkey` only the `FLUXER_VALKEY_` tuning values, and `livekit` only the key pair and the port variables. `static-proxy` reads no environment variables, and the remaining services need none.
+`api` is the one service an operator configures directly, through the shared environment block. `worker`, `gateway`, `app-proxy`, `media-proxy`, and `push` are touched rarely, `worker` for lane concurrency, `app-proxy` for CSP extras, `media-proxy` for transform limits, and `push` for queue capacity and send concurrency. The edge takes only the `FLUXER_EDGE_` variables, `postgres` only the password, `meilisearch` only the master key, `valkey` only the `FLUXER_VALKEY_` tuning values, and `livekit` only the key pair and the port variables. `static-proxy` reads no environment variables, and the remaining services need none.
 
 The internal services each run a router, which takes requests and holds no state, and one shard, which holds the caches and the database connections. The shipped stack fixes `FLUXER_SVC_SHARD_COUNT` at `1`.
 
@@ -1766,7 +1795,7 @@ The internal services each run a router, which takes requests and holds no state
 
 Every service has a memory limit and four also have a memory reservation, all under `deploy.resources`. Compose reads `gb` as 1024 MiB and `mb` as 1 MiB, so `5gb` is 5368709120 bytes. Plain `docker compose up` applies both keys on a single host, with no Swarm and no `--compatibility` flag. The engine rejects any limit below `6mb`, and rejects a limit lower than the same service's reservation with `Minimum memory limit can not be less than memory reservation limit`.
 
-A limit is a ceiling. The limits below sum to 16.75 GiB and the stack does not need a host that large, because each container uses only the memory it allocates, up to its limit.
+A limit is a ceiling. The limits below sum to 18.5 GiB and the stack does not need a host that large, because each container uses only the memory it allocates, up to its limit.
 
 `deploy.resources.reservations.memory` becomes the container's cgroup v2 `memory.low`, which biases kernel reclaim towards other containers under host pressure. It reserves nothing on its own.
 
@@ -1828,6 +1857,10 @@ Default `1gb`. The ceiling for `gateway`. The BEAM has no heap ceiling of its ow
 
 Default `512mb`. The ceiling for `media-proxy`. Image and video transforms decode into this ceiling, so a large upload is what reaches this limit.
 
+#### `FLUXER_PUSH_MEMORY_LIMIT`
+
+Default `256mb`. The ceiling for `push`. Raise it alongside `FLUXER_PUSH_SERVICE_QUEUE_CAPACITY` or `FLUXER_PUSH_SERVICE_SEND_CONCURRENCY`.
+
 #### `FLUXER_STATIC_PROXY_MEMORY_LIMIT`
 
 Default `256mb`. The ceiling for `static-proxy`. The service reads no environment variables and serves files only.

fluxer_docs/src/content/docs/operator/reverse-proxy.mdx

@@ -402,7 +402,7 @@ Pass the query string on `/gateway` through untouched. Clients always send `?v=`
 
 Apple and Google require the association files at those fixed paths for saved-password autofill and app links. A proxy that forwards all paths needs no extra rules. Include them explicitly if you use a path allowlist.
 
-`/_metrics` on the API, Media Proxy, and Gateway, plus `/_health/ready`, `/_health/drain`, and `/_health/undrain` on the Gateway, are gated to loopback and are unreachable through any proxy. The probes that work through a proxy are `/_health`, `/api/_health`, `/gateway/_health`, and `/media/_health`.
+`/_metrics` on the API, Media Proxy, Gateway, and push service, plus `/_health/ready`, `/_health/drain`, and `/_health/undrain` on the Gateway, are gated to loopback and are unreachable through any proxy. The push service has no public path. The probes that work through a proxy are `/_health`, `/api/_health`, `/gateway/_health`, and `/media/_health`.
 
 ## Trusted proxies