mirror of
https://github.com/fluxerapp/fluxer
synced 2026-10-08 03:32:27 +09:00
Compare commits
69
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
6c36d934f7 | ||
|
|
63e3be5750 | ||
|
|
cdecda7f78 | ||
|
|
4ad2858773 | ||
|
|
fda41bb57a | ||
|
|
ce08f82a92 | ||
|
|
ef067f36c6 | ||
|
|
fc2b6b5299 | ||
|
|
55846b24ea | ||
|
|
20a15ac11d | ||
|
|
667ac7da8e | ||
|
|
1b81c14c48 | ||
|
|
ceec183d38 | ||
|
|
69ddc07ebb | ||
|
|
2f008b8653 | ||
|
|
3d38d3f694 | ||
|
|
ad86a04e67 | ||
|
|
600c15e17d | ||
|
|
08c9fe9886 | ||
|
|
43924e3ac5 | ||
|
|
824b5c86c9 | ||
|
|
a2a68847fd | ||
|
|
2019909a5e | ||
|
|
d46c8d49c6 | ||
|
|
45530ebbf5 | ||
|
|
9cdad046b1 | ||
|
|
b6c6928073 | ||
|
|
dd1ee999a4 | ||
|
|
c506d6d5e3 | ||
|
|
8a65832a65 | ||
|
|
fd6ae4abd7 | ||
|
|
24b84c419c | ||
|
|
746a75187a | ||
|
|
10ba2ca896 | ||
|
|
977b6767cd | ||
|
|
f00c6ee47a | ||
|
|
82859dc2f6 | ||
|
|
328dc06ab0 | ||
|
|
ea6e4a75db | ||
|
|
69ca462930 | ||
|
|
f38619d974 | ||
|
|
6c0ce9369b | ||
|
|
00c1b19809 | ||
|
|
2fd5daf104 | ||
|
|
fbf0f6adfe | ||
|
|
d91b5bec66 | ||
|
|
0f24cfb6ef | ||
|
|
7a6691cdbe | ||
|
|
73d3a4f843 | ||
|
|
091755fe78 | ||
|
|
1fb2790bb9 | ||
|
|
798e64b224 | ||
|
|
a2d6477b42 | ||
|
|
a2ca24eeb4 | ||
|
|
8dcd00a8fe | ||
|
|
be8a52c823 | ||
|
|
43e420b0ab | ||
|
|
f8947adf62 | ||
|
|
81fccaf0ab | ||
|
|
7a42291baf | ||
|
|
d9f983b08e | ||
|
|
2f159852a7 | ||
|
|
5ef402b8ee | ||
|
|
a8d6e5ab73 | ||
|
|
e805a3797f | ||
|
|
8f4fa82a9e | ||
|
|
d2438b2fdd | ||
|
|
133640ef2b | ||
|
|
8e0516a8c3 |
@@ -36,8 +36,7 @@ body:
|
||||
label: Build information
|
||||
description: >-
|
||||
Open User Settings, scroll to the bottom of the left sidebar, and select
|
||||
the build information. Fluxer copies it to the clipboard. On mobile,
|
||||
select the build information at the bottom of the settings list.
|
||||
the build information. Fluxer copies it to the clipboard.
|
||||
validations:
|
||||
required: true
|
||||
|
||||
|
||||
@@ -1,15 +1,15 @@
|
||||
# yaml-language-server: $schema=https://www.schemastore.org/github-issue-config.json
|
||||
blank_issues_enabled: false
|
||||
contact_links:
|
||||
- name: Mobile client bugs
|
||||
url: https://github.com/fluxerapp/flutter_client#bug-reporting
|
||||
about: Read the reporting instructions for the Fluxer mobile client.
|
||||
- name: Account and billing support
|
||||
url: https://fluxer.app/help
|
||||
about: Find account help and support contact details.
|
||||
- name: Feature proposals
|
||||
url: https://github.com/orgs/fluxerapp/discussions
|
||||
about: Propose a feature in a discussion.
|
||||
- name: Security vulnerabilities
|
||||
url: https://github.com/fluxerapp/fluxer/security/advisories/new
|
||||
about: Submit a private vulnerability report.
|
||||
- name: Translations
|
||||
url: https://weblate.fluxer.tools
|
||||
about: Improve an existing locale or start a new one.
|
||||
|
||||
@@ -314,8 +314,8 @@ jobs:
|
||||
path: |
|
||||
~/.cache/rebar3
|
||||
fluxer_gateway/_build
|
||||
!fluxer_gateway/_build/default/lib/fluxer_gateway
|
||||
!fluxer_gateway/_build/test/lib/fluxer_gateway
|
||||
!fluxer_gateway/_build/default/lib/fluxer_gateway/**
|
||||
!fluxer_gateway/_build/test/lib/fluxer_gateway/**
|
||||
key: >-
|
||||
rebar3-${{ runner.os }}-otp28-rebar3.24.0-${{ hashFiles('fluxer_gateway/rebar.lock',
|
||||
'fluxer_gateway/rebar.config') }}
|
||||
@@ -345,8 +345,8 @@ jobs:
|
||||
path: |
|
||||
~/.cache/rebar3
|
||||
fluxer_gateway/_build
|
||||
!fluxer_gateway/_build/default/lib/fluxer_gateway
|
||||
!fluxer_gateway/_build/test/lib/fluxer_gateway
|
||||
!fluxer_gateway/_build/default/lib/fluxer_gateway/**
|
||||
!fluxer_gateway/_build/test/lib/fluxer_gateway/**
|
||||
key: >-
|
||||
rebar3-${{ runner.os }}-otp28-rebar3.24.0-${{ hashFiles('fluxer_gateway/rebar.lock',
|
||||
'fluxer_gateway/rebar.config') }}
|
||||
|
||||
@@ -100,6 +100,12 @@ FLUXER_MEDIA_PROXY_UPLOAD_RELAY_SECRET_BASE64=CHANGE_ME
|
||||
FLUXER_ADMIN_SECRET_KEY_BASE=CHANGE_ME
|
||||
FLUXER_ADMIN_OAUTH_CLIENT_SECRET=CHANGE_ME
|
||||
|
||||
# The token every service sends to NATS. The bundled NATS runs without
|
||||
# authentication, so this stays empty unless a Compose override points the stack
|
||||
# at an external NATS that requires a token. Compose forwards the name to every
|
||||
# container that connects.
|
||||
#FLUXER_NATS_AUTH_TOKEN=
|
||||
|
||||
FLUXER_VAPID_PUBLIC_KEY=CHANGE_ME
|
||||
FLUXER_VAPID_PRIVATE_KEY=CHANGE_ME
|
||||
|
||||
@@ -157,6 +163,16 @@ LIVEKIT_API_SECRET=CHANGE_ME
|
||||
#FLUXER_LIVEKIT_TCP_PORT=7881
|
||||
#FLUXER_LIVEKIT_UDP_PORT=7882
|
||||
|
||||
# LiveKit finds the address browsers dial by asking a STUN server. A host that
|
||||
# cannot reach one over UDP stops with "could not resolve external IP", and the
|
||||
# address is then set by hand: put it in FLUXER_LIVEKIT_NODE_IP and set
|
||||
# FLUXER_LIVEKIT_USE_EXTERNAL_IP to false. Point the two STUN entries at another
|
||||
# server to keep the lookup and leave Google out of it.
|
||||
#FLUXER_LIVEKIT_USE_EXTERNAL_IP=false
|
||||
#FLUXER_LIVEKIT_NODE_IP=203.0.113.10
|
||||
#FLUXER_LIVEKIT_STUN_PRIMARY=stun.l.google.com:19302
|
||||
#FLUXER_LIVEKIT_STUN_SECONDARY=stun1.l.google.com:19302
|
||||
|
||||
FLUXER_KLIPY_API_KEY=
|
||||
|
||||
FLUXER_EMAIL_ENABLED=false
|
||||
@@ -178,7 +194,7 @@ FLUXER_CAPTCHA_TURNSTILE_SITE_KEY=
|
||||
FLUXER_CAPTCHA_TURNSTILE_SECRET_KEY=
|
||||
FLUXER_DISCOVERY_ENABLED=true
|
||||
|
||||
# Container memory. The 25 limits sum to 16.75 GiB, which is a sum of ceilings and
|
||||
# Container memory. The 25 limits sum to 18.25 GiB, which is a sum of ceilings and
|
||||
# not an allocation, so the defaults fit a host with 8 GB and are sized for 16 GB.
|
||||
# The four reservations are cgroup memory.low, which biases the kernel away from
|
||||
# reclaiming from the services whose death takes the whole instance down. They do
|
||||
@@ -189,7 +205,7 @@ FLUXER_DISCOVERY_ENABLED=true
|
||||
#FLUXER_VALKEY_MEMORY_LIMIT=256mb
|
||||
#FLUXER_NATS_MEMORY_LIMIT=256mb
|
||||
#FLUXER_MEILISEARCH_MEMORY_LIMIT=768mb
|
||||
#FLUXER_SEAWEEDFS_MEMORY_LIMIT=512mb
|
||||
#FLUXER_SEAWEEDFS_MEMORY_LIMIT=2gb
|
||||
#FLUXER_SEAWEEDFS_INIT_MEMORY_LIMIT=128mb
|
||||
#FLUXER_LIVEKIT_MEMORY_LIMIT=512mb
|
||||
#FLUXER_API_MEMORY_LIMIT=2560mb
|
||||
@@ -217,6 +233,14 @@ FLUXER_DISCOVERY_ENABLED=true
|
||||
# which is the container ceiling the indexer shares with the search process.
|
||||
#FLUXER_MEILISEARCH_MAX_INDEXING_MEMORY=384mb
|
||||
|
||||
# SeaweedFS heap ceiling. Go collects against this value instead of against the
|
||||
# container limit, which it cannot see, so without it an upload burst grows the
|
||||
# heap past FLUXER_SEAWEEDFS_MEMORY_LIMIT and the kernel OOM-kills the container
|
||||
# mid-upload (exit 137). Keep it near three quarters of that limit, and raise both
|
||||
# together: the peak is the parts of one upload in flight at once, which is 25 MB
|
||||
# times 20 for a 500 MB attachment.
|
||||
#FLUXER_SEAWEEDFS_GOMEMLIMIT=1536MiB
|
||||
|
||||
# Node sizes its own heap from the container memory limit by default, at roughly
|
||||
# 55 percent of it, which always leaves room for the buffers and stacks that live
|
||||
# outside the heap. Leave these unset unless you have a reason to pin the value.
|
||||
|
||||
@@ -13,73 +13,51 @@
|
||||
}
|
||||
|
||||
handle_path /api/* {
|
||||
reverse_proxy api:8080 {
|
||||
header_up X-Forwarded-For {client_ip}
|
||||
}
|
||||
reverse_proxy api:8080
|
||||
}
|
||||
|
||||
handle /gateway {
|
||||
rewrite * /
|
||||
reverse_proxy gateway:8080 {
|
||||
header_up X-Forwarded-For {client_ip}
|
||||
}
|
||||
reverse_proxy gateway:8080
|
||||
}
|
||||
|
||||
handle_path /gateway/* {
|
||||
reverse_proxy gateway:8080 {
|
||||
header_up X-Forwarded-For {client_ip}
|
||||
}
|
||||
reverse_proxy gateway:8080
|
||||
}
|
||||
|
||||
handle_path /media/* {
|
||||
reverse_proxy media-proxy:8080 {
|
||||
header_up X-Forwarded-For {client_ip}
|
||||
}
|
||||
reverse_proxy media-proxy:8080
|
||||
}
|
||||
|
||||
handle_path /livekit/* {
|
||||
reverse_proxy livekit:7880 {
|
||||
header_up X-Forwarded-For {client_ip}
|
||||
}
|
||||
reverse_proxy livekit:7880
|
||||
}
|
||||
|
||||
handle /admin {
|
||||
rewrite * /
|
||||
reverse_proxy admin:8080 {
|
||||
header_up X-Forwarded-For {client_ip}
|
||||
}
|
||||
reverse_proxy admin:8080
|
||||
}
|
||||
|
||||
handle_path /admin/* {
|
||||
reverse_proxy admin:8080 {
|
||||
header_up X-Forwarded-For {client_ip}
|
||||
}
|
||||
reverse_proxy admin:8080
|
||||
}
|
||||
|
||||
@staticAssets path /web/* /emoji/* /libs/* /avatars/* /badges/* /desktop/* /embeds/*
|
||||
handle @staticAssets {
|
||||
reverse_proxy static-proxy:8080 {
|
||||
header_up X-Forwarded-For {client_ip}
|
||||
}
|
||||
reverse_proxy static-proxy:8080
|
||||
}
|
||||
|
||||
handle /.well-known/fluxer {
|
||||
reverse_proxy api:8080 {
|
||||
header_up X-Forwarded-For {client_ip}
|
||||
}
|
||||
reverse_proxy api:8080
|
||||
}
|
||||
|
||||
handle {
|
||||
reverse_proxy app-proxy:8080 {
|
||||
header_up X-Forwarded-For {client_ip}
|
||||
}
|
||||
reverse_proxy app-proxy:8080
|
||||
}
|
||||
}
|
||||
|
||||
:8088 {
|
||||
handle /.well-known/fluxer {
|
||||
reverse_proxy api:8080 {
|
||||
header_up X-Forwarded-For {client_ip}
|
||||
}
|
||||
reverse_proxy api:8080
|
||||
}
|
||||
}
|
||||
|
||||
@@ -26,6 +26,7 @@ x-fluxer-env: &fluxer-env
|
||||
FLUXER_KV_URL: redis://valkey:6379/0
|
||||
FLUXER_NATS_URL: nats://nats:4222
|
||||
FLUXER_NATS_JETSTREAM_URL: nats://nats:4222
|
||||
FLUXER_NATS_AUTH_TOKEN: ${FLUXER_NATS_AUTH_TOKEN:-}
|
||||
FLUXER_SVC_NATS_URL: nats://nats:4222
|
||||
FLUXER_SVC_SHARD_COUNT: "1"
|
||||
|
||||
@@ -258,9 +259,11 @@ services:
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
memory: ${FLUXER_SEAWEEDFS_MEMORY_LIMIT:-512mb}
|
||||
memory: ${FLUXER_SEAWEEDFS_MEMORY_LIMIT:-2gb}
|
||||
restart: unless-stopped
|
||||
networks: [fluxer]
|
||||
environment:
|
||||
GOMEMLIMIT: ${FLUXER_SEAWEEDFS_GOMEMLIMIT:-1536MiB}
|
||||
command: ["server", "-s3", "-dir=/data"]
|
||||
volumes:
|
||||
- seaweedfs-data:/data
|
||||
@@ -332,10 +335,11 @@ services:
|
||||
rtc:
|
||||
tcp_port: ${FLUXER_LIVEKIT_TCP_PORT:-7881}
|
||||
udp_port: ${FLUXER_LIVEKIT_UDP_PORT:-7882}
|
||||
use_external_ip: true
|
||||
use_external_ip: ${FLUXER_LIVEKIT_USE_EXTERNAL_IP:-true}
|
||||
node_ip: "${FLUXER_LIVEKIT_NODE_IP:-}"
|
||||
stun_servers:
|
||||
- stun.l.google.com:19302
|
||||
- stun1.l.google.com:19302
|
||||
- ${FLUXER_LIVEKIT_STUN_PRIMARY:-stun.l.google.com:19302}
|
||||
- ${FLUXER_LIVEKIT_STUN_SECONDARY:-stun1.l.google.com:19302}
|
||||
webhook:
|
||||
api_key: ${LIVEKIT_API_KEY:?set LIVEKIT_API_KEY in .env}
|
||||
urls:
|
||||
|
||||
@@ -2772,7 +2772,7 @@
|
||||
"content": {"application/json": {"schema": {"$ref": "#/components/schemas/Error"}}}
|
||||
}
|
||||
},
|
||||
"description": "Partially updates a guild. The permissions required are selected by the fields present in the body and are evaluated with all-of semantics: name requires GUILD_UPDATE_NAME, vanity_url_code requires GUILD_UPDATE_VANITY, new_owner_id requires GUILD_TRANSFER_OWNERSHIP, add_features and remove_features require GUILD_UPDATE_FEATURES, and fields together with every other setting requires GUILD_UPDATE_SETTINGS. A body carrying no field requires the wildcard permission. Every applied change is logged to the audit log.",
|
||||
"description": "Partially updates a guild. The permissions required are selected by the fields present in the body and are evaluated with all-of semantics: name requires GUILD_UPDATE_NAME, vanity_url_code requires GUILD_UPDATE_VANITY, new_owner_id requires GUILD_TRANSFER_OWNERSHIP, add_features and remove_features require GUILD_UPDATE_FEATURES, and fields together with every other setting requires GUILD_UPDATE_SETTINGS. A body with no fields applies no change. Every applied change is logged to the audit log.",
|
||||
"security": [{"adminApiKey": []}],
|
||||
"parameters": [
|
||||
{
|
||||
@@ -12287,14 +12287,7 @@
|
||||
"max_concurrent_guild_starts": {"type": "integer", "minimum": 1, "maximum": 10000, "format": "int32"},
|
||||
"gateway_dispatch_relay_shards": {"type": "integer", "minimum": 1, "maximum": 10000, "format": "int32"},
|
||||
"gateway_dispatch_relay_max_queue": {"type": "integer", "minimum": 0, "maximum": 1000000, "format": "int32"},
|
||||
"voice_e2ee_scope": {"enum": ["guild_feature_only", "platform_wide"], "type": "string"},
|
||||
"voice_reconciliation_v3_percentage": {"type": "number", "minimum": 0, "maximum": 100},
|
||||
"voice_reconciliation_v3_interval_ms": {
|
||||
"type": "integer",
|
||||
"minimum": 500,
|
||||
"maximum": 60000,
|
||||
"format": "int32"
|
||||
}
|
||||
"voice_e2ee_scope": {"enum": ["guild_feature_only", "platform_wide"], "type": "string"}
|
||||
}
|
||||
},
|
||||
"InstanceConfigUpdateRequest": {
|
||||
@@ -12534,14 +12527,7 @@
|
||||
"max_concurrent_guild_starts": {"type": "integer", "minimum": 1, "maximum": 10000, "format": "int32"},
|
||||
"gateway_dispatch_relay_shards": {"type": "integer", "minimum": 1, "maximum": 10000, "format": "int32"},
|
||||
"gateway_dispatch_relay_max_queue": {"type": "integer", "minimum": 0, "maximum": 1000000, "format": "int32"},
|
||||
"voice_e2ee_scope": {"enum": ["guild_feature_only", "platform_wide"], "type": "string"},
|
||||
"voice_reconciliation_v3_percentage": {"type": "number", "minimum": 0, "maximum": 100},
|
||||
"voice_reconciliation_v3_interval_ms": {
|
||||
"type": "integer",
|
||||
"minimum": 500,
|
||||
"maximum": 60000,
|
||||
"format": "int32"
|
||||
}
|
||||
"voice_e2ee_scope": {"enum": ["guild_feature_only", "platform_wide"], "type": "string"}
|
||||
}
|
||||
},
|
||||
"BrandingAssetUploadRequest": {
|
||||
@@ -12923,25 +12909,14 @@
|
||||
]
|
||||
},
|
||||
"AdminReportListResponse": {
|
||||
"oneOf": [
|
||||
{
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"reports": {"type": "array", "items": {"$ref": "#/components/schemas/ReportAdminResponseSchema"}},
|
||||
"total": {"type": "number"},
|
||||
"offset": {"type": "number"},
|
||||
"limit": {"type": "number"}
|
||||
},
|
||||
"required": ["reports", "total", "offset", "limit"]
|
||||
},
|
||||
{
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"reports": {"type": "array", "items": {"$ref": "#/components/schemas/ReportAdminResponseSchema"}}
|
||||
},
|
||||
"required": ["reports"]
|
||||
}
|
||||
]
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"reports": {"type": "array", "items": {"$ref": "#/components/schemas/ReportAdminResponseSchema"}},
|
||||
"total": {"type": "number"},
|
||||
"offset": {"type": "number"},
|
||||
"limit": {"type": "number"}
|
||||
},
|
||||
"required": ["reports", "total", "offset", "limit"]
|
||||
},
|
||||
"ReportAdminResponseSchema": {
|
||||
"type": "object",
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
|
||||
use crate::api::generated::types as generated_types;
|
||||
|
||||
use super::client::{AdminApiClient, ApiResult};
|
||||
use super::client::{AdminApiClient, ApiError, ApiResult};
|
||||
use super::types::{Archive, ArchiveDownloadUrlResponse, ListArchivesResponse};
|
||||
|
||||
impl AdminApiClient {
|
||||
@@ -45,16 +45,31 @@ impl AdminApiClient {
|
||||
include_expired: bool,
|
||||
requested_by: Option<&str>,
|
||||
) -> ApiResult<ListArchivesResponse> {
|
||||
let query_params = [
|
||||
("subject_type", subject_type),
|
||||
("subject_id", subject_id.unwrap_or_default()),
|
||||
("requested_by", requested_by.unwrap_or_default()),
|
||||
(
|
||||
"include_expired",
|
||||
if include_expired { "true" } else { "false" },
|
||||
),
|
||||
];
|
||||
self.get("/admin/archives", Some(&query_params)).await
|
||||
let subject_id = subject_id.filter(|id| !id.is_empty());
|
||||
let search_every_subject_type = subject_type == "all" && subject_id.is_some();
|
||||
let subject_types: &[&str] = if search_every_subject_type {
|
||||
&["user", "guild"]
|
||||
} else {
|
||||
std::slice::from_ref(&subject_type)
|
||||
};
|
||||
let mut archives = Vec::new();
|
||||
for &subject_type in subject_types {
|
||||
let query_params = [
|
||||
("subject_type", subject_type),
|
||||
("subject_id", subject_id.unwrap_or_default()),
|
||||
("requested_by", requested_by.unwrap_or_default()),
|
||||
(
|
||||
"include_expired",
|
||||
if include_expired { "true" } else { "false" },
|
||||
),
|
||||
];
|
||||
match self.get("/admin/archives", Some(&query_params)).await {
|
||||
Ok(ListArchivesResponse { archives: page }) => archives.extend(page),
|
||||
Err(ApiError::Http { status: 403, .. }) if search_every_subject_type => {}
|
||||
Err(error) => return Err(error),
|
||||
}
|
||||
}
|
||||
Ok(ListArchivesResponse { archives })
|
||||
}
|
||||
|
||||
pub async fn get_archive_download_url(
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
|
||||
use crate::api::generated::types as generated_types;
|
||||
|
||||
use super::client::{AdminApiClient, ApiResult};
|
||||
use super::client::{AdminApiClient, ApiError, ApiResult};
|
||||
use super::types::{IndexRefreshStatusResponse, RefreshSearchIndexResponse};
|
||||
|
||||
impl AdminApiClient {
|
||||
@@ -32,6 +32,85 @@ impl AdminApiClient {
|
||||
.get_admin_search_index_refresh(job_id)
|
||||
.await
|
||||
.map_err(|e| self.generated_error(e))?;
|
||||
self.generated_value(response.into_inner())
|
||||
index_refresh_status(response.into_inner())
|
||||
}
|
||||
}
|
||||
|
||||
fn index_refresh_status(
|
||||
response: generated_types::IndexRefreshStatusResponse,
|
||||
) -> ApiResult<IndexRefreshStatusResponse> {
|
||||
match response {
|
||||
generated_types::IndexRefreshStatusResponse::Variant0 { status } => {
|
||||
Ok(IndexRefreshStatusResponse::NotFound {
|
||||
status: status.to_string(),
|
||||
})
|
||||
}
|
||||
generated_types::IndexRefreshStatusResponse::Variant1 {
|
||||
status,
|
||||
index_type,
|
||||
total,
|
||||
indexed,
|
||||
started_at,
|
||||
completed_at,
|
||||
failed_at,
|
||||
error,
|
||||
} => Ok(IndexRefreshStatusResponse::Progress {
|
||||
status: status.to_string(),
|
||||
index_type: Some(index_type),
|
||||
total: total
|
||||
.map(|value| float_to_u64(value, "total"))
|
||||
.transpose()?,
|
||||
indexed: indexed
|
||||
.map(|value| float_to_u64(value, "indexed"))
|
||||
.transpose()?,
|
||||
started_at,
|
||||
completed_at,
|
||||
failed_at,
|
||||
error,
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
fn float_to_u64(value: f64, field: &str) -> ApiResult<u64> {
|
||||
crate::api::generated::number_to_u64(value, field).map_err(ApiError::Parse)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn maps_a_running_refresh_to_progress() {
|
||||
let json = r#"{"status":"in_progress","index_type":"users","total":50000,"indexed":1200,"started_at":"2026-09-06T00:00:00Z"}"#;
|
||||
let response: generated_types::IndexRefreshStatusResponse =
|
||||
serde_json::from_str(json).unwrap();
|
||||
match index_refresh_status(response).unwrap() {
|
||||
IndexRefreshStatusResponse::Progress {
|
||||
status,
|
||||
index_type,
|
||||
total,
|
||||
indexed,
|
||||
started_at,
|
||||
..
|
||||
} => {
|
||||
assert_eq!(status, "in_progress");
|
||||
assert_eq!(index_type.as_deref(), Some("users"));
|
||||
assert_eq!(total, Some(50_000));
|
||||
assert_eq!(indexed, Some(1_200));
|
||||
assert_eq!(started_at.as_deref(), Some("2026-09-06T00:00:00Z"));
|
||||
}
|
||||
other => panic!("expected a progress status, got {other:?}"),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn maps_a_missing_refresh_to_not_found() {
|
||||
let json = r#"{"status":"not_found"}"#;
|
||||
let response: generated_types::IndexRefreshStatusResponse =
|
||||
serde_json::from_str(json).unwrap();
|
||||
match index_refresh_status(response).unwrap() {
|
||||
IndexRefreshStatusResponse::NotFound { status } => assert_eq!(status, "not_found"),
|
||||
other => panic!("expected a not found status, got {other:?}"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -102,6 +102,7 @@ impl AdminApiClient {
|
||||
params: &serde_json::Value,
|
||||
) -> ApiResult<CreateVoiceServerResponse> {
|
||||
let region_id = required_field(params, "region_id")?;
|
||||
paired_coordinates(params)?;
|
||||
let body =
|
||||
serde_json::from_value::<generated_types::CreateVoiceServerRequest>(params.clone())
|
||||
.map_err(|e| ApiError::Parse(e.to_string()))?;
|
||||
@@ -119,6 +120,7 @@ impl AdminApiClient {
|
||||
) -> ApiResult<UpdateVoiceServerResponse> {
|
||||
let region_id = required_field(params, "region_id")?;
|
||||
let server_id = required_field(params, "server_id")?;
|
||||
paired_coordinates(params)?;
|
||||
let body =
|
||||
serde_json::from_value::<generated_types::UpdateVoiceServerRequest>(params.clone())
|
||||
.map_err(|e| ApiError::Parse(e.to_string()))?;
|
||||
@@ -148,6 +150,17 @@ fn bool_param(value: bool) -> &'static str {
|
||||
if value { "true" } else { "false" }
|
||||
}
|
||||
|
||||
fn paired_coordinates(params: &serde_json::Value) -> ApiResult<()> {
|
||||
let has_coordinate = |field: &str| params.get(field).is_some_and(|value| !value.is_null());
|
||||
if has_coordinate("latitude") == has_coordinate("longitude") {
|
||||
Ok(())
|
||||
} else {
|
||||
Err(ApiError::Parse(
|
||||
"latitude and longitude must both be set or both be left empty".to_owned(),
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
fn required_field(params: &serde_json::Value, field: &str) -> ApiResult<String> {
|
||||
params
|
||||
.get(field)
|
||||
|
||||
@@ -251,7 +251,7 @@ pub(crate) async fn bulk_actions_post(
|
||||
.bulk_add_guild_members(&guild_id, &user_ids, audit_log_reason.as_deref())
|
||||
.await
|
||||
}
|
||||
"bulk-schedule-user-deletion" => {
|
||||
"bulk-schedule-user-deletion" | "bulk_delete_users" => {
|
||||
let user_ids = form.list_values_any(&["user_ids[]", "user_ids"]);
|
||||
let reason_code = form.parse_u32("reason_code").unwrap_or(2);
|
||||
let days = form.parse_u32("days_until_deletion").unwrap_or(14);
|
||||
@@ -272,12 +272,6 @@ pub(crate) async fn bulk_actions_post(
|
||||
.bulk_delete_user_messages(&user_ids, audit_log_reason.as_deref())
|
||||
.await
|
||||
}
|
||||
"bulk_delete_users" => {
|
||||
let user_ids = form.list_values_any(&["user_ids[]", "user_ids"]);
|
||||
client
|
||||
.bulk_schedule_user_deletion(&user_ids, 0, 30, None, audit_log_reason.as_deref())
|
||||
.await
|
||||
}
|
||||
_ => {
|
||||
return flash::redirect_with_flash(
|
||||
&format!("{base}/bulk-actions"),
|
||||
@@ -302,7 +296,7 @@ pub(crate) async fn bulk_actions_post(
|
||||
tracing::warn!(%error, action, "admin API request failed: submit bulk action");
|
||||
flash::redirect_with_flash(
|
||||
&format!("{base}/bulk-actions"),
|
||||
FlashData::error("Failed to submit bulk action"),
|
||||
FlashData::error(format!("Failed to submit bulk action: {error}")),
|
||||
config.secure_cookies(),
|
||||
)
|
||||
}
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
|
||||
use crate::{
|
||||
api::client::{AdminApiClient, ApiResultExt},
|
||||
config::AdminConfig,
|
||||
middleware::{
|
||||
auth::AuthContext,
|
||||
csrf,
|
||||
@@ -22,6 +23,8 @@ use axum::{
|
||||
};
|
||||
use serde::Deserialize;
|
||||
|
||||
const MAX_REPORT_OFFSET: u32 = 10_000;
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct ReportsQuery {
|
||||
q: Option<String>,
|
||||
@@ -70,6 +73,14 @@ async fn reports_list(
|
||||
let page = query.page.unwrap_or(0);
|
||||
let limit = query.limit.unwrap_or(25).clamp(1, 200);
|
||||
let offset = page.saturating_mul(limit);
|
||||
if offset > MAX_REPORT_OFFSET {
|
||||
return reports_error_page(
|
||||
config,
|
||||
&auth.0,
|
||||
"That page is out of range. The reports search returns at most the first 10000 reports, so narrow the filters and start again.",
|
||||
);
|
||||
}
|
||||
let search_query = query.q.as_deref().and_then(clean_string);
|
||||
let (sort_by, sort_order) = decode_sort(query.sort.as_deref());
|
||||
let client = AdminApiClient::new(state.http_client(), config, &auth.0.session);
|
||||
let status = query.status.as_deref().and_then(|s| s.parse::<i32>().ok());
|
||||
@@ -79,7 +90,7 @@ async fn reports_list(
|
||||
.and_then(|s| s.parse::<i32>().ok());
|
||||
let reports = client
|
||||
.search_reports(
|
||||
query.q.as_deref(),
|
||||
search_query.as_deref(),
|
||||
status,
|
||||
report_type,
|
||||
query.category.as_deref(),
|
||||
@@ -102,7 +113,7 @@ async fn reports_list(
|
||||
&auth.0,
|
||||
reports.as_ref(),
|
||||
&templates::pages::reports_list::ReportFilters {
|
||||
query: query.q.as_deref(),
|
||||
query: search_query.as_deref(),
|
||||
status: query.status.as_deref(),
|
||||
report_type: query.report_type.as_deref(),
|
||||
category: query.category.as_deref(),
|
||||
@@ -120,6 +131,18 @@ async fn reports_list(
|
||||
Html(markup.into_string()).into_response()
|
||||
}
|
||||
|
||||
fn reports_error_page(config: &AdminConfig, auth: &AuthContext, message: &str) -> Response {
|
||||
let markup = templates::layout::admin_layout(
|
||||
config,
|
||||
auth,
|
||||
"Reports",
|
||||
"reports",
|
||||
None,
|
||||
templates::components::error_display::error_alert(message),
|
||||
);
|
||||
Html(markup.into_string()).into_response()
|
||||
}
|
||||
|
||||
async fn report_detail(
|
||||
State(state): State<AppState>,
|
||||
headers: HeaderMap,
|
||||
|
||||
@@ -2,7 +2,10 @@
|
||||
|
||||
use crate::{
|
||||
acl,
|
||||
api::client::{AdminApiClient, ApiResultExt},
|
||||
api::{
|
||||
client::{AdminApiClient, ApiResult, ApiResultExt},
|
||||
types::AdminUser,
|
||||
},
|
||||
middleware::{auth::AuthContext, csrf::CsrfToken, flash, htmx},
|
||||
routes::user_tabs,
|
||||
state::AppState,
|
||||
@@ -18,6 +21,8 @@ use axum::{
|
||||
};
|
||||
use serde::Deserialize;
|
||||
|
||||
const USER_ID_LOOKUP_BATCH: usize = 100;
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct UserListQuery {
|
||||
q: Option<String>,
|
||||
@@ -83,14 +88,10 @@ async fn users_list(
|
||||
let can_view_email = acl::has_permission(admin_acls, acl::USER_VIEW_EMAIL);
|
||||
let client = AdminApiClient::new(state.http_client(), config, &auth.0.session);
|
||||
let results = if params.has_id_lookup() {
|
||||
let users = client
|
||||
.lookup_users_by_ids(¶ms.requested_ids)
|
||||
lookup_users_in_batches(&client, ¶ms.requested_ids)
|
||||
.await
|
||||
.map_err(
|
||||
|error| tracing::warn!(%error, "admin API request failed: lookup users by ids"),
|
||||
)
|
||||
.unwrap_or_default();
|
||||
Some((users, false))
|
||||
.log_error("lookup users by ids")
|
||||
.map(|users| (users, false))
|
||||
} else if params.has_search() {
|
||||
let offset = params.page.saturating_mul(params.limit);
|
||||
client
|
||||
@@ -124,6 +125,17 @@ async fn users_list(
|
||||
Html(markup.into_string()).into_response()
|
||||
}
|
||||
|
||||
async fn lookup_users_in_batches(
|
||||
client: &AdminApiClient,
|
||||
user_ids: &[String],
|
||||
) -> ApiResult<Vec<AdminUser>> {
|
||||
let mut users = Vec::new();
|
||||
for batch in user_ids.chunks(USER_ID_LOOKUP_BATCH) {
|
||||
users.extend(client.lookup_users_by_ids(batch).await?);
|
||||
}
|
||||
Ok(users)
|
||||
}
|
||||
|
||||
async fn user_detail(
|
||||
State(state): State<AppState>,
|
||||
headers: HeaderMap,
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
|
||||
import {getRegionDisplayName} from '@fluxer/geo_utils/src/RegionFormatting';
|
||||
import {getSameIpDecisionKey, isValidIp, normalizeIpString} from '@fluxer/ip_utils/src/IpAddress';
|
||||
import maxmind, {type CityResponse, type Reader} from 'maxmind';
|
||||
import maxmind, {type AsnResponse, type CityResponse, type Reader} from 'maxmind';
|
||||
|
||||
export const UNKNOWN_LOCATION = 'Unknown Location';
|
||||
|
||||
@@ -15,6 +15,15 @@ export interface GeoipResult {
|
||||
countryName: string | null;
|
||||
latitude?: number | null;
|
||||
longitude?: number | null;
|
||||
accuracyRadiusKm?: number | null;
|
||||
timeZone?: string | null;
|
||||
}
|
||||
|
||||
export interface GeoipAsnResult {
|
||||
normalizedIp: string | null;
|
||||
asn: number | null;
|
||||
asnOrg: string | null;
|
||||
available: boolean;
|
||||
}
|
||||
|
||||
type CacheEntry = {
|
||||
@@ -22,12 +31,21 @@ type CacheEntry = {
|
||||
expiresAt: number;
|
||||
};
|
||||
|
||||
type AsnCacheEntry = {
|
||||
result: GeoipAsnResult;
|
||||
expiresAt: number;
|
||||
};
|
||||
|
||||
const CACHE_TTL_MS = 10 * 60 * 1000;
|
||||
const CACHE_MAX_ENTRIES = 10_000;
|
||||
const geoipCache = new Map<string, CacheEntry>();
|
||||
const asnCache = new Map<string, AsnCacheEntry>();
|
||||
|
||||
let maxmindReader: Reader<CityResponse> | null = null;
|
||||
let maxmindReaderPromise: Promise<Reader<CityResponse>> | null = null;
|
||||
let maxmindAsnReader: Reader<AsnResponse> | null = null;
|
||||
let maxmindAsnReaderPromise: Promise<Reader<AsnResponse>> | null = null;
|
||||
let maxmindAsnUnavailable = false;
|
||||
|
||||
function buildFallbackResult(normalizedIp: string): GeoipResult {
|
||||
return {
|
||||
@@ -39,6 +57,17 @@ function buildFallbackResult(normalizedIp: string): GeoipResult {
|
||||
countryName: null,
|
||||
latitude: null,
|
||||
longitude: null,
|
||||
accuracyRadiusKm: null,
|
||||
timeZone: null,
|
||||
};
|
||||
}
|
||||
|
||||
function buildAsnFallbackResult(normalizedIp: string | null): GeoipAsnResult {
|
||||
return {
|
||||
normalizedIp: normalizedIp || null,
|
||||
asn: null,
|
||||
asnOrg: null,
|
||||
available: false,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -59,6 +88,24 @@ async function ensureReader(dbPath: string): Promise<Reader<CityResponse>> {
|
||||
return maxmindReaderPromise;
|
||||
}
|
||||
|
||||
async function ensureAsnReader(dbPath: string): Promise<Reader<AsnResponse>> {
|
||||
if (maxmindAsnReader) return maxmindAsnReader;
|
||||
if (!maxmindAsnReaderPromise) {
|
||||
maxmindAsnReaderPromise = maxmind
|
||||
.open<AsnResponse>(dbPath, {watchForUpdates: true, watchForUpdatesNonPersistent: true})
|
||||
.then((reader) => {
|
||||
maxmindAsnReader = reader;
|
||||
return reader;
|
||||
})
|
||||
.catch((error) => {
|
||||
maxmindAsnReaderPromise = null;
|
||||
maxmindAsnUnavailable = true;
|
||||
throw error;
|
||||
});
|
||||
}
|
||||
return maxmindAsnReaderPromise;
|
||||
}
|
||||
|
||||
function stateLabel(record?: CityResponse): string | null {
|
||||
const subdivision = record?.subdivisions?.[0];
|
||||
if (!subdivision) return null;
|
||||
@@ -112,6 +159,31 @@ function setCachedGeoipResult(cacheKey: string, result: GeoipResult): void {
|
||||
geoipCache.set(cacheKey, {result, expiresAt: Date.now() + CACHE_TTL_MS});
|
||||
}
|
||||
|
||||
function getCachedAsnResult(cacheKey: string, normalizedIp: string): GeoipAsnResult | null {
|
||||
const cached = asnCache.get(cacheKey);
|
||||
if (!cached) {
|
||||
return null;
|
||||
}
|
||||
if (Date.now() >= cached.expiresAt) {
|
||||
asnCache.delete(cacheKey);
|
||||
return null;
|
||||
}
|
||||
asnCache.delete(cacheKey);
|
||||
asnCache.set(cacheKey, cached);
|
||||
return {...cached.result, normalizedIp};
|
||||
}
|
||||
|
||||
function setCachedAsnResult(cacheKey: string, result: GeoipAsnResult): void {
|
||||
asnCache.delete(cacheKey);
|
||||
if (asnCache.size >= CACHE_MAX_ENTRIES) {
|
||||
const oldestKey = asnCache.keys().next().value;
|
||||
if (oldestKey !== undefined) {
|
||||
asnCache.delete(oldestKey);
|
||||
}
|
||||
}
|
||||
asnCache.set(cacheKey, {result, expiresAt: Date.now() + CACHE_TTL_MS});
|
||||
}
|
||||
|
||||
async function lookupMaxmind(clean: string, dbPath: string): Promise<GeoipResult> {
|
||||
try {
|
||||
const reader = await ensureReader(dbPath);
|
||||
@@ -128,12 +200,32 @@ async function lookupMaxmind(clean: string, dbPath: string): Promise<GeoipResult
|
||||
countryName: record.country?.names?.en ?? (countryCode ? countryDisplayName(countryCode) : null) ?? null,
|
||||
latitude: record.location?.latitude ?? null,
|
||||
longitude: record.location?.longitude ?? null,
|
||||
accuracyRadiusKm: record.location?.accuracy_radius ?? null,
|
||||
timeZone: record.location?.time_zone ?? null,
|
||||
};
|
||||
} catch {
|
||||
return buildFallbackResult(clean);
|
||||
}
|
||||
}
|
||||
|
||||
async function lookupMaxmindAsn(clean: string, dbPath: string): Promise<GeoipAsnResult> {
|
||||
try {
|
||||
const reader = await ensureAsnReader(dbPath);
|
||||
const record = reader.get(clean);
|
||||
if (!record) {
|
||||
return {normalizedIp: clean, asn: null, asnOrg: null, available: true};
|
||||
}
|
||||
return {
|
||||
normalizedIp: clean,
|
||||
asn: record.autonomous_system_number ?? null,
|
||||
asnOrg: record.autonomous_system_organization ?? null,
|
||||
available: true,
|
||||
};
|
||||
} catch {
|
||||
return buildAsnFallbackResult(clean);
|
||||
}
|
||||
}
|
||||
|
||||
async function resolveGeoip(clean: string, dbPath: string): Promise<GeoipResult> {
|
||||
const cacheKey = getSameIpDecisionKey(clean) ?? clean;
|
||||
const cached = getCachedGeoipResult(cacheKey, clean);
|
||||
@@ -145,6 +237,17 @@ async function resolveGeoip(clean: string, dbPath: string): Promise<GeoipResult>
|
||||
return result;
|
||||
}
|
||||
|
||||
async function resolveAsn(clean: string, dbPath: string): Promise<GeoipAsnResult> {
|
||||
const cacheKey = getSameIpDecisionKey(clean) ?? clean;
|
||||
const cached = getCachedAsnResult(cacheKey, clean);
|
||||
if (cached) {
|
||||
return cached;
|
||||
}
|
||||
const result = await lookupMaxmindAsn(clean, dbPath);
|
||||
setCachedAsnResult(cacheKey, result);
|
||||
return result;
|
||||
}
|
||||
|
||||
export async function lookupGeoipByIp(ip: string, dbPath: string | undefined): Promise<GeoipResult> {
|
||||
if (!dbPath) {
|
||||
return buildFallbackResult(ip);
|
||||
@@ -156,6 +259,27 @@ export async function lookupGeoipByIp(ip: string, dbPath: string | undefined): P
|
||||
return resolveGeoip(clean, dbPath);
|
||||
}
|
||||
|
||||
export async function lookupAsnByIp(ip: string, asnDbPath: string | undefined): Promise<GeoipAsnResult> {
|
||||
if (!asnDbPath || maxmindAsnUnavailable) {
|
||||
return buildAsnFallbackResult(null);
|
||||
}
|
||||
const clean = normalizeIpString(ip);
|
||||
if (!isValidIp(clean)) {
|
||||
return buildAsnFallbackResult(clean);
|
||||
}
|
||||
return resolveAsn(clean, asnDbPath);
|
||||
}
|
||||
|
||||
export function resetGeoipReadersForTesting(): void {
|
||||
maxmindReader = null;
|
||||
maxmindReaderPromise = null;
|
||||
maxmindAsnReader = null;
|
||||
maxmindAsnReaderPromise = null;
|
||||
maxmindAsnUnavailable = false;
|
||||
geoipCache.clear();
|
||||
asnCache.clear();
|
||||
}
|
||||
|
||||
export function formatGeoipLocation(result: GeoipResult): string | null {
|
||||
const parts: Array<string> = [];
|
||||
if (result.city) parts.push(result.city);
|
||||
|
||||
@@ -12,6 +12,7 @@ const GEOIP_DOWNLOAD_PATH_QUERY_PARAM = 'download_path';
|
||||
const GEOIP_ASN_DOWNLOAD_PATH_QUERY_PARAM = 'asn_download_path';
|
||||
const GEOIP_ASN_KEY_QUERY_PARAM = 'asn_key';
|
||||
const DEFAULT_GEOIP_TEMPORARY_DIRECTORY = '/tmp/fluxer/geoip';
|
||||
const DEFAULT_GEOIP_ASN_DB_BASENAME = 'GeoLite2-ASN.mmdb';
|
||||
|
||||
type GeoipSourceMode = 'filesystem' | 's3';
|
||||
|
||||
@@ -167,9 +168,11 @@ async function downloadS3Object(
|
||||
}
|
||||
|
||||
function createGeoipFilesystemSourceConfig(rawValue: string | undefined): GeoipFilesystemSourceConfig {
|
||||
const maxmindDbPath = rawValue === '' ? undefined : rawValue;
|
||||
return {
|
||||
mode: 'filesystem',
|
||||
maxmindDbPath: rawValue === '' ? undefined : rawValue,
|
||||
maxmindDbPath,
|
||||
maxmindAsnDbPath: maxmindDbPath ? path.join(path.dirname(maxmindDbPath), DEFAULT_GEOIP_ASN_DB_BASENAME) : undefined,
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -9,6 +9,11 @@ const CACHE_KEY_PREFIX = 'ipinfo:max:';
|
||||
const ISO_DATE_REGEX = /^\d{4}-\d{2}-\d{2}$/u;
|
||||
const POSITIVE_CACHE_TTL_SECONDS = 7 * 24 * 60 * 60;
|
||||
const NEGATIVE_CACHE_TTL_SECONDS = 14 * 24 * 60 * 60;
|
||||
const FAILURE_TTL_REQUEST_FAILED_SECONDS = 60;
|
||||
const FAILURE_TTL_HTTP_ERROR_SECONDS = 300;
|
||||
const FAILURE_TTL_QUOTA_SECONDS = 900;
|
||||
const FAILURE_TTL_SCHEMA_MISMATCH_SECONDS = 600;
|
||||
const FAILURE_TTL_BACKGROUND_CAP_SECONDS = 120;
|
||||
|
||||
export interface IpInfoGeoBlock {
|
||||
countryCode: string | null;
|
||||
@@ -73,6 +78,41 @@ export interface IpInfoCache {
|
||||
set<T>(key: string, value: T, ttlSeconds?: number): Promise<void>;
|
||||
}
|
||||
|
||||
export type IpInfoLookupPriority = 'critical' | 'standard' | 'background';
|
||||
|
||||
export interface IpInfoLookupBudget {
|
||||
tryConsume(priority: IpInfoLookupPriority): Promise<boolean>;
|
||||
}
|
||||
|
||||
export interface CachedIpInfoFailure extends IpInfoLookupResult {
|
||||
cachedFailure: true;
|
||||
failureOutcome: 'http_error' | 'request_failed' | 'schema_mismatch';
|
||||
failureHttpStatus: number | null;
|
||||
cachedAtMs: number;
|
||||
}
|
||||
|
||||
export function resolveIpInfoLookupPriority(source: string | undefined): IpInfoLookupPriority {
|
||||
if (source === 'admin.ip_ban' || source === 'admin.scheduled_deletion_suspicious_ip') return 'critical';
|
||||
if (source === 'AbusiveIpAutoBanner') return 'background';
|
||||
return 'standard';
|
||||
}
|
||||
|
||||
export function isCachedIpInfoFailure(value: unknown): value is CachedIpInfoFailure {
|
||||
return typeof value === 'object' && value !== null && (value as {available?: unknown}).available === false;
|
||||
}
|
||||
|
||||
function failureCacheTtlSeconds(
|
||||
outcome: CachedIpInfoFailure['failureOutcome'],
|
||||
httpStatus: number | null,
|
||||
priority: IpInfoLookupPriority,
|
||||
): number {
|
||||
let ttl = FAILURE_TTL_HTTP_ERROR_SECONDS;
|
||||
if (outcome === 'request_failed') ttl = FAILURE_TTL_REQUEST_FAILED_SECONDS;
|
||||
else if (outcome === 'schema_mismatch') ttl = FAILURE_TTL_SCHEMA_MISMATCH_SECONDS;
|
||||
else if (httpStatus === 402 || httpStatus === 403 || httpStatus === 429) ttl = FAILURE_TTL_QUOTA_SECONDS;
|
||||
return priority === 'background' ? Math.min(ttl, FAILURE_TTL_BACKGROUND_CAP_SECONDS) : ttl;
|
||||
}
|
||||
|
||||
export interface IpInfoLookupContext {
|
||||
source?: string;
|
||||
reason?: string;
|
||||
@@ -86,7 +126,7 @@ export interface IpInfoRequestAuditEvent {
|
||||
source: string;
|
||||
reason: string | null;
|
||||
metadata?: Record<string, string | number | boolean | null>;
|
||||
outcome: 'http_success' | 'http_error' | 'request_failed' | 'schema_mismatch';
|
||||
outcome: 'http_success' | 'http_error' | 'request_failed' | 'schema_mismatch' | 'budget_shed';
|
||||
httpStatus: number | null;
|
||||
available: boolean;
|
||||
riskNote: string;
|
||||
@@ -110,6 +150,7 @@ interface IpInfoServiceContext {
|
||||
apiKey: string;
|
||||
cache: IpInfoCache;
|
||||
auditLogger?: IpInfoRequestAuditLogger;
|
||||
budget?: IpInfoLookupBudget;
|
||||
}
|
||||
|
||||
export interface IpInfoService {
|
||||
@@ -177,8 +218,12 @@ export function createIpInfoService(ctx: IpInfoServiceContext): IpInfoService {
|
||||
return {
|
||||
async lookup(ip: string, context?: IpInfoLookupContext): Promise<IpInfoLookupResult> {
|
||||
const cacheKey = `${CACHE_KEY_PREFIX}${getSameIpDecisionKey(ip) ?? ip}`;
|
||||
const priority = resolveIpInfoLookupPriority(context?.source);
|
||||
const cached = await ctx.cache.get<IpInfoLookupResult>(cacheKey);
|
||||
if (cached !== null) {
|
||||
if (isCachedIpInfoFailure(cached)) {
|
||||
return unavailable(ip, cached.riskNote);
|
||||
}
|
||||
return {...cached, ip};
|
||||
}
|
||||
const existing = inflight.get(cacheKey);
|
||||
@@ -222,6 +267,30 @@ export function createIpInfoService(ctx: IpInfoServiceContext): IpInfoService {
|
||||
return params.result;
|
||||
};
|
||||
const performLookup = async (): Promise<IpInfoLookupResult> => {
|
||||
if (ctx.budget && !(await ctx.budget.tryConsume(priority))) {
|
||||
return finalize({
|
||||
result: unavailable(ip, `IPInfo lookup shed (budget exhausted, priority: ${priority})`),
|
||||
outcome: 'budget_shed',
|
||||
httpStatus: null,
|
||||
});
|
||||
}
|
||||
const finalizeFailure = async (params: {
|
||||
result: IpInfoLookupResult;
|
||||
outcome: CachedIpInfoFailure['failureOutcome'];
|
||||
httpStatus: number | null;
|
||||
}): Promise<IpInfoLookupResult> => {
|
||||
const entry: CachedIpInfoFailure = {
|
||||
...params.result,
|
||||
cachedFailure: true,
|
||||
failureOutcome: params.outcome,
|
||||
failureHttpStatus: params.httpStatus,
|
||||
cachedAtMs: Date.now(),
|
||||
};
|
||||
await ctx.cache
|
||||
.set(cacheKey, entry, failureCacheTtlSeconds(params.outcome, params.httpStatus, priority))
|
||||
.catch(() => {});
|
||||
return finalize(params);
|
||||
};
|
||||
let payload: unknown;
|
||||
try {
|
||||
const res = await fetch(fetchUrl, {
|
||||
@@ -229,7 +298,7 @@ export function createIpInfoService(ctx: IpInfoServiceContext): IpInfoService {
|
||||
headers: {Accept: 'application/json'},
|
||||
});
|
||||
if (!res.ok) {
|
||||
return finalize({
|
||||
return finalizeFailure({
|
||||
result: unavailable(ip, `IPInfo HTTP ${res.status}`),
|
||||
outcome: 'http_error',
|
||||
httpStatus: res.status,
|
||||
@@ -238,7 +307,7 @@ export function createIpInfoService(ctx: IpInfoServiceContext): IpInfoService {
|
||||
payload = await res.json();
|
||||
} catch (err) {
|
||||
const detail = err instanceof Error ? err.message : String(err);
|
||||
return finalize({
|
||||
return finalizeFailure({
|
||||
result: unavailable(ip, `IPInfo request failed: ${detail}`),
|
||||
outcome: 'request_failed',
|
||||
httpStatus: null,
|
||||
@@ -246,7 +315,7 @@ export function createIpInfoService(ctx: IpInfoServiceContext): IpInfoService {
|
||||
}
|
||||
const parsedResponse = RawIpInfoResponseSchema.safeParse(payload);
|
||||
if (!parsedResponse.success) {
|
||||
return finalize({
|
||||
return finalizeFailure({
|
||||
result: unavailable(ip, formatSchemaMismatch(parsedResponse.error)),
|
||||
outcome: 'schema_mismatch',
|
||||
httpStatus: 200,
|
||||
@@ -261,8 +330,10 @@ export function createIpInfoService(ctx: IpInfoServiceContext): IpInfoService {
|
||||
httpStatus: 200,
|
||||
});
|
||||
};
|
||||
const promise = performLookup().finally(() => {
|
||||
inflight.delete(cacheKey);
|
||||
const promise: Promise<IpInfoLookupResult> = performLookup().finally(() => {
|
||||
if (inflight.get(cacheKey) === promise) {
|
||||
inflight.delete(cacheKey);
|
||||
}
|
||||
});
|
||||
inflight.set(cacheKey, promise);
|
||||
return promise;
|
||||
|
||||
@@ -8,6 +8,7 @@ interface TieredIpInfoCacheOptions {
|
||||
hot: IpInfoCache;
|
||||
cold: IpInfoCache;
|
||||
hotTtlSeconds?: number;
|
||||
skipColdWrite?: (value: unknown) => boolean;
|
||||
}
|
||||
|
||||
export function createTieredIpInfoCache(opts: TieredIpInfoCacheOptions): IpInfoCache {
|
||||
@@ -18,14 +19,17 @@ export function createTieredIpInfoCache(opts: TieredIpInfoCacheOptions): IpInfoC
|
||||
if (hit !== null) return hit;
|
||||
const cold = await opts.cold.get<T>(key).catch(() => null);
|
||||
if (cold === null) return null;
|
||||
if (opts.skipColdWrite?.(cold) === true) return cold;
|
||||
void opts.hot.set(key, cold, hotTtl).catch(() => {});
|
||||
return cold;
|
||||
},
|
||||
async set<T>(key: string, value: T, ttlSeconds?: number): Promise<void> {
|
||||
await Promise.all([
|
||||
opts.hot.set(key, value, hotTtl).catch(() => {}),
|
||||
opts.cold.set(key, value, ttlSeconds).catch(() => {}),
|
||||
]);
|
||||
const effectiveHotTtl = Math.max(1, Math.min(hotTtl, ttlSeconds ?? hotTtl));
|
||||
const writes: Array<Promise<void>> = [opts.hot.set(key, value, effectiveHotTtl).catch(() => {})];
|
||||
if (opts.skipColdWrite?.(value) !== true) {
|
||||
writes.push(opts.cold.set(key, value, ttlSeconds).catch(() => {}));
|
||||
}
|
||||
await Promise.all(writes);
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
@@ -30,6 +30,19 @@ function requireArchiveSubjectAccess(adminAcls: Set<string>, subjectType: 'user'
|
||||
throw new MissingACLError(subjectType === 'user' ? AdminACLs.ARCHIVE_TRIGGER_USER : AdminACLs.ARCHIVE_TRIGGER_GUILD);
|
||||
}
|
||||
|
||||
function resolveListSubjectType(adminAcls: Set<string>, requested: 'all' | 'user' | 'guild'): 'all' | 'user' | 'guild' {
|
||||
if (requested !== 'all') {
|
||||
requireArchiveSubjectAccess(adminAcls, requested);
|
||||
return requested;
|
||||
}
|
||||
const viewUser = canViewArchive(adminAcls, 'user');
|
||||
const viewGuild = canViewArchive(adminAcls, 'guild');
|
||||
if (viewUser && viewGuild) return 'all';
|
||||
if (viewUser) return 'user';
|
||||
if (viewGuild) return 'guild';
|
||||
throw new MissingACLError(AdminACLs.ARCHIVE_VIEW_ALL);
|
||||
}
|
||||
|
||||
export function ArchiveAdminController(app: HonoApp) {
|
||||
app.post(
|
||||
'/admin/users/:user_id/archives',
|
||||
@@ -104,18 +117,8 @@ export function ArchiveAdminController(app: HonoApp) {
|
||||
const adminArchiveService = ctx.get('adminArchiveService');
|
||||
const adminAcls = ctx.get('adminUserAcls');
|
||||
const query = ctx.req.valid('query');
|
||||
if (
|
||||
query.subject_type === 'all' &&
|
||||
!adminAcls.has(AdminACLs.ARCHIVE_VIEW_ALL) &&
|
||||
!adminAcls.has(AdminACLs.WILDCARD)
|
||||
) {
|
||||
throw new MissingACLError(AdminACLs.ARCHIVE_VIEW_ALL);
|
||||
}
|
||||
if (query.subject_type !== 'all') {
|
||||
requireArchiveSubjectAccess(adminAcls, query.subject_type);
|
||||
}
|
||||
const result = await adminArchiveService.listArchives({
|
||||
subjectType: query.subject_type,
|
||||
subjectType: resolveListSubjectType(adminAcls, query.subject_type),
|
||||
subjectId: query.subject_id ?? undefined,
|
||||
requestedBy: query.requested_by ?? undefined,
|
||||
limit: query.limit,
|
||||
|
||||
@@ -72,7 +72,10 @@ function selectGuildUpdateACLs(body: UpdateGuildRequest): Array<string> {
|
||||
if (body.new_owner_id !== undefined) {
|
||||
required.push(AdminACLs.GUILD_TRANSFER_OWNERSHIP);
|
||||
}
|
||||
return required.length > 0 ? required : [AdminACLs.WILDCARD];
|
||||
if (required.length > 0) {
|
||||
return required;
|
||||
}
|
||||
return Object.keys(body).length === 0 ? [] : [AdminACLs.WILDCARD];
|
||||
}
|
||||
|
||||
function requireAllAdminACLs(granted: ReadonlySet<string>, required: ReadonlyArray<string>): void {
|
||||
@@ -148,7 +151,7 @@ export function GuildAdminController(app: HonoApp) {
|
||||
operationId: 'update_admin_guild',
|
||||
summary: 'Update guild',
|
||||
description:
|
||||
'Partially updates a guild. The permissions required are selected by the fields present in the body and are evaluated with all-of semantics: name requires GUILD_UPDATE_NAME, vanity_url_code requires GUILD_UPDATE_VANITY, new_owner_id requires GUILD_TRANSFER_OWNERSHIP, add_features and remove_features require GUILD_UPDATE_FEATURES, and fields together with every other setting requires GUILD_UPDATE_SETTINGS. A body carrying no field requires the wildcard permission. Every applied change is logged to the audit log.',
|
||||
'Partially updates a guild. The permissions required are selected by the fields present in the body and are evaluated with all-of semantics: name requires GUILD_UPDATE_NAME, vanity_url_code requires GUILD_UPDATE_VANITY, new_owner_id requires GUILD_TRANSFER_OWNERSHIP, add_features and remove_features require GUILD_UPDATE_FEATURES, and fields together with every other setting requires GUILD_UPDATE_SETTINGS. A body with no fields applies no change. Every applied change is logged to the audit log.',
|
||||
responseSchema: GuildUpdateResponse,
|
||||
statusCode: 200,
|
||||
security: 'adminApiKey',
|
||||
|
||||
@@ -88,12 +88,12 @@ export function ReportAdminController(app: HonoApp) {
|
||||
const adminService = ctx.get('adminService');
|
||||
const adminUserAcls = ctx.get('adminUserAcls');
|
||||
const query = ctx.req.valid('query');
|
||||
if (usesReportSearchIndex(query)) {
|
||||
if (query.status === undefined || usesReportSearchIndex(query)) {
|
||||
return ctx.json(
|
||||
await adminService.reportServiceAggregate.searchReports(toSearchReportsRequest(query), adminUserAcls),
|
||||
);
|
||||
}
|
||||
const status = query.status === undefined ? 0 : REPORT_STATUS_BY_FILTER[query.status];
|
||||
const status = REPORT_STATUS_BY_FILTER[query.status];
|
||||
return ctx.json(
|
||||
await adminService.reportServiceAggregate.listReports(status, adminUserAcls, query.limit, query.offset),
|
||||
);
|
||||
|
||||
@@ -30,8 +30,8 @@ import {phraseBlocklistCache} from '../../middleware/PhraseBlocklistCache';
|
||||
import {profileSubstringBlocklistCache} from '../../middleware/ProfileSubstringBlocklistCache';
|
||||
import {urlBlocklistCache} from '../../middleware/UrlBlocklistCache';
|
||||
import {
|
||||
getIpBanBlastRadiusVerdict,
|
||||
getSuspiciousIpSkipReason,
|
||||
hasHighCgnatBlastRadiusRisk,
|
||||
isSingleIpBanCandidate,
|
||||
} from '../../risk/IpBanCgnatGuard';
|
||||
import {isIpBanExempt} from '../../risk/IpBanExemptions';
|
||||
@@ -292,7 +292,7 @@ export class AdminBanManagementService {
|
||||
return false;
|
||||
}
|
||||
try {
|
||||
const highRisk = await hasHighCgnatBlastRadiusRisk(ip, this.deps.ipInfoService, {
|
||||
const {cgnat: highRisk} = await getIpBanBlastRadiusVerdict(ip, this.deps.ipInfoService, {
|
||||
source: 'admin.ip_ban',
|
||||
reason: 'pre_write_cgnat_guard',
|
||||
});
|
||||
|
||||
@@ -74,7 +74,7 @@ export class AdminReportService {
|
||||
const {reportService} = this.deps;
|
||||
const requestedLimit = limit || 50;
|
||||
const currentOffset = offset || 0;
|
||||
const reports = await reportService.listReportsByStatus(status, requestedLimit, currentOffset);
|
||||
const {reports, total} = await reportService.listReportsByStatus(status, requestedLimit, currentOffset);
|
||||
const requestCache = createRequestCache();
|
||||
const reportNsfwLookupCache = createReportNsfwLookupCache();
|
||||
const reportResponses = await Promise.all(
|
||||
@@ -84,6 +84,9 @@ export class AdminReportService {
|
||||
);
|
||||
return {
|
||||
reports: reportResponses,
|
||||
total,
|
||||
offset: currentOffset,
|
||||
limit: requestedLimit,
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -315,7 +315,7 @@ export class AdminUserSecurityService {
|
||||
if (!user) {
|
||||
throw new UnknownUserError();
|
||||
}
|
||||
await AuthSession.terminateAllUserSessions(this.deps.apiContext, userId);
|
||||
const terminatedCount = await AuthSession.terminateAllUserSessions(this.deps.apiContext, userId);
|
||||
await auditService.createAuditLog({
|
||||
adminUserId,
|
||||
targetType: 'user',
|
||||
@@ -324,6 +324,7 @@ export class AdminUserSecurityService {
|
||||
auditLogReason,
|
||||
metadata: new Map(),
|
||||
});
|
||||
return {terminated_count: terminatedCount};
|
||||
}
|
||||
|
||||
async setUserAcls(
|
||||
|
||||
@@ -196,6 +196,18 @@ export class AdminGuildUpdateService {
|
||||
patch.nsfw_level = data.nsfw_level;
|
||||
metadata.set('nsfw_level', data.nsfw_level.toString());
|
||||
}
|
||||
if (data.nsfw !== undefined) {
|
||||
patch.nsfw = data.nsfw;
|
||||
metadata.set('nsfw', data.nsfw.toString());
|
||||
}
|
||||
if (data.content_warning_level !== undefined) {
|
||||
patch.content_warning_level = data.content_warning_level;
|
||||
metadata.set('content_warning_level', data.content_warning_level.toString());
|
||||
}
|
||||
if (data.content_warning_text !== undefined) {
|
||||
patch.content_warning_text = data.content_warning_text;
|
||||
metadata.set('content_warning_text', data.content_warning_text ?? '');
|
||||
}
|
||||
if (data.explicit_content_filter !== undefined) {
|
||||
patch.explicit_content_filter = data.explicit_content_filter;
|
||||
metadata.set('explicit_content_filter', data.explicit_content_filter.toString());
|
||||
|
||||
@@ -81,15 +81,17 @@ describe('Admin guild routes', () => {
|
||||
.expect(HTTP_STATUS.FORBIDDEN, 'MISSING_ACL')
|
||||
.execute();
|
||||
});
|
||||
test('PATCH /admin/guilds/{guild_id} rejects an empty patch without the wildcard ACL', async () => {
|
||||
test('PATCH /admin/guilds/{guild_id} applies no change for an empty patch', async () => {
|
||||
const admin = await createTestAccount(harness);
|
||||
await setUserACLs(harness, admin, ['admin:authenticate', 'guild:update:name']);
|
||||
const guild = await createGuild(harness, admin.token, `Empty Patch Guild ${Date.now()}`);
|
||||
await createBuilder(harness, `${admin.token}`)
|
||||
const name = `Empty Patch Guild ${Date.now()}`;
|
||||
const guild = await createGuild(harness, admin.token, name);
|
||||
const result = await createBuilder<AdminGuildUpdate>(harness, `${admin.token}`)
|
||||
.patch(`/admin/guilds/${guild.id}`)
|
||||
.body({})
|
||||
.expect(HTTP_STATUS.FORBIDDEN, 'MISSING_ACL')
|
||||
.expect(HTTP_STATUS.OK)
|
||||
.execute();
|
||||
expect(result.guild.name).toBe(name);
|
||||
});
|
||||
test('guild member add, ban and removal use the member and ban sub-resources', async () => {
|
||||
const admin = await createTestAccount(harness);
|
||||
|
||||
@@ -178,18 +178,19 @@ export async function logoutAuthSessions(
|
||||
});
|
||||
}
|
||||
|
||||
export async function terminateAllUserSessions(ctx: ApiContext, userId: UserID): Promise<void> {
|
||||
export async function terminateAllUserSessions(ctx: ApiContext, userId: UserID): Promise<number> {
|
||||
const {users, gateway} = ctx.services;
|
||||
const authSessions = await users.listAuthSessions(userId);
|
||||
await users.deleteAllPushSubscriptions(userId);
|
||||
await gateway.invalidatePushSubscriptions({userId});
|
||||
if (authSessions.length === 0) return;
|
||||
if (authSessions.length === 0) return 0;
|
||||
const hashes = authSessions.map((s) => s.sessionIdHash);
|
||||
await users.deleteAuthSessions(userId, hashes);
|
||||
await gateway.terminateSession({
|
||||
userId,
|
||||
sessionIdHashes: authSessions.map((s) => Buffer.from(s.sessionIdHash).toString('base64url')),
|
||||
});
|
||||
return authSessions.length;
|
||||
}
|
||||
|
||||
export async function replaceCurrentAuthSession(
|
||||
|
||||
@@ -632,6 +632,8 @@ describe('Deferred phone verification gate', () => {
|
||||
await configurePhoneGate({deferred_phone_gate_window_hours: 0.0001});
|
||||
const outsideWindow = await createGuildWithInvite(harness);
|
||||
await addFillerMember(outsideWindow.inviteCode);
|
||||
const beforeJoin = await readFlags(subject.userId);
|
||||
expect(beforeJoin & DEFERRED_PHONE_ON_COMMUNITY_JOIN).not.toBe(0);
|
||||
await createBuilder(harness, subject.token).post(`/invites/${outsideWindow.inviteCode}`).expect(200).execute();
|
||||
|
||||
const flags = await readFlags(subject.userId);
|
||||
|
||||
@@ -123,6 +123,7 @@ export function ChannelController(app: HonoApp) {
|
||||
const existing = await ctx.get('channelService').channelData.operations.getChannel({
|
||||
userId: ctx.get('user').id,
|
||||
channelId,
|
||||
skipNsfwValidation: true,
|
||||
});
|
||||
ctx.set('channelUpdateType', existing.type);
|
||||
return undefined;
|
||||
|
||||
@@ -11,6 +11,7 @@ import {
|
||||
} from '@fluxer/constants/src/LimitConstants';
|
||||
import {ValidationErrorCodes} from '@fluxer/constants/src/ValidationErrorCodes';
|
||||
import {CannotSendMessageToNonTextChannelError} from '@fluxer/errors/src/domains/channel/CannotSendMessageToNonTextChannelError';
|
||||
import {UnknownChannelError} from '@fluxer/errors/src/domains/channel/UnknownChannelError';
|
||||
import {UnknownMessageError} from '@fluxer/errors/src/domains/channel/UnknownMessageError';
|
||||
import {FeatureTemporarilyDisabledError} from '@fluxer/errors/src/domains/core/FeatureTemporarilyDisabledError';
|
||||
import {FileSizeTooLargeError} from '@fluxer/errors/src/domains/core/FileSizeTooLargeError';
|
||||
@@ -18,6 +19,7 @@ import {InputValidationError} from '@fluxer/errors/src/domains/core/InputValidat
|
||||
import {MissingPermissionsError} from '@fluxer/errors/src/domains/core/MissingPermissionsError';
|
||||
import {UnknownUserError} from '@fluxer/errors/src/domains/user/UnknownUserError';
|
||||
import {ServiceUnavailableError} from '@fluxer/errors/src/HttpErrors';
|
||||
import type {GuildResponse} from '@fluxer/schema/src/domains/guild/GuildResponseSchemas';
|
||||
import type {
|
||||
CompleteMultipartAttachmentUploadItem,
|
||||
CompleteMultipartAttachmentUploadResult,
|
||||
@@ -26,7 +28,9 @@ import type {
|
||||
} from '@fluxer/schema/src/domains/message/AttachmentUploadSchemas';
|
||||
import type {AttachmentID, ChannelID, MessageID, UserID} from '../../BrandedTypes';
|
||||
import {Config} from '../../Config';
|
||||
import {SYSTEM_USER_ID} from '../../constants/Core';
|
||||
import type {IPurgeQueue} from '../../infrastructure/BunnyPurgeQueue';
|
||||
import type {IGatewayService} from '../../infrastructure/IGatewayService';
|
||||
import type {IStorageService} from '../../infrastructure/IStorageService';
|
||||
import type {LimitConfigService} from '../../limits/LimitConfigService';
|
||||
import {resolveLimitSafe} from '../../limits/LimitConfigUtils';
|
||||
@@ -64,6 +68,8 @@ interface DeleteAttachmentParams {
|
||||
requestCache: RequestCache;
|
||||
}
|
||||
|
||||
type UploadActor = 'member' | 'webhook';
|
||||
|
||||
interface UploadFormDataAttachmentsParams {
|
||||
userId: UserID;
|
||||
channelId: ChannelID;
|
||||
@@ -76,6 +82,7 @@ interface UploadFormDataAttachmentsParams {
|
||||
id: number;
|
||||
filename: string;
|
||||
}>;
|
||||
actor?: UploadActor;
|
||||
}
|
||||
|
||||
interface RequestPresignedAttachmentUploadUrlsParams {
|
||||
@@ -104,6 +111,7 @@ export class AttachmentUploadService {
|
||||
private messageInteractionService: MessageInteractionService,
|
||||
private messageService: MessageService,
|
||||
private limitConfigService: LimitConfigService,
|
||||
private gatewayService: IGatewayService,
|
||||
) {}
|
||||
|
||||
async uploadFormDataAttachments({
|
||||
@@ -112,8 +120,9 @@ export class AttachmentUploadService {
|
||||
clientIp,
|
||||
files,
|
||||
attachmentMetadata,
|
||||
actor = 'member',
|
||||
}: UploadFormDataAttachmentsParams): Promise<Array<UploadedAttachment>> {
|
||||
const {maxFileSize} = await this.getUploadPermissionAndLimit({userId, channelId});
|
||||
const {maxFileSize} = await this.getUploadPermissionAndLimit({userId, channelId, actor});
|
||||
assertAttachmentFileSizesWithinLimit(
|
||||
files.map(({file}) => file.size),
|
||||
maxFileSize,
|
||||
@@ -168,7 +177,7 @@ export class AttachmentUploadService {
|
||||
if (!Config.presignedAttachmentUploadsEnabled) {
|
||||
throw new FeatureTemporarilyDisabledError();
|
||||
}
|
||||
const {maxFileSize} = await this.getUploadPermissionAndLimit({userId, channelId});
|
||||
const {maxFileSize} = await this.getUploadPermissionAndLimit({userId, channelId, actor: 'member'});
|
||||
assertAttachmentFileSizesWithinLimit(
|
||||
attachments.map(({file_size}) => file_size),
|
||||
maxFileSize,
|
||||
@@ -275,7 +284,7 @@ export class AttachmentUploadService {
|
||||
if (!Config.presignedAttachmentUploadsEnabled) {
|
||||
throw new FeatureTemporarilyDisabledError();
|
||||
}
|
||||
const {maxFileSize} = await this.getUploadPermissionAndLimit({userId, channelId});
|
||||
const {maxFileSize} = await this.getUploadPermissionAndLimit({userId, channelId, actor: 'member'});
|
||||
const bucket = Config.s3.buckets.uploads;
|
||||
return Promise.all(
|
||||
uploads.map(async ({upload_filename, upload_id}, index) => {
|
||||
@@ -412,21 +421,24 @@ export class AttachmentUploadService {
|
||||
}
|
||||
}
|
||||
|
||||
private async getUploadPermissionAndLimit({userId, channelId}: {userId: UserID; channelId: ChannelID}): Promise<{
|
||||
private async getUploadPermissionAndLimit({
|
||||
userId,
|
||||
channelId,
|
||||
actor,
|
||||
}: {
|
||||
userId: UserID;
|
||||
channelId: ChannelID;
|
||||
actor: UploadActor;
|
||||
}): Promise<{
|
||||
maxFileSize: number;
|
||||
}> {
|
||||
const {channel, guild, checkPermission, member} =
|
||||
await this.messageInteractionService.authService.getChannelAuthenticated({
|
||||
userId,
|
||||
channelId,
|
||||
});
|
||||
const {channel, guild} =
|
||||
actor === 'webhook'
|
||||
? await this.getWebhookUploadChannel(channelId)
|
||||
: await this.getMemberUploadChannel({userId, channelId});
|
||||
if (!TEXT_BASED_CHANNEL_TYPES.has(channel.type)) {
|
||||
throw new CannotSendMessageToNonTextChannelError();
|
||||
}
|
||||
if (guild) {
|
||||
await checkPermission(Permissions.SEND_MESSAGES | Permissions.ATTACH_FILES);
|
||||
assertGuildMemberCanCommunicate(member);
|
||||
}
|
||||
const user = await this.userRepository.findUnique(userId);
|
||||
if (!user) {
|
||||
throw new UnknownUserError();
|
||||
@@ -439,6 +451,41 @@ export class AttachmentUploadService {
|
||||
const maxFileSize = user.isBot ? Math.min(resolvedMaxFileSize, ATTACHMENT_MAX_SIZE_BOT) : resolvedMaxFileSize;
|
||||
return {maxFileSize};
|
||||
}
|
||||
|
||||
private async getMemberUploadChannel({userId, channelId}: {userId: UserID; channelId: ChannelID}): Promise<{
|
||||
channel: Channel;
|
||||
guild: GuildResponse | null;
|
||||
}> {
|
||||
const {channel, guild, checkPermission, member} =
|
||||
await this.messageInteractionService.authService.getChannelAuthenticated({
|
||||
userId,
|
||||
channelId,
|
||||
});
|
||||
if (guild) {
|
||||
await checkPermission(Permissions.SEND_MESSAGES | Permissions.ATTACH_FILES);
|
||||
assertGuildMemberCanCommunicate(member);
|
||||
}
|
||||
return {channel, guild};
|
||||
}
|
||||
|
||||
private async getWebhookUploadChannel(channelId: ChannelID): Promise<{
|
||||
channel: Channel;
|
||||
guild: GuildResponse | null;
|
||||
}> {
|
||||
const channel = await this.channelRepository.channelData.findUnique(channelId);
|
||||
if (!channel) {
|
||||
throw new UnknownChannelError();
|
||||
}
|
||||
if (!channel.guildId) {
|
||||
return {channel, guild: null};
|
||||
}
|
||||
const guild = await this.gatewayService.getGuildData({
|
||||
guildId: channel.guildId,
|
||||
userId: SYSTEM_USER_ID,
|
||||
skipMembershipCheck: true,
|
||||
});
|
||||
return {channel, guild};
|
||||
}
|
||||
}
|
||||
|
||||
async function mapWithConcurrency<T, TResult>(
|
||||
|
||||
@@ -111,7 +111,7 @@ export class ChannelDataService {
|
||||
clientFeatures: ReadonlySet<string>;
|
||||
requestCache: RequestCache;
|
||||
}): Promise<Channel> {
|
||||
const {channel} = await this.auth.getChannelAuthenticated({userId, channelId});
|
||||
const {channel} = await this.auth.getChannelAuthenticated({userId, channelId, skipNsfwValidation: true});
|
||||
if (channel.type === ChannelTypes.GROUP_DM) {
|
||||
return await this.groupDmUpdate.updateGroupDmChannel({
|
||||
userId,
|
||||
|
||||
@@ -162,6 +162,7 @@ export class ChannelService {
|
||||
this.interactions,
|
||||
this.messages,
|
||||
limitConfigService,
|
||||
gatewayService,
|
||||
);
|
||||
this.groupDms = new GroupDmOperationsService(
|
||||
channelRepository,
|
||||
|
||||
@@ -90,8 +90,20 @@ export class ChannelOperationsService {
|
||||
private rateLimitService: IRateLimitService,
|
||||
) {}
|
||||
|
||||
async getChannel({userId, channelId}: {userId: UserID; channelId: ChannelID}): Promise<Channel> {
|
||||
const {channel} = await this.channelAuthService.getChannelAuthenticated({userId, channelId});
|
||||
async getChannel({
|
||||
userId,
|
||||
channelId,
|
||||
skipNsfwValidation,
|
||||
}: {
|
||||
userId: UserID;
|
||||
channelId: ChannelID;
|
||||
skipNsfwValidation?: boolean;
|
||||
}): Promise<Channel> {
|
||||
const {channel} = await this.channelAuthService.getChannelAuthenticated({
|
||||
userId,
|
||||
channelId,
|
||||
skipNsfwValidation,
|
||||
});
|
||||
return channel;
|
||||
}
|
||||
|
||||
@@ -127,6 +139,7 @@ export class ChannelOperationsService {
|
||||
const {channel, guild, checkPermission} = await this.channelAuthService.getChannelAuthenticated({
|
||||
userId,
|
||||
channelId,
|
||||
skipNsfwValidation: true,
|
||||
});
|
||||
if (channel.type === ChannelTypes.GROUP_DM) {
|
||||
throw new InvalidChannelTypeError();
|
||||
@@ -456,7 +469,11 @@ export class ChannelOperationsService {
|
||||
if (this.voiceAvailabilityService === null) {
|
||||
return [];
|
||||
}
|
||||
const {channel, guild} = await this.channelAuthService.getChannelAuthenticated({userId, channelId});
|
||||
const {channel, guild} = await this.channelAuthService.getChannelAuthenticated({
|
||||
userId,
|
||||
channelId,
|
||||
skipNsfwValidation: true,
|
||||
});
|
||||
if (channel.type !== ChannelTypes.GUILD_VOICE) {
|
||||
throw new InvalidChannelTypeError();
|
||||
}
|
||||
|
||||
@@ -24,6 +24,8 @@ import type {MessagePersistenceService} from '../message/MessagePersistenceServi
|
||||
import {createMessageResponseDataService} from '../message/MessageResponseDataService';
|
||||
import {MessageInteractionBase} from './MessageInteractionBase';
|
||||
|
||||
const PIN_LIST_UNBOUNDED_TIMESTAMP = new Date('9999-12-31T23:59:59.999Z');
|
||||
|
||||
export class MessagePinService extends MessageInteractionBase {
|
||||
constructor(
|
||||
gatewayService: IGatewayService,
|
||||
@@ -81,7 +83,7 @@ export class MessagePinService extends MessageInteractionBase {
|
||||
const pageSize = Math.min(limit ?? 50, 50);
|
||||
const cutoffTimestamp = hasReadHistory ? null : new Date(authChannel.guild!.message_history_cutoff!).getTime();
|
||||
const filtered: Array<Message> = [];
|
||||
let before = beforeTimestamp ?? new Date();
|
||||
let before = beforeTimestamp ?? PIN_LIST_UNBOUNDED_TIMESTAMP;
|
||||
let exhausted = false;
|
||||
while (filtered.length <= pageSize && !exhausted) {
|
||||
const messages = await this.channelRepository.messageInteractions.listChannelPins(
|
||||
|
||||
@@ -0,0 +1,108 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
import {ChannelTypes} from '@fluxer/constants/src/ChannelConstants';
|
||||
import {describe, expect, it} from 'vitest';
|
||||
import {
|
||||
type ChannelID,
|
||||
createChannelID,
|
||||
createMessageID,
|
||||
createUserID,
|
||||
type MessageID,
|
||||
type UserID,
|
||||
} from '../../../BrandedTypes';
|
||||
import type {ChannelRow} from '../../../database/types/ChannelTypes';
|
||||
import type {IGatewayService} from '../../../infrastructure/IGatewayService';
|
||||
import type {UserCacheService} from '../../../infrastructure/UserCacheService';
|
||||
import type {RequestCache} from '../../../middleware/RequestCacheMiddleware';
|
||||
import {Channel} from '../../../models/Channel';
|
||||
import type {IUserRepository} from '../../../user/IUserRepository';
|
||||
import {MessageProcessingService} from './MessageProcessingService';
|
||||
|
||||
const CHANNEL_ID = createChannelID(1532860318772891648n);
|
||||
const AUTHOR_ID = createUserID(1471426754353995881n);
|
||||
const RECIPIENT_ID = createUserID(1485344055661987728n);
|
||||
const MESSAGE_ID = createMessageID(1546325276953149440n);
|
||||
|
||||
function dmChannelRow(lastMessageId: MessageID | null): ChannelRow {
|
||||
return {
|
||||
channel_id: CHANNEL_ID,
|
||||
guild_id: null,
|
||||
type: ChannelTypes.DM,
|
||||
name: null,
|
||||
topic: null,
|
||||
icon_hash: null,
|
||||
url: null,
|
||||
parent_id: null,
|
||||
position: null,
|
||||
owner_id: null,
|
||||
recipient_ids: new Set<UserID>([AUTHOR_ID, RECIPIENT_ID]),
|
||||
nsfw: null,
|
||||
content_warning_level: null,
|
||||
content_warning_text: null,
|
||||
rate_limit_per_user: null,
|
||||
bitrate: null,
|
||||
user_limit: null,
|
||||
voice_connection_limit: null,
|
||||
rtc_region: null,
|
||||
last_message_id: lastMessageId,
|
||||
last_pin_timestamp: null,
|
||||
permission_overwrites: null,
|
||||
nicks: null,
|
||||
soft_deleted: false,
|
||||
indexed_at: null,
|
||||
version: 0,
|
||||
};
|
||||
}
|
||||
|
||||
function buildService(): {service: MessageProcessingService; opened: Array<Channel>} {
|
||||
const opened: Array<Channel> = [];
|
||||
const userRepository = {
|
||||
isDmChannelOpen: async (userId: UserID, _channelId: ChannelID) => userId === AUTHOR_ID,
|
||||
openPrivateChannelForUser: async (_userId: UserID, channel: Channel) => {
|
||||
opened.push(channel);
|
||||
},
|
||||
} as unknown as IUserRepository;
|
||||
const userCacheService = {
|
||||
getUserPartialResponses: async (userIds: Array<UserID>) =>
|
||||
new Map(userIds.map((userId) => [userId, {id: userId.toString()}])),
|
||||
} as unknown as UserCacheService;
|
||||
const gatewayService = {
|
||||
dispatchPresence: async () => {},
|
||||
} as unknown as IGatewayService;
|
||||
const service = new MessageProcessingService(
|
||||
undefined as never,
|
||||
userRepository,
|
||||
userCacheService,
|
||||
gatewayService,
|
||||
undefined as never,
|
||||
undefined as never,
|
||||
);
|
||||
return {service, opened};
|
||||
}
|
||||
|
||||
describe('MessageProcessingService.updateDMRecipients', () => {
|
||||
it('snapshots the new message id when the in-request channel is stale', async () => {
|
||||
const {service, opened} = buildService();
|
||||
await service.updateDMRecipients({
|
||||
channel: new Channel(dmChannelRow(null)),
|
||||
channelId: CHANNEL_ID,
|
||||
messageId: MESSAGE_ID,
|
||||
requestCache: {} as RequestCache,
|
||||
});
|
||||
expect(opened).toHaveLength(1);
|
||||
expect(opened[0].lastMessageId).toBe(MESSAGE_ID);
|
||||
});
|
||||
|
||||
it('keeps a newer last message id already present on the channel', async () => {
|
||||
const {service, opened} = buildService();
|
||||
const newer = createMessageID(MESSAGE_ID + 10n);
|
||||
await service.updateDMRecipients({
|
||||
channel: new Channel(dmChannelRow(newer)),
|
||||
channelId: CHANNEL_ID,
|
||||
messageId: MESSAGE_ID,
|
||||
requestCache: {} as RequestCache,
|
||||
});
|
||||
expect(opened).toHaveLength(1);
|
||||
expect(opened[0].lastMessageId).toBe(newer);
|
||||
});
|
||||
});
|
||||
@@ -9,7 +9,7 @@ import type {GatewayChannelMention, IGatewayService} from '../../../infrastructu
|
||||
import type {UserCacheService} from '../../../infrastructure/UserCacheService';
|
||||
import {Logger} from '../../../Logger';
|
||||
import type {RequestCache} from '../../../middleware/RequestCacheMiddleware';
|
||||
import type {Channel} from '../../../models/Channel';
|
||||
import {Channel} from '../../../models/Channel';
|
||||
import type {Message} from '../../../models/Message';
|
||||
import type {User} from '../../../models/User';
|
||||
import type {ReadStateService} from '../../../read_state/ReadStateService';
|
||||
@@ -33,6 +33,13 @@ interface MentionProcessingResult {
|
||||
mentionChannels: Array<GatewayChannelMention>;
|
||||
}
|
||||
|
||||
function channelWithLastMessageId(channel: Channel, messageId: MessageID): Channel {
|
||||
if (channel.lastMessageId != null && channel.lastMessageId >= messageId) {
|
||||
return channel;
|
||||
}
|
||||
return new Channel({...channel.toRow(), last_message_id: messageId});
|
||||
}
|
||||
|
||||
export class MessageProcessingService {
|
||||
constructor(
|
||||
private channelRepository: IChannelRepositoryAggregate,
|
||||
@@ -64,10 +71,12 @@ export class MessageProcessingService {
|
||||
async updateDMRecipients({
|
||||
channel,
|
||||
channelId,
|
||||
messageId,
|
||||
requestCache,
|
||||
}: {
|
||||
channel: Channel;
|
||||
channelId: ChannelID;
|
||||
messageId: MessageID;
|
||||
requestCache: RequestCache;
|
||||
}): Promise<void> {
|
||||
if (channel.guildId || channel.type !== ChannelTypes.DM) return;
|
||||
@@ -76,11 +85,12 @@ export class MessageProcessingService {
|
||||
const openStates = await this.batchCheckDmChannelOpen(recipientIds, channelId);
|
||||
const closedRecipients = openStates.filter((state) => !state.isOpen);
|
||||
if (closedRecipients.length === 0) return;
|
||||
const snapshotChannel = channelWithLastMessageId(channel, messageId);
|
||||
await Promise.all(
|
||||
closedRecipients.map((state) =>
|
||||
this.openDmAndDispatch({
|
||||
recipientId: state.recipientId,
|
||||
channel,
|
||||
channel: snapshotChannel,
|
||||
requestCache,
|
||||
}),
|
||||
),
|
||||
|
||||
@@ -34,6 +34,7 @@ type AttachmentMetadata = ClientAttachmentRequest | ClientUploadedAttachmentRequ
|
||||
|
||||
interface ParseMultipartMessageDataOptions {
|
||||
onPayloadParsed?: (payload: unknown) => void;
|
||||
actor?: 'member' | 'webhook';
|
||||
}
|
||||
|
||||
export async function parseMultipartMessageData(
|
||||
@@ -158,6 +159,7 @@ export async function parseMultipartMessageData(
|
||||
clientIp,
|
||||
files: filesWithIndices,
|
||||
attachmentMetadata: inlineNewAttachments,
|
||||
actor: options?.actor,
|
||||
});
|
||||
const uploadedMap = new Map(uploadedAttachments.map((attachment) => [attachment.id, attachment]));
|
||||
const processedInlineAttachments = inlineNewAttachments.map((clientData) => {
|
||||
|
||||
@@ -9,7 +9,12 @@ import {
|
||||
SENDABLE_MESSAGE_FLAGS,
|
||||
} from '@fluxer/constants/src/ChannelConstants';
|
||||
import {GuildNSFWLevel, GuildOperations} from '@fluxer/constants/src/GuildConstants';
|
||||
import {RelationshipTypes, SensitiveMediaFilterLevel, UserFlags} from '@fluxer/constants/src/UserConstants';
|
||||
import {
|
||||
DELETED_USER_ID,
|
||||
RelationshipTypes,
|
||||
SensitiveMediaFilterLevel,
|
||||
UserFlags,
|
||||
} from '@fluxer/constants/src/UserConstants';
|
||||
import {ValidationErrorCodes} from '@fluxer/constants/src/ValidationErrorCodes';
|
||||
import {UnknownChannelError} from '@fluxer/errors/src/domains/channel/UnknownChannelError';
|
||||
import {UnknownMessageError} from '@fluxer/errors/src/domains/channel/UnknownMessageError';
|
||||
@@ -211,15 +216,18 @@ export class MessageSendService {
|
||||
return processed.length > 0 ? processed : undefined;
|
||||
}
|
||||
|
||||
private resolveWebhookAttachmentUploadUserId(
|
||||
private async resolveWebhookAttachmentUploadUserId(
|
||||
webhook: Webhook,
|
||||
attachments?: Array<AttachmentRequestData>,
|
||||
): UserID | undefined {
|
||||
const uploadUserId = webhook.creatorId ?? undefined;
|
||||
if (uploadUserId === undefined && this.attachmentsToProcess(attachments) !== undefined) {
|
||||
throw InputValidationError.fromCode('attachments', ValidationErrorCodes.INVALID_MESSAGE_DATA);
|
||||
): Promise<UserID | undefined> {
|
||||
if (this.attachmentsToProcess(attachments) === undefined) {
|
||||
return webhook.creatorId ?? undefined;
|
||||
}
|
||||
return uploadUserId;
|
||||
if (!webhook.creatorId) {
|
||||
return createUserID(DELETED_USER_ID);
|
||||
}
|
||||
const creator = await this.deps.userRepository.findUnique(webhook.creatorId);
|
||||
return creator ? webhook.creatorId : createUserID(DELETED_USER_ID);
|
||||
}
|
||||
|
||||
private getOneToOneDmRecipientId(channel: Channel, senderId: UserID): UserID | null {
|
||||
@@ -980,7 +988,7 @@ export class MessageSendService {
|
||||
await this.settlePostCreateWork(messageId, [
|
||||
{
|
||||
step: 'update_dm_recipients',
|
||||
promise: this.deps.processingService.updateDMRecipients({channel, channelId, requestCache}),
|
||||
promise: this.deps.processingService.updateDMRecipients({channel, channelId, messageId, requestCache}),
|
||||
},
|
||||
{
|
||||
step: 'process_message_after_creation',
|
||||
@@ -1184,7 +1192,7 @@ export class MessageSendService {
|
||||
flags: this.deps.validationService.calculateMessageFlags(data),
|
||||
embeds: data.embeds,
|
||||
attachments: this.attachmentsToProcess(data.attachments),
|
||||
attachmentUploadUserId: this.resolveWebhookAttachmentUploadUserId(webhook, data.attachments),
|
||||
attachmentUploadUserId: await this.resolveWebhookAttachmentUploadUserId(webhook, data.attachments),
|
||||
stickerIds: data.sticker_ids ? data.sticker_ids.flatMap((stickerId) => createStickerID(stickerId)) : undefined,
|
||||
messageReference,
|
||||
messageSnapshots,
|
||||
@@ -1269,7 +1277,7 @@ export class MessageSendService {
|
||||
data,
|
||||
channel,
|
||||
guild,
|
||||
attachmentUploadUserId: this.resolveWebhookAttachmentUploadUserId(webhook, data.attachments),
|
||||
attachmentUploadUserId: await this.resolveWebhookAttachmentUploadUserId(webhook, data.attachments),
|
||||
allowEmbeds: true,
|
||||
});
|
||||
await this.deps.dispatchService.dispatchMessageUpdate({channel, message: updatedMessage, requestCache});
|
||||
|
||||
@@ -46,6 +46,18 @@ describe('Channel Operation Permissions', () => {
|
||||
.expect(HTTP_STATUS.FORBIDDEN)
|
||||
.execute();
|
||||
});
|
||||
it('should let a minor manage a mature channel without reading it', async () => {
|
||||
const owner = await createTestAccount(harness, {dateOfBirth: '2010-01-01'});
|
||||
const guild = await createGuild(harness, owner.token, 'Mature Channel Guild');
|
||||
const systemChannel = await getChannel(harness, owner.token, guild.system_channel_id!);
|
||||
await updateChannel(harness, owner.token, systemChannel.id, {nsfw: true});
|
||||
const renamed = await updateChannel(harness, owner.token, systemChannel.id, {name: 'still-manageable'});
|
||||
expect(renamed.name).toBe('still-manageable');
|
||||
await createBuilder(harness, owner.token)
|
||||
.get(`/channels/${systemChannel.id}/messages`)
|
||||
.expect(HTTP_STATUS.FORBIDDEN)
|
||||
.execute();
|
||||
});
|
||||
it('should reject member from updating channel without MANAGE_CHANNELS', async () => {
|
||||
const owner = await createTestAccount(harness);
|
||||
const member = await createTestAccount(harness);
|
||||
|
||||
@@ -401,7 +401,7 @@ export class GuildDiscoveryService extends IGuildDiscoveryService {
|
||||
const language =
|
||||
params.primaryLanguage && isValidDiscoveryLanguage(params.primaryLanguage) ? params.primaryLanguage : undefined;
|
||||
const tag = params.tag && params.tag.trim().length > 0 ? normalizeDiscoveryTag(params.tag) : undefined;
|
||||
const sortBy = params.sortBy === 'member_count' ? 'memberCount' : 'relevance';
|
||||
const sortBy = params.sortBy === 'relevance' ? 'relevance' : 'memberCount';
|
||||
const filters: GuildSearchFilters = {
|
||||
isDiscoverable: true,
|
||||
discoveryCategory: params.categoryId,
|
||||
@@ -444,7 +444,6 @@ export class GuildDiscoveryService extends IGuildDiscoveryService {
|
||||
for (const guild of guilds) {
|
||||
const counts = freshCounts.get(BigInt(guild.id) as GuildID);
|
||||
if (counts) {
|
||||
guild.member_count = counts.memberCount;
|
||||
guild.online_count = counts.onlineCount;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -19,7 +19,7 @@ import type {UserCacheService} from '../../infrastructure/UserCacheService';
|
||||
import {Logger} from '../../Logger';
|
||||
import type {RequestCache} from '../../middleware/RequestCacheMiddleware';
|
||||
import type {GuildBan} from '../../models/GuildBan';
|
||||
import {hasHighCgnatBlastRadiusRisk, isSingleIpBanCandidate} from '../../risk/IpBanCgnatGuard';
|
||||
import {getIpBanBlastRadiusVerdict, isSingleIpBanCandidate} from '../../risk/IpBanCgnatGuard';
|
||||
import {isIpBanExempt} from '../../risk/IpBanExemptions';
|
||||
import type {IUserRepository} from '../../user/IUserRepository';
|
||||
import type {WorkerTaskName} from '../../worker/WorkerLaneConfig';
|
||||
@@ -237,19 +237,20 @@ export class GuildModerationService {
|
||||
return true;
|
||||
}
|
||||
try {
|
||||
const highRisk = await hasHighCgnatBlastRadiusRisk(userIp, this.ipInfoService, {
|
||||
const {cgnat, sharedAccess} = await getIpBanBlastRadiusVerdict(userIp, this.ipInfoService, {
|
||||
source: 'guild.ip_ban',
|
||||
reason: 'join_cgnat_guard',
|
||||
});
|
||||
const highRisk = cgnat || sharedAccess;
|
||||
if (highRisk) {
|
||||
Logger.warn(
|
||||
{userIp, bannedIp},
|
||||
'Skipping guild IP ban match because IPInfo indicates high CGNAT blast-radius risk',
|
||||
'Skipping guild IP ban match because IPInfo indicates high shared-network blast-radius risk',
|
||||
);
|
||||
}
|
||||
return !highRisk;
|
||||
} catch (error) {
|
||||
Logger.warn({error, userIp, bannedIp}, 'IPInfo CGNAT guard failed while checking guild IP ban');
|
||||
Logger.warn({error, userIp, bannedIp}, 'IPInfo blast-radius guard failed while checking guild IP ban');
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -68,23 +68,10 @@ export class GuildSearchService {
|
||||
const includeNsfwRequested = searchParams.include_nsfw ?? false;
|
||||
const canUserAccessNsfw =
|
||||
guildIsAgeRestricted || includeNsfwRequested ? await this.getCanUserAccessNsfw(userId) : false;
|
||||
if (guildIsAgeRestricted) {
|
||||
if (!canUserAccessNsfw) {
|
||||
throw new NsfwContentRequiresAgeVerificationError();
|
||||
}
|
||||
if (!includeNsfwRequested) {
|
||||
const hitsPerPage = searchParams.hits_per_page ?? 25;
|
||||
const page = searchParams.page ?? 1;
|
||||
return {
|
||||
channels: [],
|
||||
messages: [],
|
||||
total: 0,
|
||||
hits_per_page: hitsPerPage,
|
||||
page,
|
||||
};
|
||||
}
|
||||
if (guildIsAgeRestricted && !canUserAccessNsfw) {
|
||||
throw new NsfwContentRequiresAgeVerificationError();
|
||||
}
|
||||
const canIncludeNsfw = includeNsfwRequested && canUserAccessNsfw;
|
||||
const canIncludeNsfw = canUserAccessNsfw && (includeNsfwRequested || guildIsAgeRestricted);
|
||||
const guildNsfw = guildData?.nsfw ?? false;
|
||||
const channels = await this.channelRepository.listChannels(channelIds);
|
||||
const channelMap = new Map<string, Channel>();
|
||||
|
||||
@@ -14,7 +14,7 @@ import type {GuildID, RoleID, UserID} from '../../../BrandedTypes';
|
||||
import {guildIdToRoleId} from '../../../BrandedTypes';
|
||||
import {Logger} from '../../../Logger';
|
||||
import type {GuildMember} from '../../../models/GuildMember';
|
||||
import {hasHighCgnatBlastRadiusRisk, isSingleIpBanCandidate} from '../../../risk/IpBanCgnatGuard';
|
||||
import {getIpBanBlastRadiusVerdict, isSingleIpBanCandidate} from '../../../risk/IpBanCgnatGuard';
|
||||
import {isIpBanExempt} from '../../../risk/IpBanExemptions';
|
||||
import type {IUserRepository} from '../../../user/IUserRepository';
|
||||
import type {IGuildRepositoryAggregate} from '../../repositories/IGuildRepositoryAggregate';
|
||||
@@ -119,14 +119,15 @@ export class GuildMemberValidationService {
|
||||
return true;
|
||||
}
|
||||
try {
|
||||
const highRisk = await hasHighCgnatBlastRadiusRisk(userIp, this.ipInfoService, {
|
||||
const {cgnat, sharedAccess} = await getIpBanBlastRadiusVerdict(userIp, this.ipInfoService, {
|
||||
source: 'guild.member_ip_ban',
|
||||
reason: 'join_cgnat_guard',
|
||||
});
|
||||
const highRisk = cgnat || sharedAccess;
|
||||
if (highRisk) {
|
||||
Logger.warn(
|
||||
{userIp, bannedIp},
|
||||
'Skipping guild member IP ban match because IPInfo indicates high CGNAT blast-radius risk',
|
||||
'Skipping guild member IP ban match because IPInfo indicates high shared-network blast-radius risk',
|
||||
);
|
||||
}
|
||||
return !highRisk;
|
||||
|
||||
@@ -8,13 +8,21 @@ import type {
|
||||
DiscoveryCategoryResponse,
|
||||
DiscoveryGuildListResponse,
|
||||
} from '@fluxer/schema/src/domains/guild/GuildDiscoverySchemas';
|
||||
import type {WorkerTaskHelpers} from '@pkgs/worker/src/contracts/WorkerTask';
|
||||
import {afterEach, beforeEach, describe, expect, test} from 'vitest';
|
||||
import {createTestAccount, setUserACLs} from '../../auth/tests/AuthTestUtils';
|
||||
import type {GuildID} from '../../BrandedTypes';
|
||||
import {createTestBotAccount} from '../../bot/tests/BotTestUtils';
|
||||
import {setInjectedGatewayService} from '../../middleware/ServiceRegistry';
|
||||
import {getGuildRepository} from '../../middleware/ServiceSingletons';
|
||||
import {banUser} from '../../moderation/tests/ModerationTestUtils';
|
||||
import {type ApiTestHarness, createApiTestHarness} from '../../test/ApiTestHarness';
|
||||
import {NoopLogger} from '../../test/mocks/NoopLogger';
|
||||
import {NoopGatewayService} from '../../test/NoopGatewayService';
|
||||
import {HTTP_STATUS, TEST_IDS} from '../../test/TestConstants';
|
||||
import {createBuilder, createBuilderWithoutAuth} from '../../test/TestRequestBuilder';
|
||||
import syncDiscoveryIndex from '../../worker/tasks/SyncDiscoveryIndex';
|
||||
import {clearWorkerDependencies, setWorkerDependenciesForTest} from '../../worker/WorkerContext';
|
||||
import {createGuild, getUserGuilds} from './GuildTestUtils';
|
||||
|
||||
async function setGuildMemberCount(harness: ApiTestHarness, guildId: string, memberCount: number): Promise<void> {
|
||||
@@ -24,6 +32,30 @@ async function setGuildMemberCount(harness: ApiTestHarness, guildId: string, mem
|
||||
.execute();
|
||||
}
|
||||
|
||||
interface LiveGuildCounts {
|
||||
memberCount: number;
|
||||
onlineCount: number;
|
||||
}
|
||||
|
||||
const WORKER_HELPERS = {logger: new NoopLogger()} as unknown as WorkerTaskHelpers;
|
||||
|
||||
class LiveCountsGatewayService extends NoopGatewayService {
|
||||
constructor(private readonly liveCounts: Map<string, LiveGuildCounts>) {
|
||||
super();
|
||||
}
|
||||
|
||||
override async getDiscoveryGuildCounts(guildIds: Array<GuildID>): Promise<Map<GuildID, LiveGuildCounts>> {
|
||||
const counts = new Map<GuildID, LiveGuildCounts>();
|
||||
for (const guildId of guildIds) {
|
||||
const live = this.liveCounts.get(guildId.toString());
|
||||
if (live) {
|
||||
counts.set(guildId, live);
|
||||
}
|
||||
}
|
||||
return counts;
|
||||
}
|
||||
}
|
||||
|
||||
async function applyAndApprove(
|
||||
harness: ApiTestHarness,
|
||||
ownerToken: string,
|
||||
@@ -44,12 +76,39 @@ async function applyAndApprove(
|
||||
.execute();
|
||||
}
|
||||
|
||||
async function createApprovedDiscoveryGuild(
|
||||
harness: ApiTestHarness,
|
||||
adminToken: string,
|
||||
name: string,
|
||||
memberCount: number,
|
||||
): Promise<string> {
|
||||
const owner = await createTestAccount(harness);
|
||||
const guild = await createGuild(harness, owner.token, name);
|
||||
await setGuildMemberCount(harness, guild.id, memberCount);
|
||||
await applyAndApprove(
|
||||
harness,
|
||||
owner.token,
|
||||
adminToken,
|
||||
guild.id,
|
||||
`${name} welcomes everyone`,
|
||||
DiscoveryCategories.GAMING,
|
||||
);
|
||||
return guild.id;
|
||||
}
|
||||
|
||||
function expectNonIncreasing(counts: Array<number>): void {
|
||||
for (let index = 1; index < counts.length; index++) {
|
||||
expect(counts[index]).toBeLessThanOrEqual(counts[index - 1]);
|
||||
}
|
||||
}
|
||||
|
||||
describe('Discovery Search and Join', () => {
|
||||
let harness: ApiTestHarness;
|
||||
beforeEach(async () => {
|
||||
harness = await createApiTestHarness({search: 'enabled'});
|
||||
});
|
||||
afterEach(async () => {
|
||||
clearWorkerDependencies();
|
||||
await harness?.shutdown();
|
||||
});
|
||||
describe('categories', () => {
|
||||
@@ -257,6 +316,82 @@ describe('Discovery Search and Join', () => {
|
||||
.execute();
|
||||
expect(results.guilds.length).toBeLessThanOrEqual(2);
|
||||
});
|
||||
test('should order results by the member count it reports back', async () => {
|
||||
const liveCounts = new Map<string, LiveGuildCounts>();
|
||||
setInjectedGatewayService(new LiveCountsGatewayService(liveCounts));
|
||||
const admin = await createTestAccount(harness);
|
||||
await setUserACLs(harness, admin, ['admin:authenticate', 'discovery:review']);
|
||||
const guildIds: Array<string> = [];
|
||||
for (const memberCount of [50, 40, 30, 20, 10]) {
|
||||
guildIds.push(
|
||||
await createApprovedDiscoveryGuild(harness, admin.token, `Ordered Guild ${memberCount}`, memberCount),
|
||||
);
|
||||
}
|
||||
liveCounts.set(guildIds[0], {memberCount: 5, onlineCount: 3});
|
||||
liveCounts.set(guildIds[4], {memberCount: 500, onlineCount: 7});
|
||||
const searcher = await createTestAccount(harness);
|
||||
const results = await createBuilder<DiscoveryGuildListResponse>(harness, searcher.token)
|
||||
.get('/discovery/guilds?sort_by=member_count&limit=48')
|
||||
.expect(HTTP_STATUS.OK)
|
||||
.execute();
|
||||
expect(results.guilds.map((guild) => guild.id)).toEqual(guildIds);
|
||||
expectNonIncreasing(results.guilds.map((guild) => guild.member_count));
|
||||
expect(results.guilds[0].online_count).toBe(3);
|
||||
expect(results.guilds[4].online_count).toBe(7);
|
||||
});
|
||||
test('should rank by member count when the client omits sort_by', async () => {
|
||||
const admin = await createTestAccount(harness);
|
||||
await setUserACLs(harness, admin, ['admin:authenticate', 'discovery:review']);
|
||||
const guildsByCount = new Map<number, string>();
|
||||
for (const memberCount of [30, 10, 20]) {
|
||||
guildsByCount.set(
|
||||
memberCount,
|
||||
await createApprovedDiscoveryGuild(harness, admin.token, `Unsorted Guild ${memberCount}`, memberCount),
|
||||
);
|
||||
}
|
||||
const searcher = await createTestAccount(harness);
|
||||
const results = await createBuilder<DiscoveryGuildListResponse>(harness, searcher.token)
|
||||
.get('/discovery/guilds?limit=48')
|
||||
.expect(HTTP_STATUS.OK)
|
||||
.execute();
|
||||
expect(results.guilds.map((guild) => guild.id)).toEqual([
|
||||
guildsByCount.get(30),
|
||||
guildsByCount.get(20),
|
||||
guildsByCount.get(10),
|
||||
]);
|
||||
expectNonIncreasing(results.guilds.map((guild) => guild.member_count));
|
||||
});
|
||||
test('should not repeat guilds across pages when the discovery index is resynced', async () => {
|
||||
const liveCounts = new Map<string, LiveGuildCounts>();
|
||||
const gatewayService = new LiveCountsGatewayService(liveCounts);
|
||||
setInjectedGatewayService(gatewayService);
|
||||
const admin = await createTestAccount(harness);
|
||||
await setUserACLs(harness, admin, ['admin:authenticate', 'discovery:review']);
|
||||
const guildIds: Array<string> = [];
|
||||
for (const [index, memberCount] of [60, 50, 40, 40, 30, 30].entries()) {
|
||||
guildIds.push(await createApprovedDiscoveryGuild(harness, admin.token, `Paged Guild ${index}`, memberCount));
|
||||
}
|
||||
const searcher = await createTestAccount(harness);
|
||||
const firstPage = await createBuilder<DiscoveryGuildListResponse>(harness, searcher.token)
|
||||
.get('/discovery/guilds?sort_by=member_count&limit=2&offset=0')
|
||||
.expect(HTTP_STATUS.OK)
|
||||
.execute();
|
||||
expect(firstPage.guilds.map((guild) => guild.id)).toEqual([guildIds[0], guildIds[1]]);
|
||||
liveCounts.set(guildIds[0], {memberCount: 5, onlineCount: 0});
|
||||
setWorkerDependenciesForTest({guildRepository: getGuildRepository(), gatewayService});
|
||||
await syncDiscoveryIndex({}, WORKER_HELPERS);
|
||||
const secondPage = await createBuilder<DiscoveryGuildListResponse>(harness, searcher.token)
|
||||
.get('/discovery/guilds?sort_by=member_count&limit=2&offset=2')
|
||||
.expect(HTTP_STATUS.OK)
|
||||
.execute();
|
||||
const thirdPage = await createBuilder<DiscoveryGuildListResponse>(harness, searcher.token)
|
||||
.get('/discovery/guilds?sort_by=member_count&limit=2&offset=4')
|
||||
.expect(HTTP_STATUS.OK)
|
||||
.execute();
|
||||
const paged = [...firstPage.guilds, ...secondPage.guilds, ...thirdPage.guilds].map((guild) => guild.id);
|
||||
expect(new Set(paged).size).toBe(paged.length);
|
||||
expect([...paged].sort()).toEqual([...guildIds].sort());
|
||||
});
|
||||
test('should require login to search', async () => {
|
||||
await createBuilderWithoutAuth(harness).get('/discovery/guilds').expect(HTTP_STATUS.UNAUTHORIZED).execute();
|
||||
});
|
||||
|
||||
@@ -108,7 +108,7 @@ export class AvatarService {
|
||||
type: 'base64',
|
||||
base64: base64Data,
|
||||
version: 2,
|
||||
nsfw: 'block',
|
||||
nsfw: 'allow',
|
||||
}),
|
||||
kind,
|
||||
errorPath,
|
||||
@@ -163,7 +163,7 @@ export class AvatarService {
|
||||
type: 'base64',
|
||||
base64: base64Data,
|
||||
version: 2,
|
||||
nsfw: 'block',
|
||||
nsfw: 'allow',
|
||||
}),
|
||||
kind: 'avatar',
|
||||
errorPath,
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
import type {TrackSource} from 'livekit-server-sdk';
|
||||
import type {ChannelID, GuildID, UserID} from '../BrandedTypes';
|
||||
import type {VoiceRegionMetadata, VoiceServerRecord} from '../voice/VoiceModel';
|
||||
import type {ILiveKitService, ListActiveRoomsResult, ListParticipantsResult} from './ILiveKitService';
|
||||
@@ -52,27 +51,6 @@ interface UpdateParticipantPermissionsParams {
|
||||
deaf?: boolean;
|
||||
}
|
||||
|
||||
interface MuteParticipantTrackParams {
|
||||
userId: UserID;
|
||||
guildId?: GuildID;
|
||||
channelId: ChannelID;
|
||||
connectionId: string;
|
||||
regionId: string;
|
||||
serverId: string;
|
||||
trackSid: string;
|
||||
muted: boolean;
|
||||
}
|
||||
|
||||
interface RevokeParticipantPublishSourceParams {
|
||||
userId: UserID;
|
||||
guildId?: GuildID;
|
||||
channelId: ChannelID;
|
||||
connectionId: string;
|
||||
regionId: string;
|
||||
serverId: string;
|
||||
source: TrackSource;
|
||||
}
|
||||
|
||||
export class DisabledLiveKitService implements ILiveKitService {
|
||||
async createToken(_params: CreateTokenParams): Promise<{
|
||||
token: string;
|
||||
@@ -87,14 +65,6 @@ export class DisabledLiveKitService implements ILiveKitService {
|
||||
|
||||
async disconnectParticipant(_params: DisconnectParticipantParams): Promise<void> {}
|
||||
|
||||
async muteParticipantTrack(_params: MuteParticipantTrackParams): Promise<boolean> {
|
||||
return false;
|
||||
}
|
||||
|
||||
async revokeParticipantPublishSource(_params: RevokeParticipantPublishSourceParams): Promise<boolean> {
|
||||
return false;
|
||||
}
|
||||
|
||||
async listParticipants(_params: {
|
||||
guildId?: GuildID;
|
||||
channelId: ChannelID;
|
||||
|
||||
@@ -403,7 +403,7 @@ export class EntityAssetService {
|
||||
type: 'base64',
|
||||
base64: base64Data,
|
||||
version: 2,
|
||||
nsfw: 'block',
|
||||
nsfw: 'allow',
|
||||
}),
|
||||
kind,
|
||||
errorPath,
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
import type {TrackSource} from 'livekit-server-sdk';
|
||||
import type {ChannelID, GuildID, UserID} from '../BrandedTypes';
|
||||
import type {VoiceRegionMetadata, VoiceServerRecord} from '../voice/VoiceModel';
|
||||
|
||||
@@ -55,27 +54,6 @@ interface DisconnectParticipantParams {
|
||||
serverId: string;
|
||||
}
|
||||
|
||||
interface MuteParticipantTrackParams {
|
||||
userId: UserID;
|
||||
guildId?: GuildID;
|
||||
channelId: ChannelID;
|
||||
connectionId: string;
|
||||
regionId: string;
|
||||
serverId: string;
|
||||
trackSid: string;
|
||||
muted: boolean;
|
||||
}
|
||||
|
||||
interface RevokeParticipantPublishSourceParams {
|
||||
userId: UserID;
|
||||
guildId?: GuildID;
|
||||
channelId: ChannelID;
|
||||
connectionId: string;
|
||||
regionId: string;
|
||||
serverId: string;
|
||||
source: TrackSource;
|
||||
}
|
||||
|
||||
interface ListParticipantsParams {
|
||||
guildId?: GuildID;
|
||||
channelId: ChannelID;
|
||||
@@ -131,10 +109,6 @@ export abstract class ILiveKitService {
|
||||
|
||||
abstract disconnectParticipant(params: DisconnectParticipantParams): Promise<void>;
|
||||
|
||||
abstract muteParticipantTrack(params: MuteParticipantTrackParams): Promise<boolean>;
|
||||
|
||||
abstract revokeParticipantPublishSource(params: RevokeParticipantPublishSourceParams): Promise<boolean>;
|
||||
|
||||
abstract listParticipants(params: ListParticipantsParams): Promise<ListParticipantsResult>;
|
||||
|
||||
abstract listActiveRooms(): Promise<ListActiveRoomsResult>;
|
||||
|
||||
@@ -60,27 +60,6 @@ interface UpdateParticipantPermissionsParams {
|
||||
deaf?: boolean;
|
||||
}
|
||||
|
||||
interface MuteParticipantTrackParams {
|
||||
userId: UserID;
|
||||
guildId?: GuildID;
|
||||
channelId: ChannelID;
|
||||
connectionId: string;
|
||||
regionId: string;
|
||||
serverId: string;
|
||||
trackSid: string;
|
||||
muted: boolean;
|
||||
}
|
||||
|
||||
interface RevokeParticipantPublishSourceParams {
|
||||
userId: UserID;
|
||||
guildId?: GuildID;
|
||||
channelId: ChannelID;
|
||||
connectionId: string;
|
||||
regionId: string;
|
||||
serverId: string;
|
||||
source: TrackSource;
|
||||
}
|
||||
|
||||
interface ServerClientConfig {
|
||||
endpoint: string;
|
||||
apiKey: string;
|
||||
@@ -97,26 +76,6 @@ interface LiveKitPublishPermissions {
|
||||
|
||||
export const VOICE_TOKEN_TTL_SECONDS = 60 * 10;
|
||||
|
||||
const ALL_PUBLISH_SOURCES: ReadonlyArray<TrackSource> = [
|
||||
TrackSource.MICROPHONE,
|
||||
TrackSource.CAMERA,
|
||||
TrackSource.SCREEN_SHARE,
|
||||
TrackSource.SCREEN_SHARE_AUDIO,
|
||||
];
|
||||
|
||||
interface LiveKitPublishGrant {
|
||||
canPublish: boolean;
|
||||
canPublishSources: Array<TrackSource>;
|
||||
}
|
||||
|
||||
export function computeRevokedPublishGrant(current: LiveKitPublishGrant, source: TrackSource): LiveKitPublishGrant {
|
||||
const allowed = current.canPublishSources.length > 0 ? current.canPublishSources : ALL_PUBLISH_SOURCES;
|
||||
const revoked =
|
||||
source === TrackSource.SCREEN_SHARE ? [TrackSource.SCREEN_SHARE, TrackSource.SCREEN_SHARE_AUDIO] : [source];
|
||||
const canPublishSources = allowed.filter((allowedSource) => !revoked.includes(allowedSource));
|
||||
return {canPublish: current.canPublish && canPublishSources.length > 0, canPublishSources};
|
||||
}
|
||||
|
||||
export function computeLiveKitPublishSources(permissions: LiveKitPublishPermissions): Array<TrackSource> {
|
||||
const sources: Array<TrackSource> = [];
|
||||
if (permissions.canSpeak) {
|
||||
@@ -341,72 +300,6 @@ export class LiveKitService extends ILiveKitService {
|
||||
}
|
||||
}
|
||||
|
||||
async muteParticipantTrack(params: MuteParticipantTrackParams): Promise<boolean> {
|
||||
const {userId, guildId, channelId, connectionId, regionId, serverId, trackSid, muted} = params;
|
||||
const roomName = this.getRoomName(guildId, channelId);
|
||||
const participantIdentity = this.getParticipantIdentity(userId, connectionId);
|
||||
const server = this.tryResolveServerClient(regionId, serverId);
|
||||
if (server === null) {
|
||||
Logger.debug(
|
||||
{regionId, serverId, participantIdentity, roomName, trackSid},
|
||||
'LiveKit track mute skipped, pinned server no longer exists in topology',
|
||||
);
|
||||
return false;
|
||||
}
|
||||
try {
|
||||
await server.roomServiceClient.mutePublishedTrack(roomName, participantIdentity, trackSid, muted);
|
||||
return true;
|
||||
} catch (error) {
|
||||
if (LiveKitService.isHttp404(error)) {
|
||||
Logger.debug({participantIdentity, roomName, trackSid}, 'LiveKit track no longer published, nothing to mute');
|
||||
return false;
|
||||
}
|
||||
Logger.error({error, participantIdentity, roomName, trackSid, muted}, 'Error muting LiveKit published track');
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
async revokeParticipantPublishSource(params: RevokeParticipantPublishSourceParams): Promise<boolean> {
|
||||
const {userId, guildId, channelId, connectionId, regionId, serverId, source} = params;
|
||||
const roomName = this.getRoomName(guildId, channelId);
|
||||
const participantIdentity = this.getParticipantIdentity(userId, connectionId);
|
||||
const server = this.tryResolveServerClient(regionId, serverId);
|
||||
if (server === null) {
|
||||
Logger.debug(
|
||||
{regionId, serverId, participantIdentity, roomName, source},
|
||||
'LiveKit publish source revoke skipped, pinned server no longer exists in topology',
|
||||
);
|
||||
return false;
|
||||
}
|
||||
try {
|
||||
const participants = await server.roomServiceClient.listParticipants(roomName);
|
||||
const participant = participants.find((p) => p.identity === participantIdentity);
|
||||
if (!participant?.permission) {
|
||||
Logger.debug(
|
||||
{participantIdentity, roomName, source},
|
||||
'LiveKit participant no longer in room, nothing to revoke',
|
||||
);
|
||||
return false;
|
||||
}
|
||||
const grant = computeRevokedPublishGrant(participant.permission, source);
|
||||
await server.roomServiceClient.updateParticipant(roomName, participantIdentity, undefined, {
|
||||
...participant.permission,
|
||||
...grant,
|
||||
});
|
||||
return true;
|
||||
} catch (error) {
|
||||
if (LiveKitService.isHttp404(error)) {
|
||||
Logger.debug(
|
||||
{participantIdentity, roomName, source},
|
||||
'LiveKit participant no longer in room, nothing to revoke',
|
||||
);
|
||||
return false;
|
||||
}
|
||||
Logger.error({error, participantIdentity, roomName, source}, 'Error revoking LiveKit publish source');
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
async listParticipants(params: {
|
||||
guildId?: GuildID;
|
||||
channelId: ChannelID;
|
||||
@@ -431,9 +324,6 @@ export class LiveKitService extends ILiveKitService {
|
||||
participants: participants.map((participant) => ({identity: participant.identity})),
|
||||
};
|
||||
} catch (error) {
|
||||
if (LiveKitService.isHttp404(error)) {
|
||||
return {status: 'ok', participants: []};
|
||||
}
|
||||
Logger.warn({error, regionId, serverId, roomName}, 'LiveKit listParticipants failed');
|
||||
const status = LiveKitService.getHttpStatus(error);
|
||||
const isRetryable = status != null && status >= 500;
|
||||
|
||||
@@ -1,19 +1,14 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
import type {WebhookEvent} from 'livekit-server-sdk';
|
||||
import {TrackSource, WebhookReceiver} from 'livekit-server-sdk';
|
||||
import {WebhookReceiver} from 'livekit-server-sdk';
|
||||
import type {ChannelID, GuildID} from '../BrandedTypes';
|
||||
import {Config} from '../Config';
|
||||
import {Logger} from '../Logger';
|
||||
import type {LimitConfigService} from '../limits/LimitConfigService';
|
||||
import {resolveLimitSafe} from '../limits/LimitConfigUtils';
|
||||
import {createLimitMatchContext} from '../limits/LimitMatchContextBuilder';
|
||||
import type {IUserRepository} from '../user/IUserRepository';
|
||||
import type {VoiceTopology} from '../voice/VoiceTopology';
|
||||
import type {IGatewayService} from './IGatewayService';
|
||||
import type {ILiveKitService} from './ILiveKitService';
|
||||
import type {IVoiceRoomStore} from './IVoiceRoomStore';
|
||||
import {isDMRoom, parseParticipantIdentity, parseParticipantMetadataWithRaw, parseRoomName} from './VoiceRoomContext';
|
||||
import {isDMRoom, parseParticipantMetadataWithRaw, parseRoomName} from './VoiceRoomContext';
|
||||
|
||||
interface VoiceWebhookParticipantContext {
|
||||
readonly type: 'dm' | 'guild';
|
||||
@@ -34,10 +29,8 @@ export class LiveKitWebhookService {
|
||||
constructor(
|
||||
private voiceRoomStore: IVoiceRoomStore,
|
||||
private gatewayService: IGatewayService,
|
||||
private userRepository: IUserRepository,
|
||||
private liveKitService: ILiveKitService,
|
||||
private voiceTopology: VoiceTopology,
|
||||
private limitConfigService: LimitConfigService,
|
||||
) {
|
||||
this.receivers = new Map();
|
||||
this.serverMap = new Map();
|
||||
@@ -466,176 +459,6 @@ export class LiveKitWebhookService {
|
||||
}
|
||||
}
|
||||
|
||||
async handleTrackPublished(event: WebhookEvent, apiKey: string): Promise<void> {
|
||||
if (event.event !== 'track_published') {
|
||||
return;
|
||||
}
|
||||
const {room, participant, track} = event;
|
||||
if (!room || !participant || !track) {
|
||||
Logger.debug('Track published without required data, skipping');
|
||||
return;
|
||||
}
|
||||
Logger.debug(
|
||||
{
|
||||
apiKey,
|
||||
roomName: room.name,
|
||||
participantIdentity: participant.identity,
|
||||
trackType: track.type,
|
||||
width: track.width,
|
||||
height: track.height,
|
||||
},
|
||||
'Processing LiveKit track_published event',
|
||||
);
|
||||
if (track.type !== 1) {
|
||||
return;
|
||||
}
|
||||
if (track.source !== TrackSource.CAMERA && track.source !== TrackSource.SCREEN_SHARE) {
|
||||
return;
|
||||
}
|
||||
const trackSourceLabel = track.source === TrackSource.SCREEN_SHARE ? 'screen_share' : 'camera';
|
||||
try {
|
||||
const identity = parseParticipantIdentity(participant.identity);
|
||||
if (!identity) {
|
||||
Logger.warn({identity: participant.identity}, 'Unexpected participant identity format');
|
||||
return;
|
||||
}
|
||||
const {userId, connectionId} = identity;
|
||||
const user = await this.userRepository.findUnique(userId);
|
||||
if (!user) {
|
||||
Logger.warn({userId: userId.toString()}, 'User not found for track_published event');
|
||||
return;
|
||||
}
|
||||
if (Config.instance.selfHosted) {
|
||||
return;
|
||||
}
|
||||
const ctx = createLimitMatchContext({user});
|
||||
const hasHigherQuality = resolveLimitSafe(
|
||||
this.limitConfigService.getConfigSnapshot(),
|
||||
ctx,
|
||||
'feature_higher_video_quality',
|
||||
0,
|
||||
);
|
||||
const canUseHigherQuality = hasHigherQuality > 0 && !user.isBot;
|
||||
if (canUseHigherQuality) {
|
||||
return;
|
||||
}
|
||||
const FREE_MAX_WIDTH = 1280;
|
||||
const FREE_MAX_HEIGHT = 720;
|
||||
const exceedsResolution = track.width > FREE_MAX_WIDTH || track.height > FREE_MAX_HEIGHT;
|
||||
if (!exceedsResolution) {
|
||||
return;
|
||||
}
|
||||
Logger.warn(
|
||||
{
|
||||
userId: userId.toString(),
|
||||
isBot: user.isBot,
|
||||
width: track.width,
|
||||
height: track.height,
|
||||
trackSource: trackSourceLabel,
|
||||
},
|
||||
'User without higher video quality entitlement published video exceeding free tier limits - muting track and revoking source',
|
||||
);
|
||||
if (!track.sid) {
|
||||
Logger.warn(
|
||||
{userId: userId.toString(), roomName: room.name, trackSource: trackSourceLabel},
|
||||
'Track published without a sid, cannot enforce free tier video limits',
|
||||
);
|
||||
return;
|
||||
}
|
||||
const roomContext = parseRoomName(room.name);
|
||||
if (!roomContext) {
|
||||
Logger.warn({roomName: room.name}, 'Unknown room name format, cannot enforce free tier video limits');
|
||||
return;
|
||||
}
|
||||
let regionId: string | undefined;
|
||||
let serverId: string | undefined;
|
||||
if (participant.metadata) {
|
||||
const parsed = parseParticipantMetadataWithRaw(participant.metadata);
|
||||
if (parsed) {
|
||||
regionId = parsed.raw.region_id;
|
||||
serverId = parsed.raw.server_id;
|
||||
}
|
||||
}
|
||||
if (!regionId || !serverId) {
|
||||
const serverInfo = this.serverMap.get(apiKey);
|
||||
if (serverInfo) {
|
||||
regionId = serverInfo.regionId;
|
||||
serverId = serverInfo.serverId;
|
||||
}
|
||||
}
|
||||
if (!regionId || !serverId) {
|
||||
const guildId = isDMRoom(roomContext) ? undefined : roomContext.guildId;
|
||||
const pinnedServer = await this.voiceRoomStore.getPinnedRoomServer(guildId, roomContext.channelId);
|
||||
if (pinnedServer) {
|
||||
regionId = pinnedServer.regionId;
|
||||
serverId = pinnedServer.serverId;
|
||||
}
|
||||
}
|
||||
if (!regionId || !serverId) {
|
||||
Logger.warn(
|
||||
{participantId: participant.identity, roomName: room.name, apiKey},
|
||||
'Missing region or server info, cannot enforce free tier video limits',
|
||||
);
|
||||
return;
|
||||
}
|
||||
const guildId = isDMRoom(roomContext) ? undefined : roomContext.guildId;
|
||||
Logger.debug(
|
||||
{
|
||||
userId: userId.toString(),
|
||||
type: roomContext.type,
|
||||
guildId: guildId?.toString(),
|
||||
channelId: roomContext.channelId.toString(),
|
||||
regionId,
|
||||
serverId,
|
||||
isBot: user.isBot,
|
||||
width: track.width,
|
||||
height: track.height,
|
||||
trackSource: trackSourceLabel,
|
||||
},
|
||||
'Muting oversized track and revoking its publish source for user without higher video quality entitlement',
|
||||
);
|
||||
const muted = await this.liveKitService.muteParticipantTrack({
|
||||
userId,
|
||||
guildId,
|
||||
channelId: roomContext.channelId,
|
||||
connectionId,
|
||||
regionId,
|
||||
serverId,
|
||||
trackSid: track.sid,
|
||||
muted: true,
|
||||
});
|
||||
const revoked = await this.liveKitService.revokeParticipantPublishSource({
|
||||
userId,
|
||||
guildId,
|
||||
channelId: roomContext.channelId,
|
||||
connectionId,
|
||||
regionId,
|
||||
serverId,
|
||||
source: track.source,
|
||||
});
|
||||
Logger.info(
|
||||
{
|
||||
userId: userId.toString(),
|
||||
type: roomContext.type,
|
||||
guildId: guildId?.toString(),
|
||||
channelId: roomContext.channelId.toString(),
|
||||
isBot: user.isBot,
|
||||
width: track.width,
|
||||
height: track.height,
|
||||
trackSource: trackSourceLabel,
|
||||
trackSid: track.sid,
|
||||
muted,
|
||||
revoked,
|
||||
},
|
||||
muted || revoked
|
||||
? 'Enforced free tier video limits on user without higher video quality entitlement'
|
||||
: 'Failed to enforce free tier video limits on user without higher video quality entitlement',
|
||||
);
|
||||
} catch (error) {
|
||||
Logger.error({error}, 'Error processing track_published event');
|
||||
}
|
||||
}
|
||||
|
||||
async processEvent(data: {event: WebhookEvent; apiKey: string}): Promise<void> {
|
||||
const {event, apiKey} = data;
|
||||
Logger.debug({event: event.event, apiKey}, 'Dispatching LiveKit webhook event');
|
||||
@@ -650,9 +473,6 @@ export class LiveKitWebhookService {
|
||||
case 'room_finished':
|
||||
await this.handleRoomFinished(event, apiKey);
|
||||
break;
|
||||
case 'track_published':
|
||||
await this.handleTrackPublished(event, apiKey);
|
||||
break;
|
||||
default:
|
||||
Logger.debug({event: event.event}, 'Ignoring LiveKit webhook event');
|
||||
}
|
||||
|
||||
@@ -1,19 +1,9 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
import type {WebhookEvent} from 'livekit-server-sdk';
|
||||
import {AccessToken, TrackSource} from 'livekit-server-sdk';
|
||||
import {describe, expect, it, vi} from 'vitest';
|
||||
import {createUserID} from '../../BrandedTypes';
|
||||
import {getConfig} from '../../Config';
|
||||
import type {LimitConfigService} from '../../limits/LimitConfigService';
|
||||
import type {User} from '../../models/User';
|
||||
import type {IUserRepository} from '../../user/IUserRepository';
|
||||
import type {VoiceTopology} from '../../voice/VoiceTopology';
|
||||
import type {IGatewayService} from '../IGatewayService';
|
||||
import type {ILiveKitService} from '../ILiveKitService';
|
||||
import type {IVoiceRoomStore} from '../IVoiceRoomStore';
|
||||
import {computeLiveKitPublishSources, computeRevokedPublishGrant, VOICE_TOKEN_TTL_SECONDS} from '../LiveKitService';
|
||||
import {LiveKitWebhookService} from '../LiveKitWebhookService';
|
||||
import {describe, expect, it} from 'vitest';
|
||||
import {createChannelID, createGuildID} from '../../BrandedTypes';
|
||||
import {computeLiveKitPublishSources, LiveKitService, VOICE_TOKEN_TTL_SECONDS} from '../LiveKitService';
|
||||
|
||||
function decodeJwtPayload(token: string): Record<string, unknown> {
|
||||
const [, payload] = token.split('.');
|
||||
@@ -55,45 +45,6 @@ describe('LiveKitService publish permissions', () => {
|
||||
canPublishSources: ['microphone', 'screen_share', 'screen_share_audio'],
|
||||
});
|
||||
});
|
||||
it('revokes only the offending source and keeps the rest of the grant', () => {
|
||||
expect(
|
||||
computeRevokedPublishGrant(
|
||||
{
|
||||
canPublish: true,
|
||||
canPublishSources: [
|
||||
TrackSource.MICROPHONE,
|
||||
TrackSource.CAMERA,
|
||||
TrackSource.SCREEN_SHARE,
|
||||
TrackSource.SCREEN_SHARE_AUDIO,
|
||||
],
|
||||
},
|
||||
TrackSource.CAMERA,
|
||||
),
|
||||
).toEqual({
|
||||
canPublish: true,
|
||||
canPublishSources: [TrackSource.MICROPHONE, TrackSource.SCREEN_SHARE, TrackSource.SCREEN_SHARE_AUDIO],
|
||||
});
|
||||
});
|
||||
it('revokes screen share audio together with screen share video', () => {
|
||||
expect(
|
||||
computeRevokedPublishGrant(
|
||||
{
|
||||
canPublish: true,
|
||||
canPublishSources: [TrackSource.MICROPHONE, TrackSource.SCREEN_SHARE, TrackSource.SCREEN_SHARE_AUDIO],
|
||||
},
|
||||
TrackSource.SCREEN_SHARE,
|
||||
),
|
||||
).toEqual({canPublish: true, canPublishSources: [TrackSource.MICROPHONE]});
|
||||
});
|
||||
it('treats an empty source list as every source and never leaves it empty', () => {
|
||||
expect(computeRevokedPublishGrant({canPublish: true, canPublishSources: []}, TrackSource.CAMERA)).toEqual({
|
||||
canPublish: true,
|
||||
canPublishSources: [TrackSource.MICROPHONE, TrackSource.SCREEN_SHARE, TrackSource.SCREEN_SHARE_AUDIO],
|
||||
});
|
||||
expect(
|
||||
computeRevokedPublishGrant({canPublish: true, canPublishSources: [TrackSource.CAMERA]}, TrackSource.CAMERA),
|
||||
).toEqual({canPublish: false, canPublishSources: []});
|
||||
});
|
||||
it('bounds voice token lifetime to the configured TTL', async () => {
|
||||
const token = new AccessToken('test-key', 'test-secret', {
|
||||
identity: 'user_1_conn',
|
||||
@@ -108,107 +59,77 @@ describe('LiveKitService publish permissions', () => {
|
||||
});
|
||||
});
|
||||
|
||||
function createFreeUser(): User {
|
||||
return {
|
||||
id: createUserID(1n),
|
||||
isBot: false,
|
||||
premiumType: null,
|
||||
premiumUntil: null,
|
||||
premiumGiftExtensionEndsAt: null,
|
||||
premiumWillCancel: false,
|
||||
premiumGraceEndsAt: null,
|
||||
flags: 0n,
|
||||
premiumFlags: 0,
|
||||
traits: new Set<string>(),
|
||||
} as unknown as User;
|
||||
class FakeTwirpError extends Error {
|
||||
status: number;
|
||||
code?: string;
|
||||
constructor(message: string, status: number, code?: string) {
|
||||
super(message);
|
||||
this.name = 'TwirpError';
|
||||
this.status = status;
|
||||
this.code = code;
|
||||
}
|
||||
}
|
||||
|
||||
function createTrackPublishedEvent(width: number, height: number): WebhookEvent {
|
||||
return {
|
||||
event: 'track_published',
|
||||
room: {name: 'guild_2_channel_3'},
|
||||
participant: {identity: 'user_1_conn'},
|
||||
track: {type: 1, source: TrackSource.CAMERA, sid: 'TR_oversized', width, height},
|
||||
} as unknown as WebhookEvent;
|
||||
}
|
||||
|
||||
function createWebhookHarness() {
|
||||
const muteParticipantTrack = vi.fn().mockResolvedValue(true);
|
||||
const revokeParticipantPublishSource = vi.fn().mockResolvedValue(true);
|
||||
const disconnectParticipant = vi.fn().mockResolvedValue(undefined);
|
||||
const disconnectVoiceUserIfInChannel = vi.fn().mockResolvedValue(undefined);
|
||||
const service = new LiveKitWebhookService(
|
||||
{
|
||||
getPinnedRoomServer: vi.fn().mockResolvedValue({regionId: 'region-1', serverId: 'region-1-server-1'}),
|
||||
} as unknown as IVoiceRoomStore,
|
||||
{disconnectVoiceUserIfInChannel} as unknown as IGatewayService,
|
||||
{findUnique: vi.fn().mockResolvedValue(createFreeUser())} as unknown as IUserRepository,
|
||||
{muteParticipantTrack, revokeParticipantPublishSource, disconnectParticipant} as unknown as ILiveKitService,
|
||||
{
|
||||
getAllRegions: () => [],
|
||||
getServersForRegion: () => [],
|
||||
registerSubscriber: () => {},
|
||||
} as unknown as VoiceTopology,
|
||||
{getConfigSnapshot: () => null} as unknown as LimitConfigService,
|
||||
);
|
||||
return {
|
||||
function createServiceWithRoomServiceClient(roomServiceClient: unknown): LiveKitService {
|
||||
const service = Object.create(LiveKitService.prototype) as LiveKitService;
|
||||
Reflect.set(
|
||||
service,
|
||||
muteParticipantTrack,
|
||||
revokeParticipantPublishSource,
|
||||
disconnectParticipant,
|
||||
disconnectVoiceUserIfInChannel,
|
||||
};
|
||||
'serverClients',
|
||||
new Map([
|
||||
[
|
||||
'region-1',
|
||||
new Map([
|
||||
[
|
||||
'region-1-server-1',
|
||||
{
|
||||
endpoint: 'ws://livekit.test/livekit',
|
||||
apiKey: 'test-key',
|
||||
apiSecret: 'test-secret',
|
||||
isActive: true,
|
||||
roomServiceClient,
|
||||
},
|
||||
],
|
||||
]),
|
||||
],
|
||||
]),
|
||||
);
|
||||
return service;
|
||||
}
|
||||
|
||||
describe('LiveKit free tier video resolution enforcement', () => {
|
||||
it('mutes the oversized track and revokes its source instead of ending the call', async () => {
|
||||
const {
|
||||
service,
|
||||
muteParticipantTrack,
|
||||
revokeParticipantPublishSource,
|
||||
disconnectParticipant,
|
||||
disconnectVoiceUserIfInChannel,
|
||||
} = createWebhookHarness();
|
||||
describe('LiveKitService listParticipants', () => {
|
||||
const params = {
|
||||
guildId: createGuildID(1n),
|
||||
channelId: createChannelID(2n),
|
||||
regionId: 'region-1',
|
||||
serverId: 'region-1-server-1',
|
||||
};
|
||||
|
||||
await service.handleTrackPublished(createTrackPublishedEvent(1920, 1080), 'api-key');
|
||||
|
||||
expect(muteParticipantTrack).toHaveBeenCalledTimes(1);
|
||||
expect(muteParticipantTrack).toHaveBeenCalledWith(
|
||||
expect.objectContaining({trackSid: 'TR_oversized', muted: true, regionId: 'region-1'}),
|
||||
);
|
||||
expect(revokeParticipantPublishSource).toHaveBeenCalledTimes(1);
|
||||
expect(revokeParticipantPublishSource).toHaveBeenCalledWith(
|
||||
expect.objectContaining({source: TrackSource.CAMERA, connectionId: 'conn', regionId: 'region-1'}),
|
||||
);
|
||||
expect(disconnectParticipant).not.toHaveBeenCalled();
|
||||
expect(disconnectVoiceUserIfInChannel).not.toHaveBeenCalled();
|
||||
it('reports a 404 as an unreadable room instead of an empty one', async () => {
|
||||
const service = createServiceWithRoomServiceClient({
|
||||
listParticipants: async () => {
|
||||
throw new FakeTwirpError('not_found', 404, 'not_found');
|
||||
},
|
||||
});
|
||||
const result = await service.listParticipants(params);
|
||||
expect(result.status).toBe('error');
|
||||
});
|
||||
|
||||
it('leaves tracks within the free tier limits alone', async () => {
|
||||
const {service, muteParticipantTrack, revokeParticipantPublishSource, disconnectParticipant} =
|
||||
createWebhookHarness();
|
||||
|
||||
await service.handleTrackPublished(createTrackPublishedEvent(1280, 720), 'api-key');
|
||||
|
||||
expect(muteParticipantTrack).not.toHaveBeenCalled();
|
||||
expect(revokeParticipantPublishSource).not.toHaveBeenCalled();
|
||||
expect(disconnectParticipant).not.toHaveBeenCalled();
|
||||
it('reports a bad_route 404 as an unreadable room instead of an empty one', async () => {
|
||||
const service = createServiceWithRoomServiceClient({
|
||||
listParticipants: async () => {
|
||||
throw new FakeTwirpError('invalid path prefix', 404, 'bad_route');
|
||||
},
|
||||
});
|
||||
const result = await service.listParticipants(params);
|
||||
expect(result.status).toBe('error');
|
||||
expect(result.status === 'error' && result.retryable).toBe(false);
|
||||
});
|
||||
|
||||
it('does not enforce resolution limits on self-hosted instances', async () => {
|
||||
const {service, muteParticipantTrack, revokeParticipantPublishSource, disconnectParticipant} =
|
||||
createWebhookHarness();
|
||||
const config = getConfig();
|
||||
const originalSelfHosted = config.instance.selfHosted;
|
||||
config.instance.selfHosted = true;
|
||||
try {
|
||||
await service.handleTrackPublished(createTrackPublishedEvent(3840, 2160), 'api-key');
|
||||
} finally {
|
||||
config.instance.selfHosted = originalSelfHosted;
|
||||
}
|
||||
|
||||
expect(muteParticipantTrack).not.toHaveBeenCalled();
|
||||
expect(revokeParticipantPublishSource).not.toHaveBeenCalled();
|
||||
expect(disconnectParticipant).not.toHaveBeenCalled();
|
||||
it('still reports a genuinely empty room as empty', async () => {
|
||||
const service = createServiceWithRoomServiceClient({
|
||||
listParticipants: async () => [],
|
||||
});
|
||||
const result = await service.listParticipants(params);
|
||||
expect(result).toEqual({status: 'ok', participants: []});
|
||||
});
|
||||
});
|
||||
|
||||
@@ -0,0 +1,17 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
import {Config} from '../Config';
|
||||
|
||||
let cachedCollectDateOfBirth: boolean | null = null;
|
||||
|
||||
export function getDefaultDateOfBirthCollection(): boolean {
|
||||
return !Config.instance.selfHosted;
|
||||
}
|
||||
|
||||
export function instanceCollectsDateOfBirth(): boolean {
|
||||
return cachedCollectDateOfBirth ?? getDefaultDateOfBirthCollection();
|
||||
}
|
||||
|
||||
export function setCachedDateOfBirthCollection(collect: boolean): void {
|
||||
cachedCollectDateOfBirth = collect;
|
||||
}
|
||||
@@ -63,8 +63,6 @@ describe('GatewayRolloutConfigPublisher', () => {
|
||||
gateway_dispatch_relay_shards: 32,
|
||||
gateway_dispatch_relay_max_queue: 50000,
|
||||
voice_e2ee_scope: 'guild_feature_only',
|
||||
voice_reconciliation_v3_percentage: 100,
|
||||
voice_reconciliation_v3_interval_ms: 2000,
|
||||
};
|
||||
|
||||
await publisher.publish(config);
|
||||
|
||||
@@ -17,6 +17,7 @@ import {resolveDeferredPhoneGateEnabled, setCachedDeferredPhoneGateEnabled} from
|
||||
import {InstanceConfiguration} from '../Tables';
|
||||
import {DEFAULT_DECAY_CONSTANTS, DEFAULT_RENEWAL_CONSTANTS} from '../utils/AttachmentDecay';
|
||||
import {isJsonRecord, parseJsonArray, parseJsonRecord} from '../utils/JsonBoundaryUtils';
|
||||
import {getDefaultDateOfBirthCollection, setCachedDateOfBirthCollection} from './DateOfBirthCollectionCache';
|
||||
import {normalizeSsoAllowedEmailDomains} from './SsoConfigValidation';
|
||||
|
||||
const GATEWAY_ROLLOUT_CONFIG_KEY = 'gateway_rollout_config';
|
||||
@@ -41,8 +42,6 @@ const DEFAULT_GATEWAY_ROLLOUT_CONFIG: GatewayRolloutConfig = {
|
||||
gateway_dispatch_relay_shards: 32,
|
||||
gateway_dispatch_relay_max_queue: 50000,
|
||||
voice_e2ee_scope: 'guild_feature_only',
|
||||
voice_reconciliation_v3_percentage: 100,
|
||||
voice_reconciliation_v3_interval_ms: 2000,
|
||||
};
|
||||
export type InstanceRegistrationMode = 'open' | 'approval' | 'closed';
|
||||
export interface InstanceRegistrationConfig {
|
||||
@@ -357,11 +356,21 @@ function getDefaultAppPublicConfig(): InstanceAppPublicConfig {
|
||||
privacy_url: null,
|
||||
},
|
||||
registration: {
|
||||
collect_date_of_birth: !Config.instance.selfHosted,
|
||||
collect_date_of_birth: getDefaultDateOfBirthCollection(),
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function parseAppPublicConfig(raw: string): InstanceAppPublicConfig {
|
||||
try {
|
||||
const parsed: unknown = JSON.parse(raw);
|
||||
return normalizeAppPublicConfig(parsed);
|
||||
} catch (error) {
|
||||
Logger.warn({error}, 'Invalid app public config JSON, returning defaults');
|
||||
return getDefaultAppPublicConfig();
|
||||
}
|
||||
}
|
||||
|
||||
function normalizeAppPublicConfig(value: unknown): InstanceAppPublicConfig {
|
||||
const defaults = getDefaultAppPublicConfig();
|
||||
if (!isJsonRecord(value)) {
|
||||
@@ -939,6 +948,7 @@ export class InstanceConfigRepository {
|
||||
this.configCache = await this.fetchAllConfigsFromDatabase();
|
||||
} while (this.refreshRequested);
|
||||
this.syncDeferredPhoneGateCache(this.configCache.get(INSTANCE_POLICY_CONFIG_KEY) ?? null);
|
||||
this.syncDateOfBirthCollectionCache(this.configCache.get(APP_PUBLIC_CONFIG_KEY) ?? null);
|
||||
})().finally(() => {
|
||||
this.refreshPromise = null;
|
||||
});
|
||||
@@ -950,6 +960,11 @@ export class InstanceConfigRepository {
|
||||
setCachedDeferredPhoneGateEnabled(resolveDeferredPhoneGateEnabled(policy));
|
||||
}
|
||||
|
||||
private syncDateOfBirthCollectionCache(raw: string | null): void {
|
||||
const appPublic = raw ? normalizeAppPublicConfig(parseJsonRecord(raw)) : getDefaultAppPublicConfig();
|
||||
setCachedDateOfBirthCollection(appPublic.registration.collect_date_of_birth);
|
||||
}
|
||||
|
||||
private updateCachedConfigs(entries: Array<[string, string]>): void {
|
||||
if (!this.configCache) {
|
||||
return;
|
||||
@@ -1067,16 +1082,9 @@ export class InstanceConfigRepository {
|
||||
|
||||
async getAppPublicConfig(): Promise<InstanceAppPublicConfig> {
|
||||
const raw = await this.getConfig(APP_PUBLIC_CONFIG_KEY);
|
||||
if (!raw) {
|
||||
return getDefaultAppPublicConfig();
|
||||
}
|
||||
try {
|
||||
const parsed: unknown = JSON.parse(raw);
|
||||
return normalizeAppPublicConfig(parsed);
|
||||
} catch (error) {
|
||||
Logger.warn({error}, 'Invalid app public config JSON, returning defaults');
|
||||
return getDefaultAppPublicConfig();
|
||||
}
|
||||
const config = raw ? parseAppPublicConfig(raw) : getDefaultAppPublicConfig();
|
||||
setCachedDateOfBirthCollection(config.registration.collect_date_of_birth);
|
||||
return config;
|
||||
}
|
||||
|
||||
async setAppPublicConfig(config: {
|
||||
@@ -1105,6 +1113,7 @@ export class InstanceConfigRepository {
|
||||
},
|
||||
});
|
||||
await this.setConfig(APP_PUBLIC_CONFIG_KEY, JSON.stringify(next));
|
||||
setCachedDateOfBirthCollection(next.registration.collect_date_of_birth);
|
||||
return next;
|
||||
}
|
||||
|
||||
|
||||
@@ -15,6 +15,7 @@ import type {HonoEnv} from '../types/HonoEnv';
|
||||
import {parseJsonRecord} from '../utils/JsonBoundaryUtils';
|
||||
import {ipBanCache} from './IpBanMiddleware';
|
||||
import {getIpInfoService} from './ServiceMiddleware';
|
||||
import {getKVClient} from './ServiceRegistry';
|
||||
import {getCacheService} from './ServiceSingletons';
|
||||
|
||||
type IpClass = 'datacenter' | 'anonymous' | 'mobile' | 'residential' | 'unknown';
|
||||
@@ -51,6 +52,22 @@ interface AbuseSignalOptions {
|
||||
weight?: number;
|
||||
}
|
||||
|
||||
interface PeerIpClassHint {
|
||||
ipClass: IpClass;
|
||||
expiresAtMs: number;
|
||||
}
|
||||
|
||||
interface OutboundIpClass {
|
||||
lookupIp: string;
|
||||
ipClass: IpClass;
|
||||
}
|
||||
|
||||
interface ResolvedBanClass {
|
||||
ipClass: IpClass;
|
||||
authoritative: boolean;
|
||||
blocked: boolean;
|
||||
}
|
||||
|
||||
const WINDOW_MS = positiveNumberFromEnv('FLUXER_ABUSE_WINDOW_MS', 60_000);
|
||||
const THRESHOLD_DATACENTER = positiveNumberFromEnv('FLUXER_ABUSE_THRESHOLD_DATACENTER', 20);
|
||||
const THRESHOLD_ANONYMOUS = positiveNumberFromEnv('FLUXER_ABUSE_THRESHOLD_ANONYMOUS', 500);
|
||||
@@ -73,6 +90,14 @@ const REQUIRED_SCORE_WINDOWS_FOR_AUTO_BAN = positiveNumberFromEnv(
|
||||
3,
|
||||
);
|
||||
const REPLICATION_CHANNEL = 'abuse_tracker:ticks';
|
||||
const IP_CLASS_CHANNEL = 'abuse_tracker:ipclass';
|
||||
const IP_CLASS_CLAIM_PREFIX = 'abuse:ipclass:claim:';
|
||||
const IP_CLASS_CLAIM_ENABLED = process.env.FLUXER_ABUSE_IP_CLASS_CLAIM_ENABLED !== '0';
|
||||
const IP_CLASS_CLAIM_TTL_SECONDS = positiveNumberFromEnv('FLUXER_ABUSE_IP_CLASS_CLAIM_TTL_SEC', 15);
|
||||
const DEFAULT_IP_CLASS_PENDING_TTL_MS = positiveNumberFromEnv('FLUXER_ABUSE_IP_CLASS_PENDING_TTL_MS', 20_000);
|
||||
const DEFAULT_IP_CLASS_NEGATIVE_TTL_MS = positiveNumberFromEnv('FLUXER_ABUSE_IP_CLASS_NEGATIVE_TTL_MS', 300_000);
|
||||
const DEFAULT_IP_CLASS_HINT_TTL_MS = positiveNumberFromEnv('FLUXER_ABUSE_IP_CLASS_HINT_TTL_MS', 600_000);
|
||||
const IP_CLASSES = ['datacenter', 'anonymous', 'mobile', 'residential', 'unknown'] as const;
|
||||
const POD_ID = process.env.HOSTNAME ?? randomUUID();
|
||||
|
||||
type ReplicatedTick = [banKey: string, scoreDelta: number, tokenHashes: Array<string>, lookupIp: string];
|
||||
@@ -83,11 +108,23 @@ interface ReplicationMessage {
|
||||
ts: number;
|
||||
}
|
||||
|
||||
type IpClassEntry = [banKey: string, lookupIp: string, ipClass: IpClass];
|
||||
|
||||
interface IpClassMessage {
|
||||
sender: string;
|
||||
entries: Array<IpClassEntry>;
|
||||
ts: number;
|
||||
}
|
||||
|
||||
const records = new Map<string, AbuseRecord>();
|
||||
const outboundDeltas = new Map<string, OutboundEntry>();
|
||||
const persistentScoreWindows = new Map<string, PersistentScoreState>();
|
||||
const ipClassCache = new Map<string, IpClass>();
|
||||
const ipClassPending = new Set<string>();
|
||||
const ipClassPending = new Map<string, number>();
|
||||
const ipClassNegativeUntil = new Map<string, number>();
|
||||
const peerIpClassHints = new Map<string, PeerIpClassHint>();
|
||||
const outboundIpClasses = new Map<string, OutboundIpClass>();
|
||||
const pendingIpClassTasks = new Set<Promise<void>>();
|
||||
const recordedClientErrorRequests = new WeakSet<Request>();
|
||||
const pendingAutoBanTasks = new Set<Promise<void>>();
|
||||
const adminRepository = new AdminRepository();
|
||||
@@ -97,6 +134,9 @@ let flushTimer: NodeJS.Timeout | null = null;
|
||||
let kvSubscription: IKVSubscription | null = null;
|
||||
let messageHandler: ((channel: string, message: string) => void) | null = null;
|
||||
let errorHandler: ((error: Error) => void) | null = null;
|
||||
let ipClassPendingTtlMs = DEFAULT_IP_CLASS_PENDING_TTL_MS;
|
||||
let ipClassNegativeTtlMs = DEFAULT_IP_CLASS_NEGATIVE_TTL_MS;
|
||||
let ipClassHintTtlMs = DEFAULT_IP_CLASS_HINT_TTL_MS;
|
||||
|
||||
function positiveNumberFromEnv(name: string, fallback: number): number {
|
||||
const raw = process.env[name];
|
||||
@@ -164,13 +204,99 @@ function shouldSkipAutoBanForIpClass(ipClass: IpClass): boolean {
|
||||
return ipClass === 'mobile';
|
||||
}
|
||||
|
||||
function isIpClass(value: unknown): value is IpClass {
|
||||
return typeof value === 'string' && (IP_CLASSES as ReadonlyArray<string>).includes(value);
|
||||
}
|
||||
|
||||
function getOwnIpClass(key: string, now: number): IpClass | null {
|
||||
const cached = ipClassCache.get(key);
|
||||
if (cached === undefined) return null;
|
||||
const negativeUntilMs = ipClassNegativeUntil.get(key);
|
||||
if (negativeUntilMs !== undefined && negativeUntilMs <= now) {
|
||||
ipClassCache.delete(key);
|
||||
ipClassNegativeUntil.delete(key);
|
||||
return null;
|
||||
}
|
||||
return cached;
|
||||
}
|
||||
|
||||
function isOwnIpClassNegative(key: string, now: number): boolean {
|
||||
const negativeUntilMs = ipClassNegativeUntil.get(key);
|
||||
return negativeUntilMs !== undefined && negativeUntilMs > now;
|
||||
}
|
||||
|
||||
function setOwnIpClass(key: string, lookupIp: string, ipClass: IpClass, negative: boolean): void {
|
||||
ipClassCache.set(key, ipClass);
|
||||
if (negative) {
|
||||
ipClassNegativeUntil.set(key, Date.now() + ipClassNegativeTtlMs);
|
||||
} else {
|
||||
ipClassNegativeUntil.delete(key);
|
||||
peerIpClassHints.delete(key);
|
||||
}
|
||||
ipClassPending.delete(key);
|
||||
if (!negative && ipClass !== 'unknown') {
|
||||
queueOutboundIpClass(key, lookupIp, ipClass);
|
||||
}
|
||||
const rec = records.get(key);
|
||||
if (rec) maybeFireAutoBan(key, rec);
|
||||
}
|
||||
|
||||
function isIpClassPending(key: string, now: number): boolean {
|
||||
const expiresAtMs = ipClassPending.get(key);
|
||||
if (expiresAtMs === undefined) return false;
|
||||
if (expiresAtMs <= now) {
|
||||
ipClassPending.delete(key);
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
function getPeerClassHint(key: string, now: number): IpClass | null {
|
||||
const hint = peerIpClassHints.get(key);
|
||||
if (!hint) return null;
|
||||
if (hint.expiresAtMs <= now) {
|
||||
peerIpClassHints.delete(key);
|
||||
return null;
|
||||
}
|
||||
return hint.ipClass;
|
||||
}
|
||||
|
||||
function recordPeerClassHint(key: string, ipClass: IpClass): void {
|
||||
peerIpClassHints.set(key, {ipClass, expiresAtMs: Date.now() + ipClassHintTtlMs});
|
||||
}
|
||||
|
||||
function isStricterThanUnknown(ipClass: IpClass): boolean {
|
||||
return (
|
||||
scoreThresholdFor(ipClass) <= scoreThresholdFor('unknown') &&
|
||||
tokenDiversityThresholdFor(ipClass) <= tokenDiversityThresholdFor('unknown')
|
||||
);
|
||||
}
|
||||
|
||||
function resolveClassForBan(key: string, now: number): ResolvedBanClass {
|
||||
const own = getOwnIpClass(key, now);
|
||||
if (own !== null && !isOwnIpClassNegative(key, now)) return {ipClass: own, authoritative: true, blocked: false};
|
||||
const hint = getPeerClassHint(key, now);
|
||||
if (hint !== null && isStricterThanUnknown(hint)) return {ipClass: hint, authoritative: true, blocked: false};
|
||||
if (hint !== null) return {ipClass: 'unknown', authoritative: false, blocked: true};
|
||||
if (own !== null) return {ipClass: own, authoritative: true, blocked: false};
|
||||
return {ipClass: 'unknown', authoritative: false, blocked: isIpClassPending(key, now)};
|
||||
}
|
||||
|
||||
function pruneIfNeeded(now: number): void {
|
||||
if (records.size < MAX_TRACKED_IPS) return;
|
||||
for (const [key, expiresAtMs] of ipClassPending) {
|
||||
if (expiresAtMs <= now) ipClassPending.delete(key);
|
||||
}
|
||||
for (const [key, hint] of peerIpClassHints) {
|
||||
if (hint.expiresAtMs <= now) peerIpClassHints.delete(key);
|
||||
}
|
||||
for (const [key, rec] of records) {
|
||||
if (rec.windowStartMs + WINDOW_MS < now) {
|
||||
if (rec.windowStartMs + WINDOW_MS < now && !ipClassPending.has(key)) {
|
||||
records.delete(key);
|
||||
ipClassCache.delete(key);
|
||||
ipClassPending.delete(key);
|
||||
ipClassNegativeUntil.delete(key);
|
||||
peerIpClassHints.delete(key);
|
||||
outboundIpClasses.delete(key);
|
||||
}
|
||||
if (records.size < MAX_TRACKED_IPS * 0.9) return;
|
||||
}
|
||||
@@ -222,9 +348,11 @@ function queueOutboundDelta(
|
||||
}
|
||||
}
|
||||
|
||||
function shouldEnsureIpClassLookup(key: string, rec: AbuseRecord): boolean {
|
||||
if (ipClassCache.has(key) || ipClassPending.has(key)) return false;
|
||||
return rec.score >= MIN_SCORE_FOR_IP_LOOKUP || rec.distinctTokenHashes.size >= MIN_TOKENS_FOR_IP_LOOKUP;
|
||||
function shouldEnsureIpClassLookup(key: string, rec: AbuseRecord, now: number): boolean {
|
||||
if (getOwnIpClass(key, now) !== null || isIpClassPending(key, now)) return false;
|
||||
if (getPeerClassHint(key, now) !== null) return false;
|
||||
if (rec.score < MIN_SCORE_FOR_IP_LOOKUP && rec.distinctTokenHashes.size < MIN_TOKENS_FOR_IP_LOOKUP) return false;
|
||||
return ipBanCache.getMatch(rec.lookupIp) === null;
|
||||
}
|
||||
|
||||
function markScoreThresholdWindow(key: string, rec: AbuseRecord, now: number): number {
|
||||
@@ -247,33 +375,53 @@ function markScoreThresholdWindow(key: string, rec: AbuseRecord, now: number): n
|
||||
return state.count;
|
||||
}
|
||||
|
||||
async function claimIpClassLookup(key: string): Promise<boolean> {
|
||||
if (!IP_CLASS_CLAIM_ENABLED) return true;
|
||||
if (!kvPublisher) return true;
|
||||
try {
|
||||
return await getKVClient().setnx(`${IP_CLASS_CLAIM_PREFIX}${key}`, POD_ID, IP_CLASS_CLAIM_TTL_SECONDS);
|
||||
} catch {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
async function runIpClassLookup(key: string, lookupIp: string): Promise<void> {
|
||||
try {
|
||||
if (!(await claimIpClassLookup(key))) return;
|
||||
const result = await getIpInfoService().lookup(lookupIp, {source: 'AbusiveIpAutoBanner', reason: 'classify'});
|
||||
setOwnIpClass(key, lookupIp, classifyIpInfo(result), !result.available);
|
||||
} catch (err) {
|
||||
setOwnIpClass(key, lookupIp, 'unknown', true);
|
||||
Logger.warn({err, ip: lookupIp}, '[abuse-auto-ban] IP classification lookup failed');
|
||||
}
|
||||
}
|
||||
|
||||
function ensureIpClassLookup(key: string, lookupIp: string): void {
|
||||
if (ipClassCache.has(key) || ipClassPending.has(key)) return;
|
||||
ipClassPending.add(key);
|
||||
void (async () => {
|
||||
try {
|
||||
const result = await getIpInfoService().lookup(lookupIp, {source: 'AbusiveIpAutoBanner', reason: 'classify'});
|
||||
ipClassCache.set(key, classifyIpInfo(result));
|
||||
} catch (err) {
|
||||
ipClassCache.set(key, 'unknown');
|
||||
Logger.warn({err, ip: lookupIp}, '[abuse-auto-ban] IP classification lookup failed');
|
||||
} finally {
|
||||
ipClassPending.delete(key);
|
||||
const rec = records.get(key);
|
||||
if (rec) maybeFireAutoBan(key, rec);
|
||||
}
|
||||
})();
|
||||
const now = Date.now();
|
||||
if (getOwnIpClass(key, now) !== null || isIpClassPending(key, now)) return;
|
||||
ipClassPending.set(key, now + ipClassPendingTtlMs);
|
||||
const task = runIpClassLookup(key, lookupIp);
|
||||
pendingIpClassTasks.add(task);
|
||||
void task.finally(() => {
|
||||
pendingIpClassTasks.delete(task);
|
||||
});
|
||||
}
|
||||
|
||||
function maybeFireAutoBan(key: string, rec: AbuseRecord): void {
|
||||
if (rec.autoBanFired) return;
|
||||
const ipClass = ipClassCache.get(key) ?? 'unknown';
|
||||
const now = Date.now();
|
||||
if (ipBanCache.getMatch(rec.lookupIp) !== null) {
|
||||
rec.autoBanFired = true;
|
||||
return;
|
||||
}
|
||||
const resolved = resolveClassForBan(key, now);
|
||||
const ipClass = resolved.ipClass;
|
||||
const scoreThreshold = scoreThresholdFor(ipClass);
|
||||
const tokenThreshold = tokenDiversityThresholdFor(ipClass);
|
||||
const overScore = rec.score >= scoreThreshold;
|
||||
const overTokenDiversity = rec.distinctTokenHashes.size >= tokenThreshold;
|
||||
if (!overScore && !overTokenDiversity) return;
|
||||
if (!ipClassCache.has(key) && ipClassPending.has(key)) {
|
||||
if (resolved.blocked) {
|
||||
return;
|
||||
}
|
||||
if (shouldSkipAutoBanForIpClass(ipClass)) {
|
||||
@@ -369,7 +517,7 @@ export function recordAbuseSignal(ip: string | null, reason: string, opts: Abuse
|
||||
queuedTokenHash = opts.tokenHash;
|
||||
}
|
||||
queueOutboundDelta(signalIp, weight, queuedTokenHash, hadToken);
|
||||
if (shouldEnsureIpClassLookup(signalIp.banKey, rec)) {
|
||||
if (shouldEnsureIpClassLookup(signalIp.banKey, rec, now)) {
|
||||
ensureIpClassLookup(signalIp.banKey, signalIp.lookupIp);
|
||||
}
|
||||
maybeFireAutoBan(signalIp.banKey, rec);
|
||||
@@ -404,7 +552,7 @@ function applyReplicatedTick(tick: ReplicatedTick): void {
|
||||
if (rec.distinctTokenHashes.size >= MAX_TOKEN_HASHES_PER_IP) break;
|
||||
rec.distinctTokenHashes.add(tokenHash);
|
||||
}
|
||||
if (shouldEnsureIpClassLookup(banKey, rec)) {
|
||||
if (shouldEnsureIpClassLookup(banKey, rec, now)) {
|
||||
ensureIpClassLookup(banKey, lookupIp);
|
||||
}
|
||||
maybeFireAutoBan(banKey, rec);
|
||||
@@ -435,7 +583,56 @@ async function flushOutbound(): Promise<void> {
|
||||
}
|
||||
}
|
||||
|
||||
function queueOutboundIpClass(key: string, lookupIp: string, ipClass: IpClass): void {
|
||||
if (!kvPublisher) return;
|
||||
if (!outboundIpClasses.has(key) && outboundIpClasses.size >= MAX_TRACKED_IPS) return;
|
||||
outboundIpClasses.set(key, {lookupIp, ipClass});
|
||||
}
|
||||
|
||||
async function flushOutboundIpClasses(): Promise<void> {
|
||||
if (!kvPublisher || outboundIpClasses.size === 0) return;
|
||||
const entries: Array<IpClassEntry> = [];
|
||||
const selectedKeys: Array<string> = [];
|
||||
for (const [key, entry] of outboundIpClasses) {
|
||||
entries.push([key, entry.lookupIp, entry.ipClass]);
|
||||
selectedKeys.push(key);
|
||||
if (entries.length >= MAX_BATCH_TICKS) break;
|
||||
}
|
||||
const message: IpClassMessage = {sender: POD_ID, entries, ts: Date.now()};
|
||||
try {
|
||||
await kvPublisher.publish(IP_CLASS_CHANNEL, JSON.stringify(message));
|
||||
for (const key of selectedKeys) {
|
||||
outboundIpClasses.delete(key);
|
||||
}
|
||||
} catch (err) {
|
||||
Logger.warn({err, entryCount: entries.length}, '[abuse-auto-ban] Failed to publish abuse IP class batch');
|
||||
}
|
||||
}
|
||||
|
||||
function handleIpClassMessage(message: string): void {
|
||||
const msg = parseJsonRecord(message);
|
||||
if (!msg || msg.sender === POD_ID || !Array.isArray(msg.entries)) return;
|
||||
for (const rawEntry of msg.entries) {
|
||||
if (!Array.isArray(rawEntry) || rawEntry.length < 3) continue;
|
||||
const [banKey, lookupIp, ipClass] = rawEntry;
|
||||
if (typeof banKey !== 'string' || typeof lookupIp !== 'string' || !isIpClass(ipClass)) continue;
|
||||
if (ipClass === 'unknown') continue;
|
||||
const signalIp = normalizeSignalIp(lookupIp);
|
||||
if (!signalIp || signalIp.banKey !== banKey) continue;
|
||||
const rec = records.get(banKey);
|
||||
if (!rec) continue;
|
||||
const now = Date.now();
|
||||
if (getOwnIpClass(banKey, now) !== null && !isOwnIpClassNegative(banKey, now)) continue;
|
||||
recordPeerClassHint(banKey, ipClass);
|
||||
maybeFireAutoBan(banKey, rec);
|
||||
}
|
||||
}
|
||||
|
||||
function handleReplicationMessage(channel: string, message: string): void {
|
||||
if (channel === IP_CLASS_CHANNEL) {
|
||||
handleIpClassMessage(message);
|
||||
return;
|
||||
}
|
||||
if (channel !== REPLICATION_CHANNEL) return;
|
||||
const msg = parseJsonRecord(message);
|
||||
if (!msg || msg.sender === POD_ID || !Array.isArray(msg.ticks)) return;
|
||||
@@ -469,11 +666,12 @@ export async function startAbuseReplicationSubscriber(kvClient: IKVProvider | nu
|
||||
};
|
||||
try {
|
||||
await subscription.connect();
|
||||
await subscription.subscribe(REPLICATION_CHANNEL);
|
||||
await subscription.subscribe(REPLICATION_CHANNEL, IP_CLASS_CHANNEL);
|
||||
subscription.on('message', messageHandler);
|
||||
subscription.on('error', errorHandler);
|
||||
flushTimer = setInterval(() => {
|
||||
void flushOutbound();
|
||||
void flushOutboundIpClasses();
|
||||
}, BATCH_FLUSH_MS);
|
||||
if (typeof flushTimer === 'object' && flushTimer && 'unref' in flushTimer) {
|
||||
(flushTimer as {unref(): void}).unref();
|
||||
@@ -514,11 +712,28 @@ export async function drainAbuseAutoBanTasksForTests(): Promise<void> {
|
||||
await Promise.all([...pendingAutoBanTasks]);
|
||||
}
|
||||
|
||||
export async function drainAbuseIpClassLookupsForTests(): Promise<void> {
|
||||
await Promise.all([...pendingIpClassTasks]);
|
||||
}
|
||||
|
||||
export function setAbuseIpClassTtlsForTests(opts: {negativeMs?: number; hintMs?: number; pendingMs?: number}): void {
|
||||
if (opts.negativeMs !== undefined) ipClassNegativeTtlMs = opts.negativeMs;
|
||||
if (opts.hintMs !== undefined) ipClassHintTtlMs = opts.hintMs;
|
||||
if (opts.pendingMs !== undefined) ipClassPendingTtlMs = opts.pendingMs;
|
||||
}
|
||||
|
||||
export function resetAbuseTrackingForTests(): void {
|
||||
records.clear();
|
||||
outboundDeltas.clear();
|
||||
persistentScoreWindows.clear();
|
||||
ipClassCache.clear();
|
||||
ipClassPending.clear();
|
||||
ipClassNegativeUntil.clear();
|
||||
peerIpClassHints.clear();
|
||||
outboundIpClasses.clear();
|
||||
pendingIpClassTasks.clear();
|
||||
pendingAutoBanTasks.clear();
|
||||
ipClassPendingTtlMs = DEFAULT_IP_CLASS_PENDING_TTL_MS;
|
||||
ipClassNegativeTtlMs = DEFAULT_IP_CLASS_NEGATIVE_TTL_MS;
|
||||
ipClassHintTtlMs = DEFAULT_IP_CLASS_HINT_TTL_MS;
|
||||
}
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
import crypto from 'node:crypto';
|
||||
import {lookupAsnByIp, lookupGeoipByIp} from '@pkgs/geoip/src/GeoipLookup';
|
||||
import {createIpInfoService, createUnavailableIpInfoService, type IpInfoService} from '@pkgs/geoip/src/IpInfoService';
|
||||
import {createMiddleware} from 'hono/factory';
|
||||
import type {ApiContext} from '../ApiContext';
|
||||
@@ -58,8 +59,14 @@ import {createIpInfoChecker} from '../risk/adapters/IpInfoAdapter';
|
||||
import {createReverseDnsLookup} from '../risk/adapters/ReverseDnsAdapter';
|
||||
import {DeterministicRiskEngine} from '../risk/DeterministicRiskEngine';
|
||||
import {CassandraHistoricalOutcomeRepository} from '../risk/HistoricalOutcomeRepository';
|
||||
import {createKvIpInfoLookupBudget} from '../risk/IpInfoBudget';
|
||||
import {buildIpInfoCache, buildIpInfoRequestAuditLogger} from '../risk/IpInfoCacheFactory';
|
||||
import {CassandraRegistrationEventsRepository} from '../risk/RegistrationEventsRepository';
|
||||
import {
|
||||
type IpInfoPrescreenVerdict,
|
||||
ipInfoPrescreenOptionsFromEnv,
|
||||
prescreenIpInfoLookup,
|
||||
} from '../risk/RegistrationIpPrescreen';
|
||||
import {CassandraRiskAssessmentRepository} from '../risk/RiskAssessmentRepository';
|
||||
import {createRiskToolbox} from '../risk/RiskToolboxFactory';
|
||||
import {CassandraSuspiciousIpRepository} from '../risk/SuspiciousIpRepository';
|
||||
@@ -268,6 +275,7 @@ export function getIpInfoService(): IpInfoService {
|
||||
apiKey: Config.risk.ipinfoApiKey,
|
||||
cache,
|
||||
auditLogger: buildIpInfoRequestAuditLogger(),
|
||||
budget: createKvIpInfoLookupBudget({getKvClient: getKVClient}),
|
||||
});
|
||||
return _ipInfoService;
|
||||
}
|
||||
@@ -293,7 +301,14 @@ function getRegistrationRiskEvaluator(): IRegistrationRiskEvaluator {
|
||||
return _registrationRiskEvaluator;
|
||||
}
|
||||
const ipInfoService = getIpInfoService();
|
||||
const ipInfoChecker = Config.risk.ipinfoApiKey ? createIpInfoChecker({ipInfoService}) : undefined;
|
||||
const lookupLocalCity = (ip: string) => lookupGeoipByIp(ip, Config.geoip.maxmindDbPath);
|
||||
const lookupLocalAsn = (ip: string) => lookupAsnByIp(ip, Config.geoip.maxmindAsnDbPath);
|
||||
const prescreenOptions = ipInfoPrescreenOptionsFromEnv();
|
||||
const prescreen = async (ip: string): Promise<IpInfoPrescreenVerdict> => {
|
||||
const [city, asn] = await Promise.all([lookupLocalCity(ip), lookupLocalAsn(ip)]);
|
||||
return prescreenIpInfoLookup({countryIso: city.countryCode, asn: asn.asn, asnOrg: asn.asnOrg}, prescreenOptions);
|
||||
};
|
||||
const ipInfoChecker = Config.risk.ipinfoApiKey ? createIpInfoChecker({ipInfoService, prescreen}) : undefined;
|
||||
const cacheService = getCacheService();
|
||||
const reverseDnsLookup = createReverseDnsLookup({cacheService});
|
||||
const toolbox = createRiskToolbox({
|
||||
@@ -305,6 +320,8 @@ function getRegistrationRiskEvaluator(): IRegistrationRiskEvaluator {
|
||||
historicalOutcomeRepository: getHistoricalOutcomeRepository(),
|
||||
suspiciousIpRepository: getSuspiciousIpRepository(),
|
||||
cacheService,
|
||||
lookupLocalCity,
|
||||
lookupLocalAsn,
|
||||
});
|
||||
const engine = new DeterministicRiskEngine(toolbox, {
|
||||
logger: Logger,
|
||||
@@ -331,10 +348,8 @@ function getLiveKitWebhookService(): LiveKitWebhookService | null {
|
||||
_liveKitWebhookService = new LiveKitWebhookService(
|
||||
voiceRoomStore,
|
||||
getGatewayService(),
|
||||
getUserRepository(),
|
||||
liveKitService,
|
||||
voiceTopology,
|
||||
getLimitConfigService(),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,15 +1,21 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
import {getSameIpDecisionKey} from '@fluxer/ip_utils/src/IpAddress';
|
||||
import type {IpInfoLookupResult} from '@pkgs/geoip/src/IpInfoService';
|
||||
import {afterAll, beforeAll, beforeEach, describe, expect, it} from 'vitest';
|
||||
import {AdminRepository} from '../../admin/AdminRepository';
|
||||
import type {ApiTestHarness} from '../../test/ApiTestHarness';
|
||||
import {createApiTestHarness} from '../../test/ApiTestHarness';
|
||||
import type {MockKVProvider} from '../../test/mocks/MockKVProvider';
|
||||
import {
|
||||
drainAbuseAutoBanTasksForTests,
|
||||
drainAbuseIpClassLookupsForTests,
|
||||
hashAuthToken,
|
||||
recordAbuseSignal,
|
||||
resetAbuseTrackingForTests,
|
||||
setAbuseIpClassTtlsForTests,
|
||||
startAbuseReplicationSubscriber,
|
||||
stopAbuseReplicationSubscriber,
|
||||
} from '../AbusiveIpAutoBanner';
|
||||
import {ipBanCache} from '../IpBanMiddleware';
|
||||
import {setInjectedIpInfoService} from '../ServiceMiddleware';
|
||||
@@ -65,6 +71,11 @@ function ipInfoResult(ip: string, overrides: Partial<IpInfoLookupResult> = {}):
|
||||
};
|
||||
}
|
||||
|
||||
function claimCallCount(harness: ApiTestHarness, banKey: string): number {
|
||||
const kvProvider = harness.kvProvider as MockKVProvider;
|
||||
return kvProvider.setnxSpy.mock.calls.filter(([key]) => key === `abuse:ipclass:claim:${banKey}`).length;
|
||||
}
|
||||
|
||||
async function waitForAssertion(assertion: () => void): Promise<void> {
|
||||
const deadline = Date.now() + 1000;
|
||||
let lastError: unknown;
|
||||
@@ -84,6 +95,7 @@ async function waitForAssertion(assertion: () => void): Promise<void> {
|
||||
describe('AbusiveIpAutoBanner', () => {
|
||||
let harness: ApiTestHarness;
|
||||
let adminRepository: AdminRepository;
|
||||
let lookupCount = 0;
|
||||
beforeAll(async () => {
|
||||
harness = await createApiTestHarness();
|
||||
adminRepository = new AdminRepository();
|
||||
@@ -92,13 +104,18 @@ describe('AbusiveIpAutoBanner', () => {
|
||||
await harness.reset();
|
||||
resetAbuseTrackingForTests();
|
||||
ipBanCache.resetCaches();
|
||||
lookupCount = 0;
|
||||
setInjectedIpInfoService({
|
||||
async lookup(ip: string) {
|
||||
lookupCount += 1;
|
||||
return ipInfoResult(ip);
|
||||
},
|
||||
});
|
||||
await stopAbuseReplicationSubscriber();
|
||||
await startAbuseReplicationSubscriber(harness.kvProvider);
|
||||
});
|
||||
afterAll(async () => {
|
||||
await stopAbuseReplicationSubscriber();
|
||||
setInjectedIpInfoService(undefined);
|
||||
await harness.shutdown();
|
||||
});
|
||||
@@ -117,7 +134,7 @@ describe('AbusiveIpAutoBanner', () => {
|
||||
it('does not auto-ban after a single score-only spike', async () => {
|
||||
const ip = '8.8.4.4';
|
||||
recordAbuseSignal(ip, 'http_429', {weight: 150});
|
||||
await new Promise((resolve) => setTimeout(resolve, 25));
|
||||
await drainAbuseIpClassLookupsForTests();
|
||||
await drainAbuseAutoBanTasksForTests();
|
||||
expect(ipBanCache.isBanned(ip)).toBe(false);
|
||||
await expect(adminRepository.isIpBanned(ip)).resolves.toBe(false);
|
||||
@@ -135,7 +152,7 @@ describe('AbusiveIpAutoBanner', () => {
|
||||
for (let i = 0; i < 10; i += 1) {
|
||||
recordAbuseSignal(ip, 'auth_failure:session', {tokenHash: hashAuthToken(`mobile-invalid-${i}`)});
|
||||
}
|
||||
await new Promise((resolve) => setTimeout(resolve, 25));
|
||||
await drainAbuseIpClassLookupsForTests();
|
||||
await drainAbuseAutoBanTasksForTests();
|
||||
expect(ipBanCache.isBanned(ip)).toBe(false);
|
||||
await expect(adminRepository.isIpBanned(ip)).resolves.toBe(false);
|
||||
@@ -153,7 +170,7 @@ describe('AbusiveIpAutoBanner', () => {
|
||||
for (let i = 0; i < 100; i += 1) {
|
||||
recordAbuseSignal(ip, 'auth_failure:session', {tokenHash: hashAuthToken(`mobile-threshold-${i}`)});
|
||||
}
|
||||
await new Promise((resolve) => setTimeout(resolve, 25));
|
||||
await drainAbuseIpClassLookupsForTests();
|
||||
await drainAbuseAutoBanTasksForTests();
|
||||
expect(ipBanCache.isBanned(ip)).toBe(false);
|
||||
await expect(adminRepository.isIpBanned(ip)).resolves.toBe(false);
|
||||
@@ -168,4 +185,68 @@ describe('AbusiveIpAutoBanner', () => {
|
||||
await expect(adminRepository.isIpBanned(ip)).resolves.toBe(false);
|
||||
}
|
||||
});
|
||||
it('claims the class lookup exactly once for a burst on the same IP', async () => {
|
||||
const ip = '8.8.8.8';
|
||||
for (let i = 0; i < 10; i += 1) {
|
||||
recordAbuseSignal(ip, 'auth_failure:session', {tokenHash: hashAuthToken(`claim-${i}`)});
|
||||
}
|
||||
await drainAbuseIpClassLookupsForTests();
|
||||
await drainAbuseAutoBanTasksForTests();
|
||||
expect(claimCallCount(harness, ip)).toBe(1);
|
||||
expect(lookupCount).toBe(1);
|
||||
});
|
||||
it('does not pay for a lookup or ban when another pod owns the class claim', async () => {
|
||||
const ip = '8.8.8.8';
|
||||
await harness.kvProvider.setnx(`abuse:ipclass:claim:${ip}`, 'other-pod', 60);
|
||||
for (let i = 0; i < 10; i += 1) {
|
||||
recordAbuseSignal(ip, 'auth_failure:session', {tokenHash: hashAuthToken(`claim-loser-${i}`)});
|
||||
}
|
||||
await drainAbuseIpClassLookupsForTests();
|
||||
await drainAbuseAutoBanTasksForTests();
|
||||
expect(lookupCount).toBe(0);
|
||||
expect(ipBanCache.isBanned(ip)).toBe(false);
|
||||
});
|
||||
it('does not classify an IPv4 address that is already banned', async () => {
|
||||
const ip = '8.8.8.8';
|
||||
ipBanCache.banTemp(ip, 3600);
|
||||
for (let i = 0; i < 20; i += 1) {
|
||||
recordAbuseSignal(ip, 'auth_failure:session', {tokenHash: hashAuthToken(`already-banned-${i}`)});
|
||||
}
|
||||
await drainAbuseIpClassLookupsForTests();
|
||||
await drainAbuseAutoBanTasksForTests();
|
||||
expect(lookupCount).toBe(0);
|
||||
expect(claimCallCount(harness, ip)).toBe(0);
|
||||
});
|
||||
it('does not classify an IPv6 address inside an already banned /64', async () => {
|
||||
const ip = '2606:4700:4700::1111';
|
||||
const banKey = getSameIpDecisionKey(ip) ?? ip;
|
||||
ipBanCache.banTemp(banKey, 3600);
|
||||
for (let i = 0; i < 20; i += 1) {
|
||||
recordAbuseSignal(ip, 'auth_failure:session', {tokenHash: hashAuthToken(`already-banned-v6-${i}`)});
|
||||
}
|
||||
await drainAbuseIpClassLookupsForTests();
|
||||
await drainAbuseAutoBanTasksForTests();
|
||||
expect(lookupCount).toBe(0);
|
||||
expect(claimCallCount(harness, banKey)).toBe(0);
|
||||
});
|
||||
it('retries a failed classification once the negative TTL elapses', async () => {
|
||||
const ip = '9.9.9.9';
|
||||
setAbuseIpClassTtlsForTests({negativeMs: 50});
|
||||
setInjectedIpInfoService({
|
||||
async lookup(candidateIp: string) {
|
||||
lookupCount += 1;
|
||||
return ipInfoResult(candidateIp, {available: false});
|
||||
},
|
||||
});
|
||||
recordAbuseSignal(ip, 'http_429', {weight: 25});
|
||||
await drainAbuseIpClassLookupsForTests();
|
||||
await drainAbuseAutoBanTasksForTests();
|
||||
expect(lookupCount).toBe(1);
|
||||
await new Promise((resolve) => setTimeout(resolve, 60));
|
||||
await harness.kvProvider.del(`abuse:ipclass:claim:${ip}`);
|
||||
recordAbuseSignal(ip, 'http_429', {weight: 25});
|
||||
await drainAbuseIpClassLookupsForTests();
|
||||
await drainAbuseAutoBanTasksForTests();
|
||||
expect(lookupCount).toBe(2);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -0,0 +1,203 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
import type {IpInfoLookupResult} from '@pkgs/geoip/src/IpInfoService';
|
||||
import {afterAll, beforeAll, beforeEach, describe, expect, it} from 'vitest';
|
||||
import type {ApiTestHarness} from '../../test/ApiTestHarness';
|
||||
import {createApiTestHarness} from '../../test/ApiTestHarness';
|
||||
import type {MockKVProvider} from '../../test/mocks/MockKVProvider';
|
||||
import {
|
||||
drainAbuseAutoBanTasksForTests,
|
||||
drainAbuseIpClassLookupsForTests,
|
||||
hashAuthToken,
|
||||
recordAbuseSignal,
|
||||
resetAbuseTrackingForTests,
|
||||
setAbuseIpClassTtlsForTests,
|
||||
startAbuseReplicationSubscriber,
|
||||
stopAbuseReplicationSubscriber,
|
||||
} from '../AbusiveIpAutoBanner';
|
||||
import {ipBanCache} from '../IpBanMiddleware';
|
||||
import {setInjectedIpInfoService} from '../ServiceMiddleware';
|
||||
|
||||
function ipInfoResult(ip: string, overrides: Partial<IpInfoLookupResult> = {}): IpInfoLookupResult {
|
||||
return {
|
||||
ip,
|
||||
available: true,
|
||||
riskNote: 'test',
|
||||
geo: {
|
||||
countryCode: 'US',
|
||||
countryName: 'United States',
|
||||
continent: 'North America',
|
||||
continentCode: 'NA',
|
||||
region: null,
|
||||
regionCode: null,
|
||||
city: null,
|
||||
postalCode: null,
|
||||
timezone: null,
|
||||
latitude: null,
|
||||
longitude: null,
|
||||
accuracyRadiusKm: null,
|
||||
},
|
||||
asn: {
|
||||
asn: 'AS64500',
|
||||
number: 64500,
|
||||
name: 'Test ISP',
|
||||
domain: null,
|
||||
type: null,
|
||||
},
|
||||
mobile: {
|
||||
name: null,
|
||||
mcc: null,
|
||||
mnc: null,
|
||||
},
|
||||
anonymous: {
|
||||
isAnonymous: false,
|
||||
providerName: null,
|
||||
isVpn: false,
|
||||
isProxy: false,
|
||||
isResidentialProxy: false,
|
||||
isTor: false,
|
||||
isRelay: false,
|
||||
percentDaysSeen: null,
|
||||
},
|
||||
flags: {
|
||||
isAnycast: false,
|
||||
isHosting: false,
|
||||
isMobile: false,
|
||||
isSatellite: false,
|
||||
},
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function publishPeerIpClasses(harness: ApiTestHarness, entries: Array<[string, string, unknown]>): void {
|
||||
const kvProvider = harness.kvProvider as MockKVProvider;
|
||||
kvProvider
|
||||
.getSubscription()
|
||||
.simulateMessage('abuse_tracker:ipclass', JSON.stringify({sender: 'other-pod', entries, ts: Date.now()}));
|
||||
}
|
||||
|
||||
function claimCallCount(harness: ApiTestHarness, banKey: string): number {
|
||||
const kvProvider = harness.kvProvider as MockKVProvider;
|
||||
return kvProvider.setnxSpy.mock.calls.filter(([key]) => key === `abuse:ipclass:claim:${banKey}`).length;
|
||||
}
|
||||
|
||||
function recordTokenSignals(ip: string, prefix: string, from: number, to: number): void {
|
||||
for (let i = from; i < to; i += 1) {
|
||||
recordAbuseSignal(ip, 'auth_failure:session', {tokenHash: hashAuthToken(`${prefix}-${i}`)});
|
||||
}
|
||||
}
|
||||
|
||||
async function drainAbuseWork(): Promise<void> {
|
||||
await drainAbuseIpClassLookupsForTests();
|
||||
await drainAbuseAutoBanTasksForTests();
|
||||
}
|
||||
|
||||
describe('AbusiveIpAutoBanner IP class replication', () => {
|
||||
let harness: ApiTestHarness;
|
||||
let lookupCount = 0;
|
||||
beforeAll(async () => {
|
||||
harness = await createApiTestHarness();
|
||||
});
|
||||
beforeEach(async () => {
|
||||
await harness.reset();
|
||||
resetAbuseTrackingForTests();
|
||||
ipBanCache.resetCaches();
|
||||
lookupCount = 0;
|
||||
setInjectedIpInfoService({
|
||||
async lookup(ip: string) {
|
||||
lookupCount += 1;
|
||||
return ipInfoResult(ip);
|
||||
},
|
||||
});
|
||||
await stopAbuseReplicationSubscriber();
|
||||
await startAbuseReplicationSubscriber(harness.kvProvider);
|
||||
});
|
||||
afterAll(async () => {
|
||||
await stopAbuseReplicationSubscriber();
|
||||
setInjectedIpInfoService(undefined);
|
||||
await harness.shutdown();
|
||||
});
|
||||
it('adopts a datacenter class from a peer without paying for its own lookup', async () => {
|
||||
const ip = '8.8.8.8';
|
||||
recordTokenSignals(ip, 'peer-datacenter', 0, 3);
|
||||
publishPeerIpClasses(harness, [[ip, ip, 'datacenter']]);
|
||||
recordTokenSignals(ip, 'peer-datacenter', 3, 10);
|
||||
await drainAbuseWork();
|
||||
expect(lookupCount).toBe(0);
|
||||
expect(claimCallCount(harness, ip)).toBe(0);
|
||||
expect(ipBanCache.isBanned(ip)).toBe(true);
|
||||
});
|
||||
it('never adopts a mobile class from a peer and leaves the IP bannable', async () => {
|
||||
const ip = '8.8.4.4';
|
||||
recordTokenSignals(ip, 'peer-mobile', 0, 3);
|
||||
publishPeerIpClasses(harness, [[ip, ip, 'mobile']]);
|
||||
recordTokenSignals(ip, 'peer-mobile', 3, 10);
|
||||
await drainAbuseWork();
|
||||
expect(lookupCount).toBe(0);
|
||||
expect(ipBanCache.isBanned(ip)).toBe(false);
|
||||
publishPeerIpClasses(harness, [[ip, ip, 'datacenter']]);
|
||||
await drainAbuseWork();
|
||||
expect(ipBanCache.isBanned(ip)).toBe(true);
|
||||
});
|
||||
it('ignores an unrecognised class from a peer and classifies the IP itself', async () => {
|
||||
const ip = '4.4.4.4';
|
||||
recordTokenSignals(ip, 'peer-invalid', 0, 3);
|
||||
publishPeerIpClasses(harness, [
|
||||
[ip, ip, 'datacentre'],
|
||||
[ip, ip, 42],
|
||||
]);
|
||||
recordTokenSignals(ip, 'peer-invalid', 3, 10);
|
||||
await drainAbuseWork();
|
||||
expect(lookupCount).toBe(1);
|
||||
expect(ipBanCache.isBanned(ip)).toBe(true);
|
||||
});
|
||||
it('ignores an unknown class from a peer and classifies the IP itself', async () => {
|
||||
const ip = '9.9.9.9';
|
||||
recordTokenSignals(ip, 'peer-unknown', 0, 3);
|
||||
publishPeerIpClasses(harness, [[ip, ip, 'unknown']]);
|
||||
recordTokenSignals(ip, 'peer-unknown', 3, 10);
|
||||
await drainAbuseWork();
|
||||
expect(lookupCount).toBe(1);
|
||||
expect(ipBanCache.isBanned(ip)).toBe(true);
|
||||
});
|
||||
it('ignores a peer class for an IP it is not tracking', async () => {
|
||||
const ip = '208.67.222.222';
|
||||
publishPeerIpClasses(harness, [[ip, ip, 'datacenter']]);
|
||||
recordTokenSignals(ip, 'peer-untracked', 0, 10);
|
||||
await drainAbuseWork();
|
||||
expect(lookupCount).toBe(1);
|
||||
expect(claimCallCount(harness, ip)).toBe(1);
|
||||
});
|
||||
it('accepts a peer mobile class after its own lookup failed and keeps the IP unbanned', async () => {
|
||||
const ip = '199.85.126.10';
|
||||
setInjectedIpInfoService({
|
||||
async lookup(lookupIp: string) {
|
||||
lookupCount += 1;
|
||||
return ipInfoResult(lookupIp, {available: false});
|
||||
},
|
||||
});
|
||||
recordAbuseSignal(ip, 'http_429', {weight: 25});
|
||||
await drainAbuseWork();
|
||||
expect(lookupCount).toBe(1);
|
||||
publishPeerIpClasses(harness, [[ip, ip, 'mobile']]);
|
||||
recordTokenSignals(ip, 'negative-then-mobile', 0, 10);
|
||||
await drainAbuseWork();
|
||||
expect(lookupCount).toBe(1);
|
||||
expect(ipBanCache.isBanned(ip)).toBe(false);
|
||||
});
|
||||
it('resumes classifying once a peer class hint expires', async () => {
|
||||
const ip = '77.88.8.8';
|
||||
setAbuseIpClassTtlsForTests({hintMs: 50});
|
||||
recordAbuseSignal(ip, 'http_429', {weight: 3});
|
||||
publishPeerIpClasses(harness, [[ip, ip, 'datacenter']]);
|
||||
recordAbuseSignal(ip, 'http_429', {weight: 25});
|
||||
await drainAbuseWork();
|
||||
expect(lookupCount).toBe(0);
|
||||
await new Promise((resolve) => setTimeout(resolve, 80));
|
||||
recordAbuseSignal(ip, 'http_429', {weight: 25});
|
||||
await drainAbuseWork();
|
||||
expect(lookupCount).toBe(1);
|
||||
expect(claimCallCount(harness, ip)).toBe(1);
|
||||
expect(ipBanCache.isBanned(ip)).toBe(false);
|
||||
});
|
||||
});
|
||||
@@ -197,7 +197,7 @@ export class UserSettings {
|
||||
return {
|
||||
user_id: userId,
|
||||
locale,
|
||||
theme: theme ?? ThemeTypes.SYSTEM,
|
||||
theme: theme ?? ThemeTypes.DARK,
|
||||
status: 'online',
|
||||
status_resets_at: null,
|
||||
status_resets_to: null,
|
||||
|
||||
@@ -22705,7 +22705,8 @@
|
||||
"type": "string",
|
||||
"description": "Maximum number of saved messages to return (1-100, default 25)"
|
||||
}
|
||||
}
|
||||
},
|
||||
{"name": "before", "in": "query", "required": false, "schema": {"$ref": "#/components/schemas/SnowflakeType"}}
|
||||
]
|
||||
},
|
||||
"post": {
|
||||
|
||||
@@ -813,14 +813,19 @@ export class ReportService {
|
||||
return reports.filter((report): report is IARSubmission => report !== null);
|
||||
}
|
||||
|
||||
async listReportsByStatus(status: number, limit?: number, offset?: number): Promise<Array<IARSubmission>> {
|
||||
async listReportsByStatus(
|
||||
status: number,
|
||||
limit?: number,
|
||||
offset?: number,
|
||||
): Promise<{reports: Array<IARSubmission>; total: number}> {
|
||||
if (!this.reportSearchService) {
|
||||
throw new FeatureTemporarilyDisabledError();
|
||||
}
|
||||
const {hits} = await this.reportSearchService.listReportsByStatus(status, limit, offset);
|
||||
const {hits, total} = await this.reportSearchService.listReportsByStatus(status, limit, offset);
|
||||
const reportIds = hits.map((hit) => createReportID(BigInt(hit.id)));
|
||||
const reports = await Promise.all(reportIds.map((id) => this.reportRepository.getReport(id)));
|
||||
return reports.filter((report): report is IARSubmission => report !== null);
|
||||
const loaded = await Promise.all(reportIds.map((id) => this.reportRepository.getReport(id)));
|
||||
const reports = loaded.filter((report): report is IARSubmission => report !== null);
|
||||
return {reports, total: Math.max(0, total - (hits.length - reports.length))};
|
||||
}
|
||||
|
||||
async resolveReport(
|
||||
|
||||
@@ -1,6 +1,10 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
let cachedEnabled = false;
|
||||
// Null until a policy has actually been read. A deferral is only ever granted
|
||||
// while the gate is on, so reading "not known yet" as "off" revokes it from every
|
||||
// account holding one and answers their next request with the phone requirement
|
||||
// the deferral exists to hold back.
|
||||
let cachedEnabled: boolean | null = null;
|
||||
|
||||
export function resolveDeferredPhoneGateEnabled(policy: {
|
||||
deferred_phone_gate_enabled: boolean;
|
||||
@@ -9,7 +13,7 @@ export function resolveDeferredPhoneGateEnabled(policy: {
|
||||
return policy.deferred_phone_gate_enabled && !policy.single_community_enabled;
|
||||
}
|
||||
|
||||
export function getCachedDeferredPhoneGateEnabled(): boolean {
|
||||
export function getCachedDeferredPhoneGateEnabled(): boolean | null {
|
||||
return cachedEnabled;
|
||||
}
|
||||
|
||||
|
||||
@@ -7,9 +7,14 @@ import {isTrustedCommercialPrivacyProvider} from './TrustedPrivacyProviders';
|
||||
|
||||
const VERDICT_CACHE_TTL_MS = 60 * 60 * 1000;
|
||||
|
||||
interface IpBanBlastRadiusVerdict {
|
||||
cgnat: boolean;
|
||||
sharedAccess: boolean;
|
||||
}
|
||||
|
||||
interface CachedVerdict {
|
||||
expiresAtMs: number;
|
||||
highRisk: boolean;
|
||||
verdict: IpBanBlastRadiusVerdict;
|
||||
}
|
||||
|
||||
const verdictCache = new Map<string, CachedVerdict>();
|
||||
@@ -48,7 +53,7 @@ export function getSuspiciousIpSkipReason(result: IpInfoLookupResult): Suspiciou
|
||||
return null;
|
||||
}
|
||||
|
||||
function isHighSharedAccessBlastRadiusRisk(result: IpInfoLookupResult): boolean {
|
||||
export function isHighSharedAccessBlastRadiusRisk(result: IpInfoLookupResult): boolean {
|
||||
if (result.flags.isHosting || isAnonymousAccess(result)) {
|
||||
return false;
|
||||
}
|
||||
@@ -60,29 +65,32 @@ export function isSingleIpBanCandidate(value: string): boolean {
|
||||
return parseIpBanEntry(value)?.type === 'single';
|
||||
}
|
||||
|
||||
export async function hasHighCgnatBlastRadiusRisk(
|
||||
export async function getIpBanBlastRadiusVerdict(
|
||||
ip: string,
|
||||
ipInfoService: IpInfoService,
|
||||
context: {
|
||||
source: string;
|
||||
reason: string;
|
||||
},
|
||||
): Promise<boolean> {
|
||||
): Promise<IpBanBlastRadiusVerdict> {
|
||||
const now = Date.now();
|
||||
const cacheKey = getSameIpDecisionKey(ip) ?? ip;
|
||||
const cached = verdictCache.get(cacheKey);
|
||||
if (cached && cached.expiresAtMs > now) {
|
||||
return cached.highRisk;
|
||||
return cached.verdict;
|
||||
}
|
||||
const result = await ipInfoService.lookup(ip, {
|
||||
source: context.source,
|
||||
reason: context.reason,
|
||||
metadata: {policy: 'ip_ban_cgnat_guard'},
|
||||
});
|
||||
const highRisk = isHighCgnatBlastRadiusRisk(result);
|
||||
const verdict: IpBanBlastRadiusVerdict = {
|
||||
cgnat: isHighCgnatBlastRadiusRisk(result),
|
||||
sharedAccess: isHighSharedAccessBlastRadiusRisk(result),
|
||||
};
|
||||
verdictCache.set(cacheKey, {
|
||||
highRisk,
|
||||
verdict,
|
||||
expiresAtMs: now + VERDICT_CACHE_TTL_MS,
|
||||
});
|
||||
return highRisk;
|
||||
return verdict;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,105 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
import type {IpInfoLookupBudget, IpInfoLookupPriority} from '@pkgs/geoip/src/IpInfoService';
|
||||
import type {IKVProvider} from '@pkgs/kv_client/src/IKVProvider';
|
||||
import {Logger} from '../Logger';
|
||||
|
||||
const BURST_KEY_PREFIX = 'ipinfo:budget:burst:';
|
||||
const MONTH_KEY_PREFIX = 'ipinfo:budget:month:';
|
||||
const MONTH_KEY_TTL_SECONDS = 40 * 24 * 3600;
|
||||
const BURST_REFILL_INTERVAL_MS = 60_000;
|
||||
const BUDGET_LOG_INTERVAL_MS = 60_000;
|
||||
|
||||
let lastBudgetErrorLogMs = 0;
|
||||
|
||||
function positiveNumberFromEnv(name: string, fallback: number): number {
|
||||
const raw = process.env[name];
|
||||
if (!raw) return fallback;
|
||||
const parsed = Number(raw);
|
||||
return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback;
|
||||
}
|
||||
|
||||
function budgetEnabled(): boolean {
|
||||
return process.env.FLUXER_IPINFO_BUDGET_ENABLED !== '0';
|
||||
}
|
||||
|
||||
function burstConfigFor(priority: IpInfoLookupPriority): {maxTokens: number; refillPerMin: number} {
|
||||
if (priority === 'critical') {
|
||||
return {
|
||||
maxTokens: positiveNumberFromEnv('FLUXER_IPINFO_BUDGET_CRITICAL_BURST', 60),
|
||||
refillPerMin: positiveNumberFromEnv('FLUXER_IPINFO_BUDGET_CRITICAL_REFILL_PER_MIN', 60),
|
||||
};
|
||||
}
|
||||
if (priority === 'background') {
|
||||
return {
|
||||
maxTokens: positiveNumberFromEnv('FLUXER_IPINFO_BUDGET_BACKGROUND_BURST', 120),
|
||||
refillPerMin: positiveNumberFromEnv('FLUXER_IPINFO_BUDGET_BACKGROUND_REFILL_PER_MIN', 30),
|
||||
};
|
||||
}
|
||||
return {
|
||||
maxTokens: positiveNumberFromEnv('FLUXER_IPINFO_BUDGET_STANDARD_BURST', 240),
|
||||
refillPerMin: positiveNumberFromEnv('FLUXER_IPINFO_BUDGET_STANDARD_REFILL_PER_MIN', 120),
|
||||
};
|
||||
}
|
||||
|
||||
function monthlyCeilingFor(priority: IpInfoLookupPriority): number {
|
||||
const monthlyMax = positiveNumberFromEnv('FLUXER_IPINFO_BUDGET_MONTHLY_MAX', 140000);
|
||||
if (priority === 'critical') return monthlyMax;
|
||||
const percent =
|
||||
priority === 'background'
|
||||
? positiveNumberFromEnv('FLUXER_IPINFO_BUDGET_BACKGROUND_MONTHLY_PCT', 60)
|
||||
: positiveNumberFromEnv('FLUXER_IPINFO_BUDGET_STANDARD_MONTHLY_PCT', 90);
|
||||
return Math.floor((monthlyMax * Math.min(percent, 100)) / 100);
|
||||
}
|
||||
|
||||
function currentMonthKey(): string {
|
||||
const now = new Date();
|
||||
const month = String(now.getUTCMonth() + 1).padStart(2, '0');
|
||||
return `${MONTH_KEY_PREFIX}${now.getUTCFullYear()}-${month}`;
|
||||
}
|
||||
|
||||
function logThrottled(payload: Record<string, unknown>, message: string): void {
|
||||
const now = Date.now();
|
||||
if (now - lastBudgetErrorLogMs < BUDGET_LOG_INTERVAL_MS) return;
|
||||
lastBudgetErrorLogMs = now;
|
||||
Logger.warn(payload, message);
|
||||
}
|
||||
|
||||
export function createKvIpInfoLookupBudget(opts: {getKvClient: () => IKVProvider}): IpInfoLookupBudget {
|
||||
return {
|
||||
async tryConsume(priority: IpInfoLookupPriority): Promise<boolean> {
|
||||
if (!budgetEnabled()) {
|
||||
return true;
|
||||
}
|
||||
try {
|
||||
const kv = opts.getKvClient();
|
||||
const burst = burstConfigFor(priority);
|
||||
const consumed = await kv.tryConsumeTokens(
|
||||
`${BURST_KEY_PREFIX}${priority}`,
|
||||
1,
|
||||
burst.maxTokens,
|
||||
burst.refillPerMin,
|
||||
BURST_REFILL_INTERVAL_MS,
|
||||
);
|
||||
if (consumed < 1) {
|
||||
logThrottled({priority, reason: 'burst'}, 'IPInfo lookup budget shed');
|
||||
return false;
|
||||
}
|
||||
const monthKey = currentMonthKey();
|
||||
const used = Number((await kv.get(monthKey)) ?? '0');
|
||||
if (used >= monthlyCeilingFor(priority)) {
|
||||
logThrottled({priority, reason: 'monthly', used}, 'IPInfo lookup budget shed');
|
||||
return false;
|
||||
}
|
||||
const value = await kv.incr(monthKey);
|
||||
if (value === 1) {
|
||||
await kv.expire(monthKey, MONTH_KEY_TTL_SECONDS);
|
||||
}
|
||||
return true;
|
||||
} catch (error) {
|
||||
logThrottled({error, priority}, 'IPInfo lookup budget check failed, admitting lookup');
|
||||
return true;
|
||||
}
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -3,7 +3,7 @@
|
||||
import {getDefaultCassandraClient} from '@pkgs/cassandra/src/Client';
|
||||
import {createCassandraIpInfoCache} from '@pkgs/geoip/src/CassandraIpInfoCache';
|
||||
import {createCassandraIpInfoRequestAuditLogger} from '@pkgs/geoip/src/CassandraIpInfoRequestAudit';
|
||||
import type {IpInfoCache, IpInfoRequestAuditLogger} from '@pkgs/geoip/src/IpInfoService';
|
||||
import {type IpInfoCache, type IpInfoRequestAuditLogger, isCachedIpInfoFailure} from '@pkgs/geoip/src/IpInfoService';
|
||||
import {createPostgresIpInfoCache, createPostgresIpInfoRequestAuditLogger} from '@pkgs/geoip/src/PostgresIpInfoKv';
|
||||
import {createTieredIpInfoCache} from '@pkgs/geoip/src/TieredIpInfoCache';
|
||||
import {getDefaultPostgresClient} from '@pkgs/postgres/src/Client';
|
||||
@@ -22,11 +22,13 @@ export function buildIpInfoCache(options: BuildIpInfoCacheOptions): IpInfoCache
|
||||
getClient: getDefaultPostgresClient,
|
||||
onError: (error, operation) => Logger.warn({error, operation}, 'Postgres IPInfo cache operation failed'),
|
||||
}),
|
||||
skipColdWrite: isCachedIpInfoFailure,
|
||||
});
|
||||
}
|
||||
return createTieredIpInfoCache({
|
||||
hot: options.hot,
|
||||
cold: createCassandraIpInfoCache({getClient: getDefaultCassandraClient}),
|
||||
skipColdWrite: isCachedIpInfoFailure,
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,60 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
import {
|
||||
classifyAccountPolicyReverseDnsHostname,
|
||||
isAccountPolicyEducationOrganizationName,
|
||||
isAccountPolicyTrustedCommercialPrivacyProvider,
|
||||
} from './AccountPolicyService';
|
||||
|
||||
const ASN_ENTRY_REGEX = /^\d+$/u;
|
||||
|
||||
export interface LocalIpIntel {
|
||||
countryIso: string | null;
|
||||
asn: number | null;
|
||||
asnOrg: string | null;
|
||||
}
|
||||
|
||||
export interface IpInfoPrescreenOptions {
|
||||
enabled: boolean;
|
||||
allowedAsns: ReadonlySet<number>;
|
||||
}
|
||||
|
||||
export type IpInfoPrescreenVerdict = 'consult' | 'skip';
|
||||
|
||||
export function prescreenIpInfoLookup(local: LocalIpIntel, opts: IpInfoPrescreenOptions): IpInfoPrescreenVerdict {
|
||||
if (!opts.enabled) return 'consult';
|
||||
if (opts.allowedAsns.size === 0) return 'consult';
|
||||
if (local.countryIso === null) return 'consult';
|
||||
if (local.asn === null) return 'consult';
|
||||
if (!opts.allowedAsns.has(local.asn)) return 'consult';
|
||||
if (isAccountPolicyTrustedCommercialPrivacyProvider(local.asnOrg)) return 'consult';
|
||||
if (isAccountPolicyEducationOrganizationName(local.asnOrg)) return 'consult';
|
||||
if (classifyAccountPolicyReverseDnsHostname(local.asnOrg) === 'cellular') return 'consult';
|
||||
return 'skip';
|
||||
}
|
||||
|
||||
export function ipInfoPrescreenOptionsFromEnv(): IpInfoPrescreenOptions {
|
||||
return {
|
||||
enabled: booleanFromEnv(process.env.FLUXER_RISK_IPINFO_PRESCREEN_ENABLED),
|
||||
allowedAsns: asnSetFromEnv(process.env.FLUXER_RISK_IPINFO_PRESCREEN_ALLOW_ASNS),
|
||||
};
|
||||
}
|
||||
|
||||
function booleanFromEnv(rawValue: string | undefined): boolean {
|
||||
if (!rawValue) return false;
|
||||
const normalized = rawValue.trim().toLowerCase();
|
||||
return normalized === '1' || normalized === 'true';
|
||||
}
|
||||
|
||||
function asnSetFromEnv(rawValue: string | undefined): ReadonlySet<number> {
|
||||
const allowedAsns = new Set<number>();
|
||||
if (!rawValue) return allowedAsns;
|
||||
for (const entry of rawValue.split(',')) {
|
||||
const trimmed = entry.trim();
|
||||
if (!ASN_ENTRY_REGEX.test(trimmed)) continue;
|
||||
const asn = Number.parseInt(trimmed, 10);
|
||||
if (!Number.isSafeInteger(asn)) continue;
|
||||
allowedAsns.add(asn);
|
||||
}
|
||||
return allowedAsns;
|
||||
}
|
||||
@@ -1,6 +1,7 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
import type {ICacheService} from '@pkgs/cache/src/ICacheService';
|
||||
import type {GeoipAsnResult, GeoipResult} from '@pkgs/geoip/src/GeoipLookup';
|
||||
import type {IpInfoService} from '@pkgs/geoip/src/IpInfoService';
|
||||
import type {IAdminRepository} from '../admin/IAdminRepository';
|
||||
import {createDisposableDomainChecker} from './adapters/DisposableDomainChecker';
|
||||
@@ -9,6 +10,7 @@ import {createDomainAgeChecker} from './adapters/DomainAgeChecker';
|
||||
import {analyzeEmailSyntax} from './adapters/EmailSyntaxAnalyzer';
|
||||
import {createGeoIpAsnAdapter, createGeoIpCityAdapter} from './adapters/GeoIpAdapters';
|
||||
import {createHistoricalOutcomeAdapter} from './adapters/HistoricalOutcomeAdapter';
|
||||
import {unavailableIpInfoAnonymousResult} from './adapters/IpInfoAdapter';
|
||||
import {checkGeoVsLocale} from './adapters/LocaleGeoMatcher';
|
||||
import {analyzeRegistrationTiming} from './adapters/RegistrationTimingAnalyzer';
|
||||
import {analyzeUserAgent} from './adapters/UserAgentAnalyzer';
|
||||
@@ -29,12 +31,20 @@ interface RiskToolboxFactoryOptions {
|
||||
mxResolver?: MxResolver;
|
||||
mxCacheTtlMs?: number;
|
||||
cacheService?: ICacheService;
|
||||
lookupLocalCity?: (ip: string) => Promise<GeoipResult>;
|
||||
lookupLocalAsn?: (ip: string) => Promise<GeoipAsnResult>;
|
||||
}
|
||||
|
||||
export function createRiskToolbox(opts: RiskToolboxFactoryOptions): RiskToolbox {
|
||||
const checkDomainDisposable = createDisposableDomainChecker({adminRepository: opts.adminRepository});
|
||||
const lookupGeoIpCity = createGeoIpCityAdapter({ipInfoService: opts.ipInfoService});
|
||||
const lookupGeoIpAsn = createGeoIpAsnAdapter({ipInfoService: opts.ipInfoService});
|
||||
const lookupGeoIpCity = createGeoIpCityAdapter({
|
||||
ipInfoService: opts.ipInfoService,
|
||||
lookupLocalCity: opts.lookupLocalCity,
|
||||
});
|
||||
const lookupGeoIpAsn = createGeoIpAsnAdapter({
|
||||
ipInfoService: opts.ipInfoService,
|
||||
lookupLocalAsn: opts.lookupLocalAsn,
|
||||
});
|
||||
const checkMx = createDnsMxChecker({
|
||||
resolver: opts.mxResolver ?? new NodeDnsMxResolver(),
|
||||
cacheTtlMs: opts.mxCacheTtlMs,
|
||||
@@ -46,25 +56,7 @@ export function createRiskToolbox(opts: RiskToolboxFactoryOptions): RiskToolbox
|
||||
});
|
||||
const lookupIpInfo = opts.ipInfoChecker
|
||||
? async (args: {ip: string}) => opts.ipInfoChecker!(args.ip)
|
||||
: async (args: {ip: string}) =>
|
||||
({
|
||||
ip: args.ip,
|
||||
available: false,
|
||||
isAnonymous: false,
|
||||
providerName: null,
|
||||
isVpn: false,
|
||||
isProxy: false,
|
||||
isResidentialProxy: false,
|
||||
isTor: false,
|
||||
isRelay: false,
|
||||
isHosting: false,
|
||||
isMobile: false,
|
||||
asnType: null,
|
||||
asnOrg: null,
|
||||
connectionType: 'unknown',
|
||||
percentDaysSeen: null,
|
||||
riskNote: 'IPInfo not configured (no API key)',
|
||||
}) as IpInfoAnonymousResult;
|
||||
: async (args: {ip: string}) => unavailableIpInfoAnonymousResult(args.ip, 'IPInfo not configured (no API key)');
|
||||
const lookupReverseDns = opts.reverseDnsLookup
|
||||
? async (args: {ip: string}) => opts.reverseDnsLookup!(args.ip)
|
||||
: async (args: {ip: string}) => ({
|
||||
|
||||
@@ -0,0 +1,289 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
import type {GeoipAsnResult, GeoipResult} from '@pkgs/geoip/src/GeoipLookup';
|
||||
import type {IpInfoLookupResult, IpInfoService} from '@pkgs/geoip/src/IpInfoService';
|
||||
import {describe, expect, it} from 'vitest';
|
||||
import {createGeoIpAsnAdapter, createGeoIpCityAdapter} from '../adapters/GeoIpAdapters';
|
||||
|
||||
function throwingIpInfoService(): IpInfoService {
|
||||
return {
|
||||
lookup: async () => {
|
||||
throw new Error('ipinfo must not be consulted');
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function countingIpInfoService(result: IpInfoLookupResult): {service: IpInfoService; calls: () => number} {
|
||||
let calls = 0;
|
||||
return {
|
||||
service: {
|
||||
lookup: async () => {
|
||||
calls += 1;
|
||||
return result;
|
||||
},
|
||||
},
|
||||
calls: () => calls,
|
||||
};
|
||||
}
|
||||
|
||||
function ipInfoResult(overrides: Partial<IpInfoLookupResult> = {}): IpInfoLookupResult {
|
||||
return {
|
||||
ip: '198.51.100.1',
|
||||
available: true,
|
||||
riskNote: 'test',
|
||||
geo: {
|
||||
countryCode: 'US',
|
||||
countryName: 'United States',
|
||||
continent: 'North America',
|
||||
continentCode: 'NA',
|
||||
region: 'California',
|
||||
regionCode: 'CA',
|
||||
city: 'San Jose',
|
||||
postalCode: null,
|
||||
timezone: 'America/Los_Angeles',
|
||||
latitude: 37.3,
|
||||
longitude: -121.9,
|
||||
accuracyRadiusKm: 20,
|
||||
},
|
||||
asn: {
|
||||
asn: 'AS64500',
|
||||
number: 64500,
|
||||
name: 'Test ISP',
|
||||
domain: null,
|
||||
type: null,
|
||||
},
|
||||
mobile: {
|
||||
name: null,
|
||||
mcc: null,
|
||||
mnc: null,
|
||||
},
|
||||
anonymous: {
|
||||
isAnonymous: false,
|
||||
providerName: null,
|
||||
isVpn: false,
|
||||
isProxy: false,
|
||||
isResidentialProxy: false,
|
||||
isTor: false,
|
||||
isRelay: false,
|
||||
percentDaysSeen: null,
|
||||
},
|
||||
flags: {
|
||||
isAnycast: false,
|
||||
isHosting: false,
|
||||
isMobile: false,
|
||||
isSatellite: false,
|
||||
},
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function geoipResult(overrides: Partial<GeoipResult> = {}): GeoipResult {
|
||||
return {
|
||||
countryCode: 'SE',
|
||||
normalizedIp: '198.51.100.7',
|
||||
city: 'Stockholm',
|
||||
region: 'Stockholm County',
|
||||
regionCode: 'AB',
|
||||
countryName: 'Sweden',
|
||||
latitude: 59.33,
|
||||
longitude: 18.06,
|
||||
accuracyRadiusKm: 5,
|
||||
timeZone: 'Europe/Stockholm',
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function geoipAsnResult(overrides: Partial<GeoipAsnResult> = {}): GeoipAsnResult {
|
||||
return {
|
||||
normalizedIp: '198.51.100.7',
|
||||
asn: 64510,
|
||||
asnOrg: 'Example Broadband ISP',
|
||||
available: true,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
describe('createGeoIpCityAdapter', () => {
|
||||
it('answers from the local city database without touching IPInfo', async () => {
|
||||
const lookupGeoIpCity = createGeoIpCityAdapter({
|
||||
ipInfoService: throwingIpInfoService(),
|
||||
lookupLocalCity: async () => geoipResult(),
|
||||
});
|
||||
await expect(lookupGeoIpCity({ip: '198.51.100.7'})).resolves.toEqual({
|
||||
ip: '198.51.100.7',
|
||||
available: true,
|
||||
found: true,
|
||||
countryIso: 'SE',
|
||||
country: 'Sweden',
|
||||
region: 'Stockholm County',
|
||||
city: 'Stockholm',
|
||||
latitude: 59.33,
|
||||
longitude: 18.06,
|
||||
accuracyRadiusKm: 5,
|
||||
timeZone: 'Europe/Stockholm',
|
||||
});
|
||||
});
|
||||
|
||||
it('normalizes the IP before handing it to the local city database', async () => {
|
||||
const seen: Array<string> = [];
|
||||
const lookupGeoIpCity = createGeoIpCityAdapter({
|
||||
ipInfoService: throwingIpInfoService(),
|
||||
lookupLocalCity: async (ip) => {
|
||||
seen.push(ip);
|
||||
return geoipResult();
|
||||
},
|
||||
});
|
||||
await lookupGeoIpCity({ip: '[2001:0db8:0000::0001]'});
|
||||
expect(seen).toEqual(['2001:db8::1']);
|
||||
});
|
||||
|
||||
it('coalesces missing optional local fields to null', async () => {
|
||||
const lookupGeoIpCity = createGeoIpCityAdapter({
|
||||
ipInfoService: throwingIpInfoService(),
|
||||
lookupLocalCity: async () => ({
|
||||
countryCode: 'SE',
|
||||
normalizedIp: '198.51.100.7',
|
||||
city: null,
|
||||
region: null,
|
||||
countryName: null,
|
||||
}),
|
||||
});
|
||||
await expect(lookupGeoIpCity({ip: '198.51.100.7'})).resolves.toEqual({
|
||||
ip: '198.51.100.7',
|
||||
available: true,
|
||||
found: true,
|
||||
countryIso: 'SE',
|
||||
country: null,
|
||||
region: null,
|
||||
city: null,
|
||||
latitude: null,
|
||||
longitude: null,
|
||||
accuracyRadiusKm: null,
|
||||
timeZone: null,
|
||||
});
|
||||
});
|
||||
|
||||
it('falls back to IPInfo when the local city database has no country', async () => {
|
||||
const counting = countingIpInfoService(ipInfoResult());
|
||||
const lookupGeoIpCity = createGeoIpCityAdapter({
|
||||
ipInfoService: counting.service,
|
||||
lookupLocalCity: async () => geoipResult({countryCode: null}),
|
||||
});
|
||||
await expect(lookupGeoIpCity({ip: '198.51.100.1'})).resolves.toEqual({
|
||||
ip: '198.51.100.1',
|
||||
available: true,
|
||||
found: true,
|
||||
countryIso: 'US',
|
||||
country: 'United States',
|
||||
region: 'California',
|
||||
city: 'San Jose',
|
||||
latitude: 37.3,
|
||||
longitude: -121.9,
|
||||
accuracyRadiusKm: 20,
|
||||
timeZone: 'America/Los_Angeles',
|
||||
});
|
||||
expect(counting.calls()).toBe(1);
|
||||
});
|
||||
|
||||
it('falls back to IPInfo when no local city lookup is wired', async () => {
|
||||
const counting = countingIpInfoService(ipInfoResult());
|
||||
const lookupGeoIpCity = createGeoIpCityAdapter({ipInfoService: counting.service});
|
||||
const result = await lookupGeoIpCity({ip: '198.51.100.1'});
|
||||
expect(result.countryIso).toBe('US');
|
||||
expect(counting.calls()).toBe(1);
|
||||
});
|
||||
|
||||
it('returns an available not-found result for an unparseable IP without any lookup', async () => {
|
||||
const lookupGeoIpCity = createGeoIpCityAdapter({
|
||||
ipInfoService: throwingIpInfoService(),
|
||||
lookupLocalCity: async () => {
|
||||
throw new Error('local city must not be consulted');
|
||||
},
|
||||
});
|
||||
await expect(lookupGeoIpCity({ip: 'not-an-ip'})).resolves.toEqual({
|
||||
ip: 'not-an-ip',
|
||||
available: true,
|
||||
found: false,
|
||||
countryIso: null,
|
||||
country: null,
|
||||
region: null,
|
||||
city: null,
|
||||
latitude: null,
|
||||
longitude: null,
|
||||
accuracyRadiusKm: null,
|
||||
timeZone: null,
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('createGeoIpAsnAdapter', () => {
|
||||
it('answers from the local ASN database without touching IPInfo', async () => {
|
||||
const lookupGeoIpAsn = createGeoIpAsnAdapter({
|
||||
ipInfoService: throwingIpInfoService(),
|
||||
lookupLocalAsn: async () => geoipAsnResult(),
|
||||
});
|
||||
await expect(lookupGeoIpAsn({ip: '198.51.100.7'})).resolves.toEqual({
|
||||
ip: '198.51.100.7',
|
||||
available: true,
|
||||
found: true,
|
||||
asn: 64510,
|
||||
asnOrg: 'Example Broadband ISP',
|
||||
});
|
||||
});
|
||||
|
||||
it('normalizes the IP before handing it to the local ASN database', async () => {
|
||||
const seen: Array<string> = [];
|
||||
const lookupGeoIpAsn = createGeoIpAsnAdapter({
|
||||
ipInfoService: throwingIpInfoService(),
|
||||
lookupLocalAsn: async (ip) => {
|
||||
seen.push(ip);
|
||||
return geoipAsnResult();
|
||||
},
|
||||
});
|
||||
await lookupGeoIpAsn({ip: '[2001:0db8:0000::0001]'});
|
||||
expect(seen).toEqual(['2001:db8::1']);
|
||||
});
|
||||
|
||||
it('falls back to IPInfo when the local ASN database has no ASN', async () => {
|
||||
const counting = countingIpInfoService(ipInfoResult());
|
||||
const lookupGeoIpAsn = createGeoIpAsnAdapter({
|
||||
ipInfoService: counting.service,
|
||||
lookupLocalAsn: async () => geoipAsnResult({asn: null, asnOrg: null}),
|
||||
});
|
||||
await expect(lookupGeoIpAsn({ip: '198.51.100.1'})).resolves.toEqual({
|
||||
ip: '198.51.100.1',
|
||||
available: true,
|
||||
found: true,
|
||||
asn: 64500,
|
||||
asnOrg: 'Test ISP',
|
||||
});
|
||||
expect(counting.calls()).toBe(1);
|
||||
});
|
||||
|
||||
it('falls back to IPInfo when the local ASN database is unavailable', async () => {
|
||||
const counting = countingIpInfoService(ipInfoResult());
|
||||
const lookupGeoIpAsn = createGeoIpAsnAdapter({
|
||||
ipInfoService: counting.service,
|
||||
lookupLocalAsn: async () => geoipAsnResult({normalizedIp: null, asn: null, asnOrg: null, available: false}),
|
||||
});
|
||||
const result = await lookupGeoIpAsn({ip: '198.51.100.1'});
|
||||
expect(result.asn).toBe(64500);
|
||||
expect(counting.calls()).toBe(1);
|
||||
});
|
||||
|
||||
it('returns an available not-found result for an unparseable IP without any lookup', async () => {
|
||||
const lookupGeoIpAsn = createGeoIpAsnAdapter({
|
||||
ipInfoService: throwingIpInfoService(),
|
||||
lookupLocalAsn: async () => {
|
||||
throw new Error('local ASN must not be consulted');
|
||||
},
|
||||
});
|
||||
await expect(lookupGeoIpAsn({ip: 'not-an-ip'})).resolves.toEqual({
|
||||
ip: 'not-an-ip',
|
||||
available: true,
|
||||
found: false,
|
||||
asn: null,
|
||||
asnOrg: null,
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -2,7 +2,11 @@
|
||||
|
||||
import type {IpInfoLookupResult} from '@pkgs/geoip/src/IpInfoService';
|
||||
import {describe, expect, it} from 'vitest';
|
||||
import {isHighCgnatBlastRadiusRisk, isSingleIpBanCandidate} from '../IpBanCgnatGuard';
|
||||
import {
|
||||
isHighCgnatBlastRadiusRisk,
|
||||
isHighSharedAccessBlastRadiusRisk,
|
||||
isSingleIpBanCandidate,
|
||||
} from '../IpBanCgnatGuard';
|
||||
|
||||
function ipInfoResult(overrides: Partial<IpInfoLookupResult> = {}): IpInfoLookupResult {
|
||||
return {
|
||||
@@ -96,4 +100,63 @@ describe('IpBanCgnatGuard', () => {
|
||||
),
|
||||
).toBe(false);
|
||||
});
|
||||
it('flags satellite, anycast and education networks as high blast-radius risk', () => {
|
||||
expect(
|
||||
isHighSharedAccessBlastRadiusRisk(
|
||||
ipInfoResult({
|
||||
flags: {isAnycast: false, isHosting: false, isMobile: false, isSatellite: true},
|
||||
}),
|
||||
),
|
||||
).toBe(true);
|
||||
expect(
|
||||
isHighSharedAccessBlastRadiusRisk(
|
||||
ipInfoResult({
|
||||
flags: {isAnycast: true, isHosting: false, isMobile: false, isSatellite: false},
|
||||
}),
|
||||
),
|
||||
).toBe(true);
|
||||
expect(
|
||||
isHighSharedAccessBlastRadiusRisk(
|
||||
ipInfoResult({asn: {asn: 'AS64500', number: 64500, name: 'Test University', domain: null, type: 'education'}}),
|
||||
),
|
||||
).toBe(true);
|
||||
});
|
||||
it('does not flag ordinary residential networks as shared-access risk', () => {
|
||||
expect(isHighSharedAccessBlastRadiusRisk(ipInfoResult())).toBe(false);
|
||||
});
|
||||
it('does not treat shared-access networks as CGNAT risk', () => {
|
||||
expect(
|
||||
isHighCgnatBlastRadiusRisk(
|
||||
ipInfoResult({
|
||||
flags: {isAnycast: false, isHosting: false, isMobile: false, isSatellite: true},
|
||||
}),
|
||||
),
|
||||
).toBe(false);
|
||||
});
|
||||
it('does not exempt hosting or anonymous shared-access infrastructure', () => {
|
||||
expect(
|
||||
isHighSharedAccessBlastRadiusRisk(
|
||||
ipInfoResult({
|
||||
flags: {isAnycast: true, isHosting: true, isMobile: false, isSatellite: false},
|
||||
}),
|
||||
),
|
||||
).toBe(false);
|
||||
expect(
|
||||
isHighSharedAccessBlastRadiusRisk(
|
||||
ipInfoResult({
|
||||
anonymous: {
|
||||
isAnonymous: true,
|
||||
providerName: 'Example VPN',
|
||||
isVpn: true,
|
||||
isProxy: false,
|
||||
isResidentialProxy: false,
|
||||
isTor: false,
|
||||
isRelay: false,
|
||||
percentDaysSeen: null,
|
||||
},
|
||||
flags: {isAnycast: false, isHosting: false, isMobile: false, isSatellite: true},
|
||||
}),
|
||||
),
|
||||
).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -0,0 +1,156 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
import type {IpInfoLookupContext, IpInfoLookupResult, IpInfoService} from '@pkgs/geoip/src/IpInfoService';
|
||||
import {afterEach, beforeEach, describe, expect, it} from 'vitest';
|
||||
import {createCurrentBehaviorTestAccountPolicyEvaluator} from '../../test/AccountPolicyTestEvaluator';
|
||||
import {setInjectedAccountPolicyEvaluator} from '../AccountPolicyService';
|
||||
import {createIpInfoChecker, unavailableIpInfoAnonymousResult} from '../adapters/IpInfoAdapter';
|
||||
|
||||
function ipInfoResult(overrides: Partial<IpInfoLookupResult> = {}): IpInfoLookupResult {
|
||||
return {
|
||||
ip: '198.51.100.1',
|
||||
available: true,
|
||||
riskNote: 'live lookup',
|
||||
geo: {
|
||||
countryCode: 'US',
|
||||
countryName: 'United States',
|
||||
continent: 'North America',
|
||||
continentCode: 'NA',
|
||||
region: null,
|
||||
regionCode: null,
|
||||
city: null,
|
||||
postalCode: null,
|
||||
timezone: null,
|
||||
latitude: null,
|
||||
longitude: null,
|
||||
accuracyRadiusKm: null,
|
||||
},
|
||||
asn: {
|
||||
asn: 'AS64500',
|
||||
number: 64500,
|
||||
name: 'Test ISP',
|
||||
domain: null,
|
||||
type: 'isp',
|
||||
},
|
||||
mobile: {
|
||||
name: null,
|
||||
mcc: null,
|
||||
mnc: null,
|
||||
},
|
||||
anonymous: {
|
||||
isAnonymous: true,
|
||||
providerName: 'Example VPN',
|
||||
isVpn: true,
|
||||
isProxy: false,
|
||||
isResidentialProxy: false,
|
||||
isTor: false,
|
||||
isRelay: false,
|
||||
percentDaysSeen: 42,
|
||||
},
|
||||
flags: {
|
||||
isAnycast: false,
|
||||
isHosting: false,
|
||||
isMobile: false,
|
||||
isSatellite: false,
|
||||
},
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function countingIpInfoService(result: IpInfoLookupResult): {
|
||||
service: IpInfoService;
|
||||
calls: () => number;
|
||||
contexts: () => Array<IpInfoLookupContext | undefined>;
|
||||
} {
|
||||
const contexts: Array<IpInfoLookupContext | undefined> = [];
|
||||
return {
|
||||
service: {
|
||||
lookup: async (_ip, context) => {
|
||||
contexts.push(context);
|
||||
return result;
|
||||
},
|
||||
},
|
||||
calls: () => contexts.length,
|
||||
contexts: () => contexts,
|
||||
};
|
||||
}
|
||||
|
||||
beforeEach(() => {
|
||||
setInjectedAccountPolicyEvaluator(createCurrentBehaviorTestAccountPolicyEvaluator());
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
setInjectedAccountPolicyEvaluator(undefined);
|
||||
});
|
||||
|
||||
describe('createIpInfoChecker pre-screen', () => {
|
||||
it('returns an unavailable result with zero lookups when the pre-screen skips', async () => {
|
||||
const counting = countingIpInfoService(ipInfoResult());
|
||||
const checkIpInfo = createIpInfoChecker({
|
||||
ipInfoService: counting.service,
|
||||
prescreen: async () => 'skip',
|
||||
});
|
||||
await expect(checkIpInfo('198.51.100.1')).resolves.toEqual(
|
||||
unavailableIpInfoAnonymousResult('198.51.100.1', 'IPInfo skipped (local pre-screen)'),
|
||||
);
|
||||
expect(counting.calls()).toBe(0);
|
||||
});
|
||||
|
||||
it('calls IPInfo exactly once and preserves the mapping when the pre-screen consults', async () => {
|
||||
const counting = countingIpInfoService(ipInfoResult());
|
||||
const checkIpInfo = createIpInfoChecker({
|
||||
ipInfoService: counting.service,
|
||||
prescreen: async () => 'consult',
|
||||
});
|
||||
await expect(checkIpInfo('198.51.100.1')).resolves.toEqual({
|
||||
ip: '198.51.100.1',
|
||||
available: true,
|
||||
isAnonymous: true,
|
||||
providerName: 'Example VPN',
|
||||
isVpn: true,
|
||||
isProxy: false,
|
||||
isResidentialProxy: false,
|
||||
isTor: false,
|
||||
isRelay: false,
|
||||
isHosting: false,
|
||||
isMobile: false,
|
||||
asnType: 'isp',
|
||||
asnOrg: 'Test ISP',
|
||||
connectionType: 'residential',
|
||||
percentDaysSeen: 42,
|
||||
riskNote: 'live lookup',
|
||||
});
|
||||
expect(counting.calls()).toBe(1);
|
||||
expect(counting.contexts()).toEqual([{source: 'risk.ipinfo_checker', reason: 'registration_risk'}]);
|
||||
});
|
||||
|
||||
it('is identical to a consulting pre-screen when no pre-screen is wired', async () => {
|
||||
const withoutPrescreen = countingIpInfoService(ipInfoResult());
|
||||
const withPrescreen = countingIpInfoService(ipInfoResult());
|
||||
const baseline = await createIpInfoChecker({ipInfoService: withoutPrescreen.service})('198.51.100.1');
|
||||
const consulted = await createIpInfoChecker({
|
||||
ipInfoService: withPrescreen.service,
|
||||
prescreen: async () => 'consult',
|
||||
})('198.51.100.1');
|
||||
expect(baseline).toEqual(consulted);
|
||||
expect(withoutPrescreen.calls()).toBe(1);
|
||||
expect(withPrescreen.calls()).toBe(1);
|
||||
});
|
||||
|
||||
it('never synthesizes a clean attestation for a skipped lookup', () => {
|
||||
const skipped = unavailableIpInfoAnonymousResult('198.51.100.1', 'IPInfo skipped (local pre-screen)');
|
||||
expect(skipped.available).toBe(false);
|
||||
expect(skipped.isAnonymous).toBe(false);
|
||||
expect(skipped.isVpn).toBe(false);
|
||||
expect(skipped.isProxy).toBe(false);
|
||||
expect(skipped.isResidentialProxy).toBe(false);
|
||||
expect(skipped.isTor).toBe(false);
|
||||
expect(skipped.isRelay).toBe(false);
|
||||
expect(skipped.isHosting).toBe(false);
|
||||
expect(skipped.isMobile).toBe(false);
|
||||
expect(skipped.asnType).toBeNull();
|
||||
expect(skipped.asnOrg).toBeNull();
|
||||
expect(skipped.connectionType).toBe('unknown');
|
||||
expect(skipped.percentDaysSeen).toBeNull();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,134 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
import {afterEach, beforeEach, describe, expect, it, vi} from 'vitest';
|
||||
import {MockKVProvider} from '../../test/mocks/MockKVProvider';
|
||||
import {createKvIpInfoLookupBudget} from '../IpInfoBudget';
|
||||
|
||||
const BUDGET_ENV_KEYS = [
|
||||
'FLUXER_IPINFO_BUDGET_ENABLED',
|
||||
'FLUXER_IPINFO_BUDGET_MONTHLY_MAX',
|
||||
'FLUXER_IPINFO_BUDGET_BACKGROUND_MONTHLY_PCT',
|
||||
'FLUXER_IPINFO_BUDGET_STANDARD_MONTHLY_PCT',
|
||||
'FLUXER_IPINFO_BUDGET_CRITICAL_BURST',
|
||||
'FLUXER_IPINFO_BUDGET_STANDARD_BURST',
|
||||
'FLUXER_IPINFO_BUDGET_BACKGROUND_BURST',
|
||||
'FLUXER_IPINFO_BUDGET_CRITICAL_REFILL_PER_MIN',
|
||||
'FLUXER_IPINFO_BUDGET_STANDARD_REFILL_PER_MIN',
|
||||
'FLUXER_IPINFO_BUDGET_BACKGROUND_REFILL_PER_MIN',
|
||||
];
|
||||
|
||||
function monthKey(): string {
|
||||
const now = new Date();
|
||||
const month = String(now.getUTCMonth() + 1).padStart(2, '0');
|
||||
return `ipinfo:budget:month:${now.getUTCFullYear()}-${month}`;
|
||||
}
|
||||
|
||||
describe('IpInfoBudget', () => {
|
||||
const savedEnv = new Map<string, string | undefined>();
|
||||
|
||||
beforeEach(() => {
|
||||
for (const key of BUDGET_ENV_KEYS) {
|
||||
savedEnv.set(key, process.env[key]);
|
||||
delete process.env[key];
|
||||
}
|
||||
process.env.FLUXER_IPINFO_BUDGET_MONTHLY_MAX = '100';
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
for (const [key, value] of savedEnv) {
|
||||
if (value === undefined) {
|
||||
delete process.env[key];
|
||||
} else {
|
||||
process.env[key] = value;
|
||||
}
|
||||
}
|
||||
savedEnv.clear();
|
||||
});
|
||||
|
||||
it('sheds background lookups at the background monthly ceiling while standard still admits', async () => {
|
||||
const kv = new MockKVProvider();
|
||||
await kv.set(monthKey(), '60');
|
||||
const budget = createKvIpInfoLookupBudget({getKvClient: () => kv});
|
||||
|
||||
expect(await budget.tryConsume('background')).toBe(false);
|
||||
expect(await budget.tryConsume('standard')).toBe(true);
|
||||
});
|
||||
|
||||
it('sheds standard lookups at the standard monthly ceiling', async () => {
|
||||
const kv = new MockKVProvider();
|
||||
await kv.set(monthKey(), '90');
|
||||
const budget = createKvIpInfoLookupBudget({getKvClient: () => kv});
|
||||
|
||||
expect(await budget.tryConsume('standard')).toBe(false);
|
||||
});
|
||||
|
||||
it('admits critical lookups above the standard ceiling', async () => {
|
||||
const kv = new MockKVProvider();
|
||||
await kv.set(monthKey(), '95');
|
||||
const budget = createKvIpInfoLookupBudget({getKvClient: () => kv});
|
||||
|
||||
expect(await budget.tryConsume('critical')).toBe(true);
|
||||
});
|
||||
|
||||
it('never increments the monthly counter for a shed lookup', async () => {
|
||||
const kv = new MockKVProvider();
|
||||
await kv.set(monthKey(), '60');
|
||||
const budget = createKvIpInfoLookupBudget({getKvClient: () => kv});
|
||||
|
||||
const outcomes = [
|
||||
await budget.tryConsume('background'),
|
||||
await budget.tryConsume('background'),
|
||||
await budget.tryConsume('standard'),
|
||||
await budget.tryConsume('critical'),
|
||||
];
|
||||
|
||||
expect(outcomes).toEqual([false, false, true, true]);
|
||||
expect(kv.incrSpy).toHaveBeenCalledTimes(2);
|
||||
expect(await kv.get(monthKey())).toBe('62');
|
||||
});
|
||||
|
||||
it('sheds once the per-priority burst bucket is drained', async () => {
|
||||
process.env.FLUXER_IPINFO_BUDGET_BACKGROUND_BURST = '3';
|
||||
process.env.FLUXER_IPINFO_BUDGET_BACKGROUND_REFILL_PER_MIN = '1';
|
||||
const kv = new MockKVProvider();
|
||||
const budget = createKvIpInfoLookupBudget({getKvClient: () => kv});
|
||||
|
||||
expect(await budget.tryConsume('background')).toBe(true);
|
||||
expect(await budget.tryConsume('background')).toBe(true);
|
||||
expect(await budget.tryConsume('background')).toBe(true);
|
||||
expect(await budget.tryConsume('background')).toBe(false);
|
||||
expect(await budget.tryConsume('standard')).toBe(true);
|
||||
});
|
||||
|
||||
it('expires the month key only on the first increment', async () => {
|
||||
const kv = new MockKVProvider();
|
||||
const budget = createKvIpInfoLookupBudget({getKvClient: () => kv});
|
||||
|
||||
expect(await budget.tryConsume('standard')).toBe(true);
|
||||
expect(await budget.tryConsume('standard')).toBe(true);
|
||||
expect(await budget.tryConsume('standard')).toBe(true);
|
||||
|
||||
expect(kv.expireSpy).toHaveBeenCalledTimes(1);
|
||||
expect(kv.expireSpy).toHaveBeenCalledWith(monthKey(), 40 * 24 * 3600);
|
||||
});
|
||||
|
||||
it('fails open for every priority when the KV provider throws', async () => {
|
||||
const kv = new MockKVProvider();
|
||||
vi.spyOn(kv, 'get').mockRejectedValue(new Error('kv unavailable'));
|
||||
const budget = createKvIpInfoLookupBudget({getKvClient: () => kv});
|
||||
|
||||
expect(await budget.tryConsume('background')).toBe(true);
|
||||
expect(await budget.tryConsume('standard')).toBe(true);
|
||||
expect(await budget.tryConsume('critical')).toBe(true);
|
||||
});
|
||||
|
||||
it('admits everything when the budget is disabled', async () => {
|
||||
process.env.FLUXER_IPINFO_BUDGET_ENABLED = '0';
|
||||
const kv = new MockKVProvider();
|
||||
await kv.set(monthKey(), '1000');
|
||||
const budget = createKvIpInfoLookupBudget({getKvClient: () => kv});
|
||||
|
||||
expect(await budget.tryConsume('background')).toBe(true);
|
||||
expect(kv.tryConsumeTokensSpy).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,276 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
import type {
|
||||
CachedIpInfoFailure,
|
||||
IpInfoCache,
|
||||
IpInfoLookupBudget,
|
||||
IpInfoLookupPriority,
|
||||
IpInfoRequestAuditEvent,
|
||||
IpInfoRequestAuditLogger,
|
||||
} from '@pkgs/geoip/src/IpInfoService';
|
||||
import {createIpInfoService, resolveIpInfoLookupPriority} from '@pkgs/geoip/src/IpInfoService';
|
||||
import {delay, HttpResponse, http} from 'msw';
|
||||
import {describe, expect, it} from 'vitest';
|
||||
import {server} from '../../test/msw/server';
|
||||
|
||||
interface RecordedSet {
|
||||
key: string;
|
||||
value: unknown;
|
||||
ttlSeconds: number | undefined;
|
||||
}
|
||||
|
||||
interface RecordingCache {
|
||||
cache: IpInfoCache;
|
||||
sets: Array<RecordedSet>;
|
||||
}
|
||||
|
||||
function createRecordingCache(): RecordingCache {
|
||||
const store = new Map<string, unknown>();
|
||||
const sets: Array<RecordedSet> = [];
|
||||
return {
|
||||
sets,
|
||||
cache: {
|
||||
async get<T>(key: string): Promise<T | null> {
|
||||
return (store.get(key) as T | undefined) ?? null;
|
||||
},
|
||||
async set<T>(key: string, value: T, ttlSeconds?: number): Promise<void> {
|
||||
store.set(key, value);
|
||||
sets.push({key, value, ttlSeconds});
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function createRecordingAuditLogger(): {logger: IpInfoRequestAuditLogger; events: Array<IpInfoRequestAuditEvent>} {
|
||||
const events: Array<IpInfoRequestAuditEvent> = [];
|
||||
return {
|
||||
events,
|
||||
logger: {
|
||||
async record(event: IpInfoRequestAuditEvent): Promise<void> {
|
||||
events.push(event);
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function useLookupHandler(handler: () => Response | Promise<Response>): {count: () => number} {
|
||||
let calls = 0;
|
||||
server.use(
|
||||
http.get('https://api.ipinfo.io/lookup/:ip', async () => {
|
||||
calls += 1;
|
||||
return await handler();
|
||||
}),
|
||||
);
|
||||
return {count: () => calls};
|
||||
}
|
||||
|
||||
function successPayload(ip: string, anonymous: Record<string, boolean> = {}): Response {
|
||||
return HttpResponse.json({
|
||||
ip,
|
||||
geo: {country_code: 'US', country: 'United States'},
|
||||
as: {asn: 'AS64500', name: 'Test ISP'},
|
||||
anonymous,
|
||||
});
|
||||
}
|
||||
|
||||
describe('IpInfoService caching', () => {
|
||||
it('negative-caches an HTTP error and serves the second lookup without a request', async () => {
|
||||
const requests = useLookupHandler(() => new HttpResponse(null, {status: 500}));
|
||||
const {cache, sets} = createRecordingCache();
|
||||
const service = createIpInfoService({apiKey: 'token', cache});
|
||||
|
||||
const first = await service.lookup('203.0.113.1');
|
||||
const second = await service.lookup('203.0.113.1');
|
||||
|
||||
expect(first.available).toBe(false);
|
||||
expect(second.available).toBe(false);
|
||||
expect(requests.count()).toBe(1);
|
||||
expect(sets).toHaveLength(1);
|
||||
expect(sets[0]?.ttlSeconds).toBe(300);
|
||||
});
|
||||
|
||||
it('negative-caches a request failure for a short window', async () => {
|
||||
useLookupHandler(async () => {
|
||||
await delay(5000);
|
||||
return successPayload('203.0.113.2');
|
||||
});
|
||||
const {cache, sets} = createRecordingCache();
|
||||
const service = createIpInfoService({apiKey: 'token', cache});
|
||||
|
||||
const result = await service.lookup('203.0.113.2');
|
||||
|
||||
expect(result.available).toBe(false);
|
||||
expect(sets[0]?.ttlSeconds).toBe(60);
|
||||
expect((sets[0]?.value as CachedIpInfoFailure).failureOutcome).toBe('request_failed');
|
||||
});
|
||||
|
||||
it('negative-caches a schema mismatch', async () => {
|
||||
useLookupHandler(() => HttpResponse.json({}));
|
||||
const {cache, sets} = createRecordingCache();
|
||||
const service = createIpInfoService({apiKey: 'token', cache});
|
||||
|
||||
const result = await service.lookup('203.0.113.3');
|
||||
|
||||
expect(result.available).toBe(false);
|
||||
expect(sets[0]?.ttlSeconds).toBe(600);
|
||||
expect((sets[0]?.value as CachedIpInfoFailure).failureOutcome).toBe('schema_mismatch');
|
||||
expect((sets[0]?.value as CachedIpInfoFailure).failureHttpStatus).toBe(200);
|
||||
});
|
||||
|
||||
it('negative-caches a quota rejection for longer', async () => {
|
||||
useLookupHandler(() => new HttpResponse(null, {status: 429}));
|
||||
const {cache, sets} = createRecordingCache();
|
||||
const service = createIpInfoService({apiKey: 'token', cache});
|
||||
|
||||
await service.lookup('203.0.113.4');
|
||||
|
||||
expect(sets[0]?.ttlSeconds).toBe(900);
|
||||
});
|
||||
|
||||
it('caps the quota rejection TTL for background lookups', async () => {
|
||||
useLookupHandler(() => new HttpResponse(null, {status: 429}));
|
||||
const {cache, sets} = createRecordingCache();
|
||||
const service = createIpInfoService({apiKey: 'token', cache});
|
||||
|
||||
await service.lookup('203.0.113.5', {source: 'AbusiveIpAutoBanner', reason: 'classify'});
|
||||
|
||||
expect(sets[0]?.ttlSeconds).toBe(120);
|
||||
});
|
||||
|
||||
it('returns a cached failure as a clean unavailable result', async () => {
|
||||
useLookupHandler(() => new HttpResponse(null, {status: 500}));
|
||||
const {cache} = createRecordingCache();
|
||||
const service = createIpInfoService({apiKey: 'token', cache});
|
||||
|
||||
await service.lookup('203.0.113.6');
|
||||
const cached = await service.lookup('203.0.113.6');
|
||||
|
||||
expect(cached).not.toHaveProperty('cachedFailure');
|
||||
expect(cached).not.toHaveProperty('failureOutcome');
|
||||
expect(cached).not.toHaveProperty('failureHttpStatus');
|
||||
expect(cached).not.toHaveProperty('cachedAtMs');
|
||||
expect(cached.ip).toBe('203.0.113.6');
|
||||
expect(cached.riskNote).toBe('IPInfo HTTP 500');
|
||||
});
|
||||
|
||||
it('writes a cached failure that older readers can still consume', async () => {
|
||||
useLookupHandler(() => new HttpResponse(null, {status: 500}));
|
||||
const {cache, sets} = createRecordingCache();
|
||||
const service = createIpInfoService({apiKey: 'token', cache});
|
||||
|
||||
await service.lookup('203.0.113.7');
|
||||
|
||||
const entry = sets[0]?.value as CachedIpInfoFailure;
|
||||
expect(entry.cachedFailure).toBe(true);
|
||||
expect(entry.failureOutcome).toBe('http_error');
|
||||
expect(entry.failureHttpStatus).toBe(500);
|
||||
expect(typeof entry.cachedAtMs).toBe('number');
|
||||
const legacyView = {...entry, ip: '203.0.113.7'};
|
||||
expect(legacyView.available).toBe(false);
|
||||
expect(legacyView.geo.countryCode).toBeNull();
|
||||
expect(legacyView.asn.number).toBeNull();
|
||||
expect(legacyView.mobile.name).toBeNull();
|
||||
expect(legacyView.anonymous.isAnonymous).toBe(false);
|
||||
expect(legacyView.flags.isMobile).toBe(false);
|
||||
});
|
||||
|
||||
it('keeps the existing success TTL selection', async () => {
|
||||
useLookupHandler(() => successPayload('203.0.113.8'));
|
||||
const plain = createRecordingCache();
|
||||
await createIpInfoService({apiKey: 'token', cache: plain.cache}).lookup('203.0.113.8');
|
||||
|
||||
useLookupHandler(() => successPayload('203.0.113.9', {is_vpn: true}));
|
||||
const anonymous = createRecordingCache();
|
||||
await createIpInfoService({apiKey: 'token', cache: anonymous.cache}).lookup('203.0.113.9');
|
||||
|
||||
expect(plain.sets[0]?.ttlSeconds).toBe(14 * 24 * 60 * 60);
|
||||
expect(anonymous.sets[0]?.ttlSeconds).toBe(7 * 24 * 60 * 60);
|
||||
});
|
||||
|
||||
it('coalesces concurrent lookups across a failure', async () => {
|
||||
const requests = useLookupHandler(() => new HttpResponse(null, {status: 500}));
|
||||
const {cache, sets} = createRecordingCache();
|
||||
const service = createIpInfoService({apiKey: 'token', cache});
|
||||
|
||||
const [first, second] = await Promise.all([service.lookup('203.0.113.10'), service.lookup('203.0.113.10')]);
|
||||
|
||||
expect(requests.count()).toBe(1);
|
||||
expect(sets).toHaveLength(1);
|
||||
expect(first.available).toBe(false);
|
||||
expect(second.available).toBe(false);
|
||||
});
|
||||
|
||||
it('sheds a lookup when the budget refuses it', async () => {
|
||||
const requests = useLookupHandler(() => successPayload('203.0.113.11'));
|
||||
const {cache, sets} = createRecordingCache();
|
||||
const {logger, events} = createRecordingAuditLogger();
|
||||
const refused: Array<IpInfoLookupPriority> = [];
|
||||
const budget: IpInfoLookupBudget = {
|
||||
async tryConsume(priority: IpInfoLookupPriority): Promise<boolean> {
|
||||
refused.push(priority);
|
||||
return priority !== 'background';
|
||||
},
|
||||
};
|
||||
const service = createIpInfoService({apiKey: 'token', cache, auditLogger: logger, budget});
|
||||
|
||||
const result = await service.lookup('203.0.113.11', {source: 'AbusiveIpAutoBanner', reason: 'classify'});
|
||||
|
||||
expect(refused).toEqual(['background']);
|
||||
expect(requests.count()).toBe(0);
|
||||
expect(result.available).toBe(false);
|
||||
expect(result.riskNote).toContain('shed');
|
||||
expect(sets).toHaveLength(0);
|
||||
expect(events).toHaveLength(1);
|
||||
expect(events[0]?.outcome).toBe('budget_shed');
|
||||
expect(events[0]?.httpStatus).toBeNull();
|
||||
});
|
||||
|
||||
it('still coalesces concurrent lookups when a budget is configured', async () => {
|
||||
const requests = useLookupHandler(() => successPayload('203.0.113.13'));
|
||||
const {cache} = createRecordingCache();
|
||||
const consumed: Array<IpInfoLookupPriority> = [];
|
||||
const budget: IpInfoLookupBudget = {
|
||||
async tryConsume(priority: IpInfoLookupPriority): Promise<boolean> {
|
||||
consumed.push(priority);
|
||||
return true;
|
||||
},
|
||||
};
|
||||
const service = createIpInfoService({apiKey: 'token', cache, budget});
|
||||
|
||||
const results = await Promise.all([
|
||||
service.lookup('203.0.113.13', {source: 'risk.geoip_city'}),
|
||||
service.lookup('203.0.113.13', {source: 'risk.geoip_asn'}),
|
||||
service.lookup('203.0.113.13', {source: 'risk.ipinfo_checker'}),
|
||||
]);
|
||||
|
||||
expect(requests.count()).toBe(1);
|
||||
expect(consumed).toEqual(['standard']);
|
||||
expect(results.every((result) => result.available)).toBe(true);
|
||||
});
|
||||
|
||||
it('lets an admitting budget through', async () => {
|
||||
const requests = useLookupHandler(() => successPayload('203.0.113.12'));
|
||||
const {cache} = createRecordingCache();
|
||||
const budget: IpInfoLookupBudget = {
|
||||
async tryConsume(): Promise<boolean> {
|
||||
return true;
|
||||
},
|
||||
};
|
||||
const service = createIpInfoService({apiKey: 'token', cache, budget});
|
||||
|
||||
const result = await service.lookup('203.0.113.12', {source: 'risk.ipinfo_checker'});
|
||||
|
||||
expect(requests.count()).toBe(1);
|
||||
expect(result.available).toBe(true);
|
||||
});
|
||||
|
||||
it('maps every lookup source to a priority', () => {
|
||||
expect(resolveIpInfoLookupPriority('admin.ip_ban')).toBe('critical');
|
||||
expect(resolveIpInfoLookupPriority('admin.scheduled_deletion_suspicious_ip')).toBe('critical');
|
||||
expect(resolveIpInfoLookupPriority('AbusiveIpAutoBanner')).toBe('background');
|
||||
expect(resolveIpInfoLookupPriority('risk.ipinfo_checker')).toBe('standard');
|
||||
expect(resolveIpInfoLookupPriority('risk.geoip_city')).toBe('standard');
|
||||
expect(resolveIpInfoLookupPriority('risk.geoip_asn')).toBe('standard');
|
||||
expect(resolveIpInfoLookupPriority(undefined)).toBe('standard');
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,110 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
import {afterEach, beforeEach, describe, expect, it} from 'vitest';
|
||||
import {createCurrentBehaviorTestAccountPolicyEvaluator} from '../../test/AccountPolicyTestEvaluator';
|
||||
import {setInjectedAccountPolicyEvaluator} from '../AccountPolicyService';
|
||||
import {
|
||||
type IpInfoPrescreenOptions,
|
||||
ipInfoPrescreenOptionsFromEnv,
|
||||
type LocalIpIntel,
|
||||
prescreenIpInfoLookup,
|
||||
} from '../RegistrationIpPrescreen';
|
||||
|
||||
const CLEAN_LOCAL: LocalIpIntel = {countryIso: 'SE', asn: 64500, asnOrg: 'Example Broadband ISP'};
|
||||
|
||||
function options(overrides: Partial<IpInfoPrescreenOptions> = {}): IpInfoPrescreenOptions {
|
||||
return {enabled: true, allowedAsns: new Set([64500]), ...overrides};
|
||||
}
|
||||
|
||||
function local(overrides: Partial<LocalIpIntel> = {}): LocalIpIntel {
|
||||
return {...CLEAN_LOCAL, ...overrides};
|
||||
}
|
||||
|
||||
beforeEach(() => {
|
||||
setInjectedAccountPolicyEvaluator(createCurrentBehaviorTestAccountPolicyEvaluator());
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
setInjectedAccountPolicyEvaluator(undefined);
|
||||
delete process.env.FLUXER_RISK_IPINFO_PRESCREEN_ENABLED;
|
||||
delete process.env.FLUXER_RISK_IPINFO_PRESCREEN_ALLOW_ASNS;
|
||||
});
|
||||
|
||||
describe('prescreenIpInfoLookup', () => {
|
||||
it('skips the lookup only when every local signal is clean and allowlisted', () => {
|
||||
expect(prescreenIpInfoLookup(local(), options())).toBe('skip');
|
||||
});
|
||||
|
||||
it('consults when the pre-screen is disabled', () => {
|
||||
expect(prescreenIpInfoLookup(local(), options({enabled: false}))).toBe('consult');
|
||||
});
|
||||
|
||||
it('consults when the allowlist is empty', () => {
|
||||
expect(prescreenIpInfoLookup(local(), options({allowedAsns: new Set()}))).toBe('consult');
|
||||
});
|
||||
|
||||
it('consults when the local city database has no country', () => {
|
||||
expect(prescreenIpInfoLookup(local({countryIso: null}), options())).toBe('consult');
|
||||
});
|
||||
|
||||
it('consults when the local ASN database has no ASN', () => {
|
||||
expect(prescreenIpInfoLookup(local({asn: null}), options())).toBe('consult');
|
||||
});
|
||||
|
||||
it('consults when the ASN is not on the allowlist', () => {
|
||||
expect(prescreenIpInfoLookup(local({asn: 64501}), options())).toBe('consult');
|
||||
});
|
||||
|
||||
it('consults for a trusted commercial privacy provider even on the allowlist', () => {
|
||||
expect(prescreenIpInfoLookup(local({asnOrg: 'Example Privacy Relay LLC'}), options())).toBe('consult');
|
||||
});
|
||||
|
||||
it('consults for an education organization even on the allowlist', () => {
|
||||
expect(prescreenIpInfoLookup(local({asnOrg: 'North Example Academy'}), options())).toBe('consult');
|
||||
});
|
||||
|
||||
it('consults for a cellular organization even on the allowlist', () => {
|
||||
expect(prescreenIpInfoLookup(local({asnOrg: 'Example cell-net Wireless'}), options())).toBe('consult');
|
||||
});
|
||||
|
||||
it('skips when the allowlisted ASN carries no organization name to veto', () => {
|
||||
expect(prescreenIpInfoLookup(local({asnOrg: null}), options())).toBe('skip');
|
||||
});
|
||||
});
|
||||
|
||||
describe('ipInfoPrescreenOptionsFromEnv', () => {
|
||||
it('is disabled with an empty allowlist by default', () => {
|
||||
const parsed = ipInfoPrescreenOptionsFromEnv();
|
||||
expect(parsed.enabled).toBe(false);
|
||||
expect(parsed.allowedAsns.size).toBe(0);
|
||||
});
|
||||
|
||||
it('treats only 1 and true as enabled', () => {
|
||||
process.env.FLUXER_RISK_IPINFO_PRESCREEN_ENABLED = '1';
|
||||
expect(ipInfoPrescreenOptionsFromEnv().enabled).toBe(true);
|
||||
process.env.FLUXER_RISK_IPINFO_PRESCREEN_ENABLED = 'TRUE';
|
||||
expect(ipInfoPrescreenOptionsFromEnv().enabled).toBe(true);
|
||||
process.env.FLUXER_RISK_IPINFO_PRESCREEN_ENABLED = 'yes';
|
||||
expect(ipInfoPrescreenOptionsFromEnv().enabled).toBe(false);
|
||||
process.env.FLUXER_RISK_IPINFO_PRESCREEN_ENABLED = '0';
|
||||
expect(ipInfoPrescreenOptionsFromEnv().enabled).toBe(false);
|
||||
});
|
||||
|
||||
it('parses a comma separated allowlist and drops non numeric entries', () => {
|
||||
process.env.FLUXER_RISK_IPINFO_PRESCREEN_ALLOW_ASNS = ' 64500, 64501 ,,notanasn,64502x,-3,64503 ';
|
||||
expect([...ipInfoPrescreenOptionsFromEnv().allowedAsns]).toEqual([64500, 64501, 64503]);
|
||||
});
|
||||
|
||||
it('parses an empty allowlist from an empty or whitespace value', () => {
|
||||
process.env.FLUXER_RISK_IPINFO_PRESCREEN_ALLOW_ASNS = '';
|
||||
expect(ipInfoPrescreenOptionsFromEnv().allowedAsns.size).toBe(0);
|
||||
process.env.FLUXER_RISK_IPINFO_PRESCREEN_ALLOW_ASNS = ' , ,';
|
||||
expect(ipInfoPrescreenOptionsFromEnv().allowedAsns.size).toBe(0);
|
||||
});
|
||||
|
||||
it('yields a consult verdict for every input with the shipped defaults', () => {
|
||||
const shipped = ipInfoPrescreenOptionsFromEnv();
|
||||
expect(prescreenIpInfoLookup(local(), shipped)).toBe('consult');
|
||||
expect(prescreenIpInfoLookup(local({asnOrg: null}), shipped)).toBe('consult');
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,130 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
import type {IpInfoCache} from '@pkgs/geoip/src/IpInfoService';
|
||||
import {createTieredIpInfoCache} from '@pkgs/geoip/src/TieredIpInfoCache';
|
||||
import {describe, expect, it} from 'vitest';
|
||||
|
||||
interface RecordedSet {
|
||||
key: string;
|
||||
value: unknown;
|
||||
ttlSeconds: number | undefined;
|
||||
}
|
||||
|
||||
interface RecordingCache {
|
||||
cache: IpInfoCache;
|
||||
store: Map<string, unknown>;
|
||||
sets: Array<RecordedSet>;
|
||||
}
|
||||
|
||||
function createRecordingCache(): RecordingCache {
|
||||
const store = new Map<string, unknown>();
|
||||
const sets: Array<RecordedSet> = [];
|
||||
return {
|
||||
store,
|
||||
sets,
|
||||
cache: {
|
||||
async get<T>(key: string): Promise<T | null> {
|
||||
return (store.get(key) as T | undefined) ?? null;
|
||||
},
|
||||
async set<T>(key: string, value: T, ttlSeconds?: number): Promise<void> {
|
||||
store.set(key, value);
|
||||
sets.push({key, value, ttlSeconds});
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
describe('TieredIpInfoCache', () => {
|
||||
it('clamps the hot TTL to the requested TTL and passes the raw TTL to the cold tier', async () => {
|
||||
const hot = createRecordingCache();
|
||||
const cold = createRecordingCache();
|
||||
const tiered = createTieredIpInfoCache({hot: hot.cache, cold: cold.cache});
|
||||
|
||||
await tiered.set('a', {available: false}, 60);
|
||||
|
||||
expect(hot.sets).toEqual([{key: 'a', value: {available: false}, ttlSeconds: 60}]);
|
||||
expect(cold.sets).toEqual([{key: 'a', value: {available: false}, ttlSeconds: 60}]);
|
||||
});
|
||||
|
||||
it('caps the hot TTL at the configured hot window', async () => {
|
||||
const hot = createRecordingCache();
|
||||
const cold = createRecordingCache();
|
||||
const tiered = createTieredIpInfoCache({hot: hot.cache, cold: cold.cache});
|
||||
|
||||
await tiered.set('a', {available: true}, 100000);
|
||||
|
||||
expect(hot.sets[0]?.ttlSeconds).toBe(600);
|
||||
expect(cold.sets[0]?.ttlSeconds).toBe(100000);
|
||||
});
|
||||
|
||||
it('uses the hot window when no TTL is supplied', async () => {
|
||||
const hot = createRecordingCache();
|
||||
const cold = createRecordingCache();
|
||||
const tiered = createTieredIpInfoCache({hot: hot.cache, cold: cold.cache});
|
||||
|
||||
await tiered.set('a', {available: true});
|
||||
|
||||
expect(hot.sets[0]?.ttlSeconds).toBe(600);
|
||||
expect(cold.sets[0]?.ttlSeconds).toBeUndefined();
|
||||
});
|
||||
|
||||
it('skips the cold write when skipColdWrite matches', async () => {
|
||||
const hot = createRecordingCache();
|
||||
const cold = createRecordingCache();
|
||||
const tiered = createTieredIpInfoCache({
|
||||
hot: hot.cache,
|
||||
cold: cold.cache,
|
||||
skipColdWrite: (value) => (value as {available?: unknown}).available === false,
|
||||
});
|
||||
|
||||
await tiered.set('a', {available: false}, 60);
|
||||
await tiered.set('b', {available: true}, 60);
|
||||
|
||||
expect(hot.sets.map((entry) => entry.key)).toEqual(['a', 'b']);
|
||||
expect(cold.sets.map((entry) => entry.key)).toEqual(['b']);
|
||||
});
|
||||
|
||||
it('promotes a cold hit into the hot tier', async () => {
|
||||
const hot = createRecordingCache();
|
||||
const cold = createRecordingCache();
|
||||
cold.store.set('a', {available: true});
|
||||
const tiered = createTieredIpInfoCache({hot: hot.cache, cold: cold.cache});
|
||||
|
||||
const hit = await tiered.get('a');
|
||||
|
||||
expect(hit).toEqual({available: true});
|
||||
expect(hot.sets).toEqual([{key: 'a', value: {available: true}, ttlSeconds: 600}]);
|
||||
});
|
||||
|
||||
it('never promotes a cold hit that skipColdWrite matches', async () => {
|
||||
const hot = createRecordingCache();
|
||||
const cold = createRecordingCache();
|
||||
cold.store.set('a', {available: false});
|
||||
const tiered = createTieredIpInfoCache({
|
||||
hot: hot.cache,
|
||||
cold: cold.cache,
|
||||
skipColdWrite: (value) => (value as {available?: unknown}).available === false,
|
||||
});
|
||||
|
||||
const hit = await tiered.get('a');
|
||||
|
||||
expect(hit).toEqual({available: false});
|
||||
expect(hot.sets).toEqual([]);
|
||||
});
|
||||
|
||||
it('never writes a zero TTL', async () => {
|
||||
const hot = createRecordingCache();
|
||||
const cold = createRecordingCache();
|
||||
const tiered = createTieredIpInfoCache({hot: hot.cache, cold: cold.cache});
|
||||
|
||||
await tiered.set('a', {available: false}, 60);
|
||||
await tiered.set('b', {available: true}, 100000);
|
||||
await tiered.set('c', {available: true});
|
||||
cold.store.set('d', {available: true});
|
||||
await tiered.get('d');
|
||||
|
||||
for (const entry of [...hot.sets, ...cold.sets]) {
|
||||
expect(entry.ttlSeconds === undefined || entry.ttlSeconds > 0).toBe(true);
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -1,15 +1,18 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
import {parseIpAddress} from '@fluxer/ip_utils/src/IpAddress';
|
||||
import type {GeoipAsnResult, GeoipResult} from '@pkgs/geoip/src/GeoipLookup';
|
||||
import type {IpInfoService} from '@pkgs/geoip/src/IpInfoService';
|
||||
import type {GeoIpAsnResult, GeoIpCityResult} from '../RiskTypes';
|
||||
|
||||
interface GeoIpCityContext {
|
||||
ipInfoService: IpInfoService;
|
||||
lookupLocalCity?: (ip: string) => Promise<GeoipResult>;
|
||||
}
|
||||
|
||||
interface GeoIpAsnContext {
|
||||
ipInfoService: IpInfoService;
|
||||
lookupLocalAsn?: (ip: string) => Promise<GeoipAsnResult>;
|
||||
}
|
||||
|
||||
export function createGeoIpCityAdapter(ctx: GeoIpCityContext) {
|
||||
@@ -19,6 +22,22 @@ export function createGeoIpCityAdapter(ctx: GeoIpCityContext) {
|
||||
if (!parsed) {
|
||||
return notFound(ip, true);
|
||||
}
|
||||
const local = ctx.lookupLocalCity ? await ctx.lookupLocalCity(parsed.normalized) : null;
|
||||
if (local && local.countryCode !== null) {
|
||||
return {
|
||||
ip,
|
||||
available: true,
|
||||
found: true,
|
||||
countryIso: local.countryCode,
|
||||
country: local.countryName,
|
||||
region: local.region,
|
||||
city: local.city,
|
||||
latitude: local.latitude ?? null,
|
||||
longitude: local.longitude ?? null,
|
||||
accuracyRadiusKm: local.accuracyRadiusKm ?? null,
|
||||
timeZone: local.timeZone ?? null,
|
||||
};
|
||||
}
|
||||
const info = await ctx.ipInfoService.lookup(parsed.normalized, {
|
||||
source: 'risk.geoip_city',
|
||||
reason: 'registration_risk',
|
||||
@@ -63,6 +82,10 @@ export function createGeoIpAsnAdapter(ctx: GeoIpAsnContext) {
|
||||
if (!parsed) {
|
||||
return {ip, available: true, found: false, asn: null, asnOrg: null};
|
||||
}
|
||||
const local = ctx.lookupLocalAsn ? await ctx.lookupLocalAsn(parsed.normalized) : null;
|
||||
if (local && local.asn !== null) {
|
||||
return {ip, available: true, found: true, asn: local.asn, asnOrg: local.asnOrg};
|
||||
}
|
||||
const info = await ctx.ipInfoService.lookup(parsed.normalized, {
|
||||
source: 'risk.geoip_asn',
|
||||
reason: 'registration_risk',
|
||||
|
||||
@@ -2,14 +2,40 @@
|
||||
|
||||
import type {IpInfoService} from '@pkgs/geoip/src/IpInfoService';
|
||||
import {isAccountPolicyEducationOrganizationName} from '../AccountPolicyService';
|
||||
import type {IpInfoPrescreenVerdict} from '../RegistrationIpPrescreen';
|
||||
import type {IpConnectionType, IpInfoAnonymousResult} from '../RiskTypes';
|
||||
|
||||
interface IpInfoCheckerContext {
|
||||
ipInfoService: IpInfoService;
|
||||
prescreen?: (ip: string) => Promise<IpInfoPrescreenVerdict>;
|
||||
}
|
||||
|
||||
export function unavailableIpInfoAnonymousResult(ip: string, riskNote: string): IpInfoAnonymousResult {
|
||||
return {
|
||||
ip,
|
||||
available: false,
|
||||
isAnonymous: false,
|
||||
providerName: null,
|
||||
isVpn: false,
|
||||
isProxy: false,
|
||||
isResidentialProxy: false,
|
||||
isTor: false,
|
||||
isRelay: false,
|
||||
isHosting: false,
|
||||
isMobile: false,
|
||||
asnType: null,
|
||||
asnOrg: null,
|
||||
connectionType: 'unknown',
|
||||
percentDaysSeen: null,
|
||||
riskNote,
|
||||
};
|
||||
}
|
||||
|
||||
export function createIpInfoChecker(ctx: IpInfoCheckerContext) {
|
||||
return async function checkIpInfo(ip: string): Promise<IpInfoAnonymousResult> {
|
||||
if (ctx.prescreen && (await ctx.prescreen(ip)) === 'skip') {
|
||||
return unavailableIpInfoAnonymousResult(ip, 'IPInfo skipped (local pre-screen)');
|
||||
}
|
||||
const result = await ctx.ipInfoService.lookup(ip, {
|
||||
source: 'risk.ipinfo_checker',
|
||||
reason: 'registration_risk',
|
||||
|
||||
@@ -344,7 +344,7 @@ describe('Message Search Permissions', () => {
|
||||
}
|
||||
expect(result.messages.length).toBe(0);
|
||||
});
|
||||
test('age-restricted guild requires include_nsfw: true', async () => {
|
||||
test('age-restricted guild is searchable by an adult member without include_nsfw', async () => {
|
||||
const owner = await createTestAccount(harness, {dateOfBirth: '2000-01-01'});
|
||||
const guild = await createGuild(harness, owner.token, 'Age Restricted Search Guild');
|
||||
const systemChannelId = guild.system_channel_id!;
|
||||
@@ -363,7 +363,7 @@ describe('Message Search Permissions', () => {
|
||||
if (!isSearchResult(excluded)) {
|
||||
expect.fail('Expected search result but got indexing response');
|
||||
}
|
||||
expect(excluded.messages.length).toBe(0);
|
||||
expect(excluded.messages.some((m) => m.channel_id === systemChannelId)).toBe(true);
|
||||
const included = await createBuilder<MessageSearchResponse>(harness, owner.token)
|
||||
.post('/search/messages')
|
||||
.body({
|
||||
@@ -379,6 +379,30 @@ describe('Message Search Permissions', () => {
|
||||
expect(included.messages.length).toBeGreaterThan(0);
|
||||
expect(included.messages.some((m) => m.channel_id === systemChannelId)).toBe(true);
|
||||
});
|
||||
test('age-restricted guild is searchable in a channel pinned to nsfw_override: false', async () => {
|
||||
const owner = await createTestAccount(harness, {dateOfBirth: '2000-01-01'});
|
||||
const guild = await createGuild(harness, owner.token, 'Age Restricted Override Guild');
|
||||
const channel = await createBuilder<{id: string; nsfw_override?: boolean | null}>(harness, owner.token)
|
||||
.post(`/guilds/${guild.id}/channels`)
|
||||
.body({name: 'override-channel', type: ChannelTypes.GUILD_TEXT, nsfw: false})
|
||||
.execute();
|
||||
expect(channel.nsfw_override).toBe(false);
|
||||
await sendChannelMessage(harness, owner.token, channel.id, 'age restricted override searchable message');
|
||||
await updateGuild(harness, owner.token, guild.id, {nsfw_level: GuildNSFWLevel.AGE_RESTRICTED});
|
||||
await markGuildChannelsAsIndexed(harness, owner.token, guild.id);
|
||||
const result = await createBuilder<MessageSearchResponse>(harness, owner.token)
|
||||
.post('/search/messages')
|
||||
.body({
|
||||
content: 'age restricted override searchable',
|
||||
context_guild_id: guild.id,
|
||||
})
|
||||
.expect(HTTP_STATUS.OK)
|
||||
.execute();
|
||||
if (!isSearchResult(result)) {
|
||||
expect.fail('Expected search result but got indexing response');
|
||||
}
|
||||
expect(result.messages.some((m) => m.channel_id === channel.id)).toBe(true);
|
||||
});
|
||||
test('underage user cannot search messages in an age-restricted guild', async () => {
|
||||
const owner = await createTestAccount(harness, {dateOfBirth: '2000-01-01'});
|
||||
const underageMember = await createTestAccount(harness, {dateOfBirth: '2012-01-01'});
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
import {resetGeoipReadersForTesting} from '@pkgs/geoip/src/GeoipLookup';
|
||||
import {resetSudoModeServiceForTesting} from '../auth/services/SudoModeService';
|
||||
import {resetSsoRequestUrlPolicyForTesting} from '../instance/SsoConfigValidation';
|
||||
import {resetGlobalLimitConfigServiceForTesting} from '../limits/LimitConfigService';
|
||||
@@ -32,6 +33,7 @@ export function resetServiceStateForTesting(): void {
|
||||
torExitListCache.shutdown();
|
||||
torExitListCache.clearForTesting();
|
||||
urlBlocklistCache.resetForTesting();
|
||||
resetGeoipReadersForTesting();
|
||||
fileShaCache.resetForTesting();
|
||||
phraseBlocklistCache.resetForTesting();
|
||||
bannedAvatarHashCache.resetForTesting();
|
||||
|
||||
@@ -0,0 +1,28 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
import {HttpResponse, http} from 'msw';
|
||||
|
||||
export function createIpInfoLookupHandler() {
|
||||
return http.get('https://api.ipinfo.io/lookup/:ip', ({params}) => {
|
||||
const ip = typeof params.ip === 'string' ? params.ip : '198.51.100.1';
|
||||
return HttpResponse.json({
|
||||
ip,
|
||||
geo: {
|
||||
city: 'Ashburn',
|
||||
region: 'Virginia',
|
||||
region_code: 'VA',
|
||||
country: 'United States',
|
||||
country_code: 'US',
|
||||
continent: 'North America',
|
||||
continent_code: 'NA',
|
||||
},
|
||||
as: {
|
||||
asn: 'AS64500',
|
||||
name: 'Test ISP',
|
||||
domain: 'example.com',
|
||||
type: 'isp',
|
||||
},
|
||||
anonymous: {},
|
||||
});
|
||||
});
|
||||
}
|
||||
@@ -2,6 +2,7 @@
|
||||
|
||||
import {setupServer} from 'msw/node';
|
||||
import {createBunnyEdgeHandlers} from './handlers/BunnyEdgeHandlers';
|
||||
import {createIpInfoLookupHandler} from './handlers/IpInfoHandlers';
|
||||
import {createNcmecHandlers} from './handlers/NcmecHandlers';
|
||||
import {createOnionooDetailsHandler} from './handlers/OnionooHandlers';
|
||||
import {createOpenNsfwHandlers} from './handlers/OpenNsfwHandlers';
|
||||
@@ -11,6 +12,7 @@ export const server = setupServer(
|
||||
...createBunnyEdgeHandlers(),
|
||||
...createNcmecHandlers(),
|
||||
...createOpenNsfwHandlers(),
|
||||
createIpInfoLookupHandler(),
|
||||
createOnionooDetailsHandler(),
|
||||
createPwnedPasswordsRangeHandler(),
|
||||
);
|
||||
|
||||
@@ -441,16 +441,25 @@ function collectGuildText(doc: SearchableGuild): Array<string | null> {
|
||||
return [doc.name, doc.vanityUrlCode, doc.discoveryDescription, ...doc.discoveryTags];
|
||||
}
|
||||
|
||||
const sortGuildsByCreatedAt = sortNumericField<SearchableGuild, GuildSearchFilters>('createdAt', 'asc');
|
||||
const sortGuildsByMemberCount = sortNumericField<SearchableGuild, GuildSearchFilters>('memberCount', 'desc');
|
||||
|
||||
function sortGuilds(left: SearchableGuild, right: SearchableGuild, filters: GuildSearchFilters, query: string): number {
|
||||
const sorter = filters.sortBy === 'memberCount' ? sortGuildsByMemberCount : sortGuildsByCreatedAt;
|
||||
const delta = sorter(left, right, filters, query);
|
||||
if (delta !== 0) return delta;
|
||||
const leftId = BigInt(left.id);
|
||||
const rightId = BigInt(right.id);
|
||||
if (leftId === rightId) return 0;
|
||||
return leftId > rightId ? -1 : 1;
|
||||
}
|
||||
|
||||
class InMemoryGuildSearchService
|
||||
extends InMemorySearchServiceBase<GuildSearchFilters, SearchableGuild>
|
||||
implements IGuildSearchService
|
||||
{
|
||||
constructor() {
|
||||
super(
|
||||
matchesGuildFilters,
|
||||
collectGuildText,
|
||||
sortNumericField<SearchableGuild, GuildSearchFilters>('createdAt', 'asc'),
|
||||
);
|
||||
super(matchesGuildFilters, collectGuildText, sortGuilds);
|
||||
}
|
||||
|
||||
async indexGuild(guild: Guild, discovery?: GuildDiscoveryContext): Promise<void> {
|
||||
|
||||
@@ -134,7 +134,7 @@ function suppressDeferredPhoneFlags(rawFlags: number): number {
|
||||
if ((rawFlags & DEFERRED_PHONE_ON_COMMUNITY_JOIN) === 0) {
|
||||
return rawFlags;
|
||||
}
|
||||
if (!getCachedDeferredPhoneGateEnabled()) {
|
||||
if (getCachedDeferredPhoneGateEnabled() === false) {
|
||||
return rawFlags & ~DEFERRED_PHONE_ON_COMMUNITY_JOIN;
|
||||
}
|
||||
return rawFlags & ~DEFERRABLE_PHONE_FLAGS;
|
||||
|
||||
@@ -123,9 +123,11 @@ export function UserContentController(app: HonoApp) {
|
||||
'Retrieves all messages saved by the current user. Messages are saved privately for easy reference. Returns paginated list of saved messages with metadata.',
|
||||
}),
|
||||
async (ctx) => {
|
||||
const {limit, before} = ctx.req.valid('query');
|
||||
const response = await ctx.get('userContentRequestService').listSavedMessages({
|
||||
userId: ctx.get('user').id,
|
||||
limit: ctx.req.valid('query').limit,
|
||||
limit,
|
||||
before: before ? createMessageID(before) : undefined,
|
||||
requestCache: ctx.get('requestCache'),
|
||||
});
|
||||
return ctx.json(response, 200);
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
import {ValidationErrorCodes} from '@fluxer/constants/src/ValidationErrorCodes';
|
||||
import {UnknownChannelError} from '@fluxer/errors/src/domains/channel/UnknownChannelError';
|
||||
import {InputValidationError} from '@fluxer/errors/src/domains/core/InputValidationError';
|
||||
import {type ChannelID, createUserID, type EntranceSoundID, type UserID} from '../../BrandedTypes';
|
||||
import type {IChannelRepository} from '../../channel/IChannelRepository';
|
||||
@@ -25,14 +24,10 @@ export class EntranceSoundPlayService {
|
||||
async play(params: PlayEntranceSoundParams): Promise<void> {
|
||||
const {userId, channelId, soundId} = params;
|
||||
const channel = await this.channelRepository.findUnique(channelId);
|
||||
if (!channel) {
|
||||
throw new UnknownChannelError();
|
||||
}
|
||||
const guildId = channel.guildId ?? undefined;
|
||||
const {voiceStates} = await this.gatewayService.getVoiceStatesForChannel({
|
||||
guildId,
|
||||
channelId,
|
||||
});
|
||||
const guildId = channel?.guildId ?? undefined;
|
||||
const voiceStates = channel
|
||||
? (await this.gatewayService.getVoiceStatesForChannel({guildId, channelId})).voiceStates
|
||||
: [];
|
||||
const senderInChannel = voiceStates.some((state) => state.userId === userId.toString());
|
||||
if (!senderInChannel) {
|
||||
throw InputValidationError.fromCode('channel_id', ValidationErrorCodes.ENTRANCE_SOUND_INVALID_SCOPE);
|
||||
|
||||
@@ -46,6 +46,7 @@ interface UserMentionsReadParams {
|
||||
interface SavedMessagesParams {
|
||||
userId: UserID;
|
||||
limit: number;
|
||||
before?: MessageID;
|
||||
requestCache: RequestCache;
|
||||
}
|
||||
|
||||
@@ -108,7 +109,11 @@ export class UserContentRequestService {
|
||||
}
|
||||
|
||||
async listSavedMessages(params: SavedMessagesParams): Promise<SavedMessageEntryListResponse> {
|
||||
const entries = await this.userContentService.getSavedMessages({userId: params.userId, limit: params.limit});
|
||||
const entries = await this.userContentService.getSavedMessages({
|
||||
userId: params.userId,
|
||||
limit: params.limit,
|
||||
before: params.before,
|
||||
});
|
||||
const messages = entries.map((entry) => entry.message).filter((message): message is Message => message != null);
|
||||
const responses = await this.userContentService.buildMessageResponsesForUser(params.userId, messages);
|
||||
const responseByMessageId = new Map(responses.map((response) => [response.id, response] as const));
|
||||
|
||||
@@ -100,6 +100,7 @@ function createUserContentService({
|
||||
}) {
|
||||
const batchCalls: Array<ChannelBatchCall> = [];
|
||||
const deletedSavedMessageIds: Array<string> = [];
|
||||
const savedMessageListCalls: Array<{limit?: number; before?: MessageID}> = [];
|
||||
const readableByChannel = new Map<string, Map<string, Message>>();
|
||||
for (const entry of readable) {
|
||||
const key = entry.channelId.toString();
|
||||
@@ -109,7 +110,10 @@ function createUserContentService({
|
||||
}
|
||||
const userRepository = {
|
||||
listRecentMentions: async () => entries,
|
||||
listSavedMessages: async () => entries,
|
||||
listSavedMessages: async (_userId: UserID, limit?: number, before?: MessageID) => {
|
||||
savedMessageListCalls.push({limit, before});
|
||||
return entries;
|
||||
},
|
||||
deleteSavedMessage: async (_userId: UserID, messageId: MessageID) => {
|
||||
deletedSavedMessageIds.push(messageId.toString());
|
||||
},
|
||||
@@ -146,7 +150,7 @@ function createUserContentService({
|
||||
{} as unknown as KVBulkMessageDeletionQueueService,
|
||||
{} as unknown as LimitConfigService,
|
||||
);
|
||||
return {service, batchCalls, deletedSavedMessageIds};
|
||||
return {service, batchCalls, deletedSavedMessageIds, savedMessageListCalls};
|
||||
}
|
||||
|
||||
const CHANNEL_A = createChannelID(100n);
|
||||
@@ -238,6 +242,15 @@ describe('getRecentMentions', () => {
|
||||
});
|
||||
|
||||
describe('getSavedMessages', () => {
|
||||
it('passes the page cursor to the repository', async () => {
|
||||
const entries = [{channelId: CHANNEL_A, messageId: createMessageID(11n)}];
|
||||
const {service, savedMessageListCalls} = createUserContentService({entries, readable: entries});
|
||||
|
||||
await service.getSavedMessages({userId: VIEWER_ID, limit: 50, before: createMessageID(20n)});
|
||||
|
||||
expect(savedMessageListCalls).toEqual([{limit: 50, before: createMessageID(20n)}]);
|
||||
});
|
||||
|
||||
it('marks every entry of an unreachable channel as missing permissions without deleting it', async () => {
|
||||
const entries = [
|
||||
{channelId: CHANNEL_A, messageId: createMessageID(11n)},
|
||||
|
||||
@@ -229,8 +229,16 @@ export class UserContentService {
|
||||
);
|
||||
}
|
||||
|
||||
async getSavedMessages({userId, limit}: {userId: UserID; limit: number}): Promise<Array<SavedMessageEntry>> {
|
||||
const savedMessages = await this.userRepository.listSavedMessages(userId, limit);
|
||||
async getSavedMessages({
|
||||
userId,
|
||||
limit,
|
||||
before,
|
||||
}: {
|
||||
userId: UserID;
|
||||
limit: number;
|
||||
before?: MessageID;
|
||||
}): Promise<Array<SavedMessageEntry>> {
|
||||
const savedMessages = await this.userRepository.listSavedMessages(userId, limit, before);
|
||||
const messagesByChannel = await this.readMessagesByChannel(userId, savedMessages);
|
||||
const results: Array<SavedMessageEntry> = [];
|
||||
const staleMessageIds: Array<MessageID> = [];
|
||||
|
||||
@@ -0,0 +1,44 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
import {afterEach, describe, expect, it} from 'vitest';
|
||||
import {setCachedDateOfBirthCollection} from '../instance/DateOfBirthCollectionCache';
|
||||
import {canUserAccessNsfwContent} from './AgeUtils';
|
||||
|
||||
const ADULT_DATE_OF_BIRTH = '1990-01-01';
|
||||
const MINOR_DATE_OF_BIRTH = '2020-01-01';
|
||||
|
||||
describe('canUserAccessNsfwContent', () => {
|
||||
afterEach(() => {
|
||||
setCachedDateOfBirthCollection(true);
|
||||
});
|
||||
|
||||
it('allows a bot whatever the instance collects', () => {
|
||||
setCachedDateOfBirthCollection(true);
|
||||
expect(canUserAccessNsfwContent({isBot: true, dateOfBirth: null})).toBe(true);
|
||||
});
|
||||
|
||||
it('allows an adult when the instance collects a date of birth', () => {
|
||||
setCachedDateOfBirthCollection(true);
|
||||
expect(canUserAccessNsfwContent({isBot: false, dateOfBirth: ADULT_DATE_OF_BIRTH})).toBe(true);
|
||||
});
|
||||
|
||||
it('blocks a minor when the instance collects a date of birth', () => {
|
||||
setCachedDateOfBirthCollection(true);
|
||||
expect(canUserAccessNsfwContent({isBot: false, dateOfBirth: MINOR_DATE_OF_BIRTH})).toBe(false);
|
||||
});
|
||||
|
||||
it('blocks a missing date of birth when the instance collects one', () => {
|
||||
setCachedDateOfBirthCollection(true);
|
||||
expect(canUserAccessNsfwContent({isBot: false, dateOfBirth: null})).toBe(false);
|
||||
});
|
||||
|
||||
it('allows a missing date of birth when the instance collects none', () => {
|
||||
setCachedDateOfBirthCollection(false);
|
||||
expect(canUserAccessNsfwContent({isBot: false, dateOfBirth: null})).toBe(true);
|
||||
});
|
||||
|
||||
it('allows an account with a minor date of birth when the instance collects none', () => {
|
||||
setCachedDateOfBirthCollection(false);
|
||||
expect(canUserAccessNsfwContent({isBot: false, dateOfBirth: MINOR_DATE_OF_BIRTH})).toBe(true);
|
||||
});
|
||||
});
|
||||
@@ -1,5 +1,7 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
import {instanceCollectsDateOfBirth} from '../instance/DateOfBirthCollectionCache';
|
||||
|
||||
export function calculateAge(
|
||||
dateOfBirth:
|
||||
| {
|
||||
@@ -53,5 +55,8 @@ export function canUserAccessNsfwContent(user: NsfwEligibilityUser): boolean {
|
||||
if (user.isBot) {
|
||||
return true;
|
||||
}
|
||||
if (!instanceCollectsDateOfBirth()) {
|
||||
return true;
|
||||
}
|
||||
return isUserAdult(user.dateOfBirth);
|
||||
}
|
||||
|
||||
@@ -102,6 +102,7 @@ async function parseWebhookMultipartMessageData(
|
||||
onPayloadParsed(payload) {
|
||||
parsedPayload = payload;
|
||||
},
|
||||
actor: 'webhook',
|
||||
},
|
||||
);
|
||||
if (!parsedPayload) {
|
||||
|
||||
@@ -586,7 +586,7 @@ export class WebhookService {
|
||||
type: 'external',
|
||||
url: avatarUrl,
|
||||
with_base64: true,
|
||||
nsfw: 'block',
|
||||
nsfw: 'allow',
|
||||
});
|
||||
if (!metadata?.base64) {
|
||||
await this.cacheService.set(cacheKey, WEBHOOK_AVATAR_MISSING_CACHE_VALUE, seconds('5 minutes'));
|
||||
|
||||
@@ -1,12 +1,14 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
import {Permissions} from '@fluxer/constants/src/ChannelConstants';
|
||||
import {afterEach, beforeEach, describe, expect, it} from 'vitest';
|
||||
import {createTestAccount} from '../../auth/tests/AuthTestUtils';
|
||||
import {loadFixture} from '../../channel/tests/AttachmentTestUtils';
|
||||
import {createGuild} from '../../guild/tests/GuildTestUtils';
|
||||
import {createPermissionOverwrite} from '../../channel/tests/ChannelTestUtils';
|
||||
import {acceptInvite, addMemberRole, createGuild, createRole} from '../../guild/tests/GuildTestUtils';
|
||||
import {type ApiTestHarness, createApiTestHarness} from '../../test/ApiTestHarness';
|
||||
import {HTTP_STATUS} from '../../test/TestConstants';
|
||||
import {createWebhook, deleteWebhook, executeWebhookWithAttachments} from './WebhookTestUtils';
|
||||
import {createChannelInvite, createWebhook, deleteWebhook, executeWebhookWithAttachments} from './WebhookTestUtils';
|
||||
|
||||
describe('Webhook multipart attachment uploads', () => {
|
||||
let harness: ApiTestHarness;
|
||||
@@ -92,6 +94,36 @@ describe('Webhook multipart attachment uploads', () => {
|
||||
expect(response.status).toBe(HTTP_STATUS.BAD_REQUEST);
|
||||
await deleteWebhook(harness, webhook.id, owner.token);
|
||||
});
|
||||
it('executes multipart webhook requests when the creator is denied attach files', async () => {
|
||||
const owner = await createTestAccount(harness);
|
||||
const creator = await createTestAccount(harness);
|
||||
const guild = await createGuild(harness, owner.token, 'Webhook creator denied attach files guild');
|
||||
const channelId = guild.system_channel_id!;
|
||||
const invite = await createChannelInvite(harness, owner.token, channelId);
|
||||
await acceptInvite(harness, creator.token, invite.code);
|
||||
const webhookManagerRole = await createRole(harness, owner.token, guild.id, {
|
||||
name: 'Webhook Manager',
|
||||
permissions: Permissions.MANAGE_WEBHOOKS.toString(),
|
||||
});
|
||||
await addMemberRole(harness, owner.token, guild.id, creator.userId, webhookManagerRole.id);
|
||||
const webhook = await createWebhook(harness, channelId, creator.token, 'Denied Creator Webhook');
|
||||
await createPermissionOverwrite(harness, owner.token, channelId, creator.userId, {
|
||||
type: 1,
|
||||
allow: '0',
|
||||
deny: Permissions.ATTACH_FILES.toString(),
|
||||
});
|
||||
const {response, json} = await executeWebhookWithAttachments(harness, {
|
||||
webhookId: webhook.id,
|
||||
webhookToken: webhook.token,
|
||||
payload: {
|
||||
attachments: [{id: 0, filename: 'denied_creator.png'}],
|
||||
},
|
||||
files: [{index: 0, filename: 'denied_creator.png', data: loadFixture('yeah.png')}],
|
||||
});
|
||||
expect(response.status).toBe(HTTP_STATUS.OK);
|
||||
expect(json?.attachments?.length).toBe(1);
|
||||
await deleteWebhook(harness, webhook.id, owner.token);
|
||||
});
|
||||
it('rejects multipart webhook requests when file indices do not match metadata IDs', async () => {
|
||||
const owner = await createTestAccount(harness);
|
||||
const guild = await createGuild(harness, owner.token, 'Webhook multipart id mismatch guild');
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
import {APIErrorCodes} from '@fluxer/constants/src/ApiErrorCodes';
|
||||
import {DELETED_USER_ID, DELETED_USER_USERNAME} from '@fluxer/constants/src/UserConstants';
|
||||
import {afterEach, beforeEach, describe, expect, it} from 'vitest';
|
||||
import {createTestAccount} from '../../auth/tests/AuthTestUtils';
|
||||
@@ -13,10 +12,6 @@ import {createWebhook, executeWebhook, executeWebhookWithAttachments, getChannel
|
||||
|
||||
const VANISHED_CREATOR_ID = createUserID(999999999999999997n);
|
||||
|
||||
function parseErrorCode(text: string): string | undefined {
|
||||
return (JSON.parse(text) as {code?: string}).code;
|
||||
}
|
||||
|
||||
describe('Webhook whose creating account cannot be resolved', () => {
|
||||
let harness: ApiTestHarness;
|
||||
beforeEach(async () => {
|
||||
@@ -49,13 +44,13 @@ describe('Webhook whose creating account cannot be resolved', () => {
|
||||
expect(listed?.user.id).toBe(VANISHED_CREATOR_ID.toString());
|
||||
expect(listed?.user.username).toBe(DELETED_USER_USERNAME);
|
||||
});
|
||||
it('answers a multipart execution for a webhook with no creator id with an access decision', async () => {
|
||||
it('executes a multipart payload for a webhook with no creator id', async () => {
|
||||
const owner = await createTestAccount(harness);
|
||||
const guild = await createGuild(harness, owner.token, 'Null creator multipart guild');
|
||||
const channelId = guild.system_channel_id!;
|
||||
const webhook = await createWebhook(harness, channelId, owner.token, 'Null Creator Multipart Webhook');
|
||||
await new WebhookRepository().update(createWebhookID(BigInt(webhook.id)), {creatorId: null});
|
||||
const {response, text} = await executeWebhookWithAttachments(harness, {
|
||||
const {response, json} = await executeWebhookWithAttachments(harness, {
|
||||
webhookId: webhook.id,
|
||||
webhookToken: webhook.token,
|
||||
payload: {
|
||||
@@ -63,16 +58,16 @@ describe('Webhook whose creating account cannot be resolved', () => {
|
||||
},
|
||||
files: [{index: 0, filename: 'orphaned.txt', data: Buffer.from('uploaded by an orphaned webhook')}],
|
||||
});
|
||||
expect(response.status).toBe(HTTP_STATUS.FORBIDDEN);
|
||||
expect(parseErrorCode(text)).toBe(APIErrorCodes.ACCESS_DENIED);
|
||||
expect(response.status).toBe(HTTP_STATUS.OK);
|
||||
expect(json?.attachments?.[0].filename).toBe('orphaned.txt');
|
||||
});
|
||||
it('answers a multipart execution for a webhook whose creator row is gone with an access decision', async () => {
|
||||
it('executes a multipart payload for a webhook whose creator row is gone', async () => {
|
||||
const owner = await createTestAccount(harness);
|
||||
const guild = await createGuild(harness, owner.token, 'Vanished creator multipart guild');
|
||||
const channelId = guild.system_channel_id!;
|
||||
const webhook = await createWebhook(harness, channelId, owner.token, 'Vanished Creator Multipart Webhook');
|
||||
await new WebhookRepository().update(createWebhookID(BigInt(webhook.id)), {creatorId: VANISHED_CREATOR_ID});
|
||||
const {response, text} = await executeWebhookWithAttachments(harness, {
|
||||
const {response, json} = await executeWebhookWithAttachments(harness, {
|
||||
webhookId: webhook.id,
|
||||
webhookToken: webhook.token,
|
||||
payload: {
|
||||
@@ -80,8 +75,8 @@ describe('Webhook whose creating account cannot be resolved', () => {
|
||||
},
|
||||
files: [{index: 0, filename: 'vanished.txt', data: Buffer.from('uploaded by a vanished creator')}],
|
||||
});
|
||||
expect(response.status).toBe(HTTP_STATUS.FORBIDDEN);
|
||||
expect(parseErrorCode(text)).toBe(APIErrorCodes.ACCESS_DENIED);
|
||||
expect(response.status).toBe(HTTP_STATUS.OK);
|
||||
expect(json?.attachments?.[0].filename).toBe('vanished.txt');
|
||||
});
|
||||
it('executes a json payload for a webhook with no creator id', async () => {
|
||||
const owner = await createTestAccount(harness);
|
||||
|
||||
@@ -2,7 +2,6 @@
|
||||
|
||||
import {DiscoveryApplicationStatus} from '@fluxer/constants/src/DiscoveryConstants';
|
||||
import type {WorkerTaskHandler} from '@pkgs/worker/src/contracts/WorkerTask';
|
||||
import type {GuildID} from '../../BrandedTypes';
|
||||
import {GuildDiscoveryRepository} from '../../guild/repositories/GuildDiscoveryRepository';
|
||||
import {getGuildSearchService} from '../../SearchFactory';
|
||||
import {mapWithConcurrency} from '../../utils/ConcurrencyUtils';
|
||||
@@ -17,7 +16,7 @@ const syncDiscoveryIndex: WorkerTaskHandler = async (_payload, helpers) => {
|
||||
helpers.logger.warn('Search service not available, skipping discovery index sync');
|
||||
return;
|
||||
}
|
||||
const {guildRepository, gatewayService} = getWorkerDependencies();
|
||||
const {guildRepository} = getWorkerDependencies();
|
||||
const discoveryRepository = new GuildDiscoveryRepository();
|
||||
const approvedRows = await discoveryRepository.listByStatus(DiscoveryApplicationStatus.APPROVED);
|
||||
if (approvedRows.length === 0) {
|
||||
@@ -25,21 +24,6 @@ const syncDiscoveryIndex: WorkerTaskHandler = async (_payload, helpers) => {
|
||||
return;
|
||||
}
|
||||
const guildIds = approvedRows.map((row) => row.guild_id);
|
||||
let freshCounts = new Map<
|
||||
GuildID,
|
||||
{
|
||||
memberCount: number;
|
||||
onlineCount: number;
|
||||
}
|
||||
>();
|
||||
try {
|
||||
freshCounts = await gatewayService.getDiscoveryGuildCounts(guildIds);
|
||||
} catch (error) {
|
||||
helpers.logger.warn(
|
||||
{error: error instanceof Error ? error.message : String(error)},
|
||||
'Failed to fetch fresh guild counts from gateway, using database values',
|
||||
);
|
||||
}
|
||||
let synced = 0;
|
||||
for (let i = 0; i < guildIds.length; i += BATCH_SIZE) {
|
||||
const batch = guildIds.slice(i, i + BATCH_SIZE);
|
||||
@@ -54,7 +38,7 @@ const syncDiscoveryIndex: WorkerTaskHandler = async (_payload, helpers) => {
|
||||
if (!guild) return null;
|
||||
const discoveryRow = discoveryRows[index];
|
||||
if (!discoveryRow || discoveryRow.status !== DiscoveryApplicationStatus.APPROVED) return null;
|
||||
return {guild, discoveryRow, counts: freshCounts.get(guildId)};
|
||||
return {guild, discoveryRow};
|
||||
})
|
||||
.filter((update): update is NonNullable<typeof update> => update != null);
|
||||
await mapWithConcurrency(updates, UPDATE_CONCURRENCY, (update) =>
|
||||
@@ -63,7 +47,6 @@ const syncDiscoveryIndex: WorkerTaskHandler = async (_payload, helpers) => {
|
||||
categoryId: update.discoveryRow.category_type,
|
||||
primaryLanguage: update.discoveryRow.primary_language ?? null,
|
||||
tags: update.discoveryRow.custom_tags ?? [],
|
||||
memberCount: update.counts?.memberCount,
|
||||
}),
|
||||
);
|
||||
synced += updates.length;
|
||||
|
||||
@@ -28,6 +28,12 @@ const MODULE_REGISTRY_TEST_FILES = [
|
||||
'src/api/risk/__tests__/AccountPolicyService.test.ts',
|
||||
];
|
||||
|
||||
const INSTANCE_POLICY_TEST_FILES = [
|
||||
'src/api/admin/tests/InstanceConfigPendingRegistrationApproval.test.ts',
|
||||
'src/api/auth/tests/DeferredPhoneGate.test.ts',
|
||||
'src/api/instance/tests/SingleCommunityService.test.ts',
|
||||
];
|
||||
|
||||
const sharedExclude = [
|
||||
...configDefaults.exclude,
|
||||
'pkgs/**',
|
||||
@@ -76,7 +82,7 @@ export default defineConfig({
|
||||
...sharedTestConfig,
|
||||
name: 'api',
|
||||
include: ['src/**/*.{test,spec}.{ts,tsx}'],
|
||||
exclude: [...sharedExclude, ...MODULE_REGISTRY_TEST_FILES],
|
||||
exclude: [...sharedExclude, ...MODULE_REGISTRY_TEST_FILES, ...INSTANCE_POLICY_TEST_FILES],
|
||||
isolate: false,
|
||||
},
|
||||
},
|
||||
@@ -90,6 +96,16 @@ export default defineConfig({
|
||||
isolate: true,
|
||||
},
|
||||
},
|
||||
{
|
||||
plugins: [tsconfigPaths()],
|
||||
test: {
|
||||
...sharedTestConfig,
|
||||
name: 'api-instance-policy',
|
||||
include: INSTANCE_POLICY_TEST_FILES,
|
||||
exclude: sharedExclude,
|
||||
isolate: true,
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
});
|
||||
|
||||
@@ -7,114 +7,65 @@
|
||||
|
||||
## Fluxer modifications
|
||||
|
||||
Changes applied on top of the upstream v2.17.2 source. Previously maintained as
|
||||
a pnpm patch at `patches/[email protected]`; now maintained as regular
|
||||
source edits in this package.
|
||||
Changes applied on top of the upstream v2.17.2 source. These were previously a pnpm patch at `patches/[email protected]` and are now plain source edits in this package.
|
||||
|
||||
1. **AV1 E2EE support** (`src/e2ee/worker/av1Crypto.ts`, `FrameCryptor.ts`, `e2ee.worker.ts`)
|
||||
OBU-level AV1 encryption and decryption for end-to-end encrypted voice/video.
|
||||
|
||||
OBU-level AV1 encryption and decryption for end-to-end encrypted voice and video.
|
||||
|
||||
2. **UpdateTrackContext message** (`src/e2ee/types.ts`, worker dispatch)
|
||||
Replaced `updateCodec` with richer `updateTrackContext` carrying participant
|
||||
identity and track ID, preventing codec mismatch on track reuse.
|
||||
|
||||
`updateCodec` replaced with `updateTrackContext`, which carries participant identity and track id so a reused track cannot pick up the wrong codec.
|
||||
|
||||
3. **E2EEManager state tracking** (`src/e2ee/E2eeManager.ts`)
|
||||
Added `getE2EETransformState()` / `setE2EETransformState()` for proper
|
||||
transform lifecycle management.
|
||||
|
||||
4. **Screenshare scalability mode** (`src/room/participant/LocalParticipant.ts`)
|
||||
Preserve caller-supplied `scalabilityMode` for screen shares instead of forcing
|
||||
`L3T3_KEY`, so VP9/AV1 screen shares can use the browser's compatible default
|
||||
unless Fluxer explicitly requests an SVC layer layout.
|
||||
`getE2EETransformState()` and `setE2EETransformState()` for transform lifecycle management.
|
||||
|
||||
4. **Screen share scalability mode** (`src/room/participant/LocalParticipant.ts`)
|
||||
|
||||
A caller-supplied `scalabilityMode` is preserved for screen shares instead of being forced to `L3T3_KEY`, so VP9 and AV1 screen shares use the browser default unless Fluxer asks for a specific SVC layout.
|
||||
|
||||
5. **E2EE frame layout guards** (`src/e2ee/worker/FrameCryptor.ts`)
|
||||
Validate encrypted frame trailer, IV, tag, and clear-prefix bounds before
|
||||
constructing typed-array views, and drop malformed encrypted frames without
|
||||
tearing down the transform stream.
|
||||
|
||||
Encrypted frame trailer, IV, tag and clear-prefix bounds are validated before any typed-array view is constructed, and malformed frames are dropped without tearing down the transform stream.
|
||||
|
||||
6. **Encrypted backup codec publishing** (`src/room/participant/LocalParticipant.ts`, `src/e2ee/E2eeManager.ts`)
|
||||
Allows backup codec tracks to be advertised and published while E2EE is
|
||||
enabled, and attaches sender transforms to backup codec senders using their
|
||||
cloned media track ID and codec.
|
||||
|
||||
Backup codec tracks can be advertised and published while E2EE is on, with sender transforms attached using the cloned media track id and codec.
|
||||
|
||||
7. **Publisher codec preferences** (`src/room/RTCEngine.ts`)
|
||||
Applies `RTCRtpTransceiver.setCodecPreferences()` to publisher transceivers
|
||||
so browser SDP follows the selected primary or backup codec, and prefers
|
||||
H.264 profiles that use Chromium's external/hardware encoder before the
|
||||
OpenH264 software profile.
|
||||
|
||||
`setCodecPreferences()` is applied to publisher transceivers so the browser's SDP follows the selected primary or backup codec. H.264 profiles rank Baseline `42001f` first, then Constrained Baseline `42e01f`, then everything else. Main, High and Constrained High rank last on purpose.
|
||||
|
||||
livekit-server registers H.264 High `640032` on the publisher peer connection but filters it off the subscriber peer connection, and its `CodecParametersFuzzySearch` falls back to a mime-only match. A High publication therefore reaches subscribers under their `42e01f` payload type and decodes to nothing on a Constrained-Baseline-only decoder such as Firefox's OpenH264 GMP. `42001f` is the one profile Chromium's accelerated encoder factory advertises that such a decoder can still handle, because Chromium's VAAPI encoder and OpenH264 both write a Constrained Baseline SPS for `H264PROFILE_BASELINE`.
|
||||
|
||||
The trade is that livekit-server does not register `42001f` either, so Windows and macOS negotiate `42e01f`, which Chromium's accelerated encoder factory does not advertise there (`kPlatformH264CbpEncoding` is off by default on Windows, and `IsH264ConstrainedBaselineProfileAvailableForAcceleratedEncoder` returns false on Apple). Those publishers fall back to software H.264, which is what an unpatched browser does anyway. Linux, ChromeOS and Android keep hardware encoding.
|
||||
|
||||
8. **Media publishing defaults** (`src/room/defaults.ts`, `src/room/utils.ts`, `src/room/track/options.ts`)
|
||||
Falls back to H.264, then VP9, VP8, AV1, and HEVC/H.265 according to actual
|
||||
sender capabilities, pairs advanced codecs with H.264 backup simulcast, and
|
||||
uses maintain-resolution screen-share defaults with a 4K60-ready bitrate cap.
|
||||
The order puts AV1 and HEVC last because both are opt-in in Fluxer, so a
|
||||
fallback inside `publishTrack` must not land on a codec the user did not
|
||||
enable. Fluxer picks the codec itself before publishing, so this list only
|
||||
applies when the client overrides the request, such as the reconnect
|
||||
republish that runs outside Fluxer's own flows.
|
||||
|
||||
Codec fallback follows actual sender capabilities in the order H.264, VP9, VP8, AV1, HEVC. Advanced codecs are paired with an H.264 backup simulcast, and screen shares default to maintain-resolution with a 4K60-ready bitrate cap. AV1 and HEVC come last because both are opt-in in Fluxer, so a fallback inside `publishTrack` must not land on a codec the user did not enable. Fluxer picks the codec itself before publishing, so this order only applies when the client overrides the request, such as the reconnect republish that runs outside Fluxer's own flows.
|
||||
|
||||
9. **High-fidelity Opus SDP munging** (`src/room/PCTransport.ts`)
|
||||
Forces Opus RED/FEC, stereo signaling, 10 ms packet time, no DTX, and a
|
||||
510 kbps maximum average bitrate in local offers and remote answers.
|
||||
|
||||
Local offers and remote answers are munged to force Opus RED and FEC, 10 ms packet time, no DTX, and a 510 kbps maximum average bitrate. Stereo signalling stays opt-in. `stereo=1` and `sprop-stereo=1` are added only for publications whose `TrackBitrateInfo.stereo` is set (studio mode above the stereo bitrate threshold, and screen-share audio) and for the subscriber mids the server advertised as stereo, so a mono microphone is not encoded and decoded as a two-channel stream.
|
||||
|
||||
10. **Remote audio volume restore at exactly zero** (`src/room/track/RemoteAudioTrack.ts`)
|
||||
`attach()`, `connectWebAudio()` and `getVolume()` guarded the remembered
|
||||
`elementVolume` with a truthiness check, so a track deliberately held at `0`
|
||||
came back at full volume whenever it was re-attached or its Web Audio graph
|
||||
was rebuilt. All three guards now test `!== undefined`. Note that remote
|
||||
gains above `1.0` are only legal because `setVolume()` takes the Web Audio
|
||||
`gainNode` branch; the `el.volume` branch would throw `IndexSizeError`.
|
||||
`webAudioMix` must stay unconditional.
|
||||
|
||||
`attach()`, `connectWebAudio()` and `getVolume()` guarded the remembered `elementVolume` with a truthiness check, so a track deliberately held at `0` came back at full volume whenever it was re-attached or its Web Audio graph was rebuilt. All three now test `!== undefined`. Remote gains above `1.0` are only legal because `setVolume()` takes the Web Audio `gainNode` branch, as the `el.volume` branch would throw `IndexSizeError`, so `webAudioMix` must stay unconditional.
|
||||
|
||||
11. **Processor teardown before source stop** (`src/room/track/LocalTrack.ts`)
|
||||
`stop()` called `super.stop()` first, killing the source `MediaStreamTrack`
|
||||
and closing the readable feeding a track processor before `processor.destroy()`
|
||||
ran. A camera-effect worker therefore saw input EOF before its owner's stop
|
||||
command and reported an operational failure during an ordinary camera-off.
|
||||
The processor is now captured, detached, and its teardown initiated before
|
||||
`super.stop()`.
|
||||
|
||||
12. **Transactional source and processor swaps** (`src/room/track/LocalTrack.ts`,
|
||||
`LocalVideoTrack.ts`, `LocalAudioTrack.ts`)
|
||||
`setMediaStreamTrack()` applied the new source, restarted the processor and
|
||||
re-armed the sender with no unwind path, so a failure anywhere in the middle
|
||||
left a half-applied track: listeners moved, elements detached, sender pointing
|
||||
at a dead track. It now takes `SetMediaStreamTrackOptions`
|
||||
(`force`, `deferEndedListener`, `preservePreviousTrack`) and, on failure,
|
||||
restores the previous source, constraints, `enabled` state, listeners,
|
||||
processor and sender, throwing `TrackInvalidError` when the previous source is
|
||||
no longer `live` because an ended track cannot be restored. Both errors are
|
||||
surfaced together as an `AggregateError` when the unwind itself fails.
|
||||
`stageTrackReplacement()` / `commitStagedTrackReplacement()` expose a two-phase
|
||||
swap: the candidate becomes the active source with its `ended` listener
|
||||
deferred and the previous source preserved, and only the commit adopts the
|
||||
`ended` listener and clears the staged identity, so a caller can validate its
|
||||
publication before the swap is observable. `replaceTrack()` and `restart()`
|
||||
guard the `providedByUser` flip behind a `replacementCommitted` flag.
|
||||
`restart()` still detaches and stops the previous source before calling
|
||||
`getUserMedia()`, as upstream does, because Safari ends a freshly acquired
|
||||
track with a capture failure while the old track for the same device is
|
||||
live. `setSimulcastTrackSender()` routes an already-installed processor's
|
||||
`processedTrack` to a newly registered secondary sender so a backup codec
|
||||
never publishes raw frames while the primary is processed.
|
||||
Processor install and teardown in all three classes roll the processed/raw
|
||||
sender track back, including `LocalVideoTrack`'s secondary simulcast senders,
|
||||
and aggregate every cleanup failure instead of discarding it.
|
||||
`stop()` called `super.stop()` first, killing the source `MediaStreamTrack` and closing the readable that feeds a track processor before `processor.destroy()` ran. A camera-effect worker therefore saw input EOF before its owner's stop command and reported an operational failure during an ordinary camera-off. The processor is now captured, detached and torn down before `super.stop()`.
|
||||
|
||||
13. **Start bitrate for every video codec** (`src/room/PCTransport.ts`,
|
||||
`src/room/participant/LocalParticipant.ts`, `src/room/participant/publishUtils.ts`)
|
||||
`x-google-start-bitrate` was reachable only by AV1 and VP9, gated twice: the
|
||||
publish path registered a track bitrate only for SVC codecs, and the offer
|
||||
munging returned early for everything else. H264, H265 and VP8 therefore
|
||||
opened at the Chromium default and had to ramp, which showed up as a 3000 kbps
|
||||
screen share encoding at 346 kbps twenty seconds in. The bitrate is now
|
||||
registered for every video codec from the highest encoding
|
||||
(`maxEncodingBitrate()`, so a simulcast ladder contributes its top layer), and
|
||||
the offer munging applies the start bitrate whenever a max bitrate is known.
|
||||
The dependency descriptor extension stays SVC-only.
|
||||
`appendStartBitrateToFmtp()` holds the fmtp edit so it can be tested, and
|
||||
`setTrackCodecBitrate()` now replaces an entry for the same cid or transceiver
|
||||
instead of appending, since `trackBitrates` is never cleared.
|
||||
12. **Transactional source and processor swaps** (`src/room/track/LocalTrack.ts`, `LocalVideoTrack.ts`, `LocalAudioTrack.ts`)
|
||||
|
||||
`setMediaStreamTrack()` applied the new source, restarted the processor and re-armed the sender with no unwind path, so a failure part-way through left a half-applied track with listeners moved, elements detached and the sender pointing at a dead track. It now takes `SetMediaStreamTrackOptions` (`force`, `deferEndedListener`, `preservePreviousTrack`) and, on failure, restores the previous source, constraints, `enabled` state, listeners, processor and sender. It throws `TrackInvalidError` when the previous source is no longer `live`, because an ended track cannot be restored, and surfaces both failures as an `AggregateError` when the unwind itself fails.
|
||||
|
||||
`stageTrackReplacement()` and `commitStagedTrackReplacement()` add a two-phase swap. The candidate becomes the active source with its `ended` listener deferred and the previous source preserved, and only the commit adopts the `ended` listener and clears the staged identity, so a caller can validate its publication before the swap is observable. `replaceTrack()` and `restart()` guard the `providedByUser` flip behind a `replacementCommitted` flag. `restart()` still detaches and stops the previous source before calling `getUserMedia()`, as upstream does, because Safari ends a freshly acquired track with a capture failure while the old track for the same device is still live. `setSimulcastTrackSender()` routes an installed processor's `processedTrack` to a newly registered secondary sender so a backup codec never publishes raw frames while the primary is processed. Processor install and teardown in all three classes roll the processed and raw sender track back, including `LocalVideoTrack`'s secondary simulcast senders, and aggregate every cleanup failure instead of discarding it.
|
||||
|
||||
13. **Start bitrate for every video codec** (`src/room/PCTransport.ts`, `src/room/participant/LocalParticipant.ts`, `src/room/participant/publishUtils.ts`)
|
||||
|
||||
`x-google-start-bitrate` was reachable only by AV1 and VP9 because it was gated twice. The publish path registered a track bitrate only for SVC codecs, and the offer munging returned early for everything else. H.264, H.265 and VP8 therefore opened at the Chromium default and had to ramp, which showed up as a 3000 kbps screen share encoding at 346 kbps twenty seconds in. The bitrate is now registered for every video codec from the highest encoding (`maxEncodingBitrate()`, so a simulcast ladder contributes its top layer), and the offer munging applies it whenever a max bitrate is known. The dependency descriptor extension stays SVC-only. `appendStartBitrateToFmtp()` holds the fmtp edit so it can be tested, and `setTrackCodecBitrate()` replaces an entry for the same cid or transceiver instead of appending, since `trackBitrates` is never cleared.
|
||||
|
||||
## Updating from upstream
|
||||
|
||||
|
||||
@@ -3,12 +3,43 @@
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
import type {MediaDescription} from 'sdp-transform';
|
||||
import {describe, expect, it} from 'vitest';
|
||||
import {appendStartBitrateToFmtp} from './PCTransport.ts';
|
||||
import type {TrackBitrateInfo} from './PCTransport.ts';
|
||||
import {appendStartBitrateToFmtp, collectStereoMids, ensureAudioNackAndStereo, ensureOpusFmtp} from './PCTransport.ts';
|
||||
|
||||
function mediaWithFmtp(entries: Array<{payload: number; config: string}>): MediaDescription {
|
||||
return {fmtp: entries} as unknown as MediaDescription;
|
||||
}
|
||||
|
||||
function opusMedia(config: string, mid = '0'): MediaDescription {
|
||||
return {
|
||||
type: 'audio',
|
||||
mid,
|
||||
port: 9,
|
||||
protocol: 'UDP/TLS/RTP/SAVPF',
|
||||
rtp: [{payload: 109, codec: 'opus', rate: 48000, encoding: 2}],
|
||||
fmtp: [{payload: 109, config}],
|
||||
} as unknown as MediaDescription;
|
||||
}
|
||||
|
||||
function offerMedia(mid: string, trackId: string): MediaDescription {
|
||||
const media = opusMedia('useinbandfec=1', mid);
|
||||
media.msid = `- ${trackId}`;
|
||||
return media;
|
||||
}
|
||||
|
||||
function audioBitrateInfo(mid: string | null, trackId: string, stereo: boolean): TrackBitrateInfo {
|
||||
return {
|
||||
transceiver: {mid, sender: {track: {id: trackId}}} as unknown as RTCRtpTransceiver,
|
||||
codec: 'opus',
|
||||
maxbr: 320,
|
||||
stereo,
|
||||
};
|
||||
}
|
||||
|
||||
function opusConfig(media: MediaDescription): string {
|
||||
return media.fmtp.find((fmtp) => fmtp.payload === 109)?.config ?? '';
|
||||
}
|
||||
|
||||
describe('appendStartBitrateToFmtp', () => {
|
||||
it('appends the start bitrate to a non-SVC codec fmtp line', () => {
|
||||
const media = mediaWithFmtp([
|
||||
@@ -44,3 +75,72 @@ describe('appendStartBitrateToFmtp', () => {
|
||||
expect(media.fmtp[0]?.config).toBe('profile-level-id=42e01f');
|
||||
});
|
||||
});
|
||||
|
||||
describe('ensureOpusFmtp', () => {
|
||||
it('does not force stereo on a mono publication', () => {
|
||||
const media = opusMedia('maxplaybackrate=48000;stereo=0;useinbandfec=1');
|
||||
ensureOpusFmtp(media, 48000, false);
|
||||
const config = opusConfig(media);
|
||||
expect(config).toContain('minptime=10');
|
||||
expect(config).toContain('useinbandfec=1');
|
||||
expect(config).toContain('usedtx=0');
|
||||
expect(config).toContain('maxaveragebitrate=48000');
|
||||
expect(config).not.toContain('stereo=1');
|
||||
});
|
||||
|
||||
it('keeps stereo for a stereo publication', () => {
|
||||
const media = opusMedia('maxplaybackrate=48000;useinbandfec=1');
|
||||
ensureOpusFmtp(media, 320000, true);
|
||||
const config = opusConfig(media);
|
||||
expect(config).toContain('stereo=1');
|
||||
expect(config).toContain('sprop-stereo=1');
|
||||
expect(config).toContain('maxaveragebitrate=320000');
|
||||
});
|
||||
|
||||
it('preserves a stereo parameter the server negotiated', () => {
|
||||
const media = opusMedia('minptime=10;stereo=1');
|
||||
ensureOpusFmtp(media, 48000, false);
|
||||
expect(opusConfig(media)).toContain('stereo=1');
|
||||
});
|
||||
});
|
||||
|
||||
describe('ensureAudioNackAndStereo', () => {
|
||||
it('only stamps stereo on the listed mids', () => {
|
||||
const mono = opusMedia('useinbandfec=1', '0');
|
||||
ensureAudioNackAndStereo(mono as never, ['1'], []);
|
||||
expect(opusConfig(mono)).not.toContain('stereo=1');
|
||||
|
||||
const stereo = opusMedia('useinbandfec=1', '1');
|
||||
ensureAudioNackAndStereo(stereo as never, ['1'], []);
|
||||
expect(opusConfig(stereo)).toContain('stereo=1');
|
||||
expect(opusConfig(stereo)).toContain('sprop-stereo=1');
|
||||
});
|
||||
});
|
||||
|
||||
describe('collectStereoMids', () => {
|
||||
it('matches the offer media section by msid before the transceiver has a mid', () => {
|
||||
const media = [offerMedia('0', 'mic-track'), offerMedia('1', 'screenshare-track')];
|
||||
expect(collectStereoMids([audioBitrateInfo(null, 'screenshare-track', true)], media)).toEqual(['1']);
|
||||
});
|
||||
|
||||
it('stamps stereo on the first offer for a new stereo publication', () => {
|
||||
const media = [offerMedia('0', 'mic-track'), offerMedia('1', 'screenshare-track')];
|
||||
const stereoMids = collectStereoMids([audioBitrateInfo(null, 'screenshare-track', true)], media);
|
||||
for (const m of media) {
|
||||
ensureAudioNackAndStereo(m as never, stereoMids, []);
|
||||
}
|
||||
expect(opusConfig(media[0]!)).not.toContain('stereo=1');
|
||||
expect(opusConfig(media[1]!)).toContain('stereo=1');
|
||||
expect(opusConfig(media[1]!)).toContain('sprop-stereo=1');
|
||||
});
|
||||
|
||||
it('uses the assigned mid once renegotiation has one', () => {
|
||||
const media = [offerMedia('0', 'mic-track'), offerMedia('1', 'screenshare-track')];
|
||||
expect(collectStereoMids([audioBitrateInfo('1', 'screenshare-track', true)], media)).toEqual(['1']);
|
||||
});
|
||||
|
||||
it('leaves mono publications out', () => {
|
||||
const media = [offerMedia('0', 'mic-track')];
|
||||
expect(collectStereoMids([audioBitrateInfo(null, 'mic-track', false)], media)).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -11,11 +11,12 @@ import {NegotiationError, UnexpectedConnectionState} from './errors.ts';
|
||||
import type {LoggerOptions} from './types.ts';
|
||||
import {ddExtensionURI, isFireFox, isSafari, isSVCCodec} from './utils.ts';
|
||||
|
||||
interface TrackBitrateInfo {
|
||||
export interface TrackBitrateInfo {
|
||||
cid?: string;
|
||||
transceiver?: RTCRtpTransceiver;
|
||||
codec: string;
|
||||
maxbr: number;
|
||||
stereo?: boolean;
|
||||
}
|
||||
|
||||
const startBitrateFraction = 0.7;
|
||||
@@ -25,8 +26,6 @@ const requiredOpusFmtpParameters = {
|
||||
minptime: '10',
|
||||
useinbandfec: '1',
|
||||
usedtx: '0',
|
||||
stereo: '1',
|
||||
'sprop-stereo': '1',
|
||||
};
|
||||
const debounceInterval = 20;
|
||||
export const PCEvents = {
|
||||
@@ -156,20 +155,18 @@ export default class PCTransport extends EventEmitter {
|
||||
sdpParsed.media.forEach((media) => {
|
||||
const mid = getMidString(media.mid!);
|
||||
if (media.type === 'audio') {
|
||||
ensureOpusFmtp(media);
|
||||
this.trackBitrates.some((trackbr): boolean => {
|
||||
if (!trackbr.transceiver || mid !== trackbr.transceiver.mid) {
|
||||
return false;
|
||||
}
|
||||
const codecPayload = getCodecPayload(media, trackbr.codec);
|
||||
if (codecPayload === 0) {
|
||||
return true;
|
||||
}
|
||||
if (trackbr.codec.toLowerCase() === 'opus') {
|
||||
ensureOpusFmtp(media, trackbr.maxbr > 0 ? trackbr.maxbr * 1000 : opusMaxAverageBitrateBps);
|
||||
}
|
||||
return true;
|
||||
});
|
||||
const trackbr = this.trackBitrates.find(
|
||||
(br) => br.transceiver !== undefined && mid === br.transceiver.mid && br.codec.toLowerCase() === 'opus',
|
||||
);
|
||||
if (trackbr && getCodecPayload(media, trackbr.codec) !== 0) {
|
||||
ensureOpusFmtp(
|
||||
media,
|
||||
trackbr.maxbr > 0 ? trackbr.maxbr * 1000 : opusMaxAverageBitrateBps,
|
||||
trackbr.stereo === true,
|
||||
);
|
||||
} else {
|
||||
ensureOpusFmtp(media);
|
||||
}
|
||||
}
|
||||
});
|
||||
mungedSDP = write(sdpParsed);
|
||||
@@ -240,10 +237,11 @@ export default class PCTransport extends EventEmitter {
|
||||
const offer = await this.pc.createOffer(options);
|
||||
this.log.debug('original offer', {sdp: offer.sdp, ...this.logContext});
|
||||
const sdpParsed = parse(offer.sdp ?? '');
|
||||
const stereoMids = collectStereoMids(this.trackBitrates, sdpParsed.media);
|
||||
sdpParsed.media.forEach((media) => {
|
||||
ensureIPAddrMatchVersion(media);
|
||||
if (media.type === 'audio') {
|
||||
ensureAudioNackAndStereo(media, ['all'], []);
|
||||
ensureAudioNackAndStereo(media, stereoMids, []);
|
||||
} else if (media.type === 'video') {
|
||||
this.trackBitrates.some((trackbr): boolean => {
|
||||
if (!media.msid || !trackbr.cid || !media.msid.includes(trackbr.cid)) {
|
||||
@@ -587,7 +585,11 @@ function ensureAudioRedFmtp(media: MediaDescription, opusPayload: number): void
|
||||
}
|
||||
}
|
||||
|
||||
function ensureOpusFmtp(media: MediaDescription, maxAverageBitrateBps: number = opusMaxAverageBitrateBps): number {
|
||||
export function ensureOpusFmtp(
|
||||
media: MediaDescription,
|
||||
maxAverageBitrateBps: number = opusMaxAverageBitrateBps,
|
||||
stereo = false,
|
||||
): number {
|
||||
const opusPayload = getCodecPayload(media, 'opus');
|
||||
if (opusPayload <= 0) return 0;
|
||||
media.ptime = opusPacketTimeMs;
|
||||
@@ -596,6 +598,10 @@ function ensureOpusFmtp(media: MediaDescription, maxAverageBitrateBps: number =
|
||||
for (const [key, value] of Object.entries(requiredOpusFmtpParameters)) {
|
||||
config = setFmtpParameter(config, key, value);
|
||||
}
|
||||
if (stereo) {
|
||||
config = setFmtpParameter(config, 'stereo', '1');
|
||||
config = setFmtpParameter(config, 'sprop-stereo', '1');
|
||||
}
|
||||
if (maxAverageBitrateBps > 0) {
|
||||
config = setFmtpParameter(config, 'maxaveragebitrate', String(maxAverageBitrateBps));
|
||||
}
|
||||
@@ -604,18 +610,18 @@ function ensureOpusFmtp(media: MediaDescription, maxAverageBitrateBps: number =
|
||||
return opusPayload;
|
||||
}
|
||||
|
||||
function ensureAudioNackAndStereo(
|
||||
export function ensureAudioNackAndStereo(
|
||||
media: {
|
||||
type: string;
|
||||
port: number;
|
||||
protocol: string;
|
||||
payloads?: string | undefined;
|
||||
} & MediaDescription,
|
||||
_stereoMids: Array<string>,
|
||||
stereoMids: Array<string>,
|
||||
nackMids: Array<string>,
|
||||
) {
|
||||
const mid = getMidString(media.mid!);
|
||||
const opusPayload = ensureOpusFmtp(media);
|
||||
const opusPayload = ensureOpusFmtp(media, opusMaxAverageBitrateBps, stereoMids.includes(mid));
|
||||
if (opusPayload > 0) {
|
||||
if (!media.rtcpFb) {
|
||||
media.rtcpFb = [];
|
||||
@@ -629,6 +635,32 @@ function ensureAudioNackAndStereo(
|
||||
}
|
||||
}
|
||||
|
||||
export function collectStereoMids(
|
||||
trackBitrates: Array<TrackBitrateInfo>,
|
||||
media: Array<MediaDescription>,
|
||||
): Array<string> {
|
||||
const stereoMids: Array<string> = [];
|
||||
for (const trackbr of trackBitrates) {
|
||||
if (trackbr.stereo !== true || !trackbr.transceiver) {
|
||||
continue;
|
||||
}
|
||||
if (trackbr.transceiver.mid) {
|
||||
stereoMids.push(getMidString(trackbr.transceiver.mid));
|
||||
continue;
|
||||
}
|
||||
const trackId = trackbr.transceiver.sender.track?.id;
|
||||
if (trackId === undefined) {
|
||||
continue;
|
||||
}
|
||||
for (const m of media) {
|
||||
if (m.type === 'audio' && m.mid !== undefined && m.msid?.includes(trackId)) {
|
||||
stereoMids.push(getMidString(m.mid));
|
||||
}
|
||||
}
|
||||
}
|
||||
return stereoMids;
|
||||
}
|
||||
|
||||
function extractStereoAndNackAudioFromOffer(offer: RTCSessionDescriptionInit): {
|
||||
stereoMids: Array<string>;
|
||||
nackMids: Array<string>;
|
||||
|
||||
@@ -27,7 +27,59 @@ describe('selectPublisherCodecPreferences', () => {
|
||||
const highProfile = codec('video/H264', 'level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=64001f');
|
||||
const rtx = codec('video/rtx');
|
||||
const preferences = selectPublisherCodecPreferences('h264', [openH264, rtx, externalBaseline, highProfile]);
|
||||
expect(preferences).toEqual([externalBaseline, highProfile, openH264, rtx]);
|
||||
expect(preferences).toEqual([externalBaseline, openH264, highProfile, rtx]);
|
||||
});
|
||||
|
||||
it('ranks Constrained Baseline above Main, High and Constrained High', () => {
|
||||
const constrainedBaseline = codec(
|
||||
'video/H264',
|
||||
'level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42e01f',
|
||||
);
|
||||
const mainProfile = codec('video/H264', 'level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=4d001f');
|
||||
const highProfileLevel31 = codec(
|
||||
'video/H264',
|
||||
'level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=64001f',
|
||||
);
|
||||
const highProfileLevel51 = codec(
|
||||
'video/H264',
|
||||
'level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=640033',
|
||||
);
|
||||
const constrainedHigh = codec(
|
||||
'video/H264',
|
||||
'level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=640c1f',
|
||||
);
|
||||
const preferences = selectPublisherCodecPreferences('h264', [
|
||||
mainProfile,
|
||||
highProfileLevel31,
|
||||
highProfileLevel51,
|
||||
constrainedHigh,
|
||||
constrainedBaseline,
|
||||
]);
|
||||
expect(preferences).toEqual([
|
||||
constrainedBaseline,
|
||||
mainProfile,
|
||||
highProfileLevel31,
|
||||
highProfileLevel51,
|
||||
constrainedHigh,
|
||||
]);
|
||||
});
|
||||
|
||||
it('keeps Constrained Baseline packetization-mode=1 ahead of Constrained Baseline packetization-mode=0 and High', () => {
|
||||
const constrainedBaselineMode0 = codec(
|
||||
'video/H264',
|
||||
'level-asymmetry-allowed=1;packetization-mode=0;profile-level-id=42e01f',
|
||||
);
|
||||
const constrainedBaselineMode1 = codec(
|
||||
'video/H264',
|
||||
'level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42e01f',
|
||||
);
|
||||
const highProfile = codec('video/H264', 'level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=640033');
|
||||
const preferences = selectPublisherCodecPreferences('h264', [
|
||||
highProfile,
|
||||
constrainedBaselineMode0,
|
||||
constrainedBaselineMode1,
|
||||
]);
|
||||
expect(preferences).toEqual([constrainedBaselineMode1, constrainedBaselineMode0, highProfile]);
|
||||
});
|
||||
|
||||
it('keeps non-H.264 codecs in browser capability order and appends RTX', () => {
|
||||
|
||||
@@ -96,7 +96,7 @@ const videoCodecMimeTypes: Record<VideoCodec, Array<string>> = {
|
||||
vp8: ['video/vp8'],
|
||||
};
|
||||
const h264OpenH264ProfileLevelId = '42e01f';
|
||||
const h264PreferredHardwareProfileLevelIds = new Set(['42001f', '4d001f', '64001f']);
|
||||
const h264PreferredHardwareProfileLevelIds = new Set(['42001f']);
|
||||
type RtpCodecCapability = RTCRtpCapabilities['codecs'][number] & {sdpFmtpLine?: string};
|
||||
|
||||
enum PCState {
|
||||
@@ -1656,8 +1656,8 @@ function getH264PublisherCodecScore(codec: RtpCodecCapability): number {
|
||||
const packetizationMode = getFmtpParameter(codec.sdpFmtpLine, 'packetization-mode');
|
||||
const packetizationScore = packetizationMode === '1' ? 0 : 1;
|
||||
if (profileLevelId && h264PreferredHardwareProfileLevelIds.has(profileLevelId)) return packetizationScore;
|
||||
if (profileLevelId && profileLevelId !== h264OpenH264ProfileLevelId) return 10 + packetizationScore;
|
||||
if (profileLevelId === h264OpenH264ProfileLevelId) return 20 + packetizationScore;
|
||||
if (profileLevelId === h264OpenH264ProfileLevelId) return 10 + packetizationScore;
|
||||
if (profileLevelId) return 20 + packetizationScore;
|
||||
return 30 + packetizationScore;
|
||||
}
|
||||
|
||||
|
||||
@@ -1040,6 +1040,7 @@ export default class LocalParticipant extends Participant {
|
||||
transceiver: trackTransceiver,
|
||||
codec: 'opus',
|
||||
maxbr: encodings[0]?.maxBitrate ? encodings[0].maxBitrate / 1000 : 0,
|
||||
stereo: isStereo,
|
||||
});
|
||||
}
|
||||
} else if (track.codec) {
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user