fix(push): stop pushes for read, silent and muted messages (#3033)

fluxer_push/src/dedupe.rs

@@ -19,7 +19,16 @@ impl JobKey {
     }
 
     pub fn of_clear(job: &ClearJob) -> Self {
-        Self::digest(&["clear", &job.user_id, &job.channel_id, &job.message_id])
+        match &job.after_message_id {
+            Some(after_message_id) => Self::digest(&[
+                "clear",
+                &job.user_id,
+                &job.channel_id,
+                &job.message_id,
+                after_message_id,
+            ]),
+            None => Self::digest(&["clear", &job.user_id, &job.channel_id, &job.message_id]),
+        }
     }
 
     pub fn of_ring(job: &RingJob) -> Self {

fluxer_push/src/job.rs

@@ -40,6 +40,8 @@ pub struct ClearJob {
     pub user_id: String,
     pub channel_id: String,
     pub message_id: String,
+    #[serde(default)]
+    pub after_message_id: Option<String>,
 }
 
 #[derive(Clone, Debug, Deserialize, Eq, PartialEq)]

fluxer_push/src/metrics.rs

@@ -277,15 +277,17 @@ impl DeliveryRoute {
 #[repr(usize)]
 pub enum BucketKey {
     DeviceToken,
+    BackgroundDeviceToken,
     Source,
 }
 
 impl BucketKey {
-    pub const ALL: [Self; 2] = [Self::DeviceToken, Self::Source];
+    pub const ALL: [Self; 3] = [Self::DeviceToken, Self::BackgroundDeviceToken, Self::Source];
 
     pub fn label(self) -> &'static str {
         match self {
             Self::DeviceToken => "device_token",
+            Self::BackgroundDeviceToken => "background_device_token",
             Self::Source => "source",
         }
     }
@@ -354,7 +356,7 @@ pub struct Metrics {
     sends: [[AtomicU64; SEND_RESULT_COUNT]; PROVIDER_COUNT],
     token_deletions: [AtomicU64; PROVIDER_COUNT],
     payload_shrinks: [AtomicU64; PAYLOAD_SHRINK_COUNT],
-    own_relay_shortcuts: AtomicU64,
+    own_relay_shortcuts: [AtomicU64; RELAY_LEG_COUNT],
     auth_tokens_minted: [AtomicU64; AUTH_PROVIDER_COUNT],
     rpc_requests: [[AtomicU64; RPC_OUTCOME_COUNT]; RPC_METHOD_COUNT],
     relay_consent_updates: [AtomicU64; CONSENT_UPDATE_COUNT],
@@ -383,7 +385,7 @@ impl Metrics {
             sends: [const { [const { AtomicU64::new(0) }; SEND_RESULT_COUNT] }; PROVIDER_COUNT],
             token_deletions: [const { AtomicU64::new(0) }; PROVIDER_COUNT],
             payload_shrinks: [const { AtomicU64::new(0) }; PAYLOAD_SHRINK_COUNT],
-            own_relay_shortcuts: AtomicU64::new(0),
+            own_relay_shortcuts: [const { AtomicU64::new(0) }; RELAY_LEG_COUNT],
             auth_tokens_minted: [const { AtomicU64::new(0) }; AUTH_PROVIDER_COUNT],
             rpc_requests: [const { [const { AtomicU64::new(0) }; RPC_OUTCOME_COUNT] };
                 RPC_METHOD_COUNT],
@@ -436,8 +438,8 @@ impl Metrics {
         self.token_deletions[provider as usize].fetch_add(1, ORDERING);
     }
 
-    pub fn record_own_relay_shortcut(&self) {
-        self.own_relay_shortcuts.fetch_add(1, ORDERING);
+    pub fn record_own_relay_shortcut(&self, leg: RelayLeg) {
+        self.own_relay_shortcuts[leg as usize].fetch_add(1, ORDERING);
     }
 
     pub fn record_payload_shrink(&self, step: PayloadShrink) {
@@ -537,9 +539,11 @@ impl Metrics {
             Provider::ALL.map(Provider::label),
             &self.token_deletions,
         )?;
-        render_counter(
+        render_labelled_counter(
             out,
             "fluxer_push_own_relay_shortcuts_total",
+            "leg",
+            RelayLeg::ALL.map(RelayLeg::label),
             &self.own_relay_shortcuts,
         )?;
         render_labelled_counter(

fluxer_push/src/payload.rs

@@ -628,6 +628,7 @@ mod tests {
             user_id: USER_ID.to_owned(),
             channel_id: CHANNEL_ID.to_owned(),
             message_id: MESSAGE_ID.to_owned(),
+            after_message_id: None,
         }
     }
 

fluxer_push/src/providers/mod.rs

@@ -5,7 +5,8 @@ pub mod fcm;
 pub mod own_relay;
 pub mod web_push;
 
-use crate::metrics::{DeliveryRoute, Provider, SendResult, elapsed_ms};
+use crate::metrics::{DeliveryRoute, Provider, RelayLeg, SendResult, elapsed_ms};
+use crate::relay;
 use crate::server::AppState;
 use crate::subscription::{Platform, Subscription};
 use crate::vendor::VendorOutcome;
@@ -96,11 +97,24 @@ pub async fn send(state: &AppState, sub: &Subscription, envelope: &Value) -> Sen
     } else {
         let direct = in_process_hop(&sub.endpoint, &state.cfg.own_relay_hosts);
         match (route, direct) {
-            (Route::WebPush, Some(hop)) => {
+            (Route::WebPush, Some(hop)) if hop.leg == own_relay::Leg::Apns => {
                 let hopped = hop.as_subscription(sub);
-                state.metrics.record_own_relay_shortcut();
+                state.metrics.record_own_relay_shortcut(RelayLeg::Apns);
                 apns::send(state, &hopped, envelope).await
             }
+            (Route::WebPush, Some(hop)) => {
+                match relay::FcmTarget::resolve(
+                    state.cfg.fcm.as_ref(),
+                    &hop.app_id,
+                    &hop.device_token,
+                ) {
+                    Some(target) => {
+                        state.metrics.record_own_relay_shortcut(RelayLeg::Fcm);
+                        web_push::send_to_own_fcm_relay(state, sub, envelope, target).await
+                    }
+                    None => web_push::send(state, sub, envelope).await,
+                }
+            }
             (Route::WebPush, None) => web_push::send(state, sub, envelope).await,
             (Route::LegacyApns, _) => apns::send(state, sub, envelope).await,
             (Route::LegacyFcm, _) => fcm::send(state, sub, envelope).await,
@@ -123,7 +137,8 @@ fn relay_consent_missing(state: &AppState, endpoint: &str) -> bool {
 }
 
 fn in_process_hop(endpoint: &str, hosts: &[String]) -> Option<own_relay::Hop> {
-    own_relay::parse(endpoint, hosts).filter(|hop| matches!(hop.leg, own_relay::Leg::Apns))
+    own_relay::parse(endpoint, hosts)
+        .filter(|hop| matches!(hop.leg, own_relay::Leg::Apns | own_relay::Leg::Fcm))
 }
 
 fn route_label(route: Route) -> DeliveryRoute {
@@ -164,15 +179,15 @@ mod hop_tests {
     }
 
     #[test]
-    fn only_the_plain_apns_leg_is_delivered_in_process() {
+    fn the_plain_apns_leg_is_delivered_in_process() {
         let apns = format!("https://push.fluxer.com/relay/v1/apns/canary/production/{TOKEN}");
         assert!(in_process_hop(&apns, &ours()).is_some());
     }
 
     #[test]
-    fn an_fcm_relay_endpoint_keeps_its_encrypted_network_hop() {
+    fn an_fcm_relay_endpoint_is_delivered_in_process() {
         let fcm = "https://push.fluxer.com/relay/v1/fcm/canary/tok%3AAPA91bExample";
-        assert!(in_process_hop(fcm, &ours()).is_none());
+        assert!(in_process_hop(fcm, &ours()).is_some());
     }
 
     #[test]

fluxer_push/src/providers/own_relay.rs

@@ -155,14 +155,14 @@ mod tests {
     }
 
     #[test]
-    fn only_the_apns_alert_leg_is_taken_in_process() {
+    fn only_the_alert_legs_are_taken_in_process() {
         let shortcut = |path: &str| {
             parse(&format!("https://push.fluxer.com/relay/v1/{path}"), &ours())
-                .filter(|hop| matches!(hop.leg, Leg::Apns))
+                .filter(|hop| matches!(hop.leg, Leg::Apns | Leg::Fcm))
         };
         assert!(shortcut(&format!("apns/canary/production/{TOKEN}")).is_some());
         assert!(shortcut(&format!("apns-voip/canary/production/{TOKEN}")).is_none());
-        assert!(shortcut("fcm/canary/dYC_x9gXTjyyrG8_Aw3nUM%3AAPA91bExample").is_none());
+        assert!(shortcut("fcm/canary/dYC_x9gXTjyyrG8_Aw3nUM%3AAPA91bExample").is_some());
     }
 
     #[test]

fluxer_push/src/providers/web_push.rs

@@ -3,6 +3,7 @@
 use crate::crypto;
 use crate::payload::{self, RecordKind};
 use crate::providers::{SendOutcome, own_relay};
+use crate::relay;
 use crate::resolver;
 use crate::server::AppState;
 use crate::subscription::Subscription;
@@ -44,19 +45,10 @@ const MAX_HOSTNAME_BYTES: usize = 253;
 const MAX_LABEL_BYTES: usize = 63;
 
 pub async fn send(state: &AppState, sub: &Subscription, envelope: &Value) -> SendOutcome {
-    if !endpoint_is_allowed(&sub.endpoint) {
-        return SendOutcome::permanent("endpoint_rejected");
-    }
-    let (Some(p256dh), Some(auth)) = (sub.p256dh_key.as_deref(), sub.auth_key.as_deref()) else {
-        return SendOutcome::permanent("missing_keys");
+    let record = match seal(state, sub, envelope) {
+        Ok(record) => record,
+        Err(outcome) => return outcome,
     };
-    let (Ok(p256dh), Ok(auth)) = (
-        crypto::decode_subscription_key(p256dh),
-        crypto::decode_subscription_key(auth),
-    ) else {
-        return SendOutcome::permanent("invalid_keys");
-    };
-
     let vapid = &state.cfg.vapid;
     let token = match state
         .tokens
@@ -67,41 +59,72 @@ pub async fn send(state: &AppState, sub: &Subscription, envelope: &Value) -> Sen
         Err(error) => return SendOutcome::permanent(format!("vapid_token: {error}")),
     };
     let authorization = format!("vapid t={token}, k={}", vapid.public_key);
+    deliver(state, sub, &record, &Hop::Network(&authorization)).await
+}
+
+pub async fn send_to_own_fcm_relay(
+    state: &AppState,
+    sub: &Subscription,
+    envelope: &Value,
+    target: relay::FcmTarget<'_>,
+) -> SendOutcome {
+    let record = match seal(state, sub, envelope) {
+        Ok(record) => record,
+        Err(outcome) => return outcome,
+    };
+    deliver(state, sub, &record, &Hop::OwnFcmRelay(target)).await
+}
+
+enum Hop<'a> {
+    Network(&'a str),
+    OwnFcmRelay(relay::FcmTarget<'a>),
+}
+
+struct Record {
+    body: Vec<u8>,
+    ttl_seconds: &'static str,
+    urgency: &'static str,
+}
+
+fn seal(state: &AppState, sub: &Subscription, envelope: &Value) -> Result<Record, SendOutcome> {
+    if !endpoint_is_allowed(&sub.endpoint) {
+        return Err(SendOutcome::permanent("endpoint_rejected"));
+    }
+    let (Some(p256dh), Some(auth)) = (sub.p256dh_key.as_deref(), sub.auth_key.as_deref()) else {
+        return Err(SendOutcome::permanent("missing_keys"));
+    };
+    let (Ok(p256dh), Ok(auth)) = (
+        crypto::decode_subscription_key(p256dh),
+        crypto::decode_subscription_key(auth),
+    ) else {
+        return Err(SendOutcome::permanent("invalid_keys"));
+    };
 
     let (plaintext, shrunk) = payload::fit(envelope, PLAINTEXT_BUDGET);
     if let Some(step) = shrunk {
         state.metrics.record_payload_shrink(step);
     }
-    let body = match crypto::encrypt_aes128gcm(&plaintext, &p256dh, &auth, RECORD_SIZE) {
-        Ok(body) => body,
-        Err(error) => return SendOutcome::permanent(format!("encrypt: {error}")),
-    };
+    let body = crypto::encrypt_aes128gcm(&plaintext, &p256dh, &auth, RECORD_SIZE)
+        .map_err(|error| SendOutcome::permanent(format!("encrypt: {error}")))?;
     let (ttl_seconds, urgency) = delivery_headers(envelope);
+    Ok(Record {
+        body,
+        ttl_seconds,
+        urgency,
+    })
+}
 
+async fn deliver(
+    state: &AppState,
+    sub: &Subscription,
+    record: &Record,
+    hop: &Hop<'_>,
+) -> SendOutcome {
     let mut attempt: u32 = 0;
     loop {
-        let response = state
-            .web_push_http
-            .post(&sub.endpoint)
-            .header(TTL_HEADER, ttl_seconds)
-            .header(URGENCY_HEADER, urgency)
-            .header(CONTENT_TYPE, OCTET_STREAM)
-            .header(CONTENT_ENCODING, AES128GCM)
-            .header(AUTHORIZATION, &authorization)
-            .body(body.clone())
-            .send()
-            .await;
-
-        let status = match response {
-            Ok(response) => response.status().as_u16(),
-            Err(error) => {
-                let unreachable = Unreachable::of(&error);
-                warn!(
-                    error = %error.without_url(),
-                    kind = unreachable.label(),
-                    endpoint = %origin_of(&sub.endpoint),
-                    "web push request did not complete"
-                );
+        let status = match post(state, sub, record, hop).await {
+            Ok(status) => status,
+            Err(unreachable) => {
                 if unreachable.is_permanent() {
                     return SendOutcome::permanent(unreachable.label());
                 }
@@ -125,6 +148,52 @@ pub async fn send(state: &AppState, sub: &Subscription, envelope: &Value) -> Sen
     }
 }
 
+async fn post(
+    state: &AppState,
+    sub: &Subscription,
+    record: &Record,
+    hop: &Hop<'_>,
+) -> Result<u16, Unreachable> {
+    let authorization = match hop {
+        Hop::Network(authorization) => authorization,
+        Hop::OwnFcmRelay(target) => {
+            return Ok(relay::forward_fcm(
+                &state.http,
+                &state.tokens,
+                &state.metrics,
+                target,
+                &record.body,
+                record.urgency,
+                record.ttl_seconds,
+            )
+            .await);
+        }
+    };
+    let response = state
+        .web_push_http
+        .post(&sub.endpoint)
+        .header(TTL_HEADER, record.ttl_seconds)
+        .header(URGENCY_HEADER, record.urgency)
+        .header(CONTENT_TYPE, OCTET_STREAM)
+        .header(CONTENT_ENCODING, AES128GCM)
+        .header(AUTHORIZATION, *authorization)
+        .body(record.body.clone())
+        .send()
+        .await;
+    response
+        .map(|response| response.status().as_u16())
+        .map_err(|error| {
+            let unreachable = Unreachable::of(&error);
+            warn!(
+                error = %error.without_url(),
+                kind = unreachable.label(),
+                endpoint = %origin_of(&sub.endpoint),
+                "web push request did not complete"
+            );
+            unreachable
+        })
+}
+
 fn delivery_headers(envelope: &Value) -> (&'static str, &'static str) {
     match payload::record_kind(envelope) {
         RecordKind::Message => (ALERT_TTL_SECONDS, ALERT_URGENCY),

fluxer_push/src/relay/mod.rs

@@ -5,7 +5,7 @@ pub mod envelope;
 mod quota;
 pub mod reject;
 
-use crate::config::{ProviderEnvironment, RelayConfig};
+use crate::config::{FcmConfig, ProviderEnvironment, RelayConfig};
 use crate::metrics::{Metrics, RelayLeg, RelayResult};
 use crate::server::{Sidecar, serve, sidecar_router};
 use crate::tokens::{TokenCache, TokenError};
@@ -204,14 +204,71 @@ struct Delivery {
     ttl_seconds: i64,
 }
 
+impl Delivery {
+    fn parse(urgency: Option<&str>, ttl_seconds: Option<&str>) -> Result<Self, Rejection> {
+        let bad_request = Rejection::new(Reason::BadRequest);
+        Ok(Self {
+            urgency: Urgency::from_header(urgency).ok_or(bad_request)?,
+            ttl_seconds: ttl_seconds
+                .ok_or(bad_request)?
+                .parse::<i64>()
+                .map_err(|_| bad_request)?
+                .clamp(0, MAX_TTL_SECONDS),
+        })
+    }
+}
+
 enum Target<'a> {
     Apns {
         environment: ProviderEnvironment,
         topic: &'a str,
     },
-    Fcm {
-        project_id: &'a str,
-    },
+    Fcm(FcmTarget<'a>),
+}
+
+pub struct FcmTarget<'a> {
+    cfg: &'a FcmConfig,
+    project_id: &'a str,
+    device_token: &'a str,
+}
+
+impl<'a> FcmTarget<'a> {
+    pub fn resolve(
+        cfg: Option<&'a FcmConfig>,
+        app_id: &str,
+        device_token: &'a str,
+    ) -> Option<Self> {
+        let cfg = cfg?;
+        Some(Self {
+            project_id: cfg.listed_project_id(app_id)?,
+            cfg,
+            device_token,
+        })
+    }
+}
+
+pub async fn forward_fcm(
+    http: &reqwest::Client,
+    tokens: &TokenCache,
+    metrics: &Metrics,
+    target: &FcmTarget<'_>,
+    record: &[u8],
+    urgency: &str,
+    ttl_seconds: &str,
+) -> u16 {
+    let verdict = async {
+        if !device_token_is_shaped(RelayLeg::Fcm, target.device_token) {
+            return Err(Rejection::new(Reason::DeviceTokenInvalid));
+        }
+        let delivery = Delivery::parse(Some(urgency), Some(ttl_seconds))?;
+        let payload = envelope::encode_payload(record);
+        send_fcm(http, tokens, metrics, target, &delivery, &payload).await
+    }
+    .await;
+    verdict
+        .err()
+        .map_or(StatusCode::OK, |rejection| rejection.reason.status())
+        .as_u16()
 }
 
 async fn relay(state: &AppState, incoming: Incoming, headers: HeaderMap, body: Body) -> Response {
@@ -264,14 +321,14 @@ async fn forward(
     if !is_aes128gcm(headers) {
         return Err(Rejection::new(Reason::BadRequest));
     }
-    let delivery = Delivery {
-        urgency: Urgency::from_header(header(headers, URGENCY_HEADER))
-            .ok_or(Rejection::new(Reason::BadRequest))?,
-        ttl_seconds: ttl_seconds(headers)?,
-    };
+    let delivery = Delivery::parse(header(headers, URGENCY_HEADER), header(headers, TTL_HEADER))?;
     state.quota.take(
         &state.metrics,
         &incoming.device_token,
+        match incoming.leg {
+            RelayLeg::ApnsVoip => Urgency::Alert,
+            _ => delivery.urgency,
+        },
         client_ip::for_rate_limit(&state.cfg, incoming.peer, headers),
         Instant::now(),
     )?;
@@ -293,13 +350,21 @@ async fn forward(
         Target::Apns { environment, topic } => {
             send_apns(state, incoming, &delivery, &payload, environment, topic).await
         }
-        Target::Fcm { project_id } => {
-            send_fcm(state, incoming, &delivery, &payload, project_id).await
+        Target::Fcm(target) => {
+            send_fcm(
+                &state.http,
+                &state.tokens,
+                &state.metrics,
+                &target,
+                &delivery,
+                &payload,
+            )
+            .await
         }
     }
 }
 
-fn resolve<'a>(state: &'a AppState, incoming: &Incoming) -> Result<Target<'a>, Rejection> {
+fn resolve<'a>(state: &'a AppState, incoming: &'a Incoming) -> Result<Target<'a>, Rejection> {
     let unknown = Rejection::new(Reason::AppUnknown);
     match incoming.leg {
         RelayLeg::Apns | RelayLeg::ApnsVoip => {
@@ -314,12 +379,13 @@ fn resolve<'a>(state: &'a AppState, incoming: &Incoming) -> Result<Target<'a>, R
                 topic: topic.ok_or(unknown)?,
             })
         }
-        RelayLeg::Fcm => {
-            let cfg = state.cfg.fcm.as_ref().ok_or(unknown)?;
-            Ok(Target::Fcm {
-                project_id: cfg.listed_project_id(&incoming.app_id).ok_or(unknown)?,
-            })
-        }
+        RelayLeg::Fcm => FcmTarget::resolve(
+            state.cfg.fcm.as_ref(),
+            &incoming.app_id,
+            &incoming.device_token,
+        )
+        .map(Target::Fcm)
+        .ok_or(unknown),
     }
 }
 
@@ -361,41 +427,30 @@ async fn send_apns(
         request,
     )
     .await;
-    finish(state, incoming.leg, outcome)
+    finish(&state.metrics, incoming.leg, outcome)
 }
 
 async fn send_fcm(
-    state: &AppState,
-    incoming: &Incoming,
+    http: &reqwest::Client,
+    tokens: &TokenCache,
+    metrics: &Metrics,
+    target: &FcmTarget<'_>,
     delivery: &Delivery,
     payload: &str,
-    project_id: &str,
 ) -> Result<(), Rejection> {
-    let cfg = state
-        .cfg
-        .fcm
-        .as_ref()
-        .ok_or(Rejection::new(Reason::AppUnknown))?;
     let body = envelope::fcm_body(
-        &incoming.device_token,
+        target.device_token,
         payload,
         delivery.urgency,
         delivery.ttl_seconds,
     )?;
-    let outcome = vendor::send_fcm(
-        &state.http,
-        &state.tokens,
-        &state.metrics,
-        cfg,
-        project_id,
-        body,
-    )
-    .await;
-    finish(state, RelayLeg::Fcm, outcome)
+    let outcome =
+        vendor::send_fcm(http, tokens, metrics, target.cfg, target.project_id, body).await;
+    finish(metrics, RelayLeg::Fcm, outcome)
 }
 
 fn finish(
-    state: &AppState,
+    metrics: &Metrics,
     leg: RelayLeg,
     outcome: Result<VendorOutcome, TokenError>,
 ) -> Result<(), Rejection> {
@@ -418,7 +473,7 @@ fn finish(
             Err(Rejection::new(refusal_reason(&refusal))),
         ),
     };
-    state.metrics.record_relay_vendor_request(leg, result);
+    metrics.record_relay_vendor_request(leg, result);
     verdict
 }
 
@@ -455,14 +510,6 @@ fn is_aes128gcm(headers: &HeaderMap) -> bool {
         .is_some_and(|value| value.eq_ignore_ascii_case(AES128GCM))
 }
 
-fn ttl_seconds(headers: &HeaderMap) -> Result<i64, Rejection> {
-    let raw = header(headers, TTL_HEADER).ok_or(Rejection::new(Reason::BadRequest))?;
-    let parsed = raw
-        .parse::<i64>()
-        .map_err(|_| Rejection::new(Reason::BadRequest))?;
-    Ok(parsed.clamp(0, MAX_TTL_SECONDS))
-}
-
 fn header<'a>(headers: &'a HeaderMap, name: &str) -> Option<&'a str> {
     headers
         .get(name)

fluxer_push/src/relay/quota.rs

@@ -1,5 +1,6 @@
 // SPDX-License-Identifier: AGPL-3.0-or-later
 
+use super::envelope::Urgency;
 use super::reject::{Reason, Rejection};
 use crate::config::BucketConfig;
 use crate::metrics::{BucketKey, Metrics};
@@ -23,6 +24,7 @@ type Key = [u8; KEY_BYTES];
 pub struct Quota {
     admissions: Semaphore,
     device_tokens: Buckets,
+    background_device_tokens: Buckets,
     sources: Option<Buckets>,
 }
 
@@ -35,6 +37,7 @@ impl Quota {
         Self {
             admissions: Semaphore::new(max_concurrent),
             device_tokens: Buckets::new(device_tokens),
+            background_device_tokens: Buckets::new(device_tokens),
             sources: sources.map(Buckets::new),
         }
     }
@@ -49,13 +52,21 @@ impl Quota {
         &self,
         metrics: &Metrics,
         device_token: &str,
+        urgency: Urgency,
         client_ip: IpAddr,
         now: Instant,
     ) -> Result<(), Rejection> {
+        let (which, device_tokens) = match urgency {
+            Urgency::Alert => (BucketKey::DeviceToken, &self.device_tokens),
+            Urgency::Background => (
+                BucketKey::BackgroundDeviceToken,
+                &self.background_device_tokens,
+            ),
+        };
         self.check(
             metrics,
-            BucketKey::DeviceToken,
-            &self.device_tokens,
+            which,
+            device_tokens,
             device_token_key(device_token),
             now,
         )?;