1use std::{
2 ops::Range,
3 sync::atomic::{AtomicU64, Ordering},
4 time::Duration,
5};
6
7use futures::FutureExt;
8use ruma::EventId;
9use tuwunel_core::{
10 Error, implement,
11 utils::{
12 BoolExt, continue_exponential_backoff,
13 stream::{ReadyExt, TryIgnore},
14 time::now_secs,
15 },
16};
17use tuwunel_database::{Ignore, Interfix};
18
19#[cfg(test)]
20mod tests;
21
22const QUANTUM: u64 = 60;
25
26const SUPPRESS_AFTER: u32 = 3;
28
29pub(super) const UPGRADE_RETRY: Range<Duration> =
35 Duration::from_mins(5)..Duration::from_hours(24);
36
37#[derive(Clone, Copy)]
39pub(super) enum Context {
40 Fetch = 0,
41 Auth = 1,
42 Upgrade = 2,
43 Incoming = 3,
44}
45
46impl From<Context> for u8 {
47 #[inline]
48 fn from(context: Context) -> Self {
49 match context {
50 | Context::Fetch => 0,
51 | Context::Auth => 1,
52 | Context::Upgrade => 2,
53 | Context::Incoming => 3,
54 }
55 }
56}
57
58#[derive(Clone, Copy, Default)]
63pub(super) enum Disposition {
64 #[default]
65 Pending = 0,
66 Transient = 1,
67 Permanent = 2,
68}
69
70pub(super) enum Suppression {
72 Absent,
74
75 Allow,
77
78 Deny,
80}
81
82#[derive(Default)]
83struct Summary {
84 total: u32,
85 pending: u32,
86 latest_secs: u64,
87 latest_class: Disposition,
88}
89
90#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)]
98pub struct BackoffMetrics {
99 pub fetch: Verdicts,
101
102 pub auth: Verdicts,
104
105 pub upgrade: Verdicts,
108
109 pub incoming: Verdicts,
111}
112
113#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)]
118pub struct Verdicts {
119 pub absent: u64,
121
122 pub allowed: u64,
124
125 pub denied: u64,
127}
128
129#[derive(Default)]
130pub(super) struct BackoffCounters {
131 fetch: VerdictCounters,
132 auth: VerdictCounters,
133 upgrade: VerdictCounters,
134 incoming: VerdictCounters,
135}
136
137#[derive(Default)]
138struct VerdictCounters {
139 absent: AtomicU64,
140 allowed: AtomicU64,
141 denied: AtomicU64,
142}
143
144impl From<u64> for Disposition {
145 #[inline]
146 fn from(disc: u64) -> Self {
147 match disc {
148 | 1 => Self::Transient,
149 | 2 => Self::Permanent,
150 | _ => Self::Pending,
151 }
152 }
153}
154
155impl From<Disposition> for u64 {
156 #[inline]
157 fn from(disposition: Disposition) -> Self {
158 match disposition {
159 | Disposition::Pending => 0,
160 | Disposition::Transient => 1,
161 | Disposition::Permanent => 2,
162 }
163 }
164}
165
166impl Suppression {
167 #[inline]
168 pub(super) fn is_deny(&self) -> bool { matches!(self, Self::Deny) }
169}
170
171impl Summary {
172 fn tally(mut self, (_, (class, secs)): (Ignore, (u64, u64))) -> Self {
173 let class = Disposition::from(class);
174
175 self.total = self.total.saturating_add(1);
176 if matches!(class, Disposition::Pending) {
177 self.pending = self.pending.saturating_add(1);
178 }
179
180 if secs >= self.latest_secs {
181 self.latest_secs = secs;
182 self.latest_class = class;
183 }
184
185 self
186 }
187}
188
189impl Verdicts {
190 #[inline]
194 #[must_use]
195 pub fn lookups(&self) -> u64 {
196 self.absent
197 .saturating_add(self.allowed)
198 .saturating_add(self.denied)
199 }
200}
201
202#[implement(super::Service)]
206#[inline]
207#[must_use]
208pub fn backoff_metrics(&self) -> BackoffMetrics { self.backoff.snapshot() }
209
210#[implement(BackoffCounters)]
211fn snapshot(&self) -> BackoffMetrics {
212 BackoffMetrics {
213 fetch: self.fetch.snapshot(),
214 auth: self.auth.snapshot(),
215 upgrade: self.upgrade.snapshot(),
216 incoming: self.incoming.snapshot(),
217 }
218}
219
220impl VerdictCounters {
221 fn snapshot(&self) -> Verdicts {
222 Verdicts {
223 absent: self.absent.load(Ordering::Relaxed),
224 allowed: self.allowed.load(Ordering::Relaxed),
225 denied: self.denied.load(Ordering::Relaxed),
226 }
227 }
228}
229
230#[implement(super::Service)]
235pub(super) fn record_attempt(&self, ctx: Context, event_id: &EventId) -> u32 {
236 let bucket = current_bucket();
237
238 self.record_outcome_at(ctx, event_id, bucket, Disposition::Pending);
239 bucket
240}
241
242#[implement(super::Service)]
253pub(super) async fn record_completion(
254 &self,
255 ctx: Context,
256 event_id: &EventId,
257 standing: Suppression,
258 attempt: Option<u32>,
259 appended: Result<bool, &Error>,
260) {
261 match (appended, standing, attempt) {
262 | (_, Suppression::Absent, None) | (Ok(false), ..) => {},
263 | (Err(error), ..) if error.is_interrupted() || self.services.server.is_stopping() => {},
264 | (Ok(true), Suppression::Absent, Some(bucket)) =>
265 self.clear_outcome_at(ctx, event_id, bucket),
266 | (Ok(true), ..) => self.record_success(ctx, event_id).await,
267 | (Err(_), _, attempt) => {
268 let bucket = attempt.unwrap_or_else(current_bucket);
269
270 self.record_outcome_at(ctx, event_id, bucket, Disposition::Transient);
271 },
272 }
273}
274
275#[implement(super::Service)]
276pub(super) fn record_outcome(&self, ctx: Context, event_id: &EventId, disposition: Disposition) {
277 if self.services.server.is_stopping() {
278 return;
279 }
280
281 self.record_outcome_at(ctx, event_id, current_bucket(), disposition);
282}
283
284#[implement(super::Service)]
290pub async fn clear_delivery_backoff(&self, event_id: &EventId) {
291 self.record_success(Context::Upgrade, event_id)
292 .await;
293
294 self.record_success(Context::Incoming, event_id)
295 .await;
296}
297
298#[implement(super::Service)]
299pub(super) async fn record_success(&self, ctx: Context, event_id: &EventId) {
300 self.db
301 .eventid_backoff
302 .del_prefix(&(u8::from(ctx), event_id, Interfix))
303 .await;
304}
305
306#[implement(super::Service)]
311pub(super) fn is_suppressed(
312 &self,
313 ctx: Context,
314 event_id: &EventId,
315 range: Range<Duration>,
316) -> impl Future<Output = Suppression> + Send {
317 self.verdict(ctx, event_id, range)
318 .inspect(move |verdict| self.backoff.count(ctx, verdict))
319}
320
321#[implement(super::Service)]
322async fn verdict(&self, ctx: Context, event_id: &EventId, range: Range<Duration>) -> Suppression {
323 let summary = self
324 .db
325 .eventid_backoff
326 .stream_prefix::<Ignore, (u64, u64), _>(&(u8::from(ctx), event_id, Interfix))
327 .ignore_err()
328 .ready_fold(Summary::default(), Summary::tally)
329 .await;
330
331 if summary.total == 0 {
332 return Suppression::Absent;
333 }
334
335 if matches!(summary.latest_class, Disposition::Permanent) {
336 return Suppression::Deny;
337 }
338
339 let elapsed = Duration::from_secs(now_secs().saturating_sub(summary.latest_secs));
340 let (tries, rate_ok) = match summary.latest_class {
341 | Disposition::Pending => (summary.pending, summary.pending >= SUPPRESS_AFTER),
342 | _ => (summary.total, true),
343 };
344
345 let deny = rate_ok && continue_exponential_backoff(range.start, range.end, elapsed, tries);
346
347 deny.map_or(Suppression::Allow, || Suppression::Deny)
348}
349
350#[implement(BackoffCounters)]
351fn count(&self, ctx: Context, verdict: &Suppression) {
352 let counters = match ctx {
353 | Context::Fetch => &self.fetch,
354 | Context::Auth => &self.auth,
355 | Context::Upgrade => &self.upgrade,
356 | Context::Incoming => &self.incoming,
357 };
358
359 let counter = match verdict {
360 | Suppression::Absent => &counters.absent,
361 | Suppression::Allow => &counters.allowed,
362 | Suppression::Deny => &counters.denied,
363 };
364
365 counter.fetch_add(1, Ordering::Relaxed);
366}
367
368fn current_bucket() -> u32 { u32::try_from(now_secs() / QUANTUM).unwrap_or(u32::MAX) }
369
370#[implement(super::Service)]
371fn record_outcome_at(
372 &self,
373 ctx: Context,
374 event_id: &EventId,
375 bucket: u32,
376 disposition: Disposition,
377) {
378 self.db
379 .eventid_backoff
380 .put((u8::from(ctx), event_id, bucket), (u64::from(disposition), now_secs()));
381}
382
383#[implement(super::Service)]
384fn clear_outcome_at(&self, ctx: Context, event_id: &EventId, bucket: u32) {
385 self.db
386 .eventid_backoff
387 .del((u8::from(ctx), event_id, bucket));
388}