Skip to main content

tuwunel_service/pusher/email/
evaluate.rs

1use std::{
2	collections::{BTreeMap, VecDeque},
3	iter::empty,
4	pin::pin,
5};
6
7use futures::{
8	StreamExt, TryFutureExt, TryStreamExt,
9	future::{join, try_join},
10};
11use ruma::{
12	OwnedRoomId, RoomId, UserId,
13	api::client::push::{EmailPusherData, Pusher, PusherKind},
14	events::room::member::MembershipState,
15	push::Action,
16};
17use serde::{Deserialize, Serialize};
18use serde_json::Value as JsonValue;
19use tuwunel_core::{
20	Error, Result, implement, is_not_equal_to,
21	result::NotFound,
22	utils::{
23		BoolExt,
24		future::OptionFutureExt,
25		stream::{BroadbandExt, IterStream, ReadyExt, TryReadyExt, TryWidebandExt},
26	},
27	warn,
28};
29use tuwunel_database::{Cbor, Deserialized};
30use tuwunel_matrix::{
31	Event,
32	pdu::{Pdu, PduId, RawPduId},
33};
34
35use super::render::SHOWN;
36use crate::{
37	pusher::Notified, rooms::short::ShortRoomId, sendmail::Address, threepid::canonicalize_email,
38};
39
40/// What one evaluation of an email pusher found.
41///
42/// The worker drops a sent or idle pusher from its schedule, evaluates a
43/// pending one again at the time it carries, and retries a failed one with
44/// backoff.
45#[derive(Clone, Copy, Debug, Eq, PartialEq)]
46pub enum Outcome {
47	/// A mail went out, and no notification is still being written.
48	Sent,
49
50	/// Nothing is waiting, or the pusher can no longer be mailed.
51	Idle,
52
53	/// Notifications are waiting or still being written, and the pusher is
54	/// evaluated again at this time in milliseconds.
55	Pending(u64),
56
57	/// The evaluation failed and nothing was recorded, so it is retried.
58	Failed,
59}
60
61/// One room's mail throttle for one email pusher.
62///
63/// Only an evaluation that mails the room rewrites it; a later read of the room
64/// resets it when it is next loaded.
65#[derive(Clone, Copy, Debug, Default, Deserialize, Eq, PartialEq, Serialize)]
66pub(super) struct Throttle {
67	/// When the last mail naming the room went out, in milliseconds.
68	pub(super) last_sent_ms: u64,
69
70	/// How long after that mail the room stays quiet, in milliseconds.
71	pub(super) period_ms: u64,
72
73	/// The global count when that mail went out; a read of the room counted
74	/// after it resets the throttle.
75	pub(super) sent_count: u64,
76}
77
78pub(super) struct Pending {
79	pub(super) ts: u64,
80	pub(super) sroomid: ShortRoomId,
81	pub(super) pdu: Pdu,
82}
83
84/// What a scan of a pusher's notification rows gathered.
85///
86/// It keeps only what the timing and the mail need, so it stays small however
87/// many rows the scan reads.
88#[derive(Default)]
89pub(super) struct Scan {
90	/// The last row read.
91	pub(super) last: Option<u64>,
92
93	/// The first row still to mail.
94	pub(super) first: Option<u64>,
95
96	/// Each room with a notification to mail, and when its earliest one
97	/// arrived, which with the room's throttle decides when the room falls
98	/// due.
99	pub(super) rooms: PendingRooms,
100
101	/// The newest notifications to mail, up to `SHOWN`, oldest first.
102	pub(super) newest: VecDeque<Pending>,
103
104	/// How many notifications there are to mail.
105	pub(super) total: usize,
106}
107
108type PendingRooms = BTreeMap<ShortRoomId, (OwnedRoomId, u64)>;
109
110type Throttles<'a> = BTreeMap<ShortRoomId, RoomThrottle<'a>>;
111
112type RoomThrottle<'a> = (&'a RoomId, Throttle);
113
114type StoredRow<'a> = ((&'a UserId, u64), Notified);
115
116/// Factor a room's throttle grows by with each mail naming it.
117///
118/// With the default ten-minute delay, one step reaches the one-day cap, the
119/// schedule Synapse uses.
120const THROTTLE_MULTIPLIER: u64 = 144;
121
122/// The longest a room's throttle grows, one day in milliseconds.
123///
124/// A first throttle longer than this, from a configured delay over a day, is
125/// cut to it as well.
126const THROTTLE_MAX_MS: u64 = 24 * 60 * 60 * 1000;
127
128/// The soonest a pusher is evaluated again for a notification still being
129/// written, in milliseconds.
130///
131/// Without a delay, it keeps the worker from spinning until the write lands.
132const REVISIT_MIN_MS: u64 = 1000;
133
134/// Evaluates one email pusher at `now`, in milliseconds.
135///
136/// A notification after the pusher's cursor counts if it still notifies, is
137/// unread, and is in a room the user has joined or been invited to. Each waits
138/// out the delay and its room's throttle; once the earliest is due, all of them
139/// go out in one mail and the cursor moves past them. A notification still
140/// being written brings the pusher back after the delay or one second,
141/// whichever is longer.
142#[implement(super::super::Service)]
143#[tracing::instrument(level = "debug", skip(self, pushkey))]
144pub async fn evaluate_email(&self, user_id: &UserId, pushkey: &str, now: u64) -> Outcome {
145	let (address, data) = match self.email_target(user_id, pushkey).await {
146		| Ok(target) => target,
147		| Err(outcome) => return outcome,
148	};
149
150	let stored = match self.email_cursor(user_id, pushkey).await {
151		| Ok(Some(cursor)) => cursor,
152		| Ok(None) => return Outcome::Idle,
153		| Err(_) => return Outcome::Failed,
154	};
155
156	let delay = self
157		.services
158		.config
159		.push
160		.email_delay
161		.saturating_mul(1000);
162
163	let frontier = self.services.globals.current_count();
164	let Ok((scan, in_flight)) = try_join(
165		self.scan_pending(user_id, stored, frontier),
166		self.notified_past(user_id, frontier),
167	)
168	.await
169	else {
170		return Outcome::Failed;
171	};
172
173	let revisit = in_flight.then(|| now.saturating_add(delay.max(REVISIT_MIN_MS)));
174	let nothing_due = revisit.map_or(Outcome::Idle, Outcome::Pending);
175	let Some(last) = scan.last else {
176		return nothing_due;
177	};
178
179	let Some(first) = scan.first else {
180		self.record_state(user_id, pushkey, stored, last, empty())
181			.await;
182
183		return nothing_due;
184	};
185
186	let Ok(throttles) = self
187		.room_throttles(user_id, pushkey, &scan.rooms)
188		.await
189	else {
190		return Outcome::Failed;
191	};
192
193	let due = scan
194		.rooms
195		.iter()
196		.map(|(sroomid, &(_, ts))| {
197			let throttle = throttles
198				.get(sroomid)
199				.map(|&(_, throttle)| throttle)
200				.unwrap_or_default();
201
202			due_at(ts, throttle, delay)
203		})
204		.min()
205		.unwrap_or(now);
206
207	if due > now {
208		let cursor = first.saturating_sub(1);
209
210		if cursor > stored {
211			self.record_state(user_id, pushkey, stored, cursor, empty())
212				.await;
213		}
214
215		return Outcome::Pending(revisit.map_or(due, |revisit| due.min(revisit)));
216	}
217
218	let brand = data
219		.data
220		.get("brand")
221		.and_then(JsonValue::as_str)
222		.unwrap_or_else(|| self.services.globals.server_name().as_str());
223
224	let mail = self
225		.render_email(user_id, &address, brand, &scan)
226		.await;
227
228	let Ok(()) = self
229		.services
230		.sendmail
231		.send_auto_generated(&address, &mail.subject, mail.text)
232		.inspect_err(|error| warn!(%user_id, %error, "Failed to send a notification email"))
233		.await
234	else {
235		return Outcome::Failed;
236	};
237
238	let sent_count = self.services.globals.current_count();
239	let throttles = throttles
240		.into_values()
241		.map(|(room_id, throttle)| (room_id, sent_throttle(throttle, now, sent_count, delay)));
242
243	self.record_state(user_id, pushkey, stored, last, throttles)
244		.await;
245
246	revisit.map_or(Outcome::Sent, Outcome::Pending)
247}
248
249/// The address an email pusher mails and the pusher's data, or the outcome
250/// ending its evaluation.
251///
252/// A pusher that is gone, or no longer an email pusher, takes its rows with it.
253/// One whose address is no longer verified on the account, or is not a valid
254/// address, is deleted, as removing the address deletes it in Synapse.
255#[implement(super::super::Service)]
256async fn email_target(
257	&self,
258	user_id: &UserId,
259	pushkey: &str,
260) -> Result<(Address, EmailPusherData), Outcome> {
261	let Some(data) = self
262		.email_pusher(user_id, pushkey)
263		.await
264		.inspect_err(|error| warn!(%user_id, %error, "Failed to read an email pusher"))
265		.map_err(|_| Outcome::Failed)?
266	else {
267		self.forget_orphan(user_id, pushkey).await;
268		return Err(Outcome::Idle);
269	};
270
271	if !self.services.sendmail.is_enabled() {
272		return Err(Outcome::Idle);
273	}
274
275	let bound = self
276		.services
277		.threepid
278		.address_bound_to(user_id, pushkey)
279		.await
280		.inspect_err(|error| warn!(%user_id, %error, "Failed to read an email pusher's binding"))
281		.map_err(|_| Outcome::Failed)?;
282
283	let Some(address) = deliverable(bound, pushkey) else {
284		self.forget_unbound(user_id, pushkey).await;
285		return Err(Outcome::Idle);
286	};
287
288	Ok((address, data))
289}
290
291/// The pusher's email data, or `None` when it is gone or is not an email
292/// pusher.
293///
294/// Failing to read the pusher is an error, never taken for its absence.
295#[implement(super::super::Service)]
296pub(super) async fn email_pusher(
297	&self,
298	user_id: &UserId,
299	pushkey: &str,
300) -> Result<Option<EmailPusherData>> {
301	self.get_pusher(user_id, pushkey)
302		.await
303		.optional()
304		.map(|pusher| pusher.and_then(email_data))
305}
306
307/// Deletes the cursor and room throttles of a pusher that is gone or no longer
308/// mails.
309///
310/// The pusher is read again under its lock, so one created again since keeps
311/// its new cursor.
312#[implement(super::super::Service)]
313async fn forget_orphan(&self, user_id: &UserId, pushkey: &str) {
314	let _lock = self.email_lock(user_id, pushkey).await;
315
316	if matches!(self.email_pusher(user_id, pushkey).await, Ok(None)) {
317		self.forget_email_state(user_id, pushkey).await;
318	}
319}
320
321/// Deletes an email pusher whose address is no longer verified on the account
322/// or is not a valid address.
323///
324/// The pusher and its binding are read again under the pusher's lock, so a
325/// pusher replaced in the meantime, or one whose address was verified again,
326/// survives.
327#[implement(super::super::Service)]
328async fn forget_unbound(&self, user_id: &UserId, pushkey: &str) {
329	let _lock = self.email_lock(user_id, pushkey).await;
330
331	if !matches!(self.email_pusher(user_id, pushkey).await, Ok(Some(_))) {
332		return;
333	}
334
335	self.services
336		.threepid
337		.address_bound_to(user_id, pushkey)
338		.await
339		.is_ok_and(|bound| deliverable(bound, pushkey).is_none())
340		.then_async(|| self.remove_pusher(user_id, pushkey))
341		.await;
342}
343
344/// Gathers the notifications to mail between the cursor and the retired
345/// `frontier`.
346///
347/// A row past the frontier may sit above one still being written, so the scan
348/// stops before it. It reads every row up to there, a backlog the throttle cap
349/// keeps to about a day of notifications. A row, event or membership whose read
350/// may succeed on a retry ends the scan with its error; one unreadable for good
351/// is skipped.
352#[implement(super::super::Service)]
353async fn scan_pending(&self, user_id: &UserId, cursor: u64, frontier: u64) -> Result<Scan> {
354	self.db
355		.useridcount_notification
356		.stream_from(&(user_id, cursor.saturating_add(1)))
357		.ready_filter_map(|row| readable(user_id, cursor, row))
358		.ready_try_take_while(|((user, count), _): &StoredRow<'_>| {
359			Ok(*user == user_id && *count <= frontier)
360		})
361		.map_ok(|((_, count), notified): StoredRow<'_>| (count, notified))
362		.wide_and_then(async |(count, notified)| {
363			self.scanned(user_id, count, notified)
364				.map_ok(|pending| (count, pending))
365				.await
366		})
367		.ready_try_fold_default(|scan: Scan, row| Ok(gather(scan, row)))
368		.await
369}
370
371/// The notification as a pending mail, or `None` once it no longer warrants
372/// one.
373///
374/// It must still notify, survive unredacted, be unread, and be in a room the
375/// user is still joined to, or invited to when the notification is that invite.
376/// An event or membership whose read may succeed on a retry is an error; one
377/// missing or unreadable for good is skipped.
378#[implement(super::super::Service)]
379async fn scanned(
380	&self,
381	user_id: &UserId,
382	count: u64,
383	notified: Notified,
384) -> Result<Option<Pending>> {
385	if !notified.actions.iter().any(Action::should_notify) {
386		return Ok(None);
387	}
388
389	let pdu_id: RawPduId = PduId {
390		shortroomid: notified.sroomid,
391		count: count.into(),
392	}
393	.into();
394
395	let Some(pdu) = self
396		.services
397		.timeline
398		.get_pdu_from_id(&pdu_id)
399		.await
400		.present()
401		// Kept at warn: nothing else in a stock build shows the unreadable event.
402		.inspect_err(|error| warn!(%user_id, count, %error, "Failed to read a notified event"))
403		.or_else(transient_or_none)?
404		.filter(|pdu| !pdu.is_redacted())
405	else {
406		return Ok(None);
407	};
408
409	if self.notification_read(user_id, &pdu, count).await {
410		return Ok(None);
411	}
412
413	let state_cache = &self.services.state_cache;
414	let room_id = pdu.room_id();
415	let member = invites(&pdu, user_id)
416		.then_async(|| state_cache.try_is_invited(user_id, room_id))
417		.unwrap_or_else_async(|| state_cache.try_is_joined(user_id, room_id))
418		.map_ok(Some)
419		.inspect_err(|error| warn!(%user_id, %room_id, %error, "Failed to read a membership"))
420		.await
421		.or_else(transient_or_none)?
422		.unwrap_or_default();
423
424	let pending = Pending {
425		ts: notified.ts,
426		sroomid: notified.sroomid,
427		pdu,
428	};
429
430	Ok(member.then_some(pending))
431}
432
433/// Whether the user has a notification row past the retired `frontier`.
434///
435/// Such a row may sit above one still being written, which a later evaluation
436/// must not skip. A row whose read may succeed on a retry is an error, and one
437/// unreadable for good is skipped.
438#[implement(super::super::Service)]
439async fn notified_past(&self, user_id: &UserId, frontier: u64) -> Result<bool> {
440	let past = self
441		.db
442		.useridcount_notification
443		.keys_from(&(user_id, frontier.saturating_add(1)))
444		.ready_filter_map(|key| readable(user_id, frontier, key));
445
446	pin!(past)
447		.next()
448		.await
449		.transpose()
450		.map(|first| first.is_some_and(|(user, _): (&UserId, u64)| user == user_id))
451}
452
453/// Each pending room's throttle as it applies now.
454///
455/// A room the user has read since the last mail naming it has its throttle
456/// reset. A read that may succeed on a retry returns an error, while a throttle
457/// unreadable for good counts as none.
458#[implement(super::super::Service)]
459async fn room_throttles<'a>(
460	&self,
461	user_id: &UserId,
462	pushkey: &str,
463	rooms: &'a PendingRooms,
464) -> Result<Throttles<'a>> {
465	rooms
466		.iter()
467		.stream()
468		.broad_then(|(&sroomid, (room_id, _))| {
469			self.room_throttle(user_id, pushkey, sroomid, room_id)
470		})
471		.try_collect()
472		.await
473}
474
475#[implement(super::super::Service)]
476async fn room_throttle<'a>(
477	&self,
478	user_id: &UserId,
479	pushkey: &str,
480	sroomid: ShortRoomId,
481	room_id: &'a RoomId,
482) -> Result<(ShortRoomId, RoomThrottle<'a>)> {
483	let stored = self
484		.db
485		.senderkeyroom_emailthrottle
486		.qry(&(user_id, pushkey, room_id));
487
488	let last_read = self.last_notification_read(user_id, room_id);
489	let (stored, last_read) = join(stored, last_read).await;
490	let stored = stored
491		.deserialized()
492		.map(|Cbor(stored): Cbor<Throttle>| stored)
493		.present()
494		.inspect_err(
495			|error| warn!(%user_id, %room_id, %error, "Failed to read an email throttle"),
496		)
497		.or_else(transient_or_none)?
498		.unwrap_or_default();
499
500	let last_read = last_read
501		.optional()
502		.inspect_err(
503			|error| warn!(%user_id, %room_id, %error, "Failed to read a room's last read"),
504		)
505		.ok()
506		.flatten();
507
508	Ok((sroomid, (room_id, effective_throttle(stored, last_read))))
509}
510
511/// Moves the pusher's cursor on from `stored` and stores the throttles of the
512/// rooms a mail named, unless the cursor has changed since the evaluation read
513/// it.
514///
515/// Deleting the pusher, or creating it again, changes the cursor, so a result
516/// computed from the old one is dropped; the lock keeps either from landing
517/// between the check and the write. A cursor that cannot be read counts as
518/// unchanged: dropping the result would mail the same notifications again,
519/// while writing it at worst leaves a deleted pusher's cursor for its next
520/// evaluation to delete, or sets a pusher created since back to this cursor,
521/// so it also mails what came before it to the same address.
522#[implement(super::super::Service)]
523async fn record_state<'a, Rooms>(
524	&self,
525	user_id: &UserId,
526	pushkey: &str,
527	stored: u64,
528	cursor: u64,
529	rooms: Rooms,
530) where
531	Rooms: Iterator<Item = RoomThrottle<'a>> + Send,
532{
533	let _lock = self.email_lock(user_id, pushkey).await;
534	let changed = self
535		.email_cursor(user_id, pushkey)
536		.await
537		.is_ok_and(is_not_equal_to!(Some(stored)));
538
539	if !changed {
540		self.write_state(user_id, pushkey, cursor, rooms);
541	}
542}
543
544/// Writes the cursor and each room's throttle in one transaction.
545///
546/// The cursor never moves past a mail without the throttles it set.
547#[implement(super::super::Service)]
548fn write_state<'a, Rooms>(&self, user_id: &UserId, pushkey: &str, cursor: u64, rooms: Rooms)
549where
550	Rooms: Iterator<Item = RoomThrottle<'a>>,
551{
552	let mut txn = self.db.db.txn();
553
554	txn.put(&self.db.senderkey_emailcursor, (user_id, pushkey), cursor);
555	rooms.for_each(|(room_id, throttle)| {
556		let key = (user_id, pushkey, room_id);
557
558		txn.put(&self.db.senderkeyroom_emailthrottle, key, Cbor(throttle));
559	});
560
561	txn.execute();
562}
563
564/// The address a pushkey is mailed at: its canonical form, the one the binding
565/// was verified under.
566///
567/// It is `None` when the address is not bound to the account or is not a valid
568/// address. Mailing the client's spelling instead could reach another mailbox
569/// whose address folds to the same canonical form.
570fn deliverable(bound: bool, pushkey: &str) -> Option<Address> {
571	bound.and_then(|| Address::try_from(canonicalize_email(pushkey).ok()?).ok())
572}
573
574fn email_data(pusher: Pusher) -> Option<EmailPusherData> {
575	match pusher.kind {
576		| PusherKind::Email(data) => Some(data),
577		| _ => None,
578	}
579}
580
581/// A notification row or key read past `after`, or its error when a retry may
582/// read it.
583///
584/// Every failed read is logged; one unreadable for good is dropped.
585fn readable<T>(user_id: &UserId, after: u64, read: Result<T>) -> Option<Result<T>> {
586	read.map(Some)
587		// Kept at warn: nothing else in a stock build shows the unreadable row.
588		.inspect_err(|error| warn!(%user_id, after, %error, "Failed to read a notification"))
589		.or_else(transient_or_none)
590		.transpose()
591}
592
593/// The error when a retry may succeed, else `None`, which skips what failed.
594///
595/// Only an I/O error of a transient kind counts; corruption and decode errors
596/// do not.
597fn transient_or_none<T>(error: Error) -> Result<Option<T>> {
598	error
599		.is_transient_io()
600		.then_some(error)
601		.map_or(Ok(None), Err)
602}
603
604pub(super) fn invites(pdu: &Pdu, user_id: &UserId) -> bool {
605	pdu.membership_for(user_id) == Some(MembershipState::Invite)
606}
607
608/// Folds one scanned row, and the notification it leaves to mail, into the
609/// scan.
610///
611/// Rows arrive in count order, so a room's first notification is its earliest.
612pub(super) fn gather(mut scan: Scan, (count, pending): (u64, Option<Pending>)) -> Scan {
613	scan.last = Some(count);
614	pending
615		.into_iter()
616		.fold(scan, |scan, pending| keep(scan, count, pending))
617}
618
619fn keep(mut scan: Scan, count: u64, pending: Pending) -> Scan {
620	scan.first.get_or_insert(count);
621	scan.total = scan.total.saturating_add(1);
622	scan.rooms
623		.entry(pending.sroomid)
624		.or_insert_with(|| (pending.pdu.room_id().to_owned(), pending.ts));
625
626	scan.newest.push_back(pending);
627	if scan.newest.len() > SHOWN {
628		scan.newest.pop_front();
629	}
630
631	scan
632}
633
634/// A stored throttle as it applies now.
635///
636/// A read of the room counted after the last mail naming it zeroes the
637/// throttle; the stored row is left as it is.
638pub(super) fn effective_throttle(stored: Throttle, last_read: Option<u64>) -> Throttle {
639	let read_since = last_read.is_some_and(|last_read| last_read > stored.sent_count);
640
641	Throttle {
642		period_ms: read_since.copy_or(stored.period_ms, 0),
643		..stored
644	}
645}
646
647/// The earliest a notification that arrived at `ts` may be mailed.
648///
649/// Both the delay and its room's throttle, counted from the last mail naming
650/// the room, must have run out. Without a delay no room is quieted, whatever
651/// throttle an earlier delay left it.
652pub(super) fn due_at(ts: u64, throttle: Throttle, delay: u64) -> u64 {
653	let quiet_until = throttle
654		.last_sent_ms
655		.saturating_add(throttle.period_ms);
656
657	match delay {
658		| 0 => ts,
659		| _ => ts.saturating_add(delay).max(quiet_until),
660	}
661}
662
663/// A room's throttle once a mail naming it has gone out.
664///
665/// A room without one starts at the delay, then each mail multiplies it, up to
666/// a day; reading the room starts it over, and without a delay it stays at
667/// none. `sent_count` is the global count a later read is compared with.
668pub(super) fn sent_throttle(
669	effective: Throttle,
670	now: u64,
671	sent_count: u64,
672	delay: u64,
673) -> Throttle {
674	let period_ms = match (delay, effective.period_ms) {
675		| (0, _) | (_, 0) => delay,
676		| (_, period_ms) => period_ms.saturating_mul(THROTTLE_MULTIPLIER),
677	};
678
679	Throttle {
680		last_sent_ms: now,
681		period_ms: period_ms.min(THROTTLE_MAX_MS),
682		sent_count,
683	}
684}