Skip to main content

tuwunel_service/pusher/email/
mod.rs

1//! Email notifications for the `email` pusher kind.
2//!
3//! A notifying event for a user with an email pusher wakes the worker, which
4//! waits out the configured delay and then mails every notification still
5//! unread in one message. The pending work is derived from the stored
6//! notification rows; only a cursor per pusher, and a throttle per room it has
7//! mailed about, are kept beside them, so a restart loses no pending mail of
8//! a pusher whose cursor has started.
9
10mod evaluate;
11mod render;
12#[cfg(test)]
13mod tests;
14
15use std::{collections::BTreeMap, pin::pin, time::Duration};
16
17use futures::{
18	FutureExt, StreamExt,
19	future::{Either, select},
20	stream::unfold,
21};
22use loole::{Receiver, Sender, unbounded};
23use ruma::{
24	OwnedUserId, UserId,
25	api::client::push::{PusherKind, set_pusher::v3::PusherAction},
26};
27use tokio::time::{Instant, sleep_until};
28use tuwunel_core::{
29	Result, implement,
30	result::NotFound,
31	smallstr::SmallString,
32	utils::{
33		MutexMap, MutexMapGuard, OptionExt, exponential_backoff_remaining, stream::TryIgnore,
34		time::now_millis,
35	},
36	warn,
37};
38use tuwunel_database::{Deserialized, Interfix, Json};
39
40pub use self::evaluate::Outcome;
41
42/// One email pusher: its user, and its pushkey.
43///
44/// An email pusher's pushkey is the address it mails.
45type PusherKey = (OwnedUserId, PushKey);
46
47type PushKey = SmallString<[u8; 48]>;
48
49/// Serializes the writes to each email pusher's rows.
50///
51/// It holds one entry per pusher with a holder: `put_pusher`, `delete_pusher`
52/// and the worker's `start_cursor`, `record_state`, `forget_orphan` and
53/// `forget_unbound`. None holds it across sending a mail.
54pub(super) type Locks = MutexMap<PusherKey, ()>;
55
56/// The email pushers with pending work and when each is next evaluated.
57///
58/// An entry lives from a pusher's first wake to the evaluation that mails it
59/// or finds nothing left to mail, so only pushers with unread notifications
60/// not yet mailed are held. A restart loses retry counts, harmlessly, since
61/// every pusher with a cursor is evaluated again after startup. It also loses
62/// any frontier kept for a cursor not yet started, so that pusher's cursor
63/// starts at its next wake, past the notifications before it.
64type Schedule = BTreeMap<PusherKey, Entry>;
65
66/// The pusher one notification wakes, and the frontier of retired writes when
67/// the wake was sent.
68///
69/// The frontier starts a cursor for a pusher that has none.
70type Wake = (PusherKey, u64);
71
72/// The worker's channel, carrying one wake per notification.
73///
74/// It is unbounded, so the append path never waits on the worker.
75pub(super) struct Wakes {
76	sender: Sender<Wake>,
77	receiver: Receiver<Wake>,
78}
79
80#[derive(Clone, Copy, Debug)]
81struct Entry {
82	due: Instant,
83	failures: u32,
84
85	/// The frontier to start the pusher's cursor at, kept while the cursor
86	/// cannot be read.
87	frontier: Option<u64>,
88}
89
90enum Next {
91	Wake(Wake),
92	Due(PusherKey),
93}
94
95/// The wait after startup before evaluating the pushers that had work.
96///
97/// It keeps the first evaluations out of the way of startup.
98const BOOT_GRACE: Duration = Duration::from_mins(1);
99
100const RETRY_MIN: Duration = Duration::from_mins(1);
101
102const RETRY_MAX: Duration = Duration::from_hours(6);
103
104/// The longest any pusher waits, whatever the configured delay.
105///
106/// It keeps a huge delay from overflowing the timer.
107const WAIT_MAX: Duration = Duration::from_hours(30 * 24);
108
109impl Default for Wakes {
110	fn default() -> Self {
111		let (sender, receiver) = unbounded();
112
113		Self { sender, receiver }
114	}
115}
116
117/// Stores a pusher under its email lock, starting a new email pusher's cursor
118/// past every notification counted so far.
119///
120/// A stored pusher that is already an email pusher, or cannot be read, keeps
121/// its cursor, so nothing it has yet to mail is skipped; a new one drops any
122/// room throttles left by an earlier pusher with its pushkey. Taking the lock
123/// for every kind orders the store against the worker deleting the pusher it
124/// replaces.
125#[implement(super::Service)]
126pub(super) async fn put_pusher(&self, user_id: &UserId, pushkey: &str, action: &PusherAction) {
127	let email = matches!(
128		action,
129		PusherAction::Post(data) if matches!(data.pusher.kind, PusherKind::Email(_))
130	);
131
132	let _lock = self.email_lock(user_id, pushkey).await;
133	let created = email && matches!(self.email_pusher(user_id, pushkey).await, Ok(None));
134
135	if created {
136		let cursor = self.services.globals.pending_count().end;
137
138		self.forget_throttles(user_id, pushkey).await;
139		self.put_cursor(user_id, pushkey, cursor);
140	}
141
142	self.db
143		.senderkey_pusher
144		.put((user_id, pushkey), Json(action));
145}
146
147#[implement(super::Service)]
148pub(super) async fn email_lock(
149	&self,
150	user_id: &UserId,
151	pushkey: &str,
152) -> MutexMapGuard<PusherKey, ()> {
153	self.email_locks
154		.lock(&pusher_key(user_id, pushkey))
155		.await
156}
157
158fn pusher_key(user_id: &UserId, pushkey: &str) -> PusherKey {
159	(user_id.to_owned(), pushkey.into())
160}
161
162/// Deletes a pusher's cursor and room throttles.
163///
164/// Callers hold the pusher's lock.
165#[implement(super::Service)]
166pub(super) async fn forget_email_state(&self, user_id: &UserId, pushkey: &str) {
167	self.db
168		.senderkey_emailcursor
169		.del((user_id, pushkey));
170
171	self.forget_throttles(user_id, pushkey).await;
172}
173
174/// Deletes a pusher's room throttles.
175///
176/// Callers hold the pusher's lock.
177#[implement(super::Service)]
178async fn forget_throttles(&self, user_id: &UserId, pushkey: &str) {
179	self.db
180		.senderkeyroom_emailthrottle
181		.del_prefix(&(user_id, pushkey, Interfix))
182		.await;
183}
184
185/// Writes a pusher's cursor, the count of the last notification row it
186/// processed.
187///
188/// Callers hold the pusher's lock.
189#[implement(super::Service)]
190fn put_cursor(&self, user_id: &UserId, pushkey: &str, cursor: u64) {
191	self.db
192		.senderkey_emailcursor
193		.put((user_id, pushkey), cursor);
194}
195
196/// Wakes the worker for one notification an email pusher may mail.
197///
198/// Nothing is sent while mail is off, so no pusher starts a cursor that would
199/// later mail everything since; a wake sent after the worker has stopped is
200/// dropped.
201#[implement(super::Service)]
202pub(super) fn wake_email(&self, user_id: &UserId, pushkey: &str) {
203	if !self.services.sendmail.is_enabled() {
204		return;
205	}
206
207	let frontier = self.services.globals.current_count();
208	let wake = (pusher_key(user_id, pushkey), frontier);
209
210	self.email_wakes.sender.send(wake).ok();
211}
212
213/// Closes the worker's channel, which ends the worker.
214///
215/// The worker first handles the wakes already sent.
216#[implement(super::Service)]
217pub(super) fn close_email_wakes(&self) { self.email_wakes.sender.close(); }
218
219/// Service worker: schedules woken email pushers and evaluates each when due.
220///
221/// Evaluations run one at a time. Once the channel closes at shutdown, the
222/// worker finishes the evaluation in flight and starts no other. It still
223/// handles every wake already sent, so each woken pusher keeps its cursor.
224#[implement(super::Service)]
225#[tracing::instrument(name = "email", level = "debug", skip_all)]
226pub(super) async fn email_worker(&self) {
227	let schedule = self.boot_schedule().await;
228
229	unfold(schedule, async |schedule| {
230		let schedule = match self.wait_next(&schedule).await? {
231			| Next::Wake(wake) => self.schedule_wake(schedule, wake).await,
232			| Next::Due(key) => self.evaluate_due(schedule, key).await,
233		};
234
235		Some(((), schedule))
236	})
237	.collect::<()>()
238	.await;
239}
240
241/// Schedules every email pusher with a cursor.
242///
243/// Mail pending across a restart then goes out without waiting for a fresh
244/// notification; a pusher with nothing left evaluates to idle.
245#[implement(super::Service)]
246#[tracing::instrument(level = "debug", skip_all)]
247async fn boot_schedule(&self) -> Schedule {
248	let entry = entry_in(BOOT_GRACE, 0);
249
250	self.db
251		.senderkey_emailcursor
252		.keys()
253		.ignore_err()
254		.map(|(user_id, pushkey): (&UserId, &str)| (pusher_key(user_id, pushkey), entry))
255		.collect()
256		.await
257}
258
259/// Waits for the next wake, or for the earliest scheduled pusher to fall due.
260///
261/// A pusher already due is returned before any waiting wake, so a steady run
262/// of wakes cannot hold it back. Once the channel is closed, only the wakes
263/// already sent are returned, then `None`.
264#[implement(super::Service)]
265async fn wait_next(&self, schedule: &Schedule) -> Option<Next> {
266	let receiver = &self.email_wakes.receiver;
267	let received = receiver
268		.recv_async()
269		.map(|wake| wake.ok().map(Next::Wake));
270
271	let Some((key, entry)) = schedule
272		.iter()
273		.min_by_key(|(_, entry)| entry.due)
274		.filter(|_| !receiver.is_closed())
275	else {
276		return received.await;
277	};
278
279	match select(pin!(sleep_until(entry.due)), pin!(received)).await {
280		| Either::Left(((), _)) if !receiver.is_closed() => Some(Next::Due(key.clone())),
281		| Either::Left(((), received)) => received.await,
282		| Either::Right((wake, _)) => wake,
283	}
284}
285
286/// Schedules a woken pusher one delay out, unless it is already due sooner or
287/// backing off.
288///
289/// It first starts the pusher's cursor when it has none, at the earliest
290/// frontier kept by a wake that could not read the cursor, else at this wake's.
291#[implement(super::Service)]
292async fn schedule_wake(&self, schedule: Schedule, (key, frontier): Wake) -> Schedule {
293	let frontier = kept_frontier(&schedule, &key).map_or(frontier, |kept| kept.min(frontier));
294	let kept = self.start_cursor(&key, frontier).await;
295	let delay = Duration::from_secs(self.services.config.push.email_delay);
296
297	woken(schedule, key, Entry { frontier: kept, ..entry_in(delay, 0) })
298}
299
300/// The frontier kept for a pusher whose cursor could not be started, if any.
301fn kept_frontier(schedule: &Schedule, key: &PusherKey) -> Option<u64> {
302	schedule.get(key)?.frontier
303}
304
305/// Starts the cursor of a woken pusher that has none, at `frontier`, returning
306/// the frontier when its read fails but may succeed on a retry.
307///
308/// A pusher stored before email notifications existed has none, so nothing
309/// written before its first wake is mailed, while a notification still being
310/// written when the wake was sent is. A pusher deleted since the wake gets one
311/// too, which its evaluation deletes. A returned frontier is kept to start the
312/// cursor again, since evaluating without one would skip the notification,
313/// while a cursor unreadable for good is left alone.
314#[implement(super::Service)]
315async fn start_cursor(&self, key: &PusherKey, frontier: u64) -> Option<u64> {
316	let _lock = self.email_locks.lock(key).await;
317	let (user_id, pushkey) = key;
318
319	match self.email_cursor(user_id, pushkey).await {
320		| Ok(Some(_)) => None,
321		| Err(error) => error.is_transient_io().then_some(frontier),
322		| Ok(None) => {
323			self.put_cursor(user_id, pushkey, frontier);
324
325			None
326		},
327	}
328}
329
330/// The pusher's cursor, or `None` when it has none.
331///
332/// A cursor that cannot be read is logged and is an error, never taken for its
333/// absence.
334#[implement(super::Service)]
335async fn email_cursor(&self, user_id: &UserId, pushkey: &str) -> Result<Option<u64>> {
336	self.db
337		.senderkey_emailcursor
338		.qry(&(user_id, pushkey))
339		.await
340		.deserialized()
341		.present()
342		.inspect_err(|error| warn!(%user_id, %error, "Failed to read an email cursor"))
343}
344
345/// Evaluates a pusher that has fallen due.
346///
347/// A frontier kept by an earlier wake starts the cursor first. While that still
348/// fails, the pusher backs off unevaluated, keeping the frontier.
349#[implement(super::Service)]
350async fn evaluate_due(&self, schedule: Schedule, key: PusherKey) -> Schedule {
351	let kept = kept_frontier(&schedule, &key)
352		.map_async(|frontier| self.start_cursor(&key, frontier))
353		.await
354		.flatten();
355
356	if kept.is_some() {
357		return reschedule(schedule, key, Outcome::Failed, kept);
358	}
359
360	let (user_id, pushkey) = &key;
361	let outcome = self
362		.evaluate_email(user_id, pushkey, now_millis())
363		.await;
364
365	reschedule(schedule, key, outcome, None)
366}
367
368/// Adds a woken pusher, or brings its entry forward to the sooner due time.
369///
370/// An entry counting failures keeps its backoff. The wake's frontier replaces
371/// the entry's; it already folds in any kept one.
372fn woken(mut schedule: Schedule, key: PusherKey, entry: Entry) -> Schedule {
373	schedule
374		.entry(key)
375		.and_modify(|current| {
376			current.frontier = entry.frontier;
377			if current.failures == 0 {
378				current.due = current.due.min(entry.due);
379			}
380		})
381		.or_insert(entry);
382
383	schedule
384}
385
386/// Applies an outcome to a pusher that fell due.
387///
388/// The outcome, an evaluation's or a failed cursor start's, decides the
389/// pusher's next entry, if any, carrying its failure count forward; the entry
390/// keeps the frontier of a cursor start that failed.
391fn reschedule(
392	mut schedule: Schedule,
393	key: PusherKey,
394	outcome: Outcome,
395	kept: Option<u64>,
396) -> Schedule {
397	let failures = schedule
398		.remove(&key)
399		.map_or(0, |entry| entry.failures);
400
401	let entry = next_entry(outcome, failures).map(|entry| Entry { frontier: kept, ..entry });
402
403	schedule.extend(entry.map(|entry| (key, entry)));
404	schedule
405}
406
407/// The entry a pusher gets after an outcome, if any.
408///
409/// A sent or idle pusher leaves the schedule, a pending one returns at its due
410/// time, and a failed one backs off from `RETRY_MIN` up to `RETRY_MAX`.
411fn next_entry(outcome: Outcome, failures: u32) -> Option<Entry> {
412	match outcome {
413		| Outcome::Sent | Outcome::Idle => None,
414		| Outcome::Pending(due) => {
415			let wait = Duration::from_millis(due.saturating_sub(now_millis()));
416
417			Some(entry_in(wait, 0))
418		},
419		| Outcome::Failed => {
420			let failures = failures.saturating_add(1);
421			let wait =
422				exponential_backoff_remaining(RETRY_MIN, RETRY_MAX, Duration::ZERO, failures)
423					.unwrap_or(RETRY_MIN);
424
425			Some(entry_in(wait, failures))
426		},
427	}
428}
429
430fn entry_in(wait: Duration, failures: u32) -> Entry {
431	let now = Instant::now();
432	let due = now.checked_add(wait.min(WAIT_MAX)).unwrap_or(now);
433
434	Entry { due, failures, frontier: None }
435}