Skip to main content

tuwunel_service/sending/
mod.rs

1mod data;
2mod dest;
3mod device;
4mod queue;
5mod sender;
6#[cfg(test)]
7mod tests;
8mod worker;
9
10use std::{
11	collections::HashMap,
12	io::Write,
13	iter::{once, repeat_with},
14	sync::{Arc, Mutex as StdMutex},
15	time::{Duration, Instant},
16};
17
18use async_trait::async_trait;
19use futures::{Stream, StreamExt};
20use loole::{Receiver, Sender, unbounded};
21use ruma::{OwnedServerName, RoomId, ServerName, UserId};
22use tokio::task::JoinSet;
23use tuwunel_core::{
24	Result, Server, debug_warn, implement,
25	smallvec::SmallVec,
26	utils::{IterStream, ReadyExt, TryReadyExt, result::LogErr},
27};
28
29pub use self::{
30	data::{Data, Park},
31	dest::Destination,
32	sender::{EDU_LIMIT, PDU_LIMIT},
33};
34use self::{queue::Control, worker::num_senders};
35use crate::{
36	appservice::{RegistrationInfo, receivers},
37	rooms::timeline::RawPduId,
38};
39
40type Channel<T> = (Sender<T>, Receiver<T>);
41
42type StalledDestinations = StdMutex<HashMap<OwnedServerName, Option<Instant>>>;
43
44/// Outbound delivery of PDUs and EDUs to federation peers, appservices, and
45/// push gateways.
46///
47/// Requests are written as durable queue rows and dispatched to a pool of
48/// sender workers sharded by destination.
49pub struct Service {
50	pub db: Data,
51	server: Arc<Server>,
52	services: Arc<crate::services::OnceServices>,
53	channels: Vec<Channel<Msg>>,
54	controls: Vec<Channel<Control>>,
55
56	// Aborted and joined when the service stops.
57	flushes: StdMutex<JoinSet<()>>,
58
59	// One entry per worker-owned federation destination waiting for replay.
60	stalled: StalledDestinations,
61}
62
63/// One queued unit of delivery.
64///
65/// PDUs are referenced by ID; the EDU variants carry their serialized body.
66#[expect(clippy::module_name_repetitions)]
67#[derive(Clone, Debug, PartialEq, Eq, Hash)]
68pub enum SendingEvent {
69	/// A room event, fetched from the timeline by ID at dispatch.
70	Pdu(RawPduId),
71
72	/// A serialized EDU body.
73	Edu(EduBuf),
74
75	/// A serialized to-device message for an appservice (MSC4203).
76	ToDevice(EduBuf),
77
78	/// A serialized device-list change for an appservice (MSC3202).
79	DeviceListChanged(EduBuf),
80
81	/// Queue an account-wide counts-only push.
82	///
83	/// The sender recomputes the count when the row is delivered.
84	BadgeRefresh,
85
86	/// Wake the destination without queueing anything.
87	Flush,
88}
89
90#[derive(Clone, Debug, PartialEq, Eq)]
91struct Msg {
92	dest: Destination,
93	event: SendingEvent,
94	queue_id: Vec<u8>,
95}
96
97/// Inline buffer for one serialized EDU.
98///
99/// The budget keeps a typical EDU on the stack; a larger body spills to the
100/// heap.
101pub type EduBuf = SmallVec<[u8; EDU_BUF_CAP]>;
102
103/// Inline collection of the EDU buffers composed for one transaction.
104///
105/// Most transactions carry at most one EDU.
106pub type EduVec = SmallVec<[EduBuf; EDU_VEC_CAP]>;
107
108const EDU_BUF_CAP: usize = 128 - 16;
109const EDU_VEC_CAP: usize = 1;
110
111// Leading bytes on queued sending values select tagged event variants. Legacy
112// PDU and EDU rows cannot collide; the badge tag stands alone.
113const TAG_TO_DEVICE: u8 = 0x01;
114const TAG_DEVICE_LIST_CHANGED: u8 = 0x02;
115const TAG_BADGE_REFRESH: u8 = 0x03;
116const TAG_PREFIX_LEN: usize = 1 + size_of::<u64>();
117
118#[async_trait]
119impl crate::Service for Service {
120	fn build(args: &crate::Args<'_>) -> Result<Arc<Self>> {
121		let controls = repeat_with(unbounded)
122			.take(num_senders(args))
123			.collect();
124
125		let channels = repeat_with(unbounded)
126			.take(num_senders(args))
127			.collect();
128
129		Ok(Arc::new(Self {
130			db: Data::new(args),
131			server: args.server.clone(),
132			services: args.services.clone(),
133			channels,
134			controls,
135			flushes: JoinSet::new().into(),
136			stalled: HashMap::new().into(),
137		}))
138	}
139
140	async fn worker(self: Arc<Self>) -> Result { self.run().await }
141
142	async fn interrupt(&self) { self.close(); }
143
144	fn name(&self) -> &str { crate::service::make_name(std::module_path!()) }
145}
146
147/// Queue a PDU for delivery to one of a user's pushers.
148///
149/// The row is durable and the shard owning the destination is woken.
150#[implement(Service)]
151#[tracing::instrument(skip(self, pdu_id, user, pushkey), level = "debug")]
152pub fn send_pdu_push(&self, pdu_id: &RawPduId, user: &UserId, pushkey: String) -> Result {
153	let dest = Destination::Push(user.to_owned(), pushkey);
154	let event = SendingEvent::Pdu(*pdu_id);
155	let _cork = self.db.db.cork();
156
157	self.queue_and_dispatch(dest, event)
158}
159
160#[implement(Service)]
161fn queue_and_dispatch(&self, dest: Destination, event: SendingEvent) -> Result {
162	let queue_id = self
163		.db
164		.queue_requests(once((&event, &dest)))
165		.pop()
166		.expect("request queue key");
167
168	self.dispatch(Msg { dest, event, queue_id })
169}
170
171/// Queue a counts-only push refresh for every pusher owned by a user.
172///
173/// Rows are durable, coalesced, and recomputed at send time.
174#[implement(Service)]
175#[tracing::instrument(level = "debug", skip(self))]
176pub async fn refresh_push_badge(&self, user_id: &UserId) -> Result {
177	self.services
178		.pusher
179		.get_pushkeys(user_id)
180		.map(Ok)
181		.ready_try_for_each(|pushkey| {
182			let dest = Destination::Push(user_id.to_owned(), pushkey.to_owned());
183
184			self.queue_and_dispatch(dest, SendingEvent::BadgeRefresh)
185		})
186		.await
187}
188
189/// Queue a PDU for delivery to an appservice.
190///
191/// The row is durable and the shard owning the destination is woken.
192#[implement(Service)]
193#[tracing::instrument(skip(self), level = "debug")]
194pub fn send_pdu_appservice(&self, appservice_id: String, pdu_id: RawPduId) -> Result {
195	let dest = Destination::Appservice(appservice_id);
196	let event = SendingEvent::Pdu(pdu_id);
197	let _cork = self.db.db.cork();
198
199	self.queue_and_dispatch(dest, event)
200}
201
202/// Queue a PDU for delivery to every remote server in a room.
203///
204/// The fan-out is one durable row per server, dispatched under a single cork.
205#[implement(Service)]
206#[tracing::instrument(skip(self, room_id, pdu_id), level = "debug")]
207pub async fn send_pdu_room(&self, room_id: &RoomId, pdu_id: &RawPduId) -> Result {
208	let servers = self
209		.services
210		.state_cache
211		.remote_room_servers(room_id);
212
213	self.send_pdu_servers(servers, pdu_id).await
214}
215
216/// Queue a PDU for delivery to each of the given servers.
217///
218/// The fan-out is one durable row per server, dispatched under a single cork.
219#[implement(Service)]
220#[tracing::instrument(skip(self, servers, pdu_id), level = "debug")]
221pub async fn send_pdu_servers<'a, S>(&self, servers: S, pdu_id: &RawPduId) -> Result
222where
223	S: Stream<Item = &'a ServerName> + Send + 'a,
224{
225	self.queue_and_dispatch_servers(servers, SendingEvent::Pdu(*pdu_id))
226		.await
227}
228
229#[implement(Service)]
230async fn queue_and_dispatch_servers<'a, S>(&self, servers: S, event: SendingEvent) -> Result
231where
232	S: Stream<Item = &'a ServerName> + Send + 'a,
233{
234	let requests: Vec<_> = servers
235		.map(|server| (event.clone(), Destination::Federation(server.to_owned())))
236		.collect()
237		.await;
238
239	let _cork = self.db.db.cork();
240	let keys = self
241		.db
242		.queue_requests(requests.iter().map(|(event, dest)| (event, dest)));
243
244	requests
245		.into_iter()
246		.zip(keys)
247		.try_for_each(|((event, dest), queue_id)| self.dispatch(Msg { dest, event, queue_id }))
248}
249
250/// Queue an EDU for delivery to a server.
251///
252/// The row is durable and the shard owning the destination is woken.
253#[implement(Service)]
254#[tracing::instrument(skip(self, server, serialized), level = "debug")]
255pub fn send_edu_server(&self, server: &ServerName, serialized: EduBuf) -> Result {
256	let dest = Destination::Federation(server.to_owned());
257	let event = SendingEvent::Edu(serialized);
258	let _cork = self.db.db.cork();
259
260	self.queue_and_dispatch(dest, event)
261}
262
263/// Queue an EDU for delivery to every remote server in a room.
264///
265/// The fan-out is one durable row per server, dispatched under a single cork.
266#[implement(Service)]
267#[tracing::instrument(skip(self, room_id, serialized), level = "debug")]
268pub async fn send_edu_room(&self, room_id: &RoomId, serialized: EduBuf) -> Result {
269	let servers = self
270		.services
271		.state_cache
272		.remote_room_servers(room_id);
273
274	self.send_edu_servers(servers, serialized).await
275}
276
277/// Queue an EDU for delivery to each of the given servers.
278///
279/// The fan-out is one durable row per server, dispatched under a single cork.
280#[implement(Service)]
281#[tracing::instrument(skip(self, servers, serialized), level = "debug")]
282pub async fn send_edu_servers<'a, S>(&self, servers: S, serialized: EduBuf) -> Result
283where
284	S: Stream<Item = &'a ServerName> + Send + 'a,
285{
286	self.queue_and_dispatch_servers(servers, SendingEvent::Edu(serialized))
287		.await
288}
289
290/// Queue an EDU for every appservice interested in a room.
291///
292/// An appservice is interested when it has a URL and receives ephemeral
293/// events, and the room matches it: the room is in its namespace, the
294/// appservice is present in the room, or one of the room's local aliases
295/// matches. The serializer writes `EphemeralData`, not a federation `Edu`,
296/// once per matching appservice.
297#[implement(Service)]
298// Stream::filter names one future type independent of the item borrow, which
299// an async closure's future would capture.
300#[expect(closure_returning_async_block)]
301#[tracing::instrument(skip(self, serializer), level = "debug")]
302pub async fn send_edu_room_appservices<'a, F>(&self, room_id: &RoomId, serializer: F) -> Result
303where
304	F: Fn(&mut dyn Write) -> Result + Send + 'a,
305	&'a F: Send + Sync,
306{
307	let registrations = self.services.appservice.read().await;
308	// fn pointer: a closure fails the future's Send check ("FnOnce is not general enough")
309	let receives_ephemeral: fn(&&RegistrationInfo) -> bool =
310		|appservice| appservice.registration.receive_ephemeral;
311
312	receivers(&registrations)
313		.filter(receives_ephemeral)
314		.stream()
315		.filter(|&appservice| async move {
316			if appservice.rooms.is_match(room_id.as_str()) {
317				return true;
318			}
319
320			if self
321				.services
322				.state_cache
323				.appservice_in_room(room_id, appservice)
324				.await
325			{
326				return true;
327			}
328
329			self.services
330				.alias
331				.local_aliases_for_room(room_id)
332				.ready_any(|room_alias| appservice.aliases.is_match(room_alias.as_str()))
333				.await
334		})
335		.map(Ok)
336		.ready_try_for_each(|appservice| {
337			let mut buf = EduBuf::new(); // serializer out-param
338
339			serializer(&mut buf)?;
340			self.send_edu_appservice(appservice.registration.id.clone(), buf)
341				.log_err()
342				.ok();
343
344			Ok(())
345		})
346		.await
347}
348
349/// Queue an EDU for delivery to a specific appservice.
350///
351/// The row is durable and the shard owning the destination is woken.
352#[implement(Service)]
353#[tracing::instrument(skip(self, serialized), level = "debug")]
354pub fn send_edu_appservice(&self, appservice_id: String, serialized: EduBuf) -> Result {
355	let dest = Destination::Appservice(appservice_id);
356	let event = SendingEvent::Edu(serialized);
357	let _cork = self.db.db.cork();
358
359	self.queue_and_dispatch(dest, event)
360}
361
362/// Wake the sender for every remote server in a room.
363///
364/// A flush is not queued as a row; it only prompts the shard to compose a
365/// transaction from whatever is pending.
366#[implement(Service)]
367#[tracing::instrument(skip(self, room_id), level = "debug")]
368pub async fn flush_room(&self, room_id: &RoomId) -> Result {
369	let servers = self
370		.services
371		.state_cache
372		.remote_room_servers(room_id);
373
374	self.flush_servers(servers).await
375}
376
377/// Wake the sender for each of the given servers.
378///
379/// A flush is not queued as a row; it only prompts the shard to compose a
380/// transaction from whatever is pending.
381#[implement(Service)]
382#[tracing::instrument(skip(self, servers), level = "debug")]
383pub async fn flush_servers<'a, S>(&self, servers: S) -> Result
384where
385	S: Stream<Item = &'a ServerName> + Send + 'a,
386{
387	servers
388		.map(ToOwned::to_owned)
389		.map(Destination::Federation)
390		.map(Ok)
391		.ready_try_for_each(|dest| self.dispatch_flush(dest))
392		.await
393}
394
395#[implement(Service)]
396fn dispatch_flush(&self, dest: Destination) -> Result {
397	self.dispatch(Msg {
398		dest,
399		event: SendingEvent::Flush,
400		queue_id: Vec::new(),
401	})
402}
403
404/// Wake the sender for an appservice.
405///
406/// A flush is not queued as a row; it only prompts the shard to compose a
407/// transaction from whatever is pending.
408#[implement(Service)]
409#[tracing::instrument(skip(self), level = "debug")]
410pub fn flush_appservice(&self, appservice_id: String) -> Result {
411	self.dispatch_flush(Destination::Appservice(appservice_id))
412}
413
414/// Wake the sender for a federation peer that has proven reachable.
415///
416/// Reachability comes from inbound activity or an operator reset. The flush
417/// resumes a waiting sender generation after its notification floor, or
418/// immediately when peer failure rows existed. The return value reports only
419/// whether those peer rows existed.
420#[implement(Service)]
421#[tracing::instrument(
422	level = "debug",
423	skip(self),
424	fields(
425		%server,
426	),
427)]
428pub async fn notify_peer_alive(&self, server: &ServerName) -> bool {
429	let sad = self
430		.services
431		.federation
432		.note_peer_alive(server)
433		.await;
434
435	let replay = sad
436		|| self
437			.stalled
438			.lock()
439			.expect("locked")
440			.get(server)
441			.is_some_and(|last| {
442				last.is_none_or(|last| {
443					last.elapsed()
444						.ge(&Duration::from_secs(self.server.config.sending.timeout))
445				})
446			});
447
448	if replay {
449		self.dispatch_flush(Destination::Federation(server.to_owned()))
450			.log_err()
451			.ok();
452	}
453
454	sad
455}
456
457/// Clean up queued sending event data.
458///
459/// Accepts either an appservice ID alone, after its registration is removed,
460/// or a user ID with a push key, after the pusher is deleted; any other
461/// combination is ignored with a warning.
462#[implement(Service)]
463#[tracing::instrument(skip(self), level = "debug")]
464pub async fn cleanup_events(
465	&self,
466	appservice_id: Option<&str>,
467	user_id: Option<&UserId>,
468	push_key: Option<&str>,
469) -> Result {
470	let dest = match (appservice_id, user_id, push_key) {
471		| (None, Some(user_id), Some(push_key)) =>
472			Destination::Push(user_id.to_owned(), push_key.to_owned()),
473		| (Some(appservice_id), None, None) => Destination::Appservice(appservice_id.to_owned()),
474		| _ => {
475			debug_warn!("cleanup_events called with too many or too few arguments");
476			return Ok(());
477		},
478	};
479
480	self.db.delete_all_requests_for(&dest).await
481}