Skip to main content

tuwunel_service/sending/sender/
mod.rs

1mod dispatch;
2mod netburst;
3mod queue;
4mod response;
5mod select;
6mod split;
7#[cfg(test)]
8mod tests;
9mod wake;
10
11use std::{
12	cmp::Reverse,
13	collections::{BinaryHeap, HashMap},
14	sync::Arc,
15	time::{Duration, Instant},
16};
17
18use futures::{FutureExt, StreamExt, stream::FuturesUnordered};
19use tokio::{
20	select,
21	time::{Instant as TokioInstant, sleep_until},
22};
23use tuwunel_core::{
24	Result, implement,
25	smallvec::{SmallVec, smallvec},
26	trace,
27	utils::BoolExt,
28};
29
30use self::{
31	dispatch::{Completion, SendingFuture},
32	select::Selection,
33	split::Split,
34	wake::{arm_park_wake, arm_wake, is_armed},
35};
36use super::{Destination, Msg, SendingEvent, Service, data::QueueItem};
37
38/// In-flight bookkeeping for one `Destination`.
39///
40/// Federation waits out a refusal and follows the sender curve otherwise.
41/// Appservice and push retain their own destination status across retries.
42#[derive(Debug)]
43enum TransactionStatus {
44	/// A durable active generation awaiting its first dispatch after restart.
45	Pending,
46
47	/// A transaction is in flight after this many consecutive failures.
48	Running {
49		tries: u32,
50	},
51
52	/// As `Running`, with a forced retry requested while it runs.
53	RunningForceRetry {
54		tries: u32,
55	},
56
57	/// Push backoff: consecutive failures and the time of the last one.
58	Failed {
59		tries: u32,
60		last: Instant,
61	},
62
63	/// Retry state after this many consecutive failures.
64	///
65	/// For push the retry is already in flight; for other destinations the
66	/// batch waits for its replay.
67	Retrying {
68		tries: u32,
69	},
70
71	/// As `Retrying`, while a rejected transaction's rooms are sent apart.
72	Splitting {
73		tries: u32,
74		split: Split,
75	},
76}
77
78#[derive(Clone, Copy)]
79enum RetryAction {
80	None,
81	Force,
82}
83
84type SendingFutures<'a> = FuturesUnordered<SendingFuture<'a>>;
85type TransactionStatuses = HashMap<Destination, TransactionStatus>;
86
87/// The queue items one request brings to a selection.
88///
89/// A request carries one item; a badge wake dequeues up to `DEQUEUE_LIMIT`.
90type NewEvents = SmallVec<[QueueItem; 1]>;
91
92/// Per-worker retry timer keyed by earliest-retry deadline and destination.
93///
94/// Every federation, appservice, or push failure arms an entry. Traffic-triggered retries
95/// can leave stale entries, which skip an in-flight transaction or replay a
96/// waiting one. Multiple entries may therefore remain for one destination.
97type WakeQueue = BinaryHeap<Reverse<(TokioInstant, Destination)>>;
98
99const DEQUEUE_LIMIT: usize = 48;
100
101/// Most PDUs one federation transaction may carry.
102///
103/// The spec caps a `/send` body at this many PDUs; inbound bodies past it are
104/// rejected and outbound composition stays under it.
105pub const PDU_LIMIT: usize = 50;
106
107/// Most EDUs one federation transaction may carry.
108///
109/// The spec caps a `/send` body at this many EDUs; inbound bodies past it are
110/// rejected and outbound composition stays under it.
111pub const EDU_LIMIT: usize = 100;
112
113/// Most bytes of serialized EDUs one federation transaction may carry.
114///
115/// A peer refuses a body past its request limit, and refused EDU rows ride along
116/// with every later transaction to it, so EDUs past this are dropped instead.
117/// The smallest limit Synapse accepts is 200 units of 64 KiB; each PDU fits in
118/// one unit and the envelope gets one, leaving the rest for EDUs.
119pub(super) const EDU_BYTES_LIMIT: usize = (200 - PDU_LIMIT - 1) * 65_536;
120
121#[implement(Service)]
122#[tracing::instrument(skip(self), level = "debug")]
123pub(super) async fn sender(self: Arc<Self>, id: usize) -> Result {
124	// The worker's state, threaded as &mut through every phase.
125	let mut statuses = TransactionStatuses::new();
126	let mut futures = SendingFutures::new();
127	let mut wakes = WakeQueue::new();
128
129	self.startup_netburst(id, &mut futures, &mut statuses, &mut wakes)
130		.boxed() // size firewall
131		.await;
132
133	self.work_loop(id, &mut futures, &mut statuses, &mut wakes)
134		.await;
135
136	if !futures.is_empty() {
137		self.finish_responses(&mut futures)
138			.boxed() // size firewall
139			.await;
140	}
141
142	Ok(())
143}
144
145#[implement(Service)]
146#[tracing::instrument(
147	name = "work",
148	level = "trace",
149	skip_all,
150	fields(
151		futures = %futures.len(),
152		statuses = %statuses.len(),
153	),
154)]
155async fn work_loop<'a>(
156	&'a self,
157	id: usize,
158	futures: &mut SendingFutures<'a>,
159	statuses: &mut TransactionStatuses,
160	wakes: &mut WakeQueue,
161) {
162	let receiver = &self
163		.channels
164		.get(id)
165		.expect("Missing channel for sender worker")
166		.1;
167
168	let controls = &self
169		.controls
170		.get(id)
171		.expect("Missing sender controls")
172		.1;
173
174	while !receiver.is_closed() {
175		let next_due = wakes
176			.peek()
177			.map_or_else(TokioInstant::now, |Reverse((instant, _))| *instant);
178
179		select! {
180			Some(response) = futures.next() => {
181				self.handle_response(response, futures, statuses, wakes).await;
182			},
183			control = controls.recv_async() => match control {
184				Ok(control) => self.handle_control(control, futures, statuses, wakes).await,
185				Err(_) => return,
186			},
187			request = receiver.recv_async() => match request {
188				Ok(request) => self.handle_request(request, futures, statuses, wakes).await,
189				Err(_) => return,
190			},
191			() = sleep_until(next_due), if !wakes.is_empty() => {
192				self.drain_due_wakes(futures, statuses, wakes).await;
193			},
194		}
195	}
196}
197
198#[implement(Service)]
199#[tracing::instrument(name = "request", level = "debug", skip_all)]
200async fn handle_request<'a>(
201	&'a self,
202	msg: Msg,
203	futures: &mut SendingFutures<'a>,
204	statuses: &mut TransactionStatuses,
205	wakes: &mut WakeQueue,
206) {
207	let synthetic_badge =
208		msg.queue_id.is_empty() && matches!(&msg.event, SendingEvent::BadgeRefresh);
209
210	let new_events = match (synthetic_badge, statuses.contains_key(&msg.dest)) {
211		| (false, _) => smallvec![(msg.queue_id, msg.event)],
212		| (true, true) => NewEvents::new(),
213		| (true, false) =>
214			self.db
215				.queued_requests(&msg.dest)
216				.take(DEQUEUE_LIMIT)
217				.collect()
218				.await,
219	};
220
221	if let Ok(selection) = self
222		.select_events(&msg.dest, new_events, statuses)
223		.await
224	{
225		self.schedule_events(msg.dest, selection, futures, statuses, wakes);
226	}
227}
228
229#[implement(Service)]
230#[expect(
231	clippy::needless_pass_by_ref_mut,
232	reason = "mutable reference avoids requiring SendingFutures to be Sync"
233)]
234fn schedule_events<'a>(
235	&'a self,
236	dest: Destination,
237	selection: Selection,
238	futures: &mut SendingFutures<'a>,
239	statuses: &mut TransactionStatuses,
240	wakes: &mut WakeQueue,
241) {
242	match selection {
243		| Selection::Events(items) if items.is_empty() => {
244			statuses.remove(&dest);
245		},
246		| Selection::Events(items) => futures.push(self.send_events(dest, items, None)),
247		| Selection::Slice(items, split) =>
248			futures.push(self.send_events(dest, items, Some(split))),
249		| Selection::Parked { until } => {
250			statuses.remove(&dest);
251			arm_park_wake(wakes, dest, until);
252		},
253		| Selection::Refused { earliest_retry } if is_armed(wakes, &dest).is_false() =>
254			arm_wake(wakes, dest, earliest_retry),
255		| Selection::Refused { .. } | Selection::Busy => {},
256	}
257}
258
259#[implement(Service)]
260#[tracing::instrument(
261	name = "finish",
262	level = "info",
263	skip_all,
264	fields(
265		futures = %futures.len(),
266	),
267)]
268async fn finish_responses<'a>(&'a self, futures: &mut SendingFutures<'a>) {
269	let timeout = Duration::from_secs(self.server.config.sending.shutdown_timeout);
270	let now = TokioInstant::now();
271	let deadline = now.checked_add(timeout).unwrap_or(now);
272
273	loop {
274		trace!(remaining = futures.len(), "Waiting for requests to complete");
275		select! {
276			() = sleep_until(deadline) => return,
277			response = futures.next() => match response {
278				Some(Completion { result: Ok(_), keys, .. }) =>
279					self.db.delete_active_requests(&keys),
280				Some(_) => {},
281				None => return,
282			},
283		}
284	}
285}