Skip to main content

tuwunel_service/federation/
feds.rs

1//! Bounded federation fanout for surveys and broadcasts.
2//!
3//! The driver preserves one typed outcome per destination while leaving
4//! destination policy, validation, and result aggregation to its callers.
5
6mod fold;
7#[cfg(test)]
8mod tests;
9
10use std::{num::NonZeroUsize, time::Duration};
11
12use futures::{Stream, StreamExt, future::Either};
13use ruma::{OwnedServerName, RoomId, ServerName, api::OutgoingRequest};
14use tokio::time::{Instant, timeout};
15use tuwunel_core::{
16	Error, Result, implement,
17	utils::{
18		math::effective_cap,
19		stream::{BroadbandExt, ReadyExt},
20	},
21};
22
23/// Re-exports federation fanout fold results and stream helpers.
24///
25/// Callers use these types to aggregate completion-ordered outcomes while
26/// retaining per-origin failures.
27pub use self::fold::{Faults, Grid, Origins, OutcomeExt, Tally};
28use super::{
29	Classification,
30	scheme::{FedAuth, FedPath},
31};
32
33const WIDTH_DEFAULT: NonZeroUsize = NonZeroUsize::new(32).expect("width is nonzero");
34const TIMEOUT_DEFAULT: Duration = Duration::from_secs(15);
35
36/// Policy for one federation fanout.
37///
38/// Service methods resolve omitted limits from configuration, while direct
39/// [`fanout_with`] calls use built-in limits. Room sweeps include the local
40/// server by default. No sweep deadline is set and peer status is untouched.
41#[derive(Clone, Copy, Debug, Default)]
42pub struct Opts {
43	/// Maximum number of requests in flight.
44	pub width: Option<NonZeroUsize>,
45
46	/// Deadline for each destination request.
47	pub timeout: Option<Duration>,
48
49	/// Wall-clock budget for dispatching the complete sweep.
50	pub sweep_deadline: Option<Duration>,
51
52	/// Remove the local server before constructing a room sweep.
53	pub exclude_self: bool,
54
55	/// Select whether outcomes update federation peer status.
56	pub record: Record,
57}
58
59/// Peer-status behavior for one federation fanout.
60///
61/// Diagnostic surveys observe outcomes without changing routing state. Existing
62/// traffic can opt into ordinary success and failure recording.
63#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)]
64pub enum Record {
65	/// Leave peer status untouched.
66	#[default]
67	Observe,
68
69	/// Record successes and classifiable failures.
70	Contribute,
71}
72
73/// One destination's settled federation request.
74///
75/// The origin and elapsed duration remain available for successful responses
76/// and every failure class.
77#[derive(Debug)]
78pub struct Outcome<R> {
79	/// Server contacted by this request.
80	pub origin: OwnedServerName,
81
82	/// Time from dispatch until settlement.
83	pub elapsed: Duration,
84
85	/// Typed response or the reason no response was available.
86	pub result: Result<R, Fault>,
87}
88
89/// Reason a federation destination produced no response.
90///
91/// Deadlines remain distinct from transport and protocol errors so folds can
92/// account for attempted and undispatched destinations separately.
93#[derive(Debug)]
94pub enum Fault {
95	/// The destination request exceeded its deadline.
96	Elapsed,
97
98	/// The sweep deadline expired before dispatch.
99	NotAttempted,
100
101	/// Peer health suppressed dispatch until its current backoff expires.
102	Backoff {
103		/// Classification of the newest surviving peer failure.
104		class: Classification,
105
106		/// Age of the surviving failure streak.
107		age: Duration,
108
109		/// Remaining delay before the peer becomes eligible.
110		retry: Duration,
111	},
112
113	/// Federation transport or protocol failure.
114	Error(Error),
115}
116
117/// Surveys every server participating in a room.
118///
119/// The local server participates unless `exclude_self` removes it before the
120/// sweep. Cursor-backed server names become owned before concurrent polling.
121#[implement(super::Service)]
122pub fn for_room<'a, F, R>(
123	&'a self,
124	room_id: &'a RoomId,
125	make: F,
126	opts: Opts,
127) -> impl Stream<Item = Outcome<R::IncomingResponse>> + Send + 'a
128where
129	F: Fn(&ServerName) -> R + Send + 'a,
130	R: OutgoingRequest + Send + 'a,
131	R::IncomingResponse: Send,
132	R::Authentication: FedAuth,
133	R::PathBuilder: FedPath,
134{
135	let dests = self
136		.services
137		.state_cache
138		.room_servers(room_id)
139		.ready_filter(move |server| {
140			!opts.exclude_self || !self.services.globals.server_is_ours(server)
141		})
142		.map(ToOwned::to_owned);
143
144	self.fanout_to(dests, make, opts)
145}
146
147/// Builds one request for every destination and drives the resulting pairs.
148///
149/// Request construction remains sequential and lazy, while only the returned
150/// federation futures are polled concurrently.
151#[implement(super::Service)]
152pub fn fanout_to<'a, D, F, R>(
153	&'a self,
154	dests: D,
155	make: F,
156	opts: Opts,
157) -> impl Stream<Item = Outcome<R::IncomingResponse>> + Send + 'a
158where
159	D: Stream<Item = OwnedServerName> + Send + 'a,
160	F: Fn(&ServerName) -> R + Send + 'a,
161	R: OutgoingRequest + Send + 'a,
162	R::IncomingResponse: Send,
163	R::Authentication: FedAuth,
164	R::PathBuilder: FedPath,
165{
166	let pairs = dests.map(move |origin| {
167		let request = make(&origin);
168
169		(origin, request)
170	});
171
172	self.fanout(pairs, opts)
173}
174
175/// Drives arbitrary destination and federation request pairs.
176///
177/// Every consumed pair settles as one completion-ordered outcome unless the
178/// caller drops the stream and cancels remaining work. The selected recording
179/// posture applies independently to each request.
180#[implement(super::Service)]
181#[expect(
182	closure_returning_async_block,
183	reason = "the capturing async closure would not implement the reusable Fn bound"
184)]
185pub fn fanout<'a, S, R>(
186	&'a self,
187	pairs: S,
188	opts: Opts,
189) -> impl Stream<Item = Outcome<R::IncomingResponse>> + Send + 'a
190where
191	S: Stream<Item = (OwnedServerName, R)> + Send + 'a,
192	R: OutgoingRequest + Send + 'a,
193	R::IncomingResponse: Send,
194	R::Authentication: FedAuth,
195	R::PathBuilder: FedPath,
196{
197	let config = &self.services.server.config;
198	let opts =
199		resolve_opts(opts, config.federation.fanout.max_width, config.federation.fanout.timeout);
200
201	let client = &self.services.client.federation;
202	let record = opts.record;
203
204	fanout_with(
205		pairs,
206		move |dest, request| async move {
207			match record {
208				| Record::Observe =>
209					self.execute_uncounted_allow_self(client, &dest, request)
210						.await,
211				| Record::Contribute =>
212					self.execute_on_allow_self(client, &dest, request)
213						.await,
214			}
215		},
216		opts,
217	)
218	.inspect(move |outcome| {
219		if record == Record::Contribute && matches!(&outcome.result, Err(Fault::Elapsed)) {
220			self.record_failure(&outcome.origin, Classification::Transient);
221		}
222	})
223}
224
225fn resolve_opts(opts: Opts, config_width: usize, config_timeout: u64) -> Opts {
226	let width = if opts.width.is_none() && config_width == 0 {
227		WIDTH_DEFAULT
228	} else {
229		NonZeroUsize::new(effective_cap(opts.width, config_width))
230			.expect("effective cap is nonzero")
231	};
232
233	Opts {
234		width: Some(width),
235		timeout: Some(
236			opts.timeout
237				.unwrap_or_else(|| Duration::from_secs(config_timeout)),
238		),
239		..opts
240	}
241}
242
243/// Drives destination and payload pairs through a supplied asynchronous send.
244///
245/// Results are yielded in completion order. Every consumed pair produces one
246/// outcome unless the caller drops the stream and cancels the remaining work.
247pub fn fanout_with<S, I, T, F, Fut>(
248	pairs: S,
249	send: F,
250	opts: Opts,
251) -> impl Stream<Item = Outcome<T>> + Send
252where
253	S: Stream<Item = (OwnedServerName, I)> + Send,
254	F: Fn(OwnedServerName, I) -> Fut + Send,
255	Fut: Future<Output = Result<T>> + Send,
256	I: Send,
257	T: Send,
258{
259	let width = opts.width.unwrap_or(WIDTH_DEFAULT).get();
260	let request_timeout = opts.timeout.unwrap_or(TIMEOUT_DEFAULT);
261	let deadline = opts
262		.sweep_deadline
263		.and_then(|duration| Instant::now().checked_add(duration));
264
265	pairs
266		.map(move |(origin, payload)| {
267			let start = Instant::now();
268
269			if deadline.is_some_and(|deadline| start >= deadline) {
270				Either::Left(Outcome {
271					origin,
272					elapsed: Duration::ZERO,
273					result: Err(Fault::NotAttempted),
274				})
275			} else {
276				let remaining = deadline.map_or(request_timeout, |deadline| {
277					deadline
278						.saturating_duration_since(start)
279						.min(request_timeout)
280				});
281				let future = send(origin.clone(), payload);
282
283				Either::Right((origin, start, remaining, future))
284			}
285		})
286		.broadn_then(width, async move |request| match request {
287			| Either::Left(outcome) => outcome,
288			| Either::Right((origin, start, remaining, future)) => {
289				let result = match timeout(remaining, future).await {
290					| Ok(result) => result.map_err(Fault::Error),
291					| Err(_elapsed) => Err(Fault::Elapsed),
292				};
293
294				Outcome { origin, elapsed: start.elapsed(), result }
295			},
296		})
297}