Skip to main content

tuwunel_admin/query/
feds.rs

1mod event;
2mod head;
3mod ping;
4mod state;
5#[cfg(test)]
6mod tests;
7mod version;
8
9use std::{
10	borrow::Cow,
11	cmp::Ordering,
12	collections::BTreeMap,
13	fmt::{Result as FmtResult, Write as _},
14	num::NonZeroUsize,
15	time::Duration,
16};
17
18use clap::{ArgAction, Args, Subcommand, ValueEnum};
19use futures::StreamExt;
20use ruma::{OwnedEventId, OwnedRoomId, OwnedRoomOrAliasId, OwnedServerName, OwnedUserId};
21use tuwunel_core::{
22	Err, Result, implement,
23	utils::{
24		stream::ReadyExt,
25		string::{markdown_cell, plural},
26		time::Elapsed,
27	},
28};
29use tuwunel_service::federation::{
30	PeerBackoff,
31	feds::{Fault, Opts, Outcome, Record},
32};
33
34use self::version::Field;
35use crate::{Context, admin_command_dispatch};
36
37pub(super) type Backoffs = BTreeMap<OwnedServerName, PeerBackoff>;
38
39/// Run feds diagnostics against every participating server in a room.
40///
41/// Each command bounds concurrency and time while reporting every successful
42/// response, transport failure, timeout, and undispatched destination.
43#[admin_command_dispatch(handler_prefix = "feds")]
44#[derive(Debug, Subcommand)]
45pub(crate) enum FedsCommand {
46	/// Compare implementation versions reported by participating servers.
47	Version {
48		room: OwnedRoomOrAliasId,
49
50		/// Select metadata fields used to group and display versions.
51		#[arg(long = "field", value_enum)]
52		fields: Vec<Field>,
53
54		/// List servers whose request did not produce an error.
55		#[arg(long, group = "version_list")]
56		list: bool,
57
58		/// List every server.
59		#[arg(long, group = "version_list")]
60		list_all: bool,
61
62		/// List servers whose request produced an error.
63		#[arg(long, group = "version_list")]
64		list_errors: bool,
65
66		/// Order the listed servers by this column.
67		#[arg(long, value_enum, default_value_t, requires = "version_list")]
68		sort: Sort,
69
70		#[command(flatten)]
71		sweep: SweepArgs,
72	},
73
74	/// Measure request latency to participating servers.
75	///
76	/// Reports latency distribution statistics beside the peer-status record
77	/// held for each destination.
78	Ping {
79		room: OwnedRoomOrAliasId,
80
81		/// List servers whose request did not produce an error.
82		#[arg(long, group = "ping_list")]
83		list: bool,
84
85		/// List every server.
86		#[arg(long, group = "ping_list")]
87		list_all: bool,
88
89		/// List servers whose request produced an error.
90		#[arg(long, group = "ping_list")]
91		list_errors: bool,
92
93		/// Order the listed servers by this column.
94		#[arg(long, value_enum, default_value_t, requires = "ping_list")]
95		sort: Sort,
96
97		#[command(flatten)]
98		sweep: SweepArgs,
99	},
100
101	/// Compare copies of one event reported by participating servers.
102	///
103	/// Reports transport latency and enabled validation results for every
104	/// destination.
105	Event {
106		event_id: OwnedEventId,
107
108		/// Select the room containing the event.
109		///
110		/// The local PDU supplies the room when this argument is omitted.
111		room: Option<OwnedRoomOrAliasId>,
112
113		/// Control event content-hash verification.
114		///
115		/// Verification is enabled by default.
116		#[arg(long, default_value_t = true, action = ArgAction::Set)]
117		verify_hash: bool,
118
119		/// Control verification of signatures required by the room version.
120		///
121		/// Verification is enabled by default.
122		#[arg(long, default_value_t = true, action = ArgAction::Set)]
123		verify_signature: bool,
124
125		#[command(flatten)]
126		sweep: SweepArgs,
127	},
128
129	/// Compare state event identifiers at one room event.
130	State {
131		room: OwnedRoomOrAliasId,
132
133		/// Anchor the request at this event; the local latest PDU is the
134		/// default.
135		#[arg(long)]
136		at: Option<OwnedEventId>,
137
138		/// Compare the auth-chain identifiers returned with the state response.
139		#[arg(long)]
140		auth_chain: bool,
141
142		/// Print every identifier in each equivalence class.
143		#[arg(long)]
144		full: bool,
145
146		#[command(flatten)]
147		sweep: SweepArgs,
148	},
149
150	/// Probe each server's room head with a make-join request.
151	///
152	/// A remote may lock room state, sign a template, and persist an otherwise
153	/// unused short event identifier while answering this diagnostic request.
154	Head {
155		room: OwnedRoomOrAliasId,
156
157		/// Use this local user for the probe instead of the diagnostic user.
158		#[arg(long)]
159		probe_user: Option<OwnedUserId>,
160
161		#[command(flatten)]
162		sweep: SweepArgs,
163	},
164}
165
166/// Resource governors shared by feds queries.
167///
168/// The defaults are intentionally narrower than the service defaults because
169/// a feds query blocks the serial command worker until it settles.
170#[derive(Clone, Copy, Debug, Args)]
171pub(crate) struct SweepArgs {
172	/// Maximum requests in flight.
173	#[arg(long)]
174	width: Option<NonZeroUsize>,
175
176	/// Per-destination deadline in seconds.
177	#[arg(long, default_value_t = 10)]
178	timeout: u64,
179
180	/// Whole-sweep budget in seconds.
181	#[arg(long, default_value_t = 120)]
182	budget: u64,
183
184	/// Remove this homeserver from the destination set.
185	#[arg(long)]
186	no_loopback: bool,
187
188	/// Confirm a feds query whose destination count exceeds the safety cap.
189	#[arg(long)]
190	yes_i_want_to_do_this: bool,
191}
192
193/// Column ordering the detail listing.
194///
195/// Rows equal under the chosen column keep their origin order.
196#[derive(Clone, Copy, Debug, Default, Eq, PartialEq, ValueEnum)]
197pub(crate) enum Sort {
198	/// Server name.
199	#[default]
200	Origin,
201
202	/// Request latency, fastest first.
203	Elapsed,
204
205	/// Failure message.
206	Fault,
207}
208
209#[derive(Clone, Copy)]
210pub(super) enum ListMode {
211	None,
212	Successes,
213	All,
214	Errors,
215}
216
217pub(super) struct Prepared {
218	pub(super) room_id: OwnedRoomId,
219	pub(super) opts: Opts,
220}
221
222pub(super) async fn prepare(
223	context: &Context<'_>,
224	room: &OwnedRoomOrAliasId,
225	sweep: SweepArgs,
226	default_width: NonZeroUsize,
227) -> Result<Prepared> {
228	context
229		.services
230		.server
231		.config
232		.require_federation()?;
233
234	let room_id = context.services.alias.maybe_resolve(room).await?;
235	let destinations = context
236		.services
237		.state_cache
238		.room_servers(&room_id)
239		.ready_filter(|server| {
240			!sweep.no_loopback || !context.services.globals.server_is_ours(server)
241		})
242		.count()
243		.await;
244
245	let destination_limit = context
246		.services
247		.server
248		.config
249		.federation
250		.fanout
251		.destination_limit;
252
253	if destinations > destination_limit && !sweep.yes_i_want_to_do_this {
254		return Err!(
255			"Room has {destinations} feds destinations, exceeding the safety cap of \
256			 {destination_limit}. Pass the confirmation flag to continue."
257		);
258	}
259
260	writeln!(context, "Querying {destinations} servers in {room_id}.\n").await?;
261
262	let opts = Opts {
263		width: Some(sweep.width.unwrap_or(default_width)),
264		timeout: Some(Duration::from_secs(sweep.timeout)),
265		sweep_deadline: Some(Duration::from_secs(sweep.budget)),
266		exclude_self: sweep.no_loopback,
267		record: Record::Observe,
268	};
269
270	Ok(Prepared { room_id, opts })
271}
272
273#[implement(ListMode)]
274pub(super) fn new(list: bool, list_all: bool, list_errors: bool) -> Self {
275	match (list, list_all, list_errors) {
276		| (true, false, false) => Self::Successes,
277		| (false, true, false) => Self::All,
278		| (false, false, true) => Self::Errors,
279		| _ => Self::None,
280	}
281}
282
283#[implement(ListMode)]
284pub(super) fn includes<T>(self, outcome: &Outcome<T>) -> bool {
285	match self {
286		| Self::None => false,
287		| Self::Successes => outcome.result.is_ok(),
288		| Self::All => true,
289		| Self::Errors => outcome.result.is_err(),
290	}
291}
292
293/// Splits the room's destinations into dispatchable origins and held origins.
294///
295/// An origin still inside its peer backoff at `now` becomes a `Fault::Backoff`
296/// outcome with zero elapsed time instead of a destination.
297pub(super) async fn partition_backoffs<T>(
298	context: &Context<'_>,
299	prepared: &Prepared,
300	backoffs: &Backoffs,
301	now: u64,
302) -> (Vec<OwnedServerName>, Vec<Outcome<T>>) {
303	context
304		.services
305		.state_cache
306		.room_servers(&prepared.room_id)
307		.ready_filter(|server| {
308			!prepared.opts.exclude_self || !context.services.globals.server_is_ours(server)
309		})
310		.map(ToOwned::to_owned)
311		.ready_fold((Vec::new(), Vec::new()), |(mut eligible, mut outcomes), origin| {
312			match backoffs
313				.get(&origin)
314				.and_then(|backoff| backoff_fault(backoff, now))
315			{
316				| None => eligible.push(origin),
317				| Some(fault) => outcomes.push(Outcome {
318					origin,
319					elapsed: Duration::ZERO,
320					result: Err(fault),
321				}),
322			}
323
324			(eligible, outcomes)
325		})
326		.await
327}
328
329/// Describes a peer's backoff as a fault while it has not expired at `now`.
330///
331/// The age spans the oldest surviving failure bucket and the retry is the
332/// remaining delay.
333pub(super) fn backoff_fault(backoff: &PeerBackoff, now: u64) -> Option<Fault> {
334	retry_after(backoff, now).map(|retry| Fault::Backoff {
335		class: backoff.class,
336		age: Duration::from_secs(now.saturating_sub(backoff.oldest_secs)),
337		retry,
338	})
339}
340
341/// Computes the remaining delay before the peer becomes eligible.
342///
343/// The delay is measured from the newest failure; an expired backoff yields
344/// `None`.
345pub(super) fn retry_after(backoff: &PeerBackoff, now: u64) -> Option<Duration> {
346	let retry_at = backoff
347		.anchor_secs
348		.saturating_add(backoff.delay_secs);
349
350	retry_at
351		.gt(&now)
352		.then(|| Duration::from_secs(retry_at.saturating_sub(now)))
353}
354
355pub(super) fn sorted<T>(
356	mut outcomes: Vec<Outcome<T>>,
357	sort: Sort,
358	fault_key: impl Fn(&Outcome<T>) -> Cow<'static, str>,
359) -> Vec<Outcome<T>> {
360	// Both secondary sorts are stable, so origin order remains the tie-breaker.
361	outcomes.sort_by(|left, right| left.origin.cmp(&right.origin));
362	match sort {
363		| Sort::Origin => {},
364		| Sort::Elapsed => outcomes.sort_by_key(|outcome| outcome.elapsed),
365		| Sort::Fault => outcomes.sort_by_cached_key(fault_key),
366	}
367
368	outcomes
369}
370
371pub(super) fn fault_message(fault: &Fault) -> Cow<'static, str> {
372	match fault {
373		| Fault::Elapsed => Cow::Borrowed("request deadline exceeded"),
374		| Fault::NotAttempted => Cow::Borrowed("sweep budget exhausted before dispatch"),
375		| Fault::Backoff { class, age, retry } => Cow::Owned(format!(
376			"peer backoff ({class:?}, age {}, retry {})",
377			Elapsed::from(*age),
378			Elapsed::from(*retry),
379		)),
380		| Fault::Error(error) => Cow::Owned(format!("{:?}: {}", error.kind(), error.message())),
381	}
382}
383
384pub(super) fn count_results<T>(outcomes: &[Outcome<T>]) -> usize {
385	outcomes
386		.iter()
387		.filter(|outcome| outcome.result.is_ok())
388		.count()
389}
390
391pub(super) fn render_totals(
392	output: &mut String,
393	results: usize,
394	duration: Duration,
395) -> FmtResult {
396	let noun = plural(results, "result", "results");
397
398	writeln!(output, "\n{results} {noun} in {}.", Elapsed::from(duration))
399}
400
401/// Writes the elapsed cell of a listing row.
402///
403/// A destination that was never dispatched has no latency and takes a blank
404/// cell.
405pub(super) fn write_elapsed_cell<T>(output: &mut String, outcome: &Outcome<T>) -> FmtResult {
406	if matches!(&outcome.result, Err(Fault::NotAttempted | Fault::Backoff { .. })) {
407		write!(output, " |")
408	} else {
409		write!(output, " {} |", Elapsed::from(outcome.elapsed))
410	}
411}
412
413pub(super) fn write_cell(output: &mut String, value: &str) -> FmtResult {
414	if value.is_empty() {
415		write!(output, " |")
416	} else {
417		write!(output, " {value} |")
418	}
419}
420
421pub(super) fn sorted_event_id_difference(
422	remote: &[OwnedEventId],
423	local: &[OwnedEventId],
424) -> Cow<'static, str> {
425	let mut remote = remote.iter().peekable();
426	let mut local = local.iter().peekable();
427	let mut added = 0_usize;
428	let mut missing = 0_usize;
429
430	while let (Some(remote_id), Some(local_id)) = (remote.peek(), local.peek()) {
431		match remote_id.cmp(local_id) {
432			| Ordering::Less => {
433				added = added.saturating_add(1);
434				_ = remote.next();
435			},
436			| Ordering::Greater => {
437				missing = missing.saturating_add(1);
438				_ = local.next();
439			},
440			| Ordering::Equal => {
441				_ = remote.next();
442				_ = local.next();
443			},
444		}
445	}
446
447	added = added.saturating_add(remote.count());
448	missing = missing.saturating_add(local.count());
449
450	match (added, missing) {
451		| (0, 0) => Cow::Borrowed("="),
452		| _ => Cow::Owned(format!("+{added}/-{missing}")),
453	}
454}