1mod event;
2mod head;
3mod ping;
4mod state;
5#[cfg(test)]
6mod tests;
7mod version;
8
9use std::{
10 borrow::Cow,
11 cmp::Ordering,
12 collections::BTreeMap,
13 fmt::{Result as FmtResult, Write as _},
14 num::NonZeroUsize,
15 time::Duration,
16};
17
18use clap::{ArgAction, Args, Subcommand, ValueEnum};
19use futures::StreamExt;
20use ruma::{OwnedEventId, OwnedRoomId, OwnedRoomOrAliasId, OwnedServerName, OwnedUserId};
21use tuwunel_core::{
22 Err, Result, implement,
23 utils::{
24 stream::ReadyExt,
25 string::{markdown_cell, plural},
26 time::Elapsed,
27 },
28};
29use tuwunel_service::federation::{
30 PeerBackoff,
31 feds::{Fault, Opts, Outcome, Record},
32};
33
34use self::version::Field;
35use crate::{Context, admin_command_dispatch};
36
37pub(super) type Backoffs = BTreeMap<OwnedServerName, PeerBackoff>;
38
39#[admin_command_dispatch(handler_prefix = "feds")]
44#[derive(Debug, Subcommand)]
45pub(crate) enum FedsCommand {
46 Version {
48 room: OwnedRoomOrAliasId,
49
50 #[arg(long = "field", value_enum)]
52 fields: Vec<Field>,
53
54 #[arg(long, group = "version_list")]
56 list: bool,
57
58 #[arg(long, group = "version_list")]
60 list_all: bool,
61
62 #[arg(long, group = "version_list")]
64 list_errors: bool,
65
66 #[arg(long, value_enum, default_value_t, requires = "version_list")]
68 sort: Sort,
69
70 #[command(flatten)]
71 sweep: SweepArgs,
72 },
73
74 Ping {
79 room: OwnedRoomOrAliasId,
80
81 #[arg(long, group = "ping_list")]
83 list: bool,
84
85 #[arg(long, group = "ping_list")]
87 list_all: bool,
88
89 #[arg(long, group = "ping_list")]
91 list_errors: bool,
92
93 #[arg(long, value_enum, default_value_t, requires = "ping_list")]
95 sort: Sort,
96
97 #[command(flatten)]
98 sweep: SweepArgs,
99 },
100
101 Event {
106 event_id: OwnedEventId,
107
108 room: Option<OwnedRoomOrAliasId>,
112
113 #[arg(long, default_value_t = true, action = ArgAction::Set)]
117 verify_hash: bool,
118
119 #[arg(long, default_value_t = true, action = ArgAction::Set)]
123 verify_signature: bool,
124
125 #[command(flatten)]
126 sweep: SweepArgs,
127 },
128
129 State {
131 room: OwnedRoomOrAliasId,
132
133 #[arg(long)]
136 at: Option<OwnedEventId>,
137
138 #[arg(long)]
140 auth_chain: bool,
141
142 #[arg(long)]
144 full: bool,
145
146 #[command(flatten)]
147 sweep: SweepArgs,
148 },
149
150 Head {
155 room: OwnedRoomOrAliasId,
156
157 #[arg(long)]
159 probe_user: Option<OwnedUserId>,
160
161 #[command(flatten)]
162 sweep: SweepArgs,
163 },
164}
165
166#[derive(Clone, Copy, Debug, Args)]
171pub(crate) struct SweepArgs {
172 #[arg(long)]
174 width: Option<NonZeroUsize>,
175
176 #[arg(long, default_value_t = 10)]
178 timeout: u64,
179
180 #[arg(long, default_value_t = 120)]
182 budget: u64,
183
184 #[arg(long)]
186 no_loopback: bool,
187
188 #[arg(long)]
190 yes_i_want_to_do_this: bool,
191}
192
193#[derive(Clone, Copy, Debug, Default, Eq, PartialEq, ValueEnum)]
197pub(crate) enum Sort {
198 #[default]
200 Origin,
201
202 Elapsed,
204
205 Fault,
207}
208
209#[derive(Clone, Copy)]
210pub(super) enum ListMode {
211 None,
212 Successes,
213 All,
214 Errors,
215}
216
217pub(super) struct Prepared {
218 pub(super) room_id: OwnedRoomId,
219 pub(super) opts: Opts,
220}
221
222pub(super) async fn prepare(
223 context: &Context<'_>,
224 room: &OwnedRoomOrAliasId,
225 sweep: SweepArgs,
226 default_width: NonZeroUsize,
227) -> Result<Prepared> {
228 context
229 .services
230 .server
231 .config
232 .require_federation()?;
233
234 let room_id = context.services.alias.maybe_resolve(room).await?;
235 let destinations = context
236 .services
237 .state_cache
238 .room_servers(&room_id)
239 .ready_filter(|server| {
240 !sweep.no_loopback || !context.services.globals.server_is_ours(server)
241 })
242 .count()
243 .await;
244
245 let destination_limit = context
246 .services
247 .server
248 .config
249 .federation
250 .fanout
251 .destination_limit;
252
253 if destinations > destination_limit && !sweep.yes_i_want_to_do_this {
254 return Err!(
255 "Room has {destinations} feds destinations, exceeding the safety cap of \
256 {destination_limit}. Pass the confirmation flag to continue."
257 );
258 }
259
260 writeln!(context, "Querying {destinations} servers in {room_id}.\n").await?;
261
262 let opts = Opts {
263 width: Some(sweep.width.unwrap_or(default_width)),
264 timeout: Some(Duration::from_secs(sweep.timeout)),
265 sweep_deadline: Some(Duration::from_secs(sweep.budget)),
266 exclude_self: sweep.no_loopback,
267 record: Record::Observe,
268 };
269
270 Ok(Prepared { room_id, opts })
271}
272
273#[implement(ListMode)]
274pub(super) fn new(list: bool, list_all: bool, list_errors: bool) -> Self {
275 match (list, list_all, list_errors) {
276 | (true, false, false) => Self::Successes,
277 | (false, true, false) => Self::All,
278 | (false, false, true) => Self::Errors,
279 | _ => Self::None,
280 }
281}
282
283#[implement(ListMode)]
284pub(super) fn includes<T>(self, outcome: &Outcome<T>) -> bool {
285 match self {
286 | Self::None => false,
287 | Self::Successes => outcome.result.is_ok(),
288 | Self::All => true,
289 | Self::Errors => outcome.result.is_err(),
290 }
291}
292
293pub(super) async fn partition_backoffs<T>(
298 context: &Context<'_>,
299 prepared: &Prepared,
300 backoffs: &Backoffs,
301 now: u64,
302) -> (Vec<OwnedServerName>, Vec<Outcome<T>>) {
303 context
304 .services
305 .state_cache
306 .room_servers(&prepared.room_id)
307 .ready_filter(|server| {
308 !prepared.opts.exclude_self || !context.services.globals.server_is_ours(server)
309 })
310 .map(ToOwned::to_owned)
311 .ready_fold((Vec::new(), Vec::new()), |(mut eligible, mut outcomes), origin| {
312 match backoffs
313 .get(&origin)
314 .and_then(|backoff| backoff_fault(backoff, now))
315 {
316 | None => eligible.push(origin),
317 | Some(fault) => outcomes.push(Outcome {
318 origin,
319 elapsed: Duration::ZERO,
320 result: Err(fault),
321 }),
322 }
323
324 (eligible, outcomes)
325 })
326 .await
327}
328
329pub(super) fn backoff_fault(backoff: &PeerBackoff, now: u64) -> Option<Fault> {
334 retry_after(backoff, now).map(|retry| Fault::Backoff {
335 class: backoff.class,
336 age: Duration::from_secs(now.saturating_sub(backoff.oldest_secs)),
337 retry,
338 })
339}
340
341pub(super) fn retry_after(backoff: &PeerBackoff, now: u64) -> Option<Duration> {
346 let retry_at = backoff
347 .anchor_secs
348 .saturating_add(backoff.delay_secs);
349
350 retry_at
351 .gt(&now)
352 .then(|| Duration::from_secs(retry_at.saturating_sub(now)))
353}
354
355pub(super) fn sorted<T>(
356 mut outcomes: Vec<Outcome<T>>,
357 sort: Sort,
358 fault_key: impl Fn(&Outcome<T>) -> Cow<'static, str>,
359) -> Vec<Outcome<T>> {
360 outcomes.sort_by(|left, right| left.origin.cmp(&right.origin));
362 match sort {
363 | Sort::Origin => {},
364 | Sort::Elapsed => outcomes.sort_by_key(|outcome| outcome.elapsed),
365 | Sort::Fault => outcomes.sort_by_cached_key(fault_key),
366 }
367
368 outcomes
369}
370
371pub(super) fn fault_message(fault: &Fault) -> Cow<'static, str> {
372 match fault {
373 | Fault::Elapsed => Cow::Borrowed("request deadline exceeded"),
374 | Fault::NotAttempted => Cow::Borrowed("sweep budget exhausted before dispatch"),
375 | Fault::Backoff { class, age, retry } => Cow::Owned(format!(
376 "peer backoff ({class:?}, age {}, retry {})",
377 Elapsed::from(*age),
378 Elapsed::from(*retry),
379 )),
380 | Fault::Error(error) => Cow::Owned(format!("{:?}: {}", error.kind(), error.message())),
381 }
382}
383
384pub(super) fn count_results<T>(outcomes: &[Outcome<T>]) -> usize {
385 outcomes
386 .iter()
387 .filter(|outcome| outcome.result.is_ok())
388 .count()
389}
390
391pub(super) fn render_totals(
392 output: &mut String,
393 results: usize,
394 duration: Duration,
395) -> FmtResult {
396 let noun = plural(results, "result", "results");
397
398 writeln!(output, "\n{results} {noun} in {}.", Elapsed::from(duration))
399}
400
401pub(super) fn write_elapsed_cell<T>(output: &mut String, outcome: &Outcome<T>) -> FmtResult {
406 if matches!(&outcome.result, Err(Fault::NotAttempted | Fault::Backoff { .. })) {
407 write!(output, " |")
408 } else {
409 write!(output, " {} |", Elapsed::from(outcome.elapsed))
410 }
411}
412
413pub(super) fn write_cell(output: &mut String, value: &str) -> FmtResult {
414 if value.is_empty() {
415 write!(output, " |")
416 } else {
417 write!(output, " {value} |")
418 }
419}
420
421pub(super) fn sorted_event_id_difference(
422 remote: &[OwnedEventId],
423 local: &[OwnedEventId],
424) -> Cow<'static, str> {
425 let mut remote = remote.iter().peekable();
426 let mut local = local.iter().peekable();
427 let mut added = 0_usize;
428 let mut missing = 0_usize;
429
430 while let (Some(remote_id), Some(local_id)) = (remote.peek(), local.peek()) {
431 match remote_id.cmp(local_id) {
432 | Ordering::Less => {
433 added = added.saturating_add(1);
434 _ = remote.next();
435 },
436 | Ordering::Greater => {
437 missing = missing.saturating_add(1);
438 _ = local.next();
439 },
440 | Ordering::Equal => {
441 _ = remote.next();
442 _ = local.next();
443 },
444 }
445 }
446
447 added = added.saturating_add(remote.count());
448 missing = missing.saturating_add(local.count());
449
450 match (added, missing) {
451 | (0, 0) => Cow::Borrowed("="),
452 | _ => Cow::Owned(format!("+{added}/-{missing}")),
453 }
454}