1use std::{
8 collections::{HashMap, VecDeque},
9 sync::{Arc, Weak},
10};
11
12use bytes::Bytes;
13use futures::{FutureExt, StreamExt, future::BoxFuture, stream::FuturesUnordered};
14use ruma::OwnedServerName;
15use tokio::sync::watch::channel;
16use tuwunel_core::{debug_warn, implement, trace, utils::math::effective_cap};
17
18use super::{
19 Failure, Msg, Opts, Outcome, Service,
20 error::Attempted,
21 inflight::{Inflight, Key, SharedResult},
22};
23
24type FetchFuture<'a> = BoxFuture<'a, (Key, SharedResult)>;
27type FetchFutures<'a> = FuturesUnordered<FetchFuture<'a>>;
28
29#[implement(Service)]
34pub(super) async fn run_worker(self: Arc<Self>) {
35 let mut inflight: HashMap<Key, Inflight> = HashMap::new();
36 let mut pending: VecDeque<Msg> = VecDeque::new();
37 let mut futures: FetchFutures<'_> = FuturesUnordered::new();
38
39 self.work_loop(&mut inflight, &mut pending, &mut futures)
40 .await;
41}
42
43#[implement(Service)]
44async fn work_loop<'a>(
45 &'a self,
46 inflight: &mut HashMap<Key, Inflight>,
47 pending: &mut VecDeque<Msg>,
48 futures: &mut FetchFutures<'a>,
49) {
50 let rx = self.channel.1.clone();
51 while !rx.is_closed() {
52 while let Ok(msg) = rx.try_recv() {
54 self.on_request(msg, inflight, pending, futures);
55 }
56
57 tokio::select! {
58 Some((key, result)) = futures.next() =>
59 self.on_complete(key, result, inflight, pending, futures),
60 msg = rx.recv_async() => match msg {
61 | Ok(msg) => self.on_request(msg, inflight, pending, futures),
62 | Err(_) => break,
63 },
64 }
65 }
66}
67
68#[implement(Service)]
69fn on_request<'a>(
70 &'a self,
71 msg: Msg,
72 inflight: &mut HashMap<Key, Inflight>,
73 pending: &mut VecDeque<Msg>,
74 futures: &FetchFutures<'a>,
75) {
76 let Some(entry) = inflight.get_mut(&msg.key) else {
77 if futures.len() >= self.capacity {
79 pending.push_back(msg);
80 } else {
81 self.dispatch(msg, inflight, futures);
82 }
83
84 return;
85 };
86
87 match entry.interest.upgrade() {
88 | Some(strong) => {
90 msg.reply
91 .send((entry.tx.subscribe(), strong))
92 .ok();
93 },
94 | None => {
97 let interest = Arc::new(());
98 entry.interest = Arc::downgrade(&interest);
99 msg.reply
100 .send((entry.tx.subscribe(), interest))
101 .ok();
102 },
103 }
104}
105
106#[implement(Service)]
107fn dispatch<'a>(
108 &'a self,
109 msg: Msg,
110 inflight: &mut HashMap<Key, Inflight>,
111 futures: &FetchFutures<'a>,
112) {
113 let Msg { key, reply } = msg;
114 let interest = Arc::new(());
115 let (tx, rx) = channel(None);
116
117 if reply.send((rx, interest.clone())).is_err() {
119 return;
120 }
121
122 let opts = key.opts();
123 let weak = Arc::downgrade(&interest);
124 inflight.insert(key.clone(), Inflight {
125 tx,
126 interest: weak.clone(),
127 opts: opts.clone(),
128 });
129
130 self.push_attempt(futures, key, opts, weak);
131}
132
133#[implement(Service)]
137fn push_attempt<'a>(
138 &'a self,
139 futures: &FetchFutures<'a>,
140 key: Key,
141 opts: Arc<Opts>,
142 weak: Weak<()>,
143) {
144 futures.push(async move { (key, self.run_attempts(&opts, &weak).await) }.boxed());
146}
147
148#[implement(Service)]
149fn on_complete<'a>(
150 &'a self,
151 key: Key,
152 result: SharedResult,
153 inflight: &mut HashMap<Key, Inflight>,
154 pending: &mut VecDeque<Msg>,
155 futures: &FetchFutures<'a>,
156) {
157 let Some(entry) = inflight.get(&key) else {
158 return;
159 };
160
161 if matches!(&result, Err(Failure::Cancelled)) && entry.interest.upgrade().is_some() {
164 let opts = entry.opts.clone();
165 let weak = entry.interest.clone();
166 self.push_attempt(futures, key, opts, weak);
167 return;
168 }
169
170 entry.tx.send(Some(result)).ok();
171 inflight.remove(&key);
172
173 while futures.len() < self.capacity {
176 let Some(msg) = pending.pop_front() else {
177 break;
178 };
179
180 self.on_request(msg, inflight, pending, futures);
181 }
182}
183
184#[implement(Service)]
185#[tracing::instrument(
186 name = "attempts",
187 level = "debug",
188 skip_all,
189 fields(
190 op = ?opts.op,
191 room_id = ?opts.room_id,
192 event_id = ?opts.event_id,
193 ),
194)]
195async fn run_attempts(&self, opts: &Opts, interest: &Weak<()>) -> SharedResult {
196 let candidates = self.select.candidates(opts).await;
197 if candidates.is_empty() {
198 return Err(Failure::NoCandidates);
199 }
200
201 let count = candidates.len();
202 let limit = opts
203 .attempt_limit
204 .map_or(count, |n| n.get().min(count));
205
206 let (config_width, config_rounds) = self
207 .services
208 .try_get()
209 .map_or((0, 0), |services| {
210 let config = &services.server.config;
211
212 (
213 config.federation.history.fetch_fanout_max_width,
214 config.federation.history.fetch_fanout_rounds,
215 )
216 });
217
218 let max_width = effective_cap(opts.fanout_max_width, config_width);
219 let max_rounds = effective_cap(opts.fanout_rounds, config_rounds);
220
221 let mut attempted: Attempted = Attempted::new();
222 let mut remaining = candidates.into_iter();
223 let mut round: usize = 0;
224
225 while attempted.len() < limit {
226 if interest.strong_count() == 0 {
227 return Err(Failure::Cancelled);
228 }
229
230 if round >= max_rounds {
231 break;
232 }
233
234 let budget = limit.saturating_sub(attempted.len());
235 let width = opts
236 .fanout_growth
237 .round_width(round)
238 .min(max_width)
239 .min(budget);
240
241 let mut racing: FuturesUnordered<_> = remaining
244 .by_ref()
245 .take(width)
246 .map(|server| self.attempt(server, opts))
247 .collect();
248
249 if racing.is_empty() {
250 break;
251 }
252
253 while let Some((server, bytes)) = racing.next().await {
254 let Some(bytes) = bytes else {
255 attempted.push(server);
256
257 if interest.strong_count() == 0 {
258 return Err(Failure::Cancelled);
259 }
260
261 continue;
262 };
263
264 trace!(%server, "fetch satisfied");
265 return Ok(Arc::new(Outcome { bytes, origin: server }));
266 }
267
268 round = round.saturating_add(1);
269 }
270
271 Err(Failure::NotFound { attempted })
272}
273
274#[implement(Service)]
278#[tracing::instrument(
279 name = "attempt",
280 level = "trace",
281 skip_all,
282 fields(%server),
283)]
284async fn attempt(
285 &self,
286 server: OwnedServerName,
287 opts: &Opts,
288) -> (OwnedServerName, Option<Bytes>) {
289 let Some(bytes) = self
290 .transport
291 .fetch_raw(opts.op, &server, opts)
292 .await
293 .inspect_err(|error| debug_warn!(%server, "fetch attempt failed: {error}"))
294 .ok()
295 else {
296 return (server, None);
297 };
298
299 let valid = self
300 .validate(opts, &bytes)
301 .await
302 .inspect_err(|error| debug_warn!(%server, "rejecting poisoned response: {error}"))
303 .is_ok();
304
305 (server, valid.then_some(bytes))
306}