Skip to main content

tuwunel_service/rooms/event_handler/
upgrade_outlier_pdu.rs

1use std::{borrow::Borrow, collections::HashMap, iter::once, sync::Arc, time::Instant};
2
3use futures::{FutureExt, StreamExt};
4use ruma::{
5	CanonicalJsonObject, EventId, OwnedEventId, RoomId, RoomVersionId, ServerName,
6	room_version_rules::RoomVersionRules,
7};
8use tuwunel_core::{
9	Result, debug, debug_info, debug_warn, implement, is_equal_to, trace,
10	utils::{
11		BoolExt,
12		stream::{BroadbandExt, ReadyExt},
13	},
14	warn,
15};
16use tuwunel_matrix::{
17	Event, PduEvent,
18	pdu::check_rules,
19	room::state::resolution::{AuthCheckOutcome, auth_check},
20	room_version,
21};
22
23use super::{
24	backoff::{Context, Disposition, UPGRADE_RETRY},
25	handle_prev_pdu::PrevUpgrade,
26	policy_server::PolicyCheck,
27	state_local_build::WalkMode,
28};
29use crate::rooms::{
30	state::{IdMapState, RoomMutexGuard, Trigger, prune_goal},
31	state_compressor::{CompressedState, HashSetCompressStateEvent},
32	timeline::{Incoming, RawPduId},
33};
34
35#[cfg(test)]
36mod tests;
37
38/// How the state at the incoming event was obtained, deciding the memo write
39/// at the upgrade site.
40#[derive(Clone, Copy)]
41enum ResolvedVia {
42	Derived,
43	Memo,
44	Local,
45	Fetch,
46}
47
48/// Outcome of re-examining an event that already carries a soft-fail marker.
49#[derive(Clone, Copy)]
50enum Standing {
51	/// Evaluate the event. `true` when a re-check already cleared a lapsed
52	/// marker, so the soft-fail computation is settled.
53	Evaluate(bool),
54
55	/// The standing verdict holds; leave the event withheld.
56	Withheld,
57}
58
59#[implement(super::Service)]
60#[tracing::instrument(
61	name = "upgrade",
62	level = "debug",
63	ret(level = "debug"),
64	skip_all,
65	fields(
66		lev = %recursion_level,
67	),
68)]
69pub(super) async fn upgrade_outlier_to_timeline_pdu(
70	&self,
71	PrevUpgrade {
72		origin,
73		room_id,
74		room_version,
75		recursion_level,
76		create_event_id,
77		..
78	}: PrevUpgrade<'_>,
79	incoming_pdu: PduEvent,
80	mut pdu_json: CanonicalJsonObject,
81) -> Result<Option<(RawPduId, bool)>> {
82	// Skip the PDU if we already have it as a timeline event
83	if let Ok(pdu_id) = self
84		.services
85		.timeline
86		.get_pdu_id(incoming_pdu.event_id())
87		.await
88	{
89		debug!(?pdu_id, "Exists.");
90		return Ok(Some((pdu_id, false)));
91	}
92
93	trace!("Upgrading to timeline pdu");
94
95	let timer = Instant::now();
96	let room_rules = room_version::rules(room_version)?;
97
98	trace!(format = ?room_rules.event_format, "Checking format");
99	check_rules(&pdu_json, &room_rules.event_format)?;
100
101	let Standing::Evaluate(cleared) = self
102		.soft_fail_standing(&incoming_pdu, &room_rules, &mut pdu_json)
103		.await?
104	else {
105		return Ok(None);
106	};
107
108	let (state_at_incoming_event, resolved_via) = self
109		.resolve_state_at_incoming_event(
110			origin,
111			room_id,
112			&incoming_pdu,
113			room_version,
114			recursion_level,
115			create_event_id,
116		)
117		.await?;
118
119	self.auth_check_outlier_pdu(room_id, &incoming_pdu, &room_rules, &state_at_incoming_event)
120		.await?;
121
122	let soft_fail_pre = !cleared
123		&& self
124			.compute_soft_fail(&incoming_pdu, &room_rules, &mut pdu_json)
125			.await?;
126
127	// 13. Use state resolution to find new room state
128	// We start looking at current room state now, so lets lock the room
129	trace!("Locking the room");
130	let state_lock = self.services.state.mutex.lock(room_id).await;
131
132	// 14. Check if the event passes auth based on the current room state.
133	let soft_fail_current_state = !self
134		.current_state_auth_passes(room_id, &incoming_pdu, &room_rules)
135		.await?;
136
137	let soft_fail = soft_fail_pre || soft_fail_current_state;
138
139	let mut extremities = self
140		.compute_remaining_extremities(room_id, &incoming_pdu)
141		.await;
142
143	let config = &self.services.server.config;
144	let max = config.rooms.extremities.max;
145	let len = extremities
146		.len()
147		.saturating_add(usize::from(!soft_fail));
148
149	if max > 0 && len > max {
150		let goal = prune_goal(
151			len,
152			max,
153			config.rooms.extremities.emergency_max,
154			config.rooms.extremities.prune_batch,
155		);
156
157		let summary = self
158			.services
159			.state
160			.prune_forward_extremities(room_id, &mut extremities, goal, Trigger::Receive)
161			.await;
162
163		debug!(?summary, "Pruned forward extremities over the cap.");
164	}
165
166	trace!("Compressing state...");
167	let state_ids_compressed: Arc<CompressedState> = self
168		.services
169		.state_compressor
170		.compress_state_events(
171			state_at_incoming_event
172				.iter()
173				.map(|(ssk, eid)| (ssk, eid.borrow())),
174		)
175		.collect()
176		.map(Arc::new)
177		.await;
178
179	// Memoize only complete local or fetched state after positional auth succeeds;
180	// soft-failed events cannot establish reusable state.
181	if matches!(resolved_via, ResolvedVia::Local | ResolvedVia::Fetch) && !soft_fail {
182		self.cache_resolved_state(room_id, incoming_pdu.event_id(), state_ids_compressed.clone())
183			.await;
184	}
185
186	// A soft-failed event is not a forward extremity, so it never drives the
187	// room's current state; only an accepted state event resolves forward.
188	let shortstatehash = self
189		.prepare_accepted_state_after(
190			room_id,
191			room_version,
192			&incoming_pdu,
193			&state_at_incoming_event,
194			soft_fail,
195			&state_lock,
196		)
197		.await?;
198
199	// We use the `state_at_event` instead of `state_after` so we accurately
200	// represent the state for this event.
201	trace!("Appending pdu to timeline");
202
203	// Incoming event will be referenced in prev_events unless soft-failed.
204	let incoming_extremity = once(incoming_pdu.event_id()).filter(|_| !soft_fail);
205
206	let extremities = extremities
207		.iter()
208		.map(Borrow::borrow)
209		.chain(incoming_extremity);
210
211	let pdu_id = self
212		.services
213		.timeline
214		.append_incoming_pdu(Incoming {
215			pdu: &incoming_pdu,
216			pdu_json,
217			new_room_leafs: extremities,
218			state_ids_compressed,
219			soft_fail,
220			shortstatehash,
221			state_lock: &state_lock,
222		})
223		.await?;
224
225	debug_assert!(
226		pdu_id.is_some() || soft_fail,
227		"Ok(None) returned by timeline for soft-failed PDU's"
228	);
229
230	if soft_fail {
231		self.services
232			.pdu_metadata
233			.mark_event_soft_failed(incoming_pdu.event_id());
234
235		self.record_outcome(Context::Upgrade, incoming_pdu.event_id(), Disposition::Transient);
236
237		drop(state_lock);
238		warn!(
239			event_id = %incoming_pdu.event_id(),
240			redact_or_policy = soft_fail_pre,
241			current_state = soft_fail_current_state,
242			elapsed = ?timer.elapsed(),
243			"Event was soft failed.",
244		);
245
246		return Ok(None);
247	}
248
249	drop(state_lock);
250
251	if cleared {
252		self.services
253			.pdu_metadata
254			.clear_event_soft_failed(incoming_pdu.event_id());
255
256		self.record_success(Context::Upgrade, incoming_pdu.event_id())
257			.await;
258	}
259
260	debug_info!(
261		elapsed = ?timer.elapsed(),
262		"Accepted",
263	);
264
265	Ok(pdu_id.zip(Some(true)))
266}
267
268#[implement(super::Service)]
269#[tracing::instrument(
270	name = "auth_current",
271	level = "debug",
272	skip_all,
273	fields(%room_id, event_id = %incoming_pdu.event_id()),
274)]
275async fn current_state_auth_passes(
276	&self,
277	room_id: &RoomId,
278	incoming_pdu: &PduEvent,
279	room_rules: &RoomVersionRules,
280) -> Result<bool> {
281	trace!("Gathering current-state auth events.");
282	let auth_events = self
283		.services
284		.state
285		.get_auth_events(
286			room_id,
287			incoming_pdu.kind(),
288			incoming_pdu.sender(),
289			incoming_pdu.state_key(),
290			incoming_pdu.content(),
291			&room_rules.authorization,
292			true,
293		)
294		.await;
295
296	trace!("Performing current-state auth check.");
297	let outcome = current_state_auth_outcome(auth_events, async move |auth_events| {
298		auth_check(room_rules, incoming_pdu, &*self.services.timeline, &auth_events).await
299	})
300	.await;
301
302	match outcome {
303		| Ok(AuthCheckOutcome::Allow) => Ok(true),
304		| Ok(AuthCheckOutcome::Deny(error)) => {
305			warn!(
306				auth_leg = "current_state",
307				event_id = %incoming_pdu.event_id(),
308				%room_id,
309				%error,
310				"Current-state auth check failed; soft-failing event.",
311			);
312
313			Ok(false)
314		},
315		| Err(error) => {
316			warn!(
317				auth_leg = "current_state",
318				event_id = %incoming_pdu.event_id(),
319				%room_id,
320				%error,
321				"Current-state auth check could not be evaluated.",
322			);
323
324			Err(error)
325		},
326	}
327}
328
329async fn current_state_auth_outcome<AuthEvents, Check, CheckFuture>(
330	auth_events: Result<AuthEvents>,
331	check: Check,
332) -> Result<AuthCheckOutcome>
333where
334	Check: FnOnce(AuthEvents) -> CheckFuture,
335	CheckFuture: Future<Output = Result<AuthCheckOutcome>>,
336{
337	check(auth_events?).await
338}
339
340/// Re-examines an event that already carries a soft-fail marker.
341///
342/// The marker is a standing verdict rather than a permanent rejection, so it
343/// lapses on the upgrade backoff and the event is weighed again. Asking before
344/// state resolution keeps a still-refused event cheap to decline, since a
345/// cached policy answer needs no round trip.
346#[implement(super::Service)]
347async fn soft_fail_standing(
348	&self,
349	incoming_pdu: &PduEvent,
350	room_rules: &RoomVersionRules,
351	pdu_json: &mut CanonicalJsonObject,
352) -> Result<Standing> {
353	let event_id = incoming_pdu.event_id();
354
355	if !self
356		.services
357		.pdu_metadata
358		.is_event_soft_failed(event_id)
359		.await
360	{
361		return Ok(Standing::Evaluate(false));
362	}
363
364	if self
365		.is_suppressed(Context::Upgrade, event_id, UPGRADE_RETRY)
366		.await
367		.is_deny()
368	{
369		debug!(%event_id, "Soft failed; deferring re-evaluation.");
370		return Ok(Standing::Withheld);
371	}
372
373	if self
374		.compute_soft_fail(incoming_pdu, room_rules, pdu_json)
375		.await?
376	{
377		self.record_outcome(Context::Upgrade, event_id, Disposition::Transient);
378
379		debug!(%event_id, "Still soft failed.");
380		return Ok(Standing::Withheld);
381	}
382
383	Ok(Standing::Evaluate(true))
384}
385
386#[implement(super::Service)]
387async fn resolve_state_at_incoming_event(
388	&self,
389	origin: &ServerName,
390	room_id: &RoomId,
391	incoming_pdu: &PduEvent,
392	room_version: &RoomVersionId,
393	recursion_level: usize,
394	create_event_id: &EventId,
395) -> Result<(HashMap<u64, OwnedEventId>, ResolvedVia)> {
396	// 10. Fetch missing state and auth chain events by calling /state_ids at
397	//     backwards extremities doing all the checks in this list starting at 1.
398	//     These are not timeline events.
399	trace!("Resolving state at event");
400
401	let state_at_incoming_event = if incoming_pdu.prev_events().count() == 1 {
402		self.state_at_incoming_degree_one(incoming_pdu)
403			.await?
404	} else {
405		self.state_at_incoming_resolved(incoming_pdu, room_id, room_version)
406			.boxed() // cold arm: multiple prev events
407			.await?
408	};
409
410	if let Some(state) = state_at_incoming_event {
411		return Ok((state, ResolvedVia::Derived));
412	}
413
414	let config = &self.services.server.config;
415	let enabled = config.federation.resolve_state_locally
416		&& config.federation.resolve_state_locally_max > 0
417		&& !config.federation.resolve_state_locally_shadow;
418
419	if enabled {
420		match self
421			.cached_resolved_state(incoming_pdu.event_id())
422			.await
423		{
424			| Ok(Some(state)) => return Ok((state, ResolvedVia::Memo)),
425			| Ok(None) => (),
426			| Err(error) => debug_warn!(
427				event_id = %incoming_pdu.event_id(),
428				%error,
429				"Failed to load resolved state memo.",
430			),
431		}
432	}
433
434	let local = enabled
435		.then_async(|| {
436			self.state_at_incoming_local(
437				room_id,
438				incoming_pdu,
439				room_version,
440				create_event_id,
441				WalkMode::Active,
442			)
443		})
444		.await
445		.transpose()?
446		.flatten();
447
448	if let Some(state) = local {
449		return Ok((state, ResolvedVia::Local));
450	}
451
452	let state = self
453		.fetch_state(
454			origin,
455			room_id,
456			incoming_pdu.event_id(),
457			room_version,
458			recursion_level,
459			create_event_id,
460		)
461		.boxed() // cold arm: federation fallback
462		.await?
463		.expect("fetch_state always resolves state to some");
464
465	Ok((state, ResolvedVia::Fetch))
466}
467
468#[implement(super::Service)]
469async fn auth_check_outlier_pdu(
470	&self,
471	room_id: &RoomId,
472	incoming_pdu: &PduEvent,
473	room_rules: &RoomVersionRules,
474	state_at_incoming_event: &HashMap<u64, OwnedEventId>,
475) -> Result {
476	// Every event holding a `shorteventid_shortstatehash` row passed spec check 5
477	// (auth against the state at its own position) as a hard reject; soft failure
478	// (spec check 6) still writes the row, so soft-failed events are valid fold
479	// inputs while positionally rejected events never gain a row.
480
481	let state_fetch = IdMapState {
482		services: &self.services,
483		ids: state_at_incoming_event,
484	};
485
486	trace!("Performing positional auth check.");
487	auth_check(room_rules, incoming_pdu, &*self.services.timeline, state_fetch)
488		.await
489		.and_then(AuthCheckOutcome::into_result)
490		.inspect_err(|error| {
491			warn!(
492				auth_leg = "positional",
493				event_id = %incoming_pdu.event_id(),
494				%room_id,
495				%error,
496				"Positional auth check failed.",
497			);
498		})
499}
500
501#[implement(super::Service)]
502async fn compute_soft_fail(
503	&self,
504	incoming_pdu: &PduEvent,
505	room_rules: &RoomVersionRules,
506	pdu_json: &mut CanonicalJsonObject,
507) -> Result<bool> {
508	// Soft fail check before doing state res
509	trace!("Performing soft-fail check");
510	let soft_fail_redact = match incoming_pdu.redacts_id(room_rules) {
511		| None => false,
512		| Some(redact_id) =>
513			!self
514				.services
515				.state_accessor
516				.user_can_redact(&redact_id, incoming_pdu.sender(), incoming_pdu.room_id(), true)
517				.await?,
518	};
519
520	// MSC4284: soft-fail when the policy server rejects the event.
521	Ok(soft_fail_redact
522		|| matches!(
523			self.verify_or_fetch_inbound_policy_signature(pdu_json, incoming_pdu)
524				.await,
525			PolicyCheck::Invalid,
526		))
527}
528
529#[implement(super::Service)]
530async fn compute_remaining_extremities(
531	&self,
532	room_id: &RoomId,
533	incoming_pdu: &PduEvent,
534) -> Vec<OwnedEventId> {
535	// Now we calculate the set of extremities this room has after the incoming
536	// event has been applied. We start with the previous extremities (aka leaves)
537	trace!("Calculating extremities");
538	let extremities: Vec<_> = self
539		.services
540		.state
541		.get_forward_extremities(room_id)
542		.ready_filter(|&event_id| {
543			// Remove any that are referenced by this incoming event's prev_events
544			!incoming_pdu
545				.prev_events()
546				.any(is_equal_to!(event_id))
547		})
548		.map(ToOwned::to_owned)
549		.broad_filter_map(async |event_id| {
550			// Only keep those extremities were not referenced yet
551			self.services
552				.pdu_metadata
553				.is_event_referenced(room_id, &event_id)
554				.await
555				.eq(&false)
556				.then_some(event_id)
557		})
558		.collect()
559		.await;
560
561	debug!(
562		retained = extremities.len(),
563		prev_events = incoming_pdu.prev_events().count(),
564		"Retained extremities checked against prev_events.",
565	);
566
567	extremities
568}
569
570#[implement(super::Service)]
571async fn prepare_accepted_state_after(
572	&self,
573	room_id: &RoomId,
574	room_version: &RoomVersionId,
575	incoming_pdu: &PduEvent,
576	state_at_incoming_event: &HashMap<u64, OwnedEventId>,
577	soft_fail: bool,
578	state_lock: &RoomMutexGuard,
579) -> Result<Option<u64>> {
580	incoming_pdu
581		.state_key()
582		.is_some()
583		.and_is(!soft_fail)
584		.then_async(|| {
585			self.resolve_and_prepare_state_after(
586				room_id,
587				room_version,
588				incoming_pdu,
589				state_at_incoming_event,
590				state_lock,
591			)
592			.boxed() // cold arm: state event
593		})
594		.map(Option::transpose)
595		.await
596}
597
598#[implement(super::Service)]
599async fn resolve_and_prepare_state_after(
600	&self,
601	room_id: &RoomId,
602	room_version: &RoomVersionId,
603	incoming_pdu: &PduEvent,
604	state_at_incoming_event: &HashMap<u64, OwnedEventId>,
605	state_lock: &RoomMutexGuard,
606) -> Result<u64> {
607	// We also add state after incoming event to the fork states
608	let mut state_after = state_at_incoming_event.clone();
609	if let Some(state_key) = incoming_pdu.state_key() {
610		let event_id = incoming_pdu.event_id();
611		let event_type = incoming_pdu.kind();
612		let shortstatekey = self
613			.services
614			.short
615			.get_or_create_shortstatekey(&event_type.to_string().into(), state_key)
616			.await;
617
618		state_after.insert(shortstatekey, event_id.to_owned());
619		// Now it's the state after the event.
620		debug!(
621			?event_id,
622			?event_type,
623			?state_key,
624			?shortstatekey,
625			state_after = state_after.len(),
626			"Adding event to state."
627		);
628	}
629
630	trace!("Resolving new room state.");
631	let new_room_state = self
632		.resolve_state(room_id, room_version, state_after)
633		.boxed() // size firewall
634		.await?;
635
636	trace!("Saving resolved state.");
637	let HashSetCompressStateEvent { shortstatehash, added, removed } = self
638		.services
639		.state_compressor
640		.save_state(room_id, new_room_state)
641		.await?;
642
643	debug!(
644		?shortstatehash,
645		added = added.len(),
646		removed = removed.len(),
647		"Preparing new room state."
648	);
649	self.services
650		.state
651		.prepare_force_state(room_id, &added, state_lock)
652		.await?;
653
654	Ok(shortstatehash)
655}