Skip to main content

tuwunel_service/rooms/timeline/
append.rs

1//! Appends authenticated events to room timelines and applies their side effects.
2//!
3//! Incoming events first receive a state snapshot, while accepted events are
4//! assigned a normal stream count and committed to the timeline. Subsequent
5//! cache, indexing, notification, and membership effects are coordinated here.
6
7use std::{collections::btree_map::Entry, sync::Arc};
8
9use ruma::{
10	CanonicalJsonObject, CanonicalJsonValue, EventId, UserId,
11	events::{
12		TimelineEventType,
13		receipt::ReceiptThread,
14		relation::RelationType,
15		room::{
16			encrypted::Relation,
17			member::{MembershipState, RoomMemberEventContent},
18		},
19	},
20};
21use tuwunel_core::{
22	Result, debug_warn, err, error, implement,
23	smallvec::SmallVec,
24	utils::{
25		OptionExt,
26		result::{LogErr, NotFound},
27	},
28};
29use tuwunel_database::Json;
30use tuwunel_matrix::{
31	event::Event,
32	pdu::{PduCount, PduEvent, PduId, RawPduId},
33	room_version,
34};
35
36use super::{ExtractBody, ExtractRelatesTo, ExtractRelatesToEventId, RoomMutexGuard, bias_count};
37use crate::{
38	admin::CommandInput,
39	rooms::{
40		read_receipt::PrivateRead,
41		short::{ShortRoomId, ShortStateHash},
42		state_accessor::plain_text_topic,
43		state_cache::MembershipUpdate,
44		state_compressor::CompressedState,
45	},
46};
47
48/// Inputs for admitting a federated event to the timeline.
49///
50/// Event state precedes the event, while current state is installed after its
51/// accepted row exists and before its stream position becomes visible.
52pub(crate) struct Incoming<'a, Leafs> {
53	/// Event being admitted.
54	pub(crate) pdu: &'a PduEvent,
55
56	/// Canonical event representation.
57	pub(crate) pdu_json: CanonicalJsonObject,
58
59	/// Replacement forward extremities.
60	pub(crate) new_room_leafs: Leafs,
61
62	/// State immediately preceding the event.
63	pub(crate) state_ids_compressed: Arc<CompressedState>,
64
65	/// Whether current-state authorization withheld acceptance.
66	pub(crate) soft_fail: bool,
67
68	/// Prepared current state for an accepted state event.
69	pub(crate) shortstatehash: Option<ShortStateHash>,
70
71	/// Guard serializing the room's state transition.
72	pub(crate) state_lock: &'a RoomMutexGuard,
73}
74
75type Band<'a> = SmallVec<[&'a EventId; 1]>;
76
77struct Appended<'a> {
78	pdu_id: RawPduId,
79	pdu: &'a PduEvent,
80	shortroomid: ShortRoomId,
81	count: PduCount,
82
83	/// Stream position of the same user's join a member event replaced, if any.
84	replaced_join: Option<PduCount>,
85}
86
87/// Appends an incoming event with its locally resolved state snapshot.
88///
89/// The snapshot is recorded even when the event is soft-failed. A soft-failed
90/// event is not inserted into the accepted timeline, but its predecessors are
91/// marked referenced. Only a nonempty replacement extremity band is stored; an
92/// empty calculation preserves the prior band.
93#[implement(super::Service)]
94#[tracing::instrument(
95	name = "append_incoming",
96	level = "debug",
97	skip_all,
98	ret(Debug)
99)]
100pub(crate) async fn append_incoming_pdu<'a, Leafs>(
101	&'a self,
102	Incoming {
103		pdu,
104		pdu_json,
105		new_room_leafs,
106		state_ids_compressed,
107		soft_fail,
108		shortstatehash,
109		state_lock,
110	}: Incoming<'a, Leafs>,
111) -> Result<Option<RawPduId>>
112where
113	Leafs: Iterator<Item = &'a EventId> + Send + 'a,
114{
115	// The event's preceding snapshot is distinct from resolved current state.
116	// Store it before the timeline row; install current state before retirement.
117	self.services
118		.state
119		.set_event_state(&pdu.event_id, &pdu.room_id, state_ids_compressed)
120		.await?;
121
122	if soft_fail {
123		self.services
124			.pdu_metadata
125			.mark_as_referenced(&pdu.room_id, pdu.prev_events.iter().map(AsRef::as_ref));
126
127		// Keep the previous band rather than let a soft-failed event empty it; a
128		// later accepted event self-chains and heals it.
129		if let Some(new_room_leafs) = nonempty_band(new_room_leafs) {
130			self.services
131				.state
132				.set_forward_extremities(&pdu.room_id, new_room_leafs.into_iter(), state_lock)
133				.await;
134		}
135
136		return Ok(None);
137	}
138
139	let pdu_id = self
140		.append_pdu(pdu, pdu_json, new_room_leafs, shortstatehash, state_lock)
141		.await?;
142
143	Ok(Some(pdu_id))
144}
145
146fn nonempty_band<'a, Leafs>(leafs: Leafs) -> Option<Band<'a>>
147where
148	Leafs: Iterator<Item = &'a EventId>,
149{
150	let leafs: Band<'_> = leafs.collect();
151
152	(!leafs.is_empty()).then_some(leafs)
153}
154
155/// Persists an authenticated event and applies its timeline side effects.
156///
157/// This method performs no authentication. The accepted row, event mapping,
158/// outlier removal, and timestamp index are committed together, but later
159/// cache, indexing, notification, and membership work is not part of that
160/// transaction, so an error can be returned after the event is stored.
161/// Any prepared current state is installed before the event's stream count
162/// retires, including when a later timeline effect returns an error.
163#[implement(super::Service)]
164#[tracing::instrument(name = "append", level = "debug", skip_all, ret(Debug))]
165pub async fn append_pdu<'a, Leafs>(
166	&'a self,
167	pdu: &'a PduEvent,
168	pdu_json: CanonicalJsonObject,
169	leafs: Leafs,
170	shortstatehash: Option<ShortStateHash>,
171	state_lock: &'a RoomMutexGuard,
172) -> Result<RawPduId>
173where
174	Leafs: Iterator<Item = &'a EventId> + Send + 'a,
175{
176	// Coalesce database writes for the remainder of this scope.
177	let _cork = self.db.db.cork_and_flush();
178
179	let shortroomid = self
180		.services
181		.short
182		.get_shortroomid(pdu.room_id())
183		.await
184		.map_err(|_| err!(Database("Room does not exist")))?;
185
186	let (pdu_json, replaced_join) = self.read_replaced(pdu, pdu_json).await?;
187
188	// We must keep track of all events that have been referenced.
189	self.services
190		.pdu_metadata
191		.mark_as_referenced(pdu.room_id(), pdu.prev_events().map(AsRef::as_ref));
192
193	self.services
194		.state
195		.set_forward_extremities(pdu.room_id(), leafs, state_lock)
196		.await;
197
198	let insert_lock = self.mutex_insert.lock(pdu.room_id()).await;
199	let next_count = self.services.globals.next_count();
200
201	// Mark as read first so the sending client doesn't get a notification even if
202	// appending fails. Route through the dispatcher so per-thread counts are
203	// also cleared; the sender's own send subsumes any thread receipt.
204	self.services
205		.read_receipt
206		.private_read_set(PrivateRead {
207			room_id: pdu.room_id(),
208			user_id: pdu.sender(),
209			count: *next_count,
210			ts: pdu.origin_server_ts(),
211			thread: &ReceiptThread::Unthreaded,
212			announce: false,
213		})
214		.await;
215
216	self.services
217		.pusher
218		.reset_notification_counts_for_thread(
219			pdu.sender(),
220			pdu.room_id(),
221			None,
222			&ReceiptThread::Unthreaded,
223		)
224		.await;
225
226	let count = PduCount::Normal(*next_count);
227	let pdu_id: RawPduId = PduId { shortroomid, count }.into();
228
229	// Insert pdu
230	self.append_pdu_json(&pdu_id, pdu, &pdu_json);
231
232	drop(insert_lock);
233
234	// Only local senders can own pushers.
235	if self.services.globals.user_is_local(pdu.sender()) {
236		self.services
237			.sending
238			.refresh_push_badge(pdu.sender())
239			.await
240			.log_err()
241			.ok();
242	}
243
244	self.services
245		.pusher
246		.append_pdu(pdu_id, pdu)
247		.await
248		.log_err()
249		.ok();
250
251	let appended = Appended {
252		pdu_id,
253		pdu,
254		shortroomid,
255		count,
256		replaced_join,
257	};
258
259	let effects = self
260		.append_pdu_effects(appended, state_lock)
261		.await;
262
263	if let Some(shortstatehash) = shortstatehash {
264		self.services
265			.state
266			.set_room_state(pdu.room_id(), shortstatehash, state_lock);
267	}
268
269	// The cached hierarchy summary projects the installed room state.
270	if pdu.state_key().is_some() {
271		self.services.spaces.cache_evict(pdu.room_id());
272	}
273
274	drop(next_count);
275	effects?;
276
277	self.services
278		.appservice
279		.append_pdu(pdu_id, pdu)
280		.await
281		.log_err()
282		.ok();
283
284	Ok(pdu_id)
285}
286
287/// Fill a state event's `unsigned` from the event it replaced, and find the
288/// position of the same user's join that a profile change replaces.
289///
290/// The replaced event is read once from the event's own state snapshot and is
291/// dropped on return, so it holds no place across the append's later awaits.
292#[implement(super::Service)]
293async fn read_replaced(
294	&self,
295	pdu: &PduEvent,
296	pdu_json: CanonicalJsonObject,
297) -> Result<(CanonicalJsonObject, Option<PduCount>)> {
298	let state_key = pdu.state_key();
299	let prev_state = state_key
300		.map_async(|state_key| self.prev_state(pdu, state_key))
301		.await
302		.flatten();
303
304	let pdu_json = if state_key.is_some() {
305		with_prev_unsigned(pdu_json, prev_state.as_ref())?
306	} else {
307		pdu_json
308	};
309
310	let replaced_join = self.replaced_join(pdu, prev_state.as_ref()).await;
311
312	Ok((pdu_json, replaced_join))
313}
314
315#[implement(super::Service)]
316async fn prev_state(&self, pdu: &PduEvent, state_key: &str) -> Option<PduEvent> {
317	let event_id = pdu.event_id();
318	let shortstatehash = self
319		.services
320		.state
321		.pdu_shortstatehash(event_id)
322		.await
323		.optional()
324		.inspect_err(|error| debug_warn!(%event_id, %error, "State snapshot read failed."))
325		.ok()
326		.flatten()?;
327
328	let event_type = pdu.kind().to_cow_str().into();
329
330	self.services
331		.state_accessor
332		.state_get(shortstatehash, &event_type, state_key)
333		.await
334		.optional()
335		.inspect_err(|error| debug_warn!(%event_id, %error, "Replaced state read failed."))
336		.ok()
337		.flatten()
338}
339
340/// Fill a state event's `unsigned` from the event it replaced.
341///
342/// Adds the replaced event's `prev_content`, `prev_sender` and
343/// `replaces_state`, from which clients read a membership or other state
344/// change. A non-object `unsigned` is logged and left as it is.
345fn with_prev_unsigned(
346	mut pdu_json: CanonicalJsonObject,
347	prev_state: Option<&PduEvent>,
348) -> Result<CanonicalJsonObject> {
349	let replaced = || {
350		prev_state
351			.map(prev_state_unsigned)
352			.transpose()
353			.map(Option::unwrap_or_default)
354	};
355
356	match pdu_json.entry("unsigned".into()) {
357		| Entry::Vacant(entry) => {
358			entry.insert(CanonicalJsonValue::Object(replaced()?));
359		},
360		| Entry::Occupied(entry) => match entry.into_mut() {
361			| CanonicalJsonValue::Object(unsigned) => unsigned.extend(replaced()?),
362			| _ => error!("Invalid unsigned type in pdu."),
363		},
364	}
365
366	Ok(pdu_json)
367}
368
369fn prev_state_unsigned(prev_state: &PduEvent) -> Result<CanonicalJsonObject> {
370	let prev_content = prev_state
371		.get_content::<CanonicalJsonObject>()
372		.map_err(|e| {
373			err!(Database(error!("Failed to convert prev_state to canonical JSON: {e}")))
374		})?;
375
376	let unsigned = [
377		("prev_content".into(), CanonicalJsonValue::Object(prev_content)),
378		(
379			"prev_sender".into(),
380			CanonicalJsonValue::String(prev_state.sender().to_string()),
381		),
382		(
383			"replaces_state".into(),
384			CanonicalJsonValue::String(prev_state.event_id().to_string()),
385		),
386	]
387	.into();
388
389	Ok(unsigned)
390}
391
392/// Stream position of the same user's join that a member event replaces, when
393/// the event is itself a join, as a profile change is.
394///
395/// `prev_state` must come from the event's own state snapshot, never the state
396/// cache, which a forced replay may have moved ahead of this append. Any other
397/// event yields `None`, as does a replaced join without a readable position,
398/// which a failed read logs.
399#[implement(super::Service)]
400async fn replaced_join(&self, pdu: &PduEvent, prev_state: Option<&PduEvent>) -> Option<PduCount> {
401	let user_id: &UserId = pdu.state_key()?.try_into().ok()?;
402	let is_join = |event: &PduEvent| event.membership_for(user_id) == Some(MembershipState::Join);
403	let replaced = prev_state.filter(|prev| is_join(prev) && is_join(pdu))?;
404	let event_id = replaced.event_id();
405
406	self.get_pdu_count(event_id)
407		.await
408		.optional()
409		.inspect_err(
410			|error| debug_warn!(%event_id, %error, "Replaced join position read failed."),
411		)
412		.ok()
413		.flatten()
414}
415
416#[implement(super::Service)]
417async fn append_pdu_effects(
418	&self,
419	Appended {
420		pdu_id,
421		pdu,
422		shortroomid,
423		count,
424		replaced_join,
425	}: Appended<'_>,
426	state_lock: &RoomMutexGuard,
427) -> Result {
428	match *pdu.kind() {
429		| TimelineEventType::RoomMember =>
430			self.append_member_effects(pdu, replaced_join, count)
431				.await?,
432		| TimelineEventType::RoomMessage =>
433			self.append_message_effects(&pdu_id, pdu, shortroomid)
434				.await?,
435		| TimelineEventType::RoomTopic =>
436			if let Some(topic) = pdu.get_content().ok().and_then(plain_text_topic) {
437				self.services
438					.search
439					.index_pdu(shortroomid, &pdu_id, &topic);
440			},
441		| TimelineEventType::RoomRedaction => {
442			let room_version = self
443				.services
444				.state
445				.get_room_version(pdu.room_id())
446				.await?;
447
448			let room_rules = room_version::rules(&room_version)?;
449
450			let redacts_id = pdu.redacts_id(&room_rules);
451
452			if let Some(redacts_id) = &redacts_id
453				&& self
454					.services
455					.state_accessor
456					.user_can_redact(redacts_id, pdu.sender(), pdu.room_id(), false)
457					.await?
458			{
459				self.redact_pdu(redacts_id, pdu, shortroomid, state_lock)
460					.await?;
461			}
462		},
463		| _ => {},
464	}
465
466	if let Ok(content) = pdu.get_content::<ExtractRelatesToEventId>()
467		&& let Ok(related_pducount) = self
468			.get_pdu_count(&content.relates_to.event_id)
469			.await
470	{
471		self.services
472			.pdu_metadata
473			.add_relation(count, related_pducount);
474	}
475
476	if let Ok(content) = pdu.get_content::<ExtractRelatesTo>() {
477		match content.relates_to {
478			| Relation::Reply(ruma::events::relation::Reply { in_reply_to }) => {
479				// We need to do it again here, because replies don't have
480				// event_id as a top level field
481				if let Ok(related_pducount) = self.get_pdu_count(&in_reply_to.event_id).await {
482					self.services
483						.pdu_metadata
484						.add_relation(count, related_pducount);
485				}
486			},
487			| Relation::Thread(thread) => {
488				self.services
489					.threads
490					.add_to_thread(&thread.event_id, pdu_id, pdu)
491					.await?;
492			},
493			| Relation::Replacement(replacement) => {
494				self.services
495					.pdu_metadata
496					.add_typed_relation(
497						shortroomid,
498						count,
499						&replacement.event_id,
500						pdu,
501						RelationType::Replacement,
502					)
503					.await;
504			},
505			| Relation::Reference(reference) => {
506				self.services
507					.pdu_metadata
508					.add_typed_relation(
509						shortroomid,
510						count,
511						&reference.event_id,
512						pdu,
513						RelationType::Reference,
514					)
515					.await;
516			},
517			| _ => {}, // TODO: Aggregate other types
518		}
519	}
520
521	Ok(())
522}
523
524/// Record the membership transition an `m.room.member` event carries.
525///
526/// The cache is written here rather than off the resolved state so that a
527/// user who is invited or knocked and leaves immediately still leaves the
528/// earlier event on record for auth. A join that replaced the same user's join
529/// is a profile change, which keeps the user's recorded join position.
530#[implement(super::Service)]
531async fn append_member_effects(
532	&self,
533	pdu: &PduEvent,
534	replaced_join: Option<PduCount>,
535	count: PduCount,
536) -> Result {
537	let Some(state_key) = pdu.state_key() else {
538		return Ok(());
539	};
540
541	let user_id = UserId::parse(state_key).expect("This state_key was previously validated");
542	let content: RoomMemberEventContent = pdu.get_content()?;
543	let is_invite = content.membership == MembershipState::Invite;
544	let is_direct = content.is_direct;
545
546	let stripped_state = match content.membership {
547		| MembershipState::Invite | MembershipState::Knock => self
548			.services
549			.state
550			.summary_stripped(pdu)
551			.await
552			.into(),
553		| _ => None,
554	};
555
556	self.services
557		.state_cache
558		.update_membership(MembershipUpdate {
559			room_id: pdu.room_id(),
560			user_id: &user_id,
561			membership_event: content,
562			sender: pdu.sender(),
563			last_state: stripped_state,
564			invite_via: None,
565			invite_event: is_invite.then_some(pdu.event_id()),
566			update_joined_count: true,
567			replaced_join,
568			count,
569		})
570		.await?;
571
572	if is_invite {
573		self.services
574			.membership
575			.auto_accept(pdu.room_id(), &user_id, pdu.sender(), is_direct);
576	}
577
578	Ok(())
579}
580
581/// Index an `m.room.message` event's body, and queue it when it is an admin
582/// command.
583///
584/// The queued command carries the event's sender, so a handler can tell who
585/// issued it, and the event's id, which its response replies to.
586#[implement(super::Service)]
587async fn append_message_effects(
588	&self,
589	pdu_id: &RawPduId,
590	pdu: &PduEvent,
591	shortroomid: ShortRoomId,
592) -> Result {
593	let content: ExtractBody = pdu.get_content()?;
594	let Some(body) = content.body else {
595		return Ok(());
596	};
597
598	self.services
599		.search
600		.index_pdu(shortroomid, pdu_id, &body);
601
602	if self
603		.services
604		.admin
605		.is_admin_command(pdu, &body)
606		.await
607	{
608		self.services
609			.admin
610			.command(CommandInput {
611				command: body,
612				reply_id: Some(pdu.event_id().into()),
613				sender: Some(pdu.sender().into()),
614			})
615			.await?;
616	}
617
618	Ok(())
619}
620
621#[implement(super::Service)]
622fn append_pdu_json(&self, pdu_id: &RawPduId, pdu: &PduEvent, json: &CanonicalJsonObject) {
623	debug_assert!(matches!(pdu_id.pdu_count(), PduCount::Normal(_)), "PduCount not Normal");
624
625	let mut txn = self.db.db.txn();
626
627	txn.raw_put(&self.db.pduid_pdu, pdu_id, Json(json));
628	txn.insert_raw(&self.db.eventid_pduid, pdu.event_id.as_bytes(), pdu_id);
629	txn.del_raw(&self.db.eventid_outlierpdu, pdu.event_id.as_bytes());
630
631	let count_key = bias_count(pdu_id.count());
632	let ts = u64::from(pdu.origin_server_ts);
633	let key = (pdu.room_id(), ts, count_key);
634	txn.put_raw(&self.db.roomid_tscount_pducount, key, pdu_id.count());
635
636	txn.execute();
637}
638
639#[cfg(test)]
640mod tests {
641	use std::iter::empty;
642
643	use ruma::event_id;
644
645	use super::*;
646
647	#[test]
648	fn empty_band_is_skipped() {
649		assert!(nonempty_band(empty::<&EventId>()).is_none());
650	}
651
652	#[test]
653	fn nonempty_band_preserves_all_leaves() {
654		let leaves = [event_id!("$a:test.local"), event_id!("$b:test.local")];
655
656		let kept: Vec<&EventId> = nonempty_band(leaves.iter().copied())
657			.expect("non-empty band retained")
658			.into_iter()
659			.collect();
660
661		assert_eq!(kept, leaves);
662	}
663}