Skip to main content

tuwunel_service/integrity/
pdu_room_ids.rs

1//! The `room_id` of stored events.
2//!
3//! Readers decode an event together with its room, so an event without one fails
4//! to read. A room version 12 create event imported from Conduit omits it, since
5//! its room derives from the event's own id (MSC4291), whether it was stored in
6//! the timeline or as an outlier.
7
8use std::{collections::BTreeMap, fmt, str::from_utf8};
9
10use futures::{Stream, StreamExt, TryStreamExt, stream::once};
11use ruma::{CanonicalJsonObject, CanonicalJsonValue, EventId, OwnedRoomId, RoomId};
12use serde::{Deserialize, de::IgnoredAny};
13use tuwunel_core::{
14	Result,
15	utils::{BoolExt, TryReadyExt},
16};
17use tuwunel_database::KeyBuf;
18
19use super::{Check, Report, ScanExt, tally};
20use crate::{Services, rooms::short::ShortRoomId};
21
22type Rooms = BTreeMap<ShortRoomId, OwnedRoomId>;
23
24pub(super) static CHECK: Check = Check {
25	name: "pdu-room-ids",
26	about: "Every JSON event in pduid_pdu and every event in eventid_outlierpdu carries a \
27	        room_id",
28	quiescent: false,
29	walk: |services, limit| tally(scan(services), limit),
30};
31
32/// The `room_id` of a stored event, read without the rest of it.
33///
34/// Every row is read, so only an event this shows to lack its room pays for the
35/// full parse.
36#[derive(Deserialize)]
37struct Probe {
38	room_id: Option<IgnoredAny>,
39}
40
41/// A column holding stored events.
42///
43/// A finding names its column, so the repair reads the event back from the
44/// column it came from, and an outlier's room derives from its key.
45#[derive(Clone, Copy)]
46pub enum Column {
47	/// The timeline, `pduid_pdu`, whose keys lead with the room's short id.
48	Timeline,
49
50	/// The outliers, `eventid_outlierpdu`, keyed by event id.
51	Outliers,
52}
53
54/// A stored event without its `room_id`, or an outlier that cannot be read.
55///
56/// The repair writes the room into an event whose room is known and leaves the
57/// others as they are.
58pub enum Finding {
59	/// The event lacks its room, and its key or its own id names that room.
60	Missing {
61		/// The column holding the event.
62		column: Column,
63
64		/// The stored key.
65		key: KeyBuf,
66
67		/// The room the event belongs to.
68		room_id: OwnedRoomId,
69
70		/// The stored event, which the repair writes back with its room.
71		pdu: CanonicalJsonObject,
72	},
73
74	/// The event lacks its room, and nothing names that room.
75	Unresolved {
76		/// The column holding the event.
77		column: Column,
78
79		/// The stored key.
80		key: KeyBuf,
81	},
82
83	/// The event cannot be read as canonical JSON, so it cannot take a room.
84	Unparsable {
85		/// The column holding the event.
86		column: Column,
87
88		/// The stored key.
89		key: KeyBuf,
90	},
91}
92
93/// Streams each stored event lacking its `room_id`, and each outlier that is not
94/// JSON.
95///
96/// Every timeline row is read, then every outlier. A timeline row that is not
97/// JSON at all is skipped, as the checks decoding timeline events report it.
98/// Repairing one event changes no other event's finding.
99#[tracing::instrument(level = "trace", skip_all)]
100pub fn scan(services: &Services) -> impl Stream<Item = Result<Finding>> + Send + '_ {
101	timeline(services).chain(outliers(services))
102}
103
104/// Streams each timeline event lacking its `room_id`.
105///
106/// Such an event's room comes from the short id leading its key, resolved
107/// against `roomid_shortroomid` as it is read before the timeline walk starts.
108#[tracing::instrument(level = "trace", skip_all)]
109pub fn timeline(services: &Services) -> impl Stream<Item = Result<Finding>> + Send + '_ {
110	once(rooms(services))
111		.map_ok(move |rooms| {
112			services.db["pduid_pdu"]
113				.raw_scan()
114				.scanned(&services.server)
115				.ready_try_filter_map(move |(key, value)| {
116					Ok(classify(Column::Timeline, key, value, || timeline_room(&rooms, key)))
117				})
118		})
119		.try_flatten()
120}
121
122/// Streams each outlier lacking its `room_id` or not JSON at all.
123///
124/// Only a create event can lack its room and still name it, through its own id.
125#[tracing::instrument(level = "trace", skip_all)]
126pub fn outliers(services: &Services) -> impl Stream<Item = Result<Finding>> + Send + '_ {
127	services.db["eventid_outlierpdu"]
128		.raw_scan()
129		.scanned(&services.server)
130		.ready_try_filter_map(|(key, value)| Ok(classify(Column::Outliers, key, value, || None)))
131}
132
133/// Classifies one stored event as [`scan`] does.
134///
135/// A timeline event's room comes from `timeline_room`, asked only once the event
136/// proves to lack one, while an outlier's derives from its own id. A repair reads
137/// the row a finding names again and passes it back through here, so detection
138/// and repair share one predicate.
139pub(crate) fn classify<F>(
140	column: Column,
141	key: &[u8],
142	value: &[u8],
143	timeline_room: F,
144) -> Option<Finding>
145where
146	F: FnOnce() -> Option<OwnedRoomId>,
147{
148	let Ok(probe) = serde_json::from_slice::<Probe>(value) else {
149		return matches!(column, Column::Outliers)
150			.then(|| Finding::Unparsable { column, key: KeyBuf::from_slice(key) });
151	};
152
153	if probe.room_id.is_some() {
154		return None;
155	}
156
157	let Ok(pdu) = serde_json::from_slice::<CanonicalJsonObject>(value) else {
158		return Some(Finding::Unparsable { column, key: KeyBuf::from_slice(key) });
159	};
160
161	let room_id = match column {
162		| Column::Timeline => timeline_room(),
163		| Column::Outliers => outlier_room(key, &pdu),
164	};
165
166	let key = KeyBuf::from_slice(key);
167	let finding = match room_id {
168		| Some(room_id) => Finding::Missing { column, key, room_id, pdu },
169		| None => Finding::Unresolved { column, key },
170	};
171
172	Some(finding)
173}
174
175impl Column {
176	/// The name of the column.
177	///
178	/// The repair indexes the database by it, and a finding renders it.
179	#[must_use]
180	pub fn name(self) -> &'static str {
181		match self {
182			| Self::Timeline => "pduid_pdu",
183			| Self::Outliers => "eventid_outlierpdu",
184		}
185	}
186}
187
188impl Report for Finding {
189	fn kind(&self) -> &'static str {
190		match self {
191			| Self::Missing { .. } => "missing-room-id",
192			| Self::Unresolved { .. } => "unresolved-room",
193			| Self::Unparsable { .. } => "unparsable-event",
194		}
195	}
196}
197
198impl fmt::Display for Finding {
199	fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
200		match self {
201			| Self::Missing { column, key, room_id, .. } => write!(
202				f,
203				"event {} in {column} has no room_id; it belongs to {room_id}",
204				key.escape_ascii()
205			),
206			| Self::Unresolved { column, key } => write!(
207				f,
208				"event {} in {column} has no room_id and nothing names its room",
209				key.escape_ascii()
210			),
211			| Self::Unparsable { column, key } => {
212				write!(f, "event {} in {column} is not canonical JSON", key.escape_ascii())
213			},
214		}
215	}
216}
217
218impl fmt::Display for Column {
219	fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.write_str(self.name()) }
220}
221
222#[tracing::instrument(level = "debug", skip_all)]
223async fn rooms(services: &Services) -> Result<Rooms> {
224	services.db["roomid_shortroomid"]
225		.raw_scan()
226		.scanned(&services.server)
227		.ready_try_filter_map(|(key, value)| Ok(room(key, value)))
228		.try_collect()
229		.await
230}
231
232fn room(key: &[u8], value: &[u8]) -> Option<(ShortRoomId, OwnedRoomId)> {
233	let room_id: &RoomId = from_utf8(key).ok()?.try_into().ok()?;
234	let short = short_id(value)?;
235
236	Some((short, room_id.to_owned()))
237}
238
239fn timeline_room(rooms: &Rooms, key: &[u8]) -> Option<OwnedRoomId> {
240	rooms.get(&short_id(key)?).cloned()
241}
242
243fn short_id(bytes: &[u8]) -> Option<ShortRoomId> {
244	bytes
245		.first_chunk()
246		.copied()
247		.map(ShortRoomId::from_be_bytes)
248}
249
250/// The room of an outlier lacking its `room_id`.
251///
252/// Only a room version 12 create event omits it, and its room derives from the
253/// create event's own id, which keys the outlier.
254fn outlier_room(key: &[u8], pdu: &CanonicalJsonObject) -> Option<OwnedRoomId> {
255	let is_create = matches!(
256		pdu.get("type"),
257		Some(CanonicalJsonValue::String(kind)) if kind == "m.room.create"
258	);
259
260	is_create.and_then(|| {
261		let event_id: &EventId = from_utf8(key).ok()?.try_into().ok()?;
262
263		RoomId::new_v2(event_id.localpart()).ok()
264	})
265}
266
267#[cfg(test)]
268mod tests {
269	use super::Probe;
270
271	#[test]
272	fn room_id_probe_detects_presence() {
273		let with_room_id = br#"{"room_id":"!r:server","type":"m.room.message"}"#;
274		let without_room_id = br#"{"type":"m.room.create","sender":"@u:server"}"#;
275
276		let present: Probe = serde_json::from_slice(with_room_id).unwrap();
277		let absent: Probe = serde_json::from_slice(without_room_id).unwrap();
278
279		assert!(present.room_id.is_some());
280		assert!(absent.room_id.is_none());
281	}
282}