Skip to main content

tuwunel_service/integrity/
mod.rs

1//! Read-only integrity checks over the database.
2//!
3//! A check walks the columns it cares about and yields a finding for every
4//! defect it sees, writing nothing. A finding describes the database as it
5//! stands, so a check run after a repair sees that repair's work. The migrations
6//! are a caller like any other: they consume these streams and write what a
7//! finding calls for, re-deciding anything a write may have changed.
8
9pub mod account_status;
10pub mod columns;
11pub mod email_bindings;
12pub mod forbidden_names;
13pub mod joined_members;
14pub mod media_files;
15pub mod mediaid_user;
16pub mod moderation;
17pub mod pdu_room_ids;
18pub mod readreceiptid_readreceipt;
19pub mod redactions;
20pub mod referencedevents;
21pub mod relatesto_typed;
22pub mod roomid_tscount_pducount;
23pub mod roomuserid_joined;
24mod scan;
25pub mod schema;
26pub(crate) mod short_ids;
27#[cfg(test)]
28mod tests;
29pub mod thread_summaries;
30pub mod threadrootid_latestcount;
31pub mod token_expiry;
32pub mod userid_password;
33pub mod useridprofilekey_value;
34pub mod userroomid_highlightcount;
35
36use std::{collections::BTreeMap, fmt::Display};
37
38use futures::{FutureExt, Stream, StreamExt, TryFutureExt, future::BoxFuture, stream::unfold};
39use ruma::{OwnedRoomId, RoomId, api::error::ErrorKind};
40use serde::Deserialize;
41use tuwunel_core::{Err, Error, Result, implement, utils::TryReadyExt};
42use tuwunel_database::SEP;
43
44pub(crate) use self::scan::ScanExt;
45use crate::Services;
46
47type Walk = for<'a> fn(&'a Services, usize) -> BoxFuture<'a, Result<Summary>>;
48
49// The room `per_room` last answered for, and its answer.
50type Answered<V> = Option<(OwnedRoomId, V)>;
51
52// A row as `per_room` passes it: the outer result is the read, the inner one
53// whether the row names a room.
54type Row<A, B> = Result<Result<A, B>>;
55
56/// Every check, in the order a whole run takes them.
57static CHECKS: &[&Check] = &[
58	&schema::CHECK,
59	&columns::CHECK,
60	&columns::UNUSED,
61	&pdu_room_ids::CHECK,
62	&roomuserid_joined::CHECK,
63	&referencedevents::CHECK,
64	&readreceiptid_readreceipt::CHECK,
65	&userroomid_highlightcount::CHECK,
66	&mediaid_user::LEGACY,
67	&mediaid_user::REMOTE,
68	&media_files::LEGACY,
69	&media_files::CHECK,
70	&useridprofilekey_value::CHECK,
71	&userid_password::CHECK,
72	&forbidden_names::USERNAMES,
73	&forbidden_names::ALIASES,
74	&joined_members::CHECK,
75	&moderation::CHECK,
76	&account_status::CHECK,
77	&email_bindings::CHECK,
78	&token_expiry::CHECK,
79	&roomid_tscount_pducount::CHECK,
80	&relatesto_typed::CHECK,
81	&redactions::CHECK,
82	&thread_summaries::CHECK,
83	&threadrootid_latestcount::CHECK,
84	&short_ids::CHECK,
85];
86
87/// A finding of any check.
88///
89/// Rendering it describes the one defect, while its kind names the class of
90/// defect, the unit a run counts by.
91pub trait Report: Display + Send {
92	/// Names the class of defect.
93	///
94	/// A run counts its findings under this label, so every finding of one class
95	/// answers with the same text.
96	fn kind(&self) -> &'static str;
97}
98
99/// One registered check.
100///
101/// The registry is how a caller reaches a check by name, as the admin suite does.
102/// A caller that wants the findings themselves takes the typed stream of the
103/// check's own module.
104#[derive(Clone, Copy)]
105pub struct Check {
106	/// The name a caller selects it by.
107	pub name: &'static str,
108
109	/// One line saying what it verifies.
110	pub about: &'static str,
111
112	/// Whether it needs a database nothing else is writing.
113	///
114	/// A check comparing columns reads them at different moments, so on a serving
115	/// database a concurrent write would read as a defect.
116	pub quiescent: bool,
117
118	walk: Walk,
119}
120
121/// What one run of a check found.
122///
123/// Counting holds no finding in memory, so a run over a badly damaged database
124/// stays bounded by its sample limit.
125#[derive(Default)]
126pub struct Summary {
127	/// Findings of each kind.
128	pub counts: BTreeMap<&'static str, u64>,
129
130	/// The first findings rendered, up to the limit of the run.
131	pub samples: Vec<String>,
132}
133
134/// Finds the check registered under a name.
135///
136/// Names are unique, and `None` means no check has this one.
137#[must_use]
138pub fn find(name: &str) -> Option<&'static Check> { checks().find(|check| check.name == name) }
139
140/// Iterates over every registered check.
141///
142/// The order is the one a whole run takes, cheapest and most fundamental first.
143pub fn checks() -> impl Iterator<Item = &'static Check> + Clone + Send { CHECKS.iter().copied() }
144
145/// Runs this check to the end, keeping the first `limit` findings rendered.
146///
147/// A quiescent check refuses a writable database, because everything it would
148/// report could be a concurrent write. The first error ends the run, whatever it
149/// found by then.
150#[implement(Check)]
151#[tracing::instrument(
152	level = "debug",
153	skip_all,
154	fields(
155		check = %self.name,
156	),
157)]
158pub async fn run(&self, services: &Services, limit: usize) -> Result<Summary> {
159	if !self.runnable(services) {
160		return Err!("{} needs a database opened read-only", self.name);
161	}
162
163	(self.walk)(services, limit).await
164}
165
166/// Whether this check can give a true answer on this database.
167///
168/// Only a quiescent check on a writable database cannot.
169#[implement(Check)]
170#[must_use]
171pub fn runnable(&self, services: &Services) -> bool {
172	!self.quiescent || services.db.is_read_only()
173}
174
175/// The number of findings of every kind.
176///
177/// This is zero exactly when the check found the database clean.
178#[implement(Summary)]
179#[must_use]
180pub fn findings(&self) -> u64 {
181	self.counts
182		.values()
183		.copied()
184		.fold(0, u64::saturating_add)
185}
186
187fn tally<'a, S, F>(findings: S, limit: usize) -> BoxFuture<'a, Result<Summary>>
188where
189	S: Stream<Item = Result<F>> + Send + 'a,
190	F: Report + 'a,
191{
192	findings
193		.ready_try_fold(Summary::default(), move |summary, finding| {
194			Ok(summary.record(&finding, limit))
195		})
196		.boxed() // type-erasure boundary
197}
198
199#[implement(Summary)]
200fn record(mut self, finding: &impl Report, limit: usize) -> Self {
201	if self.samples.len() < limit {
202		self.samples.push(finding.to_string());
203	}
204
205	let count = self.counts.entry(finding.kind()).or_default();
206
207	*count = count.saturating_add(1);
208	self
209}
210
211/// Splits off the leading record of a stored key or value, as a release build
212/// decodes a lone string from it.
213///
214/// The rest is whatever follows the separator ending that record, or `None`
215/// when no separator does. A release build ignores the rest, where a debug build
216/// asserts there is none.
217pub(crate) fn leading_record(bytes: &[u8]) -> (&[u8], Option<&[u8]>) {
218	let split = bytes
219		.iter()
220		.position(|&byte| byte == SEP)
221		.and_then(|at| bytes.split_at_checked(at));
222
223	split.map_or((bytes, None), |(leading, rest)| (leading, rest.get(1..)))
224}
225
226/// Decodes the JSON object a stored row holds, as a release build's database
227/// decoder reads it.
228///
229/// That decoder parses the object at the start of the leading record and, in
230/// either build, ignores whatever follows the object within that record. A
231/// release build also ignores any further record, where a debug build panics on
232/// one beyond a single trailing separator.
233pub(crate) fn leading_json<T>(bytes: &[u8]) -> Result<T>
234where
235	T: for<'de> Deserialize<'de>,
236{
237	let (record, _) = leading_record(bytes);
238
239	T::deserialize(&mut serde_json::Deserializer::from_slice(record)).map_err(Into::into)
240}
241
242/// Answers `damage` for a failed read that found its data but could not decode
243/// it, and keeps any other failure.
244///
245/// A check reports such a row as damage and moves on, while a storage fault still
246/// ends it.
247pub(crate) fn undecodable_as<T>(error: Error, damage: T) -> Result<T> {
248	undecodable(&error).then_some(damage).ok_or(error)
249}
250
251/// Whether a failed read found its data but could not decode it.
252///
253/// A row that is not JSON, or content that does not fit, fails this way, and a
254/// storage fault never does.
255pub(crate) fn undecodable(error: &Error) -> bool {
256	matches!(error.kind(), ErrorKind::BadJson | ErrorKind::NotJson)
257}
258
259/// Pairs each row naming a room with what `resolve` answers for that room,
260/// asking once for each run of consecutive rows in one room.
261///
262/// A walk sorted by room asks once per room and holds only the latest answer,
263/// while a row naming no room passes through. A failed answer is yielded in
264/// place of the row that asked.
265fn per_room<'a, S, A, B, V, F, Fut>(
266	rows: S,
267	room: fn(&A) -> &RoomId,
268	resolve: F,
269) -> impl Stream<Item = Row<(A, V), B>> + Send + 'a
270where
271	S: Stream<Item = Row<A, B>> + Send + 'a,
272	A: Send + 'a,
273	B: Send + 'a,
274	V: Clone + Send + 'a,
275	F: Fn(OwnedRoomId) -> Fut + Send + Sync + 'a,
276	Fut: Future<Output = Result<V>> + Send + 'a,
277{
278	let rows = Box::pin(rows); // Unpin for StreamExt::next
279
280	unfold((rows, None, room, resolve), async |(mut rows, answered, room, resolve)| {
281		let row = rows.next().await?;
282		let (row, answered) = answer(row, answered, room, &resolve).await;
283
284		Some((row, (rows, answered, room, resolve)))
285	})
286}
287
288async fn answer<A, B, V, F, Fut>(
289	row: Row<A, B>,
290	answered: Answered<V>,
291	room: fn(&A) -> &RoomId,
292	resolve: &F,
293) -> (Row<(A, V), B>, Answered<V>)
294where
295	A: Send,
296	B: Send,
297	V: Clone + Send,
298	F: Fn(OwnedRoomId) -> Fut + Sync,
299	Fut: Future<Output = Result<V>> + Send,
300{
301	let row = match row {
302		| Ok(Ok(row)) => row,
303		| Ok(Err(other)) => return (Ok(Err(other)), answered),
304		| Err(error) => return (Err(error), answered),
305	};
306
307	let answered = match answered {
308		| Some((room_id, answer)) if &*room_id == room(&row) => Ok((room_id, answer)),
309		| _ => {
310			let room_id = room(&row).to_owned();
311
312			resolve(room_id.clone())
313				.map_ok(|answer| (room_id, answer))
314				.await
315		},
316	};
317
318	match answered {
319		| Ok((room_id, answer)) => (Ok(Ok((row, answer.clone()))), Some((room_id, answer))),
320		| Err(error) => (Err(error), None),
321	}
322}