Skip to main content

tuwunel_service/migrations/
conduit.rs

1use std::{
2	iter::from_fn,
3	path::{Path, PathBuf},
4	sync::Arc,
5	time::Duration,
6};
7
8use bytes::Bytes;
9use futures::{Stream, StreamExt, TryStreamExt};
10use object_store::Error as ObjectStoreError;
11use ruma::{
12	CanonicalJsonObject, CanonicalJsonValue, Mxc, OwnedUserId, RoomId, ServerName, UserId,
13};
14use tokio::time::sleep;
15use tuwunel_core::{
16	Err, Error, Result, debug_warn, err, error, info,
17	itertools::Itertools,
18	result::NotFound,
19	utils::{
20		ReadyExt, TryReadyExt,
21		content_disposition::make_content_disposition,
22		stream::{TryBroadbandExt, TryIgnore},
23	},
24	warn,
25};
26use tuwunel_database::{Map, SEP};
27
28use super::tally::{Outcome, Tally};
29use crate::{
30	Services,
31	integrity::pdu_room_ids::{Finding, classify, outliers, timeline},
32	storage::Provider,
33};
34
35/// Selects the source of a Conduit database's original media files.
36///
37/// The local filesystem is the default. For Conduit media backed by S3,
38/// `media.import.conduit.source_provider` names the source object store.
39enum MediaSource {
40	Filesystem(PathBuf),
41	Provider(Arc<Provider>),
42}
43
44/// One parsed `servernamemediaid_metadata` entry, borrowing the raw key/value.
45struct ConduitMediaEntry<'a> {
46	server_name: &'a ServerName,
47	media_id: &'a str,
48	sha256: &'a [u8],
49	filename: Option<&'a str>,
50	content_type: Option<&'a str>,
51}
52
53/// Resolves the configured Conduit media source. A named provider must already
54/// be defined under `[storage_provider]`; its absence is an operator error that
55/// aborts the import rather than silently dropping every file.
56fn media_source(services: &Services) -> Result<MediaSource> {
57	let config = &services.server.config;
58
59	match config
60		.media
61		.import
62		.conduit
63		.source_provider
64		.as_deref()
65	{
66		| Some(name) => services
67			.storage
68			.provider(name)
69			.map(|provider| MediaSource::Provider(provider.clone())),
70		| None => {
71			let media_dir = config
72				.media
73				.import
74				.conduit
75				.source_path
76				.clone()
77				.unwrap_or_else(|| config.database_path.join("media"));
78
79			Ok(MediaSource::Filesystem(media_dir))
80		},
81	}
82}
83
84/// Attempts to read each original from a source storage provider before the
85/// import gives up: one initial try plus one retry. The object store performs
86/// its own internal retries under each attempt, so a transient provider blip
87/// rarely reaches this outer limit.
88const PROVIDER_READ_ATTEMPTS: u32 = 2;
89
90/// Pause between provider read retries, giving a transient fault time to clear.
91const PROVIDER_READ_RETRY_DELAY: Duration = Duration::from_secs(2);
92
93/// Imports the original media files of a Conduit database, re-uploading each
94/// `servernamemediaid_metadata` entry through `media.create`.
95///
96/// Malformed entries, unreadable filesystem files, missing provider objects,
97/// and moderator-blocked media are skipped. Persistent source-provider faults,
98/// blocklist read errors, and destination creation failures abort without
99/// setting the completion marker; a restart safely overwrites deterministic
100/// partial metadata and provider objects.
101pub(super) async fn migrate_conduit_media(services: &Services) -> Result {
102	let db = &services.db;
103	let config = &services.server.config;
104
105	let Some(metadata) = db.open_cf("servernamemediaid_metadata")? else {
106		warn!("Conduit database has no media metadata; nothing to import.");
107		return Ok(());
108	};
109
110	let owners = db.open_cf("servernamemediaid_userlocalpart")?;
111	let owners = owners.as_ref();
112
113	let blocklist = db.open_cf("blocked_servername_mediaid")?;
114	let blocklist = blocklist.as_ref();
115
116	let depth = config.media.import.conduit.directory_depth;
117	let length = config.media.import.conduit.directory_length;
118	let source = media_source(services)?;
119
120	warn!("Importing Conduit media originals into tuwunel's key-addressed store...");
121
122	let cork = db.cork_and_sync();
123	let (imported, skipped, blocked) = metadata
124		.raw_stream()
125		.ignore_err()
126		.map(Ok::<_, Error>)
127		.try_fold(
128			(0_usize, 0_usize, 0_usize),
129			async |(imported, skipped, blocked), (key, value)| {
130				if conduit_media_blocked(blocklist, key).await? {
131					return Ok((imported, skipped, blocked.saturating_add(1)));
132				}
133
134				let imported_entry =
135					import_conduit_original(services, owners, &source, depth, length, key, value)
136						.await?;
137
138				Ok(if imported_entry {
139					(imported.saturating_add(1), skipped, blocked)
140				} else {
141					(imported, skipped.saturating_add(1), blocked)
142				})
143			},
144		)
145		.await?;
146
147	drop(cork);
148
149	if blocked > 0 {
150		warn!(%blocked, "Skipped Conduit media blocked by a moderator; not imported");
151	}
152
153	if skipped > 0 {
154		warn!(%imported, %skipped, "Imported Conduit media originals; some files were skipped");
155	} else {
156		info!(%imported, "Imported Conduit media originals");
157	}
158
159	Ok(())
160}
161
162/// Imports one `servernamemediaid_metadata` entry and reports whether it was
163/// stored.
164///
165/// `false` denotes an intentional source-entry skip. Destination creation
166/// errors propagate so the caller can withhold the completion marker.
167async fn import_conduit_original(
168	services: &Services,
169	owners: Option<&Arc<Map>>,
170	source: &MediaSource,
171	depth: u8,
172	length: u8,
173	key: &[u8],
174	value: &[u8],
175) -> Result<bool> {
176	let entry = match parse_conduit_media_entry(key, value) {
177		| Ok(entry) => entry,
178		| Err(e) => {
179			debug_warn!(error = %e, "skipping unimportable Conduit media entry");
180			return Ok(false);
181		},
182	};
183
184	let Some(file) = read_conduit_original(source, depth, length, entry.sha256).await? else {
185		return Ok(false);
186	};
187
188	let content_disposition = make_content_disposition(None, entry.content_type, entry.filename);
189	let owner = conduit_media_owner(owners, key, entry.server_name).await;
190	let mxc = Mxc {
191		server_name: entry.server_name,
192		media_id: entry.media_id,
193	};
194
195	services
196		.media
197		.create(&mxc, owner.as_deref(), Some(&content_disposition), entry.content_type, &file)
198		.await?;
199
200	Ok(true)
201}
202
203/// Parses a `servernamemediaid_metadata` entry: the key is
204/// `servername 0xff media_id`, the value is the digest, filename and
205/// content type.
206fn parse_conduit_media_entry<'a>(
207	key: &'a [u8],
208	value: &'a [u8],
209) -> Result<ConduitMediaEntry<'a>> {
210	let Some(sep) = key.iter().position(|&byte| byte == SEP) else {
211		return Err!(Database("Conduit media key has no server-name separator"));
212	};
213	let server_name = <&ServerName>::try_from(str::from_utf8(&key[..sep])?)
214		.map_err(|_| err!(Database("Conduit media key has an invalid server name")))?;
215
216	let media_id = str::from_utf8(&key[sep.saturating_add(1)..])?;
217
218	let (sha256, filename, content_type) = parse_conduit_media_value(value)?;
219
220	Ok(ConduitMediaEntry {
221		server_name,
222		media_id,
223		sha256,
224		filename,
225		content_type,
226	})
227}
228
229/// Reads one Conduit original, named by its content digest, from the configured
230/// media source. A filesystem file that cannot be read is reported as `None`
231/// (skipped, like a dangling metadata row). A source storage provider is
232/// retried on a transient fault; a persistent one returns `Err` so the import
233/// aborts instead of dropping reachable media.
234async fn read_conduit_original(
235	source: &MediaSource,
236	depth: u8,
237	length: u8,
238	sha256: &[u8],
239) -> Result<Option<Bytes>> {
240	let sha256_hex = sha256_hex(sha256);
241	match source {
242		| MediaSource::Filesystem(media_dir) => {
243			let path = conduit_media_path(media_dir, depth, length, &sha256_hex);
244			match tokio::fs::read(&path).await {
245				| Ok(file) => Ok(Some(file.into())),
246				| Err(e) => {
247					debug_warn!(?path, error = %e, "skipping unreadable Conduit media file");
248					Ok(None)
249				},
250			}
251		},
252		| MediaSource::Provider(provider) =>
253			read_provider_original(provider, &conduit_media_key(depth, length, &sha256_hex)).await,
254	}
255}
256
257/// Reads one original from the source storage provider. An absent object is
258/// skipped (`Ok(None)`) like a dangling filesystem row; a transient fault is
259/// retried up to `PROVIDER_READ_ATTEMPTS` times, and a persistent one aborts
260/// the import with an `Err`.
261async fn read_provider_original(provider: &Arc<Provider>, key: &str) -> Result<Option<Bytes>> {
262	let mut attempt = 0_u32;
263	loop {
264		attempt = attempt.saturating_add(1);
265		match provider.get(key).await {
266			| Ok(file) => return Ok(Some(file)),
267			| Err(e) if is_missing_object(&e) => {
268				debug_warn!(%key, error = %e, "skipping missing Conduit media object");
269				return Ok(None);
270			},
271			| Err(e) if attempt >= PROVIDER_READ_ATTEMPTS => {
272				error!(
273					%key,
274					attempts = PROVIDER_READ_ATTEMPTS,
275					error = %e,
276					"Aborting the Conduit media import: source storage provider unreachable. No \
277					 media has been imported in a way that needs cleanup; once the provider is \
278					 reachable, restart tuwunel to resume the import from the beginning."
279				);
280				return Err(e);
281			},
282			| Err(e) => {
283				warn!(%key, attempt, error = %e, "Reading Conduit media object failed; retrying");
284				sleep(PROVIDER_READ_RETRY_DELAY).await;
285			},
286		}
287	}
288}
289
290/// Whether a provider read failed because the object is absent (a 404 or a
291/// dangling metadata row), which is skipped rather than retried. tuwunel's
292/// `Error::is_not_found` does not cover the object-store variant, so match it
293/// directly.
294fn is_missing_object(error: &Error) -> bool {
295	matches!(error, Error::ObjectStore(ObjectStoreError::NotFound { .. }))
296}
297
298/// Splits a `servernamemediaid_metadata` value into its digest, filename, and
299/// content type. The value is `sha256(32) | filename | 0xff | content_type`
300/// with an optional trailing `0xff` that Conduit's media-auth migration appends
301/// to flag unauthenticated access; that flag is ignored.
302fn parse_conduit_media_value(value: &[u8]) -> Result<(&[u8], Option<&str>, Option<&str>)> {
303	let (sha256, rest) = value
304		.split_at_checked(32)
305		.ok_or_else(|| err!(Database("Conduit media value shorter than a SHA-256 digest")))?;
306
307	// Take filename and content_type, ignoring the optional trailing 0xff flag.
308	let mut parts = rest.split(|&byte| byte == SEP);
309	let filename = parts.next().unwrap_or_default();
310	let Some(content_type) = parts.next() else {
311		return Err!(Database("Conduit media value has no content-type separator"));
312	};
313	let filename = str::from_utf8(filename)?;
314	let content_type = str::from_utf8(content_type)?;
315	let filename = (!filename.is_empty()).then_some(filename);
316	let content_type = (!content_type.is_empty()).then_some(content_type);
317
318	Ok((sha256, filename, content_type))
319}
320
321/// The local owner of a Conduit media entry from
322/// `servernamemediaid_userlocalpart`; `None` for remote media, which has no
323/// such entry.
324async fn conduit_media_owner(
325	owners: Option<&Arc<Map>>,
326	key: &[u8],
327	server_name: &ServerName,
328) -> Option<OwnedUserId> {
329	let localpart = owners?.get(key).await.ok()?;
330
331	UserId::parse_with_server_name(str::from_utf8(&localpart).ok()?, server_name).ok()
332}
333
334/// Whether a Conduit media entry was blocked by a moderator. Conduit keeps the
335/// file and refuses it only at read time (`blocked_servername_mediaid`);
336/// tuwunel has no per-media blocklist, so importing a blocked original would
337/// serve it again. The blocklist key is `server_name 0xff media_id`, the same
338/// bytes as the `servernamemediaid_metadata` key, so the entry's raw key probes
339/// it directly. Only a clean miss imports; a hard read error aborts the import
340/// (like an unreachable source) rather than silently re-serving blocked media.
341async fn conduit_media_blocked(blocklist: Option<&Arc<Map>>, key: &[u8]) -> Result<bool> {
342	let Some(blocklist) = blocklist else {
343		return Ok(false);
344	};
345
346	match blocklist.exists(key).await {
347		| Ok(()) => Ok(true),
348		| Err(e) if e.is_not_found() => Ok(false),
349		| Err(e) => Err(e),
350	}
351}
352
353/// Reconstructs the on-disk path of a Conduit content-addressed media file from
354/// the lowercase SHA-256 hex digest naming it, matching Conduit's
355/// `split_media_path`: `media_dir` joined with the digest's shard segments.
356fn conduit_media_path(media_dir: &Path, depth: u8, length: u8, sha256_hex: &str) -> PathBuf {
357	let mut path = media_dir.to_path_buf();
358	path.extend(conduit_shards(depth, length, sha256_hex));
359	path
360}
361
362/// The object-store key of a Conduit content-addressed media object, the same
363/// shard segments as `conduit_media_path` joined by `/`. The source provider's
364/// `base_path` supplies any bucket prefix (Conduit's `media.path`).
365fn conduit_media_key(depth: u8, length: u8, sha256_hex: &str) -> String {
366	conduit_shards(depth, length, sha256_hex).join("/")
367}
368
369/// Splits a lowercase SHA-256 hex digest into Conduit's shard segments: `depth`
370/// segments of `length` characters then the remainder, or the whole digest when
371/// `depth` is zero (a flat layout). `config::check` bounds `depth * length`
372/// below the digest length so the segments never overrun it.
373fn conduit_shards(depth: u8, length: u8, sha256_hex: &str) -> impl Iterator<Item = &str> {
374	let mut rest = Some(sha256_hex);
375	let mut remaining = depth;
376	from_fn(move || {
377		let current = rest?;
378		if remaining == 0 {
379			rest = None;
380			return Some(current);
381		}
382
383		remaining = remaining.saturating_sub(1);
384		match current.split_at_checked(length.into()) {
385			| Some((segment, next)) => {
386				rest = Some(next);
387				Some(segment)
388			},
389			| None => {
390				rest = None;
391				Some(current)
392			},
393		}
394	})
395}
396
397/// Lowercase hex digest, matching the names Conduit gives its media files.
398fn sha256_hex(digest: &[u8]) -> String {
399	const HEX: &[u8; 16] = b"0123456789abcdef";
400
401	let mut out = String::with_capacity(digest.len().saturating_mul(2));
402	for &byte in digest {
403		out.push(char::from(HEX[usize::from(byte >> 4)]));
404		out.push(char::from(HEX[usize::from(byte & 0x0F)]));
405	}
406
407	out
408}
409
410/// Writes its room into each stored event lacking its `room_id`.
411///
412/// Runs once on every database, where a native one finds nothing: only a room
413/// version 12 create event imported from Conduit omits the field. The timeline is
414/// walked, then the outliers, and an event whose room nothing names is left as
415/// it is and counted.
416#[tracing::instrument(level = "debug", skip_all)]
417pub(super) async fn migrate_pdus(services: &Services) -> Result {
418	let db = &services.db;
419	let progress = &services.server.progress;
420
421	info!("Ensuring stored PDUs carry their room_id field...");
422	let cork = db.cork_and_sync();
423
424	progress.enter("timeline rows");
425	let tally = repair_all(services, timeline(services), Tally::default()).await?;
426
427	progress.enter("outlier rows");
428	let tally = repair_all(services, outliers(services), tally).await?;
429
430	drop(cork);
431
432	if tally.skipped > 0 {
433		warn!(
434			fixed = %tally.repaired,
435			skipped = %tally.skipped,
436			stale = %tally.stale,
437			"Injected room_id into stored PDUs; some were skipped"
438		);
439	} else {
440		info!(fixed = %tally.repaired, stale = %tally.stale, "Ensured stored PDUs carry room_id");
441	}
442
443	Ok(())
444}
445
446#[tracing::instrument(level = "debug", skip_all)]
447async fn repair_all<S>(services: &Services, findings: S, tally: Tally) -> Result<Tally>
448where
449	S: Stream<Item = Result<Finding>> + Send,
450{
451	// Each finding names its own row, so the repairs cannot interfere.
452	findings
453		.broad_and_then(async |finding| repair(services, &finding).await)
454		.ready_try_fold(tally, |tally, outcome| Ok(tally.record(outcome)))
455		.await
456}
457
458/// Writes the room into the event a finding names, as it stands now.
459///
460/// The row is read again and classified as the check does, taking a timeline
461/// event's room from the finding, since the short id leading its key never
462/// changes. An event whose room is unknown or that cannot be parsed is skipped,
463/// and one changed or removed since the scan is counted stale.
464#[tracing::instrument(level = "debug", skip_all)]
465async fn repair(services: &Services, finding: &Finding) -> Result<Outcome> {
466	let Finding::Missing { column, key, room_id, .. } = finding else {
467		debug_warn!(%finding, "skipping unreconcilable Conduit PDU");
468
469		return Ok(Outcome::Skipped);
470	};
471
472	let map = &services.db[column.name()];
473	let current = map.get(key).await.optional()?;
474	let outcome = match current
475		.as_deref()
476		.and_then(|value| classify(*column, key, value, || Some(room_id.clone())))
477	{
478		| Some(Finding::Missing { key, room_id, pdu, .. }) => write(map, &key, pdu, &room_id),
479		| _ => Outcome::Stale,
480	};
481
482	Ok(outcome)
483}
484
485/// Writes an event back with its room.
486///
487/// An event whose rooted form fails to serialize is skipped.
488fn write(map: &Map, key: &[u8], pdu: CanonicalJsonObject, room_id: &RoomId) -> Outcome {
489	match serde_json::to_vec(&rooted(pdu, room_id)) {
490		| Ok(bytes) => {
491			map.insert(key, bytes);
492
493			Outcome::Repaired
494		},
495		| Err(error) => {
496			debug_warn!(%error, "skipping unreconcilable Conduit PDU");
497
498			Outcome::Skipped
499		},
500	}
501}
502
503fn rooted(mut pdu: CanonicalJsonObject, room_id: &RoomId) -> CanonicalJsonObject {
504	pdu.insert("room_id".into(), CanonicalJsonValue::String(room_id.as_str().into()));
505	pdu
506}
507
508/// Imports Conduit's pending knocks. Conduit names the columns
509/// `roomuserid_knockcount` / `userroomid_knockstate`; tuwunel renamed them to
510/// `*knocked*` but kept the byte layout (`room_id 0xff user_id` -> u64 count;
511/// `user_id 0xff room_id` -> JSON stripped state), so each row copies verbatim.
512/// Imported once: tuwunel clears a knock on the user's join or leave, so a
513/// re-import would resurrect a knock the user has already resolved.
514pub(super) async fn migrate_conduit_knocks(services: &Services) -> Result {
515	let knocks = copy_cf(services, "roomuserid_knockcount", "roomuserid_knockedcount").await?;
516	copy_cf(services, "userroomid_knockstate", "userroomid_knockedstate").await?;
517
518	if knocks > 0 {
519		warn!(%knocks, "Imported Conduit knocks");
520	}
521
522	Ok(())
523}
524
525/// Copies every row of one column verbatim into another.
526///
527/// The two columns share the byte layout of their keys and values, so neither
528/// needs reserializing. Returns how many rows were copied.
529async fn copy_cf(
530	services: &Services,
531	source_name: &'static str,
532	target_name: &'static str,
533) -> Result<usize> {
534	let db = &services.db;
535	let Some(source) = db.open_cf(source_name)? else {
536		return Ok(0);
537	};
538
539	let target = &db[target_name];
540	let cork = db.cork_and_sync();
541	let copied = source
542		.raw_stream()
543		.ignore_err()
544		.ready_fold(0_usize, |copied, (key, value)| {
545			target.insert(key, value);
546			copied.saturating_add(1)
547		})
548		.await;
549
550	drop(cork);
551
552	Ok(copied)
553}
554
555#[cfg(test)]
556mod tests {
557	use std::path::Path;
558
559	use super::{conduit_media_key, conduit_media_path, parse_conduit_media_value, sha256_hex};
560
561	#[test]
562	fn media_path_deep_matches_conduit_default() {
563		// Conduit default Deep { length: 2, depth: 2 }: two 2-char segments of the
564		// 64-char digest, then the remaining 60 characters.
565		let hex = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef";
566		let path = conduit_media_path(Path::new("/db/media"), 2, 2, hex);
567
568		assert_eq!(
569			path,
570			Path::new(
571				"/db/media/01/23/456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"
572			)
573		);
574	}
575
576	#[test]
577	fn conduit_media_path_flat_is_unsharded() {
578		let path = conduit_media_path(Path::new("/db/media"), 0, 2, "abcdef");
579
580		assert_eq!(path, Path::new("/db/media/abcdef"));
581	}
582
583	#[test]
584	fn conduit_media_key_deep_joins_shards_with_slash() {
585		// The object key carries no media_dir; the source provider's base_path
586		// supplies any prefix. Same shard segments as the Deep on-disk path.
587		let hex = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef";
588		let key = conduit_media_key(2, 2, hex);
589
590		assert_eq!(key, "01/23/456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef");
591	}
592
593	#[test]
594	fn conduit_media_key_flat_is_bare_digest() {
595		assert_eq!(conduit_media_key(0, 2, "abcdef"), "abcdef");
596	}
597
598	#[test]
599	fn sha256_hex_encodes_lowercase_padded() {
600		assert_eq!(sha256_hex(&[0x00, 0x0F, 0xFF, 0xA5]), "000fffa5");
601	}
602
603	#[test]
604	fn conduit_media_value_ignores_unauthenticated_flag() {
605		// Conduit's media-auth migration appends a trailing 0xff after content_type.
606		let mut value = vec![7_u8; 32];
607		value.extend_from_slice(b"pic.png");
608		value.push(0xFF);
609		value.extend_from_slice(b"image/png");
610		value.push(0xFF);
611
612		let (sha256, filename, content_type) = parse_conduit_media_value(&value).unwrap();
613
614		assert_eq!(sha256, [7_u8; 32].as_slice());
615		assert_eq!(filename, Some("pic.png"));
616		assert_eq!(content_type, Some("image/png"));
617	}
618
619	#[test]
620	fn conduit_media_value_empty_filename_is_none() {
621		let mut value = vec![0_u8; 32];
622		value.push(0xFF);
623		value.extend_from_slice(b"image/png");
624
625		let (_, filename, content_type) = parse_conduit_media_value(&value).unwrap();
626
627		assert_eq!(filename, None);
628		assert_eq!(content_type, Some("image/png"));
629	}
630}