1use std::{
2 iter::from_fn,
3 path::{Path, PathBuf},
4 sync::Arc,
5 time::Duration,
6};
7
8use bytes::Bytes;
9use futures::{Stream, StreamExt, TryStreamExt};
10use object_store::Error as ObjectStoreError;
11use ruma::{
12 CanonicalJsonObject, CanonicalJsonValue, Mxc, OwnedUserId, RoomId, ServerName, UserId,
13};
14use tokio::time::sleep;
15use tuwunel_core::{
16 Err, Error, Result, debug_warn, err, error, info,
17 itertools::Itertools,
18 result::NotFound,
19 utils::{
20 ReadyExt, TryReadyExt,
21 content_disposition::make_content_disposition,
22 stream::{TryBroadbandExt, TryIgnore},
23 },
24 warn,
25};
26use tuwunel_database::{Map, SEP};
27
28use super::tally::{Outcome, Tally};
29use crate::{
30 Services,
31 integrity::pdu_room_ids::{Finding, classify, outliers, timeline},
32 storage::Provider,
33};
34
35enum MediaSource {
40 Filesystem(PathBuf),
41 Provider(Arc<Provider>),
42}
43
44struct ConduitMediaEntry<'a> {
46 server_name: &'a ServerName,
47 media_id: &'a str,
48 sha256: &'a [u8],
49 filename: Option<&'a str>,
50 content_type: Option<&'a str>,
51}
52
53fn media_source(services: &Services) -> Result<MediaSource> {
57 let config = &services.server.config;
58
59 match config
60 .media
61 .import
62 .conduit
63 .source_provider
64 .as_deref()
65 {
66 | Some(name) => services
67 .storage
68 .provider(name)
69 .map(|provider| MediaSource::Provider(provider.clone())),
70 | None => {
71 let media_dir = config
72 .media
73 .import
74 .conduit
75 .source_path
76 .clone()
77 .unwrap_or_else(|| config.database_path.join("media"));
78
79 Ok(MediaSource::Filesystem(media_dir))
80 },
81 }
82}
83
84const PROVIDER_READ_ATTEMPTS: u32 = 2;
89
90const PROVIDER_READ_RETRY_DELAY: Duration = Duration::from_secs(2);
92
93pub(super) async fn migrate_conduit_media(services: &Services) -> Result {
102 let db = &services.db;
103 let config = &services.server.config;
104
105 let Some(metadata) = db.open_cf("servernamemediaid_metadata")? else {
106 warn!("Conduit database has no media metadata; nothing to import.");
107 return Ok(());
108 };
109
110 let owners = db.open_cf("servernamemediaid_userlocalpart")?;
111 let owners = owners.as_ref();
112
113 let blocklist = db.open_cf("blocked_servername_mediaid")?;
114 let blocklist = blocklist.as_ref();
115
116 let depth = config.media.import.conduit.directory_depth;
117 let length = config.media.import.conduit.directory_length;
118 let source = media_source(services)?;
119
120 warn!("Importing Conduit media originals into tuwunel's key-addressed store...");
121
122 let cork = db.cork_and_sync();
123 let (imported, skipped, blocked) = metadata
124 .raw_stream()
125 .ignore_err()
126 .map(Ok::<_, Error>)
127 .try_fold(
128 (0_usize, 0_usize, 0_usize),
129 async |(imported, skipped, blocked), (key, value)| {
130 if conduit_media_blocked(blocklist, key).await? {
131 return Ok((imported, skipped, blocked.saturating_add(1)));
132 }
133
134 let imported_entry =
135 import_conduit_original(services, owners, &source, depth, length, key, value)
136 .await?;
137
138 Ok(if imported_entry {
139 (imported.saturating_add(1), skipped, blocked)
140 } else {
141 (imported, skipped.saturating_add(1), blocked)
142 })
143 },
144 )
145 .await?;
146
147 drop(cork);
148
149 if blocked > 0 {
150 warn!(%blocked, "Skipped Conduit media blocked by a moderator; not imported");
151 }
152
153 if skipped > 0 {
154 warn!(%imported, %skipped, "Imported Conduit media originals; some files were skipped");
155 } else {
156 info!(%imported, "Imported Conduit media originals");
157 }
158
159 Ok(())
160}
161
162async fn import_conduit_original(
168 services: &Services,
169 owners: Option<&Arc<Map>>,
170 source: &MediaSource,
171 depth: u8,
172 length: u8,
173 key: &[u8],
174 value: &[u8],
175) -> Result<bool> {
176 let entry = match parse_conduit_media_entry(key, value) {
177 | Ok(entry) => entry,
178 | Err(e) => {
179 debug_warn!(error = %e, "skipping unimportable Conduit media entry");
180 return Ok(false);
181 },
182 };
183
184 let Some(file) = read_conduit_original(source, depth, length, entry.sha256).await? else {
185 return Ok(false);
186 };
187
188 let content_disposition = make_content_disposition(None, entry.content_type, entry.filename);
189 let owner = conduit_media_owner(owners, key, entry.server_name).await;
190 let mxc = Mxc {
191 server_name: entry.server_name,
192 media_id: entry.media_id,
193 };
194
195 services
196 .media
197 .create(&mxc, owner.as_deref(), Some(&content_disposition), entry.content_type, &file)
198 .await?;
199
200 Ok(true)
201}
202
203fn parse_conduit_media_entry<'a>(
207 key: &'a [u8],
208 value: &'a [u8],
209) -> Result<ConduitMediaEntry<'a>> {
210 let Some(sep) = key.iter().position(|&byte| byte == SEP) else {
211 return Err!(Database("Conduit media key has no server-name separator"));
212 };
213 let server_name = <&ServerName>::try_from(str::from_utf8(&key[..sep])?)
214 .map_err(|_| err!(Database("Conduit media key has an invalid server name")))?;
215
216 let media_id = str::from_utf8(&key[sep.saturating_add(1)..])?;
217
218 let (sha256, filename, content_type) = parse_conduit_media_value(value)?;
219
220 Ok(ConduitMediaEntry {
221 server_name,
222 media_id,
223 sha256,
224 filename,
225 content_type,
226 })
227}
228
229async fn read_conduit_original(
235 source: &MediaSource,
236 depth: u8,
237 length: u8,
238 sha256: &[u8],
239) -> Result<Option<Bytes>> {
240 let sha256_hex = sha256_hex(sha256);
241 match source {
242 | MediaSource::Filesystem(media_dir) => {
243 let path = conduit_media_path(media_dir, depth, length, &sha256_hex);
244 match tokio::fs::read(&path).await {
245 | Ok(file) => Ok(Some(file.into())),
246 | Err(e) => {
247 debug_warn!(?path, error = %e, "skipping unreadable Conduit media file");
248 Ok(None)
249 },
250 }
251 },
252 | MediaSource::Provider(provider) =>
253 read_provider_original(provider, &conduit_media_key(depth, length, &sha256_hex)).await,
254 }
255}
256
257async fn read_provider_original(provider: &Arc<Provider>, key: &str) -> Result<Option<Bytes>> {
262 let mut attempt = 0_u32;
263 loop {
264 attempt = attempt.saturating_add(1);
265 match provider.get(key).await {
266 | Ok(file) => return Ok(Some(file)),
267 | Err(e) if is_missing_object(&e) => {
268 debug_warn!(%key, error = %e, "skipping missing Conduit media object");
269 return Ok(None);
270 },
271 | Err(e) if attempt >= PROVIDER_READ_ATTEMPTS => {
272 error!(
273 %key,
274 attempts = PROVIDER_READ_ATTEMPTS,
275 error = %e,
276 "Aborting the Conduit media import: source storage provider unreachable. No \
277 media has been imported in a way that needs cleanup; once the provider is \
278 reachable, restart tuwunel to resume the import from the beginning."
279 );
280 return Err(e);
281 },
282 | Err(e) => {
283 warn!(%key, attempt, error = %e, "Reading Conduit media object failed; retrying");
284 sleep(PROVIDER_READ_RETRY_DELAY).await;
285 },
286 }
287 }
288}
289
290fn is_missing_object(error: &Error) -> bool {
295 matches!(error, Error::ObjectStore(ObjectStoreError::NotFound { .. }))
296}
297
298fn parse_conduit_media_value(value: &[u8]) -> Result<(&[u8], Option<&str>, Option<&str>)> {
303 let (sha256, rest) = value
304 .split_at_checked(32)
305 .ok_or_else(|| err!(Database("Conduit media value shorter than a SHA-256 digest")))?;
306
307 let mut parts = rest.split(|&byte| byte == SEP);
309 let filename = parts.next().unwrap_or_default();
310 let Some(content_type) = parts.next() else {
311 return Err!(Database("Conduit media value has no content-type separator"));
312 };
313 let filename = str::from_utf8(filename)?;
314 let content_type = str::from_utf8(content_type)?;
315 let filename = (!filename.is_empty()).then_some(filename);
316 let content_type = (!content_type.is_empty()).then_some(content_type);
317
318 Ok((sha256, filename, content_type))
319}
320
321async fn conduit_media_owner(
325 owners: Option<&Arc<Map>>,
326 key: &[u8],
327 server_name: &ServerName,
328) -> Option<OwnedUserId> {
329 let localpart = owners?.get(key).await.ok()?;
330
331 UserId::parse_with_server_name(str::from_utf8(&localpart).ok()?, server_name).ok()
332}
333
334async fn conduit_media_blocked(blocklist: Option<&Arc<Map>>, key: &[u8]) -> Result<bool> {
342 let Some(blocklist) = blocklist else {
343 return Ok(false);
344 };
345
346 match blocklist.exists(key).await {
347 | Ok(()) => Ok(true),
348 | Err(e) if e.is_not_found() => Ok(false),
349 | Err(e) => Err(e),
350 }
351}
352
353fn conduit_media_path(media_dir: &Path, depth: u8, length: u8, sha256_hex: &str) -> PathBuf {
357 let mut path = media_dir.to_path_buf();
358 path.extend(conduit_shards(depth, length, sha256_hex));
359 path
360}
361
362fn conduit_media_key(depth: u8, length: u8, sha256_hex: &str) -> String {
366 conduit_shards(depth, length, sha256_hex).join("/")
367}
368
369fn conduit_shards(depth: u8, length: u8, sha256_hex: &str) -> impl Iterator<Item = &str> {
374 let mut rest = Some(sha256_hex);
375 let mut remaining = depth;
376 from_fn(move || {
377 let current = rest?;
378 if remaining == 0 {
379 rest = None;
380 return Some(current);
381 }
382
383 remaining = remaining.saturating_sub(1);
384 match current.split_at_checked(length.into()) {
385 | Some((segment, next)) => {
386 rest = Some(next);
387 Some(segment)
388 },
389 | None => {
390 rest = None;
391 Some(current)
392 },
393 }
394 })
395}
396
397fn sha256_hex(digest: &[u8]) -> String {
399 const HEX: &[u8; 16] = b"0123456789abcdef";
400
401 let mut out = String::with_capacity(digest.len().saturating_mul(2));
402 for &byte in digest {
403 out.push(char::from(HEX[usize::from(byte >> 4)]));
404 out.push(char::from(HEX[usize::from(byte & 0x0F)]));
405 }
406
407 out
408}
409
410#[tracing::instrument(level = "debug", skip_all)]
417pub(super) async fn migrate_pdus(services: &Services) -> Result {
418 let db = &services.db;
419 let progress = &services.server.progress;
420
421 info!("Ensuring stored PDUs carry their room_id field...");
422 let cork = db.cork_and_sync();
423
424 progress.enter("timeline rows");
425 let tally = repair_all(services, timeline(services), Tally::default()).await?;
426
427 progress.enter("outlier rows");
428 let tally = repair_all(services, outliers(services), tally).await?;
429
430 drop(cork);
431
432 if tally.skipped > 0 {
433 warn!(
434 fixed = %tally.repaired,
435 skipped = %tally.skipped,
436 stale = %tally.stale,
437 "Injected room_id into stored PDUs; some were skipped"
438 );
439 } else {
440 info!(fixed = %tally.repaired, stale = %tally.stale, "Ensured stored PDUs carry room_id");
441 }
442
443 Ok(())
444}
445
446#[tracing::instrument(level = "debug", skip_all)]
447async fn repair_all<S>(services: &Services, findings: S, tally: Tally) -> Result<Tally>
448where
449 S: Stream<Item = Result<Finding>> + Send,
450{
451 findings
453 .broad_and_then(async |finding| repair(services, &finding).await)
454 .ready_try_fold(tally, |tally, outcome| Ok(tally.record(outcome)))
455 .await
456}
457
458#[tracing::instrument(level = "debug", skip_all)]
465async fn repair(services: &Services, finding: &Finding) -> Result<Outcome> {
466 let Finding::Missing { column, key, room_id, .. } = finding else {
467 debug_warn!(%finding, "skipping unreconcilable Conduit PDU");
468
469 return Ok(Outcome::Skipped);
470 };
471
472 let map = &services.db[column.name()];
473 let current = map.get(key).await.optional()?;
474 let outcome = match current
475 .as_deref()
476 .and_then(|value| classify(*column, key, value, || Some(room_id.clone())))
477 {
478 | Some(Finding::Missing { key, room_id, pdu, .. }) => write(map, &key, pdu, &room_id),
479 | _ => Outcome::Stale,
480 };
481
482 Ok(outcome)
483}
484
485fn write(map: &Map, key: &[u8], pdu: CanonicalJsonObject, room_id: &RoomId) -> Outcome {
489 match serde_json::to_vec(&rooted(pdu, room_id)) {
490 | Ok(bytes) => {
491 map.insert(key, bytes);
492
493 Outcome::Repaired
494 },
495 | Err(error) => {
496 debug_warn!(%error, "skipping unreconcilable Conduit PDU");
497
498 Outcome::Skipped
499 },
500 }
501}
502
503fn rooted(mut pdu: CanonicalJsonObject, room_id: &RoomId) -> CanonicalJsonObject {
504 pdu.insert("room_id".into(), CanonicalJsonValue::String(room_id.as_str().into()));
505 pdu
506}
507
508pub(super) async fn migrate_conduit_knocks(services: &Services) -> Result {
515 let knocks = copy_cf(services, "roomuserid_knockcount", "roomuserid_knockedcount").await?;
516 copy_cf(services, "userroomid_knockstate", "userroomid_knockedstate").await?;
517
518 if knocks > 0 {
519 warn!(%knocks, "Imported Conduit knocks");
520 }
521
522 Ok(())
523}
524
525async fn copy_cf(
530 services: &Services,
531 source_name: &'static str,
532 target_name: &'static str,
533) -> Result<usize> {
534 let db = &services.db;
535 let Some(source) = db.open_cf(source_name)? else {
536 return Ok(0);
537 };
538
539 let target = &db[target_name];
540 let cork = db.cork_and_sync();
541 let copied = source
542 .raw_stream()
543 .ignore_err()
544 .ready_fold(0_usize, |copied, (key, value)| {
545 target.insert(key, value);
546 copied.saturating_add(1)
547 })
548 .await;
549
550 drop(cork);
551
552 Ok(copied)
553}
554
555#[cfg(test)]
556mod tests {
557 use std::path::Path;
558
559 use super::{conduit_media_key, conduit_media_path, parse_conduit_media_value, sha256_hex};
560
561 #[test]
562 fn media_path_deep_matches_conduit_default() {
563 let hex = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef";
566 let path = conduit_media_path(Path::new("/db/media"), 2, 2, hex);
567
568 assert_eq!(
569 path,
570 Path::new(
571 "/db/media/01/23/456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"
572 )
573 );
574 }
575
576 #[test]
577 fn conduit_media_path_flat_is_unsharded() {
578 let path = conduit_media_path(Path::new("/db/media"), 0, 2, "abcdef");
579
580 assert_eq!(path, Path::new("/db/media/abcdef"));
581 }
582
583 #[test]
584 fn conduit_media_key_deep_joins_shards_with_slash() {
585 let hex = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef";
588 let key = conduit_media_key(2, 2, hex);
589
590 assert_eq!(key, "01/23/456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef");
591 }
592
593 #[test]
594 fn conduit_media_key_flat_is_bare_digest() {
595 assert_eq!(conduit_media_key(0, 2, "abcdef"), "abcdef");
596 }
597
598 #[test]
599 fn sha256_hex_encodes_lowercase_padded() {
600 assert_eq!(sha256_hex(&[0x00, 0x0F, 0xFF, 0xA5]), "000fffa5");
601 }
602
603 #[test]
604 fn conduit_media_value_ignores_unauthenticated_flag() {
605 let mut value = vec![7_u8; 32];
607 value.extend_from_slice(b"pic.png");
608 value.push(0xFF);
609 value.extend_from_slice(b"image/png");
610 value.push(0xFF);
611
612 let (sha256, filename, content_type) = parse_conduit_media_value(&value).unwrap();
613
614 assert_eq!(sha256, [7_u8; 32].as_slice());
615 assert_eq!(filename, Some("pic.png"));
616 assert_eq!(content_type, Some("image/png"));
617 }
618
619 #[test]
620 fn conduit_media_value_empty_filename_is_none() {
621 let mut value = vec![0_u8; 32];
622 value.push(0xFF);
623 value.extend_from_slice(b"image/png");
624
625 let (_, filename, content_type) = parse_conduit_media_value(&value).unwrap();
626
627 assert_eq!(filename, None);
628 assert_eq!(content_type, Some("image/png"));
629 }
630}