1use std::{cmp::Ordering, time::Duration};
10
11use futures::{FutureExt, StreamExt, TryStreamExt};
12use ruma::{OwnedUserId, RoomId, ServerName, UserId};
13use tokio::time::sleep;
14use tuwunel_core::{
15 Err, Result, err, format_small_string, info,
16 result::NotFound,
17 smallstr::SmallString,
18 smallvec::SmallVec,
19 utils::{BoolExt, IterStream, ReadyExt, TryReadyExt},
20 warn,
21};
22use tuwunel_database::Deserialized;
23
24use self::{
25 Rerun::{Clear, Refuse},
26 account_status::migrate_account_status,
27 clear_servername_status::clear_servername_status,
28 clear_state_local_error_memos::clear_state_local_error_memos,
29 conduit::migrate_pdus,
30 email_bindings::migrate_email_bindings,
31 fix_bad_double_separator_in_state_cache::fix_bad_double_separator_in_state_cache,
32 fix_hashed_sentinel_passwords::fix_hashed_sentinel_passwords,
33 fix_readreceiptid_readreceipt_duplicates::fix_readreceiptid_readreceipt_duplicates,
34 fix_referencedevents_missing_sep::fix_referencedevents_missing_sep,
35 import_conduit_knocks::import_conduit_knocks,
36 injectivity::{fix as fix_injectivity, mark_clean as mark_clean_injectivity},
37 migrate_media::migrate_media,
38 migrate_profile_keys::migrate_profile_keys,
39 moderation::migrate as migrate_moderation,
40 rebuild_roomid_tscount_pducount::rebuild_roomid_tscount_pducount,
41 redactions::restore_redactions,
42 remove_remote_media_userid::remove_remote_media_userid,
43 retroactively_fix_bad_data_from_roomuserid_joined::retroactively_fix_bad_data_from_roomuserid_joined,
44 split_conduit_highlight_counts::split_conduit_highlight_counts,
45 token_expiry::{
46 Restore, migrate as migrate_token_expiry, restore as restore_token_expiry, restore_due,
47 },
48 upgrade_legacy_mediaid_user::upgrade_legacy_mediaid_user,
49};
50use crate::{
51 Services,
52 integrity::{
53 find as find_check,
54 forbidden_names::{
55 Username, matched as forbidden_patterns, usernames as forbidden_usernames,
56 },
57 },
58};
59
60mod account_status;
61mod clear_servername_status;
62mod clear_state_local_error_memos;
63mod conduit;
64mod email_bindings;
65mod fix_bad_double_separator_in_state_cache;
66mod fix_hashed_sentinel_passwords;
67mod fix_readreceiptid_readreceipt_duplicates;
68mod fix_referencedevents_missing_sep;
69mod import_conduit_knocks;
70pub(crate) mod injectivity;
71mod migrate_media;
72mod migrate_profile_keys;
73mod moderation;
74mod rebuild_roomid_tscount_pducount;
75mod redactions;
76mod remove_remote_media_userid;
77mod retroactively_fix_bad_data_from_roomuserid_joined;
78mod split_conduit_highlight_counts;
79mod tally;
80mod token_expiry;
81mod upgrade_legacy_mediaid_user;
82
83#[cfg(test)]
84mod tests;
85
86pub(crate) const DATABASE_VERSION: u64 = 17;
93
94const SERVER_NAME_KEY: &[u8] = b"server_name";
95
96const FORCE_MIGRATION_DELAY: Duration = Duration::from_secs(15);
97
98const FIX_PDU_MISSING_ROOM_ID: &str = "fix_pdu_missing_room_id";
99
100const SPLIT_CONDUIT_HIGHLIGHT: &str = "split_conduit_highlight";
101
102const REBUILD_RELATESTO_TYPED: &str = "rebuild_relatesto_typed";
103
104const REBUILD_THREAD_SUMMARIES: &str = "rebuild_thread_summaries";
105
106const CLEAR_STATE_LOCAL_ERROR_MEMOS: &str = "clear_state_local_error_memos";
107
108const REBUILD_LIVE_THREAD_ACTIVITY: &str = "rebuild_live_thread_activity";
109
110pub(crate) const ADOPT_ACCOUNT_STATUS: &str = "adopt_foreign_account_status";
111
112pub(crate) const ADOPT_EMAIL_BINDINGS: &str = "adopt_foreign_email_bindings";
113
114pub(crate) const SHA256_MEDIA: &str = "feat_sha256_media";
119
120const FOREIGN_LINEAGE_MARKER: &[u8] = b"populate_userroomid_leftstate_table";
125
126type UserIdBuf = SmallString<[u8; 48]>;
128
129type Stamps = SmallVec<[Stamp; 1]>;
130
131#[derive(Clone, Copy)]
136pub struct Stamp {
137 pub marker: &'static str,
139
140 pub cleared: bool,
142}
143
144#[derive(Clone, Copy)]
149enum Rerun {
150 Clear(&'static [&'static str]),
152
153 Refuse(&'static str),
155}
156
157pub(crate) const MARKERS: [&str; 21] = [
164 SHA256_MEDIA,
165 FIX_PDU_MISSING_ROOM_ID,
166 "fix_bad_double_separator_in_state_cache",
167 "retroactively_fix_bad_data_from_roomuserid_joined",
168 "fix_referencedevents_missing_sep",
169 "fix_readreceiptid_readreceipt_duplicates",
170 "fix_hashed_sentinel_passwords",
171 "upgrade_legacy_mediaid_user",
172 "remove_remote_media_userid",
173 "rebuild_roomid_tscount_pducount",
174 REBUILD_RELATESTO_TYPED,
175 "migrate_profile_keys_to_useridprofilekey",
176 REBUILD_THREAD_SUMMARIES,
177 "clear_servername_status",
178 CLEAR_STATE_LOCAL_ERROR_MEMOS,
179 ADOPT_ACCOUNT_STATUS,
180 ADOPT_EMAIL_BINDINGS,
181 token_expiry::RESTORE_MARKER,
182 token_expiry::ADOPT_MARKER,
183 redactions::MARKER,
184 REBUILD_LIVE_THREAD_ACTIVITY,
185];
186
187static RERUNS: &[(&str, Rerun)] = &[
194 ("schema", Refuse(NO_REPAIR)),
195 ("columns", Refuse(NO_REPAIR)),
196 ("unused-columns", Refuse(NO_REPAIR)),
197 ("pdu-room-ids", Clear(&[FIX_PDU_MISSING_ROOM_ID])),
198 ("joined-keys", Refuse(UNVERIFIED)),
199 ("referenced-events", Refuse(UNVERIFIED)),
200 ("read-receipts", Refuse(UNVERIFIED)),
201 ("highlight-keys", Clear(&[SPLIT_CONDUIT_HIGHLIGHT])),
202 ("media-uploader-keys", Refuse(UNVERIFIED)),
203 ("remote-media-uploaders", Refuse(UNVERIFIED)),
204 ("legacy-media-names", Refuse(MEDIA_RENAME)),
205 ("media-files", Refuse(MEDIA_CHECKUP)),
206 ("profile-values", Refuse(PROFILE_COPY)),
207 ("sentinel-passwords", Refuse(UNVERIFIED)),
208 ("forbidden-usernames", Refuse(NO_REPAIR)),
209 ("forbidden-aliases", Refuse(NO_REPAIR)),
210 ("joined-members", Refuse(UNVERIFIED)),
211 ("foreign-moderation", Refuse(EVERY_START)),
212 ("foreign-account-status", Refuse(ADOPTION)),
213 ("foreign-email-bindings", Refuse(ADOPTION)),
214 ("foreign-token-expiry", Refuse(ADOPTION)),
215 ("timestamp-index", Refuse(UNVERIFIED)),
216 ("relation-index", Clear(&[REBUILD_RELATESTO_TYPED])),
217 ("unrecorded-redactions", Refuse(REDACTIONS)),
218 (
220 "thread-summaries",
221 Clear(&[REBUILD_LIVE_THREAD_ACTIVITY, REBUILD_THREAD_SUMMARIES]),
222 ),
223 ("thread-activity", Clear(&[REBUILD_LIVE_THREAD_ACTIVITY])),
224 ("short-ids", Refuse(OUTCOME_RECORD)),
225];
226
227const NO_REPAIR: &str = "it has no repair";
228
229const UNVERIFIED: &str = "a second run of its repair has not been verified safe";
230
231const EVERY_START: &str = "its repair keeps no stamp and runs at every start";
232
233const MEDIA_CHECKUP: &str =
234 "its repair keeps no stamp and runs at every start while media.startup_check is set";
235
236const MEDIA_RENAME: &str =
237 "a second run of its repair can lose media files when media.compat_file_link is set";
238
239const PROFILE_COPY: &str =
240 "its step also copies profile fields from older columns over the current ones";
241
242const ADOPTION: &str = "its step adopts a foreign database's state once, and a second run would \
243 undo changes made here since";
244
245const REDACTIONS: &str =
246 "a second run of its repair could also record redactions this server refused";
247
248const OUTCOME_RECORD: &str = "its repair is gated by the outcome it records rather than a stamp";
249
250pub(crate) async fn migrations(services: &Services) -> Result {
251 if services.config.force_migration {
252 warn!(
253 delay = ?FORCE_MIGRATION_DELAY,
254 "The force_migration option is set. THIS IS NOT INTENDED TO BE USED UNDER ANY \
255 NORMAL CIRCUMSTANCES AND YOU MAY BE CORRUPTING YOUR DATABASE BY PROCEEDING. \
256 Remove force_migration from the configuration to clear this warning; startup \
257 continues after the delay."
258 );
259
260 sleep(FORCE_MIGRATION_DELAY).await;
261 }
262
263 if !services.config.database_migrations {
264 if !marker_present(services, CLEAR_STATE_LOCAL_ERROR_MEMOS).await? {
265 return Err!(Config(
266 "database_migrations",
267 "Local state memo invalidation is pending. Enable database_migrations for one \
268 startup, then this setting may be disabled again."
269 ));
270 }
271
272 warn!("Skipping database migrations due to configuration...");
273 return Ok(());
274 }
275
276 services.server.check_running()?;
278
279 let users_count = services.users.count().await;
280 if users_count == 0 {
281 return fresh(services).await;
282 }
283
284 let foreign_lineage = is_foreign_lineage(services).await;
287
288 check_database_version(services, foreign_lineage).await?;
289 check_server_name(services).await?;
290
291 services.server.check_running()?;
292
293 fix_injectivity(services).await;
296
297 let migrated = migrate(services, foreign_lineage).await;
298
299 services.server.progress.end();
300
301 migrated.inspect_err(|error| {
302 if error.is_interrupted() {
303 warn!(
304 "Stopped during database migrations. The steps that completed are recorded; the \
305 rest run on the next start."
306 );
307 }
308 })
309}
310
311async fn is_foreign_lineage(services: &Services) -> bool {
316 let global = &services.db["global"];
317
318 global.get(SERVER_NAME_KEY).await.is_not_found()
319 || global.get(FOREIGN_LINEAGE_MARKER).await.is_ok()
320}
321
322async fn check_database_version(services: &Services, foreign_lineage: bool) -> Result {
330 let discovered = services.globals.db.database_version().await;
331
332 if discovered < 13 {
333 return Err!(Database("Database schema version {discovered} is no longer supported"));
334 }
335
336 if discovered > DATABASE_VERSION && !foreign_lineage && !services.config.force_migration {
337 return Err!(Database(
338 "Database schema version {discovered} is newer than this build supports \
339 ({DATABASE_VERSION}). Upgrade tuwunel to a build supporting this database."
340 ));
341 }
342
343 Ok(())
344}
345
346async fn check_server_name(services: &Services) -> Result {
351 let server_name = &services.server.name;
352
353 let existing = services.db["global"]
354 .get(SERVER_NAME_KEY)
355 .await
356 .deserialized::<String>();
357
358 match existing {
359 | Err(_) => backfill_server_name(services).await,
360 | Ok(existing) if existing.eq(server_name) => Ok(()),
361 | Ok(existing) => Err!(Database(
362 "Database belongs to {existing}; configured server name is {server_name}. Cannot \
363 reuse."
364 )),
365 }
366}
367
368async fn backfill_server_name(services: &Services) -> Result {
372 let server_name = &services.server.name;
373
374 services
375 .users
376 .stream()
377 .ready_any(|user_id| services.globals.user_is_local(user_id))
378 .await
379 .into_option()
380 .ok_or_else(|| {
381 err!(Database(
382 "Database has no users from {server_name}; refusing to reuse with this \
383 server_name."
384 ))
385 })?;
386
387 services.db["global"].insert(SERVER_NAME_KEY, server_name.as_str());
388 info!(%server_name, "Stamped server_name marker on upgraded database");
389
390 Ok(())
391}
392
393pub(crate) async fn fresh(services: &Services) -> Result {
394 let db = &services.db;
395
396 services
397 .globals
398 .db
399 .bump_database_version(DATABASE_VERSION);
400
401 db["global"].insert(SERVER_NAME_KEY, services.server.name.as_str());
402 for marker in MARKERS {
403 db["global"].insert(marker, []);
404 }
405
406 db["global"].insert("rebuild_thread_activity", []);
408 mark_clean_injectivity(services);
409
410 if services.config.admin.room.create {
412 crate::admin::create_admin_room(services)
413 .boxed() .await?;
415 }
416
417 warn!("Created new RocksDB database with version {DATABASE_VERSION}");
418
419 Ok(())
420}
421
422pub(crate) async fn migrate(services: &Services, foreign_lineage: bool) -> Result {
424 let db = &services.db;
425
426 services.server.check_running()?;
427
428 let target_version = DATABASE_VERSION;
429 let discovered = services.globals.db.database_version().await;
430
431 if foreign_lineage && discovered > target_version {
438 services
439 .globals
440 .db
441 .bump_database_version(target_version);
442 }
443
444 migrate_media(services).await?;
445
446 run_once(services, FIX_PDU_MISSING_ROOM_ID, migrate_pdus).await?;
447
448 import_conduit_knocks(services).await?;
449
450 run_once(services, SPLIT_CONDUIT_HIGHLIGHT, split_conduit_highlight_counts).await?;
451
452 if db
455 .open_cf("servernamemediaid_metadata")?
456 .is_some()
457 {
458 db["global"].insert("fix_bad_double_separator_in_state_cache", []);
459 db["global"].insert("retroactively_fix_bad_data_from_roomuserid_joined", []);
460 }
461
462 run_once(
463 services,
464 "fix_bad_double_separator_in_state_cache",
465 fix_bad_double_separator_in_state_cache,
466 )
467 .await?;
468
469 run_once(
470 services,
471 "retroactively_fix_bad_data_from_roomuserid_joined",
472 retroactively_fix_bad_data_from_roomuserid_joined,
473 )
474 .await?;
475
476 run_once(services, "fix_referencedevents_missing_sep", fix_referencedevents_missing_sep)
477 .await?;
478
479 run_once(
480 services,
481 "fix_readreceiptid_readreceipt_duplicates",
482 fix_readreceiptid_readreceipt_duplicates,
483 )
484 .await?;
485
486 until_finished(services, "fix_hashed_sentinel_passwords", fix_hashed_sentinel_passwords)
487 .await?;
488
489 run_once(services, "upgrade_legacy_mediaid_user", upgrade_legacy_mediaid_user).await?;
490 run_once(services, "remove_remote_media_userid", remove_remote_media_userid).await?;
491
492 run_once(services, "rebuild_roomid_tscount_pducount", rebuild_roomid_tscount_pducount)
493 .await?;
494
495 run_once(services, REBUILD_RELATESTO_TYPED, async |services: &Services| {
496 services
497 .pdu_metadata
498 .rebuild_typed_relations()
499 .await
500 })
501 .await?;
502
503 run_once(services, "migrate_profile_keys_to_useridprofilekey", migrate_profile_keys).await?;
504
505 run_once(services, REBUILD_THREAD_SUMMARIES, async |services: &Services| {
506 rebuild_thread_summaries(services).await;
507
508 Ok(())
509 })
510 .await?;
511
512 if pending(services, "clear_servername_status").await? {
513 clear_servername_status(services).await?;
514 }
515
516 if pending(services, CLEAR_STATE_LOCAL_ERROR_MEMOS).await? {
517 clear_state_local_error_memos(services).await?;
518 }
519
520 services.server.check_running()?;
521
522 services
526 .server
527 .progress
528 .begin("migrate_moderation");
529
530 migrate_moderation(services).await?;
531
532 run_once(services, ADOPT_ACCOUNT_STATUS, migrate_account_status).await?;
533 run_once(services, ADOPT_EMAIL_BINDINGS, migrate_email_bindings).await?;
534
535 until_finished(services, token_expiry::RESTORE_MARKER, async |services: &Services| {
537 let tokenexpires = match restore_due(services)? {
538 | Restore::Finished => return Ok(true),
539 | Restore::Waiting => return Ok(false),
540 | Restore::Due(tokenexpires) => tokenexpires,
541 };
542
543 services.db["global"].remove(token_expiry::ADOPT_MARKER);
546 restore_token_expiry(services, &tokenexpires).await?;
547
548 Ok(true)
549 })
550 .await?;
551
552 until_finished(services, token_expiry::ADOPT_MARKER, migrate_token_expiry).await?;
553
554 run_once(services, redactions::MARKER, restore_redactions).await?;
555
556 run_once(services, REBUILD_LIVE_THREAD_ACTIVITY, async |services: &Services| {
558 services.threads.rebuild_thread_activity().await
559 })
560 .await?;
561
562 services.server.check_running()?;
563
564 services
568 .globals
569 .db
570 .bump_database_version(target_version);
571
572 match discovered.cmp(&target_version) {
573 | Ordering::Less =>
574 info!("Database: migrated schema version from {discovered} to {target_version}."),
575 | Ordering::Greater => warn!(
576 "Database: stamped schema version {target_version} over a higher discovered version \
577 {discovered} (forced downgrade or foreign import)."
578 ),
579 | Ordering::Equal => {},
580 }
581
582 warn_forbidden_names(services).await?;
583
584 info!("Loaded RocksDB database with schema version {DATABASE_VERSION}");
585
586 Ok(())
587}
588
589async fn rebuild_thread_summaries(services: &Services) {
594 let (changed, failed) = services.threads.rebuild_thread_summaries().await;
595
596 if failed == 0 {
597 info!(roots = changed, "Rebuilt thread summaries.");
598 } else {
599 warn!(roots = changed, failed, "Rebuilt thread summaries with failed roots.");
600 }
601}
602
603async fn until_finished<F>(services: &Services, marker: &'static str, pass: F) -> Result
608where
609 F: AsyncFnOnce(&Services) -> Result<bool>,
610{
611 if pending(services, marker).await? {
612 let finished = pass(services).await?;
613
614 if finished {
615 services.db["global"].insert(marker, []);
616 }
617 }
618
619 Ok(())
620}
621
622async fn run_once<F>(services: &Services, marker: &'static str, pass: F) -> Result
626where
627 F: AsyncFnOnce(&Services) -> Result,
628{
629 if pending(services, marker).await? {
630 pass(services).await?;
631
632 services.db["global"].insert(marker, []);
633 }
634
635 Ok(())
636}
637
638async fn warn_forbidden_names(services: &Services) -> Result {
646 let progress = &services.server.progress;
647
648 services.server.check_running()?;
649
650 if !services
651 .config
652 .registration
653 .forbidden_usernames
654 .is_empty()
655 {
656 progress.begin("scan_forbidden_usernames");
657
658 forbidden_usernames(services)
659 .ready_try_for_each(|Username { user_id, patterns }| {
660 warn!(%user_id, %patterns, "User matches forbidden username patterns");
661 Ok(())
662 })
663 .await?;
664 }
665
666 services.server.check_running()?;
667
668 if !services
669 .config
670 .rooms
671 .forbidden_alias_names
672 .is_empty()
673 {
674 progress.begin("scan_forbidden_alias_names");
675
676 services
677 .metadata
678 .iter_ids()
679 .map(|room_id| {
680 services
681 .server
682 .check_running()
683 .map(|()| room_id.to_owned())
684 })
685 .try_for_each(async |room_id| scan_room_aliases(services, &room_id).await)
686 .await?;
687 }
688
689 Ok(())
690}
691
692async fn scan_room_aliases(services: &Services, room_id: &RoomId) -> Result {
693 let patterns = &services.config.rooms.forbidden_alias_names;
694
695 services
696 .alias
697 .local_aliases_for_room(room_id)
698 .map(|alias| services.server.check_running().map(|()| alias))
699 .ready_try_for_each(|alias| {
700 if let Some(patterns) = forbidden_patterns(patterns, alias.alias()) {
701 warn!(
702 %room_id,
703 %alias,
704 %patterns,
705 "Room alias matches the following forbidden alias name patterns"
706 );
707 }
708
709 Ok(())
710 })
711 .await
712}
713
714async fn pending(services: &Services, marker: &'static str) -> Result<bool> {
723 services.server.check_running()?;
724
725 let pending = !marker_present(services, marker).await?;
726
727 if pending {
728 services.server.progress.begin(marker);
729 }
730
731 Ok(pending)
732}
733
734#[tracing::instrument(level = "debug", skip_all)]
741pub async fn rerun(services: &Services, check: &str) -> Result<impl Iterator<Item = Stamp>> {
742 let markers = rerun_markers(check)?;
743
744 if services.db.is_read_only() {
745 return Err!("scheduling a repair needs a writable database");
746 }
747
748 let stamps: Stamps = markers
750 .iter()
751 .copied()
752 .stream()
753 .then(|marker| stamp(services, marker))
754 .try_collect()
755 .await?;
756
757 for marker in markers {
758 services.db["global"].remove(marker);
759 }
760
761 Ok(stamps.into_iter())
762}
763
764fn rerun_markers(name: &str) -> Result<&'static [&'static str]> {
765 let check = find_check(name).ok_or_else(|| err!("no integrity check is named {name}"))?;
766
767 let rerun = RERUNS
768 .iter()
769 .find(|(listed, _)| *listed == check.name)
770 .map_or(Refuse(NO_REPAIR), |&(_, rerun)| rerun);
771
772 match rerun {
773 | Clear(markers) => Ok(markers),
774 | Refuse(reason) => Err!("cannot schedule a repair for {name}: {reason}"),
775 }
776}
777
778#[tracing::instrument(level = "debug", skip_all)]
779async fn stamp(services: &Services, marker: &'static str) -> Result<Stamp> {
780 let cleared = marker_present(services, marker).await?;
781
782 Ok(Stamp { marker, cleared })
783}
784
785pub(crate) async fn marker_present(services: &Services, marker: &str) -> Result<bool> {
790 services.db["global"]
791 .get(marker)
792 .await
793 .optional()
794 .inspect_err(|error| warn!(%marker, %error, "Migration marker failed to read"))
795 .map(|stamp| stamp.is_some())
796}
797
798pub(crate) fn local_user_id(localpart: &str, server_name: &ServerName) -> Option<OwnedUserId> {
804 let user_id: UserIdBuf = format_small_string!("@{localpart}:{server_name}");
805
806 UserId::parse(user_id.as_str()).ok()
807}