tuwunel_database/
maps.rs

1//! Column family descriptors used by the homeserver database.
2//!
3//! The catalog names every configured map and supplies its RocksDB tuning
4//! profile. Database startup opens these descriptors as one coherent
5//! collection.
6
7use std::{collections::BTreeMap, sync::Arc};
8
9use rocksdb::DBCompressionType as CompressionType;
10use tuwunel_core::Result;
11
12use crate::{
13	Engine, Map,
14	engine::descriptor::{self, CacheDisp, Descriptor},
15};
16
17/// Indexes opened logical maps by column-family name.
18///
19/// Keys are static names from the descriptor catalog, and values share each
20/// opened [`Map`] through an `Arc`. Dropped or unavailable families are absent.
21pub(super) type Maps = BTreeMap<MapsKey, MapsVal>;
22
23/// Names a column family in the map catalog.
24///
25/// Catalog names have static lifetime because descriptors are process-wide
26/// constants.
27pub(super) type MapsKey = &'static str;
28
29/// Holds a shared logical map opened from a catalog descriptor.
30///
31/// The shared handle keeps the map and its database engine alive for every
32/// service that uses the catalog entry.
33pub(super) type MapsVal = Arc<Map>;
34
35/// Opens every configured map in the built-in catalog.
36///
37/// The returned index contains only descriptors that are live and present in
38/// the opened database. Individual map handles share the supplied engine.
39pub(super) fn open(engine: &Arc<Engine>) -> Result<Maps> { open_list(engine, MAPS) }
40
41/// Opens maps from an explicit descriptor list.
42///
43/// Dropped descriptors and column families missing from the engine are skipped.
44/// Any failure to open a retained map aborts construction of the index.
45#[tracing::instrument(name = "maps", level = "debug", skip_all)]
46pub(super) fn open_list(engine: &Arc<Engine>, maps: &[Descriptor]) -> Result<Maps> {
47	maps.iter()
48		.filter(|desc| !desc.dropped)
49		.filter(|desc| engine.has_cf(desc.name))
50		.map(|desc| Ok((desc.name, Map::open(engine, desc.name)?)))
51		.collect()
52}
53
54#[cfg(test)]
55pub(crate) fn descriptor(name: &str) -> &'static Descriptor {
56	MAPS.iter()
57		.find(|desc| desc.name.eq(name))
58		.expect("descriptor present")
59}
60
61/// Defines the built-in column-family catalog and its RocksDB tuning.
62///
63/// Each descriptor names one logical map and inherits a workload preset with
64/// optional per-map overrides. Dropped descriptors remain as migration
65/// tombstones but are not opened for use.
66pub(super) static MAPS: &[Descriptor] = &[
67	Descriptor {
68		name: "alias_roomid",
69		..descriptor::RANDOM_SMALL
70	},
71	Descriptor {
72		name: "alias_userid",
73		..descriptor::RANDOM_SMALL
74	},
75	Descriptor {
76		name: "aliasid_alias",
77		..descriptor::RANDOM_SMALL
78	},
79	Descriptor {
80		name: "authchainkey_authchain",
81		cache_disp: CacheDisp::Unique, // fork-resolve bursts would evict the shared pool
82		compression: CompressionType::None,
83		cache_shards: 32,
84		index_size: 1024,
85		block_size: 4096,
86		key_size_hint: Some(8),
87		val_size_hint: Some(256),
88		limit_size: 1024 * 1024 * 1024 * 4,
89		..descriptor::RANDOM_CACHE
90	},
91	Descriptor {
92		name: "backupid_algorithm",
93		..descriptor::RANDOM_SMALL
94	},
95	Descriptor {
96		name: "backupid_etag",
97		..descriptor::RANDOM_SMALL
98	},
99	Descriptor {
100		name: "backupkeyid_backup",
101		..descriptor::RANDOM_SMALL
102	},
103	Descriptor {
104		name: "bannedroomids",
105		..descriptor::RANDOM_SMALL
106	},
107	Descriptor {
108		name: "disabledroomids",
109		..descriptor::RANDOM_SMALL
110	},
111	Descriptor {
112		name: "email_userid",
113		..descriptor::RANDOM_SMALL
114	},
115	Descriptor {
116		name: "eventid_backoff",
117		cache_disp: CacheDisp::Unique, // hot on every redelivery
118		key_size_hint: Some(64),
119		val_size_hint: Some(16),
120		ttl: 60 * 60 * 24 * 3, // dead after the max fetch-backoff window (24h)
121		limit_size: 1024 * 1024 * 256,
122		..descriptor::RANDOM_SMALL_CACHE
123	},
124	Descriptor {
125		name: "eventid_originalpdu",
126		key_size_hint: Some(48),
127		val_size_hint: Some(1520),
128		block_size: 2048,
129		index_size: 512,
130		..descriptor::RANDOM
131	},
132	Descriptor {
133		name: "eventid_outlierpdu",
134		cache_disp: CacheDisp::SharedWith("pduid_pdu"),
135		key_size_hint: Some(48),
136		val_size_hint: Some(1488),
137		block_size: 1024,
138		index_size: 512,
139		..descriptor::RANDOM
140	},
141	Descriptor {
142		name: "eventid_pduid",
143		cache_disp: CacheDisp::Unique,
144		key_size_hint: Some(48),
145		val_size_hint: Some(16),
146		block_size: 512,
147		index_size: 512,
148		..descriptor::RANDOM
149	},
150	Descriptor {
151		name: "eventid_policysigstate",
152		ttl: 60 * 60 * 24 * 7, // backoff/refusal hint; re-ask fails open
153		..descriptor::RANDOM_SMALL_CACHE
154	},
155	Descriptor {
156		name: "eventid_resolvedstate",
157		ttl: 60 * 60 * 24 * 7, // refetch-avoidance only; safe to evict
158		..descriptor::RANDOM_SMALL_CACHE
159	},
160	Descriptor {
161		name: "eventid_shorteventid",
162		cache_disp: CacheDisp::Unique,
163		key_size_hint: Some(48),
164		val_size_hint: Some(8),
165		block_size: 512,
166		index_size: 512,
167		..descriptor::RANDOM
168	},
169	Descriptor {
170		name: "global",
171		..descriptor::RANDOM_SMALL
172	},
173	Descriptor {
174		name: "id_appserviceregistrations",
175		..descriptor::RANDOM_SMALL
176	},
177	Descriptor {
178		name: "keychangeid_devicechange",
179		..descriptor::SEQUENTIAL_SMALL
180	},
181	Descriptor {
182		name: "keychangeid_userid",
183		..descriptor::RANDOM
184	},
185	Descriptor {
186		name: "keyid_key",
187		..descriptor::RANDOM_SMALL
188	},
189	Descriptor {
190		name: "lazyloadedids",
191		..descriptor::RANDOM_SMALL
192	},
193	Descriptor {
194		name: "logintoken_expiresatuserid",
195		..descriptor::RANDOM_SMALL
196	},
197	Descriptor {
198		name: "mediaid_file",
199		..descriptor::RANDOM_SMALL
200	},
201	Descriptor {
202		name: "mediaid_lazy",
203		ttl: 60 * 60 * 24 * 30, // raised by url_preview_cache_ttl; must outlive url_preview
204		limit_size: 1024 * 1024 * 512,
205		..descriptor::RANDOM_SMALL_CACHE
206	},
207	Descriptor {
208		name: "mediaid_lazycontent",
209		cache_disp: CacheDisp::Unique, // 256 KiB rows would evict the shared pool
210		key_size_hint: Some(64),
211		val_size_hint: Some(1024 * 256),
212		file_size: 1024 * 1024 * 64,
213		write_size: 1024 * 1024 * 64,
214		compression: CompressionType::None, // media bytes are pre-compressed
215		ttl: 60 * 60 * 24 * 14,             // staging only: rows die at promotion or here
216		..descriptor::RANDOM_CACHE
217	},
218	Descriptor {
219		name: "mediaid_pending",
220		ttl: 60 * 60 * 24 * 7,
221		..descriptor::RANDOM_SMALL_CACHE
222	},
223	Descriptor {
224		name: "mediaid_user",
225		..descriptor::RANDOM_SMALL
226	},
227	Descriptor {
228		name: "oauthid_session",
229		..descriptor::RANDOM_SMALL
230	},
231	Descriptor {
232		name: "oauthuniqid_oauthid",
233		..descriptor::RANDOM_SMALL
234	},
235	Descriptor {
236		name: "oidc_signingkey",
237		..descriptor::RANDOM_SMALL
238	},
239	Descriptor {
240		name: "oidcclientid_registration",
241		..descriptor::RANDOM_SMALL_CACHE
242	},
243	Descriptor {
244		name: "oidccode_authsession",
245		..descriptor::RANDOM_SMALL
246	},
247	Descriptor {
248		name: "oidcdevice_userdeviceid",
249		..descriptor::RANDOM_SMALL
250	},
251	Descriptor {
252		name: "oidcdevicecode_devicegrant",
253		ttl: 60 * 60 * 24,
254		..descriptor::RANDOM_SMALL_CACHE
255	},
256	Descriptor {
257		name: "oidcusercode_devicecode",
258		ttl: 60 * 60 * 24,
259		..descriptor::RANDOM_SMALL_CACHE
260	},
261	Descriptor {
262		name: "oidccskeybypass_userid",
263		..descriptor::RANDOM_SMALL
264	},
265	Descriptor {
266		name: "oidcreqid_authrequest",
267		..descriptor::RANDOM_SMALL
268	},
269	Descriptor {
270		name: "onetimekeyid_onetimekeys",
271		..descriptor::DROPPED
272	},
273	Descriptor {
274		name: "onetimekeyid4225_otk",
275		..descriptor::RANDOM_SMALL
276	},
277	Descriptor {
278		name: "openidtoken_expiresatuserid",
279		..descriptor::RANDOM_SMALL
280	},
281	Descriptor {
282		name: "pduid_pdu",
283		cache_disp: CacheDisp::SharedWith("eventid_outlierpdu"),
284		key_size_hint: Some(16),
285		val_size_hint: Some(1520),
286		block_size: 2048,
287		index_size: 512,
288		..descriptor::SEQUENTIAL
289	},
290	Descriptor {
291		name: "profilechangeid_userid",
292		..descriptor::RANDOM
293	},
294	Descriptor {
295		name: "publicroomids",
296		..descriptor::RANDOM_SMALL
297	},
298	Descriptor {
299		name: "pushkey_deviceid",
300		..descriptor::RANDOM_SMALL
301	},
302	Descriptor {
303		name: "presenceid_presence",
304		..descriptor::SEQUENTIAL_SMALL
305	},
306	Descriptor {
307		name: "readreceiptid_readreceipt",
308		..descriptor::RANDOM
309	},
310	Descriptor {
311		name: "referencedevents",
312		..descriptor::RANDOM
313	},
314	Descriptor {
315		name: "relatesto_typed",
316		key_size_hint: Some(40),
317		val_size_hint: Some(8),
318		..descriptor::RANDOM_SMALL
319	},
320	Descriptor {
321		name: "registrationtoken_info",
322		..descriptor::RANDOM_SMALL
323	},
324	Descriptor {
325		name: "roomid_knockedcount",
326		..descriptor::RANDOM_SMALL
327	},
328	Descriptor {
329		name: "roomid_invitedcount",
330		..descriptor::RANDOM_SMALL
331	},
332	Descriptor {
333		name: "roomid_inviteviaservers",
334		..descriptor::RANDOM_SMALL
335	},
336	Descriptor {
337		name: "roomid_joinedcount",
338		..descriptor::RANDOM_SMALL
339	},
340	Descriptor {
341		name: "roomid_maxremotepowerlevel",
342		..descriptor::RANDOM_SMALL
343	},
344	Descriptor {
345		name: "roomid_pduleaves",
346		..descriptor::RANDOM_SMALL
347	},
348	Descriptor {
349		name: "roomid_shortroomid",
350		val_size_hint: Some(8),
351		..descriptor::RANDOM_SMALL
352	},
353	Descriptor {
354		name: "roomid_shortstatehash",
355		val_size_hint: Some(8),
356		..descriptor::RANDOM_SMALL
357	},
358	Descriptor {
359		name: "roomid_spacehierarchy",
360		limit_size: 1024 * 1024 * 64,
361		ttl: 60 * 60 * 24 * 7, // above spacehierarchy_cache_ttl_max (18h default)
362		..descriptor::RANDOM_SMALL_CACHE
363	},
364	Descriptor {
365		name: "roomid_ts_pducount",
366		..descriptor::DROPPED
367	},
368	Descriptor {
369		name: "roomid_tscount_pducount",
370		val_size_hint: Some(8),
371		..descriptor::RANDOM
372	},
373	Descriptor {
374		name: "roomserverids",
375		..descriptor::RANDOM_SMALL
376	},
377	Descriptor {
378		name: "roomsynctoken_shortstatehash",
379		..descriptor::DROPPED
380	},
381	Descriptor {
382		name: "roomtseventid_prevwalk",
383		cache_disp: CacheDisp::Unique, // admin sweeps would evict the shared pool
384		cache_shards: 8,               // 512 KiB each, RocksDB's default minimum shard
385		block_size: 4096,              // scanned, never point-read
386		ttl: 60 * 60 * 24 * 3,         // the "up to three days" the view's docs name
387		..descriptor::RANDOM_SMALL_CACHE
388	},
389	Descriptor {
390		name: "roomuserdataid_accountdata",
391		..descriptor::RANDOM_SMALL
392	},
393	Descriptor {
394		name: "roomuserid_invitecount",
395		val_size_hint: Some(8),
396		..descriptor::RANDOM_SMALL
397	},
398	Descriptor {
399		name: "roomuserid_joined",
400		..descriptor::RANDOM_SMALL
401	},
402	Descriptor {
403		name: "roomuserid_lastprivatereadupdate",
404		..descriptor::RANDOM_SMALL
405	},
406	Descriptor {
407		name: "roomuserid_lastnotificationread",
408		..descriptor::RANDOM_SMALL
409	},
410	Descriptor {
411		name: "roomuserid_leftcount",
412		val_size_hint: Some(8),
413		..descriptor::RANDOM
414	},
415	Descriptor {
416		name: "roomuserid_knockedcount",
417		val_size_hint: Some(8),
418		..descriptor::RANDOM_SMALL
419	},
420	Descriptor {
421		name: "roomuserid_privateread",
422		..descriptor::RANDOM_SMALL
423	},
424	Descriptor {
425		name: "roomuserid_privatereadsync",
426		..descriptor::RANDOM_SMALL
427	},
428	Descriptor {
429		name: "roomuseroncejoinedids",
430		..descriptor::RANDOM
431	},
432	Descriptor {
433		name: "roomusertype_roomuserdataid",
434		..descriptor::RANDOM_SMALL
435	},
436	Descriptor {
437		name: "senderkey_pusher",
438		..descriptor::RANDOM_SMALL
439	},
440	Descriptor {
441		name: "server_signingkeys",
442		..descriptor::RANDOM
443	},
444	Descriptor {
445		name: "servercurrentevent_data",
446		..descriptor::RANDOM_SMALL
447	},
448	Descriptor {
449		name: "servername_destination",
450		cache_disp: CacheDisp::SharedWith("servername_override"), // one resolve writes both
451		key_size_hint: Some(48),
452		val_size_hint: Some(128),
453		ttl: 60 * 60 * 24 * 7, // dead after CachedDest::default_expire (<=36h)
454		limit_size: 1024 * 1024 * 96,
455		..descriptor::RANDOM_SMALL_CACHE
456	},
457	Descriptor {
458		name: "servername_educount",
459		..descriptor::RANDOM_SMALL
460	},
461	Descriptor {
462		name: "servername_override",
463		cache_disp: CacheDisp::SharedWith("servername_destination"),
464		key_size_hint: Some(48),
465		val_size_hint: Some(128),
466		ttl: 60 * 60 * 24 * 7, // dead after CachedOverride::default_expire (<=12h)
467		limit_size: 1024 * 1024 * 96,
468		..descriptor::RANDOM_SMALL_CACHE
469	},
470	Descriptor {
471		name: "servername_status",
472		cache_disp: CacheDisp::Unique, // read on every outbound attempt
473		key_size_hint: Some(48),
474		val_size_hint: Some(16),
475		ttl: 60 * 60 * 24 * 3, // dead after peer MAX_BACKOFF (24h)
476		limit_size: 1024 * 1024 * 128,
477		..descriptor::RANDOM_SMALL_CACHE
478	},
479	Descriptor {
480		name: "servernameevent_data",
481		cache_disp: CacheDisp::Unique,
482		val_size_hint: Some(128),
483		..descriptor::RANDOM
484	},
485	Descriptor {
486		name: "serverroomids",
487		..descriptor::RANDOM_SMALL
488	},
489	Descriptor {
490		name: "servershortroomid_park",
491		..descriptor::RANDOM_SMALL
492	},
493	Descriptor {
494		name: "shorteventid_authchain",
495		..descriptor::DROPPED
496	},
497	Descriptor {
498		name: "shorteventid_eventid",
499		cache_disp: CacheDisp::Unique,
500		key_size_hint: Some(8),
501		val_size_hint: Some(48),
502		..descriptor::SEQUENTIAL_SMALL
503	},
504	Descriptor {
505		name: "shorteventid_shortstatehash",
506		key_size_hint: Some(8),
507		val_size_hint: Some(8),
508		block_size: 512,
509		index_size: 512,
510		..descriptor::SEQUENTIAL
511	},
512	Descriptor {
513		name: "shortstatehash_statediff",
514		key_size_hint: Some(8),
515		..descriptor::SEQUENTIAL_SMALL
516	},
517	Descriptor {
518		name: "shortstatekey_statekey",
519		cache_disp: CacheDisp::Unique,
520		key_size_hint: Some(8),
521		val_size_hint: Some(1024),
522		..descriptor::RANDOM_SMALL
523	},
524	Descriptor {
525		name: "softfailedeventids",
526		key_size_hint: Some(48),
527		..descriptor::RANDOM_SMALL
528	},
529	Descriptor {
530		name: "statehash_shortstatehash",
531		val_size_hint: Some(8),
532		..descriptor::RANDOM
533	},
534	Descriptor {
535		name: "statekey_shortstatekey",
536		cache_disp: CacheDisp::Unique,
537		key_size_hint: Some(1024),
538		val_size_hint: Some(8),
539		..descriptor::RANDOM
540	},
541	Descriptor {
542		name: "threadactivityid_rootid",
543		key_size_hint: Some(16),
544		..descriptor::SEQUENTIAL_SMALL
545	},
546	Descriptor {
547		name: "threadid_userids",
548		..descriptor::SEQUENTIAL_SMALL
549	},
550	Descriptor {
551		name: "threadrootid_latestcount",
552		key_size_hint: Some(16),
553		val_size_hint: Some(8),
554		..descriptor::RANDOM_SMALL
555	},
556	Descriptor {
557		name: "threepidsid_pending",
558		ttl: 60 * 60 * 24, // pending validation session; minutes to complete
559		..descriptor::RANDOM_SMALL_CACHE
560	},
561	Descriptor {
562		name: "timeredacted_eventid",
563		key_size_hint: Some(64),
564		..descriptor::SEQUENTIAL_SMALL
565	},
566	Descriptor {
567		name: "todeviceid_events",
568		..descriptor::RANDOM
569	},
570	Descriptor {
571		name: "tofrom_relation",
572		key_size_hint: Some(8),
573		val_size_hint: Some(8),
574		..descriptor::RANDOM_SMALL
575	},
576	Descriptor {
577		name: "spentrefresh_userdeviceid",
578		..descriptor::RANDOM_SMALL
579	},
580	Descriptor {
581		name: "token_userdeviceid",
582		..descriptor::RANDOM_SMALL
583	},
584	Descriptor {
585		name: "tokenids",
586		block_size: 512,
587		..descriptor::RANDOM
588	},
589	Descriptor {
590		name: "url_preview",
591		ttl: 60 * 60 * 24 * 7, // floor; url_preview_cache_ttl raises it when higher
592		limit_size: 1024 * 1024 * 128,
593		..descriptor::RANDOM_SMALL_CACHE
594	},
595	Descriptor {
596		name: "url_previews",
597		..descriptor::DROPPED
598	},
599	Descriptor {
600		name: "userdeviceid_metadata",
601		..descriptor::RANDOM_SMALL
602	},
603	Descriptor {
604		name: "userdeviceconnid_conn",
605		block_size: 1024 * 16,
606		..descriptor::RANDOM_SMALL
607	},
608	Descriptor {
609		name: "userdeviceid_refresh",
610		..descriptor::RANDOM_SMALL
611	},
612	Descriptor {
613		name: "userdeviceid_spentrefresh",
614		..descriptor::RANDOM_SMALL
615	},
616	Descriptor {
617		name: "userdeviceid_token",
618		..descriptor::RANDOM_SMALL
619	},
620	Descriptor {
621		name: "userdeviceidtoken_index",
622		..descriptor::RANDOM_SMALL
623	},
624	Descriptor {
625		name: "userdeviceidalgorithm_fallback",
626		..descriptor::RANDOM_SMALL
627	},
628	Descriptor {
629		name: "userdevicesessionid_threepid",
630		ttl: 60 * 60 * 24, // interactive-auth session; minutes to complete
631		..descriptor::RANDOM_SMALL_CACHE
632	},
633	Descriptor {
634		name: "userdevicesessionid_uiaainfo",
635		ttl: 60 * 60 * 24, // interactive-auth session; minutes to complete
636		..descriptor::RANDOM_SMALL_CACHE
637	},
638	Descriptor {
639		name: "userdevicetxnid_response",
640		..descriptor::RANDOM_SMALL
641	},
642	Descriptor {
643		name: "userfilterid_filter",
644		..descriptor::RANDOM_SMALL
645	},
646	Descriptor {
647		name: "userid_avatarurl",
648		..descriptor::RANDOM_SMALL
649	},
650	Descriptor {
651		name: "userid_blurhash",
652		..descriptor::RANDOM_SMALL
653	},
654	Descriptor {
655		name: "userid_dehydrateddevice",
656		..descriptor::RANDOM_SMALL
657	},
658	Descriptor {
659		name: "userid_devicelistversion",
660		..descriptor::RANDOM_SMALL
661	},
662	Descriptor {
663		name: "userid_displayname",
664		..descriptor::RANDOM_SMALL
665	},
666	Descriptor {
667		name: "userid_email",
668		..descriptor::RANDOM_SMALL
669	},
670	Descriptor {
671		name: "userid_erased",
672		..descriptor::RANDOM_SMALL
673	},
674	Descriptor {
675		name: "userid_lastonetimekeyupdate",
676		..descriptor::RANDOM_SMALL
677	},
678	Descriptor {
679		name: "userid_locked",
680		..descriptor::RANDOM_SMALL
681	},
682	Descriptor {
683		name: "userid_masterkeyid",
684		..descriptor::RANDOM_SMALL
685	},
686	Descriptor {
687		name: "userid_oauthid",
688		..descriptor::RANDOM_SMALL
689	},
690	Descriptor {
691		name: "userid_origin",
692		..descriptor::RANDOM
693	},
694	Descriptor {
695		name: "userid_password",
696		..descriptor::RANDOM
697	},
698	Descriptor {
699		name: "userid_presenceid",
700		..descriptor::RANDOM_SMALL
701	},
702	Descriptor {
703		name: "userid_selfsigningkeyid",
704		..descriptor::RANDOM_SMALL
705	},
706	Descriptor {
707		name: "userid_suspended",
708		..descriptor::RANDOM_SMALL
709	},
710	Descriptor {
711		name: "userid_usersigningkeyid",
712		..descriptor::RANDOM_SMALL
713	},
714	Descriptor {
715		name: "useridcount_notification",
716		limit_size: 1024 * 1024 * 256,
717		..descriptor::RANDOM_SMALL_CACHE
718	},
719	Descriptor {
720		name: "useridprofilekey_value",
721		..descriptor::RANDOM_SMALL
722	},
723	Descriptor {
724		name: "userroomid_highlightcount",
725		..descriptor::RANDOM
726	},
727	Descriptor {
728		name: "userroomid_invitestate",
729		..descriptor::RANDOM_SMALL
730	},
731	Descriptor {
732		name: "userroomid_joined",
733		..descriptor::RANDOM
734	},
735	Descriptor {
736		name: "userroomid_leftstate",
737		..descriptor::RANDOM
738	},
739	Descriptor {
740		name: "userroomid_knockedstate",
741		..descriptor::RANDOM_SMALL
742	},
743	Descriptor {
744		name: "userroomid_notificationcount",
745		..descriptor::RANDOM
746	},
747];
tuwunel_database/maps.rs

tuwunel_database/
maps.rs
