Skip to main content

tuwunel_service/rooms/short/
mod.rs

1//! Compact numeric identifiers for room and event data.
2//!
3//! The service maps event IDs, state keys, state hashes, and room IDs into a shared global sequence
4//! space. Event IDs and state keys have paired forward and reverse mappings. State hashes have a
5//! partial forward index and no reverse mapping; room IDs have only a forward mapping.
6
7use std::{borrow::Borrow, sync::Arc};
8
9use futures::{FutureExt, Stream, StreamExt, pin_mut};
10use ruma::{EventId, OwnedEventId, OwnedRoomId, RoomId, events::StateEventType};
11use serde::Deserialize;
12use tuwunel_core::{
13	Err, Result, err, implement, utils,
14	utils::{
15		IterStream, MutexMap,
16		hash::sha256::Digest,
17		stream::{ReadyExt, WidebandExt},
18	},
19};
20use tuwunel_database::{Deserialized, Get, Map, Qry, Txn};
21use tuwunel_matrix::StateKey;
22/// Numeric identifier types shared with Matrix data structures.
23///
24/// Aliases name the compact identifier contexts used by event, state-key, and room mappings.
25pub use tuwunel_matrix::{ShortEventId, ShortId, ShortRoomId, ShortStateKey};
26
27/// Allocates and resolves compact identifiers used by room storage.
28///
29/// Per-identity locks serialize first allocation, and database transactions keep new forward and
30/// reverse mappings consistent where both are stored.
31pub struct Service {
32	db: Data,
33	creating: Creating,
34	services: Arc<crate::services::OnceServices>,
35}
36
37struct Data {
38	eventid_shorteventid: Arc<Map>,
39	shorteventid_eventid: Arc<Map>,
40	statekey_shortstatekey: Arc<Map>,
41	shortstatekey_statekey: Arc<Map>,
42	roomid_shortroomid: Arc<Map>,
43	statehash_shortstatehash: Arc<Map>,
44}
45
46/// Serializes concurrent allocations so one identity maps to one short id.
47///
48/// A guard is held across both the re-read that detects a competing
49/// allocation and the writes that publish this one. Entries exist only while
50/// a caller holds or awaits one, so an uncontended allocation leaves nothing
51/// behind.
52#[derive(Default)]
53struct Creating {
54	shorteventid: MutexMap<OwnedEventId, ()>,
55	shortstatekey: MutexMap<(StateEventType, StateKey), ()>,
56	shortstatehash: MutexMap<Digest, ()>,
57	shortroomid: MutexMap<OwnedRoomId, ()>,
58}
59
60/// Compact identifier assigned to a complete room state hash.
61///
62/// State-hash IDs share the global numeric sequence with the other short identifier families.
63pub type ShortStateHash = ShortId;
64
65impl crate::Service for Service {
66	fn build(args: &crate::Args<'_>) -> Result<Arc<Self>> {
67		Ok(Arc::new(Self {
68			db: Data {
69				eventid_shorteventid: args.db["eventid_shorteventid"].clone(),
70				shorteventid_eventid: args.db["shorteventid_eventid"].clone(),
71				statekey_shortstatekey: args.db["statekey_shortstatekey"].clone(),
72				shortstatekey_statekey: args.db["shortstatekey_statekey"].clone(),
73				roomid_shortroomid: args.db["roomid_shortroomid"].clone(),
74				statehash_shortstatehash: args.db["statehash_shortstatehash"].clone(),
75			},
76			creating: Creating::default(),
77			services: args.services.clone(),
78		}))
79	}
80
81	fn name(&self) -> &str { crate::service::make_name(std::module_path!()) }
82}
83
84/// Resolves an event ID to its compact identifier, allocating after an unsuccessful lookup.
85///
86/// New allocations publish the forward and reverse mappings in one transaction. Concurrent callers
87/// for the same event converge on one identifier.
88///
89/// # Panics
90///
91/// Panics when identifier dispatch or the paired mapping transaction fails.
92#[implement(Service)]
93pub async fn get_or_create_shorteventid(&self, event_id: &EventId) -> ShortEventId {
94	if let Ok(shorteventid) = self.get_shorteventid(event_id).await {
95		return shorteventid;
96	}
97
98	self.create_shorteventid(event_id).await
99}
100
101/// Resolves each event ID to its short ID, allocating after unsuccessful lookups.
102///
103/// Allocation runs ahead of consumer demand, so a caller that stops early
104/// still allocates for the events already buffered. Today's callers drain the
105/// stream in full.
106///
107/// # Panics
108///
109/// Panics when an existing mapping is malformed, identifier dispatch fails, or a paired mapping
110/// transaction fails.
111#[implement(Service)]
112pub fn multi_get_or_create_shorteventid<'a, I>(
113	&'a self,
114	event_ids: I,
115) -> impl Stream<Item = ShortEventId> + Send + '_
116where
117	I: Iterator<Item = &'a EventId> + Clone + Send + 'a,
118{
119	event_ids
120		.clone()
121		.stream()
122		.get(&self.db.eventid_shorteventid)
123		.zip(event_ids.into_iter().stream())
124		.wide_then(async |(result, event_id)| match result {
125			| Ok(ref short) => utils::u64_from_u8(short),
126			| Err(_) => self.create_shorteventid(event_id).await,
127		})
128}
129
130#[implement(Service)]
131async fn create_shorteventid(&self, event_id: &EventId) -> ShortEventId {
132	let _lock = self.creating.shorteventid.lock(event_id).await;
133
134	if let Ok(shorteventid) = self.get_shorteventid(event_id).await {
135		return shorteventid;
136	}
137
138	let short = self.services.globals.next_count();
139	let mut txn = self.services.db.txn();
140
141	txn.insert_raw(&self.db.shorteventid_eventid, (*short).to_be_bytes(), event_id);
142	txn.insert_raw(&self.db.eventid_shorteventid, event_id, (*short).to_be_bytes());
143	txn.execute();
144
145	*short
146}
147
148/// Resolves an event ID to its existing compact identifier.
149///
150/// Missing or malformed mappings are returned as database errors.
151#[implement(Service)]
152pub async fn get_shorteventid(&self, event_id: &EventId) -> Result<ShortEventId> {
153	self.db
154		.eventid_shorteventid
155		.get(event_id)
156		.await
157		.deserialized()
158}
159
160/// Resolves a state event type and key, allocating an identifier after an unsuccessful lookup.
161///
162/// New allocations publish the forward and reverse mappings in one transaction. Concurrent callers
163/// for the same pair converge on one identifier.
164///
165/// # Panics
166///
167/// Panics when identifier dispatch or the paired mapping transaction fails.
168#[implement(Service)]
169pub async fn get_or_create_shortstatekey(
170	&self,
171	event_type: &StateEventType,
172	state_key: &str,
173) -> ShortStateKey {
174	if let Ok(shortstatekey) = self
175		.get_shortstatekey(event_type, state_key)
176		.await
177	{
178		return shortstatekey;
179	}
180
181	self.create_shortstatekey(event_type, state_key)
182		.await
183}
184
185#[implement(Service)]
186async fn create_shortstatekey(
187	&self,
188	event_type: &StateEventType,
189	state_key: &str,
190) -> ShortStateKey {
191	let owned_key = (event_type.clone(), StateKey::from_str(state_key));
192	let _lock = self.creating.shortstatekey.lock(&owned_key).await;
193
194	if let Ok(shortstatekey) = self
195		.get_shortstatekey(event_type, state_key)
196		.await
197	{
198		return shortstatekey;
199	}
200
201	let key = (event_type, state_key);
202	let shortstatekey = self.services.globals.next_count();
203	let mut txn = self.services.db.txn();
204
205	txn.put(&self.db.shortstatekey_statekey, *shortstatekey, key);
206	txn.put(&self.db.statekey_shortstatekey, key, *shortstatekey);
207	txn.execute();
208
209	*shortstatekey
210}
211
212/// Resolves a state event type and key to its existing compact identifier.
213///
214/// The event type and state key together form the forward lookup key. Missing or malformed mappings
215/// are returned as database errors.
216#[implement(Service)]
217pub async fn get_shortstatekey(
218	&self,
219	event_type: &StateEventType,
220	state_key: &str,
221) -> Result<ShortStateKey> {
222	let key = (event_type, state_key);
223	self.db
224		.statekey_shortstatekey
225		.qry(&key)
226		.await
227		.deserialized()
228}
229
230/// Resolves a compact event identifier through the reverse mapping.
231///
232/// The caller selects a deserializable event-ID representation whose owned form can be borrowed as
233/// [`EventId`]. Missing or malformed rows are reported with the compact identifier.
234#[implement(Service)]
235pub async fn get_eventid_from_short<Id>(&self, shorteventid: ShortEventId) -> Result<Id>
236where
237	Id: for<'de> Deserialize<'de> + Send + Sized + ToOwned,
238	<Id as ToOwned>::Owned: Borrow<EventId>,
239{
240	const BUFSIZE: usize = size_of::<ShortEventId>();
241
242	self.db
243		.shorteventid_eventid
244		.aqry::<BUFSIZE, _>(&shorteventid)
245		.await
246		.deserialized()
247		.map_err(|e| err!(Database("Failed to find EventId from short {shorteventid:?}: {e:?}")))
248}
249
250/// Resolves a stream of compact event identifiers through the reverse mapping.
251///
252/// Input order is preserved. Each output carries its own lookup or deserialization result.
253#[implement(Service)]
254pub fn multi_get_eventid_from_short<'a, Id, S>(
255	&'a self,
256	shorteventid: S,
257) -> impl Stream<Item = Result<Id>> + Send + 'a
258where
259	S: Stream<Item = ShortEventId> + Send + 'a,
260	Id: for<'de> Deserialize<'de> + Send + Sized + ToOwned + 'a,
261	<Id as ToOwned>::Owned: Borrow<EventId>,
262{
263	shorteventid
264		.qry(&self.db.shorteventid_eventid)
265		.map(Deserialized::deserialized)
266}
267
268/// Resolves a compact state-key identifier to its event type and state key.
269///
270/// Missing or malformed reverse mappings are returned as database errors that identify the compact
271/// value.
272#[implement(Service)]
273pub async fn get_statekey_from_short(
274	&self,
275	shortstatekey: ShortStateKey,
276) -> Result<(StateEventType, StateKey)> {
277	const BUFSIZE: usize = size_of::<ShortStateKey>();
278
279	self.db
280		.shortstatekey_statekey
281		.aqry::<BUFSIZE, _>(&shortstatekey)
282		.await
283		.deserialized()
284		.map_err(|e| {
285			err!(Database(
286				"Failed to find (StateEventType, state_key) from short {shortstatekey:?}: {e:?}"
287			))
288		})
289}
290
291/// Resolves a stream of compact state-key identifiers through the reverse mapping.
292///
293/// Input order is preserved. Each output carries its own lookup or deserialization result.
294#[implement(Service)]
295pub fn multi_get_statekey_from_short<'a, S>(
296	&'a self,
297	shortstatekey: S,
298) -> impl Stream<Item = Result<(StateEventType, StateKey)>> + Send + 'a
299where
300	S: Stream<Item = ShortStateKey> + Send + 'a,
301{
302	shortstatekey
303		.qry(&self.db.shortstatekey_statekey)
304		.map(Deserialized::deserialized)
305}
306
307/// Resolves a state hash, allocating a compact identifier after an unsuccessful lookup.
308///
309/// The returned boolean is `true` when the mapping already existed, in which case the callback is
310/// skipped. For a new identifier, `write_statediff` stages its associated state diff in the same
311/// transaction as the mapping.
312///
313/// # Errors
314///
315/// Returns an error when `write_statediff` rejects the new state diff. Neither row is committed.
316///
317/// # Panics
318///
319/// Panics when identifier dispatch or the mapping and state-diff transaction fails.
320#[implement(Service)]
321pub async fn get_or_create_shortstatehash<F>(
322	&self,
323	state_hash: &Digest,
324	write_statediff: F,
325) -> Result<(ShortStateHash, bool)>
326where
327	F: FnOnce(&mut Txn, ShortStateHash) -> Result,
328{
329	if let Ok(shortstatehash) = self.get_shortstatehash(state_hash).await {
330		return Ok((shortstatehash, true));
331	}
332
333	self.create_shortstatehash(state_hash, write_statediff)
334		.await
335}
336
337#[implement(Service)]
338async fn create_shortstatehash<F>(
339	&self,
340	state_hash: &Digest,
341	write_statediff: F,
342) -> Result<(ShortStateHash, bool)>
343where
344	F: FnOnce(&mut Txn, ShortStateHash) -> Result,
345{
346	let _lock = self
347		.creating
348		.shortstatehash
349		.lock(state_hash)
350		.await;
351
352	if let Ok(shortstatehash) = self.get_shortstatehash(state_hash).await {
353		return Ok((shortstatehash, true));
354	}
355
356	let shortstatehash = self.services.globals.next_count();
357	let mut txn = self.services.db.txn();
358
359	txn.insert_raw(
360		&self.db.statehash_shortstatehash,
361		state_hash,
362		(*shortstatehash).to_be_bytes(),
363	);
364	write_statediff(&mut txn, *shortstatehash)?;
365	txn.execute();
366
367	Ok((*shortstatehash, false))
368}
369
370/// Resolves a state hash to its existing compact identifier.
371///
372/// Missing or malformed mappings are returned as database errors.
373#[implement(Service)]
374pub async fn get_shortstatehash(&self, state_hash: &Digest) -> Result<ShortStateHash> {
375	self.db
376		.statehash_shortstatehash
377		.get(state_hash)
378		.await
379		.deserialized()
380}
381
382/// Resolves a room ID to its existing compact identifier.
383///
384/// Missing or malformed mappings are returned as database errors.
385#[implement(Service)]
386pub async fn get_shortroomid(&self, room_id: &RoomId) -> Result<ShortRoomId> {
387	self.db
388		.roomid_shortroomid
389		.get(room_id)
390		.await
391		.deserialized()
392}
393
394/// Resolves a compact room identifier by scanning the forward mapping.
395///
396/// Room identifiers have no dedicated reverse map, so the first matching value is returned.
397/// Unreadable rows are skipped during the scan.
398#[implement(Service)]
399pub async fn get_roomid_from_short(&self, shortroomid_: ShortRoomId) -> Result<OwnedRoomId> {
400	let stream = self.iter_shortroomids();
401
402	pin_mut!(stream);
403	stream
404		.ready_find(|&(_, shortroomid)| shortroomid == shortroomid_)
405		.map(|found| found.map(|(room_id, _)| room_id.to_owned()))
406		.await
407		.ok_or_else(|| err!(Database("Failed to find RoomId from {shortroomid_:?}")))
408}
409
410/// Streams every room ID with its compact identifier.
411///
412/// This scan is the only way back from a compact room identifier, so resolve
413/// several in one pass. Each borrowed ID is valid only until the next poll, and
414/// unreadable rows are skipped.
415#[implement(Service)]
416pub fn iter_shortroomids(&self) -> impl Stream<Item = (&RoomId, ShortRoomId)> + Send + '_ {
417	self.db
418		.roomid_shortroomid
419		.stream()
420		.ready_filter_map(Result::ok)
421}
422
423/// Resolves a room ID to its compact identifier, allocating after an unsuccessful lookup.
424///
425/// Room mappings are stored only in the forward direction. Concurrent callers for the same room
426/// converge on one identifier.
427///
428/// # Panics
429///
430/// Panics when identifier dispatch or storing the forward mapping fails.
431#[implement(Service)]
432pub async fn get_or_create_shortroomid(&self, room_id: &RoomId) -> ShortRoomId {
433	if let Ok(shortroomid) = self.get_shortroomid(room_id).await {
434		return shortroomid;
435	}
436
437	self.create_shortroomid(room_id).await
438}
439
440#[implement(Service)]
441async fn create_shortroomid(&self, room_id: &RoomId) -> ShortRoomId {
442	const BUFSIZE: usize = size_of::<ShortRoomId>();
443
444	let _lock = self.creating.shortroomid.lock(room_id).await;
445
446	if let Ok(shortroomid) = self.get_shortroomid(room_id).await {
447		return shortroomid;
448	}
449
450	let short = self.services.globals.next_count();
451
452	debug_assert!(size_of_val(&*short) == BUFSIZE, "buffer requirement changed");
453
454	self.db
455		.roomid_shortroomid
456		.raw_aput::<BUFSIZE, _, _>(room_id, *short);
457
458	*short
459}
460
461/// Deletes the compact identifier mapping for a room.
462///
463/// Only the forward room mapping is removed. Any unsuccessful existence check is returned as a
464/// not-found database error.
465#[implement(Service)]
466pub async fn delete_shortroomid(&self, room_id: &RoomId) -> Result {
467	if self
468		.db
469		.roomid_shortroomid
470		.exists(room_id)
471		.await
472		.is_ok()
473	{
474		self.db.roomid_shortroomid.remove(room_id);
475		Ok(())
476	} else {
477		Err!(Database("not found"))
478	}
479}