Skip to main content

tuwunel_service/rooms/search/
mod.rs

1use std::sync::Arc;
2
3use futures::{Stream, StreamExt};
4use ruma::{RoomId, UserId, api::client::search::search_events::v3::Criteria};
5use tuwunel_core::{
6	Result,
7	arrayvec::ArrayVec,
8	implement, trace,
9	utils::{
10		ArrayVecExt, IterStream, ReadyExt, set,
11		stream::{TryIgnore, WidebandExt},
12	},
13};
14use tuwunel_database::{Map, Txn, keyval::Val};
15use tuwunel_matrix::{
16	PduCount,
17	event::{Event, Matches},
18};
19
20use crate::rooms::{
21	short::ShortRoomId,
22	timeline::{PduId, RawPduId},
23};
24
25pub struct Service {
26	db: Data,
27	services: Arc<crate::services::OnceServices>,
28}
29
30struct Data {
31	tokenids: Arc<Map>,
32}
33
34#[derive(Clone, Debug)]
35pub struct RoomQuery<'a> {
36	pub room_id: &'a RoomId,
37	pub user_id: Option<&'a UserId>,
38	pub criteria: &'a Criteria,
39	pub limit: usize,
40	/// Exclusive upper count bound for the returned page.
41	pub before: Option<PduCount>,
42}
43
44type TokenId = ArrayVec<u8, TOKEN_ID_MAX_LEN>;
45
46const TOKEN_ID_MAX_LEN: usize =
47	size_of::<ShortRoomId>() + WORD_MAX_LEN + 1 + size_of::<RawPduId>();
48const WORD_MAX_LEN: usize = 50;
49
50impl crate::Service for Service {
51	fn build(args: &crate::Args<'_>) -> Result<Arc<Self>> {
52		Ok(Arc::new(Self {
53			db: Data { tokenids: args.db["tokenids"].clone() },
54			services: args.services.clone(),
55		}))
56	}
57
58	fn name(&self) -> &str { crate::service::make_name(std::module_path!()) }
59}
60
61#[implement(Service)]
62pub fn index_pdu(&self, shortroomid: ShortRoomId, pdu_id: &RawPduId, message_body: &str) {
63	let items = tokenize(message_body).map(|word| {
64		let mut key = shortroomid.to_be_bytes().to_vec();
65		key.extend_from_slice(word.as_bytes());
66		key.push(0xFF);
67		key.extend_from_slice(pdu_id.as_ref()); // TODO: currently we save the room id a second time here
68
69		(key, [])
70	});
71
72	Txn::insert(&self.db.tokenids, items).execute();
73}
74
75#[implement(Service)]
76pub fn deindex_pdu(&self, shortroomid: ShortRoomId, pdu_id: &RawPduId, message_body: &str) {
77	let batch = tokenize(message_body).map(|word| {
78		let mut key = shortroomid.to_be_bytes().to_vec();
79		key.extend_from_slice(word.as_bytes());
80		key.push(0xFF);
81		key.extend_from_slice(pdu_id.as_ref()); // TODO: currently we save the room id a second time here
82		key
83	});
84
85	for token in batch {
86		self.db.tokenids.remove(&token);
87	}
88}
89
90/// Searches a room and returns its total hit count and a page of counted events.
91///
92/// The total is computed before the cursor and event filters. The stream yields
93/// visible matches in descending count order, up to the requested room limit.
94#[implement(Service)]
95pub async fn search_pdus<'a>(
96	&'a self,
97	query: &'a RoomQuery<'a>,
98) -> Result<(usize, impl Stream<Item = (PduCount, impl Event + use<>)> + Send + '_)> {
99	let pdu_ids: Vec<_> = self.search_pdu_ids(query).await?.collect().await;
100
101	let filter = &query.criteria.filter;
102	let count = pdu_ids.len();
103	let pdus = pdu_ids
104		.into_iter()
105		.stream()
106		.ready_filter(|id| {
107			query
108				.before
109				.is_none_or(|before| id.pdu_count().lt(&before))
110		})
111		.wide_filter_map(async |result_pdu_id: RawPduId| {
112			self.services
113				.timeline
114				.get_pdu_from_id(&result_pdu_id)
115				.await
116				.ok()
117				.map(|pdu| (result_pdu_id.pdu_count(), pdu))
118		})
119		.ready_filter(|(_, pdu)| !pdu.is_redacted())
120		.ready_filter(move |(_, pdu)| filter.matches(pdu))
121		.wide_filter_map(async |(count, pdu)| {
122			self.services
123				.state_accessor
124				.user_can_see_event(query.user_id?, &pdu)
125				.await
126				.then_some((count, pdu))
127		})
128		.map(|(count, pdu)| (count, pdu.without_private_unless_sender(query.user_id)))
129		.take(query.limit);
130
131	Ok((count, pdus))
132}
133
134// result is modeled as a stream such that callers don't have to be refactored
135// though an additional async/wrap still exists for now
136#[implement(Service)]
137pub async fn search_pdu_ids(
138	&self,
139	query: &RoomQuery<'_>,
140) -> Result<impl Stream<Item = RawPduId> + Send + '_ + use<'_>> {
141	let shortroomid = self
142		.services
143		.short
144		.get_shortroomid(query.room_id)
145		.await?;
146
147	let pdu_ids = self
148		.search_pdu_ids_query_room(query, shortroomid)
149		.await;
150
151	let iters = pdu_ids.into_iter().map(IntoIterator::into_iter);
152
153	Ok(set::intersection(iters).stream())
154}
155
156#[implement(Service)]
157async fn search_pdu_ids_query_room(
158	&self,
159	query: &RoomQuery<'_>,
160	shortroomid: ShortRoomId,
161) -> Vec<Vec<RawPduId>> {
162	tokenize(&query.criteria.search_term)
163		.stream()
164		.wide_then(async |word| {
165			self.search_pdu_ids_query_words(shortroomid, &word)
166				.collect::<Vec<_>>()
167				.await
168		})
169		.collect::<Vec<_>>()
170		.await
171}
172
173/// Iterate over PduId's containing a word
174#[implement(Service)]
175fn search_pdu_ids_query_words<'a>(
176	&'a self,
177	shortroomid: ShortRoomId,
178	word: &'a str,
179) -> impl Stream<Item = RawPduId> + Send + '_ {
180	self.search_pdu_ids_query_word(shortroomid, word)
181		.map(move |key| -> RawPduId {
182			let key = &key[prefix_len(word)..];
183			key.into()
184		})
185}
186
187/// Iterate over raw database results for a word
188#[implement(Service)]
189fn search_pdu_ids_query_word(
190	&self,
191	shortroomid: ShortRoomId,
192	word: &str,
193) -> impl Stream<Item = Val<'_>> + Send + '_ + use<'_> {
194	// rustc says const'ing this not yet stable
195	let end_id: RawPduId = PduId { shortroomid, count: PduCount::max() }.into();
196
197	// Newest pdus first
198	let end = make_tokenid(shortroomid, word, &end_id);
199	let prefix = make_prefix(shortroomid, word);
200	self.db
201		.tokenids
202		.rev_raw_keys_from(&end)
203		.ignore_err()
204		.ready_take_while(move |key| key.starts_with(&prefix))
205}
206
207#[implement(Service)]
208pub async fn delete_all_search_tokenids_for_room(&self, room_id: &RoomId) -> Result {
209	let Ok(shortroomid) = self.services.short.get_shortroomid(room_id).await else {
210		return Ok(());
211	};
212
213	let txn = self
214		.db
215		.tokenids
216		.keys_prefix_raw(&shortroomid)
217		.ignore_err()
218		.ready_fold(self.services.db.txn(), |mut txn, key| {
219			trace!("Removing key: {key:?}");
220			txn.del_raw(&self.db.tokenids, key);
221			txn
222		})
223		.await;
224
225	txn.execute();
226
227	Ok(())
228}
229
230/// Splits a string into tokens used as keys in the search inverted index
231///
232/// This may be used to tokenize both message bodies (for indexing) or search
233/// queries (for querying).
234fn tokenize(body: &str) -> impl Iterator<Item = String> + Send + '_ {
235	body.split_terminator(|c: char| !c.is_alphanumeric())
236		.filter(|s| !s.is_empty())
237		.filter(|word| word.len() <= WORD_MAX_LEN)
238		.map(str::to_lowercase)
239}
240
241fn make_tokenid(shortroomid: ShortRoomId, word: &str, pdu_id: &RawPduId) -> TokenId {
242	let mut key = make_prefix(shortroomid, word);
243	key.extend_from_slice(pdu_id.as_ref());
244	key
245}
246
247fn make_prefix(shortroomid: ShortRoomId, word: &str) -> TokenId {
248	let mut key = TokenId::new();
249	key.extend_from_slice(&shortroomid.to_be_bytes());
250	key.extend_from_slice(word.as_bytes());
251	key.push(tuwunel_database::SEP);
252	key
253}
254
255fn prefix_len(word: &str) -> usize {
256	size_of::<ShortRoomId>()
257		.saturating_add(word.len())
258		.saturating_add(1)
259}