tuwunel_service/rooms/search/
mod.rs1use std::sync::Arc;
2
3use futures::{Stream, StreamExt};
4use ruma::{RoomId, UserId, api::client::search::search_events::v3::Criteria};
5use tuwunel_core::{
6 Result,
7 arrayvec::ArrayVec,
8 implement, trace,
9 utils::{
10 ArrayVecExt, IterStream, ReadyExt, set,
11 stream::{TryIgnore, WidebandExt},
12 },
13};
14use tuwunel_database::{Map, Txn, keyval::Val};
15use tuwunel_matrix::{
16 PduCount,
17 event::{Event, Matches},
18};
19
20use crate::rooms::{
21 short::ShortRoomId,
22 timeline::{PduId, RawPduId},
23};
24
25pub struct Service {
26 db: Data,
27 services: Arc<crate::services::OnceServices>,
28}
29
30struct Data {
31 tokenids: Arc<Map>,
32}
33
34#[derive(Clone, Debug)]
35pub struct RoomQuery<'a> {
36 pub room_id: &'a RoomId,
37 pub user_id: Option<&'a UserId>,
38 pub criteria: &'a Criteria,
39 pub limit: usize,
40 pub before: Option<PduCount>,
42}
43
44type TokenId = ArrayVec<u8, TOKEN_ID_MAX_LEN>;
45
46const TOKEN_ID_MAX_LEN: usize =
47 size_of::<ShortRoomId>() + WORD_MAX_LEN + 1 + size_of::<RawPduId>();
48const WORD_MAX_LEN: usize = 50;
49
50impl crate::Service for Service {
51 fn build(args: &crate::Args<'_>) -> Result<Arc<Self>> {
52 Ok(Arc::new(Self {
53 db: Data { tokenids: args.db["tokenids"].clone() },
54 services: args.services.clone(),
55 }))
56 }
57
58 fn name(&self) -> &str { crate::service::make_name(std::module_path!()) }
59}
60
61#[implement(Service)]
62pub fn index_pdu(&self, shortroomid: ShortRoomId, pdu_id: &RawPduId, message_body: &str) {
63 let items = tokenize(message_body).map(|word| {
64 let mut key = shortroomid.to_be_bytes().to_vec();
65 key.extend_from_slice(word.as_bytes());
66 key.push(0xFF);
67 key.extend_from_slice(pdu_id.as_ref()); (key, [])
70 });
71
72 Txn::insert(&self.db.tokenids, items).execute();
73}
74
75#[implement(Service)]
76pub fn deindex_pdu(&self, shortroomid: ShortRoomId, pdu_id: &RawPduId, message_body: &str) {
77 let batch = tokenize(message_body).map(|word| {
78 let mut key = shortroomid.to_be_bytes().to_vec();
79 key.extend_from_slice(word.as_bytes());
80 key.push(0xFF);
81 key.extend_from_slice(pdu_id.as_ref()); key
83 });
84
85 for token in batch {
86 self.db.tokenids.remove(&token);
87 }
88}
89
90#[implement(Service)]
95pub async fn search_pdus<'a>(
96 &'a self,
97 query: &'a RoomQuery<'a>,
98) -> Result<(usize, impl Stream<Item = (PduCount, impl Event + use<>)> + Send + '_)> {
99 let pdu_ids: Vec<_> = self.search_pdu_ids(query).await?.collect().await;
100
101 let filter = &query.criteria.filter;
102 let count = pdu_ids.len();
103 let pdus = pdu_ids
104 .into_iter()
105 .stream()
106 .ready_filter(|id| {
107 query
108 .before
109 .is_none_or(|before| id.pdu_count().lt(&before))
110 })
111 .wide_filter_map(async |result_pdu_id: RawPduId| {
112 self.services
113 .timeline
114 .get_pdu_from_id(&result_pdu_id)
115 .await
116 .ok()
117 .map(|pdu| (result_pdu_id.pdu_count(), pdu))
118 })
119 .ready_filter(|(_, pdu)| !pdu.is_redacted())
120 .ready_filter(move |(_, pdu)| filter.matches(pdu))
121 .wide_filter_map(async |(count, pdu)| {
122 self.services
123 .state_accessor
124 .user_can_see_event(query.user_id?, &pdu)
125 .await
126 .then_some((count, pdu))
127 })
128 .map(|(count, pdu)| (count, pdu.without_private_unless_sender(query.user_id)))
129 .take(query.limit);
130
131 Ok((count, pdus))
132}
133
134#[implement(Service)]
137pub async fn search_pdu_ids(
138 &self,
139 query: &RoomQuery<'_>,
140) -> Result<impl Stream<Item = RawPduId> + Send + '_ + use<'_>> {
141 let shortroomid = self
142 .services
143 .short
144 .get_shortroomid(query.room_id)
145 .await?;
146
147 let pdu_ids = self
148 .search_pdu_ids_query_room(query, shortroomid)
149 .await;
150
151 let iters = pdu_ids.into_iter().map(IntoIterator::into_iter);
152
153 Ok(set::intersection(iters).stream())
154}
155
156#[implement(Service)]
157async fn search_pdu_ids_query_room(
158 &self,
159 query: &RoomQuery<'_>,
160 shortroomid: ShortRoomId,
161) -> Vec<Vec<RawPduId>> {
162 tokenize(&query.criteria.search_term)
163 .stream()
164 .wide_then(async |word| {
165 self.search_pdu_ids_query_words(shortroomid, &word)
166 .collect::<Vec<_>>()
167 .await
168 })
169 .collect::<Vec<_>>()
170 .await
171}
172
173#[implement(Service)]
175fn search_pdu_ids_query_words<'a>(
176 &'a self,
177 shortroomid: ShortRoomId,
178 word: &'a str,
179) -> impl Stream<Item = RawPduId> + Send + '_ {
180 self.search_pdu_ids_query_word(shortroomid, word)
181 .map(move |key| -> RawPduId {
182 let key = &key[prefix_len(word)..];
183 key.into()
184 })
185}
186
187#[implement(Service)]
189fn search_pdu_ids_query_word(
190 &self,
191 shortroomid: ShortRoomId,
192 word: &str,
193) -> impl Stream<Item = Val<'_>> + Send + '_ + use<'_> {
194 let end_id: RawPduId = PduId { shortroomid, count: PduCount::max() }.into();
196
197 let end = make_tokenid(shortroomid, word, &end_id);
199 let prefix = make_prefix(shortroomid, word);
200 self.db
201 .tokenids
202 .rev_raw_keys_from(&end)
203 .ignore_err()
204 .ready_take_while(move |key| key.starts_with(&prefix))
205}
206
207#[implement(Service)]
208pub async fn delete_all_search_tokenids_for_room(&self, room_id: &RoomId) -> Result {
209 let Ok(shortroomid) = self.services.short.get_shortroomid(room_id).await else {
210 return Ok(());
211 };
212
213 let txn = self
214 .db
215 .tokenids
216 .keys_prefix_raw(&shortroomid)
217 .ignore_err()
218 .ready_fold(self.services.db.txn(), |mut txn, key| {
219 trace!("Removing key: {key:?}");
220 txn.del_raw(&self.db.tokenids, key);
221 txn
222 })
223 .await;
224
225 txn.execute();
226
227 Ok(())
228}
229
230fn tokenize(body: &str) -> impl Iterator<Item = String> + Send + '_ {
235 body.split_terminator(|c: char| !c.is_alphanumeric())
236 .filter(|s| !s.is_empty())
237 .filter(|word| word.len() <= WORD_MAX_LEN)
238 .map(str::to_lowercase)
239}
240
241fn make_tokenid(shortroomid: ShortRoomId, word: &str, pdu_id: &RawPduId) -> TokenId {
242 let mut key = make_prefix(shortroomid, word);
243 key.extend_from_slice(pdu_id.as_ref());
244 key
245}
246
247fn make_prefix(shortroomid: ShortRoomId, word: &str) -> TokenId {
248 let mut key = TokenId::new();
249 key.extend_from_slice(&shortroomid.to_be_bytes());
250 key.extend_from_slice(word.as_bytes());
251 key.push(tuwunel_database::SEP);
252 key
253}
254
255fn prefix_len(word: &str) -> usize {
256 size_of::<ShortRoomId>()
257 .saturating_add(word.len())
258 .saturating_add(1)
259}