1mod data;
2pub(super) mod migrations;
3mod preview;
4mod remote;
5mod space;
6mod tests;
7mod thumbnail;
8#[cfg(feature = "media_thumbnail")]
9mod video;
10use std::{
11 collections::{HashMap, HashSet},
12 path::PathBuf,
13 sync::{Arc, Mutex},
14 time::{Duration, Instant, SystemTime},
15};
16
17use async_trait::async_trait;
18use base64::{Engine as _, engine::general_purpose};
19use futures::{FutureExt, Stream, StreamExt, TryFutureExt, TryStreamExt, pin_mut};
20use http::StatusCode;
21use object_store::ObjectMeta;
22use ruma::{
23 Mxc, OwnedMxcUri, OwnedUserId, UserId,
24 api::error::{ErrorKind, RetryAfter},
25 http_headers::ContentDisposition,
26};
27#[cfg(feature = "media_thumbnail")]
28use tokio::sync::Semaphore;
29use tokio::{fs, sync::Notify};
30use tuwunel_core::{
31 Err, Error, Result,
32 config::Media as MediaConfig,
33 debug, debug_error, debug_info, debug_warn, err, implement, trace,
34 utils::{
35 self, BoolExt, MutexMap,
36 result::LogDebugErr,
37 stream::{BroadbandExt, IterStream, ReadyExt, TryReadyExt},
38 time::now_millis,
39 },
40 warn,
41};
42use url::Url;
43
44#[cfg(feature = "media_thumbnail")]
45use self::video::{FAILURES, Failures, sweep_staging_dir};
46use self::{data::Data, preview::Agent, remote::Fetch, thumbnail::sequence};
47pub use self::{
48 data::Metadata,
49 preview::UrlPreviewData,
50 thumbnail::{Animate, Dim},
51};
52use crate::storage::Provider;
53
54#[derive(Debug)]
55pub struct Media {
56 pub content: Vec<u8>,
57 pub content_type: Option<String>,
58 pub content_disposition: Option<ContentDisposition>,
59}
60
61#[derive(Debug)]
68pub struct Fetched {
69 pub media: Media,
70 pub(crate) animates: Option<bool>,
75}
76
77#[derive(Clone, Debug)]
81pub struct UserMediaEntry {
82 pub mxc: OwnedMxcUri,
83 pub media_type: Option<String>,
84 pub upload_name: Option<String>,
85 pub media_length: Option<u64>,
86 pub created_ts: u64,
87 pub user_id: Option<OwnedUserId>,
88}
89
90#[derive(Clone, Debug)]
93pub struct UploadStat {
94 pub user_id: OwnedUserId,
95 pub media_length: u64,
96 pub created_ts: u64,
97}
98
99struct MXCState {
101 notifiers: Mutex<HashMap<OwnedMxcUri, Arc<Notify>>>,
103 ratelimiter: Mutex<HashMap<OwnedUserId, (Instant, f64)>>,
105}
106
107pub struct Service {
108 pub(super) db: Data,
109 services: Arc<crate::services::OnceServices>,
110 url_preview_mutex: MutexMap<String, ()>,
111 federation_mutex: MutexMap<String, ()>,
112 mxc_state: MXCState,
113 #[cfg(feature = "media_thumbnail")]
114 animated_thumbnail_slots: Arc<Semaphore>,
115 #[cfg(feature = "media_thumbnail")]
116 video_thumbnail_slots: Semaphore,
117 #[cfg(feature = "media_thumbnail")]
118 video_thumbnail_failures: Mutex<Failures>,
119}
120
121pub const MXC_LENGTH: usize = 32;
123
124pub const CACHE_CONTROL_IMMUTABLE: &str = "private,max-age=31536000,immutable";
126
127pub const CORP_CROSS_ORIGIN: &str = "cross-origin";
129
130const REDIRECT_TTL: Duration = Duration::from_mins(5);
132
133#[async_trait]
134impl crate::Service for Service {
135 fn build(args: &crate::Args<'_>) -> Result<Arc<Self>> {
136 #[cfg(feature = "media_thumbnail")]
137 let media = &args.server.config.media;
138
139 #[cfg(feature = "media_thumbnail")]
140 let animated_thumbnail_slots =
141 Arc::new(Semaphore::new(media.thumbnail.animated_concurrency.max(1)));
142
143 let service = Arc::new(Self {
144 db: Data::new(args.db),
145 services: args.services.clone(),
146 url_preview_mutex: MutexMap::new(),
147 federation_mutex: MutexMap::new(),
148 mxc_state: MXCState {
149 notifiers: Mutex::new(HashMap::new()),
150 ratelimiter: Mutex::new(HashMap::new()),
151 },
152 #[cfg(feature = "media_thumbnail")]
153 video_thumbnail_failures: Failures::new(FAILURES).into(),
154 #[cfg(feature = "media_thumbnail")]
155 animated_thumbnail_slots,
156 #[cfg(feature = "media_thumbnail")]
157 video_thumbnail_slots: Semaphore::new(media.video_thumbnail.concurrency.max(1)),
158 });
159
160 #[cfg(feature = "media_thumbnail")]
161 sweep_staging_dir(&args.server.config);
162
163 Ok(service)
164 }
165
166 fn name(&self) -> &str { crate::service::make_name(std::module_path!()) }
167}
168
169impl Service {
170 #[tracing::instrument(level = "debug", skip(self))]
177 pub async fn create_pending(
178 &self,
179 mxc: &Mxc<'_>,
180 user: &UserId,
181 unused_expires_at: u64,
182 ) -> Result {
183 let config = &self.services.server.config;
184
185 let rate = f64::from(config.media.rc_create_per_second);
187 let burst = f64::from(config.media.rc_create_burst_count);
188
189 if rate > 0.0 && burst > 0.0 {
191 let now = Instant::now();
192 let mut ratelimiter = self.mxc_state.ratelimiter.lock()?;
193
194 let (last_time, tokens) = ratelimiter
195 .entry(user.to_owned())
196 .or_insert_with(|| (now, burst));
197
198 let elapsed = now.duration_since(*last_time).as_secs_f64();
199 let new_tokens = elapsed.mul_add(rate, *tokens).min(burst);
200
201 if new_tokens >= 1.0 {
202 *last_time = now;
203 *tokens = new_tokens - 1.0;
204 } else {
205 return Err(Error::Request(
206 ErrorKind::LimitExceeded(ruma::api::error::LimitExceededErrorData {
207 retry_after: None,
208 }),
209 "Too many pending media creation requests.".into(),
210 StatusCode::TOO_MANY_REQUESTS,
211 ));
212 }
213 }
214
215 self.check_upload_space().await?;
216
217 let max_uploads = config.media.max_pending_uploads;
218 let (current_uploads, earliest_expiration) =
219 self.db.count_pending_mxc_for_user(user).await;
220
221 if current_uploads >= max_uploads {
223 let retry_after = earliest_expiration.saturating_sub(now_millis());
224 return Err(Error::Request(
225 ErrorKind::LimitExceeded(ruma::api::error::LimitExceededErrorData {
226 retry_after: Some(RetryAfter::Delay(Duration::from_millis(retry_after))),
227 }),
228 "Maximum number of pending media uploads reached.".into(),
229 StatusCode::TOO_MANY_REQUESTS,
230 ));
231 }
232
233 self.db
234 .insert_pending_mxc(mxc, user, unused_expires_at);
235
236 Ok(())
237 }
238
239 #[tracing::instrument(level = "debug", skip(self))]
241 pub async fn upload_pending(
242 &self,
243 mxc: &Mxc<'_>,
244 user: &UserId,
245 content_disposition: Option<&ContentDisposition>,
246 content_type: Option<&str>,
247 file: &[u8],
248 ) -> Result {
249 let Ok((owner_id, expires_at)) = self.db.search_pending_mxc(mxc).await else {
250 if self.get_metadata(mxc).await.is_some() {
251 return Err!(Request(CannotOverwriteMedia("Media ID already has content")));
252 }
253
254 return Err!(Request(NotFound("Media not found")));
255 };
256
257 if owner_id != user {
258 return Err!(Request(Forbidden("You did not create this media ID")));
259 }
260
261 let current_time = now_millis();
262 if expires_at < current_time {
263 return Err!(Request(NotFound("Pending media ID expired")));
264 }
265
266 self.create(mxc, Some(user), content_disposition, content_type, file)
267 .await?;
268
269 self.db.remove_pending_mxc(mxc);
270
271 let mxc_uri: OwnedMxcUri = mxc.to_string().into();
272 let notifier = self.mxc_state.notifiers.lock()?.remove(&mxc_uri);
273
274 if let Some(notifier) = notifier {
275 notifier.notify_waiters();
276 }
277
278 Ok(())
279 }
280
281 pub async fn create(
289 &self,
290 mxc: &Mxc<'_>,
291 user: Option<&UserId>,
292 content_disposition: Option<&ContentDisposition>,
293 content_type: Option<&str>,
294 file: &[u8],
295 ) -> Result<bool> {
296 let walk = sequence(file);
297
298 let key = self.db.create_file_metadata(
300 mxc,
301 user,
302 &Dim::default(),
303 content_disposition,
304 walk.stored_type(content_type),
305 )?;
306
307 self.create_media_file(&key, file)
309 .map_ok(|()| walk.animates())
310 .await
311 }
312
313 #[tracing::instrument(level = "trace", skip(self))]
315 pub async fn delete(&self, mxc: &Mxc<'_>) -> Result {
316 let had_lazy = self.db.search_lazy_media(mxc).await.is_ok();
319 if had_lazy {
320 let key = mxc.to_string();
321 let mut txn = self.services.db.txn();
322
323 self.db.remove_lazy_media(&mut txn, &key);
324 self.db.remove_lazy_content(&mut txn, &key);
325 txn.execute();
326 }
327
328 match self.db.search_mxc_metadata_prefix(mxc).await {
329 | Ok(keys) => {
330 for key in keys {
331 trace!(?mxc, "MXC Key: {key:?}");
332 debug_info!(?mxc, "Deleting from storage provider");
333
334 if let Err(e) = self.remove_media_file(&key).await {
335 debug_error!(?mxc, "Failed to remove media file: {e}");
336 }
337
338 debug_info!(?mxc, "Deleting from database");
339 self.db.delete_file_mxc(mxc).await;
340 }
341
342 Ok(())
343 },
344 | _ if had_lazy => Ok(()),
345 | _ => Err!(Database(error!(
346 "Failed to find any media keys for MXC {mxc} in our database."
347 ))),
348 }
349 }
350
351 #[tracing::instrument(level = "trace", skip(self))]
355 pub async fn delete_from_user(&self, user: &UserId) -> Result<usize> {
356 let mxcs = self.db.get_all_user_mxcs(user).await;
357 let mut deletion_count: usize = 0;
358
359 for mxc in mxcs {
360 let Ok(mxc) = mxc.as_str().try_into().inspect_err(|e| {
361 debug_error!(?mxc, "Failed to parse MXC URI from database: {e}");
362 }) else {
363 continue;
364 };
365
366 debug_info!(
367 %deletion_count,
368 "Deleting MXC {mxc} by user {user} from database and filesystem",
369 );
370 match self.delete(&mxc).await {
371 | Ok(()) => {
372 deletion_count = deletion_count.saturating_add(1);
373 },
374 | Err(e) => {
375 debug_error!(
376 %deletion_count,
377 "Failed to delete {mxc} from user {user}, ignoring error: {e}"
378 );
379 },
380 }
381 }
382
383 Ok(deletion_count)
384 }
385
386 #[tracing::instrument(
389 level = "debug",
390 err(level = "debug")
391 skip(self),
392 )]
393 pub async fn get_or_fetch(&self, mxc: &Mxc<'_>, timeout_ms: Duration) -> Result<Media> {
394 if let Ok(media) = self.get(mxc, Some(timeout_ms)).await {
395 return Ok(media);
396 }
397
398 if self
399 .services
400 .globals
401 .server_is_ours(mxc.server_name)
402 {
403 return Err!(Request(NotFound("Local media not found.")));
404 }
405
406 let lock = self.federation_mutex.lock(&mxc.to_string()).await;
407
408 if self
409 .db
410 .file_metadata_exists(mxc, &Dim::default())
411 .await
412 {
413 drop(lock);
414 return self.get(mxc, None).await;
415 }
416
417 self.fetch_remote_content(mxc, None, timeout_ms)
418 .map_ok(|fetched| fetched.media)
419 .await
420 }
421
422 #[tracing::instrument(
425 level = "debug",
426 err(level = "trace")
427 skip(self),
428 )]
429 pub async fn get(&self, mxc: &Mxc<'_>, timeout: Option<Duration>) -> Result<Media> {
430 if let Ok(meta) = self.get_stored(mxc).await {
431 return Ok(meta);
432 }
433
434 let Some(timeout) = timeout else {
435 return Err!(Request(NotFound("Media not found.")));
436 };
437
438 let Ok(_pending) = self.db.search_pending_mxc(mxc).await else {
439 return Err!(Request(NotFound("Media not found.")));
440 };
441
442 let notifier = self
443 .mxc_state
444 .notifiers
445 .lock()?
446 .entry(mxc.to_string().into())
447 .or_insert_with(|| Arc::new(Notify::new()))
448 .clone();
449
450 if tokio::time::timeout(timeout, notifier.notified())
451 .await
452 .is_err()
453 {
454 return Err!(Request(NotYetUploaded("Media has not been uploaded yet")));
455 }
456
457 self.get_stored(mxc).await
458 }
459
460 #[tracing::instrument(level = "debug", skip(self))]
465 pub async fn get_stored(&self, mxc: &Mxc<'_>) -> Result<Media> {
466 let meta = self
467 .db
468 .search_file_metadata(mxc, &Dim::default(), Animate::Allowed)
469 .await;
470
471 let Ok(Metadata { content_type, content_disposition, key }) = meta else {
472 return Box::pin(self.fetch_lazy_media(mxc)).await;
474 };
475
476 let path = self.get_media_name_sha256(&key);
477 let fetch = self
478 .storage_providers()
479 .stream()
480 .filter_map(async |provider| {
481 provider
482 .get(path.as_str())
483 .await
484 .log_debug_err()
485 .ok()
486 });
487
488 pin_mut!(fetch);
489 let Some(bytes) = fetch.next().await else {
490 return Err!(Request(NotFound("Media not found.")));
491 };
492
493 Ok(Media {
494 content: bytes.to_vec(),
495 content_type,
496 content_disposition,
497 })
498 }
499
500 #[tracing::instrument(level = "debug", skip(self))]
504 async fn fetch_lazy_media(&self, mxc: &Mxc<'_>) -> Result<Media> {
505 let key = mxc.to_string();
506
507 let _lock = self.federation_mutex.lock(&key).await;
510
511 if self
513 .db
514 .search_file_metadata(mxc, &Dim::default(), Animate::Allowed)
515 .await
516 .is_ok()
517 {
518 return Box::pin(self.get_stored(mxc)).await;
520 }
521
522 let media = match self.db.get_lazy_content(&key).await {
523 | Ok(media) => media,
524 | Err(_) => {
525 let Ok(url) = self.db.search_lazy_media(mxc).await else {
526 return Err!(Request(NotFound("Media not found.")));
527 };
528
529 let limit = self
530 .services
531 .config
532 .media
533 .url_preview
534 .max_media_size;
535
536 self.location_request(Fetch::Preview(Agent::Media), &url, limit)
537 .await?
538 },
539 };
540
541 if let Err(e) = self
544 .create(
545 mxc,
546 None,
547 media.content_disposition.as_ref(),
548 media.content_type.as_deref(),
549 &media.content,
550 )
551 .await
552 {
553 self.db.delete_file_mxc(mxc).await;
556
557 return Err(e);
558 }
559
560 let mut txn = self.services.db.txn();
561
562 self.db.remove_lazy_media(&mut txn, &key);
563 self.db.remove_lazy_content(&mut txn, &key);
564 txn.execute();
565
566 Ok(media)
567 }
568
569 #[tracing::instrument(level = "debug", skip(self))]
577 pub async fn redirect_url(
578 &self,
579 mxc: &Mxc<'_>,
580 dim: &Dim,
581 animate: Animate,
582 ) -> Result<Option<Url>> {
583 if !self.services.config.media.allow_redirect {
584 return Ok(None);
585 }
586
587 let Ok(Metadata { key, content_type, .. }) = self
588 .db
589 .search_file_metadata(mxc, dim, animate)
590 .await
591 else {
592 return Ok(None);
593 };
594
595 if !animate.accepts_type(content_type.as_deref()) {
598 return Ok(None);
599 }
600
601 let path = self.get_media_name_sha256(&key);
602 let urls = self
603 .storage_providers()
604 .stream()
605 .filter_map(async |provider| {
606 provider
607 .signed_get_url(path.as_str(), REDIRECT_TTL)
608 .await
609 .log_debug_err()
610 .ok()
611 .flatten()
612 });
613
614 pin_mut!(urls);
615
616 Ok(urls.next().await)
617 }
618
619 pub async fn get_all_mxcs(&self) -> Result<Vec<OwnedMxcUri>> {
621 let all_keys = self.db.get_all_media_keys().await;
622
623 let mut mxcs = Vec::with_capacity(all_keys.len());
624
625 for key in all_keys {
626 trace!("Full MXC key from database: {key:?}");
627
628 let mut parts = key.split(|&b| b == 0xFF);
629 let mxc = parts
630 .next()
631 .map(|bytes| {
632 utils::string_from_bytes(bytes).map_err(|e| {
633 err!(Database(error!(
634 "Failed to parse MXC unicode bytes from our database: {e}"
635 )))
636 })
637 })
638 .transpose()?;
639
640 let Some(mxc_s) = mxc else {
641 debug_warn!(
642 ?mxc,
643 "Parsed MXC URL unicode bytes from database but is still invalid"
644 );
645 continue;
646 };
647
648 trace!("Parsed MXC key to URL: {mxc_s}");
649 let mxc = OwnedMxcUri::from(mxc_s);
650
651 if mxc.is_valid() {
652 mxcs.push(mxc);
653 } else {
654 debug_warn!("{mxc:?} from database was found to not be valid");
655 }
656 }
657
658 Ok(mxcs)
659 }
660
661 #[tracing::instrument(level = "debug", skip(self))]
666 pub async fn user_media(&self, user: &UserId) -> Result<Vec<UserMediaEntry>> {
667 let entries = self
668 .db
669 .get_all_user_mxcs(user)
670 .await
671 .into_iter()
672 .stream()
673 .broad_filter_map(async |mxc| self.user_media_entry(Some(user), mxc).await)
674 .collect()
675 .await;
676
677 Ok(entries)
678 }
679
680 #[tracing::instrument(level = "debug", skip(self))]
683 pub async fn media_entry(&self, mxc: &Mxc<'_>) -> Option<UserMediaEntry> {
684 let user = self.db.mxc_user(mxc).await;
685
686 self.user_media_entry(user.as_deref(), mxc.to_string().into())
687 .await
688 }
689
690 async fn user_media_entry(
691 &self,
692 user: Option<&UserId>,
693 mxc: OwnedMxcUri,
694 ) -> Option<UserMediaEntry> {
695 let parts = mxc.parts().ok()?;
696 let Metadata { content_type, content_disposition, key } =
697 self.get_metadata(&parts).await?;
698
699 let object = self.head_meta(&key).await;
700 let upload_name = content_disposition.and_then(|disposition| disposition.filename);
701
702 Some(UserMediaEntry {
703 media_type: content_type,
704 upload_name,
705 media_length: object.as_ref().map(|object| object.size),
706 created_ts: object.as_ref().map(mtime_millis).unwrap_or(0),
707 user_id: user.map(ToOwned::to_owned),
708 mxc,
709 })
710 }
711
712 pub fn upload_stats(&self) -> impl Stream<Item = UploadStat> + Send + '_ {
716 self.db
717 .all_uploads()
718 .broad_filter_map(async |(mxc, user_id)| {
719 let parts = mxc.parts().ok()?;
720 let Metadata { key, .. } = self.get_metadata(&parts).await?;
721 let object = self.head_meta(&key).await?;
722
723 Some(UploadStat {
724 user_id,
725 media_length: object.size,
726 created_ts: mtime_millis(&object),
727 })
728 })
729 }
730
731 #[tracing::instrument(level = "debug", skip(self))]
737 pub async fn delete_by_date_size(
738 &self,
739 before_ts: u64,
740 size_gt: u64,
741 keep_profiles: bool,
742 ) -> Result<Vec<OwnedMxcUri>> {
743 let spared = keep_profiles
744 .then_async(|| self.avatar_mxcs())
745 .await
746 .unwrap_or_default();
747
748 let candidates = self.get_all_mxcs().await?;
749
750 let deleted: Vec<OwnedMxcUri> = candidates
751 .into_iter()
752 .stream()
753 .ready_filter(|mxc| self.is_local(mxc) && !spared.contains(mxc))
754 .broad_filter_map(async |mxc| {
755 let parts = mxc.parts().ok()?;
756 let Metadata { key, .. } = self.get_metadata(&parts).await?;
757 let object = self.head_meta(&key).await?;
758
759 let eligible = mtime_millis(&object) < before_ts && object.size > size_gt;
760
761 eligible
762 .then_async(|| self.delete(&parts))
763 .await
764 .and_then(Result::ok)
765 .map(|()| mxc)
766 })
767 .collect()
768 .await;
769
770 Ok(deleted)
771 }
772
773 async fn avatar_mxcs(&self) -> HashSet<OwnedMxcUri> {
776 let user_avatars = self
777 .services
778 .users
779 .list_local_users()
780 .map(ToOwned::to_owned)
781 .broad_filter_map(async |user| self.services.profile.avatar_url(&user).await.ok());
782
783 let room_avatars = self
784 .services
785 .metadata
786 .iter_ids()
787 .map(ToOwned::to_owned)
788 .broad_filter_map(async |room_id| {
789 self.services
790 .state_accessor
791 .get_avatar(&room_id)
792 .await
793 .ok()
794 .and_then(|avatar| avatar.url)
795 });
796
797 user_avatars.chain(room_avatars).collect().await
798 }
799
800 fn is_local(&self, mxc: &OwnedMxcUri) -> bool {
801 mxc.server_name()
802 .is_ok_and(|server| self.services.globals.server_is_ours(server))
803 }
804
805 async fn head_meta(&self, key: &[u8]) -> Option<ObjectMeta> {
809 let path = self.get_media_name_sha256(key);
810
811 let stream = self
812 .storage_providers()
813 .stream()
814 .filter_map(async |provider| provider.head(&path).await.log_debug_err().ok());
815
816 pin_mut!(stream);
817 stream.next().await
818 }
819
820 pub async fn delete_range(
823 &self,
824 time: SystemTime,
825 older_than: bool,
826 newer_than: bool,
827 yes_i_want_to_delete_local_media: bool,
828 ) -> Result<usize> {
829 let all_keys = self.db.get_all_media_keys().await;
830 let mut remote_mxcs = Vec::with_capacity(all_keys.len());
831
832 for key in all_keys {
833 trace!("Full MXC key from database: {key:?}");
834 let mut parts = key.split(|&b| b == 0xFF);
835 let mxc = parts
836 .next()
837 .map(|bytes| {
838 utils::string_from_bytes(bytes).map_err(|e| {
839 err!(Database(error!(
840 "Failed to parse MXC unicode bytes from our database: {e}"
841 )))
842 })
843 })
844 .transpose()?;
845
846 let Some(mxc_s) = mxc else {
847 debug_warn!(
848 ?mxc,
849 "Parsed MXC URL unicode bytes from database but is still invalid"
850 );
851 continue;
852 };
853
854 trace!("Parsed MXC key to URL: {mxc_s}");
855 let mxc = OwnedMxcUri::from(mxc_s);
856 if (mxc.server_name() == Ok(self.services.globals.server_name())
857 && !yes_i_want_to_delete_local_media)
858 || !mxc.is_valid()
859 {
860 debug!("Ignoring local or broken media MXC: {mxc}");
861 continue;
862 }
863
864 let file_created_at = if let Some(file_metadata) = self
865 .storage_providers()
866 .stream()
867 .filter_map(async |provider| {
868 let path = self.get_media_name_sha256(&key);
869 match provider.head(&path).await {
870 | Ok(file_metadata) => {
871 trace!(%mxc, ?path, "Provider file metadata: {file_metadata:?}");
872 Some(file_metadata)
873 },
874 | Err(e) => {
875 debug_warn!(
876 "Failed to obtain {:?} file metadata for MXC {mxc} at file path \
877 {path:?}\", skipping: {e}",
878 provider.name,
879 );
880 None
881 },
882 }
883 })
884 .boxed() .next()
886 .await
887 {
888 SystemTime::from(file_metadata.last_modified)
889 } else {
890 continue;
891 };
892
893 debug!("File created at: {file_created_at:?}");
894
895 if file_created_at <= time && older_than {
896 debug!(
897 "File is older than user duration, pushing to list of file paths and keys \
898 to delete."
899 );
900 remote_mxcs.push(mxc.to_string());
901 } else if file_created_at >= time && newer_than {
902 debug!(
903 "File is newer than user duration, pushing to list of file paths and keys \
904 to delete."
905 );
906 remote_mxcs.push(mxc.to_string());
907 }
908 }
909
910 debug_info!("Deleting media now in the past {time:?}");
911
912 let mut deletion_count: usize = 0;
913
914 for mxc in remote_mxcs {
915 let Ok(mxc) = mxc.as_str().try_into() else {
916 debug_warn!("Invalid MXC in database, skipping");
917 continue;
918 };
919
920 debug_info!("Deleting MXC {mxc} from database and filesystem");
921
922 match self.delete(&mxc).await {
923 | Ok(()) => {
924 deletion_count = deletion_count.saturating_add(1);
925 },
926 | Err(e) => {
927 warn!("Failed to delete {mxc}, ignoring error and skipping: {e}");
928 },
929 }
930 }
931
932 Ok(deletion_count)
933 }
934
935 pub async fn create_media_dir(&self) -> Result {
936 let dir = self.get_media_dir();
937 Ok(fs::create_dir_all(dir).await?)
938 }
939
940 async fn remove_media_file(&self, key: &[u8]) -> Result {
941 let path = self.get_media_name_sha256(key);
942 self.storage_providers()
943 .stream()
944 .filter_map(async |provider| {
945 debug!(
946 ?key, ?path, provider = ?provider.name,
947 "Deleting media file from provider",
948 );
949
950 provider
951 .delete_one(&path)
952 .await
953 .log_debug_err()
954 .ok()
955 })
956 .count()
957 .map(|count| {
958 count
959 .ge(&0)
960 .into_option()
961 .ok_or_else(|| err!(Request(NotFound("Failed to remove on any provider."))))
962 })
963 .await
964 }
965
966 async fn create_media_file(&self, key: &[u8], file: &[u8]) -> Result {
967 self.upload_providers()
968 .try_stream()
969 .and_then(async |provider| {
970 let path = self.get_media_name_sha256(key);
971 debug!(
972 ?key, ?path,
973 len = ?file.len(),
974 provider = ?provider.name,
975 "Creating media file on storage provider."
976 );
977
978 if let Err(e) = provider
979 .put_one(path.as_str(), file.to_vec())
980 .await
981 {
982 return Err!(Database(error!(
983 ?path,
984 ?provider,
985 "Failed to store media on provider: {e:?}"
986 )));
987 }
988
989 Ok(1)
990 })
991 .ready_try_fold(0_usize, |a, c| Ok(a.saturating_add(c)))
992 .inspect_ok(|&uploads| assert!(uploads > 0, "Successfully saved to nowhere."))
993 .map_ok(|_| ())
994 .await
995 }
996
997 #[inline]
998 pub async fn get_metadata(&self, mxc: &Mxc<'_>) -> Option<Metadata> {
999 self.db
1000 .search_file_metadata(mxc, &Dim::default(), Animate::Allowed)
1001 .await
1002 .ok()
1003 }
1004
1005 #[inline]
1006 #[must_use]
1007 pub fn get_media_path_sha256(&self, key: &[u8]) -> PathBuf {
1008 self.get_media_dir()
1009 .join(self.get_media_name_sha256(key))
1010 }
1011
1012 #[inline]
1015 #[must_use]
1016 pub fn get_media_name_sha256(&self, key: &[u8]) -> String {
1017 let digest = <sha2::Sha256 as sha2::Digest>::digest(key);
1021 encode_key(&digest)
1022 }
1023
1024 #[must_use]
1028 pub fn get_media_path_b64(&self, key: &[u8]) -> PathBuf {
1029 self.get_media_dir().join(encode_key(key))
1030 }
1031
1032 #[must_use]
1033 pub fn get_media_dir(&self) -> PathBuf {
1034 self.services
1035 .server
1036 .config
1037 .database_path
1038 .join("media")
1039 }
1040}
1041
1042#[implement(Service)]
1046#[inline]
1047fn storage_providers(&self) -> impl Iterator<Item = &Arc<Provider>> + Send + '_ {
1048 self.selected_providers(MediaConfig::reads_from)
1049}
1050
1051#[implement(Service)]
1055#[inline]
1056fn upload_providers(&self) -> impl Iterator<Item = &Arc<Provider>> + Send + '_ {
1057 self.selected_providers(MediaConfig::writes_to)
1058}
1059
1060#[implement(Service)]
1066fn selected_providers(
1067 &self,
1068 select: fn(&MediaConfig, &str) -> bool,
1069) -> impl Iterator<Item = &Arc<Provider>> + Send + '_ {
1070 let config = Arc::clone(&self.services.server.config);
1071
1072 self.services
1073 .storage
1074 .providers()
1075 .filter(move |provider| select(&config.media, &provider.name))
1076}
1077
1078#[inline]
1079#[must_use]
1080pub fn encode_key(key: &[u8]) -> String { general_purpose::URL_SAFE_NO_PAD.encode(key) }
1081
1082fn mtime_millis(object: &ObjectMeta) -> u64 {
1083 u64::try_from(object.last_modified.timestamp_millis()).unwrap_or(0)
1084}