1use std::{
9 net::IpAddr,
10 time::{Duration, SystemTime},
11};
12
13#[cfg(feature = "url_preview")]
14use reqwest::header::CONTENT_DISPOSITION;
15use reqwest::header::{ACCEPT_LANGUAGE, CONTENT_TYPE, COOKIE, HeaderValue, USER_AGENT};
16#[cfg(feature = "url_preview")]
17use ruma::Mxc;
18use serde::{Deserialize, Serialize};
19use tuwunel_core::{
20 Config, Err, Result,
21 config::media::UrlPreview,
22 debug, err, implement,
23 smallstr::SmallString,
24 utils::{
25 string::{contains_any, equals_any},
26 time::timepoint_from_now,
27 },
28};
29#[cfg(feature = "url_preview")]
30use tuwunel_core::{debug_warn, utils::random_string};
31#[cfg(feature = "url_preview")]
32use tuwunel_database::Txn;
33use url::{Host, Url};
34#[cfg(feature = "url_preview")]
35use webpage::OpengraphObject;
36
37#[cfg(feature = "url_preview")]
38use super::MXC_LENGTH;
39use super::Service;
40#[cfg(feature = "url_preview")]
41use crate::client::read_response_capped;
42
43type MediaType = SmallString<[u8; 32]>;
45
46#[derive(Debug, Default, Deserialize, Serialize)]
47pub struct UrlPreviewData {
48 #[serde(
49 default,
50 skip_serializing_if = "Option::is_none",
51 rename = "og:title"
52 )]
53 pub title: Option<String>,
54 #[serde(
55 default,
56 skip_serializing_if = "Option::is_none",
57 rename = "og:description"
58 )]
59 pub description: Option<String>,
60 #[serde(
61 default,
62 skip_serializing_if = "Option::is_none",
63 rename = "og:image"
64 )]
65 pub image: Option<String>,
66 #[serde(
67 default,
68 skip_serializing_if = "Option::is_none",
69 rename = "matrix:image:size"
70 )]
71 pub image_size: Option<usize>,
72 #[serde(
73 default,
74 skip_serializing_if = "Option::is_none",
75 rename = "og:image:width"
76 )]
77 pub image_width: Option<u32>,
78 #[serde(
79 default,
80 skip_serializing_if = "Option::is_none",
81 rename = "og:image:height"
82 )]
83 pub image_height: Option<u32>,
84 #[serde(
85 default,
86 skip_serializing_if = "Option::is_none",
87 rename = "og:video"
88 )]
89 pub video: Option<String>,
90 #[serde(
91 default,
92 skip_serializing_if = "Option::is_none",
93 rename = "og:video:type"
94 )]
95 pub video_type: Option<MediaType>,
96 #[serde(
97 default,
98 skip_serializing_if = "Option::is_none",
99 rename = "matrix:video:size"
100 )]
101 pub video_size: Option<usize>,
102 #[serde(
103 default,
104 skip_serializing_if = "Option::is_none",
105 rename = "og:video:width"
106 )]
107 pub video_width: Option<u32>,
108 #[serde(
109 default,
110 skip_serializing_if = "Option::is_none",
111 rename = "og:video:height"
112 )]
113 pub video_height: Option<u32>,
114 #[serde(
115 default,
116 skip_serializing_if = "Option::is_none",
117 rename = "og:audio"
118 )]
119 pub audio: Option<String>,
120 #[serde(
121 default,
122 skip_serializing_if = "Option::is_none",
123 rename = "matrix:audio:size"
124 )]
125 pub audio_size: Option<usize>,
126 #[serde(
127 default,
128 skip_serializing_if = "Option::is_none",
129 rename = "og:type"
130 )]
131 pub og_type: Option<String>,
132 #[serde(
133 default,
134 skip_serializing_if = "Option::is_none",
135 rename = "og:url"
136 )]
137 pub og_url: Option<String>,
138}
139
140#[derive(Debug, Deserialize, Serialize)]
141pub(super) struct CachedPreview {
142 pub(super) preview: UrlPreviewData,
143 pub(super) expire: SystemTime,
144}
145
146impl CachedPreview {
147 fn new(ttl: Duration, preview: UrlPreviewData) -> Result<Self> {
148 timepoint_from_now(ttl).map(|expire| Self { preview, expire })
149 }
150
151 #[inline]
152 #[must_use]
153 pub(super) fn valid(&self) -> bool { self.expire > SystemTime::now() }
154}
155
156#[derive(Clone, Copy)]
161pub(super) enum Agent {
162 Page,
163 Media,
164}
165
166const YOUTUBE_HOSTS: [&str; 5] = [
169 "youtu.be",
170 "youtube.com",
171 "www.youtube.com",
172 "m.youtube.com",
173 "music.youtube.com",
174];
175
176const YOUTUBE_CONSENT_COOKIE: &str = "SOCS=CAI; CONSENT=PENDING+999";
182
183#[cfg(feature = "url_preview")]
186const YOUTUBE_OEMBED: &str = "https://www.youtube.com/oembed";
187
188#[cfg(feature = "url_preview")]
191const OEMBED_MAX_SIZE: usize = 64 * 1024;
192
193#[cfg(feature = "url_preview")]
198#[derive(Deserialize)]
199struct Oembed {
200 #[serde(rename = "type")]
201 kind: Option<String>,
202 title: Option<String>,
203 author_name: Option<String>,
204 thumbnail_url: Option<String>,
205}
206
207#[implement(Service)]
208pub async fn get_url_preview(&self, url: &Url) -> Result<UrlPreviewData> {
209 if let Ok(cached) = self.db.get_url_preview(url.as_str()).await {
210 return Ok(cached.preview);
211 }
212
213 let _request_lock = self.url_preview_mutex.lock(url.as_str()).await;
215
216 match self.db.get_url_preview(url.as_str()).await {
217 | Ok(cached) => Ok(cached.preview),
218 | Err(_) => self.request_url_preview(url).await,
219 }
220}
221
222#[implement(Service)]
223pub async fn request_url_preview(&self, url: &Url) -> Result<UrlPreviewData> {
224 self.check_url_host(url)?;
225
226 let response = self.preview_get(url, Agent::Page).send().await?;
227
228 debug!(?url, "URL preview response headers: {:?}", response.headers());
229
230 self.check_remote_addr(&response)?;
231
232 let status = response.status();
238 let (response, via_media_client) = if status.is_success() {
239 (response, false)
240 } else if self
241 .services
242 .config
243 .media
244 .url_preview
245 .media_user_agent
246 .is_some()
247 {
248 (self.media_response(url).await?, true)
249 } else {
250 return Err!(Request(NotFound(debug_warn!(
251 ?status,
252 %url,
253 "URL preview request failed"
254 ))));
255 };
256
257 let content_type = response
258 .headers()
259 .get(CONTENT_TYPE)
260 .ok_or_else(|| err!(Request(Unknown("Missing Content-Type header"))))?
261 .to_str()
262 .map_err(|e| err!(Request(Unknown("Invalid Content-Type header: {e}"))))?
263 .to_owned();
264
265 let data = match content_type.as_str() {
266 | html if html.starts_with("text/html") => {
267 if via_media_client {
270 return Err!(Request(NotFound(debug_warn!(
271 ?status,
272 %url,
273 "URL preview request failed"
274 ))));
275 }
276
277 let data = self.download_html(url, response).await?;
278
279 self.oembed_recover(url, data).await
280 },
281 | img if img.starts_with("image/") => {
282 let response = self
283 .media_refetch(response, via_media_client)
284 .await?;
285
286 require_media_type(&response, "image/")?;
287 self.download_image(response).await?
288 },
289 | video if video.starts_with("video/") => {
290 let response = self
291 .media_refetch(response, via_media_client)
292 .await?;
293
294 require_media_type(&response, "video/")?;
295 self.download_video(response).await?
296 },
297 | audio if audio.starts_with("audio/") => {
298 let response = self
299 .media_refetch(response, via_media_client)
300 .await?;
301
302 require_media_type(&response, "audio/")?;
303 self.download_audio(response).await?
304 },
305 | _ => return Err!(Request(Unknown("Unsupported Content-Type"))),
306 };
307
308 let ttl = Duration::from_secs(self.services.config.media.url_preview.cache_ttl);
309 let cached = CachedPreview::new(ttl, data)?;
310
311 self.db.set_url_preview(url.as_str(), &cached)?;
312
313 Ok(cached.preview)
314}
315
316#[implement(Service)]
319fn preview_get(&self, url: &Url, agent: Agent) -> reqwest::RequestBuilder {
320 let request = self.services.client.url_preview.get(url.as_str());
321
322 self.preview_headers(request, url, agent)
323}
324
325#[implement(Service)]
333pub(super) fn preview_headers(
334 &self,
335 request: reqwest::RequestBuilder,
336 url: &Url,
337 agent: Agent,
338) -> reqwest::RequestBuilder {
339 let config: &Config = &self.services.config;
340 let default_user_agent = config.media.url_preview.user_agent.as_deref();
341 let user_agent = match agent {
342 | Agent::Page => default_user_agent,
343 | Agent::Media => config
344 .media
345 .url_preview
346 .media_user_agent
347 .as_deref()
348 .or(default_user_agent),
349 };
350
351 let request = match user_agent {
352 | Some(user_agent) => request.header(USER_AGENT, user_agent),
353 | None => request,
354 };
355
356 let request = match config
357 .media
358 .url_preview
359 .accept_language
360 .as_deref()
361 {
362 | Some(accept_language) => request.header(ACCEPT_LANGUAGE, accept_language),
363 | None => request,
364 };
365
366 match is_youtube(url) {
369 | true => request.header(COOKIE, HeaderValue::from_static(YOUTUBE_CONSENT_COOKIE)),
370 | false => request,
371 }
372}
373
374#[must_use]
375fn is_youtube(url: &Url) -> bool {
376 url.host_str()
377 .is_some_and(|host| YOUTUBE_HOSTS.contains(&host))
378}
379
380#[implement(Service)]
384fn check_remote_addr(&self, response: &reqwest::Response) -> Result {
385 let Some(remote_addr) = response.remote_addr() else {
386 return Err!(Request(Forbidden("URL preview response has no peer address")));
387 };
388
389 debug!(url = %response.url(), ?remote_addr, "URL preview response remote address");
390
391 self.services
392 .client
393 .valid_cidr_range_remote_addr(response.url(), remote_addr)
394 .then_some(())
395 .ok_or_else(|| err!(Request(Forbidden("Requesting from this address is forbidden"))))
396}
397
398#[cfg(feature = "url_preview")]
406#[implement(Service)]
407async fn oembed_recover(&self, url: &Url, data: UrlPreviewData) -> UrlPreviewData {
408 if data.title.is_some() || data.image.is_some() {
410 return data;
411 }
412
413 let Some(endpoint) = oembed_endpoint(url) else {
414 return data;
415 };
416
417 self.oembed_preview(&endpoint, url)
418 .await
419 .inspect_err(|e| debug!(%url, %e, "oEmbed recovery failed"))
420 .unwrap_or(data)
421}
422
423#[cfg(not(feature = "url_preview"))]
424#[implement(Service)]
425#[expect(clippy::unused_async)]
426async fn oembed_recover(&self, _url: &Url, data: UrlPreviewData) -> UrlPreviewData { data }
427
428#[cfg(feature = "url_preview")]
430fn oembed_endpoint(url: &Url) -> Option<Url> {
431 is_youtube(url)
432 .then(|| {
433 Url::parse_with_params(YOUTUBE_OEMBED, [("url", url.as_str()), ("format", "json")])
434 })
435 .and_then(Result::ok)
436}
437
438#[cfg(feature = "url_preview")]
443#[implement(Service)]
444async fn oembed_preview(&self, endpoint: &Url, page: &Url) -> Result<UrlPreviewData> {
445 if !self.url_preview_allowed(endpoint) {
448 return Err!(Request(Forbidden(debug_warn!(
449 %endpoint,
450 "oEmbed endpoint is not allowed for previewing"
451 ))));
452 }
453
454 self.check_url_host(endpoint)?;
455
456 let response = self
457 .preview_get(endpoint, Agent::Page)
458 .send()
459 .await?;
460
461 self.check_remote_addr(&response)?;
462
463 let status = response.status();
464
465 if !status.is_success() {
466 return Err!(Request(NotFound(debug_warn!(
467 ?status,
468 %endpoint,
469 "oEmbed request failed"
470 ))));
471 }
472
473 let body = read_response_capped(response, OEMBED_MAX_SIZE).await?;
474 let oembed: Oembed = serde_json::from_slice(&body)
475 .map_err(|e| err!(Request(Unknown("Invalid oEmbed document: {e}"))))?;
476
477 let image = self
480 .oembed_image(oembed.thumbnail_url.as_deref())
481 .await;
482
483 Ok(UrlPreviewData {
484 title: oembed.title,
485 description: oembed.author_name,
486 video_type: video_type(oembed.kind.as_deref()).map(Into::into),
487 og_type: og_type(oembed.kind.as_deref()),
488 og_url: Some(page.as_str().to_owned()),
489 ..image
490 })
491}
492
493#[cfg(feature = "url_preview")]
499fn video_type(kind: Option<&str>) -> Option<&'static str> {
500 kind.eq(&Some("video")).then_some("text/html")
501}
502
503#[cfg(feature = "url_preview")]
510fn og_type(kind: Option<&str>) -> Option<String> {
511 kind.map(|kind| match kind {
512 | "video" => "video.other",
513 | _ => "website",
514 })
515 .map(ToOwned::to_owned)
516}
517
518#[cfg(feature = "url_preview")]
524#[implement(Service)]
525async fn oembed_image(&self, thumbnail_url: Option<&str>) -> UrlPreviewData {
526 let Some(thumbnail) = thumbnail_url
527 .and_then(|thumbnail| Url::parse(thumbnail).ok())
528 .filter(|thumbnail| ["http", "https"].contains(&thumbnail.scheme()))
529 else {
530 return UrlPreviewData::default();
531 };
532
533 self.preview_image(&thumbnail)
534 .await
535 .unwrap_or_default()
536}
537
538#[cfg(feature = "url_preview")]
545#[implement(Service)]
546async fn preview_image(&self, image_url: &Url) -> Result<UrlPreviewData> {
547 self.check_url_host(image_url)?;
548
549 let response = self
550 .preview_get(image_url, Agent::Media)
551 .send()
552 .await?;
553
554 self.check_remote_addr(&response)?;
555
556 if !response.status().is_success() {
559 debug!(
560 %image_url,
561 status = ?response.status(),
562 "Skipping preview image with unsuccessful response"
563 );
564
565 return Ok(UrlPreviewData::default());
566 }
567
568 self.download_image(response).await
569}
570
571#[cfg(feature = "url_preview")]
576#[implement(Service)]
577pub async fn download_image(&self, response: reqwest::Response) -> Result<UrlPreviewData> {
578 use image::ImageReader;
579
580 let url = response.url().clone();
583 let content_type = response
584 .headers()
585 .get(CONTENT_TYPE)
586 .and_then(|value| value.to_str().ok())
587 .map(ToOwned::to_owned);
588
589 let content_disposition = response
590 .headers()
591 .get(CONTENT_DISPOSITION)
592 .and_then(|value| value.to_str().ok())
593 .map(ToOwned::to_owned);
594
595 let limit = self
596 .services
597 .config
598 .media
599 .url_preview
600 .max_media_size;
601
602 let image = read_response_capped(response, limit).await?;
603
604 let cursor = std::io::Cursor::new(&image);
605 let (width, height) = match ImageReader::new(cursor).with_guessed_format() {
606 | Err(_) => (None, None),
607 | Ok(reader) => match reader.into_dimensions() {
608 | Err(_) => (None, None),
609 | Ok((width, height)) => (Some(width), Some(height)),
610 },
611 };
612
613 let mut txn = self.services.db.txn();
614 let mxc = self.queue_lazy_media(&mut txn, url.as_str());
615
616 self.db.set_lazy_content(
617 &mut txn,
618 &mxc,
619 content_type.as_deref(),
620 content_disposition.as_deref(),
621 &image,
622 );
623
624 txn.execute();
625
626 Ok(UrlPreviewData {
627 image: Some(mxc),
628 image_size: Some(image.len()),
629 image_width: width,
630 image_height: height,
631 ..Default::default()
632 })
633}
634
635#[cfg(not(feature = "url_preview"))]
640#[implement(Service)]
641#[expect(clippy::unused_async)]
642pub async fn download_image(&self, _response: reqwest::Response) -> Result<UrlPreviewData> {
643 Err!(FeatureDisabled("url_preview"))
644}
645
646#[cfg(feature = "url_preview")]
651#[implement(Service)]
652async fn media_response(&self, url: &Url) -> Result<reqwest::Response> {
653 let response = self.preview_get(url, Agent::Media).send().await?;
654
655 self.check_remote_addr(&response)?;
656
657 if !response.status().is_success() {
658 return Err!(Request(NotFound(debug_warn!(
659 status = ?response.status(),
660 %url,
661 "URL preview media request failed"
662 ))));
663 }
664
665 Ok(response)
666}
667
668#[cfg(not(feature = "url_preview"))]
669#[implement(Service)]
670#[expect(clippy::unused_async)]
671async fn media_response(&self, _url: &Url) -> Result<reqwest::Response> {
672 Err!(FeatureDisabled("url_preview"))
673}
674
675#[implement(Service)]
686async fn media_refetch(
687 &self,
688 response: reqwest::Response,
689 via_media_client: bool,
690) -> Result<reqwest::Response> {
691 if via_media_client
692 || self
693 .services
694 .config
695 .media
696 .url_preview
697 .media_user_agent
698 .is_none()
699 {
700 return Ok(response);
701 }
702
703 let url = response.url().clone();
704 self.check_url_host(&url)?;
705 self.media_response(&url).await
706}
707
708fn require_media_type(response: &reqwest::Response, class: &str) -> Result {
712 response
713 .headers()
714 .get(CONTENT_TYPE)
715 .and_then(|value| value.to_str().ok())
716 .is_some_and(|content_type| content_type.starts_with(class))
717 .then_some(())
718 .ok_or_else(|| err!(Request(Unknown("Unsupported Content-Type"))))
719}
720
721#[cfg(feature = "url_preview")]
725#[implement(Service)]
726fn register_lazy_media(&self, url: &str) -> String {
727 let mxc = self.mint_lazy_media();
728
729 self.db.insert_lazy_media(&mxc, url);
730
731 mxc
732}
733
734#[cfg(feature = "url_preview")]
735#[implement(Service)]
736fn queue_lazy_media(&self, txn: &mut Txn, url: &str) -> String {
737 let mxc = self.mint_lazy_media();
738
739 self.db.queue_lazy_media(txn, &mxc, url);
740
741 mxc
742}
743
744#[cfg(feature = "url_preview")]
745#[implement(Service)]
746fn mint_lazy_media(&self) -> String {
747 Mxc {
748 server_name: self.services.globals.server_name(),
749 media_id: &random_string(MXC_LENGTH),
750 }
751 .to_string()
752}
753
754#[cfg(feature = "url_preview")]
755#[implement(Service)]
756#[expect(clippy::unused_async)]
757pub async fn download_video(&self, response: reqwest::Response) -> Result<UrlPreviewData> {
758 let limit = self
759 .services
760 .config
761 .media
762 .url_preview
763 .max_media_size;
764
765 let video_size = checked_media_size(&response, limit)?;
766
767 Ok(UrlPreviewData {
768 video: Some(self.register_lazy_media(response.url().as_str())),
769 video_size,
770 ..Default::default()
771 })
772}
773
774#[cfg(not(feature = "url_preview"))]
775#[implement(Service)]
776#[expect(clippy::unused_async)]
777pub async fn download_video(&self, _response: reqwest::Response) -> Result<UrlPreviewData> {
778 Err!(FeatureDisabled("url_preview"))
779}
780
781#[cfg(feature = "url_preview")]
782#[implement(Service)]
783#[expect(clippy::unused_async)]
784pub async fn download_audio(&self, response: reqwest::Response) -> Result<UrlPreviewData> {
785 let limit = self
786 .services
787 .config
788 .media
789 .url_preview
790 .max_media_size;
791
792 let audio_size = checked_media_size(&response, limit)?;
793
794 Ok(UrlPreviewData {
795 audio: Some(self.register_lazy_media(response.url().as_str())),
796 audio_size,
797 ..Default::default()
798 })
799}
800
801#[cfg(not(feature = "url_preview"))]
802#[implement(Service)]
803#[expect(clippy::unused_async)]
804pub async fn download_audio(&self, _response: reqwest::Response) -> Result<UrlPreviewData> {
805 Err!(FeatureDisabled("url_preview"))
806}
807
808#[cfg(feature = "url_preview")]
811fn checked_media_size(response: &reqwest::Response, limit: usize) -> Result<Option<usize>> {
812 let size = response
813 .content_length()
814 .and_then(|len| usize::try_from(len).ok());
815
816 if size.is_some_and(|size| size > limit) {
817 return Err!(Request(TooLarge("Media exceeds media.url_preview.max_media_size")));
818 }
819
820 Ok(size)
821}
822
823#[cfg(feature = "url_preview")]
824#[implement(Service)]
825async fn download_html(&self, url: &Url, response: reqwest::Response) -> Result<UrlPreviewData> {
826 use webpage::HTML;
827
828 let limit = self
829 .services
830 .config
831 .media
832 .url_preview
833 .max_spider_size;
834
835 let (bytes, truncated) = spider_body(response, limit).await?;
836
837 let body = String::from_utf8(bytes)
840 .unwrap_or_else(|e| String::from_utf8_lossy(e.as_bytes()).into_owned());
841
842 let Ok(html) = HTML::from_string(body, Some(url.as_str().to_owned())) else {
843 return Err!(Request(Unknown("Failed to parse HTML")));
844 };
845
846 let twitter = |key| {
849 html.meta
850 .get(key)
851 .map(String::as_str)
852 .filter(|content| !content.is_empty())
853 };
854
855 let image_url = html
858 .opengraph
859 .images
860 .first()
861 .map(|obj| obj.url.as_str())
862 .filter(|image| !image.is_empty())
863 .or_else(|| twitter("twitter:image"))
864 .or_else(|| twitter("twitter:image:src"))
865 .map(|image| url.join(image))
866 .transpose()
867 .map_err(|e| err!(Request(Unknown("Invalid preview image URL: {e}"))))?
868 .filter(|image_url| ["http", "https"].contains(&image_url.scheme()));
869
870 let mut data = match image_url {
871 | None => UrlPreviewData::default(),
872 | Some(image_url) => self.preview_image(&image_url).await?,
873 };
874
875 if let Some(obj) = html.opengraph.videos.first()
876 && !obj.url.is_empty()
877 {
878 data.video_type = obj
880 .properties
881 .get("type")
882 .map(String::as_str)
883 .map(Into::into);
884
885 data.video_width = obj
886 .properties
887 .get("width")
888 .and_then(|w| w.parse().ok());
889
890 data.video_height = obj
891 .properties
892 .get("height")
893 .and_then(|h| h.parse().ok());
894
895 data.video = self.lazy_media(url, obj, "video/");
896 }
897
898 if let Some(obj) = html.opengraph.audios.first()
899 && !obj.url.is_empty()
900 {
901 data.audio = self.lazy_media(url, obj, "audio/");
902 }
903
904 let props = html.opengraph.properties;
905
906 data.title = props
907 .get("title")
908 .cloned()
909 .filter(|title| !title.is_empty())
910 .or_else(|| twitter("twitter:title").map(ToOwned::to_owned))
911 .or(html.title);
912
913 data.description = props
914 .get("description")
915 .cloned()
916 .filter(|description| !description.is_empty())
917 .or_else(|| twitter("twitter:description").map(ToOwned::to_owned))
918 .or(html.description);
919
920 data.og_type = Some(html.opengraph.og_type);
921 data.og_url = props.get("url").cloned();
922
923 if truncated && data.title.is_none() && data.description.is_none() && data.image.is_none() {
926 debug_warn!(
927 %url,
928 %limit,
929 "Preview page was truncated before any metadata was found; a larger \
930 media.url_preview.max_spider_size or a different media.url_preview.user_agent may be needed"
931 );
932 }
933
934 Ok(data)
935}
936
937#[cfg(not(feature = "url_preview"))]
938#[implement(Service)]
939#[expect(clippy::unused_async)]
940async fn download_html(
941 &self,
942 _url: &Url,
943 _response: reqwest::Response,
944) -> Result<UrlPreviewData> {
945 Err!(FeatureDisabled("url_preview"))
946}
947
948#[cfg(feature = "url_preview")]
954async fn spider_body(mut response: reqwest::Response, limit: usize) -> Result<(Vec<u8>, bool)> {
955 let hint = response
956 .content_length()
957 .and_then(|len| usize::try_from(len).ok())
958 .map_or(0, |len| len.min(limit));
959
960 let mut bytes: Vec<u8> = Vec::with_capacity(hint);
961
962 while let Some(chunk) = response.chunk().await? {
963 let want = chunk.len().min(limit.saturating_sub(bytes.len()));
964
965 reserve_capped(&mut bytes, want, limit);
966 bytes.extend_from_slice(&chunk[..want]);
967
968 if want < chunk.len() {
969 return Ok((bytes, true));
970 }
971 }
972
973 Ok((bytes, false))
974}
975
976#[cfg(feature = "url_preview")]
982fn reserve_capped(bytes: &mut Vec<u8>, want: usize, limit: usize) {
983 let need = bytes.len().saturating_add(want);
984
985 if need <= bytes.capacity() {
986 return;
987 }
988
989 let target = bytes
990 .capacity()
991 .saturating_mul(2)
992 .clamp(need, limit.max(need));
993
994 bytes.reserve_exact(target.saturating_sub(bytes.len()));
995}
996
997#[cfg(feature = "url_preview")]
1006#[implement(Service)]
1007fn lazy_media(&self, page: &Url, obj: &OpengraphObject, class: &str) -> Option<String> {
1008 declares_media_type(obj, class)
1009 .then(|| page.join(&obj.url).ok())
1010 .flatten()
1011 .filter(|url| ["http", "https"].contains(&url.scheme()))
1012 .filter(|url| self.check_url_host(url).is_ok())
1013 .map(|url| self.register_lazy_media(url.as_str()))
1014}
1015
1016#[cfg(feature = "url_preview")]
1022fn declares_media_type(obj: &OpengraphObject, class: &str) -> bool {
1023 obj.properties
1024 .get("type")
1025 .is_none_or(|kind| kind.starts_with(class))
1026}
1027
1028#[implement(Service)]
1029pub(super) fn check_url_host(&self, url: &Url) -> Result {
1030 if self.services.client.proxy.resolver_alias(url) {
1031 return Err!(Request(Forbidden(
1032 "Requesting a locally resolved proxy endpoint is forbidden"
1033 )));
1034 }
1035
1036 let host = url
1037 .host()
1038 .ok_or_else(|| err!(Request(Unknown("URL has no host"))))?;
1039
1040 let ip = match host {
1041 | Host::Domain(_) => return Ok(()),
1042 | Host::Ipv4(v4) => IpAddr::V4(v4),
1043 | Host::Ipv6(v6) => IpAddr::V6(v6),
1044 };
1045
1046 if !self.services.client.valid_cidr_range_ip(ip) {
1047 return Err!(Request(Forbidden("Requesting from this address is forbidden")));
1048 }
1049
1050 Ok(())
1051}
1052
1053#[implement(Service)]
1057#[inline]
1058pub fn url_preview_enabled(&self) -> bool {
1059 cfg!(feature = "url_preview") && enabled(&self.services.config.media.url_preview)
1060}
1061
1062fn enabled(config: &UrlPreview) -> bool {
1063 !config.domain_contains_allowlist.is_empty()
1064 || !config.domain_explicit_allowlist.is_empty()
1065 || !config.url_contains_allowlist.is_empty()
1066}
1067
1068#[implement(Service)]
1075#[inline]
1076pub fn url_preview_allowed(&self, url: &Url) -> bool {
1077 cfg!(feature = "url_preview") && url_allowed(&self.services.config.media.url_preview, url)
1078}
1079
1080fn url_allowed(url_preview: &UrlPreview, url: &Url) -> bool {
1081 if ["http", "https"]
1082 .iter()
1083 .all(|&scheme| !scheme.eq_ignore_ascii_case(url.scheme()))
1084 {
1085 debug!(%url, "Ignoring a non-HTTP/HTTPS URL for preview");
1086 return false;
1087 }
1088
1089 let Some(host) = url.host_str() else {
1090 debug!(%url, "Ignoring URL preview for a URL without a host");
1091 return false;
1092 };
1093
1094 let UrlPreview {
1095 domain_contains_allowlist,
1096 domain_explicit_allowlist,
1097 domain_explicit_denylist,
1098 url_contains_allowlist,
1099 check_root_domain,
1100 ..
1101 } = url_preview;
1102
1103 if equals_any("*", domain_contains_allowlist)
1104 || equals_any("*", domain_explicit_allowlist)
1105 || equals_any("*", url_contains_allowlist)
1106 {
1107 debug!(%url, "An allowlist holding \"*\" allows every URL preview");
1108 return true;
1109 }
1110
1111 if host.is_empty() {
1112 return false;
1113 }
1114
1115 if equals_any(host, domain_explicit_denylist) {
1116 debug!(%host, "Host is not allowed by media.url_preview.domain_explicit_denylist (check \
1117 1/4)");
1118
1119 return false;
1120 }
1121
1122 if equals_any(host, domain_explicit_allowlist) {
1123 debug!(%host, "Host is allowed by media.url_preview.domain_explicit_allowlist (check 2/4)");
1124
1125 return true;
1126 }
1127
1128 if contains_any(host, domain_contains_allowlist) {
1129 debug!(%host, "Host is allowed by media.url_preview.domain_contains_allowlist (check 3/4)");
1130
1131 return true;
1132 }
1133
1134 if url_contains_allowlist
1137 .iter()
1138 .any(|entry| url.as_str().contains(entry.as_str()))
1139 {
1140 debug!(%host, "URL is allowed by media.url_preview.url_contains_allowlist (check 4/4)");
1141
1142 return true;
1143 }
1144
1145 if !check_root_domain {
1146 return false;
1147 }
1148
1149 debug!("Checking root domain");
1150 let Some((_, root_domain)) = host.split_once('.') else {
1151 return false;
1152 };
1153
1154 if equals_any(root_domain, domain_explicit_denylist) {
1155 debug!(%root_domain, "Root domain is not allowed by \
1156 media.url_preview.domain_explicit_denylist (check 1/2)");
1157
1158 return false;
1159 }
1160
1161 if equals_any(root_domain, domain_explicit_allowlist) {
1162 debug!(%root_domain, "Root domain is allowed by \
1163 media.url_preview.domain_explicit_allowlist (check 2/2)");
1164
1165 return true;
1166 }
1167
1168 false
1169}
1170
1171#[cfg(test)]
1172mod tests {
1173 use std::time::Duration;
1174
1175 use minicbor_serde::{from_slice, to_vec};
1176 use tuwunel_core::utils::time::timepoint_ago;
1177 use url::Url;
1178
1179 use super::{CachedPreview, UrlPreview, UrlPreviewData, enabled, is_youtube, url_allowed};
1180 #[cfg(feature = "url_preview")]
1181 use super::{oembed_endpoint, reserve_capped, video_type};
1182
1183 const TTL: Duration = Duration::from_hours(24);
1184
1185 #[test]
1186 fn enabled_by_any_allowlist() {
1187 assert!(!enabled(&UrlPreview::default()));
1188 let denied = UrlPreview {
1189 domain_explicit_denylist: vec!["example.org".to_owned()],
1190 ..Default::default()
1191 };
1192
1193 assert!(!enabled(&denied));
1194 let entry = || vec!["example.org".to_owned()];
1195
1196 for (domain_contains_allowlist, domain_explicit_allowlist, url_contains_allowlist) in
1197 [(entry(), vec![], vec![]), (vec![], entry(), vec![]), (vec![], vec![], entry())]
1198 {
1199 let config = UrlPreview {
1200 domain_contains_allowlist,
1201 domain_explicit_allowlist,
1202 url_contains_allowlist,
1203 ..Default::default()
1204 };
1205
1206 assert!(enabled(&config));
1207 }
1208 }
1209
1210 #[test]
1211 fn url_allowed_by_the_preview_lists() {
1212 let base = UrlPreview {
1213 domain_contains_allowlist: vec!["google.com".to_owned()],
1214 domain_explicit_allowlist: vec!["wikipedia.org".to_owned()],
1215 domain_explicit_denylist: vec!["ads.google.com".to_owned()],
1216 url_contains_allowlist: vec!["example.org/allowed/".to_owned()],
1217 check_root_domain: true,
1218 ..Default::default()
1219 };
1220
1221 let no_root = UrlPreview { check_root_domain: false, ..base.clone() };
1222
1223 let root_denied = UrlPreview {
1224 domain_explicit_denylist: vec!["wikipedia.org".to_owned()],
1225 ..base.clone()
1226 };
1227
1228 let wildcard = UrlPreview {
1229 url_contains_allowlist: vec!["*".to_owned()],
1230 ..base.clone()
1231 };
1232
1233 let empty_url_entry = UrlPreview {
1234 url_contains_allowlist: vec![String::new()],
1235 ..base.clone()
1236 };
1237
1238 let cases = [
1239 ("base", &base, "https://google.com/", true),
1240 ("base", &base, "https://www.google.com/search", true),
1241 ("base", &base, "https://oogle.com/", false),
1242 ("base", &base, "https://ads.google.com/", false),
1243 ("base", &base, "ftp://google.com/", false),
1244 ("base", &base, "https://example.org/allowed/page", true),
1245 ("base", &base, "https://example.org/other", false),
1246 ("base", &base, "https://wikipedia.org/", true),
1247 ("base", &base, "https://en.wikipedia.org/", true),
1248 ("base", &base, "https://en.m.wikipedia.org/", false),
1249 ("no_root", &no_root, "https://en.wikipedia.org/", false),
1250 ("root_denied", &root_denied, "https://en.wikipedia.org/", false),
1251 ("wildcard", &wildcard, "https://ads.google.com/", true),
1252 ("empty_url_entry", &empty_url_entry, "https://unlisted.example/", true),
1253 ("empty_url_entry", &empty_url_entry, "https://ads.google.com/", false),
1254 ];
1255
1256 for (name, url_preview, url, allowed) in cases {
1257 let url = Url::parse(url).expect("a valid URL");
1258
1259 assert_eq!(url_allowed(url_preview, &url), allowed, "{name}: {url}");
1260 }
1261 }
1262
1263 #[test]
1264 fn cached_preview_roundtrip() {
1265 let cached = CachedPreview::new(TTL, sample()).expect("representable");
1266 let bytes = to_vec(&cached).expect("encodes");
1267 let decoded: CachedPreview = from_slice(&bytes).expect("decodes");
1268
1269 assert_eq!(
1270 serde_json::to_value(&decoded.preview).expect("json"),
1271 serde_json::to_value(&cached.preview).expect("json"),
1272 );
1273 assert_eq!(decoded.preview.image_size, Some(0xFF01));
1274 assert_eq!(decoded.preview.image_height, Some(0xFF));
1275 assert_eq!(decoded.expire, cached.expire);
1276 }
1277
1278 #[test]
1279 fn preview_wire_keys_unchanged() {
1280 let value = serde_json::to_value(sample()).expect("json");
1281 let object = value.as_object().expect("object");
1282
1283 assert!(object.contains_key("og:title"));
1284 assert!(object.contains_key("matrix:image:size"));
1285 assert!(object.contains_key("og:video:width"));
1286 assert!(object.contains_key("og:video:type"));
1287 assert!(object.contains_key("og:url"));
1288 assert!(!object.contains_key("title"));
1289
1290 let empty = serde_json::to_value(UrlPreviewData::default()).expect("json");
1291 assert!(empty.as_object().expect("object").is_empty());
1292 }
1293
1294 fn sample() -> UrlPreviewData {
1295 UrlPreviewData {
1296 title: Some("Title".to_owned()),
1297 description: Some("Description".to_owned()),
1298 image: Some("mxc://example.org/image".to_owned()),
1299 image_size: Some(0xFF01),
1301 image_width: Some(640),
1302 image_height: Some(0xFF),
1303 video: Some("mxc://example.org/video".to_owned()),
1304 video_type: Some("video/mp4".into()),
1305 video_size: Some(123_456),
1306 video_width: Some(1920),
1307 video_height: Some(1080),
1308 audio: Some("mxc://example.org/audio".to_owned()),
1309 audio_size: Some(4096),
1310 og_type: Some("website".to_owned()),
1311 og_url: Some("https://example.org/".to_owned()),
1312 }
1313 }
1314
1315 #[test]
1316 fn preview_cbor_missing_fields_default() {
1317 let sparse = UrlPreviewData {
1318 title: Some("Only a title".to_owned()),
1319 ..Default::default()
1320 };
1321
1322 let bytes = to_vec(&sparse).expect("encodes");
1323 let decoded: UrlPreviewData = from_slice(&bytes).expect("decodes");
1324
1325 assert_eq!(decoded.title.as_deref(), Some("Only a title"));
1326 assert!(decoded.description.is_none());
1327 assert!(decoded.image.is_none());
1328 assert!(decoded.og_url.is_none());
1329 }
1330
1331 #[test]
1332 fn preview_cbor_unknown_key_skipped() {
1333 #[derive(serde::Serialize)]
1334 struct Superset {
1335 #[serde(rename = "og:title")]
1336 title: &'static str,
1337 #[serde(rename = "og:unknown")]
1338 unknown: &'static str,
1339 }
1340
1341 let bytes = to_vec(Superset { title: "Kept", unknown: "Discarded" }).expect("encodes");
1342 let decoded: UrlPreviewData = from_slice(&bytes).expect("decodes");
1343
1344 assert_eq!(decoded.title.as_deref(), Some("Kept"));
1345 assert!(decoded.description.is_none());
1346 }
1347
1348 #[test]
1349 fn cached_preview_expiry() {
1350 let cached = CachedPreview::new(TTL, UrlPreviewData::default()).expect("representable");
1351
1352 assert!(cached.valid());
1353
1354 let expired = CachedPreview {
1355 preview: UrlPreviewData::default(),
1356 expire: timepoint_ago(Duration::from_secs(1)).expect("representable"),
1357 };
1358
1359 assert!(!expired.valid());
1360 }
1361
1362 #[test]
1363 fn cached_preview_honors_configured_lifetime() {
1364 let day = CachedPreview::new(TTL, UrlPreviewData::default()).expect("representable");
1365 let month =
1366 CachedPreview::new(TTL * 30, UrlPreviewData::default()).expect("representable");
1367
1368 assert!(month.expire > day.expire);
1369 }
1370
1371 #[test]
1372 fn cached_preview_unrepresentable_lifetime_refused() {
1373 let refused =
1374 CachedPreview::new(Duration::from_secs(u64::MAX), UrlPreviewData::default());
1375
1376 assert!(refused.is_err(), "a lifetime past representable time errors rather than panics");
1377 }
1378
1379 #[test]
1380 fn youtube_hosts_matched() {
1381 let youtube = [
1382 "https://www.youtube.com/watch?v=abc",
1383 "https://youtu.be/abc",
1384 "https://music.youtube.com/watch?v=abc",
1385 "https://m.youtube.com/watch?v=abc",
1386 "https://youtube.com/watch?v=abc",
1387 "https://WWW.YOUTUBE.COM/watch?v=abc",
1388 ];
1389
1390 for url in youtube {
1391 assert!(is_youtube(&Url::parse(url).expect("parses")), "{url}");
1392 }
1393
1394 let other = [
1396 "https://youtube.com.evil.example/watch?v=abc",
1397 "https://notyoutube.com/watch?v=abc",
1398 "https://i.ytimg.com/vi/abc/hqdefault.jpg",
1399 "https://example.org/",
1400 ];
1401
1402 for url in other {
1403 assert!(!is_youtube(&Url::parse(url).expect("parses")), "{url}");
1404 }
1405 }
1406
1407 #[cfg(feature = "url_preview")]
1408 #[test]
1409 fn oembed_endpoint_carries_the_page_url() {
1410 let url = Url::parse("https://www.youtube.com/watch?v=a&b=c").expect("parses");
1411 let endpoint = oembed_endpoint(&url).expect("youtube has an endpoint");
1412
1413 assert_eq!(endpoint.path(), "/oembed");
1414
1415 let params: Vec<_> = endpoint.query_pairs().collect();
1416 assert_eq!(params, [
1417 ("url".into(), url.as_str().into()),
1418 ("format".into(), "json".into())
1419 ]);
1420
1421 assert!(oembed_endpoint(&Url::parse("https://example.org/").expect("parses")).is_none());
1422 }
1423
1424 #[cfg(feature = "url_preview")]
1425 #[test]
1426 fn oembed_video_declares_a_player() {
1427 assert_eq!(video_type(Some("video")), Some("text/html"));
1428
1429 for kind in [Some("photo"), Some("rich"), Some("link"), None] {
1430 assert!(video_type(kind).is_none(), "{kind:?}");
1431 }
1432 }
1433
1434 #[cfg(feature = "url_preview")]
1435 #[test]
1436 fn reserve_capped_never_exceeds_the_cap() {
1437 const LIMIT: usize = 768 * 1024;
1438
1439 let mut bytes: Vec<u8> = Vec::new();
1440 let chunk = vec![0_u8; 16 * 1024];
1441 let mut reallocs = 0;
1442
1443 while bytes.len() < LIMIT {
1444 let want = chunk.len().min(LIMIT.saturating_sub(bytes.len()));
1445 let before = bytes.capacity();
1446
1447 reserve_capped(&mut bytes, want, LIMIT);
1448 bytes.extend_from_slice(&chunk[..want]);
1449
1450 if bytes.capacity() != before {
1451 reallocs += 1;
1452 }
1453
1454 assert!(bytes.capacity() <= LIMIT, "capacity {} past cap", bytes.capacity());
1455 }
1456
1457 assert_eq!(bytes.len(), LIMIT);
1458
1459 assert!(reallocs < 12, "{reallocs} reallocations");
1461 }
1462
1463 #[cfg(feature = "url_preview")]
1464 #[test]
1465 fn reserve_capped_honors_an_unclamped_request() {
1466 let mut bytes: Vec<u8> = Vec::new();
1467
1468 reserve_capped(&mut bytes, 64, 16);
1469
1470 assert!(bytes.capacity() >= 64);
1471 }
1472}