1pub mod local;
13
14#[cfg(feature = "gcs_storage")]
21pub mod gcs;
22
23#[cfg(feature = "s3_storage")]
29pub mod s3;
30
31#[cfg(test)]
32mod tests;
33
34use std::{
35 iter::{from_fn, once},
36 ops::Range,
37 sync::Arc,
38 time::Duration,
39};
40
41use bytes::Bytes;
42use derive_more::Debug;
43use futures::{FutureExt, Stream, StreamExt, TryFutureExt, TryStreamExt};
44#[cfg(any(feature = "gcs_storage", feature = "s3_storage"))]
45use http::Method;
46#[cfg(any(feature = "gcs_storage", feature = "s3_storage"))]
47use object_store::signer::Signer;
48use object_store::{
49 Attributes, CopyMode, DynObjectStore, GetResult, MultipartUpload, ObjectMeta, ObjectStore,
50 ObjectStoreExt, PutPayload, PutResult, path::Path,
51};
52use tuwunel_core::{
53 Error, Result,
54 config::StorageProvider,
55 debug, err, error,
56 error::error_chain,
57 extract_variant, implement, info, trace,
58 utils::{
59 BoolExt,
60 result::FlatOk,
61 stream::{IterStream, TryReadyExt},
62 },
63};
64use url::Url;
65
66#[derive(Debug)]
72pub struct Provider {
73 pub name: String,
75
76 pub config: StorageProvider,
78
79 pub(crate) provider: Box<DynObjectStore>,
81
82 #[cfg(any(feature = "gcs_storage", feature = "s3_storage"))]
83 #[debug(skip)]
84 pub(crate) signer: Option<Arc<dyn Signer>>,
86
87 pub(crate) base_path: Option<Path>,
89
90 startup_check: bool,
91}
92
93pub type FetchItem = (Bytes, (Range<u64>, u64));
99
100pub type FetchMetaItem = (Bytes, Arc<(Range<u64>, ObjectMeta, Attributes)>);
106
107#[implement(Provider)]
112#[tracing::instrument(skip_all, err)]
113pub(super) async fn start(self: &Arc<Self>) -> Result {
114 if self.startup_check {
115 self.startup_check().await?;
116 }
117
118 Ok(())
119}
120
121#[implement(Provider)]
122#[tracing::instrument(name = "check", skip_all, err)]
123async fn startup_check(self: &Arc<Self>) -> Result {
124 debug!(
125 name = ?self.name,
126 "Checking storage provider client connection...",
127 );
128 self.ping()
129 .inspect_ok(|()| {
130 info!(
131 name = %self.name,
132 "Connected to storage provider"
133 );
134 })
135 .await
136}
137
138#[implement(Provider)]
143#[tracing::instrument(
144 level = "debug",
145 err(level = "debug"),
146 skip_all,
147 fields(
148 provider = %self.name,
149 ?path,
150 ?size,
151 )
152)]
153pub async fn put<S, T>(&self, path: &str, size: Option<usize>, input: S) -> Result<PutResult>
154where
155 S: Stream<Item = Result<T>> + Send,
156 PutPayload: From<T> + From<PutPayload>,
157{
158 if size.is_none_or(|size| size >= self.multipart_threshold()) {
159 return self.put_multi(path, input).await;
160 }
161
162 debug!(
163 ?size,
164 threshold = ?self.multipart_threshold(),
165 "Selecting single-part upload..."
166 );
167
168 let payload: PutPayload = input
169 .map_ok(PutPayload::from)
170 .try_collect::<Vec<_>>()
171 .await?
172 .into_iter()
173 .map(Bytes::from)
174 .collect();
175
176 self.put_single(path, payload).await
177}
178
179#[implement(Provider)]
184#[tracing::instrument(
185 level = "debug",
186 err(level = "debug"),
187 skip_all,
188 fields(
189 provider = %self.name,
190 ?path,
191 )
192)]
193pub async fn put_one<T>(&self, path: &str, input: T) -> Result<PutResult>
194where
195 PutPayload: From<T> + From<PutPayload>,
196{
197 let payload: PutPayload = input.into();
198
199 if payload.content_length() < self.multipart_threshold() {
200 return self.put_single(path, payload).await;
201 }
202
203 let part_size = self.multipart_part_size();
204
205 debug!(
206 len = ?payload.content_length(),
207 threshold = ?self.multipart_threshold(),
208 ?part_size,
209 "Selecting multi-part upload..."
210 );
211
212 self.put_multi(path, chunked(payload, part_size).try_stream())
213 .await
214}
215
216#[implement(Provider)]
221#[tracing::instrument(
222 level = "debug",
223 err(level = "debug"),
224 skip_all,
225 fields(
226 provider = %self.name,
227 ?path,
228 )
229)]
230async fn put_multi<S, T>(&self, path: &str, input: S) -> Result<PutResult>
231where
232 S: Stream<Item = Result<T>> + Send,
233 PutPayload: From<T> + From<PutPayload>,
234{
235 let path = self.to_abs_path(path)?;
236 let mut handle = self
237 .provider
238 .put_multipart(&path)
239 .map_err(Error::from)
240 .await?;
241
242 match input
243 .try_for_each(|t| handle.put_part(t.into()).map_err(Error::from))
244 .inspect_err(|e| error!(?path, chain = %error_chain(e), "Failed to store object"))
245 .await
246 {
247 | Ok(()) =>
248 handle
249 .complete()
250 .map_err(Error::from)
251 .inspect_err(|e| {
252 error!(
253 ?path,
254 chain = %error_chain(e),
255 "Failed to store object during completion",
256 );
257 })
258 .await,
259
260 | Err(e) =>
261 handle
262 .abort()
263 .map_ok(|()| Err(e))
264 .map_err(Error::from)
265 .inspect_err(|e| {
266 error!(
267 ?path,
268 chain = %error_chain(e),
269 "Additional errors during error handling",
270 );
271 })
272 .await?,
273 }
274}
275
276#[implement(Provider)]
281#[tracing::instrument(
282 level = "debug",
283 err(level = "debug"),
284 skip_all,
285 fields(
286 provider = %self.name,
287 ?path,
288 )
289)]
290async fn put_single(&self, path: &str, input: PutPayload) -> Result<PutResult> {
291 let path = self.to_abs_path(path)?;
292
293 self.provider
294 .put(&path, input)
295 .map_err(Error::from)
296 .await
297}
298
299#[implement(Provider)]
305#[tracing::instrument(
306 level = "debug",
307 skip_all,
308 fields(
309 provider = %self.name,
310 ?path,
311 )
312)]
313pub fn fetch_with_metadata(
314 &self,
315 path: &str,
316) -> impl Stream<Item = Result<FetchMetaItem>> + Send {
317 self.load(path)
318 .map_ok(|result| {
319 let meta = (result.range.clone(), result.meta.clone(), result.attributes.clone());
320 let data = Arc::new(meta);
321
322 result
323 .into_stream()
324 .map_err(Error::from)
325 .map_ok(move |bytes| (bytes, data.clone()))
326 })
327 .map_err(Error::from)
328 .try_flatten_stream()
329}
330
331#[implement(Provider)]
337#[tracing::instrument(
338 level = "debug",
339 skip_all,
340 fields(
341 provider = %self.name,
342 ?path,
343 )
344)]
345pub fn fetch(&self, path: &str) -> impl Stream<Item = Result<FetchItem>> + Send {
346 self.load(path)
347 .map_ok(|result| {
348 let size = result.meta.size;
349 let range = result.range.clone();
350
351 result
352 .into_stream()
353 .map_err(Error::from)
354 .map_ok(move |bytes| (bytes, (range.clone(), size)))
355 })
356 .map_err(Error::from)
357 .try_flatten_stream()
358}
359
360#[implement(Provider)]
365#[tracing::instrument(
366 level = "debug",
367 err(level = "debug"),
368 skip_all,
369 fields(
370 provider = %self.name,
371 ?path,
372 )
373)]
374pub async fn get(&self, path: &str) -> Result<Bytes> {
375 self.load(path)
376 .map_ok(GetResult::bytes)
377 .await?
378 .map_err(Error::from)
379 .await
380}
381
382#[implement(Provider)]
387#[tracing::instrument(
388 level = "debug",
389 err(level = "debug"),
390 skip_all,
391 fields(
392 provider = %self.name,
393 ?path,
394 )
395)]
396pub async fn load(&self, path: &str) -> Result<GetResult> {
397 let path = self.to_abs_path(path)?;
398
399 self.provider
400 .get(&path)
401 .map_err(Error::from)
402 .await
403}
404
405#[cfg(any(feature = "gcs_storage", feature = "s3_storage"))]
410#[implement(Provider)]
411#[tracing::instrument(
412 level = "debug",
413 err(level = "debug"),
414 skip_all,
415 fields(
416 provider = %self.name,
417 ?path,
418 ?ttl,
419 )
420)]
421pub async fn signed_get_url(&self, path: &str, ttl: Duration) -> Result<Option<Url>> {
422 let Some(signer) = self.signer.as_ref() else {
423 return Ok(None);
424 };
425
426 let path = self.to_abs_path(path)?;
427
428 signer
429 .signed_url(Method::GET, &path, ttl)
430 .map_err(Error::from)
431 .map_ok(Some)
432 .await
433}
434
435#[cfg(not(any(feature = "gcs_storage", feature = "s3_storage")))]
440#[implement(Provider)]
441#[expect(clippy::unused_async)]
442pub async fn signed_get_url(&self, _path: &str, _ttl: Duration) -> Result<Option<Url>> {
443 Ok(None)
444}
445
446#[implement(Provider)]
451#[tracing::instrument(
452 level = "debug",
453 err(level = "debug"),
454 skip_all,
455 fields(
456 provider = %self.name,
457 ?path,
458 )
459)]
460pub async fn delete_one(self: &Arc<Self>, path: &str) -> Result {
461 self.delete(once(path.to_owned()).stream())
462 .map_ok(|_| ())
463 .try_collect()
464 .await
465}
466
467#[implement(Provider)]
472#[tracing::instrument(
473 level = "debug",
474 skip_all,
475 fields(
476 provider = %self.name,
477 )
478)]
479pub fn delete<S>(self: &Arc<Self>, paths: S) -> impl Stream<Item = Result<Path>> + Send
480where
481 S: Stream<Item = String> + Send + 'static,
482{
483 let this = self.clone();
484 let paths = paths
485 .map(Ok)
486 .ready_and_then(move |path| {
487 use object_store::{Error, path};
488
489 this.to_abs_path(&path)
490 .map_err(|_| Error::InvalidPath {
491 source: path::Error::InvalidPath { path: path.into() },
492 })
493 })
494 .boxed(); self.provider
497 .delete_stream(paths)
498 .map_err(Error::from)
499}
500
501#[implement(Provider)]
506#[tracing::instrument(
507 level = "debug",
508 err(level = "debug"),
509 skip_all,
510 fields(
511 provider = %self.name,
512 ?src,
513 ?dst,
514 ?overwrite,
515 )
516)]
517pub async fn rename(&self, src: &str, dst: &str, overwrite: CopyMode) -> Result {
518 let src = self.to_abs_path(src)?;
519 let dst = self.to_abs_path(dst)?;
520
521 match overwrite {
522 | CopyMode::Overwrite => self.provider.rename(&src, &dst).left_future(),
523 | CopyMode::Create => self
524 .provider
525 .rename_if_not_exists(&src, &dst)
526 .right_future(),
527 }
528 .map_err(Error::from)
529 .await
530}
531
532#[implement(Provider)]
537#[tracing::instrument(
538 level = "debug",
539 err(level = "debug"),
540 skip_all,
541 fields(
542 provider = %self.name,
543 ?src,
544 ?dst,
545 ?overwrite,
546 )
547)]
548pub async fn copy(&self, src: &str, dst: &str, overwrite: CopyMode) -> Result {
549 let src = self.to_abs_path(src)?;
550 let dst = self.to_abs_path(dst)?;
551
552 match overwrite {
553 | CopyMode::Overwrite => self.provider.copy(&src, &dst).left_future(),
554 | CopyMode::Create => self
555 .provider
556 .copy_if_not_exists(&src, &dst)
557 .right_future(),
558 }
559 .map_err(Error::from)
560 .await
561}
562
563#[implement(Provider)]
568#[tracing::instrument(
569 level = "debug",
570 skip_all,
571 fields(
572 provider = %self.name,
573 ?prefix,
574 )
575)]
576pub fn list(&self, prefix: Option<&str>) -> impl Stream<Item = Result<ObjectMeta>> + Send {
577 let abs_prefix = prefix
578 .map(Path::from)
579 .map(|p| self.prepend_base_path(p))
580 .or_else(|| self.base_path.clone());
581
582 self.provider
583 .list(abs_prefix.as_ref())
584 .map_err(Error::from)
585 .map_ok(|meta| ObjectMeta {
586 location: self.strip_base_path(meta.location),
587 ..meta
588 })
589}
590
591#[implement(Provider)]
596#[tracing::instrument(
597 level = "debug",
598 err(level = "debug"),
599 skip_all,
600 fields(
601 provider = %self.name,
602 ?path,
603 )
604)]
605pub async fn head(&self, path: &str) -> Result<ObjectMeta> {
606 self.provider
607 .head(&self.to_abs_path(path)?)
608 .map_err(Error::from)
609 .await
610}
611
612#[implement(Provider)]
617#[tracing::instrument(
618 level = "debug",
619 err(level = "error"),
620 skip_all,
621 fields(
622 provider = %self.name,
623 )
624)]
625pub async fn ping(&self) -> Result {
626 self.list(None)
627 .try_next()
628 .inspect_err(|e| {
629 error!(chain = %error_chain(e), "Failed to connect to storage provider");
630 })
631 .await
632 .map(|_| ())
633}
634
635#[implement(Provider)]
636fn to_abs_path(&self, location: &str) -> Result<Path> {
637 let location = Path::parse(location)
638 .map_err(|e| err!("Failed to parse location into canonical PathPart: {e}"))?;
639
640 let path = self.prepend_base_path(location);
641
642 trace!(
643 provider = ?self.name,
644 base_path = ?self.base_path,
645 ?path,
646 "Computed absolute path for object on provider.",
647 );
648
649 Ok(path)
650}
651
652#[implement(Provider)]
653fn prepend_base_path(&self, location: Path) -> Path {
654 match self.base_path.as_ref() {
655 | Some(base_path) if !location.prefix_matches(base_path) => base_path
656 .parts()
657 .chain(location.parts())
658 .collect(),
659
660 | _ => location,
661 }
662}
663
664#[implement(Provider)]
665fn strip_base_path(&self, location: Path) -> Path {
666 self.base_path
667 .as_ref()
668 .and_then(|base_path| location.prefix_match(base_path))
669 .map(Iterator::collect)
670 .unwrap_or(location)
671}
672
673#[implement(Provider)]
674fn multipart_threshold(&self) -> usize {
675 extract_variant!(&self.config, StorageProvider::s3)
676 .map(|config| config.multipart_threshold.as_u64())
677 .or_else(|| {
678 extract_variant!(&self.config, StorageProvider::gcs)
679 .map(|config| config.multipart_threshold.as_u64())
680 })
681 .map(TryInto::try_into)
682 .flat_ok()
683 .unwrap_or(usize::MAX)
684}
685
686#[implement(Provider)]
687fn multipart_part_size(&self) -> usize {
688 extract_variant!(&self.config, StorageProvider::s3)
689 .map(|config| config.multipart_part_size.as_u64())
690 .or_else(|| {
691 extract_variant!(&self.config, StorageProvider::gcs)
692 .map(|config| config.multipart_part_size.as_u64())
693 })
694 .map(TryInto::try_into)
695 .flat_ok()
696 .unwrap_or(usize::MAX)
697}
698
699fn chunked(payload: PutPayload, part_size: usize) -> impl Iterator<Item = PutPayload> {
704 let mut buf: Bytes = payload.into();
705 from_fn(move || {
706 buf.is_empty()
707 .is_false()
708 .then(|| buf.split_to(part_size.min(buf.len())).into())
709 })
710}