Skip to main content

tuwunel_service/storage/
provider.rs

1//! Common interface over local and S3-compatible object stores.
2//!
3//! Providers apply an optional base path, choose single-part or multipart
4//! uploads, and expose streaming reads and deletes. Backend failures are
5//! translated into the service's shared error type.
6
7/// Local-filesystem storage-provider construction.
8///
9/// The constructor validates the configured directory and can create it when
10/// requested before wrapping it as a provider.
11/// Disabled local configurations return no provider.
12pub mod local;
13
14/// Google Cloud Storage storage-provider construction.
15///
16/// The constructor applies bucket and prefix options before wrapping the
17/// object-store client as a provider. Credentials come from the environment,
18/// ending at the instance metadata server, so GKE Workload Identity needs no
19/// credential settings. Configurations without a bucket return no provider.
20#[cfg(feature = "gcs_storage")]
21pub mod gcs;
22
23/// S3-compatible storage-provider construction.
24///
25/// The constructor applies endpoint, credential, transport, and signing
26/// options before wrapping the object-store client as a provider.
27/// Configurations with neither a URL nor a bucket return no provider.
28#[cfg(feature = "s3_storage")]
29pub mod s3;
30
31#[cfg(test)]
32mod tests;
33
34use std::{
35	iter::{from_fn, once},
36	ops::Range,
37	sync::Arc,
38	time::Duration,
39};
40
41use bytes::Bytes;
42use derive_more::Debug;
43use futures::{FutureExt, Stream, StreamExt, TryFutureExt, TryStreamExt};
44#[cfg(any(feature = "gcs_storage", feature = "s3_storage"))]
45use http::Method;
46#[cfg(any(feature = "gcs_storage", feature = "s3_storage"))]
47use object_store::signer::Signer;
48use object_store::{
49	Attributes, CopyMode, DynObjectStore, GetResult, MultipartUpload, ObjectMeta, ObjectStore,
50	ObjectStoreExt, PutPayload, PutResult, path::Path,
51};
52use tuwunel_core::{
53	Error, Result,
54	config::StorageProvider,
55	debug, err, error,
56	error::error_chain,
57	extract_variant, implement, info, trace,
58	utils::{
59		BoolExt,
60		result::FlatOk,
61		stream::{IterStream, TryReadyExt},
62	},
63};
64use url::Url;
65
66/// One configured object-storage backend.
67///
68/// The provider normalizes configured paths and transfer policies before
69/// delegating operations to its local or S3-compatible object store.
70/// Optional startup checks and URL signing remain backend capabilities.
71#[derive(Debug)]
72pub struct Provider {
73	/// Configuration identifier for this provider.
74	pub name: String,
75
76	/// Backend-specific configuration used to construct this provider.
77	pub config: StorageProvider,
78
79	/// Erased object-store implementation receiving provider operations.
80	pub(crate) provider: Box<DynObjectStore>,
81
82	#[cfg(any(feature = "gcs_storage", feature = "s3_storage"))]
83	#[debug(skip)]
84	/// Optional backend signer used to create time-limited object URLs.
85	pub(crate) signer: Option<Arc<dyn Signer>>,
86
87	/// Prefix prepended to logical object paths before backend operations.
88	pub(crate) base_path: Option<Path>,
89
90	startup_check: bool,
91}
92
93/// One streamed object chunk with its returned range and complete object size.
94///
95/// Every chunk from one fetch carries a clone of the range and size metadata
96/// reported by the backend for that request.
97/// Stream and backend failures are represented separately as error items.
98pub type FetchItem = (Bytes, (Range<u64>, u64));
99
100/// One streamed object chunk with shared response metadata and attributes.
101///
102/// The metadata tuple is shared by [`Arc`] across every chunk from the same
103/// fetch, avoiding a per-chunk clone of the backend response details.
104/// Stream and backend failures are represented separately as error items.
105pub type FetchMetaItem = (Bytes, Arc<(Range<u64>, ObjectMeta, Attributes)>);
106
107/// Starts this provider and performs its configured connectivity check.
108///
109/// Providers with startup checks disabled become ready without backend I/O.
110/// An enabled check lists at most one object and propagates any backend error.
111#[implement(Provider)]
112#[tracing::instrument(skip_all, err)]
113pub(super) async fn start(self: &Arc<Self>) -> Result {
114	if self.startup_check {
115		self.startup_check().await?;
116	}
117
118	Ok(())
119}
120
121#[implement(Provider)]
122#[tracing::instrument(name = "check", skip_all, err)]
123async fn startup_check(self: &Arc<Self>) -> Result {
124	debug!(
125		name = ?self.name,
126		"Checking storage provider client connection...",
127	);
128	self.ping()
129		.inspect_ok(|()| {
130			info!(
131				name = %self.name,
132				"Connected to storage provider"
133			);
134		})
135		.await
136}
137
138/// Stores a streamed object under `path`.
139///
140/// Supplying the total size permits a single-part upload below the configured
141/// threshold. A missing or large size selects multipart upload instead.
142#[implement(Provider)]
143#[tracing::instrument(
144	level = "debug",
145	err(level = "debug"),
146	skip_all,
147	fields(
148		provider = %self.name,
149		?path,
150		?size,
151	)
152)]
153pub async fn put<S, T>(&self, path: &str, size: Option<usize>, input: S) -> Result<PutResult>
154where
155	S: Stream<Item = Result<T>> + Send,
156	PutPayload: From<T> + From<PutPayload>,
157{
158	if size.is_none_or(|size| size >= self.multipart_threshold()) {
159		return self.put_multi(path, input).await;
160	}
161
162	debug!(
163		?size,
164		threshold = ?self.multipart_threshold(),
165		"Selecting single-part upload..."
166	);
167
168	let payload: PutPayload = input
169		.map_ok(PutPayload::from)
170		.try_collect::<Vec<_>>()
171		.await?
172		.into_iter()
173		.map(Bytes::from)
174		.collect();
175
176	self.put_single(path, payload).await
177}
178
179/// Stores one contiguous object under `path`.
180///
181/// The input length selects single-part or multipart upload against the
182/// configured threshold. Backend upload failures are propagated.
183#[implement(Provider)]
184#[tracing::instrument(
185	level = "debug",
186	err(level = "debug"),
187	skip_all,
188	fields(
189		provider = %self.name,
190		?path,
191	)
192)]
193pub async fn put_one<T>(&self, path: &str, input: T) -> Result<PutResult>
194where
195	PutPayload: From<T> + From<PutPayload>,
196{
197	let payload: PutPayload = input.into();
198
199	if payload.content_length() < self.multipart_threshold() {
200		return self.put_single(path, payload).await;
201	}
202
203	let part_size = self.multipart_part_size();
204
205	debug!(
206		len = ?payload.content_length(),
207		threshold = ?self.multipart_threshold(),
208		?part_size,
209		"Selecting multi-part upload..."
210	);
211
212	self.put_multi(path, chunked(payload, part_size).try_stream())
213		.await
214}
215
216/// Stores streamed input through a multipart upload.
217///
218/// Input chunks are written as ordered multipart parts. Upload cleanup and
219/// backend failures are delegated to the object-store implementation.
220#[implement(Provider)]
221#[tracing::instrument(
222	level = "debug",
223	err(level = "debug"),
224	skip_all,
225	fields(
226		provider = %self.name,
227		?path,
228	)
229)]
230async fn put_multi<S, T>(&self, path: &str, input: S) -> Result<PutResult>
231where
232	S: Stream<Item = Result<T>> + Send,
233	PutPayload: From<T> + From<PutPayload>,
234{
235	let path = self.to_abs_path(path)?;
236	let mut handle = self
237		.provider
238		.put_multipart(&path)
239		.map_err(Error::from)
240		.await?;
241
242	match input
243		.try_for_each(|t| handle.put_part(t.into()).map_err(Error::from))
244		.inspect_err(|e| error!(?path, chain = %error_chain(e), "Failed to store object"))
245		.await
246	{
247		| Ok(()) =>
248			handle
249				.complete()
250				.map_err(Error::from)
251				.inspect_err(|e| {
252					error!(
253						?path,
254						chain = %error_chain(e),
255						"Failed to store object during completion",
256					);
257				})
258				.await,
259
260		| Err(e) =>
261			handle
262				.abort()
263				.map_ok(|()| Err(e))
264				.map_err(Error::from)
265				.inspect_err(|e| {
266					error!(
267						?path,
268						chain = %error_chain(e),
269						"Additional errors during error handling",
270					);
271				})
272				.await?,
273	}
274}
275
276/// Stores contiguous input through a single-part upload.
277///
278/// The provider prefix is applied before the backend request. Backend failures
279/// are propagated without retrying as multipart upload.
280#[implement(Provider)]
281#[tracing::instrument(
282	level = "debug",
283	err(level = "debug"),
284	skip_all,
285	fields(
286		provider = %self.name,
287		?path,
288	)
289)]
290async fn put_single(&self, path: &str, input: PutPayload) -> Result<PutResult> {
291	let path = self.to_abs_path(path)?;
292
293	self.provider
294		.put(&path, input)
295		.map_err(Error::from)
296		.await
297}
298
299/// Streams an object's bytes together with shared response metadata.
300///
301/// The provider prefix is applied to `path`, and each successful chunk shares
302/// the same range, object metadata, and attributes. Load and stream failures
303/// are returned as items rather than being discarded.
304#[implement(Provider)]
305#[tracing::instrument(
306	level = "debug",
307	skip_all,
308	fields(
309		provider = %self.name,
310		?path,
311	)
312)]
313pub fn fetch_with_metadata(
314	&self,
315	path: &str,
316) -> impl Stream<Item = Result<FetchMetaItem>> + Send {
317	self.load(path)
318		.map_ok(|result| {
319			let meta = (result.range.clone(), result.meta.clone(), result.attributes.clone());
320			let data = Arc::new(meta);
321
322			result
323				.into_stream()
324				.map_err(Error::from)
325				.map_ok(move |bytes| (bytes, data.clone()))
326		})
327		.map_err(Error::from)
328		.try_flatten_stream()
329}
330
331/// Streams an object's bytes together with its returned range and total size.
332///
333/// The provider prefix is applied to `path`. Load and stream failures are
334/// returned as items, allowing callers to consume the body without buffering
335/// the complete object.
336#[implement(Provider)]
337#[tracing::instrument(
338	level = "debug",
339	skip_all,
340	fields(
341		provider = %self.name,
342		?path,
343	)
344)]
345pub fn fetch(&self, path: &str) -> impl Stream<Item = Result<FetchItem>> + Send {
346	self.load(path)
347		.map_ok(|result| {
348			let size = result.meta.size;
349			let range = result.range.clone();
350
351			result
352				.into_stream()
353				.map_err(Error::from)
354				.map_ok(move |bytes| (bytes, (range.clone(), size)))
355		})
356		.map_err(Error::from)
357		.try_flatten_stream()
358}
359
360/// Loads an entire object into one contiguous byte buffer.
361///
362/// The provider prefix is applied before the backend request. Backend and body
363/// streaming failures are propagated to the caller.
364#[implement(Provider)]
365#[tracing::instrument(
366	level = "debug",
367	err(level = "debug"),
368	skip_all,
369	fields(
370		provider = %self.name,
371		?path,
372	)
373)]
374pub async fn get(&self, path: &str) -> Result<Bytes> {
375	self.load(path)
376		.map_ok(GetResult::bytes)
377		.await?
378		.map_err(Error::from)
379		.await
380}
381
382/// Opens an object and returns the backend's raw read result.
383///
384/// The provider prefix is applied before the request. Callers can inspect the
385/// returned range and metadata or consume its body as a stream.
386#[implement(Provider)]
387#[tracing::instrument(
388	level = "debug",
389	err(level = "debug"),
390	skip_all,
391	fields(
392		provider = %self.name,
393		?path,
394	)
395)]
396pub async fn load(&self, path: &str) -> Result<GetResult> {
397	let path = self.to_abs_path(path)?;
398
399	self.provider
400		.get(&path)
401		.map_err(Error::from)
402		.await
403}
404
405/// Creates a time-limited GET URL when the backend supports signing.
406///
407/// The provider prefix is applied before signing. Backends without a signer,
408/// such as local filesystem providers, return `None` without performing I/O.
409#[cfg(any(feature = "gcs_storage", feature = "s3_storage"))]
410#[implement(Provider)]
411#[tracing::instrument(
412	level = "debug",
413	err(level = "debug"),
414	skip_all,
415	fields(
416		provider = %self.name,
417		?path,
418		?ttl,
419	)
420)]
421pub async fn signed_get_url(&self, path: &str, ttl: Duration) -> Result<Option<Url>> {
422	let Some(signer) = self.signer.as_ref() else {
423		return Ok(None);
424	};
425
426	let path = self.to_abs_path(path)?;
427
428	signer
429		.signed_url(Method::GET, &path, ttl)
430		.map_err(Error::from)
431		.map_ok(Some)
432		.await
433}
434
435/// Creates no URL, since a build without S3 or GCS support has no signer.
436///
437/// Such a build has only local filesystem providers, so callers serve every
438/// object directly, as they do for local providers in any build.
439#[cfg(not(any(feature = "gcs_storage", feature = "s3_storage")))]
440#[implement(Provider)]
441#[expect(clippy::unused_async)]
442pub async fn signed_get_url(&self, _path: &str, _ttl: Duration) -> Result<Option<Url>> {
443	Ok(None)
444}
445
446/// Deletes one object from this provider.
447///
448/// This consumes [`Self::delete`] to completion and discards its yielded path.
449/// Invalid paths and backend failures are propagated.
450#[implement(Provider)]
451#[tracing::instrument(
452	level = "debug",
453	err(level = "debug"),
454	skip_all,
455	fields(
456		provider = %self.name,
457		?path,
458	)
459)]
460pub async fn delete_one(self: &Arc<Self>, path: &str) -> Result {
461	self.delete(once(path.to_owned()).stream())
462		.map_ok(|_| ())
463		.try_collect()
464		.await
465}
466
467/// Lazily deletes each supplied object path.
468///
469/// The provider prefix is applied to every path before it reaches the backend.
470/// Invalid paths and backend failures are emitted by the returned stream.
471#[implement(Provider)]
472#[tracing::instrument(
473	level = "debug",
474	skip_all,
475	fields(
476		provider = %self.name,
477	)
478)]
479pub fn delete<S>(self: &Arc<Self>, paths: S) -> impl Stream<Item = Result<Path>> + Send
480where
481	S: Stream<Item = String> + Send + 'static,
482{
483	let this = self.clone();
484	let paths = paths
485		.map(Ok)
486		.ready_and_then(move |path| {
487			use object_store::{Error, path};
488
489			this.to_abs_path(&path)
490				.map_err(|_| Error::InvalidPath {
491					source: path::Error::InvalidPath { path: path.into() },
492				})
493		})
494		.boxed(); // type-erasure boundary
495
496	self.provider
497		.delete_stream(paths)
498		.map_err(Error::from)
499}
500
501/// Renames an object within this provider.
502///
503/// Both paths receive the provider prefix. [`CopyMode::Create`] refuses an
504/// existing destination, while [`CopyMode::Overwrite`] permits replacement.
505#[implement(Provider)]
506#[tracing::instrument(
507	level = "debug",
508	err(level = "debug"),
509	skip_all,
510	fields(
511		provider = %self.name,
512		?src,
513		?dst,
514		?overwrite,
515	)
516)]
517pub async fn rename(&self, src: &str, dst: &str, overwrite: CopyMode) -> Result {
518	let src = self.to_abs_path(src)?;
519	let dst = self.to_abs_path(dst)?;
520
521	match overwrite {
522		| CopyMode::Overwrite => self.provider.rename(&src, &dst).left_future(),
523		| CopyMode::Create => self
524			.provider
525			.rename_if_not_exists(&src, &dst)
526			.right_future(),
527	}
528	.map_err(Error::from)
529	.await
530}
531
532/// Copies an object within this provider.
533///
534/// Both paths receive the provider prefix. [`CopyMode::Create`] refuses an
535/// existing destination, while [`CopyMode::Overwrite`] permits replacement.
536#[implement(Provider)]
537#[tracing::instrument(
538	level = "debug",
539	err(level = "debug"),
540	skip_all,
541	fields(
542		provider = %self.name,
543		?src,
544		?dst,
545		?overwrite,
546	)
547)]
548pub async fn copy(&self, src: &str, dst: &str, overwrite: CopyMode) -> Result {
549	let src = self.to_abs_path(src)?;
550	let dst = self.to_abs_path(dst)?;
551
552	match overwrite {
553		| CopyMode::Overwrite => self.provider.copy(&src, &dst).left_future(),
554		| CopyMode::Create => self
555			.provider
556			.copy_if_not_exists(&src, &dst)
557			.right_future(),
558	}
559	.map_err(Error::from)
560	.await
561}
562
563/// Streams object metadata beneath an optional logical prefix.
564///
565/// The configured provider prefix is applied to the backend query and removed
566/// from each returned location. Backend failures remain stream items.
567#[implement(Provider)]
568#[tracing::instrument(
569	level = "debug",
570	skip_all,
571	fields(
572		provider = %self.name,
573		?prefix,
574	)
575)]
576pub fn list(&self, prefix: Option<&str>) -> impl Stream<Item = Result<ObjectMeta>> + Send {
577	let abs_prefix = prefix
578		.map(Path::from)
579		.map(|p| self.prepend_base_path(p))
580		.or_else(|| self.base_path.clone());
581
582	self.provider
583		.list(abs_prefix.as_ref())
584		.map_err(Error::from)
585		.map_ok(|meta| ObjectMeta {
586			location: self.strip_base_path(meta.location),
587			..meta
588		})
589}
590
591/// Returns metadata for one object.
592///
593/// The provider prefix is applied before the backend request. Missing objects
594/// and backend failures are propagated.
595#[implement(Provider)]
596#[tracing::instrument(
597	level = "debug",
598	err(level = "debug"),
599	skip_all,
600	fields(
601		provider = %self.name,
602		?path,
603	)
604)]
605pub async fn head(&self, path: &str) -> Result<ObjectMeta> {
606	self.provider
607		.head(&self.to_abs_path(path)?)
608		.map_err(Error::from)
609		.await
610}
611
612/// Probes whether this provider can service a listing request.
613///
614/// The probe consumes at most the first result, so an empty store succeeds.
615/// Any path or backend error is logged and returned.
616#[implement(Provider)]
617#[tracing::instrument(
618	level = "debug",
619	err(level = "error"),
620	skip_all,
621	fields(
622		provider = %self.name,
623	)
624)]
625pub async fn ping(&self) -> Result {
626	self.list(None)
627		.try_next()
628		.inspect_err(|e| {
629			error!(chain = %error_chain(e), "Failed to connect to storage provider");
630		})
631		.await
632		.map(|_| ())
633}
634
635#[implement(Provider)]
636fn to_abs_path(&self, location: &str) -> Result<Path> {
637	let location = Path::parse(location)
638		.map_err(|e| err!("Failed to parse location into canonical PathPart: {e}"))?;
639
640	let path = self.prepend_base_path(location);
641
642	trace!(
643		provider = ?self.name,
644		base_path = ?self.base_path,
645		?path,
646		"Computed absolute path for object on provider.",
647	);
648
649	Ok(path)
650}
651
652#[implement(Provider)]
653fn prepend_base_path(&self, location: Path) -> Path {
654	match self.base_path.as_ref() {
655		| Some(base_path) if !location.prefix_matches(base_path) => base_path
656			.parts()
657			.chain(location.parts())
658			.collect(),
659
660		| _ => location,
661	}
662}
663
664#[implement(Provider)]
665fn strip_base_path(&self, location: Path) -> Path {
666	self.base_path
667		.as_ref()
668		.and_then(|base_path| location.prefix_match(base_path))
669		.map(Iterator::collect)
670		.unwrap_or(location)
671}
672
673#[implement(Provider)]
674fn multipart_threshold(&self) -> usize {
675	extract_variant!(&self.config, StorageProvider::s3)
676		.map(|config| config.multipart_threshold.as_u64())
677		.or_else(|| {
678			extract_variant!(&self.config, StorageProvider::gcs)
679				.map(|config| config.multipart_threshold.as_u64())
680		})
681		.map(TryInto::try_into)
682		.flat_ok()
683		.unwrap_or(usize::MAX)
684}
685
686#[implement(Provider)]
687fn multipart_part_size(&self) -> usize {
688	extract_variant!(&self.config, StorageProvider::s3)
689		.map(|config| config.multipart_part_size.as_u64())
690		.or_else(|| {
691			extract_variant!(&self.config, StorageProvider::gcs)
692				.map(|config| config.multipart_part_size.as_u64())
693		})
694		.map(TryInto::try_into)
695		.flat_ok()
696		.unwrap_or(usize::MAX)
697}
698
699/// Splits a payload into nonempty parts no larger than `part_size`.
700///
701/// The iterator owns the payload buffer and advances it without copying the
702/// bytes in each yielded part.
703fn chunked(payload: PutPayload, part_size: usize) -> impl Iterator<Item = PutPayload> {
704	let mut buf: Bytes = payload.into();
705	from_fn(move || {
706		buf.is_empty()
707			.is_false()
708			.then(|| buf.split_to(part_size.min(buf.len())).into())
709	})
710}