tuwunel_database/engine/
descriptor.rs

1//! Database column descriptors
2//!
3//! Templates classifying column families along three axes:
4//!
5//! - Write pattern: `RANDOM` (writes scatter across the keyspace) vs
6//!   `SEQUENTIAL` (writes append to the end). Drives compaction priority
7//!   (`OldestSmallestSeqFirst` vs `OldestLargestSeqFirst`) and write-buffer
8//!   sizing.
9//! - Dataset size: plain (level compaction, MB-scale files) vs `_SMALL`
10//!   (universal compaction, KB-scale files and blocks).
11//! - Retention: plain (unbounded) vs `_CACHE` (FIFO eviction bounded by
12//!   `limit_size` and `ttl`).
13//!
14//! Each CF in `maps::MAPS` picks one template and overrides individual
15//! knobs as needed.
16
17#![allow(unused)]
18
19use rocksdb::{
20	DBCompactionPri as CompactionPri, DBCompactionStyle as CompactionStyle,
21	DBCompressionType as CompressionType,
22};
23use tuwunel_core::utils::string::EMPTY;
24
25use super::cf_opts::SENTINEL_COMPRESSION_LEVEL;
26
27/// Column Descriptor
28#[derive(Debug, Clone, Copy)]
29pub(crate) struct Descriptor {
30	pub(crate) name: &'static str,
31	pub(crate) ignored: bool,
32	pub(crate) dropped: bool,
33	pub(crate) cache_disp: CacheDisp,
34	pub(crate) key_size_hint: Option<usize>,
35	pub(crate) val_size_hint: Option<usize>,
36	pub(crate) block_size: usize,
37	pub(crate) index_size: usize,
38	pub(crate) write_size: usize,
39	pub(crate) cache_size: usize,
40	pub(crate) level_size: u64,
41	pub(crate) level_shape: [i32; 7],
42	pub(crate) file_size: u64,
43	pub(crate) file_shape: i32,
44	pub(crate) level0_width: i32,
45	pub(crate) merge_width: (i32, i32),
46	pub(crate) limit_size: u64,
47	pub(crate) ttl: u64,
48	pub(crate) compaction: CompactionStyle,
49	pub(crate) compaction_pri: CompactionPri,
50	pub(crate) compression: CompressionType,
51	pub(crate) compressed_index: bool,
52	pub(crate) compression_shape: [i32; 7],
53	pub(crate) compression_level: i32,
54	pub(crate) bottommost_level: Option<i32>,
55	pub(crate) block_index_hashing: Option<bool>,
56	pub(crate) cache_shards: u32,
57	pub(crate) write_to_cache: bool,
58	pub(crate) auto_readahead_thresh: u32,
59	pub(crate) auto_readahead_init: usize,
60	pub(crate) auto_readahead_max: usize,
61}
62
63/// Cache Disposition
64#[derive(Debug, Clone, Copy)]
65pub(crate) enum CacheDisp {
66	Unique,
67	Shared,
68	SharedWith(&'static str),
69}
70
71/// Base descriptor supplying common defaults to all derived descriptors.
72static BASE: Descriptor = Descriptor {
73	name: EMPTY,
74	ignored: false,
75	dropped: false,
76	cache_disp: CacheDisp::Shared,
77	key_size_hint: None,
78	val_size_hint: None,
79	block_size: 1024 * 4,
80	index_size: 1024 * 4,
81	write_size: 1024 * 1024 * 2,
82	cache_size: 1024 * 1024 * 4,
83	level_size: 1024 * 1024 * 8,
84	level_shape: [1, 1, 1, 3, 7, 15, 31],
85	file_size: 1024 * 1024,
86	file_shape: 2,
87	level0_width: 2,
88	merge_width: (2, 16),
89	limit_size: 0,
90	ttl: 60 * 60 * 24 * 21,
91	compaction: CompactionStyle::Level,
92	compaction_pri: CompactionPri::MinOverlappingRatio,
93	compression: CompressionType::Zstd,
94	compressed_index: true,
95	compression_shape: [0, 0, 0, 1, 1, 1, 1],
96	compression_level: SENTINEL_COMPRESSION_LEVEL,
97	bottommost_level: Some(SENTINEL_COMPRESSION_LEVEL),
98	block_index_hashing: None,
99	cache_shards: 64,
100	write_to_cache: false,
101	auto_readahead_thresh: 0,
102	auto_readahead_init: 1024 * 16,
103	auto_readahead_max: 1024 * 1024 * 2,
104};
105
106/// Placeholder descriptor for existing columns which have no description.
107/// Automatically generated on db open; should not appear in any schema.
108pub(crate) static IGNORED: Descriptor = Descriptor { ignored: true, ..BASE };
109
110/// Tombstone descriptor for columns which have been or will be deleted.
111/// Descriptors of this kind are explicitly set to delete data. Care should be
112/// taken when using this as it inhibits downgrading and other migrations.
113pub(crate) static DROPPED: Descriptor = Descriptor { dropped: true, ..IGNORED };
114
115/// Descriptor for large datasets where writes scatter across the keyspace.
116pub(crate) static RANDOM: Descriptor = Descriptor {
117	compaction_pri: CompactionPri::OldestSmallestSeqFirst,
118	write_size: 1024 * 1024 * 32,
119	cache_shards: 128,
120	compression_level: -3,
121	bottommost_level: Some(2),
122	compressed_index: true,
123	..BASE
124};
125
126/// Descriptor for large datasets where writes append to the end of the
127/// keyspace.
128pub(crate) static SEQUENTIAL: Descriptor = Descriptor {
129	compaction_pri: CompactionPri::OldestLargestSeqFirst,
130	write_size: 1024 * 1024 * 64,
131	level_size: 1024 * 1024 * 32,
132	file_size: 1024 * 1024 * 2,
133	cache_shards: 128,
134	compression_level: -2,
135	bottommost_level: Some(2),
136	compression_shape: [0, 0, 1, 1, 1, 1, 1],
137	compressed_index: false,
138	..BASE
139};
140
141/// Descriptor for small datasets where writes scatter across the keyspace.
142pub(crate) static RANDOM_SMALL: Descriptor = Descriptor {
143	compaction: CompactionStyle::Universal,
144	write_size: 1024 * 1024 * 16,
145	level_size: 1024 * 512,
146	file_size: 1024 * 128,
147	file_shape: 3,
148	index_size: 512,
149	block_size: 512,
150	cache_shards: 64,
151	compression_level: -4,
152	bottommost_level: Some(-1),
153	compression_shape: [0, 0, 0, 0, 0, 1, 1],
154	compressed_index: false,
155	..RANDOM
156};
157
158/// Descriptor for small datasets where writes append to the end of the
159/// keyspace.
160pub(crate) static SEQUENTIAL_SMALL: Descriptor = Descriptor {
161	compaction: CompactionStyle::Universal,
162	write_size: 1024 * 1024 * 16,
163	level_size: 1024 * 1024,
164	file_size: 1024 * 512,
165	file_shape: 3,
166	block_size: 512,
167	cache_shards: 64,
168	block_index_hashing: Some(false),
169	compression_level: -4,
170	bottommost_level: Some(-2),
171	compression_shape: [0, 0, 0, 0, 1, 1, 1],
172	compressed_index: false,
173	..SEQUENTIAL
174};
175
176/// Descriptor for large persistent caches where writes scatter across the
177/// keyspace. Oldest entries are evicted by FIFO compaction once `limit_size`
178/// is reached.
179pub(crate) static RANDOM_CACHE: Descriptor = Descriptor {
180	compaction: CompactionStyle::Fifo,
181	cache_disp: CacheDisp::Unique,
182	limit_size: 1024 * 1024 * 1024 * 2,
183	ttl: 60 * 60 * 24 * 180,
184	..RANDOM
185};
186
187/// Descriptor for large persistent ring/queue caches where writes append to
188/// the end of the keyspace. Lowest keys are evicted off the front once
189/// `limit_size` is reached.
190pub(crate) static SEQUENTIAL_CACHE: Descriptor = Descriptor {
191	compaction: CompactionStyle::Fifo,
192	cache_disp: CacheDisp::Unique,
193	limit_size: 1024 * 1024 * 1024 * 2,
194	ttl: 60 * 60 * 24 * 180,
195	..SEQUENTIAL
196};
197
198/// Descriptor for small persistent caches where writes scatter across the
199/// keyspace. Oldest entries are evicted by FIFO compaction once `limit_size`
200/// is reached.
201pub(crate) static RANDOM_SMALL_CACHE: Descriptor = Descriptor {
202	compaction: CompactionStyle::Fifo,
203	cache_disp: CacheDisp::Unique,
204	compression: CompressionType::None,
205	limit_size: 1024 * 1024 * 64,
206	ttl: 60 * 60 * 24 * 180,
207	file_shape: 2,
208	..RANDOM_SMALL
209};
210
211/// Descriptor for small persistent ring/queue caches where writes append to
212/// the end of the keyspace. Lowest keys are evicted off the front once
213/// `limit_size` is reached.
214pub(crate) static SEQUENTIAL_SMALL_CACHE: Descriptor = Descriptor {
215	compaction: CompactionStyle::Fifo,
216	cache_disp: CacheDisp::Unique,
217	compression: CompressionType::None,
218	limit_size: 1024 * 1024 * 64,
219	ttl: 60 * 60 * 24 * 180,
220	file_shape: 2,
221	..SEQUENTIAL_SMALL
222};
tuwunel_database/engine/descriptor.rs

tuwunel_database/engine/
descriptor.rs
