Skip to main content

tuwunel_database/engine/
descriptor.rs

1//! Database column descriptors
2//!
3//! Templates classifying column families along three axes:
4//!
5//! - Write pattern: `RANDOM` (writes scatter across the keyspace) vs
6//!   `SEQUENTIAL` (writes append to the end). Drives compaction priority
7//!   (`OldestSmallestSeqFirst` vs `OldestLargestSeqFirst`) and write-buffer
8//!   sizing.
9//! - Dataset size: plain (level compaction, MB-scale files) vs `_SMALL`
10//!   (universal compaction, KB-scale files and blocks).
11//! - Retention: plain (unbounded) vs `_CACHE` (FIFO eviction bounded by
12//!   `limit_size` and `ttl`).
13//!
14//! Each CF in `maps::MAPS` picks one template and overrides individual
15//! knobs as needed.
16
17#![allow(unused)]
18
19use rocksdb::{
20	DBCompactionPri as CompactionPri, DBCompactionStyle as CompactionStyle,
21	DBCompressionType as CompressionType,
22};
23use tuwunel_core::utils::string::EMPTY;
24
25use super::cf_opts::SENTINEL_COMPRESSION_LEVEL;
26
27/// Describes a column family and its RocksDB tuning.
28///
29/// Catalog entries inherit workload presets and override fields for their key,
30/// value, cache, compaction, and compression characteristics. Lifecycle flags
31/// identify families retained only for compatibility or removal.
32#[derive(Debug, Clone, Copy)]
33pub(crate) struct Descriptor {
34	pub(crate) name: &'static str,
35	pub(crate) ignored: bool,
36	pub(crate) dropped: bool,
37	pub(crate) cache_disp: CacheDisp,
38	pub(crate) key_size_hint: Option<usize>,
39	pub(crate) val_size_hint: Option<usize>,
40	pub(crate) block_size: usize,
41	pub(crate) index_size: usize,
42	pub(crate) write_size: usize,
43	pub(crate) cache_size: usize,
44	pub(crate) level_size: u64,
45	pub(crate) level_shape: [i32; 7],
46	pub(crate) file_size: u64,
47	pub(crate) file_shape: i32,
48	pub(crate) level0_width: i32,
49	pub(crate) merge_width: (i32, i32),
50	pub(crate) prefix_size: usize,
51	pub(crate) limit_size: u64,
52	pub(crate) ttl: u64,
53	pub(crate) compaction: CompactionStyle,
54	pub(crate) compaction_pri: CompactionPri,
55	pub(crate) compaction_size: u64,
56	pub(crate) compression: CompressionType,
57	pub(crate) compressed_index: bool,
58	pub(crate) compression_shape: [i32; 7],
59	pub(crate) compression_level: i32,
60	pub(crate) bottommost_level: Option<i32>,
61	pub(crate) access_pattern: AccessPattern,
62	pub(crate) cache_shards: u32,
63	pub(crate) write_to_cache: bool,
64	pub(crate) auto_readahead_thresh: u32,
65	pub(crate) auto_readahead_init: usize,
66	pub(crate) auto_readahead_max: usize,
67}
68
69/// Selects block-cache ownership for a column family.
70///
71/// A family can own a unique cache, join the global shared pool, or pair its
72/// cache with one named family.
73#[derive(Debug, Clone, Copy)]
74pub(crate) enum CacheDisp {
75	Unique,
76	Shared,
77	SharedWith(&'static str),
78}
79
80#[derive(Debug, Default, Clone, Copy)]
81pub(crate) enum AccessPattern {
82	#[default]
83	RandomScan, // binary + hash
84	FastScan,   // binary + binary
85	FromScan,   // twolev + binary
86	ShortScan,  // twolev + hash
87	PrefixScan, // hash + binary
88	PointQuery, // hash + hash
89}
90
91/// Base descriptor supplying common defaults to all derived descriptors.
92static BASE: Descriptor = Descriptor {
93	name: EMPTY,
94	ignored: false,
95	dropped: false,
96	cache_disp: CacheDisp::Shared,
97	key_size_hint: None,
98	val_size_hint: None,
99	block_size: 1024 * 4,
100	index_size: 1024 * 4,
101	write_size: 1024 * 1024 * 2,
102	cache_size: 1024 * 1024 * 4,
103	level_size: 1024 * 1024 * 8,
104	level_shape: [1, 1, 1, 3, 7, 15, 31],
105	file_size: 1024 * 1024,
106	file_shape: 2,
107	level0_width: 2,
108	merge_width: (2, 16),
109	prefix_size: 0,
110	limit_size: 0,
111	ttl: 60 * 60 * 24 * 21,
112	compaction: CompactionStyle::Level,
113	compaction_pri: CompactionPri::MinOverlappingRatio,
114	compaction_size: 1024 * 1024 * 64,
115	compression: CompressionType::Zstd,
116	compressed_index: true,
117	compression_shape: [0, 0, 0, 1, 1, 1, 1],
118	compression_level: SENTINEL_COMPRESSION_LEVEL,
119	bottommost_level: Some(SENTINEL_COMPRESSION_LEVEL),
120	access_pattern: AccessPattern::RandomScan,
121	cache_shards: 32,
122	write_to_cache: false,
123	auto_readahead_thresh: 0,
124	auto_readahead_init: 1024 * 16,
125	auto_readahead_max: 1024 * 1024 * 2,
126};
127
128/// Placeholder descriptor for existing columns which have no description.
129/// Automatically generated on db open; should not appear in any schema.
130pub(crate) static IGNORED: Descriptor = Descriptor { ignored: true, ..BASE };
131
132/// Tombstone descriptor for columns which have been or will be deleted.
133/// Descriptors of this kind are explicitly set to delete data. Care should be
134/// taken when using this as it inhibits downgrading and other migrations.
135pub(crate) static DROPPED: Descriptor = Descriptor { dropped: true, ..IGNORED };
136
137/// Descriptor for large datasets where writes scatter across the keyspace.
138pub(crate) static RANDOM: Descriptor = Descriptor {
139	compaction_pri: CompactionPri::OldestSmallestSeqFirst,
140	compaction_size: 1024 * 1024 * 256,
141	write_size: 1024 * 1024 * 32,
142	cache_shards: 64,
143	compression_level: -3,
144	bottommost_level: Some(2),
145	compressed_index: true,
146	..BASE
147};
148
149/// Descriptor for large datasets where writes append to the end of the
150/// keyspace.
151pub(crate) static SEQUENTIAL: Descriptor = Descriptor {
152	compaction_pri: CompactionPri::OldestLargestSeqFirst,
153	compaction_size: 1024 * 1024 * 512,
154	write_size: 1024 * 1024 * 64,
155	level_size: 1024 * 1024 * 32,
156	file_size: 1024 * 1024 * 2,
157	cache_size: 1024 * 1024 * 8,
158	compression_level: -2,
159	bottommost_level: Some(2),
160	compression_shape: [0, 0, 1, 1, 1, 1, 1],
161	compressed_index: false,
162	access_pattern: AccessPattern::FromScan,
163	..BASE
164};
165
166/// Descriptor for small datasets where writes scatter across the keyspace.
167pub(crate) static RANDOM_SMALL: Descriptor = Descriptor {
168	compaction: CompactionStyle::Universal,
169	compaction_size: 1024 * 1024 * 128,
170	write_size: 1024 * 1024 * 16,
171	level_size: 1024 * 512,
172	file_size: 1024 * 128,
173	file_shape: 3,
174	index_size: 512,
175	block_size: 512,
176	compression_level: -4,
177	bottommost_level: Some(-1),
178	compression_shape: [0, 0, 0, 0, 0, 1, 1],
179	compressed_index: false,
180	..RANDOM
181};
182
183/// Descriptor for small datasets where writes append to the end of the
184/// keyspace.
185pub(crate) static SEQUENTIAL_SMALL: Descriptor = Descriptor {
186	compaction: CompactionStyle::Universal,
187	compaction_size: 1024 * 1024 * 128,
188	write_size: 1024 * 1024 * 16,
189	level_size: 1024 * 1024,
190	file_size: 1024 * 512,
191	file_shape: 3,
192	block_size: 512,
193	compression_level: -4,
194	bottommost_level: Some(-2),
195	compression_shape: [0, 0, 0, 0, 1, 1, 1],
196	compressed_index: false,
197	access_pattern: AccessPattern::FastScan,
198	..SEQUENTIAL
199};
200
201/// Descriptor for large persistent caches where writes scatter across the
202/// keyspace. Oldest entries are evicted by FIFO compaction once `limit_size`
203/// is reached.
204pub(crate) static RANDOM_CACHE: Descriptor = Descriptor {
205	compaction: CompactionStyle::Fifo,
206	compaction_size: 1024 * 1024 * 32,
207	limit_size: 1024 * 1024 * 1024 * 2,
208	ttl: 60 * 60 * 24 * 180,
209	..RANDOM
210};
211
212/// Descriptor for large persistent ring/queue caches where writes append to
213/// the end of the keyspace. Lowest keys are evicted off the front once
214/// `limit_size` is reached.
215pub(crate) static SEQUENTIAL_CACHE: Descriptor = Descriptor {
216	cache_disp: CacheDisp::Unique,
217	compaction: CompactionStyle::Fifo,
218	compaction_size: 1024 * 1024 * 64,
219	limit_size: 1024 * 1024 * 1024 * 2,
220	ttl: 60 * 60 * 24 * 180,
221	..SEQUENTIAL
222};
223
224/// Descriptor for small persistent caches where writes scatter across the
225/// keyspace. Oldest entries are evicted by FIFO compaction once `limit_size`
226/// is reached.
227pub(crate) static RANDOM_SMALL_CACHE: Descriptor = Descriptor {
228	compression: CompressionType::None,
229	compaction: CompactionStyle::Fifo,
230	compaction_size: 1024 * 1024 * 16,
231	limit_size: 1024 * 1024 * 64,
232	ttl: 60 * 60 * 24 * 180,
233	file_shape: 2,
234	..RANDOM_SMALL
235};
236
237/// Descriptor for small persistent ring/queue caches where writes append to
238/// the end of the keyspace. Lowest keys are evicted off the front once
239/// `limit_size` is reached.
240pub(crate) static SEQUENTIAL_SMALL_CACHE: Descriptor = Descriptor {
241	cache_disp: CacheDisp::Unique,
242	compression: CompressionType::None,
243	compaction: CompactionStyle::Fifo,
244	compaction_size: 1024 * 1024 * 16,
245	limit_size: 1024 * 1024 * 64,
246	ttl: 60 * 60 * 24 * 180,
247	file_shape: 2,
248	..SEQUENTIAL_SMALL
249};