tuwunel_database/engine/
descriptor.rs

1//! Database column descriptors
2//!
3//! Templates classifying column families along three axes:
4//!
5//! - Write pattern: `RANDOM` (writes scatter across the keyspace) vs
6//!   `SEQUENTIAL` (writes append to the end). Drives compaction priority
7//!   (`OldestSmallestSeqFirst` vs `OldestLargestSeqFirst`) and write-buffer
8//!   sizing.
9//! - Dataset size: plain (level compaction, MB-scale files) vs `_SMALL`
10//!   (universal compaction, KB-scale files and blocks).
11//! - Retention: plain (unbounded) vs `_CACHE` (FIFO eviction bounded by
12//!   `limit_size` and `ttl`).
13//!
14//! Each CF in `maps::MAPS` picks one template and overrides individual
15//! knobs as needed.
16
17#![allow(unused)]
18
19use rocksdb::{
20	DBCompactionPri as CompactionPri, DBCompactionStyle as CompactionStyle,
21	DBCompressionType as CompressionType,
22};
23use tuwunel_core::utils::string::EMPTY;
24
25use super::cf_opts::SENTINEL_COMPRESSION_LEVEL;
26
27/// Describes a column family and its RocksDB tuning.
28///
29/// Catalog entries inherit workload presets and override fields for their key,
30/// value, cache, compaction, and compression characteristics. Lifecycle flags
31/// identify families retained only for compatibility or removal.
32#[derive(Debug, Clone, Copy)]
33pub(crate) struct Descriptor {
34	pub(crate) name: &'static str,
35	pub(crate) ignored: bool,
36	pub(crate) dropped: bool,
37	pub(crate) cache_disp: CacheDisp,
38	pub(crate) key_size_hint: Option<usize>,
39	pub(crate) val_size_hint: Option<usize>,
40	pub(crate) block_size: usize,
41	pub(crate) index_size: usize,
42	pub(crate) write_size: usize,
43	pub(crate) cache_size: usize,
44	pub(crate) level_size: u64,
45	pub(crate) level_shape: [i32; 7],
46	pub(crate) file_size: u64,
47	pub(crate) file_shape: i32,
48	pub(crate) level0_width: i32,
49	pub(crate) merge_width: (i32, i32),
50	pub(crate) limit_size: u64,
51	pub(crate) ttl: u64,
52	pub(crate) compaction: CompactionStyle,
53	pub(crate) compaction_pri: CompactionPri,
54	pub(crate) compaction_size: u64,
55	pub(crate) compression: CompressionType,
56	pub(crate) compressed_index: bool,
57	pub(crate) compression_shape: [i32; 7],
58	pub(crate) compression_level: i32,
59	pub(crate) bottommost_level: Option<i32>,
60	pub(crate) block_index_hashing: Option<bool>,
61	pub(crate) cache_shards: u32,
62	pub(crate) write_to_cache: bool,
63	pub(crate) auto_readahead_thresh: u32,
64	pub(crate) auto_readahead_init: usize,
65	pub(crate) auto_readahead_max: usize,
66}
67
68/// Selects block-cache ownership for a column family.
69///
70/// A family can own a unique cache, join the global shared pool, or pair its
71/// cache with one named family.
72#[derive(Debug, Clone, Copy)]
73pub(crate) enum CacheDisp {
74	Unique,
75	Shared,
76	SharedWith(&'static str),
77}
78
79/// Base descriptor supplying common defaults to all derived descriptors.
80static BASE: Descriptor = Descriptor {
81	name: EMPTY,
82	ignored: false,
83	dropped: false,
84	cache_disp: CacheDisp::Shared,
85	key_size_hint: None,
86	val_size_hint: None,
87	block_size: 1024 * 4,
88	index_size: 1024 * 4,
89	write_size: 1024 * 1024 * 2,
90	cache_size: 1024 * 1024 * 4,
91	level_size: 1024 * 1024 * 8,
92	level_shape: [1, 1, 1, 3, 7, 15, 31],
93	file_size: 1024 * 1024,
94	file_shape: 2,
95	level0_width: 2,
96	merge_width: (2, 16),
97	limit_size: 0,
98	ttl: 60 * 60 * 24 * 21,
99	compaction: CompactionStyle::Level,
100	compaction_pri: CompactionPri::MinOverlappingRatio,
101	compaction_size: 1024 * 1024 * 64,
102	compression: CompressionType::Zstd,
103	compressed_index: true,
104	compression_shape: [0, 0, 0, 1, 1, 1, 1],
105	compression_level: SENTINEL_COMPRESSION_LEVEL,
106	bottommost_level: Some(SENTINEL_COMPRESSION_LEVEL),
107	block_index_hashing: None,
108	cache_shards: 32,
109	write_to_cache: false,
110	auto_readahead_thresh: 0,
111	auto_readahead_init: 1024 * 16,
112	auto_readahead_max: 1024 * 1024 * 2,
113};
114
115/// Placeholder descriptor for existing columns which have no description.
116/// Automatically generated on db open; should not appear in any schema.
117pub(crate) static IGNORED: Descriptor = Descriptor { ignored: true, ..BASE };
118
119/// Tombstone descriptor for columns which have been or will be deleted.
120/// Descriptors of this kind are explicitly set to delete data. Care should be
121/// taken when using this as it inhibits downgrading and other migrations.
122pub(crate) static DROPPED: Descriptor = Descriptor { dropped: true, ..IGNORED };
123
124/// Descriptor for large datasets where writes scatter across the keyspace.
125pub(crate) static RANDOM: Descriptor = Descriptor {
126	compaction_pri: CompactionPri::OldestSmallestSeqFirst,
127	compaction_size: 1024 * 1024 * 256,
128	write_size: 1024 * 1024 * 32,
129	cache_shards: 64,
130	compression_level: -3,
131	bottommost_level: Some(2),
132	compressed_index: true,
133	..BASE
134};
135
136/// Descriptor for large datasets where writes append to the end of the
137/// keyspace.
138pub(crate) static SEQUENTIAL: Descriptor = Descriptor {
139	compaction_pri: CompactionPri::OldestLargestSeqFirst,
140	compaction_size: 1024 * 1024 * 512,
141	write_size: 1024 * 1024 * 64,
142	level_size: 1024 * 1024 * 32,
143	file_size: 1024 * 1024 * 2,
144	cache_size: 1024 * 1024 * 8,
145	compression_level: -2,
146	bottommost_level: Some(2),
147	compression_shape: [0, 0, 1, 1, 1, 1, 1],
148	compressed_index: false,
149	..BASE
150};
151
152/// Descriptor for small datasets where writes scatter across the keyspace.
153pub(crate) static RANDOM_SMALL: Descriptor = Descriptor {
154	compaction: CompactionStyle::Universal,
155	compaction_size: 1024 * 1024 * 128,
156	write_size: 1024 * 1024 * 16,
157	level_size: 1024 * 512,
158	file_size: 1024 * 128,
159	file_shape: 3,
160	index_size: 512,
161	block_size: 512,
162	compression_level: -4,
163	bottommost_level: Some(-1),
164	compression_shape: [0, 0, 0, 0, 0, 1, 1],
165	compressed_index: false,
166	..RANDOM
167};
168
169/// Descriptor for small datasets where writes append to the end of the
170/// keyspace.
171pub(crate) static SEQUENTIAL_SMALL: Descriptor = Descriptor {
172	compaction: CompactionStyle::Universal,
173	compaction_size: 1024 * 1024 * 128,
174	write_size: 1024 * 1024 * 16,
175	level_size: 1024 * 1024,
176	file_size: 1024 * 512,
177	file_shape: 3,
178	block_size: 512,
179	block_index_hashing: Some(false),
180	compression_level: -4,
181	bottommost_level: Some(-2),
182	compression_shape: [0, 0, 0, 0, 1, 1, 1],
183	compressed_index: false,
184	..SEQUENTIAL
185};
186
187/// Descriptor for large persistent caches where writes scatter across the
188/// keyspace. Oldest entries are evicted by FIFO compaction once `limit_size`
189/// is reached.
190pub(crate) static RANDOM_CACHE: Descriptor = Descriptor {
191	compaction: CompactionStyle::Fifo,
192	compaction_size: 1024 * 1024 * 32,
193	limit_size: 1024 * 1024 * 1024 * 2,
194	ttl: 60 * 60 * 24 * 180,
195	..RANDOM
196};
197
198/// Descriptor for large persistent ring/queue caches where writes append to
199/// the end of the keyspace. Lowest keys are evicted off the front once
200/// `limit_size` is reached.
201pub(crate) static SEQUENTIAL_CACHE: Descriptor = Descriptor {
202	cache_disp: CacheDisp::Unique,
203	compaction: CompactionStyle::Fifo,
204	compaction_size: 1024 * 1024 * 64,
205	limit_size: 1024 * 1024 * 1024 * 2,
206	ttl: 60 * 60 * 24 * 180,
207	..SEQUENTIAL
208};
209
210/// Descriptor for small persistent caches where writes scatter across the
211/// keyspace. Oldest entries are evicted by FIFO compaction once `limit_size`
212/// is reached.
213pub(crate) static RANDOM_SMALL_CACHE: Descriptor = Descriptor {
214	compression: CompressionType::None,
215	compaction: CompactionStyle::Fifo,
216	compaction_size: 1024 * 1024 * 16,
217	limit_size: 1024 * 1024 * 64,
218	ttl: 60 * 60 * 24 * 180,
219	file_shape: 2,
220	..RANDOM_SMALL
221};
222
223/// Descriptor for small persistent ring/queue caches where writes append to
224/// the end of the keyspace. Lowest keys are evicted off the front once
225/// `limit_size` is reached.
226pub(crate) static SEQUENTIAL_SMALL_CACHE: Descriptor = Descriptor {
227	cache_disp: CacheDisp::Unique,
228	compression: CompressionType::None,
229	compaction: CompactionStyle::Fifo,
230	compaction_size: 1024 * 1024 * 16,
231	limit_size: 1024 * 1024 * 64,
232	ttl: 60 * 60 * 24 * 180,
233	file_shape: 2,
234	..SEQUENTIAL_SMALL
235};
tuwunel_database/engine/descriptor.rs

tuwunel_database/engine/
descriptor.rs
