tuwunel_database/engine/
descriptor.rs

1//! Database column descriptors
2//!
3//! Templates classifying column families along three axes:
4//!
5//! - Write pattern: `RANDOM` (writes scatter across the keyspace) vs
6//!   `SEQUENTIAL` (writes append to the end). Drives compaction priority
7//!   (`OldestSmallestSeqFirst` vs `OldestLargestSeqFirst`) and write-buffer
8//!   sizing.
9//! - Dataset size: plain (level compaction, MB-scale files) vs `_SMALL`
10//!   (universal compaction, KB-scale files and blocks).
11//! - Retention: plain (unbounded) vs `_CACHE` (FIFO eviction bounded by
12//!   `limit_size` and `ttl`).
13//!
14//! Each CF in `maps::MAPS` picks one template and overrides individual
15//! knobs as needed.
16
17#![allow(unused)]
18
19use rocksdb::{
20	DBCompactionPri as CompactionPri, DBCompactionStyle as CompactionStyle,
21	DBCompressionType as CompressionType,
22};
23use tuwunel_core::utils::string::EMPTY;
24
25use super::cf_opts::SENTINEL_COMPRESSION_LEVEL;
26
27/// Describes a column family and its RocksDB tuning.
28///
29/// Catalog entries inherit workload presets and override fields for their key,
30/// value, cache, compaction, and compression characteristics. Lifecycle flags
31/// identify families retained only for compatibility or removal.
32#[derive(Debug, Clone, Copy)]
33pub(crate) struct Descriptor {
34	pub(crate) name: &'static str,
35	pub(crate) ignored: bool,
36	pub(crate) dropped: bool,
37	pub(crate) cache_disp: CacheDisp,
38	pub(crate) key_size_hint: Option<usize>,
39	pub(crate) val_size_hint: Option<usize>,
40	pub(crate) block_size: usize,
41	pub(crate) index_size: usize,
42	pub(crate) write_size: usize,
43	pub(crate) cache_size: usize,
44	pub(crate) level_size: u64,
45	pub(crate) level_shape: [i32; 7],
46	pub(crate) file_size: u64,
47	pub(crate) file_shape: i32,
48	pub(crate) level0_width: i32,
49	pub(crate) merge_width: (i32, i32),
50	pub(crate) limit_size: u64,
51	pub(crate) ttl: u64,
52	pub(crate) compaction: CompactionStyle,
53	pub(crate) compaction_pri: CompactionPri,
54	pub(crate) compression: CompressionType,
55	pub(crate) compressed_index: bool,
56	pub(crate) compression_shape: [i32; 7],
57	pub(crate) compression_level: i32,
58	pub(crate) bottommost_level: Option<i32>,
59	pub(crate) block_index_hashing: Option<bool>,
60	pub(crate) cache_shards: u32,
61	pub(crate) write_to_cache: bool,
62	pub(crate) auto_readahead_thresh: u32,
63	pub(crate) auto_readahead_init: usize,
64	pub(crate) auto_readahead_max: usize,
65}
66
67/// Selects block-cache ownership for a column family.
68///
69/// A family can own a unique cache, join the global shared pool, or pair its
70/// cache with one named family.
71#[derive(Debug, Clone, Copy)]
72pub(crate) enum CacheDisp {
73	Unique,
74	Shared,
75	SharedWith(&'static str),
76}
77
78/// Base descriptor supplying common defaults to all derived descriptors.
79static BASE: Descriptor = Descriptor {
80	name: EMPTY,
81	ignored: false,
82	dropped: false,
83	cache_disp: CacheDisp::Shared,
84	key_size_hint: None,
85	val_size_hint: None,
86	block_size: 1024 * 4,
87	index_size: 1024 * 4,
88	write_size: 1024 * 1024 * 2,
89	cache_size: 1024 * 1024 * 4,
90	level_size: 1024 * 1024 * 8,
91	level_shape: [1, 1, 1, 3, 7, 15, 31],
92	file_size: 1024 * 1024,
93	file_shape: 2,
94	level0_width: 2,
95	merge_width: (2, 16),
96	limit_size: 0,
97	ttl: 60 * 60 * 24 * 21,
98	compaction: CompactionStyle::Level,
99	compaction_pri: CompactionPri::MinOverlappingRatio,
100	compression: CompressionType::Zstd,
101	compressed_index: true,
102	compression_shape: [0, 0, 0, 1, 1, 1, 1],
103	compression_level: SENTINEL_COMPRESSION_LEVEL,
104	bottommost_level: Some(SENTINEL_COMPRESSION_LEVEL),
105	block_index_hashing: None,
106	cache_shards: 32,
107	write_to_cache: false,
108	auto_readahead_thresh: 0,
109	auto_readahead_init: 1024 * 16,
110	auto_readahead_max: 1024 * 1024 * 2,
111};
112
113/// Placeholder descriptor for existing columns which have no description.
114/// Automatically generated on db open; should not appear in any schema.
115pub(crate) static IGNORED: Descriptor = Descriptor { ignored: true, ..BASE };
116
117/// Tombstone descriptor for columns which have been or will be deleted.
118/// Descriptors of this kind are explicitly set to delete data. Care should be
119/// taken when using this as it inhibits downgrading and other migrations.
120pub(crate) static DROPPED: Descriptor = Descriptor { dropped: true, ..IGNORED };
121
122/// Descriptor for large datasets where writes scatter across the keyspace.
123pub(crate) static RANDOM: Descriptor = Descriptor {
124	compaction_pri: CompactionPri::OldestSmallestSeqFirst,
125	write_size: 1024 * 1024 * 32,
126	cache_shards: 64,
127	compression_level: -3,
128	bottommost_level: Some(2),
129	compressed_index: true,
130	..BASE
131};
132
133/// Descriptor for large datasets where writes append to the end of the
134/// keyspace.
135pub(crate) static SEQUENTIAL: Descriptor = Descriptor {
136	compaction_pri: CompactionPri::OldestLargestSeqFirst,
137	write_size: 1024 * 1024 * 64,
138	level_size: 1024 * 1024 * 32,
139	file_size: 1024 * 1024 * 2,
140	cache_size: 1024 * 1024 * 8,
141	compression_level: -2,
142	bottommost_level: Some(2),
143	compression_shape: [0, 0, 1, 1, 1, 1, 1],
144	compressed_index: false,
145	..BASE
146};
147
148/// Descriptor for small datasets where writes scatter across the keyspace.
149pub(crate) static RANDOM_SMALL: Descriptor = Descriptor {
150	compaction: CompactionStyle::Universal,
151	write_size: 1024 * 1024 * 16,
152	level_size: 1024 * 512,
153	file_size: 1024 * 128,
154	file_shape: 3,
155	index_size: 512,
156	block_size: 512,
157	compression_level: -4,
158	bottommost_level: Some(-1),
159	compression_shape: [0, 0, 0, 0, 0, 1, 1],
160	compressed_index: false,
161	..RANDOM
162};
163
164/// Descriptor for small datasets where writes append to the end of the
165/// keyspace.
166pub(crate) static SEQUENTIAL_SMALL: Descriptor = Descriptor {
167	compaction: CompactionStyle::Universal,
168	write_size: 1024 * 1024 * 16,
169	level_size: 1024 * 1024,
170	file_size: 1024 * 512,
171	file_shape: 3,
172	block_size: 512,
173	block_index_hashing: Some(false),
174	compression_level: -4,
175	bottommost_level: Some(-2),
176	compression_shape: [0, 0, 0, 0, 1, 1, 1],
177	compressed_index: false,
178	..SEQUENTIAL
179};
180
181/// Descriptor for large persistent caches where writes scatter across the
182/// keyspace. Oldest entries are evicted by FIFO compaction once `limit_size`
183/// is reached.
184pub(crate) static RANDOM_CACHE: Descriptor = Descriptor {
185	compaction: CompactionStyle::Fifo,
186	limit_size: 1024 * 1024 * 1024 * 2,
187	ttl: 60 * 60 * 24 * 180,
188	..RANDOM
189};
190
191/// Descriptor for large persistent ring/queue caches where writes append to
192/// the end of the keyspace. Lowest keys are evicted off the front once
193/// `limit_size` is reached.
194pub(crate) static SEQUENTIAL_CACHE: Descriptor = Descriptor {
195	compaction: CompactionStyle::Fifo,
196	cache_disp: CacheDisp::Unique,
197	limit_size: 1024 * 1024 * 1024 * 2,
198	ttl: 60 * 60 * 24 * 180,
199	..SEQUENTIAL
200};
201
202/// Descriptor for small persistent caches where writes scatter across the
203/// keyspace. Oldest entries are evicted by FIFO compaction once `limit_size`
204/// is reached.
205pub(crate) static RANDOM_SMALL_CACHE: Descriptor = Descriptor {
206	compaction: CompactionStyle::Fifo,
207	compression: CompressionType::None,
208	limit_size: 1024 * 1024 * 64,
209	ttl: 60 * 60 * 24 * 180,
210	file_shape: 2,
211	..RANDOM_SMALL
212};
213
214/// Descriptor for small persistent ring/queue caches where writes append to
215/// the end of the keyspace. Lowest keys are evicted off the front once
216/// `limit_size` is reached.
217pub(crate) static SEQUENTIAL_SMALL_CACHE: Descriptor = Descriptor {
218	compaction: CompactionStyle::Fifo,
219	cache_disp: CacheDisp::Unique,
220	compression: CompressionType::None,
221	limit_size: 1024 * 1024 * 64,
222	ttl: 60 * 60 * 24 * 180,
223	file_shape: 2,
224	..SEQUENTIAL_SMALL
225};
tuwunel_database/engine/descriptor.rs

tuwunel_database/engine/
descriptor.rs
