Skip to main content

tuwunel_database/pool/
configure.rs

1use std::{
2	path::{Path, PathBuf},
3	sync::Arc,
4};
5
6use tuwunel_core::{
7	Config, Server, at, debug,
8	debug::INFO_SPAN_LEVEL,
9	debug_info, debug_warn, expected, info, is_equal_to,
10	utils::{
11		BoolExt,
12		math::usize_from_f64,
13		result::LogDebugErr,
14		stream,
15		stream::{AMPLIFICATION_LIMIT, WIDTH_LIMIT},
16		sys::{
17			cgroup_max_tasks,
18			compute::{
19				CORE_IDS_FROM_OS, available_parallelism, cores_available, is_core_available,
20			},
21			max_threads,
22			storage::{self, MultiDevice, md_discover},
23		},
24	},
25	warn,
26};
27
28use super::{QUEUE_LIMIT, WORKER_LIMIT};
29
30/// Determine storage hardware capabilities of the system for configuring the
31/// shape of the database frontend threadpool.
32///
33/// Returns a tuple of:
34/// - `topology` Vector mapping hardware cores to hardware queues. Systems with
35///   fewer queues than cores will see queue ID's repeated. Systems with the
36///   same or more queues as cores will usually see a 1:1 association of core
37///   ID's to queue ID's. Systems with sparse core assignments will see 0 for
38///   core ID positions not available to the process. Systems where detection
39///   failed will see a default of 1:1 core identity as a best-guess maintaining
40///   core locality.
41/// - `workers` Vector mapping hardware queues to the number of threads to spawn
42///   in service of that queue. Systems with fewer queues than cores will set an
43///   affinity mask for each thread to multiple cores based on the topology.
44///   Systems with equal or more hardware queues than cores will set a single
45///   affinity for each thread.
46/// - `queues` Vector of software mpmc queues to create and the size of each
47///   queue. Each indice is associated with a thread-pool of workers which it
48///   feeds requests from various tokio tasks. When this queue reaches capacity
49///   the tokio task must yield.
50#[tracing::instrument(
51	level = INFO_SPAN_LEVEL,
52	skip_all,
53	ret(level = "trace"),
54)]
55pub(super) fn configure(server: &Arc<Server>) -> (Vec<usize>, Vec<usize>, Vec<usize>) {
56	let config = &server.config;
57	let num_cores = available_parallelism();
58
59	let cores_max = cores_available()
60		.last()
61		.unwrap_or(0)
62		.saturating_add(1);
63
64	let path: PathBuf = config.database_path.clone();
65	let device_name = storage::name_from_path(&path)
66		.log_debug_err()
67		.ok();
68
69	let devices = discover_devices(&path);
70	let topology_detected = devices.md.is_empty().is_false();
71	debug!(?topology_detected, ?device_name, ?devices);
72
73	let default_worker_count = topology_detected
74		.is_false()
75		.then_some(config.database.pool.workers)
76		.map(|workers| workers.saturating_mul(num_cores));
77
78	let total_tags = sum_total_tags(&devices, default_worker_count);
79	let topology = compute_topology(&devices, topology_detected, cores_max);
80	let max_workers =
81		compute_max_workers(&devices, default_worker_count, config.database.pool.max_workers);
82
83	let chan_limit = expected!(max_workers / num_cores)
84		.saturating_sub(8)
85		.saturating_add(1)
86		.next_multiple_of(8);
87
88	let workers =
89		compute_workers(&devices, config, default_worker_count, topology.len(), chan_limit);
90
91	let workers = constrain_workers(workers, max_workers);
92
93	let queues: Vec<usize> = workers
94		.iter()
95		.map(|count| {
96			count
97				.saturating_mul(config.database.pool.queue_mult)
98				.min(QUEUE_LIMIT.1)
99		})
100		.collect();
101
102	let total_workers = workers.iter().sum::<usize>();
103	let total_capacity = queues.iter().sum::<usize>();
104	let num_queues = queues.iter().filter(|&&cap| cap > 0).count();
105
106	if config.runtime.stream.width_scale > 0.0 {
107		update_stream_width(server, num_queues, total_workers, total_capacity);
108	}
109
110	log_topology(
111		topology_detected,
112		device_name.as_deref(),
113		num_cores,
114		&topology,
115		&workers,
116		&queues,
117		num_queues,
118		total_workers,
119		total_tags,
120		total_capacity,
121	);
122
123	warn_task_budget(total_workers, num_cores);
124
125	assert!(total_workers > 0, "some workers expected");
126	// A queue's capacity is positive exactly when its group is populated, so
127	// num_queues is the floor constrain_workers guarantees.
128	debug_assert!(total_workers <= max_workers.max(num_queues), "spawning too many workers");
129
130	assert!(!queues.is_empty(), "some queues expected");
131	assert!(!queues.iter().copied().all(is_equal_to!(0)), "positive queue capacity expected");
132
133	(topology, workers, queues)
134}
135
136/// Discovers the storage queues backing `path` when core IDs can be matched
137/// against their CPU lists.
138///
139/// Nominal core IDs would match the wrong CPUs, so the topology then stays
140/// undetected and the pool takes its defaults.
141fn discover_devices(path: &Path) -> MultiDevice {
142	CORE_IDS_FROM_OS
143		.then(|| md_discover(path))
144		.unwrap_or_default()
145}
146
147/// Sum the total number of possible tags. Without hardware detection this
148/// reduces to the default worker count. The thread-worker model never
149/// approaches actual NVMe capacity, but the value still informs request
150/// capacity downstream.
151fn sum_total_tags(devices: &MultiDevice, default_worker_count: Option<usize>) -> usize {
152	devices
153		.md
154		.iter()
155		.flat_map(|md| md.mq.iter())
156		.filter(|mq| mq.cpu_list.iter().copied().any(is_core_available))
157		.filter_map(|mq| mq.nr_tags)
158		.chain(default_worker_count)
159		.fold(0_usize, usize::saturating_add)
160}
161
162/// Map cores to their associated hardware queue. Shared queues repeat across
163/// cores; sparse unavailable cores default to 0; undetected hardware falls back
164/// to the core identity as a best-guess maintaining core locality.
165fn compute_topology(
166	devices: &MultiDevice,
167	topology_detected: bool,
168	cores_max: usize,
169) -> Vec<usize> {
170	devices
171		.md
172		.iter()
173		.flat_map(|md| md.mq.iter())
174		.fold(vec![0; cores_max], |mut topology, mq| {
175			mq.cpu_list
176				.iter()
177				.filter(|&&id| id < cores_max)
178				.filter(|&&id| is_core_available(id))
179				.for_each(|&id| {
180					topology[id] = mq.id;
181				});
182
183			topology
184		})
185		.into_iter()
186		.enumerate()
187		.map(|(core_id, queue_id)| {
188			topology_detected
189				.then_some(queue_id)
190				.unwrap_or(core_id)
191		})
192		.collect()
193}
194
195/// Determine an ideal max worker count based on true capacity. The true value
196/// is rarely attainable in a thread-worker model so the result is clamped by
197/// both the rlimit-derived budget and the static `WORKER_LIMIT` range.
198fn compute_max_workers(
199	devices: &MultiDevice,
200	default_worker_count: Option<usize>,
201	max_workers_cfg: usize,
202) -> usize {
203	let max_threads = max_threads()
204		.map(at!(0))
205		.unwrap_or(usize::MAX)
206		.saturating_div(3);
207
208	devices
209		.md
210		.iter()
211		.flat_map(|md| md.mq.iter())
212		.filter_map(|mq| mq.nr_tags)
213		.chain(default_worker_count)
214		.fold(0_usize, usize::saturating_add)
215		.min(max_workers_cfg)
216		.clamp(WORKER_LIMIT.0, max_threads)
217		.clamp(WORKER_LIMIT.0, WORKER_LIMIT.1)
218}
219
220/// Determine the worker groupings. Each indice represents a hardware queue and
221/// contains the number of workers which will service it. The vector is
222/// truncated to the number of cores on systems with multiple hardware queues
223/// per core, and the per-pool count is capped well below NVMe capacity.
224fn compute_workers(
225	devices: &MultiDevice,
226	config: &Config,
227	default_worker_count: Option<usize>,
228	topology_len: usize,
229	chan_limit: usize,
230) -> Vec<usize> {
231	let default_workers = default_worker_count
232		.into_iter()
233		.cycle()
234		.enumerate()
235		.map(move |(core_id, count)| {
236			is_core_available(core_id)
237				.then_some(count)
238				.unwrap_or(0)
239				.min(chan_limit)
240		});
241
242	devices
243		.md
244		.iter()
245		.inspect(|md| debug!(?md))
246		.flat_map(|md| md.mq.iter())
247		.map(|mq| {
248			let shares = mq
249				.cpu_list
250				.iter()
251				.filter(|&&id| is_core_available(id))
252				.count();
253
254			let conf_limit = config
255				.database
256				.pool
257				.workers_limit
258				.saturating_mul(shares);
259
260			let hard_limit = devices
261				.md
262				.iter()
263				.filter(|_| shares > 0)
264				.fold(0_usize, |acc, mq| {
265					mq.nr_requests
266						.map(|nr| nr.min(conf_limit))
267						.or(Some(conf_limit))
268						.map(|nr| acc.saturating_add(nr))
269						.unwrap_or(acc)
270				});
271
272			let tags = mq
273				.nr_tags
274				.unwrap_or(WORKER_LIMIT.0)
275				.min(hard_limit)
276				.min(chan_limit);
277
278			debug!(?mq, ?shares, ?tags, ?conf_limit, ?hard_limit, ?chan_limit);
279
280			tags
281		})
282		.chain(default_workers)
283		.take(topology_len)
284		.collect()
285}
286
287/// Reduce the worker groups until their total satisfies the configured
288/// maximum.
289///
290/// Group counts are levelled down toward an even share of the maximum, with
291/// the remainder spread one apiece over the leading populated groups. Every
292/// populated group retains at least one worker so its queue keeps service,
293/// making the number of populated groups the effective floor.
294fn constrain_workers(mut workers: Vec<usize>, max_workers: usize) -> Vec<usize> {
295	let total: usize = workers.iter().sum();
296
297	if total <= max_workers {
298		return workers;
299	}
300
301	let num_groups = workers.iter().filter(|&&count| count > 0).count();
302
303	let share = expected!(max_workers / num_groups);
304	let extra = expected!(max_workers % num_groups);
305
306	workers
307		.iter_mut()
308		.filter(|count| **count > 0)
309		.enumerate()
310		.for_each(|(rank, count)| {
311			*count = share
312				.saturating_add(usize::from(rank < extra))
313				.max(1)
314				.min(*count);
315		});
316
317	workers
318}
319
320#[expect(clippy::too_many_arguments)]
321fn log_topology(
322	topology_detected: bool,
323	device_name: Option<&str>,
324	num_cores: usize,
325	topology: &[usize],
326	workers: &[usize],
327	queues: &[usize],
328	num_queues: usize,
329	total_workers: usize,
330	total_tags: usize,
331	total_capacity: usize,
332) {
333	if topology_detected {
334		debug_info!(?num_cores, ?topology, ?workers, ?queues, "Frontend topology",);
335		info!(
336			device_name = ?device_name.unwrap_or("None"),
337			?num_queues,
338			?total_workers,
339			?total_tags,
340			?total_capacity,
341			stream_width = ?stream::automatic_width(),
342			amplification = ?stream::automatic_amplification(),
343			"Frontend topology",
344		);
345	} else {
346		debug_info!(?num_cores, ?topology, ?workers, ?queues, "Frontend topology (defaults)");
347		debug_warn!(
348			device_name = ?device_name.unwrap_or("None"),
349			?total_workers,
350			?total_capacity,
351			stream_width = ?stream::automatic_width(),
352			amplification = ?stream::automatic_amplification(),
353			"Storage hardware not detected for database directory; assuming defaults.",
354		);
355	}
356}
357
358/// Warn when the pool's threads and the rest of the process may exceed a
359/// container task limit.
360///
361/// The check is advisory and the pool is not resized: which of the two
362/// numbers is the wrong one is the operator's call. A host whose cgroup sets
363/// no task limit is skipped.
364fn warn_task_budget(total_workers: usize, num_cores: usize) {
365	// Threads outside the pool measured 6 on two cores and 22 on eight, the
366	// tokio workers being num_cores of them.
367	const TASK_RESERVE: usize = 32;
368
369	let Some(max_tasks) = cgroup_max_tasks() else {
370		return;
371	};
372
373	let tasks = total_workers
374		.saturating_add(num_cores)
375		.saturating_add(TASK_RESERVE);
376
377	if tasks > max_tasks {
378		warn!(
379			?total_workers,
380			?max_tasks,
381			"The database pool may exceed this container's task limit; raise the task limit \
382			 (--pids-limit for docker and podman) or lower database.pool.max_workers."
383		);
384	}
385}
386
387#[expect(clippy::as_conversions, clippy::cast_precision_loss)]
388fn update_stream_width(
389	server: &Arc<Server>,
390	num_queues: usize,
391	total_workers: usize,
392	_total_capacity: usize,
393) {
394	assert!(num_queues > 0, "Expected at least one queue.");
395	assert!(total_workers > 0, "Expected some workers.");
396
397	let config = &server.config;
398	let scale: f64 = config
399		.runtime
400		.stream
401		.width_scale
402		.min(100.0)
403		.into();
404
405	let max_width = expected!(total_workers / num_queues);
406
407	let old_width = stream::automatic_width();
408	let old_scale_width = expected!(old_width * num_queues);
409
410	let new_scale = total_workers as f64 / old_scale_width as f64;
411	let new_scale = new_scale.clamp(1.0, 4.0);
412	let new_scale_width = new_scale * old_width as f64;
413	let new_scale_width = usize_from_f64(new_scale_width)
414		.expect("failed to convert f64 to usize")
415		.next_multiple_of(8);
416
417	let req_width = usize_from_f64(scale * new_scale_width as f64)
418		.expect("failed to convert f64 to usize")
419		.next_multiple_of(4)
420		.min(max_width)
421		.clamp(WIDTH_LIMIT.0, WIDTH_LIMIT.1);
422
423	let req_amp = new_scale * config.runtime.stream.amplification as f64;
424	let req_amp = usize_from_f64(req_amp * scale)
425		.expect("failed to convert f64 to usize")
426		.next_multiple_of(64)
427		.clamp(AMPLIFICATION_LIMIT.0, AMPLIFICATION_LIMIT.1);
428
429	let (old_width, new_width) = stream::set_width(req_width);
430	let (old_amp, new_amp) = stream::set_amplification(req_amp);
431	debug!(
432		config_scale = ?config.runtime.stream.width_scale,
433		?old_width,
434		?new_scale,
435		?new_width,
436		?old_amp,
437		?new_amp,
438		"Updated global stream width"
439	);
440}
441
442#[cfg(test)]
443mod tests {
444	use std::env::current_dir;
445
446	use super::{CORE_IDS_FROM_OS, constrain_workers, discover_devices};
447
448	#[test]
449	fn constrain_respects_the_maximum() {
450		let constrained = constrain_workers(vec![8; 24], 64);
451
452		assert_eq!(constrained.iter().sum::<usize>(), 64);
453		assert!(constrained.iter().all(|&count| count > 0));
454	}
455
456	#[test]
457	fn constrain_floors_populated_groups() {
458		assert_eq!(constrain_workers(vec![8, 0, 8, 8], 2), vec![1, 0, 1, 1]);
459	}
460
461	// Nominal IDs matched to queues 0 -> [2,3] and 1 -> [0,1] under a one-CPU quota give no workers.
462	#[test]
463	fn nominal_core_ids_skip_storage_discovery() {
464		let devices = discover_devices(&current_dir().expect("current directory"));
465
466		assert!(CORE_IDS_FROM_OS || devices.md.is_empty());
467	}
468
469	#[test]
470	fn constrain_passes_a_satisfied_total() {
471		let workers = vec![4, 0, 4];
472
473		assert_eq!(constrain_workers(workers.clone(), 8), workers);
474	}
475}