Skip to main content

tuwunel_core/utils/sys/
compute.rs

1//! CPU topology, affinity, and parallelism utilities.
2//!
3//! The helpers inspect logical-core availability and derive sibling sets for
4//! simultaneous multithreading and hardware nodes. Platform-specific
5//! implementations provide available parallelism and current-CPU data.
6
7#[cfg(all(not(feature = "cpu_affinity"), not(target_os = "openbsd")))]
8use std::thread::available_parallelism as std_parallelism;
9use std::{cell::Cell, fmt::Debug, path::PathBuf, sync::LazyLock, thread};
10
11#[cfg(all(feature = "cpu_affinity", not(target_os = "openbsd")))]
12use core_affinity::get_core_ids;
13#[cfg(feature = "cpu_affinity")]
14use core_affinity::{CoreId, set_each_for_current, set_for_current};
15
16use crate::{Result, is_equal_to};
17
18type Id = usize;
19
20type Mask = u128;
21type Masks = [Mask; MASK_BITS];
22
23const MASK_BITS: usize = CORES_MAX;
24
25/// Maximum number of cores we support; for now limited to bits of our mask
26/// integral.
27pub const CORES_MAX: usize = 128;
28
29/// Whether `cores_available` lists the operating system's CPU IDs.
30///
31/// Builds without the `cpu_affinity` feature, and OpenBSD, list nominal IDs
32/// `0..n` instead, which hardware topology such as a storage queue's CPU list
33/// cannot be matched against.
34pub const CORE_IDS_FROM_OS: bool =
35	cfg!(all(feature = "cpu_affinity", not(target_os = "openbsd")));
36
37/// The mask of logical cores available to the process (at startup).
38static CORES_AVAILABLE: LazyLock<Mask> = LazyLock::new(|| into_mask(query_cores_available()));
39
40/// Stores the mask of logical-cores with thread/HT/SMT association. Each group
41/// here makes up a physical-core.
42static SMT_TOPOLOGY: LazyLock<Masks> = LazyLock::new(init_smt_topology);
43
44/// Stores the mask of logical-core associations on a node/socket. Bits are set
45/// for all logical cores within all physical cores of the node.
46static NODE_TOPOLOGY: LazyLock<Masks> = LazyLock::new(init_node_topology);
47
48thread_local! {
49	/// Tracks the affinity for this thread. This is updated when affinities
50	/// are set via our set_affinity() interface.
51	static CORE_AFFINITY: Cell<Mask> = const { Cell::new(0) };
52}
53
54/// Sets the core affinity for this thread.
55///
56/// The IDs should be listed in `CORES_AVAILABLE`; empty input is a no-op that
57/// leaves the prior affinity unchanged. The mask is always recorded for
58/// `get_affinity`, but the thread is pinned only in builds with the
59/// `cpu_affinity` feature.
60#[tracing::instrument(
61	level = "debug",
62	skip_all,
63	fields(
64		id = ?thread::current().id(),
65		name = %thread::current().name().unwrap_or("None"),
66		set = ?ids.clone().collect::<Vec<_>>(),
67		CURRENT = %format!("[b{:b}]", CORE_AFFINITY.get()),
68		AVAILABLE = %format!("[b{:b}]", *CORES_AVAILABLE),
69	),
70)]
71pub fn set_affinity<I>(ids: I)
72where
73	I: Iterator<Item = Id> + Clone + Debug,
74{
75	let mask = into_mask(ids.inspect(|&id| {
76		debug_assert!(is_core_available(id), "setting affinity to unavailable core");
77	}));
78
79	if mask != 0 {
80		pin(mask);
81		CORE_AFFINITY.set(mask);
82	}
83}
84
85#[cfg(feature = "cpu_affinity")]
86fn pin(mask: Mask) {
87	// set_each_for_current is a no-op off Linux and Android; set_for_current is not.
88	if mask.count_ones() > 1 {
89		set_each_for_current(from_mask(mask).map(|id| CoreId { id }));
90	} else if let Some(id) = from_mask(mask).next() {
91		set_for_current(CoreId { id });
92	}
93}
94
95#[cfg(not(feature = "cpu_affinity"))]
96fn pin(_mask: Mask) {}
97
98/// Get the core affinity for this thread.
99pub fn get_affinity() -> impl Iterator<Item = Id> {
100	CORE_AFFINITY
101		.get()
102		.ne(&0)
103		.then_some(from_mask(CORE_AFFINITY.get()))
104		.or_else(|| Some(from_mask(*CORES_AVAILABLE)))
105		.into_iter()
106		.flatten()
107}
108
109/// List the cores sharing SMT-tier resources
110pub fn smt_siblings() -> impl Iterator<Item = Id> {
111	from_mask(get_affinity().fold(0_u128, |mask, id| {
112		mask | SMT_TOPOLOGY
113			.get(id)
114			.expect("ID must not exceed max cpus")
115	}))
116}
117
118/// List the cores sharing Node-tier resources relative to this threads current
119/// affinity.
120pub fn node_siblings() -> impl Iterator<Item = Id> {
121	from_mask(get_affinity().fold(0_u128, |mask, id| {
122		mask | NODE_TOPOLOGY
123			.get(id)
124			.expect("Id must not exceed max cpus")
125	}))
126}
127
128/// Get the cores sharing SMT resources relative to id.
129#[inline]
130pub fn smt_affinity(id: Id) -> impl Iterator<Item = Id> {
131	from_mask(
132		*SMT_TOPOLOGY
133			.get(id)
134			.expect("ID must not exceed max cpus"),
135	)
136}
137
138/// Get the cores sharing Node resources relative to id.
139#[inline]
140pub fn node_affinity(id: Id) -> impl Iterator<Item = Id> {
141	from_mask(
142		*NODE_TOPOLOGY
143			.get(id)
144			.expect("ID must not exceed max cpus"),
145	)
146}
147
148/// Get the number of threads which could execute in parallel based on hardware
149/// constraints of this system.
150#[cfg(not(target_os = "openbsd"))]
151#[inline]
152#[must_use]
153pub fn available_parallelism() -> usize { cores_available().count() }
154
155/// Reports the number of CPUs OpenBSD currently has online.
156///
157/// The count is read from the system directly rather than from the core mask
158/// recorded at startup, and is not narrowed by process affinity, which OpenBSD
159/// does not expose.
160#[cfg(target_os = "openbsd")]
161#[inline]
162#[must_use]
163pub fn available_parallelism() -> usize { num_cpus::get() }
164
165/// Gets the ID of the nth core available. This bijects our sequence of cores to
166/// actual ID's which may have gaps for cores which are not available.
167#[inline]
168#[must_use]
169pub fn nth_core_available(i: usize) -> Option<Id> { cores_available().nth(i) }
170
171/// Determine if core (by id) is available to the process.
172#[inline]
173#[must_use]
174pub fn is_core_available(id: Id) -> bool { cores_available().any(is_equal_to!(id)) }
175
176/// Get the list of cores available. The values were recorded at program start.
177#[inline]
178pub fn cores_available() -> impl Iterator<Item = Id> { from_mask(*CORES_AVAILABLE) }
179
180/// Returns the logical CPU currently executing the calling thread.
181///
182/// The value is a point-in-time scheduler observation and may become stale on
183/// the next instruction boundary. Linux obtains it through `sched_getcpu()`.
184#[cfg(target_os = "linux")]
185#[inline]
186pub fn getcpu() -> Result<usize> {
187	use crate::{Error, utils::math};
188
189	// SAFETY: This is part of an interface with many low-level calls taking many
190	// raw params, but it's unclear why this specific call is unsafe. Nevertheless
191	// the value obtained here is semantically unsafe because it can change on the
192	// instruction boundary trailing its own acquisition and also any other time.
193	let ret: i32 = unsafe { libc::sched_getcpu() };
194
195	#[cfg(target_os = "linux")]
196	// SAFETY: On modern linux systems with a vdso if we can optimize away the branch checking
197	// for error (see getcpu(2)) then this system call becomes a memory access.
198	unsafe {
199		std::hint::assert_unchecked(ret >= 0);
200	};
201
202	if ret == -1 {
203		return Err(Error::from_errno());
204	}
205
206	math::try_into(ret)
207}
208
209/// Reports that current-CPU queries are unsupported on this platform.
210///
211/// No scheduler observation is attempted. The result contains an I/O error with
212/// the unsupported error kind.
213#[cfg(not(target_os = "linux"))]
214#[inline]
215pub fn getcpu() -> Result<usize> { Err(crate::Error::Io(std::io::ErrorKind::Unsupported.into())) }
216
217#[cfg(all(feature = "cpu_affinity", not(target_os = "openbsd")))]
218fn query_cores_available() -> impl Iterator<Item = Id> {
219	get_core_ids()
220		.unwrap_or_default()
221		.into_iter()
222		.map(|core_id| core_id.id)
223}
224
225// Nominal IDs: without the feature nothing is pinned to them.
226#[cfg(all(not(feature = "cpu_affinity"), not(target_os = "openbsd")))]
227fn query_cores_available() -> impl Iterator<Item = Id> {
228	0..std_parallelism().map_or(1, usize::from)
229}
230
231#[cfg(target_os = "openbsd")]
232fn query_cores_available() -> impl Iterator<Item = Id> { 0..num_cpus::get() }
233
234fn init_smt_topology() -> [Mask; MASK_BITS] { [Mask::default(); MASK_BITS] }
235
236fn init_node_topology() -> [Mask; MASK_BITS] { [Mask::default(); MASK_BITS] }
237
238fn into_mask<I>(ids: I) -> Mask
239where
240	I: Iterator<Item = Id>,
241{
242	ids.inspect(|&id| {
243		debug_assert!(id < MASK_BITS, "Core ID must be < Mask::BITS at least for now");
244	})
245	.fold(Mask::default(), |mask, id| mask | (1 << id))
246}
247
248fn from_mask(v: Mask) -> impl Iterator<Item = Id> {
249	(0..MASK_BITS).filter(move |&i| (v & (1 << i)) != 0)
250}
251
252fn _sys_path(id: usize, suffix: &str) -> PathBuf {
253	format!("/sys/devices/system/cpu/cpu{id}/{suffix}").into()
254}