Skip to main content

tuwunel_core/alloc/
je.rs

1//! jemalloc allocator
2
3use std::{
4	alloc::Layout,
5	io::Write,
6	panic::catch_unwind,
7	process::abort,
8	sync::atomic::{AtomicBool, AtomicU64, Ordering},
9};
10
11#[cfg(feature = "jemalloc_conf")]
12use const_str::concat_bytes;
13use jevmalloc::{
14	Jemalloc,
15	global::hook::{ALLOC, ALLOC_ZEROED},
16	stats::print as print_stats,
17};
18pub use jevmalloc::{arenas::trim, background_thread_enable};
19#[cfg(feature = "jemalloc_conf")]
20use libc::c_char;
21use libc::{STDOUT_FILENO, c_void, write};
22
23use crate::{arrayvec::ArrayVec, utils::BoolExt};
24
25/// Line buffer for one allocation-trace record. A record of three integers at
26/// their maximum widths occupies 74 bytes.
27type TraceLine = ArrayVec<u8, 128>;
28
29/// Provides the process-wide jemalloc startup configuration.
30///
31/// Jemalloc declares this symbol as a mutable `const char *` and reads it
32/// during allocator initialization, which can occur before `main`. The
33/// NUL-terminated options tune the cache and decay thresholds. CPU-affine
34/// arenas, metadata huge pages and background thread settings are requested
35/// only on the platforms whose jemalloc provides them.
36#[cfg(feature = "jemalloc_conf")]
37#[used]
38#[unsafe(no_mangle)]
39pub static mut malloc_conf: *const c_char = MALLOC_CONF.as_ptr().cast();
40
41/// Provides the jemalloc startup configuration under the symbol name a
42/// prefixed build reads.
43///
44/// Jemalloc renames its public symbols when configured with a prefix, which
45/// `jevmalloc-sys` does for musl, Apple, Android, DragonFly and OpenBSD, and
46/// which a substituted `JEMALLOC_OVERRIDE` library may do on any target.
47/// Defining both names lets the linked allocator take whichever one it declares
48/// and leaves the other unreferenced.
49#[cfg(feature = "jemalloc_conf")]
50#[used]
51#[unsafe(export_name = "_rjem_malloc_conf")]
52pub static mut MALLOC_CONF_PREFIXED: *const c_char = MALLOC_CONF.as_ptr().cast();
53
54#[cfg(feature = "jemalloc_conf")]
55const MALLOC_CONF: &[u8] = concat_bytes!(
56	"tcache:true",
57	MALLOC_CONF_PERCPU_ARENA,
58	MALLOC_CONF_METADATA_THP,
59	MALLOC_CONF_BACKGROUND,
60	",lg_extent_max_active_fit:4",
61	",oversize_threshold:2097152",
62	",tcache_max:8192",
63	",dirty_decay_ms:16000",
64	",muzzy_decay_ms:144000",
65	//MALLOC_CONF_PROF,
66	0
67);
68
69/// Assigns arenas by CPU where jemalloc can tell which CPU a thread runs on.
70///
71/// Jemalloc needs `sched_getcpu`, or its own Apple and Windows paths. Without
72/// them a release build falls back to its default arena count and prints a
73/// notice, and a debug build aborts during allocator initialization.
74#[cfg(feature = "jemalloc_conf")]
75const MALLOC_CONF_PERCPU_ARENA: &str = cfg_select! {
76	any(
77		target_os = "linux",
78		target_os = "android",
79		target_os = "freebsd",
80		target_os = "dragonfly",
81		target_vendor = "apple",
82		target_os = "windows",
83	) => ",percpu_arena:percpu",
84	_ => "",
85};
86
87/// Asks jemalloc to back allocator metadata with huge pages where it can.
88///
89/// Jemalloc needs `MADV_HUGEPAGE` (which its build ignores on 32-bit ARM) or
90/// `memcntl`. Without either, a debug build aborts during allocator
91/// initialization, and the key's only remaining effect in release is
92/// whether the tcache stacks come from the never-purging base allocator,
93/// since base blocks keep the 2 MiB alignment and rounding of
94/// `BASE_BLOCK_MIN_ALIGN` either way.
95#[cfg(feature = "jemalloc_conf")]
96const MALLOC_CONF_METADATA_THP: &str = cfg_select! {
97	any(
98		target_os = "solaris",
99		target_os = "illumos",
100		all(
101			any(target_os = "linux", target_os = "android"),
102			not(target_arch = "arm"),
103		),
104	) => ",metadata_thp:always",
105	_ => "",
106};
107
108#[cfg(feature = "jemalloc_conf")]
109const _MALLOC_CONF_PROF: &str = cfg_select! {
110	all(feature = "jemalloc_prof", target_arch = "x86_64") => ",prof_active:false",
111	_ => "",
112};
113
114// Apple's jemalloc has no background threads and prints a notice when asked.
115#[cfg(feature = "jemalloc_conf")]
116const MALLOC_CONF_BACKGROUND: &str = cfg_select! {
117	target_vendor = "apple" => "",
118	_ => ",background_thread:false,max_background_threads:-1",
119};
120
121#[global_allocator]
122static JEMALLOC: Jemalloc = Jemalloc;
123
124static GLOBAL_ALLOCS: AtomicU64 = AtomicU64::new(0);
125static COUNT_GLOBAL_ALLOCS: AtomicBool = AtomicBool::new(false);
126static TRACE_GLOBAL_ALLOCS: AtomicBool = AtomicBool::new(false);
127
128/// Registers the allocation-observer callbacks during process startup.
129///
130/// Normal and zeroed allocations made after registration feed the same counting
131/// and tracing instrumentation. The allocator reads these slots only when
132/// `jevmalloc` is built with its `global_hooks` feature.
133#[crate::ctor(unsafe)]
134fn _static_initialization() {
135	// SAFETY: Mutable static globals in jemalloc crate; must be initialized
136	// properly and uniquely.
137	unsafe { ALLOC = Some(global_alloc_hook) };
138
139	// SAFETY: As above.
140	unsafe { ALLOC_ZEROED = Some(global_alloc_zeroed_hook) };
141}
142
143fn global_alloc_hook(layout: Layout) {
144	catch_unwind(move || handle_global_alloc(layout))
145		.map_err(|_| abort())
146		.ok();
147}
148
149fn global_alloc_zeroed_hook(layout: Layout) {
150	catch_unwind(move || handle_global_alloc(layout))
151		.map_err(|_| abort())
152		.ok();
153}
154
155fn handle_global_alloc(layout: Layout) {
156	let do_count = COUNT_GLOBAL_ALLOCS.load(Ordering::Relaxed);
157	let count = GLOBAL_ALLOCS.fetch_add(do_count.into(), Ordering::Relaxed);
158
159	if TRACE_GLOBAL_ALLOCS.load(Ordering::Relaxed) {
160		let mut buf = TraceLine::new();
161
162		writeln!(&mut buf, "{count} align={} size={}", layout.align(), layout.size())
163			.expect("writeln! to buffer failed");
164
165		// SAFETY: Valid ptr and len from buf for writing to stdout.
166		unsafe { write(STDOUT_FILENO, buf.as_ptr().cast::<c_void>(), buf.len()) }
167			.ge(&0)
168			.into_result()
169			.expect("write(2) error");
170	}
171}
172
173/// Returns the process allocation count observed by the allocator hook.
174///
175/// The counter uses relaxed ordering and advances only when internal allocation
176/// counting is enabled. It is intended for allocation measurements rather than
177/// synchronized accounting.
178#[inline]
179#[must_use]
180pub fn global_alloc_count() -> u64 { GLOBAL_ALLOCS.load(Ordering::Relaxed) }
181
182/// Collects jemalloc's UTF-8 statistics report with the supplied print options.
183///
184/// Returns `None` if jemalloc produces no report, the report exceeds 1 MiB, or
185/// the report contains invalid UTF-8.
186#[must_use]
187pub fn memory_stats(opts: &str) -> Option<String> {
188	const MAX_LENGTH: usize = 1_048_576;
189
190	let mut stats = vec![0; MAX_LENGTH];
191	let length = print_stats(opts, &mut stats).ok()?.len();
192	if length == 0 {
193		return None;
194	}
195
196	stats.truncate(length);
197	String::from_utf8(stats).ok()
198}
199
200/// Exposes jemalloc state controls associated with the calling thread.
201///
202/// These functions resolve the thread's own arena before applying the
203/// operation. They return jevmalloc's control errors unchanged.
204pub mod this_thread {
205	pub use jevmalloc::thread::this::{decay, set_muzzy_decay};
206}