diff --git a/src/core/alloc/je.rs b/src/core/alloc/je.rs index cb83b0bf6..2ee2fba46 100644 --- a/src/core/alloc/je.rs +++ b/src/core/alloc/je.rs @@ -2,7 +2,7 @@ use std::{ any::Any, - ffi::{CStr, CString, c_char, c_void}, + ffi::{CStr, CString, c_char, c_uint, c_void}, fmt::Debug, panic::{self, AssertUnwindSafe}, ptr, @@ -218,107 +218,165 @@ macro_rules! mallctl { } pub mod this_thread { - use super::{Debug, Key, OnceLock, Result, is_nonzero, key, math}; + use super::{Debug, Key, OnceLock, Result, c_uint, is_nonzero, key, math}; pub fn trim() -> Result { decay().and_then(|()| purge()) } - pub fn purge() -> Result { notify(mallctl!("arena.0.purge")) } + pub fn purge() -> Result { + // SAFETY: `arena..purge` is an `arena..*` notification ctl. + unsafe { notify(mallctl!("arena.0.purge")) } + } - pub fn decay() -> Result { notify(mallctl!("arena.0.decay")) } + pub fn decay() -> Result { + // SAFETY: `arena..decay` is an `arena..*` notification ctl. + unsafe { notify(mallctl!("arena.0.decay")) } + } - pub fn idle() -> Result { super::notify(&mallctl!("thread.idle")) } + pub fn idle() -> Result { + // SAFETY: `thread.idle` is a non-indexed notification ctl. + unsafe { super::notify(&mallctl!("thread.idle")) } + } - pub fn flush() -> Result { super::notify(&mallctl!("thread.tcache.flush")) } + pub fn flush() -> Result { + // SAFETY: `thread.tcache.flush` is a non-indexed notification ctl. + unsafe { super::notify(&mallctl!("thread.tcache.flush")) } + } pub fn set_muzzy_decay(decay_ms: isize) -> Result { - set(mallctl!("arena.0.muzzy_decay_ms"), decay_ms) + // SAFETY: `arena..muzzy_decay_ms` has C type `ssize_t`. + unsafe { set(mallctl!("arena.0.muzzy_decay_ms"), decay_ms) } } - pub fn get_muzzy_decay() -> Result { get(mallctl!("arena.0.muzzy_decay_ms")) } + pub fn get_muzzy_decay() -> Result { + // SAFETY: `arena..muzzy_decay_ms` has C type `ssize_t`. + unsafe { get(mallctl!("arena.0.muzzy_decay_ms")) } + } pub fn set_dirty_decay(decay_ms: isize) -> Result { - set(mallctl!("arena.0.dirty_decay_ms"), decay_ms) + // SAFETY: `arena..dirty_decay_ms` has C type `ssize_t`. + unsafe { set(mallctl!("arena.0.dirty_decay_ms"), decay_ms) } } - pub fn get_dirty_decay() -> Result { get(mallctl!("arena.0.dirty_decay_ms")) } + pub fn get_dirty_decay() -> Result { + // SAFETY: `arena..dirty_decay_ms` has C type `ssize_t`. + unsafe { get(mallctl!("arena.0.dirty_decay_ms")) } + } pub fn cache_enable(enable: bool) -> Result { - super::set::(&mallctl!("thread.tcache.enabled"), enable.into()).map(is_nonzero!()) + // SAFETY: `thread.tcache.enabled` has C type `bool` (1 byte on non-MSVC). + unsafe { super::set::(&mallctl!("thread.tcache.enabled"), enable.into()) } + .map(is_nonzero!()) } pub fn is_cache_enabled() -> Result { - super::get::(&mallctl!("thread.tcache.enabled")).map(is_nonzero!()) + // SAFETY: `thread.tcache.enabled` has C type `bool` (1 byte on non-MSVC). + unsafe { super::get::(&mallctl!("thread.tcache.enabled")) }.map(is_nonzero!()) } pub fn set_arena(id: usize) -> Result { - super::set::(&mallctl!("thread.arena"), id.try_into()?).and_then(math::try_into) + // SAFETY: `thread.arena` has C type `unsigned`. + unsafe { super::set::(&mallctl!("thread.arena"), id.try_into()?) } + .and_then(math::try_into) } pub fn arena_id() -> Result { - super::get::(&mallctl!("thread.arena")).and_then(math::try_into) + // SAFETY: `thread.arena` has C type `unsigned`. + unsafe { super::get::(&mallctl!("thread.arena")) }.and_then(math::try_into) } pub fn prof_enable(enable: bool) -> Result { - super::set::(&mallctl!("thread.prof.active"), enable.into()).map(is_nonzero!()) + // SAFETY: `thread.prof.active` has C type `bool` (1 byte on non-MSVC). + unsafe { super::set::(&mallctl!("thread.prof.active"), enable.into()) } + .map(is_nonzero!()) } pub fn is_prof_enabled() -> Result { - super::get::(&mallctl!("thread.prof.active")).map(is_nonzero!()) + // SAFETY: `thread.prof.active` has C type `bool` (1 byte on non-MSVC). + unsafe { super::get::(&mallctl!("thread.prof.active")) }.map(is_nonzero!()) } - pub fn reset_peak() -> Result { super::notify(&mallctl!("thread.peak.reset")) } + pub fn reset_peak() -> Result { + // SAFETY: `thread.peak.reset` is a non-indexed notification ctl. + unsafe { super::notify(&mallctl!("thread.peak.reset")) } + } - pub fn peak() -> Result { super::get(&mallctl!("thread.peak.read")) } + pub fn peak() -> Result { + // SAFETY: `thread.peak.read` has C type `uint64_t`. + unsafe { super::get(&mallctl!("thread.peak.read")) } + } #[inline] #[must_use] pub fn allocated() -> u64 { - super::get(&mallctl!("thread.allocated")).unwrap_or_default() + // SAFETY: `thread.allocated` has C type `uint64_t`. + unsafe { super::get(&mallctl!("thread.allocated")) }.unwrap_or_default() } #[inline] #[must_use] pub fn deallocated() -> u64 { - super::get(&mallctl!("thread.deallocated")).unwrap_or_default() + // SAFETY: `thread.deallocated` has C type `uint64_t`. + unsafe { super::get(&mallctl!("thread.deallocated")) }.unwrap_or_default() } - // `key` must be an `arena..*` MIB (`key.len() >= 2`). `arena_id()` returns - // this thread's own valid arena (< narenas < ARENA_LIMIT), so we skip - // `validate_arena_id()` to avoid locking `ctl_mtx` on hot paths like `decay()`. - fn notify(mut key: Key) -> Result { + /// # Safety + /// Caller must ensure `key` is an `arena..*` notification MIB + /// (`key.len() >= 2`). + // `arena_id()` returns this thread's own valid arena (< narenas < ARENA_LIMIT), + // so we skip `validate_arena_id()` to avoid locking `ctl_mtx` on hot paths like + // `decay()`. + unsafe fn notify(mut key: Key) -> Result { key[1] = arena_id()?; - super::notify(&key) + // SAFETY: Caller guarantees `key` is an `arena..*` notification MIB + // and `arena_id()` returns a valid arena index. + unsafe { super::notify(&key) } } - fn set(mut key: Key, val: T) -> Result + /// # Safety + /// Caller must ensure `key` is an `arena..*` MIB and `T` matches its + /// exact C type. + unsafe fn set(mut key: Key, val: T) -> Result where T: Copy + Debug, { // `arena_id()` is already a valid arena index; skip `validate_arena_id()`. key[1] = arena_id()?; - super::set(&key, val) + // SAFETY: Caller guarantees `T` matches `key`. + unsafe { super::set(&key, val) } } - fn get(mut key: Key) -> Result + /// # Safety + /// Caller must ensure `key` is an `arena..*` MIB and `T` matches its + /// exact C type. + unsafe fn get(mut key: Key) -> Result where T: Copy + Debug, { // `arena_id()` is already a valid arena index; skip `validate_arena_id()`. key[1] = arena_id()?; - super::get(&key) + // SAFETY: Caller guarantees `T` matches `key`. + unsafe { super::get(&key) } } } -pub fn stats_reset() -> Result { notify(&mallctl!("stats.mutexes.reset")) } +pub fn stats_reset() -> Result { + // SAFETY: `stats.mutexes.reset` is a non-indexed notification ctl. + unsafe { notify(&mallctl!("stats.mutexes.reset")) } +} -pub fn prof_reset() -> Result { notify(&mallctl!("prof.reset")) } +pub fn prof_reset() -> Result { + // SAFETY: `prof.reset` is a non-indexed notification ctl. + unsafe { notify(&mallctl!("prof.reset")) } +} pub fn prof_enable(enable: bool) -> Result { - set::(&mallctl!("prof.active"), enable.into()).map(is_nonzero!()) + // SAFETY: `prof.active` has C type `bool` (1 byte on non-MSVC). + unsafe { set::(&mallctl!("prof.active"), enable.into()) }.map(is_nonzero!()) } pub fn is_prof_enabled() -> Result { - get::(&mallctl!("prof.active")).map(is_nonzero!()) + // SAFETY: `prof.active` has C type `bool` (1 byte on non-MSVC). + unsafe { get::(&mallctl!("prof.active")) }.map(is_nonzero!()) } pub fn trim> + Copy>(arena: I) -> Result { @@ -326,29 +384,40 @@ pub fn trim> + Copy>(arena: I) -> Result { } pub fn purge>>(arena: I) -> Result { - notify_by_arena(arena.into(), mallctl!("arena.4096.purge")) + // SAFETY: `arena..purge` is an `arena..*` notification ctl. + unsafe { notify_by_arena(arena.into(), mallctl!("arena.4096.purge")) } } pub fn decay>>(arena: I) -> Result { - notify_by_arena(arena.into(), mallctl!("arena.4096.decay")) + // SAFETY: `arena..decay` is an `arena..*` notification ctl. + unsafe { notify_by_arena(arena.into(), mallctl!("arena.4096.decay")) } } pub fn set_muzzy_decay>>(arena: I, decay_ms: isize) -> Result { match arena.into() { - | Some(arena) => set_by_arena(arena, mallctl!("arena.0.muzzy_decay_ms"), decay_ms), - | _ => set(&mallctl!("arenas.muzzy_decay_ms"), decay_ms), + // SAFETY: `arena..muzzy_decay_ms` has C type `ssize_t`. + | Some(arena) => unsafe { + set_by_arena(arena, mallctl!("arena.0.muzzy_decay_ms"), decay_ms) + }, + // SAFETY: `arenas.muzzy_decay_ms` has C type `ssize_t`. + | _ => unsafe { set(&mallctl!("arenas.muzzy_decay_ms"), decay_ms) }, } } pub fn set_dirty_decay>>(arena: I, decay_ms: isize) -> Result { match arena.into() { - | Some(arena) => set_by_arena(arena, mallctl!("arena.0.dirty_decay_ms"), decay_ms), - | _ => set(&mallctl!("arenas.dirty_decay_ms"), decay_ms), + // SAFETY: `arena..dirty_decay_ms` has C type `ssize_t`. + | Some(arena) => unsafe { + set_by_arena(arena, mallctl!("arena.0.dirty_decay_ms"), decay_ms) + }, + // SAFETY: `arenas.dirty_decay_ms` has C type `ssize_t`. + | _ => unsafe { set(&mallctl!("arenas.dirty_decay_ms"), decay_ms) }, } } pub fn background_thread_enable(enable: bool) -> Result { - set::(&mallctl!("background_thread"), enable.into()).map(is_nonzero!()) + // SAFETY: `background_thread` has C type `bool` (1 byte on non-MSVC). + unsafe { set::(&mallctl!("background_thread"), enable.into()) }.map(is_nonzero!()) } #[inline] @@ -364,20 +433,32 @@ pub fn is_percpu_arena() -> bool { percpu_arenas().is_ok_and(is_equal_to!("percp pub fn is_phycpu_arena() -> bool { percpu_arenas().is_ok_and(is_equal_to!("phycpu")) } pub fn percpu_arenas() -> Result<&'static str> { - let ptr = get::<*const c_char>(&mallctl!("opt.percpu_arena"))?; - //SAFETY: ptr points to a null-terminated string returned for - // opt.percpu_arena. + // SAFETY: `opt.percpu_arena` has C type `const char *`. + let ptr = unsafe { get::<*const c_char>(&mallctl!("opt.percpu_arena")) }?; + if ptr.is_null() { + return Err(err!("mallctl: opt.percpu_arena returned NULL")); + } + + // SAFETY: `ptr` is non-null and points to a static NUL-terminated string + // returned by jemalloc for `opt.percpu_arena`. let cstr = unsafe { CStr::from_ptr(ptr) }; cstr.to_str().map_err(Into::into) } pub fn arenas() -> Result { - get::(&mallctl!("arenas.narenas")).and_then(math::try_into) + // SAFETY: `arenas.narenas` has C type `unsigned`. + unsafe { get::(&mallctl!("arenas.narenas")) }.and_then(math::try_into) } -pub fn inc_epoch() -> Result { xchg(&mallctl!("epoch"), 1_u64) } +pub fn inc_epoch() -> Result { + // SAFETY: `epoch` has C type `uint64_t`. + unsafe { xchg(&mallctl!("epoch"), 1_u64) } +} -pub fn acq_epoch() -> Result { xchg(&mallctl!("epoch"), 0_u64) } +pub fn acq_epoch() -> Result { + // SAFETY: `epoch` has C type `uint64_t`. + unsafe { xchg(&mallctl!("epoch"), 0_u64) } +} fn validate_arena_id(id: usize, allow_all: bool) -> Result { if allow_all && id == ARENAS_ALL { @@ -392,23 +473,35 @@ fn validate_arena_id(id: usize, allow_all: bool) -> Result { } } -fn notify_by_arena(id: Option, mut key: Key) -> Result { +/// # Safety +/// Caller must ensure `key` is an `arena..*` notification MIB +/// (`key.len() >= 2`). +unsafe fn notify_by_arena(id: Option, mut key: Key) -> Result { key[1] = match id { | Some(id) => validate_arena_id(id, true)?, | None => ARENAS_ALL, }; - notify(&key) + // SAFETY: Caller guarantees `key` is an `arena..*` notification MIB and + // `key[1]` has been validated above. + unsafe { notify(&key) } } -fn set_by_arena(id: usize, mut key: Key, val: T) -> Result +/// # Safety +/// Caller must ensure `key` is an `arena..*` MIB (`key.len() >= 2`) and `T` +/// matches its exact C type. +unsafe fn set_by_arena(id: usize, mut key: Key, val: T) -> Result where T: Copy + Debug, { key[1] = validate_arena_id(id, false)?; - set(&key, val) + // SAFETY: Caller guarantees `T` matches `key`. + unsafe { set(&key, val) } } -fn notify(key: &Key) -> Result { +/// # Safety +/// Caller must ensure `key` is a valid notification MIB and any indexed arena +/// component is within bounds. +unsafe fn notify(key: &Key) -> Result { // SAFETY: jemalloc notification ctls require `oldp`, `oldlenp`, and `newp` // to be NULL and `newlen` to be 0 (otherwise `EPERM` is returned). let ret = unsafe { @@ -428,44 +521,51 @@ fn notify(key: &Key) -> Result { } } -fn set(key: &Key, val: T) -> Result +/// # Safety +/// Caller must ensure `T` matches the exact C type expected by `key`. +unsafe fn set(key: &Key, val: T) -> Result where T: Copy + Debug, { let _lock = CONTROL.write(); - let res = xchg(key, val)?; + // SAFETY: Caller guarantees `T` matches `key`. + let res = unsafe { xchg(key, val) }?; inc_epoch()?; Ok(res) } +/// # Safety +/// Caller must ensure `T` matches the exact C type expected by `key`. #[tracing::instrument( name = "get", level = "trace", skip_all, fields(?key) )] -fn get(key: &Key) -> Result +unsafe fn get(key: &Key) -> Result where T: Copy + Debug, { acq_epoch()?; - // SAFETY: T must be perfectly valid to receive value. + // SAFETY: Caller guarantees `T` matches the C type returned for `key`. unsafe { mallctl::raw::read_mib(key.as_slice()) }.map_err(map_err) } +/// # Safety +/// Caller must ensure `T` matches the exact C type expected by `key`. #[tracing::instrument( name = "xchg", level = "trace", skip_all, fields(?key, ?val) )] -fn xchg(key: &Key, val: T) -> Result +unsafe fn xchg(key: &Key, val: T) -> Result where T: Copy + Debug, { - // SAFETY: T must be the exact expected type. + // SAFETY: Caller guarantees `T` matches the C type expected by `key`. unsafe { mallctl::raw::update_mib(key.as_slice(), val) }.map_err(map_err) }