mirror of
https://forgejo.ellis.link/continuwuation/continuwuity.git
synced 2026-11-21 07:57:34 +00:00
Compare commits
12
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
cf8d6091f1 | ||
|
|
b4f163aacb | ||
|
|
6c1a6099b9 | ||
|
|
97585f872c | ||
|
|
08eef0c00f | ||
|
|
b73eace66f | ||
|
|
fc4ad66af1 | ||
|
|
daabb53092 | ||
|
|
806a63cb29 | ||
|
|
cd51a01fac | ||
|
|
ffa535e0d5 | ||
|
|
4fa966212e |
No files matched your search
Generated
+6
-6
@@ -497,7 +497,7 @@ dependencies = [
|
|||||||
"regex",
|
"regex",
|
||||||
"rustc-hash",
|
"rustc-hash",
|
||||||
"shlex",
|
"shlex",
|
||||||
"syn 2.0.119",
|
"syn 3.0.10",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
@@ -5189,7 +5189,7 @@ dependencies = [
|
|||||||
[[package]]
|
[[package]]
|
||||||
name = "rust-librocksdb-sys"
|
name = "rust-librocksdb-sys"
|
||||||
version = "0.49.0+11.12.0"
|
version = "0.49.0+11.12.0"
|
||||||
source = "git+https://forgejo.ellis.link/continuwuation/rust-rocksdb-zaidoon1?rev=6021299f3e0e9455dc3f2e39c65e1dc217be8903#6021299f3e0e9455dc3f2e39c65e1dc217be8903"
|
source = "git+https://forgejo.ellis.link/continuwuation/rust-rocksdb-zaidoon1?rev=85ded20348e3dab902c239967075795e9c6d101b#85ded20348e3dab902c239967075795e9c6d101b"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"bindgen",
|
"bindgen",
|
||||||
"bzip2-sys",
|
"bzip2-sys",
|
||||||
@@ -5207,7 +5207,7 @@ dependencies = [
|
|||||||
[[package]]
|
[[package]]
|
||||||
name = "rust-rocksdb"
|
name = "rust-rocksdb"
|
||||||
version = "0.54.0"
|
version = "0.54.0"
|
||||||
source = "git+https://forgejo.ellis.link/continuwuation/rust-rocksdb-zaidoon1?rev=6021299f3e0e9455dc3f2e39c65e1dc217be8903#6021299f3e0e9455dc3f2e39c65e1dc217be8903"
|
source = "git+https://forgejo.ellis.link/continuwuation/rust-rocksdb-zaidoon1?rev=85ded20348e3dab902c239967075795e9c6d101b#85ded20348e3dab902c239967075795e9c6d101b"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"libc",
|
"libc",
|
||||||
"parking_lot",
|
"parking_lot",
|
||||||
@@ -6228,7 +6228,7 @@ dependencies = [
|
|||||||
[[package]]
|
[[package]]
|
||||||
name = "tikv-jemalloc-ctl"
|
name = "tikv-jemalloc-ctl"
|
||||||
version = "0.7.0"
|
version = "0.7.0"
|
||||||
source = "git+https://forgejo.ellis.link/continuwuation/jemallocator?rev=ac4d4861b335b199f812807996e8c9166961041a#ac4d4861b335b199f812807996e8c9166961041a"
|
source = "git+https://forgejo.ellis.link/continuwuation/jemallocator?rev=46e7821917b06636cdb5b203d312799970eaea92#46e7821917b06636cdb5b203d312799970eaea92"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"libc",
|
"libc",
|
||||||
"tikv-jemalloc-sys",
|
"tikv-jemalloc-sys",
|
||||||
@@ -6237,7 +6237,7 @@ dependencies = [
|
|||||||
[[package]]
|
[[package]]
|
||||||
name = "tikv-jemalloc-sys"
|
name = "tikv-jemalloc-sys"
|
||||||
version = "0.7.0+5.4.0-0-g7a34f18502e7b222724097cdcd499b437d189acc"
|
version = "0.7.0+5.4.0-0-g7a34f18502e7b222724097cdcd499b437d189acc"
|
||||||
source = "git+https://forgejo.ellis.link/continuwuation/jemallocator?rev=ac4d4861b335b199f812807996e8c9166961041a#ac4d4861b335b199f812807996e8c9166961041a"
|
source = "git+https://forgejo.ellis.link/continuwuation/jemallocator?rev=46e7821917b06636cdb5b203d312799970eaea92#46e7821917b06636cdb5b203d312799970eaea92"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"cc",
|
"cc",
|
||||||
"libc",
|
"libc",
|
||||||
@@ -6246,7 +6246,7 @@ dependencies = [
|
|||||||
[[package]]
|
[[package]]
|
||||||
name = "tikv-jemallocator"
|
name = "tikv-jemallocator"
|
||||||
version = "0.7.0"
|
version = "0.7.0"
|
||||||
source = "git+https://forgejo.ellis.link/continuwuation/jemallocator?rev=ac4d4861b335b199f812807996e8c9166961041a#ac4d4861b335b199f812807996e8c9166961041a"
|
source = "git+https://forgejo.ellis.link/continuwuation/jemallocator?rev=46e7821917b06636cdb5b203d312799970eaea92#46e7821917b06636cdb5b203d312799970eaea92"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"libc",
|
"libc",
|
||||||
"tikv-jemalloc-sys",
|
"tikv-jemalloc-sys",
|
||||||
|
|||||||
+4
-4
@@ -407,7 +407,7 @@ features = [
|
|||||||
|
|
||||||
[workspace.dependencies.rust-rocksdb]
|
[workspace.dependencies.rust-rocksdb]
|
||||||
git = "https://forgejo.ellis.link/continuwuation/rust-rocksdb-zaidoon1"
|
git = "https://forgejo.ellis.link/continuwuation/rust-rocksdb-zaidoon1"
|
||||||
rev = "6021299f3e0e9455dc3f2e39c65e1dc217be8903"
|
rev = "85ded20348e3dab902c239967075795e9c6d101b"
|
||||||
default-features = false
|
default-features = false
|
||||||
features = [
|
features = [
|
||||||
"multi-threaded-cf",
|
"multi-threaded-cf",
|
||||||
@@ -490,7 +490,7 @@ version = "0.49.5"
|
|||||||
# jemalloc usage
|
# jemalloc usage
|
||||||
[workspace.dependencies.tikv-jemalloc-sys]
|
[workspace.dependencies.tikv-jemalloc-sys]
|
||||||
git = "https://forgejo.ellis.link/continuwuation/jemallocator"
|
git = "https://forgejo.ellis.link/continuwuation/jemallocator"
|
||||||
rev = "ac4d4861b335b199f812807996e8c9166961041a"
|
rev = "46e7821917b06636cdb5b203d312799970eaea92"
|
||||||
default-features = false
|
default-features = false
|
||||||
features = [
|
features = [
|
||||||
"background_threads_runtime_support",
|
"background_threads_runtime_support",
|
||||||
@@ -498,7 +498,7 @@ features = [
|
|||||||
]
|
]
|
||||||
[workspace.dependencies.tikv-jemallocator]
|
[workspace.dependencies.tikv-jemallocator]
|
||||||
git = "https://forgejo.ellis.link/continuwuation/jemallocator"
|
git = "https://forgejo.ellis.link/continuwuation/jemallocator"
|
||||||
rev = "ac4d4861b335b199f812807996e8c9166961041a"
|
rev = "46e7821917b06636cdb5b203d312799970eaea92"
|
||||||
default-features = false
|
default-features = false
|
||||||
features = [
|
features = [
|
||||||
"background_threads_runtime_support",
|
"background_threads_runtime_support",
|
||||||
@@ -506,7 +506,7 @@ features = [
|
|||||||
]
|
]
|
||||||
[workspace.dependencies.tikv-jemalloc-ctl]
|
[workspace.dependencies.tikv-jemalloc-ctl]
|
||||||
git = "https://forgejo.ellis.link/continuwuation/jemallocator"
|
git = "https://forgejo.ellis.link/continuwuation/jemallocator"
|
||||||
rev = "ac4d4861b335b199f812807996e8c9166961041a"
|
rev = "46e7821917b06636cdb5b203d312799970eaea92"
|
||||||
default-features = false
|
default-features = false
|
||||||
features = ["use_std"]
|
features = ["use_std"]
|
||||||
|
|
||||||
|
|||||||
@@ -84,8 +84,9 @@ All forked dependencies are maintained under the
|
|||||||
- [rocksdb][continuwuation-rocksdb] - Fork of [facebook/rocksdb][rocksdb] via
|
- [rocksdb][continuwuation-rocksdb] - Fork of [facebook/rocksdb][rocksdb] via
|
||||||
[`@zaidoon1`][8] with liburing build fixes and GCC debug build fixes
|
[`@zaidoon1`][8] with liburing build fixes and GCC debug build fixes
|
||||||
- [jemallocator][continuwuation-jemallocator] - Fork of
|
- [jemallocator][continuwuation-jemallocator] - Fork of
|
||||||
[tikv/jemallocator][jemallocator] fixing musl builds, suspicious code, and
|
[tikv/jemallocator][jemallocator] fixing musl builds and using jemalloc's
|
||||||
adding support for redzones in Valgrind
|
`*x` APIs consistently in `GlobalAlloc` to avoid interposer/Valgrind
|
||||||
|
mismatches
|
||||||
- [rustyline-async][continuwuation-rustyline-async] - Fork of
|
- [rustyline-async][continuwuation-rustyline-async] - Fork of
|
||||||
[zyansheep/rustyline-async][rustyline-async] with tab completion callback and
|
[zyansheep/rustyline-async][rustyline-async] with tab completion callback and
|
||||||
`CTRL+\` signal quit event for Continuwuity console CLI
|
`CTRL+\` signal quit event for Continuwuity console CLI
|
||||||
|
|||||||
+428
-137
@@ -1,9 +1,12 @@
|
|||||||
//! jemalloc allocator
|
//! jemalloc allocator
|
||||||
|
|
||||||
use std::{
|
use std::{
|
||||||
cell::OnceCell,
|
any::Any,
|
||||||
ffi::{CStr, c_char, c_void},
|
ffi::{CStr, CString, c_char, c_uint, c_void},
|
||||||
fmt::Debug,
|
fmt::Debug,
|
||||||
|
panic::{self, AssertUnwindSafe},
|
||||||
|
ptr,
|
||||||
|
sync::OnceLock,
|
||||||
};
|
};
|
||||||
|
|
||||||
use arrayvec::ArrayVec;
|
use arrayvec::ArrayVec;
|
||||||
@@ -12,26 +15,61 @@ use tikv_jemalloc_sys as ffi;
|
|||||||
use tikv_jemallocator as jemalloc;
|
use tikv_jemallocator as jemalloc;
|
||||||
|
|
||||||
use crate::{
|
use crate::{
|
||||||
Result, SyncRwLock, err, is_equal_to, is_nonzero,
|
Result, err, is_equal_to, is_nonzero,
|
||||||
utils::{math, math::Tried},
|
utils::{math, math::Tried},
|
||||||
};
|
};
|
||||||
|
|
||||||
#[cfg(feature = "jemalloc_conf")]
|
#[cfg(feature = "jemalloc_conf")]
|
||||||
#[unsafe(no_mangle)]
|
#[repr(transparent)]
|
||||||
pub static malloc_conf: &[u8] = const_str::concat_bytes!(
|
struct MallocConf(*const c_char);
|
||||||
|
|
||||||
|
// SAFETY: MallocConf points to static immutable NUL-terminated bytes.
|
||||||
|
#[cfg(feature = "jemalloc_conf")]
|
||||||
|
unsafe impl Sync for MallocConf {}
|
||||||
|
|
||||||
|
#[cfg(feature = "jemalloc_conf")]
|
||||||
|
const MALLOC_CONF_BYTES: &[u8] = const_str::concat_bytes!(
|
||||||
"lg_extent_max_active_fit:4",
|
"lg_extent_max_active_fit:4",
|
||||||
",oversize_threshold:16777216",
|
",oversize_threshold:16777216",
|
||||||
",tcache_max:2097152",
|
",tcache_max:2097152",
|
||||||
",dirty_decay_ms:16000",
|
",dirty_decay_ms:16000",
|
||||||
",muzzy_decay_ms:144000",
|
",muzzy_decay_ms:144000",
|
||||||
",percpu_arena:percpu",
|
MALLOC_CONF_PLATFORM,
|
||||||
",metadata_thp:always",
|
|
||||||
",background_thread:true",
|
|
||||||
",max_background_threads:-1",
|
|
||||||
MALLOC_CONF_PROF,
|
MALLOC_CONF_PROF,
|
||||||
0
|
0
|
||||||
);
|
);
|
||||||
|
|
||||||
|
#[cfg(all(feature = "jemalloc_conf", target_os = "linux"))]
|
||||||
|
const MALLOC_CONF_PLATFORM: &str =
|
||||||
|
",percpu_arena:percpu,metadata_thp:always,background_thread:true";
|
||||||
|
#[cfg(all(feature = "jemalloc_conf", not(target_os = "linux")))]
|
||||||
|
const MALLOC_CONF_PLATFORM: &str = "";
|
||||||
|
|
||||||
|
// Targets match `NO_UNPREFIXED_MALLOC_TARGETS` in `tikv-jemalloc-sys` when
|
||||||
|
// `unprefixed_malloc_on_supported_platforms` is enabled.
|
||||||
|
#[cfg(feature = "jemalloc_conf")]
|
||||||
|
#[used]
|
||||||
|
#[allow(non_upper_case_globals)]
|
||||||
|
#[cfg_attr(
|
||||||
|
any(
|
||||||
|
target_vendor = "apple",
|
||||||
|
target_env = "musl",
|
||||||
|
target_os = "android",
|
||||||
|
target_os = "dragonfly",
|
||||||
|
),
|
||||||
|
unsafe(export_name = "_rjem_malloc_conf")
|
||||||
|
)]
|
||||||
|
#[cfg_attr(
|
||||||
|
not(any(
|
||||||
|
target_vendor = "apple",
|
||||||
|
target_env = "musl",
|
||||||
|
target_os = "android",
|
||||||
|
target_os = "dragonfly",
|
||||||
|
)),
|
||||||
|
unsafe(no_mangle)
|
||||||
|
)]
|
||||||
|
static malloc_conf: MallocConf = MallocConf(MALLOC_CONF_BYTES.as_ptr().cast());
|
||||||
|
|
||||||
#[cfg(all(feature = "jemalloc_conf", feature = "jemalloc_prof"))]
|
#[cfg(all(feature = "jemalloc_conf", feature = "jemalloc_prof"))]
|
||||||
const MALLOC_CONF_PROF: &str = ",prof_active:false";
|
const MALLOC_CONF_PROF: &str = ",prof_active:false";
|
||||||
#[cfg(all(feature = "jemalloc_conf", not(feature = "jemalloc_prof")))]
|
#[cfg(all(feature = "jemalloc_conf", not(feature = "jemalloc_prof")))]
|
||||||
@@ -39,28 +77,39 @@ const MALLOC_CONF_PROF: &str = "";
|
|||||||
|
|
||||||
#[global_allocator]
|
#[global_allocator]
|
||||||
static JEMALLOC: jemalloc::Jemalloc = jemalloc::Jemalloc;
|
static JEMALLOC: jemalloc::Jemalloc = jemalloc::Jemalloc;
|
||||||
static CONTROL: SyncRwLock<()> = SyncRwLock::new(());
|
|
||||||
|
|
||||||
type Name = ArrayVec<u8, NAME_MAX>;
|
type Name = ArrayVec<u8, NAME_MAX>;
|
||||||
type Key = ArrayVec<usize, KEY_SEGS>;
|
type Key = ArrayVec<usize, KEY_SEGS>;
|
||||||
|
|
||||||
const NAME_MAX: usize = 128;
|
const NAME_MAX: usize = 128;
|
||||||
const KEY_SEGS: usize = 8;
|
const KEY_SEGS: usize = 8;
|
||||||
|
const ARENA_LIMIT: usize = 4095;
|
||||||
|
const ARENAS_ALL: usize = 4096;
|
||||||
|
|
||||||
#[ctor::ctor(unsafe)]
|
#[ctor::ctor(unsafe)]
|
||||||
fn _static_initialization() {
|
fn _static_initialization() {
|
||||||
acq_epoch().expect("pre-initialization of jemalloc failed");
|
let mut epoch: u64 = 0;
|
||||||
acq_epoch().expect("pre-initialization of jemalloc failed");
|
let mut len = size_of::<u64>();
|
||||||
|
// SAFETY: Pre-initialize jemalloc before main() without touching Rust TLS
|
||||||
|
// or unwinding across the constructor boundary.
|
||||||
|
unsafe {
|
||||||
|
ffi::mallctl(
|
||||||
|
c"epoch".as_ptr(),
|
||||||
|
ptr::from_mut(&mut epoch).cast::<c_void>(),
|
||||||
|
&raw mut len,
|
||||||
|
ptr::null_mut(),
|
||||||
|
0,
|
||||||
|
);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[must_use]
|
#[must_use]
|
||||||
#[cfg(feature = "jemalloc_stats")]
|
#[cfg(feature = "jemalloc_stats")]
|
||||||
pub fn memory_usage() -> Option<String> {
|
pub fn memory_usage() -> Option<String> {
|
||||||
let mibs = |input: usize| {
|
let mibs = |input: usize| {
|
||||||
let kibs = input / 1024;
|
let whole = u32::try_from(input / (1024 * 1024)).unwrap_or(u32::MAX);
|
||||||
let kibs = u32::try_from(kibs).unwrap_or_default();
|
let rem_kibs = u32::try_from((input / 1024) % 1024).unwrap_or_default();
|
||||||
let kibs = f64::from(kibs);
|
f64::from(whole) + f64::from(rem_kibs) / 1024.0
|
||||||
kibs / 1024.0
|
|
||||||
};
|
};
|
||||||
|
|
||||||
let [allocated, active, mapped, metadata, resident, retained] =
|
let [allocated, active, mapped, metadata, resident, retained] =
|
||||||
@@ -81,8 +130,8 @@ pub fn memory_usage() -> Option<String> { None }
|
|||||||
pub fn memory_usage_stats() -> Option<[(&'static str, usize); 6]> {
|
pub fn memory_usage_stats() -> Option<[(&'static str, usize); 6]> {
|
||||||
use mallctl::stats;
|
use mallctl::stats;
|
||||||
|
|
||||||
// Acquire the epoch; ensure latest stats are pulled in
|
// Advance the epoch; ensure latest stats are pulled in
|
||||||
acq_epoch().ok()?;
|
inc_epoch().ok()?;
|
||||||
|
|
||||||
Some([
|
Some([
|
||||||
("allocated", stats::allocated::read().unwrap_or_default()),
|
("allocated", stats::allocated::read().unwrap_or_default()),
|
||||||
@@ -98,164 +147,239 @@ pub fn memory_usage_stats() -> Option<[(&'static str, usize); 6]> {
|
|||||||
#[cfg(not(feature = "jemalloc_stats"))]
|
#[cfg(not(feature = "jemalloc_stats"))]
|
||||||
pub fn memory_usage_stats() -> Option<[(&'static str, usize); 6]> { None }
|
pub fn memory_usage_stats() -> Option<[(&'static str, usize); 6]> { None }
|
||||||
|
|
||||||
|
struct StatsPrintState {
|
||||||
|
buf: String,
|
||||||
|
max_len: usize,
|
||||||
|
panic: Option<Box<dyn Any + Send>>,
|
||||||
|
}
|
||||||
|
|
||||||
pub fn memory_stats(opts: &str) -> Option<String> {
|
pub fn memory_stats(opts: &str) -> Option<String> {
|
||||||
const MAX_LENGTH: usize = 1_048_576;
|
const MAX_LENGTH: usize = 1_048_576;
|
||||||
|
|
||||||
let mut str = String::new();
|
let opts = CString::new(opts).ok()?;
|
||||||
let opaque = std::ptr::from_mut(&mut str).cast::<c_void>();
|
|
||||||
let opts_p: *const c_char = std::ffi::CString::new(opts)
|
|
||||||
.expect("cstring")
|
|
||||||
.into_raw()
|
|
||||||
.cast_const();
|
|
||||||
|
|
||||||
// Acquire the epoch; ensure latest stats are pulled in
|
let mut state = StatsPrintState {
|
||||||
acq_epoch().ok()?;
|
buf: String::new(),
|
||||||
|
max_len: MAX_LENGTH,
|
||||||
|
panic: None,
|
||||||
|
};
|
||||||
|
let opaque = ptr::from_mut(&mut state).cast::<c_void>();
|
||||||
|
|
||||||
// SAFETY: calls malloc_stats_print() with our string instance which must
|
// SAFETY: `state` and `opts` remain live in this frame for the duration of
|
||||||
// remain in this frame. https://docs.rs/tikv-jemalloc-sys/latest/tikv_jemalloc_sys/fn.malloc_stats_print.html
|
// `malloc_stats_print()`, which invokes `malloc_stats_cb` synchronously.
|
||||||
unsafe { ffi::malloc_stats_print(Some(malloc_stats_cb), opaque, opts_p) };
|
unsafe { ffi::malloc_stats_print(Some(malloc_stats_cb), opaque, opts.as_ptr()) };
|
||||||
|
|
||||||
str.truncate(MAX_LENGTH);
|
if let Some(payload) = state.panic {
|
||||||
|
panic::resume_unwind(payload);
|
||||||
|
}
|
||||||
|
|
||||||
Some(str)
|
Some(state.buf)
|
||||||
}
|
}
|
||||||
|
|
||||||
unsafe extern "C" fn malloc_stats_cb(opaque: *mut c_void, msg: *const c_char) {
|
unsafe extern "C" fn malloc_stats_cb(opaque: *mut c_void, msg: *const c_char) {
|
||||||
// SAFETY: we have to trust the opaque points to our String
|
// SAFETY: `opaque` points to `StatsPrintState` on the caller's stack.
|
||||||
let res: &mut String = unsafe {
|
let Some(state) = (unsafe { opaque.cast::<StatsPrintState>().as_mut() }) else {
|
||||||
opaque
|
return;
|
||||||
.cast::<String>()
|
|
||||||
.as_mut()
|
|
||||||
.expect("failed to cast void* to &mut String")
|
|
||||||
};
|
};
|
||||||
|
|
||||||
// SAFETY: we have to trust the string is null terminated.
|
if state.panic.is_some() || msg.is_null() || state.buf.len() >= state.max_len {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
// SAFETY: jemalloc guarantees `msg` is a valid NUL-terminated C string.
|
||||||
let msg = unsafe { CStr::from_ptr(msg) };
|
let msg = unsafe { CStr::from_ptr(msg) };
|
||||||
|
|
||||||
let msg = String::from_utf8_lossy(msg.to_bytes());
|
if let Err(err) = panic::catch_unwind(AssertUnwindSafe(|| {
|
||||||
res.push_str(msg.as_ref());
|
let msg = String::from_utf8_lossy(msg.to_bytes());
|
||||||
|
let remaining = state.max_len.saturating_sub(state.buf.len());
|
||||||
|
if msg.len() <= remaining {
|
||||||
|
state.buf.push_str(msg.as_ref());
|
||||||
|
} else {
|
||||||
|
if let Some(prefix) = msg.get(..msg.floor_char_boundary(remaining)) {
|
||||||
|
state.buf.push_str(prefix);
|
||||||
|
}
|
||||||
|
state.max_len = state.buf.len();
|
||||||
|
}
|
||||||
|
})) {
|
||||||
|
state.panic = Some(err);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
macro_rules! mallctl {
|
macro_rules! mallctl {
|
||||||
($name:expr_2021) => {{
|
($name:literal) => {{
|
||||||
thread_local! {
|
static KEY: OnceLock<Key> = OnceLock::new();
|
||||||
static KEY: OnceCell<Key> = OnceCell::default();
|
|
||||||
};
|
|
||||||
|
|
||||||
KEY.with(|once| {
|
KEY.get_or_init(|| key($name).expect("failed to translate name into mib key"))
|
||||||
once.get_or_init(move || key($name).expect("failed to translate name into mib key"))
|
|
||||||
.clone()
|
|
||||||
})
|
|
||||||
}};
|
}};
|
||||||
}
|
}
|
||||||
|
|
||||||
pub mod this_thread {
|
pub mod this_thread {
|
||||||
use super::{Debug, Key, OnceCell, Result, is_nonzero, key, math};
|
use super::{Debug, Key, OnceLock, Result, c_uint, is_nonzero, key, math};
|
||||||
|
|
||||||
thread_local! {
|
|
||||||
static ALLOCATED_BYTES: OnceCell<&'static u64> = const { OnceCell::new() };
|
|
||||||
static DEALLOCATED_BYTES: OnceCell<&'static u64> = const { OnceCell::new() };
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn trim() -> Result { decay().and_then(|()| purge()) }
|
pub fn trim() -> Result { decay().and_then(|()| purge()) }
|
||||||
|
|
||||||
pub fn purge() -> Result { notify(mallctl!("arena.0.purge")) }
|
pub fn purge() -> Result {
|
||||||
|
// SAFETY: `arena.<i>.purge` is an `arena.<i>.*` notification ctl.
|
||||||
|
unsafe { notify(mallctl!("arena.0.purge")) }
|
||||||
|
}
|
||||||
|
|
||||||
pub fn decay() -> Result { notify(mallctl!("arena.0.decay")) }
|
pub fn decay() -> Result {
|
||||||
|
// SAFETY: `arena.<i>.decay` is an `arena.<i>.*` notification ctl.
|
||||||
|
unsafe { notify(mallctl!("arena.0.decay")) }
|
||||||
|
}
|
||||||
|
|
||||||
pub fn idle() -> Result { super::notify(&mallctl!("thread.idle")) }
|
pub fn idle() -> Result {
|
||||||
|
// SAFETY: `thread.idle` is a non-indexed notification ctl.
|
||||||
|
unsafe { super::notify(mallctl!("thread.idle")) }
|
||||||
|
}
|
||||||
|
|
||||||
pub fn flush() -> Result { super::notify(&mallctl!("thread.tcache.flush")) }
|
pub fn flush() -> Result {
|
||||||
|
// SAFETY: `thread.tcache.flush` is a non-indexed notification ctl.
|
||||||
|
unsafe { super::notify(mallctl!("thread.tcache.flush")) }
|
||||||
|
}
|
||||||
|
|
||||||
pub fn set_muzzy_decay(decay_ms: isize) -> Result<isize> {
|
pub fn set_muzzy_decay(decay_ms: isize) -> Result<isize> {
|
||||||
set(mallctl!("arena.0.muzzy_decay_ms"), decay_ms)
|
// SAFETY: `arena.<i>.muzzy_decay_ms` has C type `ssize_t`.
|
||||||
|
unsafe { set(mallctl!("arena.0.muzzy_decay_ms"), decay_ms) }
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn get_muzzy_decay() -> Result<isize> { get(mallctl!("arena.0.muzzy_decay_ms")) }
|
pub fn get_muzzy_decay() -> Result<isize> {
|
||||||
|
// SAFETY: `arena.<i>.muzzy_decay_ms` has C type `ssize_t`.
|
||||||
|
unsafe { get(mallctl!("arena.0.muzzy_decay_ms")) }
|
||||||
|
}
|
||||||
|
|
||||||
pub fn set_dirty_decay(decay_ms: isize) -> Result<isize> {
|
pub fn set_dirty_decay(decay_ms: isize) -> Result<isize> {
|
||||||
set(mallctl!("arena.0.dirty_decay_ms"), decay_ms)
|
// SAFETY: `arena.<i>.dirty_decay_ms` has C type `ssize_t`.
|
||||||
|
unsafe { set(mallctl!("arena.0.dirty_decay_ms"), decay_ms) }
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn get_dirty_decay() -> Result<isize> { get(mallctl!("arena.0.dirty_decay_ms")) }
|
pub fn get_dirty_decay() -> Result<isize> {
|
||||||
|
// SAFETY: `arena.<i>.dirty_decay_ms` has C type `ssize_t`.
|
||||||
|
unsafe { get(mallctl!("arena.0.dirty_decay_ms")) }
|
||||||
|
}
|
||||||
|
|
||||||
pub fn cache_enable(enable: bool) -> Result<bool> {
|
pub fn cache_enable(enable: bool) -> Result<bool> {
|
||||||
super::set::<u8>(&mallctl!("thread.tcache.enabled"), enable.into()).map(is_nonzero!())
|
// SAFETY: `thread.tcache.enabled` has C type `bool` (1 byte on
|
||||||
|
// non-MSVC).
|
||||||
|
unsafe { super::set::<u8>(mallctl!("thread.tcache.enabled"), enable.into()) }
|
||||||
|
.map(is_nonzero!())
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn is_cache_enabled() -> Result<bool> {
|
pub fn is_cache_enabled() -> Result<bool> {
|
||||||
super::get::<u8>(&mallctl!("thread.tcache.enabled")).map(is_nonzero!())
|
// SAFETY: `thread.tcache.enabled` has C type `bool` (1 byte on
|
||||||
|
// non-MSVC).
|
||||||
|
unsafe { super::get::<u8>(mallctl!("thread.tcache.enabled")) }.map(is_nonzero!())
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn set_arena(id: usize) -> Result<usize> {
|
pub fn set_arena(id: usize) -> Result<usize> {
|
||||||
super::set::<u32>(&mallctl!("thread.arena"), id.try_into()?).and_then(math::try_into)
|
// SAFETY: `thread.arena` has C type `unsigned`.
|
||||||
|
unsafe { super::set::<c_uint>(mallctl!("thread.arena"), id.try_into()?) }
|
||||||
|
.and_then(math::try_into)
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn arena_id() -> Result<usize> {
|
pub fn arena_id() -> Result<usize> {
|
||||||
super::get::<u32>(&mallctl!("thread.arena")).and_then(math::try_into)
|
// SAFETY: `thread.arena` has C type `unsigned`.
|
||||||
|
unsafe { super::get::<c_uint>(mallctl!("thread.arena")) }.and_then(math::try_into)
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn prof_enable(enable: bool) -> Result<bool> {
|
pub fn prof_enable(enable: bool) -> Result<bool> {
|
||||||
super::set::<u8>(&mallctl!("thread.prof.active"), enable.into()).map(is_nonzero!())
|
// SAFETY: `thread.prof.active` has C type `bool` (1 byte on non-MSVC).
|
||||||
|
unsafe { super::set::<u8>(mallctl!("thread.prof.active"), enable.into()) }
|
||||||
|
.map(is_nonzero!())
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn is_prof_enabled() -> Result<bool> {
|
pub fn is_prof_enabled() -> Result<bool> {
|
||||||
super::get::<u8>(&mallctl!("thread.prof.active")).map(is_nonzero!())
|
// SAFETY: `thread.prof.active` has C type `bool` (1 byte on non-MSVC).
|
||||||
|
unsafe { super::get::<u8>(mallctl!("thread.prof.active")) }.map(is_nonzero!())
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn reset_peak() -> Result { super::notify(&mallctl!("thread.peak.reset")) }
|
pub fn reset_peak() -> Result {
|
||||||
|
// SAFETY: `thread.peak.reset` is a non-indexed notification ctl.
|
||||||
|
unsafe { super::notify(mallctl!("thread.peak.reset")) }
|
||||||
|
}
|
||||||
|
|
||||||
pub fn peak() -> Result<u64> { super::get(&mallctl!("thread.peak.read")) }
|
pub fn peak() -> Result<u64> {
|
||||||
|
// SAFETY: `thread.peak.read` has C type `uint64_t`.
|
||||||
|
unsafe { super::get(mallctl!("thread.peak.read")) }
|
||||||
|
}
|
||||||
|
|
||||||
#[inline]
|
#[inline]
|
||||||
#[must_use]
|
#[must_use]
|
||||||
pub fn allocated() -> u64 {
|
pub fn allocated() -> u64 {
|
||||||
*ALLOCATED_BYTES.with(|once| init_tls_cell(once, "thread.allocatedp"))
|
// SAFETY: `thread.allocated` has C type `uint64_t`.
|
||||||
|
unsafe { super::get(mallctl!("thread.allocated")) }.unwrap_or_default()
|
||||||
}
|
}
|
||||||
|
|
||||||
#[inline]
|
#[inline]
|
||||||
#[must_use]
|
#[must_use]
|
||||||
pub fn deallocated() -> u64 {
|
pub fn deallocated() -> u64 {
|
||||||
*DEALLOCATED_BYTES.with(|once| init_tls_cell(once, "thread.deallocatedp"))
|
// SAFETY: `thread.deallocated` has C type `uint64_t`.
|
||||||
|
unsafe { super::get(mallctl!("thread.deallocated")) }.unwrap_or_default()
|
||||||
}
|
}
|
||||||
|
|
||||||
fn notify(key: Key) -> Result { super::notify_by_arena(Some(arena_id()?), key) }
|
/// # Safety
|
||||||
|
/// Caller must ensure `key` is an `arena.<i>.*` notification MIB
|
||||||
|
/// (`key.len() >= 2`).
|
||||||
|
unsafe fn notify(key: &Key) -> Result {
|
||||||
|
let mut key = key.clone();
|
||||||
|
// `arena_id()` returns this thread's own valid arena (< narenas <
|
||||||
|
// ARENA_LIMIT), so we skip `validate_arena_id()` to avoid locking
|
||||||
|
// `ctl_mtx` on hot paths like `decay()`.
|
||||||
|
key[1] = arena_id()?;
|
||||||
|
// SAFETY: Caller guarantees `key` is an `arena.<i>.*` notification MIB
|
||||||
|
// and `arena_id()` returns a valid arena index.
|
||||||
|
unsafe { super::notify(&key) }
|
||||||
|
}
|
||||||
|
|
||||||
fn set<T>(key: Key, val: T) -> Result<T>
|
/// # Safety
|
||||||
|
/// Caller must ensure `key` is an `arena.<i>.*` MIB and `T` matches its
|
||||||
|
/// exact C type.
|
||||||
|
unsafe fn set<T>(key: &Key, val: T) -> Result<T>
|
||||||
where
|
where
|
||||||
T: Copy + Debug,
|
T: Copy + Debug,
|
||||||
{
|
{
|
||||||
super::set_by_arena(Some(arena_id()?), key, val)
|
let mut key = key.clone();
|
||||||
|
// `arena_id()` is already a valid arena index; skip
|
||||||
|
// `validate_arena_id()`.
|
||||||
|
key[1] = arena_id()?;
|
||||||
|
// SAFETY: Caller guarantees `T` matches `key`.
|
||||||
|
unsafe { super::set(&key, val) }
|
||||||
}
|
}
|
||||||
|
|
||||||
fn get<T>(key: Key) -> Result<T>
|
/// # Safety
|
||||||
|
/// Caller must ensure `key` is an `arena.<i>.*` MIB and `T` matches its
|
||||||
|
/// exact C type.
|
||||||
|
unsafe fn get<T>(key: &Key) -> Result<T>
|
||||||
where
|
where
|
||||||
T: Copy + Debug,
|
T: Copy + Debug,
|
||||||
{
|
{
|
||||||
super::get_by_arena(Some(arena_id()?), key)
|
let mut key = key.clone();
|
||||||
}
|
// `arena_id()` is already a valid arena index; skip
|
||||||
|
// `validate_arena_id()`.
|
||||||
fn init_tls_cell(cell: &OnceCell<&'static u64>, name: &str) -> &'static u64 {
|
key[1] = arena_id()?;
|
||||||
cell.get_or_init(|| {
|
// SAFETY: Caller guarantees `T` matches `key`.
|
||||||
let ptr: *const u64 = super::get(&mallctl!(name)).expect("failed to obtain pointer");
|
unsafe { super::get(&key) }
|
||||||
|
|
||||||
// SAFETY: ptr points directly to the internal state of jemalloc for
|
|
||||||
// this thread
|
|
||||||
unsafe { ptr.as_ref() }.expect("pointer must not be null")
|
|
||||||
})
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn stats_reset() -> Result { notify(&mallctl!("stats.mutexes.reset")) }
|
pub fn stats_reset() -> Result {
|
||||||
|
// SAFETY: `stats.mutexes.reset` is a non-indexed notification ctl.
|
||||||
|
unsafe { notify(mallctl!("stats.mutexes.reset")) }
|
||||||
|
}
|
||||||
|
|
||||||
pub fn prof_reset() -> Result { notify(&mallctl!("prof.reset")) }
|
pub fn prof_reset() -> Result {
|
||||||
|
// SAFETY: `prof.reset` is a non-indexed notification ctl.
|
||||||
|
unsafe { notify(mallctl!("prof.reset")) }
|
||||||
|
}
|
||||||
|
|
||||||
pub fn prof_enable(enable: bool) -> Result<bool> {
|
pub fn prof_enable(enable: bool) -> Result<bool> {
|
||||||
set::<u8>(&mallctl!("prof.active"), enable.into()).map(is_nonzero!())
|
// SAFETY: `prof.active` has C type `bool` (1 byte on non-MSVC).
|
||||||
|
unsafe { set::<u8>(mallctl!("prof.active"), enable.into()) }.map(is_nonzero!())
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn is_prof_enabled() -> Result<bool> {
|
pub fn is_prof_enabled() -> Result<bool> {
|
||||||
get::<u8>(&mallctl!("prof.active")).map(is_nonzero!())
|
// SAFETY: `prof.active` has C type `bool` (1 byte on non-MSVC).
|
||||||
|
unsafe { get::<u8>(mallctl!("prof.active")) }.map(is_nonzero!())
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn trim<I: Into<Option<usize>> + Copy>(arena: I) -> Result {
|
pub fn trim<I: Into<Option<usize>> + Copy>(arena: I) -> Result {
|
||||||
@@ -263,31 +387,40 @@ pub fn trim<I: Into<Option<usize>> + Copy>(arena: I) -> Result {
|
|||||||
}
|
}
|
||||||
|
|
||||||
pub fn purge<I: Into<Option<usize>>>(arena: I) -> Result {
|
pub fn purge<I: Into<Option<usize>>>(arena: I) -> Result {
|
||||||
notify_by_arena(arena.into(), mallctl!("arena.4096.purge"))
|
// SAFETY: `arena.<i>.purge` is an `arena.<i>.*` notification ctl.
|
||||||
|
unsafe { notify_by_arena(arena.into(), mallctl!("arena.4096.purge")) }
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn decay<I: Into<Option<usize>>>(arena: I) -> Result {
|
pub fn decay<I: Into<Option<usize>>>(arena: I) -> Result {
|
||||||
notify_by_arena(arena.into(), mallctl!("arena.4096.decay"))
|
// SAFETY: `arena.<i>.decay` is an `arena.<i>.*` notification ctl.
|
||||||
|
unsafe { notify_by_arena(arena.into(), mallctl!("arena.4096.decay")) }
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn set_muzzy_decay<I: Into<Option<usize>>>(arena: I, decay_ms: isize) -> Result<isize> {
|
pub fn set_muzzy_decay<I: Into<Option<usize>>>(arena: I, decay_ms: isize) -> Result<isize> {
|
||||||
match arena.into() {
|
match arena.into() {
|
||||||
| Some(arena) =>
|
// SAFETY: `arena.<i>.muzzy_decay_ms` has C type `ssize_t`.
|
||||||
set_by_arena(Some(arena), mallctl!("arena.4096.muzzy_decay_ms"), decay_ms),
|
| Some(arena) => unsafe {
|
||||||
| _ => set(&mallctl!("arenas.muzzy_decay_ms"), decay_ms),
|
set_by_arena(arena, mallctl!("arena.0.muzzy_decay_ms"), decay_ms)
|
||||||
|
},
|
||||||
|
// SAFETY: `arenas.muzzy_decay_ms` has C type `ssize_t`.
|
||||||
|
| _ => unsafe { set(mallctl!("arenas.muzzy_decay_ms"), decay_ms) },
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn set_dirty_decay<I: Into<Option<usize>>>(arena: I, decay_ms: isize) -> Result<isize> {
|
pub fn set_dirty_decay<I: Into<Option<usize>>>(arena: I, decay_ms: isize) -> Result<isize> {
|
||||||
match arena.into() {
|
match arena.into() {
|
||||||
| Some(arena) =>
|
// SAFETY: `arena.<i>.dirty_decay_ms` has C type `ssize_t`.
|
||||||
set_by_arena(Some(arena), mallctl!("arena.4096.dirty_decay_ms"), decay_ms),
|
| Some(arena) => unsafe {
|
||||||
| _ => set(&mallctl!("arenas.dirty_decay_ms"), decay_ms),
|
set_by_arena(arena, mallctl!("arena.0.dirty_decay_ms"), decay_ms)
|
||||||
|
},
|
||||||
|
// SAFETY: `arenas.dirty_decay_ms` has C type `ssize_t`.
|
||||||
|
| _ => unsafe { set(mallctl!("arenas.dirty_decay_ms"), decay_ms) },
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn background_thread_enable(enable: bool) -> Result<bool> {
|
pub fn background_thread_enable(enable: bool) -> Result<bool> {
|
||||||
set::<u8>(&mallctl!("background_thread"), enable.into()).map(is_nonzero!())
|
// SAFETY: `background_thread` has C type `bool` (1 byte on non-MSVC).
|
||||||
|
unsafe { set::<u8>(mallctl!("background_thread"), enable.into()) }.map(is_nonzero!())
|
||||||
}
|
}
|
||||||
|
|
||||||
#[inline]
|
#[inline]
|
||||||
@@ -303,83 +436,138 @@ pub fn is_percpu_arena() -> bool { percpu_arenas().is_ok_and(is_equal_to!("percp
|
|||||||
pub fn is_phycpu_arena() -> bool { percpu_arenas().is_ok_and(is_equal_to!("phycpu")) }
|
pub fn is_phycpu_arena() -> bool { percpu_arenas().is_ok_and(is_equal_to!("phycpu")) }
|
||||||
|
|
||||||
pub fn percpu_arenas() -> Result<&'static str> {
|
pub fn percpu_arenas() -> Result<&'static str> {
|
||||||
let ptr = get::<*const c_char>(&mallctl!("opt.percpu_arena"))?;
|
// SAFETY: `opt.percpu_arena` has C type `const char *`.
|
||||||
//SAFETY: ptr points to a null-terminated string returned for
|
let ptr = unsafe { get::<*const c_char>(mallctl!("opt.percpu_arena")) }?;
|
||||||
// opt.percpu_arena.
|
if ptr.is_null() {
|
||||||
|
return Err(err!("mallctl: opt.percpu_arena returned NULL"));
|
||||||
|
}
|
||||||
|
|
||||||
|
// SAFETY: `ptr` is non-null and points to a static NUL-terminated string
|
||||||
|
// returned by jemalloc for `opt.percpu_arena`.
|
||||||
let cstr = unsafe { CStr::from_ptr(ptr) };
|
let cstr = unsafe { CStr::from_ptr(ptr) };
|
||||||
cstr.to_str().map_err(Into::into)
|
cstr.to_str().map_err(Into::into)
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn arenas() -> Result<usize> {
|
pub fn arenas() -> Result<usize> {
|
||||||
get::<u32>(&mallctl!("arenas.narenas")).and_then(math::try_into)
|
// SAFETY: `arenas.narenas` has C type `unsigned`.
|
||||||
|
unsafe { get::<c_uint>(mallctl!("arenas.narenas")) }.and_then(math::try_into)
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn inc_epoch() -> Result<u64> { xchg(&mallctl!("epoch"), 1_u64) }
|
pub fn inc_epoch() -> Result<u64> {
|
||||||
|
// SAFETY: `epoch` has C type `uint64_t`.
|
||||||
pub fn acq_epoch() -> Result<u64> { xchg(&mallctl!("epoch"), 0_u64) }
|
unsafe { xchg(mallctl!("epoch"), 1_u64) }
|
||||||
|
|
||||||
fn notify_by_arena(id: Option<usize>, mut key: Key) -> Result {
|
|
||||||
key[1] = id.unwrap_or(4096);
|
|
||||||
notify(&key)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
fn set_by_arena<T>(id: Option<usize>, mut key: Key, val: T) -> Result<T>
|
pub fn acq_epoch() -> Result<u64> {
|
||||||
|
// SAFETY: `epoch` has C type `uint64_t`.
|
||||||
|
unsafe { get(mallctl!("epoch")) }
|
||||||
|
}
|
||||||
|
|
||||||
|
fn validate_arena_id(id: usize, allow_all: bool) -> Result<usize> {
|
||||||
|
if allow_all && id == ARENAS_ALL {
|
||||||
|
return Ok(id);
|
||||||
|
}
|
||||||
|
|
||||||
|
let narenas = arenas()?.min(ARENA_LIMIT);
|
||||||
|
if id < narenas {
|
||||||
|
Ok(id)
|
||||||
|
} else {
|
||||||
|
Err(err!("mallctl: invalid arena index {id} (narenas: {narenas})"))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// # Safety
|
||||||
|
/// Caller must ensure `key` is an `arena.<i>.*` notification MIB
|
||||||
|
/// (`key.len() >= 2`).
|
||||||
|
unsafe fn notify_by_arena(id: Option<usize>, key: &Key) -> Result {
|
||||||
|
let mut key = key.clone();
|
||||||
|
key[1] = match id {
|
||||||
|
| Some(id) => validate_arena_id(id, true)?,
|
||||||
|
| None => ARENAS_ALL,
|
||||||
|
};
|
||||||
|
// SAFETY: Caller guarantees `key` is an `arena.<i>.*` notification MIB and
|
||||||
|
// `key[1]` has been validated above.
|
||||||
|
unsafe { notify(&key) }
|
||||||
|
}
|
||||||
|
|
||||||
|
/// # Safety
|
||||||
|
/// Caller must ensure `key` is an `arena.<i>.*` MIB (`key.len() >= 2`) and `T`
|
||||||
|
/// matches its exact C type.
|
||||||
|
unsafe fn set_by_arena<T>(id: usize, key: &Key, val: T) -> Result<T>
|
||||||
where
|
where
|
||||||
T: Copy + Debug,
|
T: Copy + Debug,
|
||||||
{
|
{
|
||||||
key[1] = id.unwrap_or(4096);
|
let mut key = key.clone();
|
||||||
set(&key, val)
|
key[1] = validate_arena_id(id, false)?;
|
||||||
|
// SAFETY: Caller guarantees `T` matches `key`.
|
||||||
|
unsafe { set(&key, val) }
|
||||||
}
|
}
|
||||||
|
|
||||||
fn get_by_arena<T>(id: Option<usize>, mut key: Key) -> Result<T>
|
/// # Safety
|
||||||
|
/// Caller must ensure `key` is a valid notification MIB and any indexed arena
|
||||||
|
/// component is within bounds.
|
||||||
|
unsafe fn notify(key: &Key) -> Result {
|
||||||
|
// SAFETY: jemalloc notification ctls require `oldp`, `oldlenp`, and `newp`
|
||||||
|
// to be NULL and `newlen` to be 0 (otherwise `EPERM` is returned).
|
||||||
|
let ret = unsafe {
|
||||||
|
ffi::mallctlbymib(
|
||||||
|
key.as_ptr(),
|
||||||
|
key.len(),
|
||||||
|
ptr::null_mut(),
|
||||||
|
ptr::null_mut(),
|
||||||
|
ptr::null_mut(),
|
||||||
|
0,
|
||||||
|
)
|
||||||
|
};
|
||||||
|
if ret == 0 {
|
||||||
|
Ok(())
|
||||||
|
} else {
|
||||||
|
Err(err!("mallctl: {}", std::io::Error::from_raw_os_error(ret)))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Does not advance `epoch`; callers reading cached `stats.*` must call
|
||||||
|
/// `inc_epoch()` explicitly.
|
||||||
|
///
|
||||||
|
/// # Safety
|
||||||
|
/// Caller must ensure `T` matches the exact C type expected by `key`.
|
||||||
|
unsafe fn set<T>(key: &Key, val: T) -> Result<T>
|
||||||
where
|
where
|
||||||
T: Copy + Debug,
|
T: Copy + Debug,
|
||||||
{
|
{
|
||||||
key[1] = id.unwrap_or(4096);
|
// SAFETY: Caller guarantees `T` matches `key`.
|
||||||
get(&key)
|
unsafe { xchg(key, val) }
|
||||||
}
|
|
||||||
|
|
||||||
fn notify(key: &Key) -> Result { xchg(key, ()) }
|
|
||||||
|
|
||||||
fn set<T>(key: &Key, val: T) -> Result<T>
|
|
||||||
where
|
|
||||||
T: Copy + Debug,
|
|
||||||
{
|
|
||||||
let _lock = CONTROL.write();
|
|
||||||
let res = xchg(key, val)?;
|
|
||||||
inc_epoch()?;
|
|
||||||
|
|
||||||
Ok(res)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// # Safety
|
||||||
|
/// Caller must ensure `T` matches the exact C type expected by `key`.
|
||||||
#[tracing::instrument(
|
#[tracing::instrument(
|
||||||
name = "get",
|
name = "get",
|
||||||
level = "trace",
|
level = "trace",
|
||||||
skip_all,
|
skip_all,
|
||||||
fields(?key)
|
fields(?key)
|
||||||
)]
|
)]
|
||||||
fn get<T>(key: &Key) -> Result<T>
|
unsafe fn get<T>(key: &Key) -> Result<T>
|
||||||
where
|
where
|
||||||
T: Copy + Debug,
|
T: Copy + Debug,
|
||||||
{
|
{
|
||||||
acq_epoch()?;
|
// SAFETY: Caller guarantees `T` matches the C type returned for `key`.
|
||||||
acq_epoch()?;
|
|
||||||
|
|
||||||
// SAFETY: T must be perfectly valid to receive value.
|
|
||||||
unsafe { mallctl::raw::read_mib(key.as_slice()) }.map_err(map_err)
|
unsafe { mallctl::raw::read_mib(key.as_slice()) }.map_err(map_err)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// # Safety
|
||||||
|
/// Caller must ensure `T` matches the exact C type expected by `key`.
|
||||||
#[tracing::instrument(
|
#[tracing::instrument(
|
||||||
name = "xchg",
|
name = "xchg",
|
||||||
level = "trace",
|
level = "trace",
|
||||||
skip_all,
|
skip_all,
|
||||||
fields(?key, ?val)
|
fields(?key, ?val)
|
||||||
)]
|
)]
|
||||||
fn xchg<T>(key: &Key, val: T) -> Result<T>
|
unsafe fn xchg<T>(key: &Key, val: T) -> Result<T>
|
||||||
where
|
where
|
||||||
T: Copy + Debug,
|
T: Copy + Debug,
|
||||||
{
|
{
|
||||||
// SAFETY: T must be the exact expected type.
|
// SAFETY: Caller guarantees `T` matches the C type expected by `key`.
|
||||||
unsafe { mallctl::raw::update_mib(key.as_slice(), val) }.map_err(map_err)
|
unsafe { mallctl::raw::update_mib(key.as_slice(), val) }.map_err(map_err)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -387,6 +575,9 @@ fn key(name: &str) -> Result<Key> {
|
|||||||
// tikv asserts the output buffer length is tight to the number of required
|
// tikv asserts the output buffer length is tight to the number of required
|
||||||
// mibs so we slice that down here.
|
// mibs so we slice that down here.
|
||||||
let segs = name.chars().filter(is_equal_to!(&'.')).count().try_add(1)?;
|
let segs = name.chars().filter(is_equal_to!(&'.')).count().try_add(1)?;
|
||||||
|
if segs > KEY_SEGS {
|
||||||
|
return Err(err!("mallctl: key has too many segments ({segs} > {KEY_SEGS})"));
|
||||||
|
}
|
||||||
|
|
||||||
let name = self::name(name)?;
|
let name = self::name(name)?;
|
||||||
let mut buf = [0_usize; KEY_SEGS];
|
let mut buf = [0_usize; KEY_SEGS];
|
||||||
@@ -396,6 +587,10 @@ fn key(name: &str) -> Result<Key> {
|
|||||||
}
|
}
|
||||||
|
|
||||||
fn name(name: &str) -> Result<Name> {
|
fn name(name: &str) -> Result<Name> {
|
||||||
|
if name.as_bytes().contains(&0) {
|
||||||
|
return Err(err!("mallctl: key contains interior NUL byte"));
|
||||||
|
}
|
||||||
|
|
||||||
let mut buf = Name::new();
|
let mut buf = Name::new();
|
||||||
buf.try_extend_from_slice(name.as_bytes())?;
|
buf.try_extend_from_slice(name.as_bytes())?;
|
||||||
buf.try_extend_from_slice(b"\0")?;
|
buf.try_extend_from_slice(b"\0")?;
|
||||||
@@ -406,3 +601,99 @@ fn name(name: &str) -> Result<Name> {
|
|||||||
fn map_err(error: tikv_jemalloc_ctl::Error) -> crate::Error {
|
fn map_err(error: tikv_jemalloc_ctl::Error) -> crate::Error {
|
||||||
err!("mallctl: {}", error.to_string())
|
err!("mallctl: {}", error.to_string())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn notify_ctls_succeed() {
|
||||||
|
this_thread::flush().expect("thread.tcache.flush");
|
||||||
|
this_thread::decay().expect("arena.<i>.decay");
|
||||||
|
this_thread::purge().expect("arena.<i>.purge");
|
||||||
|
this_thread::trim().expect("this_thread::trim");
|
||||||
|
trim(None).expect("trim(None)");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
#[cfg(feature = "jemalloc_stats")]
|
||||||
|
fn allocated_and_deallocated_do_not_alias() {
|
||||||
|
let a0 = this_thread::allocated();
|
||||||
|
let d0 = this_thread::deallocated();
|
||||||
|
|
||||||
|
let buf = std::hint::black_box(vec![0_u8; 64 * 1024]);
|
||||||
|
let a1 = this_thread::allocated();
|
||||||
|
assert!(a1 > a0, "expected allocated to increase: {a0} -> {a1}");
|
||||||
|
|
||||||
|
drop(buf);
|
||||||
|
let d1 = this_thread::deallocated();
|
||||||
|
assert!(d1 > d0, "expected deallocated to increase: {d0} -> {d1}");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn memory_stats_handles_valid_and_nul_opts() {
|
||||||
|
assert!(memory_stats("invalid\0opts").is_none());
|
||||||
|
let stats = memory_stats("gmal").expect("memory_stats");
|
||||||
|
assert!(!stats.is_empty());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn key_too_many_segments_returns_err() { key("a.b.c.d.e.f.g.h.i").unwrap_err(); }
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn key_interior_nul_returns_err() { key("a.b\0.c").unwrap_err(); }
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn arena_index_bounds_rejected() {
|
||||||
|
purge(Some(ARENAS_ALL)).expect("purge(MALLCTL_ARENAS_ALL)");
|
||||||
|
decay(Some(ARENAS_ALL)).expect("decay(MALLCTL_ARENAS_ALL)");
|
||||||
|
|
||||||
|
let narenas = arenas().expect("arenas.narenas");
|
||||||
|
purge(Some(narenas)).unwrap_err();
|
||||||
|
decay(Some(narenas)).unwrap_err();
|
||||||
|
set_dirty_decay(Some(narenas), -1).unwrap_err();
|
||||||
|
set_muzzy_decay(Some(narenas), -1).unwrap_err();
|
||||||
|
|
||||||
|
purge(Some(4097)).unwrap_err();
|
||||||
|
decay(Some(4097)).unwrap_err();
|
||||||
|
trim(Some(4097)).unwrap_err();
|
||||||
|
purge(Some(usize::MAX)).unwrap_err();
|
||||||
|
|
||||||
|
set_dirty_decay(Some(ARENAS_ALL), -1).unwrap_err();
|
||||||
|
set_muzzy_decay(Some(ARENAS_ALL), -1).unwrap_err();
|
||||||
|
set_dirty_decay(Some(4097), -1).unwrap_err();
|
||||||
|
set_muzzy_decay(Some(4097), -1).unwrap_err();
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
#[cfg(feature = "jemalloc_conf")]
|
||||||
|
fn malloc_conf_is_applied() {
|
||||||
|
if std::env::var_os("MALLOC_CONF").is_some()
|
||||||
|
|| std::env::var_os("_RJEM_MALLOC_CONF").is_some()
|
||||||
|
{
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
// SAFETY: `opt.dirty_decay_ms` and `opt.muzzy_decay_ms` have C type
|
||||||
|
// `ssize_t`.
|
||||||
|
let dirty =
|
||||||
|
unsafe { get::<isize>(mallctl!("opt.dirty_decay_ms")) }.expect("opt.dirty_decay_ms");
|
||||||
|
let muzzy =
|
||||||
|
unsafe { get::<isize>(mallctl!("opt.muzzy_decay_ms")) }.expect("opt.muzzy_decay_ms");
|
||||||
|
assert_eq!(dirty, 16_000, "expected malloc_conf dirty_decay_ms:16000");
|
||||||
|
assert_eq!(muzzy, 144_000, "expected malloc_conf muzzy_decay_ms:144000");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn percpu_arenas_returns_valid_mode() {
|
||||||
|
let mode = percpu_arenas().expect("opt.percpu_arena");
|
||||||
|
assert!(matches!(mode, "disabled" | "percpu" | "phycpu"), "unexpected mode: {mode}");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn epoch_read_and_advance() {
|
||||||
|
let e0 = acq_epoch().expect("acq_epoch");
|
||||||
|
let e1 = inc_epoch().expect("inc_epoch");
|
||||||
|
assert!(e1 > e0, "expected inc_epoch to advance epoch: {e0} -> {e1}");
|
||||||
|
}
|
||||||
|
}
|
||||||
Reference in new issue
Block a user