Compare commits

...
Author SHA1 Message Date
timedout 08098c5e88 chore: Add newsfrag 2026-11-21 15:15:28 +00:00
Jade Ellis 8e54ea0853 fix: Use mimalloc for c/c++ code by default 2026-11-21 14:54:10 +00:00
Jade Ellis 5bb55a1285 refactor: Unify memory stats and create feature parity 2026-11-21 14:12:45 +00:00
Jade Ellis 3a849c2687 docs(main): Clarify mimalloc feature scope and jemalloc precedence 2026-11-21 13:38:29 +00:00
Jade Ellis 9e02f5b607 feat(alloc): Implement trim, memory_usage, and memory_stats for mimalloc 2026-11-21 13:38:28 +00:00
Jade Ellis 98d4022768 fix(config): Warn in release builds when both mimalloc and jemalloc are enabled 2026-11-21 13:38:28 +00:00
Jade Ellis 7294a3589d fix(alloc): Restore not(target_env = "msvc") cfg gates and gate default module 2026-11-21 13:38:28 +00:00
timedout 6aed1895b0 feat: Add mimalloc support, drop hardened_malloc 2026-11-21 13:38:27 +00:00
timedout a9377fb896 fix(service/admin): Gate IsTerminal and warn imports on console feature 2026-11-21 13:38:27 +00:00
Jade Ellis 94f5026d57 chore: Add changelog 2026-11-21 13:36:35 +00:00
Jade Ellis c04446b86e docs(alloc): Fix interrupted doc comments in je.rs 2026-11-21 13:33:55 +00:00
Jade Ellis b4f163aacb chore(deps): Upgrade jemalloc with soundness fixes 2026-11-21 01:42:15 +00:00
Jade Ellis 6c1a6099b9 test(alloc): add unit tests for jemalloc wrapper fixes 2026-11-21 01:42:15 +00:00
Jade Ellis 97585f872c perf(alloc): return &'static Key from mallctl! macro 2026-11-21 01:42:14 +00:00
Jade Ellis 08eef0c00f fix(alloc): avoid unnecessary global epoch refreshes in mallctl get and memory_stats 2026-11-21 01:42:14 +00:00
Jade Ellis b73eace66f fix(alloc): mark raw mallctl helpers unsafe and check null in percpu_arenas 2026-11-21 01:42:13 +00:00
Jade Ellis fc4ad66af1 fix(alloc): validate arena index bounds before arena.<i>.* mallctls 2026-11-21 01:42:13 +00:00
Jade Ellis daabb53092 fix(alloc): pass NULL pointers in jemalloc notify ctls 2026-11-21 01:41:17 +00:00
Jade Ellis 806a63cb29 fix(alloc): fix UB and MIB aliasing in thread::allocated/deallocated 2026-11-21 01:41:16 +00:00
Jade Ellis cd51a01fac fix(alloc): prevent leak, panic, and FFI unwind in memory_stats 2026-11-21 01:41:16 +00:00
Jade Ellis ffa535e0d5 fix(alloc): avoid TLS and panics in jemalloc ctor and key lookup 2026-11-21 01:41:15 +00:00
Jade Ellis 4fa966212e fix(alloc): export malloc_conf as thin pointer and fix options 2026-11-21 01:41:14 +00:00
19 changed files with 807 additions and 237 deletions

No files matched your search

Generated
+33 -13
View File
@@ -497,7 +497,7 @@ dependencies = [
"regex",
"rustc-hash",
"shlex",
"syn 2.0.119",
"syn 3.0.10",
]
[[package]]
@@ -991,7 +991,6 @@ dependencies = [
"figment",
"futures",
"governor",
"hardened_malloc-rs",
"http",
"http-body-util",
"hyper-util",
@@ -999,8 +998,10 @@ dependencies = [
"lettre",
"libc",
"libloading 0.9.1",
"libmimalloc-sys",
"lock_api 0.5.0",
"maplit",
"mimalloc",
"nix",
"num-traits",
"openidconnect",
@@ -1552,6 +1553,12 @@ dependencies = [
"subtle",
]
[[package]]
name = "cty"
version = "0.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b365fabc795046672053e29c954733ec3b05e4be654ab130fe8f1f94d7051f35"
[[package]]
name = "curve25519-dalek"
version = "5.0.0"
@@ -2395,12 +2402,6 @@ dependencies = [
"tokio-util",
]
[[package]]
name = "hardened_malloc-rs"
version = "0.1.2+12"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "647deb1583b14d160f85f3ff626f20b6edd366e3852c9843b06077388f794cb6"
[[package]]
name = "hashbrown"
version = "0.12.3"
@@ -3256,6 +3257,16 @@ version = "0.2.16"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981"
[[package]]
name = "libmimalloc-sys"
version = "0.1.49"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6a45a52f43e1c16f667ccfe4dd8c85b7f7c204fd5e3bf46c5b0db9a5c3c0b8e9"
dependencies = [
"cc",
"cty",
]
[[package]]
name = "libz-sys"
version = "1.1.32"
@@ -3429,6 +3440,15 @@ dependencies = [
"walkdir",
]
[[package]]
name = "mimalloc"
version = "0.1.52"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2d4139bb28d14ad1facf21d5eb8825051b326e172d216b39f6d31df53cc97862"
dependencies = [
"libmimalloc-sys",
]
[[package]]
name = "mime"
version = "0.3.18"
@@ -5189,7 +5209,7 @@ dependencies = [
[[package]]
name = "rust-librocksdb-sys"
version = "0.49.0+11.12.0"
source = "git+https://forgejo.ellis.link/continuwuation/rust-rocksdb-zaidoon1?rev=6021299f3e0e9455dc3f2e39c65e1dc217be8903#6021299f3e0e9455dc3f2e39c65e1dc217be8903"
source = "git+https://forgejo.ellis.link/continuwuation/rust-rocksdb-zaidoon1?rev=85ded20348e3dab902c239967075795e9c6d101b#85ded20348e3dab902c239967075795e9c6d101b"
dependencies = [
"bindgen",
"bzip2-sys",
@@ -5207,7 +5227,7 @@ dependencies = [
[[package]]
name = "rust-rocksdb"
version = "0.54.0"
source = "git+https://forgejo.ellis.link/continuwuation/rust-rocksdb-zaidoon1?rev=6021299f3e0e9455dc3f2e39c65e1dc217be8903#6021299f3e0e9455dc3f2e39c65e1dc217be8903"
source = "git+https://forgejo.ellis.link/continuwuation/rust-rocksdb-zaidoon1?rev=85ded20348e3dab902c239967075795e9c6d101b#85ded20348e3dab902c239967075795e9c6d101b"
dependencies = [
"libc",
"parking_lot",
@@ -6228,7 +6248,7 @@ dependencies = [
[[package]]
name = "tikv-jemalloc-ctl"
version = "0.7.0"
source = "git+https://forgejo.ellis.link/continuwuation/jemallocator?rev=ac4d4861b335b199f812807996e8c9166961041a#ac4d4861b335b199f812807996e8c9166961041a"
source = "git+https://forgejo.ellis.link/continuwuation/jemallocator?rev=46e7821917b06636cdb5b203d312799970eaea92#46e7821917b06636cdb5b203d312799970eaea92"
dependencies = [
"libc",
"tikv-jemalloc-sys",
@@ -6237,7 +6257,7 @@ dependencies = [
[[package]]
name = "tikv-jemalloc-sys"
version = "0.7.0+5.4.0-0-g7a34f18502e7b222724097cdcd499b437d189acc"
source = "git+https://forgejo.ellis.link/continuwuation/jemallocator?rev=ac4d4861b335b199f812807996e8c9166961041a#ac4d4861b335b199f812807996e8c9166961041a"
source = "git+https://forgejo.ellis.link/continuwuation/jemallocator?rev=46e7821917b06636cdb5b203d312799970eaea92#46e7821917b06636cdb5b203d312799970eaea92"
dependencies = [
"cc",
"libc",
@@ -6246,7 +6266,7 @@ dependencies = [
[[package]]
name = "tikv-jemallocator"
version = "0.7.0"
source = "git+https://forgejo.ellis.link/continuwuation/jemallocator?rev=ac4d4861b335b199f812807996e8c9166961041a#ac4d4861b335b199f812807996e8c9166961041a"
source = "git+https://forgejo.ellis.link/continuwuation/jemallocator?rev=46e7821917b06636cdb5b203d312799970eaea92#46e7821917b06636cdb5b203d312799970eaea92"
dependencies = [
"libc",
"tikv-jemalloc-sys",
+12 -13
View File
@@ -407,7 +407,7 @@ features = [
[workspace.dependencies.rust-rocksdb]
git = "https://forgejo.ellis.link/continuwuation/rust-rocksdb-zaidoon1"
rev = "6021299f3e0e9455dc3f2e39c65e1dc217be8903"
rev = "85ded20348e3dab902c239967075795e9c6d101b"
default-features = false
features = [
"multi-threaded-cf",
@@ -490,7 +490,7 @@ version = "0.49.5"
# jemalloc usage
[workspace.dependencies.tikv-jemalloc-sys]
git = "https://forgejo.ellis.link/continuwuation/jemallocator"
rev = "ac4d4861b335b199f812807996e8c9166961041a"
rev = "46e7821917b06636cdb5b203d312799970eaea92"
default-features = false
features = [
"background_threads_runtime_support",
@@ -498,7 +498,7 @@ features = [
]
[workspace.dependencies.tikv-jemallocator]
git = "https://forgejo.ellis.link/continuwuation/jemallocator"
rev = "ac4d4861b335b199f812807996e8c9166961041a"
rev = "46e7821917b06636cdb5b203d312799970eaea92"
default-features = false
features = [
"background_threads_runtime_support",
@@ -506,7 +506,7 @@ features = [
]
[workspace.dependencies.tikv-jemalloc-ctl]
git = "https://forgejo.ellis.link/continuwuation/jemallocator"
rev = "ac4d4861b335b199f812807996e8c9166961041a"
rev = "46e7821917b06636cdb5b203d312799970eaea92"
default-features = false
features = ["use_std"]
@@ -522,15 +522,6 @@ features = ["resource"]
version = "0.5.0"
default-features = false
[workspace.dependencies.hardened_malloc-rs]
version = "0.1.2"
default-features = false
features = [
"static",
"gcc",
"light",
]
[workspace.dependencies.rustyline-async]
version = "0.4.9"
default-features = false
@@ -610,6 +601,14 @@ features = ["unstable-msc4188"]
[workspace.dependencies.serde_urlencoded]
version = "0.7.1"
[workspace.dependencies.mimalloc]
version = "0.1.52"
features = ["extended"]
[workspace.dependencies.libmimalloc-sys]
version = "0.1.49"
features = ["extended"]
#
# Patches
#
+1
View File
@@ -0,0 +1 @@
Dropped support for `hardened_malloc` and add support for `mimalloc` instead. Contributed by @nex and @jade.
+1
View File
@@ -0,0 +1 @@
Fix soundness and correctness issues in the jemalloc interface, improving performance. Contributed by @Jade
+3 -2
View File
@@ -84,8 +84,9 @@ All forked dependencies are maintained under the
- [rocksdb][continuwuation-rocksdb] - Fork of [facebook/rocksdb][rocksdb] via
[`@zaidoon1`][8] with liburing build fixes and GCC debug build fixes
- [jemallocator][continuwuation-jemallocator] - Fork of
[tikv/jemallocator][jemallocator] fixing musl builds, suspicious code, and
adding support for redzones in Valgrind
[tikv/jemallocator][jemallocator] fixing musl builds and using jemalloc's
`*x` APIs consistently in `GlobalAlloc` to avoid interposer/Valgrind
mismatches
- [rustyline-async][continuwuation-rustyline-async] - Fork of
[zyansheep/rustyline-async][rustyline-async] with tab completion callback and
`CTRL+\` signal quit event for Continuwuity console CLI
+1
View File
@@ -82,6 +82,7 @@ Names below are as they appear in Prometheus. OTLP uses the dotted OpenTelemetry
| `continuwuity_database_block_cache_bytes` | gauge | `column` | RocksDB block cache usage |
| `continuwuity_cache_entries` | gauge | `name` | Entries in the auth chain and state info caches |
| `continuwuity_jemalloc_memory_bytes` | gauge | `kind` | jemalloc statistics, only with the `jemalloc_stats` feature |
| `continuwuity_mimalloc_memory_bytes` | gauge | `kind` | mimalloc statistics, only with the `mimalloc` feature |
| `process_uptime_seconds` | gauge | | Time since the server started |
| `process_cpu_time_seconds_total` | counter | `cpu_mode` | CPU time used by the process |
| `process_memory_usage_bytes` | gauge | | Resident memory (Linux) |
+6 -2
View File
@@ -1007,7 +1007,7 @@ impl crate::Context<'_> {
pub(super) async fn memory_stats(&self, opts: Option<String>) -> Result {
const OPTS: &str = "abcdefghijklmnopqrstuvwxyz";
let opts: String = OPTS
let mut mapped_opts: String = OPTS
.chars()
.filter(|&c| {
let allow_any = opts.as_ref().is_some_and(|opts| opts == "*");
@@ -1018,7 +1018,11 @@ impl crate::Context<'_> {
})
.collect();
let stats = conduwuit::alloc::memory_stats(&opts).unwrap_or_default();
if opts.as_ref().is_some_and(|opts| opts.contains('J')) {
mapped_opts.push('J');
}
let stats = conduwuit::alloc::memory_stats(&mapped_opts).unwrap_or_default();
self.write_str("```\n").await?;
self.write_str(&stats).await?;
+2 -1
View File
@@ -192,7 +192,8 @@ pub enum DebugCommand {
///
/// Optional argument is a character mask (a sequence of characters in any
/// order) which enable additional extended statistics. Known characters are
/// "abdeglmx". For convenience, a '*' will enable everything.
/// "abdeglmx" and 'J' for JSON output. For convenience, a '*' will enable
/// everything.
MemoryStats {
opts: Option<String>,
},
+12 -7
View File
@@ -26,14 +26,19 @@ conduwuit_mods = [
gzip_compression = [
"reqwest/gzip",
]
hardened_malloc = [
"dep:hardened_malloc-rs"
]
jemalloc = [
"dep:tikv-jemalloc-sys",
"dep:tikv-jemalloc-ctl",
"dep:tikv-jemallocator",
]
# Uses mimalloc for Rust allocations and, via `override`, exports
# malloc/free/etc. so C/C++ dependencies (RocksDB, zstd, lz4, ...) use it too.
# Mutually exclusive with `jemalloc` on non-MSVC targets.
mimalloc = [
"dep:libmimalloc-sys",
"dep:mimalloc",
"libmimalloc-sys/override",
]
jemalloc_conf = []
jemalloc_prof = [
"tikv-jemalloc-sys/profiling",
@@ -118,14 +123,14 @@ lock_api.workspace = true
hyper-util.workspace = true
resolvematrix.workspace = true
openidconnect.workspace = true
libmimalloc-sys.optional = true
libmimalloc-sys.workspace = true
mimalloc.optional = true
mimalloc.workspace = true
[target.'cfg(unix)'.dependencies]
nix.workspace = true
[target.'cfg(all(not(target_env = "msvc"), target_os = "linux"))'.dependencies]
hardened_malloc-rs.workspace = true
hardened_malloc-rs.optional = true
[dev-dependencies]
maplit.workspace = true
opentelemetry_sdk.workspace = true
+5 -1
View File
@@ -1,5 +1,9 @@
//! Default allocator with no special features
use arrayvec::ArrayVec;
pub(crate) fn metric() -> Option<(&'static str, &'static str)> { None }
/// Always returns Ok
pub fn trim<I: Into<Option<usize>>>(_: I) -> crate::Result { Ok(()) }
@@ -12,4 +16,4 @@ pub fn memory_stats(_opts: &str) -> Option<String> { None }
pub fn memory_usage() -> Option<String> { None }
#[must_use]
pub fn memory_usage_stats() -> Option<[(&'static str, usize); 6]> { None }
pub fn memory_usage_stats() -> Option<ArrayVec<(&'static str, usize), 6>> { None }
-18
View File
@@ -1,18 +0,0 @@
//! hmalloc allocator
#[global_allocator]
static HMALLOC: hardened_malloc_rs::HardenedMalloc = hardened_malloc_rs::HardenedMalloc;
pub fn trim<I: Into<Option<usize>>>(_: I) -> crate::Result { Ok(()) }
#[must_use]
//TODO: get usage
pub fn memory_usage() -> Option<String> { None }
#[must_use]
pub fn memory_usage_stats() -> Option<[(&'static str, usize); 6]> { None }
#[must_use]
pub fn memory_stats(_opts: &str) -> Option<String> {
Some("Extended statistics are not available from hardened_malloc.".to_owned())
}
+448 -148
View File
@@ -1,9 +1,12 @@
//! jemalloc allocator
use std::{
cell::OnceCell,
ffi::{CStr, c_char, c_void},
any::Any,
ffi::{CStr, CString, c_char, c_uint, c_void},
fmt::Debug,
panic::{self, AssertUnwindSafe},
ptr,
sync::OnceLock,
};
use arrayvec::ArrayVec;
@@ -12,26 +15,61 @@ use tikv_jemalloc_sys as ffi;
use tikv_jemallocator as jemalloc;
use crate::{
Result, SyncRwLock, err, is_equal_to, is_nonzero,
Result, err, is_equal_to, is_nonzero,
utils::{math, math::Tried},
};
#[cfg(feature = "jemalloc_conf")]
#[unsafe(no_mangle)]
pub static malloc_conf: &[u8] = const_str::concat_bytes!(
#[repr(transparent)]
struct MallocConf(*const c_char);
// SAFETY: MallocConf points to static immutable NUL-terminated bytes.
#[cfg(feature = "jemalloc_conf")]
unsafe impl Sync for MallocConf {}
#[cfg(feature = "jemalloc_conf")]
const MALLOC_CONF_BYTES: &[u8] = const_str::concat_bytes!(
"lg_extent_max_active_fit:4",
",oversize_threshold:16777216",
",tcache_max:2097152",
",dirty_decay_ms:16000",
",muzzy_decay_ms:144000",
",percpu_arena:percpu",
",metadata_thp:always",
",background_thread:true",
",max_background_threads:-1",
MALLOC_CONF_PLATFORM,
MALLOC_CONF_PROF,
0
);
#[cfg(all(feature = "jemalloc_conf", target_os = "linux"))]
const MALLOC_CONF_PLATFORM: &str =
",percpu_arena:percpu,metadata_thp:always,background_thread:true";
#[cfg(all(feature = "jemalloc_conf", not(target_os = "linux")))]
const MALLOC_CONF_PLATFORM: &str = "";
// Targets match `NO_UNPREFIXED_MALLOC_TARGETS` in `tikv-jemalloc-sys` when
// `unprefixed_malloc_on_supported_platforms` is enabled.
#[cfg(feature = "jemalloc_conf")]
#[used]
#[allow(non_upper_case_globals)]
#[cfg_attr(
any(
target_vendor = "apple",
target_env = "musl",
target_os = "android",
target_os = "dragonfly",
),
unsafe(export_name = "_rjem_malloc_conf")
)]
#[cfg_attr(
not(any(
target_vendor = "apple",
target_env = "musl",
target_os = "android",
target_os = "dragonfly",
)),
unsafe(no_mangle)
)]
static malloc_conf: MallocConf = MallocConf(MALLOC_CONF_BYTES.as_ptr().cast());
#[cfg(all(feature = "jemalloc_conf", feature = "jemalloc_prof"))]
const MALLOC_CONF_PROF: &str = ",prof_active:false";
#[cfg(all(feature = "jemalloc_conf", not(feature = "jemalloc_prof")))]
@@ -39,37 +77,56 @@ const MALLOC_CONF_PROF: &str = "";
#[global_allocator]
static JEMALLOC: jemalloc::Jemalloc = jemalloc::Jemalloc;
static CONTROL: SyncRwLock<()> = SyncRwLock::new(());
type Name = ArrayVec<u8, NAME_MAX>;
type Key = ArrayVec<usize, KEY_SEGS>;
const NAME_MAX: usize = 128;
const KEY_SEGS: usize = 8;
const ARENA_LIMIT: usize = 4095;
const ARENAS_ALL: usize = 4096;
#[ctor::ctor(unsafe)]
fn _static_initialization() {
acq_epoch().expect("pre-initialization of jemalloc failed");
acq_epoch().expect("pre-initialization of jemalloc failed");
let mut epoch: u64 = 0;
let mut len = size_of::<u64>();
// SAFETY: Pre-initialize jemalloc before main() without touching Rust TLS
// or unwinding across the constructor boundary.
unsafe {
ffi::mallctl(
c"epoch".as_ptr(),
ptr::from_mut(&mut epoch).cast::<c_void>(),
&raw mut len,
ptr::null_mut(),
0,
);
}
}
pub(crate) fn metric() -> Option<(&'static str, &'static str)> {
Some((
"continuwuity.jemalloc.memory",
"jemalloc memory statistics, when built with jemalloc_stats",
))
}
#[must_use]
#[cfg(feature = "jemalloc_stats")]
pub fn memory_usage() -> Option<String> {
use std::fmt::Write;
let mibs = |input: usize| {
let kibs = input / 1024;
let kibs = u32::try_from(kibs).unwrap_or_default();
let kibs = f64::from(kibs);
kibs / 1024.0
let whole = u32::try_from(input / (1024 * 1024)).unwrap_or(u32::MAX);
let rem_kibs = u32::try_from((input / 1024) % 1024).unwrap_or_default();
f64::from(whole) + f64::from(rem_kibs) / 1024.0
};
let [allocated, active, mapped, metadata, resident, retained] =
memory_usage_stats()?.map(|(_, bytes)| mibs(bytes));
Some(format!(
"allocated: {allocated:.2} MiB\nactive: {active:.2} MiB\nmapped: {mapped:.2} \
MiB\nmetadata: {metadata:.2} MiB\nresident: {resident:.2} MiB\nretained: {retained:.2} \
MiB\n"
))
let mut out = String::new();
for (name, bytes) in memory_usage_stats()? {
writeln!(out, "{name}: {:.2} MiB", mibs(bytes)).ok()?;
}
Some(out)
}
#[must_use]
@@ -78,184 +135,259 @@ pub fn memory_usage() -> Option<String> { None }
#[must_use]
#[cfg(feature = "jemalloc_stats")]
pub fn memory_usage_stats() -> Option<[(&'static str, usize); 6]> {
pub fn memory_usage_stats() -> Option<ArrayVec<(&'static str, usize), 6>> {
use mallctl::stats;
// Acquire the epoch; ensure latest stats are pulled in
acq_epoch().ok()?;
// Advance the epoch; ensure latest stats are pulled in
inc_epoch().ok()?;
Some([
Some(ArrayVec::from([
("allocated", stats::allocated::read().unwrap_or_default()),
("active", stats::active::read().unwrap_or_default()),
("mapped", stats::mapped::read().unwrap_or_default()),
("metadata", stats::metadata::read().unwrap_or_default()),
("resident", stats::resident::read().unwrap_or_default()),
("retained", stats::retained::read().unwrap_or_default()),
])
]))
}
#[must_use]
#[cfg(not(feature = "jemalloc_stats"))]
pub fn memory_usage_stats() -> Option<[(&'static str, usize); 6]> { None }
pub fn memory_usage_stats() -> Option<ArrayVec<(&'static str, usize), 6>> { None }
struct StatsPrintState {
buf: String,
max_len: usize,
panic: Option<Box<dyn Any + Send>>,
}
pub fn memory_stats(opts: &str) -> Option<String> {
const MAX_LENGTH: usize = 1_048_576;
let mut str = String::new();
let opaque = std::ptr::from_mut(&mut str).cast::<c_void>();
let opts_p: *const c_char = std::ffi::CString::new(opts)
.expect("cstring")
.into_raw()
.cast_const();
let opts = CString::new(opts).ok()?;
// Acquire the epoch; ensure latest stats are pulled in
acq_epoch().ok()?;
let mut state = StatsPrintState {
buf: String::new(),
max_len: MAX_LENGTH,
panic: None,
};
let opaque = ptr::from_mut(&mut state).cast::<c_void>();
// SAFETY: calls malloc_stats_print() with our string instance which must
// remain in this frame. https://docs.rs/tikv-jemalloc-sys/latest/tikv_jemalloc_sys/fn.malloc_stats_print.html
unsafe { ffi::malloc_stats_print(Some(malloc_stats_cb), opaque, opts_p) };
// SAFETY: `state` and `opts` remain live in this frame for the duration of
// `malloc_stats_print()`, which invokes `malloc_stats_cb` synchronously.
unsafe { ffi::malloc_stats_print(Some(malloc_stats_cb), opaque, opts.as_ptr()) };
str.truncate(MAX_LENGTH);
if let Some(payload) = state.panic {
panic::resume_unwind(payload);
}
Some(str)
Some(state.buf)
}
unsafe extern "C" fn malloc_stats_cb(opaque: *mut c_void, msg: *const c_char) {
// SAFETY: we have to trust the opaque points to our String
let res: &mut String = unsafe {
opaque
.cast::<String>()
.as_mut()
.expect("failed to cast void* to &mut String")
// SAFETY: `opaque` points to `StatsPrintState` on the caller's stack.
let Some(state) = (unsafe { opaque.cast::<StatsPrintState>().as_mut() }) else {
return;
};
// SAFETY: we have to trust the string is null terminated.
if state.panic.is_some() || msg.is_null() || state.buf.len() >= state.max_len {
return;
}
// SAFETY: jemalloc guarantees `msg` is a valid NUL-terminated C string.
let msg = unsafe { CStr::from_ptr(msg) };
let msg = String::from_utf8_lossy(msg.to_bytes());
res.push_str(msg.as_ref());
if let Err(err) = panic::catch_unwind(AssertUnwindSafe(|| {
let msg = String::from_utf8_lossy(msg.to_bytes());
let remaining = state.max_len.saturating_sub(state.buf.len());
if msg.len() <= remaining {
state.buf.push_str(msg.as_ref());
} else {
if let Some(prefix) = msg.get(..msg.floor_char_boundary(remaining)) {
state.buf.push_str(prefix);
}
state.max_len = state.buf.len();
}
})) {
state.panic = Some(err);
}
}
macro_rules! mallctl {
($name:expr_2021) => {{
thread_local! {
static KEY: OnceCell<Key> = OnceCell::default();
};
($name:literal) => {{
static KEY: OnceLock<Key> = OnceLock::new();
KEY.with(|once| {
once.get_or_init(move || key($name).expect("failed to translate name into mib key"))
.clone()
})
KEY.get_or_init(|| key($name).expect("failed to translate name into mib key"))
}};
}
pub mod this_thread {
use super::{Debug, Key, OnceCell, Result, is_nonzero, key, math};
thread_local! {
static ALLOCATED_BYTES: OnceCell<&'static u64> = const { OnceCell::new() };
static DEALLOCATED_BYTES: OnceCell<&'static u64> = const { OnceCell::new() };
}
use super::{Debug, Key, OnceLock, Result, c_uint, is_nonzero, key, math};
pub fn trim() -> Result { decay().and_then(|()| purge()) }
pub fn purge() -> Result { notify(mallctl!("arena.0.purge")) }
pub fn purge() -> Result {
// SAFETY: `arena.<i>.purge` is an `arena.<i>.*` notification ctl.
unsafe { notify(mallctl!("arena.0.purge")) }
}
pub fn decay() -> Result { notify(mallctl!("arena.0.decay")) }
pub fn decay() -> Result {
// SAFETY: `arena.<i>.decay` is an `arena.<i>.*` notification ctl.
unsafe { notify(mallctl!("arena.0.decay")) }
}
pub fn idle() -> Result { super::notify(&mallctl!("thread.idle")) }
pub fn idle() -> Result {
// SAFETY: `thread.idle` is a non-indexed notification ctl.
unsafe { super::notify(mallctl!("thread.idle")) }
}
pub fn flush() -> Result { super::notify(&mallctl!("thread.tcache.flush")) }
pub fn flush() -> Result {
// SAFETY: `thread.tcache.flush` is a non-indexed notification ctl.
unsafe { super::notify(mallctl!("thread.tcache.flush")) }
}
pub fn set_muzzy_decay(decay_ms: isize) -> Result<isize> {
set(mallctl!("arena.0.muzzy_decay_ms"), decay_ms)
// SAFETY: `arena.<i>.muzzy_decay_ms` has C type `ssize_t`.
unsafe { set(mallctl!("arena.0.muzzy_decay_ms"), decay_ms) }
}
pub fn get_muzzy_decay() -> Result<isize> { get(mallctl!("arena.0.muzzy_decay_ms")) }
pub fn get_muzzy_decay() -> Result<isize> {
// SAFETY: `arena.<i>.muzzy_decay_ms` has C type `ssize_t`.
unsafe { get(mallctl!("arena.0.muzzy_decay_ms")) }
}
pub fn set_dirty_decay(decay_ms: isize) -> Result<isize> {
set(mallctl!("arena.0.dirty_decay_ms"), decay_ms)
// SAFETY: `arena.<i>.dirty_decay_ms` has C type `ssize_t`.
unsafe { set(mallctl!("arena.0.dirty_decay_ms"), decay_ms) }
}
pub fn get_dirty_decay() -> Result<isize> { get(mallctl!("arena.0.dirty_decay_ms")) }
pub fn get_dirty_decay() -> Result<isize> {
// SAFETY: `arena.<i>.dirty_decay_ms` has C type `ssize_t`.
unsafe { get(mallctl!("arena.0.dirty_decay_ms")) }
}
pub fn cache_enable(enable: bool) -> Result<bool> {
super::set::<u8>(&mallctl!("thread.tcache.enabled"), enable.into()).map(is_nonzero!())
// SAFETY: `thread.tcache.enabled` has C type `bool` (1 byte on
// non-MSVC).
unsafe { super::set::<u8>(mallctl!("thread.tcache.enabled"), enable.into()) }
.map(is_nonzero!())
}
pub fn is_cache_enabled() -> Result<bool> {
super::get::<u8>(&mallctl!("thread.tcache.enabled")).map(is_nonzero!())
// SAFETY: `thread.tcache.enabled` has C type `bool` (1 byte on
// non-MSVC).
unsafe { super::get::<u8>(mallctl!("thread.tcache.enabled")) }.map(is_nonzero!())
}
pub fn set_arena(id: usize) -> Result<usize> {
super::set::<u32>(&mallctl!("thread.arena"), id.try_into()?).and_then(math::try_into)
// SAFETY: `thread.arena` has C type `unsigned`.
unsafe { super::set::<c_uint>(mallctl!("thread.arena"), id.try_into()?) }
.and_then(math::try_into)
}
pub fn arena_id() -> Result<usize> {
super::get::<u32>(&mallctl!("thread.arena")).and_then(math::try_into)
// SAFETY: `thread.arena` has C type `unsigned`.
unsafe { super::get::<c_uint>(mallctl!("thread.arena")) }.and_then(math::try_into)
}
pub fn prof_enable(enable: bool) -> Result<bool> {
super::set::<u8>(&mallctl!("thread.prof.active"), enable.into()).map(is_nonzero!())
// SAFETY: `thread.prof.active` has C type `bool` (1 byte on non-MSVC).
unsafe { super::set::<u8>(mallctl!("thread.prof.active"), enable.into()) }
.map(is_nonzero!())
}
pub fn is_prof_enabled() -> Result<bool> {
super::get::<u8>(&mallctl!("thread.prof.active")).map(is_nonzero!())
// SAFETY: `thread.prof.active` has C type `bool` (1 byte on non-MSVC).
unsafe { super::get::<u8>(mallctl!("thread.prof.active")) }.map(is_nonzero!())
}
pub fn reset_peak() -> Result { super::notify(&mallctl!("thread.peak.reset")) }
pub fn reset_peak() -> Result {
// SAFETY: `thread.peak.reset` is a non-indexed notification ctl.
unsafe { super::notify(mallctl!("thread.peak.reset")) }
}
pub fn peak() -> Result<u64> { super::get(&mallctl!("thread.peak.read")) }
pub fn peak() -> Result<u64> {
// SAFETY: `thread.peak.read` has C type `uint64_t`.
unsafe { super::get(mallctl!("thread.peak.read")) }
}
#[inline]
#[must_use]
pub fn allocated() -> u64 {
*ALLOCATED_BYTES.with(|once| init_tls_cell(once, "thread.allocatedp"))
// SAFETY: `thread.allocated` has C type `uint64_t`.
unsafe { super::get(mallctl!("thread.allocated")) }.unwrap_or_default()
}
#[inline]
#[must_use]
pub fn deallocated() -> u64 {
*DEALLOCATED_BYTES.with(|once| init_tls_cell(once, "thread.deallocatedp"))
// SAFETY: `thread.deallocated` has C type `uint64_t`.
unsafe { super::get(mallctl!("thread.deallocated")) }.unwrap_or_default()
}
fn notify(key: Key) -> Result { super::notify_by_arena(Some(arena_id()?), key) }
/// # Safety
/// Caller must ensure `key` is an `arena.<i>.*` notification MIB
/// (`key.len() >= 2`).
unsafe fn notify(key: &Key) -> Result {
let mut key = key.clone();
// `arena_id()` returns this thread's own valid arena (< narenas <
// ARENA_LIMIT), so we skip `validate_arena_id()` to avoid locking
// `ctl_mtx` on hot paths like `decay()`.
key[1] = arena_id()?;
// SAFETY: Caller guarantees `key` is an `arena.<i>.*` notification MIB
// and `arena_id()` returns a valid arena index.
unsafe { super::notify(&key) }
}
fn set<T>(key: Key, val: T) -> Result<T>
/// # Safety
/// Caller must ensure `key` is an `arena.<i>.*` MIB and `T` matches its
/// exact C type.
unsafe fn set<T>(key: &Key, val: T) -> Result<T>
where
T: Copy + Debug,
{
super::set_by_arena(Some(arena_id()?), key, val)
let mut key = key.clone();
// `arena_id()` is already a valid arena index; skip
// `validate_arena_id()`.
key[1] = arena_id()?;
// SAFETY: Caller guarantees `T` matches `key`.
unsafe { super::set(&key, val) }
}
fn get<T>(key: Key) -> Result<T>
/// # Safety
/// Caller must ensure `key` is an `arena.<i>.*` MIB and `T` matches its
/// exact C type.
unsafe fn get<T>(key: &Key) -> Result<T>
where
T: Copy + Debug,
{
super::get_by_arena(Some(arena_id()?), key)
}
fn init_tls_cell(cell: &OnceCell<&'static u64>, name: &str) -> &'static u64 {
cell.get_or_init(|| {
let ptr: *const u64 = super::get(&mallctl!(name)).expect("failed to obtain pointer");
// SAFETY: ptr points directly to the internal state of jemalloc for
// this thread
unsafe { ptr.as_ref() }.expect("pointer must not be null")
})
let mut key = key.clone();
// `arena_id()` is already a valid arena index; skip
// `validate_arena_id()`.
key[1] = arena_id()?;
// SAFETY: Caller guarantees `T` matches `key`.
unsafe { super::get(&key) }
}
}
pub fn stats_reset() -> Result { notify(&mallctl!("stats.mutexes.reset")) }
pub fn stats_reset() -> Result {
// SAFETY: `stats.mutexes.reset` is a non-indexed notification ctl.
unsafe { notify(mallctl!("stats.mutexes.reset")) }
}
pub fn prof_reset() -> Result { notify(&mallctl!("prof.reset")) }
pub fn prof_reset() -> Result {
// SAFETY: `prof.reset` is a non-indexed notification ctl.
unsafe { notify(mallctl!("prof.reset")) }
}
pub fn prof_enable(enable: bool) -> Result<bool> {
set::<u8>(&mallctl!("prof.active"), enable.into()).map(is_nonzero!())
// SAFETY: `prof.active` has C type `bool` (1 byte on non-MSVC).
unsafe { set::<u8>(mallctl!("prof.active"), enable.into()) }.map(is_nonzero!())
}
pub fn is_prof_enabled() -> Result<bool> {
get::<u8>(&mallctl!("prof.active")).map(is_nonzero!())
// SAFETY: `prof.active` has C type `bool` (1 byte on non-MSVC).
unsafe { get::<u8>(mallctl!("prof.active")) }.map(is_nonzero!())
}
pub fn trim<I: Into<Option<usize>> + Copy>(arena: I) -> Result {
@@ -263,31 +395,40 @@ pub fn trim<I: Into<Option<usize>> + Copy>(arena: I) -> Result {
}
pub fn purge<I: Into<Option<usize>>>(arena: I) -> Result {
notify_by_arena(arena.into(), mallctl!("arena.4096.purge"))
// SAFETY: `arena.<i>.purge` is an `arena.<i>.*` notification ctl.
unsafe { notify_by_arena(arena.into(), mallctl!("arena.4096.purge")) }
}
pub fn decay<I: Into<Option<usize>>>(arena: I) -> Result {
notify_by_arena(arena.into(), mallctl!("arena.4096.decay"))
// SAFETY: `arena.<i>.decay` is an `arena.<i>.*` notification ctl.
unsafe { notify_by_arena(arena.into(), mallctl!("arena.4096.decay")) }
}
pub fn set_muzzy_decay<I: Into<Option<usize>>>(arena: I, decay_ms: isize) -> Result<isize> {
match arena.into() {
| Some(arena) =>
set_by_arena(Some(arena), mallctl!("arena.4096.muzzy_decay_ms"), decay_ms),
| _ => set(&mallctl!("arenas.muzzy_decay_ms"), decay_ms),
// SAFETY: `arena.<i>.muzzy_decay_ms` has C type `ssize_t`.
| Some(arena) => unsafe {
set_by_arena(arena, mallctl!("arena.0.muzzy_decay_ms"), decay_ms)
},
// SAFETY: `arenas.muzzy_decay_ms` has C type `ssize_t`.
| _ => unsafe { set(mallctl!("arenas.muzzy_decay_ms"), decay_ms) },
}
}
pub fn set_dirty_decay<I: Into<Option<usize>>>(arena: I, decay_ms: isize) -> Result<isize> {
match arena.into() {
| Some(arena) =>
set_by_arena(Some(arena), mallctl!("arena.4096.dirty_decay_ms"), decay_ms),
| _ => set(&mallctl!("arenas.dirty_decay_ms"), decay_ms),
// SAFETY: `arena.<i>.dirty_decay_ms` has C type `ssize_t`.
| Some(arena) => unsafe {
set_by_arena(arena, mallctl!("arena.0.dirty_decay_ms"), decay_ms)
},
// SAFETY: `arenas.dirty_decay_ms` has C type `ssize_t`.
| _ => unsafe { set(mallctl!("arenas.dirty_decay_ms"), decay_ms) },
}
}
pub fn background_thread_enable(enable: bool) -> Result<bool> {
set::<u8>(&mallctl!("background_thread"), enable.into()).map(is_nonzero!())
// SAFETY: `background_thread` has C type `bool` (1 byte on non-MSVC).
unsafe { set::<u8>(mallctl!("background_thread"), enable.into()) }.map(is_nonzero!())
}
#[inline]
@@ -303,83 +444,138 @@ pub fn is_percpu_arena() -> bool { percpu_arenas().is_ok_and(is_equal_to!("percp
pub fn is_phycpu_arena() -> bool { percpu_arenas().is_ok_and(is_equal_to!("phycpu")) }
pub fn percpu_arenas() -> Result<&'static str> {
let ptr = get::<*const c_char>(&mallctl!("opt.percpu_arena"))?;
//SAFETY: ptr points to a null-terminated string returned for
// opt.percpu_arena.
// SAFETY: `opt.percpu_arena` has C type `const char *`.
let ptr = unsafe { get::<*const c_char>(mallctl!("opt.percpu_arena")) }?;
if ptr.is_null() {
return Err(err!("mallctl: opt.percpu_arena returned NULL"));
}
// SAFETY: `ptr` is non-null and points to a static NUL-terminated string
// returned by jemalloc for `opt.percpu_arena`.
let cstr = unsafe { CStr::from_ptr(ptr) };
cstr.to_str().map_err(Into::into)
}
pub fn arenas() -> Result<usize> {
get::<u32>(&mallctl!("arenas.narenas")).and_then(math::try_into)
// SAFETY: `arenas.narenas` has C type `unsigned`.
unsafe { get::<c_uint>(mallctl!("arenas.narenas")) }.and_then(math::try_into)
}
pub fn inc_epoch() -> Result<u64> { xchg(&mallctl!("epoch"), 1_u64) }
pub fn acq_epoch() -> Result<u64> { xchg(&mallctl!("epoch"), 0_u64) }
fn notify_by_arena(id: Option<usize>, mut key: Key) -> Result {
key[1] = id.unwrap_or(4096);
notify(&key)
pub fn inc_epoch() -> Result<u64> {
// SAFETY: `epoch` has C type `uint64_t`.
unsafe { xchg(mallctl!("epoch"), 1_u64) }
}
fn set_by_arena<T>(id: Option<usize>, mut key: Key, val: T) -> Result<T>
pub fn acq_epoch() -> Result<u64> {
// SAFETY: `epoch` has C type `uint64_t`.
unsafe { get(mallctl!("epoch")) }
}
fn validate_arena_id(id: usize, allow_all: bool) -> Result<usize> {
if allow_all && id == ARENAS_ALL {
return Ok(id);
}
let narenas = arenas()?.min(ARENA_LIMIT);
if id < narenas {
Ok(id)
} else {
Err(err!("mallctl: invalid arena index {id} (narenas: {narenas})"))
}
}
/// # Safety
/// Caller must ensure `key` is an `arena.<i>.*` notification MIB
/// (`key.len() >= 2`).
unsafe fn notify_by_arena(id: Option<usize>, key: &Key) -> Result {
let mut key = key.clone();
key[1] = match id {
| Some(id) => validate_arena_id(id, true)?,
| None => ARENAS_ALL,
};
// SAFETY: Caller guarantees `key` is an `arena.<i>.*` notification MIB and
// `key[1]` has been validated above.
unsafe { notify(&key) }
}
/// # Safety
/// Caller must ensure `key` is an `arena.<i>.*` MIB (`key.len() >= 2`) and `T`
/// matches its exact C type.
unsafe fn set_by_arena<T>(id: usize, key: &Key, val: T) -> Result<T>
where
T: Copy + Debug,
{
key[1] = id.unwrap_or(4096);
set(&key, val)
let mut key = key.clone();
key[1] = validate_arena_id(id, false)?;
// SAFETY: Caller guarantees `T` matches `key`.
unsafe { set(&key, val) }
}
fn get_by_arena<T>(id: Option<usize>, mut key: Key) -> Result<T>
/// # Safety
/// Caller must ensure `key` is a valid notification MIB and any indexed arena
/// component is within bounds.
unsafe fn notify(key: &Key) -> Result {
// SAFETY: jemalloc notification ctls require `oldp`, `oldlenp`, and `newp`
// to be NULL and `newlen` to be 0 (otherwise `EPERM` is returned).
let ret = unsafe {
ffi::mallctlbymib(
key.as_ptr(),
key.len(),
ptr::null_mut(),
ptr::null_mut(),
ptr::null_mut(),
0,
)
};
if ret == 0 {
Ok(())
} else {
Err(err!("mallctl: {}", std::io::Error::from_raw_os_error(ret)))
}
}
/// Does not advance `epoch`; callers reading cached `stats.*` must call
/// `inc_epoch()` explicitly.
///
/// # Safety
/// Caller must ensure `T` matches the exact C type expected by `key`.
unsafe fn set<T>(key: &Key, val: T) -> Result<T>
where
T: Copy + Debug,
{
key[1] = id.unwrap_or(4096);
get(&key)
}
fn notify(key: &Key) -> Result { xchg(key, ()) }
fn set<T>(key: &Key, val: T) -> Result<T>
where
T: Copy + Debug,
{
let _lock = CONTROL.write();
let res = xchg(key, val)?;
inc_epoch()?;
Ok(res)
// SAFETY: Caller guarantees `T` matches `key`.
unsafe { xchg(key, val) }
}
/// # Safety
/// Caller must ensure `T` matches the exact C type expected by `key`.
#[tracing::instrument(
name = "get",
level = "trace",
skip_all,
fields(?key)
)]
fn get<T>(key: &Key) -> Result<T>
unsafe fn get<T>(key: &Key) -> Result<T>
where
T: Copy + Debug,
{
acq_epoch()?;
acq_epoch()?;
// SAFETY: T must be perfectly valid to receive value.
// SAFETY: Caller guarantees `T` matches the C type returned for `key`.
unsafe { mallctl::raw::read_mib(key.as_slice()) }.map_err(map_err)
}
/// # Safety
/// Caller must ensure `T` matches the exact C type expected by `key`.
#[tracing::instrument(
name = "xchg",
level = "trace",
skip_all,
fields(?key, ?val)
)]
fn xchg<T>(key: &Key, val: T) -> Result<T>
unsafe fn xchg<T>(key: &Key, val: T) -> Result<T>
where
T: Copy + Debug,
{
// SAFETY: T must be the exact expected type.
// SAFETY: Caller guarantees `T` matches the C type expected by `key`.
unsafe { mallctl::raw::update_mib(key.as_slice(), val) }.map_err(map_err)
}
@@ -387,6 +583,9 @@ fn key(name: &str) -> Result<Key> {
// tikv asserts the output buffer length is tight to the number of required
// mibs so we slice that down here.
let segs = name.chars().filter(is_equal_to!(&'.')).count().try_add(1)?;
if segs > KEY_SEGS {
return Err(err!("mallctl: key has too many segments ({segs} > {KEY_SEGS})"));
}
let name = self::name(name)?;
let mut buf = [0_usize; KEY_SEGS];
@@ -396,6 +595,10 @@ fn key(name: &str) -> Result<Key> {
}
fn name(name: &str) -> Result<Name> {
if name.as_bytes().contains(&0) {
return Err(err!("mallctl: key contains interior NUL byte"));
}
let mut buf = Name::new();
buf.try_extend_from_slice(name.as_bytes())?;
buf.try_extend_from_slice(b"\0")?;
@@ -406,3 +609,100 @@ fn name(name: &str) -> Result<Name> {
fn map_err(error: tikv_jemalloc_ctl::Error) -> crate::Error {
err!("mallctl: {}", error.to_string())
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn notify_ctls_succeed() {
this_thread::flush().expect("thread.tcache.flush");
this_thread::decay().expect("arena.<i>.decay");
this_thread::purge().expect("arena.<i>.purge");
this_thread::trim().expect("this_thread::trim");
trim(None).expect("trim(None)");
}
#[test]
#[cfg(feature = "jemalloc_stats")]
fn allocated_and_deallocated_do_not_alias() {
let a0 = this_thread::allocated();
let d0 = this_thread::deallocated();
let buf = std::hint::black_box(vec![0_u8; 64 * 1024]);
let a1 = this_thread::allocated();
assert!(a1 > a0, "expected allocated to increase: {a0} -> {a1}");
drop(buf);
let d1 = this_thread::deallocated();
assert!(d1 > d0, "expected deallocated to increase: {d0} -> {d1}");
}
#[test]
fn memory_stats_handles_valid_and_nul_opts() {
assert!(memory_stats("invalid\0opts").is_none());
let stats = memory_stats("gmal").expect("memory_stats");
assert!(!stats.is_empty());
}
#[test]
fn key_too_many_segments_returns_err() { key("a.b.c.d.e.f.g.h.i").unwrap_err(); }
#[test]
fn key_interior_nul_returns_err() { key("a.b\0.c").unwrap_err(); }
#[test]
fn arena_index_bounds_rejected() {
purge(Some(ARENAS_ALL)).expect("purge(MALLCTL_ARENAS_ALL)");
decay(Some(ARENAS_ALL)).expect("decay(MALLCTL_ARENAS_ALL)");
let narenas = arenas().expect("arenas.narenas");
purge(Some(narenas)).unwrap_err();
decay(Some(narenas)).unwrap_err();
set_dirty_decay(Some(narenas), -1).unwrap_err();
set_muzzy_decay(Some(narenas), -1).unwrap_err();
purge(Some(4097)).unwrap_err();
decay(Some(4097)).unwrap_err();
trim(Some(4097)).unwrap_err();
purge(Some(usize::MAX)).unwrap_err();
set_dirty_decay(Some(ARENAS_ALL), -1).unwrap_err();
set_muzzy_decay(Some(ARENAS_ALL), -1).unwrap_err();
set_dirty_decay(Some(4097), -1).unwrap_err();
set_muzzy_decay(Some(4097), -1).unwrap_err();
}
#[test]
#[cfg(feature = "jemalloc_conf")]
fn malloc_conf_is_applied() {
if std::env::var_os("MALLOC_CONF").is_some()
|| std::env::var_os("_RJEM_MALLOC_CONF").is_some()
{
return;
}
// SAFETY: `opt.dirty_decay_ms` and `opt.muzzy_decay_ms` have C type
// `ssize_t`.
let dirty =
unsafe { get::<isize>(mallctl!("opt.dirty_decay_ms")) }.expect("opt.dirty_decay_ms");
// SAFETY: Same as above
let muzzy =
unsafe { get::<isize>(mallctl!("opt.muzzy_decay_ms")) }.expect("opt.muzzy_decay_ms");
assert_eq!(dirty, 16_000, "expected malloc_conf dirty_decay_ms:16000");
assert_eq!(muzzy, 144_000, "expected malloc_conf muzzy_decay_ms:144000");
}
#[test]
fn percpu_arenas_returns_valid_mode() {
let mode = percpu_arenas().expect("opt.percpu_arena");
assert!(matches!(mode, "disabled" | "percpu" | "phycpu"), "unexpected mode: {mode}");
}
#[test]
fn epoch_read_and_advance() {
let e0 = acq_epoch().expect("acq_epoch");
let e1 = inc_epoch().expect("inc_epoch");
assert!(e1 > e0, "expected inc_epoch to advance epoch: {e0} -> {e1}");
}
}
+234
View File
@@ -0,0 +1,234 @@
//! mimalloc allocator
use std::{
any::Any,
ffi::{CStr, c_void},
panic::{self, AssertUnwindSafe},
ptr,
};
use arrayvec::ArrayVec;
use libmimalloc_sys as ffi;
use mimalloc::MiMalloc;
use serde::Deserialize;
#[global_allocator]
static MIMALLOC: MiMalloc = MiMalloc;
#[derive(Deserialize)]
struct StatCount {
current: i64,
peak: i64,
}
#[derive(Deserialize)]
struct Stats {
reserved: StatCount,
committed: StatCount,
purged: i64,
reset: i64,
}
trait OutputFnChar {
type Char;
}
impl<C> OutputFnChar for Option<unsafe extern "C" fn(*const C, *mut c_void)> {
type Char = C;
}
type MiChar = <ffi::mi_output_fun as OutputFnChar>::Char;
struct StatsPrintState {
buf: String,
max_len: usize,
panic: Option<Box<dyn Any + Send>>,
}
pub(crate) fn metric() -> Option<(&'static str, &'static str)> {
Some(("continuwuity.mimalloc.memory", "mimalloc memory statistics"))
}
pub fn trim<I: Into<Option<usize>>>(_: I) -> crate::Result {
// SAFETY: `mi_collect` is thread-safe and takes a single boolean flag.
unsafe { ffi::mi_collect(true) };
Ok(())
}
fn read_stats() -> Option<Stats> {
let stats = MiMalloc::stats_json().ok()?;
let json = stats.to_str().ok()?;
serde_json::from_str(json).ok()
}
fn non_neg_usize(value: i64) -> usize { usize::try_from(value.max(0)).unwrap_or(usize::MAX) }
fn mibs(input: usize) -> f64 {
let whole = u32::try_from(input / (1024 * 1024)).unwrap_or(u32::MAX);
let rem_kibs = u32::try_from((input / 1024) % 1024).unwrap_or_default();
f64::from(whole) + f64::from(rem_kibs) / 1024.0
}
#[must_use]
pub fn memory_usage() -> Option<String> {
let stats = read_stats()?;
Some(format!(
"reserved: {:.2} MiB\nreserved_peak: {:.2} MiB\ncommitted: {:.2} MiB\ncommitted_peak: \
{:.2} MiB\npurged_total: {:.2} MiB\nreset_total: {:.2} MiB\n",
mibs(non_neg_usize(stats.reserved.current)),
mibs(non_neg_usize(stats.reserved.peak)),
mibs(non_neg_usize(stats.committed.current)),
mibs(non_neg_usize(stats.committed.peak)),
mibs(non_neg_usize(stats.purged)),
mibs(non_neg_usize(stats.reset)),
))
}
#[must_use]
pub fn memory_usage_stats() -> Option<ArrayVec<(&'static str, usize), 6>> {
let stats = read_stats()?;
let mut out = ArrayVec::new();
out.extend([
("reserved", non_neg_usize(stats.reserved.current)),
("reserved_peak", non_neg_usize(stats.reserved.peak)),
("committed", non_neg_usize(stats.committed.current)),
("committed_peak", non_neg_usize(stats.committed.peak)),
]);
Some(out)
}
#[must_use]
pub fn memory_stats(opts: &str) -> Option<String> {
const MAX_LENGTH: usize = 1_048_576;
if opts.contains('\0') {
return None;
}
if opts.contains('J') {
let stats = MiMalloc::stats_json().ok()?;
let json = String::from_utf8_lossy(stats.to_bytes());
let end = json.floor_char_boundary(json.len().min(MAX_LENGTH));
return json.get(..end).map(str::to_owned);
}
let mut state = StatsPrintState {
buf: String::new(),
max_len: MAX_LENGTH,
panic: None,
};
let opaque = ptr::from_mut(&mut state).cast::<c_void>();
// SAFETY: `state` remains live in this frame for the duration of
// `mi_stats_print_out()`, which invokes `mi_stats_cb` synchronously.
unsafe { ffi::mi_stats_print_out(Some(mi_stats_cb), opaque) };
if let Some(payload) = state.panic {
panic::resume_unwind(payload);
}
Some(state.buf)
}
unsafe extern "C" fn mi_stats_cb(msg: *const MiChar, arg: *mut c_void) {
// SAFETY: `arg` points to `StatsPrintState` on the caller's stack.
let Some(state) = (unsafe { arg.cast::<StatsPrintState>().as_mut() }) else {
return;
};
if state.panic.is_some() || msg.is_null() || state.buf.len() >= state.max_len {
return;
}
// SAFETY: mimalloc guarantees `msg` is a valid NUL-terminated C string.
let msg = unsafe { CStr::from_ptr(msg.cast()) };
if let Err(err) = panic::catch_unwind(AssertUnwindSafe(|| {
let msg = String::from_utf8_lossy(msg.to_bytes());
let remaining = state.max_len.saturating_sub(state.buf.len());
if msg.len() <= remaining {
state.buf.push_str(msg.as_ref());
} else {
if let Some(prefix) = msg.get(..msg.floor_char_boundary(remaining)) {
state.buf.push_str(prefix);
}
state.max_len = state.buf.len();
}
})) {
state.panic = Some(err);
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn trim_succeeds() { trim(None).expect("trim(None)"); }
#[test]
fn non_neg_usize_clamps_negative_values() {
assert_eq!(non_neg_usize(-1), 0, "expected -1 to clamp to 0");
assert_eq!(non_neg_usize(i64::MIN), 0, "expected i64::MIN to clamp to 0");
assert_eq!(non_neg_usize(0), 0, "expected 0 to remain 0");
assert_eq!(non_neg_usize(4096), 4096, "expected positive value unchanged");
}
#[test]
fn memory_usage_stats_reports_allocator_bytes() {
let buf = std::hint::black_box(vec![0_u8; 64 * 1024]);
let stats = memory_usage_stats().expect("memory_usage_stats");
drop(buf);
let keys: Vec<_> = stats.iter().map(|&(key, _)| key).collect();
assert_eq!(
keys,
["reserved", "reserved_peak", "committed", "committed_peak"],
"unexpected stat keys"
);
for (key, bytes) in stats {
assert!(bytes > 0, "expected {key} > 0, got {bytes}");
}
}
#[test]
fn memory_usage_returns_allocator_summary() {
let usage = memory_usage().expect("memory_usage");
for expected in [
"reserved:",
"reserved_peak:",
"committed:",
"committed_peak:",
"purged_total:",
"reset_total:",
"MiB",
] {
assert!(usage.contains(expected), "missing {expected:?} in usage:\n{usage}");
}
assert!(!usage.contains("current_rss:"), "unexpected current_rss in usage:\n{usage}");
assert!(!usage.contains("peak_rss:"), "unexpected peak_rss in usage:\n{usage}");
}
#[test]
fn memory_stats_supports_text_json_and_rejects_nul() {
assert!(memory_stats("invalid\0opts").is_none(), "expected NUL opts to return None");
for opts in ["", "abcdefghijklmnopqrstuvwxyz"] {
let stats = memory_stats(opts).expect("memory_stats text");
assert!(!stats.is_empty(), "expected non-empty stats for opts={opts:?}");
assert!(!stats.starts_with('{'), "expected formatted text, got JSON:\n{stats}");
assert!(stats.contains("reserved"), "missing reserved in stats:\n{stats}");
assert!(stats.contains("committed"), "missing committed in stats:\n{stats}");
}
let json = memory_stats("J").expect("memory_stats JSON");
assert!(json.starts_with('{'), "expected JSON object, got:\n{json}");
let parsed: serde_json::Value =
serde_json::from_str(&json).expect("expected valid JSON from memory_stats(\"J\")");
assert!(parsed.get("committed").is_some(), "expected committed field in JSON");
}
}
+16 -16
View File
@@ -4,24 +4,24 @@
#[cfg(all(not(target_env = "msvc"), feature = "jemalloc"))]
pub mod je;
#[cfg(all(not(target_env = "msvc"), feature = "jemalloc"))]
pub(crate) use je::metric;
#[cfg(all(not(target_env = "msvc"), feature = "jemalloc"))]
pub use je::{memory_stats, memory_usage, memory_usage_stats, trim};
#[cfg(all(not(target_env = "msvc"), feature = "hardened_malloc", not(feature = "jemalloc")))]
pub mod hardened;
#[cfg(all(
not(target_env = "msvc"),
feature = "hardened_malloc",
not(feature = "jemalloc")
))]
pub use hardened::{memory_stats, memory_usage, memory_usage_stats, trim};
// mimalloc
#[cfg(all(feature = "mimalloc", feature = "jemalloc", not(target_env = "msvc")))]
compile_error!("the `mimalloc` and `jemalloc` features are mutually exclusive");
#[cfg(all(feature = "mimalloc", any(target_env = "msvc", not(feature = "jemalloc"))))]
pub mod mimalloc;
#[cfg(all(feature = "mimalloc", any(target_env = "msvc", not(feature = "jemalloc"))))]
pub(crate) use mimalloc::metric;
#[cfg(all(feature = "mimalloc", any(target_env = "msvc", not(feature = "jemalloc"))))]
pub use mimalloc::{memory_stats, memory_usage, memory_usage_stats, trim};
#[cfg(any(
target_env = "msvc",
all(not(feature = "hardened_malloc"), not(feature = "jemalloc"))
))]
// default
#[cfg(all(not(feature = "mimalloc"), any(target_env = "msvc", not(feature = "jemalloc"))))]
pub mod default;
#[cfg(any(
target_env = "msvc",
all(not(feature = "hardened_malloc"), not(feature = "jemalloc"))
))]
#[cfg(all(not(feature = "mimalloc"), any(target_env = "msvc", not(feature = "jemalloc"))))]
pub(crate) use default::metric;
#[cfg(all(not(feature = "mimalloc"), any(target_env = "msvc", not(feature = "jemalloc"))))]
pub use default::{memory_stats, memory_usage, memory_usage_stats, trim};
+1 -8
View File
@@ -5,7 +5,7 @@ use figment::Figment;
use ruma::{OwnedServerName, events::room::server_acl::RoomServerAclEventContent};
use super::DEPRECATED_KEYS;
use crate::{Config, Err, Result, Server, debug, debug_info, debug_warn, error, warn};
use crate::{Config, Err, Result, Server, debug, debug_info, error, warn};
/// Performs check() with additional checks specific to reloading old config
/// with new config.
@@ -43,13 +43,6 @@ pub fn check(config: &Config) -> Result {
));
}
if cfg!(all(feature = "hardened_malloc", feature = "jemalloc", not(target_env = "msvc"))) {
debug_warn!(
"hardened_malloc and jemalloc compile-time features are both enabled, this causes \
jemalloc to be used."
);
}
if cfg!(not(unix)) && config.unix_socket_path.is_some() {
return Err!(Config(
"unix_socket_path",
+6 -2
View File
@@ -196,9 +196,13 @@ fn register_process(meter: &Meter) {
}
fn register_allocator(meter: &Meter) {
let Some((name, description)) = crate::alloc::metric() else {
return;
};
meter
.u64_observable_gauge("continuwuity.jemalloc.memory")
.with_description("jemalloc memory statistics, when built with jemalloc_stats")
.u64_observable_gauge(name)
.with_description(description)
.with_unit("By")
.with_callback(|observer| {
for (kind, bytes) in crate::alloc::memory_usage_stats().into_iter().flatten() {
+20
View File
@@ -38,3 +38,23 @@ async fn renders_prometheus_names() {
}
assert!(!body.contains("otel_scope_name"));
}
#[tokio::test]
#[cfg(all(feature = "mimalloc", any(target_env = "msvc", not(feature = "jemalloc"))))]
async fn renders_mimalloc_memory_metric() {
let (metrics, _provider) = metrics();
let body = metrics.render().unwrap();
for expected in [
"continuwuity_mimalloc_memory_bytes{kind=\"reserved\"}",
"continuwuity_mimalloc_memory_bytes{kind=\"reserved_peak\"}",
"continuwuity_mimalloc_memory_bytes{kind=\"committed\"}",
"continuwuity_mimalloc_memory_bytes{kind=\"committed_peak\"}",
] {
assert!(body.contains(expected), "missing {expected:?} in:\n{body}");
}
assert!(
!body.contains("continuwuity_jemalloc_memory_bytes"),
"unexpected jemalloc metric in mimalloc build:\n{body}"
);
}
+3 -4
View File
@@ -70,7 +70,6 @@ standard = [
]
full = [
"standard",
# "hardened_malloc", # Conflicts with jemalloc
"jemalloc_prof",
"perf_measurements",
"tokio_console",
@@ -96,9 +95,6 @@ gzip_compression = [
"conduwuit-router/gzip_compression",
"conduwuit-service/gzip_compression",
]
hardened_malloc = [
"conduwuit-core/hardened_malloc",
]
http3 = [
"conduwuit-service/http3"
]
@@ -116,6 +112,9 @@ jemalloc = [
"conduwuit-core/jemalloc",
"conduwuit-service/jemalloc",
]
mimalloc = [
"conduwuit-core/mimalloc",
]
jemalloc_prof = [
"conduwuit-core/jemalloc_prof",
]
+3 -2
View File
@@ -1,6 +1,7 @@
#[cfg(feature = "console")]
use std::io::IsTerminal;
use conduwuit::{Err, Result, debug, debug_info, error, info, warn};
use conduwuit::{Err, Result, debug, debug_info, error, info};
use ruma::events::room::message::RoomMessageEventContent;
use tokio::time::{Duration, sleep};
@@ -15,7 +16,7 @@ impl super::Service {
#[cfg(feature = "console")]
if self.services.server.config.admin_console_automatic {
if !std::io::stdin().is_terminal() {
warn!("Console enabled without a tty available; Not enabling console");
conduwuit::warn!("Console enabled without a tty available; Not enabling console");
return;
}