feat(metrics): Record metrics with OpenTelemetry and export over OTLP

src/core/Cargo.toml

@@ -113,7 +113,8 @@ tracing-subscriber.workspace = true
 tracing.workspace = true
 url.workspace = true
 parking_lot.workspace = true
-prometheus-client.workspace = true
+opentelemetry.workspace = true
+prometheus.workspace = true
 lock_api.workspace = true
 hyper-util.workspace = true
 resolvematrix.workspace = true
@@ -128,6 +129,8 @@ hardened_malloc-rs.optional = true
 
 [dev-dependencies]
 maplit.workspace = true
+opentelemetry_sdk.workspace = true
+opentelemetry-prometheus.workspace = true
 
 [lints]
 workspace = true

src/core/alloc/default.rs

@@ -10,3 +10,6 @@ pub fn memory_stats(_opts: &str) -> Option<String> { None }
 /// Always returns None
 #[must_use]
 pub fn memory_usage() -> Option<String> { None }
+
+#[must_use]
+pub fn memory_usage_stats() -> Option<[(&'static str, usize); 6]> { None }

src/core/alloc/hardened.rs

@@ -9,6 +9,9 @@ pub fn trim<I: Into<Option<usize>>>(_: I) -> crate::Result { Ok(()) }
 //TODO: get usage
 pub fn memory_usage() -> Option<String> { None }
 
+#[must_use]
+pub fn memory_usage_stats() -> Option<[(&'static str, usize); 6]> { None }
+
 #[must_use]
 pub fn memory_stats(_opts: &str) -> Option<String> {
 	Some("Extended statistics are not available from hardened_malloc.".to_owned())

src/core/alloc/je.rs

@@ -56,25 +56,15 @@ fn _static_initialization() {
 #[must_use]
 #[cfg(feature = "jemalloc_stats")]
 pub fn memory_usage() -> Option<String> {
-	use mallctl::stats;
-
-	let mibs = |input: Result<usize, mallctl::Error>| {
-		let input = input.unwrap_or_default();
+	let mibs = |input: usize| {
 		let kibs = input / 1024;
 		let kibs = u32::try_from(kibs).unwrap_or_default();
 		let kibs = f64::from(kibs);
 		kibs / 1024.0
 	};
 
-	// Acquire the epoch; ensure latest stats are pulled in
-	acq_epoch().ok()?;
-
-	let allocated = mibs(stats::allocated::read());
-	let active = mibs(stats::active::read());
-	let mapped = mibs(stats::mapped::read());
-	let metadata = mibs(stats::metadata::read());
-	let resident = mibs(stats::resident::read());
-	let retained = mibs(stats::retained::read());
+	let [allocated, active, mapped, metadata, resident, retained] =
+		memory_usage_stats()?.map(|(_, bytes)| mibs(bytes));
 	Some(format!(
 		"allocated: {allocated:.2} MiB\nactive: {active:.2} MiB\nmapped: {mapped:.2} \
 		 MiB\nmetadata: {metadata:.2} MiB\nresident: {resident:.2} MiB\nretained: {retained:.2} \
@@ -86,6 +76,28 @@ pub fn memory_usage() -> Option<String> {
 #[cfg(not(feature = "jemalloc_stats"))]
 pub fn memory_usage() -> Option<String> { None }
 
+#[must_use]
+#[cfg(feature = "jemalloc_stats")]
+pub fn memory_usage_stats() -> Option<[(&'static str, usize); 6]> {
+	use mallctl::stats;
+
+	// Acquire the epoch; ensure latest stats are pulled in
+	acq_epoch().ok()?;
+
+	Some([
+		("allocated", stats::allocated::read().unwrap_or_default()),
+		("active", stats::active::read().unwrap_or_default()),
+		("mapped", stats::mapped::read().unwrap_or_default()),
+		("metadata", stats::metadata::read().unwrap_or_default()),
+		("resident", stats::resident::read().unwrap_or_default()),
+		("retained", stats::retained::read().unwrap_or_default()),
+	])
+}
+
+#[must_use]
+#[cfg(not(feature = "jemalloc_stats"))]
+pub fn memory_usage_stats() -> Option<[(&'static str, usize); 6]> { None }
+
 pub fn memory_stats(opts: &str) -> Option<String> {
 	const MAX_LENGTH: usize = 1_048_576;
 

src/core/alloc/mod.rs

@@ -4,7 +4,7 @@
 #[cfg(all(not(target_env = "msvc"), feature = "jemalloc"))]
 pub mod je;
 #[cfg(all(not(target_env = "msvc"), feature = "jemalloc"))]
-pub use je::{memory_stats, memory_usage, trim};
+pub use je::{memory_stats, memory_usage, memory_usage_stats, trim};
 
 #[cfg(all(not(target_env = "msvc"), feature = "hardened_malloc", not(feature = "jemalloc")))]
 pub mod hardened;
@@ -13,7 +13,7 @@ pub mod hardened;
 	feature = "hardened_malloc",
 	not(feature = "jemalloc")
 ))]
-pub use hardened::{memory_stats, memory_usage, trim};
+pub use hardened::{memory_stats, memory_usage, memory_usage_stats, trim};
 
 #[cfg(any(
 	target_env = "msvc",
@@ -24,4 +24,4 @@ pub mod default;
 	target_env = "msvc",
 	all(not(feature = "hardened_malloc"), not(feature = "jemalloc"))
 ))]
-pub use default::{memory_stats, memory_usage, trim};
+pub use default::{memory_stats, memory_usage, memory_usage_stats, trim};

src/core/config/mod.rs

@@ -1026,6 +1026,15 @@ pub struct Config {
 	#[serde(default = "default_otlp_protocol")]
 	pub otlp_protocol: String,
 
+	/// Enable OpenTelemetry OTLP metrics export, using the same endpoint and
+	/// `otlp_protocol` as traces.
+	///
+	/// The export interval can be changed with the
+	/// OTEL_METRIC_EXPORT_INTERVAL environment variable (milliseconds,
+	/// defaults to 60000).
+	#[serde(default)]
+	pub allow_otlp_metrics: bool,
+
 	/// Enable the Prometheus metrics endpoint at `/_continuwuity/metrics`.
 	///
 	/// The endpoint is served on the same listener as the rest of the server.

src/core/metrics/instruments.rs

@@ -0,0 +1,296 @@
+use std::time::{Duration, SystemTime};
+
+use opentelemetry::{
+	KeyValue,
+	metrics::{Counter, Histogram, Meter, UpDownCounter},
+};
+use tokio::runtime;
+
+use super::process;
+
+pub struct Instruments {
+	pub http_server_request_duration: Histogram<f64>,
+	pub http_server_active_requests: UpDownCounter<i64>,
+
+	pub federation_outgoing_request_duration: Histogram<f64>,
+	pub federation_incoming_transaction_duration: Histogram<f64>,
+	pub federation_incoming_pdus: Counter<u64>,
+	pub federation_incoming_pdu_duration: Histogram<f64>,
+	pub federation_incoming_pdu_lag: Histogram<f64>,
+	pub federation_incoming_edus: Counter<u64>,
+	pub federation_incoming_edu_duration: Histogram<f64>,
+
+	pub sender_transaction_duration: Histogram<f64>,
+	pub sender_events: Counter<u64>,
+	pub sender_events_queued: Counter<u64>,
+
+	pub media_write_size: Histogram<u64>,
+	pub media_read_size: Histogram<u64>,
+
+	pub antispam_checks: Counter<u64>,
+	pub invites: Counter<u64>,
+}
+
+impl Instruments {
+	pub(super) fn new(meter: &Meter, runtime_metrics: Option<runtime::RuntimeMetrics>) -> Self {
+		register_process(meter);
+		register_allocator(meter);
+		if let Some(runtime_metrics) = runtime_metrics {
+			register_runtime(meter, runtime_metrics);
+		}
+
+		let duration = |name: &'static str, description: &'static str| {
+			meter
+				.f64_histogram(name)
+				.with_description(description)
+				.with_unit("s")
+				.with_boundaries(duration_buckets())
+				.build()
+		};
+		let size = |name: &'static str, description: &'static str| {
+			meter
+				.u64_histogram(name)
+				.with_description(description)
+				.with_unit("By")
+				.with_boundaries(size_buckets())
+				.build()
+		};
+
+		Self {
+			http_server_request_duration: duration(
+				"http.server.request.duration",
+				"Duration of handled HTTP requests",
+			),
+			http_server_active_requests: meter
+				.i64_up_down_counter("http.server.active_requests")
+				.with_description("Number of HTTP requests currently being handled")
+				.with_unit("{request}")
+				.build(),
+
+			federation_outgoing_request_duration: duration(
+				"continuwuity.federation.outgoing.request.duration",
+				"Duration of outgoing federation requests",
+			),
+			federation_incoming_transaction_duration: duration(
+				"continuwuity.federation.incoming.transaction.duration",
+				"Time taken to process incoming federation transactions",
+			),
+			federation_incoming_pdus: meter
+				.u64_counter("continuwuity.federation.incoming.pdus")
+				.with_description("Incoming federation PDUs by event type and outcome")
+				.with_unit("{pdu}")
+				.build(),
+			federation_incoming_pdu_duration: duration(
+				"continuwuity.federation.incoming.pdu.duration",
+				"Time taken to handle an incoming federation PDU",
+			),
+			federation_incoming_pdu_lag: meter
+				.f64_histogram("continuwuity.federation.incoming.pdu.lag")
+				.with_description(
+					"Time between an accepted incoming PDU's origin_server_ts and its arrival",
+				)
+				.with_unit("s")
+				.with_boundaries(vec![
+					0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0, 30.0, 60.0, 300.0, 900.0, 3600.0,
+				])
+				.build(),
+			federation_incoming_edus: meter
+				.u64_counter("continuwuity.federation.incoming.edus")
+				.with_description("Incoming federation EDUs by type")
+				.with_unit("{edu}")
+				.build(),
+			federation_incoming_edu_duration: duration(
+				"continuwuity.federation.incoming.edu.duration",
+				"Time taken to handle an incoming federation EDU",
+			),
+
+			sender_transaction_duration: duration(
+				"continuwuity.sender.transaction.duration",
+				"Duration of outgoing transactions by destination type and outcome",
+			),
+			sender_events: meter
+				.u64_counter("continuwuity.sender.events")
+				.with_description("PDUs and EDUs sent in outgoing transactions by outcome")
+				.with_unit("{event}")
+				.build(),
+			sender_events_queued: meter
+				.u64_counter("continuwuity.sender.events.queued")
+				.with_description("PDUs and EDUs queued for sending")
+				.with_unit("{event}")
+				.build(),
+
+			media_write_size: size(
+				"continuwuity.media.write.size",
+				"Size of media files written, from local uploads or cached remote media",
+			),
+			media_read_size: size("continuwuity.media.read.size", "Size of media files served"),
+
+			antispam_checks: meter
+				.u64_counter("continuwuity.antispam.checks")
+				.with_description("Antispam service checks by check and outcome")
+				.with_unit("{check}")
+				.build(),
+			invites: meter
+				.u64_counter("continuwuity.invites")
+				.with_description("Invites sent by local users or received over federation")
+				.with_unit("{invite}")
+				.build(),
+		}
+	}
+}
+
+/// Collapse non-standard methods so arbitrary client input cannot create new
+/// attribute values.
+#[must_use]
+pub fn method_label(method: &http::Method) -> &'static str {
+	match *method {
+		| http::Method::GET => "GET",
+		| http::Method::HEAD => "HEAD",
+		| http::Method::POST => "POST",
+		| http::Method::PUT => "PUT",
+		| http::Method::DELETE => "DELETE",
+		| http::Method::OPTIONS => "OPTIONS",
+		| http::Method::PATCH => "PATCH",
+		| http::Method::CONNECT => "CONNECT",
+		| http::Method::TRACE => "TRACE",
+		| _ if method.as_str() == "QUERY" => "QUERY",
+		| _ => "_OTHER",
+	}
+}
+
+fn duration_buckets() -> Vec<f64> { (0..16).map(|i| 0.001 * f64::from(1_u32 << i)).collect() }
+
+fn size_buckets() -> Vec<f64> { (0..11).map(|i| 1024.0 * f64::from(4_u32.pow(i))).collect() }
+
+fn register_process(meter: &Meter) {
+	let started = SystemTime::now();
+	meter
+		.f64_observable_gauge("process.uptime")
+		.with_description("Time since the server started")
+		.with_unit("s")
+		.with_callback(move |observer| {
+			let uptime = started.elapsed().unwrap_or_default();
+			observer.observe(uptime.as_secs_f64(), &[]);
+		})
+		.build();
+
+	meter
+		.f64_observable_counter("process.cpu.time")
+		.with_description("Total CPU time spent by the process")
+		.with_unit("s")
+		.with_callback(|observer| {
+			if let Some((user, system)) = process::cpu_time() {
+				observer.observe(user.as_secs_f64(), &[KeyValue::new("cpu.mode", "user")]);
+				observer.observe(system.as_secs_f64(), &[KeyValue::new("cpu.mode", "system")]);
+			}
+		})
+		.build();
+
+	meter
+		.i64_observable_up_down_counter("process.memory.usage")
+		.with_description("Resident memory of the process")
+		.with_unit("By")
+		.with_callback(|observer| {
+			if let Some(resident) =
+				process::memory().and_then(|(resident, _)| resident.try_into().ok())
+			{
+				observer.observe(resident, &[]);
+			}
+		})
+		.build();
+
+	meter
+		.i64_observable_up_down_counter("process.memory.virtual")
+		.with_description("Virtual memory of the process")
+		.with_unit("By")
+		.with_callback(|observer| {
+			if let Some(virt) = process::memory().and_then(|(_, virt)| virt.try_into().ok()) {
+				observer.observe(virt, &[]);
+			}
+		})
+		.build();
+
+	meter
+		.i64_observable_up_down_counter("process.unix.file_descriptor.count")
+		.with_description("Number of file descriptors in use by the process")
+		.with_unit("{file_descriptor}")
+		.with_callback(|observer| {
+			if let Some(fds) = process::open_fds().and_then(|fds| fds.try_into().ok()) {
+				observer.observe(fds, &[]);
+			}
+		})
+		.build();
+}
+
+fn register_allocator(meter: &Meter) {
+	meter
+		.u64_observable_gauge("continuwuity.jemalloc.memory")
+		.with_description("jemalloc memory statistics, when built with jemalloc_stats")
+		.with_unit("By")
+		.with_callback(|observer| {
+			for (kind, bytes) in crate::alloc::memory_usage_stats().into_iter().flatten() {
+				if let Ok(bytes) = bytes.try_into() {
+					observer.observe(bytes, &[KeyValue::new("kind", kind)]);
+				}
+			}
+		})
+		.build();
+}
+
+fn register_runtime(meter: &Meter, metrics: runtime::RuntimeMetrics) {
+	let gauge = |name: &'static str,
+	             description: &'static str,
+	             read: fn(&runtime::RuntimeMetrics) -> usize| {
+		let metrics = metrics.clone();
+		meter
+			.u64_observable_gauge(name)
+			.with_description(description)
+			.with_callback(move |observer| {
+				if let Ok(value) = read(&metrics).try_into() {
+					observer.observe(value, &[]);
+				}
+			})
+			.build();
+	};
+
+	gauge(
+		"continuwuity.tokio.workers",
+		"Number of tokio runtime worker threads",
+		runtime::RuntimeMetrics::num_workers,
+	);
+	gauge(
+		"continuwuity.tokio.alive_tasks",
+		"Number of alive tasks in the tokio runtime",
+		runtime::RuntimeMetrics::num_alive_tasks,
+	);
+	gauge(
+		"continuwuity.tokio.global_queue_depth",
+		"Number of tasks in the tokio runtime global queue",
+		runtime::RuntimeMetrics::global_queue_depth,
+	);
+
+	let busy_metrics = metrics.clone();
+	meter
+		.f64_observable_counter("continuwuity.tokio.worker.busy_time")
+		.with_description("Total time tokio worker threads spent busy")
+		.with_unit("s")
+		.with_callback(move |observer| {
+			let busy: Duration = (0..busy_metrics.num_workers())
+				.map(|worker| busy_metrics.worker_total_busy_duration(worker))
+				.sum();
+			observer.observe(busy.as_secs_f64(), &[]);
+		})
+		.build();
+
+	meter
+		.u64_observable_counter("continuwuity.tokio.worker.parks")
+		.with_description("Total number of times tokio worker threads parked")
+		.with_unit("{park}")
+		.with_callback(move |observer| {
+			let parks = (0..metrics.num_workers())
+				.map(|worker| metrics.worker_park_count(worker))
+				.sum();
+			observer.observe(parks, &[]);
+		})
+		.build();
+}

src/core/metrics/mod.rs

@@ -1,16 +1,20 @@
-mod prometheus;
+mod instruments;
+mod process;
+mod tests;
 
 use std::sync::atomic::AtomicU32;
 
+use opentelemetry::metrics::Meter;
+pub use opentelemetry::{KeyValue, metrics::AsyncInstrument};
+pub use prometheus::TEXT_FORMAT as PROMETHEUS_CONTENT_TYPE;
+use prometheus::{Encoder, TextEncoder};
 use tokio::runtime;
 use tokio_metrics::TaskMonitor;
 #[cfg(tokio_unstable)]
 use tokio_metrics::{RuntimeIntervals, RuntimeMonitor};
 
-pub use self::prometheus::{
-	ColumnLabels, HttpLabels, NameLabels, OutcomeLabels, Prometheus, TransactionLabels,
-	method_label,
-};
+pub use self::instruments::{Instruments, method_label};
+use crate::{Err, Result};
 
 pub struct Metrics {
 	_runtime: Option<runtime::Handle>,
@@ -30,12 +34,18 @@ pub struct Metrics {
 	pub requests_handle_finished: AtomicU32,
 	pub requests_panic: AtomicU32,
 
-	pub prometheus: Prometheus,
+	pub meter: Meter,
+	pub instruments: Instruments,
+	registry: Option<prometheus::Registry>,
 }
 
 impl Metrics {
 	#[must_use]
-	pub fn new(runtime: Option<runtime::Handle>) -> Self {
+	pub fn new(
+		runtime: Option<runtime::Handle>,
+		meter: Meter,
+		registry: Option<prometheus::Registry>,
+	) -> Self {
 		#[cfg(tokio_unstable)]
 		let runtime_monitor = runtime.as_ref().map(RuntimeMonitor::new);
 
@@ -47,7 +57,9 @@ impl Metrics {
 		Self {
 			_runtime: runtime.clone(),
 
-			prometheus: Prometheus::new(runtime_metrics.clone()),
+			instruments: Instruments::new(&meter, runtime_metrics.clone()),
+			meter,
+			registry,
 
 			runtime_metrics,
 
@@ -75,6 +87,19 @@ impl Metrics {
 			.expect("next interval")
 	}
 
+	pub fn render(&self) -> Result<String> {
+		let Some(registry) = &self.registry else {
+			return Err!("Prometheus metrics are not enabled");
+		};
+
+		let mut buf = Vec::new();
+		TextEncoder::new()
+			.encode(&registry.gather(), &mut buf)
+			.map_err(|e| crate::err!("{e}"))?;
+
+		Ok(String::from_utf8(buf)?)
+	}
+
 	#[inline]
 	pub fn task_root(&self) -> Option<&TaskMonitor> { self.task_monitor.as_ref() }
 

src/core/metrics/process.rs

@@ -0,0 +1,43 @@
+use std::time::Duration;
+
+/// User and system CPU time.
+#[cfg(unix)]
+pub(super) fn cpu_time() -> Option<(Duration, Duration)> {
+	use nix::sys::resource::{UsageWho, getrusage};
+
+	let usage = getrusage(UsageWho::RUSAGE_SELF).ok()?;
+	let to_duration = |tv: nix::sys::time::TimeVal| {
+		Duration::from_secs(tv.tv_sec().try_into().ok()?)
+			.checked_add(Duration::from_micros(tv.tv_usec().try_into().ok()?))
+	};
+
+	Some((to_duration(usage.user_time())?, to_duration(usage.system_time())?))
+}
+
+#[cfg(not(unix))]
+pub(super) fn cpu_time() -> Option<(Duration, Duration)> { None }
+
+/// Resident and virtual memory in bytes, read from `/proc/self/status`.
+#[cfg(target_os = "linux")]
+pub(super) fn memory() -> Option<(u64, u64)> {
+	let status = std::fs::read_to_string("/proc/self/status").ok()?;
+	let field = |name: &str| -> Option<u64> {
+		let value = status.lines().find_map(|line| line.strip_prefix(name))?;
+		let kib: u64 = value.trim().trim_end_matches(" kB").parse().ok()?;
+		kib.checked_mul(1024)
+	};
+
+	Some((field("VmRSS:")?, field("VmSize:")?))
+}
+
+#[cfg(not(target_os = "linux"))]
+pub(super) fn memory() -> Option<(u64, u64)> { None }
+
+#[cfg(target_os = "linux")]
+pub(super) fn open_fds() -> Option<u64> {
+	let count = std::fs::read_dir("/proc/self/fd").ok()?.count();
+	count.try_into().ok()
+}
+
+#[cfg(not(target_os = "linux"))]
+pub(super) fn open_fds() -> Option<u64> { None }

src/core/metrics/prometheus.rs

@@ -1,261 +0,0 @@
-use std::time::Duration;
-
-use prometheus_client::{
-	collector::Collector,
-	encoding::{DescriptorEncoder, EncodeLabelSet, EncodeMetric, text},
-	metrics::{
-		counter::{ConstCounter, Counter},
-		family::Family,
-		gauge::{ConstGauge, Gauge},
-		histogram::{Histogram, exponential_buckets},
-		info::Info,
-	},
-	registry::{Registry, Unit},
-};
-use tokio::runtime;
-
-use crate::Result;
-
-#[derive(Clone, Debug, Hash, PartialEq, Eq, EncodeLabelSet)]
-pub struct HttpLabels {
-	pub method: &'static str,
-	pub route: String,
-	pub status: u16,
-}
-
-#[derive(Clone, Debug, Hash, PartialEq, Eq, EncodeLabelSet)]
-pub struct OutcomeLabels {
-	pub outcome: &'static str,
-}
-
-#[derive(Clone, Debug, Hash, PartialEq, Eq, EncodeLabelSet)]
-pub struct TransactionLabels {
-	pub destination: &'static str,
-	pub outcome: &'static str,
-}
-
-#[derive(Clone, Debug, Hash, PartialEq, Eq, EncodeLabelSet)]
-pub struct NameLabels {
-	pub name: &'static str,
-}
-
-#[derive(Clone, Debug, Hash, PartialEq, Eq, EncodeLabelSet)]
-pub struct ColumnLabels {
-	pub column: String,
-}
-
-pub struct Prometheus {
-	registry: Registry,
-
-	pub http_requests_in_flight: Gauge,
-	pub http_request_duration: Family<HttpLabels, Histogram>,
-
-	pub federation_request_duration: Family<OutcomeLabels, Histogram>,
-	pub federation_unhealthy_destinations: Gauge,
-	pub federation_stale_destinations: Gauge,
-
-	pub sender_transactions: Family<TransactionLabels, Counter>,
-	pub sender_queue_depth: Gauge,
-
-	pub database_memory: Family<NameLabels, Gauge>,
-	pub database_block_cache: Family<ColumnLabels, Gauge>,
-	pub cache_entries: Family<NameLabels, Gauge>,
-}
-
-impl Prometheus {
-	pub(super) fn new(runtime_metrics: Option<runtime::RuntimeMetrics>) -> Self {
-		let mut this = Self {
-			registry: Registry::with_prefix("continuwuity"),
-			http_requests_in_flight: Gauge::default(),
-			http_request_duration: Family::new_with_constructor(duration_histogram),
-			federation_request_duration: Family::new_with_constructor(duration_histogram),
-			federation_unhealthy_destinations: Gauge::default(),
-			federation_stale_destinations: Gauge::default(),
-			sender_transactions: Family::default(),
-			sender_queue_depth: Gauge::default(),
-			database_memory: Family::default(),
-			database_block_cache: Family::default(),
-			cache_entries: Family::default(),
-		};
-
-		this.registry.register(
-			"http_requests_in_flight",
-			"Number of client and federation requests currently being handled",
-			this.http_requests_in_flight.clone(),
-		);
-		this.registry.register_with_unit(
-			"http_request_duration",
-			"Duration of handled HTTP requests",
-			Unit::Seconds,
-			this.http_request_duration.clone(),
-		);
-		this.registry.register_with_unit(
-			"federation_request_duration",
-			"Duration of outgoing federation requests",
-			Unit::Seconds,
-			this.federation_request_duration.clone(),
-		);
-		this.registry.register(
-			"federation_unhealthy_destinations",
-			"Number of remote servers currently in retry backoff",
-			this.federation_unhealthy_destinations.clone(),
-		);
-		this.registry.register(
-			"federation_stale_destinations",
-			"Number of remote servers with a stale destination cache entry",
-			this.federation_stale_destinations.clone(),
-		);
-		this.registry.register(
-			"sender_transactions",
-			"Outgoing transactions completed by the sender, by destination type and outcome",
-			this.sender_transactions.clone(),
-		);
-		this.registry.register(
-			"sender_queue_depth",
-			"Number of messages waiting in the sender worker channels",
-			this.sender_queue_depth.clone(),
-		);
-		this.registry.register_with_unit(
-			"database_memory",
-			"RocksDB memory usage by component",
-			Unit::Bytes,
-			this.database_memory.clone(),
-		);
-		this.registry.register_with_unit(
-			"database_block_cache",
-			"RocksDB block cache usage by column family",
-			Unit::Bytes,
-			this.database_block_cache.clone(),
-		);
-		this.registry.register(
-			"cache_entries",
-			"Number of entries held in in-memory caches",
-			this.cache_entries.clone(),
-		);
-
-		this.registry.register(
-			"build",
-			"Build information",
-			Info::new([("version", crate::version())]),
-		);
-
-		if let Some(runtime_metrics) = runtime_metrics {
-			this.registry
-				.register_collector(Box::new(RuntimeCollector(runtime_metrics)));
-		}
-
-		this
-	}
-
-	pub fn encode(&self) -> Result<String> {
-		let mut buf = String::new();
-		text::encode(&mut buf, &self.registry)?;
-		Ok(buf)
-	}
-}
-
-/// Collapse non-standard methods so arbitrary client input cannot create new
-/// label values.
-#[must_use]
-pub fn method_label(method: &http::Method) -> &'static str {
-	match *method {
-		| http::Method::GET => "GET",
-		| http::Method::HEAD => "HEAD",
-		| http::Method::POST => "POST",
-		| http::Method::PUT => "PUT",
-		| http::Method::DELETE => "DELETE",
-		| http::Method::OPTIONS => "OPTIONS",
-		| http::Method::PATCH => "PATCH",
-		| _ => "OTHER",
-	}
-}
-
-fn duration_histogram() -> Histogram { Histogram::new(exponential_buckets(0.001, 2.0, 16)) }
-
-#[derive(Debug)]
-struct RuntimeCollector(runtime::RuntimeMetrics);
-
-impl Collector for RuntimeCollector {
-	fn encode(&self, mut encoder: DescriptorEncoder<'_>) -> Result<(), std::fmt::Error> {
-		let metrics = &self.0;
-		let workers = metrics.num_workers();
-		let busy: Duration = (0..workers)
-			.map(|worker| metrics.worker_total_busy_duration(worker))
-			.sum();
-		let parks: u64 = (0..workers)
-			.map(|worker| metrics.worker_park_count(worker))
-			.sum();
-
-		let gauges = [
-			("tokio_workers", "Number of tokio runtime worker threads.", workers),
-			(
-				"tokio_alive_tasks",
-				"Number of alive tasks in the tokio runtime.",
-				metrics.num_alive_tasks(),
-			),
-			(
-				"tokio_global_queue_depth",
-				"Number of tasks in the tokio runtime global queue.",
-				metrics.global_queue_depth(),
-			),
-		];
-		for (name, help, value) in gauges {
-			let gauge = ConstGauge::new(value);
-			gauge.encode(encoder.encode_descriptor(name, help, None, gauge.metric_type())?)?;
-		}
-
-		let busy = ConstCounter::new(busy.as_secs_f64());
-		busy.encode(encoder.encode_descriptor(
-			"tokio_worker_busy",
-			"Total time tokio worker threads spent busy.",
-			Some(&Unit::Seconds),
-			busy.metric_type(),
-		)?)?;
-
-		let parks = ConstCounter::new(parks);
-		parks.encode(encoder.encode_descriptor(
-			"tokio_worker_parks",
-			"Total number of times tokio worker threads parked.",
-			None,
-			parks.metric_type(),
-		)?)?;
-
-		Ok(())
-	}
-}
-
-#[cfg(test)]
-mod tests {
-	use super::{HttpLabels, Prometheus, method_label};
-
-	#[tokio::test]
-	async fn encodes_openmetrics() {
-		let prometheus = Prometheus::new(Some(tokio::runtime::Handle::current().metrics()));
-		prometheus
-			.http_request_duration
-			.get_or_create(&HttpLabels {
-				method: method_label(&http::Method::GET),
-				route: "/_matrix/client/versions".to_owned(),
-				status: 200,
-			})
-			.observe(0.01);
-
-		let body = prometheus.encode().unwrap();
-		for name in [
-			"continuwuity_http_request_duration_seconds_count{method=\"GET\",route=\"/_matrix/\
-			 client/versions\",status=\"200\"} 1",
-			"continuwuity_http_requests_in_flight 0",
-			"continuwuity_build_info{version=",
-			"# TYPE continuwuity_tokio_worker_busy_seconds counter",
-		] {
-			assert!(body.contains(name), "missing {name:?} in:\n{body}");
-		}
-		assert!(body.ends_with("# EOF\n"));
-	}
-
-	#[test]
-	fn method_label_is_bounded() {
-		let custom = http::Method::from_bytes(b"BREW").unwrap();
-		assert_eq!(method_label(&custom), "OTHER");
-	}
-}

src/core/metrics/tests.rs

@@ -0,0 +1,54 @@
+#![cfg(test)]
+
+use opentelemetry::metrics::MeterProvider;
+use opentelemetry_sdk::metrics::SdkMeterProvider;
+
+use super::{KeyValue, Metrics, method_label};
+
+fn metrics() -> (Metrics, SdkMeterProvider) {
+	let registry = prometheus::Registry::new();
+	let exporter = opentelemetry_prometheus::exporter()
+		.with_registry(registry.clone())
+		.scope_info_enabled(false)
+		.build()
+		.unwrap();
+	let provider = SdkMeterProvider::builder().with_reader(exporter).build();
+	let meter = provider.meter("continuwuity");
+	let handle = tokio::runtime::Handle::current();
+
+	(Metrics::new(Some(handle), meter, Some(registry)), provider)
+}
+
+#[tokio::test]
+async fn renders_prometheus_names() {
+	let (metrics, _provider) = metrics();
+	let instruments = &metrics.instruments;
+	instruments.http_server_request_duration.record(0.01, &[
+		KeyValue::new("http.request.method", method_label(&http::Method::GET)),
+		KeyValue::new("http.route", "/_matrix/client/versions"),
+		KeyValue::new("http.response.status_code", 200_i64),
+	]);
+	instruments
+		.sender_events
+		.add(3, &[KeyValue::new("destination", "federation"), KeyValue::new("kind", "pdu")]);
+
+	let body = metrics.render().unwrap();
+	for expected in [
+		"http_server_request_duration_seconds_count{http_request_method=\"GET\",\
+		 http_response_status_code=\"200\",http_route=\"/_matrix/client/versions\"} 1",
+		"continuwuity_sender_events_total{destination=\"federation\",kind=\"pdu\"} 3",
+		"# TYPE process_uptime_seconds gauge",
+		"process_cpu_time_seconds_total{cpu_mode=\"user\"}",
+		"# TYPE continuwuity_tokio_worker_busy_time_seconds_total counter",
+	] {
+		assert!(body.contains(expected), "missing {expected:?} in:\n{body}");
+	}
+	assert!(!body.contains("otel_scope_name"));
+}
+
+#[test]
+fn method_label_is_bounded() {
+	let method = |name: &[u8]| http::Method::from_bytes(name).unwrap();
+	assert_eq!(method_label(&method(b"QUERY")), "QUERY");
+	assert_eq!(method_label(&method(b"BREW")), "_OTHER");
+}

src/core/server.rs

@@ -49,7 +49,12 @@ pub struct Server {
 
 impl Server {
 	#[must_use]
-	pub fn new(config: Config, runtime: Option<runtime::Handle>, log: Log) -> Self {
+	pub fn new(
+		config: Config,
+		runtime: Option<runtime::Handle>,
+		log: Log,
+		metrics: Metrics,
+	) -> Self {
 		Self {
 			name: config.server_name.clone(),
 			config: config::Manager::new(config),
@@ -57,10 +62,10 @@ impl Server {
 			stopping: AtomicBool::new(false),
 			reloading: AtomicBool::new(false),
 			restarting: AtomicBool::new(false),
-			runtime: runtime.clone(),
+			runtime,
 			signal: broadcast::channel::<&'static str>(1).0,
 			log,
-			metrics: Metrics::new(runtime),
+			metrics,
 		}
 	}