feat(metrics): Record metrics with OpenTelemetry and export over OTLP

src/api/server/invite.rs

@@ -5,6 +5,7 @@ use base64::{Engine as _, engine::general_purpose};
 use conduwuit::{
 	Err, Error, EventTypeExt, PduEvent, Result, debug, err, error,
 	matrix::{Event, StateKey},
+	metrics::KeyValue,
 	result::FlatOk,
 	state_res, trace,
 	utils::hash::sha256,
@@ -31,6 +32,21 @@ use crate::{Ruma, server::utils::validate_any_membership_event};
 pub(crate) async fn create_invite_route(
 	State(services): State<crate::State>,
 	body: Ruma<create_invite::v2::Request>,
+) -> Result<create_invite::v2::Response> {
+	let result = create_invite(&services, body).await;
+
+	services.server.metrics.instruments.invites.add(1, &[
+		KeyValue::new("direction", "incoming"),
+		KeyValue::new("destination", "local"),
+		KeyValue::new("outcome", if result.is_ok() { "success" } else { "failure" }),
+	]);
+
+	result
+}
+
+async fn create_invite(
+	services: &crate::State,
+	body: Ruma<create_invite::v2::Request>,
 ) -> Result<create_invite::v2::Response> {
 	if !services.server.supported_room_version(&body.room_version) {
 		return Err(Error::BadRequest(
@@ -64,7 +80,7 @@ pub(crate) async fn create_invite_route(
 		"Validating invite room state for invite request"
 	);
 	let (create_event_id, state) = validate_invite_state(
-		&services,
+		services,
 		&body.invite_room_state,
 		&room_version_rules,
 		body.room_id.clone(),
@@ -92,7 +108,7 @@ pub(crate) async fn create_invite_route(
 
 	// And then we can validate the member event itself
 	let (mut signed_event, sender_user, recipient_user) = validate_invite_membership_event(
-		&services,
+		services,
 		&body.event,
 		&room_version_rules,
 		&body.identity,

src/api/server/send.rs

@@ -6,6 +6,7 @@ use std::{
 use axum::extract::State;
 use conduwuit::{
 	Err, Error, Result, debug, debug_error, debug_warn, err, error,
+	metrics::KeyValue,
 	result::LogErr,
 	trace,
 	utils::{
@@ -22,7 +23,8 @@ use futures::{FutureExt, Stream, StreamExt, TryFutureExt, TryStreamExt};
 use http::StatusCode;
 use itertools::Itertools;
 use ruma::{
-	CanonicalJsonObject, OwnedEventId, OwnedRoomId, OwnedUserId, RoomId, ServerName, UserId,
+	CanonicalJsonObject, CanonicalJsonValue, OwnedEventId, OwnedRoomId, OwnedUserId, RoomId,
+	ServerName, UserId,
 	api::{
 		error::{ErrorKind, LimitExceededErrorData},
 		federation::transactions::{
@@ -34,7 +36,10 @@ use ruma::{
 			send_transaction_message,
 		},
 	},
-	events::receipt::{ReceiptEvent, ReceiptEventContent, ReceiptType},
+	events::{
+		TimelineEventType,
+		receipt::{ReceiptEvent, ReceiptEventContent, ReceiptType},
+	},
 	serde::Raw,
 	to_device::DeviceIdOrAllDevices,
 };
@@ -151,6 +156,12 @@ async fn process_inbound_transaction(
 		{
 			| Ok(pdu) => Some(pdu),
 			| Err(error) => {
+				services
+					.server
+					.metrics
+					.instruments
+					.federation_incoming_pdus
+					.add(1, &[KeyValue::new("outcome", "unparsable")]);
 				let queued = services
 					.rooms
 					.event_handler
@@ -181,6 +192,17 @@ async fn process_inbound_transaction(
 			.catch_unwind()
 			.map_err(Error::from_panic)
 			.await;
+	let outcome = if matches!(handle_result, Ok(Ok(_))) {
+		"success"
+	} else {
+		"failure"
+	};
+	services
+		.server
+		.metrics
+		.instruments
+		.federation_incoming_transaction_duration
+		.record(txn_start_time.elapsed().as_secs_f64(), &[KeyValue::new("outcome", outcome)]);
 	let results = match handle_result {
 		| Ok(Ok(results)) => results,
 		| Ok(Err(err)) => {
@@ -337,6 +359,13 @@ async fn handle_room(
 			.server
 			.check_running()
 			.map_err(|_| TransactionError::ShuttingDown)?;
+		let event_type = KeyValue::new("type", pdu_type_label(&value));
+		let origin_server_ts = value
+			.get("origin_server_ts")
+			.and_then(CanonicalJsonValue::as_integer)
+			.and_then(|ts| u64::try_from(i64::from(ts)).ok());
+
+		let start = Instant::now();
 		let result = services
 			.rooms
 			.event_handler
@@ -344,12 +373,44 @@ async fn handle_room(
 			.boxed()
 			.await
 			.map(|_| ());
+
+		let outcome = KeyValue::new("outcome", match &result {
+			| Ok(()) => "accepted",
+			| Err(Error::BadRequest(ErrorKind::Forbidden, ..)) => "forbidden",
+			| Err(Error::BadRequest(ErrorKind::InvalidParam | ErrorKind::BadJson, ..)) =>
+				"invalid",
+			| Err(_) => "error",
+		});
+		let instruments = &services.server.metrics.instruments;
+		instruments
+			.federation_incoming_pdu_duration
+			.record(start.elapsed().as_secs_f64(), std::slice::from_ref(&outcome));
+		if let (Ok(()), Some(ts)) = (&result, origin_server_ts) {
+			let lag = millis_since_unix_epoch().saturating_sub(ts);
+			instruments
+				.federation_incoming_pdu_lag
+				.record(Duration::from_millis(lag).as_secs_f64(), &[]);
+		}
+		instruments
+			.federation_incoming_pdus
+			.add(1, &[event_type, outcome]);
+
 		results.push((event_id, result));
 	}
 	Ok(results)
 }
 
 async fn handle_edu(services: &Services, origin: &ServerName, edu: Edu) {
+	let edu_type = match &edu {
+		| Edu::Presence(_) => "m.presence",
+		| Edu::Receipt(_) => "m.receipt",
+		| Edu::Typing(_) => "m.typing",
+		| Edu::DeviceListUpdate(_) => "m.device_list_update",
+		| Edu::DirectToDevice(_) => "m.direct_to_device",
+		| Edu::SigningKeyUpdate(_) => "m.signing_key_update",
+		| _ => "other",
+	};
+	let start = Instant::now();
 	match edu {
 		| Edu::Presence(presence) if services.server.config.allow_incoming_presence =>
 			handle_edu_presence(services, origin, presence).await,
@@ -373,6 +434,26 @@ async fn handle_edu(services: &Services, origin: &ServerName, edu: Edu) {
 
 		| _ => trace!(?edu, "skipped"),
 	}
+
+	let edu_type = [KeyValue::new("type", edu_type)];
+	let instruments = &services.server.metrics.instruments;
+	instruments
+		.federation_incoming_edu_duration
+		.record(start.elapsed().as_secs_f64(), &edu_type);
+	instruments.federation_incoming_edus.add(1, &edu_type);
+}
+
+/// Known event types, or `other`, so remote servers cannot create new
+/// attribute values.
+fn pdu_type_label(value: &CanonicalJsonObject) -> String {
+	match value
+		.get("type")
+		.and_then(CanonicalJsonValue::as_str)
+		.map(TimelineEventType::from)
+	{
+		| Some(TimelineEventType::_Custom(_)) | None => "other".to_owned(),
+		| Some(event_type) => event_type.to_string(),
+	}
 }
 
 async fn handle_edu_presence(