feat(service/sending): Sender Supervision

This commit is contained in:
Jacob Taylor committed 2026-11-20 20:29:42 -08:00
1 parent 42a66fecb6
commit 3ab25fff0a
12 files changed
+408 -79

No files matched your search

+1
View File
@@ -0,0 +1 @@
Fix outgoing federation stalling after a sender worker panics. Contributed by @eleboucher.
+3
View File
@@ -71,6 +71,9 @@ Names below are as they appear in Prometheus. OTLP uses the dotted OpenTelemetry
| `continuwuity_sender_events_total` | counter | `destination`, `kind`, `outcome` | PDUs and EDUs sent in outgoing transactions |
| `continuwuity_sender_events_queued_total` | counter | `destination`, `kind` | PDUs and EDUs queued for sending |
| `continuwuity_sender_queue_depth` | gauge | | Messages waiting for the sender workers |
| `continuwuity_sender_requests` | gauge | `state` | Estimated outgoing requests stored as `queued` or `active` (in flight) |
| `continuwuity_sender_workers` | gauge | | Running sender workers; below the configured count means a worker is restarting |
| `continuwuity_sender_panics_total` | counter | `scope` | Panics caught in a single outgoing `transaction` or a whole sender `worker` |
| `continuwuity_users` | gauge | `origin` | Known `local` users, including deactivated, and `remote` users |
| `continuwuity_invites_total` | counter | `direction`, `destination`, `outcome` | Invites sent by local users or received over federation |
| `continuwuity_media_write_size_bytes` | histogram | `origin` | Media files written, from local uploads or cached remote media |
+6
View File
@@ -21,6 +21,7 @@ pub struct Instruments {
pub sender_transaction_duration: Histogram<f64>,
pub sender_events: Counter<u64>,
pub sender_events_queued: Counter<u64>,
pub sender_panics: Counter<u64>,
pub media_write_size: Histogram<u64>,
pub media_read_size: Histogram<u64>,
@@ -106,6 +107,11 @@ impl Instruments {
.with_description("PDUs and EDUs queued for sending")
.with_unit("{event}")
.build(),
sender_panics: meter
.u64_counter("continuwuity.sender.panics")
.with_description("Panics caught in outgoing transactions and sender workers")
.with_unit("{panic}")
.build(),
media_write_size: size(
"continuwuity.media.write.size",
+22
View File
@@ -18,6 +18,28 @@ impl Services {
},
);
self.gauge(
"continuwuity.sender.requests",
"Estimated outgoing requests stored as queued or in flight",
"{request}",
|services, observer| {
for (state, count) in services.sending.db.request_counts() {
if let Some(count) = count {
observer.observe(count, &[KeyValue::new("state", state)]);
}
}
},
);
self.gauge(
"continuwuity.sender.workers",
"Running sender workers",
"{worker}",
|services, observer| {
observe(observer, services.sending.workers_alive(), &[]);
},
);
self.gauge(
"continuwuity.federation.incoming.transactions.active",
"Incoming federation transactions currently being processed",
+2
View File
@@ -13,6 +13,8 @@ mod migrations;
mod service;
pub mod services;
pub mod state;
#[cfg(test)]
mod test_utils;
pub mod account_data;
pub mod admin;
@@ -1,14 +1,10 @@
use std::sync::Arc;
use conduwuit::{
Config, PduEvent, Server,
log::{Log, LogLevelReloadHandles},
metrics::Metrics,
};
use conduwuit::PduEvent;
use futures::StreamExt;
use ruma::{RoomId, RoomVersionId, events::StateEventType};
use serde_json::json;
use crate::test_utils::test_services;
#[tokio::test]
async fn admitted_joins_do_not_disappear_after_a_merge() {
use conduwuit::matrix::PartialPdu;
@@ -20,7 +16,7 @@ async fn admitted_joins_do_not_disappear_after_a_merge() {
},
};
let (services, path) = test_services().await;
let (services, path, _database) = test_services().await;
let alice = UserId::parse("@alice:test.example").unwrap();
let bob = UserId::parse("@bob:bob.example").unwrap();
let charlie = UserId::parse("@charlie:charlie.example").unwrap();
@@ -212,35 +208,6 @@ async fn admitted_joins_do_not_disappear_after_a_merge() {
assert_eq!(results, [true, true], "both joined users must be able to send messages");
}
async fn test_services() -> (Arc<crate::Services>, std::path::PathBuf) {
let _ = rustls::crypto::ring::default_provider().install_default();
let path = std::env::temp_dir()
.join(format!("continuwuity-shared-state-{:016x}", rand::random::<u64>()));
std::fs::create_dir(&path).unwrap();
let config: Config = serde_json::from_value(json!({
"server_name": "test.example",
"database_path": path,
"db_cache_capacity_mb": 16.0,
"rocksdb_write_buffer_capacity_mb": 4.0,
"allow_local_presence": false,
"dns_servers": ["127.0.0.1"],
"allow_invalid_tls_certificates_yes_i_know_what_the_fuck_i_am_doing_with_this_and_i_know_this_is_insecure": true
}))
.unwrap();
let runtime = tokio::runtime::Handle::current();
let server = Arc::new(Server::new(
config,
Some(runtime.clone()),
Log {
reload: LogLevelReloadHandles::default(),
capture: Arc::default(),
},
Metrics::new(Some(runtime), opentelemetry::global::meter("test"), None),
));
let services = crate::Services::build(server).await.unwrap();
(services, path)
}
fn cache_key(
services: &crate::Services,
server: &ruma::ServerName,
+28
View File
@@ -3,6 +3,7 @@ use std::{fmt::Debug, sync::Arc};
use conduwuit::{
Error, Result, at, utils,
utils::{ReadyExt, stream::TryIgnore},
warn,
};
use database::{Database, Deserialized, Map};
use futures::{Stream, StreamExt};
@@ -169,6 +170,33 @@ impl Data {
})
}
/// The oldest queued request of each destination, not yet in flight.
pub(super) async fn queued_heads(&self) -> Vec<(Destination, QueueItem)> {
let mut heads: Vec<(Destination, QueueItem)> = Vec::new();
self.servernameevent_data
.raw_stream()
.ignore_err()
.ready_for_each(|(key, val)| match parse_servercurrentevent(key, val) {
| Ok((dest, event)) =>
if heads.last().is_none_or(|(last, _)| *last != dest) {
heads.push((dest, (key.to_vec(), event)));
},
| Err(e) => warn!(?key, "invalid queued request key: {e}"),
})
.await;
heads
}
/// Estimated number of requests waiting to be sent and in flight.
pub(crate) fn request_counts(&self) -> [(&'static str, Option<u64>); 2] {
[
("queued", &self.servernameevent_data),
("active", &self.servercurrentevent_data),
]
.map(|(state, map)| (state, map.property_integer(c"rocksdb.estimate-num-keys").ok()))
}
pub(super) fn set_latest_educount(&self, server_name: &ServerName, last_count: u64) {
self.servername_educount.raw_put(server_name, last_count);
}
+106 -24
View File
@@ -3,12 +3,19 @@ mod appservice;
mod data;
mod dest;
mod sender;
#[cfg(test)]
mod tests;
use std::{
collections::HashMap,
fmt::Debug,
hash::{DefaultHasher, Hash, Hasher},
iter::once,
sync::Arc,
sync::{
Arc,
atomic::{AtomicUsize, Ordering},
},
time::{Duration, Instant},
};
use async_trait::async_trait;
@@ -19,7 +26,7 @@ use conduwuit::{
utils::{ReadyExt, TryReadyExt, available_parallelism, math::usize_from_u64_truncated},
warn,
};
use futures::{FutureExt, Stream, StreamExt};
use futures::{FutureExt, Stream, StreamExt, stream::FuturesUnordered};
use ruma::{
OwnedServerName, RoomId, ServerName, UserId,
api::{
@@ -27,7 +34,7 @@ use ruma::{
federation::authentication::ServerSignatures, path_builder::PathBuilder,
},
};
use tokio::{task, task::JoinSet};
use tokio::{task, task::JoinSet, time::sleep};
use self::data::Data;
pub use self::{
@@ -49,6 +56,7 @@ pub struct Service {
server: Arc<Server>,
services: Services,
channels: Vec<(loole::Sender<Msg>, loole::Receiver<Msg>)>,
workers_alive: AtomicUsize,
}
struct Services {
@@ -87,6 +95,8 @@ pub type EduVec = SmallVec<[EduBuf; EDU_VEC_CAP]>;
const EDU_BUF_CAP: usize = 128;
const EDU_VEC_CAP: usize = 1;
const WORKER_RESTART_DELAY: Duration = Duration::from_millis(2500);
const WORKER_RESTART_DELAY_MAX: Duration = Duration::from_secs(30);
#[async_trait]
impl crate::Service for Service {
@@ -111,36 +121,67 @@ impl crate::Service for Service {
federation: args.depend::<federation::Service>("federation"),
},
channels: (0..num_senders).map(|_| loole::unbounded()).collect(),
workers_alive: AtomicUsize::new(0),
}))
}
async fn worker(self: Arc<Self>) -> Result {
let mut senders =
self.channels
.iter()
.enumerate()
.fold(JoinSet::new(), |mut joinset, (id, _)| {
let self_ = self.clone();
let worker = self_.sender(id);
let worker = if self.unconstrained() {
task::unconstrained(worker).boxed()
} else {
worker.boxed()
if self.server.config.startup_netburst {
self.dispatch_queued(None).await;
}
let num_senders = self.channels.len();
let mut senders = JoinSet::new();
let mut shards = HashMap::new();
let mut started = vec![Instant::now(); num_senders];
let mut failures = vec![0_u32; num_senders];
let mut restarts = FuturesUnordered::new();
for id in 0..num_senders {
self.spawn_sender(&mut senders, &mut shards, id, false);
}
loop {
tokio::select! {
Some(ret) = senders.join_next_with_id() => {
self.workers_alive.fetch_sub(1, Ordering::Relaxed);
let (task_id, panicked) = match &ret {
| Ok((task_id, _)) => (*task_id, false),
| Err(error) => (error.id(), error.is_panic()),
};
let shard = shards.remove(&task_id).expect("sender worker has a shard");
let runtime = self.server.runtime();
let _abort = joinset.spawn_on(worker, runtime);
joinset
});
if panicked {
self.server
.metrics
.instruments
.sender_panics
.add(1, &[KeyValue::new("scope", "worker")]);
}
while let Some(ret) = senders.join_next_with_id().await {
match ret {
| Ok((id, error)) => {
debug!(?id, ?error, "sender worker finished");
if self.channels[shard].1.is_closed() || !self.server.running() {
debug!(shard, ?ret, "sender worker finished");
continue;
}
// A worker that ran a while before failing starts over from
// the shortest delay.
failures[shard] = if started[shard].elapsed() > WORKER_RESTART_DELAY_MAX {
1
} else {
failures[shard].saturating_add(1)
};
let delay = restart_delay(failures[shard]);
error!(shard, ?ret, ?delay, "sender worker unexpectedly finished, restarting");
restarts.push(sleep(delay).map(move |()| shard));
},
| Err(error) => {
error!(id = ?error.id(), ?error, "sender worker failed");
Some(shard) = restarts.next() => {
if !self.channels[shard].1.is_closed() {
started[shard] = Instant::now();
self.spawn_sender(&mut senders, &mut shards, shard, true);
}
},
() = self.server.until_shutdown(), if !restarts.is_empty() => restarts.clear(),
else => break,
}
}
@@ -161,6 +202,25 @@ impl crate::Service for Service {
}
impl Service {
fn spawn_sender(
self: &Arc<Self>,
senders: &mut JoinSet<Result>,
shards: &mut HashMap<task::Id, usize>,
id: usize,
restarted: bool,
) {
let worker = self.clone().sender(id, restarted);
let worker = if crate::Service::unconstrained(&**self) {
task::unconstrained(worker).boxed()
} else {
worker.boxed()
};
let handle = senders.spawn_on(worker, self.server.runtime());
shards.insert(handle.id(), id);
self.workers_alive.fetch_add(1, Ordering::Relaxed);
}
#[tracing::instrument(skip(self, pdu_id, user, pushkey), level = "debug")]
pub fn send_pdu_push(&self, pdu_id: &RawPduId, user: &UserId, pushkey: String) -> Result {
let dest = Destination::Push(user.to_owned(), pushkey);
@@ -480,10 +540,24 @@ impl Service {
sender.send(msg).map_err(|e| err!("{e}"))
}
/// Hands the oldest unsent request of each destination to its worker, so
/// a backlog with nothing in flight is sent without waiting for a new
/// event. Bypasses `dispatch` as these requests were already counted.
async fn dispatch_queued(&self, shard: Option<usize>) {
for (dest, (queue_id, event)) in self.db.queued_heads().await {
let id = self.shard_id(&dest);
if shard.is_none_or(|shard| shard == id) {
self.channels[id].0.send(Msg { dest, event, queue_id }).ok();
}
}
}
pub fn queue_depth(&self) -> usize {
self.channels.iter().map(|(sender, _)| sender.len()).sum()
}
pub fn workers_alive(&self) -> usize { self.workers_alive.load(Ordering::Relaxed) }
pub(super) fn shard_id(&self, dest: &Destination) -> usize {
if self.channels.len() <= 1 {
return 0;
@@ -519,3 +593,11 @@ fn num_senders(args: &crate::Args<'_>) -> usize {
worker_count.clamp(MIN_SENDERS, max_senders)
}
}
/// Delay before restarting a worker after its `failures`-th consecutive
/// failure, doubling each time.
fn restart_delay(failures: u32) -> Duration {
WORKER_RESTART_DELAY
.saturating_mul(2_u32.saturating_pow(failures.saturating_sub(1)))
.min(WORKER_RESTART_DELAY_MAX)
}
+56 -18
View File
@@ -1,6 +1,7 @@
use std::{
collections::{BTreeMap, HashMap, HashSet},
fmt::Debug,
panic::AssertUnwindSafe,
sync::{
Arc,
atomic::{AtomicU64, AtomicUsize, Ordering},
@@ -55,6 +56,9 @@ use serde_json::value::{RawValue as RawJsonValue, to_raw_value};
use super::{Destination, EduBuf, EduVec, Msg, SendingEvent, Service, data::QueueItem};
#[cfg(test)]
mod tests;
#[derive(Debug)]
enum TransactionStatus {
Running,
@@ -78,12 +82,18 @@ pub const EDU_LIMIT: usize = 100;
impl Service {
#[tracing::instrument(skip(self), level = "debug")]
pub(super) async fn sender(self: Arc<Self>, id: usize) -> Result {
pub(super) async fn sender(self: Arc<Self>, id: usize, restarted: bool) -> Result {
let mut statuses: CurTransactionStatus = CurTransactionStatus::new();
let mut futures: SendingFutures<'_> = FuturesUnordered::new();
// The previous worker of this shard may have dropped requests it had
// taken from the channel.
if restarted {
self.dispatch_queued(Some(id)).await;
}
info!(shard = id, "Running startup netburst");
self.startup_netburst(id, &mut futures, &mut statuses)
self.startup_netburst(id, restarted, &mut futures, &mut statuses)
.boxed()
.await;
info!(shard = id, "Finished startup netburst");
@@ -142,6 +152,9 @@ impl Service {
) {
match response {
| Ok(dest) => self.handle_response_ok(&dest, futures, statuses).await,
// A bug on our side, not the remote's fault: retry with backoff
// without marking the remote unhealthy.
| Err((dest, e)) if e.is_panic() => Self::handle_response_err(dest, statuses, &e),
| Err((dest, e)) => {
if let Destination::Federation(dest) = &dest {
if self.services.federation.should_mark_stale(&e)
@@ -274,7 +287,8 @@ impl Service {
}
/// Performs startup netburst, which sends any previously queued up but not
/// yet sent requests immediately.
/// yet sent requests immediately. A restarted worker always resends its
/// shard's requests in flight.
#[tracing::instrument(
name = "netburst",
level = "debug",
@@ -285,6 +299,7 @@ impl Service {
async fn startup_netburst<'a>(
&'a self,
id: usize,
restarted: bool,
futures: &mut SendingFutures<'a>,
statuses: &mut CurTransactionStatus,
) {
@@ -315,8 +330,12 @@ impl Service {
}
}
if !restarted && !self.server.config.startup_netburst {
return;
}
for (dest, events) in txns {
if self.server.config.startup_netburst && !events.is_empty() {
if !events.is_empty() {
statuses.insert(dest.clone(), TransactionStatus::Running);
futures.push(self.send_events(dest, events));
}
@@ -772,6 +791,7 @@ impl Service {
});
let start = Instant::now();
let panic_dest = dest.clone();
let send = match dest {
| Destination::Federation(server) =>
self.send_events_dest_federation(server, events).boxed(),
@@ -780,24 +800,42 @@ impl Service {
self.send_events_dest_push(user_id, pushkey, events).boxed(),
};
send.inspect(move |result| {
let instruments = &self.server.metrics.instruments;
let outcome =
KeyValue::new("outcome", if result.is_ok() { "success" } else { "failure" });
instruments
.sender_transaction_duration
.record(start.elapsed().as_secs_f64(), &[destination.clone(), outcome.clone()]);
for (kind, count) in [("pdu", pdus), ("edu", edus)] {
if count > 0 {
instruments.sender_events.add(count, &[
AssertUnwindSafe(send)
.catch_unwind()
.map(move |result| {
result.unwrap_or_else(|panic| {
let error = Error::from_panic(panic);
error!(dest = ?panic_dest, ?error, "Panicked while sending transaction");
self.server
.metrics
.instruments
.sender_panics
.add(1, &[KeyValue::new("scope", "transaction")]);
Err((panic_dest, Box::new(error)))
})
})
.inspect(move |result| {
let instruments = &self.server.metrics.instruments;
let outcome =
KeyValue::new("outcome", if result.is_ok() { "success" } else { "failure" });
instruments
.sender_transaction_duration
.record(start.elapsed().as_secs_f64(), &[
destination.clone(),
KeyValue::new("kind", kind),
outcome.clone(),
]);
for (kind, count) in [("pdu", pdus), ("edu", edus)] {
if count > 0 {
instruments.sender_events.add(count, &[
destination.clone(),
KeyValue::new("kind", kind),
outcome.clone(),
]);
}
}
}
})
.boxed()
})
.boxed()
}
#[tracing::instrument(
+86
View File
@@ -0,0 +1,86 @@
use std::iter::once;
use conduwuit::Error;
use futures::{StreamExt, stream::FuturesUnordered};
use ruma::server_name;
use serde_json::json;
use super::{CurTransactionStatus, TransactionStatus};
use crate::{
sending::{Destination, EduBuf, SendingEvent},
test_utils::{test_services, test_services_with},
};
#[tokio::test]
async fn panicked_transaction_is_kept_for_retry_without_blaming_the_remote() {
let (services, path, _database) = test_services().await;
let sending = &services.sending;
let server = server_name!("remote.example");
let dest = Destination::Federation(server.to_owned());
let event = SendingEvent::Edu(EduBuf::from_slice(br#"{"edu_type":"m.typing"}"#));
sending.db.queue_requests(once((&event, &dest)));
let in_flight: Vec<_> = sending.db.queued_requests(&dest).collect().await;
sending.db.mark_as_active(in_flight.iter());
let mut futures = FuturesUnordered::new();
let mut statuses = CurTransactionStatus::from([(dest.clone(), TransactionStatus::Running)]);
let panic = Error::from_panic(Box::new("boom"));
sending
.handle_response(Err((dest.clone(), Box::new(panic))), &mut futures, &mut statuses)
.await;
let active: Vec<_> = sending
.db
.active_requests_for(&dest)
.map(|(key, _)| key)
.collect()
.await;
let retry_after = sending.services.federation.retry_after(server);
let sending_next = futures.len();
let failed = matches!(statuses.get(&dest), Some(TransactionStatus::Failed(1, _)));
drop(futures);
drop(services);
std::fs::remove_dir_all(path).unwrap();
assert_eq!(
active,
in_flight
.into_iter()
.map(|(key, _)| key)
.collect::<Vec<_>>()
);
assert_eq!(sending_next, 0);
assert!(failed);
assert_eq!(retry_after, None);
}
#[tokio::test]
async fn restarted_worker_resends_trimmed_in_flight_requests() {
let (services, path, _database) =
test_services_with(json!({ "startup_netburst_keep": 1 })).await;
let sending = &services.sending;
let dest = Destination::Federation(server_name!("remote.example").to_owned());
let event = SendingEvent::Edu(EduBuf::from_slice(br#"{"edu_type":"m.typing"}"#));
for _ in 0..3 {
sending.db.queue_requests(once((&event, &dest)));
}
let queued: Vec<_> = sending.db.queued_requests(&dest).collect().await;
sending.db.mark_as_active(queued.iter());
let id = sending.shard_id(&dest);
let mut futures = FuturesUnordered::new();
let mut statuses = CurTransactionStatus::new();
sending
.startup_netburst(id, true, &mut futures, &mut statuses)
.await;
let active = sending.db.active_requests_for(&dest).count().await;
let sending_now = futures.len();
drop(futures);
drop(services);
std::fs::remove_dir_all(path).unwrap();
assert_eq!(active, 1);
assert_eq!(sending_now, 1);
}
+42
View File
@@ -0,0 +1,42 @@
use std::{iter::once, time::Duration};
use futures::StreamExt;
use ruma::server_name;
use super::{Destination, EduBuf, SendingEvent, restart_delay};
use crate::test_utils::test_services;
#[tokio::test]
async fn queued_heads_lists_the_oldest_unsent_request_per_destination() {
let (services, path, _database) = test_services().await;
let db = &services.sending.db;
let queued = Destination::Federation(server_name!("queued.example").to_owned());
let active = Destination::Federation(server_name!("active.example").to_owned());
let event = SendingEvent::Edu(EduBuf::from_slice(br#"{"edu_type":"m.typing"}"#));
let first = db.queue_requests(once((&event, &queued)));
db.queue_requests(once((&event, &queued)));
db.queue_requests(once((&event, &active)));
let in_flight: Vec<_> = db.queued_requests(&active).collect().await;
db.mark_as_active(in_flight.iter());
let heads: Vec<_> = db
.queued_heads()
.await
.into_iter()
.map(|(dest, (key, _))| (dest, key))
.collect();
drop(services);
std::fs::remove_dir_all(path).unwrap();
assert_eq!(heads, [(queued, first[0].clone())]);
}
#[test]
fn restart_delay_doubles_up_to_the_cap() {
assert_eq!(restart_delay(1), Duration::from_millis(2500));
assert_eq!(restart_delay(2), Duration::from_secs(5));
assert_eq!(restart_delay(4), Duration::from_secs(20));
assert_eq!(restart_delay(5), Duration::from_secs(30));
assert_eq!(restart_delay(u32::MAX), Duration::from_secs(30));
}
+52
View File
@@ -0,0 +1,52 @@
use std::{path::PathBuf, sync::Arc};
use conduwuit::{
Config, Server,
log::{Log, LogLevelReloadHandles},
metrics::Metrics,
};
use serde_json::json;
use tokio::sync::{Mutex, MutexGuard};
/// RocksDB's default `Env` is shared by the whole process and closing a
/// database stops its background threads, so tests must not keep databases
/// open in parallel. Hold the returned guard until the services are dropped.
static DATABASE: Mutex<()> = Mutex::const_new(());
type TestServices = (Arc<crate::Services>, PathBuf, MutexGuard<'static, ()>);
pub(crate) async fn test_services() -> TestServices { test_services_with(json!({})).await }
pub(crate) async fn test_services_with(overrides: serde_json::Value) -> TestServices {
let guard = DATABASE.lock().await;
let _ = rustls::crypto::ring::default_provider().install_default();
let path = std::env::temp_dir()
.join(format!("continuwuity-shared-state-{:016x}", rand::random::<u64>()));
std::fs::create_dir(&path).unwrap();
let mut config = json!({
"server_name": "test.example",
"database_path": path,
"db_cache_capacity_mb": 16.0,
"rocksdb_write_buffer_capacity_mb": 4.0,
"allow_local_presence": false,
"dns_servers": ["127.0.0.1"],
"allow_invalid_tls_certificates_yes_i_know_what_the_fuck_i_am_doing_with_this_and_i_know_this_is_insecure": true
});
config
.as_object_mut()
.unwrap()
.extend(overrides.as_object().unwrap().clone());
let config: Config = serde_json::from_value(config).unwrap();
let runtime = tokio::runtime::Handle::current();
let server = Arc::new(Server::new(
config,
Some(runtime.clone()),
Log {
reload: LogLevelReloadHandles::default(),
capture: Arc::default(),
},
Metrics::new(Some(runtime), opentelemetry::global::meter("test"), None),
));
let services = crate::Services::build(server).await.unwrap();
(services, path, guard)
}