diff --git a/changelog.d/sender-supervision.bugfix.md b/changelog.d/sender-supervision.bugfix.md new file mode 100644 index 000000000..bbd9bce83 --- /dev/null +++ b/changelog.d/sender-supervision.bugfix.md @@ -0,0 +1 @@ +Fix outgoing federation stalling after a sender worker panics. Contributed by @eleboucher. diff --git a/docs/guides/metrics.mdx b/docs/guides/metrics.mdx index cb5df8c70..cbc706ee1 100644 --- a/docs/guides/metrics.mdx +++ b/docs/guides/metrics.mdx @@ -71,6 +71,9 @@ Names below are as they appear in Prometheus. OTLP uses the dotted OpenTelemetry | `continuwuity_sender_events_total` | counter | `destination`, `kind`, `outcome` | PDUs and EDUs sent in outgoing transactions | | `continuwuity_sender_events_queued_total` | counter | `destination`, `kind` | PDUs and EDUs queued for sending | | `continuwuity_sender_queue_depth` | gauge | | Messages waiting for the sender workers | +| `continuwuity_sender_requests` | gauge | `state` | Estimated outgoing requests stored as `queued` or `active` (in flight) | +| `continuwuity_sender_workers` | gauge | | Running sender workers; below the configured count means a worker is restarting | +| `continuwuity_sender_panics_total` | counter | `scope` | Panics caught in a single outgoing `transaction` or a whole sender `worker` | | `continuwuity_users` | gauge | `origin` | Known `local` users, including deactivated, and `remote` users | | `continuwuity_invites_total` | counter | `direction`, `destination`, `outcome` | Invites sent by local users or received over federation | | `continuwuity_media_write_size_bytes` | histogram | `origin` | Media files written, from local uploads or cached remote media | diff --git a/src/core/metrics/instruments.rs b/src/core/metrics/instruments.rs index b9ba973b2..3f8d6b97a 100644 --- a/src/core/metrics/instruments.rs +++ b/src/core/metrics/instruments.rs @@ -21,6 +21,7 @@ pub struct Instruments { pub sender_transaction_duration: Histogram, pub sender_events: Counter, pub sender_events_queued: Counter, + pub sender_panics: Counter, pub media_write_size: Histogram, pub media_read_size: Histogram, @@ -106,6 +107,11 @@ impl Instruments { .with_description("PDUs and EDUs queued for sending") .with_unit("{event}") .build(), + sender_panics: meter + .u64_counter("continuwuity.sender.panics") + .with_description("Panics caught in outgoing transactions and sender workers") + .with_unit("{panic}") + .build(), media_write_size: size( "continuwuity.media.write.size", diff --git a/src/service/metrics.rs b/src/service/metrics.rs index a1291de06..80a985f4e 100644 --- a/src/service/metrics.rs +++ b/src/service/metrics.rs @@ -18,6 +18,28 @@ impl Services { }, ); + self.gauge( + "continuwuity.sender.requests", + "Estimated outgoing requests stored as queued or in flight", + "{request}", + |services, observer| { + for (state, count) in services.sending.db.request_counts() { + if let Some(count) = count { + observer.observe(count, &[KeyValue::new("state", state)]); + } + } + }, + ); + + self.gauge( + "continuwuity.sender.workers", + "Running sender workers", + "{worker}", + |services, observer| { + observe(observer, services.sending.workers_alive(), &[]); + }, + ); + self.gauge( "continuwuity.federation.incoming.transactions.active", "Incoming federation transactions currently being processed", diff --git a/src/service/mod.rs b/src/service/mod.rs index 0cb476cb0..aaf7973ab 100644 --- a/src/service/mod.rs +++ b/src/service/mod.rs @@ -13,6 +13,8 @@ mod migrations; mod service; pub mod services; pub mod state; +#[cfg(test)] +mod test_utils; pub mod account_data; pub mod admin; diff --git a/src/service/rooms/event_handler/state_at_incoming/tests.rs b/src/service/rooms/event_handler/state_at_incoming/tests.rs index d46395cee..fc90905e3 100644 --- a/src/service/rooms/event_handler/state_at_incoming/tests.rs +++ b/src/service/rooms/event_handler/state_at_incoming/tests.rs @@ -1,14 +1,10 @@ -use std::sync::Arc; - -use conduwuit::{ - Config, PduEvent, Server, - log::{Log, LogLevelReloadHandles}, - metrics::Metrics, -}; +use conduwuit::PduEvent; use futures::StreamExt; use ruma::{RoomId, RoomVersionId, events::StateEventType}; use serde_json::json; +use crate::test_utils::test_services; + #[tokio::test] async fn admitted_joins_do_not_disappear_after_a_merge() { use conduwuit::matrix::PartialPdu; @@ -20,7 +16,7 @@ async fn admitted_joins_do_not_disappear_after_a_merge() { }, }; - let (services, path) = test_services().await; + let (services, path, _database) = test_services().await; let alice = UserId::parse("@alice:test.example").unwrap(); let bob = UserId::parse("@bob:bob.example").unwrap(); let charlie = UserId::parse("@charlie:charlie.example").unwrap(); @@ -212,35 +208,6 @@ async fn admitted_joins_do_not_disappear_after_a_merge() { assert_eq!(results, [true, true], "both joined users must be able to send messages"); } -async fn test_services() -> (Arc, std::path::PathBuf) { - let _ = rustls::crypto::ring::default_provider().install_default(); - let path = std::env::temp_dir() - .join(format!("continuwuity-shared-state-{:016x}", rand::random::())); - std::fs::create_dir(&path).unwrap(); - let config: Config = serde_json::from_value(json!({ - "server_name": "test.example", - "database_path": path, - "db_cache_capacity_mb": 16.0, - "rocksdb_write_buffer_capacity_mb": 4.0, - "allow_local_presence": false, - "dns_servers": ["127.0.0.1"], - "allow_invalid_tls_certificates_yes_i_know_what_the_fuck_i_am_doing_with_this_and_i_know_this_is_insecure": true - })) - .unwrap(); - let runtime = tokio::runtime::Handle::current(); - let server = Arc::new(Server::new( - config, - Some(runtime.clone()), - Log { - reload: LogLevelReloadHandles::default(), - capture: Arc::default(), - }, - Metrics::new(Some(runtime), opentelemetry::global::meter("test"), None), - )); - let services = crate::Services::build(server).await.unwrap(); - (services, path) -} - fn cache_key( services: &crate::Services, server: &ruma::ServerName, diff --git a/src/service/sending/data.rs b/src/service/sending/data.rs index 59f5c0647..3f34bca91 100644 --- a/src/service/sending/data.rs +++ b/src/service/sending/data.rs @@ -3,6 +3,7 @@ use std::{fmt::Debug, sync::Arc}; use conduwuit::{ Error, Result, at, utils, utils::{ReadyExt, stream::TryIgnore}, + warn, }; use database::{Database, Deserialized, Map}; use futures::{Stream, StreamExt}; @@ -169,6 +170,33 @@ impl Data { }) } + /// The oldest queued request of each destination, not yet in flight. + pub(super) async fn queued_heads(&self) -> Vec<(Destination, QueueItem)> { + let mut heads: Vec<(Destination, QueueItem)> = Vec::new(); + self.servernameevent_data + .raw_stream() + .ignore_err() + .ready_for_each(|(key, val)| match parse_servercurrentevent(key, val) { + | Ok((dest, event)) => + if heads.last().is_none_or(|(last, _)| *last != dest) { + heads.push((dest, (key.to_vec(), event))); + }, + | Err(e) => warn!(?key, "invalid queued request key: {e}"), + }) + .await; + + heads + } + + /// Estimated number of requests waiting to be sent and in flight. + pub(crate) fn request_counts(&self) -> [(&'static str, Option); 2] { + [ + ("queued", &self.servernameevent_data), + ("active", &self.servercurrentevent_data), + ] + .map(|(state, map)| (state, map.property_integer(c"rocksdb.estimate-num-keys").ok())) + } + pub(super) fn set_latest_educount(&self, server_name: &ServerName, last_count: u64) { self.servername_educount.raw_put(server_name, last_count); } diff --git a/src/service/sending/mod.rs b/src/service/sending/mod.rs index 4bd1f76a1..180f8da12 100644 --- a/src/service/sending/mod.rs +++ b/src/service/sending/mod.rs @@ -3,12 +3,19 @@ mod appservice; mod data; mod dest; mod sender; +#[cfg(test)] +mod tests; use std::{ + collections::HashMap, fmt::Debug, hash::{DefaultHasher, Hash, Hasher}, iter::once, - sync::Arc, + sync::{ + Arc, + atomic::{AtomicUsize, Ordering}, + }, + time::{Duration, Instant}, }; use async_trait::async_trait; @@ -19,7 +26,7 @@ use conduwuit::{ utils::{ReadyExt, TryReadyExt, available_parallelism, math::usize_from_u64_truncated}, warn, }; -use futures::{FutureExt, Stream, StreamExt}; +use futures::{FutureExt, Stream, StreamExt, stream::FuturesUnordered}; use ruma::{ OwnedServerName, RoomId, ServerName, UserId, api::{ @@ -27,7 +34,7 @@ use ruma::{ federation::authentication::ServerSignatures, path_builder::PathBuilder, }, }; -use tokio::{task, task::JoinSet}; +use tokio::{task, task::JoinSet, time::sleep}; use self::data::Data; pub use self::{ @@ -49,6 +56,7 @@ pub struct Service { server: Arc, services: Services, channels: Vec<(loole::Sender, loole::Receiver)>, + workers_alive: AtomicUsize, } struct Services { @@ -87,6 +95,8 @@ pub type EduVec = SmallVec<[EduBuf; EDU_VEC_CAP]>; const EDU_BUF_CAP: usize = 128; const EDU_VEC_CAP: usize = 1; +const WORKER_RESTART_DELAY: Duration = Duration::from_millis(2500); +const WORKER_RESTART_DELAY_MAX: Duration = Duration::from_secs(30); #[async_trait] impl crate::Service for Service { @@ -111,36 +121,67 @@ impl crate::Service for Service { federation: args.depend::("federation"), }, channels: (0..num_senders).map(|_| loole::unbounded()).collect(), + workers_alive: AtomicUsize::new(0), })) } async fn worker(self: Arc) -> Result { - let mut senders = - self.channels - .iter() - .enumerate() - .fold(JoinSet::new(), |mut joinset, (id, _)| { - let self_ = self.clone(); - let worker = self_.sender(id); - let worker = if self.unconstrained() { - task::unconstrained(worker).boxed() - } else { - worker.boxed() + if self.server.config.startup_netburst { + self.dispatch_queued(None).await; + } + + let num_senders = self.channels.len(); + let mut senders = JoinSet::new(); + let mut shards = HashMap::new(); + let mut started = vec![Instant::now(); num_senders]; + let mut failures = vec![0_u32; num_senders]; + let mut restarts = FuturesUnordered::new(); + for id in 0..num_senders { + self.spawn_sender(&mut senders, &mut shards, id, false); + } + + loop { + tokio::select! { + Some(ret) = senders.join_next_with_id() => { + self.workers_alive.fetch_sub(1, Ordering::Relaxed); + let (task_id, panicked) = match &ret { + | Ok((task_id, _)) => (*task_id, false), + | Err(error) => (error.id(), error.is_panic()), }; + let shard = shards.remove(&task_id).expect("sender worker has a shard"); - let runtime = self.server.runtime(); - let _abort = joinset.spawn_on(worker, runtime); - joinset - }); + if panicked { + self.server + .metrics + .instruments + .sender_panics + .add(1, &[KeyValue::new("scope", "worker")]); + } - while let Some(ret) = senders.join_next_with_id().await { - match ret { - | Ok((id, error)) => { - debug!(?id, ?error, "sender worker finished"); + if self.channels[shard].1.is_closed() || !self.server.running() { + debug!(shard, ?ret, "sender worker finished"); + continue; + } + + // A worker that ran a while before failing starts over from + // the shortest delay. + failures[shard] = if started[shard].elapsed() > WORKER_RESTART_DELAY_MAX { + 1 + } else { + failures[shard].saturating_add(1) + }; + let delay = restart_delay(failures[shard]); + error!(shard, ?ret, ?delay, "sender worker unexpectedly finished, restarting"); + restarts.push(sleep(delay).map(move |()| shard)); }, - | Err(error) => { - error!(id = ?error.id(), ?error, "sender worker failed"); + Some(shard) = restarts.next() => { + if !self.channels[shard].1.is_closed() { + started[shard] = Instant::now(); + self.spawn_sender(&mut senders, &mut shards, shard, true); + } }, + () = self.server.until_shutdown(), if !restarts.is_empty() => restarts.clear(), + else => break, } } @@ -161,6 +202,25 @@ impl crate::Service for Service { } impl Service { + fn spawn_sender( + self: &Arc, + senders: &mut JoinSet, + shards: &mut HashMap, + id: usize, + restarted: bool, + ) { + let worker = self.clone().sender(id, restarted); + let worker = if crate::Service::unconstrained(&**self) { + task::unconstrained(worker).boxed() + } else { + worker.boxed() + }; + + let handle = senders.spawn_on(worker, self.server.runtime()); + shards.insert(handle.id(), id); + self.workers_alive.fetch_add(1, Ordering::Relaxed); + } + #[tracing::instrument(skip(self, pdu_id, user, pushkey), level = "debug")] pub fn send_pdu_push(&self, pdu_id: &RawPduId, user: &UserId, pushkey: String) -> Result { let dest = Destination::Push(user.to_owned(), pushkey); @@ -480,10 +540,24 @@ impl Service { sender.send(msg).map_err(|e| err!("{e}")) } + /// Hands the oldest unsent request of each destination to its worker, so + /// a backlog with nothing in flight is sent without waiting for a new + /// event. Bypasses `dispatch` as these requests were already counted. + async fn dispatch_queued(&self, shard: Option) { + for (dest, (queue_id, event)) in self.db.queued_heads().await { + let id = self.shard_id(&dest); + if shard.is_none_or(|shard| shard == id) { + self.channels[id].0.send(Msg { dest, event, queue_id }).ok(); + } + } + } + pub fn queue_depth(&self) -> usize { self.channels.iter().map(|(sender, _)| sender.len()).sum() } + pub fn workers_alive(&self) -> usize { self.workers_alive.load(Ordering::Relaxed) } + pub(super) fn shard_id(&self, dest: &Destination) -> usize { if self.channels.len() <= 1 { return 0; @@ -519,3 +593,11 @@ fn num_senders(args: &crate::Args<'_>) -> usize { worker_count.clamp(MIN_SENDERS, max_senders) } } + +/// Delay before restarting a worker after its `failures`-th consecutive +/// failure, doubling each time. +fn restart_delay(failures: u32) -> Duration { + WORKER_RESTART_DELAY + .saturating_mul(2_u32.saturating_pow(failures.saturating_sub(1))) + .min(WORKER_RESTART_DELAY_MAX) +} diff --git a/src/service/sending/sender.rs b/src/service/sending/sender.rs index 26cb371b1..fe94a8ff7 100644 --- a/src/service/sending/sender.rs +++ b/src/service/sending/sender.rs @@ -1,6 +1,7 @@ use std::{ collections::{BTreeMap, HashMap, HashSet}, fmt::Debug, + panic::AssertUnwindSafe, sync::{ Arc, atomic::{AtomicU64, AtomicUsize, Ordering}, @@ -55,6 +56,9 @@ use serde_json::value::{RawValue as RawJsonValue, to_raw_value}; use super::{Destination, EduBuf, EduVec, Msg, SendingEvent, Service, data::QueueItem}; +#[cfg(test)] +mod tests; + #[derive(Debug)] enum TransactionStatus { Running, @@ -78,12 +82,18 @@ pub const EDU_LIMIT: usize = 100; impl Service { #[tracing::instrument(skip(self), level = "debug")] - pub(super) async fn sender(self: Arc, id: usize) -> Result { + pub(super) async fn sender(self: Arc, id: usize, restarted: bool) -> Result { let mut statuses: CurTransactionStatus = CurTransactionStatus::new(); let mut futures: SendingFutures<'_> = FuturesUnordered::new(); + // The previous worker of this shard may have dropped requests it had + // taken from the channel. + if restarted { + self.dispatch_queued(Some(id)).await; + } + info!(shard = id, "Running startup netburst"); - self.startup_netburst(id, &mut futures, &mut statuses) + self.startup_netburst(id, restarted, &mut futures, &mut statuses) .boxed() .await; info!(shard = id, "Finished startup netburst"); @@ -142,6 +152,9 @@ impl Service { ) { match response { | Ok(dest) => self.handle_response_ok(&dest, futures, statuses).await, + // A bug on our side, not the remote's fault: retry with backoff + // without marking the remote unhealthy. + | Err((dest, e)) if e.is_panic() => Self::handle_response_err(dest, statuses, &e), | Err((dest, e)) => { if let Destination::Federation(dest) = &dest { if self.services.federation.should_mark_stale(&e) @@ -274,7 +287,8 @@ impl Service { } /// Performs startup netburst, which sends any previously queued up but not - /// yet sent requests immediately. + /// yet sent requests immediately. A restarted worker always resends its + /// shard's requests in flight. #[tracing::instrument( name = "netburst", level = "debug", @@ -285,6 +299,7 @@ impl Service { async fn startup_netburst<'a>( &'a self, id: usize, + restarted: bool, futures: &mut SendingFutures<'a>, statuses: &mut CurTransactionStatus, ) { @@ -315,8 +330,12 @@ impl Service { } } + if !restarted && !self.server.config.startup_netburst { + return; + } + for (dest, events) in txns { - if self.server.config.startup_netburst && !events.is_empty() { + if !events.is_empty() { statuses.insert(dest.clone(), TransactionStatus::Running); futures.push(self.send_events(dest, events)); } @@ -772,6 +791,7 @@ impl Service { }); let start = Instant::now(); + let panic_dest = dest.clone(); let send = match dest { | Destination::Federation(server) => self.send_events_dest_federation(server, events).boxed(), @@ -780,24 +800,42 @@ impl Service { self.send_events_dest_push(user_id, pushkey, events).boxed(), }; - send.inspect(move |result| { - let instruments = &self.server.metrics.instruments; - let outcome = - KeyValue::new("outcome", if result.is_ok() { "success" } else { "failure" }); - instruments - .sender_transaction_duration - .record(start.elapsed().as_secs_f64(), &[destination.clone(), outcome.clone()]); - for (kind, count) in [("pdu", pdus), ("edu", edus)] { - if count > 0 { - instruments.sender_events.add(count, &[ + AssertUnwindSafe(send) + .catch_unwind() + .map(move |result| { + result.unwrap_or_else(|panic| { + let error = Error::from_panic(panic); + error!(dest = ?panic_dest, ?error, "Panicked while sending transaction"); + self.server + .metrics + .instruments + .sender_panics + .add(1, &[KeyValue::new("scope", "transaction")]); + + Err((panic_dest, Box::new(error))) + }) + }) + .inspect(move |result| { + let instruments = &self.server.metrics.instruments; + let outcome = + KeyValue::new("outcome", if result.is_ok() { "success" } else { "failure" }); + instruments + .sender_transaction_duration + .record(start.elapsed().as_secs_f64(), &[ destination.clone(), - KeyValue::new("kind", kind), outcome.clone(), ]); + for (kind, count) in [("pdu", pdus), ("edu", edus)] { + if count > 0 { + instruments.sender_events.add(count, &[ + destination.clone(), + KeyValue::new("kind", kind), + outcome.clone(), + ]); + } } - } - }) - .boxed() + }) + .boxed() } #[tracing::instrument( diff --git a/src/service/sending/sender/tests.rs b/src/service/sending/sender/tests.rs new file mode 100644 index 000000000..e5104158d --- /dev/null +++ b/src/service/sending/sender/tests.rs @@ -0,0 +1,86 @@ +use std::iter::once; + +use conduwuit::Error; +use futures::{StreamExt, stream::FuturesUnordered}; +use ruma::server_name; +use serde_json::json; + +use super::{CurTransactionStatus, TransactionStatus}; +use crate::{ + sending::{Destination, EduBuf, SendingEvent}, + test_utils::{test_services, test_services_with}, +}; + +#[tokio::test] +async fn panicked_transaction_is_kept_for_retry_without_blaming_the_remote() { + let (services, path, _database) = test_services().await; + let sending = &services.sending; + let server = server_name!("remote.example"); + let dest = Destination::Federation(server.to_owned()); + let event = SendingEvent::Edu(EduBuf::from_slice(br#"{"edu_type":"m.typing"}"#)); + + sending.db.queue_requests(once((&event, &dest))); + let in_flight: Vec<_> = sending.db.queued_requests(&dest).collect().await; + sending.db.mark_as_active(in_flight.iter()); + + let mut futures = FuturesUnordered::new(); + let mut statuses = CurTransactionStatus::from([(dest.clone(), TransactionStatus::Running)]); + let panic = Error::from_panic(Box::new("boom")); + sending + .handle_response(Err((dest.clone(), Box::new(panic))), &mut futures, &mut statuses) + .await; + + let active: Vec<_> = sending + .db + .active_requests_for(&dest) + .map(|(key, _)| key) + .collect() + .await; + let retry_after = sending.services.federation.retry_after(server); + let sending_next = futures.len(); + let failed = matches!(statuses.get(&dest), Some(TransactionStatus::Failed(1, _))); + + drop(futures); + drop(services); + std::fs::remove_dir_all(path).unwrap(); + assert_eq!( + active, + in_flight + .into_iter() + .map(|(key, _)| key) + .collect::>() + ); + assert_eq!(sending_next, 0); + assert!(failed); + assert_eq!(retry_after, None); +} + +#[tokio::test] +async fn restarted_worker_resends_trimmed_in_flight_requests() { + let (services, path, _database) = + test_services_with(json!({ "startup_netburst_keep": 1 })).await; + let sending = &services.sending; + let dest = Destination::Federation(server_name!("remote.example").to_owned()); + let event = SendingEvent::Edu(EduBuf::from_slice(br#"{"edu_type":"m.typing"}"#)); + + for _ in 0..3 { + sending.db.queue_requests(once((&event, &dest))); + } + let queued: Vec<_> = sending.db.queued_requests(&dest).collect().await; + sending.db.mark_as_active(queued.iter()); + + let id = sending.shard_id(&dest); + let mut futures = FuturesUnordered::new(); + let mut statuses = CurTransactionStatus::new(); + sending + .startup_netburst(id, true, &mut futures, &mut statuses) + .await; + let active = sending.db.active_requests_for(&dest).count().await; + let sending_now = futures.len(); + + drop(futures); + drop(services); + std::fs::remove_dir_all(path).unwrap(); + assert_eq!(active, 1); + assert_eq!(sending_now, 1); +} diff --git a/src/service/sending/tests.rs b/src/service/sending/tests.rs new file mode 100644 index 000000000..40c1cb957 --- /dev/null +++ b/src/service/sending/tests.rs @@ -0,0 +1,42 @@ +use std::{iter::once, time::Duration}; + +use futures::StreamExt; +use ruma::server_name; + +use super::{Destination, EduBuf, SendingEvent, restart_delay}; +use crate::test_utils::test_services; + +#[tokio::test] +async fn queued_heads_lists_the_oldest_unsent_request_per_destination() { + let (services, path, _database) = test_services().await; + let db = &services.sending.db; + let queued = Destination::Federation(server_name!("queued.example").to_owned()); + let active = Destination::Federation(server_name!("active.example").to_owned()); + let event = SendingEvent::Edu(EduBuf::from_slice(br#"{"edu_type":"m.typing"}"#)); + + let first = db.queue_requests(once((&event, &queued))); + db.queue_requests(once((&event, &queued))); + db.queue_requests(once((&event, &active))); + let in_flight: Vec<_> = db.queued_requests(&active).collect().await; + db.mark_as_active(in_flight.iter()); + + let heads: Vec<_> = db + .queued_heads() + .await + .into_iter() + .map(|(dest, (key, _))| (dest, key)) + .collect(); + + drop(services); + std::fs::remove_dir_all(path).unwrap(); + assert_eq!(heads, [(queued, first[0].clone())]); +} + +#[test] +fn restart_delay_doubles_up_to_the_cap() { + assert_eq!(restart_delay(1), Duration::from_millis(2500)); + assert_eq!(restart_delay(2), Duration::from_secs(5)); + assert_eq!(restart_delay(4), Duration::from_secs(20)); + assert_eq!(restart_delay(5), Duration::from_secs(30)); + assert_eq!(restart_delay(u32::MAX), Duration::from_secs(30)); +} diff --git a/src/service/test_utils.rs b/src/service/test_utils.rs new file mode 100644 index 000000000..e71fd2959 --- /dev/null +++ b/src/service/test_utils.rs @@ -0,0 +1,52 @@ +use std::{path::PathBuf, sync::Arc}; + +use conduwuit::{ + Config, Server, + log::{Log, LogLevelReloadHandles}, + metrics::Metrics, +}; +use serde_json::json; +use tokio::sync::{Mutex, MutexGuard}; + +/// RocksDB's default `Env` is shared by the whole process and closing a +/// database stops its background threads, so tests must not keep databases +/// open in parallel. Hold the returned guard until the services are dropped. +static DATABASE: Mutex<()> = Mutex::const_new(()); + +type TestServices = (Arc, PathBuf, MutexGuard<'static, ()>); + +pub(crate) async fn test_services() -> TestServices { test_services_with(json!({})).await } + +pub(crate) async fn test_services_with(overrides: serde_json::Value) -> TestServices { + let guard = DATABASE.lock().await; + let _ = rustls::crypto::ring::default_provider().install_default(); + let path = std::env::temp_dir() + .join(format!("continuwuity-shared-state-{:016x}", rand::random::())); + std::fs::create_dir(&path).unwrap(); + let mut config = json!({ + "server_name": "test.example", + "database_path": path, + "db_cache_capacity_mb": 16.0, + "rocksdb_write_buffer_capacity_mb": 4.0, + "allow_local_presence": false, + "dns_servers": ["127.0.0.1"], + "allow_invalid_tls_certificates_yes_i_know_what_the_fuck_i_am_doing_with_this_and_i_know_this_is_insecure": true + }); + config + .as_object_mut() + .unwrap() + .extend(overrides.as_object().unwrap().clone()); + let config: Config = serde_json::from_value(config).unwrap(); + let runtime = tokio::runtime::Handle::current(); + let server = Arc::new(Server::new( + config, + Some(runtime.clone()), + Log { + reload: LogLevelReloadHandles::default(), + capture: Arc::default(), + }, + Metrics::new(Some(runtime), opentelemetry::global::meter("test"), None), + )); + let services = crate::Services::build(server).await.unwrap(); + (services, path, guard) +}