mirror of
https://forgejo.ellis.link/continuwuation/continuwuity.git
synced 2026-11-22 16:37:32 +00:00
feat(service/sending): Sender Supervision
This commit is contained in:
1 parent
42a66fecb6
commit
3ab25fff0a
12 files changed
+408
-79
No files matched your search
@@ -0,0 +1 @@
|
||||
Fix outgoing federation stalling after a sender worker panics. Contributed by @eleboucher.
|
||||
@@ -71,6 +71,9 @@ Names below are as they appear in Prometheus. OTLP uses the dotted OpenTelemetry
|
||||
| `continuwuity_sender_events_total` | counter | `destination`, `kind`, `outcome` | PDUs and EDUs sent in outgoing transactions |
|
||||
| `continuwuity_sender_events_queued_total` | counter | `destination`, `kind` | PDUs and EDUs queued for sending |
|
||||
| `continuwuity_sender_queue_depth` | gauge | | Messages waiting for the sender workers |
|
||||
| `continuwuity_sender_requests` | gauge | `state` | Estimated outgoing requests stored as `queued` or `active` (in flight) |
|
||||
| `continuwuity_sender_workers` | gauge | | Running sender workers; below the configured count means a worker is restarting |
|
||||
| `continuwuity_sender_panics_total` | counter | `scope` | Panics caught in a single outgoing `transaction` or a whole sender `worker` |
|
||||
| `continuwuity_users` | gauge | `origin` | Known `local` users, including deactivated, and `remote` users |
|
||||
| `continuwuity_invites_total` | counter | `direction`, `destination`, `outcome` | Invites sent by local users or received over federation |
|
||||
| `continuwuity_media_write_size_bytes` | histogram | `origin` | Media files written, from local uploads or cached remote media |
|
||||
|
||||
@@ -21,6 +21,7 @@ pub struct Instruments {
|
||||
pub sender_transaction_duration: Histogram<f64>,
|
||||
pub sender_events: Counter<u64>,
|
||||
pub sender_events_queued: Counter<u64>,
|
||||
pub sender_panics: Counter<u64>,
|
||||
|
||||
pub media_write_size: Histogram<u64>,
|
||||
pub media_read_size: Histogram<u64>,
|
||||
@@ -106,6 +107,11 @@ impl Instruments {
|
||||
.with_description("PDUs and EDUs queued for sending")
|
||||
.with_unit("{event}")
|
||||
.build(),
|
||||
sender_panics: meter
|
||||
.u64_counter("continuwuity.sender.panics")
|
||||
.with_description("Panics caught in outgoing transactions and sender workers")
|
||||
.with_unit("{panic}")
|
||||
.build(),
|
||||
|
||||
media_write_size: size(
|
||||
"continuwuity.media.write.size",
|
||||
|
||||
@@ -18,6 +18,28 @@ impl Services {
|
||||
},
|
||||
);
|
||||
|
||||
self.gauge(
|
||||
"continuwuity.sender.requests",
|
||||
"Estimated outgoing requests stored as queued or in flight",
|
||||
"{request}",
|
||||
|services, observer| {
|
||||
for (state, count) in services.sending.db.request_counts() {
|
||||
if let Some(count) = count {
|
||||
observer.observe(count, &[KeyValue::new("state", state)]);
|
||||
}
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
self.gauge(
|
||||
"continuwuity.sender.workers",
|
||||
"Running sender workers",
|
||||
"{worker}",
|
||||
|services, observer| {
|
||||
observe(observer, services.sending.workers_alive(), &[]);
|
||||
},
|
||||
);
|
||||
|
||||
self.gauge(
|
||||
"continuwuity.federation.incoming.transactions.active",
|
||||
"Incoming federation transactions currently being processed",
|
||||
|
||||
@@ -13,6 +13,8 @@ mod migrations;
|
||||
mod service;
|
||||
pub mod services;
|
||||
pub mod state;
|
||||
#[cfg(test)]
|
||||
mod test_utils;
|
||||
|
||||
pub mod account_data;
|
||||
pub mod admin;
|
||||
|
||||
@@ -1,14 +1,10 @@
|
||||
use std::sync::Arc;
|
||||
|
||||
use conduwuit::{
|
||||
Config, PduEvent, Server,
|
||||
log::{Log, LogLevelReloadHandles},
|
||||
metrics::Metrics,
|
||||
};
|
||||
use conduwuit::PduEvent;
|
||||
use futures::StreamExt;
|
||||
use ruma::{RoomId, RoomVersionId, events::StateEventType};
|
||||
use serde_json::json;
|
||||
|
||||
use crate::test_utils::test_services;
|
||||
|
||||
#[tokio::test]
|
||||
async fn admitted_joins_do_not_disappear_after_a_merge() {
|
||||
use conduwuit::matrix::PartialPdu;
|
||||
@@ -20,7 +16,7 @@ async fn admitted_joins_do_not_disappear_after_a_merge() {
|
||||
},
|
||||
};
|
||||
|
||||
let (services, path) = test_services().await;
|
||||
let (services, path, _database) = test_services().await;
|
||||
let alice = UserId::parse("@alice:test.example").unwrap();
|
||||
let bob = UserId::parse("@bob:bob.example").unwrap();
|
||||
let charlie = UserId::parse("@charlie:charlie.example").unwrap();
|
||||
@@ -212,35 +208,6 @@ async fn admitted_joins_do_not_disappear_after_a_merge() {
|
||||
assert_eq!(results, [true, true], "both joined users must be able to send messages");
|
||||
}
|
||||
|
||||
async fn test_services() -> (Arc<crate::Services>, std::path::PathBuf) {
|
||||
let _ = rustls::crypto::ring::default_provider().install_default();
|
||||
let path = std::env::temp_dir()
|
||||
.join(format!("continuwuity-shared-state-{:016x}", rand::random::<u64>()));
|
||||
std::fs::create_dir(&path).unwrap();
|
||||
let config: Config = serde_json::from_value(json!({
|
||||
"server_name": "test.example",
|
||||
"database_path": path,
|
||||
"db_cache_capacity_mb": 16.0,
|
||||
"rocksdb_write_buffer_capacity_mb": 4.0,
|
||||
"allow_local_presence": false,
|
||||
"dns_servers": ["127.0.0.1"],
|
||||
"allow_invalid_tls_certificates_yes_i_know_what_the_fuck_i_am_doing_with_this_and_i_know_this_is_insecure": true
|
||||
}))
|
||||
.unwrap();
|
||||
let runtime = tokio::runtime::Handle::current();
|
||||
let server = Arc::new(Server::new(
|
||||
config,
|
||||
Some(runtime.clone()),
|
||||
Log {
|
||||
reload: LogLevelReloadHandles::default(),
|
||||
capture: Arc::default(),
|
||||
},
|
||||
Metrics::new(Some(runtime), opentelemetry::global::meter("test"), None),
|
||||
));
|
||||
let services = crate::Services::build(server).await.unwrap();
|
||||
(services, path)
|
||||
}
|
||||
|
||||
fn cache_key(
|
||||
services: &crate::Services,
|
||||
server: &ruma::ServerName,
|
||||
|
||||
@@ -3,6 +3,7 @@ use std::{fmt::Debug, sync::Arc};
|
||||
use conduwuit::{
|
||||
Error, Result, at, utils,
|
||||
utils::{ReadyExt, stream::TryIgnore},
|
||||
warn,
|
||||
};
|
||||
use database::{Database, Deserialized, Map};
|
||||
use futures::{Stream, StreamExt};
|
||||
@@ -169,6 +170,33 @@ impl Data {
|
||||
})
|
||||
}
|
||||
|
||||
/// The oldest queued request of each destination, not yet in flight.
|
||||
pub(super) async fn queued_heads(&self) -> Vec<(Destination, QueueItem)> {
|
||||
let mut heads: Vec<(Destination, QueueItem)> = Vec::new();
|
||||
self.servernameevent_data
|
||||
.raw_stream()
|
||||
.ignore_err()
|
||||
.ready_for_each(|(key, val)| match parse_servercurrentevent(key, val) {
|
||||
| Ok((dest, event)) =>
|
||||
if heads.last().is_none_or(|(last, _)| *last != dest) {
|
||||
heads.push((dest, (key.to_vec(), event)));
|
||||
},
|
||||
| Err(e) => warn!(?key, "invalid queued request key: {e}"),
|
||||
})
|
||||
.await;
|
||||
|
||||
heads
|
||||
}
|
||||
|
||||
/// Estimated number of requests waiting to be sent and in flight.
|
||||
pub(crate) fn request_counts(&self) -> [(&'static str, Option<u64>); 2] {
|
||||
[
|
||||
("queued", &self.servernameevent_data),
|
||||
("active", &self.servercurrentevent_data),
|
||||
]
|
||||
.map(|(state, map)| (state, map.property_integer(c"rocksdb.estimate-num-keys").ok()))
|
||||
}
|
||||
|
||||
pub(super) fn set_latest_educount(&self, server_name: &ServerName, last_count: u64) {
|
||||
self.servername_educount.raw_put(server_name, last_count);
|
||||
}
|
||||
|
||||
+106
-24
@@ -3,12 +3,19 @@ mod appservice;
|
||||
mod data;
|
||||
mod dest;
|
||||
mod sender;
|
||||
#[cfg(test)]
|
||||
mod tests;
|
||||
|
||||
use std::{
|
||||
collections::HashMap,
|
||||
fmt::Debug,
|
||||
hash::{DefaultHasher, Hash, Hasher},
|
||||
iter::once,
|
||||
sync::Arc,
|
||||
sync::{
|
||||
Arc,
|
||||
atomic::{AtomicUsize, Ordering},
|
||||
},
|
||||
time::{Duration, Instant},
|
||||
};
|
||||
|
||||
use async_trait::async_trait;
|
||||
@@ -19,7 +26,7 @@ use conduwuit::{
|
||||
utils::{ReadyExt, TryReadyExt, available_parallelism, math::usize_from_u64_truncated},
|
||||
warn,
|
||||
};
|
||||
use futures::{FutureExt, Stream, StreamExt};
|
||||
use futures::{FutureExt, Stream, StreamExt, stream::FuturesUnordered};
|
||||
use ruma::{
|
||||
OwnedServerName, RoomId, ServerName, UserId,
|
||||
api::{
|
||||
@@ -27,7 +34,7 @@ use ruma::{
|
||||
federation::authentication::ServerSignatures, path_builder::PathBuilder,
|
||||
},
|
||||
};
|
||||
use tokio::{task, task::JoinSet};
|
||||
use tokio::{task, task::JoinSet, time::sleep};
|
||||
|
||||
use self::data::Data;
|
||||
pub use self::{
|
||||
@@ -49,6 +56,7 @@ pub struct Service {
|
||||
server: Arc<Server>,
|
||||
services: Services,
|
||||
channels: Vec<(loole::Sender<Msg>, loole::Receiver<Msg>)>,
|
||||
workers_alive: AtomicUsize,
|
||||
}
|
||||
|
||||
struct Services {
|
||||
@@ -87,6 +95,8 @@ pub type EduVec = SmallVec<[EduBuf; EDU_VEC_CAP]>;
|
||||
|
||||
const EDU_BUF_CAP: usize = 128;
|
||||
const EDU_VEC_CAP: usize = 1;
|
||||
const WORKER_RESTART_DELAY: Duration = Duration::from_millis(2500);
|
||||
const WORKER_RESTART_DELAY_MAX: Duration = Duration::from_secs(30);
|
||||
|
||||
#[async_trait]
|
||||
impl crate::Service for Service {
|
||||
@@ -111,36 +121,67 @@ impl crate::Service for Service {
|
||||
federation: args.depend::<federation::Service>("federation"),
|
||||
},
|
||||
channels: (0..num_senders).map(|_| loole::unbounded()).collect(),
|
||||
workers_alive: AtomicUsize::new(0),
|
||||
}))
|
||||
}
|
||||
|
||||
async fn worker(self: Arc<Self>) -> Result {
|
||||
let mut senders =
|
||||
self.channels
|
||||
.iter()
|
||||
.enumerate()
|
||||
.fold(JoinSet::new(), |mut joinset, (id, _)| {
|
||||
let self_ = self.clone();
|
||||
let worker = self_.sender(id);
|
||||
let worker = if self.unconstrained() {
|
||||
task::unconstrained(worker).boxed()
|
||||
} else {
|
||||
worker.boxed()
|
||||
if self.server.config.startup_netburst {
|
||||
self.dispatch_queued(None).await;
|
||||
}
|
||||
|
||||
let num_senders = self.channels.len();
|
||||
let mut senders = JoinSet::new();
|
||||
let mut shards = HashMap::new();
|
||||
let mut started = vec![Instant::now(); num_senders];
|
||||
let mut failures = vec![0_u32; num_senders];
|
||||
let mut restarts = FuturesUnordered::new();
|
||||
for id in 0..num_senders {
|
||||
self.spawn_sender(&mut senders, &mut shards, id, false);
|
||||
}
|
||||
|
||||
loop {
|
||||
tokio::select! {
|
||||
Some(ret) = senders.join_next_with_id() => {
|
||||
self.workers_alive.fetch_sub(1, Ordering::Relaxed);
|
||||
let (task_id, panicked) = match &ret {
|
||||
| Ok((task_id, _)) => (*task_id, false),
|
||||
| Err(error) => (error.id(), error.is_panic()),
|
||||
};
|
||||
let shard = shards.remove(&task_id).expect("sender worker has a shard");
|
||||
|
||||
let runtime = self.server.runtime();
|
||||
let _abort = joinset.spawn_on(worker, runtime);
|
||||
joinset
|
||||
});
|
||||
if panicked {
|
||||
self.server
|
||||
.metrics
|
||||
.instruments
|
||||
.sender_panics
|
||||
.add(1, &[KeyValue::new("scope", "worker")]);
|
||||
}
|
||||
|
||||
while let Some(ret) = senders.join_next_with_id().await {
|
||||
match ret {
|
||||
| Ok((id, error)) => {
|
||||
debug!(?id, ?error, "sender worker finished");
|
||||
if self.channels[shard].1.is_closed() || !self.server.running() {
|
||||
debug!(shard, ?ret, "sender worker finished");
|
||||
continue;
|
||||
}
|
||||
|
||||
// A worker that ran a while before failing starts over from
|
||||
// the shortest delay.
|
||||
failures[shard] = if started[shard].elapsed() > WORKER_RESTART_DELAY_MAX {
|
||||
1
|
||||
} else {
|
||||
failures[shard].saturating_add(1)
|
||||
};
|
||||
let delay = restart_delay(failures[shard]);
|
||||
error!(shard, ?ret, ?delay, "sender worker unexpectedly finished, restarting");
|
||||
restarts.push(sleep(delay).map(move |()| shard));
|
||||
},
|
||||
| Err(error) => {
|
||||
error!(id = ?error.id(), ?error, "sender worker failed");
|
||||
Some(shard) = restarts.next() => {
|
||||
if !self.channels[shard].1.is_closed() {
|
||||
started[shard] = Instant::now();
|
||||
self.spawn_sender(&mut senders, &mut shards, shard, true);
|
||||
}
|
||||
},
|
||||
() = self.server.until_shutdown(), if !restarts.is_empty() => restarts.clear(),
|
||||
else => break,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -161,6 +202,25 @@ impl crate::Service for Service {
|
||||
}
|
||||
|
||||
impl Service {
|
||||
fn spawn_sender(
|
||||
self: &Arc<Self>,
|
||||
senders: &mut JoinSet<Result>,
|
||||
shards: &mut HashMap<task::Id, usize>,
|
||||
id: usize,
|
||||
restarted: bool,
|
||||
) {
|
||||
let worker = self.clone().sender(id, restarted);
|
||||
let worker = if crate::Service::unconstrained(&**self) {
|
||||
task::unconstrained(worker).boxed()
|
||||
} else {
|
||||
worker.boxed()
|
||||
};
|
||||
|
||||
let handle = senders.spawn_on(worker, self.server.runtime());
|
||||
shards.insert(handle.id(), id);
|
||||
self.workers_alive.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
#[tracing::instrument(skip(self, pdu_id, user, pushkey), level = "debug")]
|
||||
pub fn send_pdu_push(&self, pdu_id: &RawPduId, user: &UserId, pushkey: String) -> Result {
|
||||
let dest = Destination::Push(user.to_owned(), pushkey);
|
||||
@@ -480,10 +540,24 @@ impl Service {
|
||||
sender.send(msg).map_err(|e| err!("{e}"))
|
||||
}
|
||||
|
||||
/// Hands the oldest unsent request of each destination to its worker, so
|
||||
/// a backlog with nothing in flight is sent without waiting for a new
|
||||
/// event. Bypasses `dispatch` as these requests were already counted.
|
||||
async fn dispatch_queued(&self, shard: Option<usize>) {
|
||||
for (dest, (queue_id, event)) in self.db.queued_heads().await {
|
||||
let id = self.shard_id(&dest);
|
||||
if shard.is_none_or(|shard| shard == id) {
|
||||
self.channels[id].0.send(Msg { dest, event, queue_id }).ok();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn queue_depth(&self) -> usize {
|
||||
self.channels.iter().map(|(sender, _)| sender.len()).sum()
|
||||
}
|
||||
|
||||
pub fn workers_alive(&self) -> usize { self.workers_alive.load(Ordering::Relaxed) }
|
||||
|
||||
pub(super) fn shard_id(&self, dest: &Destination) -> usize {
|
||||
if self.channels.len() <= 1 {
|
||||
return 0;
|
||||
@@ -519,3 +593,11 @@ fn num_senders(args: &crate::Args<'_>) -> usize {
|
||||
worker_count.clamp(MIN_SENDERS, max_senders)
|
||||
}
|
||||
}
|
||||
|
||||
/// Delay before restarting a worker after its `failures`-th consecutive
|
||||
/// failure, doubling each time.
|
||||
fn restart_delay(failures: u32) -> Duration {
|
||||
WORKER_RESTART_DELAY
|
||||
.saturating_mul(2_u32.saturating_pow(failures.saturating_sub(1)))
|
||||
.min(WORKER_RESTART_DELAY_MAX)
|
||||
}
|
||||
@@ -1,6 +1,7 @@
|
||||
use std::{
|
||||
collections::{BTreeMap, HashMap, HashSet},
|
||||
fmt::Debug,
|
||||
panic::AssertUnwindSafe,
|
||||
sync::{
|
||||
Arc,
|
||||
atomic::{AtomicU64, AtomicUsize, Ordering},
|
||||
@@ -55,6 +56,9 @@ use serde_json::value::{RawValue as RawJsonValue, to_raw_value};
|
||||
|
||||
use super::{Destination, EduBuf, EduVec, Msg, SendingEvent, Service, data::QueueItem};
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests;
|
||||
|
||||
#[derive(Debug)]
|
||||
enum TransactionStatus {
|
||||
Running,
|
||||
@@ -78,12 +82,18 @@ pub const EDU_LIMIT: usize = 100;
|
||||
|
||||
impl Service {
|
||||
#[tracing::instrument(skip(self), level = "debug")]
|
||||
pub(super) async fn sender(self: Arc<Self>, id: usize) -> Result {
|
||||
pub(super) async fn sender(self: Arc<Self>, id: usize, restarted: bool) -> Result {
|
||||
let mut statuses: CurTransactionStatus = CurTransactionStatus::new();
|
||||
let mut futures: SendingFutures<'_> = FuturesUnordered::new();
|
||||
|
||||
// The previous worker of this shard may have dropped requests it had
|
||||
// taken from the channel.
|
||||
if restarted {
|
||||
self.dispatch_queued(Some(id)).await;
|
||||
}
|
||||
|
||||
info!(shard = id, "Running startup netburst");
|
||||
self.startup_netburst(id, &mut futures, &mut statuses)
|
||||
self.startup_netburst(id, restarted, &mut futures, &mut statuses)
|
||||
.boxed()
|
||||
.await;
|
||||
info!(shard = id, "Finished startup netburst");
|
||||
@@ -142,6 +152,9 @@ impl Service {
|
||||
) {
|
||||
match response {
|
||||
| Ok(dest) => self.handle_response_ok(&dest, futures, statuses).await,
|
||||
// A bug on our side, not the remote's fault: retry with backoff
|
||||
// without marking the remote unhealthy.
|
||||
| Err((dest, e)) if e.is_panic() => Self::handle_response_err(dest, statuses, &e),
|
||||
| Err((dest, e)) => {
|
||||
if let Destination::Federation(dest) = &dest {
|
||||
if self.services.federation.should_mark_stale(&e)
|
||||
@@ -274,7 +287,8 @@ impl Service {
|
||||
}
|
||||
|
||||
/// Performs startup netburst, which sends any previously queued up but not
|
||||
/// yet sent requests immediately.
|
||||
/// yet sent requests immediately. A restarted worker always resends its
|
||||
/// shard's requests in flight.
|
||||
#[tracing::instrument(
|
||||
name = "netburst",
|
||||
level = "debug",
|
||||
@@ -285,6 +299,7 @@ impl Service {
|
||||
async fn startup_netburst<'a>(
|
||||
&'a self,
|
||||
id: usize,
|
||||
restarted: bool,
|
||||
futures: &mut SendingFutures<'a>,
|
||||
statuses: &mut CurTransactionStatus,
|
||||
) {
|
||||
@@ -315,8 +330,12 @@ impl Service {
|
||||
}
|
||||
}
|
||||
|
||||
if !restarted && !self.server.config.startup_netburst {
|
||||
return;
|
||||
}
|
||||
|
||||
for (dest, events) in txns {
|
||||
if self.server.config.startup_netburst && !events.is_empty() {
|
||||
if !events.is_empty() {
|
||||
statuses.insert(dest.clone(), TransactionStatus::Running);
|
||||
futures.push(self.send_events(dest, events));
|
||||
}
|
||||
@@ -772,6 +791,7 @@ impl Service {
|
||||
});
|
||||
|
||||
let start = Instant::now();
|
||||
let panic_dest = dest.clone();
|
||||
let send = match dest {
|
||||
| Destination::Federation(server) =>
|
||||
self.send_events_dest_federation(server, events).boxed(),
|
||||
@@ -780,24 +800,42 @@ impl Service {
|
||||
self.send_events_dest_push(user_id, pushkey, events).boxed(),
|
||||
};
|
||||
|
||||
send.inspect(move |result| {
|
||||
let instruments = &self.server.metrics.instruments;
|
||||
let outcome =
|
||||
KeyValue::new("outcome", if result.is_ok() { "success" } else { "failure" });
|
||||
instruments
|
||||
.sender_transaction_duration
|
||||
.record(start.elapsed().as_secs_f64(), &[destination.clone(), outcome.clone()]);
|
||||
for (kind, count) in [("pdu", pdus), ("edu", edus)] {
|
||||
if count > 0 {
|
||||
instruments.sender_events.add(count, &[
|
||||
AssertUnwindSafe(send)
|
||||
.catch_unwind()
|
||||
.map(move |result| {
|
||||
result.unwrap_or_else(|panic| {
|
||||
let error = Error::from_panic(panic);
|
||||
error!(dest = ?panic_dest, ?error, "Panicked while sending transaction");
|
||||
self.server
|
||||
.metrics
|
||||
.instruments
|
||||
.sender_panics
|
||||
.add(1, &[KeyValue::new("scope", "transaction")]);
|
||||
|
||||
Err((panic_dest, Box::new(error)))
|
||||
})
|
||||
})
|
||||
.inspect(move |result| {
|
||||
let instruments = &self.server.metrics.instruments;
|
||||
let outcome =
|
||||
KeyValue::new("outcome", if result.is_ok() { "success" } else { "failure" });
|
||||
instruments
|
||||
.sender_transaction_duration
|
||||
.record(start.elapsed().as_secs_f64(), &[
|
||||
destination.clone(),
|
||||
KeyValue::new("kind", kind),
|
||||
outcome.clone(),
|
||||
]);
|
||||
for (kind, count) in [("pdu", pdus), ("edu", edus)] {
|
||||
if count > 0 {
|
||||
instruments.sender_events.add(count, &[
|
||||
destination.clone(),
|
||||
KeyValue::new("kind", kind),
|
||||
outcome.clone(),
|
||||
]);
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
.boxed()
|
||||
})
|
||||
.boxed()
|
||||
}
|
||||
|
||||
#[tracing::instrument(
|
||||
|
||||
@@ -0,0 +1,86 @@
|
||||
use std::iter::once;
|
||||
|
||||
use conduwuit::Error;
|
||||
use futures::{StreamExt, stream::FuturesUnordered};
|
||||
use ruma::server_name;
|
||||
use serde_json::json;
|
||||
|
||||
use super::{CurTransactionStatus, TransactionStatus};
|
||||
use crate::{
|
||||
sending::{Destination, EduBuf, SendingEvent},
|
||||
test_utils::{test_services, test_services_with},
|
||||
};
|
||||
|
||||
#[tokio::test]
|
||||
async fn panicked_transaction_is_kept_for_retry_without_blaming_the_remote() {
|
||||
let (services, path, _database) = test_services().await;
|
||||
let sending = &services.sending;
|
||||
let server = server_name!("remote.example");
|
||||
let dest = Destination::Federation(server.to_owned());
|
||||
let event = SendingEvent::Edu(EduBuf::from_slice(br#"{"edu_type":"m.typing"}"#));
|
||||
|
||||
sending.db.queue_requests(once((&event, &dest)));
|
||||
let in_flight: Vec<_> = sending.db.queued_requests(&dest).collect().await;
|
||||
sending.db.mark_as_active(in_flight.iter());
|
||||
|
||||
let mut futures = FuturesUnordered::new();
|
||||
let mut statuses = CurTransactionStatus::from([(dest.clone(), TransactionStatus::Running)]);
|
||||
let panic = Error::from_panic(Box::new("boom"));
|
||||
sending
|
||||
.handle_response(Err((dest.clone(), Box::new(panic))), &mut futures, &mut statuses)
|
||||
.await;
|
||||
|
||||
let active: Vec<_> = sending
|
||||
.db
|
||||
.active_requests_for(&dest)
|
||||
.map(|(key, _)| key)
|
||||
.collect()
|
||||
.await;
|
||||
let retry_after = sending.services.federation.retry_after(server);
|
||||
let sending_next = futures.len();
|
||||
let failed = matches!(statuses.get(&dest), Some(TransactionStatus::Failed(1, _)));
|
||||
|
||||
drop(futures);
|
||||
drop(services);
|
||||
std::fs::remove_dir_all(path).unwrap();
|
||||
assert_eq!(
|
||||
active,
|
||||
in_flight
|
||||
.into_iter()
|
||||
.map(|(key, _)| key)
|
||||
.collect::<Vec<_>>()
|
||||
);
|
||||
assert_eq!(sending_next, 0);
|
||||
assert!(failed);
|
||||
assert_eq!(retry_after, None);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn restarted_worker_resends_trimmed_in_flight_requests() {
|
||||
let (services, path, _database) =
|
||||
test_services_with(json!({ "startup_netburst_keep": 1 })).await;
|
||||
let sending = &services.sending;
|
||||
let dest = Destination::Federation(server_name!("remote.example").to_owned());
|
||||
let event = SendingEvent::Edu(EduBuf::from_slice(br#"{"edu_type":"m.typing"}"#));
|
||||
|
||||
for _ in 0..3 {
|
||||
sending.db.queue_requests(once((&event, &dest)));
|
||||
}
|
||||
let queued: Vec<_> = sending.db.queued_requests(&dest).collect().await;
|
||||
sending.db.mark_as_active(queued.iter());
|
||||
|
||||
let id = sending.shard_id(&dest);
|
||||
let mut futures = FuturesUnordered::new();
|
||||
let mut statuses = CurTransactionStatus::new();
|
||||
sending
|
||||
.startup_netburst(id, true, &mut futures, &mut statuses)
|
||||
.await;
|
||||
let active = sending.db.active_requests_for(&dest).count().await;
|
||||
let sending_now = futures.len();
|
||||
|
||||
drop(futures);
|
||||
drop(services);
|
||||
std::fs::remove_dir_all(path).unwrap();
|
||||
assert_eq!(active, 1);
|
||||
assert_eq!(sending_now, 1);
|
||||
}
|
||||
@@ -0,0 +1,42 @@
|
||||
use std::{iter::once, time::Duration};
|
||||
|
||||
use futures::StreamExt;
|
||||
use ruma::server_name;
|
||||
|
||||
use super::{Destination, EduBuf, SendingEvent, restart_delay};
|
||||
use crate::test_utils::test_services;
|
||||
|
||||
#[tokio::test]
|
||||
async fn queued_heads_lists_the_oldest_unsent_request_per_destination() {
|
||||
let (services, path, _database) = test_services().await;
|
||||
let db = &services.sending.db;
|
||||
let queued = Destination::Federation(server_name!("queued.example").to_owned());
|
||||
let active = Destination::Federation(server_name!("active.example").to_owned());
|
||||
let event = SendingEvent::Edu(EduBuf::from_slice(br#"{"edu_type":"m.typing"}"#));
|
||||
|
||||
let first = db.queue_requests(once((&event, &queued)));
|
||||
db.queue_requests(once((&event, &queued)));
|
||||
db.queue_requests(once((&event, &active)));
|
||||
let in_flight: Vec<_> = db.queued_requests(&active).collect().await;
|
||||
db.mark_as_active(in_flight.iter());
|
||||
|
||||
let heads: Vec<_> = db
|
||||
.queued_heads()
|
||||
.await
|
||||
.into_iter()
|
||||
.map(|(dest, (key, _))| (dest, key))
|
||||
.collect();
|
||||
|
||||
drop(services);
|
||||
std::fs::remove_dir_all(path).unwrap();
|
||||
assert_eq!(heads, [(queued, first[0].clone())]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn restart_delay_doubles_up_to_the_cap() {
|
||||
assert_eq!(restart_delay(1), Duration::from_millis(2500));
|
||||
assert_eq!(restart_delay(2), Duration::from_secs(5));
|
||||
assert_eq!(restart_delay(4), Duration::from_secs(20));
|
||||
assert_eq!(restart_delay(5), Duration::from_secs(30));
|
||||
assert_eq!(restart_delay(u32::MAX), Duration::from_secs(30));
|
||||
}
|
||||
@@ -0,0 +1,52 @@
|
||||
use std::{path::PathBuf, sync::Arc};
|
||||
|
||||
use conduwuit::{
|
||||
Config, Server,
|
||||
log::{Log, LogLevelReloadHandles},
|
||||
metrics::Metrics,
|
||||
};
|
||||
use serde_json::json;
|
||||
use tokio::sync::{Mutex, MutexGuard};
|
||||
|
||||
/// RocksDB's default `Env` is shared by the whole process and closing a
|
||||
/// database stops its background threads, so tests must not keep databases
|
||||
/// open in parallel. Hold the returned guard until the services are dropped.
|
||||
static DATABASE: Mutex<()> = Mutex::const_new(());
|
||||
|
||||
type TestServices = (Arc<crate::Services>, PathBuf, MutexGuard<'static, ()>);
|
||||
|
||||
pub(crate) async fn test_services() -> TestServices { test_services_with(json!({})).await }
|
||||
|
||||
pub(crate) async fn test_services_with(overrides: serde_json::Value) -> TestServices {
|
||||
let guard = DATABASE.lock().await;
|
||||
let _ = rustls::crypto::ring::default_provider().install_default();
|
||||
let path = std::env::temp_dir()
|
||||
.join(format!("continuwuity-shared-state-{:016x}", rand::random::<u64>()));
|
||||
std::fs::create_dir(&path).unwrap();
|
||||
let mut config = json!({
|
||||
"server_name": "test.example",
|
||||
"database_path": path,
|
||||
"db_cache_capacity_mb": 16.0,
|
||||
"rocksdb_write_buffer_capacity_mb": 4.0,
|
||||
"allow_local_presence": false,
|
||||
"dns_servers": ["127.0.0.1"],
|
||||
"allow_invalid_tls_certificates_yes_i_know_what_the_fuck_i_am_doing_with_this_and_i_know_this_is_insecure": true
|
||||
});
|
||||
config
|
||||
.as_object_mut()
|
||||
.unwrap()
|
||||
.extend(overrides.as_object().unwrap().clone());
|
||||
let config: Config = serde_json::from_value(config).unwrap();
|
||||
let runtime = tokio::runtime::Handle::current();
|
||||
let server = Arc::new(Server::new(
|
||||
config,
|
||||
Some(runtime.clone()),
|
||||
Log {
|
||||
reload: LogLevelReloadHandles::default(),
|
||||
capture: Arc::default(),
|
||||
},
|
||||
Metrics::new(Some(runtime), opentelemetry::global::meter("test"), None),
|
||||
));
|
||||
let services = crate::Services::build(server).await.unwrap();
|
||||
(services, path, guard)
|
||||
}
|
||||
Reference in new issue
Block a user