Compare commits

...
5 changed files with 300 additions and 80 deletions

No files matched your search

+27 -51
View File
@@ -44,7 +44,10 @@ use ruma::{
to_device::DeviceIdOrAllDevices,
};
use service::{
rooms::event_handler::{DagBuilderTree, build_local_dag},
rooms::{
event_handler::{DagBuilderTree, build_local_dag},
queue::QueueJob,
},
transactions::{FederationTxnState, TransactionError, TxnKey, WrappedTransactionResponse},
};
use tokio::sync::watch::{Receiver, Sender};
@@ -332,13 +335,6 @@ async fn handle_room(
room_id: OwnedRoomId,
pdus: impl Iterator<Item = Pdu> + Send,
) -> std::result::Result<Vec<(OwnedEventId, Result)>, TransactionError> {
let _room_lock = services
.rooms
.event_handler
.mutex_federation
.lock(room_id.as_str())
.await;
let room_id = &room_id;
let mut pdu_map: HashMap<OwnedEventId, CanonicalJsonObject> = pdus
.into_iter()
@@ -358,50 +354,30 @@ async fn handle_room(
let value = pdu_map
.remove(&event_id)
.expect("sorted event IDs must be from the original map");
services
.server
.check_running()
.map_err(|_| TransactionError::ShuttingDown)?;
let event_type = KeyValue::new(
"type",
pdu_type_label(value.get("type").and_then(CanonicalJsonValue::as_str)),
);
let origin_server_ts = value
.get("origin_server_ts")
.and_then(CanonicalJsonValue::as_integer)
.and_then(|ts| u64::try_from(i64::from(ts)).ok());
let room_id = match value
.get("room_id")
.and_then(CanonicalJsonValue::as_str)
.map(|v| v.try_into().map_err(Into::into))
{
| Some(Ok(v)) => v,
| Some(Err(e)) => {
results.push((event_id, Err(e)));
continue;
},
| None => {
results
.push((event_id, Err!(Request(BadJson("PDU is missing a room_id field")))));
continue;
},
};
let job = QueueJob {
room_id,
event_id: event_id.clone(),
origin: origin.to_owned(),
object: value,
};
let start = Instant::now();
let result = services
.rooms
.event_handler
.handle_incoming_pdu(origin, room_id, &event_id, value, false)
.boxed()
.await
.map(|_| ());
let outcome = KeyValue::new("outcome", match &result {
| Ok(()) => "accepted",
| Err(Error::BadRequest(ErrorKind::Forbidden, ..)) => "forbidden",
| Err(Error::BadRequest(ErrorKind::InvalidParam | ErrorKind::BadJson, ..)) =>
"invalid",
| Err(_) => "error",
});
let instruments = &services.server.metrics.instruments;
instruments
.federation_incoming_pdu_duration
.record(start.elapsed().as_secs_f64(), std::slice::from_ref(&outcome));
if let (Ok(()), Some(ts)) = (&result, origin_server_ts) {
let lag = millis_since_unix_epoch().saturating_sub(ts);
instruments
.federation_incoming_pdu_lag
.record(Duration::from_millis(lag).as_secs_f64(), &[]);
}
instruments
.federation_incoming_pdus
.add(1, &[event_type, outcome]);
results.push((event_id, result));
results.push((event_id, services.rooms.queue.new_job(job).await));
}
Ok(results)
}
+30 -29
View File
@@ -11,8 +11,8 @@ version.workspace = true
[lib]
path = "mod.rs"
crate-type = [
"rlib",
# "dylib",
"rlib",
# "dylib",
]
[features]
@@ -23,61 +23,61 @@ bindgen-static = [
"conduwuit-database/bindgen-static"
]
brotli_compression = [
"conduwuit-core/brotli_compression",
"reqwest/brotli",
"conduwuit-core/brotli_compression",
"reqwest/brotli",
]
console = [
"dep:rustyline-async",
"dep:termimad",
"dep:rustyline-async",
"dep:termimad",
]
element_hacks = []
gzip_compression = [
"conduwuit-core/gzip_compression",
"reqwest/gzip",
"conduwuit-core/gzip_compression",
"reqwest/gzip",
]
http3 = [
"reqwest/http3"
]
io_uring = [
"conduwuit-database/io_uring",
"conduwuit-database/io_uring",
]
jemalloc = [
"conduwuit-core/jemalloc",
"conduwuit-database/jemalloc",
"conduwuit-core/jemalloc",
"conduwuit-database/jemalloc",
]
jemalloc_conf = [
"conduwuit-core/jemalloc_conf",
"conduwuit-database/jemalloc_conf",
"conduwuit-core/jemalloc_conf",
"conduwuit-database/jemalloc_conf",
]
jemalloc_prof = [
"conduwuit-core/jemalloc_prof",
"conduwuit-database/jemalloc_prof",
"conduwuit-core/jemalloc_prof",
"conduwuit-database/jemalloc_prof",
]
jemalloc_stats = [
"conduwuit-core/jemalloc_stats",
"conduwuit-database/jemalloc_stats",
"conduwuit-core/jemalloc_stats",
"conduwuit-database/jemalloc_stats",
]
media_thumbnail = [
"dep:image",
"dep:image",
]
release_max_log_level = [
"conduwuit-core/release_max_log_level",
"conduwuit-database/release_max_log_level",
"log/max_level_trace",
"log/release_max_level_info",
"tracing/max_level_trace",
"tracing/release_max_level_info",
"conduwuit-core/release_max_log_level",
"conduwuit-database/release_max_log_level",
"log/max_level_trace",
"log/release_max_level_info",
"tracing/max_level_trace",
"tracing/release_max_level_info",
]
systemd = [
"dep:sd-notify",
"dep:sd-notify",
]
url_preview = [
"dep:image",
"dep:webpage",
"dep:image",
"dep:webpage",
]
zstd_compression = [
"conduwuit-core/zstd_compression",
"conduwuit-database/zstd_compression",
"conduwuit-core/zstd_compression",
"conduwuit-database/zstd_compression",
"reqwest/zstd",
]
@@ -135,6 +135,7 @@ openidconnect.workspace = true
# which is necessary to call `danger_accept_invalid_certs`.
reqwest_0_12.workspace = true
rand.workspace = true
opentelemetry.workspace = true
[target.'cfg(all(unix, target_os = "linux"))'.dependencies]
sd-notify.workspace = true
+2
View File
@@ -7,6 +7,7 @@ pub mod membership;
pub mod metadata;
pub mod outlier;
pub mod pdu_metadata;
pub mod queue;
pub mod read_receipt;
pub mod search;
pub mod short;
@@ -32,6 +33,7 @@ pub struct Service {
pub metadata: Arc<metadata::Service>,
pub outlier: Arc<outlier::Service>,
pub pdu_metadata: Arc<pdu_metadata::Service>,
pub queue: Arc<queue::Service>,
pub read_receipt: Arc<read_receipt::Service>,
pub search: Arc<search::Service>,
pub short: Arc<short::Service>,
+240
View File
@@ -0,0 +1,240 @@
use std::{
collections::{HashMap, hash_map::Entry},
sync::Arc,
time::{Duration, Instant},
};
use async_trait::async_trait;
use conduwuit::{
Error, Result, Server, SyncMutex, debug, err, error, info, utils::millis_since_unix_epoch,
};
use opentelemetry::KeyValue;
use ruma::{
CanonicalJsonObject, CanonicalJsonValue, OwnedEventId, OwnedRoomId, OwnedServerName,
api::error::ErrorKind, events::TimelineEventType,
};
use tokio::{
sync::{
Notify, mpsc,
mpsc::{Sender, UnboundedReceiver, UnboundedSender},
},
task::JoinSet,
};
use crate::{
rooms,
service::{Args, Dep},
};
pub struct Services {
event_handler: Dep<rooms::event_handler::Service>,
server: Arc<Server>,
}
pub struct Service {
queues: SyncMutex<HashMap<OwnedRoomId, UnboundedSender<QueueJob>>>,
send: Sender<QueueJob>,
recv: tokio::sync::Mutex<mpsc::Receiver<QueueJob>>,
shutdown: Notify,
services: Services,
}
#[async_trait]
impl crate::Service for Service {
fn build(args: Args<'_>) -> Result<Arc<Self>>
where
Self: Sized,
{
let (send, recv) = mpsc::channel(1);
Ok(Arc::new(Self {
queues: SyncMutex::new(HashMap::new()),
shutdown: Notify::new(),
send,
recv: tokio::sync::Mutex::new(recv),
services: Services {
event_handler: args.depend("rooms::event_handler"),
server: args.server.clone(),
},
}))
}
async fn worker(self: Arc<Self>) -> Result<()> {
let mut recv = self.recv.lock().await;
let mut handles = JoinSet::new();
let (yielder, mut yielding) = mpsc::channel(5);
loop {
let job = tokio::select! {
() = self.shutdown.notified() => {
info!("Shutting down");
yielding.close();
recv.close();
break;
},
room_id = yielding.recv() => {
if let Some(room_id) = room_id {
self.queues.lock().remove(&room_id);
}
continue;
}
job = recv.recv() => job
};
let Some(job) = job else {
debug!("No job (channel closed)");
break;
};
let mut queues = self.queues.lock();
match queues.entry(job.room_id.clone()) {
| Entry::Occupied(entry) => {
entry.get().send(job).unwrap();
// TODO: handle error
// It might be possible for a self-reaping worker to close
// at the same time we try to send to it, which will cause
// this send to return a "channel closed" error.
},
| Entry::Vacant(entry) => {
let room_id = entry.key().to_owned();
let (sender, receiver) = mpsc::unbounded_channel();
sender.send(job).unwrap(); // TODO: handle error
handles.spawn_on(
Self::room_worker(self.clone(), room_id, receiver, yielder.clone()),
self.services.server.runtime(),
);
entry.insert(sender);
},
}
}
// If the loop broke, make sure that the shutdown signal was sent, so
// that any room workers are guaranteed to shut down as soon as
// possible.
self.interrupt();
while let Some(ret) = handles.join_next_with_id().await {
match ret {
| Ok((id, ())) => {
debug!(?id, "room queue worker finished");
},
| Err(error) => {
error!(id = ?error.id(), ?error, "room queue worker failed");
},
}
}
Ok(())
}
fn interrupt(&self) { self.shutdown.notify_waiters(); }
fn name(&self) -> &'static str { module_path!() }
}
impl Service {
pub async fn new_job(&self, job: QueueJob) -> Result<()> {
self.send
.send(job)
.await
.map_err(|e| err!("failed to send PDU to worker: {e}"))
}
#[tracing::instrument(skip(self,channel,yielder), fields(%room_id))]
async fn room_worker(
self: Arc<Self>,
room_id: OwnedRoomId,
mut channel: UnboundedReceiver<QueueJob>,
yielder: Sender<OwnedRoomId>,
) {
debug!("Room worker starting up");
loop {
let job = tokio::select! {
() = self.shutdown.notified() => {
debug!("Room worker shutting down");
channel.close();
return;
},
() = tokio::time::sleep(Duration::from_mins(5)) => {
debug!("Room worker self-reaping (no activity in 5 minutes)");
yielder.send(room_id).await.ok();
debug!("Room worker reaped. Goodbye.");
return;
},
job = channel.recv() => job
};
let Some(job) = job else {
debug!("Room worker channel closed without shutdown?");
break;
};
debug!(
event_id=%job.event_id,
origin=%job.origin,
backpressure=%channel.len(),
"Handling incoming job"
);
let _room_lock = self
.services
.event_handler
.mutex_federation
.lock(room_id.as_str())
.await;
debug!(event_id=%job.event_id, origin=%job.origin, "Acquired room federation lock");
let event_type = KeyValue::new(
"type",
pdu_type_label(job.object.get("type").and_then(CanonicalJsonValue::as_str)),
);
let origin_server_ts = job
.object
.get("origin_server_ts")
.and_then(CanonicalJsonValue::as_integer)
.and_then(|ts| u64::try_from(i64::from(ts)).ok());
let start = Instant::now();
let result = self
.services
.event_handler
.handle_incoming_pdu(&job.origin, &job.room_id, &job.event_id, job.object, false)
.await;
let outcome = KeyValue::new("outcome", match &result {
| Ok(_) => "accepted",
| Err(Error::BadRequest(ErrorKind::Forbidden, ..)) => "forbidden",
| Err(Error::BadRequest(ErrorKind::InvalidParam | ErrorKind::BadJson, ..)) =>
"invalid",
| Err(_) => "error",
});
let instruments = &self.services.server.metrics.instruments;
instruments
.federation_incoming_pdu_duration
.record(start.elapsed().as_secs_f64(), std::slice::from_ref(&outcome));
if let (Ok(_), Some(ts)) = (&result, origin_server_ts) {
let lag = millis_since_unix_epoch().saturating_sub(ts);
instruments
.federation_incoming_pdu_lag
.record(Duration::from_millis(lag).as_secs_f64(), &[]);
}
instruments
.federation_incoming_pdus
.add(1, &[event_type, outcome]);
debug!(event_id=%job.event_id, origin=%job.origin, "Job finished");
}
}
}
pub struct QueueJob {
pub origin: OwnedServerName,
pub event_id: OwnedEventId,
pub room_id: OwnedRoomId,
pub object: CanonicalJsonObject,
}
/// Known event types, or `other`, so remote servers cannot create new
/// attribute values.
fn pdu_type_label(event_type: Option<&str>) -> String {
match event_type.map(TimelineEventType::from) {
| Some(TimelineEventType::_Custom(_)) | None => "other".to_owned(),
| Some(event_type) => event_type.to_string(),
}
}
+1
View File
@@ -99,6 +99,7 @@ impl Services {
metadata: build!(rooms::metadata::Service),
outlier: build!(rooms::outlier::Service),
pdu_metadata: build!(rooms::pdu_metadata::Service),
queue: build!(rooms::queue::Service),
read_receipt: build!(rooms::read_receipt::Service),
search: build!(rooms::search::Service),
short: build!(rooms::short::Service),