diff --git a/src/api/server/send.rs b/src/api/server/send.rs index 3599c105b..2d54d9c15 100644 --- a/src/api/server/send.rs +++ b/src/api/server/send.rs @@ -159,12 +159,26 @@ async fn process_inbound_transaction( .stream(); debug!(pdus = body.pdus.len(), edus = body.edus.len(), "Processing transaction",); - let results = match handle(&services, &body.identity, pdus, edus).await { - | Ok(results) => results, - | Err(err) => { + // We have an additional panic catch here because if there's a panic in the + // handle, the sender will effectively become defederated until restart after + // never being removed from the tracking map. Ideally the handle function + // doesn't panic, but when has "ideally" ever worked for us. + let handle_result = + std::panic::AssertUnwindSafe(handle(&services, &body.identity, pdus, edus)) + .catch_unwind() + .map_err(Error::from_panic) + .await; + let results = match handle_result { + | Ok(Ok(results)) => results, + | Ok(Err(err)) => { fail_federation_txn(services, &txn_key, &sender, err); return; }, + | Err(err) => { + error!(error=?err, "Fatal error while processing incoming transaction"); + fail_federation_txn(services, &txn_key, &sender, TransactionError::Unexpected); + return; + }, }; for (id, result) in &results { @@ -228,6 +242,11 @@ fn transaction_error_to_response(err: &TransactionError) -> Error { "Server is shutting down, please retry later".into(), StatusCode::SERVICE_UNAVAILABLE, ), + | TransactionError::Unexpected => Error::Request( + ErrorKind::Unknown, + err.to_string().into(), + StatusCode::INTERNAL_SERVER_ERROR, + ), } } async fn handle( diff --git a/src/service/transactions/mod.rs b/src/service/transactions/mod.rs index ae03499d7..86f4bd6b6 100644 --- a/src/service/transactions/mod.rs +++ b/src/service/transactions/mod.rs @@ -32,12 +32,14 @@ pub enum TransactionError { /// Server is shutting down - the sender should retry the entire /// transaction. ShuttingDown, + Unexpected, } impl fmt::Display for TransactionError { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { | Self::ShuttingDown => write!(f, "Server is shutting down"), + | Self::Unexpected => write!(f, "Fatal Internal Server Error"), } } }