fix(federation): Make transactions infallible

Previously, a transaction panicking in the handle() call would cause the
transaction to be indefinitely logged as "running" in the transaction
handler. This means subsequent transactions from the sending server
would be met with the "You're still sending me a txn!" 429 response,
forever, effectively causing defederation between the two servers, until
continuwuity is fully restarted.

I could just fix the known panics in the handle function and internal
calls, however there's so many subsystems that could reasonably panic
that it is just better to have a safer approach and ensure that the
handle call is infallible.

src/api/server/send.rs

@@ -159,12 +159,26 @@ async fn process_inbound_transaction(
 		.stream();
 
 	debug!(pdus = body.pdus.len(), edus = body.edus.len(), "Processing transaction",);
-	let results = match handle(&services, &body.identity, pdus, edus).await {
-		| Ok(results) => results,
-		| Err(err) => {
+	// We have an additional panic catch here because if there's a panic in the
+	// handle, the sender will effectively become defederated until restart after
+	// never being removed from the tracking map. Ideally the handle function
+	// doesn't panic, but when has "ideally" ever worked for us.
+	let handle_result =
+		std::panic::AssertUnwindSafe(handle(&services, &body.identity, pdus, edus))
+			.catch_unwind()
+			.map_err(Error::from_panic)
+			.await;
+	let results = match handle_result {
+		| Ok(Ok(results)) => results,
+		| Ok(Err(err)) => {
 			fail_federation_txn(services, &txn_key, &sender, err);
 			return;
 		},
+		| Err(err) => {
+			error!(error=?err, "Fatal error while processing incoming transaction");
+			fail_federation_txn(services, &txn_key, &sender, TransactionError::Unexpected);
+			return;
+		},
 	};
 
 	for (id, result) in &results {
@@ -228,6 +242,11 @@ fn transaction_error_to_response(err: &TransactionError) -> Error {
 			"Server is shutting down, please retry later".into(),
 			StatusCode::SERVICE_UNAVAILABLE,
 		),
+		| TransactionError::Unexpected => Error::Request(
+			ErrorKind::Unknown,
+			err.to_string().into(),
+			StatusCode::INTERNAL_SERVER_ERROR,
+		),
 	}
 }
 async fn handle(

src/service/transactions/mod.rs

@@ -32,12 +32,14 @@ pub enum TransactionError {
 	/// Server is shutting down - the sender should retry the entire
 	/// transaction.
 	ShuttingDown,
+	Unexpected,
 }
 
 impl fmt::Display for TransactionError {
 	fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
 		match self {
 			| Self::ShuttingDown => write!(f, "Server is shutting down"),
+			| Self::Unexpected => write!(f, "Fatal Internal Server Error"),
 		}
 	}
 }