diff --git a/src/service/sending/sender.rs b/src/service/sending/sender.rs
index 1007bc067..d81c3fd3e 100644
--- a/src/service/sending/sender.rs
+++ b/src/service/sending/sender.rs
@@ -414,7 +414,10 @@ impl Service {
 		let (mut allow, mut retry) = (true, false);
 		if let Destination::Federation(server_name) = dest {
 			if let Some(retry_after) = self.services.federation.retry_after(server_name) {
-				allow = retry_after.is_zero();
+				if !retry_after.is_zero() {
+					trace!("{dest:?} is in backoff, refusing send");
+					return Ok((false, false));
+				}
 			}
 			statuses
 				.entry(dest.clone())