diff --git a/src/service/federation/execute.rs b/src/service/federation/execute.rs index 2363cca40..152433cb7 100644 --- a/src/service/federation/execute.rs +++ b/src/service/federation/execute.rs @@ -1,4 +1,9 @@ -use std::{borrow::Cow, fmt::Debug, mem, time::Instant}; +use std::{ + borrow::Cow, + fmt::Debug, + mem, + time::{Duration, Instant}, +}; use conduwuit::{ Err, Error, Result, debug, debug_error, debug_info, debug_warn, err, metrics::KeyValue, @@ -133,6 +138,10 @@ impl super::Service { self.ensure_remote_is_healthy(dest)?; + /// How long to mark a server as unhealthy before attempting + /// re-resolution + const UNHEALTHY_TIMEOUT: Duration = Duration::from_hours(24 * 90); + let actual = self .services .client @@ -140,7 +149,7 @@ impl super::Service { .resolve_server(dest.as_str()) .await .inspect_err(|_| { - self.hit_unhealthy(dest.to_owned()); + self.hit_unhealthy(dest.to_owned(), Some(UNHEALTHY_TIMEOUT)); self.mark_destination_stale(dest); })?; @@ -214,7 +223,7 @@ impl super::Service { if self.should_mark_stale(&wrapped) { debug_info!("{dest} is unhealthy & stale due to a connect error"); self.mark_destination_stale(dest); - self.hit_unhealthy(dest.to_owned()); + self.hit_unhealthy(dest.to_owned(), None); } let Error::Reqwest(unwrapped) = wrapped else { unreachable!("wrapped reqwest error must unwrap to a reqwest error"); diff --git a/src/service/federation/mod.rs b/src/service/federation/mod.rs index 76d0e124a..17a5d2d32 100644 --- a/src/service/federation/mod.rs +++ b/src/service/federation/mod.rs @@ -18,6 +18,7 @@ use ruma::{ OwnedServerName, ServerName, api::error::{ErrorKind, LimitExceededErrorData, RetryAfter}, }; +use yansi::Paint; use crate::{Dep, client, moderation, server_keys}; @@ -92,7 +93,7 @@ impl Service { /// retry count is incremented and next retry window is updated. /// /// Does not update the marker if the backoff period is already in effect. - pub fn hit_unhealthy(&self, server_name: OwnedServerName) { + pub fn hit_unhealthy(&self, server_name: OwnedServerName, next_attempt: Option) { if server_name == self.services.server.name { return; } @@ -109,7 +110,8 @@ impl Service { } *retries = retries.saturating_add(1); - let next_interval = self.services.server.config.sender_retry_backoff(*retries); + let next_interval = next_attempt + .unwrap_or_else(|| self.services.server.config.sender_retry_backoff(*retries)); *next_retry = millis_from_now(next_interval); debug!( "{} is (now) unhealthy ({} retries, blocked until: {})", @@ -200,14 +202,14 @@ impl Service { } match error.status_code() { - // Some special servers account for this specifically - | StatusCode::MISDIRECTED_REQUEST - // Common error codes observed for misdirected requests - // | StatusCode::NOT_FOUND This one can be encountered naturally - | StatusCode::METHOD_NOT_ALLOWED - | StatusCode::IM_A_TEAPOT => true, - _ => false, - } + // Some special servers account for this specifically + | StatusCode::MISDIRECTED_REQUEST + // Common error codes observed for misdirected requests + // | StatusCode::NOT_FOUND This one can be encountered naturally + | StatusCode::METHOD_NOT_ALLOWED + | StatusCode::IM_A_TEAPOT => true, + _ => false, + } } /// Returns a clone of the internal remote health tracking map. diff --git a/src/service/sending/sender.rs b/src/service/sending/sender.rs index 5467b36f4..03be9e2ce 100644 --- a/src/service/sending/sender.rs +++ b/src/service/sending/sender.rs @@ -149,10 +149,10 @@ impl Service { { debug!("{dest} is now unhealthy & stale due to a connect error: {e:?}"); self.services.federation.mark_destination_stale(dest); - self.services.federation.hit_unhealthy(dest.clone()); + self.services.federation.hit_unhealthy(dest.clone(), None); } else if e.status_code() != StatusCode::OK { debug!("{dest} is now unhealthy due to error response: {e:?}"); - self.services.federation.hit_unhealthy(dest.clone()); + self.services.federation.hit_unhealthy(dest.clone(), None); } } Self::handle_response_err(dest, statuses, &e);