From 4f128af9212c0c317c72ded3a9fdb0abb1c26280 Mon Sep 17 00:00:00 2001 From: timedout Date: Sat, 31 Oct 2026 22:17:11 +0000 Subject: [PATCH] feat: Move & destructure servers-in-back command to federation group --- src/admin/debug/commands.rs | 56 ------------ src/admin/debug/mod.rs | 14 --- src/admin/federation/health.rs | 158 +++++++++++++++++++++++++++++++++ src/admin/federation/mod.rs | 8 +- 4 files changed, 164 insertions(+), 72 deletions(-) create mode 100644 src/admin/federation/health.rs diff --git a/src/admin/debug/commands.rs b/src/admin/debug/commands.rs index 97ba37fa7..eba80b214 100644 --- a/src/admin/debug/commands.rs +++ b/src/admin/debug/commands.rs @@ -1124,60 +1124,4 @@ impl crate::Context<'_> { Ok(()) } - - pub(super) async fn servers_in_backoff( - &self, - room_id: Option, - name: Option, - ) -> Result { - let backoff_map = self.services.federation.remote_health(); - - if backoff_map.is_empty() { - return self.write_str("No servers are currently in backoff.").await; - } - - let stale_map = self.services.federation.stale_destinations(); - - if let Some(specific_server_name) = name { - let entry = backoff_map.get(&specific_server_name); - return match entry { - | Some((retries, next_retry)) => - self.write_str(&format!( - "{specific_server_name}: {retries} retries, next retry at {next_retry}" - )) - .await, - | None => - self.write_str(&format!("Server {specific_server_name} is not in backoff.")) - .await, - }; - } - - self.write_str("| Server Name | Retries | Next connection attempt | Is stale |\n") - .await?; - self.write_str("| ----------- | ------- | ----------------------- | -------- |\n") - .await?; - - if let Some(specific_room) = room_id { - let mut servers_in_room = - self.services.rooms.state_cache.room_servers(&specific_room); - while let Some(server_name) = servers_in_room.next().await { - let Some((retries, retry_at)) = backoff_map.get(&server_name) else { continue }; - let is_stale = if stale_map.contains(&server_name) { "Yes" } else { "No" }; - self.write_str(&format!( - "| {server_name} | {retries} | {retry_at} | {is_stale} |\n" - )) - .await?; - } - - return Ok(()); - } - - for (server_name, (retries, retry_at)) in &backoff_map { - let is_stale = if stale_map.contains(server_name) { "Yes" } else { "No" }; - self.write_str(&format!("| {server_name} | {retries} | {retry_at} | {is_stale} |\n")) - .await?; - } - - Ok(()) - } } diff --git a/src/admin/debug/mod.rs b/src/admin/debug/mod.rs index d61f15407..6cf211109 100644 --- a/src/admin/debug/mod.rs +++ b/src/admin/debug/mod.rs @@ -208,20 +208,6 @@ pub enum DebugCommand { room_id: OwnedRoomId, }, - /// Lists remote servers that are currently in a "backoff" state. - /// - /// Servers in this list will not be contacted until the retry time elapses, - /// or they make contact first. - /// - /// "Stale" indicates that routing information may be out of date, and the - /// remote will be re-resolved immediately before the next contact attempt. - ServersInBackoff { - #[arg(short, long, alias = "room")] - room_id: Option, - #[arg(short, long)] - name: Option, - }, - /// Developer test stubs #[command(subcommand)] #[allow(non_snake_case)] diff --git a/src/admin/federation/health.rs b/src/admin/federation/health.rs new file mode 100644 index 000000000..8b1feaec7 --- /dev/null +++ b/src/admin/federation/health.rs @@ -0,0 +1,158 @@ +use std::time::Duration; + +use clap::Subcommand; +use conduwuit::Result; +use conduwuit_macros::admin_command_dispatch; +use futures::StreamExt; +use ruma::{OwnedRoomOrAliasId, OwnedServerName}; + +#[admin_command_dispatch] +#[derive(Debug, Subcommand)] +pub enum HealthCommand { + /// Lists all servers that are currently marked as unhealthy (i.e. we + /// previously failed to establish direct federation and have not succeeded + /// since). + #[clap(name = "list")] + ListUnhealthyServers { + /// An optional filter to only check servers that are in a specific room + /// (ID or alias). + room: Option, + }, + + /// Displays the current health status of the given remote server. + #[clap(name = "status")] + ServerHealthStatus { + server_name: OwnedServerName, + }, + + /// Forces the server to consider the remote server healthy. + /// + /// This is generally unnecessary as this already happens when the remote + /// server makes contact with this server, or if this server successfully + /// makes a request to the remote server after the backoff window. + ForceHealthy { + server_name: OwnedServerName, + }, + + /// Forces the server to consider the remote server unhealthy. + /// + /// This is generally unnecessary as this already happens when the remote + /// server fails to resolve or connect, or returns unusual errors. + /// If you are looking to stop contacting a specific server, take a look at + /// the `forbidden_remote_server_names` configuration option. + ForceUnhealthy { + server_name: OwnedServerName, + #[arg(long)] + stale: bool, + }, +} + +impl crate::Context<'_> { + pub(super) async fn list_unhealthy_servers( + &self, + room: Option, + ) -> Result { + let unhealthy = self.services.federation.remote_health(); + + if unhealthy.is_empty() { + return self + .write_str("No servers are currently tracked as unhealthy.") + .await; + } + + let stale_destinations = self.services.federation.stale_destinations(); + + let mut candidates: Vec<_> = if let Some(room) = room { + let room_id = self.services.rooms.alias.resolve(&room).await?; + self.services + .rooms + .state_cache + .room_servers(&room_id) + .collect() + .await + } else { + unhealthy + .keys() + .chain(stale_destinations.iter()) + .cloned() + .collect() + }; + candidates.retain(|name| unhealthy.contains_key(name)); + candidates.sort(); + candidates.dedup(); + if candidates.is_empty() { + return self + .write_str("No servers are currently unhealthy meeting the given criteria.") + .await; + } + for server_name in candidates { + let Some((retries, retry_at)) = unhealthy.get(&server_name) else { continue }; + let is_stale = if stale_destinations.contains(&server_name) { + "Yes" + } else { + "No" + }; + self.write_str(&format!("| {server_name} | {retries} | {retry_at} | {is_stale} |\n")) + .await?; + } + + Ok(()) + } + + pub(super) async fn server_health_status(&self, server_name: OwnedServerName) -> Result { + let stale = self + .services + .federation + .stale_destinations() + .contains(&server_name); + let Some((retries, retry_at_ts)) = self + .services + .federation + .remote_health() + .get(&server_name) + .copied() + else { + return self + .write_str(&if stale { + format!( + "{server_name} is currently healthy, but may have stale routing \ + information." + ) + } else { + format!("{server_name} is currently healthy.") + }) + .await; + }; + + let state = if stale { + "unhealthy with potentially stale routing information" + } else { + "unhealthy" + }; + self.write_str(&format!( + "{server_name} is currently {state}: {retries} consecutive connections have \ + failed,and no new connection will be attempted for another {}.", + conduwuit::utils::time::pretty(Duration::from_millis(retry_at_ts)), + )) + .await + } + + pub(super) async fn force_healthy(&self, server_name: OwnedServerName) -> Result { + self.services.federation.mark_healthy(&server_name); + self.write_str("Done.").await + } + + pub(super) async fn force_unhealthy( + &self, + server_name: OwnedServerName, + stale: bool, + ) -> Result { + if stale { + self.services + .federation + .mark_destination_stale(&server_name); + } + self.services.federation.hit_unhealthy(server_name); + self.write_str("Done.").await + } +} diff --git a/src/admin/federation/mod.rs b/src/admin/federation/mod.rs index df1c49cbc..8107cf362 100644 --- a/src/admin/federation/mod.rs +++ b/src/admin/federation/mod.rs @@ -1,10 +1,11 @@ mod commands; +mod health; use clap::Subcommand; use conduwuit::Result; -use ruma::{OwnedRoomId, OwnedRoomOrAliasId, OwnedServerName, OwnedUserId}; +use ruma::{OwnedRoomOrAliasId, OwnedServerName}; -use crate::admin_command_dispatch; +use crate::{admin_command_dispatch, federation::health::HealthCommand}; #[admin_command_dispatch] #[derive(Debug, Subcommand)] @@ -49,4 +50,7 @@ pub enum FederationCommand { Ping { server_name: OwnedServerName, }, + + #[command(subcommand)] + Health(HealthCommand), }