chore: Add newsfrag

Should prevent weird situations where we accidentally gapfill into backfill territory

changelog.d/1818.feature.md

@@ -0,0 +1,2 @@
+Improved the performance and reliability of fetching missing events, improving network partition recovery. Contributed
+by @nex.

conduwuit-example.toml

@@ -297,7 +297,7 @@
 
 # This item is undocumented. Please contribute documentation for it.
 #
-#max_fetch_prev_events = 192
+#max_fetch_prev_events = 256
 
 # How many incoming federation transactions the server is willing to be
 # processing at any given time before it becomes overloaded and starts

src/api/server/send.rs

@@ -381,6 +381,7 @@ async fn handle_room(
 			.rooms
 			.event_handler
 			.handle_incoming_pdu(origin, room_id, &event_id, value, true)
+			.boxed()
 			.await
 			.map(|_| ());
 		results.push((event_id, result));

src/core/config/mod.rs

@@ -375,7 +375,7 @@ pub struct Config {
 	#[serde(default = "default_max_request_size")]
 	pub max_request_size: usize,
 
-	/// default: 192
+	/// default: 256
 	#[serde(default = "default_max_fetch_prev_events")]
 	pub max_fetch_prev_events: u16,
 
@@ -2549,7 +2549,7 @@ fn default_pusher_timeout() -> u64 { 60 }
 
 fn default_pusher_idle_timeout() -> u64 { 15 }
 
-fn default_max_fetch_prev_events() -> u16 { 192_u16 }
+fn default_max_fetch_prev_events() -> u16 { 256_u16 }
 
 fn default_max_concurrent_inbound_transactions() -> usize { 150 }
 

src/database/maps.rs

@@ -187,6 +187,10 @@ pub(super) static MAPS: &[Descriptor] = &[
 		val_size_hint: Some(8),
 		..descriptor::RANDOM_SMALL
 	},
+	Descriptor {
+		name: "roomid_mindepth",
+		..descriptor::RANDOM_SMALL
+	},
 	Descriptor {
 		name: "roomserverids",
 		..descriptor::RANDOM_SMALL

src/service/rooms/event_handler/fetch_and_handle_outliers.rs

@@ -1,233 +1,456 @@
 use std::{
-	collections::{BTreeMap, HashSet, VecDeque, hash_map},
+	collections::{BTreeMap, HashMap, HashSet, VecDeque, hash_map},
 	time::Instant,
 };
 
+use assign::assign;
 use conduwuit::{
-	Event, PduEvent, debug, debug_warn, implement, matrix::event::gen_event_id_canonical_json,
-	trace, utils::continue_exponential_backoff_secs, warn,
+	Event, PduEvent, debug, debug_info, debug_warn, err, error,
+	matrix::event::gen_event_id_canonical_json,
+	state_res::lexicographical_topological_sort,
+	trace,
+	utils::{IterStream, continue_exponential_backoff_secs, stream::BroadbandExt},
+	warn,
 };
+use futures::StreamExt;
 use ruma::{
-	CanonicalJsonValue, EventId, OwnedEventId, RoomId, ServerName,
-	api::federation::event::get_event,
+	CanonicalJsonObject, CanonicalJsonValue, EventId, MilliSecondsSinceUnixEpoch, OwnedEventId,
+	RoomId, ServerName, UInt,
+	api::federation::event::{get_event, get_missing_events},
+	int,
 };
 
 use super::get_room_version_rules;
 
-/// Find the event and auth it. Once the event is validated (steps 1 - 8)
-/// it is appended to the outliers Tree.
+/// Attempts to build a localised directed acyclic graph out of the given PDUs,
+/// returning them in a topologically sorted order.
 ///
-/// Returns pdu and if we fetched it over federation the raw json.
-///
-/// a. Look in the main timeline (pduid_pdu tree)
-/// b. Look at outlier pdu tree
-/// c. Ask origin server over federation
-/// d. TODO: Ask other servers over federation?
-#[implement(super::Service)]
-pub(super) async fn fetch_and_handle_outliers<'a, Pdu, Events>(
-	&self,
-	origin: &'a ServerName,
-	events: Events,
-	create_event: &'a Pdu,
-	room_id: &'a RoomId,
-) -> Vec<(PduEvent, Option<BTreeMap<String, CanonicalJsonValue>>)>
-where
-	Pdu: Event + Send + Sync,
-	Events: Iterator<Item = &'a EventId> + Clone + Send,
-{
-	let back_off = |id| match self
-		.services
-		.globals
-		.bad_event_ratelimiter
-		.write()
-		.entry(id)
-	{
-		| hash_map::Entry::Vacant(e) => {
-			e.insert((Instant::now(), 1));
-		},
-		| hash_map::Entry::Occupied(mut e) => {
-			*e.get_mut() = (Instant::now(), e.get().1.saturating_add(1));
-		},
-	};
+/// This is used to attempt to process PDUs in an order that respects their
+/// dependencies, however it is ultimately the sender's responsibility to send
+/// them in a processable order, so this is just a best effort attempt. It does
+/// not account for power levels or other tie breaks.
+pub async fn build_local_dag<S: std::hash::BuildHasher>(
+	pdu_map: &HashMap<OwnedEventId, CanonicalJsonObject, S>,
+) -> conduwuit::Result<Vec<OwnedEventId>> {
+	debug_assert!(pdu_map.len() >= 2, "needless call to build_local_dag with less than 2 PDUs");
+	let mut dag: HashMap<OwnedEventId, HashSet<OwnedEventId>> =
+		HashMap::with_capacity(pdu_map.len());
+	let mut id_origin_ts: HashMap<OwnedEventId, _> = HashMap::with_capacity(pdu_map.len());
 
-	let mut events_with_auth_events = Vec::with_capacity(events.clone().count());
-	trace!("Fetching {} outlier pdus", events.clone().count());
+	for (event_id, value) in pdu_map {
+		// We already checked that these properties are correct in parse_incoming_pdu,
+		// so it's safe to unwrap here.
+		// We also filter to remove any prev_events that are not in this pdu_map, as we
+		// need to have at least one event with zero out degrees for the lexico-topo
+		// sort below. If there are multiple events with omitted prevs, they will be
+		// ordered by timestamp, then event ID. At that point though, it's unlikely to
+		// matter.
+		let prev_events = value
+			.get("prev_events")
+			.unwrap()
+			.as_array()
+			.unwrap()
+			.iter()
+			.map(|v| EventId::parse(v.as_str().unwrap()).unwrap())
+			.filter(|id| pdu_map.contains_key(id))
+			.collect();
 
-	for id in events {
-		// a. Look in the main timeline (pduid_pdu tree)
-		// b. Look at outlier pdu tree
-		// (get_pdu_json checks both)
-		if let Ok(local_pdu) = self.services.timeline.get_pdu(id).await {
-			trace!("Found {id} in main timeline or outlier tree");
-			events_with_auth_events.push((id.to_owned(), Some(local_pdu), vec![]));
-			continue;
-		}
+		dag.insert(event_id.clone(), prev_events);
+		let origin_server_ts = value
+			.get("origin_server_ts")
+			.and_then(CanonicalJsonValue::as_integer)
+			.unwrap_or_default();
+		id_origin_ts.insert(event_id.clone(), origin_server_ts);
+	}
 
-		// c. Ask origin server over federation
-		// We also handle its auth chain here so we don't get a stack overflow in
-		// handle_outlier_pdu.
-		let mut todo_auth_events: VecDeque<_> = [id.to_owned()].into();
-		let mut events_in_reverse_order = Vec::with_capacity(todo_auth_events.len());
+	debug!(count = dag.len(), "Sorting incoming events with partial graph");
+	lexicographical_topological_sort(&dag, &async |node_id| {
+		// Note: we don't bother fetching power levels because that would massively slow
+		// this function down. This is a best-effort attempt to order events correctly
+		// for processing, however ultimately that should be the sender's job.
+		let ts = id_origin_ts
+			.get(&node_id)
+			.copied()
+			.unwrap_or_else(|| int!(0))
+			.to_string()
+			.parse::<u64>()
+			.ok()
+			.and_then(UInt::new)
+			.unwrap_or_default();
+		Ok((int!(0), MilliSecondsSinceUnixEpoch(ts)))
+	})
+	.await
+	.inspect(|sorted| {
+		debug_assert_eq!(
+			sorted.len(),
+			pdu_map.len(),
+			"Sorted graph was not the same size as the input graph"
+		);
+	})
+	.map_err(|e| err!("failed to resolve local graph: {e}"))
+}
 
-		let mut events_all = HashSet::with_capacity(todo_auth_events.len());
-		while let Some(next_id) = todo_auth_events.pop_front() {
-			if let Some((time, tries)) = self
-				.services
-				.globals
-				.bad_event_ratelimiter
-				.read()
-				.get(&*next_id)
-			{
-				// Exponential backoff
-				const MIN_DURATION: u64 = 60 * 2;
-				const MAX_DURATION: u64 = 60 * 60 * 8;
-				if continue_exponential_backoff_secs(
-					MIN_DURATION,
-					MAX_DURATION,
-					time.elapsed(),
-					*tries,
-				) {
-					debug_warn!(
-						tried = ?*tries,
-						elapsed = ?time.elapsed(),
-						"Backing off from {next_id}",
-					);
-					continue;
-				}
-			}
+impl super::Service {
+	/// Uses `/_matrix/federation/v1/get_missing_events` to fill gaps in the
+	/// DAG.
+	///
+	/// When this function is called, the "earliest events" (current forward
+	/// extremities) will be collected, and the function will loop with an
+	/// exponentially incrementing limit (up to 100 per request) until it has
+	/// filled the gap, i.e. when the remote says there's no more events.
+	///
+	/// This function will iterate until the remote returns no more events,
+	/// increasing the limit by a factor of 10. If 100 iterations are reached or
+	/// max_fetch_prev_events events are backfilled, the function will give up
+	/// and return what it has, to avoid pulling in too much data (for example,
+	/// absurdly large gaps).
+	///
+	/// This function does not persist the events. The caller is responsible for
+	/// passing them through handle_incoming_pdu.
+	///
+	/// ## Parameters
+	///
+	/// - `room_id`: The room's ID.
+	/// - `head`: The event we are potentially missing prev_events for.
+	/// - `tail`: The most recently known events in the graph (typically forward
+	///   extremities).
+	/// - `via`: The server to ask for missing events.
+	/// - `min_depth`: Don't process events with a `depth` lower than this
+	///   value. Not massively useful, but can help short-circuit infinite loops
+	///   and weird edge paths.
+	pub async fn get_missing_events(
+		&self,
+		room_id: &RoomId,
+		head: &PduEvent,
+		tail: Vec<OwnedEventId>,
+		via: &ServerName,
+		min_depth: UInt,
+	) -> conduwuit::Result<HashMap<OwnedEventId, PduEvent>> {
+		#[cfg(debug_assertions)]
+		{
+			let missing_count = head
+				.prev_events()
+				.stream()
+				.broad_filter_map(|event_id| async move {
+					match self
+						.services
+						.timeline
+						.get_non_outlier_pdu_json(event_id)
+						.await
+						.inspect(|_| debug!("Found prev_event {event_id} locally."))
+						.inspect_err(
+							|e| debug!(%e, "Could not find prev_event {event_id} locally."),
+						) {
+						| Ok(_) => None,
+						| Err(_) => Some(event_id),
+					}
+				})
+				.count()
+				.await;
+			debug_assert_ne!(
+				missing_count, 0,
+				"event passed to get_missing_events is not missing any events (wasteful call)"
+			);
+		};
 
-			if events_all.contains(&next_id) {
-				continue;
-			}
-
-			if self.services.timeline.pdu_exists(&next_id).await {
-				trace!("Found {next_id} in db");
-				continue;
-			}
-
-			debug!("Fetching {next_id} over federation from {origin}.");
-			match self
+		let mut discovered = HashMap::with_capacity(20);
+		let mut latest_events = vec![head.event_id().to_owned()];
+		let mut iterations = 0_u8;
+		loop {
+			iterations = iterations.saturating_add(1);
+			let limit = iterations.saturating_mul(10).min(100);
+			debug_info!(%limit, %via, %iterations, discovered=discovered.len(), %min_depth, "Attempting to gap fill missing events");
+			let response: get_missing_events::v1::Response = self
 				.services
 				.sending
 				.send_federation_request(
-					origin,
-					get_event::v1::Request::new((*next_id).to_owned()),
+					via,
+					assign!(
+						get_missing_events::v1::Request::new(
+							room_id.to_owned(),
+							tail.clone(),
+							latest_events.clone()
+						),
+						{limit: limit.into(), min_depth}
+					),
 				)
-				.await
-			{
-				| Ok(res) => {
-					debug!("Got {next_id} over federation from {origin}");
-					let Ok(room_version_rules) = get_room_version_rules(create_event) else {
-						back_off((*next_id).to_owned());
-						continue;
-					};
+				.await?;
 
-					let Ok((calculated_event_id, value)) =
-						gen_event_id_canonical_json(&res.pdu, &room_version_rules)
-					else {
-						back_off((*next_id).to_owned());
-						continue;
-					};
+			if response.events.is_empty() {
+				debug_info!(%via, "Finished gap filling missing events (remote returned no more events).");
+				break;
+			}
+			debug_info!("Got {} events back from remote", response.events.len());
 
-					if calculated_event_id != *next_id {
-						warn!(
-							"Server didn't return event id we requested: requested: {next_id}, \
-							 we got {calculated_event_id}. Event: {:?}",
-							&res.pdu
-						);
+			latest_events.clear();
+			for raw_event in response.events {
+				let (_, event_id, pdu_json) = self.parse_incoming_pdu(&raw_event).await?;
+				let pdu = PduEvent::from_id_val(&event_id, pdu_json).map_err(|e| {
+					err!(Request(BadJson("Failed to parse backfilled event {event_id}: {e}")))
+				})?;
+
+				if pdu.depth < min_depth {
+					debug_warn!(
+						"Received PDU with depth {} below min_depth {}, ignoring",
+						pdu.depth,
+						min_depth
+					);
+					continue;
+				}
+
+				for prev_event_id in pdu.prev_events() {
+					if discovered.contains_key(prev_event_id) {
+						continue;
 					}
-
-					if let Some(auth_events) = value
-						.get("auth_events")
-						.and_then(CanonicalJsonValue::as_array)
+					if self
+						.services
+						.timeline
+						.non_outlier_pdu_exists(prev_event_id)
+						.await
 					{
-						for auth_event in auth_events {
-							match serde_json::from_value::<OwnedEventId>(
-								auth_event.clone().into(),
-							) {
-								| Ok(auth_event) => {
-									trace!(
-										"Found auth event id {auth_event} for event {next_id}"
-									);
-									todo_auth_events.push_back(auth_event);
-								},
-								| _ => {
-									warn!("Auth event id is not valid");
-								},
-							}
-						}
-					} else {
-						warn!("Auth event list invalid");
+						continue;
 					}
+					latest_events.push(prev_event_id.to_owned());
+					break;
+				}
 
-					events_in_reverse_order.push((next_id.clone(), value));
-					events_all.insert(next_id);
-				},
-				| Err(e) => {
-					warn!("Failed to fetch auth event {next_id} from {origin}: {e}");
-					back_off((*next_id).to_owned());
-				},
+				discovered.insert(event_id.clone(), pdu);
+			}
+
+			if latest_events.is_empty() {
+				break;
+			} else if discovered.len() > self.services.server.config.max_fetch_prev_events.into()
+				|| iterations >= 20
+			{
+				error!(
+					filled=discovered.len(),
+					max_fetch_prev_events=self.services.server.config.max_fetch_prev_events,
+					%iterations,
+					"Gap too large, giving up"
+				);
+				break;
 			}
 		}
 
-		events_with_auth_events.push((id.to_owned(), None, events_in_reverse_order));
+		Ok(discovered)
 	}
 
-	let mut pdus = Vec::with_capacity(events_with_auth_events.len());
-	for (id, local_pdu, events_in_reverse_order) in events_with_auth_events {
-		// a. Look in the main timeline (pduid_pdu tree)
-		// b. Look at outlier pdu tree
-		// (get_pdu_json checks both)
-		if let Some(local_pdu) = local_pdu {
-			trace!("Found {id} in main timeline or outlier tree");
-			pdus.push((local_pdu.clone(), None));
-		}
+	/// Find the event and auth it. Once the event is validated (steps 1 - 8)
+	/// it is appended to the outliers Tree.
+	///
+	/// Returns pdu and if we fetched it over federation the raw json.
+	///
+	/// a. Look in the main timeline (pduid_pdu tree)
+	/// b. Look at outlier pdu tree
+	/// c. Ask origin server over federation
+	/// d. TODO: Ask other servers over federation?
+	#[deprecated]
+	pub(super) async fn fetch_and_handle_outliers<'a, Pdu, Events>(
+		&self,
+		origin: &'a ServerName,
+		events: Events,
+		create_event: &'a Pdu,
+		room_id: &'a RoomId,
+	) -> Vec<(PduEvent, Option<BTreeMap<String, CanonicalJsonValue>>)>
+	where
+		Pdu: Event + Send + Sync,
+		Events: Iterator<Item = &'a EventId> + Clone + Send,
+	{
+		let back_off = |id| match self
+			.services
+			.globals
+			.bad_event_ratelimiter
+			.write()
+			.entry(id)
+		{
+			| hash_map::Entry::Vacant(e) => {
+				e.insert((Instant::now(), 1));
+			},
+			| hash_map::Entry::Occupied(mut e) => {
+				*e.get_mut() = (Instant::now(), e.get().1.saturating_add(1));
+			},
+		};
 
-		for (next_id, value) in events_in_reverse_order.into_iter().rev() {
-			if let Some((time, tries)) = self
-				.services
-				.globals
-				.bad_event_ratelimiter
-				.read()
-				.get(&*next_id)
-			{
-				// Exponential backoff
-				const MIN_DURATION: u64 = 5 * 60;
-				const MAX_DURATION: u64 = 60 * 60 * 24;
-				if continue_exponential_backoff_secs(
-					MIN_DURATION,
-					MAX_DURATION,
-					time.elapsed(),
-					*tries,
-				) {
-					debug!("Backing off from {next_id}");
+		let mut events_with_auth_events = Vec::with_capacity(events.clone().count());
+		trace!("Fetching {} outlier pdus", events.clone().count());
+
+		for id in events {
+			// a. Look in the main timeline (pduid_pdu tree)
+			// b. Look at outlier pdu tree
+			// (get_pdu_json checks both)
+			if let Ok(local_pdu) = self.services.timeline.get_pdu(id).await {
+				trace!("Found {id} in main timeline or outlier tree");
+				events_with_auth_events.push((id.to_owned(), Some(local_pdu), vec![]));
+				continue;
+			}
+
+			// c. Ask origin server over federation
+			// We also handle its auth chain here so we don't get a stack overflow in
+			// handle_outlier_pdu.
+			let mut todo_auth_events: VecDeque<_> = [id.to_owned()].into();
+			let mut events_in_reverse_order = Vec::with_capacity(todo_auth_events.len());
+
+			let mut events_all = HashSet::with_capacity(todo_auth_events.len());
+			while let Some(next_id) = todo_auth_events.pop_front() {
+				if let Some((time, tries)) = self
+					.services
+					.globals
+					.bad_event_ratelimiter
+					.read()
+					.get(&*next_id)
+				{
+					// Exponential backoff
+					const MIN_DURATION: u64 = 60 * 2;
+					const MAX_DURATION: u64 = 60 * 60 * 8;
+					if continue_exponential_backoff_secs(
+						MIN_DURATION,
+						MAX_DURATION,
+						time.elapsed(),
+						*tries,
+					) {
+						debug_warn!(
+							tried = ?*tries,
+							elapsed = ?time.elapsed(),
+							"Backing off from {next_id}",
+						);
+						continue;
+					}
+				}
+
+				if events_all.contains(&next_id) {
 					continue;
 				}
+
+				if self.services.timeline.pdu_exists(&next_id).await {
+					trace!("Found {next_id} in db");
+					continue;
+				}
+
+				debug!("Fetching {next_id} over federation from {origin}.");
+				match self
+					.services
+					.sending
+					.send_federation_request(
+						origin,
+						get_event::v1::Request::new((*next_id).to_owned()),
+					)
+					.await
+				{
+					| Ok(res) => {
+						debug!("Got {next_id} over federation from {origin}");
+						let Ok(room_version_rules) = get_room_version_rules(create_event) else {
+							back_off((*next_id).to_owned());
+							continue;
+						};
+
+						let Ok((calculated_event_id, value)) =
+							gen_event_id_canonical_json(&res.pdu, &room_version_rules)
+						else {
+							back_off((*next_id).to_owned());
+							continue;
+						};
+
+						if calculated_event_id != *next_id {
+							warn!(
+								"Server didn't return event id we requested: requested: \
+								 {next_id}, we got {calculated_event_id}. Event: {:?}",
+								&res.pdu
+							);
+						}
+
+						if let Some(auth_events) = value
+							.get("auth_events")
+							.and_then(CanonicalJsonValue::as_array)
+						{
+							for auth_event in auth_events {
+								match serde_json::from_value::<OwnedEventId>(
+									auth_event.clone().into(),
+								) {
+									| Ok(auth_event) => {
+										trace!(
+											"Found auth event id {auth_event} for event \
+											 {next_id}"
+										);
+										todo_auth_events.push_back(auth_event);
+									},
+									| _ => {
+										warn!("Auth event id is not valid");
+									},
+								}
+							}
+						} else {
+							warn!("Auth event list invalid");
+						}
+
+						events_in_reverse_order.push((next_id.clone(), value));
+						events_all.insert(next_id);
+					},
+					| Err(e) => {
+						warn!("Failed to fetch auth event {next_id} from {origin}: {e}");
+						back_off((*next_id).to_owned());
+					},
+				}
 			}
 
-			trace!("Handling outlier {next_id}");
-			match Box::pin(self.handle_outlier_pdu(
-				origin,
-				create_event,
-				&next_id,
-				room_id,
-				value.clone(),
-				true,
-			))
-			.await
-			{
-				| Ok((pdu, json)) =>
-					if next_id == *id {
-						trace!("Handled outlier {next_id} (original request)");
-						pdus.push((pdu, Some(json)));
+			events_with_auth_events.push((id.to_owned(), None, events_in_reverse_order));
+		}
+
+		let mut pdus = Vec::with_capacity(events_with_auth_events.len());
+		for (id, local_pdu, events_in_reverse_order) in events_with_auth_events {
+			// a. Look in the main timeline (pduid_pdu tree)
+			// b. Look at outlier pdu tree
+			// (get_pdu_json checks both)
+			if let Some(local_pdu) = local_pdu {
+				trace!("Found {id} in main timeline or outlier tree");
+				pdus.push((local_pdu.clone(), None));
+			}
+
+			for (next_id, value) in events_in_reverse_order.into_iter().rev() {
+				if let Some((time, tries)) = self
+					.services
+					.globals
+					.bad_event_ratelimiter
+					.read()
+					.get(&*next_id)
+				{
+					// Exponential backoff
+					const MIN_DURATION: u64 = 5 * 60;
+					const MAX_DURATION: u64 = 60 * 60 * 24;
+					if continue_exponential_backoff_secs(
+						MIN_DURATION,
+						MAX_DURATION,
+						time.elapsed(),
+						*tries,
+					) {
+						debug!("Backing off from {next_id}");
+						continue;
+					}
+				}
+
+				trace!("Handling outlier {next_id}");
+				match Box::pin(self.handle_outlier_pdu(
+					origin,
+					create_event,
+					&next_id,
+					room_id,
+					value.clone(),
+					true,
+				))
+				.await
+				{
+					| Ok((pdu, json)) =>
+						if next_id == *id {
+							trace!("Handled outlier {next_id} (original request)");
+							pdus.push((pdu, Some(json)));
+						},
+					| Err(e) => {
+						warn!("Authentication of event {next_id} failed: {e:?}");
+						back_off(next_id);
 					},
-				| Err(e) => {
-					warn!("Authentication of event {next_id} failed: {e:?}");
-					back_off(next_id);
-				},
+				}
 			}
 		}
+		trace!("Fetched and handled {} outlier pdus", pdus.len());
+		pdus
 	}
-	trace!("Fetched and handled {} outlier pdus", pdus.len());
-	pdus
 }

src/service/rooms/event_handler/fetch_prev.rs

@@ -1,128 +1,96 @@
-use std::{
-	collections::{BTreeMap, HashMap, HashSet, VecDeque},
-	iter::once,
-};
+use std::collections::HashMap;
 
 use conduwuit::{
-	Event, PduEvent, Result, debug_warn, err, implement,
-	state_res::{self},
-};
-use futures::{FutureExt, future};
-use ruma::{
-	CanonicalJsonValue, EventId, MilliSecondsSinceUnixEpoch, OwnedEventId, RoomId, ServerName,
-	int, uint,
+	Event, PduEvent, debug, debug_info,
+	utils::{BoolExt, IterStream, stream::BroadbandExt},
+	warn,
 };
+use futures::StreamExt;
+use ruma::{RoomId, ServerName};
 
-use super::check_room_id;
+use crate::rooms::event_handler::build_local_dag;
 
-#[implement(super::Service)]
-#[tracing::instrument(
-    level = "debug",
-	skip_all,
-	fields(%origin),
-)]
-#[allow(clippy::type_complexity)]
-pub(super) async fn fetch_prev<'a, Pdu, Events>(
-	&self,
-	origin: &ServerName,
-	create_event: &Pdu,
-	room_id: &RoomId,
-	first_ts_in_room: MilliSecondsSinceUnixEpoch,
-	initial_set: Events,
-) -> Result<(
-	Vec<OwnedEventId>,
-	HashMap<OwnedEventId, (PduEvent, BTreeMap<String, CanonicalJsonValue>)>,
-)>
-where
-	Pdu: Event + Send + Sync,
-	Events: Iterator<Item = &'a EventId> + Clone + Send,
-{
-	let num_ids = initial_set.clone().count();
-	let mut eventid_info = HashMap::new();
-	let mut graph: HashMap<OwnedEventId, _> = HashMap::with_capacity(num_ids);
-	let mut todo_outlier_stack: VecDeque<OwnedEventId> =
-		initial_set.map(ToOwned::to_owned).collect();
-
-	let mut amount = 0;
-
-	while let Some(prev_event_id) = todo_outlier_stack.pop_front() {
-		self.services.server.check_running()?;
-
-		match self
-			.fetch_and_handle_outliers(
-				origin,
-				once(prev_event_id.as_ref()),
-				create_event,
-				room_id,
-			)
-			.boxed()
-			.await
-			.pop()
-		{
-			| Some((pdu, mut json_opt)) => {
-				check_room_id(room_id, &pdu)?;
-
-				let limit = self.services.server.config.max_fetch_prev_events;
-				if amount > limit {
-					debug_warn!("Max prev event limit reached! Limit: {limit}");
-					graph.insert(prev_event_id.clone(), HashSet::new());
-					continue;
-				}
-
-				if json_opt.is_none() {
-					json_opt = self
-						.services
-						.outlier
-						.get_outlier_pdu_json(&prev_event_id)
-						.await
-						.ok();
-				}
-
-				if let Some(json) = json_opt {
-					if pdu.origin_server_ts() > first_ts_in_room {
-						amount = amount.saturating_add(1);
-						for prev_prev in pdu.prev_events() {
-							if !graph.contains_key(prev_prev) {
-								todo_outlier_stack.push_back(prev_prev.to_owned());
-							}
-						}
-
-						graph.insert(
-							prev_event_id.clone(),
-							pdu.prev_events().map(ToOwned::to_owned).collect(),
-						);
-					} else {
-						// Time based check failed
-						graph.insert(prev_event_id.clone(), HashSet::new());
-					}
-
-					eventid_info.insert(prev_event_id.clone(), (pdu, json));
-				} else {
-					// Get json failed, so this was not fetched over federation
-					graph.insert(prev_event_id.clone(), HashSet::new());
-				}
-			},
-			| _ => {
-				// Fetch and handle failed
-				graph.insert(prev_event_id.clone(), HashSet::new());
-			},
+impl super::Service {
+	/// Fetches any missing prev_events for this event and persists them before
+	/// returning.
+	pub(super) async fn fetch_prevs(
+		&self,
+		room_id: &RoomId,
+		create_event: &PduEvent,
+		incoming_pdu: &PduEvent,
+		origin: &ServerName,
+	) -> conduwuit::Result<()> {
+		let missing = incoming_pdu
+			.prev_events()
+			.stream()
+			.broad_filter_map(|event_id| async move {
+				self.services
+					.timeline
+					.get_non_outlier_pdu_json(event_id)
+					.await
+					.is_ok()
+					.or(|| event_id.to_owned())
+			})
+			.collect::<Vec<_>>()
+			.await;
+		if missing.is_empty() {
+			debug!(event_id=%incoming_pdu.event_id(), "No missing prev events.");
+			return Ok(());
 		}
+		debug!(%room_id, event_id=%incoming_pdu.event_id(), ?missing, "Fetching previous events");
+		let tail = self
+			.services
+			.state
+			.get_forward_extremities(room_id)
+			.collect::<Vec<_>>()
+			.await;
+
+		let backfilled = self
+			.get_missing_events(
+				room_id,
+				incoming_pdu,
+				tail,
+				origin,
+				self.services.metadata.get_mindepth(room_id).await,
+			)
+			.await?;
+		debug_info!("Fetched {} missing events", backfilled.len());
+
+		// Persist all fetched events
+		let mapped = backfilled
+			.iter()
+			.map(|(eid, evt)| {
+				let mut obj = evt.to_canonical_object();
+				obj.remove("event_id"); // event_id is inserted by backfill_missing_events
+				(eid.clone(), obj)
+			})
+			.collect::<HashMap<_, _>>();
+
+		let to_persist = if mapped.len() <= 1 {
+			mapped.keys().map(ToOwned::to_owned).collect()
+		} else {
+			build_local_dag(&mapped).await?
+		};
+
+		for event_id in to_persist {
+			debug_info!("Persisting fetched prev event {event_id}");
+			let obj = mapped.get(&event_id).cloned().unwrap();
+			match self
+				.handle_outlier_pdu(origin, create_event, &event_id, room_id, obj, false)
+				.await
+			{
+				| Ok((pdu, val)) =>
+					self.upgrade_outlier_to_timeline_pdu(pdu, val, create_event, origin, room_id)
+						.await,
+				| Err(e) => {
+					warn!("Failed to persist prev_event {event_id}: {e}");
+					continue;
+				},
+			}?;
+		}
+
+		// NOTE because i keep forgetting: the caller persists incoming_pdu.
+		// we only care about its prev events
+		Ok(())
 	}
-
-	let event_fetch = |event_id| {
-		let origin_server_ts = eventid_info
-			.get(&event_id)
-			.map_or_else(|| uint!(0), |info| info.0.origin_server_ts().get());
-
-		// This return value is the key used for sorting events,
-		// events are then sorted by power level, time,
-		// and lexically by event_id.
-		future::ok((int!(0), MilliSecondsSinceUnixEpoch(origin_server_ts)))
-	};
-
-	let sorted = state_res::lexicographical_topological_sort(&graph, &event_fetch)
-		.await
-		.map_err(|e| err!(Database(error!("Error sorting prev events: {e}"))))?;
-
-	Ok((sorted, eventid_info))
 }

src/service/rooms/event_handler/fetch_state.rs

@@ -1,7 +1,6 @@
 use std::collections::{HashMap, hash_map};
 
 use conduwuit::{Err, Event, Result, debug, debug_warn, err, implement};
-use futures::FutureExt;
 use ruma::{
 	EventId, OwnedEventId, RoomId, ServerName, api::federation::event::get_room_state_ids,
 	events::StateEventType,
@@ -42,7 +41,6 @@ where
 	let state_ids = res.pdu_ids.iter().map(AsRef::as_ref);
 	let state_vec = self
 		.fetch_and_handle_outliers(origin, state_ids, create_event, room_id)
-		.boxed()
 		.await;
 
 	let mut state: HashMap<ShortStateKey, OwnedEventId> = HashMap::with_capacity(state_vec.len());

src/service/rooms/event_handler/handle_incoming_pdu.rs

@@ -1,14 +1,11 @@
-use std::{
-	collections::{BTreeMap, hash_map},
-	time::Instant,
-};
+use std::{collections::BTreeMap, time::Instant};
 
 use conduwuit::{
 	Err, Event, PduEvent, Result, debug::INFO_SPAN_LEVEL, debug_error, debug_info, defer, err,
-	implement, info, trace, utils::stream::IterStream, warn,
+	implement, info, trace, warn,
 };
 use futures::{
-	FutureExt, TryFutureExt, TryStreamExt,
+	FutureExt,
 	future::{OptionFuture, try_join4},
 };
 use ruma::{
@@ -236,63 +233,21 @@ pub async fn handle_incoming_pdu<'a>(
 	}
 
 	// Skip old events
-	let first_ts_in_room = self
-		.services
-		.timeline
-		.first_pdu_in_room(room_id)
-		.await?
-		.origin_server_ts();
+	// let first_ts_in_room = self
+	// 	.services
+	// 	.timeline
+	// 	.first_pdu_in_room(room_id)
+	// 	.await?
+	// 	.origin_server_ts();
 
 	// 9. Fetch any missing prev events doing all checks listed here starting at 1.
 	//    These are timeline events
-	let (sorted_prev_events, mut eventid_info) = self
-		.fetch_prev(origin, create_event, room_id, first_ts_in_room, incoming_pdu.prev_events())
-		.await?;
+	debug!("Handling previous events");
 
-	debug!(
-		events = ?sorted_prev_events,
-		"Handling previous events"
-	);
-
-	sorted_prev_events
-		.iter()
-		.try_stream()
-		.map_ok(AsRef::as_ref)
-		.try_for_each(|prev_id| {
-			self.handle_prev_pdu(
-				origin,
-				event_id,
-				room_id,
-				eventid_info.remove(prev_id),
-				create_event,
-				first_ts_in_room,
-				prev_id,
-			)
-			.inspect_err(move |e| {
-				warn!("Prev {prev_id} failed: {e}");
-				match self
-					.services
-					.globals
-					.bad_event_ratelimiter
-					.write()
-					.entry(prev_id.into())
-				{
-					| hash_map::Entry::Vacant(e) => {
-						e.insert((Instant::now(), 1));
-					},
-					| hash_map::Entry::Occupied(mut e) => {
-						let tries = e.get().1.saturating_add(1);
-						*e.get_mut() = (Instant::now(), tries);
-					},
-				}
-			})
-			.map(|_| self.services.server.check_running())
-		})
-		.boxed()
+	self.fetch_prevs(room_id, create_event, &incoming_pdu, origin)
 		.await?;
 
 	// Done with prev events, now handling the incoming event
 	self.upgrade_outlier_to_timeline_pdu(incoming_pdu, val, create_event, origin, room_id)
-		.boxed()
 		.await
 }

src/service/rooms/event_handler/handle_outlier_pdu.rs

@@ -1,13 +1,13 @@
 use std::collections::{BTreeMap, HashMap, hash_map};
 
 use conduwuit::{
-	Err, Event, PduEvent, Result, debug, debug_info, debug_warn, err, implement, state_res,
+	Err, Event, PduEvent, Result, debug, debug_info, debug_warn, err, implement, info, state_res,
 	trace, warn,
 };
 use futures::future::ready;
 use ruma::{
 	CanonicalJsonObject, CanonicalJsonValue, EventId, OwnedEventId, RoomId, ServerName,
-	events::StateEventType,
+	api::federation::authorization::get_event_authorization, events::StateEventType,
 };
 
 use super::{check_room_id, get_room_version_rules};
@@ -22,7 +22,7 @@ pub(super) async fn handle_outlier_pdu<'a, Pdu>(
 	event_id: &'a EventId,
 	room_id: &'a RoomId,
 	mut value: CanonicalJsonObject,
-	auth_events_known: bool,
+	_auth_events_known: bool,
 ) -> Result<(PduEvent, BTreeMap<String, CanonicalJsonValue>)>
 where
 	Pdu: Event + Send + Sync,
@@ -107,45 +107,52 @@ where
 	}
 
 	// Fetch any missing ones & reject invalid ones
-	let missing_auth_events = if auth_events_known {
-		pdu_event
-			.auth_events()
-			.filter(|id| !auth_events.contains_key(*id))
-			.collect::<Vec<_>>()
-	} else {
-		pdu_event.auth_events().collect::<Vec<_>>()
-	};
-	if !missing_auth_events.is_empty() || !auth_events_known {
-		debug_info!(
-			"Fetching {} missing auth events for outlier event {event_id}",
-			missing_auth_events.len()
-		);
-		for (pdu, _) in self
-			.fetch_and_handle_outliers(
+	if auth_events.len() != pdu_event.auth_events().count() {
+		info!("Missing some auth events, asking remote for auth chain");
+		let response: get_event_authorization::v1::Response = self
+			.services
+			.sending
+			.send_federation_request(
 				origin,
-				missing_auth_events.iter().copied(),
-				create_event,
-				room_id,
+				get_event_authorization::v1::Request::new(
+					room_id.to_owned(),
+					event_id.to_owned(),
+				),
 			)
 			.await
-		{
-			auth_events.insert(pdu.event_id().to_owned(), pdu);
+			.map_err(|e| {
+				err!(Request(Forbidden(
+					"Remote server is not divulging incoming event's auth chain: {e}"
+				)))
+			})?;
+		let mut auth_chain_map = HashMap::with_capacity(response.auth_chain.len());
+		for auth_pdu_json in response.auth_chain {
+			let (auth_event_room_id, auth_event_id, auth_pdu_json) =
+				self.parse_incoming_pdu(&auth_pdu_json).await?;
+			if auth_event_room_id != room_id {
+				return Err!(Request(BadJson(
+					"Auth event {auth_event_id} is in {auth_event_room_id}, not {room_id}."
+				)));
+			}
+			let auth_pdu = PduEvent::from_id_val(&auth_event_id, auth_pdu_json)
+				.map_err(|e| err!(Request(BadJson("Invalid PDU {auth_event_id}: {e}"))))?;
+			auth_chain_map.insert(auth_event_id, auth_pdu);
 		}
-	} else {
-		debug!("No missing auth events for outlier event {event_id}");
-	}
-	// reject if we are still missing some
-	let still_missing = pdu_event
-		.auth_events()
-		.filter(|id| !auth_events.contains_key(*id))
-		.collect::<Vec<_>>();
-	if !still_missing.is_empty() {
-		// Don't reject: this could be a temporary condition
-		// TODO: use get_missing_events?
-		return Err!(Request(InvalidParam(
-			"Could not fetch all auth events for outlier event {event_id}, still missing: \
-			 {still_missing:?}"
-		)));
+		for aid in pdu_event.auth_events() {
+			if auth_events.contains_key(aid) {
+				continue;
+			}
+			if let Some(auth_event) = auth_chain_map.get(aid) {
+				auth_events.insert(aid.to_owned(), auth_event.clone());
+			} else {
+				return Err!(Request(Forbidden(
+					"Remote server is not divulging incoming event's auth events (missing: \
+					 {aid})"
+				)));
+			}
+		}
+		// TODO: do events received from auth chain need persisting? that sounds
+		// awfully slow
 	}
 
 	// 6. Reject "due to auth events" if the event doesn't pass auth based on the

src/service/rooms/event_handler/handle_prev_pdu.rs

@@ -1,89 +0,0 @@
-use std::{collections::BTreeMap, time::Instant};
-
-use conduwuit::{
-	Err, Event, PduEvent, Result, debug::INFO_SPAN_LEVEL, defer, implement,
-	utils::continue_exponential_backoff_secs,
-};
-use ruma::{CanonicalJsonValue, EventId, MilliSecondsSinceUnixEpoch, RoomId, ServerName};
-use tracing::debug;
-
-#[implement(super::Service)]
-#[allow(clippy::type_complexity)]
-#[allow(clippy::too_many_arguments)]
-#[tracing::instrument(
-	name = "prev",
-	level = INFO_SPAN_LEVEL,
-	skip_all,
-	fields(%prev_id),
-)]
-pub(super) async fn handle_prev_pdu<'a, Pdu>(
-	&self,
-	origin: &'a ServerName,
-	event_id: &'a EventId,
-	room_id: &'a RoomId,
-	eventid_info: Option<(PduEvent, BTreeMap<String, CanonicalJsonValue>)>,
-	create_event: &'a Pdu,
-	first_ts_in_room: MilliSecondsSinceUnixEpoch,
-	prev_id: &'a EventId,
-) -> Result
-where
-	Pdu: Event + Send + Sync,
-{
-	// Check for disabled again because it might have changed
-	if self.services.metadata.is_disabled(room_id).await {
-		return Err!(Request(Forbidden(debug_warn!(
-			"Federaton of room {room_id} is currently disabled on this server. Request by \
-			 origin {origin} and event ID {event_id}"
-		))));
-	}
-
-	if let Some((time, tries)) = self
-		.services
-		.globals
-		.bad_event_ratelimiter
-		.read()
-		.get(prev_id)
-	{
-		// Exponential backoff
-		const MIN_DURATION: u64 = 5 * 60;
-		const MAX_DURATION: u64 = 60 * 60 * 24;
-		if continue_exponential_backoff_secs(MIN_DURATION, MAX_DURATION, time.elapsed(), *tries) {
-			debug!(
-				?tries,
-				duration = ?time.elapsed(),
-				"Backing off from prev_event"
-			);
-			return Ok(());
-		}
-	}
-
-	let Some((pdu, json)) = eventid_info else {
-		return Ok(());
-	};
-
-	// Skip old events
-	if pdu.origin_server_ts() < first_ts_in_room {
-		return Ok(());
-	}
-
-	let start_time = Instant::now();
-	self.federation_handletime
-		.write()
-		.insert(room_id.into(), ((*prev_id).to_owned(), start_time));
-
-	defer! {{
-		self.federation_handletime
-			.write()
-			.remove(room_id);
-	}};
-
-	self.upgrade_outlier_to_timeline_pdu(pdu, json, create_event, origin, room_id)
-		.await?;
-
-	debug!(
-		elapsed = ?start_time.elapsed(),
-		"Handled prev_event",
-	);
-
-	Ok(())
-}

src/service/rooms/event_handler/mod.rs

@@ -4,7 +4,6 @@ mod fetch_prev;
 mod fetch_state;
 mod handle_incoming_pdu;
 mod handle_outlier_pdu;
-mod handle_prev_pdu;
 mod parse_incoming_pdu;
 mod policy_server;
 mod resolve_state;
@@ -15,6 +14,7 @@ use std::{collections::HashMap, fmt::Write, sync::Arc, time::Instant};
 
 use async_trait::async_trait;
 use conduwuit::{Err, Event, PduEvent, Result, Server, SyncRwLock, utils::MutexMap};
+pub use fetch_and_handle_outliers::build_local_dag;
 use ruma::{
 	OwnedEventId, OwnedRoomId, RoomId, events::room::create::RoomCreateEventContent,
 	room_version_rules::RoomVersionRules,
@@ -22,7 +22,6 @@ use ruma::{
 use tokio::sync::Notify;
 
 use crate::{Dep, globals, rooms, sending, server_keys};
-
 pub struct Service {
 	pub mutex_federation: RoomMutexMap,
 	pub federation_handletime: SyncRwLock<HandleTimeMap>,

src/service/rooms/event_handler/parse_incoming_pdu.rs

@@ -56,7 +56,10 @@ fn extract_room_id(event_type: &str, pdu: &CanonicalJsonObject) -> Result<OwnedR
 
 /// Parses every entry in an array as an event ID, returning an error if any
 /// step fails.
-fn expect_event_id_array(value: &CanonicalJsonObject, field: &str) -> Result<Vec<OwnedEventId>> {
+pub(super) fn expect_event_id_array(
+	value: &CanonicalJsonObject,
+	field: &str,
+) -> Result<Vec<OwnedEventId>> {
 	value
 		.get(field)
 		.ok_or_else(|| err!(Request(BadJson("missing field `{field}` on PDU"))))?

src/service/rooms/event_handler/state_at_incoming.rs

@@ -5,7 +5,7 @@ use std::{
 };
 
 use conduwuit::{
-	Result, debug, err, error, implement,
+	Result, debug, debug_error, err, error, implement,
 	matrix::{Event, StateMap},
 	trace,
 	utils::stream::{BroadbandExt, IterStream, ReadyExt, TryBroadbandExt, TryWidebandExt},
@@ -37,6 +37,7 @@ where
 		.pdu_shortstatehash(prev_event)
 		.await
 	else {
+		trace!("No shortstatehash for {prev_event}, cannot calculate one-degree state.");
 		return Ok(None);
 	};
 
@@ -99,6 +100,7 @@ where
 				.map_ok(move |sstatehash| (sstatehash, prev_event))
 		})
 		.try_collect::<HashMap<_, _>>()
+		.inspect_err(|e| debug_error!("failed to calculate N-degree short state hashes: {e}"))
 		.await
 	else {
 		return Ok(None);

src/service/rooms/event_handler/upgrade_outlier_pdu.rs

@@ -41,6 +41,7 @@ where
 		.get_pdu_id(incoming_pdu.event_id())
 		.await
 	{
+		trace!(event_id=%incoming_pdu.event_id(), "Skipping upgrade of already upgraded PDU");
 		return Ok(Some(pduid));
 	}
 
@@ -63,6 +64,7 @@ where
 		"Upgrading PDU from outlier to timeline"
 	);
 	let timer = Instant::now();
+	let min_depth = self.services.metadata.get_mindepth(room_id).await;
 	let room_version_rules = get_room_version_rules(create_event)?;
 
 	// 10. Fetch missing state and auth chain events by calling /state_ids at
@@ -81,6 +83,7 @@ where
 	};
 
 	if state_at_incoming_event.is_none() {
+		trace!("Could not calculate incoming state, asking remote {origin} for it");
 		state_at_incoming_event = self
 			.fetch_state(origin, create_event, room_id, incoming_pdu.event_id())
 			.await?;
@@ -382,6 +385,11 @@ where
 
 	// Event has passed all auth/stateres checks
 	drop(state_lock);
+	if incoming_pdu.depth > min_depth {
+		self.services
+			.metadata
+			.set_mindepth(room_id, incoming_pdu.depth.into());
+	}
 
 	Ok(pdu_id)
 }

src/service/rooms/membership/mod.rs

@@ -626,6 +626,10 @@ impl Service {
 				room_id,
 			)
 			.await?;
+		self.services
+			.metadata
+			.maybe_set_mindepth(room_id, parsed_join_pdu.depth.into())
+			.await;
 
 		info!("Setting final room state for new room");
 		// We set the room state after inserting the pdu, so that we never have a moment

src/service/rooms/metadata/mod.rs

@@ -1,9 +1,9 @@
 use std::sync::Arc;
 
 use conduwuit::{Result, implement, utils::stream::TryIgnore};
-use database::Map;
+use database::{Deserialized, Map};
 use futures::{Stream, StreamExt};
-use ruma::{OwnedRoomId, RoomId};
+use ruma::{OwnedRoomId, RoomId, UInt, uint};
 
 use crate::{Dep, rooms};
 
@@ -17,6 +17,7 @@ struct Data {
 	bannedroomids: Arc<Map>,
 	roomid_shortroomid: Arc<Map>,
 	pduid_pdu: Arc<Map>,
+	roomid_mindepth: Arc<Map>,
 }
 
 struct Services {
@@ -31,6 +32,7 @@ impl crate::Service for Service {
 				bannedroomids: args.db["bannedroomids"].clone(),
 				roomid_shortroomid: args.db["roomid_shortroomid"].clone(),
 				pduid_pdu: args.db["pduid_pdu"].clone(),
+				roomid_mindepth: args.db["roomid_mindepth"].clone(),
 			},
 			services: Services {
 				short: args.depend::<rooms::short::Service>("rooms::short"),
@@ -98,3 +100,27 @@ pub async fn is_disabled(&self, room_id: &RoomId) -> bool {
 pub async fn is_banned(&self, room_id: &RoomId) -> bool {
 	self.db.bannedroomids.get(room_id).await.is_ok()
 }
+
+#[implement(Service)]
+pub async fn get_mindepth(&self, room_id: &RoomId) -> UInt {
+	self.db
+		.roomid_mindepth
+		.get(room_id)
+		.await
+		.deserialized::<UInt>()
+		.unwrap_or_else(|_| uint!(0))
+}
+
+#[implement(Service)]
+pub fn set_mindepth(&self, room_id: &RoomId, min_depth: u64) {
+	self.db
+		.roomid_mindepth
+		.put_raw(room_id.as_bytes(), min_depth.to_be_bytes());
+}
+
+#[implement(Service)]
+pub async fn maybe_set_mindepth(&self, room_id: &RoomId, min_depth: u64) {
+	if min_depth > self.get_mindepth(room_id).await.into() {
+		self.set_mindepth(room_id, min_depth);
+	}
+}

src/service/rooms/timeline/mod.rs

@@ -173,6 +173,10 @@ impl Service {
 		self.db.get_non_outlier_pdu_json(event_id).await
 	}
 
+	pub async fn non_outlier_pdu_exists(&self, event_id: &EventId) -> bool {
+		self.db.non_outlier_pdu_exists(event_id).await.is_ok()
+	}
+
 	/// Returns the pdu's id.
 	#[inline]
 	pub async fn get_pdu_id(&self, event_id: &EventId) -> Result<RawPduId> {

src/service/server_keys/request.rs

@@ -34,8 +34,9 @@ where
 
 		batch
 	});
-
-	debug_assert!(!server_keys.is_empty(), "empty batch request to notary");
+	if server_keys.is_empty() {
+		return Ok(vec![]);
+	}
 
 	let mut results = Vec::new();
 	while let Some(batch) = server_keys