perf(wip): Improve individual events fetcher

src/service/rooms/event_handler/fetch_state.rs

@@ -1,12 +1,13 @@
 use std::collections::{HashMap, hash_map};
 
-use conduwuit::{Err, Event, Result, debug, debug_warn, err};
+use conduwuit::{Err, Event, PduEvent, Result, debug, debug_warn, err, utils::IterStream};
+use futures::StreamExt;
 use ruma::{
 	EventId, OwnedEventId, RoomId, ServerName, api::federation::event::get_room_state_ids,
 	events::StateEventType,
 };
 
-use crate::rooms::short::ShortStateKey;
+use crate::{conduwuit::utils::stream::BroadbandExt, rooms::short::ShortStateKey};
 
 impl super::Service {
 	/// Call /state_ids to find out what the state at this pdu is. We trust the
@@ -27,7 +28,7 @@ impl super::Service {
 	where
 		Pdu: Event + Send + Sync,
 	{
-		let res = self
+		let res: get_room_state_ids::v1::Response = self
 			.services
 			.sending
 			.send_federation_request(
@@ -38,17 +39,39 @@ impl super::Service {
 			.inspect_err(|e| debug_warn!("Fetching state for event failed: {e}"))?;
 
 		debug!("Fetching state events");
-		let state_ids = res.pdu_ids.iter().map(AsRef::as_ref);
-		let state_vec = self
-			.fetch_and_handle_outliers(origin, state_ids, create_event, room_id)
+		let mut state_events: HashMap<OwnedEventId, PduEvent> =
+			HashMap::with_capacity(res.pdu_ids.len());
+		let to_fetch: Vec<OwnedEventId> = res
+			.pdu_ids
+			.clone()
+			.into_iter()
+			.stream()
+			.broad_filter_map(|event_id| async move {
+				if self.services.timeline.pdu_exists(&event_id).await {
+					None
+				} else {
+					Some(event_id)
+				}
+			})
+			.collect()
 			.await;
+		if !to_fetch.is_empty() {
+			if to_fetch.len() >= 100 {
+				// That's a lot of events to fetch, just ask for the full state
+				// at that point. TODO: fetch /state
+			}
+			state_events.extend(
+				self.fetch_and_handle_missing_events(origin, to_fetch, create_event, room_id)
+					.await,
+			);
+		}
 
 		let mut state: HashMap<ShortStateKey, OwnedEventId> =
-			HashMap::with_capacity(state_vec.len());
-		for (pdu, _) in state_vec {
-			let state_key = pdu
-				.state_key()
-				.ok_or_else(|| err!(Database("Found non-state pdu in state events.")))?;
+			HashMap::with_capacity(state_events.len());
+		for (event_id, pdu) in state_events {
+			let state_key = pdu.state_key().ok_or_else(|| {
+				err!(Database("Found non-state pdu in state events: {event_id}"))
+			})?;
 
 			let shortstatekey = self
 				.services
@@ -62,8 +85,8 @@ impl super::Service {
 				},
 				| hash_map::Entry::Occupied(_) => {
 					return Err!(Database(
-						"State event's type and state_key combination exists multiple times: \
-						 {}, {}",
+						"State event's type and state_key combination exists multiple times \
+						 ({event_id}): {}, {}",
 						pdu.kind(),
 						state_key
 					));