From a1e89eb04dde2f0e2360ef72a38121b60eddd5c5 Mon Sep 17 00:00:00 2001 From: Jacob Taylor Date: Wed, 18 Nov 2026 20:47:48 -0800 Subject: [PATCH] feat: MSC3030 --- changelog.d/+timestamp-to-event.feature.md | 1 + src/api/client/room/mod.rs | 4 +- src/api/client/room/timestamp_to_event.rs | 61 ++++++++++++++ src/api/router.rs | 1 + src/database/maps.rs | 4 + src/service/migrations.rs | 72 +++++++++++++++- src/service/rooms/timeline/append.rs | 2 +- src/service/rooms/timeline/backfill.rs | 3 +- src/service/rooms/timeline/data.rs | 96 +++++++++++++++++++++- src/service/rooms/timeline/mod.rs | 20 ++++- 10 files changed, 254 insertions(+), 10 deletions(-) create mode 100644 changelog.d/+timestamp-to-event.feature.md create mode 100644 src/api/client/room/timestamp_to_event.rs diff --git a/changelog.d/+timestamp-to-event.feature.md b/changelog.d/+timestamp-to-event.feature.md new file mode 100644 index 000000000..c1f903968 --- /dev/null +++ b/changelog.d/+timestamp-to-event.feature.md @@ -0,0 +1 @@ +Added the `GET /_matrix/client/v1/rooms/{roomId}/timestamp_to_event` endpoint as per MSC3030, letting clients find the event closest to a given timestamp. Contributed by @eleboucher. diff --git a/src/api/client/room/mod.rs b/src/api/client/room/mod.rs index 1d3125d3e..78487f8da 100644 --- a/src/api/client/room/mod.rs +++ b/src/api/client/room/mod.rs @@ -4,10 +4,12 @@ mod event; mod initial_sync; mod stripped_state; mod summary; +mod timestamp_to_event; mod upgrade; pub(crate) use self::{ aliases::get_room_aliases_route, create::create_room_route, event::get_room_event_route, initial_sync::room_initial_sync_route, stripped_state::get_stripped_state_route, - summary::get_room_summary, upgrade::upgrade_room_route, + summary::get_room_summary, timestamp_to_event::get_event_by_timestamp_route, + upgrade::upgrade_room_route, }; diff --git a/src/api/client/room/timestamp_to_event.rs b/src/api/client/room/timestamp_to_event.rs new file mode 100644 index 000000000..0ce077a9b --- /dev/null +++ b/src/api/client/room/timestamp_to_event.rs @@ -0,0 +1,61 @@ +use axum::extract::State; +use conduwuit::{Err, Event, Result, utils::stream::TryIgnore}; +use futures::{StreamExt, pin_mut}; +use ruma::api::client::room::get_event_by_timestamp; + +use crate::Ruma; + +const MAX_CANDIDATES: usize = 20; + +/// # `GET /_matrix/client/v1/rooms/{roomId}/timestamp_to_event` +/// +/// Gets the ID of the event closest to the given timestamp, in the direction +/// specified by `dir`. +pub(crate) async fn get_event_by_timestamp_route( + State(ref services): State, + body: Ruma, +) -> Result { + let sender_user = body.identity.expect_sender_user()?; + let room_id = &body.room_id; + + if !services + .rooms + .state_accessor + .user_can_see_state_events(sender_user, room_id) + .await + { + return Err!(Request(Forbidden("You don't have permission to view this room."))); + } + + if !services.rooms.timeline.timestamp_index_ready().await { + return Err!(FeatureDisabled("timestamp_to_event")); + } + + let candidates = services + .rooms + .timeline + .pdus_by_timestamp(room_id, body.ts, body.dir) + .ignore_err() + .take(MAX_CANDIDATES); + + pin_mut!(candidates); + while let Some((_, pdu)) = candidates.next().await { + if services + .rooms + .state_accessor + .user_can_see_event(sender_user, room_id, pdu.event_id()) + .await + { + return Ok(get_event_by_timestamp::v1::Response::new( + pdu.event_id().to_owned(), + pdu.origin_server_ts(), + )); + } + } + + Err!(Request(NotFound( + "Unable to find event from {} in direction {:?}", + body.ts.get(), + body.dir + ))) +} diff --git a/src/api/router.rs b/src/api/router.rs index c6cec4a59..e966939ba 100644 --- a/src/api/router.rs +++ b/src/api/router.rs @@ -52,6 +52,7 @@ pub fn build(router: Router, state: State) -> Router { .ruma_route(&client::set_pushrule_actions_route) .ruma_route(&client::delete_pushrule_route) .ruma_route(&client::get_room_event_route) + .ruma_route(&client::get_event_by_timestamp_route) .ruma_route(&client::get_room_aliases_route) .ruma_route(&client::get_filter_route) .ruma_route(&client::create_filter_route) diff --git a/src/database/maps.rs b/src/database/maps.rs index 183355d06..35bc75f5e 100644 --- a/src/database/maps.rs +++ b/src/database/maps.rs @@ -232,6 +232,10 @@ pub(super) static MAPS: &[Descriptor] = &[ name: "roomsynctoken_shortstatehash", ..descriptor::DROPPED }, + Descriptor { + name: "roomts_pduid", + ..descriptor::SEQUENTIAL_SMALL + }, Descriptor { name: "roomuserdataid_accountdata", ..descriptor::RANDOM_SMALL diff --git a/src/service/migrations.rs b/src/service/migrations.rs index 666994d9e..ac4825d8d 100644 --- a/src/service/migrations.rs +++ b/src/service/migrations.rs @@ -18,11 +18,11 @@ use conduwuit::{ }, warn, }; -use database::{Deserialized, Ignore, Json}; +use database::{Database, Deserialized, Ignore, Json}; use futures::{FutureExt, StreamExt, TryStreamExt}; use itertools::Itertools; use ruma::{ - OwnedDeviceId, OwnedEventId, OwnedRoomId, OwnedUserId, UserId, + MilliSecondsSinceUnixEpoch, OwnedDeviceId, OwnedEventId, OwnedRoomId, OwnedUserId, UserId, api::OAuthClientScope, events::{ AnyStrippedStateEvent, GlobalAccountDataEventType, StateEventType, @@ -32,9 +32,13 @@ use ruma::{ push::Ruleset, serde::{Raw, from_raw_json_value}, }; +use serde::Deserialize; use serde_json::{Value, value::to_raw_value}; -use crate::{Services, media, rooms, rooms::short::ShortStateHash}; +use crate::{ + Services, media, rooms, + rooms::{short::ShortStateHash, timeline::RawPduId}, +}; type MigrationFn = for<'a> fn(&'a Services) -> Pin> + Send + 'a>>; @@ -176,6 +180,36 @@ migration!(stringify!($name) => $name) // This is backwards compatible because old versions never read the // account data entries it removes. migration!(remove_replaced_account_data), + // This is backwards compatible because old versions ignore the + // `roomts_pduid` keyspace. + Migration(rooms::timeline::TIMESTAMP_INDEX_MIGRATION, |services| { + Box::pin(async { + let global = &services.db["global"]; + if global + .exists(rooms::timeline::TIMESTAMP_INDEX_MIGRATION) + .await + .is_ok() + { + return Ok(()); + } + + let until = match global.get(ROOMTS_UNTIL).await.deserialized::() { + | Ok(until) => until, + | Err(_) => { + let until = services.globals.current_count()?; + global.insert(ROOMTS_UNTIL, until.to_be_bytes()); + until + }, + }; + + services + .server + .runtime() + .spawn(populate_roomts_pduid(services.db.clone(), until)); + + Ok(()) + }) + }), ] }; @@ -961,6 +995,38 @@ async fn drop_server_signingkeys(services: &Services) -> Result<()> { Ok(()) } +const ROOMTS_UNTIL: &str = "populate_roomts_pduid_until"; + +async fn populate_roomts_pduid(db: Arc, until: u64) { + #[derive(Deserialize)] + struct ExtractTs { + origin_server_ts: MilliSecondsSinceUnixEpoch, + } + + info!("Starting background migration to index PDUs by timestamp."); + let start = Instant::now(); + let roomts_pduid = db["roomts_pduid"].clone(); + + db["pduid_pdu"] + .raw_stream() + .ignore_err() + .ready_for_each(|(key, val)| { + let pdu_id = RawPduId::from(key); + if matches!(pdu_id.pdu_count(), PduCount::Normal(count) if count > until) { + return; + } + if let Ok(pdu) = serde_json::from_slice::(val) { + roomts_pduid + .insert(&rooms::timeline::roomts_key(&pdu_id, pdu.origin_server_ts), []); + } + }) + .await; + + db["global"].insert(rooms::timeline::TIMESTAMP_INDEX_MIGRATION, []); + db["global"].remove(ROOMTS_UNTIL); + info!(elapsed = ?start.elapsed(), "Finished indexing PDUs by timestamp."); +} + async fn fix_oauth_scope_names(services: &Services) -> Result { let db = &services.db; let userdeviceid_oauthsessioninfo = db["userdeviceid_oauthsessioninfo"].clone(); diff --git a/src/service/rooms/timeline/append.rs b/src/service/rooms/timeline/append.rs index c4b66e866..887dca516 100644 --- a/src/service/rooms/timeline/append.rs +++ b/src/service/rooms/timeline/append.rs @@ -349,7 +349,7 @@ impl super::Service { // Insert pdu self.db - .append_pdu(&pdu_id, pdu, room_id, &pdu_json, count2) + .append_pdu(&pdu_id, pdu, room_id, &pdu_json, count2, pdu.origin_server_ts()) .await; self.progress_extremities(pdu) .await diff --git a/src/service/rooms/timeline/backfill.rs b/src/service/rooms/timeline/backfill.rs index 85efcd5a1..ff78e9ff9 100644 --- a/src/service/rooms/timeline/backfill.rs +++ b/src/service/rooms/timeline/backfill.rs @@ -225,7 +225,8 @@ impl super::Service { .into(); // Insert pdu - self.db.prepend_backfill_pdu(&pdu_id, &event_id, &value); + self.db + .prepend_backfill_pdu(&pdu_id, &event_id, &value, pdu.origin_server_ts()); drop(insert_lock); if pdu.kind == TimelineEventType::RoomMessage { diff --git a/src/service/rooms/timeline/data.rs b/src/service/rooms/timeline/data.rs index f416582fc..f0184533b 100644 --- a/src/service/rooms/timeline/data.rs +++ b/src/service/rooms/timeline/data.rs @@ -12,7 +12,9 @@ use database::{Database, Deserialized, Json, KeyVal, Map}; use futures::{ FutureExt, Stream, StreamExt, TryFutureExt, TryStreamExt, future::select_ok, pin_mut, }; -use ruma::{CanonicalJsonObject, EventId, OwnedUserId, RoomId, api::Direction}; +use ruma::{ + CanonicalJsonObject, EventId, MilliSecondsSinceUnixEpoch, OwnedUserId, RoomId, api::Direction, +}; use serde::{Deserialize, Serialize}; use serde_json::value::RawValue as RawJsonValue; @@ -25,6 +27,7 @@ pub(super) struct Data { pduid_pdu: Arc, pduid_unredactedcontent: Arc, roomid_lasttimelinecount: Arc, + roomts_pduid: Arc, userroomid_highlightcount: Arc, userroomid_notificationcount: Arc, pub(super) db: Arc, @@ -52,6 +55,7 @@ impl Data { pduid_pdu: db["pduid_pdu"].clone(), pduid_unredactedcontent: db["pduid_unredactedcontent"].clone(), roomid_lasttimelinecount: db["roomid_lasttimelinecount"].clone(), + roomts_pduid: db["roomts_pduid"].clone(), userroomid_highlightcount: db["userroomid_highlightcount"].clone(), userroomid_notificationcount: db["userroomid_notificationcount"].clone(), db: args.db.clone(), @@ -213,10 +217,12 @@ impl Data { room_id: &RoomId, json: &CanonicalJsonObject, count: PduCount, + index_ts: MilliSecondsSinceUnixEpoch, ) { debug_assert!(matches!(count, PduCount::Normal(_)), "PduCount not Normal"); self.pduid_pdu.raw_put(pdu_id, Json(json)); + self.roomts_pduid.insert(&roomts_key(pdu_id, index_ts), []); self.eventid_pduid.insert(pdu.event_id.as_bytes(), pdu_id); self.eventid_outlierpdu.remove(pdu.event_id.as_bytes()); self.roomid_lasttimelinecount @@ -228,8 +234,10 @@ impl Data { pdu_id: &RawPduId, event_id: &EventId, json: &CanonicalJsonObject, + index_ts: MilliSecondsSinceUnixEpoch, ) { self.pduid_pdu.raw_put(pdu_id, Json(json)); + self.roomts_pduid.insert(&roomts_key(pdu_id, index_ts), []); self.eventid_pduid.insert(event_id, pdu_id); self.eventid_outlierpdu.remove(event_id); } @@ -322,6 +330,49 @@ impl Data { .try_flatten_stream() } + pub(super) fn pdus_by_timestamp<'a>( + &'a self, + room_id: &'a RoomId, + timestamp: MilliSecondsSinceUnixEpoch, + dir: Direction, + ) -> impl Stream> + Send + 'a { + self.services + .short + .get_shortroomid(room_id) + .map_err(move |e| err!(Request(NotFound("Room {room_id:?} not found: {e:?}")))) + .map_ok(move |shortroomid| { + let seek = |shorteventid| { + roomts_key(&PduId { shortroomid, shorteventid }.into(), timestamp) + }; + let keys = match dir { + | Direction::Forward => self + .roomts_pduid + .raw_keys_from(&seek(PduCount::min())) + .left_stream(), + | Direction::Backward => self + .roomts_pduid + .rev_raw_keys_from(&seek(PduCount::max())) + .right_stream(), + }; + + keys.ready_try_take_while(move |key| { + Ok(key.starts_with(&shortroomid.to_be_bytes())) + }) + .map_ok(roomts_key_to_pdu_id) + .and_then(async move |pdu_id| { + Ok((pdu_id.pdu_count(), self.get_pdu_from_id(&pdu_id).await?)) + }) + }) + .try_flatten_stream() + } + + pub(super) async fn timestamp_index_ready(&self) -> bool { + self.db["global"] + .exists(TIMESTAMP_INDEX_MIGRATION) + .await + .is_ok() + } + fn from_json_slice((pdu_id, pdu): KeyVal<'_>) -> Result { let pdu_id: RawPduId = pdu_id.into(); @@ -376,9 +427,52 @@ impl Data { } } +pub const TIMESTAMP_INDEX_MIGRATION: &str = "populate_roomts_pduid"; + +#[must_use] +pub fn roomts_key(pdu_id: &RawPduId, ts: MilliSecondsSinceUnixEpoch) -> Vec { + let count = pdu_id.pdu_count().into_signed().cast_unsigned() ^ (1 << 63); + [pdu_id.shortroomid(), u64::from(ts.get()).to_be_bytes(), count.to_be_bytes()].concat() +} + +fn roomts_key_to_pdu_id(key: &[u8]) -> RawPduId { + let shortroomid = u64::from_be_bytes(key[0..8].try_into().expect("roomts key shortroomid")); + let count = u64::from_be_bytes(key[16..24].try_into().expect("roomts key count")) ^ (1 << 63); + + PduId { + shortroomid, + shorteventid: PduCount::from_signed(count.cast_signed()), + } + .into() +} + //TODO: this is an ABA fn increment(db: &Arc, key: &[u8]) { let old = db.get_blocking(key); let new = utils::increment(old.ok().as_deref()); db.insert(key, new); } + +#[cfg(test)] +mod tests { + use ruma::uint; + + use super::*; + + #[test] + fn roomts_key_orders_by_ts_then_count() { + let ids = [ + (PduCount::Normal(9), uint!(1)), + (PduCount::Backfilled(-5), uint!(2)), + (PduCount::Normal(3), uint!(2)), + (PduCount::Normal(1), uint!(3)), + ] + .map(|(count, ts)| (RawPduId::from(PduId { shortroomid: 7, shorteventid: count }), ts)); + let keys = ids.map(|(id, ts)| roomts_key(&id, MilliSecondsSinceUnixEpoch(ts))); + + assert!(keys.is_sorted()); + for ((id, _), key) in ids.iter().zip(&keys) { + assert_eq!(roomts_key_to_pdu_id(key), *id); + } + } +} diff --git a/src/service/rooms/timeline/mod.rs b/src/service/rooms/timeline/mod.rs index 6c206d9bf..15774c2e5 100644 --- a/src/service/rooms/timeline/mod.rs +++ b/src/service/rooms/timeline/mod.rs @@ -21,8 +21,8 @@ use conduwuit_core::{ }; use futures::{Future, Stream, TryStreamExt, pin_mut}; use ruma::{ - CanonicalJsonObject, EventId, OwnedEventId, OwnedRoomId, RoomId, - events::room::encrypted::Relation, + CanonicalJsonObject, EventId, MilliSecondsSinceUnixEpoch, OwnedEventId, OwnedRoomId, RoomId, + api::Direction, events::room::encrypted::Relation, }; use serde::Deserialize; use serde_json::value::RawValue as RawJsonValue; @@ -32,7 +32,10 @@ use tokio::{ }; use self::data::Data; -pub use self::{create::pdu_fits, data::PdusIterItem}; +pub use self::{ + create::pdu_fits, + data::{PdusIterItem, TIMESTAMP_INDEX_MIGRATION, roomts_key}, +}; use crate::{ Dep, account_data, admin, appservice, config, globals, pusher, rooms, sending, server_keys, sync, users, @@ -307,6 +310,17 @@ impl Service { self.pdus(room_id, None).ignore_err() } + pub fn pdus_by_timestamp<'a>( + &'a self, + room_id: &'a RoomId, + timestamp: MilliSecondsSinceUnixEpoch, + dir: Direction, + ) -> impl Stream> + Send + 'a { + self.db.pdus_by_timestamp(room_id, timestamp, dir) + } + + pub async fn timestamp_index_ready(&self) -> bool { self.db.timestamp_index_ready().await } + /// Reverse iteration starting after `until`. #[tracing::instrument(skip(self), level = "debug")] pub fn pdus_rev<'a>(