feat: MSC3030

This commit is contained in:
Jacob Taylor committed 2026-11-18 20:47:48 -08:00
1 parent 8db6f7a2e2
commit a1e89eb04d
10 files changed
+254 -10

No files matched your search

@@ -0,0 +1 @@
Added the `GET /_matrix/client/v1/rooms/{roomId}/timestamp_to_event` endpoint as per MSC3030, letting clients find the event closest to a given timestamp. Contributed by @eleboucher.
+3 -1
View File
@@ -4,10 +4,12 @@ mod event;
mod initial_sync;
mod stripped_state;
mod summary;
mod timestamp_to_event;
mod upgrade;
pub(crate) use self::{
aliases::get_room_aliases_route, create::create_room_route, event::get_room_event_route,
initial_sync::room_initial_sync_route, stripped_state::get_stripped_state_route,
summary::get_room_summary, upgrade::upgrade_room_route,
summary::get_room_summary, timestamp_to_event::get_event_by_timestamp_route,
upgrade::upgrade_room_route,
};
+61
View File
@@ -0,0 +1,61 @@
use axum::extract::State;
use conduwuit::{Err, Event, Result, utils::stream::TryIgnore};
use futures::{StreamExt, pin_mut};
use ruma::api::client::room::get_event_by_timestamp;
use crate::Ruma;
const MAX_CANDIDATES: usize = 20;
/// # `GET /_matrix/client/v1/rooms/{roomId}/timestamp_to_event`
///
/// Gets the ID of the event closest to the given timestamp, in the direction
/// specified by `dir`.
pub(crate) async fn get_event_by_timestamp_route(
State(ref services): State<crate::State>,
body: Ruma<get_event_by_timestamp::v1::Request>,
) -> Result<get_event_by_timestamp::v1::Response> {
let sender_user = body.identity.expect_sender_user()?;
let room_id = &body.room_id;
if !services
.rooms
.state_accessor
.user_can_see_state_events(sender_user, room_id)
.await
{
return Err!(Request(Forbidden("You don't have permission to view this room.")));
}
if !services.rooms.timeline.timestamp_index_ready().await {
return Err!(FeatureDisabled("timestamp_to_event"));
}
let candidates = services
.rooms
.timeline
.pdus_by_timestamp(room_id, body.ts, body.dir)
.ignore_err()
.take(MAX_CANDIDATES);
pin_mut!(candidates);
while let Some((_, pdu)) = candidates.next().await {
if services
.rooms
.state_accessor
.user_can_see_event(sender_user, room_id, pdu.event_id())
.await
{
return Ok(get_event_by_timestamp::v1::Response::new(
pdu.event_id().to_owned(),
pdu.origin_server_ts(),
));
}
}
Err!(Request(NotFound(
"Unable to find event from {} in direction {:?}",
body.ts.get(),
body.dir
)))
}
+1
View File
@@ -52,6 +52,7 @@ pub fn build(router: Router<State>, state: State) -> Router<State> {
.ruma_route(&client::set_pushrule_actions_route)
.ruma_route(&client::delete_pushrule_route)
.ruma_route(&client::get_room_event_route)
.ruma_route(&client::get_event_by_timestamp_route)
.ruma_route(&client::get_room_aliases_route)
.ruma_route(&client::get_filter_route)
.ruma_route(&client::create_filter_route)
+4
View File
@@ -232,6 +232,10 @@ pub(super) static MAPS: &[Descriptor] = &[
name: "roomsynctoken_shortstatehash",
..descriptor::DROPPED
},
Descriptor {
name: "roomts_pduid",
..descriptor::SEQUENTIAL_SMALL
},
Descriptor {
name: "roomuserdataid_accountdata",
..descriptor::RANDOM_SMALL
+69 -3
View File
@@ -18,11 +18,11 @@ use conduwuit::{
},
warn,
};
use database::{Deserialized, Ignore, Json};
use database::{Database, Deserialized, Ignore, Json};
use futures::{FutureExt, StreamExt, TryStreamExt};
use itertools::Itertools;
use ruma::{
OwnedDeviceId, OwnedEventId, OwnedRoomId, OwnedUserId, UserId,
MilliSecondsSinceUnixEpoch, OwnedDeviceId, OwnedEventId, OwnedRoomId, OwnedUserId, UserId,
api::OAuthClientScope,
events::{
AnyStrippedStateEvent, GlobalAccountDataEventType, StateEventType,
@@ -32,9 +32,13 @@ use ruma::{
push::Ruleset,
serde::{Raw, from_raw_json_value},
};
use serde::Deserialize;
use serde_json::{Value, value::to_raw_value};
use crate::{Services, media, rooms, rooms::short::ShortStateHash};
use crate::{
Services, media, rooms,
rooms::{short::ShortStateHash, timeline::RawPduId},
};
type MigrationFn =
for<'a> fn(&'a Services) -> Pin<Box<dyn Future<Output = Result<()>> + Send + 'a>>;
@@ -176,6 +180,36 @@ migration!(stringify!($name) => $name)
// This is backwards compatible because old versions never read the
// account data entries it removes.
migration!(remove_replaced_account_data),
// This is backwards compatible because old versions ignore the
// `roomts_pduid` keyspace.
Migration(rooms::timeline::TIMESTAMP_INDEX_MIGRATION, |services| {
Box::pin(async {
let global = &services.db["global"];
if global
.exists(rooms::timeline::TIMESTAMP_INDEX_MIGRATION)
.await
.is_ok()
{
return Ok(());
}
let until = match global.get(ROOMTS_UNTIL).await.deserialized::<u64>() {
| Ok(until) => until,
| Err(_) => {
let until = services.globals.current_count()?;
global.insert(ROOMTS_UNTIL, until.to_be_bytes());
until
},
};
services
.server
.runtime()
.spawn(populate_roomts_pduid(services.db.clone(), until));
Ok(())
})
}),
]
};
@@ -961,6 +995,38 @@ async fn drop_server_signingkeys(services: &Services) -> Result<()> {
Ok(())
}
const ROOMTS_UNTIL: &str = "populate_roomts_pduid_until";
async fn populate_roomts_pduid(db: Arc<Database>, until: u64) {
#[derive(Deserialize)]
struct ExtractTs {
origin_server_ts: MilliSecondsSinceUnixEpoch,
}
info!("Starting background migration to index PDUs by timestamp.");
let start = Instant::now();
let roomts_pduid = db["roomts_pduid"].clone();
db["pduid_pdu"]
.raw_stream()
.ignore_err()
.ready_for_each(|(key, val)| {
let pdu_id = RawPduId::from(key);
if matches!(pdu_id.pdu_count(), PduCount::Normal(count) if count > until) {
return;
}
if let Ok(pdu) = serde_json::from_slice::<ExtractTs>(val) {
roomts_pduid
.insert(&rooms::timeline::roomts_key(&pdu_id, pdu.origin_server_ts), []);
}
})
.await;
db["global"].insert(rooms::timeline::TIMESTAMP_INDEX_MIGRATION, []);
db["global"].remove(ROOMTS_UNTIL);
info!(elapsed = ?start.elapsed(), "Finished indexing PDUs by timestamp.");
}
async fn fix_oauth_scope_names(services: &Services) -> Result {
let db = &services.db;
let userdeviceid_oauthsessioninfo = db["userdeviceid_oauthsessioninfo"].clone();
+1 -1
View File
@@ -349,7 +349,7 @@ impl super::Service {
// Insert pdu
self.db
.append_pdu(&pdu_id, pdu, room_id, &pdu_json, count2)
.append_pdu(&pdu_id, pdu, room_id, &pdu_json, count2, pdu.origin_server_ts())
.await;
self.progress_extremities(pdu)
.await
+2 -1
View File
@@ -225,7 +225,8 @@ impl super::Service {
.into();
// Insert pdu
self.db.prepend_backfill_pdu(&pdu_id, &event_id, &value);
self.db
.prepend_backfill_pdu(&pdu_id, &event_id, &value, pdu.origin_server_ts());
drop(insert_lock);
if pdu.kind == TimelineEventType::RoomMessage {
+95 -1
View File
@@ -12,7 +12,9 @@ use database::{Database, Deserialized, Json, KeyVal, Map};
use futures::{
FutureExt, Stream, StreamExt, TryFutureExt, TryStreamExt, future::select_ok, pin_mut,
};
use ruma::{CanonicalJsonObject, EventId, OwnedUserId, RoomId, api::Direction};
use ruma::{
CanonicalJsonObject, EventId, MilliSecondsSinceUnixEpoch, OwnedUserId, RoomId, api::Direction,
};
use serde::{Deserialize, Serialize};
use serde_json::value::RawValue as RawJsonValue;
@@ -25,6 +27,7 @@ pub(super) struct Data {
pduid_pdu: Arc<Map>,
pduid_unredactedcontent: Arc<Map>,
roomid_lasttimelinecount: Arc<Map>,
roomts_pduid: Arc<Map>,
userroomid_highlightcount: Arc<Map>,
userroomid_notificationcount: Arc<Map>,
pub(super) db: Arc<Database>,
@@ -52,6 +55,7 @@ impl Data {
pduid_pdu: db["pduid_pdu"].clone(),
pduid_unredactedcontent: db["pduid_unredactedcontent"].clone(),
roomid_lasttimelinecount: db["roomid_lasttimelinecount"].clone(),
roomts_pduid: db["roomts_pduid"].clone(),
userroomid_highlightcount: db["userroomid_highlightcount"].clone(),
userroomid_notificationcount: db["userroomid_notificationcount"].clone(),
db: args.db.clone(),
@@ -213,10 +217,12 @@ impl Data {
room_id: &RoomId,
json: &CanonicalJsonObject,
count: PduCount,
index_ts: MilliSecondsSinceUnixEpoch,
) {
debug_assert!(matches!(count, PduCount::Normal(_)), "PduCount not Normal");
self.pduid_pdu.raw_put(pdu_id, Json(json));
self.roomts_pduid.insert(&roomts_key(pdu_id, index_ts), []);
self.eventid_pduid.insert(pdu.event_id.as_bytes(), pdu_id);
self.eventid_outlierpdu.remove(pdu.event_id.as_bytes());
self.roomid_lasttimelinecount
@@ -228,8 +234,10 @@ impl Data {
pdu_id: &RawPduId,
event_id: &EventId,
json: &CanonicalJsonObject,
index_ts: MilliSecondsSinceUnixEpoch,
) {
self.pduid_pdu.raw_put(pdu_id, Json(json));
self.roomts_pduid.insert(&roomts_key(pdu_id, index_ts), []);
self.eventid_pduid.insert(event_id, pdu_id);
self.eventid_outlierpdu.remove(event_id);
}
@@ -322,6 +330,49 @@ impl Data {
.try_flatten_stream()
}
pub(super) fn pdus_by_timestamp<'a>(
&'a self,
room_id: &'a RoomId,
timestamp: MilliSecondsSinceUnixEpoch,
dir: Direction,
) -> impl Stream<Item = Result<PdusIterItem>> + Send + 'a {
self.services
.short
.get_shortroomid(room_id)
.map_err(move |e| err!(Request(NotFound("Room {room_id:?} not found: {e:?}"))))
.map_ok(move |shortroomid| {
let seek = |shorteventid| {
roomts_key(&PduId { shortroomid, shorteventid }.into(), timestamp)
};
let keys = match dir {
| Direction::Forward => self
.roomts_pduid
.raw_keys_from(&seek(PduCount::min()))
.left_stream(),
| Direction::Backward => self
.roomts_pduid
.rev_raw_keys_from(&seek(PduCount::max()))
.right_stream(),
};
keys.ready_try_take_while(move |key| {
Ok(key.starts_with(&shortroomid.to_be_bytes()))
})
.map_ok(roomts_key_to_pdu_id)
.and_then(async move |pdu_id| {
Ok((pdu_id.pdu_count(), self.get_pdu_from_id(&pdu_id).await?))
})
})
.try_flatten_stream()
}
pub(super) async fn timestamp_index_ready(&self) -> bool {
self.db["global"]
.exists(TIMESTAMP_INDEX_MIGRATION)
.await
.is_ok()
}
fn from_json_slice((pdu_id, pdu): KeyVal<'_>) -> Result<PdusIterItem> {
let pdu_id: RawPduId = pdu_id.into();
@@ -376,9 +427,52 @@ impl Data {
}
}
pub const TIMESTAMP_INDEX_MIGRATION: &str = "populate_roomts_pduid";
#[must_use]
pub fn roomts_key(pdu_id: &RawPduId, ts: MilliSecondsSinceUnixEpoch) -> Vec<u8> {
let count = pdu_id.pdu_count().into_signed().cast_unsigned() ^ (1 << 63);
[pdu_id.shortroomid(), u64::from(ts.get()).to_be_bytes(), count.to_be_bytes()].concat()
}
fn roomts_key_to_pdu_id(key: &[u8]) -> RawPduId {
let shortroomid = u64::from_be_bytes(key[0..8].try_into().expect("roomts key shortroomid"));
let count = u64::from_be_bytes(key[16..24].try_into().expect("roomts key count")) ^ (1 << 63);
PduId {
shortroomid,
shorteventid: PduCount::from_signed(count.cast_signed()),
}
.into()
}
//TODO: this is an ABA
fn increment(db: &Arc<Map>, key: &[u8]) {
let old = db.get_blocking(key);
let new = utils::increment(old.ok().as_deref());
db.insert(key, new);
}
#[cfg(test)]
mod tests {
use ruma::uint;
use super::*;
#[test]
fn roomts_key_orders_by_ts_then_count() {
let ids = [
(PduCount::Normal(9), uint!(1)),
(PduCount::Backfilled(-5), uint!(2)),
(PduCount::Normal(3), uint!(2)),
(PduCount::Normal(1), uint!(3)),
]
.map(|(count, ts)| (RawPduId::from(PduId { shortroomid: 7, shorteventid: count }), ts));
let keys = ids.map(|(id, ts)| roomts_key(&id, MilliSecondsSinceUnixEpoch(ts)));
assert!(keys.is_sorted());
for ((id, _), key) in ids.iter().zip(&keys) {
assert_eq!(roomts_key_to_pdu_id(key), *id);
}
}
}
+17 -3
View File
@@ -21,8 +21,8 @@ use conduwuit_core::{
};
use futures::{Future, Stream, TryStreamExt, pin_mut};
use ruma::{
CanonicalJsonObject, EventId, OwnedEventId, OwnedRoomId, RoomId,
events::room::encrypted::Relation,
CanonicalJsonObject, EventId, MilliSecondsSinceUnixEpoch, OwnedEventId, OwnedRoomId, RoomId,
api::Direction, events::room::encrypted::Relation,
};
use serde::Deserialize;
use serde_json::value::RawValue as RawJsonValue;
@@ -32,7 +32,10 @@ use tokio::{
};
use self::data::Data;
pub use self::{create::pdu_fits, data::PdusIterItem};
pub use self::{
create::pdu_fits,
data::{PdusIterItem, TIMESTAMP_INDEX_MIGRATION, roomts_key},
};
use crate::{
Dep, account_data, admin, appservice, config, globals, pusher, rooms, sending, server_keys,
sync, users,
@@ -307,6 +310,17 @@ impl Service {
self.pdus(room_id, None).ignore_err()
}
pub fn pdus_by_timestamp<'a>(
&'a self,
room_id: &'a RoomId,
timestamp: MilliSecondsSinceUnixEpoch,
dir: Direction,
) -> impl Stream<Item = Result<PdusIterItem>> + Send + 'a {
self.db.pdus_by_timestamp(room_id, timestamp, dir)
}
pub async fn timestamp_index_ready(&self) -> bool { self.db.timestamp_index_ready().await }
/// Reverse iteration starting after `until`.
#[tracing::instrument(skip(self), level = "debug")]
pub fn pdus_rev<'a>(