mirror of
https://forgejo.ellis.link/continuwuation/continuwuity.git
synced 2026-11-20 23:47:33 +00:00
feat: MSC3030
This commit is contained in:
1 parent
8db6f7a2e2
commit
a1e89eb04d
10 files changed
+254
-10
No files matched your search
@@ -0,0 +1 @@
|
||||
Added the `GET /_matrix/client/v1/rooms/{roomId}/timestamp_to_event` endpoint as per MSC3030, letting clients find the event closest to a given timestamp. Contributed by @eleboucher.
|
||||
@@ -4,10 +4,12 @@ mod event;
|
||||
mod initial_sync;
|
||||
mod stripped_state;
|
||||
mod summary;
|
||||
mod timestamp_to_event;
|
||||
mod upgrade;
|
||||
|
||||
pub(crate) use self::{
|
||||
aliases::get_room_aliases_route, create::create_room_route, event::get_room_event_route,
|
||||
initial_sync::room_initial_sync_route, stripped_state::get_stripped_state_route,
|
||||
summary::get_room_summary, upgrade::upgrade_room_route,
|
||||
summary::get_room_summary, timestamp_to_event::get_event_by_timestamp_route,
|
||||
upgrade::upgrade_room_route,
|
||||
};
|
||||
@@ -0,0 +1,61 @@
|
||||
use axum::extract::State;
|
||||
use conduwuit::{Err, Event, Result, utils::stream::TryIgnore};
|
||||
use futures::{StreamExt, pin_mut};
|
||||
use ruma::api::client::room::get_event_by_timestamp;
|
||||
|
||||
use crate::Ruma;
|
||||
|
||||
const MAX_CANDIDATES: usize = 20;
|
||||
|
||||
/// # `GET /_matrix/client/v1/rooms/{roomId}/timestamp_to_event`
|
||||
///
|
||||
/// Gets the ID of the event closest to the given timestamp, in the direction
|
||||
/// specified by `dir`.
|
||||
pub(crate) async fn get_event_by_timestamp_route(
|
||||
State(ref services): State<crate::State>,
|
||||
body: Ruma<get_event_by_timestamp::v1::Request>,
|
||||
) -> Result<get_event_by_timestamp::v1::Response> {
|
||||
let sender_user = body.identity.expect_sender_user()?;
|
||||
let room_id = &body.room_id;
|
||||
|
||||
if !services
|
||||
.rooms
|
||||
.state_accessor
|
||||
.user_can_see_state_events(sender_user, room_id)
|
||||
.await
|
||||
{
|
||||
return Err!(Request(Forbidden("You don't have permission to view this room.")));
|
||||
}
|
||||
|
||||
if !services.rooms.timeline.timestamp_index_ready().await {
|
||||
return Err!(FeatureDisabled("timestamp_to_event"));
|
||||
}
|
||||
|
||||
let candidates = services
|
||||
.rooms
|
||||
.timeline
|
||||
.pdus_by_timestamp(room_id, body.ts, body.dir)
|
||||
.ignore_err()
|
||||
.take(MAX_CANDIDATES);
|
||||
|
||||
pin_mut!(candidates);
|
||||
while let Some((_, pdu)) = candidates.next().await {
|
||||
if services
|
||||
.rooms
|
||||
.state_accessor
|
||||
.user_can_see_event(sender_user, room_id, pdu.event_id())
|
||||
.await
|
||||
{
|
||||
return Ok(get_event_by_timestamp::v1::Response::new(
|
||||
pdu.event_id().to_owned(),
|
||||
pdu.origin_server_ts(),
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
Err!(Request(NotFound(
|
||||
"Unable to find event from {} in direction {:?}",
|
||||
body.ts.get(),
|
||||
body.dir
|
||||
)))
|
||||
}
|
||||
@@ -52,6 +52,7 @@ pub fn build(router: Router<State>, state: State) -> Router<State> {
|
||||
.ruma_route(&client::set_pushrule_actions_route)
|
||||
.ruma_route(&client::delete_pushrule_route)
|
||||
.ruma_route(&client::get_room_event_route)
|
||||
.ruma_route(&client::get_event_by_timestamp_route)
|
||||
.ruma_route(&client::get_room_aliases_route)
|
||||
.ruma_route(&client::get_filter_route)
|
||||
.ruma_route(&client::create_filter_route)
|
||||
|
||||
@@ -232,6 +232,10 @@ pub(super) static MAPS: &[Descriptor] = &[
|
||||
name: "roomsynctoken_shortstatehash",
|
||||
..descriptor::DROPPED
|
||||
},
|
||||
Descriptor {
|
||||
name: "roomts_pduid",
|
||||
..descriptor::SEQUENTIAL_SMALL
|
||||
},
|
||||
Descriptor {
|
||||
name: "roomuserdataid_accountdata",
|
||||
..descriptor::RANDOM_SMALL
|
||||
|
||||
@@ -18,11 +18,11 @@ use conduwuit::{
|
||||
},
|
||||
warn,
|
||||
};
|
||||
use database::{Deserialized, Ignore, Json};
|
||||
use database::{Database, Deserialized, Ignore, Json};
|
||||
use futures::{FutureExt, StreamExt, TryStreamExt};
|
||||
use itertools::Itertools;
|
||||
use ruma::{
|
||||
OwnedDeviceId, OwnedEventId, OwnedRoomId, OwnedUserId, UserId,
|
||||
MilliSecondsSinceUnixEpoch, OwnedDeviceId, OwnedEventId, OwnedRoomId, OwnedUserId, UserId,
|
||||
api::OAuthClientScope,
|
||||
events::{
|
||||
AnyStrippedStateEvent, GlobalAccountDataEventType, StateEventType,
|
||||
@@ -32,9 +32,13 @@ use ruma::{
|
||||
push::Ruleset,
|
||||
serde::{Raw, from_raw_json_value},
|
||||
};
|
||||
use serde::Deserialize;
|
||||
use serde_json::{Value, value::to_raw_value};
|
||||
|
||||
use crate::{Services, media, rooms, rooms::short::ShortStateHash};
|
||||
use crate::{
|
||||
Services, media, rooms,
|
||||
rooms::{short::ShortStateHash, timeline::RawPduId},
|
||||
};
|
||||
|
||||
type MigrationFn =
|
||||
for<'a> fn(&'a Services) -> Pin<Box<dyn Future<Output = Result<()>> + Send + 'a>>;
|
||||
@@ -176,6 +180,36 @@ migration!(stringify!($name) => $name)
|
||||
// This is backwards compatible because old versions never read the
|
||||
// account data entries it removes.
|
||||
migration!(remove_replaced_account_data),
|
||||
// This is backwards compatible because old versions ignore the
|
||||
// `roomts_pduid` keyspace.
|
||||
Migration(rooms::timeline::TIMESTAMP_INDEX_MIGRATION, |services| {
|
||||
Box::pin(async {
|
||||
let global = &services.db["global"];
|
||||
if global
|
||||
.exists(rooms::timeline::TIMESTAMP_INDEX_MIGRATION)
|
||||
.await
|
||||
.is_ok()
|
||||
{
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let until = match global.get(ROOMTS_UNTIL).await.deserialized::<u64>() {
|
||||
| Ok(until) => until,
|
||||
| Err(_) => {
|
||||
let until = services.globals.current_count()?;
|
||||
global.insert(ROOMTS_UNTIL, until.to_be_bytes());
|
||||
until
|
||||
},
|
||||
};
|
||||
|
||||
services
|
||||
.server
|
||||
.runtime()
|
||||
.spawn(populate_roomts_pduid(services.db.clone(), until));
|
||||
|
||||
Ok(())
|
||||
})
|
||||
}),
|
||||
]
|
||||
};
|
||||
|
||||
@@ -961,6 +995,38 @@ async fn drop_server_signingkeys(services: &Services) -> Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
const ROOMTS_UNTIL: &str = "populate_roomts_pduid_until";
|
||||
|
||||
async fn populate_roomts_pduid(db: Arc<Database>, until: u64) {
|
||||
#[derive(Deserialize)]
|
||||
struct ExtractTs {
|
||||
origin_server_ts: MilliSecondsSinceUnixEpoch,
|
||||
}
|
||||
|
||||
info!("Starting background migration to index PDUs by timestamp.");
|
||||
let start = Instant::now();
|
||||
let roomts_pduid = db["roomts_pduid"].clone();
|
||||
|
||||
db["pduid_pdu"]
|
||||
.raw_stream()
|
||||
.ignore_err()
|
||||
.ready_for_each(|(key, val)| {
|
||||
let pdu_id = RawPduId::from(key);
|
||||
if matches!(pdu_id.pdu_count(), PduCount::Normal(count) if count > until) {
|
||||
return;
|
||||
}
|
||||
if let Ok(pdu) = serde_json::from_slice::<ExtractTs>(val) {
|
||||
roomts_pduid
|
||||
.insert(&rooms::timeline::roomts_key(&pdu_id, pdu.origin_server_ts), []);
|
||||
}
|
||||
})
|
||||
.await;
|
||||
|
||||
db["global"].insert(rooms::timeline::TIMESTAMP_INDEX_MIGRATION, []);
|
||||
db["global"].remove(ROOMTS_UNTIL);
|
||||
info!(elapsed = ?start.elapsed(), "Finished indexing PDUs by timestamp.");
|
||||
}
|
||||
|
||||
async fn fix_oauth_scope_names(services: &Services) -> Result {
|
||||
let db = &services.db;
|
||||
let userdeviceid_oauthsessioninfo = db["userdeviceid_oauthsessioninfo"].clone();
|
||||
|
||||
@@ -349,7 +349,7 @@ impl super::Service {
|
||||
|
||||
// Insert pdu
|
||||
self.db
|
||||
.append_pdu(&pdu_id, pdu, room_id, &pdu_json, count2)
|
||||
.append_pdu(&pdu_id, pdu, room_id, &pdu_json, count2, pdu.origin_server_ts())
|
||||
.await;
|
||||
self.progress_extremities(pdu)
|
||||
.await
|
||||
|
||||
@@ -225,7 +225,8 @@ impl super::Service {
|
||||
.into();
|
||||
|
||||
// Insert pdu
|
||||
self.db.prepend_backfill_pdu(&pdu_id, &event_id, &value);
|
||||
self.db
|
||||
.prepend_backfill_pdu(&pdu_id, &event_id, &value, pdu.origin_server_ts());
|
||||
|
||||
drop(insert_lock);
|
||||
if pdu.kind == TimelineEventType::RoomMessage {
|
||||
|
||||
@@ -12,7 +12,9 @@ use database::{Database, Deserialized, Json, KeyVal, Map};
|
||||
use futures::{
|
||||
FutureExt, Stream, StreamExt, TryFutureExt, TryStreamExt, future::select_ok, pin_mut,
|
||||
};
|
||||
use ruma::{CanonicalJsonObject, EventId, OwnedUserId, RoomId, api::Direction};
|
||||
use ruma::{
|
||||
CanonicalJsonObject, EventId, MilliSecondsSinceUnixEpoch, OwnedUserId, RoomId, api::Direction,
|
||||
};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::value::RawValue as RawJsonValue;
|
||||
|
||||
@@ -25,6 +27,7 @@ pub(super) struct Data {
|
||||
pduid_pdu: Arc<Map>,
|
||||
pduid_unredactedcontent: Arc<Map>,
|
||||
roomid_lasttimelinecount: Arc<Map>,
|
||||
roomts_pduid: Arc<Map>,
|
||||
userroomid_highlightcount: Arc<Map>,
|
||||
userroomid_notificationcount: Arc<Map>,
|
||||
pub(super) db: Arc<Database>,
|
||||
@@ -52,6 +55,7 @@ impl Data {
|
||||
pduid_pdu: db["pduid_pdu"].clone(),
|
||||
pduid_unredactedcontent: db["pduid_unredactedcontent"].clone(),
|
||||
roomid_lasttimelinecount: db["roomid_lasttimelinecount"].clone(),
|
||||
roomts_pduid: db["roomts_pduid"].clone(),
|
||||
userroomid_highlightcount: db["userroomid_highlightcount"].clone(),
|
||||
userroomid_notificationcount: db["userroomid_notificationcount"].clone(),
|
||||
db: args.db.clone(),
|
||||
@@ -213,10 +217,12 @@ impl Data {
|
||||
room_id: &RoomId,
|
||||
json: &CanonicalJsonObject,
|
||||
count: PduCount,
|
||||
index_ts: MilliSecondsSinceUnixEpoch,
|
||||
) {
|
||||
debug_assert!(matches!(count, PduCount::Normal(_)), "PduCount not Normal");
|
||||
|
||||
self.pduid_pdu.raw_put(pdu_id, Json(json));
|
||||
self.roomts_pduid.insert(&roomts_key(pdu_id, index_ts), []);
|
||||
self.eventid_pduid.insert(pdu.event_id.as_bytes(), pdu_id);
|
||||
self.eventid_outlierpdu.remove(pdu.event_id.as_bytes());
|
||||
self.roomid_lasttimelinecount
|
||||
@@ -228,8 +234,10 @@ impl Data {
|
||||
pdu_id: &RawPduId,
|
||||
event_id: &EventId,
|
||||
json: &CanonicalJsonObject,
|
||||
index_ts: MilliSecondsSinceUnixEpoch,
|
||||
) {
|
||||
self.pduid_pdu.raw_put(pdu_id, Json(json));
|
||||
self.roomts_pduid.insert(&roomts_key(pdu_id, index_ts), []);
|
||||
self.eventid_pduid.insert(event_id, pdu_id);
|
||||
self.eventid_outlierpdu.remove(event_id);
|
||||
}
|
||||
@@ -322,6 +330,49 @@ impl Data {
|
||||
.try_flatten_stream()
|
||||
}
|
||||
|
||||
pub(super) fn pdus_by_timestamp<'a>(
|
||||
&'a self,
|
||||
room_id: &'a RoomId,
|
||||
timestamp: MilliSecondsSinceUnixEpoch,
|
||||
dir: Direction,
|
||||
) -> impl Stream<Item = Result<PdusIterItem>> + Send + 'a {
|
||||
self.services
|
||||
.short
|
||||
.get_shortroomid(room_id)
|
||||
.map_err(move |e| err!(Request(NotFound("Room {room_id:?} not found: {e:?}"))))
|
||||
.map_ok(move |shortroomid| {
|
||||
let seek = |shorteventid| {
|
||||
roomts_key(&PduId { shortroomid, shorteventid }.into(), timestamp)
|
||||
};
|
||||
let keys = match dir {
|
||||
| Direction::Forward => self
|
||||
.roomts_pduid
|
||||
.raw_keys_from(&seek(PduCount::min()))
|
||||
.left_stream(),
|
||||
| Direction::Backward => self
|
||||
.roomts_pduid
|
||||
.rev_raw_keys_from(&seek(PduCount::max()))
|
||||
.right_stream(),
|
||||
};
|
||||
|
||||
keys.ready_try_take_while(move |key| {
|
||||
Ok(key.starts_with(&shortroomid.to_be_bytes()))
|
||||
})
|
||||
.map_ok(roomts_key_to_pdu_id)
|
||||
.and_then(async move |pdu_id| {
|
||||
Ok((pdu_id.pdu_count(), self.get_pdu_from_id(&pdu_id).await?))
|
||||
})
|
||||
})
|
||||
.try_flatten_stream()
|
||||
}
|
||||
|
||||
pub(super) async fn timestamp_index_ready(&self) -> bool {
|
||||
self.db["global"]
|
||||
.exists(TIMESTAMP_INDEX_MIGRATION)
|
||||
.await
|
||||
.is_ok()
|
||||
}
|
||||
|
||||
fn from_json_slice((pdu_id, pdu): KeyVal<'_>) -> Result<PdusIterItem> {
|
||||
let pdu_id: RawPduId = pdu_id.into();
|
||||
|
||||
@@ -376,9 +427,52 @@ impl Data {
|
||||
}
|
||||
}
|
||||
|
||||
pub const TIMESTAMP_INDEX_MIGRATION: &str = "populate_roomts_pduid";
|
||||
|
||||
#[must_use]
|
||||
pub fn roomts_key(pdu_id: &RawPduId, ts: MilliSecondsSinceUnixEpoch) -> Vec<u8> {
|
||||
let count = pdu_id.pdu_count().into_signed().cast_unsigned() ^ (1 << 63);
|
||||
[pdu_id.shortroomid(), u64::from(ts.get()).to_be_bytes(), count.to_be_bytes()].concat()
|
||||
}
|
||||
|
||||
fn roomts_key_to_pdu_id(key: &[u8]) -> RawPduId {
|
||||
let shortroomid = u64::from_be_bytes(key[0..8].try_into().expect("roomts key shortroomid"));
|
||||
let count = u64::from_be_bytes(key[16..24].try_into().expect("roomts key count")) ^ (1 << 63);
|
||||
|
||||
PduId {
|
||||
shortroomid,
|
||||
shorteventid: PduCount::from_signed(count.cast_signed()),
|
||||
}
|
||||
.into()
|
||||
}
|
||||
|
||||
//TODO: this is an ABA
|
||||
fn increment(db: &Arc<Map>, key: &[u8]) {
|
||||
let old = db.get_blocking(key);
|
||||
let new = utils::increment(old.ok().as_deref());
|
||||
db.insert(key, new);
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use ruma::uint;
|
||||
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn roomts_key_orders_by_ts_then_count() {
|
||||
let ids = [
|
||||
(PduCount::Normal(9), uint!(1)),
|
||||
(PduCount::Backfilled(-5), uint!(2)),
|
||||
(PduCount::Normal(3), uint!(2)),
|
||||
(PduCount::Normal(1), uint!(3)),
|
||||
]
|
||||
.map(|(count, ts)| (RawPduId::from(PduId { shortroomid: 7, shorteventid: count }), ts));
|
||||
let keys = ids.map(|(id, ts)| roomts_key(&id, MilliSecondsSinceUnixEpoch(ts)));
|
||||
|
||||
assert!(keys.is_sorted());
|
||||
for ((id, _), key) in ids.iter().zip(&keys) {
|
||||
assert_eq!(roomts_key_to_pdu_id(key), *id);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -21,8 +21,8 @@ use conduwuit_core::{
|
||||
};
|
||||
use futures::{Future, Stream, TryStreamExt, pin_mut};
|
||||
use ruma::{
|
||||
CanonicalJsonObject, EventId, OwnedEventId, OwnedRoomId, RoomId,
|
||||
events::room::encrypted::Relation,
|
||||
CanonicalJsonObject, EventId, MilliSecondsSinceUnixEpoch, OwnedEventId, OwnedRoomId, RoomId,
|
||||
api::Direction, events::room::encrypted::Relation,
|
||||
};
|
||||
use serde::Deserialize;
|
||||
use serde_json::value::RawValue as RawJsonValue;
|
||||
@@ -32,7 +32,10 @@ use tokio::{
|
||||
};
|
||||
|
||||
use self::data::Data;
|
||||
pub use self::{create::pdu_fits, data::PdusIterItem};
|
||||
pub use self::{
|
||||
create::pdu_fits,
|
||||
data::{PdusIterItem, TIMESTAMP_INDEX_MIGRATION, roomts_key},
|
||||
};
|
||||
use crate::{
|
||||
Dep, account_data, admin, appservice, config, globals, pusher, rooms, sending, server_keys,
|
||||
sync, users,
|
||||
@@ -307,6 +310,17 @@ impl Service {
|
||||
self.pdus(room_id, None).ignore_err()
|
||||
}
|
||||
|
||||
pub fn pdus_by_timestamp<'a>(
|
||||
&'a self,
|
||||
room_id: &'a RoomId,
|
||||
timestamp: MilliSecondsSinceUnixEpoch,
|
||||
dir: Direction,
|
||||
) -> impl Stream<Item = Result<PdusIterItem>> + Send + 'a {
|
||||
self.db.pdus_by_timestamp(room_id, timestamp, dir)
|
||||
}
|
||||
|
||||
pub async fn timestamp_index_ready(&self) -> bool { self.db.timestamp_index_ready().await }
|
||||
|
||||
/// Reverse iteration starting after `until`.
|
||||
#[tracing::instrument(skip(self), level = "debug")]
|
||||
pub fn pdus_rev<'a>(
|
||||
|
||||
Reference in new issue
Block a user