fix: Don't store events that have already been redacted

This prevents clobbering

.typos.toml

@@ -1,5 +1,5 @@
 [files]
-extend-exclude = ["*.csr", "*.lock"]
+extend-exclude = ["*.csr"]
 
 [default.extend-words]
 "allocatedp" = "allocatedp"

Cargo.lock

@@ -3695,7 +3695,7 @@ dependencies = [
 [[package]]
 name = "ruma"
 version = "0.10.1"
-source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=a48665b682be1016cea53ea5e7787442dfe7c1de#a48665b682be1016cea53ea5e7787442dfe7c1de"
+source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=d6870a7fb7f6cccff63f7fd0ff6c581bad80e983#d6870a7fb7f6cccff63f7fd0ff6c581bad80e983"
 dependencies = [
  "assign",
  "js_int",
@@ -3715,7 +3715,7 @@ dependencies = [
 [[package]]
 name = "ruma-appservice-api"
 version = "0.10.0"
-source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=a48665b682be1016cea53ea5e7787442dfe7c1de#a48665b682be1016cea53ea5e7787442dfe7c1de"
+source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=d6870a7fb7f6cccff63f7fd0ff6c581bad80e983#d6870a7fb7f6cccff63f7fd0ff6c581bad80e983"
 dependencies = [
  "js_int",
  "ruma-common",
@@ -3727,7 +3727,7 @@ dependencies = [
 [[package]]
 name = "ruma-client-api"
 version = "0.18.0"
-source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=a48665b682be1016cea53ea5e7787442dfe7c1de#a48665b682be1016cea53ea5e7787442dfe7c1de"
+source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=d6870a7fb7f6cccff63f7fd0ff6c581bad80e983#d6870a7fb7f6cccff63f7fd0ff6c581bad80e983"
 dependencies = [
  "as_variant",
  "assign",
@@ -3750,7 +3750,7 @@ dependencies = [
 [[package]]
 name = "ruma-common"
 version = "0.13.0"
-source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=a48665b682be1016cea53ea5e7787442dfe7c1de#a48665b682be1016cea53ea5e7787442dfe7c1de"
+source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=d6870a7fb7f6cccff63f7fd0ff6c581bad80e983#d6870a7fb7f6cccff63f7fd0ff6c581bad80e983"
 dependencies = [
  "as_variant",
  "base64 0.22.1",
@@ -3782,7 +3782,7 @@ dependencies = [
 [[package]]
 name = "ruma-events"
 version = "0.28.1"
-source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=a48665b682be1016cea53ea5e7787442dfe7c1de#a48665b682be1016cea53ea5e7787442dfe7c1de"
+source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=d6870a7fb7f6cccff63f7fd0ff6c581bad80e983#d6870a7fb7f6cccff63f7fd0ff6c581bad80e983"
 dependencies = [
  "as_variant",
  "indexmap 2.9.0",
@@ -3807,7 +3807,7 @@ dependencies = [
 [[package]]
 name = "ruma-federation-api"
 version = "0.9.0"
-source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=a48665b682be1016cea53ea5e7787442dfe7c1de#a48665b682be1016cea53ea5e7787442dfe7c1de"
+source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=d6870a7fb7f6cccff63f7fd0ff6c581bad80e983#d6870a7fb7f6cccff63f7fd0ff6c581bad80e983"
 dependencies = [
  "bytes",
  "headers",
@@ -3829,7 +3829,7 @@ dependencies = [
 [[package]]
 name = "ruma-identifiers-validation"
 version = "0.9.5"
-source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=a48665b682be1016cea53ea5e7787442dfe7c1de#a48665b682be1016cea53ea5e7787442dfe7c1de"
+source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=d6870a7fb7f6cccff63f7fd0ff6c581bad80e983#d6870a7fb7f6cccff63f7fd0ff6c581bad80e983"
 dependencies = [
  "js_int",
  "thiserror 2.0.12",
@@ -3838,7 +3838,7 @@ dependencies = [
 [[package]]
 name = "ruma-identity-service-api"
 version = "0.9.0"
-source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=a48665b682be1016cea53ea5e7787442dfe7c1de#a48665b682be1016cea53ea5e7787442dfe7c1de"
+source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=d6870a7fb7f6cccff63f7fd0ff6c581bad80e983#d6870a7fb7f6cccff63f7fd0ff6c581bad80e983"
 dependencies = [
  "js_int",
  "ruma-common",
@@ -3848,7 +3848,7 @@ dependencies = [
 [[package]]
 name = "ruma-macros"
 version = "0.13.0"
-source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=a48665b682be1016cea53ea5e7787442dfe7c1de#a48665b682be1016cea53ea5e7787442dfe7c1de"
+source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=d6870a7fb7f6cccff63f7fd0ff6c581bad80e983#d6870a7fb7f6cccff63f7fd0ff6c581bad80e983"
 dependencies = [
  "cfg-if",
  "proc-macro-crate",
@@ -3863,7 +3863,7 @@ dependencies = [
 [[package]]
 name = "ruma-push-gateway-api"
 version = "0.9.0"
-source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=a48665b682be1016cea53ea5e7787442dfe7c1de#a48665b682be1016cea53ea5e7787442dfe7c1de"
+source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=d6870a7fb7f6cccff63f7fd0ff6c581bad80e983#d6870a7fb7f6cccff63f7fd0ff6c581bad80e983"
 dependencies = [
  "js_int",
  "ruma-common",
@@ -3875,7 +3875,7 @@ dependencies = [
 [[package]]
 name = "ruma-signatures"
 version = "0.15.0"
-source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=a48665b682be1016cea53ea5e7787442dfe7c1de#a48665b682be1016cea53ea5e7787442dfe7c1de"
+source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=d6870a7fb7f6cccff63f7fd0ff6c581bad80e983#d6870a7fb7f6cccff63f7fd0ff6c581bad80e983"
 dependencies = [
  "base64 0.22.1",
  "ed25519-dalek",

Cargo.toml

@@ -350,7 +350,7 @@ version = "0.1.2"
 [workspace.dependencies.ruma]
 git = "https://forgejo.ellis.link/continuwuation/ruwuma"
 #branch = "conduwuit-changes"
-rev = "a48665b682be1016cea53ea5e7787442dfe7c1de"
+rev = "d6870a7fb7f6cccff63f7fd0ff6c581bad80e983"
 features = [
     "compat",
     "rand",
@@ -381,7 +381,7 @@ features = [
     "unstable-msc4121",
     "unstable-msc4125",
     "unstable-msc4186",
-    "unstable-msc4203", # sending to-device events to appservices
+    "unstable-msc4203", # sending to-device events to appservices 
     "unstable-msc4210", # remove legacy mentions
     "unstable-extensible-events",
     "unstable-pdu",

src/api/client/room/event.rs

@@ -1,7 +1,7 @@
 use axum::extract::State;
-use conduwuit::{Err, Event, Result, err};
+use conduwuit::{Err, Event, PduEvent, Result, err};
 use futures::{FutureExt, TryFutureExt, future::try_join};
-use ruma::api::client::room::get_room_event;
+use ruma::api::client::{error::ErrorKind, room::get_room_event};
 
 use crate::{Ruma, client::is_ignored_pdu};
 
@@ -14,6 +14,7 @@ pub(crate) async fn get_room_event_route(
 ) -> Result<get_room_event::v3::Response> {
 	let event_id = &body.event_id;
 	let room_id = &body.room_id;
+	let sender_user = body.sender_user();
 
 	let event = services
 		.rooms
@@ -33,6 +34,52 @@ pub(crate) async fn get_room_event_route(
 		return Err!(Request(Forbidden("You don't have permission to view this event.")));
 	}
 
+	let include_unredacted_content = body
+		.include_unredacted_content // User's file has this field name
+		.unwrap_or(false);
+
+	if include_unredacted_content && event.is_redacted() {
+		let is_server_admin = services
+			.users
+			.is_admin(sender_user)
+			.map(|is_admin| Ok(is_admin));
+		let can_redact_privilege = services
+			.rooms
+			.state_accessor
+			.user_can_redact(event_id, sender_user, room_id, false) // federation=false for local check
+			;
+		let (is_server_admin, can_redact_privilege) =
+			try_join(is_server_admin, can_redact_privilege).await?;
+
+		if !is_server_admin && !can_redact_privilege {
+			return Err!(Request(Forbidden(
+				"You don't have permission to view redacted content.",
+			)));
+		}
+
+		let pdu_id = match services.rooms.timeline.get_pdu_id(event_id).await {
+			| Ok(id) => id,
+			| Err(e) => {
+				return Err(e);
+			},
+		};
+		let original_content = services
+			.rooms
+			.timeline
+			.get_original_pdu_content(&pdu_id)
+			.await?;
+		if let Some(original_content) = original_content {
+			// If the original content is available, we can return it.
+			// event.content = to_raw_value(&original_content)?;
+			event = PduEvent::from_id_val(event_id, original_content)?;
+		} else {
+			return Err(conduwuit::Error::BadRequest(
+				ErrorKind::UnredactedContentDeleted { content_keep_ms: None },
+				"The original unredacted content is not in the database.",
+			));
+		}
+	}
+
 	debug_assert!(
 		event.event_id() == event_id && event.room_id() == room_id,
 		"Fetched PDU must match requested"

src/api/client/sync/v3.rs

@@ -808,7 +808,7 @@ async fn load_joined_room(
 			let typings = services
 				.rooms
 				.typing
-				.typings_event_for_user(room_id, sender_user)
+				.typings_all(room_id, sender_user)
 				.await?;
 
 			Ok(vec![serde_json::from_str(&serde_json::to_string(&typings)?)?])

src/api/client/sync/v5.rs

@@ -33,7 +33,6 @@ use ruma::{
 	events::{
 		AnyRawAccountDataEvent, AnySyncEphemeralRoomEvent, StateEventType, TimelineEventType,
 		room::member::{MembershipState, RoomMemberEventContent},
-		typing::TypingEventContent,
 	},
 	serde::Raw,
 	uint,
@@ -206,9 +205,6 @@ pub(crate) async fn sync_events_v5_route(
 		_ = tokio::time::timeout(duration, watcher).await;
 	}
 
-	let typing = collect_typing_events(services, sender_user, &body, &todo_rooms).await?;
-	response.extensions.typing = typing;
-
 	trace!(
 		rooms = ?response.rooms.len(),
 		account_data = ?response.extensions.account_data.rooms.len(),
@@ -292,8 +288,6 @@ where
 	Rooms: Iterator<Item = &'a RoomId> + Clone + Send + 'a,
 	AllRooms: Iterator<Item = &'a RoomId> + Clone + Send + 'a,
 {
-	// TODO MSC4186: Implement remaining list filters: is_dm, is_encrypted,
-	// room_types.
 	for (list_id, list) in &body.lists {
 		let active_rooms: Vec<_> = match list.filters.as_ref().and_then(|f| f.is_invite) {
 			| None => all_rooms.clone().collect(),
@@ -671,62 +665,6 @@ where
 	}
 	Ok(rooms)
 }
-
-async fn collect_typing_events(
-	services: &Services,
-	sender_user: &UserId,
-	body: &sync_events::v5::Request,
-	todo_rooms: &TodoRooms,
-) -> Result<sync_events::v5::response::Typing> {
-	if !body.extensions.typing.enabled.unwrap_or(false) {
-		return Ok(sync_events::v5::response::Typing::default());
-	}
-	let rooms: Vec<_> = body.extensions.typing.rooms.clone().unwrap_or_else(|| {
-		body.room_subscriptions
-			.keys()
-			.map(ToOwned::to_owned)
-			.collect()
-	});
-	let lists: Vec<_> = body
-		.extensions
-		.typing
-		.lists
-		.clone()
-		.unwrap_or_else(|| body.lists.keys().map(ToOwned::to_owned).collect::<Vec<_>>());
-
-	if rooms.is_empty() && lists.is_empty() {
-		return Ok(sync_events::v5::response::Typing::default());
-	}
-
-	let mut typing_response = sync_events::v5::response::Typing::default();
-	for (room_id, (required_state_request, timeline_limit, roomsince)) in todo_rooms {
-		if services.rooms.typing.last_typing_update(room_id).await? <= *roomsince {
-			continue;
-		}
-
-		match services
-			.rooms
-			.typing
-			.typing_users_for_user(room_id, sender_user)
-			.await
-		{
-			| Ok(typing_users) => {
-				typing_response.rooms.insert(
-					room_id.to_owned(), // Already OwnedRoomId
-					Raw::new(&sync_events::v5::response::SyncTypingEvent {
-						content: TypingEventContent::new(typing_users),
-					})?,
-				);
-			},
-			| Err(e) => {
-				warn!(%room_id, "Failed to get typing events for room: {}", e);
-			},
-		}
-	}
-
-	Ok(typing_response)
-}
-
 async fn collect_account_data(
 	services: &Services,
 	(sender_user, _, globalsince, body): (&UserId, &DeviceId, u64, &sync_events::v5::Request),

src/api/client/unversioned.rs

@@ -40,6 +40,7 @@ pub(crate) async fn get_supported_versions_route(
 			"v1.11".to_owned(),
 		],
 		unstable_features: BTreeMap::from_iter([
+			("fi.mau.msc2815".to_owned(), true),
 			("org.matrix.e2e_cross_signing".to_owned(), true),
 			("org.matrix.msc2285.stable".to_owned(), true), /* private read receipts (https://github.com/matrix-org/matrix-spec-proposals/pull/2285) */
 			("uk.half-shot.msc2666.query_mutual_rooms".to_owned(), true), /* query mutual rooms (https://github.com/matrix-org/matrix-spec-proposals/pull/2666) */

src/database/maps.rs

@@ -121,6 +121,15 @@ pub(super) static MAPS: &[Descriptor] = &[
 		index_size: 512,
 		..descriptor::SEQUENTIAL
 	},
+	Descriptor {
+		name: "pduid_originalcontent",
+		cache_disp: CacheDisp::SharedWith("pduid_pdu"),
+		key_size_hint: Some(16),
+		val_size_hint: Some(1520),
+		block_size: 2048,
+		index_size: 512,
+		..descriptor::RANDOM
+	},
 	Descriptor {
 		name: "publicroomids",
 		..descriptor::RANDOM_SMALL

src/service/rooms/timeline/data.rs

@@ -19,6 +19,8 @@ pub(super) struct Data {
 	pduid_pdu: Arc<Map>,
 	userroomid_highlightcount: Arc<Map>,
 	userroomid_notificationcount: Arc<Map>,
+	/// Stores the original content of redacted PDUs.
+	pduid_originalcontent: Arc<Map>,
 	pub(super) db: Arc<Database>,
 	services: Services,
 }
@@ -38,6 +40,7 @@ impl Data {
 			pduid_pdu: db["pduid_pdu"].clone(),
 			userroomid_highlightcount: db["userroomid_highlightcount"].clone(),
 			userroomid_notificationcount: db["userroomid_notificationcount"].clone(),
+			pduid_originalcontent: db["pduid_originalcontent"].clone(), // Initialize new table
 			db: args.db.clone(),
 			services: Services {
 				short: args.depend::<rooms::short::Service>("rooms::short"),
@@ -177,6 +180,24 @@ impl Data {
 		self.pduid_pdu.get(pdu_id).await.deserialized()
 	}
 
+	/// Stores the original content of a PDU that is about to be redacted.
+	pub(super) async fn store_redacted_pdu_content(
+		&self,
+		pdu_id: &RawPduId,
+		pdu_json: &CanonicalJsonObject,
+	) -> Result<()> {
+		self.pduid_originalcontent.raw_put(pdu_id, Json(pdu_json));
+		Ok(())
+	}
+
+	/// Returns the original content of a redacted PDU.
+	pub(super) async fn get_original_pdu_content(
+		&self,
+		pdu_id: &RawPduId,
+	) -> Result<Option<CanonicalJsonObject>> {
+		self.pduid_originalcontent.get(pdu_id).await.deserialized()
+	}
+
 	pub(super) async fn append_pdu(
 		&self,
 		pdu_id: &RawPduId,

src/service/rooms/timeline/mod.rs

@@ -260,6 +260,25 @@ impl Service {
 		self.db.replace_pdu(pdu_id, pdu_json, pdu).await
 	}
 
+	/// Stores the content of a to-be redacted pdu.
+	#[tracing::instrument(skip(self), level = "debug")]
+	pub async fn store_redacted_pdu_content(
+		&self,
+		pdu_id: &RawPduId,
+		pdu_json: &CanonicalJsonObject,
+	) -> Result<()> {
+		self.db.store_redacted_pdu_content(pdu_id, pdu_json).await
+	}
+
+	/// Returns the original content of a redacted PDU.
+	#[tracing::instrument(skip(self), level = "debug")]
+	pub async fn get_original_pdu_content(
+		&self,
+		pdu_id: &RawPduId,
+	) -> Result<Option<CanonicalJsonObject>> {
+		self.db.get_original_pdu_content(pdu_id).await
+	}
+
 	/// Creates a new persisted data unit and adds it to a room.
 	///
 	/// By this point the incoming event should be fully authenticated, no auth
@@ -472,7 +491,7 @@ impl Service {
 								.user_can_redact(redact_id, &pdu.sender, &pdu.room_id, false)
 								.await?
 							{
-								self.redact_pdu(redact_id, pdu, shortroomid).await?;
+								self.redact_pdu(redact_id, pdu, shortroomid, true).await?;
 							}
 						}
 					},
@@ -485,7 +504,7 @@ impl Service {
 								.user_can_redact(redact_id, &pdu.sender, &pdu.room_id, false)
 								.await?
 							{
-								self.redact_pdu(redact_id, pdu, shortroomid).await?;
+								self.redact_pdu(redact_id, pdu, shortroomid, true).await?;
 							}
 						}
 					},
@@ -1033,6 +1052,7 @@ impl Service {
 		event_id: &EventId,
 		reason: &PduEvent,
 		shortroomid: ShortRoomId,
+		keep_original_content: bool,
 	) -> Result {
 		// TODO: Don't reserialize, keep original json
 		let Ok(pdu_id) = self.get_pdu_id(event_id).await else {
@@ -1054,6 +1074,19 @@ impl Service {
 
 		let room_version_id = self.services.state.get_room_version(&pdu.room_id).await?;
 
+		if keep_original_content && !pdu.is_redacted() {
+			let original_pdu_json = utils::to_canonical_object(&pdu).map_err(|e| {
+				err!(Database(error!(
+					?event_id,
+					?e,
+					"Failed to convert PDU to canonical JSON for original content storage"
+				)))
+			})?;
+			self.db
+				.store_redacted_pdu_content(&pdu_id, &original_pdu_json)
+				.await?;
+		}
+
 		pdu.redact(&room_version_id, reason)?;
 
 		let obj = utils::to_canonical_object(&pdu).map_err(|e| {

src/service/rooms/typing/mod.rs

@@ -179,15 +179,18 @@ impl Service {
 			.unwrap_or(0))
 	}
 
-	pub async fn typing_users_for_user(
+	/// Returns a new typing EDU.
+	pub async fn typings_all(
 		&self,
 		room_id: &RoomId,
 		sender_user: &UserId,
-	) -> Result<Vec<OwnedUserId>> {
+	) -> Result<SyncEphemeralRoomEvent<ruma::events::typing::TypingEventContent>> {
 		let room_typing_indicators = self.typing.read().await.get(room_id).cloned();
 
 		let Some(typing_indicators) = room_typing_indicators else {
-			return Ok(Vec::new());
+			return Ok(SyncEphemeralRoomEvent {
+				content: ruma::events::typing::TypingEventContent { user_ids: Vec::new() },
+			});
 		};
 
 		let user_ids: Vec<_> = typing_indicators
@@ -204,19 +207,8 @@ impl Service {
 			.collect()
 			.await;
 
-		Ok(user_ids)
-	}
-
-	/// Returns a new typing EDU.
-	pub async fn typings_event_for_user(
-		&self,
-		room_id: &RoomId,
-		sender_user: &UserId,
-	) -> Result<SyncEphemeralRoomEvent<ruma::events::typing::TypingEventContent>> {
 		Ok(SyncEphemeralRoomEvent {
-			content: ruma::events::typing::TypingEventContent {
-				user_ids: self.typing_users_for_user(room_id, sender_user).await?,
-			},
+			content: ruma::events::typing::TypingEventContent { user_ids },
 		})
 	}