Compare commits

..
Author SHA1 Message Date
Jade Ellis 0aaa42188e fix(service): Prevent potential corruption in state diffs
This fixes a non-atomicity issue. Also fixes incorrect
calculation of a buffer size that would result in
reallocation.
2026-11-21 23:51:20 +00:00
4 changed files with 108 additions and 153 deletions

No files matched your search

+15 -79
View File
@@ -1,8 +1,10 @@
//! Extended external extensions to futures::FutureExt
use std::marker::Unpin;
use futures::{
Future, FutureExt,
future::{try_join, try_join_all},
future::{select_ok, try_join, try_join_all, try_select},
};
pub trait BoolExt
@@ -16,8 +18,8 @@ where
fn or<B>(self, b: B) -> impl Future<Output = bool> + Send
where
B: Future<Output = bool> + Send,
Self: Sized;
B: Future<Output = bool> + Send + Unpin,
Self: Sized + Unpin;
}
impl<Fut> BoolExt for Fut
@@ -42,20 +44,20 @@ where
#[inline]
fn or<B>(self, b: B) -> impl Future<Output = bool> + Send
where
B: Future<Output = bool> + Send,
Self: Sized,
B: Future<Output = bool> + Send + Unpin,
Self: Sized + Unpin,
{
type Result = crate::Result<(), ()>;
let a = self.map(|a| (!a).then_some(()).ok_or(Result::Err(())));
let a = self.map(|a| a.then_some(()).ok_or(Result::Err(())));
let b = b.map(|b| (!b).then_some(()).ok_or(Result::Err(())));
let b = b.map(|b| b.then_some(()).ok_or(Result::Err(())));
try_join(a, b).map(|result| result.is_err())
try_select(a, b).map(|result| result.is_ok())
}
}
pub fn and<I, F>(args: I) -> impl Future<Output = bool> + Send
pub async fn and<I, F>(args: I) -> impl Future<Output = bool> + Send
where
I: Iterator<Item = F> + Send,
F: Future<Output = bool> + Send,
@@ -67,80 +69,14 @@ where
try_join_all(args).map(|result| result.is_ok())
}
pub fn or<I, F>(args: I) -> impl Future<Output = bool> + Send
pub async fn or<I, F>(args: I) -> impl Future<Output = bool> + Send
where
I: Iterator<Item = F> + Send,
F: Future<Output = bool> + Send,
F: Future<Output = bool> + Send + Unpin,
{
type Result = crate::Result<(), ()>;
let args = args.map(|a| a.map(|a| (!a).then_some(()).ok_or(Result::Err(()))));
let args = args.map(|a| a.map(|a| a.then_some(()).ok_or(Result::Err(()))));
try_join_all(args).map(|result| result.is_err())
}
#[cfg(test)]
mod tests {
use std::pin::pin;
use futures::future;
use super::{BoolExt, and, or};
#[tokio::test]
async fn bool_ext_truth_tables() {
for a in [false, true] {
for b in [false, true] {
assert_eq!(future::ready(a).and(future::ready(b)).await, a && b, "and({a}, {b})");
assert_eq!(future::ready(a).or(future::ready(b)).await, a || b, "or({a}, {b})");
}
}
assert!(
future::ready(false)
.or(future::ready(false))
.or(future::ready(true))
.await,
"chained or with trailing true must return true"
);
assert!(
!future::ready(false)
.or(future::ready(false))
.or(future::ready(false))
.await,
"chained or with all false must return false"
);
}
#[tokio::test]
async fn bool_ext_short_circuits() {
let pend = pin!(future::pending::<bool>());
assert!(
future::ready(true).or(pend).await,
"or must short-circuit on true without polling pending"
);
let pend = pin!(future::pending::<bool>());
assert!(
!future::ready(false).and(pend).await,
"and must short-circuit on false without polling pending"
);
}
#[tokio::test]
async fn free_and_or() {
assert!(and(std::iter::empty::<future::Ready<bool>>()).await, "empty and is true");
assert!(!or(std::iter::empty::<future::Ready<bool>>()).await, "empty or is false");
assert!(and([future::ready(true), future::ready(true)].into_iter()).await, "all true");
assert!(
!and([future::ready(true), future::ready(false)].into_iter()).await,
"and with false"
);
assert!(
or([future::ready(false), future::ready(true)].into_iter()).await,
"or with true"
);
assert!(!or([future::ready(false), future::ready(false)].into_iter()).await, "all false");
}
select_ok(args).map(|result| result.is_ok())
}
+19 -19
View File
@@ -1,4 +1,4 @@
use std::{borrow::Borrow, fmt::Debug, mem::size_of_val, sync::Arc};
use std::{borrow::Borrow, collections::HashMap, fmt::Debug, mem::size_of_val, sync::Arc};
pub use conduwuit::matrix::pdu::{ShortEventId, ShortId, ShortRoomId, ShortStateKey};
use conduwuit::{
@@ -72,14 +72,22 @@ impl Service {
where
I: Iterator<Item = &'a EventId> + Clone + Debug + Send + 'a,
{
let mut created: HashMap<OwnedEventId, ShortEventId> = HashMap::new();
event_ids
.clone()
.stream()
.get(&self.db.eventid_shorteventid)
.zip(event_ids.into_iter().stream())
.map(|(result, event_id)| match result {
.map(move |(result, event_id)| match result {
| Ok(ref short) => utils::u64_from_u8(short),
| Err(_) => self.create_shorteventid(event_id),
| Err(_) =>
if let Some(&short) = created.get(event_id) {
short
} else {
let short = self.create_shorteventid(event_id);
created.insert(event_id.to_owned(), short);
short
},
})
}
@@ -217,32 +225,24 @@ impl Service {
.map(Deserialized::deserialized)
}
/// Gets or creates a short state hash ID. The boolean indicates whether a
/// new short ID was created.
pub async fn get_or_create_shortstatehash(
&self,
state_hash: &[u8],
) -> (ShortStateHash, bool) {
const BUFSIZE: usize = size_of::<ShortStateHash>();
if let Ok(shortstatehash) = self
.db
/// Gets an existing short state hash ID without creating one.
pub async fn get_shortstatehash(&self, state_hash: &[u8]) -> Result<ShortStateHash> {
self.db
.statehash_shortstatehash
.get(state_hash)
.await
.deserialized()
{
return (shortstatehash, true);
}
}
let shortstatehash = self.services.globals.next_count().unwrap();
/// Publishes the mapping from state hash to short state hash after the
/// corresponding state diff has been written.
pub fn set_shortstatehash(&self, state_hash: &[u8], shortstatehash: ShortStateHash) {
const BUFSIZE: usize = size_of::<ShortStateHash>();
debug_assert!(size_of_val(&shortstatehash) == BUFSIZE, "buffer requirement changed");
self.db
.statehash_shortstatehash
.raw_aput::<BUFSIZE, _, _>(state_hash, shortstatehash);
(shortstatehash, false)
}
/// Gets a short room ID.
+43 -40
View File
@@ -170,47 +170,50 @@ impl Service {
let state_hash = calculate_hash(state_ids_compressed.iter().map(|s| &s[..]));
let (shortstatehash, already_existed) = self
.services
.short
.get_or_create_shortstatehash(&state_hash)
.await;
if !already_existed {
let states_parents = match previous_shortstatehash {
| Ok(p) =>
self.services
.state_compressor
.load_shortstatehash_info(p)
.await?,
| _ => Vec::new(),
};
let (statediffnew, statediffremoved) =
if let Some(parent_stateinfo) = states_parents.last() {
let statediffnew: CompressedState = state_ids_compressed
.difference(&parent_stateinfo.full_state)
.copied()
.collect();
let statediffremoved: CompressedState = parent_stateinfo
.full_state
.difference(&state_ids_compressed)
.copied()
.collect();
(Arc::new(statediffnew), Arc::new(statediffremoved))
} else {
(state_ids_compressed, Arc::new(CompressedState::new()))
let shortstatehash =
if let Ok(existing) = self.services.short.get_shortstatehash(&state_hash).await {
existing
} else {
let states_parents = match previous_shortstatehash {
| Ok(p) =>
self.services
.state_compressor
.load_shortstatehash_info(p)
.await?,
| _ => Vec::new(),
};
self.services.state_compressor.save_state_from_diff(
shortstatehash,
statediffnew,
statediffremoved,
1_000_000, // high number because no state will be based on this one
states_parents,
)?;
}
let (statediffnew, statediffremoved) =
if let Some(parent_stateinfo) = states_parents.last() {
let statediffnew: CompressedState = state_ids_compressed
.difference(&parent_stateinfo.full_state)
.copied()
.collect();
let statediffremoved: CompressedState = parent_stateinfo
.full_state
.difference(&state_ids_compressed)
.copied()
.collect();
(Arc::new(statediffnew), Arc::new(statediffremoved))
} else {
(state_ids_compressed, Arc::new(CompressedState::new()))
};
let new_shortstatehash = self.services.globals.next_count()?;
self.services.state_compressor.save_state_from_diff(
new_shortstatehash,
statediffnew,
statediffremoved,
1_000_000, // high number because no state will be based on this one
states_parents,
)?;
self.services
.short
.set_shortstatehash(&state_hash, new_shortstatehash);
new_shortstatehash
};
self.db
.shorteventid_shortstatehash
+31 -15
View File
@@ -18,7 +18,7 @@ use lru_cache::LruCache;
use ruma::{EventId, RoomId};
use crate::{
Dep, rooms,
Dep, globals, rooms,
rooms::short::{ShortEventId, ShortId, ShortStateHash, ShortStateKey},
};
@@ -29,6 +29,7 @@ pub struct Service {
}
struct Services {
globals: Dep<globals::Service>,
short: Dep<rooms::short::Service>,
state: Dep<rooms::state::Service>,
}
@@ -78,6 +79,7 @@ impl crate::Service for Service {
shortstatehash_statediff: args.db["shortstatehash_statediff"].clone(),
},
services: Services {
globals: args.depend::<globals::Service>("globals"),
short: args.depend::<rooms::short::Service>("rooms::short"),
state: args.depend::<rooms::state::Service>("rooms::state"),
},
@@ -368,17 +370,17 @@ impl Service {
let state_hash =
utils::calculate_hash(new_state_ids_compressed.iter().map(|bytes| &bytes[..]));
let (new_shortstatehash, already_existed) = self
let existing = self
.services
.short
.get_or_create_shortstatehash(&state_hash)
.await;
.get_shortstatehash(&state_hash)
.await
.ok();
if Some(new_shortstatehash) == previous_shortstatehash {
return Ok(HashSetCompressStateEvent {
shortstatehash: new_shortstatehash,
..Default::default()
});
if let Some(shortstatehash) = existing
&& Some(shortstatehash) == previous_shortstatehash
{
return Ok(HashSetCompressStateEvent { shortstatehash, ..Default::default() });
}
let states_parents = if let Some(p) = previous_shortstatehash {
@@ -405,7 +407,10 @@ impl Service {
(new_state_ids_compressed, Arc::new(CompressedState::new()))
};
if !already_existed {
let new_shortstatehash = if let Some(existing) = existing {
existing
} else {
let new_shortstatehash = self.services.globals.next_count()?;
self.save_state_from_diff(
new_shortstatehash,
statediffnew.clone(),
@@ -413,7 +418,11 @@ impl Service {
2, // every state change is 2 event changes on average
states_parents,
)?;
}
self.services
.short
.set_shortstatehash(&state_hash, new_shortstatehash);
new_shortstatehash
};
Ok(HashSetCompressStateEvent {
shortstatehash: new_shortstatehash,
@@ -437,7 +446,10 @@ impl Service {
err!(Database("Failed to find StateDiff from short {shortstatehash:?}: {e}"))
})?;
let parent = utils::u64_from_bytes(&value[0..size_of::<u64>()])
let parent_bytes = value
.get(0..size_of::<u64>())
.ok_or_else(|| err!(Database("Truncated StateDiff for short {shortstatehash:?}")))?;
let parent = utils::u64_from_bytes(parent_bytes)
.ok()
.take_if(|parent| *parent != 0);
@@ -472,10 +484,14 @@ impl Service {
/// Stores a new state diff with a shortstatehash.
fn save_statediff(&self, shortstatehash: ShortStateHash, diff: &StateDiff) {
const STRIDE: usize = size_of::<ShortStateHash>();
let mut value = Vec::<u8>::with_capacity(
2_usize
.saturating_add(diff.added.len())
.saturating_add(diff.removed.len()),
1_usize
.saturating_add(usize::from(!diff.removed.is_empty()))
.saturating_add(diff.added.len().saturating_mul(2))
.saturating_add(diff.removed.len().saturating_mul(2))
.saturating_mul(STRIDE),
);
let parent = diff.parent.unwrap_or(0_u64);