From c4f5a0a6316200f797bf81d77c3cd2815a73706d Mon Sep 17 00:00:00 2001 From: Devin Ragotzy Date: Thu, 6 Aug 2020 08:29:59 -0400 Subject: [PATCH 01/51] Keep track of State at event for state resolution feat: first steps towards joining rooms over federation Add state-res as a dependency of conduit Add reverse_topological_power_sort before append_pdu Implement statehashstatid_pduid tree for keeping track of state Clean up implementation of state_hash as key for tracking state --- Cargo.lock | 206 +++++++++++- Cargo.toml | 7 +- src/client_server/account.rs | 6 +- src/client_server/alias.rs | 16 +- src/client_server/context.rs | 30 +- src/client_server/device.rs | 8 +- src/client_server/directory.rs | 13 +- src/client_server/filter.rs | 21 +- src/client_server/keys.rs | 2 +- src/client_server/media.rs | 4 +- src/client_server/membership.rs | 125 ++++++-- src/client_server/message.rs | 43 +-- src/client_server/room.rs | 2 +- src/client_server/session.rs | 20 +- src/client_server/state.rs | 48 ++- src/client_server/sync.rs | 2 +- src/client_server/unversioned.rs | 13 +- src/database.rs | 5 +- src/database/rooms.rs | 525 +++++++++++++++++++++---------- src/database/uiaa.rs | 6 +- src/pdu.rs | 29 ++ src/ruma_wrapper.rs | 10 +- src/server_server.rs | 27 +- src/utils.rs | 6 + 24 files changed, 818 insertions(+), 356 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 0a7334ca0..ffee8ea59 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -75,6 +75,15 @@ dependencies = [ "opaque-debug 0.2.3", ] +[[package]] +name = "ansi_term" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d52a9bb7ec0cf484c551830a7ce27bd20d67eac647e1befb56b0be4ee39a55d2" +dependencies = [ + "winapi 0.3.9", +] + [[package]] name = "arc-swap" version = "0.4.7" @@ -248,6 +257,17 @@ version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822" +[[package]] +name = "chrono" +version = "0.4.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "942f72db697d8767c22d46a598e01f2d3b475501ea43d0db4f16d90259182d0b" +dependencies = [ + "num-integer", + "num-traits", + "time 0.1.43", +] + [[package]] name = "cloudabi" version = "0.1.0" @@ -281,6 +301,7 @@ dependencies = [ "serde", "serde_json", "sled", + "state-res", "thiserror", "tokio", ] @@ -456,6 +477,12 @@ version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "134951f4028bdadb9b84baf4232681efbf277da25144b9b0ad65df75946c422b" +[[package]] +name = "either" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd56b59865bce947ac5958779cfa508f6c3b9497cc762b7e24a12d11ccde2c4f" + [[package]] name = "encoding_rs" version = "0.8.23" @@ -872,6 +899,15 @@ version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "47be2f14c678be2fdcab04ab1171db51b2762ce6f0a8ee87c8dd4a04ed216135" +[[package]] +name = "itertools" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "284f18f85651fe11e8a991b2adb42cb078325c996ed026d994719efcfca1d54b" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "0.4.6" @@ -951,6 +987,21 @@ version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7d947cbb889ed21c2a84be6ffbaebf5b4e0f4340638cba0444907e38b56be084" +[[package]] +name = "maplit" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e2e65a1a2e43cfcb47a895c4c8b10d1f4a61097f9f254f183aee60cad9c651d" + +[[package]] +name = "matchers" +version = "0.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f099785f7595cc4b4553a174ce30dd7589ef93391ff414dbb67f62392b9e0ce1" +dependencies = [ + "regex-automata", +] + [[package]] name = "matches" version = "0.1.8" @@ -1439,6 +1490,31 @@ dependencies = [ "syn", ] +[[package]] +name = "regex" +version = "1.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c3780fcf44b193bc4d09f36d2a3c87b251da4a046c87795a0d35f4f927ad8e6" +dependencies = [ + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae1ded71d66a4a97f5e961fd0cb25a5f366a42a41570d16a763a69c092c26ae4" +dependencies = [ + "byteorder", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.6.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26412eb97c6b088a6997e05f69403a802a92d520de2f8e63c2b65f9e0f47c4e8" + [[package]] name = "remove_dir_all" version = "0.5.3" @@ -1560,21 +1636,23 @@ dependencies = [ [[package]] name = "ruma" version = "0.0.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fixes#c2adc9ecb85538505ff351dbd883c9106f651744" +source = "git+https://github.com/ruma/ruma?rev=aff914050eb297bd82b8aafb12158c88a9e480e1#aff914050eb297bd82b8aafb12158c88a9e480e1" dependencies = [ "ruma-api", + "ruma-appservice-api", "ruma-client-api", "ruma-common", "ruma-events", "ruma-federation-api", "ruma-identifiers", + "ruma-serde", "ruma-signatures", ] [[package]] name = "ruma-api" version = "0.17.0-alpha.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fixes#c2adc9ecb85538505ff351dbd883c9106f651744" +source = "git+https://github.com/ruma/ruma?rev=aff914050eb297bd82b8aafb12158c88a9e480e1#aff914050eb297bd82b8aafb12158c88a9e480e1" dependencies = [ "http", "percent-encoding", @@ -1589,7 +1667,7 @@ dependencies = [ [[package]] name = "ruma-api-macros" version = "0.17.0-alpha.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fixes#c2adc9ecb85538505ff351dbd883c9106f651744" +source = "git+https://github.com/ruma/ruma?rev=aff914050eb297bd82b8aafb12158c88a9e480e1#aff914050eb297bd82b8aafb12158c88a9e480e1" dependencies = [ "proc-macro-crate", "proc-macro2", @@ -1597,14 +1675,28 @@ dependencies = [ "syn", ] +[[package]] +name = "ruma-appservice-api" +version = "0.2.0-alpha.1" +source = "git+https://github.com/ruma/ruma?rev=aff914050eb297bd82b8aafb12158c88a9e480e1#aff914050eb297bd82b8aafb12158c88a9e480e1" +dependencies = [ + "ruma-api", + "ruma-common", + "ruma-events", + "ruma-identifiers", + "serde", + "serde_json", +] + [[package]] name = "ruma-client-api" version = "0.10.0-alpha.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fixes#c2adc9ecb85538505ff351dbd883c9106f651744" +source = "git+https://github.com/ruma/ruma?rev=aff914050eb297bd82b8aafb12158c88a9e480e1#aff914050eb297bd82b8aafb12158c88a9e480e1" dependencies = [ "assign", "http", "js_int", + "percent-encoding", "ruma-api", "ruma-common", "ruma-events", @@ -1618,7 +1710,7 @@ dependencies = [ [[package]] name = "ruma-common" version = "0.2.0" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fixes#c2adc9ecb85538505ff351dbd883c9106f651744" +source = "git+https://github.com/ruma/ruma?rev=aff914050eb297bd82b8aafb12158c88a9e480e1#aff914050eb297bd82b8aafb12158c88a9e480e1" dependencies = [ "js_int", "ruma-identifiers", @@ -1631,7 +1723,7 @@ dependencies = [ [[package]] name = "ruma-events" version = "0.22.0-alpha.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fixes#c2adc9ecb85538505ff351dbd883c9106f651744" +source = "git+https://github.com/ruma/ruma?rev=aff914050eb297bd82b8aafb12158c88a9e480e1#aff914050eb297bd82b8aafb12158c88a9e480e1" dependencies = [ "js_int", "ruma-common", @@ -1646,7 +1738,7 @@ dependencies = [ [[package]] name = "ruma-events-macros" version = "0.22.0-alpha.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fixes#c2adc9ecb85538505ff351dbd883c9106f651744" +source = "git+https://github.com/ruma/ruma?rev=aff914050eb297bd82b8aafb12158c88a9e480e1#aff914050eb297bd82b8aafb12158c88a9e480e1" dependencies = [ "proc-macro-crate", "proc-macro2", @@ -1657,7 +1749,7 @@ dependencies = [ [[package]] name = "ruma-federation-api" version = "0.0.3" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fixes#c2adc9ecb85538505ff351dbd883c9106f651744" +source = "git+https://github.com/ruma/ruma?rev=aff914050eb297bd82b8aafb12158c88a9e480e1#aff914050eb297bd82b8aafb12158c88a9e480e1" dependencies = [ "js_int", "ruma-api", @@ -1672,7 +1764,7 @@ dependencies = [ [[package]] name = "ruma-identifiers" version = "0.17.4" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fixes#c2adc9ecb85538505ff351dbd883c9106f651744" +source = "git+https://github.com/ruma/ruma?rev=aff914050eb297bd82b8aafb12158c88a9e480e1#aff914050eb297bd82b8aafb12158c88a9e480e1" dependencies = [ "rand", "ruma-identifiers-macros", @@ -1684,7 +1776,7 @@ dependencies = [ [[package]] name = "ruma-identifiers-macros" version = "0.17.4" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fixes#c2adc9ecb85538505ff351dbd883c9106f651744" +source = "git+https://github.com/ruma/ruma?rev=aff914050eb297bd82b8aafb12158c88a9e480e1#aff914050eb297bd82b8aafb12158c88a9e480e1" dependencies = [ "proc-macro2", "quote", @@ -1695,7 +1787,7 @@ dependencies = [ [[package]] name = "ruma-identifiers-validation" version = "0.1.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fixes#c2adc9ecb85538505ff351dbd883c9106f651744" +source = "git+https://github.com/ruma/ruma?rev=aff914050eb297bd82b8aafb12158c88a9e480e1#aff914050eb297bd82b8aafb12158c88a9e480e1" dependencies = [ "ruma-serde", "serde", @@ -1706,7 +1798,7 @@ dependencies = [ [[package]] name = "ruma-serde" version = "0.2.3" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fixes#c2adc9ecb85538505ff351dbd883c9106f651744" +source = "git+https://github.com/ruma/ruma?rev=aff914050eb297bd82b8aafb12158c88a9e480e1#aff914050eb297bd82b8aafb12158c88a9e480e1" dependencies = [ "form_urlencoded", "itoa", @@ -1718,7 +1810,7 @@ dependencies = [ [[package]] name = "ruma-signatures" version = "0.6.0-dev.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fixes#c2adc9ecb85538505ff351dbd883c9106f651744" +source = "git+https://github.com/ruma/ruma?rev=aff914050eb297bd82b8aafb12158c88a9e480e1#aff914050eb297bd82b8aafb12158c88a9e480e1" dependencies = [ "base64 0.12.3", "ring", @@ -1910,6 +2002,15 @@ dependencies = [ "opaque-debug 0.3.0", ] +[[package]] +name = "sharded-slab" +version = "0.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06d5a3f5166fb5b42a5439f2eee8b9de149e235961e3eb21c5808fc3ea17ff3e" +dependencies = [ + "lazy_static", +] + [[package]] name = "signal-hook-registry" version = "1.2.1" @@ -1983,6 +2084,22 @@ version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7345c971d1ef21ffdbd103a75990a15eb03604fc8b8852ca8cb418ee1a099028" +[[package]] +name = "state-res" +version = "0.1.0" +source = "git+https://github.com/ruma/state-res#789c8140890e076d38b23fa1147c4ff0500c0d38" +dependencies = [ + "itertools", + "js_int", + "maplit", + "ruma", + "serde", + "serde_json", + "thiserror", + "tracing", + "tracing-subscriber", +] + [[package]] name = "stdweb" version = "0.4.20" @@ -2104,6 +2221,15 @@ dependencies = [ "syn", ] +[[package]] +name = "thread_local" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d40c6d1b69745a6ec6fb1ca717914848da4b44ae29d9b3080cbee91d72a69b14" +dependencies = [ + "lazy_static", +] + [[package]] name = "time" version = "0.1.43" @@ -2251,9 +2377,21 @@ checksum = "6d79ca061b032d6ce30c660fded31189ca0b9922bf483cd70759f13a2d86786c" dependencies = [ "cfg-if", "log", + "tracing-attributes", "tracing-core", ] +[[package]] +name = "tracing-attributes" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fe233f4227389ab7df5b32649239da7ebe0b281824b4e84b342d04d3fd8c25e" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "tracing-core" version = "0.1.14" @@ -2263,6 +2401,48 @@ dependencies = [ "lazy_static", ] +[[package]] +name = "tracing-log" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e0f8c7178e13481ff6765bd169b33e8d554c5d2bbede5e32c356194be02b9b9" +dependencies = [ + "lazy_static", + "log", + "tracing-core", +] + +[[package]] +name = "tracing-serde" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6ccba2f8f16e0ed268fc765d9b7ff22e965e7185d32f8f1ec8294fe17d86e79" +dependencies = [ + "serde", + "tracing-core", +] + +[[package]] +name = "tracing-subscriber" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "abd165311cc4d7a555ad11cc77a37756df836182db0d81aac908c8184c584f40" +dependencies = [ + "ansi_term", + "chrono", + "lazy_static", + "matchers", + "regex", + "serde", + "serde_json", + "sharded-slab", + "smallvec", + "thread_local", + "tracing-core", + "tracing-log", + "tracing-serde", +] + [[package]] name = "try-lock" version = "0.2.3" diff --git a/Cargo.toml b/Cargo.toml index 4945e3c88..4c14d712b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -16,9 +16,7 @@ edition = "2018" #rocket = { git = "https://github.com/SergioBenitez/Rocket.git", rev = "8d779caa22c63b15a6c3ceb75d8f6d4971b2eb67", features = ["tls"] } # Used to handle requests rocket = { git = "https://github.com/timokoesters/Rocket.git", branch = "empty_parameters", features = ["tls"] } -#ruma = { git = "https://github.com/ruma/ruma", features = ["rand", "client-api", "federation-api", "unstable-pre-spec", "unstable-synapse-quirks"], rev = "987d48666cf166cf12100b5dbc61b5e3385c4014" } # Used for matrix spec type definitions and helpers -ruma = { git = "https://github.com/timokoesters/ruma", features = ["rand", "client-api", "federation-api", "unstable-pre-spec", "unstable-synapse-quirks"], branch = "timo-fixes" } # Used for matrix spec type definitions and helpers -#ruma = { path = "../ruma/ruma", features = ["rand", "client-api", "federation-api", "unstable-pre-spec", "unstable-synapse-quirks"] } +ruma = { git = "https://github.com/ruma/ruma", features = ["rand", "client-api", "federation-api", "unstable-pre-spec", "unstable-synapse-quirks"], rev = "aff914050eb297bd82b8aafb12158c88a9e480e1" } # Used for matrix spec type definitions and helpers tokio = "0.2.22" # Used for long polling sled = "0.32.0" # Used for storing data permanently log = "0.4.8" # Used for emitting log entries @@ -33,6 +31,9 @@ reqwest = "0.10.6" # Used to send requests thiserror = "1.0.19" # Used for conduit::Error type image = { version = "0.23.4", default-features = false, features = ["jpeg", "png", "gif"] } # Used to generate thumbnails for images base64 = "0.12.3" # Used to encode server public key +# state-res = { path = "../../state-res" } +state-res = { git = "https://github.com/ruma/state-res", version = "0.1.0" } + [features] default = ["conduit_bin"] diff --git a/src/client_server/account.rs b/src/client_server/account.rs index 9837d1b95..9fa1a9c74 100644 --- a/src/client_server/account.rs +++ b/src/client_server/account.rs @@ -75,7 +75,7 @@ pub fn get_register_available_route( )] pub fn register_route( db: State<'_, Database>, - body: Ruma, + body: Ruma, ) -> ConduitResult { if db.globals.registration_disabled() { return Err(Error::BadRequest( @@ -223,7 +223,7 @@ pub fn register_route( )] pub fn change_password_route( db: State<'_, Database>, - body: Ruma, + body: Ruma, ) -> ConduitResult { let sender_id = body.sender_id.as_ref().expect("user is authenticated"); let device_id = body.device_id.as_ref().expect("user is authenticated"); @@ -305,7 +305,7 @@ pub fn whoami_route(body: Ruma) -> ConduitResult, - body: Ruma, + body: Ruma, ) -> ConduitResult { let sender_id = body.sender_id.as_ref().expect("user is authenticated"); let device_id = body.device_id.as_ref().expect("user is authenticated"); diff --git a/src/client_server/alias.rs b/src/client_server/alias.rs index a02938816..7dc90783d 100644 --- a/src/client_server/alias.rs +++ b/src/client_server/alias.rs @@ -26,7 +26,7 @@ pub fn create_alias_route( db.rooms .set_alias(&body.room_alias, Some(&body.room_id), &db.globals)?; - Ok(create_alias::Response.into()) + Ok(create_alias::Response::new().into()) } #[cfg_attr( @@ -39,7 +39,7 @@ pub fn delete_alias_route( ) -> ConduitResult { db.rooms.set_alias(&body.room_alias, None, &db.globals)?; - Ok(delete_alias::Response.into()) + Ok(delete_alias::Response::new().into()) } #[cfg_attr( @@ -60,11 +60,7 @@ pub async fn get_alias_route( ) .await?; - return Ok(get_alias::Response { - room_id: response.room_id, - servers: response.servers, - } - .into()); + return Ok(get_alias::Response::new(response.room_id, response.servers).into()); } let room_id = db @@ -75,9 +71,5 @@ pub async fn get_alias_route( "Room with alias not found.", ))?; - Ok(get_alias::Response { - room_id, - servers: vec![db.globals.server_name().to_string()], - } - .into()) + Ok(get_alias::Response::new(room_id, vec![db.globals.server_name().to_string()]).into()) } diff --git a/src/client_server/context.rs b/src/client_server/context.rs index 7a6cbce0a..7b1fad917 100644 --- a/src/client_server/context.rs +++ b/src/client_server/context.rs @@ -12,7 +12,7 @@ use rocket::get; )] pub fn get_context_route( db: State<'_, Database>, - body: Ruma, + body: Ruma, ) -> ConduitResult { let sender_id = body.sender_id.as_ref().expect("user is authenticated"); @@ -75,18 +75,18 @@ pub fn get_context_route( .map(|(_, pdu)| pdu.to_room_event()) .collect::>(); - Ok(get_context::Response { - start: start_token, - end: end_token, - events_before, - event: Some(base_event), - events_after, - state: db // TODO: State at event - .rooms - .room_state_full(&body.room_id)? - .values() - .map(|pdu| pdu.to_state_event()) - .collect(), - } - .into()) + let mut resp = get_context::Response::new(); + resp.start = start_token; + resp.end = end_token; + resp.events_before = events_before; + resp.event = Some(base_event); + resp.events_after = events_after; + resp.state = db // TODO: State at event + .rooms + .room_state_full(&body.room_id)? + .values() + .map(|pdu| pdu.to_state_event()) + .collect(); + + Ok(resp.into()) } diff --git a/src/client_server/device.rs b/src/client_server/device.rs index 379f827ea..89033f06f 100644 --- a/src/client_server/device.rs +++ b/src/client_server/device.rs @@ -37,7 +37,7 @@ pub fn get_devices_route( )] pub fn get_device_route( db: State<'_, Database>, - body: Ruma, + body: Ruma, _device_id: String, ) -> ConduitResult { let sender_id = body.sender_id.as_ref().expect("user is authenticated"); @@ -56,7 +56,7 @@ pub fn get_device_route( )] pub fn update_device_route( db: State<'_, Database>, - body: Ruma, + body: Ruma, _device_id: String, ) -> ConduitResult { let sender_id = body.sender_id.as_ref().expect("user is authenticated"); @@ -80,7 +80,7 @@ pub fn update_device_route( )] pub fn delete_device_route( db: State<'_, Database>, - body: Ruma, + body: Ruma, _device_id: String, ) -> ConduitResult { let sender_id = body.sender_id.as_ref().expect("user is authenticated"); @@ -127,7 +127,7 @@ pub fn delete_device_route( )] pub fn delete_devices_route( db: State<'_, Database>, - body: Ruma, + body: Ruma, ) -> ConduitResult { let sender_id = body.sender_id.as_ref().expect("user is authenticated"); let device_id = body.device_id.as_ref().expect("user is authenticated"); diff --git a/src/client_server/directory.rs b/src/client_server/directory.rs index 26188f733..0aace15d1 100644 --- a/src/client_server/directory.rs +++ b/src/client_server/directory.rs @@ -6,7 +6,7 @@ use ruma::{ error::ErrorKind, r0::{ directory::{ - self, get_public_rooms, get_public_rooms_filtered, get_room_visibility, + get_public_rooms, get_public_rooms_filtered, get_room_visibility, set_room_visibility, }, room, @@ -14,6 +14,7 @@ use ruma::{ }, federation, }, + directory::PublicRoomsChunk, events::{ room::{avatar, canonical_alias, guest_access, history_visibility, name, topic}, EventType, @@ -35,15 +36,15 @@ pub async fn get_public_rooms_filtered_route( if let Some(other_server) = body .server .clone() - .filter(|server| server != &db.globals.server_name().as_str()) + .filter(|server| server != db.globals.server_name().as_str()) { let response = server_server::send_request( &db, other_server, federation::directory::get_public_rooms::v1::Request { limit: body.limit, - since: body.since.clone(), - room_network: federation::directory::get_public_rooms::v1::RoomNetwork::Matrix, + since: body.since.as_deref(), + room_network: ruma::directory::RoomNetwork::Matrix, }, ) .await?; @@ -107,7 +108,7 @@ pub async fn get_public_rooms_filtered_route( // TODO: Do not load full state? let state = db.rooms.room_state_full(&room_id)?; - let chunk = directory::PublicRoomsChunk { + let chunk = PublicRoomsChunk { aliases: Vec::new(), canonical_alias: state .get(&(EventType::RoomCanonicalAlias, "".to_owned())) @@ -272,7 +273,7 @@ pub async fn get_public_rooms_route( body: get_public_rooms_filtered::IncomingRequest { filter: None, limit, - room_network: get_public_rooms_filtered::RoomNetwork::Matrix, + room_network: ruma::directory::RoomNetwork::Matrix, server, since, }, diff --git a/src/client_server/filter.rs b/src/client_server/filter.rs index 165419a81..4322de3e5 100644 --- a/src/client_server/filter.rs +++ b/src/client_server/filter.rs @@ -7,23 +7,18 @@ use rocket::{get, post}; #[cfg_attr(feature = "conduit_bin", get("/_matrix/client/r0/user/<_>/filter/<_>"))] pub fn get_filter_route() -> ConduitResult { // TODO - Ok(get_filter::Response { - filter: filter::FilterDefinition { - event_fields: None, - event_format: None, - account_data: None, - room: None, - presence: None, - }, - } + Ok(get_filter::Response::new(filter::FilterDefinition { + event_fields: None, + event_format: None, + account_data: None, + room: None, + presence: None, + }) .into()) } #[cfg_attr(feature = "conduit_bin", post("/_matrix/client/r0/user/<_>/filter"))] pub fn create_filter_route() -> ConduitResult { // TODO - Ok(create_filter::Response { - filter_id: utils::random_string(10), - } - .into()) + Ok(create_filter::Response::new(utils::random_string(10)).into()) } diff --git a/src/client_server/keys.rs b/src/client_server/keys.rs index f88878ce6..331152964 100644 --- a/src/client_server/keys.rs +++ b/src/client_server/keys.rs @@ -167,7 +167,7 @@ pub fn claim_keys_route( )] pub fn upload_signing_keys_route( db: State<'_, Database>, - body: Ruma, + body: Ruma, ) -> ConduitResult { let sender_id = body.sender_id.as_ref().expect("user is authenticated"); let device_id = body.device_id.as_ref().expect("user is authenticated"); diff --git a/src/client_server/media.rs b/src/client_server/media.rs index efcb3a6d3..79c1f080a 100644 --- a/src/client_server/media.rs +++ b/src/client_server/media.rs @@ -53,7 +53,7 @@ pub fn create_content_route( )] pub fn get_content_route( db: State<'_, Database>, - body: Ruma, + body: Ruma, _server_name: String, _media_id: String, ) -> ConduitResult { @@ -85,7 +85,7 @@ pub fn get_content_route( )] pub fn get_content_thumbnail_route( db: State<'_, Database>, - body: Ruma, + body: Ruma, _server_name: String, _media_id: String, ) -> ConduitResult { diff --git a/src/client_server/membership.rs b/src/client_server/membership.rs index 84c0ebd31..c04cf7f6c 100644 --- a/src/client_server/membership.rs +++ b/src/client_server/membership.rs @@ -20,6 +20,8 @@ use ruma::{ events::{room::member, EventType}, EventId, Raw, RoomId, RoomVersionId, }; +use state_res::StateEvent; + use std::{collections::BTreeMap, convert::TryFrom}; #[cfg(feature = "conduit_bin")] @@ -92,17 +94,73 @@ pub async fn join_room_by_id_route( let send_join_response = server_server::send_request( &db, body.room_id.server_name().to_string(), - federation::membership::create_join_event::v2::Request { + federation::membership::create_join_event::v1::Request { room_id: body.room_id.clone(), event_id, - pdu_stub: serde_json::from_value::>(join_event_stub_value) + pdu_stub: serde_json::from_value(join_event_stub_value) .expect("Raw::from_value always works"), }, ) .await?; - dbg!(send_join_response); - todo!("Take send_join_response and 'create' the room using that data"); + dbg!(&send_join_response); + // todo!("Take send_join_response and 'create' the room using that data"); + + let mut event_map = send_join_response + .room_state + .state + .iter() + .map(|pdu| pdu.deserialize().map(StateEvent::Full)) + .map(|ev| { + let ev = ev?; + Ok::<_, serde_json::Error>((ev.event_id(), ev)) + }) + .collect::, _>>() + .map_err(|_| Error::bad_database("Invalid PDU found in db."))?; + + let _auth_chain = send_join_response + .room_state + .auth_chain + .iter() + .flat_map(|pdu| pdu.deserialize().ok()) + .map(StateEvent::Full) + .collect::>(); + + // TODO make StateResolution's methods free functions ? or no self param ? + let sorted_events_ids = state_res::StateResolution::default() + .reverse_topological_power_sort( + &body.room_id, + &event_map.keys().cloned().collect::>(), + &mut event_map, + &db.rooms, + &[], // TODO auth_diff: is this none since we have a set of resolved events we only want to sort + ); + + for ev_id in &sorted_events_ids { + // this is a `state_res::StateEvent` that holds a `ruma::Pdu` + let pdu = event_map.get(ev_id).ok_or_else(|| { + Error::Conflict("Found event_id in sorted events that is not in resolved state") + })?; + + db.rooms.append_pdu( + PduBuilder { + room_id: pdu.room_id().unwrap_or(&body.room_id).clone(), + sender: pdu.sender().clone(), + event_type: pdu.kind(), + content: pdu.content().clone(), + unsigned: Some( + pdu.unsigned() + .iter() + .map(|(k, v)| (k.clone(), v.clone())) + .collect(), + ), + state_key: pdu.state_key(), + redacts: pdu.redacts().cloned(), + }, + &db.globals, + &db.account_data, + )?; + } } let event = member::MemberEventContent { @@ -127,10 +185,7 @@ pub async fn join_room_by_id_route( &db.account_data, )?; - Ok(join_room_by_id::Response { - room_id: body.room_id.clone(), - } - .into()) + Ok(join_room_by_id::Response::new(body.room_id.clone()).into()) } #[cfg_attr( @@ -140,7 +195,7 @@ pub async fn join_room_by_id_route( pub async fn join_room_by_id_or_alias_route( db: State<'_, Database>, db2: State<'_, Database>, - body: Ruma, + body: Ruma, ) -> ConduitResult { let room_id = match RoomId::try_from(body.room_id_or_alias.clone()) { Ok(room_id) => room_id, @@ -148,7 +203,13 @@ pub async fn join_room_by_id_or_alias_route( client_server::get_alias_route( db, Ruma { - body: alias::get_alias::IncomingRequest { room_alias }, + body: alias::get_alias::IncomingRequest::try_from(http::Request::new( + serde_json::json!({ "room_alias": room_alias }) + .to_string() + .as_bytes() + .to_vec(), + )) + .unwrap(), sender_id: body.sender_id.clone(), device_id: body.device_id.clone(), json_body: None, @@ -160,14 +221,32 @@ pub async fn join_room_by_id_or_alias_route( } }; + // TODO ruma needs to implement the same constructors for the Incoming variants + let tps = if let Some(in_tps) = &body.third_party_signed { + Some(ruma::api::client::r0::membership::ThirdPartySigned { + token: &in_tps.token, + sender: &in_tps.sender, + signatures: in_tps.signatures.clone(), + mxid: &in_tps.mxid, + }) + } else { + None + }; + let body = Ruma { sender_id: body.sender_id.clone(), device_id: body.device_id.clone(), json_body: None, - body: join_room_by_id::IncomingRequest { - room_id, - third_party_signed: body.third_party_signed.clone(), - }, + body: join_room_by_id::IncomingRequest::try_from(http::Request::new( + serde_json::json!({ + "room_id": room_id, + "third_party_signed": tps, + }) + .to_string() + .as_bytes() + .to_vec(), + )) + .unwrap(), }; Ok(join_room_by_id_or_alias::Response { @@ -219,7 +298,7 @@ pub fn leave_room_route( &db.account_data, )?; - Ok(leave_room::Response.into()) + Ok(leave_room::Response::new().into()) } #[cfg_attr( @@ -266,7 +345,7 @@ pub fn invite_user_route( )] pub fn kick_user_route( db: State<'_, Database>, - body: Ruma, + body: Ruma, ) -> ConduitResult { let sender_id = body.sender_id.as_ref().expect("user is authenticated"); @@ -304,7 +383,7 @@ pub fn kick_user_route( &db.account_data, )?; - Ok(kick_user::Response.into()) + Ok(kick_user::Response::new().into()) } #[cfg_attr( @@ -313,7 +392,7 @@ pub fn kick_user_route( )] pub fn ban_user_route( db: State<'_, Database>, - body: Ruma, + body: Ruma, ) -> ConduitResult { let sender_id = body.sender_id.as_ref().expect("user is authenticated"); @@ -359,7 +438,7 @@ pub fn ban_user_route( &db.account_data, )?; - Ok(ban_user::Response.into()) + Ok(ban_user::Response::new().into()) } #[cfg_attr( @@ -368,7 +447,7 @@ pub fn ban_user_route( )] pub fn unban_user_route( db: State<'_, Database>, - body: Ruma, + body: Ruma, ) -> ConduitResult { let sender_id = body.sender_id.as_ref().expect("user is authenticated"); @@ -405,7 +484,7 @@ pub fn unban_user_route( &db.account_data, )?; - Ok(unban_user::Response.into()) + Ok(unban_user::Response::new().into()) } #[cfg_attr( @@ -414,13 +493,13 @@ pub fn unban_user_route( )] pub fn forget_room_route( db: State<'_, Database>, - body: Ruma, + body: Ruma, ) -> ConduitResult { let sender_id = body.sender_id.as_ref().expect("user is authenticated"); db.rooms.forget(&body.room_id, &sender_id)?; - Ok(forget_room::Response.into()) + Ok(forget_room::Response::new().into()) } #[cfg_attr( diff --git a/src/client_server/message.rs b/src/client_server/message.rs index d85121485..1b461d240 100644 --- a/src/client_server/message.rs +++ b/src/client_server/message.rs @@ -1,8 +1,11 @@ use super::State; use crate::{pdu::PduBuilder, ConduitResult, Database, Error, Ruma}; -use ruma::api::client::{ - error::ErrorKind, - r0::message::{get_message_events, send_message_event}, +use ruma::{ + api::client::{ + error::ErrorKind, + r0::message::{get_message_events, send_message_event}, + }, + events::EventContent, }; use std::convert::TryInto; @@ -26,7 +29,7 @@ pub fn send_message_event_route( PduBuilder { room_id: body.room_id.clone(), sender: sender_id.clone(), - event_type: body.event_type.clone(), + event_type: body.content.event_type().into(), content: serde_json::from_str( body.json_body .ok_or(Error::BadRequest(ErrorKind::BadJson, "Invalid JSON body."))? @@ -41,7 +44,7 @@ pub fn send_message_event_route( &db.account_data, )?; - Ok(send_message_event::Response { event_id }.into()) + Ok(send_message_event::Response::new(event_id).into()) } #[cfg_attr( @@ -50,7 +53,7 @@ pub fn send_message_event_route( )] pub fn get_message_events_route( db: State<'_, Database>, - body: Ruma, + body: Ruma, ) -> ConduitResult { let sender_id = body.sender_id.as_ref().expect("user is authenticated"); @@ -92,13 +95,13 @@ pub fn get_message_events_route( .map(|(_, pdu)| pdu.to_room_event()) .collect::>(); - Ok(get_message_events::Response { - start: Some(body.from.clone()), - end: end_token, - chunk: events_after, - state: Vec::new(), - } - .into()) + let mut resp = get_message_events::Response::new(); + resp.start = Some(body.from.clone()); + resp.end = end_token; + resp.chunk = events_after; + resp.state = Vec::new(); + + Ok(resp.into()) } get_message_events::Direction::Backward => { let events_before = db @@ -116,13 +119,13 @@ pub fn get_message_events_route( .map(|(_, pdu)| pdu.to_room_event()) .collect::>(); - Ok(get_message_events::Response { - start: Some(body.from.clone()), - end: start_token, - chunk: events_before, - state: Vec::new(), - } - .into()) + let mut resp = get_message_events::Response::new(); + resp.start = Some(body.from.clone()); + resp.end = start_token; + resp.chunk = events_before; + resp.state = Vec::new(); + + Ok(resp.into()) } } } diff --git a/src/client_server/room.rs b/src/client_server/room.rs index b5f1529e5..589a2dcdf 100644 --- a/src/client_server/room.rs +++ b/src/client_server/room.rs @@ -315,7 +315,7 @@ pub fn create_room_route( db.rooms.set_public(&room_id, true)?; } - Ok(create_room::Response { room_id }.into()) + Ok(create_room::Response::new(room_id).into()) } #[cfg_attr( diff --git a/src/client_server/session.rs b/src/client_server/session.rs index 401105882..948b455b7 100644 --- a/src/client_server/session.rs +++ b/src/client_server/session.rs @@ -1,5 +1,4 @@ -use super::State; -use super::{DEVICE_ID_LENGTH, TOKEN_LENGTH}; +use super::{State, DEVICE_ID_LENGTH, TOKEN_LENGTH}; use crate::{utils, ConduitResult, Database, Error, Ruma}; use ruma::{ api::client::{ @@ -18,10 +17,7 @@ use rocket::{get, post}; /// when logging in. #[cfg_attr(feature = "conduit_bin", get("/_matrix/client/r0/login"))] pub fn get_login_types_route() -> ConduitResult { - Ok(get_login_types::Response { - flows: vec![get_login_types::LoginType::Password], - } - .into()) + Ok(get_login_types::Response::new(vec![get_login_types::LoginType::Password]).into()) } /// # `POST /_matrix/client/r0/login` @@ -40,15 +36,15 @@ pub fn get_login_types_route() -> ConduitResult { )] pub fn login_route( db: State<'_, Database>, - body: Ruma, + body: Ruma, ) -> ConduitResult { // Validate login method let user_id = // TODO: Other login methods - if let (login::UserInfo::MatrixId(username), login::LoginInfo::Password { password }) = - (body.user.clone(), body.login_info.clone()) + if let (login::IncomingUserInfo::MatrixId(username), login::IncomingLoginInfo::Password { password }) = + (&body.user, &body.login_info) { - let user_id = UserId::parse_with_server_name(username, db.globals.server_name()) + let user_id = UserId::parse_with_server_name(username.to_string(), db.globals.server_name()) .map_err(|_| Error::BadRequest( ErrorKind::InvalidUsername, "Username is invalid." @@ -126,7 +122,7 @@ pub fn logout_route( db.users.remove_device(&sender_id, device_id)?; - Ok(logout::Response.into()) + Ok(logout::Response::new().into()) } /// # `POST /_matrix/client/r0/logout/all` @@ -154,5 +150,5 @@ pub fn logout_all_route( } } - Ok(logout_all::Response.into()) + Ok(logout_all::Response::new().into()) } diff --git a/src/client_server/state.rs b/src/client_server/state.rs index 60b3e9f11..14cc49727 100644 --- a/src/client_server/state.rs +++ b/src/client_server/state.rs @@ -8,9 +8,9 @@ use ruma::{ send_state_event_for_empty_key, send_state_event_for_key, }, }, - events::{room::canonical_alias, EventType}, - Raw, + events::{AnyStateEventContent, EventContent}, }; +use std::convert::TryFrom; #[cfg(feature = "conduit_bin")] use rocket::{get, put}; @@ -33,17 +33,10 @@ pub fn send_state_event_for_key_route( ) .map_err(|_| Error::BadRequest(ErrorKind::BadJson, "Invalid JSON body."))?; - if body.event_type == EventType::RoomCanonicalAlias { - let canonical_alias = serde_json::from_value::< - Raw, - >(content.clone()) - .expect("from_value::> can never fail") - .deserialize() - .map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "Invalid canonical alias."))?; + if let AnyStateEventContent::RoomCanonicalAlias(canonical_alias) = &body.content { + let mut aliases = canonical_alias.alt_aliases.clone(); - let mut aliases = canonical_alias.alt_aliases; - - if let Some(alias) = canonical_alias.alias { + if let Some(alias) = canonical_alias.alias.clone() { aliases.push(alias); } @@ -68,7 +61,7 @@ pub fn send_state_event_for_key_route( PduBuilder { room_id: body.room_id.clone(), sender: sender_id.clone(), - event_type: body.event_type.clone(), + event_type: body.content.event_type().into(), content, unsigned: None, state_key: Some(body.state_key.clone()), @@ -78,7 +71,7 @@ pub fn send_state_event_for_key_route( &db.account_data, )?; - Ok(send_state_event_for_key::Response { event_id }.into()) + Ok(send_state_event_for_key::Response::new(event_id).into()) } #[cfg_attr( @@ -93,25 +86,28 @@ pub fn send_state_event_for_empty_key_route( let Ruma { body: send_state_event_for_empty_key::IncomingRequest { - room_id, - event_type, - data, + room_id, content, .. }, sender_id, device_id, json_body, } = body; - Ok(send_state_event_for_empty_key::Response { - event_id: send_state_event_for_key_route( + Ok(send_state_event_for_empty_key::Response::new( + send_state_event_for_key_route( db, Ruma { - body: send_state_event_for_key::IncomingRequest { - room_id, - event_type, - data, - state_key: "".to_owned(), - }, + body: send_state_event_for_key::IncomingRequest::try_from(http::Request::new( + serde_json::json!({ + "room_id": room_id, + "state_key": "", + "content": content, + }) + .to_string() + .as_bytes() + .to_vec(), + )) + .unwrap(), sender_id, device_id, json_body, @@ -119,7 +115,7 @@ pub fn send_state_event_for_empty_key_route( )? .0 .event_id, - } + ) .into()) } diff --git a/src/client_server/sync.rs b/src/client_server/sync.rs index 2307f0281..ae4c22426 100644 --- a/src/client_server/sync.rs +++ b/src/client_server/sync.rs @@ -31,7 +31,7 @@ use std::{ )] pub async fn sync_events_route( db: State<'_, Database>, - body: Ruma, + body: Ruma, ) -> ConduitResult { let sender_id = body.sender_id.as_ref().expect("user is authenticated"); let device_id = body.device_id.as_ref().expect("user is authenticated"); diff --git a/src/client_server/unversioned.rs b/src/client_server/unversioned.rs index 3ff8bec2b..ea7f63381 100644 --- a/src/client_server/unversioned.rs +++ b/src/client_server/unversioned.rs @@ -1,6 +1,5 @@ use crate::ConduitResult; use ruma::api::client::unversioned::get_supported_versions; -use std::collections::BTreeMap; #[cfg(feature = "conduit_bin")] use rocket::get; @@ -17,13 +16,11 @@ use rocket::get; /// unstable features in their stable releases #[cfg_attr(feature = "conduit_bin", get("/_matrix/client/versions"))] pub fn get_supported_versions_route() -> ConduitResult { - let mut unstable_features = BTreeMap::new(); + let mut resp = + get_supported_versions::Response::new(vec!["r0.5.0".to_owned(), "r0.6.0".to_owned()]); - unstable_features.insert("org.matrix.e2e_cross_signing".to_owned(), true); + resp.unstable_features + .insert("org.matrix.e2e_cross_signing".to_owned(), true); - Ok(get_supported_versions::Response { - versions: vec!["r0.5.0".to_owned(), "r0.6.0".to_owned()], - unstable_features, - } - .into()) + Ok(resp.into()) } diff --git a/src/database.rs b/src/database.rs index 7bbb6dd7e..6cd65c316 100644 --- a/src/database.rs +++ b/src/database.rs @@ -97,8 +97,8 @@ impl Database { }, pduid_pdu: db.open_tree("pduid_pdu")?, eventid_pduid: db.open_tree("eventid_pduid")?, + roomstateid_pduid: db.open_tree("roomstateid_pduid")?, roomid_pduleaves: db.open_tree("roomid_pduleaves")?, - roomstateid_pdu: db.open_tree("roomstateid_pdu")?, alias_roomid: db.open_tree("alias_roomid")?, aliasid_alias: db.open_tree("alias_roomid")?, @@ -111,6 +111,9 @@ impl Database { userroomid_invited: db.open_tree("userroomid_invited")?, roomuserid_invited: db.open_tree("roomuserid_invited")?, userroomid_left: db.open_tree("userroomid_left")?, + + stateid_pduid: db.open_tree("stateid_pduid")?, + pduid_statehash: db.open_tree("pduid_statehash")?, }, account_data: account_data::AccountData { roomuserdataid_accountdata: db.open_tree("roomuserdataid_accountdata")?, diff --git a/src/database/rooms.rs b/src/database/rooms.rs index d2cd5e9fb..0d3632623 100644 --- a/src/database/rooms.rs +++ b/src/database/rooms.rs @@ -9,7 +9,7 @@ use ruma::{ events::{ ignored_user_list, room::{ - join_rules, member, + member, power_levels::{self, PowerLevelsEventContent}, redaction, }, @@ -18,19 +18,31 @@ use ruma::{ EventId, Raw, RoomAliasId, RoomId, UserId, }; use sled::IVec; +use state_res::{event_auth, Requester, StateEvent, StateMap, StateStore}; + use std::{ - collections::{BTreeMap, HashMap}, + collections::{hash_map::DefaultHasher, BTreeMap, HashMap}, convert::{TryFrom, TryInto}, + hash::{Hash, Hasher}, mem, + result::Result as StdResult, }; +/// The unique identifier of each state group. +/// +/// This is created when a state group is added to the database by +/// hashing the entire state. +pub type StateHashId = String; + +/// This identifier consists of roomId + count. It represents a +/// unique event, it will never be overwritten or removed. +pub type PduId = IVec; + pub struct Rooms { pub edus: edus::RoomEdus, pub(super) pduid_pdu: sled::Tree, // PduId = RoomId + Count pub(super) eventid_pduid: sled::Tree, pub(super) roomid_pduleaves: sled::Tree, - pub(super) roomstateid_pdu: sled::Tree, // RoomStateId = Room + StateType + StateKey - pub(super) alias_roomid: sled::Tree, pub(super) aliasid_alias: sled::Tree, // AliasId = RoomId + Count pub(super) publicroomids: sled::Tree, @@ -42,9 +54,263 @@ pub struct Rooms { pub(super) userroomid_invited: sled::Tree, pub(super) roomuserid_invited: sled::Tree, pub(super) userroomid_left: sled::Tree, + + // STATE TREES + /// This holds the full current state, including the latest event. + pub(super) roomstateid_pduid: sled::Tree, // RoomStateId = Room + StateType + StateKey + /// This holds the full room state minus the latest event. + pub(super) pduid_statehash: sled::Tree, // PDU id -> StateHash + /// Also holds the full room state minus the latest event. + pub(super) stateid_pduid: sled::Tree, // StateId = StateHash + (EventType, StateKey) } +impl StateStore for Rooms { + fn get_event(&self, room_id: &RoomId, event_id: &EventId) -> StdResult { + let pid = self + .eventid_pduid + .get(event_id.as_bytes()) + .map_err(|e| e.to_string())? + .ok_or_else(|| "PDU via room_id and event_id not found in the db.".to_owned())?; + + utils::deserialize( + &self + .pduid_pdu + .get(pid) + .map_err(|e| e.to_string())? + .ok_or_else(|| "PDU via pduid not found in db.".to_owned())?, + ) + .and_then(|pdu: StateEvent| { + // conduit's PDU's always contain a room_id but some + // of ruma's do not so this must be an Option + if pdu.room_id() == Some(room_id) { + Ok(pdu) + } else { + Err(Error::bad_database("Found PDU for incorrect room in db.")) + } + }) + .map_err(|e| e.to_string()) + } +} + +// These are the methods related to STATE resolution. impl Rooms { + /// Generates a new StateHash and associates it with the incoming event. + /// + /// This adds all current state events (not including the incoming event) + /// to `stateid_pduid` and adds the incoming event to `pduid_statehash`. + /// The incoming event is the `pdu_id` passed to this method. + pub fn append_state_pdu(&self, room_id: &RoomId, pdu_id: &[u8]) -> Result { + let state_hash = self.new_state_hash_id(room_id)?; + let state = self.current_state_pduids(room_id)?; + + let mut key = state_hash.as_bytes().to_vec(); + key.push(0xff); + + // TODO eventually we could avoid writing to the DB so much on every event + // by keeping track of the delta and write that every so often + for ((ev_ty, state_key), pid) in state { + let mut state_id = key.to_vec(); + state_id.extend_from_slice(ev_ty.to_string().as_bytes()); + key.push(0xff); + state_id.extend_from_slice(state_key.expect("state event").as_bytes()); + key.push(0xff); + + self.stateid_pduid.insert(&state_id, &pid)?; + } + + // This event's state does not include the event itself. `current_state_pduids` + // uses `roomstateid_pduid` before the current event is inserted to the tree so the state + // will be everything up to but not including the incoming event. + self.pduid_statehash.insert(pdu_id, state_hash.as_bytes())?; + + Ok(state_hash) + } + + /// Builds a `StateMap` by iterating over all keys that start + /// with `state_hash`, this gives the full state at event "x". + pub fn get_statemap_by_hash(&self, state_hash: StateHashId) -> Result> { + self.stateid_pduid + .scan_prefix(state_hash.as_bytes()) + .values() + .map(|pduid| { + self.pduid_pdu.get(&pduid?)?.map_or_else( + || Err(Error::bad_database("Failed to find StateMap.")), + |b| { + serde_json::from_slice::(&b) + .map_err(|_| Error::bad_database("Invalid PDU in db.")) + }, + ) + }) + .map(|pdu| { + let pdu = pdu?; + Ok(((pdu.kind, pdu.state_key), pdu.event_id)) + }) + .collect::>>() + } + + // TODO make this return Result + /// Fetches the previous StateHash ID to `current`. + pub fn prev_state_hash(&self, current: StateHashId) -> Option { + let mut found = false; + for pair in self.pduid_statehash.iter().rev() { + let prev = utils::string_from_bytes(&pair.ok()?.1).ok()?; + if current == prev { + found = true; + } + if current != prev && found { + return Some(prev); + } + } + None + } + + /// Fetch the current State using the `roomstateid_pduid` tree. + pub fn current_state_pduids(&self, room_id: &RoomId) -> Result> { + // TODO this could also scan roomstateid_pduid if we passed in room_id ? + self.roomstateid_pduid + .scan_prefix(room_id.as_bytes()) + .values() + .map(|pduid| { + let pduid = &pduid?; + self.pduid_pdu.get(pduid)?.map_or_else( + || { + Err(Error::bad_database( + "Failed to find current state of pduid's.", + )) + }, + |b| { + Ok(( + serde_json::from_slice::(&b) + .map_err(|_| Error::bad_database("Invalid PDU in db."))?, + pduid.clone(), + )) + }, + ) + }) + .map(|pair| { + let (pdu, id) = pair?; + Ok(((pdu.kind, pdu.state_key), id)) + }) + .collect::>>() + } + + /// Returns the last state hash key added to the db. + pub fn current_state_hash(&self, room_id: &RoomId) -> Result { + let mut prefix = room_id.as_bytes().to_vec(); + prefix.push(0xff); + + // We must check here because this method is called outside and before + // `append_state_pdu` so the DB can be empty + if self.pduid_statehash.scan_prefix(prefix).next().is_none() { + // TODO use ring crate to hash + return Ok(room_id.as_str().to_owned()); + } + + self.pduid_statehash + .iter() + .next_back() + .map(|pair| { + utils::string_from_bytes(&pair?.1) + .map_err(|_| Error::bad_database("Invalid state hash string in db.")) + }) + .ok_or_else(|| Error::bad_database("No PDU's found for this room."))? + } + + /// This fetches auth event_ids from the current state using the + /// full `roomstateid_pdu` tree. + pub fn get_auth_event_ids( + &self, + room_id: &RoomId, + kind: &EventType, + sender: &UserId, + state_key: Option<&str>, + content: serde_json::Value, + ) -> Result> { + let auth_events = state_res::auth_types_for_event( + kind.clone(), + sender, + state_key.map(|s| s.to_string()), + content, + ); + + let mut events = vec![]; + for (event_type, state_key) in auth_events { + if let Some(state_key) = state_key.as_ref() { + if let Some(id) = self.room_state_get(room_id, &event_type, state_key)? { + events.push(id.event_id); + } + } + } + Ok(events) + } + + // This fetches auth events from the current state using the + /// full `roomstateid_pdu` tree. + pub fn get_auth_events( + &self, + room_id: &RoomId, + kind: &EventType, + sender: &UserId, + state_key: Option<&str>, + content: serde_json::Value, + ) -> Result> { + let auth_events = state_res::auth_types_for_event( + kind.clone(), + sender, + state_key.map(|s| s.to_string()), + content, + ); + + let mut events = StateMap::new(); + for (event_type, state_key) in auth_events { + if let Some(s_key) = state_key.as_ref() { + if let Some(pdu) = self.room_state_get(room_id, &event_type, s_key)? { + events.insert((event_type, state_key), pdu); + } + } + } + Ok(events) + } + + /// Generate a new StateHash. + /// + /// A unique hash made from hashing the current states pduid's. + /// Because `append_state_pdu` handles the empty state db case it does not + /// have to be here. + fn new_state_hash_id(&self, room_id: &RoomId) -> Result { + // Use hashed roomId as the first StateHash key for first state event in room + if self + .pduid_statehash + .scan_prefix(room_id.as_bytes()) + .next() + .is_none() + { + // TODO use ring crate to hash + return Ok(room_id.as_str().to_owned()); + } + + let pdu_ids_to_hash = self + .pduid_statehash + .scan_prefix(room_id.as_bytes()) + .values() + .next_back() + .unwrap() // We just checked if the tree was empty + .map(|hash| { + self.stateid_pduid + .scan_prefix(hash) + .values() + // pduid is roomId + count so just hash the whole thing + .map(|pid| Ok(pid?.to_vec())) + .collect::>>>() + })??; + + let mut hasher = DefaultHasher::new(); + pdu_ids_to_hash.hash(&mut hasher); + let hash = hasher.finish().to_string(); + // TODO not sure how you want to hash this + Ok(hash) + } + /// Checks if a room exists. pub fn exists(&self, room_id: &RoomId) -> Result { let mut prefix = room_id.to_string().as_bytes().to_vec(); @@ -64,16 +330,20 @@ impl Rooms { room_id: &RoomId, ) -> Result> { let mut hashmap = HashMap::new(); - for pdu in self - .roomstateid_pdu - .scan_prefix(&room_id.to_string().as_bytes()) - .values() - .map(|value| { - Ok::<_, Error>( - serde_json::from_slice::(&value?) + for pdu in + self.roomstateid_pduid + .scan_prefix(&room_id.to_string().as_bytes()) + .values() + .map(|value| { + Ok::<_, Error>( + serde_json::from_slice::( + &self.pduid_pdu.get(value?)?.ok_or_else(|| { + Error::bad_database("PDU not found for ID in db.") + })?, + ) .map_err(|_| Error::bad_database("Invalid PDU in db."))?, - ) - }) + ) + }) { let pdu = pdu?; let state_key = pdu.state_key.clone().ok_or_else(|| { @@ -95,16 +365,20 @@ impl Rooms { prefix.extend_from_slice(&event_type.to_string().as_bytes()); let mut hashmap = HashMap::new(); - for pdu in self - .roomstateid_pdu - .scan_prefix(&prefix) - .values() - .map(|value| { - Ok::<_, Error>( - serde_json::from_slice::(&value?) + for pdu in + self.roomstateid_pduid + .scan_prefix(&prefix) + .values() + .map(|value| { + Ok::<_, Error>( + serde_json::from_slice::( + &self.pduid_pdu.get(value?)?.ok_or_else(|| { + Error::bad_database("PDU not found for ID in db.") + })?, + ) .map_err(|_| Error::bad_database("Invalid PDU in db."))?, - ) - }) + ) + }) { let pdu = pdu?; let state_key = pdu.state_key.clone().ok_or_else(|| { @@ -115,23 +389,28 @@ impl Rooms { Ok(hashmap) } - /// Returns the full room state. + /// Returns a single PDU in `room_id` with key (`event_type`, `state_key`). pub fn room_state_get( &self, room_id: &RoomId, event_type: &EventType, state_key: &str, ) -> Result> { - let mut key = room_id.to_string().as_bytes().to_vec(); + let mut key = room_id.as_bytes().to_vec(); key.push(0xff); key.extend_from_slice(&event_type.to_string().as_bytes()); key.push(0xff); key.extend_from_slice(&state_key.as_bytes()); - self.roomstateid_pdu.get(&key)?.map_or(Ok(None), |value| { + self.roomstateid_pduid.get(&key)?.map_or(Ok(None), |value| { Ok::<_, Error>(Some( - serde_json::from_slice::(&value) - .map_err(|_| Error::bad_database("Invalid PDU in db."))?, + serde_json::from_slice::( + &self + .pduid_pdu + .get(value)? + .ok_or_else(|| Error::bad_database("PDU not found for ID in db."))?, + ) + .map_err(|_| Error::bad_database("Invalid PDU in db."))?, )) }) } @@ -139,7 +418,7 @@ impl Rooms { /// Returns the `count` of this pdu's id. pub fn get_pdu_count(&self, event_id: &EventId) -> Result> { self.eventid_pduid - .get(event_id.to_string().as_bytes())? + .get(event_id.as_bytes())? .map_or(Ok(None), |pdu_id| { Ok(Some( utils::u64_from_bytes( @@ -153,7 +432,7 @@ impl Rooms { /// Returns the json of a pdu. pub fn get_pdu_json(&self, event_id: &EventId) -> Result> { self.eventid_pduid - .get(event_id.to_string().as_bytes())? + .get(event_id.as_bytes())? .map_or(Ok(None), |pdu_id| { Ok(Some( serde_json::from_slice(&self.pduid_pdu.get(pdu_id)?.ok_or_else(|| { @@ -174,7 +453,7 @@ impl Rooms { /// Returns the pdu. pub fn get_pdu(&self, event_id: &EventId) -> Result> { self.eventid_pduid - .get(event_id.to_string().as_bytes())? + .get(event_id.as_bytes())? .map_or(Ok(None), |pdu_id| { Ok(Some( serde_json::from_slice(&self.pduid_pdu.get(pdu_id)?.ok_or_else(|| { @@ -238,16 +517,15 @@ impl Rooms { /// Replace the leaves of a room with a new event. pub fn replace_pdu_leaves(&self, room_id: &RoomId, event_id: &EventId) -> Result<()> { - let mut prefix = room_id.to_string().as_bytes().to_vec(); + let mut prefix = room_id.as_bytes().to_vec(); prefix.push(0xff); for key in self.roomid_pduleaves.scan_prefix(&prefix).keys() { self.roomid_pduleaves.remove(key?)?; } - prefix.extend_from_slice(event_id.to_string().as_bytes()); - self.roomid_pduleaves - .insert(&prefix, &*event_id.to_string())?; + prefix.extend_from_slice(event_id.as_bytes()); + self.roomid_pduleaves.insert(&prefix, event_id.as_bytes())?; Ok(()) } @@ -272,6 +550,14 @@ impl Rooms { // TODO: Make sure this isn't called twice in parallel let prev_events = self.get_pdu_leaves(&room_id)?; + let auth_events = self.get_auth_events( + &room_id, + &event_type, + &sender, + state_key.as_deref(), + content.clone(), + )?; + // Is the event authorized? if let Some(state_key) = &state_key { let power_levels = self @@ -333,138 +619,24 @@ impl Rooms { // Don't allow encryption events when it's disabled !globals.encryption_disabled() } - EventType::RoomMember => { - let target_user_id = UserId::try_from(&**state_key).map_err(|_| { - Error::BadRequest( - ErrorKind::InvalidParam, - "State key of member event does not contain user id.", - ) - })?; - - let current_membership = self - .room_state_get( - &room_id, - &EventType::RoomMember, - &target_user_id.to_string(), - )? - .map_or(Ok::<_, Error>(member::MembershipState::Leave), |pdu| { - Ok(serde_json::from_value::>( - pdu.content, - ) - .expect("Raw::from_value always works.") - .deserialize() - .map_err(|_| Error::bad_database("Invalid Member event in db."))? - .membership) - })?; - - let target_membership = - serde_json::from_value::>(content.clone()) - .expect("Raw::from_value always works.") - .deserialize() - .map_err(|_| Error::bad_database("Invalid Member event in db."))? - .membership; - - let target_power = power_levels.users.get(&target_user_id).map_or_else( - || { - if target_membership != member::MembershipState::Join { - None - } else { - Some(&power_levels.users_default) - } - }, - // If it's okay, wrap with Some(_) - Some, - ); - - let join_rules = - self.room_state_get(&room_id, &EventType::RoomJoinRules, "")? - .map_or(Ok::<_, Error>(join_rules::JoinRule::Public), |pdu| { - Ok(serde_json::from_value::< - Raw, - >(pdu.content) - .expect("Raw::from_value always works.") - .deserialize() - .map_err(|_| { - Error::bad_database("Database contains invalid JoinRules event") - })? - .join_rule) - })?; - - if target_membership == member::MembershipState::Join { - let mut prev_events = prev_events.iter(); - let prev_event = self - .get_pdu(prev_events.next().ok_or(Error::BadRequest( - ErrorKind::Unknown, - "Membership can't be the first event", - ))?)? - .ok_or_else(|| { - Error::bad_database("PDU leaf points to invalid event!") - })?; - if prev_event.kind == EventType::RoomCreate - && prev_event.prev_events.is_empty() - { - true - } else if sender != target_user_id { - false - } else if let member::MembershipState::Ban = current_membership { - false - } else { - join_rules == join_rules::JoinRule::Invite - && (current_membership == member::MembershipState::Join - || current_membership == member::MembershipState::Invite) - || join_rules == join_rules::JoinRule::Public - } - } else if target_membership == member::MembershipState::Invite { - if let Some(third_party_invite_json) = content.get("third_party_invite") { - if current_membership == member::MembershipState::Ban { - false - } else { - let _third_party_invite = - serde_json::from_value::( - third_party_invite_json.clone(), - ) - .map_err(|_| { - Error::BadRequest( - ErrorKind::InvalidParam, - "ThirdPartyInvite is invalid", - ) - })?; - todo!("handle third party invites"); - } - } else if sender_membership != member::MembershipState::Join - || current_membership == member::MembershipState::Join - || current_membership == member::MembershipState::Ban - { - false - } else { - sender_power - .filter(|&p| p >= &power_levels.invite) - .is_some() - } - } else if target_membership == member::MembershipState::Leave { - if sender == target_user_id { - current_membership == member::MembershipState::Join - || current_membership == member::MembershipState::Invite - } else if sender_membership != member::MembershipState::Join - || current_membership == member::MembershipState::Ban - && sender_power.filter(|&p| p < &power_levels.ban).is_some() - { - false - } else { - sender_power.filter(|&p| p >= &power_levels.kick).is_some() - && target_power < sender_power - } - } else if target_membership == member::MembershipState::Ban { - if sender_membership != member::MembershipState::Join { - false - } else { - sender_power.filter(|&p| p >= &power_levels.ban).is_some() - && target_power < sender_power - } - } else { - false - } - } + EventType::RoomMember => event_auth::is_membership_change_allowed( + // TODO this is a bit of a hack but not sure how to have a type + // declared in `state_res` crate be + Requester { + prev_event_ids: prev_events.to_owned(), + room_id: &room_id, + content: &content, + state_key: Some(state_key.to_owned()), + sender: &sender, + }, + &auth_events + .iter() + .map(|((ty, key), pdu)| { + Ok(((ty.clone(), key.clone()), pdu.convert_for_state_res()?)) + }) + .collect::>>()?, + ) + .ok_or(Error::Conflict("Found incoming PDU with invalid data."))?, EventType::RoomCreate => prev_events.is_empty(), // Not allow any of the following events if the sender is not joined. _ if sender_membership != member::MembershipState::Join => false, @@ -474,7 +646,7 @@ impl Rooms { >= &power_levels.state_default } } { - error!("Unauthorized"); + error!("Unauthorized {}", event_type); // Not authorized return Err(Error::BadRequest( ErrorKind::Forbidden, @@ -483,7 +655,7 @@ impl Rooms { } } else if !self.is_joined(&sender, &room_id)? { // TODO: auth rules apply to all events, not only those with a state key - error!("Unauthorized"); + error!("Unauthorized {}", event_type); return Err(Error::BadRequest( ErrorKind::Forbidden, "Event is not authorized", @@ -524,7 +696,10 @@ impl Rooms { depth: depth .try_into() .map_err(|_| Error::bad_database("Depth is invalid"))?, - auth_events: Vec::new(), + auth_events: auth_events + .into_iter() + .map(|(_, pdu)| pdu.event_id) + .collect(), redacts: redacts.clone(), unsigned, hashes: ruma::events::pdu::EventHash { @@ -564,15 +739,19 @@ impl Rooms { self.pduid_pdu.insert(&pdu_id, &*pdu_json.to_string())?; self.eventid_pduid - .insert(pdu.event_id.to_string(), pdu_id.clone())?; + .insert(pdu.event_id.to_string(), &*pdu_id)?; - if let Some(state_key) = pdu.state_key { - let mut key = room_id.to_string().as_bytes().to_vec(); + if let Some(state_key) = &pdu.state_key { + // We call this first because our StateHash relies on the + // state before the new event + self.append_state_pdu(&room_id, &pdu_id)?; + + let mut key = room_id.as_bytes().to_vec(); key.push(0xff); key.extend_from_slice(pdu.kind.to_string().as_bytes()); key.push(0xff); key.extend_from_slice(state_key.as_bytes()); - self.roomstateid_pdu.insert(key, &*pdu_json.to_string())?; + self.roomstateid_pduid.insert(key, pdu_id.as_slice())?; } match event_type { diff --git a/src/database/uiaa.rs b/src/database/uiaa.rs index cece8dbe3..e318f4368 100644 --- a/src/database/uiaa.rs +++ b/src/database/uiaa.rs @@ -2,7 +2,7 @@ use crate::{Error, Result}; use ruma::{ api::client::{ error::ErrorKind, - r0::uiaa::{AuthData, UiaaInfo}, + r0::uiaa::{IncomingAuthData, UiaaInfo}, }, DeviceId, UserId, }; @@ -26,12 +26,12 @@ impl Uiaa { &self, user_id: &UserId, device_id: &DeviceId, - auth: &AuthData, + auth: &IncomingAuthData, uiaainfo: &UiaaInfo, users: &super::users::Users, globals: &super::globals::Globals, ) -> Result<(bool, UiaaInfo)> { - if let AuthData::DirectRequest { + if let IncomingAuthData::DirectRequest { kind, session, auth_parameters, diff --git a/src/pdu.rs b/src/pdu.rs index 9936802e0..5485f2363 100644 --- a/src/pdu.rs +++ b/src/pdu.rs @@ -177,6 +177,35 @@ impl PduEvent { } } +impl PduEvent { + pub fn convert_for_state_res(&self) -> Result { + serde_json::from_value(json!({ + "event_id": self.event_id, + "room_id": self.room_id, + "sender": self.sender, + "origin": self.origin, + "origin_server_ts": self.origin_server_ts, + "type": self.kind, + "content": self.content, + "state_key": self.state_key, + "prev_events": self.prev_events + .iter() + .map(|id| (id, EventHash { sha256: "hello".into() })) + .collect::>(), + "depth": self.depth, + "auth_events": self.auth_events + .iter() + .map(|id| (id, EventHash { sha256: "hello".into() })) + .collect::>(), + "redacts": self.redacts, + "unsigned": self.unsigned, + "hashes": self.hashes, + "signatures": self.signatures, + })) + .map_err(|_| Error::bad_database("Failed to convert PDU to ruma::Pdu type.")) + } +} + /// Build the start of a PDU in order to add it to the `Database`. #[derive(Debug)] pub struct PduBuilder { diff --git a/src/ruma_wrapper.rs b/src/ruma_wrapper.rs index 8d8620435..80e6e5829 100644 --- a/src/ruma_wrapper.rs +++ b/src/ruma_wrapper.rs @@ -1,5 +1,8 @@ use crate::Error; -use ruma::identifiers::{DeviceId, UserId}; +use ruma::{ + api::IncomingRequest, + identifiers::{DeviceId, UserId}, +}; use std::{convert::TryInto, ops::Deref}; #[cfg(feature = "conduit_bin")] @@ -16,13 +19,12 @@ use { tokio::io::AsyncReadExt, Request, State, }, - ruma::api::IncomingRequest, std::io::Cursor, }; /// This struct converts rocket requests into ruma structs by converting them into http requests /// first. -pub struct Ruma { +pub struct Ruma { pub body: T, pub sender_id: Option, pub device_id: Option>, @@ -110,7 +112,7 @@ impl<'a, T: IncomingRequest> FromTransformedData<'a> for Ruma { } } -impl Deref for Ruma { +impl Deref for Ruma { type Target = T; fn deref(&self) -> &Self::Target { diff --git a/src/server_server.rs b/src/server_server.rs index f48f502b9..e47b50a52 100644 --- a/src/server_server.rs +++ b/src/server_server.rs @@ -1,14 +1,17 @@ use crate::{client_server, ConduitResult, Database, Error, Result, Ruma}; use http::header::{HeaderValue, AUTHORIZATION}; use rocket::{get, post, put, response::content::Json, State}; -use ruma::api::federation::{ - directory::get_public_rooms, - discovery::{ - get_server_keys, get_server_version::v1 as get_server_version, ServerKey, VerifyKey, +use ruma::api::{ + client, + federation::{ + directory::get_public_rooms, + discovery::{ + get_server_keys, get_server_version::v1 as get_server_version, ServerKey, VerifyKey, + }, + transactions::send_transaction_message, }, - transactions::send_transaction_message, + OutgoingRequest, }; -use ruma::api::{client, OutgoingRequest}; use serde_json::json; use std::{ collections::BTreeMap, @@ -204,11 +207,11 @@ pub fn get_server_keys_deprecated(db: State<'_, Database>) -> Json { )] pub async fn get_public_rooms_route( db: State<'_, Database>, - body: Ruma, + body: Ruma, ) -> ConduitResult { let Ruma { body: - get_public_rooms::v1::Request { + get_public_rooms::v1::IncomingRequest { room_network: _room_network, // TODO limit, since, @@ -229,7 +232,7 @@ pub async fn get_public_rooms_route( body: client::r0::directory::get_public_rooms_filtered::IncomingRequest { filter: None, limit, - room_network: client::r0::directory::get_public_rooms_filtered::RoomNetwork::Matrix, + room_network: ruma::directory::RoomNetwork::Matrix, server: None, since, }, @@ -268,9 +271,9 @@ pub async fn get_public_rooms_route( feature = "conduit_bin", put("/_matrix/federation/v1/send/<_>", data = "") )] -pub fn send_transaction_message_route( - db: State<'_, Database>, - body: Ruma, +pub fn send_transaction_message_route<'a>( + _db: State<'a, Database>, + body: Ruma, ) -> ConduitResult { dbg!(&*body); Ok(send_transaction_message::v1::Response { diff --git a/src/utils.rs b/src/utils.rs index 8cf1b2ce1..77a7d1fad 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -1,3 +1,4 @@ +use crate::Error; use argon2::{Config, Variant}; use cmp::Ordering; use rand::prelude::*; @@ -90,3 +91,8 @@ pub fn common_elements( .all(|b| b) })) } + +pub fn deserialize<'de, T: serde::Deserialize<'de>>(val: &'de sled::IVec) -> Result { + serde_json::from_slice::(val.as_ref()) + .map_err(|_| Error::bad_database("PDU in db is invalid.")) +} From cb68bf9e0c073aa060b3fe6d9aa14bbfaebc5dae Mon Sep 17 00:00:00 2001 From: Devin Ragotzy Date: Tue, 18 Aug 2020 14:32:38 -0400 Subject: [PATCH 02/51] Use ring crate to generate StatHashes when saving stateid/statehash --- Cargo.lock | 1 + Cargo.toml | 2 +- src/database/rooms.rs | 20 ++++++++++++-------- src/utils.rs | 2 +- 4 files changed, 15 insertions(+), 10 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ffee8ea59..faa9e8983 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -295,6 +295,7 @@ dependencies = [ "log", "rand", "reqwest", + "ring", "rocket", "ruma", "rust-argon2 0.8.2", diff --git a/Cargo.toml b/Cargo.toml index 4c14d712b..78d8f7606 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -33,7 +33,7 @@ image = { version = "0.23.4", default-features = false, features = ["jpeg", "png base64 = "0.12.3" # Used to encode server public key # state-res = { path = "../../state-res" } state-res = { git = "https://github.com/ruma/state-res", version = "0.1.0" } - +ring = "0.16.15" [features] default = ["conduit_bin"] diff --git a/src/database/rooms.rs b/src/database/rooms.rs index 0d3632623..6366c8cdc 100644 --- a/src/database/rooms.rs +++ b/src/database/rooms.rs @@ -4,6 +4,7 @@ pub use edus::RoomEdus; use crate::{pdu::PduBuilder, utils, Error, PduEvent, Result}; use log::error; +use ring::digest; use ruma::{ api::client::error::ErrorKind, events::{ @@ -21,9 +22,8 @@ use sled::IVec; use state_res::{event_auth, Requester, StateEvent, StateMap, StateStore}; use std::{ - collections::{hash_map::DefaultHasher, BTreeMap, HashMap}, + collections::{BTreeMap, HashMap}, convert::{TryFrom, TryInto}, - hash::{Hash, Hasher}, mem, result::Result as StdResult, }; @@ -285,8 +285,10 @@ impl Rooms { .next() .is_none() { - // TODO use ring crate to hash - return Ok(room_id.as_str().to_owned()); + return utils::string_from_bytes( + digest::digest(&digest::SHA256, room_id.as_bytes()).as_ref(), + ) + .map_err(|_| Error::bad_database("Empty state generated invalid string from hash.")); } let pdu_ids_to_hash = self @@ -304,11 +306,13 @@ impl Rooms { .collect::>>>() })??; - let mut hasher = DefaultHasher::new(); - pdu_ids_to_hash.hash(&mut hasher); - let hash = hasher.finish().to_string(); + let hash = digest::digest( + &digest::SHA256, + &pdu_ids_to_hash.into_iter().flatten().collect::>(), + ); // TODO not sure how you want to hash this - Ok(hash) + utils::string_from_bytes(hash.as_ref()) + .map_err(|_| Error::bad_database("State generated invalid string from hash.")) } /// Checks if a room exists. diff --git a/src/utils.rs b/src/utils.rs index 77a7d1fad..b549153ab 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -94,5 +94,5 @@ pub fn common_elements( pub fn deserialize<'de, T: serde::Deserialize<'de>>(val: &'de sled::IVec) -> Result { serde_json::from_slice::(val.as_ref()) - .map_err(|_| Error::bad_database("PDU in db is invalid.")) + .map_err(|_| Error::bad_database("Found invalid bytes as PDU in db.")) } From 846a0098c182272a5669bacf0d27fa988eaa4c23 Mon Sep 17 00:00:00 2001 From: Devin Ragotzy Date: Tue, 18 Aug 2020 16:26:03 -0400 Subject: [PATCH 03/51] Split append_pdu -> append_pdu and build_and_append Move all state event appending to append_state_pdu. --- Cargo.lock | 22 ++-- src/client_server/account.rs | 2 +- src/client_server/membership.rs | 37 ++---- src/client_server/message.rs | 2 +- src/client_server/profile.rs | 4 +- src/client_server/redact.rs | 2 +- src/client_server/room.rs | 20 +-- src/client_server/state.rs | 2 +- src/database/rooms.rs | 217 ++++++++++++++++---------------- src/pdu.rs | 28 ++++- 10 files changed, 175 insertions(+), 161 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index faa9e8983..3c5d836ba 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -116,9 +116,9 @@ checksum = "4af5687fe33aec5e70ef14caac5e0d363e335e5e5d6385fb75978d0c241b1d67" [[package]] name = "async-trait" -version = "0.1.37" +version = "0.1.38" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "caae68055714ff28740f310927e04f2eba76ff580b16fb18ed90073ee71646f7" +checksum = "6e1a4a2f97ce50c9d0282c1468816208588441492b40d813b2e0419c22c05e7f" dependencies = [ "proc-macro2", "quote", @@ -1194,9 +1194,9 @@ checksum = "1ab52be62400ca80aa00285d25253d7f7c437b7375c4de678f5405d3afe82ca5" [[package]] name = "once_cell" -version = "1.4.0" +version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b631f7e854af39a1739f401cf34a8a013dfe09eac4fa4dba91e9768bd28168d" +checksum = "260e51e7efe62b592207e9e13a68e43692a7a279171d6ba57abd208bf23645ad" [[package]] name = "opaque-debug" @@ -1860,9 +1860,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.18.0" +version = "0.18.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cac94b333ee2aac3284c5b8a1b7fb4dd11cba88c244e3fe33cdbd047af0eb693" +checksum = "5d1126dcf58e93cee7d098dbda643b5f92ed724f1f6a63007c1116eed6700c81" dependencies = [ "base64 0.12.3", "log", @@ -2088,7 +2088,7 @@ checksum = "7345c971d1ef21ffdbd103a75990a15eb03604fc8b8852ca8cb418ee1a099028" [[package]] name = "state-res" version = "0.1.0" -source = "git+https://github.com/ruma/state-res#789c8140890e076d38b23fa1147c4ff0500c0d38" +source = "git+https://github.com/ruma/state-res#4e9b428c0db50ac3a3421ced12a6fd202a1c36a3" dependencies = [ "itertools", "js_int", @@ -2281,9 +2281,9 @@ dependencies = [ [[package]] name = "tinyvec" -version = "0.3.3" +version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53953d2d3a5ad81d9f844a32f14ebb121f50b650cd59d0ee2a07cf13c617efed" +checksum = "238ce071d267c5710f9d31451efec16c5ee22de34df17cc05e56cbc92e967117" [[package]] name = "tokio" @@ -2384,9 +2384,9 @@ dependencies = [ [[package]] name = "tracing-attributes" -version = "0.1.10" +version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fe233f4227389ab7df5b32649239da7ebe0b281824b4e84b342d04d3fd8c25e" +checksum = "80e0ccfc3378da0cce270c946b676a376943f5cd16aeba64568e7939806f4ada" dependencies = [ "proc-macro2", "quote", diff --git a/src/client_server/account.rs b/src/client_server/account.rs index 9fa1a9c74..9e52f6d23 100644 --- a/src/client_server/account.rs +++ b/src/client_server/account.rs @@ -354,7 +354,7 @@ pub fn deactivate_route( third_party_invite: None, }; - db.rooms.append_pdu( + db.rooms.build_and_append_pdu( PduBuilder { room_id: room_id.clone(), sender: sender_id.clone(), diff --git a/src/client_server/membership.rs b/src/client_server/membership.rs index c04cf7f6c..824e8718a 100644 --- a/src/client_server/membership.rs +++ b/src/client_server/membership.rs @@ -1,6 +1,8 @@ use super::State; use crate::{ - client_server, pdu::PduBuilder, server_server, utils, ConduitResult, Database, Error, Ruma, + client_server, + pdu::{PduBuilder, PduEvent}, + server_server, utils, ConduitResult, Database, Error, Ruma, }; use ruma::{ api::{ @@ -142,24 +144,9 @@ pub async fn join_room_by_id_route( Error::Conflict("Found event_id in sorted events that is not in resolved state") })?; - db.rooms.append_pdu( - PduBuilder { - room_id: pdu.room_id().unwrap_or(&body.room_id).clone(), - sender: pdu.sender().clone(), - event_type: pdu.kind(), - content: pdu.content().clone(), - unsigned: Some( - pdu.unsigned() - .iter() - .map(|(k, v)| (k.clone(), v.clone())) - .collect(), - ), - state_key: pdu.state_key(), - redacts: pdu.redacts().cloned(), - }, - &db.globals, - &db.account_data, - )?; + // We do not rebuild the PDU in this case only insert to DB + db.rooms + .append_pdu(PduEvent::try_from(pdu)?, &db.globals, &db.account_data)?; } } @@ -171,7 +158,7 @@ pub async fn join_room_by_id_route( third_party_invite: None, }; - db.rooms.append_pdu( + db.rooms.build_and_append_pdu( PduBuilder { room_id: body.room_id.clone(), sender: sender_id.clone(), @@ -284,7 +271,7 @@ pub fn leave_room_route( event.membership = member::MembershipState::Leave; - db.rooms.append_pdu( + db.rooms.build_and_append_pdu( PduBuilder { room_id: body.room_id.clone(), sender: sender_id.clone(), @@ -312,7 +299,7 @@ pub fn invite_user_route( let sender_id = body.sender_id.as_ref().expect("user is authenticated"); if let invite_user::InvitationRecipient::UserId { user_id } = &body.recipient { - db.rooms.append_pdu( + db.rooms.build_and_append_pdu( PduBuilder { room_id: body.room_id.clone(), sender: sender_id.clone(), @@ -369,7 +356,7 @@ pub fn kick_user_route( event.membership = ruma::events::room::member::MembershipState::Leave; // TODO: reason - db.rooms.append_pdu( + db.rooms.build_and_append_pdu( PduBuilder { room_id: body.room_id.clone(), sender: sender_id.clone(), @@ -424,7 +411,7 @@ pub fn ban_user_route( }, )?; - db.rooms.append_pdu( + db.rooms.build_and_append_pdu( PduBuilder { room_id: body.room_id.clone(), sender: sender_id.clone(), @@ -470,7 +457,7 @@ pub fn unban_user_route( event.membership = ruma::events::room::member::MembershipState::Leave; - db.rooms.append_pdu( + db.rooms.build_and_append_pdu( PduBuilder { room_id: body.room_id.clone(), sender: sender_id.clone(), diff --git a/src/client_server/message.rs b/src/client_server/message.rs index 1b461d240..03832d86f 100644 --- a/src/client_server/message.rs +++ b/src/client_server/message.rs @@ -25,7 +25,7 @@ pub fn send_message_event_route( let mut unsigned = serde_json::Map::new(); unsigned.insert("transaction_id".to_owned(), body.txn_id.clone().into()); - let event_id = db.rooms.append_pdu( + let event_id = db.rooms.build_and_append_pdu( PduBuilder { room_id: body.room_id.clone(), sender: sender_id.clone(), diff --git a/src/client_server/profile.rs b/src/client_server/profile.rs index 1313db776..0707b342d 100644 --- a/src/client_server/profile.rs +++ b/src/client_server/profile.rs @@ -31,7 +31,7 @@ pub fn set_displayname_route( // Send a new membership event and presence update into all joined rooms for room_id in db.rooms.rooms_joined(&sender_id) { let room_id = room_id?; - db.rooms.append_pdu( + db.rooms.build_and_append_pdu( PduBuilder { room_id: room_id.clone(), sender: sender_id.clone(), @@ -134,7 +134,7 @@ pub fn set_avatar_url_route( // Send a new membership event and presence update into all joined rooms for room_id in db.rooms.rooms_joined(&sender_id) { let room_id = room_id?; - db.rooms.append_pdu( + db.rooms.build_and_append_pdu( PduBuilder { room_id: room_id.clone(), sender: sender_id.clone(), diff --git a/src/client_server/redact.rs b/src/client_server/redact.rs index fc65c23f5..8708692f3 100644 --- a/src/client_server/redact.rs +++ b/src/client_server/redact.rs @@ -18,7 +18,7 @@ pub fn redact_event_route( ) -> ConduitResult { let sender_id = body.sender_id.as_ref().expect("user is authenticated"); - let event_id = db.rooms.append_pdu( + let event_id = db.rooms.build_and_append_pdu( PduBuilder { room_id: body.room_id.clone(), sender: sender_id.clone(), diff --git a/src/client_server/room.rs b/src/client_server/room.rs index 589a2dcdf..3ee21b6d8 100644 --- a/src/client_server/room.rs +++ b/src/client_server/room.rs @@ -56,7 +56,7 @@ pub fn create_room_route( content.room_version = RoomVersionId::Version6; // 1. The room create event - db.rooms.append_pdu( + db.rooms.build_and_append_pdu( PduBuilder { room_id: room_id.clone(), sender: sender_id.clone(), @@ -71,7 +71,7 @@ pub fn create_room_route( )?; // 2. Let the room creator join - db.rooms.append_pdu( + db.rooms.build_and_append_pdu( PduBuilder { room_id: room_id.clone(), sender: sender_id.clone(), @@ -120,7 +120,7 @@ pub fn create_room_route( }) .expect("event is valid, we just created it") }; - db.rooms.append_pdu( + db.rooms.build_and_append_pdu( PduBuilder { room_id: room_id.clone(), sender: sender_id.clone(), @@ -144,7 +144,7 @@ pub fn create_room_route( }); // 4.1 Join Rules - db.rooms.append_pdu( + db.rooms.build_and_append_pdu( PduBuilder { room_id: room_id.clone(), sender: sender_id.clone(), @@ -169,7 +169,7 @@ pub fn create_room_route( )?; // 4.2 History Visibility - db.rooms.append_pdu( + db.rooms.build_and_append_pdu( PduBuilder { room_id: room_id.clone(), sender: sender_id.clone(), @@ -187,7 +187,7 @@ pub fn create_room_route( )?; // 4.3 Guest Access - db.rooms.append_pdu( + db.rooms.build_and_append_pdu( PduBuilder { room_id: room_id.clone(), sender: sender_id.clone(), @@ -224,7 +224,7 @@ pub fn create_room_route( continue; } - db.rooms.append_pdu( + db.rooms.build_and_append_pdu( PduBuilder { room_id: room_id.clone(), sender: sender_id.clone(), @@ -243,7 +243,7 @@ pub fn create_room_route( // 6. Events implied by name and topic if let Some(name) = &body.name { - db.rooms.append_pdu( + db.rooms.build_and_append_pdu( PduBuilder { room_id: room_id.clone(), sender: sender_id.clone(), @@ -264,7 +264,7 @@ pub fn create_room_route( } if let Some(topic) = &body.topic { - db.rooms.append_pdu( + db.rooms.build_and_append_pdu( PduBuilder { room_id: room_id.clone(), sender: sender_id.clone(), @@ -284,7 +284,7 @@ pub fn create_room_route( // 7. Events implied by invite (and TODO: invite_3pid) for user in &body.invite { - db.rooms.append_pdu( + db.rooms.build_and_append_pdu( PduBuilder { room_id: room_id.clone(), sender: sender_id.clone(), diff --git a/src/client_server/state.rs b/src/client_server/state.rs index 14cc49727..2920de2f6 100644 --- a/src/client_server/state.rs +++ b/src/client_server/state.rs @@ -57,7 +57,7 @@ pub fn send_state_event_for_key_route( } } - let event_id = db.rooms.append_pdu( + let event_id = db.rooms.build_and_append_pdu( PduBuilder { room_id: body.room_id.clone(), sender: sender_id.clone(), diff --git a/src/database/rooms.rs b/src/database/rooms.rs index 6366c8cdc..0339b7f9a 100644 --- a/src/database/rooms.rs +++ b/src/database/rooms.rs @@ -4,6 +4,7 @@ pub use edus::RoomEdus; use crate::{pdu::PduBuilder, utils, Error, PduEvent, Result}; use log::error; +// TODO if ruma-signatures re-exports `use ruma::signatures::digest;` use ring::digest; use ruma::{ api::client::error::ErrorKind, @@ -99,7 +100,13 @@ impl Rooms { /// This adds all current state events (not including the incoming event) /// to `stateid_pduid` and adds the incoming event to `pduid_statehash`. /// The incoming event is the `pdu_id` passed to this method. - pub fn append_state_pdu(&self, room_id: &RoomId, pdu_id: &[u8]) -> Result { + pub fn append_state_pdu( + &self, + room_id: &RoomId, + pdu_id: &[u8], + state_key: &str, + kind: &EventType, + ) -> Result { let state_hash = self.new_state_hash_id(room_id)?; let state = self.current_state_pduids(room_id)?; @@ -123,6 +130,13 @@ impl Rooms { // will be everything up to but not including the incoming event. self.pduid_statehash.insert(pdu_id, state_hash.as_bytes())?; + let mut key = room_id.as_bytes().to_vec(); + key.push(0xff); + key.extend_from_slice(kind.to_string().as_bytes()); + key.push(0xff); + key.extend_from_slice(state_key.as_bytes()); + self.roomstateid_pduid.insert(key, pdu_id)?; + Ok(state_hash) } @@ -535,8 +549,92 @@ impl Rooms { } /// Creates a new persisted data unit and adds it to a room. - #[allow(clippy::blocks_in_if_conditions)] pub fn append_pdu( + &self, + pdu: PduEvent, + globals: &super::globals::Globals, + account_data: &super::account_data::AccountData, + ) -> Result { + let mut pdu_json = serde_json::to_value(&pdu).expect("event is valid, we just created it"); + ruma::signatures::hash_and_sign_event( + globals.server_name().as_str(), + globals.keypair(), + &mut pdu_json, + ) + .expect("event is valid, we just created it"); + + self.replace_pdu_leaves(&pdu.room_id, &pdu.event_id)?; + + // Increment the last index and use that + // This is also the next_batch/since value + let index = globals.next_count()?; + + let mut pdu_id = pdu.room_id.as_bytes().to_vec(); + pdu_id.push(0xff); + pdu_id.extend_from_slice(&index.to_be_bytes()); + + self.pduid_pdu.insert(&pdu_id, &*pdu_json.to_string())?; + + self.eventid_pduid + .insert(pdu.event_id.as_bytes(), &*pdu_id)?; + + if let Some(state_key) = &pdu.state_key { + self.append_state_pdu(&pdu.room_id, &pdu_id, state_key, &pdu.kind)?; + } + + match pdu.kind { + EventType::RoomRedaction => { + if let Some(redact_id) = &pdu.redacts { + // TODO: Reason + let _reason = serde_json::from_value::>( + pdu.content, + ) + .expect("Raw::from_value always works.") + .deserialize() + .map_err(|_| { + Error::BadRequest( + ErrorKind::InvalidParam, + "Invalid redaction event content.", + ) + })? + .reason; + + self.redact_pdu(&redact_id)?; + } + } + EventType::RoomMember => { + if let Some(state_key) = &pdu.state_key { + // if the state_key fails + let target_user_id = UserId::try_from(state_key.as_str()) + .expect("This state_key was previously validated"); + // Update our membership info, we do this here incase a user is invited + // and immediately leaves we need the DB to record the invite event for auth + self.update_membership( + &pdu.room_id, + &target_user_id, + serde_json::from_value::(pdu.content).map_err( + |_| { + Error::BadRequest( + ErrorKind::InvalidParam, + "Invalid redaction event content.", + ) + }, + )?, + &pdu.sender, + account_data, + globals, + )?; + } + } + _ => {} + } + self.edus.room_read_set(&pdu.room_id, &pdu.sender, index)?; + + Ok(pdu.event_id) + } + + /// Creates a new persisted data unit and adds it to a room. + pub fn build_and_append_pdu( &self, pdu_builder: PduBuilder, globals: &super::globals::Globals, @@ -618,6 +716,7 @@ impl Rooms { ); // Is the event allowed? + #[allow(clippy::blocks_in_if_conditions)] if !match event_type { EventType::RoomEncryption => { // Don't allow encryption events when it's disabled @@ -687,15 +786,15 @@ impl Rooms { let mut pdu = PduEvent { event_id: EventId::try_from("$thiswillbefilledinlater").expect("we know this is valid"), - room_id: room_id.clone(), - sender: sender.clone(), + room_id, + sender, origin: globals.server_name().to_owned(), origin_server_ts: utils::millis_since_unix_epoch() .try_into() .expect("time is valid"), - kind: event_type.clone(), - content: content.clone(), - state_key: state_key.clone(), + kind: event_type, + content, + state_key, prev_events, depth: depth .try_into() @@ -704,7 +803,7 @@ impl Rooms { .into_iter() .map(|(_, pdu)| pdu.event_id) .collect(), - redacts: redacts.clone(), + redacts, unsigned, hashes: ruma::events::pdu::EventHash { sha256: "aaa".to_owned(), @@ -722,105 +821,7 @@ impl Rooms { )) .expect("ruma's reference hashes are valid event ids"); - let mut pdu_json = serde_json::to_value(&pdu).expect("event is valid, we just created it"); - ruma::signatures::hash_and_sign_event( - globals.server_name().as_str(), - globals.keypair(), - &mut pdu_json, - ) - .expect("event is valid, we just created it"); - - self.replace_pdu_leaves(&room_id, &pdu.event_id)?; - - // Increment the last index and use that - // This is also the next_batch/since value - let index = globals.next_count()?; - - let mut pdu_id = room_id.to_string().as_bytes().to_vec(); - pdu_id.push(0xff); - pdu_id.extend_from_slice(&index.to_be_bytes()); - - self.pduid_pdu.insert(&pdu_id, &*pdu_json.to_string())?; - - self.eventid_pduid - .insert(pdu.event_id.to_string(), &*pdu_id)?; - - if let Some(state_key) = &pdu.state_key { - // We call this first because our StateHash relies on the - // state before the new event - self.append_state_pdu(&room_id, &pdu_id)?; - - let mut key = room_id.as_bytes().to_vec(); - key.push(0xff); - key.extend_from_slice(pdu.kind.to_string().as_bytes()); - key.push(0xff); - key.extend_from_slice(state_key.as_bytes()); - self.roomstateid_pduid.insert(key, pdu_id.as_slice())?; - } - - match event_type { - EventType::RoomRedaction => { - if let Some(redact_id) = &redacts { - // TODO: Reason - let _reason = - serde_json::from_value::>(content) - .expect("Raw::from_value always works.") - .deserialize() - .map_err(|_| { - Error::BadRequest( - ErrorKind::InvalidParam, - "Invalid redaction event content.", - ) - })? - .reason; - - self.redact_pdu(&redact_id)?; - } - } - EventType::RoomMember => { - if let Some(state_key) = state_key { - // if the state_key fails - let target_user_id = UserId::try_from(state_key) - .expect("This state_key was previously validated"); - // Update our membership info, we do this here incase a user is invited - // and immediately leaves we need the DB to record the invite event for auth - self.update_membership( - &room_id, - &target_user_id, - serde_json::from_value::(content).map_err( - |_| { - Error::BadRequest( - ErrorKind::InvalidParam, - "Invalid redaction event content.", - ) - }, - )?, - &sender, - account_data, - globals, - )?; - } - } - EventType::RoomMessage => { - if let Some(body) = content.get("body").and_then(|b| b.as_str()) { - for word in body - .split_terminator(|c: char| !c.is_alphanumeric()) - .map(str::to_lowercase) - { - let mut key = room_id.to_string().as_bytes().to_vec(); - key.push(0xff); - key.extend_from_slice(word.as_bytes()); - key.push(0xff); - key.extend_from_slice(&pdu_id); - self.tokenids.insert(key, &[])?; - } - } - } - _ => {} - } - self.edus.room_read_set(&room_id, &sender, index)?; - - Ok(pdu.event_id) + self.append_pdu(pdu, globals, account_data) } /// Returns an iterator over all PDUs in a room. @@ -999,7 +1000,7 @@ impl Rooms { if is_ignored { member_content.membership = member::MembershipState::Leave; - self.append_pdu( + self.build_and_append_pdu( PduBuilder { room_id: room_id.clone(), sender: user_id.clone(), diff --git a/src/pdu.rs b/src/pdu.rs index 5485f2363..eec8e4964 100644 --- a/src/pdu.rs +++ b/src/pdu.rs @@ -9,7 +9,7 @@ use ruma::{ }; use serde::{Deserialize, Serialize}; use serde_json::json; -use std::collections::HashMap; +use std::{collections::HashMap, convert::TryFrom}; #[derive(Deserialize, Serialize)] pub struct PduEvent { @@ -177,6 +177,30 @@ impl PduEvent { } } +impl TryFrom<&state_res::StateEvent> for PduEvent { + type Error = Error; + fn try_from(pdu: &state_res::StateEvent) -> Result { + serde_json::from_value(json!({ + "event_id": pdu.event_id(), + "room_id": pdu.room_id(), + "sender": pdu.sender(), + "origin": pdu.origin(), + "origin_server_ts": pdu.origin_server_ts(), + "event_type": pdu.kind(), + "content": pdu.content(), + "state_key": pdu.state_key(), + "prev_events": pdu.prev_event_ids(), + "depth": pdu.depth(), + "auth_events": pdu.auth_events(), + "redacts": pdu.redacts(), + "unsigned": pdu.unsigned(), + "hashes": pdu.hashes(), + "signatures": pdu.signatures(), + })) + .map_err(|_| Error::bad_database("Failed to convert PDU to ruma::Pdu type.")) + } +} + impl PduEvent { pub fn convert_for_state_res(&self) -> Result { serde_json::from_value(json!({ @@ -190,11 +214,13 @@ impl PduEvent { "state_key": self.state_key, "prev_events": self.prev_events .iter() + // TODO How do we create one of these .map(|id| (id, EventHash { sha256: "hello".into() })) .collect::>(), "depth": self.depth, "auth_events": self.auth_events .iter() + // TODO How do we create one of these .map(|id| (id, EventHash { sha256: "hello".into() })) .collect::>(), "redacts": self.redacts, From d73c6aa8addcaac5f13cbaa009960659a4365bb2 Mon Sep 17 00:00:00 2001 From: Devin Ragotzy Date: Wed, 19 Aug 2020 17:27:24 -0400 Subject: [PATCH 04/51] Add roomid_statehash tree, clean up review issues --- src/client_server/membership.rs | 10 +--- src/database.rs | 1 + src/database/rooms.rs | 95 +++++++++++++++++---------------- 3 files changed, 52 insertions(+), 54 deletions(-) diff --git a/src/client_server/membership.rs b/src/client_server/membership.rs index 824e8718a..555291e56 100644 --- a/src/client_server/membership.rs +++ b/src/client_server/membership.rs @@ -112,11 +112,7 @@ pub async fn join_room_by_id_route( .room_state .state .iter() - .map(|pdu| pdu.deserialize().map(StateEvent::Full)) - .map(|ev| { - let ev = ev?; - Ok::<_, serde_json::Error>((ev.event_id(), ev)) - }) + .map(|pdu| pdu.deserialize().map(StateEvent::Full).map(|ev| (ev.event_id(), ev))) .collect::, _>>() .map_err(|_| Error::bad_database("Invalid PDU found in db."))?; @@ -140,9 +136,7 @@ pub async fn join_room_by_id_route( for ev_id in &sorted_events_ids { // this is a `state_res::StateEvent` that holds a `ruma::Pdu` - let pdu = event_map.get(ev_id).ok_or_else(|| { - Error::Conflict("Found event_id in sorted events that is not in resolved state") - })?; + let pdu = event_map.get(ev_id).expect("Found event_id in sorted events that is not in resolved state"); // We do not rebuild the PDU in this case only insert to DB db.rooms diff --git a/src/database.rs b/src/database.rs index 6cd65c316..a105058cb 100644 --- a/src/database.rs +++ b/src/database.rs @@ -114,6 +114,7 @@ impl Database { stateid_pduid: db.open_tree("stateid_pduid")?, pduid_statehash: db.open_tree("pduid_statehash")?, + roomid_statehash: db.open_tree("roomid_statehash")?, }, account_data: account_data::AccountData { roomuserdataid_accountdata: db.open_tree("roomuserdataid_accountdata")?, diff --git a/src/database/rooms.rs b/src/database/rooms.rs index 0339b7f9a..62730054c 100644 --- a/src/database/rooms.rs +++ b/src/database/rooms.rs @@ -63,6 +63,8 @@ pub struct Rooms { pub(super) pduid_statehash: sled::Tree, // PDU id -> StateHash /// Also holds the full room state minus the latest event. pub(super) stateid_pduid: sled::Tree, // StateId = StateHash + (EventType, StateKey) + /// The room_id -> the latest StateHash + pub(super) roomid_statehash: sled::Tree, } impl StateStore for Rooms { @@ -93,53 +95,7 @@ impl StateStore for Rooms { } } -// These are the methods related to STATE resolution. impl Rooms { - /// Generates a new StateHash and associates it with the incoming event. - /// - /// This adds all current state events (not including the incoming event) - /// to `stateid_pduid` and adds the incoming event to `pduid_statehash`. - /// The incoming event is the `pdu_id` passed to this method. - pub fn append_state_pdu( - &self, - room_id: &RoomId, - pdu_id: &[u8], - state_key: &str, - kind: &EventType, - ) -> Result { - let state_hash = self.new_state_hash_id(room_id)?; - let state = self.current_state_pduids(room_id)?; - - let mut key = state_hash.as_bytes().to_vec(); - key.push(0xff); - - // TODO eventually we could avoid writing to the DB so much on every event - // by keeping track of the delta and write that every so often - for ((ev_ty, state_key), pid) in state { - let mut state_id = key.to_vec(); - state_id.extend_from_slice(ev_ty.to_string().as_bytes()); - key.push(0xff); - state_id.extend_from_slice(state_key.expect("state event").as_bytes()); - key.push(0xff); - - self.stateid_pduid.insert(&state_id, &pid)?; - } - - // This event's state does not include the event itself. `current_state_pduids` - // uses `roomstateid_pduid` before the current event is inserted to the tree so the state - // will be everything up to but not including the incoming event. - self.pduid_statehash.insert(pdu_id, state_hash.as_bytes())?; - - let mut key = room_id.as_bytes().to_vec(); - key.push(0xff); - key.extend_from_slice(kind.to_string().as_bytes()); - key.push(0xff); - key.extend_from_slice(state_key.as_bytes()); - self.roomstateid_pduid.insert(key, pdu_id)?; - - Ok(state_hash) - } - /// Builds a `StateMap` by iterating over all keys that start /// with `state_hash`, this gives the full state at event "x". pub fn get_statemap_by_hash(&self, state_hash: StateHashId) -> Result> { @@ -633,6 +589,53 @@ impl Rooms { Ok(pdu.event_id) } + /// Generates a new StateHash and associates it with the incoming event. + /// + /// This adds all current state events (not including the incoming event) + /// to `stateid_pduid` and adds the incoming event to `pduid_statehash`. + /// The incoming event is the `pdu_id` passed to this method. + fn append_state_pdu( + &self, + room_id: &RoomId, + pdu_id: &[u8], + state_key: &str, + kind: &EventType, + ) -> Result { + let state_hash = self.new_state_hash_id(room_id)?; + let state = self.current_state_pduids(room_id)?; + + let mut key = state_hash.as_bytes().to_vec(); + key.push(0xff); + + // TODO eventually we could avoid writing to the DB so much on every event + // by keeping track of the delta and write that every so often + for ((ev_ty, state_key), pid) in state { + let mut state_id = key.to_vec(); + state_id.extend_from_slice(ev_ty.to_string().as_bytes()); + key.push(0xff); + state_id.extend_from_slice(state_key.expect("state event").as_bytes()); + key.push(0xff); + + self.stateid_pduid.insert(&state_id, &pid)?; + } + + // This event's state does not include the event itself. `current_state_pduids` + // uses `roomstateid_pduid` before the current event is inserted to the tree so the state + // will be everything up to but not including the incoming event. + self.pduid_statehash.insert(pdu_id, state_hash.as_bytes())?; + + self.roomid_statehash.insert(room_id.as_bytes(), state_hash.as_bytes())?; + + let mut key = room_id.as_bytes().to_vec(); + key.push(0xff); + key.extend_from_slice(kind.to_string().as_bytes()); + key.push(0xff); + key.extend_from_slice(state_key.as_bytes()); + self.roomstateid_pduid.insert(key, pdu_id)?; + + Ok(state_hash) + } + /// Creates a new persisted data unit and adds it to a room. pub fn build_and_append_pdu( &self, From 64fb0374b60461ad1b7600e92032dfc1e30b39d3 Mon Sep 17 00:00:00 2001 From: Devin Ragotzy Date: Wed, 19 Aug 2020 19:29:39 -0400 Subject: [PATCH 05/51] Use Vec instead of string for digest bytes and add roomid_statehash --- src/database/rooms.rs | 39 +++++++++++++++++---------------------- 1 file changed, 17 insertions(+), 22 deletions(-) diff --git a/src/database/rooms.rs b/src/database/rooms.rs index 62730054c..c8ff198b4 100644 --- a/src/database/rooms.rs +++ b/src/database/rooms.rs @@ -33,7 +33,7 @@ use std::{ /// /// This is created when a state group is added to the database by /// hashing the entire state. -pub type StateHashId = String; +pub type StateHashId = Vec; /// This identifier consists of roomId + count. It represents a /// unique event, it will never be overwritten or removed. @@ -100,7 +100,7 @@ impl Rooms { /// with `state_hash`, this gives the full state at event "x". pub fn get_statemap_by_hash(&self, state_hash: StateHashId) -> Result> { self.stateid_pduid - .scan_prefix(state_hash.as_bytes()) + .scan_prefix(&state_hash) .values() .map(|pduid| { self.pduid_pdu.get(&pduid?)?.map_or_else( @@ -123,12 +123,12 @@ impl Rooms { pub fn prev_state_hash(&self, current: StateHashId) -> Option { let mut found = false; for pair in self.pduid_statehash.iter().rev() { - let prev = utils::string_from_bytes(&pair.ok()?.1).ok()?; - if current == prev { + let prev = pair.ok()?.1; + if current == prev.as_ref() { found = true; } - if current != prev && found { - return Some(prev); + if current != prev.as_ref() && found { + return Some(prev.to_vec()); } } None @@ -172,17 +172,14 @@ impl Rooms { // We must check here because this method is called outside and before // `append_state_pdu` so the DB can be empty if self.pduid_statehash.scan_prefix(prefix).next().is_none() { - // TODO use ring crate to hash - return Ok(room_id.as_str().to_owned()); + // return the hash of the room_id, this represents a room with no state + return self.new_state_hash_id(room_id); } self.pduid_statehash .iter() .next_back() - .map(|pair| { - utils::string_from_bytes(&pair?.1) - .map_err(|_| Error::bad_database("Invalid state hash string in db.")) - }) + .map(|pair| Ok(pair?.1.to_vec())) .ok_or_else(|| Error::bad_database("No PDU's found for this room."))? } @@ -255,10 +252,9 @@ impl Rooms { .next() .is_none() { - return utils::string_from_bytes( - digest::digest(&digest::SHA256, room_id.as_bytes()).as_ref(), - ) - .map_err(|_| Error::bad_database("Empty state generated invalid string from hash.")); + return Ok(digest::digest(&digest::SHA256, room_id.as_bytes()) + .as_ref() + .to_vec()); } let pdu_ids_to_hash = self @@ -280,9 +276,7 @@ impl Rooms { &digest::SHA256, &pdu_ids_to_hash.into_iter().flatten().collect::>(), ); - // TODO not sure how you want to hash this - utils::string_from_bytes(hash.as_ref()) - .map_err(|_| Error::bad_database("State generated invalid string from hash.")) + Ok(hash.as_ref().to_vec()) } /// Checks if a room exists. @@ -604,7 +598,7 @@ impl Rooms { let state_hash = self.new_state_hash_id(room_id)?; let state = self.current_state_pduids(room_id)?; - let mut key = state_hash.as_bytes().to_vec(); + let mut key = state_hash.to_vec(); key.push(0xff); // TODO eventually we could avoid writing to the DB so much on every event @@ -622,9 +616,10 @@ impl Rooms { // This event's state does not include the event itself. `current_state_pduids` // uses `roomstateid_pduid` before the current event is inserted to the tree so the state // will be everything up to but not including the incoming event. - self.pduid_statehash.insert(pdu_id, state_hash.as_bytes())?; + self.pduid_statehash.insert(pdu_id, state_hash.as_slice())?; - self.roomid_statehash.insert(room_id.as_bytes(), state_hash.as_bytes())?; + self.roomid_statehash + .insert(room_id.as_bytes(), state_hash.as_slice())?; let mut key = room_id.as_bytes().to_vec(); key.push(0xff); From ea2003240400a5870f1f755d8a13bfb618101aed Mon Sep 17 00:00:00 2001 From: Devin Ragotzy Date: Wed, 19 Aug 2020 19:30:28 -0400 Subject: [PATCH 06/51] Helper for join_room_by_id route so routes aren't calling routes --- src/client_server/membership.rs | 325 ++++++++++++++++---------------- 1 file changed, 163 insertions(+), 162 deletions(-) diff --git a/src/client_server/membership.rs b/src/client_server/membership.rs index 555291e56..0d9fa1250 100644 --- a/src/client_server/membership.rs +++ b/src/client_server/membership.rs @@ -13,14 +13,14 @@ use ruma::{ membership::{ ban_user, forget_room, get_member_events, invite_user, join_room_by_id, join_room_by_id_or_alias, joined_members, joined_rooms, kick_user, leave_room, - unban_user, + unban_user, IncomingThirdPartySigned, }, }, }, federation, }, events::{room::member, EventType}, - EventId, Raw, RoomId, RoomVersionId, + EventId, Raw, RoomId, RoomVersionId, UserId, }; use state_res::StateEvent; @@ -37,136 +37,13 @@ pub async fn join_room_by_id_route( db: State<'_, Database>, body: Ruma, ) -> ConduitResult { - let sender_id = body.sender_id.as_ref().expect("user is authenticated"); - - // Ask a remote server if we don't have this room - if !db.rooms.exists(&body.room_id)? && body.room_id.server_name() != db.globals.server_name() { - let make_join_response = server_server::send_request( - &db, - body.room_id.server_name().to_string(), - federation::membership::create_join_event_template::v1::Request { - room_id: body.room_id.clone(), - user_id: sender_id.clone(), - ver: vec![RoomVersionId::Version5, RoomVersionId::Version6], - }, - ) - .await?; - - let mut join_event_stub_value = - serde_json::from_str::(make_join_response.event.json().get()) - .map_err(|_| { - Error::BadServerResponse("Invalid make_join event json received from server.") - })?; - - let join_event_stub = - join_event_stub_value - .as_object_mut() - .ok_or(Error::BadServerResponse( - "Invalid make join event object received from server.", - ))?; - - join_event_stub.insert( - "origin".to_owned(), - db.globals.server_name().to_owned().to_string().into(), - ); - join_event_stub.insert( - "origin_server_ts".to_owned(), - utils::millis_since_unix_epoch().into(), - ); - - // Generate event id - let event_id = EventId::try_from(&*format!( - "${}", - ruma::signatures::reference_hash(&join_event_stub_value) - .expect("ruma can calculate reference hashes") - )) - .expect("ruma's reference hashes are valid event ids"); - - // We don't leave the event id into the pdu because that's only allowed in v1 or v2 rooms - let join_event_stub = join_event_stub_value.as_object_mut().unwrap(); - join_event_stub.remove("event_id"); - - ruma::signatures::hash_and_sign_event( - db.globals.server_name().as_str(), - db.globals.keypair(), - &mut join_event_stub_value, - ) - .expect("event is valid, we just created it"); - - let send_join_response = server_server::send_request( - &db, - body.room_id.server_name().to_string(), - federation::membership::create_join_event::v1::Request { - room_id: body.room_id.clone(), - event_id, - pdu_stub: serde_json::from_value(join_event_stub_value) - .expect("Raw::from_value always works"), - }, - ) - .await?; - - dbg!(&send_join_response); - // todo!("Take send_join_response and 'create' the room using that data"); - - let mut event_map = send_join_response - .room_state - .state - .iter() - .map(|pdu| pdu.deserialize().map(StateEvent::Full).map(|ev| (ev.event_id(), ev))) - .collect::, _>>() - .map_err(|_| Error::bad_database("Invalid PDU found in db."))?; - - let _auth_chain = send_join_response - .room_state - .auth_chain - .iter() - .flat_map(|pdu| pdu.deserialize().ok()) - .map(StateEvent::Full) - .collect::>(); - - // TODO make StateResolution's methods free functions ? or no self param ? - let sorted_events_ids = state_res::StateResolution::default() - .reverse_topological_power_sort( - &body.room_id, - &event_map.keys().cloned().collect::>(), - &mut event_map, - &db.rooms, - &[], // TODO auth_diff: is this none since we have a set of resolved events we only want to sort - ); - - for ev_id in &sorted_events_ids { - // this is a `state_res::StateEvent` that holds a `ruma::Pdu` - let pdu = event_map.get(ev_id).expect("Found event_id in sorted events that is not in resolved state"); - - // We do not rebuild the PDU in this case only insert to DB - db.rooms - .append_pdu(PduEvent::try_from(pdu)?, &db.globals, &db.account_data)?; - } - } - - let event = member::MemberEventContent { - membership: member::MembershipState::Join, - displayname: db.users.displayname(&sender_id)?, - avatar_url: db.users.avatar_url(&sender_id)?, - is_direct: None, - third_party_invite: None, - }; - - db.rooms.build_and_append_pdu( - PduBuilder { - room_id: body.room_id.clone(), - sender: sender_id.clone(), - event_type: EventType::RoomMember, - content: serde_json::to_value(event).expect("event is valid, we just created it"), - unsigned: None, - state_key: Some(sender_id.to_string()), - redacts: None, - }, - &db.globals, - &db.account_data, - )?; - - Ok(join_room_by_id::Response::new(body.room_id.clone()).into()) + join_room_by_id_helper( + &db, + body.sender_id.as_ref(), + &body.room_id, + body.third_party_signed.as_ref(), + ) + .await } #[cfg_attr( @@ -185,7 +62,7 @@ pub async fn join_room_by_id_or_alias_route( db, Ruma { body: alias::get_alias::IncomingRequest::try_from(http::Request::new( - serde_json::json!({ "room_alias": room_alias }) + serde_json::json!({ "room_alias": room_alias, }) .to_string() .as_bytes() .to_vec(), @@ -202,36 +79,16 @@ pub async fn join_room_by_id_or_alias_route( } }; - // TODO ruma needs to implement the same constructors for the Incoming variants - let tps = if let Some(in_tps) = &body.third_party_signed { - Some(ruma::api::client::r0::membership::ThirdPartySigned { - token: &in_tps.token, - sender: &in_tps.sender, - signatures: in_tps.signatures.clone(), - mxid: &in_tps.mxid, - }) - } else { - None - }; - - let body = Ruma { - sender_id: body.sender_id.clone(), - device_id: body.device_id.clone(), - json_body: None, - body: join_room_by_id::IncomingRequest::try_from(http::Request::new( - serde_json::json!({ - "room_id": room_id, - "third_party_signed": tps, - }) - .to_string() - .as_bytes() - .to_vec(), - )) - .unwrap(), - }; - Ok(join_room_by_id_or_alias::Response { - room_id: join_room_by_id_route(db2, body).await?.0.room_id, + room_id: join_room_by_id_helper( + &db2, + body.sender_id.as_ref(), + &room_id, + body.third_party_signed.as_ref(), + ) + .await? + .0 + .room_id, } .into()) } @@ -568,3 +425,147 @@ pub fn joined_members_route( Ok(joined_members::Response { joined }.into()) } + +async fn join_room_by_id_helper( + db: &Database, + sender_id: Option<&UserId>, + room_id: &RoomId, + _third_party_signed: Option<&IncomingThirdPartySigned>, +) -> ConduitResult { + let sender_id = sender_id.expect("user is authenticated"); + + // Ask a remote server if we don't have this room + if !db.rooms.exists(&room_id)? && room_id.server_name() != db.globals.server_name() { + let make_join_response = server_server::send_request( + &db, + room_id.server_name().to_string(), + federation::membership::create_join_event_template::v1::Request { + room_id: room_id.clone(), + user_id: sender_id.clone(), + ver: vec![RoomVersionId::Version5, RoomVersionId::Version6], + }, + ) + .await?; + + let mut join_event_stub_value = + serde_json::from_str::(make_join_response.event.json().get()) + .map_err(|_| { + Error::BadServerResponse("Invalid make_join event json received from server.") + })?; + + let join_event_stub = + join_event_stub_value + .as_object_mut() + .ok_or(Error::BadServerResponse( + "Invalid make join event object received from server.", + ))?; + + join_event_stub.insert( + "origin".to_owned(), + db.globals.server_name().to_owned().to_string().into(), + ); + join_event_stub.insert( + "origin_server_ts".to_owned(), + utils::millis_since_unix_epoch().into(), + ); + + // Generate event id + let event_id = EventId::try_from(&*format!( + "${}", + ruma::signatures::reference_hash(&join_event_stub_value) + .expect("ruma can calculate reference hashes") + )) + .expect("ruma's reference hashes are valid event ids"); + + // We don't leave the event id into the pdu because that's only allowed in v1 or v2 rooms + let join_event_stub = join_event_stub_value.as_object_mut().unwrap(); + join_event_stub.remove("event_id"); + + ruma::signatures::hash_and_sign_event( + db.globals.server_name().as_str(), + db.globals.keypair(), + &mut join_event_stub_value, + ) + .expect("event is valid, we just created it"); + + let send_join_response = server_server::send_request( + &db, + room_id.server_name().to_string(), + federation::membership::create_join_event::v1::Request { + room_id: room_id.clone(), + event_id, + pdu_stub: serde_json::from_value(join_event_stub_value) + .expect("Raw::from_value always works"), + }, + ) + .await?; + + dbg!(&send_join_response); + // todo!("Take send_join_response and 'create' the room using that data"); + + let mut event_map = send_join_response + .room_state + .state + .iter() + .map(|pdu| { + pdu.deserialize() + .map(StateEvent::Full) + .map(|ev| (ev.event_id(), ev)) + }) + .collect::, _>>() + .map_err(|_| Error::bad_database("Invalid PDU found in db."))?; + + let _auth_chain = send_join_response + .room_state + .auth_chain + .iter() + .flat_map(|pdu| pdu.deserialize().ok()) + .map(StateEvent::Full) + .collect::>(); + + // TODO make StateResolution's methods free functions ? or no self param ? + let sorted_events_ids = state_res::StateResolution::default() + .reverse_topological_power_sort( + &room_id, + &event_map.keys().cloned().collect::>(), + &mut event_map, + &db.rooms, + &[], // TODO auth_diff: is this none since we have a set of resolved events we only want to sort + ); + + for ev_id in &sorted_events_ids { + // this is a `state_res::StateEvent` that holds a `ruma::Pdu` + let pdu = event_map + .get(ev_id) + .expect("Found event_id in sorted events that is not in resolved state"); + + // We do not rebuild the PDU in this case only insert to DB + db.rooms + .append_pdu(PduEvent::try_from(pdu)?, &db.globals, &db.account_data)?; + } + } + + let event = member::MemberEventContent { + membership: member::MembershipState::Join, + displayname: db.users.displayname(&sender_id)?, + avatar_url: db.users.avatar_url(&sender_id)?, + is_direct: None, + third_party_invite: None, + }; + + db.rooms.build_and_append_pdu( + PduBuilder { + room_id: room_id.clone(), + sender: sender_id.clone(), + event_type: EventType::RoomMember, + content: serde_json::to_value(event).expect("event is valid, we just created it"), + unsigned: None, + state_key: Some(sender_id.to_string()), + redacts: None, + }, + &db.globals, + &db.account_data, + )?; + + Ok(join_room_by_id::Response::new(room_id.clone()).into()) +} From 5ccdd3694bd86b7048d089d6a2101ae740a1d0f9 Mon Sep 17 00:00:00 2001 From: Devin Ragotzy Date: Thu, 20 Aug 2020 12:12:02 -0400 Subject: [PATCH 07/51] Add helper function for get_alias route --- src/client_server/alias.rs | 28 +++++++++++++++++--------- src/client_server/membership.rs | 35 +++++++++------------------------ 2 files changed, 28 insertions(+), 35 deletions(-) diff --git a/src/client_server/alias.rs b/src/client_server/alias.rs index 7dc90783d..12bb8df37 100644 --- a/src/client_server/alias.rs +++ b/src/client_server/alias.rs @@ -1,11 +1,14 @@ use super::State; use crate::{server_server, ConduitResult, Database, Error, Ruma}; -use ruma::api::{ - client::{ - error::ErrorKind, - r0::alias::{create_alias, delete_alias, get_alias}, +use ruma::{ + api::{ + client::{ + error::ErrorKind, + r0::alias::{create_alias, delete_alias, get_alias}, + }, + federation, }, - federation, + RoomAliasId, }; #[cfg(feature = "conduit_bin")] @@ -50,12 +53,19 @@ pub async fn get_alias_route( db: State<'_, Database>, body: Ruma, ) -> ConduitResult { - if body.room_alias.server_name() != db.globals.server_name() { + get_alias_helper(db, &body.room_alias).await +} + +pub async fn get_alias_helper( + db: State<'_, Database>, + room_alias: &RoomAliasId, +) -> ConduitResult { + if room_alias.server_name() != db.globals.server_name() { let response = server_server::send_request( &db, - body.room_alias.server_name().to_string(), + room_alias.server_name().to_string(), federation::query::get_room_information::v1::Request { - room_alias: body.room_alias.to_string(), + room_alias: room_alias.to_string(), }, ) .await?; @@ -65,7 +75,7 @@ pub async fn get_alias_route( let room_id = db .rooms - .id_from_alias(&body.room_alias)? + .id_from_alias(&room_alias)? .ok_or(Error::BadRequest( ErrorKind::NotFound, "Room with alias not found.", diff --git a/src/client_server/membership.rs b/src/client_server/membership.rs index 0d9fa1250..0075861a3 100644 --- a/src/client_server/membership.rs +++ b/src/client_server/membership.rs @@ -8,13 +8,10 @@ use ruma::{ api::{ client::{ error::ErrorKind, - r0::{ - alias, - membership::{ - ban_user, forget_room, get_member_events, invite_user, join_room_by_id, - join_room_by_id_or_alias, joined_members, joined_rooms, kick_user, leave_room, - unban_user, IncomingThirdPartySigned, - }, + r0::membership::{ + ban_user, forget_room, get_member_events, invite_user, join_room_by_id, + join_room_by_id_or_alias, joined_members, joined_rooms, kick_user, leave_room, + unban_user, IncomingThirdPartySigned, }, }, federation, @@ -58,24 +55,10 @@ pub async fn join_room_by_id_or_alias_route( let room_id = match RoomId::try_from(body.room_id_or_alias.clone()) { Ok(room_id) => room_id, Err(room_alias) => { - client_server::get_alias_route( - db, - Ruma { - body: alias::get_alias::IncomingRequest::try_from(http::Request::new( - serde_json::json!({ "room_alias": room_alias, }) - .to_string() - .as_bytes() - .to_vec(), - )) - .unwrap(), - sender_id: body.sender_id.clone(), - device_id: body.device_id.clone(), - json_body: None, - }, - ) - .await? - .0 - .room_id + client_server::get_alias_helper(db, &room_alias) + .await? + .0 + .room_id } }; @@ -495,7 +478,7 @@ async fn join_room_by_id_helper( room_id: room_id.clone(), event_id, pdu_stub: serde_json::from_value(join_event_stub_value) - .expect("Raw::from_value always works"), + .expect("we just created this event"), }, ) .await?; From fe795d38ead8b83c413187f68e83ccea4185c8c7 Mon Sep 17 00:00:00 2001 From: Devin Ragotzy Date: Fri, 21 Aug 2020 17:19:18 -0400 Subject: [PATCH 08/51] Replace route calling routes with helpers This fixes the panic from ruma "index out of bounds" --- src/client_server/alias.rs | 4 +- src/client_server/membership.rs | 20 ++--- src/client_server/state.rs | 142 ++++++++++++++++++-------------- 3 files changed, 90 insertions(+), 76 deletions(-) diff --git a/src/client_server/alias.rs b/src/client_server/alias.rs index 12bb8df37..669f558d6 100644 --- a/src/client_server/alias.rs +++ b/src/client_server/alias.rs @@ -53,11 +53,11 @@ pub async fn get_alias_route( db: State<'_, Database>, body: Ruma, ) -> ConduitResult { - get_alias_helper(db, &body.room_alias).await + get_alias_helper(&db, &body.room_alias).await } pub async fn get_alias_helper( - db: State<'_, Database>, + db: &Database, room_alias: &RoomAliasId, ) -> ConduitResult { if room_alias.server_name() != db.globals.server_name() { diff --git a/src/client_server/membership.rs b/src/client_server/membership.rs index 0075861a3..996d3c489 100644 --- a/src/client_server/membership.rs +++ b/src/client_server/membership.rs @@ -49,13 +49,12 @@ pub async fn join_room_by_id_route( )] pub async fn join_room_by_id_or_alias_route( db: State<'_, Database>, - db2: State<'_, Database>, body: Ruma, ) -> ConduitResult { let room_id = match RoomId::try_from(body.room_id_or_alias.clone()) { Ok(room_id) => room_id, Err(room_alias) => { - client_server::get_alias_helper(db, &room_alias) + client_server::get_alias_helper(&db, &room_alias) .await? .0 .room_id @@ -64,7 +63,7 @@ pub async fn join_room_by_id_or_alias_route( Ok(join_room_by_id_or_alias::Response { room_id: join_room_by_id_helper( - &db2, + &db, body.sender_id.as_ref(), &room_id, body.third_party_signed.as_ref(), @@ -507,14 +506,13 @@ async fn join_room_by_id_helper( .collect::>(); // TODO make StateResolution's methods free functions ? or no self param ? - let sorted_events_ids = state_res::StateResolution::default() - .reverse_topological_power_sort( - &room_id, - &event_map.keys().cloned().collect::>(), - &mut event_map, - &db.rooms, - &[], // TODO auth_diff: is this none since we have a set of resolved events we only want to sort - ); + let sorted_events_ids = state_res::StateResolution::reverse_topological_power_sort( + &room_id, + &event_map.keys().cloned().collect::>(), + &mut event_map, + &db.rooms, + &[], // TODO auth_diff: is this none since we have a set of resolved events we only want to sort + ); for ev_id in &sorted_events_ids { // this is a `state_res::StateEvent` that holds a `ruma::Pdu` diff --git a/src/client_server/state.rs b/src/client_server/state.rs index 2920de2f6..867b051ea 100644 --- a/src/client_server/state.rs +++ b/src/client_server/state.rs @@ -9,8 +9,8 @@ use ruma::{ }, }, events::{AnyStateEventContent, EventContent}, + RoomId, UserId, }; -use std::convert::TryFrom; #[cfg(feature = "conduit_bin")] use rocket::{get, put}; @@ -33,45 +33,14 @@ pub fn send_state_event_for_key_route( ) .map_err(|_| Error::BadRequest(ErrorKind::BadJson, "Invalid JSON body."))?; - if let AnyStateEventContent::RoomCanonicalAlias(canonical_alias) = &body.content { - let mut aliases = canonical_alias.alt_aliases.clone(); - - if let Some(alias) = canonical_alias.alias.clone() { - aliases.push(alias); - } - - for alias in aliases { - if alias.server_name() != db.globals.server_name() - || db - .rooms - .id_from_alias(&alias)? - .filter(|room| room == &body.room_id) // Make sure it's the right room - .is_none() - { - return Err(Error::BadRequest( - ErrorKind::Forbidden, - "You are only allowed to send canonical_alias \ - events when it's aliases already exists", - )); - } - } - } - - let event_id = db.rooms.build_and_append_pdu( - PduBuilder { - room_id: body.room_id.clone(), - sender: sender_id.clone(), - event_type: body.content.event_type().into(), - content, - unsigned: None, - state_key: Some(body.state_key.clone()), - redacts: None, - }, - &db.globals, - &db.account_data, - )?; - - Ok(send_state_event_for_key::Response::new(event_id).into()) + send_state_event_for_key_helper( + &db, + sender_id, + &body.content, + content, + &body.room_id, + Some(body.state_key.clone()), + ) } #[cfg_attr( @@ -84,34 +53,30 @@ pub fn send_state_event_for_empty_key_route( ) -> ConduitResult { // This just calls send_state_event_for_key_route let Ruma { - body: - send_state_event_for_empty_key::IncomingRequest { - room_id, content, .. - }, + body, sender_id, - device_id, + device_id: _, json_body, } = body; + let json = serde_json::from_str::( + json_body + .as_ref() + .ok_or(Error::BadRequest(ErrorKind::BadJson, "Invalid JSON body."))? + .get(), + ) + .map_err(|_| Error::BadRequest(ErrorKind::BadJson, "Invalid JSON body."))?; + Ok(send_state_event_for_empty_key::Response::new( - send_state_event_for_key_route( - db, - Ruma { - body: send_state_event_for_key::IncomingRequest::try_from(http::Request::new( - serde_json::json!({ - "room_id": room_id, - "state_key": "", - "content": content, - }) - .to_string() - .as_bytes() - .to_vec(), - )) - .unwrap(), - sender_id, - device_id, - json_body, - }, + send_state_event_for_key_helper( + &db, + sender_id + .as_ref() + .expect("no user for send state empty key rout"), + &body.content, + json, + &body.room_id, + None, )? .0 .event_id, @@ -210,3 +175,54 @@ pub fn get_state_events_for_empty_key_route( } .into()) } + +pub fn send_state_event_for_key_helper( + db: &Database, + sender: &UserId, + content: &AnyStateEventContent, + json: serde_json::Value, + room_id: &RoomId, + state_key: Option, +) -> ConduitResult { + let sender_id = sender; + + if let AnyStateEventContent::RoomCanonicalAlias(canonical_alias) = content { + let mut aliases = canonical_alias.alt_aliases.clone(); + + if let Some(alias) = canonical_alias.alias.clone() { + aliases.push(alias); + } + + for alias in aliases { + if alias.server_name() != db.globals.server_name() + || db + .rooms + .id_from_alias(&alias)? + .filter(|room| room == room_id) // Make sure it's the right room + .is_none() + { + return Err(Error::BadRequest( + ErrorKind::Forbidden, + "You are only allowed to send canonical_alias \ + events when it's aliases already exists", + )); + } + } + } + + let event_id = db.rooms.build_and_append_pdu( + PduBuilder { + room_id: room_id.clone(), + sender: sender_id.clone(), + event_type: content.event_type().into(), + content: json, + unsigned: None, + state_key, + redacts: None, + }, + &db.globals, + &db.account_data, + )?; + + Ok(send_state_event_for_key::Response::new(event_id).into()) +} From 672bf4f47376eea520c19fa1b9108478ec56974f Mon Sep 17 00:00:00 2001 From: Devin Ragotzy Date: Fri, 21 Aug 2020 20:18:56 -0400 Subject: [PATCH 09/51] Cargo lock update and a few doc additions --- Cargo.lock | 22 +++++++++++----------- src/client_server/config.rs | 6 +----- src/client_server/room.rs | 1 + src/database/rooms.rs | 2 +- 4 files changed, 14 insertions(+), 17 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3c5d836ba..0e5da6f6f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -144,9 +144,9 @@ dependencies = [ [[package]] name = "autocfg" -version = "1.0.0" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8aac770f1885fd7e387acedd76065302551364496e46b3dd00860b2f8359b9d" +checksum = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a" [[package]] name = "backtrace" @@ -247,9 +247,9 @@ checksum = "0e4cec68f03f32e44924783795810fa50a7035d8c8ebe78580ad7e6c703fba38" [[package]] name = "cc" -version = "1.0.58" +version = "1.0.59" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9a06fb2e53271d7c279ec1efea6ab691c35a2ae67ec0d91d7acec0caf13b518" +checksum = "66120af515773fb005778dc07c261bd201ec8ce50bd6e7144c927753fe013381" [[package]] name = "cfg-if" @@ -960,9 +960,9 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] name = "libc" -version = "0.2.74" +version = "0.2.76" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2f02823cf78b754822df5f7f268fb59822e7296276d3e069d8e8cb26a14bd10" +checksum = "755456fae044e6fa1ebbbd1b3e902ae19e73097ed4ed87bb79934a867c007bc3" [[package]] name = "lock_api" @@ -1358,9 +1358,9 @@ dependencies = [ [[package]] name = "ppv-lite86" -version = "0.2.8" +version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "237a5ed80e274dbc66f86bd59c1e25edc039660be53194b5fe0a482e0f2612ea" +checksum = "c36fa947111f5c62a733b652544dd0016a43ce89619538a8ef92724a6f501a20" [[package]] name = "proc-macro-crate" @@ -2088,7 +2088,7 @@ checksum = "7345c971d1ef21ffdbd103a75990a15eb03604fc8b8852ca8cb418ee1a099028" [[package]] name = "state-res" version = "0.1.0" -source = "git+https://github.com/ruma/state-res#4e9b428c0db50ac3a3421ced12a6fd202a1c36a3" +source = "git+https://github.com/ruma/state-res#d93a965ad17781fa9554bb3cea71673c054b9f3f" dependencies = [ "itertools", "js_int", @@ -2179,9 +2179,9 @@ checksum = "502d53007c02d7605a05df1c1a73ee436952781653da5d0bf57ad608f66932c1" [[package]] name = "syn" -version = "1.0.38" +version = "1.0.39" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e69abc24912995b3038597a7a593be5053eb0fb44f3cc5beec0deb421790c1f4" +checksum = "891d8d6567fe7c7f8835a3a98af4208f3846fba258c1bc3c31d6e506239f11f9" dependencies = [ "proc-macro2", "quote", diff --git a/src/client_server/config.rs b/src/client_server/config.rs index 8cb6a0d72..45aec33e8 100644 --- a/src/client_server/config.rs +++ b/src/client_server/config.rs @@ -56,11 +56,7 @@ pub fn get_global_account_data_route( let data = db .account_data - .get::>( - None, - sender_id, - EventType::try_from(&body.event_type).expect("EventType::try_from can never fail"), - )? + .get::>(None, sender_id, body.event_type.clone().into())? .ok_or(Error::BadRequest(ErrorKind::NotFound, "Data not found."))?; Ok(get_global_account_data::Response { account_data: data }.into()) diff --git a/src/client_server/room.rs b/src/client_server/room.rs index 3ee21b6d8..c0603d361 100644 --- a/src/client_server/room.rs +++ b/src/client_server/room.rs @@ -195,6 +195,7 @@ pub fn create_room_route( content: match preset { create_room::RoomPreset::PublicChat => { serde_json::to_value(guest_access::GuestAccessEventContent::new( + // In a public room a joining is the only way to access guest_access::GuestAccess::Forbidden, )) .expect("event is valid, we just created it") diff --git a/src/database/rooms.rs b/src/database/rooms.rs index c8ff198b4..012974214 100644 --- a/src/database/rooms.rs +++ b/src/database/rooms.rs @@ -722,7 +722,7 @@ impl Rooms { } EventType::RoomMember => event_auth::is_membership_change_allowed( // TODO this is a bit of a hack but not sure how to have a type - // declared in `state_res` crate be + // declared in `state_res` crate easily convert to/from conduit::PduEvent Requester { prev_event_ids: prev_events.to_owned(), room_id: &room_id, From 972babbc795c9bfc6f3aee7b6351c0deb7e51b5d Mon Sep 17 00:00:00 2001 From: Timo Date: Wed, 19 Aug 2020 18:26:39 +0200 Subject: [PATCH 10/51] fix: set limited to true when skipping messages in /sync --- src/client_server/config.rs | 3 +-- src/client_server/search.rs | 27 +++++++++++++-------------- src/database/rooms.rs | 19 +++++++++++++++++-- src/pdu.rs | 24 +++++++++++++++++++++++- 4 files changed, 54 insertions(+), 19 deletions(-) diff --git a/src/client_server/config.rs b/src/client_server/config.rs index 45aec33e8..baa9381ba 100644 --- a/src/client_server/config.rs +++ b/src/client_server/config.rs @@ -5,10 +5,9 @@ use ruma::{ error::ErrorKind, r0::config::{get_global_account_data, set_global_account_data}, }, - events::{custom::CustomEventContent, BasicEvent, EventType}, + events::{custom::CustomEventContent, BasicEvent}, Raw, }; -use std::convert::TryFrom; #[cfg(feature = "conduit_bin")] use rocket::{get, put}; diff --git a/src/client_server/search.rs b/src/client_server/search.rs index dec1ec9f6..082711d13 100644 --- a/src/client_server/search.rs +++ b/src/client_server/search.rs @@ -14,7 +14,7 @@ use std::collections::BTreeMap; )] pub fn search_events_route( db: State<'_, Database>, - body: Ruma, + body: Ruma, ) -> ConduitResult { let sender_id = body.sender_id.as_ref().expect("user is authenticated"); @@ -56,7 +56,8 @@ pub fn search_events_route( result: db .rooms .get_pdu_from_id(&result)? - .map(|pdu| pdu.to_room_event()), + // TODO this is an awkward type conversion see method + .map(|pdu| pdu.to_any_event()), }) }) .filter_map(|r| r.ok()) @@ -70,17 +71,15 @@ pub fn search_events_route( Some((skip + limit).to_string()) }; - Ok(search_events::Response { - search_categories: ResultCategories { - room_events: Some(ResultRoomEvents { - count: uint!(0), // TODO - groups: BTreeMap::new(), // TODO - next_batch, - results, - state: BTreeMap::new(), // TODO - highlights: search.1, - }), - }, - } + Ok(search_events::Response::new(ResultCategories { + room_events: Some(ResultRoomEvents { + count: uint!(0), // TODO + groups: BTreeMap::new(), // TODO + next_batch, + results, + state: BTreeMap::new(), // TODO + highlights: search.1, + }), + }) .into()) } diff --git a/src/database/rooms.rs b/src/database/rooms.rs index 012974214..d087d6525 100644 --- a/src/database/rooms.rs +++ b/src/database/rooms.rs @@ -532,7 +532,7 @@ impl Rooms { self.append_state_pdu(&pdu.room_id, &pdu_id, state_key, &pdu.kind)?; } - match pdu.kind { + match &pdu.kind { EventType::RoomRedaction => { if let Some(redact_id) = &pdu.redacts { // TODO: Reason @@ -553,7 +553,7 @@ impl Rooms { } } EventType::RoomMember => { - if let Some(state_key) = &pdu.state_key { + if let Some(state_key) = pdu.state_key.as_ref() { // if the state_key fails let target_user_id = UserId::try_from(state_key.as_str()) .expect("This state_key was previously validated"); @@ -576,6 +576,21 @@ impl Rooms { )?; } } + EventType::RoomMessage => { + if let Some(body) = pdu.content.get("body").and_then(|b| b.as_str()) { + for word in body + .split_terminator(|c: char| !c.is_alphanumeric()) + .map(str::to_lowercase) + { + let mut key = pdu.room_id.to_string().as_bytes().to_vec(); + key.push(0xff); + key.extend_from_slice(word.as_bytes()); + key.push(0xff); + key.extend_from_slice(&pdu_id); + self.tokenids.insert(key, &[])?; + } + } + } _ => {} } self.edus.room_read_set(&pdu.room_id, &pdu.sender, index)?; diff --git a/src/pdu.rs b/src/pdu.rs index eec8e4964..b565a24c6 100644 --- a/src/pdu.rs +++ b/src/pdu.rs @@ -2,7 +2,7 @@ use crate::{Error, Result}; use js_int::UInt; use ruma::{ events::{ - pdu::EventHash, room::member::MemberEventContent, AnyRoomEvent, AnyStateEvent, + pdu::EventHash, room::member::MemberEventContent, AnyEvent, AnyRoomEvent, AnyStateEvent, AnyStrippedStateEvent, AnySyncRoomEvent, AnySyncStateEvent, EventType, StateEvent, }, EventId, Raw, RoomId, ServerName, UserId, @@ -99,6 +99,28 @@ impl PduEvent { serde_json::from_value(json).expect("Raw::from_value always works") } + /// This only works for events that are also AnyRoomEvents. + pub fn to_any_event(&self) -> Raw { + let mut json = json!({ + "content": self.content, + "type": self.kind, + "event_id": self.event_id, + "sender": self.sender, + "origin_server_ts": self.origin_server_ts, + "unsigned": self.unsigned, + "room_id": self.room_id, + }); + + if let Some(state_key) = &self.state_key { + json["state_key"] = json!(state_key); + } + if let Some(redacts) = &self.redacts { + json["redacts"] = json!(redacts); + } + + serde_json::from_value(json).expect("Raw::from_value always works") + } + pub fn to_room_event(&self) -> Raw { let mut json = json!({ "content": self.content, From 3c26166fb59d78cf887645144ffbb108328247df Mon Sep 17 00:00:00 2001 From: Timo Date: Fri, 21 Aug 2020 21:22:59 +0200 Subject: [PATCH 11/51] improvement: device list works better The only situation that isn't working yet is sending `left` events for users when the sender leaves the room --- src/client_server/sync.rs | 20 ++++++-------------- src/database/rooms.rs | 9 ++++----- src/utils.rs | 6 ------ 3 files changed, 10 insertions(+), 25 deletions(-) diff --git a/src/client_server/sync.rs b/src/client_server/sync.rs index ae4c22426..accb199bf 100644 --- a/src/client_server/sync.rs +++ b/src/client_server/sync.rs @@ -149,15 +149,7 @@ pub async fn sync_events_route( device_list_updates.extend( db.rooms .room_members(&room_id) - .filter_map(|user_id| { - Some( - UserId::try_from(user_id.ok()?.clone()) - .map_err(|_| { - Error::bad_database("Invalid member event state key in db.") - }) - .ok()?, - ) - }) + .filter_map(|user_id| Some(user_id.ok()?)) .filter(|user_id| { // Don't send key updates from the sender to the sender sender_id != user_id @@ -491,9 +483,7 @@ pub async fn sync_events_route( } for user_id in left_encrypted_users { - // If the user doesn't share an encrypted room with the target anymore, we need to tell - // them - if db + let user_target_encrypted = db .rooms .get_shared_rooms(vec![sender_id.clone(), user_id.clone()]) .filter_map(|r| r.ok()) @@ -505,8 +495,10 @@ pub async fn sync_events_route( .is_some(), ) }) - .all(|encrypted| !encrypted) - { + .all(|encrypted| !encrypted); + // If the user doesn't share an encrypted room with the target anymore, we need to tell + // them + if user_target_encrypted { device_list_left.insert(user_id); } } diff --git a/src/database/rooms.rs b/src/database/rooms.rs index d087d6525..575a2bf09 100644 --- a/src/database/rooms.rs +++ b/src/database/rooms.rs @@ -75,23 +75,23 @@ impl StateStore for Rooms { .map_err(|e| e.to_string())? .ok_or_else(|| "PDU via room_id and event_id not found in the db.".to_owned())?; - utils::deserialize( + serde_json::from_slice( &self .pduid_pdu .get(pid) .map_err(|e| e.to_string())? .ok_or_else(|| "PDU via pduid not found in db.".to_owned())?, ) + .map_err(|e| e.to_string()) .and_then(|pdu: StateEvent| { // conduit's PDU's always contain a room_id but some // of ruma's do not so this must be an Option if pdu.room_id() == Some(room_id) { Ok(pdu) } else { - Err(Error::bad_database("Found PDU for incorrect room in db.")) + Err("Found PDU for incorrect room in db.".into()) } }) - .map_err(|e| e.to_string()) } } @@ -1207,8 +1207,7 @@ impl Rooms { let roomid_index = key .iter() .enumerate() - .filter(|(_, &b)| b == 0xff) - .nth(0) + .find(|(_, &b)| b == 0xff) .ok_or_else(|| Error::bad_database("Invalid userroomid_joined in db."))? .0 + 1; // +1 because the room id starts AFTER the separator diff --git a/src/utils.rs b/src/utils.rs index b549153ab..8cf1b2ce1 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -1,4 +1,3 @@ -use crate::Error; use argon2::{Config, Variant}; use cmp::Ordering; use rand::prelude::*; @@ -91,8 +90,3 @@ pub fn common_elements( .all(|b| b) })) } - -pub fn deserialize<'de, T: serde::Deserialize<'de>>(val: &'de sled::IVec) -> Result { - serde_json::from_slice::(val.as_ref()) - .map_err(|_| Error::bad_database("Found invalid bytes as PDU in db.")) -} From 1848f08292875030f100bc4ffa987e31c15d6912 Mon Sep 17 00:00:00 2001 From: Devin Ragotzy Date: Fri, 21 Aug 2020 21:44:55 -0400 Subject: [PATCH 12/51] Use full sorting algorithm on incoming PDU's in membership --- src/client_server/membership.rs | 50 +++++++++++++++++++++++++++++---- 1 file changed, 44 insertions(+), 6 deletions(-) diff --git a/src/client_server/membership.rs b/src/client_server/membership.rs index 996d3c489..3fa3b6ae7 100644 --- a/src/client_server/membership.rs +++ b/src/client_server/membership.rs @@ -497,7 +497,7 @@ async fn join_room_by_id_helper( .collect::, _>>() .map_err(|_| Error::bad_database("Invalid PDU found in db."))?; - let _auth_chain = send_join_response + let auth_chain = send_join_response .room_state .auth_chain .iter() @@ -505,16 +505,54 @@ async fn join_room_by_id_helper( .map(StateEvent::Full) .collect::>(); - // TODO make StateResolution's methods free functions ? or no self param ? - let sorted_events_ids = state_res::StateResolution::reverse_topological_power_sort( + let power_events = event_map + .values() + .filter(|pdu| pdu.is_power_event()) + .map(|pdu| pdu.event_id()) + .collect::>(); + + // TODO these events are not guaranteed to be sorted but they are resolved, do + // we need the auth_chain + let sorted_power_events = state_res::StateResolution::reverse_topological_power_sort( &room_id, - &event_map.keys().cloned().collect::>(), + &power_events, &mut event_map, &db.rooms, - &[], // TODO auth_diff: is this none since we have a set of resolved events we only want to sort + &auth_chain // if we only use it here just build this list in the first place + .iter() + .map(|pdu| pdu.event_id()) + .collect::>(), ); - for ev_id in &sorted_events_ids { + // TODO we may be able to skip this since they are resolved according to spec + let resolved_power = state_res::StateResolution::iterative_auth_check( + room_id, + &RoomVersionId::Version6, + &sorted_power_events, + &BTreeMap::new(), // unconflicted events + &mut event_map, + &db.rooms, + ) + .expect("iterative auth check failed on resolved events"); + // TODO do we need to dedup them + + let events_to_sort = event_map + .keys() + .filter(|id| !sorted_power_events.contains(id)) + .cloned() + .collect::>(); + + let power_level = resolved_power.get(&(EventType::RoomPowerLevels, Some("".into()))); + + let sorted_event_ids = state_res::StateResolution::mainline_sort( + room_id, + &events_to_sort, + power_level, + &mut event_map, + &db.rooms, + ); + + for ev_id in &sorted_event_ids { // this is a `state_res::StateEvent` that holds a `ruma::Pdu` let pdu = event_map .get(ev_id) From 27ffe778233370347d50b914862b6e866dae557b Mon Sep 17 00:00:00 2001 From: Devin Ragotzy Date: Sun, 23 Aug 2020 08:32:43 -0400 Subject: [PATCH 13/51] Use helper instead of route for get_public_rooms_filtered --- src/client_server/directory.rs | 224 ++++++++++++++++++--------------- src/client_server/sync.rs | 10 +- src/server_server.rs | 25 ++-- 3 files changed, 131 insertions(+), 128 deletions(-) diff --git a/src/client_server/directory.rs b/src/client_server/directory.rs index 0aace15d1..5e03274b8 100644 --- a/src/client_server/directory.rs +++ b/src/client_server/directory.rs @@ -14,7 +14,7 @@ use ruma::{ }, federation, }, - directory::PublicRoomsChunk, + directory::{Filter, PublicRoomsChunk, RoomNetwork}, events::{ room::{avatar, canonical_alias, guest_access, history_visibility, name, topic}, EventType, @@ -33,17 +33,123 @@ pub async fn get_public_rooms_filtered_route( db: State<'_, Database>, body: Ruma, ) -> ConduitResult { - if let Some(other_server) = body - .server + let Ruma { + body: + get_public_rooms_filtered::IncomingRequest { + limit, + server, + since, + filter, + room_network, + }, + .. + } = body; + get_public_rooms_filtered_helper( + &db, + server.as_deref(), + limit, + since.as_deref(), + filter, // This is not used yet + Some(room_network), // This is not used + ) + .await +} + +#[cfg_attr( + feature = "conduit_bin", + get("/_matrix/client/r0/publicRooms", data = "") +)] +pub async fn get_public_rooms_route( + db: State<'_, Database>, + body: Ruma, +) -> ConduitResult { + let Ruma { + body: + get_public_rooms::IncomingRequest { + limit, + server, + since, + }, + .. + } = body; + + let get_public_rooms_filtered::Response { + chunk, + prev_batch, + next_batch, + total_room_count_estimate, + } = get_public_rooms_filtered_helper( + &db, + server.as_deref(), + limit, + since.as_deref(), + None, // This is not used + None, // This is not used + ) + .await? + .0; + + Ok(get_public_rooms::Response { + chunk, + prev_batch, + next_batch, + total_room_count_estimate, + } + .into()) +} + +#[cfg_attr( + feature = "conduit_bin", + put("/_matrix/client/r0/directory/list/room/<_>", data = "") +)] +pub async fn set_room_visibility_route( + db: State<'_, Database>, + body: Ruma, +) -> ConduitResult { + match body.visibility { + room::Visibility::Public => db.rooms.set_public(&body.room_id, true)?, + room::Visibility::Private => db.rooms.set_public(&body.room_id, false)?, + } + + Ok(set_room_visibility::Response.into()) +} + +#[cfg_attr( + feature = "conduit_bin", + get("/_matrix/client/r0/directory/list/room/<_>", data = "") +)] +pub async fn get_room_visibility_route( + db: State<'_, Database>, + body: Ruma, +) -> ConduitResult { + Ok(get_room_visibility::Response { + visibility: if db.rooms.is_public_room(&body.room_id)? { + room::Visibility::Public + } else { + room::Visibility::Private + }, + } + .into()) +} + +pub async fn get_public_rooms_filtered_helper( + db: &Database, + server: Option<&str>, + limit: Option, + since: Option<&str>, + _filter: Option, + _network: Option, +) -> ConduitResult { + if let Some(other_server) = server .clone() - .filter(|server| server != db.globals.server_name().as_str()) + .filter(|server| *server != db.globals.server_name().as_str()) { let response = server_server::send_request( &db, - other_server, + other_server.to_owned(), federation::directory::get_public_rooms::v1::Request { - limit: body.limit, - since: body.since.as_deref(), + limit, + since: since.as_deref(), room_network: ruma::directory::RoomNetwork::Matrix, }, ) @@ -73,10 +179,10 @@ pub async fn get_public_rooms_filtered_route( .into()); } - let limit = body.limit.map_or(10, u64::from); - let mut since = 0_u64; + let limit = limit.map_or(10, u64::from); + let mut num_since = 0_u64; - if let Some(s) = &body.since { + if let Some(s) = &since { let mut characters = s.chars(); let backwards = match characters.next() { Some('n') => false, @@ -89,13 +195,13 @@ pub async fn get_public_rooms_filtered_route( } }; - since = characters + num_since = characters .collect::() .parse() .map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "Invalid `since` token."))?; if backwards { - since = since.saturating_sub(limit); + num_since = num_since.saturating_sub(limit); } } @@ -217,20 +323,20 @@ pub async fn get_public_rooms_filtered_route( let chunk = all_rooms .into_iter() - .skip(since as usize) + .skip(num_since as usize) .take(limit as usize) .collect::>(); - let prev_batch = if since == 0 { + let prev_batch = if num_since == 0 { None } else { - Some(format!("p{}", since)) + Some(format!("p{}", num_since)) }; let next_batch = if chunk.len() < limit as usize { None } else { - Some(format!("n{}", since + limit)) + Some(format!("n{}", num_since + limit)) }; Ok(get_public_rooms_filtered::Response { @@ -241,89 +347,3 @@ pub async fn get_public_rooms_filtered_route( } .into()) } - -#[cfg_attr( - feature = "conduit_bin", - get("/_matrix/client/r0/publicRooms", data = "") -)] -pub async fn get_public_rooms_route( - db: State<'_, Database>, - body: Ruma, -) -> ConduitResult { - let Ruma { - body: - get_public_rooms::IncomingRequest { - limit, - server, - since, - }, - sender_id, - device_id, - json_body, - } = body; - - let get_public_rooms_filtered::Response { - chunk, - prev_batch, - next_batch, - total_room_count_estimate, - } = get_public_rooms_filtered_route( - db, - Ruma { - body: get_public_rooms_filtered::IncomingRequest { - filter: None, - limit, - room_network: ruma::directory::RoomNetwork::Matrix, - server, - since, - }, - sender_id, - device_id, - json_body, - }, - ) - .await? - .0; - - Ok(get_public_rooms::Response { - chunk, - prev_batch, - next_batch, - total_room_count_estimate, - } - .into()) -} - -#[cfg_attr( - feature = "conduit_bin", - put("/_matrix/client/r0/directory/list/room/<_>", data = "") -)] -pub async fn set_room_visibility_route( - db: State<'_, Database>, - body: Ruma, -) -> ConduitResult { - match body.visibility { - room::Visibility::Public => db.rooms.set_public(&body.room_id, true)?, - room::Visibility::Private => db.rooms.set_public(&body.room_id, false)?, - } - - Ok(set_room_visibility::Response.into()) -} - -#[cfg_attr( - feature = "conduit_bin", - get("/_matrix/client/r0/directory/list/room/<_>", data = "") -)] -pub async fn get_room_visibility_route( - db: State<'_, Database>, - body: Ruma, -) -> ConduitResult { - Ok(get_room_visibility::Response { - visibility: if db.rooms.is_public_room(&body.room_id)? { - room::Visibility::Public - } else { - room::Visibility::Private - }, - } - .into()) -} diff --git a/src/client_server/sync.rs b/src/client_server/sync.rs index accb199bf..ccb25d19b 100644 --- a/src/client_server/sync.rs +++ b/src/client_server/sync.rs @@ -415,15 +415,7 @@ pub async fn sync_events_route( device_list_left.extend( db.rooms .room_members(&room_id) - .filter_map(|user_id| { - Some( - UserId::try_from(user_id.ok()?.clone()) - .map_err(|_| { - Error::bad_database("Invalid member event state key in db.") - }) - .ok()?, - ) - }) + .filter_map(|user_id| Some(user_id.ok()?)) .filter(|user_id| { // Don't send key updates from the sender to the sender sender_id != user_id diff --git a/src/server_server.rs b/src/server_server.rs index e47b50a52..ac4407baa 100644 --- a/src/server_server.rs +++ b/src/server_server.rs @@ -216,9 +216,7 @@ pub async fn get_public_rooms_route( limit, since, }, - sender_id, - device_id, - json_body, + .. } = body; let client::r0::directory::get_public_rooms_filtered::Response { @@ -226,20 +224,13 @@ pub async fn get_public_rooms_route( prev_batch, next_batch, total_room_count_estimate, - } = client_server::get_public_rooms_filtered_route( - db, - Ruma { - body: client::r0::directory::get_public_rooms_filtered::IncomingRequest { - filter: None, - limit, - room_network: ruma::directory::RoomNetwork::Matrix, - server: None, - since, - }, - sender_id, - device_id, - json_body, - }, + } = client_server::get_public_rooms_filtered_helper( + &db, + None, + limit, + since.as_deref(), + None, + Some(ruma::directory::RoomNetwork::Matrix), ) .await? .0; From d9a29e3e5c2b8390bb1a4b99a631b3ac615b0763 Mon Sep 17 00:00:00 2001 From: Devin Ragotzy Date: Tue, 25 Aug 2020 15:30:25 -0400 Subject: [PATCH 14/51] Fix state for empty key route Replace None with Some("") for state_key --- src/client_server/room.rs | 2 +- src/client_server/state.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/client_server/room.rs b/src/client_server/room.rs index c0603d361..7f4a15fab 100644 --- a/src/client_server/room.rs +++ b/src/client_server/room.rs @@ -195,7 +195,7 @@ pub fn create_room_route( content: match preset { create_room::RoomPreset::PublicChat => { serde_json::to_value(guest_access::GuestAccessEventContent::new( - // In a public room a joining is the only way to access + // In a public room, joining is the only way to access guest_access::GuestAccess::Forbidden, )) .expect("event is valid, we just created it") diff --git a/src/client_server/state.rs b/src/client_server/state.rs index 867b051ea..12c5cacb4 100644 --- a/src/client_server/state.rs +++ b/src/client_server/state.rs @@ -76,7 +76,7 @@ pub fn send_state_event_for_empty_key_route( &body.content, json, &body.room_id, - None, + Some("".into()), )? .0 .event_id, From f46c2d1eec808658f671a4845b0ecd39221c8826 Mon Sep 17 00:00:00 2001 From: Devin Ragotzy Date: Wed, 26 Aug 2020 11:15:52 -0400 Subject: [PATCH 15/51] Fix review issues, move state-res to spec-comp branch --- Cargo.lock | 79 ++++++++++++++------------------- Cargo.toml | 2 +- src/client_server/directory.rs | 31 ++++--------- src/client_server/membership.rs | 16 +++---- src/client_server/room.rs | 1 - src/client_server/state.rs | 35 ++++++++------- src/client_server/sync.rs | 4 +- src/database/rooms.rs | 76 +++++++++++++++---------------- 8 files changed, 106 insertions(+), 138 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 0e5da6f6f..62b13d00b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -157,7 +157,7 @@ dependencies = [ "addr2line", "cfg-if", "libc", - "miniz_oxide 0.4.0", + "miniz_oxide 0.4.1", "object", "rustc-demangle", ] @@ -168,12 +168,6 @@ version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1b20b618342cf9891c292c4f5ac2cde7287cc5c87e87e9c769d617793607dec1" -[[package]] -name = "base64" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b41b7ea54a0c9d92199de89e20e58d49f02f8e699814ef3fdf266f6f748d15c7" - [[package]] name = "base64" version = "0.12.3" @@ -265,7 +259,7 @@ checksum = "942f72db697d8767c22d46a598e01f2d3b475501ea43d0db4f16d90259182d0b" dependencies = [ "num-integer", "num-traits", - "time 0.1.43", + "time 0.1.44", ] [[package]] @@ -287,7 +281,7 @@ checksum = "0dbbb57365263e881e805dc77d94697c9118fd94d8da011240555aa7b23445bd" name = "conduit" version = "0.1.0" dependencies = [ - "base64 0.12.3", + "base64", "directories", "http", "image", @@ -298,7 +292,7 @@ dependencies = [ "ring", "rocket", "ruma", - "rust-argon2 0.8.2", + "rust-argon2", "serde", "serde_json", "sled", @@ -320,7 +314,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1373a16a4937bc34efec7b391f9c1500c30b8478a701a4f44c9165cc0475a6e0" dependencies = [ "aes-gcm", - "base64 0.12.3", + "base64", "hkdf", "percent-encoding", "rand", @@ -486,9 +480,9 @@ checksum = "cd56b59865bce947ac5958779cfa508f6c3b9497cc762b7e24a12d11ccde2c4f" [[package]] name = "encoding_rs" -version = "0.8.23" +version = "0.8.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8ac63f94732332f44fe654443c46f6375d1939684c17b0afb6cb56b0456e171" +checksum = "a51b8cf747471cb9499b6d59e59b0444f4c90eba8968c4e44874e92b5b64ace2" dependencies = [ "cfg-if", ] @@ -672,7 +666,7 @@ checksum = "7abc8dd8451921606d809ba32e95b6111925cd2906060d2dcc29c070220503eb" dependencies = [ "cfg-if", "libc", - "wasi", + "wasi 0.9.0+wasi-snapshot-preview1", ] [[package]] @@ -816,7 +810,7 @@ dependencies = [ "itoa", "pin-project", "socket2", - "time 0.1.43", + "time 0.1.44", "tokio", "tower-service", "tracing", @@ -1057,9 +1051,9 @@ dependencies = [ [[package]] name = "miniz_oxide" -version = "0.4.0" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be0f75932c1f6cfae3c04000e40114adf955636e19040f9c0a2c380702aa1c7f" +checksum = "4d7559a8a40d0f97e1edea3220f698f78b1c5ab67532e49f68fde3910323b722" dependencies = [ "adler", ] @@ -1462,13 +1456,13 @@ checksum = "41cc0f7e4d5d4544e8861606a285bb08d3e70712ccc7d2b84d7c0ccfaf4b05ce" [[package]] name = "redox_users" -version = "0.3.4" +version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09b23093265f8d200fa7b4c2c76297f47e681c655f6f1285a8780d6a022f7431" +checksum = "de0737333e7a9502c789a36d7c7fa6092a49895d4faa31ca5df163857ded2e9d" dependencies = [ "getrandom", "redox_syscall", - "rust-argon2 0.7.0", + "rust-argon2", ] [[package]] @@ -1527,11 +1521,11 @@ dependencies = [ [[package]] name = "reqwest" -version = "0.10.7" +version = "0.10.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12427a5577082c24419c9c417db35cfeb65962efc7675bb6b0d5f1f9d315bfe6" +checksum = "e9eaa17ac5d7b838b7503d118fa16ad88f440498bf9ffe5424e621f93190d61e" dependencies = [ - "base64 0.12.3", + "base64", "bytes", "encoding_rs", "futures-core", @@ -1813,31 +1807,19 @@ name = "ruma-signatures" version = "0.6.0-dev.1" source = "git+https://github.com/ruma/ruma?rev=aff914050eb297bd82b8aafb12158c88a9e480e1#aff914050eb297bd82b8aafb12158c88a9e480e1" dependencies = [ - "base64 0.12.3", + "base64", "ring", "serde_json", "untrusted", ] -[[package]] -name = "rust-argon2" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2bc8af4bda8e1ff4932523b94d3dd20ee30a87232323eda55903ffd71d2fb017" -dependencies = [ - "base64 0.11.0", - "blake2b_simd", - "constant_time_eq", - "crossbeam-utils", -] - [[package]] name = "rust-argon2" version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9dab61250775933275e84053ac235621dfb739556d5c54a2f2e9313b7cf43a19" dependencies = [ - "base64 0.12.3", + "base64", "blake2b_simd", "constant_time_eq", "crossbeam-utils", @@ -1864,7 +1846,7 @@ version = "0.18.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5d1126dcf58e93cee7d098dbda643b5f92ed724f1f6a63007c1116eed6700c81" dependencies = [ - "base64 0.12.3", + "base64", "log", "ring", "sct", @@ -2072,9 +2054,9 @@ checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" [[package]] name = "standback" -version = "0.2.9" +version = "0.2.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0437cfb83762844799a60e1e3b489d5ceb6a650fbacb86437badc1b6d87b246" +checksum = "33a71ea1ea5f8747d1af1979bfb7e65c3a025a70609f04ceb78425bc5adad8e6" dependencies = [ "version_check", ] @@ -2088,7 +2070,7 @@ checksum = "7345c971d1ef21ffdbd103a75990a15eb03604fc8b8852ca8cb418ee1a099028" [[package]] name = "state-res" version = "0.1.0" -source = "git+https://github.com/ruma/state-res#d93a965ad17781fa9554bb3cea71673c054b9f3f" +source = "git+https://github.com/ruma/state-res?branch=spec-comp#17958665f6592af3ef478024fd1d75c384a30e7f" dependencies = [ "itertools", "js_int", @@ -2233,11 +2215,12 @@ dependencies = [ [[package]] name = "time" -version = "0.1.43" +version = "0.1.44" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca8a50ef2360fbd1eeb0ecd46795a87a19024eb4b53c5dc916ca1fd95fe62438" +checksum = "6db9e6914ab8b1ae1c260a4ae7a49b6c5611b40328a735b21862567685e73255" dependencies = [ "libc", + "wasi 0.10.0+wasi-snapshot-preview1", "winapi 0.3.9", ] @@ -2395,9 +2378,9 @@ dependencies = [ [[package]] name = "tracing-core" -version = "0.1.14" +version = "0.1.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db63662723c316b43ca36d833707cc93dff82a02ba3d7e354f342682cc8b3545" +checksum = "4f0e00789804e99b20f12bc7003ca416309d28a6f495d6af58d1e2c2842461b5" dependencies = [ "lazy_static", ] @@ -2550,6 +2533,12 @@ version = "0.9.0+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519" +[[package]] +name = "wasi" +version = "0.10.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a143597ca7c7793eff794def352d41792a93c481eb1042423ff7ff72ba2c31f" + [[package]] name = "wasm-bindgen" version = "0.2.67" diff --git a/Cargo.toml b/Cargo.toml index 78d8f7606..15cee7259 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -32,7 +32,7 @@ thiserror = "1.0.19" # Used for conduit::Error type image = { version = "0.23.4", default-features = false, features = ["jpeg", "png", "gif"] } # Used to generate thumbnails for images base64 = "0.12.3" # Used to encode server public key # state-res = { path = "../../state-res" } -state-res = { git = "https://github.com/ruma/state-res", version = "0.1.0" } +state-res = { git = "https://github.com/ruma/state-res", version = "0.1.0", branch = "spec-comp" } ring = "0.16.15" [features] diff --git a/src/client_server/directory.rs b/src/client_server/directory.rs index 5e03274b8..3b1068668 100644 --- a/src/client_server/directory.rs +++ b/src/client_server/directory.rs @@ -63,26 +63,11 @@ pub async fn get_public_rooms_route( db: State<'_, Database>, body: Ruma, ) -> ConduitResult { - let Ruma { - body: - get_public_rooms::IncomingRequest { - limit, - server, - since, - }, - .. - } = body; - - let get_public_rooms_filtered::Response { - chunk, - prev_batch, - next_batch, - total_room_count_estimate, - } = get_public_rooms_filtered_helper( + let response = get_public_rooms_filtered_helper( &db, - server.as_deref(), - limit, - since.as_deref(), + body.body.server.as_deref(), + body.body.limit, + body.body.since.as_deref(), None, // This is not used None, // This is not used ) @@ -90,10 +75,10 @@ pub async fn get_public_rooms_route( .0; Ok(get_public_rooms::Response { - chunk, - prev_batch, - next_batch, - total_room_count_estimate, + chunk: response.chunk, + prev_batch: response.prev_batch, + next_batch: response.next_batch, + total_room_count_estimate: response.total_room_count_estimate, } .into()) } diff --git a/src/client_server/membership.rs b/src/client_server/membership.rs index 3fa3b6ae7..90683e61f 100644 --- a/src/client_server/membership.rs +++ b/src/client_server/membership.rs @@ -483,12 +483,12 @@ async fn join_room_by_id_helper( .await?; dbg!(&send_join_response); - // todo!("Take send_join_response and 'create' the room using that data"); let mut event_map = send_join_response .room_state .state .iter() + .chain(send_join_response.room_state.auth_chain.iter()) .map(|pdu| { pdu.deserialize() .map(StateEvent::Full) @@ -497,14 +497,6 @@ async fn join_room_by_id_helper( .collect::, _>>() .map_err(|_| Error::bad_database("Invalid PDU found in db."))?; - let auth_chain = send_join_response - .room_state - .auth_chain - .iter() - .flat_map(|pdu| pdu.deserialize().ok()) - .map(StateEvent::Full) - .collect::>(); - let power_events = event_map .values() .filter(|pdu| pdu.is_power_event()) @@ -518,9 +510,11 @@ async fn join_room_by_id_helper( &power_events, &mut event_map, &db.rooms, - &auth_chain // if we only use it here just build this list in the first place + &send_join_response + .room_state + .auth_chain .iter() - .map(|pdu| pdu.event_id()) + .filter_map(|pdu| Some(StateEvent::Full(pdu.deserialize().ok()?).event_id())) .collect::>(), ); diff --git a/src/client_server/room.rs b/src/client_server/room.rs index 7f4a15fab..3ee21b6d8 100644 --- a/src/client_server/room.rs +++ b/src/client_server/room.rs @@ -195,7 +195,6 @@ pub fn create_room_route( content: match preset { create_room::RoomPreset::PublicChat => { serde_json::to_value(guest_access::GuestAccessEventContent::new( - // In a public room, joining is the only way to access guest_access::GuestAccess::Forbidden, )) .expect("event is valid, we just created it") diff --git a/src/client_server/state.rs b/src/client_server/state.rs index 12c5cacb4..e7d2bcf3e 100644 --- a/src/client_server/state.rs +++ b/src/client_server/state.rs @@ -1,5 +1,5 @@ use super::State; -use crate::{pdu::PduBuilder, ConduitResult, Database, Error, Ruma}; +use crate::{pdu::PduBuilder, ConduitResult, Database, Error, Result, Ruma}; use ruma::{ api::client::{ error::ErrorKind, @@ -9,7 +9,7 @@ use ruma::{ }, }, events::{AnyStateEventContent, EventContent}, - RoomId, UserId, + EventId, RoomId, UserId, }; #[cfg(feature = "conduit_bin")] @@ -33,13 +33,16 @@ pub fn send_state_event_for_key_route( ) .map_err(|_| Error::BadRequest(ErrorKind::BadJson, "Invalid JSON body."))?; - send_state_event_for_key_helper( - &db, - sender_id, - &body.content, - content, - &body.room_id, - Some(body.state_key.clone()), + Ok( + send_state_event_for_key::Response::new(send_state_event_for_key_helper( + &db, + sender_id, + &body.content, + content, + &body.room_id, + Some(body.state_key.clone()), + )?) + .into(), ) } @@ -67,8 +70,8 @@ pub fn send_state_event_for_empty_key_route( ) .map_err(|_| Error::BadRequest(ErrorKind::BadJson, "Invalid JSON body."))?; - Ok(send_state_event_for_empty_key::Response::new( - send_state_event_for_key_helper( + Ok( + send_state_event_for_empty_key::Response::new(send_state_event_for_key_helper( &db, sender_id .as_ref() @@ -77,11 +80,9 @@ pub fn send_state_event_for_empty_key_route( json, &body.room_id, Some("".into()), - )? - .0 - .event_id, + )?) + .into(), ) - .into()) } #[cfg_attr( @@ -183,7 +184,7 @@ pub fn send_state_event_for_key_helper( json: serde_json::Value, room_id: &RoomId, state_key: Option, -) -> ConduitResult { +) -> Result { let sender_id = sender; if let AnyStateEventContent::RoomCanonicalAlias(canonical_alias) = content { @@ -224,5 +225,5 @@ pub fn send_state_event_for_key_helper( &db.account_data, )?; - Ok(send_state_event_for_key::Response::new(event_id).into()) + Ok(event_id) } diff --git a/src/client_server/sync.rs b/src/client_server/sync.rs index ccb25d19b..74329605e 100644 --- a/src/client_server/sync.rs +++ b/src/client_server/sync.rs @@ -475,7 +475,7 @@ pub async fn sync_events_route( } for user_id in left_encrypted_users { - let user_target_encrypted = db + let still_share_encrypted_room = db .rooms .get_shared_rooms(vec![sender_id.clone(), user_id.clone()]) .filter_map(|r| r.ok()) @@ -490,7 +490,7 @@ pub async fn sync_events_route( .all(|encrypted| !encrypted); // If the user doesn't share an encrypted room with the target anymore, we need to tell // them - if user_target_encrypted { + if still_share_encrypted_room { device_list_left.insert(user_id); } } diff --git a/src/database/rooms.rs b/src/database/rooms.rs index 575a2bf09..66af73637 100644 --- a/src/database/rooms.rs +++ b/src/database/rooms.rs @@ -4,7 +4,6 @@ pub use edus::RoomEdus; use crate::{pdu::PduBuilder, utils, Error, PduEvent, Result}; use log::error; -// TODO if ruma-signatures re-exports `use ruma::signatures::digest;` use ring::digest; use ruma::{ api::client::error::ErrorKind, @@ -96,9 +95,9 @@ impl StateStore for Rooms { } impl Rooms { - /// Builds a `StateMap` by iterating over all keys that start - /// with `state_hash`, this gives the full state at event "x". - pub fn get_statemap_by_hash(&self, state_hash: StateHashId) -> Result> { + /// Builds a StateMap by iterating over all keys that start + /// with state_hash, this gives the full state for the given state_hash. + pub fn state_full(&self, state_hash: StateHashId) -> Result> { self.stateid_pduid .scan_prefix(&state_hash) .values() @@ -242,8 +241,6 @@ impl Rooms { /// Generate a new StateHash. /// /// A unique hash made from hashing the current states pduid's. - /// Because `append_state_pdu` handles the empty state db case it does not - /// have to be here. fn new_state_hash_id(&self, room_id: &RoomId) -> Result { // Use hashed roomId as the first StateHash key for first state event in room if self @@ -281,7 +278,7 @@ impl Rooms { /// Checks if a room exists. pub fn exists(&self, room_id: &RoomId) -> Result { - let mut prefix = room_id.to_string().as_bytes().to_vec(); + let mut prefix = room_id.as_bytes().to_vec(); prefix.push(0xff); // Look for PDUs in that room. @@ -300,7 +297,7 @@ impl Rooms { let mut hashmap = HashMap::new(); for pdu in self.roomstateid_pduid - .scan_prefix(&room_id.to_string().as_bytes()) + .scan_prefix(&room_id.as_bytes()) .values() .map(|value| { Ok::<_, Error>( @@ -322,13 +319,13 @@ impl Rooms { Ok(hashmap) } - /// Returns the all state entries for this type. + /// Returns all state entries for this type. pub fn room_state_type( &self, room_id: &RoomId, event_type: &EventType, ) -> Result> { - let mut prefix = room_id.to_string().as_bytes().to_vec(); + let mut prefix = room_id.as_bytes().to_vec(); prefix.push(0xff); prefix.extend_from_slice(&event_type.to_string().as_bytes()); @@ -357,7 +354,7 @@ impl Rooms { Ok(hashmap) } - /// Returns a single PDU in `room_id` with key (`event_type`, `state_key`). + /// Returns a single PDU from `room_id` with key (`event_type`, `state_key`). pub fn room_state_get( &self, room_id: &RoomId, @@ -459,7 +456,7 @@ impl Rooms { /// Returns the leaf pdus of a room. pub fn get_pdu_leaves(&self, room_id: &RoomId) -> Result> { - let mut prefix = room_id.to_string().as_bytes().to_vec(); + let mut prefix = room_id.as_bytes().to_vec(); prefix.push(0xff); let mut events = Vec::new(); @@ -582,7 +579,7 @@ impl Rooms { .split_terminator(|c: char| !c.is_alphanumeric()) .map(str::to_lowercase) { - let mut key = pdu.room_id.to_string().as_bytes().to_vec(); + let mut key = pdu.room_id.as_bytes().to_vec(); key.push(0xff); key.extend_from_slice(word.as_bytes()); key.push(0xff); @@ -752,7 +749,10 @@ impl Rooms { }) .collect::>>()?, ) - .ok_or(Error::Conflict("Found incoming PDU with invalid data."))?, + .map_err(|e| { + log::error!("{}", e); + Error::Conflict("Found incoming PDU with invalid data.") + })?, EventType::RoomCreate => prev_events.is_empty(), // Not allow any of the following events if the sender is not joined. _ if sender_membership != member::MembershipState::Join => false, @@ -982,13 +982,13 @@ impl Rooms { globals: &super::globals::Globals, ) -> Result<()> { let membership = member_content.membership; - let mut userroom_id = user_id.to_string().as_bytes().to_vec(); + let mut userroom_id = user_id.as_bytes().to_vec(); userroom_id.push(0xff); - userroom_id.extend_from_slice(room_id.to_string().as_bytes()); + userroom_id.extend_from_slice(room_id.as_bytes()); - let mut roomuser_id = room_id.to_string().as_bytes().to_vec(); + let mut roomuser_id = room_id.as_bytes().to_vec(); roomuser_id.push(0xff); - roomuser_id.extend_from_slice(user_id.to_string().as_bytes()); + roomuser_id.extend_from_slice(user_id.as_bytes()); match &membership { member::MembershipState::Join => { @@ -1051,9 +1051,9 @@ impl Rooms { /// Makes a user forget a room. pub fn forget(&self, room_id: &RoomId, user_id: &UserId) -> Result<()> { - let mut userroom_id = user_id.to_string().as_bytes().to_vec(); + let mut userroom_id = user_id.as_bytes().to_vec(); userroom_id.push(0xff); - userroom_id.extend_from_slice(room_id.to_string().as_bytes()); + userroom_id.extend_from_slice(room_id.as_bytes()); self.userroomid_left.remove(userroom_id)?; @@ -1069,8 +1069,8 @@ impl Rooms { if let Some(room_id) = room_id { // New alias self.alias_roomid - .insert(alias.alias(), &*room_id.to_string())?; - let mut aliasid = room_id.to_string().as_bytes().to_vec(); + .insert(alias.alias(), room_id.as_bytes())?; + let mut aliasid = room_id.as_bytes().to_vec(); aliasid.extend_from_slice(&globals.next_count()?.to_be_bytes()); self.aliasid_alias.insert(aliasid, &*alias.alias())?; } else { @@ -1105,7 +1105,7 @@ impl Rooms { } pub fn room_aliases(&self, room_id: &RoomId) -> impl Iterator> { - let mut prefix = room_id.to_string().as_bytes().to_vec(); + let mut prefix = room_id.as_bytes().to_vec(); prefix.push(0xff); self.aliasid_alias @@ -1119,16 +1119,16 @@ impl Rooms { pub fn set_public(&self, room_id: &RoomId, public: bool) -> Result<()> { if public { - self.publicroomids.insert(room_id.to_string(), &[])?; + self.publicroomids.insert(room_id.as_bytes(), &[])?; } else { - self.publicroomids.remove(room_id.to_string())?; + self.publicroomids.remove(room_id.as_bytes())?; } Ok(()) } pub fn is_public_room(&self, room_id: &RoomId) -> Result { - Ok(self.publicroomids.contains_key(room_id.to_string())?) + Ok(self.publicroomids.contains_key(room_id.as_bytes())?) } pub fn public_rooms(&self) -> impl Iterator> { @@ -1147,7 +1147,7 @@ impl Rooms { room_id: &RoomId, search_string: &str, ) -> Result<(impl Iterator + 'a, Vec)> { - let mut prefix = room_id.to_string().as_bytes().to_vec(); + let mut prefix = room_id.as_bytes().to_vec(); prefix.push(0xff); let words = search_string @@ -1233,7 +1233,7 @@ impl Rooms { /// Returns an iterator over all joined members of a room. pub fn room_members(&self, room_id: &RoomId) -> impl Iterator> { self.roomuserid_joined - .scan_prefix(room_id.to_string()) + .scan_prefix(room_id.as_bytes()) .keys() .map(|key| { Ok(UserId::try_from( @@ -1254,7 +1254,7 @@ impl Rooms { /// Returns an iterator over all invited members of a room. pub fn room_members_invited(&self, room_id: &RoomId) -> impl Iterator> { self.roomuserid_invited - .scan_prefix(room_id.to_string()) + .scan_prefix(room_id.as_bytes()) .keys() .map(|key| { Ok(UserId::try_from( @@ -1275,7 +1275,7 @@ impl Rooms { /// Returns an iterator over all rooms this user joined. pub fn rooms_joined(&self, user_id: &UserId) -> impl Iterator> { self.userroomid_joined - .scan_prefix(user_id.to_string()) + .scan_prefix(user_id.as_bytes()) .keys() .map(|key| { Ok(RoomId::try_from( @@ -1296,7 +1296,7 @@ impl Rooms { /// Returns an iterator over all rooms a user was invited to. pub fn rooms_invited(&self, user_id: &UserId) -> impl Iterator> { self.userroomid_invited - .scan_prefix(&user_id.to_string()) + .scan_prefix(&user_id.as_bytes()) .keys() .map(|key| { Ok(RoomId::try_from( @@ -1317,7 +1317,7 @@ impl Rooms { /// Returns an iterator over all rooms a user left. pub fn rooms_left(&self, user_id: &UserId) -> impl Iterator> { self.userroomid_left - .scan_prefix(&user_id.to_string()) + .scan_prefix(&user_id.as_bytes()) .keys() .map(|key| { Ok(RoomId::try_from( @@ -1336,25 +1336,25 @@ impl Rooms { } pub fn is_joined(&self, user_id: &UserId, room_id: &RoomId) -> Result { - let mut userroom_id = user_id.to_string().as_bytes().to_vec(); + let mut userroom_id = user_id.as_bytes().to_vec(); userroom_id.push(0xff); - userroom_id.extend_from_slice(room_id.to_string().as_bytes()); + userroom_id.extend_from_slice(room_id.as_bytes()); Ok(self.userroomid_joined.get(userroom_id)?.is_some()) } pub fn is_invited(&self, user_id: &UserId, room_id: &RoomId) -> Result { - let mut userroom_id = user_id.to_string().as_bytes().to_vec(); + let mut userroom_id = user_id.as_bytes().to_vec(); userroom_id.push(0xff); - userroom_id.extend_from_slice(room_id.to_string().as_bytes()); + userroom_id.extend_from_slice(room_id.as_bytes()); Ok(self.userroomid_invited.get(userroom_id)?.is_some()) } pub fn is_left(&self, user_id: &UserId, room_id: &RoomId) -> Result { - let mut userroom_id = user_id.to_string().as_bytes().to_vec(); + let mut userroom_id = user_id.as_bytes().to_vec(); userroom_id.push(0xff); - userroom_id.extend_from_slice(room_id.to_string().as_bytes()); + userroom_id.extend_from_slice(room_id.as_bytes()); Ok(self.userroomid_left.get(userroom_id)?.is_some()) } From 3b40f3d60ef9839c72763169f6ae1daed7b04895 Mon Sep 17 00:00:00 2001 From: Devin Ragotzy Date: Thu, 27 Aug 2020 16:10:20 -0400 Subject: [PATCH 16/51] Update state-res crate --- Cargo.lock | 2 +- src/database/rooms.rs | 73 +++++++++++++++++++++++++------------------ 2 files changed, 44 insertions(+), 31 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 62b13d00b..eebaddc0a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2070,7 +2070,7 @@ checksum = "7345c971d1ef21ffdbd103a75990a15eb03604fc8b8852ca8cb418ee1a099028" [[package]] name = "state-res" version = "0.1.0" -source = "git+https://github.com/ruma/state-res?branch=spec-comp#17958665f6592af3ef478024fd1d75c384a30e7f" +source = "git+https://github.com/ruma/state-res?branch=spec-comp#394d26744a6586ccdc01838964bb27dab289eee5" dependencies = [ "itertools", "js_int", diff --git a/src/database/rooms.rs b/src/database/rooms.rs index 66af73637..ee070b38b 100644 --- a/src/database/rooms.rs +++ b/src/database/rooms.rs @@ -19,13 +19,12 @@ use ruma::{ EventId, Raw, RoomAliasId, RoomId, UserId, }; use sled::IVec; -use state_res::{event_auth, Requester, StateEvent, StateMap, StateStore}; +use state_res::{event_auth, Error as StateError, Requester, StateEvent, StateMap, StateStore}; use std::{ collections::{BTreeMap, HashMap}, convert::{TryFrom, TryInto}, mem, - result::Result as StdResult, }; /// The unique identifier of each state group. @@ -67,28 +66,32 @@ pub struct Rooms { } impl StateStore for Rooms { - fn get_event(&self, room_id: &RoomId, event_id: &EventId) -> StdResult { + fn get_event(&self, room_id: &RoomId, event_id: &EventId) -> state_res::Result { let pid = self .eventid_pduid .get(event_id.as_bytes()) - .map_err(|e| e.to_string())? - .ok_or_else(|| "PDU via room_id and event_id not found in the db.".to_owned())?; + .map_err(StateError::custom)? + .ok_or_else(|| { + StateError::NotFound("PDU via room_id and event_id not found in the db.".into()) + })?; serde_json::from_slice( &self .pduid_pdu .get(pid) - .map_err(|e| e.to_string())? - .ok_or_else(|| "PDU via pduid not found in db.".to_owned())?, + .map_err(StateError::custom)? + .ok_or_else(|| StateError::NotFound("PDU via pduid not found in db.".into()))?, ) - .map_err(|e| e.to_string()) + .map_err(Into::into) .and_then(|pdu: StateEvent| { // conduit's PDU's always contain a room_id but some // of ruma's do not so this must be an Option if pdu.room_id() == Some(room_id) { Ok(pdu) } else { - Err("Found PDU for incorrect room in db.".into()) + Err(StateError::NotFound( + "Found PDU for incorrect room in db.".into(), + )) } }) } @@ -732,27 +735,37 @@ impl Rooms { // Don't allow encryption events when it's disabled !globals.encryption_disabled() } - EventType::RoomMember => event_auth::is_membership_change_allowed( - // TODO this is a bit of a hack but not sure how to have a type - // declared in `state_res` crate easily convert to/from conduit::PduEvent - Requester { - prev_event_ids: prev_events.to_owned(), - room_id: &room_id, - content: &content, - state_key: Some(state_key.to_owned()), - sender: &sender, - }, - &auth_events - .iter() - .map(|((ty, key), pdu)| { - Ok(((ty.clone(), key.clone()), pdu.convert_for_state_res()?)) - }) - .collect::>>()?, - ) - .map_err(|e| { - log::error!("{}", e); - Error::Conflict("Found incoming PDU with invalid data.") - })?, + EventType::RoomMember => { + let prev_event = self + .get_pdu(prev_events.iter().next().ok_or(Error::BadRequest( + ErrorKind::Unknown, + "Membership can't be the first event", + ))?)? + .map(|pdu| pdu.convert_for_state_res()) + .transpose()?; + event_auth::valid_membership_change( + // TODO this is a bit of a hack but not sure how to have a type + // declared in `state_res` crate easily convert to/from conduit::PduEvent + Requester { + prev_event_ids: prev_events.to_owned(), + room_id: &room_id, + content: &content, + state_key: Some(state_key.to_owned()), + sender: &sender, + }, + prev_event.as_ref(), + &auth_events + .iter() + .map(|((ty, key), pdu)| { + Ok(((ty.clone(), key.clone()), pdu.convert_for_state_res()?)) + }) + .collect::>>()?, + ) + .map_err(|e| { + log::error!("{}", e); + Error::Conflict("Found incoming PDU with invalid data.") + })? + } EventType::RoomCreate => prev_events.is_empty(), // Not allow any of the following events if the sender is not joined. _ if sender_membership != member::MembershipState::Join => false, From 2a63d0955ab06d1d1cbcae8fdf078d02a49fa65b Mon Sep 17 00:00:00 2001 From: Devin Ragotzy Date: Sun, 30 Aug 2020 16:08:47 -0400 Subject: [PATCH 17/51] Sort and authenticate the events from /send_join response --- src/client_server/membership.rs | 49 +++++++++++++++++++++++---------- 1 file changed, 35 insertions(+), 14 deletions(-) diff --git a/src/client_server/membership.rs b/src/client_server/membership.rs index 90683e61f..6d1931bf1 100644 --- a/src/client_server/membership.rs +++ b/src/client_server/membership.rs @@ -497,17 +497,18 @@ async fn join_room_by_id_helper( .collect::, _>>() .map_err(|_| Error::bad_database("Invalid PDU found in db."))?; - let power_events = event_map + let control_events = event_map .values() .filter(|pdu| pdu.is_power_event()) .map(|pdu| pdu.event_id()) .collect::>(); - // TODO these events are not guaranteed to be sorted but they are resolved, do - // we need the auth_chain - let sorted_power_events = state_res::StateResolution::reverse_topological_power_sort( + // These events are not guaranteed to be sorted but they are resolved according to spec + // we auth them anyways to weed out faulty/malicious server. The following is basically the + // full state resolution algorithm. + let sorted_control_events = state_res::StateResolution::reverse_topological_power_sort( &room_id, - &power_events, + &control_events, &mut event_map, &db.rooms, &send_join_response @@ -518,26 +519,31 @@ async fn join_room_by_id_helper( .collect::>(), ); - // TODO we may be able to skip this since they are resolved according to spec - let resolved_power = state_res::StateResolution::iterative_auth_check( + // Auth check each event against the "partial" state created by the preceding events + let resolved_control_events = state_res::StateResolution::iterative_auth_check( room_id, &RoomVersionId::Version6, - &sorted_power_events, - &BTreeMap::new(), // unconflicted events + &sorted_control_events, + &BTreeMap::new(), // We have no "clean/resolved" events to add (these extend the `resolved_control_events`) &mut event_map, &db.rooms, ) .expect("iterative auth check failed on resolved events"); - // TODO do we need to dedup them + // This removes the control events that failed auth, leaving the resolved + // to be mainline sorted let events_to_sort = event_map .keys() - .filter(|id| !sorted_power_events.contains(id)) + .filter(|id| { + !sorted_control_events.contains(id) + || resolved_control_events.values().any(|rid| *id == rid) + }) .cloned() .collect::>(); - let power_level = resolved_power.get(&(EventType::RoomPowerLevels, Some("".into()))); - + let power_level = + resolved_control_events.get(&(EventType::RoomPowerLevels, Some("".into()))); + // Sort the remaining non control events let sorted_event_ids = state_res::StateResolution::mainline_sort( room_id, &events_to_sort, @@ -546,7 +552,22 @@ async fn join_room_by_id_helper( &db.rooms, ); - for ev_id in &sorted_event_ids { + let resolved_events = state_res::StateResolution::iterative_auth_check( + room_id, + &RoomVersionId::Version6, + &sorted_event_ids, + &resolved_control_events, + &mut event_map, + &db.rooms, + ) + .expect("iterative auth check failed on resolved events"); + + // filter the events that failed the auth check keeping the remaining events + // sorted correctly + for ev_id in sorted_event_ids + .iter() + .filter(|id| resolved_events.values().any(|rid| rid == *id)) + { // this is a `state_res::StateEvent` that holds a `ruma::Pdu` let pdu = event_map .get(ev_id) From 1e8fbd8d50c1bf0fe80cb77b8df8b5a45c3698f5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20K=C3=B6sters?= Date: Tue, 8 Sep 2020 17:32:03 +0200 Subject: [PATCH 18/51] Update ruma version --- Cargo.lock | 18 +--------- Cargo.toml | 9 ++--- src/client_server/account.rs | 10 +++--- src/client_server/alias.rs | 8 ++--- src/client_server/backup.rs | 10 +++--- src/client_server/config.rs | 4 +-- src/client_server/context.rs | 2 +- src/client_server/device.rs | 8 ++--- src/client_server/directory.rs | 14 ++++---- src/client_server/filter.rs | 2 +- src/client_server/keys.rs | 18 +++++----- src/client_server/media.rs | 8 ++--- src/client_server/membership.rs | 32 +++++++++--------- src/client_server/message.rs | 8 ++--- src/client_server/presence.rs | 2 +- src/client_server/profile.rs | 10 +++--- src/client_server/read_marker.rs | 4 +-- src/client_server/redact.rs | 2 +- src/client_server/room.rs | 51 ++++++++++------------------- src/client_server/search.rs | 22 ++++++++----- src/client_server/session.rs | 2 +- src/client_server/state.rs | 6 ++-- src/client_server/sync.rs | 2 +- src/client_server/tag.rs | 6 ++-- src/client_server/to_device.rs | 2 +- src/client_server/typing.rs | 8 ++--- src/client_server/user_directory.rs | 2 +- src/database/media.rs | 4 +-- src/database/users.rs | 6 ++-- src/error.rs | 6 ++-- src/pdu.rs | 2 +- src/ruma_wrapper.rs | 22 ++++++++----- src/server_server.rs | 6 ++-- 33 files changed, 147 insertions(+), 169 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index eebaddc0a..c88c5786d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1631,7 +1631,6 @@ dependencies = [ [[package]] name = "ruma" version = "0.0.1" -source = "git+https://github.com/ruma/ruma?rev=aff914050eb297bd82b8aafb12158c88a9e480e1#aff914050eb297bd82b8aafb12158c88a9e480e1" dependencies = [ "ruma-api", "ruma-appservice-api", @@ -1647,7 +1646,6 @@ dependencies = [ [[package]] name = "ruma-api" version = "0.17.0-alpha.1" -source = "git+https://github.com/ruma/ruma?rev=aff914050eb297bd82b8aafb12158c88a9e480e1#aff914050eb297bd82b8aafb12158c88a9e480e1" dependencies = [ "http", "percent-encoding", @@ -1662,7 +1660,6 @@ dependencies = [ [[package]] name = "ruma-api-macros" version = "0.17.0-alpha.1" -source = "git+https://github.com/ruma/ruma?rev=aff914050eb297bd82b8aafb12158c88a9e480e1#aff914050eb297bd82b8aafb12158c88a9e480e1" dependencies = [ "proc-macro-crate", "proc-macro2", @@ -1673,7 +1670,6 @@ dependencies = [ [[package]] name = "ruma-appservice-api" version = "0.2.0-alpha.1" -source = "git+https://github.com/ruma/ruma?rev=aff914050eb297bd82b8aafb12158c88a9e480e1#aff914050eb297bd82b8aafb12158c88a9e480e1" dependencies = [ "ruma-api", "ruma-common", @@ -1686,7 +1682,6 @@ dependencies = [ [[package]] name = "ruma-client-api" version = "0.10.0-alpha.1" -source = "git+https://github.com/ruma/ruma?rev=aff914050eb297bd82b8aafb12158c88a9e480e1#aff914050eb297bd82b8aafb12158c88a9e480e1" dependencies = [ "assign", "http", @@ -1705,9 +1700,9 @@ dependencies = [ [[package]] name = "ruma-common" version = "0.2.0" -source = "git+https://github.com/ruma/ruma?rev=aff914050eb297bd82b8aafb12158c88a9e480e1#aff914050eb297bd82b8aafb12158c88a9e480e1" dependencies = [ "js_int", + "ruma-api", "ruma-identifiers", "ruma-serde", "serde", @@ -1718,7 +1713,6 @@ dependencies = [ [[package]] name = "ruma-events" version = "0.22.0-alpha.1" -source = "git+https://github.com/ruma/ruma?rev=aff914050eb297bd82b8aafb12158c88a9e480e1#aff914050eb297bd82b8aafb12158c88a9e480e1" dependencies = [ "js_int", "ruma-common", @@ -1733,7 +1727,6 @@ dependencies = [ [[package]] name = "ruma-events-macros" version = "0.22.0-alpha.1" -source = "git+https://github.com/ruma/ruma?rev=aff914050eb297bd82b8aafb12158c88a9e480e1#aff914050eb297bd82b8aafb12158c88a9e480e1" dependencies = [ "proc-macro-crate", "proc-macro2", @@ -1744,7 +1737,6 @@ dependencies = [ [[package]] name = "ruma-federation-api" version = "0.0.3" -source = "git+https://github.com/ruma/ruma?rev=aff914050eb297bd82b8aafb12158c88a9e480e1#aff914050eb297bd82b8aafb12158c88a9e480e1" dependencies = [ "js_int", "ruma-api", @@ -1759,7 +1751,6 @@ dependencies = [ [[package]] name = "ruma-identifiers" version = "0.17.4" -source = "git+https://github.com/ruma/ruma?rev=aff914050eb297bd82b8aafb12158c88a9e480e1#aff914050eb297bd82b8aafb12158c88a9e480e1" dependencies = [ "rand", "ruma-identifiers-macros", @@ -1771,7 +1762,6 @@ dependencies = [ [[package]] name = "ruma-identifiers-macros" version = "0.17.4" -source = "git+https://github.com/ruma/ruma?rev=aff914050eb297bd82b8aafb12158c88a9e480e1#aff914050eb297bd82b8aafb12158c88a9e480e1" dependencies = [ "proc-macro2", "quote", @@ -1782,18 +1772,14 @@ dependencies = [ [[package]] name = "ruma-identifiers-validation" version = "0.1.1" -source = "git+https://github.com/ruma/ruma?rev=aff914050eb297bd82b8aafb12158c88a9e480e1#aff914050eb297bd82b8aafb12158c88a9e480e1" dependencies = [ - "ruma-serde", "serde", - "serde_json", "strum", ] [[package]] name = "ruma-serde" version = "0.2.3" -source = "git+https://github.com/ruma/ruma?rev=aff914050eb297bd82b8aafb12158c88a9e480e1#aff914050eb297bd82b8aafb12158c88a9e480e1" dependencies = [ "form_urlencoded", "itoa", @@ -1805,7 +1791,6 @@ dependencies = [ [[package]] name = "ruma-signatures" version = "0.6.0-dev.1" -source = "git+https://github.com/ruma/ruma?rev=aff914050eb297bd82b8aafb12158c88a9e480e1#aff914050eb297bd82b8aafb12158c88a9e480e1" dependencies = [ "base64", "ring", @@ -2070,7 +2055,6 @@ checksum = "7345c971d1ef21ffdbd103a75990a15eb03604fc8b8852ca8cb418ee1a099028" [[package]] name = "state-res" version = "0.1.0" -source = "git+https://github.com/ruma/state-res?branch=spec-comp#394d26744a6586ccdc01838964bb27dab289eee5" dependencies = [ "itertools", "js_int", diff --git a/Cargo.toml b/Cargo.toml index 15cee7259..5d3543338 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -15,8 +15,9 @@ edition = "2018" # TODO: This can become optional as soon as proper configs are supported #rocket = { git = "https://github.com/SergioBenitez/Rocket.git", rev = "8d779caa22c63b15a6c3ceb75d8f6d4971b2eb67", features = ["tls"] } # Used to handle requests rocket = { git = "https://github.com/timokoesters/Rocket.git", branch = "empty_parameters", features = ["tls"] } - -ruma = { git = "https://github.com/ruma/ruma", features = ["rand", "client-api", "federation-api", "unstable-pre-spec", "unstable-synapse-quirks"], rev = "aff914050eb297bd82b8aafb12158c88a9e480e1" } # Used for matrix spec type definitions and helpers +#ruma = { git = "https://github.com/ruma/ruma", features = ["rand", "client-api", "federation-api", "unstable-pre-spec", "unstable-synapse-quirks"], rev = "aff914050eb297bd82b8aafb12158c88a9e480e1" } # Used for matrix spec type definitions and helpers +#ruma = { git = "https://github.com/timokoesters/ruma", features = ["rand", "client-api", "federation-api", "unstable-pre-spec", "unstable-synapse-quirks"], branch = "timo-fixes" } # Used for matrix spec type definitions and helpers +ruma = { path = "../ruma/ruma", features = ["unstable-exhaustive-types", "rand", "client-api", "federation-api", "unstable-pre-spec", "unstable-synapse-quirks"] } tokio = "0.2.22" # Used for long polling sled = "0.32.0" # Used for storing data permanently log = "0.4.8" # Used for emitting log entries @@ -31,8 +32,8 @@ reqwest = "0.10.6" # Used to send requests thiserror = "1.0.19" # Used for conduit::Error type image = { version = "0.23.4", default-features = false, features = ["jpeg", "png", "gif"] } # Used to generate thumbnails for images base64 = "0.12.3" # Used to encode server public key -# state-res = { path = "../../state-res" } -state-res = { git = "https://github.com/ruma/state-res", version = "0.1.0", branch = "spec-comp" } +#state-res = { git = "https://github.com/ruma/state-res", version = "0.1.0", branch = "spec-comp" } +state-res = { path = "../state-res" } ring = "0.16.15" [features] diff --git a/src/client_server/account.rs b/src/client_server/account.rs index 9e52f6d23..cb77a15d4 100644 --- a/src/client_server/account.rs +++ b/src/client_server/account.rs @@ -33,7 +33,7 @@ const GUEST_NAME_LENGTH: usize = 10; )] pub fn get_register_available_route( db: State<'_, Database>, - body: Ruma, + body: Ruma>, ) -> ConduitResult { // Validate user id let user_id = UserId::parse_with_server_name(body.username.clone(), db.globals.server_name()) @@ -75,7 +75,7 @@ pub fn get_register_available_route( )] pub fn register_route( db: State<'_, Database>, - body: Ruma, + body: Ruma>, ) -> ConduitResult { if db.globals.registration_disabled() { return Err(Error::BadRequest( @@ -84,7 +84,7 @@ pub fn register_route( )); } - let is_guest = matches!(body.kind, Some(RegistrationKind::Guest)); + let is_guest = body.kind == RegistrationKind::Guest; let mut missing_username = false; @@ -223,7 +223,7 @@ pub fn register_route( )] pub fn change_password_route( db: State<'_, Database>, - body: Ruma, + body: Ruma>, ) -> ConduitResult { let sender_id = body.sender_id.as_ref().expect("user is authenticated"); let device_id = body.device_id.as_ref().expect("user is authenticated"); @@ -305,7 +305,7 @@ pub fn whoami_route(body: Ruma) -> ConduitResult, - body: Ruma, + body: Ruma>, ) -> ConduitResult { let sender_id = body.sender_id.as_ref().expect("user is authenticated"); let device_id = body.device_id.as_ref().expect("user is authenticated"); diff --git a/src/client_server/alias.rs b/src/client_server/alias.rs index 669f558d6..1d30261b6 100644 --- a/src/client_server/alias.rs +++ b/src/client_server/alias.rs @@ -20,7 +20,7 @@ use rocket::{delete, get, put}; )] pub fn create_alias_route( db: State<'_, Database>, - body: Ruma, + body: Ruma>, ) -> ConduitResult { if db.rooms.id_from_alias(&body.room_alias)?.is_some() { return Err(Error::Conflict("Alias already exists.")); @@ -38,7 +38,7 @@ pub fn create_alias_route( )] pub fn delete_alias_route( db: State<'_, Database>, - body: Ruma, + body: Ruma>, ) -> ConduitResult { db.rooms.set_alias(&body.room_alias, None, &db.globals)?; @@ -51,7 +51,7 @@ pub fn delete_alias_route( )] pub async fn get_alias_route( db: State<'_, Database>, - body: Ruma, + body: Ruma>, ) -> ConduitResult { get_alias_helper(&db, &body.room_alias).await } @@ -65,7 +65,7 @@ pub async fn get_alias_helper( &db, room_alias.server_name().to_string(), federation::query::get_room_information::v1::Request { - room_alias: room_alias.to_string(), + room_alias, }, ) .await?; diff --git a/src/client_server/backup.rs b/src/client_server/backup.rs index a10496494..8966c017c 100644 --- a/src/client_server/backup.rs +++ b/src/client_server/backup.rs @@ -33,7 +33,7 @@ pub fn create_backup_route( )] pub fn update_backup_route( db: State<'_, Database>, - body: Ruma, + body: Ruma>, ) -> ConduitResult { let sender_id = body.sender_id.as_ref().expect("user is authenticated"); db.key_backups @@ -75,7 +75,7 @@ pub fn get_latest_backup_route( )] pub fn get_backup_route( db: State<'_, Database>, - body: Ruma, + body: Ruma>, ) -> ConduitResult { let sender_id = body.sender_id.as_ref().expect("user is authenticated"); let algorithm = db @@ -90,7 +90,7 @@ pub fn get_backup_route( algorithm, count: (db.key_backups.count_keys(sender_id, &body.version)? as u32).into(), etag: db.key_backups.get_etag(sender_id, &body.version)?, - version: body.version.clone(), + version: body.version.to_owned(), } .into()) } @@ -102,7 +102,7 @@ pub fn get_backup_route( )] pub fn add_backup_keys_route( db: State<'_, Database>, - body: Ruma, + body: Ruma>, ) -> ConduitResult { let sender_id = body.sender_id.as_ref().expect("user is authenticated"); @@ -132,7 +132,7 @@ pub fn add_backup_keys_route( )] pub fn get_backup_keys_route( db: State<'_, Database>, - body: Ruma, + body: Ruma>, ) -> ConduitResult { let sender_id = body.sender_id.as_ref().expect("user is authenticated"); diff --git a/src/client_server/config.rs b/src/client_server/config.rs index baa9381ba..515ad1607 100644 --- a/src/client_server/config.rs +++ b/src/client_server/config.rs @@ -18,7 +18,7 @@ use rocket::{get, put}; )] pub fn set_global_account_data_route( db: State<'_, Database>, - body: Ruma, + body: Ruma>, ) -> ConduitResult { let sender_id = body.sender_id.as_ref().expect("user is authenticated"); @@ -49,7 +49,7 @@ pub fn set_global_account_data_route( )] pub fn get_global_account_data_route( db: State<'_, Database>, - body: Ruma, + body: Ruma>, ) -> ConduitResult { let sender_id = body.sender_id.as_ref().expect("user is authenticated"); diff --git a/src/client_server/context.rs b/src/client_server/context.rs index 7b1fad917..959372646 100644 --- a/src/client_server/context.rs +++ b/src/client_server/context.rs @@ -12,7 +12,7 @@ use rocket::get; )] pub fn get_context_route( db: State<'_, Database>, - body: Ruma, + body: Ruma>, ) -> ConduitResult { let sender_id = body.sender_id.as_ref().expect("user is authenticated"); diff --git a/src/client_server/device.rs b/src/client_server/device.rs index 89033f06f..6352d0d1d 100644 --- a/src/client_server/device.rs +++ b/src/client_server/device.rs @@ -37,7 +37,7 @@ pub fn get_devices_route( )] pub fn get_device_route( db: State<'_, Database>, - body: Ruma, + body: Ruma>, _device_id: String, ) -> ConduitResult { let sender_id = body.sender_id.as_ref().expect("user is authenticated"); @@ -56,7 +56,7 @@ pub fn get_device_route( )] pub fn update_device_route( db: State<'_, Database>, - body: Ruma, + body: Ruma>, _device_id: String, ) -> ConduitResult { let sender_id = body.sender_id.as_ref().expect("user is authenticated"); @@ -80,7 +80,7 @@ pub fn update_device_route( )] pub fn delete_device_route( db: State<'_, Database>, - body: Ruma, + body: Ruma>, _device_id: String, ) -> ConduitResult { let sender_id = body.sender_id.as_ref().expect("user is authenticated"); @@ -127,7 +127,7 @@ pub fn delete_device_route( )] pub fn delete_devices_route( db: State<'_, Database>, - body: Ruma, + body: Ruma>, ) -> ConduitResult { let sender_id = body.sender_id.as_ref().expect("user is authenticated"); let device_id = body.device_id.as_ref().expect("user is authenticated"); diff --git a/src/client_server/directory.rs b/src/client_server/directory.rs index 3b1068668..34feb7180 100644 --- a/src/client_server/directory.rs +++ b/src/client_server/directory.rs @@ -14,7 +14,7 @@ use ruma::{ }, federation, }, - directory::{Filter, PublicRoomsChunk, RoomNetwork}, + directory::{IncomingFilter, PublicRoomsChunk, IncomingRoomNetwork}, events::{ room::{avatar, canonical_alias, guest_access, history_visibility, name, topic}, EventType, @@ -31,7 +31,7 @@ use rocket::{get, post, put}; )] pub async fn get_public_rooms_filtered_route( db: State<'_, Database>, - body: Ruma, + body: Ruma>, ) -> ConduitResult { let Ruma { body: @@ -61,7 +61,7 @@ pub async fn get_public_rooms_filtered_route( )] pub async fn get_public_rooms_route( db: State<'_, Database>, - body: Ruma, + body: Ruma>, ) -> ConduitResult { let response = get_public_rooms_filtered_helper( &db, @@ -89,7 +89,7 @@ pub async fn get_public_rooms_route( )] pub async fn set_room_visibility_route( db: State<'_, Database>, - body: Ruma, + body: Ruma>, ) -> ConduitResult { match body.visibility { room::Visibility::Public => db.rooms.set_public(&body.room_id, true)?, @@ -105,7 +105,7 @@ pub async fn set_room_visibility_route( )] pub async fn get_room_visibility_route( db: State<'_, Database>, - body: Ruma, + body: Ruma>, ) -> ConduitResult { Ok(get_room_visibility::Response { visibility: if db.rooms.is_public_room(&body.room_id)? { @@ -122,8 +122,8 @@ pub async fn get_public_rooms_filtered_helper( server: Option<&str>, limit: Option, since: Option<&str>, - _filter: Option, - _network: Option, + _filter: Option, + _network: Option, ) -> ConduitResult { if let Some(other_server) = server .clone() diff --git a/src/client_server/filter.rs b/src/client_server/filter.rs index 4322de3e5..4b1c3a000 100644 --- a/src/client_server/filter.rs +++ b/src/client_server/filter.rs @@ -7,7 +7,7 @@ use rocket::{get, post}; #[cfg_attr(feature = "conduit_bin", get("/_matrix/client/r0/user/<_>/filter/<_>"))] pub fn get_filter_route() -> ConduitResult { // TODO - Ok(get_filter::Response::new(filter::FilterDefinition { + Ok(get_filter::Response::new(filter::IncomingFilterDefinition { event_fields: None, event_format: None, account_data: None, diff --git a/src/client_server/keys.rs b/src/client_server/keys.rs index 331152964..0e7b1eff1 100644 --- a/src/client_server/keys.rs +++ b/src/client_server/keys.rs @@ -11,7 +11,7 @@ use ruma::{ uiaa::{AuthFlow, UiaaInfo}, }, }, - encryption::UnsignedDeviceInfo, + encryption::IncomingUnsignedDeviceInfo, }; use std::collections::{BTreeMap, HashSet}; @@ -24,7 +24,7 @@ use rocket::{get, post}; )] pub fn upload_keys_route( db: State<'_, Database>, - body: Ruma, + body: Ruma>, ) -> ConduitResult { let sender_id = body.sender_id.as_ref().expect("user is authenticated"); let device_id = body.device_id.as_ref().expect("user is authenticated"); @@ -56,7 +56,7 @@ pub fn upload_keys_route( )] pub fn get_keys_route( db: State<'_, Database>, - body: Ruma, + body: Ruma>, ) -> ConduitResult { let sender_id = body.sender_id.as_ref().expect("user is authenticated"); @@ -78,9 +78,9 @@ pub fn get_keys_route( Error::bad_database("all_device_keys contained nonexistent device.") })?; - keys.unsigned = Some(UnsignedDeviceInfo { + keys.unsigned = IncomingUnsignedDeviceInfo { device_display_name: metadata.display_name, - }); + }; container.insert(device_id, keys); } @@ -97,9 +97,9 @@ pub fn get_keys_route( ), )?; - keys.unsigned = Some(UnsignedDeviceInfo { + keys.unsigned = IncomingUnsignedDeviceInfo { device_display_name: metadata.display_name, - }); + }; container.insert(device_id.clone(), keys); } @@ -167,7 +167,7 @@ pub fn claim_keys_route( )] pub fn upload_signing_keys_route( db: State<'_, Database>, - body: Ruma, + body: Ruma>, ) -> ConduitResult { let sender_id = body.sender_id.as_ref().expect("user is authenticated"); let device_id = body.device_id.as_ref().expect("user is authenticated"); @@ -280,7 +280,7 @@ pub fn upload_signatures_route( )] pub fn get_key_changes_route( db: State<'_, Database>, - body: Ruma, + body: Ruma>, ) -> ConduitResult { let sender_id = body.sender_id.as_ref().expect("user is authenticated"); diff --git a/src/client_server/media.rs b/src/client_server/media.rs index 79c1f080a..038012e23 100644 --- a/src/client_server/media.rs +++ b/src/client_server/media.rs @@ -27,7 +27,7 @@ pub fn get_media_config_route( )] pub fn create_content_route( db: State<'_, Database>, - body: Ruma, + body: Ruma>, ) -> ConduitResult { let mxc = format!( "mxc://{}/{}", @@ -36,7 +36,7 @@ pub fn create_content_route( ); db.media.create( mxc.clone(), - body.filename.as_ref(), + &body.filename, &body.content_type, &body.file, )?; @@ -53,7 +53,7 @@ pub fn create_content_route( )] pub fn get_content_route( db: State<'_, Database>, - body: Ruma, + body: Ruma>, _server_name: String, _media_id: String, ) -> ConduitResult { @@ -85,7 +85,7 @@ pub fn get_content_route( )] pub fn get_content_thumbnail_route( db: State<'_, Database>, - body: Ruma, + body: Ruma>, _server_name: String, _media_id: String, ) -> ConduitResult { diff --git a/src/client_server/membership.rs b/src/client_server/membership.rs index 6d1931bf1..606e4700a 100644 --- a/src/client_server/membership.rs +++ b/src/client_server/membership.rs @@ -32,7 +32,7 @@ use rocket::{get, post}; )] pub async fn join_room_by_id_route( db: State<'_, Database>, - body: Ruma, + body: Ruma>, ) -> ConduitResult { join_room_by_id_helper( &db, @@ -49,7 +49,7 @@ pub async fn join_room_by_id_route( )] pub async fn join_room_by_id_or_alias_route( db: State<'_, Database>, - body: Ruma, + body: Ruma>, ) -> ConduitResult { let room_id = match RoomId::try_from(body.room_id_or_alias.clone()) { Ok(room_id) => room_id, @@ -81,7 +81,7 @@ pub async fn join_room_by_id_or_alias_route( )] pub fn leave_room_route( db: State<'_, Database>, - body: Ruma, + body: Ruma>, ) -> ConduitResult { let sender_id = body.sender_id.as_ref().expect("user is authenticated"); @@ -127,11 +127,11 @@ pub fn leave_room_route( )] pub fn invite_user_route( db: State<'_, Database>, - body: Ruma, + body: Ruma>, ) -> ConduitResult { let sender_id = body.sender_id.as_ref().expect("user is authenticated"); - if let invite_user::InvitationRecipient::UserId { user_id } = &body.recipient { + if let invite_user::IncomingInvitationRecipient::UserId { user_id } = &body.recipient { db.rooms.build_and_append_pdu( PduBuilder { room_id: body.room_id.clone(), @@ -165,7 +165,7 @@ pub fn invite_user_route( )] pub fn kick_user_route( db: State<'_, Database>, - body: Ruma, + body: Ruma>, ) -> ConduitResult { let sender_id = body.sender_id.as_ref().expect("user is authenticated"); @@ -212,7 +212,7 @@ pub fn kick_user_route( )] pub fn ban_user_route( db: State<'_, Database>, - body: Ruma, + body: Ruma>, ) -> ConduitResult { let sender_id = body.sender_id.as_ref().expect("user is authenticated"); @@ -267,7 +267,7 @@ pub fn ban_user_route( )] pub fn unban_user_route( db: State<'_, Database>, - body: Ruma, + body: Ruma>, ) -> ConduitResult { let sender_id = body.sender_id.as_ref().expect("user is authenticated"); @@ -313,7 +313,7 @@ pub fn unban_user_route( )] pub fn forget_room_route( db: State<'_, Database>, - body: Ruma, + body: Ruma>, ) -> ConduitResult { let sender_id = body.sender_id.as_ref().expect("user is authenticated"); @@ -348,7 +348,7 @@ pub fn joined_rooms_route( )] pub fn get_member_events_route( db: State<'_, Database>, - body: Ruma, + body: Ruma>, ) -> ConduitResult { let sender_id = body.sender_id.as_ref().expect("user is authenticated"); @@ -376,7 +376,7 @@ pub fn get_member_events_route( )] pub fn joined_members_route( db: State<'_, Database>, - body: Ruma, + body: Ruma>, ) -> ConduitResult { let sender_id = body.sender_id.as_ref().expect("user is authenticated"); @@ -422,9 +422,9 @@ async fn join_room_by_id_helper( &db, room_id.server_name().to_string(), federation::membership::create_join_event_template::v1::Request { - room_id: room_id.clone(), - user_id: sender_id.clone(), - ver: vec![RoomVersionId::Version5, RoomVersionId::Version6], + room_id, + user_id: sender_id, + ver: &[RoomVersionId::Version5, RoomVersionId::Version6], }, ) .await?; @@ -474,8 +474,8 @@ async fn join_room_by_id_helper( &db, room_id.server_name().to_string(), federation::membership::create_join_event::v1::Request { - room_id: room_id.clone(), - event_id, + room_id, + event_id: &event_id, pdu_stub: serde_json::from_value(join_event_stub_value) .expect("we just created this event"), }, diff --git a/src/client_server/message.rs b/src/client_server/message.rs index 03832d86f..09c3517fa 100644 --- a/src/client_server/message.rs +++ b/src/client_server/message.rs @@ -18,7 +18,7 @@ use rocket::{get, put}; )] pub fn send_message_event_route( db: State<'_, Database>, - body: Ruma, + body: Ruma>, ) -> ConduitResult { let sender_id = body.sender_id.as_ref().expect("user is authenticated"); @@ -53,7 +53,7 @@ pub fn send_message_event_route( )] pub fn get_message_events_route( db: State<'_, Database>, - body: Ruma, + body: Ruma>, ) -> ConduitResult { let sender_id = body.sender_id.as_ref().expect("user is authenticated"); @@ -96,7 +96,7 @@ pub fn get_message_events_route( .collect::>(); let mut resp = get_message_events::Response::new(); - resp.start = Some(body.from.clone()); + resp.start = Some(body.from.to_owned()); resp.end = end_token; resp.chunk = events_after; resp.state = Vec::new(); @@ -120,7 +120,7 @@ pub fn get_message_events_route( .collect::>(); let mut resp = get_message_events::Response::new(); - resp.start = Some(body.from.clone()); + resp.start = Some(body.from.to_owned()); resp.end = start_token; resp.chunk = events_before; resp.state = Vec::new(); diff --git a/src/client_server/presence.rs b/src/client_server/presence.rs index 0b6a51f4e..d105eb6ab 100644 --- a/src/client_server/presence.rs +++ b/src/client_server/presence.rs @@ -12,7 +12,7 @@ use rocket::put; )] pub fn set_presence_route( db: State<'_, Database>, - body: Ruma, + body: Ruma>, ) -> ConduitResult { let sender_id = body.sender_id.as_ref().expect("user is authenticated"); diff --git a/src/client_server/profile.rs b/src/client_server/profile.rs index 0707b342d..386d89885 100644 --- a/src/client_server/profile.rs +++ b/src/client_server/profile.rs @@ -21,7 +21,7 @@ use std::convert::TryInto; )] pub fn set_displayname_route( db: State<'_, Database>, - body: Ruma, + body: Ruma>, ) -> ConduitResult { let sender_id = body.sender_id.as_ref().expect("user is authenticated"); @@ -98,7 +98,7 @@ pub fn set_displayname_route( )] pub fn get_displayname_route( db: State<'_, Database>, - body: Ruma, + body: Ruma>, ) -> ConduitResult { Ok(get_display_name::Response { displayname: db.users.displayname(&body.user_id)?, @@ -112,7 +112,7 @@ pub fn get_displayname_route( )] pub fn set_avatar_url_route( db: State<'_, Database>, - body: Ruma, + body: Ruma>, ) -> ConduitResult { let sender_id = body.sender_id.as_ref().expect("user is authenticated"); @@ -201,7 +201,7 @@ pub fn set_avatar_url_route( )] pub fn get_avatar_url_route( db: State<'_, Database>, - body: Ruma, + body: Ruma>, ) -> ConduitResult { Ok(get_avatar_url::Response { avatar_url: db.users.avatar_url(&body.user_id)?, @@ -215,7 +215,7 @@ pub fn get_avatar_url_route( )] pub fn get_profile_route( db: State<'_, Database>, - body: Ruma, + body: Ruma>, ) -> ConduitResult { let avatar_url = db.users.avatar_url(&body.user_id)?; let displayname = db.users.displayname(&body.user_id)?; diff --git a/src/client_server/read_marker.rs b/src/client_server/read_marker.rs index ff72765f3..023eece2b 100644 --- a/src/client_server/read_marker.rs +++ b/src/client_server/read_marker.rs @@ -15,7 +15,7 @@ use std::{collections::BTreeMap, time::SystemTime}; )] pub fn set_read_marker_route( db: State<'_, Database>, - body: Ruma, + body: Ruma>, ) -> ConduitResult { let sender_id = body.sender_id.as_ref().expect("user is authenticated"); @@ -52,7 +52,7 @@ pub fn set_read_marker_route( ); let mut receipt_content = BTreeMap::new(); receipt_content.insert( - event.clone(), + event.to_owned(), ruma::events::receipt::Receipts { read: Some(user_receipts), }, diff --git a/src/client_server/redact.rs b/src/client_server/redact.rs index 8708692f3..cd1b44384 100644 --- a/src/client_server/redact.rs +++ b/src/client_server/redact.rs @@ -14,7 +14,7 @@ use rocket::put; )] pub fn redact_event_route( db: State<'_, Database>, - body: Ruma, + body: Ruma>, ) -> ConduitResult { let sender_id = body.sender_id.as_ref().expect("user is authenticated"); diff --git a/src/client_server/room.rs b/src/client_server/room.rs index 3ee21b6d8..9918123f8 100644 --- a/src/client_server/room.rs +++ b/src/client_server/room.rs @@ -22,7 +22,7 @@ use rocket::{get, post}; )] pub fn create_room_route( db: State<'_, Database>, - body: Ruma, + body: Ruma>, ) -> ConduitResult { let sender_id = body.sender_id.as_ref().expect("user is authenticated"); @@ -48,11 +48,8 @@ pub fn create_room_route( })?; let mut content = ruma::events::room::create::CreateEventContent::new(sender_id.clone()); - content.federate = body.creation_content.as_ref().map_or(true, |c| c.federate); - content.predecessor = body - .creation_content - .as_ref() - .and_then(|c| c.predecessor.clone()); + content.federate = body.creation_content.federate; + content.predecessor = body.creation_content.predecessor.clone(); content.room_version = RoomVersionId::Version6; // 1. The room create event @@ -80,7 +77,7 @@ pub fn create_room_route( membership: member::MembershipState::Join, displayname: db.users.displayname(&sender_id)?, avatar_url: db.users.avatar_url(&sender_id)?, - is_direct: body.is_direct, + is_direct: Some(body.is_direct), third_party_invite: None, }) .expect("event is valid, we just created it"), @@ -137,8 +134,7 @@ pub fn create_room_route( // 4. Events set by preset // Figure out preset. We need it for preset specific events - let visibility = body.visibility.unwrap_or(room::Visibility::Private); - let preset = body.preset.unwrap_or_else(|| match visibility { + let preset = body.preset.unwrap_or_else(|| match body.visibility { room::Visibility::Private => create_room::RoomPreset::PrivateChat, room::Visibility::Public => create_room::RoomPreset::PublicChat, }); @@ -213,32 +209,19 @@ pub fn create_room_route( )?; // 5. Events listed in initial_state - for create_room::InitialStateEvent { - event_type, - state_key, - content, - } in &body.initial_state - { + for event in &body.initial_state { + let pdu_builder = serde_json::from_str::( + &serde_json::to_string(&event).expect("AnyInitialStateEvent::to_string always works"), + ).map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "Invalid initial state event."))?; + // Silently skip encryption events if they are not allowed - if event_type == &EventType::RoomEncryption && db.globals.encryption_disabled() { + if pdu_builder.event_type == EventType::RoomEncryption && db.globals.encryption_disabled() + { continue; } - db.rooms.build_and_append_pdu( - PduBuilder { - room_id: room_id.clone(), - sender: sender_id.clone(), - event_type: event_type.clone(), - content: serde_json::from_str(content.get()).map_err(|_| { - Error::BadRequest(ErrorKind::BadJson, "Invalid initial_state content.") - })?, - unsigned: None, - state_key: state_key.clone(), - redacts: None, - }, - &db.globals, - &db.account_data, - )?; + db.rooms + .build_and_append_pdu(pdu_builder, &db.globals, &db.account_data)?; } // 6. Events implied by name and topic @@ -293,7 +276,7 @@ pub fn create_room_route( membership: member::MembershipState::Invite, displayname: db.users.displayname(&user)?, avatar_url: db.users.avatar_url(&user)?, - is_direct: body.is_direct, + is_direct: Some(body.is_direct), third_party_invite: None, }) .expect("event is valid, we just created it"), @@ -311,7 +294,7 @@ pub fn create_room_route( db.rooms.set_alias(&alias, Some(&room_id), &db.globals)?; } - if let Some(room::Visibility::Public) = body.visibility { + if body.visibility == room::Visibility::Public { db.rooms.set_public(&room_id, true)?; } @@ -324,7 +307,7 @@ pub fn create_room_route( )] pub fn get_room_event_route( db: State<'_, Database>, - body: Ruma, + body: Ruma>, ) -> ConduitResult { let sender_id = body.sender_id.as_ref().expect("user is authenticated"); diff --git a/src/client_server/search.rs b/src/client_server/search.rs index 082711d13..2967e001e 100644 --- a/src/client_server/search.rs +++ b/src/client_server/search.rs @@ -1,11 +1,10 @@ use super::State; use crate::{ConduitResult, Database, Error, Ruma}; -use js_int::uint; use ruma::api::client::{error::ErrorKind, r0::search::search_events}; #[cfg(feature = "conduit_bin")] use rocket::post; -use search_events::{ResultCategories, ResultRoomEvents, SearchResult}; +use search_events::{EventContextResult, ResultCategories, ResultRoomEvents, SearchResult}; use std::collections::BTreeMap; #[cfg_attr( @@ -14,7 +13,7 @@ use std::collections::BTreeMap; )] pub fn search_events_route( db: State<'_, Database>, - body: Ruma, + body: Ruma>, ) -> ConduitResult { let sender_id = body.sender_id.as_ref().expect("user is authenticated"); @@ -51,13 +50,18 @@ pub fn search_events_route( .0 .map(|result| { Ok::<_, Error>(SearchResult { - context: None, + context: EventContextResult { + end: None, + events_after: Vec::new(), + events_before: Vec::new(), + profile_info: BTreeMap::new(), + start: None, + }, rank: None, result: db .rooms .get_pdu_from_id(&result)? - // TODO this is an awkward type conversion see method - .map(|pdu| pdu.to_any_event()), + .map(|pdu| pdu.to_room_event()), }) }) .filter_map(|r| r.ok()) @@ -72,14 +76,14 @@ pub fn search_events_route( }; Ok(search_events::Response::new(ResultCategories { - room_events: Some(ResultRoomEvents { - count: uint!(0), // TODO + room_events: ResultRoomEvents { + count: None, // TODO? maybe not groups: BTreeMap::new(), // TODO next_batch, results, state: BTreeMap::new(), // TODO highlights: search.1, - }), + }, }) .into()) } diff --git a/src/client_server/session.rs b/src/client_server/session.rs index 948b455b7..9cd051ce1 100644 --- a/src/client_server/session.rs +++ b/src/client_server/session.rs @@ -36,7 +36,7 @@ pub fn get_login_types_route() -> ConduitResult { )] pub fn login_route( db: State<'_, Database>, - body: Ruma, + body: Ruma>, ) -> ConduitResult { // Validate login method let user_id = diff --git a/src/client_server/state.rs b/src/client_server/state.rs index e7d2bcf3e..75463cbac 100644 --- a/src/client_server/state.rs +++ b/src/client_server/state.rs @@ -21,7 +21,7 @@ use rocket::{get, put}; )] pub fn send_state_event_for_key_route( db: State<'_, Database>, - body: Ruma, + body: Ruma>, ) -> ConduitResult { let sender_id = body.sender_id.as_ref().expect("user is authenticated"); @@ -40,7 +40,7 @@ pub fn send_state_event_for_key_route( &body.content, content, &body.room_id, - Some(body.state_key.clone()), + Some(body.state_key.to_owned()), )?) .into(), ) @@ -52,7 +52,7 @@ pub fn send_state_event_for_key_route( )] pub fn send_state_event_for_empty_key_route( db: State<'_, Database>, - body: Ruma, + body: Ruma>, ) -> ConduitResult { // This just calls send_state_event_for_key_route let Ruma { diff --git a/src/client_server/sync.rs b/src/client_server/sync.rs index 74329605e..167ee75ef 100644 --- a/src/client_server/sync.rs +++ b/src/client_server/sync.rs @@ -31,7 +31,7 @@ use std::{ )] pub async fn sync_events_route( db: State<'_, Database>, - body: Ruma, + body: Ruma>, ) -> ConduitResult { let sender_id = body.sender_id.as_ref().expect("user is authenticated"); let device_id = body.device_id.as_ref().expect("user is authenticated"); diff --git a/src/client_server/tag.rs b/src/client_server/tag.rs index 99ee6e349..d04dd3aeb 100644 --- a/src/client_server/tag.rs +++ b/src/client_server/tag.rs @@ -15,7 +15,7 @@ use rocket::{delete, get, put}; )] pub fn update_tag_route( db: State<'_, Database>, - body: Ruma, + body: Ruma>, ) -> ConduitResult { let sender_id = body.sender_id.as_ref().expect("user is authenticated"); @@ -49,7 +49,7 @@ pub fn update_tag_route( )] pub fn delete_tag_route( db: State<'_, Database>, - body: Ruma, + body: Ruma>, ) -> ConduitResult { let sender_id = body.sender_id.as_ref().expect("user is authenticated"); @@ -80,7 +80,7 @@ pub fn delete_tag_route( )] pub fn get_tags_route( db: State<'_, Database>, - body: Ruma, + body: Ruma>, ) -> ConduitResult { let sender_id = body.sender_id.as_ref().expect("user is authenticated"); diff --git a/src/client_server/to_device.rs b/src/client_server/to_device.rs index ca423fefd..fe7410111 100644 --- a/src/client_server/to_device.rs +++ b/src/client_server/to_device.rs @@ -14,7 +14,7 @@ use rocket::put; )] pub fn send_event_to_device_route( db: State<'_, Database>, - body: Ruma, + body: Ruma>, ) -> ConduitResult { let sender_id = body.sender_id.as_ref().expect("user is authenticated"); diff --git a/src/client_server/typing.rs b/src/client_server/typing.rs index 7eba13e17..b15121c02 100644 --- a/src/client_server/typing.rs +++ b/src/client_server/typing.rs @@ -1,5 +1,6 @@ use super::State; use crate::{utils, ConduitResult, Database, Ruma}; +use create_typing_event::Typing; use ruma::api::client::r0::typing::create_typing_event; #[cfg(feature = "conduit_bin")] @@ -11,16 +12,15 @@ use rocket::put; )] pub fn create_typing_event_route( db: State<'_, Database>, - body: Ruma, + body: Ruma>, ) -> ConduitResult { let sender_id = body.sender_id.as_ref().expect("user is authenticated"); - if body.typing { + if let Typing::Yes(duration) = body.state { db.rooms.edus.roomactive_add( &sender_id, &body.room_id, - body.timeout.map(|d| d.as_millis() as u64).unwrap_or(30000) - + utils::millis_since_unix_epoch(), + duration.as_millis() as u64 + utils::millis_since_unix_epoch(), &db.globals, )?; } else { diff --git a/src/client_server/user_directory.rs b/src/client_server/user_directory.rs index f47643c1a..dcf48fe3c 100644 --- a/src/client_server/user_directory.rs +++ b/src/client_server/user_directory.rs @@ -11,7 +11,7 @@ use rocket::post; )] pub fn search_users_route( db: State<'_, Database>, - body: Ruma, + body: Ruma>, ) -> ConduitResult { let limit = u64::from(body.limit) as usize; diff --git a/src/database/media.rs b/src/database/media.rs index e9dcb4a00..63fa11c64 100644 --- a/src/database/media.rs +++ b/src/database/media.rs @@ -16,7 +16,7 @@ impl Media { pub fn create( &self, mxc: String, - filename: Option<&String>, + filename: &Option, content_type: &str, file: &[u8], ) -> Result<()> { @@ -25,7 +25,7 @@ impl Media { key.extend_from_slice(&0_u32.to_be_bytes()); // Width = 0 if it's not a thumbnail key.extend_from_slice(&0_u32.to_be_bytes()); // Height = 0 if it's not a thumbnail key.push(0xff); - key.extend_from_slice(filename.map(|f| f.as_bytes()).unwrap_or_default()); + key.extend_from_slice(filename.as_ref().map(|f| f.as_bytes()).unwrap_or_default()); key.push(0xff); key.extend_from_slice(content_type.as_bytes()); diff --git a/src/database/users.rs b/src/database/users.rs index 1b6a6812e..10e1ef301 100644 --- a/src/database/users.rs +++ b/src/database/users.rs @@ -8,7 +8,7 @@ use ruma::{ keys::{CrossSigningKey, OneTimeKey}, }, }, - encryption::DeviceKeys, + encryption::IncomingDeviceKeys, events::{AnyToDeviceEvent, EventType}, DeviceId, DeviceKeyAlgorithm, DeviceKeyId, Raw, UserId, }; @@ -395,7 +395,7 @@ impl Users { &self, user_id: &UserId, device_id: &DeviceId, - device_keys: &DeviceKeys, + device_keys: &IncomingDeviceKeys, rooms: &super::rooms::Rooms, globals: &super::globals::Globals, ) -> Result<()> { @@ -625,7 +625,7 @@ impl Users { &self, user_id: &UserId, device_id: &DeviceId, - ) -> Result> { + ) -> Result> { let mut key = user_id.to_string().as_bytes().to_vec(); key.push(0xff); key.extend_from_slice(device_id.as_bytes()); diff --git a/src/error.rs b/src/error.rs index 623aa0ef9..f521da43d 100644 --- a/src/error.rs +++ b/src/error.rs @@ -70,14 +70,14 @@ where use ErrorKind::*; let (kind, status_code) = match self { Self::BadRequest(kind, _) => ( - kind, + kind.clone(), match kind { Forbidden | GuestAccessForbidden | ThreepidAuthFailed | ThreepidDenied => { StatusCode::FORBIDDEN } - Unauthorized | UnknownToken | MissingToken => StatusCode::UNAUTHORIZED, + Unauthorized | UnknownToken { .. } | MissingToken => StatusCode::UNAUTHORIZED, NotFound => StatusCode::NOT_FOUND, - LimitExceeded => StatusCode::TOO_MANY_REQUESTS, + LimitExceeded { .. } => StatusCode::TOO_MANY_REQUESTS, UserDeactivated => StatusCode::FORBIDDEN, TooLarge => StatusCode::PAYLOAD_TOO_LARGE, _ => StatusCode::BAD_REQUEST, diff --git a/src/pdu.rs b/src/pdu.rs index b565a24c6..152648423 100644 --- a/src/pdu.rs +++ b/src/pdu.rs @@ -255,7 +255,7 @@ impl PduEvent { } /// Build the start of a PDU in order to add it to the `Database`. -#[derive(Debug)] +#[derive(Debug, Deserialize)] pub struct PduBuilder { pub room_id: RoomId, pub sender: UserId, diff --git a/src/ruma_wrapper.rs b/src/ruma_wrapper.rs index 80e6e5829..734d2144c 100644 --- a/src/ruma_wrapper.rs +++ b/src/ruma_wrapper.rs @@ -1,9 +1,9 @@ use crate::Error; use ruma::{ - api::IncomingRequest, + api::{Outgoing, OutgoingRequest}, identifiers::{DeviceId, UserId}, }; -use std::{convert::TryInto, ops::Deref}; +use std::{convert::TryFrom, convert::TryInto, ops::Deref}; #[cfg(feature = "conduit_bin")] use { @@ -24,15 +24,21 @@ use { /// This struct converts rocket requests into ruma structs by converting them into http requests /// first. -pub struct Ruma { - pub body: T, +pub struct Ruma { + pub body: T::Incoming, pub sender_id: Option, pub device_id: Option>, pub json_body: Option>, // This is None when body is not a valid string } #[cfg(feature = "conduit_bin")] -impl<'a, T: IncomingRequest> FromTransformedData<'a> for Ruma { +impl<'a, T: Outgoing + OutgoingRequest> FromTransformedData<'a> for Ruma +where + ::Incoming: TryFrom>> + std::fmt::Debug, + <::Incoming as std::convert::TryFrom< + http::request::Request>, + >>::Error: std::fmt::Debug, +{ type Error = (); // TODO: Better error handling type Owned = Data; type Borrowed = Self::Owned; @@ -93,7 +99,7 @@ impl<'a, T: IncomingRequest> FromTransformedData<'a> for Ruma { let http_request = http_request.body(body.clone()).unwrap(); log::info!("{:?}", http_request); - match T::try_from(http_request) { + match ::Incoming::try_from(http_request) { Ok(t) => Success(Ruma { body: t, sender_id: user_id, @@ -112,8 +118,8 @@ impl<'a, T: IncomingRequest> FromTransformedData<'a> for Ruma { } } -impl Deref for Ruma { - type Target = T; +impl Deref for Ruma { + type Target = T::Incoming; fn deref(&self) -> &Self::Target { &self.body diff --git a/src/server_server.rs b/src/server_server.rs index ac4407baa..d39abe645 100644 --- a/src/server_server.rs +++ b/src/server_server.rs @@ -207,7 +207,7 @@ pub fn get_server_keys_deprecated(db: State<'_, Database>) -> Json { )] pub async fn get_public_rooms_route( db: State<'_, Database>, - body: Ruma, + body: Ruma>, ) -> ConduitResult { let Ruma { body: @@ -230,7 +230,7 @@ pub async fn get_public_rooms_route( limit, since.as_deref(), None, - Some(ruma::directory::RoomNetwork::Matrix), + Some(ruma::directory::IncomingRoomNetwork::Matrix), ) .await? .0; @@ -264,7 +264,7 @@ pub async fn get_public_rooms_route( )] pub fn send_transaction_message_route<'a>( _db: State<'a, Database>, - body: Ruma, + body: Ruma>, ) -> ConduitResult { dbg!(&*body); Ok(send_transaction_message::v1::Response { From 12a8c9badd202f3019cab71a1f3a3134c8fb75ec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20K=C3=B6sters?= Date: Sat, 12 Sep 2020 21:30:07 +0200 Subject: [PATCH 19/51] fix: join rooms over federation --- Cargo.lock | 140 +++++++------ Cargo.toml | 8 +- src/client_server/account.rs | 4 +- src/client_server/alias.rs | 4 +- src/client_server/directory.rs | 2 +- src/client_server/media.rs | 8 +- src/client_server/membership.rs | 106 +++++----- src/client_server/message.rs | 5 +- src/client_server/profile.rs | 8 +- src/client_server/redact.rs | 4 +- src/client_server/room.rs | 51 ++--- src/client_server/search.rs | 2 +- src/client_server/state.rs | 4 +- src/database.rs | 1 - src/database/rooms.rs | 340 ++++++++++++++------------------ src/pdu.rs | 112 +++++------ src/server_server.rs | 1 + 17 files changed, 395 insertions(+), 405 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c88c5786d..dc215c396 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -116,9 +116,9 @@ checksum = "4af5687fe33aec5e70ef14caac5e0d363e335e5e5d6385fb75978d0c241b1d67" [[package]] name = "async-trait" -version = "0.1.38" +version = "0.1.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e1a4a2f97ce50c9d0282c1468816208588441492b40d813b2e0419c22c05e7f" +checksum = "687c230d85c0a52504709705fc8a53e4a692b83a2184f03dae73e38e1e93a783" dependencies = [ "proc-macro2", "quote", @@ -223,9 +223,9 @@ checksum = "2e8c087f005730276d1096a652e92a8bacee2e2472bcc9715a74d2bec38b5820" [[package]] name = "bytemuck" -version = "1.3.1" +version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db7a1029718df60331e557c9e83a55523c955e5dd2a7bfeffad6bbd50b538ae9" +checksum = "41aa2ec95ca3b5c54cf73c91acf06d24f4495d5f1b1c12506ae3483d646177ac" [[package]] name = "byteorder" @@ -301,6 +301,12 @@ dependencies = [ "tokio", ] +[[package]] +name = "const_fn" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce90df4c658c62f12d78f7508cf92f9173e5184a539c10bfe54a3107b3ffd0f2" + [[package]] name = "constant_time_eq" version = "0.1.5" @@ -319,7 +325,7 @@ dependencies = [ "percent-encoding", "rand", "sha2", - "time 0.2.16", + "time 0.2.19", "version_check", ] @@ -660,9 +666,9 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.1.14" +version = "0.1.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7abc8dd8451921606d809ba32e95b6111925cd2906060d2dcc29c070220503eb" +checksum = "fc587bc0ec293155d5bfa6b9891ec18a1e330c234f896ea47fbada4cadbe47e6" dependencies = [ "cfg-if", "libc", @@ -721,12 +727,9 @@ dependencies = [ [[package]] name = "hashbrown" -version = "0.8.2" +version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e91b62f79061a0bc2e046024cb7ba44b08419ed238ecbd9adbd787434b9e8c25" -dependencies = [ - "autocfg", -] +checksum = "00d63df3d41950fb462ed38308eea019113ad1508da725bbedcd0fa5a85ef5f7" [[package]] name = "heck" @@ -843,9 +846,9 @@ dependencies = [ [[package]] name = "image" -version = "0.23.8" +version = "0.23.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "543904170510c1b5fb65140485d84de4a57fddb2ed685481e9020ce3d2c9f64c" +checksum = "974e194911d1f7efe3cd8a8f9db3b767e43536327e899e8bc9a12ef5711b74d2" dependencies = [ "bytemuck", "byteorder", @@ -859,9 +862,9 @@ dependencies = [ [[package]] name = "indexmap" -version = "1.5.1" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86b45e59b16c76b11bf9738fd5d38879d3bd28ad292d7b313608becb17ae2df9" +checksum = "55e2e4c765aa53a0424761bf9f41aa7a6ac1efa87238f59560640e27fca028f2" dependencies = [ "autocfg", "hashbrown", @@ -920,9 +923,9 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.44" +version = "0.3.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85a7e2c92a4804dd459b86c339278d0fe87cf93757fae222c3fa3ae75458bc73" +checksum = "ca059e81d9486668f12d455a4ea6daa600bd408134cd17e3d3fb5a32d1f016f8" dependencies = [ "wasm-bindgen", ] @@ -954,9 +957,9 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] name = "libc" -version = "0.2.76" +version = "0.2.77" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "755456fae044e6fa1ebbbd1b3e902ae19e73097ed4ed87bb79934a867c007bc3" +checksum = "f2f96b10ec2560088a8e76961b00d47107b3a625fecb76dedb29ee7ccbf98235" [[package]] name = "lock_api" @@ -1120,9 +1123,9 @@ dependencies = [ [[package]] name = "net2" -version = "0.2.34" +version = "0.2.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ba7c918ac76704fb42afcbbb43891e72731f3dcca3bef2a19786297baf14af7" +checksum = "3ebc3ec692ed7c9a255596c67808dee269f64655d8baf7b4f0638e51ba1d6853" dependencies = [ "cfg-if", "libc", @@ -1379,9 +1382,9 @@ checksum = "eba180dafb9038b050a4c280019bbedf9f2467b61e5d892dcad585bb57aadc5a" [[package]] name = "proc-macro2" -version = "1.0.19" +version = "1.0.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04f5f085b5d71e2188cb8271e5da0161ad52c3f227a661a3c135fdf28e258b12" +checksum = "36e28516df94f3dd551a587da5357459d9b36d945a7c37c3557928c1c2ff2a2c" dependencies = [ "unicode-xid", ] @@ -1587,7 +1590,7 @@ dependencies = [ "rocket_codegen", "rocket_http", "state", - "time 0.2.16", + "time 0.2.19", "tokio", "toml", "version_check", @@ -1622,7 +1625,7 @@ dependencies = [ "ref-cast", "smallvec", "state", - "time 0.2.16", + "time 0.2.19", "tokio", "tokio-rustls", "unicode-xid", @@ -1631,6 +1634,7 @@ dependencies = [ [[package]] name = "ruma" version = "0.0.1" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#ca07bb61d88fd665464dab9707de6d47048fc225" dependencies = [ "ruma-api", "ruma-appservice-api", @@ -1646,6 +1650,7 @@ dependencies = [ [[package]] name = "ruma-api" version = "0.17.0-alpha.1" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#ca07bb61d88fd665464dab9707de6d47048fc225" dependencies = [ "http", "percent-encoding", @@ -1660,6 +1665,7 @@ dependencies = [ [[package]] name = "ruma-api-macros" version = "0.17.0-alpha.1" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#ca07bb61d88fd665464dab9707de6d47048fc225" dependencies = [ "proc-macro-crate", "proc-macro2", @@ -1670,6 +1676,7 @@ dependencies = [ [[package]] name = "ruma-appservice-api" version = "0.2.0-alpha.1" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#ca07bb61d88fd665464dab9707de6d47048fc225" dependencies = [ "ruma-api", "ruma-common", @@ -1682,6 +1689,7 @@ dependencies = [ [[package]] name = "ruma-client-api" version = "0.10.0-alpha.1" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#ca07bb61d88fd665464dab9707de6d47048fc225" dependencies = [ "assign", "http", @@ -1700,6 +1708,7 @@ dependencies = [ [[package]] name = "ruma-common" version = "0.2.0" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#ca07bb61d88fd665464dab9707de6d47048fc225" dependencies = [ "js_int", "ruma-api", @@ -1713,6 +1722,7 @@ dependencies = [ [[package]] name = "ruma-events" version = "0.22.0-alpha.1" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#ca07bb61d88fd665464dab9707de6d47048fc225" dependencies = [ "js_int", "ruma-common", @@ -1727,6 +1737,7 @@ dependencies = [ [[package]] name = "ruma-events-macros" version = "0.22.0-alpha.1" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#ca07bb61d88fd665464dab9707de6d47048fc225" dependencies = [ "proc-macro-crate", "proc-macro2", @@ -1737,6 +1748,7 @@ dependencies = [ [[package]] name = "ruma-federation-api" version = "0.0.3" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#ca07bb61d88fd665464dab9707de6d47048fc225" dependencies = [ "js_int", "ruma-api", @@ -1751,6 +1763,7 @@ dependencies = [ [[package]] name = "ruma-identifiers" version = "0.17.4" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#ca07bb61d88fd665464dab9707de6d47048fc225" dependencies = [ "rand", "ruma-identifiers-macros", @@ -1762,6 +1775,7 @@ dependencies = [ [[package]] name = "ruma-identifiers-macros" version = "0.17.4" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#ca07bb61d88fd665464dab9707de6d47048fc225" dependencies = [ "proc-macro2", "quote", @@ -1772,6 +1786,7 @@ dependencies = [ [[package]] name = "ruma-identifiers-validation" version = "0.1.1" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#ca07bb61d88fd665464dab9707de6d47048fc225" dependencies = [ "serde", "strum", @@ -1780,6 +1795,7 @@ dependencies = [ [[package]] name = "ruma-serde" version = "0.2.3" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#ca07bb61d88fd665464dab9707de6d47048fc225" dependencies = [ "form_urlencoded", "itoa", @@ -1791,6 +1807,7 @@ dependencies = [ [[package]] name = "ruma-signatures" version = "0.6.0-dev.1" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#ca07bb61d88fd665464dab9707de6d47048fc225" dependencies = [ "base64", "ring", @@ -1910,18 +1927,18 @@ checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" [[package]] name = "serde" -version = "1.0.115" +version = "1.0.116" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e54c9a88f2da7238af84b5101443f0c0d0a3bbdc455e34a5c9497b1903ed55d5" +checksum = "96fe57af81d28386a513cbc6858332abc6117cfdb5999647c6444b8f43a370a5" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.115" +version = "1.0.116" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "609feed1d0a73cc36a0182a840a9b37b4a82f0b1150369f0536a9e3f2a31dc48" +checksum = "f630a6370fd8e457873b4bd2ffdae75408bc291ba72be773772a4c2a065d9ae8" dependencies = [ "proc-macro2", "quote", @@ -2021,9 +2038,9 @@ checksum = "fbee7696b84bbf3d89a1c2eccff0850e3047ed46bfcd2e92c29a2d074d57e252" [[package]] name = "socket2" -version = "0.3.12" +version = "0.3.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03088793f677dce356f3ccc2edb1b314ad191ab702a5de3faf49304f7e104918" +checksum = "b1fa70dc5c8104ec096f4fe7ede7a221d35ae13dcd19ba1ad9a81d2cab9a1c44" dependencies = [ "cfg-if", "libc", @@ -2055,6 +2072,7 @@ checksum = "7345c971d1ef21ffdbd103a75990a15eb03604fc8b8852ca8cb418ee1a099028" [[package]] name = "state-res" version = "0.1.0" +source = "git+https://github.com/timokoesters/state-res?branch=spec-comp#0081081604b051d412a2365b68357e064c33320c" dependencies = [ "itertools", "js_int", @@ -2139,15 +2157,15 @@ dependencies = [ [[package]] name = "subtle" -version = "2.2.3" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "502d53007c02d7605a05df1c1a73ee436952781653da5d0bf57ad608f66932c1" +checksum = "343f3f510c2915908f155e94f17220b19ccfacf2a64a2a5d8004f2c3e311e7fd" [[package]] name = "syn" -version = "1.0.39" +version = "1.0.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "891d8d6567fe7c7f8835a3a98af4208f3846fba258c1bc3c31d6e506239f11f9" +checksum = "963f7d3cc59b59b9325165add223142bbf1df27655d07789f109896d353d8350" dependencies = [ "proc-macro2", "quote", @@ -2210,11 +2228,11 @@ dependencies = [ [[package]] name = "time" -version = "0.2.16" +version = "0.2.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a51cadc5b1eec673a685ff7c33192ff7b7603d0b75446fb354939ee615acb15" +checksum = "80c1a1fd93112fc50b11c43a1def21f926be3c18884fad676ea879572da070a1" dependencies = [ - "cfg-if", + "const_fn", "libc", "standback", "stdweb", @@ -2288,9 +2306,9 @@ dependencies = [ [[package]] name = "tokio-rustls" -version = "0.14.0" +version = "0.14.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "228139ddd4fea3fa345a29233009635235833e52807af7ea6448ead03890d6a9" +checksum = "e12831b255bcfa39dc0436b01e19fea231a37db570686c06ee72c423479f889a" dependencies = [ "futures-core", "rustls", @@ -2362,9 +2380,9 @@ dependencies = [ [[package]] name = "tracing-core" -version = "0.1.15" +version = "0.1.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f0e00789804e99b20f12bc7003ca416309d28a6f495d6af58d1e2c2842461b5" +checksum = "5bcf46c1f1f06aeea2d6b81f3c863d0930a596c86ad1920d4e5bad6dd1d7119a" dependencies = [ "lazy_static", ] @@ -2382,9 +2400,9 @@ dependencies = [ [[package]] name = "tracing-serde" -version = "0.1.1" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6ccba2f8f16e0ed268fc765d9b7ff22e965e7185d32f8f1ec8294fe17d86e79" +checksum = "fb65ea441fbb84f9f6748fd496cf7f63ec9af5bca94dd86456978d055e8eb28b" dependencies = [ "serde", "tracing-core", @@ -2392,9 +2410,9 @@ dependencies = [ [[package]] name = "tracing-subscriber" -version = "0.2.11" +version = "0.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "abd165311cc4d7a555ad11cc77a37756df836182db0d81aac908c8184c584f40" +checksum = "82bb5079aa76438620837198db8a5c529fb9878c730bc2b28179b0241cf04c10" dependencies = [ "ansi_term", "chrono", @@ -2525,9 +2543,9 @@ checksum = "1a143597ca7c7793eff794def352d41792a93c481eb1042423ff7ff72ba2c31f" [[package]] name = "wasm-bindgen" -version = "0.2.67" +version = "0.2.68" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0563a9a4b071746dd5aedbc3a28c6fe9be4586fb3fbadb67c400d4f53c6b16c" +checksum = "1ac64ead5ea5f05873d7c12b545865ca2b8d28adfc50a49b84770a3a97265d42" dependencies = [ "cfg-if", "serde", @@ -2537,9 +2555,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.67" +version = "0.2.68" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc71e4c5efa60fb9e74160e89b93353bc24059999c0ae0fb03affc39770310b0" +checksum = "f22b422e2a757c35a73774860af8e112bff612ce6cb604224e8e47641a9e4f68" dependencies = [ "bumpalo", "lazy_static", @@ -2552,9 +2570,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.17" +version = "0.4.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95f8d235a77f880bcef268d379810ea6c0af2eacfa90b1ad5af731776e0c4699" +checksum = "b7866cab0aa01de1edf8b5d7936938a7e397ee50ce24119aef3e1eaa3b6171da" dependencies = [ "cfg-if", "js-sys", @@ -2564,9 +2582,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.67" +version = "0.2.68" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97c57cefa5fa80e2ba15641578b44d36e7a64279bc5ed43c6dbaf329457a2ed2" +checksum = "6b13312a745c08c469f0b292dd2fcd6411dba5f7160f593da6ef69b64e407038" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -2574,9 +2592,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.67" +version = "0.2.68" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "841a6d1c35c6f596ccea1f82504a192a60378f64b3bb0261904ad8f2f5657556" +checksum = "f249f06ef7ee334cc3b8ff031bfc11ec99d00f34d86da7498396dc1e3b1498fe" dependencies = [ "proc-macro2", "quote", @@ -2587,15 +2605,15 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.67" +version = "0.2.68" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93b162580e34310e5931c4b792560108b10fd14d64915d7fff8ff00180e70092" +checksum = "1d649a3145108d7d3fbcde896a468d1bd636791823c9921135218ad89be08307" [[package]] name = "web-sys" -version = "0.3.44" +version = "0.3.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dda38f4e5ca63eda02c059d243aa25b5f35ab98451e518c51612cd0f1bd19a47" +checksum = "4bf6ef87ad7ae8008e15a355ce696bed26012b7caa21605188cfd8214ab51e2d" dependencies = [ "js-sys", "wasm-bindgen", diff --git a/Cargo.toml b/Cargo.toml index 5d3543338..1b7a70079 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -16,8 +16,8 @@ edition = "2018" #rocket = { git = "https://github.com/SergioBenitez/Rocket.git", rev = "8d779caa22c63b15a6c3ceb75d8f6d4971b2eb67", features = ["tls"] } # Used to handle requests rocket = { git = "https://github.com/timokoesters/Rocket.git", branch = "empty_parameters", features = ["tls"] } #ruma = { git = "https://github.com/ruma/ruma", features = ["rand", "client-api", "federation-api", "unstable-pre-spec", "unstable-synapse-quirks"], rev = "aff914050eb297bd82b8aafb12158c88a9e480e1" } # Used for matrix spec type definitions and helpers -#ruma = { git = "https://github.com/timokoesters/ruma", features = ["rand", "client-api", "federation-api", "unstable-pre-spec", "unstable-synapse-quirks"], branch = "timo-fixes" } # Used for matrix spec type definitions and helpers -ruma = { path = "../ruma/ruma", features = ["unstable-exhaustive-types", "rand", "client-api", "federation-api", "unstable-pre-spec", "unstable-synapse-quirks"] } +ruma = { git = "https://github.com/timokoesters/ruma", features = ["rand", "client-api", "federation-api", "unstable-exhaustive-types", "unstable-pre-spec", "unstable-synapse-quirks"], branch = "timo-fed-fixes" } # Used for matrix spec type definitions and helpers +#ruma = { path = "../ruma/ruma", features = ["unstable-exhaustive-types", "rand", "client-api", "federation-api", "unstable-pre-spec", "unstable-synapse-quirks"] } tokio = "0.2.22" # Used for long polling sled = "0.32.0" # Used for storing data permanently log = "0.4.8" # Used for emitting log entries @@ -32,8 +32,8 @@ reqwest = "0.10.6" # Used to send requests thiserror = "1.0.19" # Used for conduit::Error type image = { version = "0.23.4", default-features = false, features = ["jpeg", "png", "gif"] } # Used to generate thumbnails for images base64 = "0.12.3" # Used to encode server public key -#state-res = { git = "https://github.com/ruma/state-res", version = "0.1.0", branch = "spec-comp" } -state-res = { path = "../state-res" } +state-res = { git = "https://github.com/timokoesters/state-res", branch = "spec-comp", features = ["unstable-pre-spec"] } +#state-res = { path = "../state-res", features = ["unstable-pre-spec"] } ring = "0.16.15" [features] diff --git a/src/client_server/account.rs b/src/client_server/account.rs index cb77a15d4..3db933c17 100644 --- a/src/client_server/account.rs +++ b/src/client_server/account.rs @@ -356,14 +356,14 @@ pub fn deactivate_route( db.rooms.build_and_append_pdu( PduBuilder { - room_id: room_id.clone(), - sender: sender_id.clone(), event_type: EventType::RoomMember, content: serde_json::to_value(event).expect("event is valid, we just created it"), unsigned: None, state_key: Some(sender_id.to_string()), redacts: None, }, + &sender_id, + &room_id, &db.globals, &db.account_data, )?; diff --git a/src/client_server/alias.rs b/src/client_server/alias.rs index 1d30261b6..bfdaecac4 100644 --- a/src/client_server/alias.rs +++ b/src/client_server/alias.rs @@ -64,9 +64,7 @@ pub async fn get_alias_helper( let response = server_server::send_request( &db, room_alias.server_name().to_string(), - federation::query::get_room_information::v1::Request { - room_alias, - }, + federation::query::get_room_information::v1::Request { room_alias }, ) .await?; diff --git a/src/client_server/directory.rs b/src/client_server/directory.rs index 34feb7180..2764d2c52 100644 --- a/src/client_server/directory.rs +++ b/src/client_server/directory.rs @@ -14,7 +14,7 @@ use ruma::{ }, federation, }, - directory::{IncomingFilter, PublicRoomsChunk, IncomingRoomNetwork}, + directory::{IncomingFilter, IncomingRoomNetwork, PublicRoomsChunk}, events::{ room::{avatar, canonical_alias, guest_access, history_visibility, name, topic}, EventType, diff --git a/src/client_server/media.rs b/src/client_server/media.rs index 038012e23..d07744727 100644 --- a/src/client_server/media.rs +++ b/src/client_server/media.rs @@ -34,12 +34,8 @@ pub fn create_content_route( db.globals.server_name(), utils::random_string(MXC_LENGTH) ); - db.media.create( - mxc.clone(), - &body.filename, - &body.content_type, - &body.file, - )?; + db.media + .create(mxc.clone(), &body.filename, &body.content_type, &body.file)?; Ok(create_content::Response { content_uri: mxc }.into()) } diff --git a/src/client_server/membership.rs b/src/client_server/membership.rs index 606e4700a..ea2271bb3 100644 --- a/src/client_server/membership.rs +++ b/src/client_server/membership.rs @@ -4,6 +4,7 @@ use crate::{ pdu::{PduBuilder, PduEvent}, server_server, utils, ConduitResult, Database, Error, Ruma, }; +use log::warn; use ruma::{ api::{ client::{ @@ -20,8 +21,7 @@ use ruma::{ EventId, Raw, RoomId, RoomVersionId, UserId, }; use state_res::StateEvent; - -use std::{collections::BTreeMap, convert::TryFrom}; +use std::{collections::BTreeMap, convert::TryFrom, sync::Arc}; #[cfg(feature = "conduit_bin")] use rocket::{get, post}; @@ -106,14 +106,14 @@ pub fn leave_room_route( db.rooms.build_and_append_pdu( PduBuilder { - room_id: body.room_id.clone(), - sender: sender_id.clone(), event_type: EventType::RoomMember, content: serde_json::to_value(event).expect("event is valid, we just created it"), unsigned: None, state_key: Some(sender_id.to_string()), redacts: None, }, + &sender_id, + &body.room_id, &db.globals, &db.account_data, )?; @@ -134,8 +134,6 @@ pub fn invite_user_route( if let invite_user::IncomingInvitationRecipient::UserId { user_id } = &body.recipient { db.rooms.build_and_append_pdu( PduBuilder { - room_id: body.room_id.clone(), - sender: sender_id.clone(), event_type: EventType::RoomMember, content: serde_json::to_value(member::MemberEventContent { membership: member::MembershipState::Invite, @@ -149,6 +147,8 @@ pub fn invite_user_route( state_key: Some(user_id.to_string()), redacts: None, }, + &sender_id, + &body.room_id, &db.globals, &db.account_data, )?; @@ -191,14 +191,14 @@ pub fn kick_user_route( db.rooms.build_and_append_pdu( PduBuilder { - room_id: body.room_id.clone(), - sender: sender_id.clone(), event_type: EventType::RoomMember, content: serde_json::to_value(event).expect("event is valid, we just created it"), unsigned: None, state_key: Some(body.user_id.to_string()), redacts: None, }, + &sender_id, + &body.room_id, &db.globals, &db.account_data, )?; @@ -246,14 +246,14 @@ pub fn ban_user_route( db.rooms.build_and_append_pdu( PduBuilder { - room_id: body.room_id.clone(), - sender: sender_id.clone(), event_type: EventType::RoomMember, content: serde_json::to_value(event).expect("event is valid, we just created it"), unsigned: None, state_key: Some(body.user_id.to_string()), redacts: None, }, + &sender_id, + &body.room_id, &db.globals, &db.account_data, )?; @@ -292,14 +292,14 @@ pub fn unban_user_route( db.rooms.build_and_append_pdu( PduBuilder { - room_id: body.room_id.clone(), - sender: sender_id.clone(), event_type: EventType::RoomMember, content: serde_json::to_value(event).expect("event is valid, we just created it"), unsigned: None, state_key: Some(body.user_id.to_string()), redacts: None, }, + &sender_id, + &body.room_id, &db.globals, &db.account_data, )?; @@ -473,7 +473,7 @@ async fn join_room_by_id_helper( let send_join_response = server_server::send_request( &db, room_id.server_name().to_string(), - federation::membership::create_join_event::v1::Request { + federation::membership::create_join_event::v2::Request { room_id, event_id: &event_id, pdu_stub: serde_json::from_value(join_event_stub_value) @@ -482,25 +482,39 @@ async fn join_room_by_id_helper( ) .await?; - dbg!(&send_join_response); - let mut event_map = send_join_response .room_state .state .iter() .chain(send_join_response.room_state.auth_chain.iter()) .map(|pdu| { - pdu.deserialize() - .map(StateEvent::Full) - .map(|ev| (ev.event_id(), ev)) + let mut value = serde_json::from_str(pdu.json().get()) + .expect("converting raw jsons to values always works"); + let event_id = EventId::try_from(&*format!( + "${}", + ruma::signatures::reference_hash(&value) + .expect("ruma can calculate reference hashes") + )) + .expect("ruma's reference hashes are valid event ids"); + + value + .as_object_mut() + .ok_or_else(|| Error::BadServerResponse("PDU is not an object."))? + .insert("event_id".to_owned(), event_id.to_string().into()); + + serde_json::from_value::(value) + .map(|ev| (event_id, Arc::new(ev))) + .map_err(|e| { + warn!("{}", e); + Error::BadServerResponse("Invalid PDU bytes in send_join response.") + }) }) - .collect::, _>>() - .map_err(|_| Error::bad_database("Invalid PDU found in db."))?; + .collect::>, _>>()?; let control_events = event_map .values() .filter(|pdu| pdu.is_power_event()) - .map(|pdu| pdu.event_id()) + .map(|pdu| pdu.event_id().clone()) .collect::>(); // These events are not guaranteed to be sorted but they are resolved according to spec @@ -515,7 +529,9 @@ async fn join_room_by_id_helper( .room_state .auth_chain .iter() - .filter_map(|pdu| Some(StateEvent::Full(pdu.deserialize().ok()?).event_id())) + .filter_map(|pdu| { + Some(StateEvent::Full(pdu.deserialize().ok()?).event_id().clone()) + }) .collect::>(), ); @@ -575,31 +591,31 @@ async fn join_room_by_id_helper( // We do not rebuild the PDU in this case only insert to DB db.rooms - .append_pdu(PduEvent::try_from(pdu)?, &db.globals, &db.account_data)?; + .append_pdu(PduEvent::from(&**pdu), &db.globals, &db.account_data)?; } + } else { + let event = member::MemberEventContent { + membership: member::MembershipState::Join, + displayname: db.users.displayname(&sender_id)?, + avatar_url: db.users.avatar_url(&sender_id)?, + is_direct: None, + third_party_invite: None, + }; + + db.rooms.build_and_append_pdu( + PduBuilder { + event_type: EventType::RoomMember, + content: serde_json::to_value(event).expect("event is valid, we just created it"), + unsigned: None, + state_key: Some(sender_id.to_string()), + redacts: None, + }, + &sender_id, + &room_id, + &db.globals, + &db.account_data, + )?; } - let event = member::MemberEventContent { - membership: member::MembershipState::Join, - displayname: db.users.displayname(&sender_id)?, - avatar_url: db.users.avatar_url(&sender_id)?, - is_direct: None, - third_party_invite: None, - }; - - db.rooms.build_and_append_pdu( - PduBuilder { - room_id: room_id.clone(), - sender: sender_id.clone(), - event_type: EventType::RoomMember, - content: serde_json::to_value(event).expect("event is valid, we just created it"), - unsigned: None, - state_key: Some(sender_id.to_string()), - redacts: None, - }, - &db.globals, - &db.account_data, - )?; - Ok(join_room_by_id::Response::new(room_id.clone()).into()) } diff --git a/src/client_server/message.rs b/src/client_server/message.rs index 09c3517fa..025331e69 100644 --- a/src/client_server/message.rs +++ b/src/client_server/message.rs @@ -27,11 +27,10 @@ pub fn send_message_event_route( let event_id = db.rooms.build_and_append_pdu( PduBuilder { - room_id: body.room_id.clone(), - sender: sender_id.clone(), event_type: body.content.event_type().into(), content: serde_json::from_str( body.json_body + .as_ref() .ok_or(Error::BadRequest(ErrorKind::BadJson, "Invalid JSON body."))? .get(), ) @@ -40,6 +39,8 @@ pub fn send_message_event_route( state_key: None, redacts: None, }, + &sender_id, + &body.room_id, &db.globals, &db.account_data, )?; diff --git a/src/client_server/profile.rs b/src/client_server/profile.rs index 386d89885..2a2e05ef3 100644 --- a/src/client_server/profile.rs +++ b/src/client_server/profile.rs @@ -33,8 +33,6 @@ pub fn set_displayname_route( let room_id = room_id?; db.rooms.build_and_append_pdu( PduBuilder { - room_id: room_id.clone(), - sender: sender_id.clone(), event_type: EventType::RoomMember, content: serde_json::to_value(ruma::events::room::member::MemberEventContent { displayname: body.displayname.clone(), @@ -62,6 +60,8 @@ pub fn set_displayname_route( state_key: Some(sender_id.to_string()), redacts: None, }, + &sender_id, + &room_id, &db.globals, &db.account_data, )?; @@ -136,8 +136,6 @@ pub fn set_avatar_url_route( let room_id = room_id?; db.rooms.build_and_append_pdu( PduBuilder { - room_id: room_id.clone(), - sender: sender_id.clone(), event_type: EventType::RoomMember, content: serde_json::to_value(ruma::events::room::member::MemberEventContent { avatar_url: body.avatar_url.clone(), @@ -165,6 +163,8 @@ pub fn set_avatar_url_route( state_key: Some(sender_id.to_string()), redacts: None, }, + &sender_id, + &room_id, &db.globals, &db.account_data, )?; diff --git a/src/client_server/redact.rs b/src/client_server/redact.rs index cd1b44384..511734899 100644 --- a/src/client_server/redact.rs +++ b/src/client_server/redact.rs @@ -20,8 +20,6 @@ pub fn redact_event_route( let event_id = db.rooms.build_and_append_pdu( PduBuilder { - room_id: body.room_id.clone(), - sender: sender_id.clone(), event_type: EventType::RoomRedaction, content: serde_json::to_value(redaction::RedactionEventContent { reason: body.reason.clone(), @@ -31,6 +29,8 @@ pub fn redact_event_route( state_key: None, redacts: Some(body.event_id.clone()), }, + &sender_id, + &body.room_id, &db.globals, &db.account_data, )?; diff --git a/src/client_server/room.rs b/src/client_server/room.rs index 9918123f8..9a83f81b0 100644 --- a/src/client_server/room.rs +++ b/src/client_server/room.rs @@ -55,14 +55,14 @@ pub fn create_room_route( // 1. The room create event db.rooms.build_and_append_pdu( PduBuilder { - room_id: room_id.clone(), - sender: sender_id.clone(), event_type: EventType::RoomCreate, content: serde_json::to_value(content).expect("event is valid, we just created it"), unsigned: None, state_key: Some("".to_owned()), redacts: None, }, + &sender_id, + &room_id, &db.globals, &db.account_data, )?; @@ -70,8 +70,6 @@ pub fn create_room_route( // 2. Let the room creator join db.rooms.build_and_append_pdu( PduBuilder { - room_id: room_id.clone(), - sender: sender_id.clone(), event_type: EventType::RoomMember, content: serde_json::to_value(member::MemberEventContent { membership: member::MembershipState::Join, @@ -85,6 +83,8 @@ pub fn create_room_route( state_key: Some(sender_id.to_string()), redacts: None, }, + &sender_id, + &room_id, &db.globals, &db.account_data, )?; @@ -119,14 +119,14 @@ pub fn create_room_route( }; db.rooms.build_and_append_pdu( PduBuilder { - room_id: room_id.clone(), - sender: sender_id.clone(), event_type: EventType::RoomPowerLevels, content: power_levels_content, unsigned: None, state_key: Some("".to_owned()), redacts: None, }, + &sender_id, + &room_id, &db.globals, &db.account_data, )?; @@ -142,8 +142,6 @@ pub fn create_room_route( // 4.1 Join Rules db.rooms.build_and_append_pdu( PduBuilder { - room_id: room_id.clone(), - sender: sender_id.clone(), event_type: EventType::RoomJoinRules, content: match preset { create_room::RoomPreset::PublicChat => serde_json::to_value( @@ -160,6 +158,8 @@ pub fn create_room_route( state_key: Some("".to_owned()), redacts: None, }, + &sender_id, + &room_id, &db.globals, &db.account_data, )?; @@ -167,8 +167,6 @@ pub fn create_room_route( // 4.2 History Visibility db.rooms.build_and_append_pdu( PduBuilder { - room_id: room_id.clone(), - sender: sender_id.clone(), event_type: EventType::RoomHistoryVisibility, content: serde_json::to_value(history_visibility::HistoryVisibilityEventContent::new( history_visibility::HistoryVisibility::Shared, @@ -178,6 +176,8 @@ pub fn create_room_route( state_key: Some("".to_owned()), redacts: None, }, + &sender_id, + &room_id, &db.globals, &db.account_data, )?; @@ -185,8 +185,6 @@ pub fn create_room_route( // 4.3 Guest Access db.rooms.build_and_append_pdu( PduBuilder { - room_id: room_id.clone(), - sender: sender_id.clone(), event_type: EventType::RoomGuestAccess, content: match preset { create_room::RoomPreset::PublicChat => { @@ -204,6 +202,8 @@ pub fn create_room_route( state_key: Some("".to_owned()), redacts: None, }, + &sender_id, + &room_id, &db.globals, &db.account_data, )?; @@ -212,24 +212,27 @@ pub fn create_room_route( for event in &body.initial_state { let pdu_builder = serde_json::from_str::( &serde_json::to_string(&event).expect("AnyInitialStateEvent::to_string always works"), - ).map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "Invalid initial state event."))?; + ) + .map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "Invalid initial state event."))?; // Silently skip encryption events if they are not allowed - if pdu_builder.event_type == EventType::RoomEncryption && db.globals.encryption_disabled() - { + if pdu_builder.event_type == EventType::RoomEncryption && db.globals.encryption_disabled() { continue; } - db.rooms - .build_and_append_pdu(pdu_builder, &db.globals, &db.account_data)?; + db.rooms.build_and_append_pdu( + pdu_builder, + &sender_id, + &room_id, + &db.globals, + &db.account_data, + )?; } // 6. Events implied by name and topic if let Some(name) = &body.name { db.rooms.build_and_append_pdu( PduBuilder { - room_id: room_id.clone(), - sender: sender_id.clone(), event_type: EventType::RoomName, content: serde_json::to_value( name::NameEventContent::new(name.clone()).map_err(|_| { @@ -241,6 +244,8 @@ pub fn create_room_route( state_key: Some("".to_owned()), redacts: None, }, + &sender_id, + &room_id, &db.globals, &db.account_data, )?; @@ -249,8 +254,6 @@ pub fn create_room_route( if let Some(topic) = &body.topic { db.rooms.build_and_append_pdu( PduBuilder { - room_id: room_id.clone(), - sender: sender_id.clone(), event_type: EventType::RoomTopic, content: serde_json::to_value(topic::TopicEventContent { topic: topic.clone(), @@ -260,6 +263,8 @@ pub fn create_room_route( state_key: Some("".to_owned()), redacts: None, }, + &sender_id, + &room_id, &db.globals, &db.account_data, )?; @@ -269,8 +274,6 @@ pub fn create_room_route( for user in &body.invite { db.rooms.build_and_append_pdu( PduBuilder { - room_id: room_id.clone(), - sender: sender_id.clone(), event_type: EventType::RoomMember, content: serde_json::to_value(member::MemberEventContent { membership: member::MembershipState::Invite, @@ -284,6 +287,8 @@ pub fn create_room_route( state_key: Some(user.to_string()), redacts: None, }, + &sender_id, + &room_id, &db.globals, &db.account_data, )?; diff --git a/src/client_server/search.rs b/src/client_server/search.rs index 2967e001e..3b03e7ace 100644 --- a/src/client_server/search.rs +++ b/src/client_server/search.rs @@ -77,7 +77,7 @@ pub fn search_events_route( Ok(search_events::Response::new(ResultCategories { room_events: ResultRoomEvents { - count: None, // TODO? maybe not + count: None, // TODO? maybe not groups: BTreeMap::new(), // TODO next_batch, results, diff --git a/src/client_server/state.rs b/src/client_server/state.rs index 75463cbac..1fe3cd6c2 100644 --- a/src/client_server/state.rs +++ b/src/client_server/state.rs @@ -213,14 +213,14 @@ pub fn send_state_event_for_key_helper( let event_id = db.rooms.build_and_append_pdu( PduBuilder { - room_id: room_id.clone(), - sender: sender_id.clone(), event_type: content.event_type().into(), content: json, unsigned: None, state_key, redacts: None, }, + &sender_id, + &room_id, &db.globals, &db.account_data, )?; diff --git a/src/database.rs b/src/database.rs index a105058cb..0d18020bc 100644 --- a/src/database.rs +++ b/src/database.rs @@ -97,7 +97,6 @@ impl Database { }, pduid_pdu: db.open_tree("pduid_pdu")?, eventid_pduid: db.open_tree("eventid_pduid")?, - roomstateid_pduid: db.open_tree("roomstateid_pduid")?, roomid_pduleaves: db.open_tree("roomid_pduleaves")?, alias_roomid: db.open_tree("alias_roomid")?, diff --git a/src/database/rooms.rs b/src/database/rooms.rs index ee070b38b..60333782d 100644 --- a/src/database/rooms.rs +++ b/src/database/rooms.rs @@ -25,6 +25,7 @@ use std::{ collections::{BTreeMap, HashMap}, convert::{TryFrom, TryInto}, mem, + sync::Arc, }; /// The unique identifier of each state group. @@ -33,10 +34,6 @@ use std::{ /// hashing the entire state. pub type StateHashId = Vec; -/// This identifier consists of roomId + count. It represents a -/// unique event, it will never be overwritten or removed. -pub type PduId = IVec; - pub struct Rooms { pub edus: edus::RoomEdus, pub(super) pduid_pdu: sled::Tree, // PduId = RoomId + Count @@ -54,22 +51,22 @@ pub struct Rooms { pub(super) roomuserid_invited: sled::Tree, pub(super) userroomid_left: sled::Tree, - // STATE TREES - /// This holds the full current state, including the latest event. - pub(super) roomstateid_pduid: sled::Tree, // RoomStateId = Room + StateType + StateKey - /// This holds the full room state minus the latest event. - pub(super) pduid_statehash: sled::Tree, // PDU id -> StateHash - /// Also holds the full room state minus the latest event. - pub(super) stateid_pduid: sled::Tree, // StateId = StateHash + (EventType, StateKey) - /// The room_id -> the latest StateHash + /// Remember the current state hash of a room. pub(super) roomid_statehash: sled::Tree, + /// Remember the state hash at events in the past. + pub(super) pduid_statehash: sled::Tree, + /// The state for a given state hash. + pub(super) stateid_pduid: sled::Tree, // StateId = StateHash + EventType + StateKey } impl StateStore for Rooms { - fn get_event(&self, room_id: &RoomId, event_id: &EventId) -> state_res::Result { + fn get_event( + &self, + room_id: &RoomId, + event_id: &EventId, + ) -> state_res::Result> { let pid = self - .eventid_pduid - .get(event_id.as_bytes()) + .get_pdu_id(event_id) .map_err(StateError::custom)? .ok_or_else(|| { StateError::NotFound("PDU via room_id and event_id not found in the db.".into()) @@ -87,7 +84,7 @@ impl StateStore for Rooms { // conduit's PDU's always contain a room_id but some // of ruma's do not so this must be an Option if pdu.room_id() == Some(room_id) { - Ok(pdu) + Ok(Arc::new(pdu)) } else { Err(StateError::NotFound( "Found PDU for incorrect room in db.".into(), @@ -136,53 +133,12 @@ impl Rooms { None } - /// Fetch the current State using the `roomstateid_pduid` tree. - pub fn current_state_pduids(&self, room_id: &RoomId) -> Result> { - // TODO this could also scan roomstateid_pduid if we passed in room_id ? - self.roomstateid_pduid - .scan_prefix(room_id.as_bytes()) - .values() - .map(|pduid| { - let pduid = &pduid?; - self.pduid_pdu.get(pduid)?.map_or_else( - || { - Err(Error::bad_database( - "Failed to find current state of pduid's.", - )) - }, - |b| { - Ok(( - serde_json::from_slice::(&b) - .map_err(|_| Error::bad_database("Invalid PDU in db."))?, - pduid.clone(), - )) - }, - ) - }) - .map(|pair| { - let (pdu, id) = pair?; - Ok(((pdu.kind, pdu.state_key), id)) - }) - .collect::>>() - } - /// Returns the last state hash key added to the db. - pub fn current_state_hash(&self, room_id: &RoomId) -> Result { - let mut prefix = room_id.as_bytes().to_vec(); - prefix.push(0xff); - - // We must check here because this method is called outside and before - // `append_state_pdu` so the DB can be empty - if self.pduid_statehash.scan_prefix(prefix).next().is_none() { - // return the hash of the room_id, this represents a room with no state - return self.new_state_hash_id(room_id); - } - - self.pduid_statehash - .iter() - .next_back() - .map(|pair| Ok(pair?.1.to_vec())) - .ok_or_else(|| Error::bad_database("No PDU's found for this room."))? + pub fn current_state_hash(&self, room_id: &RoomId) -> Result> { + Ok(self + .roomid_statehash + .get(room_id.as_bytes())? + .map(|bytes| bytes.to_vec())) } /// This fetches auth event_ids from the current state using the @@ -243,39 +199,11 @@ impl Rooms { /// Generate a new StateHash. /// - /// A unique hash made from hashing the current states pduid's. - fn new_state_hash_id(&self, room_id: &RoomId) -> Result { - // Use hashed roomId as the first StateHash key for first state event in room - if self - .pduid_statehash - .scan_prefix(room_id.as_bytes()) - .next() - .is_none() - { - return Ok(digest::digest(&digest::SHA256, room_id.as_bytes()) - .as_ref() - .to_vec()); - } - - let pdu_ids_to_hash = self - .pduid_statehash - .scan_prefix(room_id.as_bytes()) - .values() - .next_back() - .unwrap() // We just checked if the tree was empty - .map(|hash| { - self.stateid_pduid - .scan_prefix(hash) - .values() - // pduid is roomId + count so just hash the whole thing - .map(|pid| Ok(pid?.to_vec())) - .collect::>>>() - })??; - - let hash = digest::digest( - &digest::SHA256, - &pdu_ids_to_hash.into_iter().flatten().collect::>(), - ); + /// A unique hash made from hashing all PDU ids of the state joined with 0xff. + fn calculate_hash(&self, pdu_id_bytes: &[&[u8]]) -> Result { + // We only hash the pdu's event ids, not the whole pdu + let bytes = pdu_id_bytes.join(&0xff); + let hash = digest::digest(&digest::SHA256, &bytes); Ok(hash.as_ref().to_vec()) } @@ -297,29 +225,38 @@ impl Rooms { &self, room_id: &RoomId, ) -> Result> { - let mut hashmap = HashMap::new(); - for pdu in - self.roomstateid_pduid - .scan_prefix(&room_id.as_bytes()) + if let Some(current_state_hash) = self.current_state_hash(room_id)? { + let mut prefix = current_state_hash; + prefix.push(0xff); + + let mut hashmap = HashMap::new(); + for pdu in self + .stateid_pduid + .scan_prefix(prefix) .values() - .map(|value| { + .map(|pdu_id| { Ok::<_, Error>( serde_json::from_slice::( - &self.pduid_pdu.get(value?)?.ok_or_else(|| { - Error::bad_database("PDU not found for ID in db.") + &self.pduid_pdu.get(pdu_id?)?.ok_or_else(|| { + Error::bad_database("PDU in state not found in database.") })?, ) - .map_err(|_| Error::bad_database("Invalid PDU in db."))?, + .map_err(|_| { + Error::bad_database("Invalid PDU bytes in current room state.") + })?, ) }) - { - let pdu = pdu?; - let state_key = pdu.state_key.clone().ok_or_else(|| { - Error::bad_database("Room state contains event without state_key.") - })?; - hashmap.insert((pdu.kind.clone(), state_key), pdu); + { + let pdu = pdu?; + let state_key = pdu.state_key.clone().ok_or_else(|| { + Error::bad_database("Room state contains event without state_key.") + })?; + hashmap.insert((pdu.kind.clone(), state_key), pdu); + } + Ok(hashmap) + } else { + Ok(HashMap::new()) } - Ok(hashmap) } /// Returns all state entries for this type. @@ -328,33 +265,40 @@ impl Rooms { room_id: &RoomId, event_type: &EventType, ) -> Result> { - let mut prefix = room_id.as_bytes().to_vec(); - prefix.push(0xff); - prefix.extend_from_slice(&event_type.to_string().as_bytes()); + if let Some(current_state_hash) = self.current_state_hash(room_id)? { + let mut prefix = current_state_hash; + prefix.push(0xff); + prefix.extend_from_slice(&event_type.to_string().as_bytes()); + prefix.push(0xff); - let mut hashmap = HashMap::new(); - for pdu in - self.roomstateid_pduid + let mut hashmap = HashMap::new(); + for pdu in self + .stateid_pduid .scan_prefix(&prefix) .values() - .map(|value| { + .map(|pdu_id| { Ok::<_, Error>( serde_json::from_slice::( - &self.pduid_pdu.get(value?)?.ok_or_else(|| { - Error::bad_database("PDU not found for ID in db.") + &self.pduid_pdu.get(pdu_id?)?.ok_or_else(|| { + Error::bad_database("PDU in state not found in database.") })?, ) - .map_err(|_| Error::bad_database("Invalid PDU in db."))?, + .map_err(|_| { + Error::bad_database("Invalid PDU bytes in current room state.") + })?, ) }) - { - let pdu = pdu?; - let state_key = pdu.state_key.clone().ok_or_else(|| { - Error::bad_database("Room state contains event without state_key.") - })?; - hashmap.insert(state_key, pdu); + { + let pdu = pdu?; + let state_key = pdu.state_key.clone().ok_or_else(|| { + Error::bad_database("Room state contains event without state_key.") + })?; + hashmap.insert(state_key, pdu); + } + Ok(hashmap) + } else { + Ok(HashMap::new()) } - Ok(hashmap) } /// Returns a single PDU from `room_id` with key (`event_type`, `state_key`). @@ -364,23 +308,24 @@ impl Rooms { event_type: &EventType, state_key: &str, ) -> Result> { - let mut key = room_id.as_bytes().to_vec(); - key.push(0xff); - key.extend_from_slice(&event_type.to_string().as_bytes()); - key.push(0xff); - key.extend_from_slice(&state_key.as_bytes()); + if let Some(current_state_hash) = self.current_state_hash(room_id)? { + let mut key = current_state_hash; + key.push(0xff); + key.extend_from_slice(&event_type.to_string().as_bytes()); + key.push(0xff); + key.extend_from_slice(&state_key.as_bytes()); - self.roomstateid_pduid.get(&key)?.map_or(Ok(None), |value| { - Ok::<_, Error>(Some( - serde_json::from_slice::( - &self - .pduid_pdu - .get(value)? - .ok_or_else(|| Error::bad_database("PDU not found for ID in db."))?, - ) - .map_err(|_| Error::bad_database("Invalid PDU in db."))?, - )) - }) + self.stateid_pduid.get(&key)?.map_or(Ok(None), |pdu_id| { + Ok::<_, Error>(Some( + serde_json::from_slice::(&self.pduid_pdu.get(pdu_id)?.ok_or_else( + || Error::bad_database("PDU in state not found in database."), + )?) + .map_err(|_| Error::bad_database("Invalid PDU bytes in current room state."))?, + )) + }) + } else { + Ok(None) + } } /// Returns the `count` of this pdu's id. @@ -528,8 +473,8 @@ impl Rooms { self.eventid_pduid .insert(pdu.event_id.as_bytes(), &*pdu_id)?; - if let Some(state_key) = &pdu.state_key { - self.append_state_pdu(&pdu.room_id, &pdu_id, state_key, &pdu.kind)?; + if pdu.state_key.is_some() { + self.append_to_state(&pdu_id, &pdu)?; } match &pdu.kind { @@ -603,59 +548,69 @@ impl Rooms { /// This adds all current state events (not including the incoming event) /// to `stateid_pduid` and adds the incoming event to `pduid_statehash`. /// The incoming event is the `pdu_id` passed to this method. - fn append_state_pdu( - &self, - room_id: &RoomId, - pdu_id: &[u8], - state_key: &str, - kind: &EventType, - ) -> Result { - let state_hash = self.new_state_hash_id(room_id)?; - let state = self.current_state_pduids(room_id)?; + fn append_to_state(&self, new_pdu_id: &[u8], new_pdu: &PduEvent) -> Result { + let old_state = + if let Some(old_state_hash) = self.roomid_statehash.get(new_pdu.room_id.as_bytes())? { + // Store state for event. The state does not include the event itself. + // Instead it's the state before the pdu, so the room's old state. + self.pduid_statehash.insert(new_pdu_id, &old_state_hash)?; + if new_pdu.state_key.is_none() { + return Ok(old_state_hash.to_vec()); + } - let mut key = state_hash.to_vec(); - key.push(0xff); + let mut prefix = old_state_hash.to_vec(); + prefix.push(0xff); + self.stateid_pduid + .scan_prefix(&prefix) + .filter_map(|pdu| pdu.map_err(|e| error!("{}", e)).ok()) + .map(|(k, v)| (k.subslice(prefix.len(), k.len() - prefix.len()), v)) + .collect::>() + } else { + HashMap::new() + }; - // TODO eventually we could avoid writing to the DB so much on every event - // by keeping track of the delta and write that every so often - for ((ev_ty, state_key), pid) in state { - let mut state_id = key.to_vec(); - state_id.extend_from_slice(ev_ty.to_string().as_bytes()); - key.push(0xff); - state_id.extend_from_slice(state_key.expect("state event").as_bytes()); + if let Some(state_key) = &new_pdu.state_key { + let mut new_state = old_state; + let mut pdu_key = new_pdu.kind.as_str().as_bytes().to_vec(); + pdu_key.push(0xff); + pdu_key.extend_from_slice(state_key.as_bytes()); + new_state.insert(pdu_key.into(), new_pdu_id.into()); + + let new_state_hash = + self.calculate_hash(&new_state.values().map(|b| &**b).collect::>())?; + + let mut key = new_state_hash.to_vec(); key.push(0xff); - self.stateid_pduid.insert(&state_id, &pid)?; + // TODO: we could avoid writing to the DB on every state event by keeping + // track of the delta and write that every so often + for (key_without_prefix, pdu_id) in new_state { + let mut state_id = key.clone(); + state_id.extend_from_slice(&key_without_prefix); + self.stateid_pduid.insert(&state_id, &pdu_id)?; + } + + self.roomid_statehash + .insert(new_pdu.room_id.as_bytes(), &*new_state_hash)?; + + Ok(new_state_hash) + } else { + Err(Error::bad_database( + "Tried to insert non-state event into room without a state.", + )) } - - // This event's state does not include the event itself. `current_state_pduids` - // uses `roomstateid_pduid` before the current event is inserted to the tree so the state - // will be everything up to but not including the incoming event. - self.pduid_statehash.insert(pdu_id, state_hash.as_slice())?; - - self.roomid_statehash - .insert(room_id.as_bytes(), state_hash.as_slice())?; - - let mut key = room_id.as_bytes().to_vec(); - key.push(0xff); - key.extend_from_slice(kind.to_string().as_bytes()); - key.push(0xff); - key.extend_from_slice(state_key.as_bytes()); - self.roomstateid_pduid.insert(key, pdu_id)?; - - Ok(state_hash) } /// Creates a new persisted data unit and adds it to a room. pub fn build_and_append_pdu( &self, pdu_builder: PduBuilder, + sender: &UserId, + room_id: &RoomId, globals: &super::globals::Globals, account_data: &super::account_data::AccountData, ) -> Result { let PduBuilder { - room_id, - sender, event_type, content, unsigned, @@ -741,8 +696,7 @@ impl Rooms { ErrorKind::Unknown, "Membership can't be the first event", ))?)? - .map(|pdu| pdu.convert_for_state_res()) - .transpose()?; + .map(|pdu| pdu.convert_for_state_res()); event_auth::valid_membership_change( // TODO this is a bit of a hack but not sure how to have a type // declared in `state_res` crate easily convert to/from conduit::PduEvent @@ -753,11 +707,12 @@ impl Rooms { state_key: Some(state_key.to_owned()), sender: &sender, }, - prev_event.as_ref(), + prev_event, + None, &auth_events .iter() .map(|((ty, key), pdu)| { - Ok(((ty.clone(), key.clone()), pdu.convert_for_state_res()?)) + Ok(((ty.clone(), key.clone()), pdu.convert_for_state_res())) }) .collect::>>()?, ) @@ -812,9 +767,8 @@ impl Rooms { let mut pdu = PduEvent { event_id: EventId::try_from("$thiswillbefilledinlater").expect("we know this is valid"), - room_id, - sender, - origin: globals.server_name().to_owned(), + room_id: room_id.clone(), + sender: sender.clone(), origin_server_ts: utils::millis_since_unix_epoch() .try_into() .expect("time is valid"), @@ -834,7 +788,7 @@ impl Rooms { hashes: ruma::events::pdu::EventHash { sha256: "aaa".to_owned(), }, - signatures: HashMap::new(), + signatures: BTreeMap::new(), }; // Generate event id @@ -1028,8 +982,6 @@ impl Rooms { self.build_and_append_pdu( PduBuilder { - room_id: room_id.clone(), - sender: user_id.clone(), event_type: EventType::RoomMember, content: serde_json::to_value(member_content) .expect("event is valid, we just created it"), @@ -1037,6 +989,8 @@ impl Rooms { state_key: Some(user_id.to_string()), redacts: None, }, + &user_id, + &room_id, globals, account_data, )?; diff --git a/src/pdu.rs b/src/pdu.rs index 152648423..f23e688c1 100644 --- a/src/pdu.rs +++ b/src/pdu.rs @@ -5,18 +5,17 @@ use ruma::{ pdu::EventHash, room::member::MemberEventContent, AnyEvent, AnyRoomEvent, AnyStateEvent, AnyStrippedStateEvent, AnySyncRoomEvent, AnySyncStateEvent, EventType, StateEvent, }, - EventId, Raw, RoomId, ServerName, UserId, + EventId, Raw, RoomId, ServerKeyId, ServerName, UserId, }; use serde::{Deserialize, Serialize}; use serde_json::json; -use std::{collections::HashMap, convert::TryFrom}; +use std::{collections::BTreeMap, convert::TryInto, sync::Arc, time::UNIX_EPOCH}; -#[derive(Deserialize, Serialize)] +#[derive(Deserialize, Serialize, Debug)] pub struct PduEvent { pub event_id: EventId, pub room_id: RoomId, pub sender: UserId, - pub origin: Box, pub origin_server_ts: UInt, #[serde(rename = "type")] pub kind: EventType, @@ -31,7 +30,7 @@ pub struct PduEvent { #[serde(default, skip_serializing_if = "serde_json::Map::is_empty")] pub unsigned: serde_json::Map, pub hashes: EventHash, - pub signatures: HashMap>, + pub signatures: BTreeMap, BTreeMap>, } impl PduEvent { @@ -199,66 +198,69 @@ impl PduEvent { } } -impl TryFrom<&state_res::StateEvent> for PduEvent { - type Error = Error; - fn try_from(pdu: &state_res::StateEvent) -> Result { - serde_json::from_value(json!({ - "event_id": pdu.event_id(), - "room_id": pdu.room_id(), - "sender": pdu.sender(), - "origin": pdu.origin(), - "origin_server_ts": pdu.origin_server_ts(), - "event_type": pdu.kind(), - "content": pdu.content(), - "state_key": pdu.state_key(), - "prev_events": pdu.prev_event_ids(), - "depth": pdu.depth(), - "auth_events": pdu.auth_events(), - "redacts": pdu.redacts(), - "unsigned": pdu.unsigned(), - "hashes": pdu.hashes(), - "signatures": pdu.signatures(), - })) - .map_err(|_| Error::bad_database("Failed to convert PDU to ruma::Pdu type.")) +impl From<&state_res::StateEvent> for PduEvent { + fn from(pdu: &state_res::StateEvent) -> Self { + Self { + event_id: pdu.event_id().clone(), + room_id: pdu.room_id().unwrap().clone(), + sender: pdu.sender().clone(), + origin_server_ts: (pdu + .origin_server_ts() + .duration_since(UNIX_EPOCH) + .expect("time is valid") + .as_millis() as u64) + .try_into() + .expect("time is valid"), + kind: pdu.kind(), + content: pdu.content().clone(), + state_key: pdu.state_key(), + prev_events: pdu.prev_event_ids(), + depth: pdu.depth().clone(), + auth_events: pdu.auth_events(), + redacts: pdu.redacts().cloned(), + unsigned: pdu.unsigned().clone().into_iter().collect(), + hashes: pdu.hashes().clone(), + signatures: pdu.signatures(), + } } } impl PduEvent { - pub fn convert_for_state_res(&self) -> Result { - serde_json::from_value(json!({ - "event_id": self.event_id, - "room_id": self.room_id, - "sender": self.sender, - "origin": self.origin, - "origin_server_ts": self.origin_server_ts, - "type": self.kind, - "content": self.content, - "state_key": self.state_key, - "prev_events": self.prev_events - .iter() - // TODO How do we create one of these - .map(|id| (id, EventHash { sha256: "hello".into() })) - .collect::>(), - "depth": self.depth, - "auth_events": self.auth_events - .iter() - // TODO How do we create one of these - .map(|id| (id, EventHash { sha256: "hello".into() })) - .collect::>(), - "redacts": self.redacts, - "unsigned": self.unsigned, - "hashes": self.hashes, - "signatures": self.signatures, - })) - .map_err(|_| Error::bad_database("Failed to convert PDU to ruma::Pdu type.")) + pub fn convert_for_state_res(&self) -> Arc { + Arc::new( + serde_json::from_value(json!({ + "event_id": self.event_id, + "room_id": self.room_id, + "sender": self.sender, + "origin_server_ts": self.origin_server_ts, + "type": self.kind, + "content": self.content, + "state_key": self.state_key, + "prev_events": self.prev_events + .iter() + // TODO How do we create one of these + .map(|id| (id, EventHash { sha256: "hello".into() })) + .collect::>(), + "depth": self.depth, + "auth_events": self.auth_events + .iter() + // TODO How do we create one of these + .map(|id| (id, EventHash { sha256: "hello".into() })) + .collect::>(), + "redacts": self.redacts, + "unsigned": self.unsigned, + "hashes": self.hashes, + "signatures": self.signatures, + })) + .expect("all conduit PDUs are state events"), + ) } } /// Build the start of a PDU in order to add it to the `Database`. #[derive(Debug, Deserialize)] pub struct PduBuilder { - pub room_id: RoomId, - pub sender: UserId, + #[serde(rename = "type")] pub event_type: EventType, pub content: serde_json::Value, pub unsigned: Option>, diff --git a/src/server_server.rs b/src/server_server.rs index d39abe645..ffa5c5be3 100644 --- a/src/server_server.rs +++ b/src/server_server.rs @@ -139,6 +139,7 @@ where .unwrap() .into_iter() .collect(); + Ok( T::IncomingResponse::try_from(http_response.body(body).unwrap()) .expect("TODO: error handle other server errors"), From 1f2843498863065ca2775bd0a02a37db3f8f0002 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20K=C3=B6sters?= Date: Sat, 12 Sep 2020 22:41:33 +0200 Subject: [PATCH 20/51] feat: hacky transactions --- src/server_server.rs | 40 +++++++++++++++++++++++++++++++--------- 1 file changed, 31 insertions(+), 9 deletions(-) diff --git a/src/server_server.rs b/src/server_server.rs index ffa5c5be3..d7d0e23fe 100644 --- a/src/server_server.rs +++ b/src/server_server.rs @@ -1,16 +1,19 @@ use crate::{client_server, ConduitResult, Database, Error, Result, Ruma}; use http::header::{HeaderValue, AUTHORIZATION}; use rocket::{get, post, put, response::content::Json, State}; -use ruma::api::{ - client, - federation::{ - directory::get_public_rooms, - discovery::{ - get_server_keys, get_server_version::v1 as get_server_version, ServerKey, VerifyKey, +use ruma::{ + api::{ + client, + federation::{ + directory::get_public_rooms, + discovery::{ + get_server_keys, get_server_version::v1 as get_server_version, ServerKey, VerifyKey, + }, + transactions::send_transaction_message, }, - transactions::send_transaction_message, + OutgoingRequest, }, - OutgoingRequest, + EventId, }; use serde_json::json; use std::{ @@ -264,10 +267,29 @@ pub async fn get_public_rooms_route( put("/_matrix/federation/v1/send/<_>", data = "") )] pub fn send_transaction_message_route<'a>( - _db: State<'a, Database>, + db: State<'a, Database>, body: Ruma>, ) -> ConduitResult { dbg!(&*body); + for pdu in &body.pdus { + let mut value = serde_json::to_value(pdu).expect("all ruma pdus are json values"); + let event_id = EventId::try_from(&*format!( + "${}", + ruma::signatures::reference_hash(&value).expect("ruma can calculate reference hashes") + )) + .expect("ruma's reference hashes are valid event ids"); + + value + .as_object_mut() + .expect("ruma pdus are json objects") + .insert("event_id".to_owned(), event_id.to_string().into()); + + db.rooms.append_pdu( + serde_json::from_value(value).expect("all ruma pdus are conduit pdus"), + &db.globals, + &db.account_data, + )?; + } Ok(send_transaction_message::v1::Response { pdus: BTreeMap::new(), } From af53485d70d474468b02d10badd6959fa05f1068 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20K=C3=B6sters?= Date: Sat, 12 Sep 2020 23:38:52 +0200 Subject: [PATCH 21/51] fix: avoid pdus without event ids --- src/client_server/membership.rs | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/src/client_server/membership.rs b/src/client_server/membership.rs index ea2271bb3..20cf315b2 100644 --- a/src/client_server/membership.rs +++ b/src/client_server/membership.rs @@ -502,8 +502,10 @@ async fn join_room_by_id_helper( .ok_or_else(|| Error::BadServerResponse("PDU is not an object."))? .insert("event_id".to_owned(), event_id.to_string().into()); + dbg!(&value); + serde_json::from_value::(value) - .map(|ev| (event_id, Arc::new(ev))) + .map(|ev| (dbg!(&ev).event_id().clone(), Arc::new(ev))) .map_err(|e| { warn!("{}", e); Error::BadServerResponse("Invalid PDU bytes in send_join response.") @@ -520,19 +522,14 @@ async fn join_room_by_id_helper( // These events are not guaranteed to be sorted but they are resolved according to spec // we auth them anyways to weed out faulty/malicious server. The following is basically the // full state resolution algorithm. + let event_ids = event_map.keys().cloned().collect::>(); + let sorted_control_events = state_res::StateResolution::reverse_topological_power_sort( &room_id, &control_events, &mut event_map, &db.rooms, - &send_join_response - .room_state - .auth_chain - .iter() - .filter_map(|pdu| { - Some(StateEvent::Full(pdu.deserialize().ok()?).event_id().clone()) - }) - .collect::>(), + &event_ids, ); // Auth check each event against the "partial" state created by the preceding events From 1f292c09f2e8a0679467ec5f3b699918d60cea64 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20K=C3=B6sters?= Date: Sun, 13 Sep 2020 22:24:36 +0200 Subject: [PATCH 22/51] improvement: better federation joins --- Cargo.lock | 30 ++++++------ src/client_server/membership.rs | 81 +++++++++++++++++++++---------- src/database/rooms.rs | 86 +++++++++++++++++++++------------ src/server_server.rs | 18 ++++--- 4 files changed, 136 insertions(+), 79 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 646cdcc66..6ffd3473c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1634,7 +1634,7 @@ dependencies = [ [[package]] name = "ruma" version = "0.0.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#63341000fbabce9b230b6665ce65c617944408fa" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#8d763abaecb13f4799a31ecf1e0da77d2bc956a6" dependencies = [ "ruma-api", "ruma-appservice-api", @@ -1650,7 +1650,7 @@ dependencies = [ [[package]] name = "ruma-api" version = "0.17.0-alpha.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#63341000fbabce9b230b6665ce65c617944408fa" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#8d763abaecb13f4799a31ecf1e0da77d2bc956a6" dependencies = [ "http", "percent-encoding", @@ -1665,7 +1665,7 @@ dependencies = [ [[package]] name = "ruma-api-macros" version = "0.17.0-alpha.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#63341000fbabce9b230b6665ce65c617944408fa" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#8d763abaecb13f4799a31ecf1e0da77d2bc956a6" dependencies = [ "proc-macro-crate", "proc-macro2", @@ -1676,7 +1676,7 @@ dependencies = [ [[package]] name = "ruma-appservice-api" version = "0.2.0-alpha.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#63341000fbabce9b230b6665ce65c617944408fa" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#8d763abaecb13f4799a31ecf1e0da77d2bc956a6" dependencies = [ "ruma-api", "ruma-common", @@ -1689,7 +1689,7 @@ dependencies = [ [[package]] name = "ruma-client-api" version = "0.10.0-alpha.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#63341000fbabce9b230b6665ce65c617944408fa" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#8d763abaecb13f4799a31ecf1e0da77d2bc956a6" dependencies = [ "assign", "http", @@ -1708,7 +1708,7 @@ dependencies = [ [[package]] name = "ruma-common" version = "0.2.0" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#63341000fbabce9b230b6665ce65c617944408fa" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#8d763abaecb13f4799a31ecf1e0da77d2bc956a6" dependencies = [ "js_int", "ruma-api", @@ -1722,7 +1722,7 @@ dependencies = [ [[package]] name = "ruma-events" version = "0.22.0-alpha.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#63341000fbabce9b230b6665ce65c617944408fa" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#8d763abaecb13f4799a31ecf1e0da77d2bc956a6" dependencies = [ "js_int", "ruma-common", @@ -1737,7 +1737,7 @@ dependencies = [ [[package]] name = "ruma-events-macros" version = "0.22.0-alpha.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#63341000fbabce9b230b6665ce65c617944408fa" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#8d763abaecb13f4799a31ecf1e0da77d2bc956a6" dependencies = [ "proc-macro-crate", "proc-macro2", @@ -1748,7 +1748,7 @@ dependencies = [ [[package]] name = "ruma-federation-api" version = "0.0.3" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#63341000fbabce9b230b6665ce65c617944408fa" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#8d763abaecb13f4799a31ecf1e0da77d2bc956a6" dependencies = [ "js_int", "ruma-api", @@ -1763,7 +1763,7 @@ dependencies = [ [[package]] name = "ruma-identifiers" version = "0.17.4" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#63341000fbabce9b230b6665ce65c617944408fa" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#8d763abaecb13f4799a31ecf1e0da77d2bc956a6" dependencies = [ "rand", "ruma-identifiers-macros", @@ -1775,7 +1775,7 @@ dependencies = [ [[package]] name = "ruma-identifiers-macros" version = "0.17.4" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#63341000fbabce9b230b6665ce65c617944408fa" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#8d763abaecb13f4799a31ecf1e0da77d2bc956a6" dependencies = [ "proc-macro2", "quote", @@ -1786,7 +1786,7 @@ dependencies = [ [[package]] name = "ruma-identifiers-validation" version = "0.1.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#63341000fbabce9b230b6665ce65c617944408fa" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#8d763abaecb13f4799a31ecf1e0da77d2bc956a6" dependencies = [ "serde", "strum", @@ -1795,7 +1795,7 @@ dependencies = [ [[package]] name = "ruma-serde" version = "0.2.3" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#63341000fbabce9b230b6665ce65c617944408fa" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#8d763abaecb13f4799a31ecf1e0da77d2bc956a6" dependencies = [ "form_urlencoded", "itoa", @@ -1807,7 +1807,7 @@ dependencies = [ [[package]] name = "ruma-signatures" version = "0.6.0-dev.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#63341000fbabce9b230b6665ce65c617944408fa" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#8d763abaecb13f4799a31ecf1e0da77d2bc956a6" dependencies = [ "base64", "ring", @@ -2072,7 +2072,7 @@ checksum = "7345c971d1ef21ffdbd103a75990a15eb03604fc8b8852ca8cb418ee1a099028" [[package]] name = "state-res" version = "0.1.0" -source = "git+https://github.com/timokoesters/state-res?branch=spec-comp#0081081604b051d412a2365b68357e064c33320c" +source = "git+https://github.com/timokoesters/state-res?branch=spec-comp#a9186476b748c901fbf4356414247a0b3ac01b5f" dependencies = [ "itertools", "js_int", diff --git a/src/client_server/membership.rs b/src/client_server/membership.rs index 20cf315b2..9285648cb 100644 --- a/src/client_server/membership.rs +++ b/src/client_server/membership.rs @@ -2,7 +2,7 @@ use super::State; use crate::{ client_server, pdu::{PduBuilder, PduEvent}, - server_server, utils, ConduitResult, Database, Error, Ruma, + server_server, utils, ConduitResult, Database, Error, Result, Ruma, }; use log::warn; use ruma::{ @@ -17,11 +17,14 @@ use ruma::{ }, federation, }, + events::pdu::Pdu, events::{room::member, EventType}, EventId, Raw, RoomId, RoomVersionId, UserId, }; use state_res::StateEvent; -use std::{collections::BTreeMap, convert::TryFrom, sync::Arc}; +use std::{ + collections::BTreeMap, collections::HashMap, collections::HashSet, convert::TryFrom, sync::Arc, +}; #[cfg(feature = "conduit_bin")] use rocket::{get, post}; @@ -482,36 +485,49 @@ async fn join_room_by_id_helper( ) .await?; - let mut event_map = send_join_response + let add_event_id = |pdu: &Raw| { + let mut value = serde_json::from_str(pdu.json().get()) + .expect("converting raw jsons to values always works"); + let event_id = EventId::try_from(&*format!( + "${}", + ruma::signatures::reference_hash(&value) + .expect("ruma can calculate reference hashes") + )) + .expect("ruma's reference hashes are valid event ids"); + + value + .as_object_mut() + .ok_or_else(|| Error::BadServerResponse("PDU is not an object."))? + .insert("event_id".to_owned(), event_id.to_string().into()); + + Ok((event_id, value)) + }; + + let room_state = send_join_response.room_state.state.iter().map(add_event_id); + + let state_events = room_state + .clone() + .map(|pdu: Result<(EventId, serde_json::Value)>| Ok(pdu?.0)) + .collect::>>()?; + + let auth_chain = send_join_response .room_state - .state + .auth_chain .iter() - .chain(send_join_response.room_state.auth_chain.iter()) - .map(|pdu| { - let mut value = serde_json::from_str(pdu.json().get()) - .expect("converting raw jsons to values always works"); - let event_id = EventId::try_from(&*format!( - "${}", - ruma::signatures::reference_hash(&value) - .expect("ruma can calculate reference hashes") - )) - .expect("ruma's reference hashes are valid event ids"); - - value - .as_object_mut() - .ok_or_else(|| Error::BadServerResponse("PDU is not an object."))? - .insert("event_id".to_owned(), event_id.to_string().into()); - - dbg!(&value); + .map(add_event_id); + let mut event_map = room_state + .chain(auth_chain) + .map(|r| { + let (event_id, value) = r?; serde_json::from_value::(value) - .map(|ev| (dbg!(&ev).event_id().clone(), Arc::new(ev))) + .map(|ev| (event_id, Arc::new(ev))) .map_err(|e| { warn!("{}", e); Error::BadServerResponse("Invalid PDU bytes in send_join response.") }) }) - .collect::>, _>>()?; + .collect::>>>()?; let control_events = event_map .values() @@ -575,6 +591,8 @@ async fn join_room_by_id_helper( ) .expect("iterative auth check failed on resolved events"); + let mut state = HashMap::new(); + // filter the events that failed the auth check keeping the remaining events // sorted correctly for ev_id in sorted_event_ids @@ -587,9 +605,22 @@ async fn join_room_by_id_helper( .expect("Found event_id in sorted events that is not in resolved state"); // We do not rebuild the PDU in this case only insert to DB - db.rooms - .append_pdu(PduEvent::from(&**pdu), &db.globals, &db.account_data)?; + let pdu_id = + db.rooms + .append_pdu(&PduEvent::from(&**pdu), &db.globals, &db.account_data)?; + + if state_events.contains(ev_id) { + state.insert( + ( + pdu.kind(), + pdu.state_key().expect("State events have a state key"), + ), + pdu_id, + ); + } } + + db.rooms.force_state(room_id, state)?; } else { let event = member::MemberEventContent { membership: member::MembershipState::Join, diff --git a/src/database/rooms.rs b/src/database/rooms.rs index 87f4dcd47..b538c854a 100644 --- a/src/database/rooms.rs +++ b/src/database/rooms.rs @@ -220,6 +220,31 @@ impl Rooms { .is_some()) } + /// Returns the full room state. + pub fn force_state( + &self, + room_id: &RoomId, + state: HashMap<(EventType, String), Vec>, + ) -> Result<()> { + let state_hash = + self.calculate_hash(&state.values().map(|pdu_id| &**pdu_id).collect::>())?; + let mut prefix = state_hash.clone(); + prefix.push(0xff); + + for ((event_type, state_key), pdu_id) in state { + let mut state_id = prefix.clone(); + state_id.extend_from_slice(&event_type.as_str().as_bytes()); + state_id.push(0xff); + state_id.extend_from_slice(&state_key.as_bytes()); + self.stateid_pduid.insert(state_id, pdu_id)?; + } + + self.roomid_statehash + .insert(room_id.as_bytes(), &*state_hash)?; + + Ok(()) + } + /// Returns the full room state. pub fn room_state_full( &self, @@ -446,10 +471,10 @@ impl Rooms { /// Creates a new persisted data unit and adds it to a room. pub fn append_pdu( &self, - pdu: PduEvent, + pdu: &PduEvent, globals: &super::globals::Globals, account_data: &super::account_data::AccountData, - ) -> Result { + ) -> Result> { let mut pdu_json = serde_json::to_value(&pdu).expect("event is valid, we just created it"); ruma::signatures::hash_and_sign_event( globals.server_name().as_str(), @@ -473,10 +498,6 @@ impl Rooms { self.eventid_pduid .insert(pdu.event_id.as_bytes(), &*pdu_id)?; - if pdu.state_key.is_some() { - self.append_to_state(&pdu_id, &pdu)?; - } - match pdu.kind { EventType::RoomRedaction => { if let Some(redact_id) = &pdu.redacts { @@ -484,23 +505,22 @@ impl Rooms { } } EventType::RoomMember => { - if let Some(state_key) = pdu.state_key { + if let Some(state_key) = &pdu.state_key { // if the state_key fails - let target_user_id = UserId::try_from(state_key) + let target_user_id = UserId::try_from(state_key.clone()) .expect("This state_key was previously validated"); // Update our membership info, we do this here incase a user is invited // and immediately leaves we need the DB to record the invite event for auth self.update_membership( &pdu.room_id, &target_user_id, - serde_json::from_value::(pdu.content).map_err( - |_| { - Error::BadRequest( - ErrorKind::InvalidParam, - "Invalid redaction event content.", - ) - }, - )?, + serde_json::from_value::(pdu.content.clone()) + .map_err(|_| { + Error::BadRequest( + ErrorKind::InvalidParam, + "Invalid redaction event content.", + ) + })?, &pdu.sender, account_data, globals, @@ -528,7 +548,7 @@ impl Rooms { self.edus .private_read_set(&pdu.room_id, &pdu.sender, index, &globals)?; - Ok(pdu.event_id) + Ok(pdu_id) } /// Generates a new StateHash and associates it with the incoming event. @@ -789,7 +809,13 @@ impl Rooms { )) .expect("ruma's reference hashes are valid event ids"); - self.append_pdu(pdu, globals, account_data) + let pdu_id = self.append_pdu(&pdu, globals, account_data)?; + + if pdu.state_key.is_some() { + self.append_to_state(&pdu_id, &pdu)?; + } + + Ok(pdu.event_id) } /// Returns an iterator over all PDUs in a room. @@ -953,19 +979,17 @@ impl Rooms { self.roomuseroncejoinedids.insert(&userroom_id, &[])?; // Check if the room has a predecessor - if let Some(predecessor) = serde_json::from_value::< - Raw, - >( - self.room_state_get(&room_id, &EventType::RoomCreate, "")? - .ok_or_else(|| { - Error::bad_database("Found room without m.room.create event.") - })? - .content, - ) - .expect("Raw::from_value always works") - .deserialize() - .map_err(|_| Error::bad_database("Invalid room event in database."))? - .predecessor + if let Some(predecessor) = self + .room_state_get(&room_id, &EventType::RoomCreate, "")? + .and_then(|create| { + serde_json::from_value::< + Raw, + >(create.content) + .expect("Raw::from_value always works") + .deserialize() + .ok() + }) + .and_then(|content| content.predecessor) { // Copy user settings from predecessor to the current room: // - Push rules diff --git a/src/server_server.rs b/src/server_server.rs index d7d0e23fe..6634d5a22 100644 --- a/src/server_server.rs +++ b/src/server_server.rs @@ -1,4 +1,4 @@ -use crate::{client_server, ConduitResult, Database, Error, Result, Ruma}; +use crate::{client_server, ConduitResult, Database, Error, PduEvent, Result, Ruma}; use http::header::{HeaderValue, AUTHORIZATION}; use rocket::{get, post, put, response::content::Json, State}; use ruma::{ @@ -270,9 +270,11 @@ pub fn send_transaction_message_route<'a>( db: State<'a, Database>, body: Ruma>, ) -> ConduitResult { - dbg!(&*body); + //dbg!(&*body); for pdu in &body.pdus { - let mut value = serde_json::to_value(pdu).expect("all ruma pdus are json values"); + let mut value = serde_json::from_str(pdu.json().get()) + .expect("converting raw jsons to values always works"); + let event_id = EventId::try_from(&*format!( "${}", ruma::signatures::reference_hash(&value).expect("ruma can calculate reference hashes") @@ -284,11 +286,11 @@ pub fn send_transaction_message_route<'a>( .expect("ruma pdus are json objects") .insert("event_id".to_owned(), event_id.to_string().into()); - db.rooms.append_pdu( - serde_json::from_value(value).expect("all ruma pdus are conduit pdus"), - &db.globals, - &db.account_data, - )?; + let pdu = + serde_json::from_value::(value).expect("all ruma pdus are conduit pdus"); + if db.rooms.exists(&pdu.room_id)? { + db.rooms.append_pdu(&pdu, &db.globals, &db.account_data)?; + } } Ok(send_transaction_message::v1::Response { pdus: BTreeMap::new(), From c5313b3e8f8d024c540d0428a6c75828d82f95c9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20K=C3=B6sters?= Date: Mon, 14 Sep 2020 11:00:31 +0200 Subject: [PATCH 23/51] improvement: try out multiple servers when joining remote rooms --- Cargo.lock | 35 +++++++++++----------- src/client_server/alias.rs | 4 +-- src/client_server/directory.rs | 12 ++++---- src/client_server/membership.rs | 52 +++++++++++++++++++++------------ src/server_server.rs | 10 +++---- 5 files changed, 65 insertions(+), 48 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 6ffd3473c..28a4395e1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -157,7 +157,7 @@ dependencies = [ "addr2line", "cfg-if", "libc", - "miniz_oxide 0.4.1", + "miniz_oxide 0.4.2", "object", "rustc-demangle", ] @@ -1054,11 +1054,12 @@ dependencies = [ [[package]] name = "miniz_oxide" -version = "0.4.1" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d7559a8a40d0f97e1edea3220f698f78b1c5ab67532e49f68fde3910323b722" +checksum = "c60c0dfe32c10b43a144bad8fc83538c52f58302c92300ea7ec7bf7b38d5a7b9" dependencies = [ "adler", + "autocfg", ] [[package]] @@ -1634,7 +1635,7 @@ dependencies = [ [[package]] name = "ruma" version = "0.0.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#8d763abaecb13f4799a31ecf1e0da77d2bc956a6" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#ee66e30cbd58aecbbfde1d7008d7d6457deef87b" dependencies = [ "ruma-api", "ruma-appservice-api", @@ -1650,7 +1651,7 @@ dependencies = [ [[package]] name = "ruma-api" version = "0.17.0-alpha.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#8d763abaecb13f4799a31ecf1e0da77d2bc956a6" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#ee66e30cbd58aecbbfde1d7008d7d6457deef87b" dependencies = [ "http", "percent-encoding", @@ -1665,7 +1666,7 @@ dependencies = [ [[package]] name = "ruma-api-macros" version = "0.17.0-alpha.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#8d763abaecb13f4799a31ecf1e0da77d2bc956a6" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#ee66e30cbd58aecbbfde1d7008d7d6457deef87b" dependencies = [ "proc-macro-crate", "proc-macro2", @@ -1676,7 +1677,7 @@ dependencies = [ [[package]] name = "ruma-appservice-api" version = "0.2.0-alpha.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#8d763abaecb13f4799a31ecf1e0da77d2bc956a6" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#ee66e30cbd58aecbbfde1d7008d7d6457deef87b" dependencies = [ "ruma-api", "ruma-common", @@ -1689,7 +1690,7 @@ dependencies = [ [[package]] name = "ruma-client-api" version = "0.10.0-alpha.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#8d763abaecb13f4799a31ecf1e0da77d2bc956a6" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#ee66e30cbd58aecbbfde1d7008d7d6457deef87b" dependencies = [ "assign", "http", @@ -1708,7 +1709,7 @@ dependencies = [ [[package]] name = "ruma-common" version = "0.2.0" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#8d763abaecb13f4799a31ecf1e0da77d2bc956a6" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#ee66e30cbd58aecbbfde1d7008d7d6457deef87b" dependencies = [ "js_int", "ruma-api", @@ -1722,7 +1723,7 @@ dependencies = [ [[package]] name = "ruma-events" version = "0.22.0-alpha.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#8d763abaecb13f4799a31ecf1e0da77d2bc956a6" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#ee66e30cbd58aecbbfde1d7008d7d6457deef87b" dependencies = [ "js_int", "ruma-common", @@ -1737,7 +1738,7 @@ dependencies = [ [[package]] name = "ruma-events-macros" version = "0.22.0-alpha.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#8d763abaecb13f4799a31ecf1e0da77d2bc956a6" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#ee66e30cbd58aecbbfde1d7008d7d6457deef87b" dependencies = [ "proc-macro-crate", "proc-macro2", @@ -1748,7 +1749,7 @@ dependencies = [ [[package]] name = "ruma-federation-api" version = "0.0.3" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#8d763abaecb13f4799a31ecf1e0da77d2bc956a6" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#ee66e30cbd58aecbbfde1d7008d7d6457deef87b" dependencies = [ "js_int", "ruma-api", @@ -1763,7 +1764,7 @@ dependencies = [ [[package]] name = "ruma-identifiers" version = "0.17.4" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#8d763abaecb13f4799a31ecf1e0da77d2bc956a6" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#ee66e30cbd58aecbbfde1d7008d7d6457deef87b" dependencies = [ "rand", "ruma-identifiers-macros", @@ -1775,7 +1776,7 @@ dependencies = [ [[package]] name = "ruma-identifiers-macros" version = "0.17.4" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#8d763abaecb13f4799a31ecf1e0da77d2bc956a6" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#ee66e30cbd58aecbbfde1d7008d7d6457deef87b" dependencies = [ "proc-macro2", "quote", @@ -1786,7 +1787,7 @@ dependencies = [ [[package]] name = "ruma-identifiers-validation" version = "0.1.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#8d763abaecb13f4799a31ecf1e0da77d2bc956a6" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#ee66e30cbd58aecbbfde1d7008d7d6457deef87b" dependencies = [ "serde", "strum", @@ -1795,7 +1796,7 @@ dependencies = [ [[package]] name = "ruma-serde" version = "0.2.3" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#8d763abaecb13f4799a31ecf1e0da77d2bc956a6" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#ee66e30cbd58aecbbfde1d7008d7d6457deef87b" dependencies = [ "form_urlencoded", "itoa", @@ -1807,7 +1808,7 @@ dependencies = [ [[package]] name = "ruma-signatures" version = "0.6.0-dev.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#8d763abaecb13f4799a31ecf1e0da77d2bc956a6" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#ee66e30cbd58aecbbfde1d7008d7d6457deef87b" dependencies = [ "base64", "ring", diff --git a/src/client_server/alias.rs b/src/client_server/alias.rs index bfdaecac4..0ec43f576 100644 --- a/src/client_server/alias.rs +++ b/src/client_server/alias.rs @@ -63,7 +63,7 @@ pub async fn get_alias_helper( if room_alias.server_name() != db.globals.server_name() { let response = server_server::send_request( &db, - room_alias.server_name().to_string(), + room_alias.server_name(), federation::query::get_room_information::v1::Request { room_alias }, ) .await?; @@ -79,5 +79,5 @@ pub async fn get_alias_helper( "Room with alias not found.", ))?; - Ok(get_alias::Response::new(room_id, vec![db.globals.server_name().to_string()]).into()) + Ok(get_alias::Response::new(room_id, vec![db.globals.server_name().to_owned()]).into()) } diff --git a/src/client_server/directory.rs b/src/client_server/directory.rs index 2764d2c52..a68d8dd13 100644 --- a/src/client_server/directory.rs +++ b/src/client_server/directory.rs @@ -19,7 +19,7 @@ use ruma::{ room::{avatar, canonical_alias, guest_access, history_visibility, name, topic}, EventType, }, - Raw, + Raw, ServerName, }; #[cfg(feature = "conduit_bin")] @@ -65,9 +65,9 @@ pub async fn get_public_rooms_route( ) -> ConduitResult { let response = get_public_rooms_filtered_helper( &db, - body.body.server.as_deref(), - body.body.limit, - body.body.since.as_deref(), + body.server.as_deref(), + body.limit, + body.since.as_deref(), None, // This is not used None, // This is not used ) @@ -119,7 +119,7 @@ pub async fn get_room_visibility_route( pub async fn get_public_rooms_filtered_helper( db: &Database, - server: Option<&str>, + server: Option<&ServerName>, limit: Option, since: Option<&str>, _filter: Option, @@ -131,7 +131,7 @@ pub async fn get_public_rooms_filtered_helper( { let response = server_server::send_request( &db, - other_server.to_owned(), + other_server, federation::directory::get_public_rooms::v1::Request { limit, since: since.as_deref(), diff --git a/src/client_server/membership.rs b/src/client_server/membership.rs index 9285648cb..8d1940243 100644 --- a/src/client_server/membership.rs +++ b/src/client_server/membership.rs @@ -19,7 +19,7 @@ use ruma::{ }, events::pdu::Pdu, events::{room::member, EventType}, - EventId, Raw, RoomId, RoomVersionId, UserId, + EventId, Raw, RoomId, RoomVersionId, ServerName, UserId, }; use state_res::StateEvent; use std::{ @@ -41,6 +41,7 @@ pub async fn join_room_by_id_route( &db, body.sender_id.as_ref(), &body.room_id, + &[body.room_id.server_name().to_owned()], body.third_party_signed.as_ref(), ) .await @@ -54,13 +55,12 @@ pub async fn join_room_by_id_or_alias_route( db: State<'_, Database>, body: Ruma>, ) -> ConduitResult { - let room_id = match RoomId::try_from(body.room_id_or_alias.clone()) { - Ok(room_id) => room_id, + let (servers, room_id) = match RoomId::try_from(body.room_id_or_alias.clone()) { + Ok(room_id) => (vec![room_id.server_name().to_owned()], room_id), Err(room_alias) => { - client_server::get_alias_helper(&db, &room_alias) - .await? - .0 - .room_id + let response = client_server::get_alias_helper(&db, &room_alias).await?; + + (response.0.servers, response.0.room_id) } }; @@ -69,6 +69,7 @@ pub async fn join_room_by_id_or_alias_route( &db, body.sender_id.as_ref(), &room_id, + &servers, body.third_party_signed.as_ref(), ) .await? @@ -415,22 +416,37 @@ async fn join_room_by_id_helper( db: &Database, sender_id: Option<&UserId>, room_id: &RoomId, + servers: &[Box], _third_party_signed: Option<&IncomingThirdPartySigned>, ) -> ConduitResult { let sender_id = sender_id.expect("user is authenticated"); // Ask a remote server if we don't have this room if !db.rooms.exists(&room_id)? && room_id.server_name() != db.globals.server_name() { - let make_join_response = server_server::send_request( - &db, - room_id.server_name().to_string(), - federation::membership::create_join_event_template::v1::Request { - room_id, - user_id: sender_id, - ver: &[RoomVersionId::Version5, RoomVersionId::Version6], - }, - ) - .await?; + let mut make_join_response_and_server = Err(Error::BadServerResponse( + "No server available to assist in joining.", + )); + + for remote_server in servers { + let make_join_response = server_server::send_request( + &db, + remote_server, + federation::membership::create_join_event_template::v1::Request { + room_id, + user_id: sender_id, + ver: &[RoomVersionId::Version5, RoomVersionId::Version6], + }, + ) + .await; + + make_join_response_and_server = make_join_response.map(|r| (r, remote_server)); + + if make_join_response_and_server.is_ok() { + break; + } + } + + let (make_join_response, remote_server) = make_join_response_and_server?; let mut join_event_stub_value = serde_json::from_str::(make_join_response.event.json().get()) @@ -475,7 +491,7 @@ async fn join_room_by_id_helper( let send_join_response = server_server::send_request( &db, - room_id.server_name().to_string(), + remote_server, federation::membership::create_join_event::v2::Request { room_id, event_id: &event_id, diff --git a/src/server_server.rs b/src/server_server.rs index 6634d5a22..fc1da001d 100644 --- a/src/server_server.rs +++ b/src/server_server.rs @@ -13,7 +13,7 @@ use ruma::{ }, OutgoingRequest, }, - EventId, + EventId, ServerName, }; use serde_json::json; use std::{ @@ -44,16 +44,16 @@ pub async fn request_well_known(db: &crate::Database, destination: &str) -> Opti pub async fn send_request( db: &crate::Database, - destination: String, + destination: &ServerName, request: T, ) -> Result where T: Debug, { let actual_destination = "https://".to_owned() - + &request_well_known(db, &destination) + + &request_well_known(db, &destination.as_str()) .await - .unwrap_or(destination.clone() + ":8448"); + .unwrap_or(destination.as_str().to_owned() + ":8448"); let mut http_request = request .try_into_http_request(&actual_destination, Some("")) @@ -82,7 +82,7 @@ where "origin".to_owned(), db.globals.server_name().as_str().into(), ); - request_map.insert("destination".to_owned(), destination.into()); + request_map.insert("destination".to_owned(), destination.as_str().into()); let mut request_json = request_map.into(); ruma::signatures::sign_json( From 4e44fedbcd823876862315ac3590cc3d21a2825e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20K=C3=B6sters?= Date: Mon, 14 Sep 2020 11:42:16 +0200 Subject: [PATCH 24/51] fix: room list over federation --- Cargo.lock | 28 +++++------ src/client_server/directory.rs | 32 +++++-------- src/server_server.rs | 88 +++++++++++++++++++++++----------- 3 files changed, 84 insertions(+), 64 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 28a4395e1..865540c86 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1635,7 +1635,7 @@ dependencies = [ [[package]] name = "ruma" version = "0.0.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#ee66e30cbd58aecbbfde1d7008d7d6457deef87b" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#088382dbdc176e61fa5bde679ae38093865e7053" dependencies = [ "ruma-api", "ruma-appservice-api", @@ -1651,7 +1651,7 @@ dependencies = [ [[package]] name = "ruma-api" version = "0.17.0-alpha.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#ee66e30cbd58aecbbfde1d7008d7d6457deef87b" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#088382dbdc176e61fa5bde679ae38093865e7053" dependencies = [ "http", "percent-encoding", @@ -1666,7 +1666,7 @@ dependencies = [ [[package]] name = "ruma-api-macros" version = "0.17.0-alpha.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#ee66e30cbd58aecbbfde1d7008d7d6457deef87b" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#088382dbdc176e61fa5bde679ae38093865e7053" dependencies = [ "proc-macro-crate", "proc-macro2", @@ -1677,7 +1677,7 @@ dependencies = [ [[package]] name = "ruma-appservice-api" version = "0.2.0-alpha.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#ee66e30cbd58aecbbfde1d7008d7d6457deef87b" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#088382dbdc176e61fa5bde679ae38093865e7053" dependencies = [ "ruma-api", "ruma-common", @@ -1690,7 +1690,7 @@ dependencies = [ [[package]] name = "ruma-client-api" version = "0.10.0-alpha.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#ee66e30cbd58aecbbfde1d7008d7d6457deef87b" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#088382dbdc176e61fa5bde679ae38093865e7053" dependencies = [ "assign", "http", @@ -1709,7 +1709,7 @@ dependencies = [ [[package]] name = "ruma-common" version = "0.2.0" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#ee66e30cbd58aecbbfde1d7008d7d6457deef87b" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#088382dbdc176e61fa5bde679ae38093865e7053" dependencies = [ "js_int", "ruma-api", @@ -1723,7 +1723,7 @@ dependencies = [ [[package]] name = "ruma-events" version = "0.22.0-alpha.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#ee66e30cbd58aecbbfde1d7008d7d6457deef87b" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#088382dbdc176e61fa5bde679ae38093865e7053" dependencies = [ "js_int", "ruma-common", @@ -1738,7 +1738,7 @@ dependencies = [ [[package]] name = "ruma-events-macros" version = "0.22.0-alpha.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#ee66e30cbd58aecbbfde1d7008d7d6457deef87b" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#088382dbdc176e61fa5bde679ae38093865e7053" dependencies = [ "proc-macro-crate", "proc-macro2", @@ -1749,7 +1749,7 @@ dependencies = [ [[package]] name = "ruma-federation-api" version = "0.0.3" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#ee66e30cbd58aecbbfde1d7008d7d6457deef87b" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#088382dbdc176e61fa5bde679ae38093865e7053" dependencies = [ "js_int", "ruma-api", @@ -1764,7 +1764,7 @@ dependencies = [ [[package]] name = "ruma-identifiers" version = "0.17.4" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#ee66e30cbd58aecbbfde1d7008d7d6457deef87b" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#088382dbdc176e61fa5bde679ae38093865e7053" dependencies = [ "rand", "ruma-identifiers-macros", @@ -1776,7 +1776,7 @@ dependencies = [ [[package]] name = "ruma-identifiers-macros" version = "0.17.4" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#ee66e30cbd58aecbbfde1d7008d7d6457deef87b" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#088382dbdc176e61fa5bde679ae38093865e7053" dependencies = [ "proc-macro2", "quote", @@ -1787,7 +1787,7 @@ dependencies = [ [[package]] name = "ruma-identifiers-validation" version = "0.1.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#ee66e30cbd58aecbbfde1d7008d7d6457deef87b" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#088382dbdc176e61fa5bde679ae38093865e7053" dependencies = [ "serde", "strum", @@ -1796,7 +1796,7 @@ dependencies = [ [[package]] name = "ruma-serde" version = "0.2.3" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#ee66e30cbd58aecbbfde1d7008d7d6457deef87b" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#088382dbdc176e61fa5bde679ae38093865e7053" dependencies = [ "form_urlencoded", "itoa", @@ -1808,7 +1808,7 @@ dependencies = [ [[package]] name = "ruma-signatures" version = "0.6.0-dev.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#ee66e30cbd58aecbbfde1d7008d7d6457deef87b" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#088382dbdc176e61fa5bde679ae38093865e7053" dependencies = [ "base64", "ring", diff --git a/src/client_server/directory.rs b/src/client_server/directory.rs index a68d8dd13..f30825d13 100644 --- a/src/client_server/directory.rs +++ b/src/client_server/directory.rs @@ -14,6 +14,7 @@ use ruma::{ }, federation, }, + directory::RoomNetwork, directory::{IncomingFilter, IncomingRoomNetwork, PublicRoomsChunk}, events::{ room::{avatar, canonical_alias, guest_access, history_visibility, name, topic}, @@ -33,24 +34,13 @@ pub async fn get_public_rooms_filtered_route( db: State<'_, Database>, body: Ruma>, ) -> ConduitResult { - let Ruma { - body: - get_public_rooms_filtered::IncomingRequest { - limit, - server, - since, - filter, - room_network, - }, - .. - } = body; get_public_rooms_filtered_helper( &db, - server.as_deref(), - limit, - since.as_deref(), - filter, // This is not used yet - Some(room_network), // This is not used + body.server.as_deref(), + body.limit, + body.since.as_deref(), + &body.filter, + &body.room_network, ) .await } @@ -68,8 +58,8 @@ pub async fn get_public_rooms_route( body.server.as_deref(), body.limit, body.since.as_deref(), - None, // This is not used - None, // This is not used + &IncomingFilter::default(), + &IncomingRoomNetwork::Matrix, ) .await? .0; @@ -122,8 +112,8 @@ pub async fn get_public_rooms_filtered_helper( server: Option<&ServerName>, limit: Option, since: Option<&str>, - _filter: Option, - _network: Option, + _filter: &IncomingFilter, + _network: &IncomingRoomNetwork, ) -> ConduitResult { if let Some(other_server) = server .clone() @@ -135,7 +125,7 @@ pub async fn get_public_rooms_filtered_helper( federation::directory::get_public_rooms::v1::Request { limit, since: since.as_deref(), - room_network: ruma::directory::RoomNetwork::Matrix, + room_network: RoomNetwork::Matrix, }, ) .await?; diff --git a/src/server_server.rs b/src/server_server.rs index fc1da001d..6c53aed29 100644 --- a/src/server_server.rs +++ b/src/server_server.rs @@ -2,8 +2,8 @@ use crate::{client_server, ConduitResult, Database, Error, PduEvent, Result, Rum use http::header::{HeaderValue, AUTHORIZATION}; use rocket::{get, post, put, response::content::Json, State}; use ruma::{ + api::federation::directory::get_public_rooms_filtered, api::{ - client, federation::{ directory::get_public_rooms, discovery::{ @@ -13,6 +13,7 @@ use ruma::{ }, OutgoingRequest, }, + directory::{IncomingFilter, IncomingRoomNetwork}, EventId, ServerName, }; use serde_json::json; @@ -209,38 +210,24 @@ pub fn get_server_keys_deprecated(db: State<'_, Database>) -> Json { feature = "conduit_bin", post("/_matrix/federation/v1/publicRooms", data = "") )] -pub async fn get_public_rooms_route( +pub async fn get_public_rooms_filtered_route( db: State<'_, Database>, - body: Ruma>, -) -> ConduitResult { - let Ruma { - body: - get_public_rooms::v1::IncomingRequest { - room_network: _room_network, // TODO - limit, - since, - }, - .. - } = body; - - let client::r0::directory::get_public_rooms_filtered::Response { - chunk, - prev_batch, - next_batch, - total_room_count_estimate, - } = client_server::get_public_rooms_filtered_helper( + body: Ruma>, +) -> ConduitResult { + let response = client_server::get_public_rooms_filtered_helper( &db, None, - limit, - since.as_deref(), - None, - Some(ruma::directory::IncomingRoomNetwork::Matrix), + body.limit, + body.since.as_deref(), + &body.filter, + &body.room_network, ) .await? .0; - Ok(get_public_rooms::v1::Response { - chunk: chunk + Ok(get_public_rooms_filtered::v1::Response { + chunk: response + .chunk .into_iter() .map(|c| { // Convert ruma::api::federation::directory::get_public_rooms::v1::PublicRoomsChunk @@ -255,9 +242,52 @@ pub async fn get_public_rooms_route( }) .filter_map(|r| r.ok()) .collect(), - prev_batch, - next_batch, - total_room_count_estimate, + prev_batch: response.prev_batch, + next_batch: response.next_batch, + total_room_count_estimate: response.total_room_count_estimate, + } + .into()) +} + +#[cfg_attr( + feature = "conduit_bin", + get("/_matrix/federation/v1/publicRooms", data = "") +)] +pub async fn get_public_rooms_route( + db: State<'_, Database>, + body: Ruma>, +) -> ConduitResult { + let response = client_server::get_public_rooms_filtered_helper( + &db, + None, + body.limit, + body.since.as_deref(), + &IncomingFilter::default(), + &IncomingRoomNetwork::Matrix, + ) + .await? + .0; + + Ok(get_public_rooms::v1::Response { + chunk: response + .chunk + .into_iter() + .map(|c| { + // Convert ruma::api::federation::directory::get_public_rooms::v1::PublicRoomsChunk + // to ruma::api::client::r0::directory::PublicRoomsChunk + Ok::<_, Error>( + serde_json::from_str( + &serde_json::to_string(&c) + .expect("PublicRoomsChunk::to_string always works"), + ) + .expect("federation and client-server PublicRoomsChunk are the same type"), + ) + }) + .filter_map(|r| r.ok()) + .collect(), + prev_batch: response.prev_batch, + next_batch: response.next_batch, + total_room_count_estimate: response.total_room_count_estimate, } .into()) } From aa5e9e607ecc739d0d991ea7221dadd0125f6d64 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20K=C3=B6sters?= Date: Mon, 14 Sep 2020 14:20:38 +0200 Subject: [PATCH 25/51] feat: download media and thumbnails over federation --- src/client_server/media.rs | 69 +++++++++++++++++++++++++++++++++++--- src/database/media.rs | 28 ++++++++++++++-- 2 files changed, 90 insertions(+), 7 deletions(-) diff --git a/src/client_server/media.rs b/src/client_server/media.rs index d07744727..8f3374352 100644 --- a/src/client_server/media.rs +++ b/src/client_server/media.rs @@ -1,5 +1,7 @@ use super::State; -use crate::{database::media::FileMeta, utils, ConduitResult, Database, Error, Ruma}; +use crate::{ + database::media::FileMeta, server_server, utils, ConduitResult, Database, Error, Ruma, +}; use ruma::api::client::{ error::ErrorKind, r0::media::{create_content, get_content, get_content_thumbnail, get_media_config}, @@ -35,7 +37,7 @@ pub fn create_content_route( utils::random_string(MXC_LENGTH) ); db.media - .create(mxc.clone(), &body.filename, &body.content_type, &body.file)?; + .create(mxc.clone(), &body.filename.as_deref(), &body.content_type, &body.file)?; Ok(create_content::Response { content_uri: mxc }.into()) } @@ -47,19 +49,25 @@ pub fn create_content_route( data = "" ) )] -pub fn get_content_route( +pub async fn get_content_route( db: State<'_, Database>, body: Ruma>, _server_name: String, _media_id: String, ) -> ConduitResult { + let mxc = format!( + "mxc://{}/{}", + db.globals.server_name(), + utils::random_string(MXC_LENGTH) + ); + if let Some(FileMeta { filename, content_type, file, }) = db .media - .get(format!("mxc://{}/{}", body.server_name, body.media_id))? + .get(&mxc)? { Ok(get_content::Response { file, @@ -67,6 +75,26 @@ pub fn get_content_route( content_disposition: filename.unwrap_or_default(), // TODO: Spec says this should be optional } .into()) + } else if body.allow_remote { + let get_content_response = server_server::send_request( + &db, + body.server_name.as_ref(), + get_content::Request { + allow_remote: false, + server_name: &body.server_name, + media_id: &body.media_id, + }, + ) + .await?; + + db.media.create( + mxc, + &Some(&get_content_response.content_disposition), + &get_content_response.content_type, + &get_content_response.file, + )?; + + Ok(get_content_response.into()) } else { Err(Error::BadRequest(ErrorKind::NotFound, "Media not found.")) } @@ -79,7 +107,7 @@ pub fn get_content_route( data = "" ) )] -pub fn get_content_thumbnail_route( +pub async fn get_content_thumbnail_route( db: State<'_, Database>, body: Ruma>, _server_name: String, @@ -97,6 +125,37 @@ pub fn get_content_thumbnail_route( .map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "Width is invalid."))?, )? { Ok(get_content_thumbnail::Response { file, content_type }.into()) + } else if body.allow_remote { + let get_thumbnail_response = server_server::send_request( + &db, + body.server_name.as_ref(), + get_content_thumbnail::Request { + allow_remote: false, + height: body.height, + width: body.width, + method: body.method, + server_name: &body.server_name, + media_id: &body.media_id, + }, + ) + .await?; + + let mxc = format!( + "mxc://{}/{}", + db.globals.server_name(), + utils::random_string(MXC_LENGTH) + ); + + db.media.upload_thumbnail( + mxc, + &None, + &get_thumbnail_response.content_type, + body.width.try_into().expect("all UInts are valid u32s"), + body.height.try_into().expect("all UInts are valid u32s"), + &get_thumbnail_response.file, + )?; + + Ok(get_thumbnail_response.into()) } else { Err(Error::BadRequest(ErrorKind::NotFound, "Media not found.")) } diff --git a/src/database/media.rs b/src/database/media.rs index 63fa11c64..869d5d807 100644 --- a/src/database/media.rs +++ b/src/database/media.rs @@ -16,7 +16,7 @@ impl Media { pub fn create( &self, mxc: String, - filename: &Option, + filename: &Option<&str>, content_type: &str, file: &[u8], ) -> Result<()> { @@ -34,8 +34,32 @@ impl Media { Ok(()) } + /// Uploads or replaces a file thumbnail. + pub fn upload_thumbnail( + &self, + mxc: String, + filename: &Option, + content_type: &str, + width: u32, + height: u32, + file: &[u8], + ) -> Result<()> { + let mut key = mxc.as_bytes().to_vec(); + key.push(0xff); + key.extend_from_slice(&width.to_be_bytes()); + key.extend_from_slice(&height.to_be_bytes()); + key.push(0xff); + key.extend_from_slice(filename.as_ref().map(|f| f.as_bytes()).unwrap_or_default()); + key.push(0xff); + key.extend_from_slice(content_type.as_bytes()); + + self.mediaid_file.insert(key, file)?; + + Ok(()) + } + /// Downloads a file. - pub fn get(&self, mxc: String) -> Result> { + pub fn get(&self, mxc: &str) -> Result> { let mut prefix = mxc.as_bytes().to_vec(); prefix.push(0xff); prefix.extend_from_slice(&0_u32.to_be_bytes()); // Width = 0 if it's not a thumbnail From d1099e9224f3b47d9de9135ab751edd9152dc3b0 Mon Sep 17 00:00:00 2001 From: Timo Date: Tue, 25 Aug 2020 11:49:51 +0200 Subject: [PATCH 26/51] Update dependencies --- Cargo.lock | 274 +------------------------------------ Cargo.toml | 57 +++++--- src/client_server/media.rs | 22 +-- 3 files changed, 56 insertions(+), 297 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 865540c86..bde0b0dd0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1,80 +1,11 @@ # This file is automatically @generated by Cargo. # It is not intended for manual editing. -[[package]] -name = "addr2line" -version = "0.13.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b6a2d3371669ab3ca9797670853d61402b03d0b4b9ebf33d677dfa720203072" -dependencies = [ - "gimli", -] - -[[package]] -name = "adler" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee2a4ec343196209d6594e19543ae87a39f96d5534d7174822a3ad825dd6ed7e" - [[package]] name = "adler32" version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "aae1277d39aeec15cb388266ecc24b11c80469deae6067e17a1a7aa9e5c1f234" -[[package]] -name = "aead" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fc95d1bdb8e6666b2b217308eeeb09f2d6728d104be3e31916cc74d15420331" -dependencies = [ - "generic-array", -] - -[[package]] -name = "aes" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7001367fde4c768a19d1029f0a8be5abd9308e1119846d5bd9ad26297b8faf5" -dependencies = [ - "aes-soft", - "aesni", - "block-cipher", -] - -[[package]] -name = "aes-gcm" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86f5007801316299f922a6198d1d09a0bae95786815d066d5880d13f7c45ead1" -dependencies = [ - "aead", - "aes", - "block-cipher", - "ghash", - "subtle", -] - -[[package]] -name = "aes-soft" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4925647ee64e5056cf231608957ce7c81e12d6d6e316b9ce1404778cc1d35fa7" -dependencies = [ - "block-cipher", - "byteorder", - "opaque-debug 0.2.3", -] - -[[package]] -name = "aesni" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d050d39b0b7688b3a3254394c3e30a9d66c41dcf9b05b0e2dbdc623f6505d264" -dependencies = [ - "block-cipher", - "opaque-debug 0.2.3", -] - [[package]] name = "ansi_term" version = "0.12.1" @@ -90,12 +21,6 @@ version = "0.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4d25d88fd6b8041580a654f9d0c581a047baee2b3efee13275f2fc392fc75034" -[[package]] -name = "array-init" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f30bbe2f5e3d117f55bd8c7a1f9191e4a5deba9f15f595bbea4f670c59c765db" - [[package]] name = "arrayref" version = "0.3.6" @@ -148,20 +73,6 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a" -[[package]] -name = "backtrace" -version = "0.3.50" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46254cf2fdcdf1badb5934448c1bcbe046a56537b3987d96c51a7afc5d03f293" -dependencies = [ - "addr2line", - "cfg-if", - "libc", - "miniz_oxide 0.4.2", - "object", - "rustc-demangle", -] - [[package]] name = "base-x" version = "0.2.6" @@ -197,24 +108,6 @@ dependencies = [ "constant_time_eq", ] -[[package]] -name = "block-buffer" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4152116fd6e9dadb291ae18fc1ec3575ed6d84c29642d97890f4b4a3417297e4" -dependencies = [ - "generic-array", -] - -[[package]] -name = "block-cipher" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa136449e765dc7faa244561ccae839c394048667929af599b5d931ebe7b7f10" -dependencies = [ - "generic-array", -] - [[package]] name = "bumpalo" version = "3.4.0" @@ -319,12 +212,7 @@ version = "0.14.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1373a16a4937bc34efec7b391f9c1500c30b8478a701a4f44c9165cc0475a6e0" dependencies = [ - "aes-gcm", - "base64", - "hkdf", "percent-encoding", - "rand", - "sha2", "time 0.2.19", "version_check", ] @@ -345,12 +233,6 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b3a71ab494c0b5b860bdc8407ae08978052417070c2ced38573a9157ad75b8ac" -[[package]] -name = "cpuid-bool" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8aebca1129a03dc6dc2b127edd729435bbc4a37e1d5f4d7513165089ceb02634" - [[package]] name = "crc32fast" version = "1.2.0" @@ -386,16 +268,6 @@ dependencies = [ "lazy_static", ] -[[package]] -name = "crypto-mac" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b584a330336237c1eecd3e94266efb216c56ed91225d634cb2991c5f3fd1aeab" -dependencies = [ - "generic-array", - "subtle", -] - [[package]] name = "deflate" version = "0.8.6" @@ -436,22 +308,12 @@ dependencies = [ "syn", ] -[[package]] -name = "digest" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3dd60d1080a57a05ab032377049e0591415d2b31afd7028356dbf3cc6dcb066" -dependencies = [ - "generic-array", -] - [[package]] name = "directories" -version = "2.0.2" +version = "3.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "551a778172a450d7fc12e629ca3b0428d00f6afa9a43da1b630d54604e97371c" +checksum = "f8fed639d60b58d0f53498ab13d26f621fd77569cc6edb031f4cc36a2ad9da0f" dependencies = [ - "cfg-if", "dirs-sys", ] @@ -654,16 +516,6 @@ dependencies = [ "byteorder", ] -[[package]] -name = "generic-array" -version = "0.14.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "501466ecc8a30d1d3b7fc9229b122b2ce8ed6e9d9223f1138d4babb253e51817" -dependencies = [ - "typenum", - "version_check", -] - [[package]] name = "getrandom" version = "0.1.15" @@ -675,15 +527,6 @@ dependencies = [ "wasi 0.9.0+wasi-snapshot-preview1", ] -[[package]] -name = "ghash" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6e27f0689a6e15944bdce7e45425efb87eaa8ab0c6e87f11d0987a9133e2531" -dependencies = [ - "polyval", -] - [[package]] name = "gif" version = "0.10.3" @@ -694,12 +537,6 @@ dependencies = [ "lzw", ] -[[package]] -name = "gimli" -version = "0.22.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aaf91faf136cb47367fa430cd46e37a788775e7fa104f8b4bcb3861dc389b724" - [[package]] name = "glob" version = "0.3.0" @@ -749,26 +586,6 @@ dependencies = [ "libc", ] -[[package]] -name = "hkdf" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fe1149865383e4526a43aee8495f9a325f0b806c63ce6427d06336a590abbbc9" -dependencies = [ - "digest", - "hmac", -] - -[[package]] -name = "hmac" -version = "0.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "126888268dcc288495a26bf004b38c5fdbb31682f992c84ceb046a1f0fe38840" -dependencies = [ - "crypto-mac", - "digest", -] - [[package]] name = "http" version = "0.2.1" @@ -1052,16 +869,6 @@ dependencies = [ "adler32", ] -[[package]] -name = "miniz_oxide" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c60c0dfe32c10b43a144bad8fc83538c52f58302c92300ea7ec7bf7b38d5a7b9" -dependencies = [ - "adler", - "autocfg", -] - [[package]] name = "mio" version = "0.6.22" @@ -1184,30 +991,12 @@ dependencies = [ "libc", ] -[[package]] -name = "object" -version = "0.20.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ab52be62400ca80aa00285d25253d7f7c437b7375c4de678f5405d3afe82ca5" - [[package]] name = "once_cell" version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "260e51e7efe62b592207e9e13a68e43692a7a279171d6ba57abd208bf23645ad" -[[package]] -name = "opaque-debug" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2839e79665f131bdb5782e51f2c6c9599c133c6098982a54c794358bf432529c" - -[[package]] -name = "opaque-debug" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5" - [[package]] name = "openssl" version = "0.10.30" @@ -1341,17 +1130,7 @@ dependencies = [ "bitflags", "crc32fast", "deflate", - "miniz_oxide 0.3.7", -] - -[[package]] -name = "polyval" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9a50142b55ab3ed0e9f68dfb3709f1d90d29da24e91033f28b96330643107dc" -dependencies = [ - "cfg-if", - "universal-hash", + "miniz_oxide", ] [[package]] @@ -1828,12 +1607,6 @@ dependencies = [ "crossbeam-utils", ] -[[package]] -name = "rustc-demangle" -version = "0.1.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c691c0e608126e00913e33f0ccf3727d5fc84573623b8d65b2df340b5201783" - [[package]] name = "rustc_version" version = "0.2.3" @@ -1975,19 +1748,6 @@ version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2579985fda508104f7587689507983eadd6a6e84dd35d6d115361f530916fa0d" -[[package]] -name = "sha2" -version = "0.9.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2933378ddfeda7ea26f48c555bdad8bb446bf8a3d17832dc83e380d444cfb8c1" -dependencies = [ - "block-buffer", - "cfg-if", - "cpuid-bool", - "digest", - "opaque-debug 0.3.0", -] - [[package]] name = "sharded-slab" version = "0.0.9" @@ -2015,12 +1775,10 @@ checksum = "c111b5bd5695e56cffe5129854aa230b39c93a305372fdbb2668ca2394eea9f8" [[package]] name = "sled" -version = "0.32.1" +version = "0.34.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e3dbbb8ee10611bd1d020767c27599ccbbf8365f7e0ed7e54429cc8b9433ad8" +checksum = "f72c064e63fbca3138ad07f3588c58093f1684f3a99f60dcfa6d46b87e60fde7" dependencies = [ - "array-init", - "backtrace", "crc32fast", "crossbeam-epoch", "crossbeam-utils", @@ -2156,12 +1914,6 @@ dependencies = [ "syn", ] -[[package]] -name = "subtle" -version = "2.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "343f3f510c2915908f155e94f17220b19ccfacf2a64a2a5d8004f2c3e311e7fd" - [[package]] name = "syn" version = "1.0.40" @@ -2436,12 +2188,6 @@ version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "59547bce71d9c38b83d9c0e92b6066c4253371f15005def0c30d9657f50c7642" -[[package]] -name = "typenum" -version = "1.12.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "373c8a200f9e67a0c95e62a4f52fbf80c23b4381c05a17845531982fa99e6b33" - [[package]] name = "unicase" version = "2.6.0" @@ -2481,16 +2227,6 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f7fe0bb3479651439c9112f72b6c505038574c9fbb575ed1bf3b797fa39dd564" -[[package]] -name = "universal-hash" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8326b2c654932e3e4f9196e69d08fdf7cfd718e1dc6f66b347e6024a0c961402" -dependencies = [ - "generic-array", - "subtle", -] - [[package]] name = "untrusted" version = "0.7.1" diff --git a/Cargo.toml b/Cargo.toml index 1b7a70079..60296a2ca 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -12,28 +12,49 @@ edition = "2018" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] +# Used to handle requests # TODO: This can become optional as soon as proper configs are supported -#rocket = { git = "https://github.com/SergioBenitez/Rocket.git", rev = "8d779caa22c63b15a6c3ceb75d8f6d4971b2eb67", features = ["tls"] } # Used to handle requests -rocket = { git = "https://github.com/timokoesters/Rocket.git", branch = "empty_parameters", features = ["tls"] } -#ruma = { git = "https://github.com/ruma/ruma", features = ["rand", "client-api", "federation-api", "unstable-pre-spec", "unstable-synapse-quirks"], rev = "aff914050eb297bd82b8aafb12158c88a9e480e1" } # Used for matrix spec type definitions and helpers -ruma = { git = "https://github.com/timokoesters/ruma", features = ["rand", "client-api", "federation-api", "unstable-exhaustive-types", "unstable-pre-spec", "unstable-synapse-quirks"], branch = "timo-fed-fixes" } # Used for matrix spec type definitions and helpers +#rocket = { git = "https://github.com/SergioBenitez/Rocket.git", rev = "8d779caa22c63b15a6c3ceb75d8f6d4971b2eb67", default-features = false, features = ["tls"] } # Used to handle requests +rocket = { git = "https://github.com/timokoesters/Rocket.git", branch = "empty_parameters", default-features = false, features = ["tls"] } + +# Used for matrix spec type definitions and helpers +#ruma = { git = "https://github.com/ruma/ruma", features = ["rand", "client-api", "federation-api", "unstable-pre-spec", "unstable-synapse-quirks"], rev = "aff914050eb297bd82b8aafb12158c88a9e480e1" } +ruma = { git = "https://github.com/timokoesters/ruma", features = ["rand", "client-api", "federation-api", "unstable-exhaustive-types", "unstable-pre-spec", "unstable-synapse-quirks"], branch = "timo-fed-fixes" } #ruma = { path = "../ruma/ruma", features = ["unstable-exhaustive-types", "rand", "client-api", "federation-api", "unstable-pre-spec", "unstable-synapse-quirks"] } -tokio = "0.2.22" # Used for long polling -sled = "0.32.0" # Used for storing data permanently -log = "0.4.8" # Used for emitting log entries -http = "0.2.1" # Used for rocket<->ruma conversions -directories = "2.0.2" # Used to find data directory for default db path -js_int = "0.1.5" # Used for number types for ruma -serde_json = { version = "1.0.53", features = ["raw_value"] } # Used for ruma wrapper -serde = "1.0.111" # Used for pdu definition -rand = "0.7.3" # Used for secure identifiers -rust-argon2 = "0.8.2" # Used to hash passwords -reqwest = "0.10.6" # Used to send requests -thiserror = "1.0.19" # Used for conduit::Error type -image = { version = "0.23.4", default-features = false, features = ["jpeg", "png", "gif"] } # Used to generate thumbnails for images -base64 = "0.12.3" # Used to encode server public key + +# Used when doing state resolution state-res = { git = "https://github.com/timokoesters/state-res", branch = "spec-comp", features = ["unstable-pre-spec"] } #state-res = { path = "../state-res", features = ["unstable-pre-spec"] } + +# Used for long polling +tokio = "0.2.22" +# Used for storing data permanently +sled = "0.34.4" +# Used for emitting log entries +log = "0.4.11" +# Used for rocket<->ruma conversions +http = "0.2.1" +# Used to find data directory for default db path +directories = "3.0.1" +# Used for number types for ruma +js_int = "0.1.9" +# Used for ruma wrapper +serde_json = { version = "1.0.57", features = ["raw_value"] } +# Used for pdu definition +serde = "1.0.116" +# Used for secure identifiers +rand = "0.7.3" +# Used to hash passwords +rust-argon2 = "0.8.2" +# Used to send requests +reqwest = "0.10.8" +# Used for conduit::Error type +thiserror = "1.0.20" +# Used to generate thumbnails for images +image = { version = "0.23.9", default-features = false, features = ["jpeg", "png", "gif"] } +# Used to encode server public key +base64 = "0.12.3" +# Used when hashing the state ring = "0.16.15" [features] diff --git a/src/client_server/media.rs b/src/client_server/media.rs index 8f3374352..f897a6780 100644 --- a/src/client_server/media.rs +++ b/src/client_server/media.rs @@ -36,8 +36,12 @@ pub fn create_content_route( db.globals.server_name(), utils::random_string(MXC_LENGTH) ); - db.media - .create(mxc.clone(), &body.filename.as_deref(), &body.content_type, &body.file)?; + db.media.create( + mxc.clone(), + &body.filename.as_deref(), + &body.content_type, + &body.file, + )?; Ok(create_content::Response { content_uri: mxc }.into()) } @@ -55,19 +59,17 @@ pub async fn get_content_route( _server_name: String, _media_id: String, ) -> ConduitResult { - let mxc = format!( - "mxc://{}/{}", - db.globals.server_name(), - utils::random_string(MXC_LENGTH) - ); + let mxc = format!( + "mxc://{}/{}", + db.globals.server_name(), + utils::random_string(MXC_LENGTH) + ); if let Some(FileMeta { filename, content_type, file, - }) = db - .media - .get(&mxc)? + }) = db.media.get(&mxc)? { Ok(get_content::Response { file, From 9f05ef926af6f0c7c7d2886b99b87ccd6e218e7c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20K=C3=B6sters?= Date: Mon, 14 Sep 2020 16:23:15 +0200 Subject: [PATCH 27/51] fix: filter public room dir --- src/client_server/directory.rs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/client_server/directory.rs b/src/client_server/directory.rs index f30825d13..871a780f2 100644 --- a/src/client_server/directory.rs +++ b/src/client_server/directory.rs @@ -14,6 +14,7 @@ use ruma::{ }, federation, }, + directory::Filter, directory::RoomNetwork, directory::{IncomingFilter, IncomingRoomNetwork, PublicRoomsChunk}, events::{ @@ -112,7 +113,7 @@ pub async fn get_public_rooms_filtered_helper( server: Option<&ServerName>, limit: Option, since: Option<&str>, - _filter: &IncomingFilter, + filter: &IncomingFilter, _network: &IncomingRoomNetwork, ) -> ConduitResult { if let Some(other_server) = server @@ -122,9 +123,12 @@ pub async fn get_public_rooms_filtered_helper( let response = server_server::send_request( &db, other_server, - federation::directory::get_public_rooms::v1::Request { + federation::directory::get_public_rooms_filtered::v1::Request { limit, since: since.as_deref(), + filter: Filter { + generic_search_term: filter.generic_search_term.as_deref(), + }, room_network: RoomNetwork::Matrix, }, ) From f7816b11de0889fca761f55510a3313dcfa78a42 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20K=C3=B6sters?= Date: Mon, 14 Sep 2020 20:23:19 +0200 Subject: [PATCH 28/51] feat: send messages over federation --- Cargo.lock | 32 ++--- src/client_server/account.rs | 4 +- src/client_server/alias.rs | 2 +- src/client_server/directory.rs | 2 +- src/client_server/media.rs | 4 +- src/client_server/membership.rs | 206 +++++++++++++++++--------------- src/client_server/message.rs | 4 +- src/client_server/profile.rs | 8 +- src/client_server/redact.rs | 4 +- src/client_server/room.rs | 37 +++--- src/client_server/state.rs | 12 +- src/database.rs | 1 + src/database/rooms.rs | 177 ++++++++++++++++++--------- src/pdu.rs | 27 ++++- src/server_server.rs | 22 ++-- 15 files changed, 324 insertions(+), 218 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index bde0b0dd0..e0de2a7bc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1414,7 +1414,7 @@ dependencies = [ [[package]] name = "ruma" version = "0.0.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#088382dbdc176e61fa5bde679ae38093865e7053" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#6ccb3ecaf69167ba405379826a9d87a98f168df8" dependencies = [ "ruma-api", "ruma-appservice-api", @@ -1430,7 +1430,7 @@ dependencies = [ [[package]] name = "ruma-api" version = "0.17.0-alpha.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#088382dbdc176e61fa5bde679ae38093865e7053" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#6ccb3ecaf69167ba405379826a9d87a98f168df8" dependencies = [ "http", "percent-encoding", @@ -1445,7 +1445,7 @@ dependencies = [ [[package]] name = "ruma-api-macros" version = "0.17.0-alpha.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#088382dbdc176e61fa5bde679ae38093865e7053" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#6ccb3ecaf69167ba405379826a9d87a98f168df8" dependencies = [ "proc-macro-crate", "proc-macro2", @@ -1456,7 +1456,7 @@ dependencies = [ [[package]] name = "ruma-appservice-api" version = "0.2.0-alpha.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#088382dbdc176e61fa5bde679ae38093865e7053" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#6ccb3ecaf69167ba405379826a9d87a98f168df8" dependencies = [ "ruma-api", "ruma-common", @@ -1469,7 +1469,7 @@ dependencies = [ [[package]] name = "ruma-client-api" version = "0.10.0-alpha.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#088382dbdc176e61fa5bde679ae38093865e7053" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#6ccb3ecaf69167ba405379826a9d87a98f168df8" dependencies = [ "assign", "http", @@ -1488,7 +1488,7 @@ dependencies = [ [[package]] name = "ruma-common" version = "0.2.0" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#088382dbdc176e61fa5bde679ae38093865e7053" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#6ccb3ecaf69167ba405379826a9d87a98f168df8" dependencies = [ "js_int", "ruma-api", @@ -1502,7 +1502,7 @@ dependencies = [ [[package]] name = "ruma-events" version = "0.22.0-alpha.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#088382dbdc176e61fa5bde679ae38093865e7053" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#6ccb3ecaf69167ba405379826a9d87a98f168df8" dependencies = [ "js_int", "ruma-common", @@ -1517,7 +1517,7 @@ dependencies = [ [[package]] name = "ruma-events-macros" version = "0.22.0-alpha.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#088382dbdc176e61fa5bde679ae38093865e7053" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#6ccb3ecaf69167ba405379826a9d87a98f168df8" dependencies = [ "proc-macro-crate", "proc-macro2", @@ -1528,7 +1528,7 @@ dependencies = [ [[package]] name = "ruma-federation-api" version = "0.0.3" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#088382dbdc176e61fa5bde679ae38093865e7053" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#6ccb3ecaf69167ba405379826a9d87a98f168df8" dependencies = [ "js_int", "ruma-api", @@ -1543,7 +1543,7 @@ dependencies = [ [[package]] name = "ruma-identifiers" version = "0.17.4" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#088382dbdc176e61fa5bde679ae38093865e7053" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#6ccb3ecaf69167ba405379826a9d87a98f168df8" dependencies = [ "rand", "ruma-identifiers-macros", @@ -1555,7 +1555,7 @@ dependencies = [ [[package]] name = "ruma-identifiers-macros" version = "0.17.4" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#088382dbdc176e61fa5bde679ae38093865e7053" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#6ccb3ecaf69167ba405379826a9d87a98f168df8" dependencies = [ "proc-macro2", "quote", @@ -1566,7 +1566,7 @@ dependencies = [ [[package]] name = "ruma-identifiers-validation" version = "0.1.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#088382dbdc176e61fa5bde679ae38093865e7053" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#6ccb3ecaf69167ba405379826a9d87a98f168df8" dependencies = [ "serde", "strum", @@ -1575,7 +1575,7 @@ dependencies = [ [[package]] name = "ruma-serde" version = "0.2.3" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#088382dbdc176e61fa5bde679ae38093865e7053" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#6ccb3ecaf69167ba405379826a9d87a98f168df8" dependencies = [ "form_urlencoded", "itoa", @@ -1587,7 +1587,7 @@ dependencies = [ [[package]] name = "ruma-signatures" version = "0.6.0-dev.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#088382dbdc176e61fa5bde679ae38093865e7053" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#6ccb3ecaf69167ba405379826a9d87a98f168df8" dependencies = [ "base64", "ring", @@ -1916,9 +1916,9 @@ dependencies = [ [[package]] name = "syn" -version = "1.0.40" +version = "1.0.41" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "963f7d3cc59b59b9325165add223142bbf1df27655d07789f109896d353d8350" +checksum = "6690e3e9f692504b941dc6c3b188fd28df054f7fb8469ab40680df52fdcc842b" dependencies = [ "proc-macro2", "quote", diff --git a/src/client_server/account.rs b/src/client_server/account.rs index 3db933c17..2ec9282fe 100644 --- a/src/client_server/account.rs +++ b/src/client_server/account.rs @@ -303,7 +303,7 @@ pub fn whoami_route(body: Ruma) -> ConduitResult, body: Ruma>, ) -> ConduitResult { @@ -366,7 +366,7 @@ pub fn deactivate_route( &room_id, &db.globals, &db.account_data, - )?; + ).await?; } // Remove devices and mark account as deactivated diff --git a/src/client_server/alias.rs b/src/client_server/alias.rs index 0ec43f576..c5c514e09 100644 --- a/src/client_server/alias.rs +++ b/src/client_server/alias.rs @@ -62,7 +62,7 @@ pub async fn get_alias_helper( ) -> ConduitResult { if room_alias.server_name() != db.globals.server_name() { let response = server_server::send_request( - &db, + &db.globals, room_alias.server_name(), federation::query::get_room_information::v1::Request { room_alias }, ) diff --git a/src/client_server/directory.rs b/src/client_server/directory.rs index 871a780f2..372ce9838 100644 --- a/src/client_server/directory.rs +++ b/src/client_server/directory.rs @@ -121,7 +121,7 @@ pub async fn get_public_rooms_filtered_helper( .filter(|server| *server != db.globals.server_name().as_str()) { let response = server_server::send_request( - &db, + &db.globals, other_server, federation::directory::get_public_rooms_filtered::v1::Request { limit, diff --git a/src/client_server/media.rs b/src/client_server/media.rs index f897a6780..8f7a9b96f 100644 --- a/src/client_server/media.rs +++ b/src/client_server/media.rs @@ -79,7 +79,7 @@ pub async fn get_content_route( .into()) } else if body.allow_remote { let get_content_response = server_server::send_request( - &db, + &db.globals, body.server_name.as_ref(), get_content::Request { allow_remote: false, @@ -129,7 +129,7 @@ pub async fn get_content_thumbnail_route( Ok(get_content_thumbnail::Response { file, content_type }.into()) } else if body.allow_remote { let get_thumbnail_response = server_server::send_request( - &db, + &db.globals, body.server_name.as_ref(), get_content_thumbnail::Request { allow_remote: false, diff --git a/src/client_server/membership.rs b/src/client_server/membership.rs index 8d1940243..18fb5a9ef 100644 --- a/src/client_server/membership.rs +++ b/src/client_server/membership.rs @@ -83,7 +83,7 @@ pub async fn join_room_by_id_or_alias_route( feature = "conduit_bin", post("/_matrix/client/r0/rooms/<_>/leave", data = "") )] -pub fn leave_room_route( +pub async fn leave_room_route( db: State<'_, Database>, body: Ruma>, ) -> ConduitResult { @@ -108,19 +108,21 @@ pub fn leave_room_route( event.membership = member::MembershipState::Leave; - db.rooms.build_and_append_pdu( - PduBuilder { - event_type: EventType::RoomMember, - content: serde_json::to_value(event).expect("event is valid, we just created it"), - unsigned: None, - state_key: Some(sender_id.to_string()), - redacts: None, - }, - &sender_id, - &body.room_id, - &db.globals, - &db.account_data, - )?; + db.rooms + .build_and_append_pdu( + PduBuilder { + event_type: EventType::RoomMember, + content: serde_json::to_value(event).expect("event is valid, we just created it"), + unsigned: None, + state_key: Some(sender_id.to_string()), + redacts: None, + }, + &sender_id, + &body.room_id, + &db.globals, + &db.account_data, + ) + .await?; Ok(leave_room::Response::new().into()) } @@ -129,33 +131,35 @@ pub fn leave_room_route( feature = "conduit_bin", post("/_matrix/client/r0/rooms/<_>/invite", data = "") )] -pub fn invite_user_route( +pub async fn invite_user_route( db: State<'_, Database>, body: Ruma>, ) -> ConduitResult { let sender_id = body.sender_id.as_ref().expect("user is authenticated"); if let invite_user::IncomingInvitationRecipient::UserId { user_id } = &body.recipient { - db.rooms.build_and_append_pdu( - PduBuilder { - event_type: EventType::RoomMember, - content: serde_json::to_value(member::MemberEventContent { - membership: member::MembershipState::Invite, - displayname: db.users.displayname(&user_id)?, - avatar_url: db.users.avatar_url(&user_id)?, - is_direct: None, - third_party_invite: None, - }) - .expect("event is valid, we just created it"), - unsigned: None, - state_key: Some(user_id.to_string()), - redacts: None, - }, - &sender_id, - &body.room_id, - &db.globals, - &db.account_data, - )?; + db.rooms + .build_and_append_pdu( + PduBuilder { + event_type: EventType::RoomMember, + content: serde_json::to_value(member::MemberEventContent { + membership: member::MembershipState::Invite, + displayname: db.users.displayname(&user_id)?, + avatar_url: db.users.avatar_url(&user_id)?, + is_direct: None, + third_party_invite: None, + }) + .expect("event is valid, we just created it"), + unsigned: None, + state_key: Some(user_id.to_string()), + redacts: None, + }, + &sender_id, + &body.room_id, + &db.globals, + &db.account_data, + ) + .await?; Ok(invite_user::Response.into()) } else { @@ -167,7 +171,7 @@ pub fn invite_user_route( feature = "conduit_bin", post("/_matrix/client/r0/rooms/<_>/kick", data = "") )] -pub fn kick_user_route( +pub async fn kick_user_route( db: State<'_, Database>, body: Ruma>, ) -> ConduitResult { @@ -193,19 +197,21 @@ pub fn kick_user_route( event.membership = ruma::events::room::member::MembershipState::Leave; // TODO: reason - db.rooms.build_and_append_pdu( - PduBuilder { - event_type: EventType::RoomMember, - content: serde_json::to_value(event).expect("event is valid, we just created it"), - unsigned: None, - state_key: Some(body.user_id.to_string()), - redacts: None, - }, - &sender_id, - &body.room_id, - &db.globals, - &db.account_data, - )?; + db.rooms + .build_and_append_pdu( + PduBuilder { + event_type: EventType::RoomMember, + content: serde_json::to_value(event).expect("event is valid, we just created it"), + unsigned: None, + state_key: Some(body.user_id.to_string()), + redacts: None, + }, + &sender_id, + &body.room_id, + &db.globals, + &db.account_data, + ) + .await?; Ok(kick_user::Response::new().into()) } @@ -214,7 +220,7 @@ pub fn kick_user_route( feature = "conduit_bin", post("/_matrix/client/r0/rooms/<_>/ban", data = "") )] -pub fn ban_user_route( +pub async fn ban_user_route( db: State<'_, Database>, body: Ruma>, ) -> ConduitResult { @@ -248,19 +254,21 @@ pub fn ban_user_route( }, )?; - db.rooms.build_and_append_pdu( - PduBuilder { - event_type: EventType::RoomMember, - content: serde_json::to_value(event).expect("event is valid, we just created it"), - unsigned: None, - state_key: Some(body.user_id.to_string()), - redacts: None, - }, - &sender_id, - &body.room_id, - &db.globals, - &db.account_data, - )?; + db.rooms + .build_and_append_pdu( + PduBuilder { + event_type: EventType::RoomMember, + content: serde_json::to_value(event).expect("event is valid, we just created it"), + unsigned: None, + state_key: Some(body.user_id.to_string()), + redacts: None, + }, + &sender_id, + &body.room_id, + &db.globals, + &db.account_data, + ) + .await?; Ok(ban_user::Response::new().into()) } @@ -269,7 +277,7 @@ pub fn ban_user_route( feature = "conduit_bin", post("/_matrix/client/r0/rooms/<_>/unban", data = "") )] -pub fn unban_user_route( +pub async fn unban_user_route( db: State<'_, Database>, body: Ruma>, ) -> ConduitResult { @@ -294,19 +302,21 @@ pub fn unban_user_route( event.membership = ruma::events::room::member::MembershipState::Leave; - db.rooms.build_and_append_pdu( - PduBuilder { - event_type: EventType::RoomMember, - content: serde_json::to_value(event).expect("event is valid, we just created it"), - unsigned: None, - state_key: Some(body.user_id.to_string()), - redacts: None, - }, - &sender_id, - &body.room_id, - &db.globals, - &db.account_data, - )?; + db.rooms + .build_and_append_pdu( + PduBuilder { + event_type: EventType::RoomMember, + content: serde_json::to_value(event).expect("event is valid, we just created it"), + unsigned: None, + state_key: Some(body.user_id.to_string()), + redacts: None, + }, + &sender_id, + &body.room_id, + &db.globals, + &db.account_data, + ) + .await?; Ok(unban_user::Response::new().into()) } @@ -429,7 +439,7 @@ async fn join_room_by_id_helper( for remote_server in servers { let make_join_response = server_server::send_request( - &db, + &db.globals, remote_server, federation::membership::create_join_event_template::v1::Request { room_id, @@ -490,7 +500,7 @@ async fn join_room_by_id_helper( .expect("event is valid, we just created it"); let send_join_response = server_server::send_request( - &db, + &db.globals, remote_server, federation::membership::create_join_event::v2::Request { room_id, @@ -621,9 +631,12 @@ async fn join_room_by_id_helper( .expect("Found event_id in sorted events that is not in resolved state"); // We do not rebuild the PDU in this case only insert to DB - let pdu_id = - db.rooms - .append_pdu(&PduEvent::from(&**pdu), &db.globals, &db.account_data)?; + let pdu_id = db.rooms.append_pdu( + &PduEvent::from(&**pdu), + &serde_json::to_value(&**pdu).expect("PDU is valid value"), + &db.globals, + &db.account_data, + )?; if state_events.contains(ev_id) { state.insert( @@ -646,19 +659,22 @@ async fn join_room_by_id_helper( third_party_invite: None, }; - db.rooms.build_and_append_pdu( - PduBuilder { - event_type: EventType::RoomMember, - content: serde_json::to_value(event).expect("event is valid, we just created it"), - unsigned: None, - state_key: Some(sender_id.to_string()), - redacts: None, - }, - &sender_id, - &room_id, - &db.globals, - &db.account_data, - )?; + db.rooms + .build_and_append_pdu( + PduBuilder { + event_type: EventType::RoomMember, + content: serde_json::to_value(event) + .expect("event is valid, we just created it"), + unsigned: None, + state_key: Some(sender_id.to_string()), + redacts: None, + }, + &sender_id, + &room_id, + &db.globals, + &db.account_data, + ) + .await?; } Ok(join_room_by_id::Response::new(room_id.clone()).into()) diff --git a/src/client_server/message.rs b/src/client_server/message.rs index 8a09aba57..4ba0d9fd8 100644 --- a/src/client_server/message.rs +++ b/src/client_server/message.rs @@ -17,7 +17,7 @@ use rocket::{get, put}; feature = "conduit_bin", put("/_matrix/client/r0/rooms/<_>/send/<_>/<_>", data = "") )] -pub fn send_message_event_route( +pub async fn send_message_event_route( db: State<'_, Database>, body: Ruma>, ) -> ConduitResult { @@ -67,7 +67,7 @@ pub fn send_message_event_route( &body.room_id, &db.globals, &db.account_data, - )?; + ).await?; db.transaction_ids .add_txnid(sender_id, device_id, &body.txn_id, event_id.as_bytes())?; diff --git a/src/client_server/profile.rs b/src/client_server/profile.rs index c1c0253ed..be893e1b5 100644 --- a/src/client_server/profile.rs +++ b/src/client_server/profile.rs @@ -19,7 +19,7 @@ use std::convert::TryInto; feature = "conduit_bin", put("/_matrix/client/r0/profile/<_>/displayname", data = "") )] -pub fn set_displayname_route( +pub async fn set_displayname_route( db: State<'_, Database>, body: Ruma>, ) -> ConduitResult { @@ -64,7 +64,7 @@ pub fn set_displayname_route( &room_id, &db.globals, &db.account_data, - )?; + ).await?; // Presence update db.rooms.edus.update_presence( @@ -110,7 +110,7 @@ pub fn get_displayname_route( feature = "conduit_bin", put("/_matrix/client/r0/profile/<_>/avatar_url", data = "") )] -pub fn set_avatar_url_route( +pub async fn set_avatar_url_route( db: State<'_, Database>, body: Ruma>, ) -> ConduitResult { @@ -167,7 +167,7 @@ pub fn set_avatar_url_route( &room_id, &db.globals, &db.account_data, - )?; + ).await?; // Presence update db.rooms.edus.update_presence( diff --git a/src/client_server/redact.rs b/src/client_server/redact.rs index 511734899..701fc00de 100644 --- a/src/client_server/redact.rs +++ b/src/client_server/redact.rs @@ -12,7 +12,7 @@ use rocket::put; feature = "conduit_bin", put("/_matrix/client/r0/rooms/<_>/redact/<_>/<_>", data = "") )] -pub fn redact_event_route( +pub async fn redact_event_route( db: State<'_, Database>, body: Ruma>, ) -> ConduitResult { @@ -33,7 +33,7 @@ pub fn redact_event_route( &body.room_id, &db.globals, &db.account_data, - )?; + ).await?; Ok(redact_event::Response { event_id }.into()) } diff --git a/src/client_server/room.rs b/src/client_server/room.rs index a5280cf54..0e5c5716e 100644 --- a/src/client_server/room.rs +++ b/src/client_server/room.rs @@ -20,7 +20,7 @@ use rocket::{get, post}; feature = "conduit_bin", post("/_matrix/client/r0/createRoom", data = "") )] -pub fn create_room_route( +pub async fn create_room_route( db: State<'_, Database>, body: Ruma>, ) -> ConduitResult { @@ -65,7 +65,7 @@ pub fn create_room_route( &room_id, &db.globals, &db.account_data, - )?; + ).await?; // 2. Let the room creator join db.rooms.build_and_append_pdu( @@ -87,7 +87,7 @@ pub fn create_room_route( &room_id, &db.globals, &db.account_data, - )?; + ).await?; // 3. Power levels let mut users = BTreeMap::new(); @@ -129,7 +129,7 @@ pub fn create_room_route( &room_id, &db.globals, &db.account_data, - )?; + ).await?; // 4. Events set by preset @@ -162,7 +162,7 @@ pub fn create_room_route( &room_id, &db.globals, &db.account_data, - )?; + ).await?; // 4.2 History Visibility db.rooms.build_and_append_pdu( @@ -180,7 +180,7 @@ pub fn create_room_route( &room_id, &db.globals, &db.account_data, - )?; + ).await?; // 4.3 Guest Access db.rooms.build_and_append_pdu( @@ -206,7 +206,7 @@ pub fn create_room_route( &room_id, &db.globals, &db.account_data, - )?; + ).await?; // 5. Events listed in initial_state for event in &body.initial_state { @@ -226,7 +226,7 @@ pub fn create_room_route( &room_id, &db.globals, &db.account_data, - )?; + ).await?; } // 6. Events implied by name and topic @@ -248,7 +248,7 @@ pub fn create_room_route( &room_id, &db.globals, &db.account_data, - )?; + ).await?; } if let Some(topic) = &body.topic { @@ -267,7 +267,7 @@ pub fn create_room_route( &room_id, &db.globals, &db.account_data, - )?; + ).await?; } // 7. Events implied by invite (and TODO: invite_3pid) @@ -291,7 +291,7 @@ pub fn create_room_route( &room_id, &db.globals, &db.account_data, - )?; + ).await?; } // Homeserver specific stuff @@ -337,7 +337,7 @@ pub fn get_room_event_route( feature = "conduit_bin", post("/_matrix/client/r0/rooms/<_room_id>/upgrade", data = "") )] -pub fn upgrade_room_route( +pub async fn upgrade_room_route( db: State<'_, Database>, body: Ruma>, _room_id: String, @@ -379,7 +379,7 @@ pub fn upgrade_room_route( &body.room_id, &db.globals, &db.account_data, - )?; + ).await?; // Get the old room federations status let federate = serde_json::from_value::>( @@ -419,7 +419,7 @@ pub fn upgrade_room_route( &replacement_room, &db.globals, &db.account_data, - )?; + ).await?; // Join the new room db.rooms.build_and_append_pdu( @@ -441,7 +441,7 @@ pub fn upgrade_room_route( &replacement_room, &db.globals, &db.account_data, - )?; + ).await?; // Recommended transferable state events list from the specs let transferable_state_events = vec![ @@ -475,7 +475,7 @@ pub fn upgrade_room_route( &replacement_room, &db.globals, &db.account_data, - )?; + ).await?; } // Moves any local aliases to the new room @@ -505,7 +505,7 @@ pub fn upgrade_room_route( power_levels_event_content.invite = new_level; // Modify the power levels in the old room to prevent sending of events and inviting new users - db.rooms + let _ = db.rooms .build_and_append_pdu( PduBuilder { event_type: EventType::RoomPowerLevels, @@ -519,8 +519,7 @@ pub fn upgrade_room_route( &body.room_id, &db.globals, &db.account_data, - ) - .ok(); + ).await; // Return the replacement room id Ok(upgrade_room::Response { replacement_room }.into()) diff --git a/src/client_server/state.rs b/src/client_server/state.rs index 1fe3cd6c2..e9d20e2f6 100644 --- a/src/client_server/state.rs +++ b/src/client_server/state.rs @@ -19,7 +19,7 @@ use rocket::{get, put}; feature = "conduit_bin", put("/_matrix/client/r0/rooms/<_>/state/<_>/<_>", data = "") )] -pub fn send_state_event_for_key_route( +pub async fn send_state_event_for_key_route( db: State<'_, Database>, body: Ruma>, ) -> ConduitResult { @@ -41,7 +41,7 @@ pub fn send_state_event_for_key_route( content, &body.room_id, Some(body.state_key.to_owned()), - )?) + ).await?) .into(), ) } @@ -50,7 +50,7 @@ pub fn send_state_event_for_key_route( feature = "conduit_bin", put("/_matrix/client/r0/rooms/<_>/state/<_>", data = "") )] -pub fn send_state_event_for_empty_key_route( +pub async fn send_state_event_for_empty_key_route( db: State<'_, Database>, body: Ruma>, ) -> ConduitResult { @@ -80,7 +80,7 @@ pub fn send_state_event_for_empty_key_route( json, &body.room_id, Some("".into()), - )?) + ).await?) .into(), ) } @@ -177,7 +177,7 @@ pub fn get_state_events_for_empty_key_route( .into()) } -pub fn send_state_event_for_key_helper( +pub async fn send_state_event_for_key_helper( db: &Database, sender: &UserId, content: &AnyStateEventContent, @@ -223,7 +223,7 @@ pub fn send_state_event_for_key_helper( &room_id, &db.globals, &db.account_data, - )?; + ).await?; Ok(event_id) } diff --git a/src/database.rs b/src/database.rs index 83f30c967..e1a356c78 100644 --- a/src/database.rs +++ b/src/database.rs @@ -109,6 +109,7 @@ impl Database { tokenids: db.open_tree("tokenids")?, + roomserverids: db.open_tree("roomserverids")?, userroomid_joined: db.open_tree("userroomid_joined")?, roomuserid_joined: db.open_tree("roomuserid_joined")?, roomuseroncejoinedids: db.open_tree("roomuseroncejoinedids")?, diff --git a/src/database/rooms.rs b/src/database/rooms.rs index b538c854a..ba54e7f27 100644 --- a/src/database/rooms.rs +++ b/src/database/rooms.rs @@ -2,11 +2,12 @@ mod edus; pub use edus::RoomEdus; -use crate::{pdu::PduBuilder, utils, Error, PduEvent, Result}; +use crate::{pdu::PduBuilder, server_server, utils, Error, PduEvent, Result}; use log::error; use ring::digest; use ruma::{ api::client::error::ErrorKind, + api::federation, events::{ ignored_user_list, room::{ @@ -15,7 +16,7 @@ use ruma::{ }, EventType, }, - EventId, Raw, RoomAliasId, RoomId, UserId, + EventId, Raw, RoomAliasId, RoomId, ServerName, UserId, }; use sled::IVec; use state_res::{event_auth, Error as StateError, Requester, StateEvent, StateMap, StateStore}; @@ -25,6 +26,7 @@ use std::{ convert::{TryFrom, TryInto}, mem, sync::Arc, + time::SystemTime, }; /// The unique identifier of each state group. @@ -44,6 +46,8 @@ pub struct Rooms { pub(super) tokenids: sled::Tree, // TokenId = RoomId + Token + PduId + /// Participating servers in a room. + pub(super) roomserverids: sled::Tree, // RoomServerId = RoomId + ServerName pub(super) userroomid_joined: sled::Tree, pub(super) roomuserid_joined: sled::Tree, pub(super) roomuseroncejoinedids: sled::Tree, @@ -169,8 +173,7 @@ impl Rooms { Ok(events) } - // This fetches auth events from the current state using the - /// full `roomstateid_pdu` tree. + /// This fetches auth events from the current state. pub fn get_auth_events( &self, room_id: &RoomId, @@ -472,17 +475,10 @@ impl Rooms { pub fn append_pdu( &self, pdu: &PduEvent, + pdu_json: &serde_json::Value, globals: &super::globals::Globals, account_data: &super::account_data::AccountData, ) -> Result> { - let mut pdu_json = serde_json::to_value(&pdu).expect("event is valid, we just created it"); - ruma::signatures::hash_and_sign_event( - globals.server_name().as_str(), - globals.keypair(), - &mut pdu_json, - ) - .expect("event is valid, we just created it"); - self.replace_pdu_leaves(&pdu.room_id, &pdu.event_id)?; // Increment the last index and use that @@ -610,7 +606,7 @@ impl Rooms { } /// Creates a new persisted data unit and adds it to a room. - pub fn build_and_append_pdu( + pub async fn build_and_append_pdu( &self, pdu_builder: PduBuilder, sender: &UserId, @@ -799,22 +795,59 @@ impl Rooms { signatures: BTreeMap::new(), }; + // Hash and sign + let mut pdu_json = serde_json::to_value(&pdu).expect("event is valid, we just created it"); + pdu_json + .as_object_mut() + .expect("json is object") + .remove("event_id"); + + ruma::signatures::hash_and_sign_event( + globals.server_name().as_str(), + globals.keypair(), + &mut pdu_json, + ) + .expect("event is valid, we just created it"); + // Generate event id pdu.event_id = EventId::try_from(&*format!( "${}", - ruma::signatures::reference_hash( - &serde_json::to_value(&pdu).expect("event is valid, we just created it") - ) - .expect("ruma can calculate reference hashes") + ruma::signatures::reference_hash(&pdu_json) + .expect("ruma can calculate reference hashes") )) .expect("ruma's reference hashes are valid event ids"); - let pdu_id = self.append_pdu(&pdu, globals, account_data)?; + pdu_json + .as_object_mut() + .expect("json is object") + .insert("event_id".to_owned(), pdu.event_id.to_string().into()); + + let pdu_id = self.append_pdu(&pdu, &pdu_json, globals, account_data)?; if pdu.state_key.is_some() { self.append_to_state(&pdu_id, &pdu)?; } + pdu_json + .as_object_mut() + .expect("json is object") + .remove("event_id"); + + let response = server_server::send_request( + &globals, + "koesters.xyz".try_into().unwrap(), + federation::transactions::send_transaction_message::v1::Request { + origin: globals.server_name(), + pdus: &[serde_json::from_value(pdu_json).expect("Raw::from_value always works")], + edus: &[], + origin_server_ts: SystemTime::now(), + transaction_id: &utils::random_string(16), + }, + ) + .await; + + let _ = dbg!(response); + Ok(pdu.event_id) } @@ -957,12 +990,17 @@ impl Rooms { &self, room_id: &RoomId, user_id: &UserId, - mut member_content: member::MemberEventContent, + member_content: member::MemberEventContent, sender: &UserId, account_data: &super::account_data::AccountData, globals: &super::globals::Globals, ) -> Result<()> { let membership = member_content.membership; + + let mut roomserver_id = room_id.as_bytes().to_vec(); + roomserver_id.push(0xff); + roomserver_id.extend_from_slice(user_id.server_name().as_bytes()); + let mut userroom_id = user_id.as_bytes().to_vec(); userroom_id.push(0xff); userroom_id.extend_from_slice(room_id.as_bytes()); @@ -1056,6 +1094,7 @@ impl Rooms { } } + self.roomserverids.insert(&roomserver_id, &[])?; self.userroomid_joined.insert(&userroom_id, &[])?; self.roomuserid_joined.insert(&roomuser_id, &[])?; self.userroomid_invited.remove(&userroom_id)?; @@ -1075,25 +1114,10 @@ impl Rooms { }); if is_ignored { - member_content.membership = member::MembershipState::Leave; - - self.build_and_append_pdu( - PduBuilder { - event_type: EventType::RoomMember, - content: serde_json::to_value(member_content) - .expect("event is valid, we just created it"), - unsigned: None, - state_key: Some(user_id.to_string()), - redacts: None, - }, - &user_id, - &room_id, - globals, - account_data, - )?; - return Ok(()); } + + self.roomserverids.insert(&roomserver_id, &[])?; self.userroomid_invited.insert(&userroom_id, &[])?; self.roomuserid_invited.insert(&roomuser_id, &[])?; self.userroomid_joined.remove(&userroom_id)?; @@ -1101,6 +1125,14 @@ impl Rooms { self.userroomid_left.remove(&userroom_id)?; } member::MembershipState::Leave | member::MembershipState::Ban => { + if self + .room_members(room_id) + .chain(self.room_members_invited(room_id)) + .filter_map(|r| r.ok()) + .all(|u| u.server_name() != user_id.server_name()) + { + self.roomserverids.remove(&roomserver_id)?; + } self.userroomid_left.insert(&userroom_id, &[])?; self.userroomid_joined.remove(&userroom_id)?; self.roomuserid_joined.remove(&roomuser_id)?; @@ -1294,10 +1326,34 @@ impl Rooms { }) } + /// Returns an iterator over all joined members of a room. + pub fn room_servers(&self, room_id: &RoomId) -> impl Iterator>> { + let mut prefix = room_id.as_bytes().to_vec(); + prefix.push(0xff); + + self.roomserverids.scan_prefix(prefix).keys().map(|key| { + Ok(Box::::try_from( + utils::string_from_bytes( + &key? + .rsplit(|&b| b == 0xff) + .next() + .expect("rsplit always returns an element"), + ) + .map_err(|_| { + Error::bad_database("Server name in roomserverids is invalid unicode.") + })?, + ) + .map_err(|_| Error::bad_database("Server name in roomserverids is invalid."))?) + }) + } + /// Returns an iterator over all joined members of a room. pub fn room_members(&self, room_id: &RoomId) -> impl Iterator> { + let mut prefix = room_id.as_bytes().to_vec(); + prefix.push(0xff); + self.roomuserid_joined - .scan_prefix(room_id.as_bytes()) + .scan_prefix(prefix) .keys() .map(|key| { Ok(UserId::try_from( @@ -1317,8 +1373,11 @@ impl Rooms { /// Returns an iterator over all User IDs who ever joined a room. pub fn room_useroncejoined(&self, room_id: &RoomId) -> impl Iterator> { + let mut prefix = room_id.as_bytes().to_vec(); + prefix.push(0xff); + self.roomuseroncejoinedids - .scan_prefix(room_id.to_string()) + .scan_prefix(prefix) .keys() .map(|key| { Ok(UserId::try_from( @@ -1338,8 +1397,11 @@ impl Rooms { /// Returns an iterator over all invited members of a room. pub fn room_members_invited(&self, room_id: &RoomId) -> impl Iterator> { + let mut prefix = room_id.as_bytes().to_vec(); + prefix.push(0xff); + self.roomuserid_invited - .scan_prefix(room_id.as_bytes()) + .scan_prefix(prefix) .keys() .map(|key| { Ok(UserId::try_from( @@ -1380,8 +1442,11 @@ impl Rooms { /// Returns an iterator over all rooms a user was invited to. pub fn rooms_invited(&self, user_id: &UserId) -> impl Iterator> { + let mut prefix = user_id.as_bytes().to_vec(); + prefix.push(0xff); + self.userroomid_invited - .scan_prefix(&user_id.as_bytes()) + .scan_prefix(prefix) .keys() .map(|key| { Ok(RoomId::try_from( @@ -1401,23 +1466,23 @@ impl Rooms { /// Returns an iterator over all rooms a user left. pub fn rooms_left(&self, user_id: &UserId) -> impl Iterator> { - self.userroomid_left - .scan_prefix(&user_id.as_bytes()) - .keys() - .map(|key| { - Ok(RoomId::try_from( - utils::string_from_bytes( - &key? - .rsplit(|&b| b == 0xff) - .next() - .expect("rsplit always returns an element"), - ) - .map_err(|_| { - Error::bad_database("Room ID in userroomid_left is invalid unicode.") - })?, + let mut prefix = user_id.as_bytes().to_vec(); + prefix.push(0xff); + + self.userroomid_left.scan_prefix(prefix).keys().map(|key| { + Ok(RoomId::try_from( + utils::string_from_bytes( + &key? + .rsplit(|&b| b == 0xff) + .next() + .expect("rsplit always returns an element"), ) - .map_err(|_| Error::bad_database("Room ID in userroomid_left is invalid."))?) - }) + .map_err(|_| { + Error::bad_database("Room ID in userroomid_left is invalid unicode.") + })?, + ) + .map_err(|_| Error::bad_database("Room ID in userroomid_left is invalid."))?) + }) } pub fn once_joined(&self, user_id: &UserId, room_id: &RoomId) -> Result { diff --git a/src/pdu.rs b/src/pdu.rs index 7f842e2d7..c9042306e 100644 --- a/src/pdu.rs +++ b/src/pdu.rs @@ -6,7 +6,7 @@ use ruma::{ AnyStrippedStateEvent, AnySyncRoomEvent, AnySyncStateEvent, EventType, StateEvent, }, EventId, Raw, RoomId, ServerKeyId, ServerName, UserId, -}; +events::pdu::PduStub}; use serde::{Deserialize, Serialize}; use serde_json::json; use std::{collections::BTreeMap, convert::TryInto, sync::Arc, time::UNIX_EPOCH}; @@ -198,6 +198,31 @@ impl PduEvent { serde_json::from_value(json).expect("Raw::from_value always works") } + + pub fn to_outgoing_federation_event(&self) -> Raw { + let mut json = json!({ + "room_id": self.room_id, + "sender": self.sender, + "origin_server_ts": self.origin_server_ts, + "type": self.kind, + "content": self.content, + "prev_events": self.prev_events, + "depth": self.depth, + "auth_events": self.auth_events, + "unsigned": self.unsigned, + "hashes": self.hashes, + "signatures": self.signatures, + }); + + if let Some(state_key) = &self.state_key { + json["state_key"] = json!(state_key); + } + if let Some(redacts) = &self.redacts { + json["redacts"] = json!(redacts); + } + + serde_json::from_value(json).expect("Raw::from_value always works") + } } impl From<&state_res::StateEvent> for PduEvent { diff --git a/src/server_server.rs b/src/server_server.rs index 6c53aed29..9f4be132f 100644 --- a/src/server_server.rs +++ b/src/server_server.rs @@ -24,9 +24,9 @@ use std::{ time::{Duration, SystemTime}, }; -pub async fn request_well_known(db: &crate::Database, destination: &str) -> Option { +pub async fn request_well_known(globals: &crate::database::globals::Globals, destination: &str) -> Option { let body: serde_json::Value = serde_json::from_str( - &db.globals + &globals .reqwest_client() .get(&format!( "https://{}/.well-known/matrix/server", @@ -44,7 +44,7 @@ pub async fn request_well_known(db: &crate::Database, destination: &str) -> Opti } pub async fn send_request( - db: &crate::Database, + globals: &crate::database::globals::Globals, destination: &ServerName, request: T, ) -> Result @@ -52,7 +52,7 @@ where T: Debug, { let actual_destination = "https://".to_owned() - + &request_well_known(db, &destination.as_str()) + + &request_well_known(globals, &destination.as_str()) .await .unwrap_or(destination.as_str().to_owned() + ":8448"); @@ -81,14 +81,14 @@ where ); request_map.insert( "origin".to_owned(), - db.globals.server_name().as_str().into(), + globals.server_name().as_str().into(), ); request_map.insert("destination".to_owned(), destination.as_str().into()); let mut request_json = request_map.into(); ruma::signatures::sign_json( - db.globals.server_name().as_str(), - db.globals.keypair(), + globals.server_name().as_str(), + globals.keypair(), &mut request_json, ) .unwrap(); @@ -110,7 +110,7 @@ where AUTHORIZATION, HeaderValue::from_str(&format!( "X-Matrix origin={},key=\"{}\",sig=\"{}\"", - db.globals.server_name(), + globals.server_name(), s.0, s.1 )) @@ -122,7 +122,7 @@ where let reqwest_request = reqwest::Request::try_from(http_request) .expect("all http requests are valid reqwest requests"); - let reqwest_response = db.globals.reqwest_client().execute(reqwest_request).await; + let reqwest_response = globals.reqwest_client().execute(reqwest_request).await; // Because reqwest::Response -> http::Response is complicated: match reqwest_response { @@ -317,9 +317,9 @@ pub fn send_transaction_message_route<'a>( .insert("event_id".to_owned(), event_id.to_string().into()); let pdu = - serde_json::from_value::(value).expect("all ruma pdus are conduit pdus"); + serde_json::from_value::(value.clone()).expect("all ruma pdus are conduit pdus"); if db.rooms.exists(&pdu.room_id)? { - db.rooms.append_pdu(&pdu, &db.globals, &db.account_data)?; + db.rooms.append_pdu(&pdu, &value, &db.globals, &db.account_data)?; } } Ok(send_transaction_message::v1::Response { From 71500b14b902321e91cab432d55dd3f3ae7aedfc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20K=C3=B6sters?= Date: Tue, 15 Sep 2020 08:16:20 +0200 Subject: [PATCH 29/51] fix: send to all servers and fix media store --- src/client_server/alias.rs | 2 +- src/client_server/directory.rs | 2 +- src/client_server/media.rs | 38 ++++++++-------------------- src/client_server/membership.rs | 4 +-- src/database/rooms.rs | 44 ++++++++++++++++++++++----------- src/server_server.rs | 29 ++++++++++++---------- 6 files changed, 60 insertions(+), 59 deletions(-) diff --git a/src/client_server/alias.rs b/src/client_server/alias.rs index c5c514e09..c2c3eb9c5 100644 --- a/src/client_server/alias.rs +++ b/src/client_server/alias.rs @@ -63,7 +63,7 @@ pub async fn get_alias_helper( if room_alias.server_name() != db.globals.server_name() { let response = server_server::send_request( &db.globals, - room_alias.server_name(), + room_alias.server_name().to_owned(), federation::query::get_room_information::v1::Request { room_alias }, ) .await?; diff --git a/src/client_server/directory.rs b/src/client_server/directory.rs index 372ce9838..c82a15f02 100644 --- a/src/client_server/directory.rs +++ b/src/client_server/directory.rs @@ -122,7 +122,7 @@ pub async fn get_public_rooms_filtered_helper( { let response = server_server::send_request( &db.globals, - other_server, + other_server.to_owned(), federation::directory::get_public_rooms_filtered::v1::Request { limit, since: since.as_deref(), diff --git a/src/client_server/media.rs b/src/client_server/media.rs index 8f7a9b96f..8a93d4924 100644 --- a/src/client_server/media.rs +++ b/src/client_server/media.rs @@ -48,22 +48,13 @@ pub fn create_content_route( #[cfg_attr( feature = "conduit_bin", - get( - "/_matrix/media/r0/download/<_server_name>/<_media_id>", - data = "" - ) + get("/_matrix/media/r0/download/<_>/<_>", data = "") )] pub async fn get_content_route( db: State<'_, Database>, body: Ruma>, - _server_name: String, - _media_id: String, ) -> ConduitResult { - let mxc = format!( - "mxc://{}/{}", - db.globals.server_name(), - utils::random_string(MXC_LENGTH) - ); + let mxc = format!("mxc://{}/{}", body.server_name, body.media_id); if let Some(FileMeta { filename, @@ -77,10 +68,10 @@ pub async fn get_content_route( content_disposition: filename.unwrap_or_default(), // TODO: Spec says this should be optional } .into()) - } else if body.allow_remote { + } else if &*body.server_name != db.globals.server_name() && body.allow_remote { let get_content_response = server_server::send_request( &db.globals, - body.server_name.as_ref(), + body.server_name.clone(), get_content::Request { allow_remote: false, server_name: &body.server_name, @@ -104,21 +95,18 @@ pub async fn get_content_route( #[cfg_attr( feature = "conduit_bin", - get( - "/_matrix/media/r0/thumbnail/<_server_name>/<_media_id>", - data = "" - ) + get("/_matrix/media/r0/thumbnail/<_>/<_>", data = "") )] pub async fn get_content_thumbnail_route( db: State<'_, Database>, body: Ruma>, - _server_name: String, - _media_id: String, ) -> ConduitResult { + let mxc = format!("mxc://{}/{}", body.server_name, body.media_id); + if let Some(FileMeta { content_type, file, .. }) = db.media.get_thumbnail( - format!("mxc://{}/{}", body.server_name, body.media_id), + mxc.clone(), body.width .try_into() .map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "Width is invalid."))?, @@ -127,10 +115,10 @@ pub async fn get_content_thumbnail_route( .map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "Width is invalid."))?, )? { Ok(get_content_thumbnail::Response { file, content_type }.into()) - } else if body.allow_remote { + } else if &*body.server_name != db.globals.server_name() && body.allow_remote { let get_thumbnail_response = server_server::send_request( &db.globals, - body.server_name.as_ref(), + body.server_name.clone(), get_content_thumbnail::Request { allow_remote: false, height: body.height, @@ -142,12 +130,6 @@ pub async fn get_content_thumbnail_route( ) .await?; - let mxc = format!( - "mxc://{}/{}", - db.globals.server_name(), - utils::random_string(MXC_LENGTH) - ); - db.media.upload_thumbnail( mxc, &None, diff --git a/src/client_server/membership.rs b/src/client_server/membership.rs index 18fb5a9ef..f60601fd6 100644 --- a/src/client_server/membership.rs +++ b/src/client_server/membership.rs @@ -440,7 +440,7 @@ async fn join_room_by_id_helper( for remote_server in servers { let make_join_response = server_server::send_request( &db.globals, - remote_server, + remote_server.clone(), federation::membership::create_join_event_template::v1::Request { room_id, user_id: sender_id, @@ -501,7 +501,7 @@ async fn join_room_by_id_helper( let send_join_response = server_server::send_request( &db.globals, - remote_server, + remote_server.clone(), federation::membership::create_join_event::v2::Request { room_id, event_id: &event_id, diff --git a/src/database/rooms.rs b/src/database/rooms.rs index ba54e7f27..3c3a0b272 100644 --- a/src/database/rooms.rs +++ b/src/database/rooms.rs @@ -1,9 +1,10 @@ mod edus; pub use edus::RoomEdus; +use rocket::futures; use crate::{pdu::PduBuilder, server_server, utils, Error, PduEvent, Result}; -use log::error; +use log::{error, warn}; use ring::digest; use ruma::{ api::client::error::ErrorKind, @@ -833,20 +834,35 @@ impl Rooms { .expect("json is object") .remove("event_id"); - let response = server_server::send_request( - &globals, - "koesters.xyz".try_into().unwrap(), - federation::transactions::send_transaction_message::v1::Request { - origin: globals.server_name(), - pdus: &[serde_json::from_value(pdu_json).expect("Raw::from_value always works")], - edus: &[], - origin_server_ts: SystemTime::now(), - transaction_id: &utils::random_string(16), - }, - ) - .await; + let raw_json = + serde_json::from_value::>(pdu_json).expect("Raw::from_value always works"); - let _ = dbg!(response); + let pdus = &[raw_json]; + let transaction_id = utils::random_string(16); + + for result in futures::future::join_all( + self.room_servers(room_id) + .filter_map(|r| r.ok()) + .filter(|server| &**server != globals.server_name()) + .map(|server| { + server_server::send_request( + &globals, + server, + federation::transactions::send_transaction_message::v1::Request { + origin: globals.server_name(), + pdus, + edus: &[], + origin_server_ts: SystemTime::now(), + transaction_id: &transaction_id, + }, + ) + }), + ) + .await { + if let Err(e) = result { + warn!("{}", e); + } + } Ok(pdu.event_id) } diff --git a/src/server_server.rs b/src/server_server.rs index 9f4be132f..da5a6c1e2 100644 --- a/src/server_server.rs +++ b/src/server_server.rs @@ -1,5 +1,6 @@ use crate::{client_server, ConduitResult, Database, Error, PduEvent, Result, Ruma}; use http::header::{HeaderValue, AUTHORIZATION}; +use log::warn; use rocket::{get, post, put, response::content::Json, State}; use ruma::{ api::federation::directory::get_public_rooms_filtered, @@ -24,7 +25,10 @@ use std::{ time::{Duration, SystemTime}, }; -pub async fn request_well_known(globals: &crate::database::globals::Globals, destination: &str) -> Option { +pub async fn request_well_known( + globals: &crate::database::globals::Globals, + destination: &str, +) -> Option { let body: serde_json::Value = serde_json::from_str( &globals .reqwest_client() @@ -45,7 +49,7 @@ pub async fn request_well_known(globals: &crate::database::globals::Globals, des pub async fn send_request( globals: &crate::database::globals::Globals, - destination: &ServerName, + destination: Box, request: T, ) -> Result where @@ -79,10 +83,7 @@ where .to_string() .into(), ); - request_map.insert( - "origin".to_owned(), - globals.server_name().as_str().into(), - ); + request_map.insert("origin".to_owned(), globals.server_name().as_str().into()); request_map.insert("destination".to_owned(), destination.as_str().into()); let mut request_json = request_map.into(); @@ -144,10 +145,11 @@ where .into_iter() .collect(); - Ok( - T::IncomingResponse::try_from(http_response.body(body).unwrap()) - .expect("TODO: error handle other server errors"), - ) + let response = T::IncomingResponse::try_from(http_response.body(body).unwrap()); + response.map_err(|e| { + warn!("{}", e); + Error::BadServerResponse("Server returned bad response.") + }) } Err(e) => Err(e.into()), } @@ -316,10 +318,11 @@ pub fn send_transaction_message_route<'a>( .expect("ruma pdus are json objects") .insert("event_id".to_owned(), event_id.to_string().into()); - let pdu = - serde_json::from_value::(value.clone()).expect("all ruma pdus are conduit pdus"); + let pdu = serde_json::from_value::(value.clone()) + .expect("all ruma pdus are conduit pdus"); if db.rooms.exists(&pdu.room_id)? { - db.rooms.append_pdu(&pdu, &value, &db.globals, &db.account_data)?; + db.rooms + .append_pdu(&pdu, &value, &db.globals, &db.account_data)?; } } Ok(send_transaction_message::v1::Response { From 0b263208e39a735fcb6970d168494783ff9994a7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20K=C3=B6sters?= Date: Tue, 15 Sep 2020 08:55:02 +0200 Subject: [PATCH 30/51] fix: don't panic on bad server names --- src/server_server.rs | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/src/server_server.rs b/src/server_server.rs index da5a6c1e2..40ad654fc 100644 --- a/src/server_server.rs +++ b/src/server_server.rs @@ -62,14 +62,18 @@ where let mut http_request = request .try_into_http_request(&actual_destination, Some("")) - .unwrap(); + .map_err(|e| { + warn!("{}: {}", actual_destination, e); + Error::BadServerResponse("Invalid destination") + })?; let mut request_map = serde_json::Map::new(); if !http_request.body().is_empty() { request_map.insert( "content".to_owned(), - serde_json::from_slice(http_request.body()).unwrap(), + serde_json::from_slice(http_request.body()) + .expect("body is valid json, we just created it"), ); }; @@ -92,7 +96,7 @@ where globals.keypair(), &mut request_json, ) - .unwrap(); + .expect("our request json is what ruma expects"); let signatures = request_json["signatures"] .as_object() @@ -145,7 +149,11 @@ where .into_iter() .collect(); - let response = T::IncomingResponse::try_from(http_response.body(body).unwrap()); + let response = T::IncomingResponse::try_from( + http_response + .body(body) + .expect("reqwest body is valid http body"), + ); response.map_err(|e| { warn!("{}", e); Error::BadServerResponse("Server returned bad response.") From b7ab57897bc96e468421cf82ecd7d49e75c3f7dd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20K=C3=B6sters?= Date: Tue, 15 Sep 2020 16:13:54 +0200 Subject: [PATCH 31/51] fix: sending slowness --- Cargo.lock | 2 +- src/client_server/account.rs | 30 +- src/client_server/membership.rs | 6 + src/client_server/message.rs | 42 +-- src/client_server/profile.rs | 138 +++++---- src/client_server/redact.rs | 36 ++- src/client_server/room.rs | 516 ++++++++++++++++++-------------- src/client_server/state.rs | 48 +-- src/database.rs | 5 + src/database/globals.rs | 21 +- src/database/rooms.rs | 58 ++-- src/database/rooms/edus.rs | 1 + src/database/sending.rs | 83 +++++ src/main.rs | 2 + src/pdu.rs | 3 +- 15 files changed, 574 insertions(+), 417 deletions(-) create mode 100644 src/database/sending.rs diff --git a/Cargo.lock b/Cargo.lock index e0de2a7bc..30144ca54 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1831,7 +1831,7 @@ checksum = "7345c971d1ef21ffdbd103a75990a15eb03604fc8b8852ca8cb418ee1a099028" [[package]] name = "state-res" version = "0.1.0" -source = "git+https://github.com/timokoesters/state-res?branch=spec-comp#a9186476b748c901fbf4356414247a0b3ac01b5f" +source = "git+https://github.com/timokoesters/state-res?branch=spec-comp#1d01b6e65b6afd50e65085fb40f1e7d2782f519e" dependencies = [ "itertools", "js_int", diff --git a/src/client_server/account.rs b/src/client_server/account.rs index 2ec9282fe..7e0f942e6 100644 --- a/src/client_server/account.rs +++ b/src/client_server/account.rs @@ -354,19 +354,23 @@ pub async fn deactivate_route( third_party_invite: None, }; - db.rooms.build_and_append_pdu( - PduBuilder { - event_type: EventType::RoomMember, - content: serde_json::to_value(event).expect("event is valid, we just created it"), - unsigned: None, - state_key: Some(sender_id.to_string()), - redacts: None, - }, - &sender_id, - &room_id, - &db.globals, - &db.account_data, - ).await?; + db.rooms + .build_and_append_pdu( + PduBuilder { + event_type: EventType::RoomMember, + content: serde_json::to_value(event) + .expect("event is valid, we just created it"), + unsigned: None, + state_key: Some(sender_id.to_string()), + redacts: None, + }, + &sender_id, + &room_id, + &db.globals, + &db.sending, + &db.account_data, + ) + .await?; } // Remove devices and mark account as deactivated diff --git a/src/client_server/membership.rs b/src/client_server/membership.rs index f60601fd6..c4eed95c1 100644 --- a/src/client_server/membership.rs +++ b/src/client_server/membership.rs @@ -120,6 +120,7 @@ pub async fn leave_room_route( &sender_id, &body.room_id, &db.globals, + &db.sending, &db.account_data, ) .await?; @@ -157,6 +158,7 @@ pub async fn invite_user_route( &sender_id, &body.room_id, &db.globals, + &db.sending, &db.account_data, ) .await?; @@ -209,6 +211,7 @@ pub async fn kick_user_route( &sender_id, &body.room_id, &db.globals, + &db.sending, &db.account_data, ) .await?; @@ -266,6 +269,7 @@ pub async fn ban_user_route( &sender_id, &body.room_id, &db.globals, + &db.sending, &db.account_data, ) .await?; @@ -314,6 +318,7 @@ pub async fn unban_user_route( &sender_id, &body.room_id, &db.globals, + &db.sending, &db.account_data, ) .await?; @@ -672,6 +677,7 @@ async fn join_room_by_id_helper( &sender_id, &room_id, &db.globals, + &db.sending, &db.account_data, ) .await?; diff --git a/src/client_server/message.rs b/src/client_server/message.rs index 4ba0d9fd8..3944d5bdf 100644 --- a/src/client_server/message.rs +++ b/src/client_server/message.rs @@ -49,25 +49,29 @@ pub async fn send_message_event_route( let mut unsigned = serde_json::Map::new(); unsigned.insert("transaction_id".to_owned(), body.txn_id.clone().into()); - let event_id = db.rooms.build_and_append_pdu( - PduBuilder { - event_type: body.content.event_type().into(), - content: serde_json::from_str( - body.json_body - .as_ref() - .ok_or(Error::BadRequest(ErrorKind::BadJson, "Invalid JSON body."))? - .get(), - ) - .map_err(|_| Error::BadRequest(ErrorKind::BadJson, "Invalid JSON body."))?, - unsigned: Some(unsigned), - state_key: None, - redacts: None, - }, - &sender_id, - &body.room_id, - &db.globals, - &db.account_data, - ).await?; + let event_id = db + .rooms + .build_and_append_pdu( + PduBuilder { + event_type: body.content.event_type().into(), + content: serde_json::from_str( + body.json_body + .as_ref() + .ok_or(Error::BadRequest(ErrorKind::BadJson, "Invalid JSON body."))? + .get(), + ) + .map_err(|_| Error::BadRequest(ErrorKind::BadJson, "Invalid JSON body."))?, + unsigned: Some(unsigned), + state_key: None, + redacts: None, + }, + &sender_id, + &body.room_id, + &db.globals, + &db.sending, + &db.account_data, + ) + .await?; db.transaction_ids .add_txnid(sender_id, device_id, &body.txn_id, event_id.as_bytes())?; diff --git a/src/client_server/profile.rs b/src/client_server/profile.rs index be893e1b5..53893c0fb 100644 --- a/src/client_server/profile.rs +++ b/src/client_server/profile.rs @@ -31,40 +31,43 @@ pub async fn set_displayname_route( // Send a new membership event and presence update into all joined rooms for room_id in db.rooms.rooms_joined(&sender_id) { let room_id = room_id?; - db.rooms.build_and_append_pdu( - PduBuilder { - event_type: EventType::RoomMember, - content: serde_json::to_value(ruma::events::room::member::MemberEventContent { - displayname: body.displayname.clone(), - ..serde_json::from_value::>( - db.rooms - .room_state_get( - &room_id, - &EventType::RoomMember, - &sender_id.to_string(), - )? - .ok_or_else(|| { - Error::bad_database( + db.rooms + .build_and_append_pdu( + PduBuilder { + event_type: EventType::RoomMember, + content: serde_json::to_value(ruma::events::room::member::MemberEventContent { + displayname: body.displayname.clone(), + ..serde_json::from_value::>( + db.rooms + .room_state_get( + &room_id, + &EventType::RoomMember, + &sender_id.to_string(), + )? + .ok_or_else(|| { + Error::bad_database( "Tried to send displayname update for user not in the room.", ) - })? - .content - .clone(), - ) - .expect("from_value::> can never fail") - .deserialize() - .map_err(|_| Error::bad_database("Database contains invalid PDU."))? - }) - .expect("event is valid, we just created it"), - unsigned: None, - state_key: Some(sender_id.to_string()), - redacts: None, - }, - &sender_id, - &room_id, - &db.globals, - &db.account_data, - ).await?; + })? + .content + .clone(), + ) + .expect("from_value::> can never fail") + .deserialize() + .map_err(|_| Error::bad_database("Database contains invalid PDU."))? + }) + .expect("event is valid, we just created it"), + unsigned: None, + state_key: Some(sender_id.to_string()), + redacts: None, + }, + &sender_id, + &room_id, + &db.globals, + &db.sending, + &db.account_data, + ) + .await?; // Presence update db.rooms.edus.update_presence( @@ -134,40 +137,43 @@ pub async fn set_avatar_url_route( // Send a new membership event and presence update into all joined rooms for room_id in db.rooms.rooms_joined(&sender_id) { let room_id = room_id?; - db.rooms.build_and_append_pdu( - PduBuilder { - event_type: EventType::RoomMember, - content: serde_json::to_value(ruma::events::room::member::MemberEventContent { - avatar_url: body.avatar_url.clone(), - ..serde_json::from_value::>( - db.rooms - .room_state_get( - &room_id, - &EventType::RoomMember, - &sender_id.to_string(), - )? - .ok_or_else(|| { - Error::bad_database( - "Tried to send avatar url update for user not in the room.", - ) - })? - .content - .clone(), - ) - .expect("from_value::> can never fail") - .deserialize() - .map_err(|_| Error::bad_database("Database contains invalid PDU."))? - }) - .expect("event is valid, we just created it"), - unsigned: None, - state_key: Some(sender_id.to_string()), - redacts: None, - }, - &sender_id, - &room_id, - &db.globals, - &db.account_data, - ).await?; + db.rooms + .build_and_append_pdu( + PduBuilder { + event_type: EventType::RoomMember, + content: serde_json::to_value(ruma::events::room::member::MemberEventContent { + avatar_url: body.avatar_url.clone(), + ..serde_json::from_value::>( + db.rooms + .room_state_get( + &room_id, + &EventType::RoomMember, + &sender_id.to_string(), + )? + .ok_or_else(|| { + Error::bad_database( + "Tried to send avatar url update for user not in the room.", + ) + })? + .content + .clone(), + ) + .expect("from_value::> can never fail") + .deserialize() + .map_err(|_| Error::bad_database("Database contains invalid PDU."))? + }) + .expect("event is valid, we just created it"), + unsigned: None, + state_key: Some(sender_id.to_string()), + redacts: None, + }, + &sender_id, + &room_id, + &db.globals, + &db.sending, + &db.account_data, + ) + .await?; // Presence update db.rooms.edus.update_presence( diff --git a/src/client_server/redact.rs b/src/client_server/redact.rs index 701fc00de..24df8dd7d 100644 --- a/src/client_server/redact.rs +++ b/src/client_server/redact.rs @@ -18,22 +18,26 @@ pub async fn redact_event_route( ) -> ConduitResult { let sender_id = body.sender_id.as_ref().expect("user is authenticated"); - let event_id = db.rooms.build_and_append_pdu( - PduBuilder { - event_type: EventType::RoomRedaction, - content: serde_json::to_value(redaction::RedactionEventContent { - reason: body.reason.clone(), - }) - .expect("event is valid, we just created it"), - unsigned: None, - state_key: None, - redacts: Some(body.event_id.clone()), - }, - &sender_id, - &body.room_id, - &db.globals, - &db.account_data, - ).await?; + let event_id = db + .rooms + .build_and_append_pdu( + PduBuilder { + event_type: EventType::RoomRedaction, + content: serde_json::to_value(redaction::RedactionEventContent { + reason: body.reason.clone(), + }) + .expect("event is valid, we just created it"), + unsigned: None, + state_key: None, + redacts: Some(body.event_id.clone()), + }, + &sender_id, + &body.room_id, + &db.globals, + &db.sending, + &db.account_data, + ) + .await?; Ok(redact_event::Response { event_id }.into()) } diff --git a/src/client_server/room.rs b/src/client_server/room.rs index 0e5c5716e..d21148bb7 100644 --- a/src/client_server/room.rs +++ b/src/client_server/room.rs @@ -53,41 +53,47 @@ pub async fn create_room_route( content.room_version = RoomVersionId::Version6; // 1. The room create event - db.rooms.build_and_append_pdu( - PduBuilder { - event_type: EventType::RoomCreate, - content: serde_json::to_value(content).expect("event is valid, we just created it"), - unsigned: None, - state_key: Some("".to_owned()), - redacts: None, - }, - &sender_id, - &room_id, - &db.globals, - &db.account_data, - ).await?; + db.rooms + .build_and_append_pdu( + PduBuilder { + event_type: EventType::RoomCreate, + content: serde_json::to_value(content).expect("event is valid, we just created it"), + unsigned: None, + state_key: Some("".to_owned()), + redacts: None, + }, + &sender_id, + &room_id, + &db.globals, + &db.sending, + &db.account_data, + ) + .await?; // 2. Let the room creator join - db.rooms.build_and_append_pdu( - PduBuilder { - event_type: EventType::RoomMember, - content: serde_json::to_value(member::MemberEventContent { - membership: member::MembershipState::Join, - displayname: db.users.displayname(&sender_id)?, - avatar_url: db.users.avatar_url(&sender_id)?, - is_direct: Some(body.is_direct), - third_party_invite: None, - }) - .expect("event is valid, we just created it"), - unsigned: None, - state_key: Some(sender_id.to_string()), - redacts: None, - }, - &sender_id, - &room_id, - &db.globals, - &db.account_data, - ).await?; + db.rooms + .build_and_append_pdu( + PduBuilder { + event_type: EventType::RoomMember, + content: serde_json::to_value(member::MemberEventContent { + membership: member::MembershipState::Join, + displayname: db.users.displayname(&sender_id)?, + avatar_url: db.users.avatar_url(&sender_id)?, + is_direct: Some(body.is_direct), + third_party_invite: None, + }) + .expect("event is valid, we just created it"), + unsigned: None, + state_key: Some(sender_id.to_string()), + redacts: None, + }, + &sender_id, + &room_id, + &db.globals, + &db.sending, + &db.account_data, + ) + .await?; // 3. Power levels let mut users = BTreeMap::new(); @@ -117,19 +123,22 @@ pub async fn create_room_route( }) .expect("event is valid, we just created it") }; - db.rooms.build_and_append_pdu( - PduBuilder { - event_type: EventType::RoomPowerLevels, - content: power_levels_content, - unsigned: None, - state_key: Some("".to_owned()), - redacts: None, - }, - &sender_id, - &room_id, - &db.globals, - &db.account_data, - ).await?; + db.rooms + .build_and_append_pdu( + PduBuilder { + event_type: EventType::RoomPowerLevels, + content: power_levels_content, + unsigned: None, + state_key: Some("".to_owned()), + redacts: None, + }, + &sender_id, + &room_id, + &db.globals, + &db.sending, + &db.account_data, + ) + .await?; // 4. Events set by preset @@ -140,73 +149,84 @@ pub async fn create_room_route( }); // 4.1 Join Rules - db.rooms.build_and_append_pdu( - PduBuilder { - event_type: EventType::RoomJoinRules, - content: match preset { - create_room::RoomPreset::PublicChat => serde_json::to_value( - join_rules::JoinRulesEventContent::new(join_rules::JoinRule::Public), - ) - .expect("event is valid, we just created it"), - // according to spec "invite" is the default - _ => serde_json::to_value(join_rules::JoinRulesEventContent::new( - join_rules::JoinRule::Invite, - )) - .expect("event is valid, we just created it"), + db.rooms + .build_and_append_pdu( + PduBuilder { + event_type: EventType::RoomJoinRules, + content: match preset { + create_room::RoomPreset::PublicChat => serde_json::to_value( + join_rules::JoinRulesEventContent::new(join_rules::JoinRule::Public), + ) + .expect("event is valid, we just created it"), + // according to spec "invite" is the default + _ => serde_json::to_value(join_rules::JoinRulesEventContent::new( + join_rules::JoinRule::Invite, + )) + .expect("event is valid, we just created it"), + }, + unsigned: None, + state_key: Some("".to_owned()), + redacts: None, }, - unsigned: None, - state_key: Some("".to_owned()), - redacts: None, - }, - &sender_id, - &room_id, - &db.globals, - &db.account_data, - ).await?; + &sender_id, + &room_id, + &db.globals, + &db.sending, + &db.account_data, + ) + .await?; // 4.2 History Visibility - db.rooms.build_and_append_pdu( - PduBuilder { - event_type: EventType::RoomHistoryVisibility, - content: serde_json::to_value(history_visibility::HistoryVisibilityEventContent::new( - history_visibility::HistoryVisibility::Shared, - )) - .expect("event is valid, we just created it"), - unsigned: None, - state_key: Some("".to_owned()), - redacts: None, - }, - &sender_id, - &room_id, - &db.globals, - &db.account_data, - ).await?; + db.rooms + .build_and_append_pdu( + PduBuilder { + event_type: EventType::RoomHistoryVisibility, + content: serde_json::to_value( + history_visibility::HistoryVisibilityEventContent::new( + history_visibility::HistoryVisibility::Shared, + ), + ) + .expect("event is valid, we just created it"), + unsigned: None, + state_key: Some("".to_owned()), + redacts: None, + }, + &sender_id, + &room_id, + &db.globals, + &db.sending, + &db.account_data, + ) + .await?; // 4.3 Guest Access - db.rooms.build_and_append_pdu( - PduBuilder { - event_type: EventType::RoomGuestAccess, - content: match preset { - create_room::RoomPreset::PublicChat => { - serde_json::to_value(guest_access::GuestAccessEventContent::new( - guest_access::GuestAccess::Forbidden, + db.rooms + .build_and_append_pdu( + PduBuilder { + event_type: EventType::RoomGuestAccess, + content: match preset { + create_room::RoomPreset::PublicChat => { + serde_json::to_value(guest_access::GuestAccessEventContent::new( + guest_access::GuestAccess::Forbidden, + )) + .expect("event is valid, we just created it") + } + _ => serde_json::to_value(guest_access::GuestAccessEventContent::new( + guest_access::GuestAccess::CanJoin, )) - .expect("event is valid, we just created it") - } - _ => serde_json::to_value(guest_access::GuestAccessEventContent::new( - guest_access::GuestAccess::CanJoin, - )) - .expect("event is valid, we just created it"), + .expect("event is valid, we just created it"), + }, + unsigned: None, + state_key: Some("".to_owned()), + redacts: None, }, - unsigned: None, - state_key: Some("".to_owned()), - redacts: None, - }, - &sender_id, - &room_id, - &db.globals, - &db.account_data, - ).await?; + &sender_id, + &room_id, + &db.globals, + &db.sending, + &db.account_data, + ) + .await?; // 5. Events listed in initial_state for event in &body.initial_state { @@ -220,78 +240,90 @@ pub async fn create_room_route( continue; } - db.rooms.build_and_append_pdu( - pdu_builder, - &sender_id, - &room_id, - &db.globals, - &db.account_data, - ).await?; + db.rooms + .build_and_append_pdu( + pdu_builder, + &sender_id, + &room_id, + &db.globals, + &db.sending, + &db.account_data, + ) + .await?; } // 6. Events implied by name and topic if let Some(name) = &body.name { - db.rooms.build_and_append_pdu( - PduBuilder { - event_type: EventType::RoomName, - content: serde_json::to_value( - name::NameEventContent::new(name.clone()).map_err(|_| { - Error::BadRequest(ErrorKind::InvalidParam, "Name is invalid.") - })?, - ) - .expect("event is valid, we just created it"), - unsigned: None, - state_key: Some("".to_owned()), - redacts: None, - }, - &sender_id, - &room_id, - &db.globals, - &db.account_data, - ).await?; + db.rooms + .build_and_append_pdu( + PduBuilder { + event_type: EventType::RoomName, + content: serde_json::to_value( + name::NameEventContent::new(name.clone()).map_err(|_| { + Error::BadRequest(ErrorKind::InvalidParam, "Name is invalid.") + })?, + ) + .expect("event is valid, we just created it"), + unsigned: None, + state_key: Some("".to_owned()), + redacts: None, + }, + &sender_id, + &room_id, + &db.globals, + &db.sending, + &db.account_data, + ) + .await?; } if let Some(topic) = &body.topic { - db.rooms.build_and_append_pdu( - PduBuilder { - event_type: EventType::RoomTopic, - content: serde_json::to_value(topic::TopicEventContent { - topic: topic.clone(), - }) - .expect("event is valid, we just created it"), - unsigned: None, - state_key: Some("".to_owned()), - redacts: None, - }, - &sender_id, - &room_id, - &db.globals, - &db.account_data, - ).await?; + db.rooms + .build_and_append_pdu( + PduBuilder { + event_type: EventType::RoomTopic, + content: serde_json::to_value(topic::TopicEventContent { + topic: topic.clone(), + }) + .expect("event is valid, we just created it"), + unsigned: None, + state_key: Some("".to_owned()), + redacts: None, + }, + &sender_id, + &room_id, + &db.globals, + &db.sending, + &db.account_data, + ) + .await?; } // 7. Events implied by invite (and TODO: invite_3pid) for user in &body.invite { - db.rooms.build_and_append_pdu( - PduBuilder { - event_type: EventType::RoomMember, - content: serde_json::to_value(member::MemberEventContent { - membership: member::MembershipState::Invite, - displayname: db.users.displayname(&user)?, - avatar_url: db.users.avatar_url(&user)?, - is_direct: Some(body.is_direct), - third_party_invite: None, - }) - .expect("event is valid, we just created it"), - unsigned: None, - state_key: Some(user.to_string()), - redacts: None, - }, - &sender_id, - &room_id, - &db.globals, - &db.account_data, - ).await?; + db.rooms + .build_and_append_pdu( + PduBuilder { + event_type: EventType::RoomMember, + content: serde_json::to_value(member::MemberEventContent { + membership: member::MembershipState::Invite, + displayname: db.users.displayname(&user)?, + avatar_url: db.users.avatar_url(&user)?, + is_direct: Some(body.is_direct), + third_party_invite: None, + }) + .expect("event is valid, we just created it"), + unsigned: None, + state_key: Some(user.to_string()), + redacts: None, + }, + &sender_id, + &room_id, + &db.globals, + &db.sending, + &db.account_data, + ) + .await?; } // Homeserver specific stuff @@ -363,23 +395,29 @@ pub async fn upgrade_room_route( // Send a m.room.tombstone event to the old room to indicate that it is not intended to be used any further // Fail if the sender does not have the required permissions - let tombstone_event_id = db.rooms.build_and_append_pdu( - PduBuilder { - event_type: EventType::RoomTombstone, - content: serde_json::to_value(ruma::events::room::tombstone::TombstoneEventContent { - body: "This room has been replaced".to_string(), - replacement_room: replacement_room.clone(), - }) - .expect("event is valid, we just created it"), - unsigned: None, - state_key: Some("".to_owned()), - redacts: None, - }, - sender_id, - &body.room_id, - &db.globals, - &db.account_data, - ).await?; + let tombstone_event_id = db + .rooms + .build_and_append_pdu( + PduBuilder { + event_type: EventType::RoomTombstone, + content: serde_json::to_value( + ruma::events::room::tombstone::TombstoneEventContent { + body: "This room has been replaced".to_string(), + replacement_room: replacement_room.clone(), + }, + ) + .expect("event is valid, we just created it"), + unsigned: None, + state_key: Some("".to_owned()), + redacts: None, + }, + sender_id, + &body.room_id, + &db.globals, + &db.sending, + &db.account_data, + ) + .await?; // Get the old room federations status let federate = serde_json::from_value::>( @@ -406,42 +444,48 @@ pub async fn upgrade_room_route( create_event_content.room_version = new_version; create_event_content.predecessor = predecessor; - db.rooms.build_and_append_pdu( - PduBuilder { - event_type: EventType::RoomCreate, - content: serde_json::to_value(create_event_content) - .expect("event is valid, we just created it"), - unsigned: None, - state_key: Some("".to_owned()), - redacts: None, - }, - sender_id, - &replacement_room, - &db.globals, - &db.account_data, - ).await?; + db.rooms + .build_and_append_pdu( + PduBuilder { + event_type: EventType::RoomCreate, + content: serde_json::to_value(create_event_content) + .expect("event is valid, we just created it"), + unsigned: None, + state_key: Some("".to_owned()), + redacts: None, + }, + sender_id, + &replacement_room, + &db.globals, + &db.sending, + &db.account_data, + ) + .await?; // Join the new room - db.rooms.build_and_append_pdu( - PduBuilder { - event_type: EventType::RoomMember, - content: serde_json::to_value(member::MemberEventContent { - membership: member::MembershipState::Join, - displayname: db.users.displayname(&sender_id)?, - avatar_url: db.users.avatar_url(&sender_id)?, - is_direct: None, - third_party_invite: None, - }) - .expect("event is valid, we just created it"), - unsigned: None, - state_key: Some(sender_id.to_string()), - redacts: None, - }, - sender_id, - &replacement_room, - &db.globals, - &db.account_data, - ).await?; + db.rooms + .build_and_append_pdu( + PduBuilder { + event_type: EventType::RoomMember, + content: serde_json::to_value(member::MemberEventContent { + membership: member::MembershipState::Join, + displayname: db.users.displayname(&sender_id)?, + avatar_url: db.users.avatar_url(&sender_id)?, + is_direct: None, + third_party_invite: None, + }) + .expect("event is valid, we just created it"), + unsigned: None, + state_key: Some(sender_id.to_string()), + redacts: None, + }, + sender_id, + &replacement_room, + &db.globals, + &db.sending, + &db.account_data, + ) + .await?; // Recommended transferable state events list from the specs let transferable_state_events = vec![ @@ -463,19 +507,22 @@ pub async fn upgrade_room_route( None => continue, // Skipping missing events. }; - db.rooms.build_and_append_pdu( - PduBuilder { - event_type, - content: event_content, - unsigned: None, - state_key: Some("".to_owned()), - redacts: None, - }, - sender_id, - &replacement_room, - &db.globals, - &db.account_data, - ).await?; + db.rooms + .build_and_append_pdu( + PduBuilder { + event_type, + content: event_content, + unsigned: None, + state_key: Some("".to_owned()), + redacts: None, + }, + sender_id, + &replacement_room, + &db.globals, + &db.sending, + &db.account_data, + ) + .await?; } // Moves any local aliases to the new room @@ -505,7 +552,8 @@ pub async fn upgrade_room_route( power_levels_event_content.invite = new_level; // Modify the power levels in the old room to prevent sending of events and inviting new users - let _ = db.rooms + let _ = db + .rooms .build_and_append_pdu( PduBuilder { event_type: EventType::RoomPowerLevels, @@ -518,8 +566,10 @@ pub async fn upgrade_room_route( sender_id, &body.room_id, &db.globals, + &db.sending, &db.account_data, - ).await; + ) + .await; // Return the replacement room id Ok(upgrade_room::Response { replacement_room }.into()) diff --git a/src/client_server/state.rs b/src/client_server/state.rs index e9d20e2f6..46182a12e 100644 --- a/src/client_server/state.rs +++ b/src/client_server/state.rs @@ -33,17 +33,18 @@ pub async fn send_state_event_for_key_route( ) .map_err(|_| Error::BadRequest(ErrorKind::BadJson, "Invalid JSON body."))?; - Ok( - send_state_event_for_key::Response::new(send_state_event_for_key_helper( + Ok(send_state_event_for_key::Response::new( + send_state_event_for_key_helper( &db, sender_id, &body.content, content, &body.room_id, Some(body.state_key.to_owned()), - ).await?) - .into(), + ) + .await?, ) + .into()) } #[cfg_attr( @@ -70,8 +71,8 @@ pub async fn send_state_event_for_empty_key_route( ) .map_err(|_| Error::BadRequest(ErrorKind::BadJson, "Invalid JSON body."))?; - Ok( - send_state_event_for_empty_key::Response::new(send_state_event_for_key_helper( + Ok(send_state_event_for_empty_key::Response::new( + send_state_event_for_key_helper( &db, sender_id .as_ref() @@ -80,9 +81,10 @@ pub async fn send_state_event_for_empty_key_route( json, &body.room_id, Some("".into()), - ).await?) - .into(), + ) + .await?, ) + .into()) } #[cfg_attr( @@ -211,19 +213,23 @@ pub async fn send_state_event_for_key_helper( } } - let event_id = db.rooms.build_and_append_pdu( - PduBuilder { - event_type: content.event_type().into(), - content: json, - unsigned: None, - state_key, - redacts: None, - }, - &sender_id, - &room_id, - &db.globals, - &db.account_data, - ).await?; + let event_id = db + .rooms + .build_and_append_pdu( + PduBuilder { + event_type: content.event_type().into(), + content: json, + unsigned: None, + state_key, + redacts: None, + }, + &sender_id, + &room_id, + &db.globals, + &db.sending, + &db.account_data, + ) + .await?; Ok(event_id) } diff --git a/src/database.rs b/src/database.rs index e1a356c78..4b2cba10a 100644 --- a/src/database.rs +++ b/src/database.rs @@ -3,6 +3,7 @@ pub mod globals; pub mod key_backups; pub mod media; pub mod rooms; +pub mod sending; pub mod transaction_ids; pub mod uiaa; pub mod users; @@ -25,6 +26,7 @@ pub struct Database { pub media: media::Media, pub key_backups: key_backups::KeyBackups, pub transaction_ids: transaction_ids::TransactionIds, + pub sending: sending::Sending, pub _db: sled::Db, } @@ -135,6 +137,9 @@ impl Database { transaction_ids: transaction_ids::TransactionIds { userdevicetxnid_response: db.open_tree("userdevicetxnid_response")?, }, + sending: sending::Sending { + serverpduids: db.open_tree("serverpduids")?, + }, _db: db, }) } diff --git a/src/database/globals.rs b/src/database/globals.rs index 5db28069f..895142515 100644 --- a/src/database/globals.rs +++ b/src/database/globals.rs @@ -1,12 +1,13 @@ use crate::{utils, Error, Result}; use ruma::ServerName; -use std::convert::TryInto; +use std::{convert::TryInto, sync::Arc}; pub const COUNTER: &str = "c"; +#[derive(Clone)] pub struct Globals { pub(super) globals: sled::Tree, - keypair: ruma::signatures::Ed25519KeyPair, + keypair: Arc, reqwest_client: reqwest::Client, server_name: Box, max_request_size: u32, @@ -16,13 +17,15 @@ pub struct Globals { impl Globals { pub fn load(globals: sled::Tree, config: &rocket::Config) -> Result { - let keypair = ruma::signatures::Ed25519KeyPair::new( - &*globals - .update_and_fetch("keypair", utils::generate_keypair)? - .expect("utils::generate_keypair always returns Some"), - "key1".to_owned(), - ) - .map_err(|_| Error::bad_database("Private or public keys are invalid."))?; + let keypair = Arc::new( + ruma::signatures::Ed25519KeyPair::new( + &*globals + .update_and_fetch("keypair", utils::generate_keypair)? + .expect("utils::generate_keypair always returns Some"), + "key1".to_owned(), + ) + .map_err(|_| Error::bad_database("Private or public keys are invalid."))?, + ); Ok(Self { globals, diff --git a/src/database/rooms.rs b/src/database/rooms.rs index 3c3a0b272..2246a61fe 100644 --- a/src/database/rooms.rs +++ b/src/database/rooms.rs @@ -1,14 +1,12 @@ mod edus; pub use edus::RoomEdus; -use rocket::futures; -use crate::{pdu::PduBuilder, server_server, utils, Error, PduEvent, Result}; -use log::{error, warn}; +use crate::{pdu::PduBuilder, utils, Error, PduEvent, Result}; +use log::error; use ring::digest; use ruma::{ api::client::error::ErrorKind, - api::federation, events::{ ignored_user_list, room::{ @@ -27,7 +25,6 @@ use std::{ convert::{TryFrom, TryInto}, mem, sync::Arc, - time::SystemTime, }; /// The unique identifier of each state group. @@ -36,6 +33,7 @@ use std::{ /// hashing the entire state. pub type StateHashId = Vec; +#[derive(Clone)] pub struct Rooms { pub edus: edus::RoomEdus, pub(super) pduid_pdu: sled::Tree, // PduId = RoomId + Count @@ -415,6 +413,16 @@ impl Rooms { }) } + /// Returns the pdu. + pub fn get_pdu_json_from_id(&self, pdu_id: &IVec) -> Result> { + self.pduid_pdu.get(pdu_id)?.map_or(Ok(None), |pdu| { + Ok(Some( + serde_json::from_slice(&pdu) + .map_err(|_| Error::bad_database("Invalid PDU in db."))?, + )) + }) + } + /// Removes a pdu and creates a new one with the same id. fn replace_pdu(&self, pdu_id: &IVec, pdu: &PduEvent) -> Result<()> { if self.pduid_pdu.get(&pdu_id)?.is_some() { @@ -613,6 +621,7 @@ impl Rooms { sender: &UserId, room_id: &RoomId, globals: &super::globals::Globals, + sending: &super::sending::Sending, account_data: &super::account_data::AccountData, ) -> Result { let PduBuilder { @@ -829,39 +838,12 @@ impl Rooms { self.append_to_state(&pdu_id, &pdu)?; } - pdu_json - .as_object_mut() - .expect("json is object") - .remove("event_id"); - - let raw_json = - serde_json::from_value::>(pdu_json).expect("Raw::from_value always works"); - - let pdus = &[raw_json]; - let transaction_id = utils::random_string(16); - - for result in futures::future::join_all( - self.room_servers(room_id) - .filter_map(|r| r.ok()) - .filter(|server| &**server != globals.server_name()) - .map(|server| { - server_server::send_request( - &globals, - server, - federation::transactions::send_transaction_message::v1::Request { - origin: globals.server_name(), - pdus, - edus: &[], - origin_server_ts: SystemTime::now(), - transaction_id: &transaction_id, - }, - ) - }), - ) - .await { - if let Err(e) = result { - warn!("{}", e); - } + for server in self + .room_servers(room_id) + .filter_map(|r| r.ok()) + .filter(|server| &**server != globals.server_name()) + { + sending.send_pdu(server, &pdu_id)?; } Ok(pdu.event_id) diff --git a/src/database/rooms/edus.rs b/src/database/rooms/edus.rs index d60e1f168..a794c690a 100644 --- a/src/database/rooms/edus.rs +++ b/src/database/rooms/edus.rs @@ -13,6 +13,7 @@ use std::{ convert::{TryFrom, TryInto}, }; +#[derive(Clone)] pub struct RoomEdus { pub(in super::super) readreceiptid_readreceipt: sled::Tree, // ReadReceiptId = RoomId + Count + UserId pub(in super::super) roomuserid_privateread: sled::Tree, // RoomUserId = Room + User, PrivateRead = Count diff --git a/src/database/sending.rs b/src/database/sending.rs new file mode 100644 index 000000000..187fd5754 --- /dev/null +++ b/src/database/sending.rs @@ -0,0 +1,83 @@ +use std::{convert::TryFrom, time::SystemTime}; + +use crate::{server_server, utils, Error, Result}; +use rocket::futures::stream::{FuturesUnordered, StreamExt}; +use ruma::{api::federation, Raw, ServerName}; +use tokio::select; + +pub struct Sending { + /// The state for a given state hash. + pub(super) serverpduids: sled::Tree, // ServerPduId = ServerName + PduId +} + +impl Sending { + pub fn start_handler(&self, globals: &super::globals::Globals, rooms: &super::rooms::Rooms) { + let serverpduids = self.serverpduids.clone(); + let rooms = rooms.clone(); + let globals = globals.clone(); + + tokio::spawn(async move { + let mut futures = FuturesUnordered::new(); + let mut subscriber = serverpduids.watch_prefix(b""); + loop { + select! { + Some(_) = futures.next() => {}, + Some(event) = &mut subscriber => { + let serverpduid = if let sled::Event::Insert {key, ..} = event { + key + } else + { return Err::<(), Error>(Error::bad_database("")); }; + let mut parts = serverpduid.splitn(2, |&b| b == 0xff); + let server = Box::::try_from( + utils::string_from_bytes(parts.next().expect("splitn will always return 1 or more elements")) + .map_err(|_| Error::bad_database("ServerName in serverpduid bytes are invalid."))? + ).map_err(|_| Error::bad_database("ServerName in serverpduid is invalid."))?; + + let pdu_id = parts.next().ok_or_else(|| Error::bad_database("Invalid serverpduid in db."))?; + let mut pdu_json = rooms.get_pdu_json_from_id(&pdu_id.into())?.ok_or_else(|| Error::bad_database("Event in serverpduids not found in db."))?; + + pdu_json + .as_object_mut() + .expect("json is object") + .remove("event_id"); + + let raw_json = + serde_json::from_value::>(pdu_json).expect("Raw::from_value always works"); + + let globals = &globals; + + futures.push( + async move { + let pdus = vec![raw_json]; + let transaction_id = utils::random_string(16); + + server_server::send_request( + &globals, + server, + federation::transactions::send_transaction_message::v1::Request { + origin: globals.server_name(), + pdus: &pdus, + edus: &[], + origin_server_ts: SystemTime::now(), + transaction_id: &transaction_id, + }, + ).await + } + ); + }, + } + } + }); + } + /* + */ + + pub fn send_pdu(&self, server: Box, pdu_id: &[u8]) -> Result<()> { + let mut key = server.as_bytes().to_vec(); + key.push(0xff); + key.extend_from_slice(pdu_id); + self.serverpduids.insert(key, b"")?; + + Ok(()) + } +} diff --git a/src/main.rs b/src/main.rs index eb060e3ec..2817ab974 100644 --- a/src/main.rs +++ b/src/main.rs @@ -130,6 +130,8 @@ fn setup_rocket() -> rocket::Rocket { .attach(AdHoc::on_attach("Config", |mut rocket| async { let data = Database::load_or_create(rocket.config().await).expect("valid config"); + data.sending.start_handler(&data.globals, &data.rooms); + Ok(rocket.manage(data)) })) } diff --git a/src/pdu.rs b/src/pdu.rs index c9042306e..6d78092f9 100644 --- a/src/pdu.rs +++ b/src/pdu.rs @@ -1,12 +1,13 @@ use crate::{Error, Result}; use js_int::UInt; use ruma::{ + events::pdu::PduStub, events::{ pdu::EventHash, room::member::MemberEventContent, AnyEvent, AnyRoomEvent, AnyStateEvent, AnyStrippedStateEvent, AnySyncRoomEvent, AnySyncStateEvent, EventType, StateEvent, }, EventId, Raw, RoomId, ServerKeyId, ServerName, UserId, -events::pdu::PduStub}; +}; use serde::{Deserialize, Serialize}; use serde_json::json; use std::{collections::BTreeMap, convert::TryInto, sync::Arc, time::UNIX_EPOCH}; From 1bf614b0f57a023cbc467f1e5bf03d1eae87b755 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20K=C3=B6sters?= Date: Tue, 15 Sep 2020 17:02:20 +0200 Subject: [PATCH 32/51] fix: remove transaction_id from pdus over federation --- src/database/sending.rs | 6 ++++++ src/pdu.rs | 5 ++++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/src/database/sending.rs b/src/database/sending.rs index 187fd5754..77998e79b 100644 --- a/src/database/sending.rs +++ b/src/database/sending.rs @@ -36,6 +36,12 @@ impl Sending { let pdu_id = parts.next().ok_or_else(|| Error::bad_database("Invalid serverpduid in db."))?; let mut pdu_json = rooms.get_pdu_json_from_id(&pdu_id.into())?.ok_or_else(|| Error::bad_database("Event in serverpduids not found in db."))?; + if let Some(unsigned) = pdu_json + .as_object_mut() + .expect("json is object") + .get_mut("unsigned") { + unsigned.as_object_mut().expect("unsigned is object").remove("transaction_id"); + } pdu_json .as_object_mut() .expect("json is object") diff --git a/src/pdu.rs b/src/pdu.rs index 6d78092f9..957d9e0a2 100644 --- a/src/pdu.rs +++ b/src/pdu.rs @@ -201,6 +201,9 @@ impl PduEvent { } pub fn to_outgoing_federation_event(&self) -> Raw { + let mut unsigned = self.unsigned.clone(); + unsigned.remove("transaction_id"); + let mut json = json!({ "room_id": self.room_id, "sender": self.sender, @@ -210,7 +213,7 @@ impl PduEvent { "prev_events": self.prev_events, "depth": self.depth, "auth_events": self.auth_events, - "unsigned": self.unsigned, + "unsigned": unsigned, "hashes": self.hashes, "signatures": self.signatures, }); From 005e00e9b18459d569cca1993138a85d10dfc271 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20K=C3=B6sters?= Date: Tue, 15 Sep 2020 17:16:55 +0200 Subject: [PATCH 33/51] fix: remove well-known --- src/main.rs | 1 - src/server_server.rs | 5 ----- 2 files changed, 6 deletions(-) diff --git a/src/main.rs b/src/main.rs index 2817ab974..f81c7f463 100644 --- a/src/main.rs +++ b/src/main.rs @@ -119,7 +119,6 @@ fn setup_rocket() -> rocket::Rocket { client_server::get_pushers_route, client_server::set_pushers_route, client_server::upgrade_room_route, - server_server::well_known_server, server_server::get_server_version, server_server::get_server_keys, server_server::get_server_keys_deprecated, diff --git a/src/server_server.rs b/src/server_server.rs index 40ad654fc..106f60eb2 100644 --- a/src/server_server.rs +++ b/src/server_server.rs @@ -163,11 +163,6 @@ where } } -#[cfg_attr(feature = "conduit_bin", get("/.well-known/matrix/server"))] -pub fn well_known_server() -> Json { - rocket::response::content::Json(json!({ "m.server": "pc.koesters.xyz:59003"}).to_string()) -} - #[cfg_attr(feature = "conduit_bin", get("/_matrix/federation/v1/version"))] pub fn get_server_version() -> ConduitResult { Ok(get_server_version::Response { From dd749b8aee7c09ca8084059f91cd922e95fb6424 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20K=C3=B6sters?= Date: Tue, 15 Sep 2020 21:46:10 +0200 Subject: [PATCH 34/51] fix: server keys and destination resolution when server name contains port --- src/database/globals.rs | 45 ++++++++++++++++++++++++++++++++--------- src/server_server.rs | 9 +++++++-- src/utils.rs | 9 +++++++-- 3 files changed, 49 insertions(+), 14 deletions(-) diff --git a/src/database/globals.rs b/src/database/globals.rs index 895142515..8ce9c0113 100644 --- a/src/database/globals.rs +++ b/src/database/globals.rs @@ -1,4 +1,5 @@ use crate::{utils, Error, Result}; +use log::error; use ruma::ServerName; use std::{convert::TryInto, sync::Arc}; @@ -17,19 +18,43 @@ pub struct Globals { impl Globals { pub fn load(globals: sled::Tree, config: &rocket::Config) -> Result { - let keypair = Arc::new( - ruma::signatures::Ed25519KeyPair::new( - &*globals - .update_and_fetch("keypair", utils::generate_keypair)? - .expect("utils::generate_keypair always returns Some"), - "key1".to_owned(), - ) - .map_err(|_| Error::bad_database("Private or public keys are invalid."))?, - ); + let bytes = &*globals + .update_and_fetch("keypair", utils::generate_keypair)? + .expect("utils::generate_keypair always returns Some"); + + let mut parts = bytes.splitn(2, |&b| b == 0xff); + + let keypair = utils::string_from_bytes( + // 1. version + parts + .next() + .expect("splitn always returns at least one element"), + ) + .map_err(|_| Error::bad_database("Invalid version bytes in keypair.")) + .and_then(|version| { + // 2. key + parts + .next() + .ok_or_else(|| Error::bad_database("Invalid keypair format in database.")) + .map(|key| (version, key)) + }) + .and_then(|(version, key)| { + ruma::signatures::Ed25519KeyPair::new(&key, version) + .map_err(|_| Error::bad_database("Private or public keys are invalid.")) + }); + + let keypair = match keypair { + Ok(k) => k, + Err(e) => { + error!("Keypair invalid. Deleting..."); + globals.remove("keypair")?; + return Err(e); + } + }; Ok(Self { globals, - keypair, + keypair: Arc::new(keypair), reqwest_client: reqwest::Client::new(), server_name: config .get_str("server_name") diff --git a/src/server_server.rs b/src/server_server.rs index 106f60eb2..f334d6b5e 100644 --- a/src/server_server.rs +++ b/src/server_server.rs @@ -17,7 +17,6 @@ use ruma::{ directory::{IncomingFilter, IncomingRoomNetwork}, EventId, ServerName, }; -use serde_json::json; use std::{ collections::BTreeMap, convert::TryFrom, @@ -58,7 +57,13 @@ where let actual_destination = "https://".to_owned() + &request_well_known(globals, &destination.as_str()) .await - .unwrap_or(destination.as_str().to_owned() + ":8448"); + .unwrap_or_else(|| { + let mut destination = destination.as_str().to_owned(); + if destination.find(':').is_none() { + destination += ":8448"; + } + destination + }); let mut http_request = request .try_into_http_request(&actual_destination, Some("")) diff --git a/src/utils.rs b/src/utils.rs index 8cf1b2ce1..452b7c5af 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -29,8 +29,13 @@ pub fn increment(old: Option<&[u8]>) -> Option> { pub fn generate_keypair(old: Option<&[u8]>) -> Option> { Some(old.map(|s| s.to_vec()).unwrap_or_else(|| { - ruma::signatures::Ed25519KeyPair::generate() - .expect("Ed25519KeyPair generation always works (?)") + let mut value = random_string(8).as_bytes().to_vec(); + value.push(0xff); + value.extend_from_slice( + &ruma::signatures::Ed25519KeyPair::generate() + .expect("Ed25519KeyPair generation always works (?)"), + ); + value })) } From f4078a29eb2e4975bc5664aab718875ce67da6a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20K=C3=B6sters?= Date: Wed, 16 Sep 2020 10:49:54 +0200 Subject: [PATCH 35/51] fix: synapse complains about missing origin --- src/database/rooms.rs | 6 ++++++ src/database/sending.rs | 1 + 2 files changed, 7 insertions(+) diff --git a/src/database/rooms.rs b/src/database/rooms.rs index 2246a61fe..8e6803301 100644 --- a/src/database/rooms.rs +++ b/src/database/rooms.rs @@ -812,6 +812,12 @@ impl Rooms { .expect("json is object") .remove("event_id"); + // Add origin because synapse likes that (and it's required in the spec) + pdu_json + .as_object_mut() + .expect("json is object") + .insert("origin".to_owned(), globals.server_name().as_str().into()); + ruma::signatures::hash_and_sign_event( globals.server_name().as_str(), globals.keypair(), diff --git a/src/database/sending.rs b/src/database/sending.rs index 77998e79b..a3f15742b 100644 --- a/src/database/sending.rs +++ b/src/database/sending.rs @@ -42,6 +42,7 @@ impl Sending { .get_mut("unsigned") { unsigned.as_object_mut().expect("unsigned is object").remove("transaction_id"); } + pdu_json .as_object_mut() .expect("json is object") From a567cd81d5e849285e6ef14b4d7ac41dd436c8fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20K=C3=B6sters?= Date: Wed, 16 Sep 2020 15:08:51 +0200 Subject: [PATCH 36/51] improvement: better logs on deserialization errors --- src/server_server.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/server_server.rs b/src/server_server.rs index f334d6b5e..aef3991e2 100644 --- a/src/server_server.rs +++ b/src/server_server.rs @@ -160,7 +160,7 @@ where .expect("reqwest body is valid http body"), ); response.map_err(|e| { - warn!("{}", e); + warn!("Server returned bad response: {:?}", e); Error::BadServerResponse("Server returned bad response.") }) } From 4db6d7e4308b206e0f34e291f1d23e7a54ea254a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20K=C3=B6sters?= Date: Wed, 16 Sep 2020 18:10:17 +0200 Subject: [PATCH 37/51] fix: remove avatar url checks They are not in the spec and maubot relies on that --- src/client_server/profile.rs | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/src/client_server/profile.rs b/src/client_server/profile.rs index 53893c0fb..686d4c3e3 100644 --- a/src/client_server/profile.rs +++ b/src/client_server/profile.rs @@ -119,18 +119,6 @@ pub async fn set_avatar_url_route( ) -> ConduitResult { let sender_id = body.sender_id.as_ref().expect("user is authenticated"); - if let Some(avatar_url) = &body.avatar_url { - if !avatar_url.starts_with("mxc://") { - return Err(Error::BadRequest( - ErrorKind::InvalidParam, - "avatar_url has to start with mxc://.", - )); - } - - // TODO in the future when we can handle media uploads make sure that this url is our own server - // TODO also make sure this is valid mxc:// format (not only starting with it) - } - db.users .set_avatar_url(&sender_id, body.avatar_url.clone())?; From 506c2a3146bb5314c95ad75ed069869a724ce628 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20K=C3=B6sters?= Date: Wed, 16 Sep 2020 21:11:38 +0200 Subject: [PATCH 38/51] fix: can't find count from event in db --- src/client_server/sync.rs | 31 ++++++++++++++----------------- src/database/rooms.rs | 25 +++++++++++++------------ 2 files changed, 27 insertions(+), 29 deletions(-) diff --git a/src/client_server/sync.rs b/src/client_server/sync.rs index eeeec005e..6ece18092 100644 --- a/src/client_server/sync.rs +++ b/src/client_server/sync.rs @@ -93,7 +93,7 @@ pub async fn sync_events_route( let mut limited = false; let mut state_pdus = Vec::new(); - for pdu in non_timeline_pdus { + for (_, pdu) in non_timeline_pdus { if pdu.state_key.is_some() { state_pdus.push(pdu); } @@ -113,7 +113,7 @@ pub async fn sync_events_route( .rooms .pdus_since(&sender_id, &room_id, since)? .filter_map(|r| r.ok()) - .filter_map(|pdu| Some((pdu.state_key.clone()?, pdu))) + .filter_map(|(_, pdu)| Some((pdu.state_key.clone()?, pdu))) { if pdu.kind == EventType::RoomMember { send_member_count = true; @@ -188,8 +188,8 @@ pub async fn sync_events_route( .rooms .all_pdus(&sender_id, &room_id)? .filter_map(|pdu| pdu.ok()) // Ignore all broken pdus - .filter(|pdu| pdu.kind == EventType::RoomMember) - .map(|pdu| { + .filter(|(_, pdu)| pdu.kind == EventType::RoomMember) + .map(|(_, pdu)| { let content = serde_json::from_value::< Raw, >(pdu.content.clone()) @@ -244,7 +244,7 @@ pub async fn sync_events_route( (db.rooms .pdus_since(&sender_id, &room_id, last_read)? .filter_map(|pdu| pdu.ok()) // Filter out buggy events - .filter(|pdu| { + .filter(|(_, pdu)| { matches!( pdu.kind.clone(), EventType::RoomMessage | EventType::RoomEncrypted @@ -260,18 +260,15 @@ pub async fn sync_events_route( None }; - let prev_batch = timeline_pdus.first().map_or(Ok::<_, Error>(None), |e| { - Ok(Some( - db.rooms - .get_pdu_count(&e.event_id)? - .ok_or_else(|| Error::bad_database("Can't find count from event in db."))? - .to_string(), - )) - })?; + let prev_batch = timeline_pdus + .first() + .map_or(Ok::<_, Error>(None), |(pdu_id, _)| { + Ok(Some(db.rooms.pdu_count(pdu_id)?.to_string())) + })?; let room_events = timeline_pdus .into_iter() - .map(|pdu| pdu.to_sync_room_event()) + .map(|(_, pdu)| pdu.to_sync_room_event()) .collect::>(); let mut edus = db @@ -380,7 +377,7 @@ pub async fn sync_events_route( let pdus = db.rooms.pdus_since(&sender_id, &room_id, since)?; let room_events = pdus .filter_map(|pdu| pdu.ok()) // Filter out buggy events - .map(|pdu| pdu.to_sync_room_event()) + .map(|(_, pdu)| pdu.to_sync_room_event()) .collect(); let left_room = sync_events::LeftRoom { @@ -395,7 +392,7 @@ pub async fn sync_events_route( let mut left_since_last_sync = false; for pdu in db.rooms.pdus_since(&sender_id, &room_id, since)? { - let pdu = pdu?; + let (_, pdu) = pdu?; if pdu.kind == EventType::RoomMember && pdu.state_key == Some(sender_id.to_string()) { let content = serde_json::from_value::< Raw, @@ -438,7 +435,7 @@ pub async fn sync_events_route( let room_id = room_id?; let mut invited_since_last_sync = false; for pdu in db.rooms.pdus_since(&sender_id, &room_id, since)? { - let pdu = pdu?; + let (_, pdu) = pdu?; if pdu.kind == EventType::RoomMember && pdu.state_key == Some(sender_id.to_string()) { let content = serde_json::from_value::< Raw, diff --git a/src/database/rooms.rs b/src/database/rooms.rs index 8e6803301..263f51b92 100644 --- a/src/database/rooms.rs +++ b/src/database/rooms.rs @@ -355,18 +355,19 @@ impl Rooms { } } + /// Returns the `count` of this pdu's id. + pub fn pdu_count(&self, pdu_id: &[u8]) -> Result { + Ok( + utils::u64_from_bytes(&pdu_id[pdu_id.len() - mem::size_of::()..pdu_id.len()]) + .map_err(|_| Error::bad_database("PDU has invalid count bytes."))?, + ) + } + /// Returns the `count` of this pdu's id. pub fn get_pdu_count(&self, event_id: &EventId) -> Result> { self.eventid_pduid .get(event_id.as_bytes())? - .map_or(Ok(None), |pdu_id| { - Ok(Some( - utils::u64_from_bytes( - &pdu_id[pdu_id.len() - mem::size_of::()..pdu_id.len()], - ) - .map_err(|_| Error::bad_database("PDU has invalid count bytes."))?, - )) - }) + .map_or(Ok(None), |pdu_id| self.pdu_count(&pdu_id).map(Some)) } /// Returns the json of a pdu. @@ -860,7 +861,7 @@ impl Rooms { &self, user_id: &UserId, room_id: &RoomId, - ) -> Result>> { + ) -> Result>> { self.pdus_since(user_id, room_id, 0) } @@ -871,7 +872,7 @@ impl Rooms { user_id: &UserId, room_id: &RoomId, since: u64, - ) -> Result>> { + ) -> Result>> { let mut prefix = room_id.to_string().as_bytes().to_vec(); prefix.push(0xff); @@ -887,13 +888,13 @@ impl Rooms { .pduid_pdu .range(first_pdu_id..last_pdu_id) .filter_map(|r| r.ok()) - .map(move |(_, v)| { + .map(move |(pdu_id, v)| { let mut pdu = serde_json::from_slice::(&v) .map_err(|_| Error::bad_database("PDU in db is invalid."))?; if pdu.sender != user_id { pdu.unsigned.remove("transaction_id"); } - Ok(pdu) + Ok((pdu_id, pdu)) })) } From ea3aaa6b5c06e01bef52a66b64fe45d74d5f60c5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20K=C3=B6sters?= Date: Thu, 17 Sep 2020 14:44:47 +0200 Subject: [PATCH 39/51] improvement: more efficient /sync with gaps --- Cargo.lock | 48 +++---- src/client_server/context.rs | 10 +- src/client_server/membership.rs | 11 +- src/client_server/message.rs | 12 ++ src/client_server/sync.rs | 152 +++++++++++++------- src/database/rooms.rs | 247 ++++++++++++-------------------- src/main.rs | 1 + src/pdu.rs | 6 +- src/server_server.rs | 4 +- src/stateres.rs | 59 -------- 10 files changed, 251 insertions(+), 299 deletions(-) delete mode 100644 src/stateres.rs diff --git a/Cargo.lock b/Cargo.lock index 30144ca54..e142d7250 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -134,9 +134,9 @@ checksum = "0e4cec68f03f32e44924783795810fa50a7035d8c8ebe78580ad7e6c703fba38" [[package]] name = "cc" -version = "1.0.59" +version = "1.0.60" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "66120af515773fb005778dc07c261bd201ec8ce50bd6e7144c927753fe013381" +checksum = "ef611cc68ff783f18535d77ddd080185275713d852c4f5cbb6122c462a7a825c" [[package]] name = "cfg-if" @@ -213,7 +213,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1373a16a4937bc34efec7b391f9c1500c30b8478a701a4f44c9165cc0475a6e0" dependencies = [ "percent-encoding", - "time 0.2.19", + "time 0.2.20", "version_check", ] @@ -342,9 +342,9 @@ checksum = "134951f4028bdadb9b84baf4232681efbf277da25144b9b0ad65df75946c422b" [[package]] name = "either" -version = "1.6.0" +version = "1.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd56b59865bce947ac5958779cfa508f6c3b9497cc762b7e24a12d11ccde2c4f" +checksum = "e78d4f1cc4ae33bbfc157ed5d5a5ef3bc29227303d595861deb238fcec4e9457" [[package]] name = "encoding_rs" @@ -1370,7 +1370,7 @@ dependencies = [ "rocket_codegen", "rocket_http", "state", - "time 0.2.19", + "time 0.2.20", "tokio", "toml", "version_check", @@ -1405,7 +1405,7 @@ dependencies = [ "ref-cast", "smallvec", "state", - "time 0.2.19", + "time 0.2.20", "tokio", "tokio-rustls", "unicode-xid", @@ -1414,7 +1414,7 @@ dependencies = [ [[package]] name = "ruma" version = "0.0.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#6ccb3ecaf69167ba405379826a9d87a98f168df8" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#425d34d4cfb5aefe5bab6957d71bc9389384c1e5" dependencies = [ "ruma-api", "ruma-appservice-api", @@ -1430,7 +1430,7 @@ dependencies = [ [[package]] name = "ruma-api" version = "0.17.0-alpha.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#6ccb3ecaf69167ba405379826a9d87a98f168df8" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#425d34d4cfb5aefe5bab6957d71bc9389384c1e5" dependencies = [ "http", "percent-encoding", @@ -1445,7 +1445,7 @@ dependencies = [ [[package]] name = "ruma-api-macros" version = "0.17.0-alpha.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#6ccb3ecaf69167ba405379826a9d87a98f168df8" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#425d34d4cfb5aefe5bab6957d71bc9389384c1e5" dependencies = [ "proc-macro-crate", "proc-macro2", @@ -1456,7 +1456,7 @@ dependencies = [ [[package]] name = "ruma-appservice-api" version = "0.2.0-alpha.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#6ccb3ecaf69167ba405379826a9d87a98f168df8" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#425d34d4cfb5aefe5bab6957d71bc9389384c1e5" dependencies = [ "ruma-api", "ruma-common", @@ -1469,7 +1469,7 @@ dependencies = [ [[package]] name = "ruma-client-api" version = "0.10.0-alpha.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#6ccb3ecaf69167ba405379826a9d87a98f168df8" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#425d34d4cfb5aefe5bab6957d71bc9389384c1e5" dependencies = [ "assign", "http", @@ -1488,7 +1488,7 @@ dependencies = [ [[package]] name = "ruma-common" version = "0.2.0" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#6ccb3ecaf69167ba405379826a9d87a98f168df8" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#425d34d4cfb5aefe5bab6957d71bc9389384c1e5" dependencies = [ "js_int", "ruma-api", @@ -1502,7 +1502,7 @@ dependencies = [ [[package]] name = "ruma-events" version = "0.22.0-alpha.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#6ccb3ecaf69167ba405379826a9d87a98f168df8" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#425d34d4cfb5aefe5bab6957d71bc9389384c1e5" dependencies = [ "js_int", "ruma-common", @@ -1517,7 +1517,7 @@ dependencies = [ [[package]] name = "ruma-events-macros" version = "0.22.0-alpha.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#6ccb3ecaf69167ba405379826a9d87a98f168df8" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#425d34d4cfb5aefe5bab6957d71bc9389384c1e5" dependencies = [ "proc-macro-crate", "proc-macro2", @@ -1528,7 +1528,7 @@ dependencies = [ [[package]] name = "ruma-federation-api" version = "0.0.3" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#6ccb3ecaf69167ba405379826a9d87a98f168df8" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#425d34d4cfb5aefe5bab6957d71bc9389384c1e5" dependencies = [ "js_int", "ruma-api", @@ -1543,7 +1543,7 @@ dependencies = [ [[package]] name = "ruma-identifiers" version = "0.17.4" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#6ccb3ecaf69167ba405379826a9d87a98f168df8" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#425d34d4cfb5aefe5bab6957d71bc9389384c1e5" dependencies = [ "rand", "ruma-identifiers-macros", @@ -1555,7 +1555,7 @@ dependencies = [ [[package]] name = "ruma-identifiers-macros" version = "0.17.4" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#6ccb3ecaf69167ba405379826a9d87a98f168df8" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#425d34d4cfb5aefe5bab6957d71bc9389384c1e5" dependencies = [ "proc-macro2", "quote", @@ -1566,7 +1566,7 @@ dependencies = [ [[package]] name = "ruma-identifiers-validation" version = "0.1.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#6ccb3ecaf69167ba405379826a9d87a98f168df8" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#425d34d4cfb5aefe5bab6957d71bc9389384c1e5" dependencies = [ "serde", "strum", @@ -1575,7 +1575,7 @@ dependencies = [ [[package]] name = "ruma-serde" version = "0.2.3" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#6ccb3ecaf69167ba405379826a9d87a98f168df8" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#425d34d4cfb5aefe5bab6957d71bc9389384c1e5" dependencies = [ "form_urlencoded", "itoa", @@ -1587,7 +1587,7 @@ dependencies = [ [[package]] name = "ruma-signatures" version = "0.6.0-dev.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#6ccb3ecaf69167ba405379826a9d87a98f168df8" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#425d34d4cfb5aefe5bab6957d71bc9389384c1e5" dependencies = [ "base64", "ring", @@ -1831,7 +1831,7 @@ checksum = "7345c971d1ef21ffdbd103a75990a15eb03604fc8b8852ca8cb418ee1a099028" [[package]] name = "state-res" version = "0.1.0" -source = "git+https://github.com/timokoesters/state-res?branch=spec-comp#1d01b6e65b6afd50e65085fb40f1e7d2782f519e" +source = "git+https://github.com/timokoesters/state-res?branch=spec-comp#d11a3feb5307715ab5d86af8f25d4bccfee6264b" dependencies = [ "itertools", "js_int", @@ -1981,9 +1981,9 @@ dependencies = [ [[package]] name = "time" -version = "0.2.19" +version = "0.2.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "80c1a1fd93112fc50b11c43a1def21f926be3c18884fad676ea879572da070a1" +checksum = "0d4953c513c9bf1b97e9cdd83f11d60c4b0a83462880a360d80d96953a953fee" dependencies = [ "const_fn", "libc", diff --git a/src/client_server/context.rs b/src/client_server/context.rs index 959372646..4c9be20fb 100644 --- a/src/client_server/context.rs +++ b/src/client_server/context.rs @@ -49,7 +49,10 @@ pub fn get_context_route( .filter_map(|r| r.ok()) // Remove buggy events .collect::>(); - let start_token = events_before.last().map(|(count, _)| count.to_string()); + let start_token = events_before + .last() + .and_then(|(pdu_id, _)| db.rooms.pdu_count(pdu_id).ok()) + .map(|count| count.to_string()); let events_before = events_before .into_iter() @@ -68,7 +71,10 @@ pub fn get_context_route( .filter_map(|r| r.ok()) // Remove buggy events .collect::>(); - let end_token = events_after.last().map(|(count, _)| count.to_string()); + let end_token = events_after + .last() + .and_then(|(pdu_id, _)| db.rooms.pdu_count(pdu_id).ok()) + .map(|count| count.to_string()); let events_after = events_after .into_iter() diff --git a/src/client_server/membership.rs b/src/client_server/membership.rs index c4eed95c1..628045dee 100644 --- a/src/client_server/membership.rs +++ b/src/client_server/membership.rs @@ -601,8 +601,7 @@ async fn join_room_by_id_helper( .cloned() .collect::>(); - let power_level = - resolved_control_events.get(&(EventType::RoomPowerLevels, Some("".into()))); + let power_level = resolved_control_events.get(&(EventType::RoomPowerLevels, "".into())); // Sort the remaining non control events let sorted_event_ids = state_res::StateResolution::mainline_sort( room_id, @@ -644,13 +643,7 @@ async fn join_room_by_id_helper( )?; if state_events.contains(ev_id) { - state.insert( - ( - pdu.kind(), - pdu.state_key().expect("State events have a state key"), - ), - pdu_id, - ); + state.insert((pdu.kind(), pdu.state_key()), pdu_id); } } diff --git a/src/client_server/message.rs b/src/client_server/message.rs index 3944d5bdf..5a4488fe5 100644 --- a/src/client_server/message.rs +++ b/src/client_server/message.rs @@ -117,6 +117,12 @@ pub fn get_message_events_route( .pdus_after(&sender_id, &body.room_id, from) .take(limit) .filter_map(|r| r.ok()) // Filter out buggy events + .filter_map(|(pdu_id, pdu)| { + db.rooms + .pdu_count(&pdu_id) + .map(|pdu_count| (pdu_count, pdu)) + .ok() + }) .take_while(|&(k, _)| Some(Ok(k)) != to) // Stop at `to` .collect::>(); @@ -141,6 +147,12 @@ pub fn get_message_events_route( .pdus_until(&sender_id, &body.room_id, from) .take(limit) .filter_map(|r| r.ok()) // Filter out buggy events + .filter_map(|(pdu_id, pdu)| { + db.rooms + .pdu_count(&pdu_id) + .map(|pdu_count| (pdu_count, pdu)) + .ok() + }) .take_while(|&(k, _)| Some(Ok(k)) != to) // Stop at `to` .collect::>(); diff --git a/src/client_server/sync.rs b/src/client_server/sync.rs index 6ece18092..0e40bfbb1 100644 --- a/src/client_server/sync.rs +++ b/src/client_server/sync.rs @@ -105,50 +105,92 @@ pub async fn sync_events_route( .room_state_get(&room_id, &EventType::RoomEncryption, "")? .is_some(); - // TODO: optimize this? - let mut send_member_count = false; - let mut joined_since_last_sync = false; - let mut new_encrypted_room = false; - for (state_key, pdu) in db + // Database queries: + let since_state_hash = db .rooms - .pdus_since(&sender_id, &room_id, since)? - .filter_map(|r| r.ok()) - .filter_map(|(_, pdu)| Some((pdu.state_key.clone()?, pdu))) - { - if pdu.kind == EventType::RoomMember { - send_member_count = true; + .pdus_until(sender_id, &room_id, since) + .next() + .and_then(|pdu| pdu.ok()) + .and_then(|pdu| db.rooms.pdu_state_hash(&pdu.0).ok()?); - let content = serde_json::from_value::< - Raw, - >(pdu.content.clone()) + let since_members = since_state_hash + .as_ref() + .and_then(|state_hash| db.rooms.state_type(state_hash, &EventType::RoomMember).ok()); + + let since_encryption = since_state_hash.as_ref().and_then(|state_hash| { + db.rooms + .state_get(&state_hash, &EventType::RoomEncryption, "") + .ok() + }); + + let current_members = db.rooms.room_state_type(&room_id, &EventType::RoomMember)?; + + // Calculations: + let new_encrypted_room = encrypted_room && since_encryption.is_none(); + + let send_member_count = since_members.as_ref().map_or(true, |since_members| { + current_members.len() != since_members.len() + }); + + let since_sender_member = since_members.as_ref().and_then(|members| { + members.get(sender_id.as_str()).and_then(|pdu| { + serde_json::from_value::>( + pdu.content.clone(), + ) .expect("Raw::from_value always works") .deserialize() - .map_err(|_| Error::bad_database("Invalid PDU in database."))?; + .map_err(|_| Error::bad_database("Invalid PDU in database.")) + .ok() + }) + }); - if pdu.state_key == Some(sender_id.to_string()) - && content.membership == MembershipState::Join - { - joined_since_last_sync = true; - } else if encrypted_room && content.membership == MembershipState::Join { - // A new user joined an encrypted room - let user_id = UserId::try_from(state_key) - .map_err(|_| Error::bad_database("Invalid UserId in member PDU."))?; - // Add encryption update if we didn't share an encrypted room already - if !share_encrypted_room(&db, &sender_id, &user_id, &room_id) { - device_list_updates.insert(user_id); + if encrypted_room { + for (user_id, current_member) in current_members { + let current_membership = serde_json::from_value::< + Raw, + >(current_member.content.clone()) + .expect("Raw::from_value always works") + .deserialize() + .map_err(|_| Error::bad_database("Invalid PDU in database."))? + .membership; + + let since_membership = since_members + .as_ref() + .and_then(|members| { + members.get(&user_id).and_then(|since_member| { + serde_json::from_value::< + Raw, + >(since_member.content.clone()) + .expect("Raw::from_value always works") + .deserialize() + .map_err(|_| Error::bad_database("Invalid PDU in database.")) + .ok() + }) + }) + .map_or(MembershipState::Leave, |member| member.membership); + + let user_id = UserId::try_from(user_id) + .map_err(|_| Error::bad_database("Invalid UserId in member PDU."))?; + + match (since_membership, current_membership) { + (MembershipState::Leave, MembershipState::Join) => { + // A new user joined an encrypted room + if !share_encrypted_room(&db, &sender_id, &user_id, &room_id) { + device_list_updates.insert(user_id); + } } - } else if encrypted_room && content.membership == MembershipState::Leave { - // Write down users that have left encrypted rooms we are in - left_encrypted_users.insert( - UserId::try_from(state_key) - .map_err(|_| Error::bad_database("Invalid UserId in member PDU."))?, - ); + (MembershipState::Join, MembershipState::Leave) => { + // Write down users that have left encrypted rooms we are in + left_encrypted_users.insert(user_id); + } + _ => {} } - } else if pdu.kind == EventType::RoomEncryption { - new_encrypted_room = true; } } + let joined_since_last_sync = + since_sender_member.map_or(true, |member| member.membership != MembershipState::Join); + if joined_since_last_sync && encrypted_room || new_encrypted_room { // If the user is in a new encrypted room, give them all joined users device_list_updates.extend( @@ -390,23 +432,37 @@ pub async fn sync_events_route( state: sync_events::State { events: Vec::new() }, }; - let mut left_since_last_sync = false; - for pdu in db.rooms.pdus_since(&sender_id, &room_id, since)? { - let (_, pdu) = pdu?; - if pdu.kind == EventType::RoomMember && pdu.state_key == Some(sender_id.to_string()) { - let content = serde_json::from_value::< - Raw, - >(pdu.content.clone()) + let since_member = db + .rooms + .pdus_until(sender_id, &room_id, since) + .next() + .and_then(|pdu| pdu.ok()) + .and_then(|pdu| { + db.rooms + .pdu_state_hash(&pdu.0) + .ok()? + .ok_or_else(|| Error::bad_database("Pdu in db doesn't have a state hash.")) + .ok() + }) + .and_then(|state_hash| { + db.rooms + .state_get(&state_hash, &EventType::RoomMember, sender_id.as_str()) + .ok()? + .ok_or_else(|| Error::bad_database("State hash in db doesn't have a state.")) + .ok() + }) + .and_then(|pdu| { + serde_json::from_value::>( + pdu.content.clone(), + ) .expect("Raw::from_value always works") .deserialize() - .map_err(|_| Error::bad_database("Invalid PDU in database."))?; + .map_err(|_| Error::bad_database("Invalid PDU in database.")) + .ok() + }); - if content.membership == MembershipState::Leave { - left_since_last_sync = true; - break; - } - } - } + let left_since_last_sync = + since_member.map_or(false, |member| member.membership == MembershipState::Join); if left_since_last_sync { device_list_left.extend( diff --git a/src/database/rooms.rs b/src/database/rooms.rs index 263f51b92..595862624 100644 --- a/src/database/rooms.rs +++ b/src/database/rooms.rs @@ -31,7 +31,7 @@ use std::{ /// /// This is created when a state group is added to the database by /// hashing the entire state. -pub type StateHashId = Vec; +pub type StateHashId = IVec; #[derive(Clone)] pub struct Rooms { @@ -100,7 +100,7 @@ impl StateStore for Rooms { impl Rooms { /// Builds a StateMap by iterating over all keys that start /// with state_hash, this gives the full state for the given state_hash. - pub fn state_full(&self, state_hash: StateHashId) -> Result> { + pub fn state_full(&self, state_hash: &StateHashId) -> Result> { self.stateid_pduid .scan_prefix(&state_hash) .values() @@ -115,61 +115,87 @@ impl Rooms { }) .map(|pdu| { let pdu = pdu?; - Ok(((pdu.kind, pdu.state_key), pdu.event_id)) + Ok(( + ( + pdu.kind.clone(), + pdu.state_key + .as_ref() + .ok_or_else(|| Error::bad_database("State event has no state key."))? + .clone(), + ), + pdu, + )) }) .collect::>>() } - // TODO make this return Result - /// Fetches the previous StateHash ID to `current`. - pub fn prev_state_hash(&self, current: StateHashId) -> Option { - let mut found = false; - for pair in self.pduid_statehash.iter().rev() { - let prev = pair.ok()?.1; - if current == prev.as_ref() { - found = true; - } - if current != prev.as_ref() && found { - return Some(prev.to_vec()); - } + /// Returns all state entries for this type. + pub fn state_type( + &self, + state_hash: &StateHashId, + event_type: &EventType, + ) -> Result> { + let mut prefix = state_hash.to_vec(); + prefix.push(0xff); + prefix.extend_from_slice(&event_type.to_string().as_bytes()); + prefix.push(0xff); + + let mut hashmap = HashMap::new(); + for pdu in self + .stateid_pduid + .scan_prefix(&prefix) + .values() + .map(|pdu_id| { + Ok::<_, Error>( + serde_json::from_slice::(&self.pduid_pdu.get(pdu_id?)?.ok_or_else( + || Error::bad_database("PDU in state not found in database."), + )?) + .map_err(|_| Error::bad_database("Invalid PDU bytes in room state."))?, + ) + }) + { + let pdu = pdu?; + let state_key = pdu.state_key.clone().ok_or_else(|| { + Error::bad_database("Room state contains event without state_key.") + })?; + hashmap.insert(state_key, pdu); } - None + Ok(hashmap) + } + + /// Returns a single PDU from `room_id` with key (`event_type`, `state_key`). + pub fn state_get( + &self, + state_hash: &StateHashId, + event_type: &EventType, + state_key: &str, + ) -> Result> { + let mut key = state_hash.to_vec(); + key.push(0xff); + key.extend_from_slice(&event_type.to_string().as_bytes()); + key.push(0xff); + key.extend_from_slice(&state_key.as_bytes()); + + self.stateid_pduid.get(&key)?.map_or(Ok(None), |pdu_id| { + Ok::<_, Error>(Some( + serde_json::from_slice::( + &self.pduid_pdu.get(pdu_id)?.ok_or_else(|| { + Error::bad_database("PDU in state not found in database.") + })?, + ) + .map_err(|_| Error::bad_database("Invalid PDU bytes in room state."))?, + )) + }) + } + + /// Returns the last state hash key added to the db. + pub fn pdu_state_hash(&self, pdu_id: &[u8]) -> Result> { + Ok(self.pduid_statehash.get(pdu_id)?) } /// Returns the last state hash key added to the db. pub fn current_state_hash(&self, room_id: &RoomId) -> Result> { - Ok(self - .roomid_statehash - .get(room_id.as_bytes())? - .map(|bytes| bytes.to_vec())) - } - - /// This fetches auth event_ids from the current state using the - /// full `roomstateid_pdu` tree. - pub fn get_auth_event_ids( - &self, - room_id: &RoomId, - kind: &EventType, - sender: &UserId, - state_key: Option<&str>, - content: serde_json::Value, - ) -> Result> { - let auth_events = state_res::auth_types_for_event( - kind.clone(), - sender, - state_key.map(|s| s.to_string()), - content, - ); - - let mut events = vec![]; - for (event_type, state_key) in auth_events { - if let Some(state_key) = state_key.as_ref() { - if let Some(id) = self.room_state_get(room_id, &event_type, state_key)? { - events.push(id.event_id); - } - } - } - Ok(events) + Ok(self.roomid_statehash.get(room_id.as_bytes())?) } /// This fetches auth events from the current state. @@ -190,10 +216,8 @@ impl Rooms { let mut events = StateMap::new(); for (event_type, state_key) in auth_events { - if let Some(s_key) = state_key.as_ref() { - if let Some(pdu) = self.room_state_get(room_id, &event_type, s_key)? { - events.insert((event_type, state_key), pdu); - } + if let Some(pdu) = self.room_state_get(room_id, &event_type, &state_key)? { + events.insert((event_type, state_key), pdu); } } Ok(events) @@ -206,7 +230,7 @@ impl Rooms { // We only hash the pdu's event ids, not the whole pdu let bytes = pdu_id_bytes.join(&0xff); let hash = digest::digest(&digest::SHA256, &bytes); - Ok(hash.as_ref().to_vec()) + Ok(hash.as_ref().into()) } /// Checks if a room exists. @@ -230,7 +254,7 @@ impl Rooms { ) -> Result<()> { let state_hash = self.calculate_hash(&state.values().map(|pdu_id| &**pdu_id).collect::>())?; - let mut prefix = state_hash.clone(); + let mut prefix = state_hash.to_vec(); prefix.push(0xff); for ((event_type, state_key), pdu_id) in state { @@ -248,41 +272,11 @@ impl Rooms { } /// Returns the full room state. - pub fn room_state_full( - &self, - room_id: &RoomId, - ) -> Result> { + pub fn room_state_full(&self, room_id: &RoomId) -> Result> { if let Some(current_state_hash) = self.current_state_hash(room_id)? { - let mut prefix = current_state_hash; - prefix.push(0xff); - - let mut hashmap = HashMap::new(); - for pdu in self - .stateid_pduid - .scan_prefix(prefix) - .values() - .map(|pdu_id| { - Ok::<_, Error>( - serde_json::from_slice::( - &self.pduid_pdu.get(pdu_id?)?.ok_or_else(|| { - Error::bad_database("PDU in state not found in database.") - })?, - ) - .map_err(|_| { - Error::bad_database("Invalid PDU bytes in current room state.") - })?, - ) - }) - { - let pdu = pdu?; - let state_key = pdu.state_key.clone().ok_or_else(|| { - Error::bad_database("Room state contains event without state_key.") - })?; - hashmap.insert((pdu.kind.clone(), state_key), pdu); - } - Ok(hashmap) + self.state_full(¤t_state_hash) } else { - Ok(HashMap::new()) + Ok(BTreeMap::new()) } } @@ -293,36 +287,7 @@ impl Rooms { event_type: &EventType, ) -> Result> { if let Some(current_state_hash) = self.current_state_hash(room_id)? { - let mut prefix = current_state_hash; - prefix.push(0xff); - prefix.extend_from_slice(&event_type.to_string().as_bytes()); - prefix.push(0xff); - - let mut hashmap = HashMap::new(); - for pdu in self - .stateid_pduid - .scan_prefix(&prefix) - .values() - .map(|pdu_id| { - Ok::<_, Error>( - serde_json::from_slice::( - &self.pduid_pdu.get(pdu_id?)?.ok_or_else(|| { - Error::bad_database("PDU in state not found in database.") - })?, - ) - .map_err(|_| { - Error::bad_database("Invalid PDU bytes in current room state.") - })?, - ) - }) - { - let pdu = pdu?; - let state_key = pdu.state_key.clone().ok_or_else(|| { - Error::bad_database("Room state contains event without state_key.") - })?; - hashmap.insert(state_key, pdu); - } - Ok(hashmap) + self.state_type(¤t_state_hash, event_type) } else { Ok(HashMap::new()) } @@ -336,20 +301,7 @@ impl Rooms { state_key: &str, ) -> Result> { if let Some(current_state_hash) = self.current_state_hash(room_id)? { - let mut key = current_state_hash; - key.push(0xff); - key.extend_from_slice(&event_type.to_string().as_bytes()); - key.push(0xff); - key.extend_from_slice(&state_key.as_bytes()); - - self.stateid_pduid.get(&key)?.map_or(Ok(None), |pdu_id| { - Ok::<_, Error>(Some( - serde_json::from_slice::(&self.pduid_pdu.get(pdu_id)?.ok_or_else( - || Error::bad_database("PDU in state not found in database."), - )?) - .map_err(|_| Error::bad_database("Invalid PDU bytes in current room state."))?, - )) - }) + self.state_get(¤t_state_hash, event_type, state_key) } else { Ok(None) } @@ -562,14 +514,15 @@ impl Rooms { /// This adds all current state events (not including the incoming event) /// to `stateid_pduid` and adds the incoming event to `pduid_statehash`. /// The incoming event is the `pdu_id` passed to this method. - fn append_to_state(&self, new_pdu_id: &[u8], new_pdu: &PduEvent) -> Result { + pub fn append_to_state(&self, new_pdu_id: &[u8], new_pdu: &PduEvent) -> Result { let old_state = if let Some(old_state_hash) = self.roomid_statehash.get(new_pdu.room_id.as_bytes())? { // Store state for event. The state does not include the event itself. // Instead it's the state before the pdu, so the room's old state. - self.pduid_statehash.insert(new_pdu_id, &old_state_hash)?; + self.pduid_statehash + .insert(dbg!(new_pdu_id), &old_state_hash)?; if new_pdu.state_key.is_none() { - return Ok(old_state_hash.to_vec()); + return Ok(old_state_hash); } let mut prefix = old_state_hash.to_vec(); @@ -841,9 +794,7 @@ impl Rooms { let pdu_id = self.append_pdu(&pdu, &pdu_json, globals, account_data)?; - if pdu.state_key.is_some() { - self.append_to_state(&pdu_id, &pdu)?; - } + self.append_to_state(&pdu_id, &pdu)?; for server in self .room_servers(room_id) @@ -905,7 +856,7 @@ impl Rooms { user_id: &UserId, room_id: &RoomId, until: u64, - ) -> impl Iterator> { + ) -> impl Iterator> { // Create the first part of the full pdu id let mut prefix = room_id.to_string().as_bytes().to_vec(); prefix.push(0xff); @@ -916,23 +867,18 @@ impl Rooms { let current: &[u8] = ¤t; let user_id = user_id.clone(); - let prefixlen = prefix.len(); self.pduid_pdu .range(..current) .rev() .filter_map(|r| r.ok()) .take_while(move |(k, _)| k.starts_with(&prefix)) - .map(move |(k, v)| { + .map(move |(pdu_id, v)| { let mut pdu = serde_json::from_slice::(&v) .map_err(|_| Error::bad_database("PDU in db is invalid."))?; if pdu.sender != user_id { pdu.unsigned.remove("transaction_id"); } - Ok(( - utils::u64_from_bytes(&k[prefixlen..]) - .map_err(|_| Error::bad_database("Invalid pdu id in db."))?, - pdu, - )) + Ok((pdu_id, pdu)) }) } @@ -943,7 +889,7 @@ impl Rooms { user_id: &UserId, room_id: &RoomId, from: u64, - ) -> impl Iterator> { + ) -> impl Iterator> { // Create the first part of the full pdu id let mut prefix = room_id.to_string().as_bytes().to_vec(); prefix.push(0xff); @@ -954,22 +900,17 @@ impl Rooms { let current: &[u8] = ¤t; let user_id = user_id.clone(); - let prefixlen = prefix.len(); self.pduid_pdu .range(current..) .filter_map(|r| r.ok()) .take_while(move |(k, _)| k.starts_with(&prefix)) - .map(move |(k, v)| { + .map(move |(pdu_id, v)| { let mut pdu = serde_json::from_slice::(&v) .map_err(|_| Error::bad_database("PDU in db is invalid."))?; if pdu.sender != user_id { pdu.unsigned.remove("transaction_id"); } - Ok(( - utils::u64_from_bytes(&k[prefixlen..]) - .map_err(|_| Error::bad_database("Invalid pdu id in db."))?, - pdu, - )) + Ok((pdu_id, pdu)) }) } diff --git a/src/main.rs b/src/main.rs index f81c7f463..06fda59a0 100644 --- a/src/main.rs +++ b/src/main.rs @@ -123,6 +123,7 @@ fn setup_rocket() -> rocket::Rocket { server_server::get_server_keys, server_server::get_server_keys_deprecated, server_server::get_public_rooms_route, + server_server::get_public_rooms_filtered_route, server_server::send_transaction_message_route, ], ) diff --git a/src/pdu.rs b/src/pdu.rs index 957d9e0a2..d5b5415c5 100644 --- a/src/pdu.rs +++ b/src/pdu.rs @@ -1,4 +1,4 @@ -use crate::{Error, Result}; +use crate::Error; use js_int::UInt; use ruma::{ events::pdu::PduStub, @@ -35,7 +35,7 @@ pub struct PduEvent { } impl PduEvent { - pub fn redact(&mut self, reason: &PduEvent) -> Result<()> { + pub fn redact(&mut self, reason: &PduEvent) -> crate::Result<()> { self.unsigned.clear(); let allowed: &[&str] = match self.kind { @@ -244,7 +244,7 @@ impl From<&state_res::StateEvent> for PduEvent { .expect("time is valid"), kind: pdu.kind(), content: pdu.content().clone(), - state_key: pdu.state_key(), + state_key: Some(pdu.state_key()), prev_events: pdu.prev_event_ids(), depth: pdu.depth().clone(), auth_events: pdu.auth_events(), diff --git a/src/server_server.rs b/src/server_server.rs index aef3991e2..6f2b17903 100644 --- a/src/server_server.rs +++ b/src/server_server.rs @@ -329,8 +329,10 @@ pub fn send_transaction_message_route<'a>( let pdu = serde_json::from_value::(value.clone()) .expect("all ruma pdus are conduit pdus"); if db.rooms.exists(&pdu.room_id)? { - db.rooms + let pdu_id = db + .rooms .append_pdu(&pdu, &value, &db.globals, &db.account_data)?; + db.rooms.append_to_state(&pdu_id, &pdu)?; } } Ok(send_transaction_message::v1::Response { diff --git a/src/stateres.rs b/src/stateres.rs deleted file mode 100644 index ee47099a2..000000000 --- a/src/stateres.rs +++ /dev/null @@ -1,59 +0,0 @@ -use std::collections::HashMap; - -fn stateres(state_a: HashMap, state_b: HashMap) { - let mut unconflicted = todo!("state at fork event"); - - let mut conflicted: HashMap = state_a - .iter() - .filter(|(key_a, value_a)| match state_b.remove(key_a) { - Some(value_b) if value_a == value_b => unconflicted.insert(key_a, value_a), - _ => false, - }) - .collect(); - - // We removed unconflicted from state_b, now we can easily insert all events that are only in fork b - conflicted.extend(state_b); - - let partial_state = unconflicted.clone(); - - let full_conflicted = conflicted.clone(); // TODO: auth events - - let output_rev = Vec::new(); - let event_map = HashMap::new(); - let incoming_edges = HashMap::new(); - - for event in full_conflicted { - event_map.insert(event.event_id, event); - incoming_edges.insert(event.event_id, 0); - } - - for e in conflicted_control_events { - for a in e.auth_events { - incoming_edges[a.event_id] += 1; - } - } - - while incoming_edges.len() > 0 { - let mut count_0 = incoming_edges - .iter() - .filter(|(_, c)| c == 0) - .collect::>(); - - count_0.sort_by(|(x, _), (y, _)| { - x.power_level - .cmp(&a.power_level) - .then_with(|| x.origin_server.ts.cmp(&y.origin_server_ts)) - .then_with(|| x.event_id.cmp(&y.event_id)) - }); - - for (id, count) in count_0 { - output_rev.push(event_map[id]); - - for auth_event in event_map[id].auth_events { - incoming_edges[auth_event.event_id] -= 1; - } - - incoming_edges.remove(id); - } - } -} From 8bcfff276652e20fee14e13109d80a514e1a107d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20K=C3=B6sters?= Date: Thu, 17 Sep 2020 19:58:19 +0200 Subject: [PATCH 40/51] fix: no notification counts for fast /syncs --- src/database/rooms.rs | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/database/rooms.rs b/src/database/rooms.rs index 595862624..18881dd84 100644 --- a/src/database/rooms.rs +++ b/src/database/rooms.rs @@ -447,6 +447,11 @@ impl Rooms { // This is also the next_batch/since value let index = globals.next_count()?; + // Mark as read first so the sending client doesn't get a notification even if appending + // fails + self.edus + .private_read_set(&pdu.room_id, &pdu.sender, index, &globals)?; + let mut pdu_id = pdu.room_id.as_bytes().to_vec(); pdu_id.push(0xff); pdu_id.extend_from_slice(&index.to_be_bytes()); @@ -503,9 +508,6 @@ impl Rooms { _ => {} } - self.edus - .private_read_set(&pdu.room_id, &pdu.sender, index, &globals)?; - Ok(pdu_id) } @@ -520,7 +522,7 @@ impl Rooms { // Store state for event. The state does not include the event itself. // Instead it's the state before the pdu, so the room's old state. self.pduid_statehash - .insert(dbg!(new_pdu_id), &old_state_hash)?; + .insert(new_pdu_id, &old_state_hash)?; if new_pdu.state_key.is_none() { return Ok(old_state_hash); } From 267c7216162932c9f6e35d69be4c6303d5134952 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20K=C3=B6sters?= Date: Thu, 17 Sep 2020 22:41:43 +0200 Subject: [PATCH 41/51] fix: encryption and sync spam --- src/client_server/sync.rs | 106 +++++++++++++++++++++++--------------- src/database/rooms.rs | 3 +- src/database/users.rs | 2 +- 3 files changed, 67 insertions(+), 44 deletions(-) diff --git a/src/client_server/sync.rs b/src/client_server/sync.rs index 0e40bfbb1..2f2c8eacf 100644 --- a/src/client_server/sync.rs +++ b/src/client_server/sync.rs @@ -100,47 +100,61 @@ pub async fn sync_events_route( limited = true; } + // Database queries: let encrypted_room = db .rooms .room_state_get(&room_id, &EventType::RoomEncryption, "")? .is_some(); - // Database queries: + // These type is Option>. The outer Option is None when there is no event between + // since and the current room state, meaning there should be no updates. + // The inner Option is None when there is an event, but there is no state hash associated + // with it. This can happen for the RoomCreate event, so all updates should arrive. let since_state_hash = db .rooms - .pdus_until(sender_id, &room_id, since) + .pdus_after(sender_id, &room_id, since) // - 1 So we can get the event at since .next() - .and_then(|pdu| pdu.ok()) - .and_then(|pdu| db.rooms.pdu_state_hash(&pdu.0).ok()?); + .map(|pdu| db.rooms.pdu_state_hash(&pdu.ok()?.0).ok()?); - let since_members = since_state_hash - .as_ref() - .and_then(|state_hash| db.rooms.state_type(state_hash, &EventType::RoomMember).ok()); + let since_members = since_state_hash.as_ref().map(|state_hash| { + state_hash.as_ref().and_then(|state_hash| { + db.rooms + .state_type(&state_hash, &EventType::RoomMember) + .ok() + }) + }); - let since_encryption = since_state_hash.as_ref().and_then(|state_hash| { - db.rooms - .state_get(&state_hash, &EventType::RoomEncryption, "") - .ok() + let since_encryption = since_state_hash.as_ref().map(|state_hash| { + state_hash.as_ref().and_then(|state_hash| { + db.rooms + .state_get(&state_hash, &EventType::RoomEncryption, "") + .ok() + }) }); let current_members = db.rooms.room_state_type(&room_id, &EventType::RoomMember)?; // Calculations: - let new_encrypted_room = encrypted_room && since_encryption.is_none(); + let new_encrypted_room = + encrypted_room && since_encryption.map_or(false, |encryption| encryption.is_none()); - let send_member_count = since_members.as_ref().map_or(true, |since_members| { - current_members.len() != since_members.len() + let send_member_count = since_members.as_ref().map_or(false, |since_members| { + since_members.as_ref().map_or(true, |since_members| { + current_members.len() != since_members.len() + }) }); - let since_sender_member = since_members.as_ref().and_then(|members| { - members.get(sender_id.as_str()).and_then(|pdu| { - serde_json::from_value::>( - pdu.content.clone(), - ) - .expect("Raw::from_value always works") - .deserialize() - .map_err(|_| Error::bad_database("Invalid PDU in database.")) - .ok() + let since_sender_member = since_members.as_ref().map(|since_members| { + since_members.as_ref().and_then(|members| { + members.get(sender_id.as_str()).and_then(|pdu| { + serde_json::from_value::>( + pdu.content.clone(), + ) + .expect("Raw::from_value always works") + .deserialize() + .map_err(|_| Error::bad_database("Invalid PDU in database.")) + .ok() + }) }) }); @@ -154,20 +168,29 @@ pub async fn sync_events_route( .map_err(|_| Error::bad_database("Invalid PDU in database."))? .membership; - let since_membership = since_members - .as_ref() - .and_then(|members| { - members.get(&user_id).and_then(|since_member| { - serde_json::from_value::< - Raw, - >(since_member.content.clone()) - .expect("Raw::from_value always works") - .deserialize() - .map_err(|_| Error::bad_database("Invalid PDU in database.")) - .ok() - }) - }) - .map_or(MembershipState::Leave, |member| member.membership); + let since_membership = + since_members + .as_ref() + .map_or(MembershipState::Join, |members| { + members + .as_ref() + .and_then(|members| { + members.get(&user_id).and_then(|since_member| { + serde_json::from_value::< + Raw, + >( + since_member.content.clone() + ) + .expect("Raw::from_value always works") + .deserialize() + .map_err(|_| { + Error::bad_database("Invalid PDU in database.") + }) + .ok() + }) + }) + .map_or(MembershipState::Leave, |member| member.membership) + }); let user_id = UserId::try_from(user_id) .map_err(|_| Error::bad_database("Invalid UserId in member PDU."))?; @@ -188,8 +211,9 @@ pub async fn sync_events_route( } } - let joined_since_last_sync = - since_sender_member.map_or(true, |member| member.membership != MembershipState::Join); + let joined_since_last_sync = since_sender_member.map_or(false, |member| { + member.map_or(true, |member| member.membership != MembershipState::Join) + }); if joined_since_last_sync && encrypted_room || new_encrypted_room { // If the user is in a new encrypted room, give them all joined users @@ -434,7 +458,7 @@ pub async fn sync_events_route( let since_member = db .rooms - .pdus_until(sender_id, &room_id, since) + .pdus_after(sender_id, &room_id, since) .next() .and_then(|pdu| pdu.ok()) .and_then(|pdu| { @@ -581,7 +605,7 @@ pub async fn sync_events_route( changed: device_list_updates.into_iter().collect(), left: device_list_left.into_iter().collect(), }, - device_one_time_keys_count: if db.users.last_one_time_keys_update(sender_id)? > since { + device_one_time_keys_count: if db.users.last_one_time_keys_update(sender_id)? > since || since == 0 { db.users.count_one_time_keys(sender_id, device_id)? } else { BTreeMap::new() diff --git a/src/database/rooms.rs b/src/database/rooms.rs index 18881dd84..108edb5dd 100644 --- a/src/database/rooms.rs +++ b/src/database/rooms.rs @@ -521,8 +521,7 @@ impl Rooms { if let Some(old_state_hash) = self.roomid_statehash.get(new_pdu.room_id.as_bytes())? { // Store state for event. The state does not include the event itself. // Instead it's the state before the pdu, so the room's old state. - self.pduid_statehash - .insert(new_pdu_id, &old_state_hash)?; + self.pduid_statehash.insert(new_pdu_id, &old_state_hash)?; if new_pdu.state_key.is_none() { return Ok(old_state_hash); } diff --git a/src/database/users.rs b/src/database/users.rs index 10e1ef301..2e26c1ea4 100644 --- a/src/database/users.rs +++ b/src/database/users.rs @@ -603,7 +603,7 @@ impl Users { .room_state_get(&room_id, &EventType::RoomEncryption, "")? .is_none() { - return Ok(()); + continue; } let mut key = room_id.to_string().as_bytes().to_vec(); From 19207845bc2ec7db6d0ede2ff2b345162bc817ec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20K=C3=B6sters?= Date: Sun, 20 Sep 2020 13:49:13 +0200 Subject: [PATCH 42/51] Fix ruma dependency --- Cargo.lock | 26 +++++++++++++------------- Cargo.toml | 2 +- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 98dbac9c5..9ea58b465 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1554,7 +1554,7 @@ dependencies = [ [[package]] name = "ruma" version = "0.0.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fixes#195b15be25ba1f2d4e0b520f01ecb77143c01eb0" +source = "git+https://github.com/timokoesters/ruma?branch=timo-old-fixes#195b15be25ba1f2d4e0b520f01ecb77143c01eb0" dependencies = [ "ruma-api", "ruma-client-api", @@ -1568,7 +1568,7 @@ dependencies = [ [[package]] name = "ruma-api" version = "0.17.0-alpha.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fixes#195b15be25ba1f2d4e0b520f01ecb77143c01eb0" +source = "git+https://github.com/timokoesters/ruma?branch=timo-old-fixes#195b15be25ba1f2d4e0b520f01ecb77143c01eb0" dependencies = [ "http", "percent-encoding", @@ -1583,7 +1583,7 @@ dependencies = [ [[package]] name = "ruma-api-macros" version = "0.17.0-alpha.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fixes#195b15be25ba1f2d4e0b520f01ecb77143c01eb0" +source = "git+https://github.com/timokoesters/ruma?branch=timo-old-fixes#195b15be25ba1f2d4e0b520f01ecb77143c01eb0" dependencies = [ "proc-macro-crate", "proc-macro2", @@ -1594,7 +1594,7 @@ dependencies = [ [[package]] name = "ruma-client-api" version = "0.10.0-alpha.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fixes#195b15be25ba1f2d4e0b520f01ecb77143c01eb0" +source = "git+https://github.com/timokoesters/ruma?branch=timo-old-fixes#195b15be25ba1f2d4e0b520f01ecb77143c01eb0" dependencies = [ "assign", "http", @@ -1612,7 +1612,7 @@ dependencies = [ [[package]] name = "ruma-common" version = "0.2.0" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fixes#195b15be25ba1f2d4e0b520f01ecb77143c01eb0" +source = "git+https://github.com/timokoesters/ruma?branch=timo-old-fixes#195b15be25ba1f2d4e0b520f01ecb77143c01eb0" dependencies = [ "js_int", "ruma-identifiers", @@ -1625,7 +1625,7 @@ dependencies = [ [[package]] name = "ruma-events" version = "0.22.0-alpha.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fixes#195b15be25ba1f2d4e0b520f01ecb77143c01eb0" +source = "git+https://github.com/timokoesters/ruma?branch=timo-old-fixes#195b15be25ba1f2d4e0b520f01ecb77143c01eb0" dependencies = [ "js_int", "ruma-common", @@ -1640,7 +1640,7 @@ dependencies = [ [[package]] name = "ruma-events-macros" version = "0.22.0-alpha.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fixes#195b15be25ba1f2d4e0b520f01ecb77143c01eb0" +source = "git+https://github.com/timokoesters/ruma?branch=timo-old-fixes#195b15be25ba1f2d4e0b520f01ecb77143c01eb0" dependencies = [ "proc-macro-crate", "proc-macro2", @@ -1651,7 +1651,7 @@ dependencies = [ [[package]] name = "ruma-federation-api" version = "0.0.3" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fixes#195b15be25ba1f2d4e0b520f01ecb77143c01eb0" +source = "git+https://github.com/timokoesters/ruma?branch=timo-old-fixes#195b15be25ba1f2d4e0b520f01ecb77143c01eb0" dependencies = [ "js_int", "ruma-api", @@ -1666,7 +1666,7 @@ dependencies = [ [[package]] name = "ruma-identifiers" version = "0.17.4" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fixes#195b15be25ba1f2d4e0b520f01ecb77143c01eb0" +source = "git+https://github.com/timokoesters/ruma?branch=timo-old-fixes#195b15be25ba1f2d4e0b520f01ecb77143c01eb0" dependencies = [ "rand", "ruma-identifiers-macros", @@ -1678,7 +1678,7 @@ dependencies = [ [[package]] name = "ruma-identifiers-macros" version = "0.17.4" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fixes#195b15be25ba1f2d4e0b520f01ecb77143c01eb0" +source = "git+https://github.com/timokoesters/ruma?branch=timo-old-fixes#195b15be25ba1f2d4e0b520f01ecb77143c01eb0" dependencies = [ "proc-macro2", "quote", @@ -1689,7 +1689,7 @@ dependencies = [ [[package]] name = "ruma-identifiers-validation" version = "0.1.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fixes#195b15be25ba1f2d4e0b520f01ecb77143c01eb0" +source = "git+https://github.com/timokoesters/ruma?branch=timo-old-fixes#195b15be25ba1f2d4e0b520f01ecb77143c01eb0" dependencies = [ "ruma-serde", "serde", @@ -1700,7 +1700,7 @@ dependencies = [ [[package]] name = "ruma-serde" version = "0.2.3" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fixes#195b15be25ba1f2d4e0b520f01ecb77143c01eb0" +source = "git+https://github.com/timokoesters/ruma?branch=timo-old-fixes#195b15be25ba1f2d4e0b520f01ecb77143c01eb0" dependencies = [ "form_urlencoded", "itoa", @@ -1712,7 +1712,7 @@ dependencies = [ [[package]] name = "ruma-signatures" version = "0.6.0-dev.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fixes#195b15be25ba1f2d4e0b520f01ecb77143c01eb0" +source = "git+https://github.com/timokoesters/ruma?branch=timo-old-fixes#195b15be25ba1f2d4e0b520f01ecb77143c01eb0" dependencies = [ "base64", "ring", diff --git a/Cargo.toml b/Cargo.toml index 4945e3c88..db3c68524 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -17,7 +17,7 @@ edition = "2018" rocket = { git = "https://github.com/timokoesters/Rocket.git", branch = "empty_parameters", features = ["tls"] } #ruma = { git = "https://github.com/ruma/ruma", features = ["rand", "client-api", "federation-api", "unstable-pre-spec", "unstable-synapse-quirks"], rev = "987d48666cf166cf12100b5dbc61b5e3385c4014" } # Used for matrix spec type definitions and helpers -ruma = { git = "https://github.com/timokoesters/ruma", features = ["rand", "client-api", "federation-api", "unstable-pre-spec", "unstable-synapse-quirks"], branch = "timo-fixes" } # Used for matrix spec type definitions and helpers +ruma = { git = "https://github.com/timokoesters/ruma", features = ["rand", "client-api", "federation-api", "unstable-pre-spec", "unstable-synapse-quirks"], branch = "timo-old-fixes" } # Used for matrix spec type definitions and helpers #ruma = { path = "../ruma/ruma", features = ["rand", "client-api", "federation-api", "unstable-pre-spec", "unstable-synapse-quirks"] } tokio = "0.2.22" # Used for long polling sled = "0.32.0" # Used for storing data permanently From e08dfd982b87b632590bf125cdc48ee83bad9ebe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20K=C3=B6sters?= Date: Wed, 23 Sep 2020 12:03:08 +0200 Subject: [PATCH 43/51] improvement: look at SRV record when sending requests --- Cargo.lock | 239 +++++++++++++++++++++++++++++++++----- Cargo.toml | 2 + src/client_server/sync.rs | 4 +- src/server_server.rs | 49 ++++++-- 4 files changed, 253 insertions(+), 41 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e142d7250..4cce6668b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1,5 +1,20 @@ # This file is automatically @generated by Cargo. # It is not intended for manual editing. +[[package]] +name = "addr2line" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b6a2d3371669ab3ca9797670853d61402b03d0b4b9ebf33d677dfa720203072" +dependencies = [ + "gimli", +] + +[[package]] +name = "adler" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee2a4ec343196209d6594e19543ae87a39f96d5534d7174822a3ad825dd6ed7e" + [[package]] name = "adler32" version = "1.2.0" @@ -73,6 +88,20 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a" +[[package]] +name = "backtrace" +version = "0.3.50" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46254cf2fdcdf1badb5934448c1bcbe046a56537b3987d96c51a7afc5d03f293" +dependencies = [ + "addr2line", + "cfg-if", + "libc", + "miniz_oxide 0.4.2", + "object", + "rustc-demangle", +] + [[package]] name = "base-x" version = "0.2.6" @@ -192,6 +221,7 @@ dependencies = [ "state-res", "thiserror", "tokio", + "trust-dns-resolver", ] [[package]] @@ -213,7 +243,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1373a16a4937bc34efec7b391f9c1500c30b8478a701a4f44c9165cc0475a6e0" dependencies = [ "percent-encoding", - "time 0.2.20", + "time 0.2.21", "version_check", ] @@ -355,6 +385,18 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "enum-as-inner" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c5f0096a91d210159eceb2ff5e1c4da18388a170e1e3ce948aac9c8fdbbf595" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "fnv" version = "1.0.7" @@ -537,6 +579,12 @@ dependencies = [ "lzw", ] +[[package]] +name = "gimli" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aaf91faf136cb47367fa430cd46e37a788775e7fa104f8b4bcb3861dc389b724" + [[package]] name = "glob" version = "0.3.0" @@ -586,6 +634,17 @@ dependencies = [ "libc", ] +[[package]] +name = "hostname" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c731c3e10504cc8ed35cfe2f1db4c9274c3d35fa486e3b31df46f068ef3e867" +dependencies = [ + "libc", + "match_cfg", + "winapi 0.3.9", +] + [[package]] name = "http" version = "0.2.1" @@ -614,10 +673,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cd179ae861f0c2e53da70d892f5f3029f9594be0c41dc5269cd371691b1dc2f9" [[package]] -name = "hyper" -version = "0.13.7" +name = "httpdate" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e68a8dd9716185d9e64ea473ea6ef63529252e3e27623295a0378a19665d5eb" +checksum = "494b4d60369511e7dea41cf646832512a94e542f68bb9c49e54518e0f468eb47" + +[[package]] +name = "hyper" +version = "0.13.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f3afcfae8af5ad0576a31e768415edb627824129e8e5a29b8bfccb2f234e835" dependencies = [ "bytes", "futures-channel", @@ -627,10 +692,10 @@ dependencies = [ "http", "http-body", "httparse", + "httpdate", "itoa", "pin-project", "socket2", - "time 0.1.44", "tokio", "tower-service", "tracing", @@ -695,9 +760,12 @@ checksum = "cb6ee2a7da03bfc3b66ca47c92c2e392fcc053ea040a85561749b026f7aad09a" [[package]] name = "instant" -version = "0.1.6" +version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b141fdc7836c525d4d594027d318c84161ca17aaf8113ab1f81ab93ae897485" +checksum = "63312a18f7ea8760cdd0a7c5aac1a619752a246b833545e3e36d1f81f7cd9e66" +dependencies = [ + "cfg-if", +] [[package]] name = "iovec" @@ -708,6 +776,18 @@ dependencies = [ "libc", ] +[[package]] +name = "ipconfig" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7e2f18aece9709094573a9f24f483c4f65caa4298e2f7ae1b71cc65d853fad7" +dependencies = [ + "socket2", + "widestring", + "winapi 0.3.9", + "winreg 0.6.2", +] + [[package]] name = "ipnet" version = "2.3.0" @@ -778,6 +858,12 @@ version = "0.2.77" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f2f96b10ec2560088a8e76961b00d47107b3a625fecb76dedb29ee7ccbf98235" +[[package]] +name = "linked-hash-map" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8dd5a6d5999d9907cda8ed67bbd137d3af8085216c2ac62de5be860bd41f304a" + [[package]] name = "lock_api" version = "0.4.1" @@ -796,6 +882,15 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "lru-cache" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31e24f1ad8321ca0e8a1e0ac13f23cb668e6f5466c2c57319f6a5cf1cc8e3b1c" +dependencies = [ + "linked-hash-map", +] + [[package]] name = "lzw" version = "0.10.0" @@ -808,6 +903,12 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3e2e65a1a2e43cfcb47a895c4c8b10d1f4a61097f9f254f183aee60cad9c651d" +[[package]] +name = "match_cfg" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ffbee8634e0d45d258acb448e7eaab3fce7a0a467395d4d9f228e3c1f01fb2e4" + [[package]] name = "matchers" version = "0.0.1" @@ -837,9 +938,9 @@ checksum = "3728d817d99e5ac407411fa471ff9800a778d88a24685968b36824eaf4bee400" [[package]] name = "memoffset" -version = "0.5.5" +version = "0.5.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c198b026e1bbf08a937e94c6c60f9ec4a2267f5b0d2eec9c1b21b061ce2be55f" +checksum = "043175f069eda7b85febe4a74abbaeff828d9f8b448515d3151a14a3542811aa" dependencies = [ "autocfg", ] @@ -869,6 +970,16 @@ dependencies = [ "adler32", ] +[[package]] +name = "miniz_oxide" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c60c0dfe32c10b43a144bad8fc83538c52f58302c92300ea7ec7bf7b38d5a7b9" +dependencies = [ + "adler", + "autocfg", +] + [[package]] name = "mio" version = "0.6.22" @@ -991,6 +1102,12 @@ dependencies = [ "libc", ] +[[package]] +name = "object" +version = "0.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ab52be62400ca80aa00285d25253d7f7c437b7375c4de678f5405d3afe82ca5" + [[package]] name = "once_cell" version = "1.4.1" @@ -1130,7 +1247,7 @@ dependencies = [ "bitflags", "crc32fast", "deflate", - "miniz_oxide", + "miniz_oxide 0.3.7", ] [[package]] @@ -1181,6 +1298,12 @@ dependencies = [ "yansi", ] +[[package]] +name = "quick-error" +version = "1.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" + [[package]] name = "quote" version = "1.0.7" @@ -1334,7 +1457,17 @@ dependencies = [ "wasm-bindgen", "wasm-bindgen-futures", "web-sys", - "winreg", + "winreg 0.7.0", +] + +[[package]] +name = "resolv-conf" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11834e137f3b14e309437a8276714eed3a80d1ef894869e510f2c0c0b98b9f4a" +dependencies = [ + "hostname", + "quick-error", ] [[package]] @@ -1370,7 +1503,7 @@ dependencies = [ "rocket_codegen", "rocket_http", "state", - "time 0.2.20", + "time 0.2.21", "tokio", "toml", "version_check", @@ -1405,7 +1538,7 @@ dependencies = [ "ref-cast", "smallvec", "state", - "time 0.2.20", + "time 0.2.21", "tokio", "tokio-rustls", "unicode-xid", @@ -1414,7 +1547,6 @@ dependencies = [ [[package]] name = "ruma" version = "0.0.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#425d34d4cfb5aefe5bab6957d71bc9389384c1e5" dependencies = [ "ruma-api", "ruma-appservice-api", @@ -1430,7 +1562,6 @@ dependencies = [ [[package]] name = "ruma-api" version = "0.17.0-alpha.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#425d34d4cfb5aefe5bab6957d71bc9389384c1e5" dependencies = [ "http", "percent-encoding", @@ -1445,7 +1576,6 @@ dependencies = [ [[package]] name = "ruma-api-macros" version = "0.17.0-alpha.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#425d34d4cfb5aefe5bab6957d71bc9389384c1e5" dependencies = [ "proc-macro-crate", "proc-macro2", @@ -1456,7 +1586,6 @@ dependencies = [ [[package]] name = "ruma-appservice-api" version = "0.2.0-alpha.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#425d34d4cfb5aefe5bab6957d71bc9389384c1e5" dependencies = [ "ruma-api", "ruma-common", @@ -1469,7 +1598,6 @@ dependencies = [ [[package]] name = "ruma-client-api" version = "0.10.0-alpha.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#425d34d4cfb5aefe5bab6957d71bc9389384c1e5" dependencies = [ "assign", "http", @@ -1488,7 +1616,6 @@ dependencies = [ [[package]] name = "ruma-common" version = "0.2.0" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#425d34d4cfb5aefe5bab6957d71bc9389384c1e5" dependencies = [ "js_int", "ruma-api", @@ -1502,7 +1629,6 @@ dependencies = [ [[package]] name = "ruma-events" version = "0.22.0-alpha.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#425d34d4cfb5aefe5bab6957d71bc9389384c1e5" dependencies = [ "js_int", "ruma-common", @@ -1517,7 +1643,6 @@ dependencies = [ [[package]] name = "ruma-events-macros" version = "0.22.0-alpha.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#425d34d4cfb5aefe5bab6957d71bc9389384c1e5" dependencies = [ "proc-macro-crate", "proc-macro2", @@ -1528,7 +1653,6 @@ dependencies = [ [[package]] name = "ruma-federation-api" version = "0.0.3" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#425d34d4cfb5aefe5bab6957d71bc9389384c1e5" dependencies = [ "js_int", "ruma-api", @@ -1543,7 +1667,6 @@ dependencies = [ [[package]] name = "ruma-identifiers" version = "0.17.4" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#425d34d4cfb5aefe5bab6957d71bc9389384c1e5" dependencies = [ "rand", "ruma-identifiers-macros", @@ -1555,7 +1678,6 @@ dependencies = [ [[package]] name = "ruma-identifiers-macros" version = "0.17.4" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#425d34d4cfb5aefe5bab6957d71bc9389384c1e5" dependencies = [ "proc-macro2", "quote", @@ -1566,7 +1688,6 @@ dependencies = [ [[package]] name = "ruma-identifiers-validation" version = "0.1.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#425d34d4cfb5aefe5bab6957d71bc9389384c1e5" dependencies = [ "serde", "strum", @@ -1575,7 +1696,6 @@ dependencies = [ [[package]] name = "ruma-serde" version = "0.2.3" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#425d34d4cfb5aefe5bab6957d71bc9389384c1e5" dependencies = [ "form_urlencoded", "itoa", @@ -1587,7 +1707,6 @@ dependencies = [ [[package]] name = "ruma-signatures" version = "0.6.0-dev.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#425d34d4cfb5aefe5bab6957d71bc9389384c1e5" dependencies = [ "base64", "ring", @@ -1607,6 +1726,12 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "rustc-demangle" +version = "0.1.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c691c0e608126e00913e33f0ccf3727d5fc84573623b8d65b2df340b5201783" + [[package]] name = "rustc_version" version = "0.2.3" @@ -1831,7 +1956,6 @@ checksum = "7345c971d1ef21ffdbd103a75990a15eb03604fc8b8852ca8cb418ee1a099028" [[package]] name = "state-res" version = "0.1.0" -source = "git+https://github.com/timokoesters/state-res?branch=spec-comp#d11a3feb5307715ab5d86af8f25d4bccfee6264b" dependencies = [ "itertools", "js_int", @@ -1981,9 +2105,9 @@ dependencies = [ [[package]] name = "time" -version = "0.2.20" +version = "0.2.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d4953c513c9bf1b97e9cdd83f11d60c4b0a83462880a360d80d96953a953fee" +checksum = "2c2e31fb28e2a9f01f5ed6901b066c1ba2333c04b64dc61254142bafcb3feb2c" dependencies = [ "const_fn", "libc", @@ -2182,6 +2306,46 @@ dependencies = [ "tracing-serde", ] +[[package]] +name = "trust-dns-proto" +version = "0.19.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdd7061ba6f4d4d9721afedffbfd403f20f39a4301fee1b70d6fcd09cca69f28" +dependencies = [ + "async-trait", + "backtrace", + "enum-as-inner", + "futures", + "idna", + "lazy_static", + "log", + "rand", + "smallvec", + "thiserror", + "tokio", + "url", +] + +[[package]] +name = "trust-dns-resolver" +version = "0.19.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f23cdfdc3d8300b3c50c9e84302d3bd6d860fb9529af84ace6cf9665f181b77" +dependencies = [ + "backtrace", + "cfg-if", + "futures", + "ipconfig", + "lazy_static", + "log", + "lru-cache", + "resolv-conf", + "smallvec", + "thiserror", + "tokio", + "trust-dns-proto", +] + [[package]] name = "try-lock" version = "0.2.3" @@ -2366,6 +2530,12 @@ dependencies = [ "untrusted", ] +[[package]] +name = "widestring" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a763e303c0e0f23b0da40888724762e802a8ffefbc22de4127ef42493c2ea68c" + [[package]] name = "winapi" version = "0.2.8" @@ -2400,6 +2570,15 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +[[package]] +name = "winreg" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2986deb581c4fe11b621998a5e53361efe6b48a151178d0cd9eeffa4dc6acc9" +dependencies = [ + "winapi 0.3.9", +] + [[package]] name = "winreg" version = "0.7.0" diff --git a/Cargo.toml b/Cargo.toml index 60296a2ca..2126e4278 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -56,6 +56,8 @@ image = { version = "0.23.9", default-features = false, features = ["jpeg", "png base64 = "0.12.3" # Used when hashing the state ring = "0.16.15" +# Used when querying the SRV record of other servers +trust-dns-resolver = "0.19.5" [features] default = ["conduit_bin"] diff --git a/src/client_server/sync.rs b/src/client_server/sync.rs index 2f2c8eacf..aec03afae 100644 --- a/src/client_server/sync.rs +++ b/src/client_server/sync.rs @@ -605,7 +605,9 @@ pub async fn sync_events_route( changed: device_list_updates.into_iter().collect(), left: device_list_left.into_iter().collect(), }, - device_one_time_keys_count: if db.users.last_one_time_keys_update(sender_id)? > since || since == 0 { + device_one_time_keys_count: if db.users.last_one_time_keys_update(sender_id)? > since + || since == 0 + { db.users.count_one_time_keys(sender_id, device_id)? } else { BTreeMap::new() diff --git a/src/server_server.rs b/src/server_server.rs index 6f2b17903..3ebbeac46 100644 --- a/src/server_server.rs +++ b/src/server_server.rs @@ -1,5 +1,5 @@ use crate::{client_server, ConduitResult, Database, Error, PduEvent, Result, Ruma}; -use http::header::{HeaderValue, AUTHORIZATION}; +use http::header::{HeaderValue, AUTHORIZATION, HOST}; use log::warn; use rocket::{get, post, put, response::content::Json, State}; use ruma::{ @@ -23,6 +23,7 @@ use std::{ fmt::Debug, time::{Duration, SystemTime}, }; +use trust_dns_resolver::AsyncResolver; pub async fn request_well_known( globals: &crate::database::globals::Globals, @@ -54,16 +55,36 @@ pub async fn send_request( where T: Debug, { + let resolver = AsyncResolver::tokio_from_system_conf() + .await + .map_err(|_| Error::BadConfig("Failed to set up trust dns resolver with system config."))?; + + let mut host = None; + let actual_destination = "https://".to_owned() - + &request_well_known(globals, &destination.as_str()) - .await - .unwrap_or_else(|| { - let mut destination = destination.as_str().to_owned(); - if destination.find(':').is_none() { - destination += ":8448"; + + &if let Some(mut delegated_hostname) = + request_well_known(globals, &destination.as_str()).await + { + if let Ok(Some(srv)) = resolver + .srv_lookup(format!("_matrix._tcp.{}", delegated_hostname)) + .await + .map(|srv| srv.iter().next().map(|result| result.target().to_string())) + { + host = Some(delegated_hostname); + srv.trim_end_matches('.').to_owned() + } else { + if delegated_hostname.find(':').is_none() { + delegated_hostname += ":8448"; } - destination - }); + delegated_hostname + } + } else { + let mut destination = destination.as_str().to_owned(); + if destination.find(':').is_none() { + destination += ":8448"; + } + destination + }; let mut http_request = request .try_into_http_request(&actual_destination, Some("")) @@ -129,9 +150,17 @@ where } } - let reqwest_request = reqwest::Request::try_from(http_request) + if let Some(host) = host { + http_request + .headers_mut() + .insert(HOST, HeaderValue::from_str(&host).unwrap()); + } + + let mut reqwest_request = reqwest::Request::try_from(http_request) .expect("all http requests are valid reqwest requests"); + *reqwest_request.timeout_mut() = Some(Duration::from_secs(30)); + let reqwest_response = globals.reqwest_client().execute(reqwest_request).await; // Because reqwest::Response -> http::Response is complicated: From 26e200e290133e0e2424da0d1e354eaa764c6d4c Mon Sep 17 00:00:00 2001 From: miruka Date: Fri, 25 Sep 2020 14:18:36 -0400 Subject: [PATCH 44/51] Reduce media ID length from 256 to 32 Most common filesystems limit paths to 255 bytes. This change brings down the media ID length to be similar to Synapse servers (25), and makes it possible for clients to download media with the ID included in the filename. --- src/client_server/media.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/client_server/media.rs b/src/client_server/media.rs index efcb3a6d3..394cf7409 100644 --- a/src/client_server/media.rs +++ b/src/client_server/media.rs @@ -9,7 +9,7 @@ use ruma::api::client::{ use rocket::{get, post}; use std::convert::TryInto; -const MXC_LENGTH: usize = 256; +const MXC_LENGTH: usize = 32; #[cfg_attr(feature = "conduit_bin", get("/_matrix/media/r0/config"))] pub fn get_media_config_route( From ab332363ce3af716694cde7bf35cf421e91707d3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20K=C3=B6sters?= Date: Wed, 23 Sep 2020 15:23:29 +0200 Subject: [PATCH 45/51] fix: don't send new requests to servers if we are already waiting --- Cargo.lock | 15 ++++ src/database/sending.rs | 149 ++++++++++++++++++++++++++-------------- src/server_server.rs | 6 +- 3 files changed, 117 insertions(+), 53 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 4cce6668b..5f6b21c5a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1547,6 +1547,7 @@ dependencies = [ [[package]] name = "ruma" version = "0.0.1" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#a6486e7a00183b4578650528d65e83318da53b23" dependencies = [ "ruma-api", "ruma-appservice-api", @@ -1562,6 +1563,7 @@ dependencies = [ [[package]] name = "ruma-api" version = "0.17.0-alpha.1" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#a6486e7a00183b4578650528d65e83318da53b23" dependencies = [ "http", "percent-encoding", @@ -1576,6 +1578,7 @@ dependencies = [ [[package]] name = "ruma-api-macros" version = "0.17.0-alpha.1" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#a6486e7a00183b4578650528d65e83318da53b23" dependencies = [ "proc-macro-crate", "proc-macro2", @@ -1586,6 +1589,7 @@ dependencies = [ [[package]] name = "ruma-appservice-api" version = "0.2.0-alpha.1" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#a6486e7a00183b4578650528d65e83318da53b23" dependencies = [ "ruma-api", "ruma-common", @@ -1598,6 +1602,7 @@ dependencies = [ [[package]] name = "ruma-client-api" version = "0.10.0-alpha.1" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#a6486e7a00183b4578650528d65e83318da53b23" dependencies = [ "assign", "http", @@ -1616,6 +1621,7 @@ dependencies = [ [[package]] name = "ruma-common" version = "0.2.0" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#a6486e7a00183b4578650528d65e83318da53b23" dependencies = [ "js_int", "ruma-api", @@ -1629,6 +1635,7 @@ dependencies = [ [[package]] name = "ruma-events" version = "0.22.0-alpha.1" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#a6486e7a00183b4578650528d65e83318da53b23" dependencies = [ "js_int", "ruma-common", @@ -1643,6 +1650,7 @@ dependencies = [ [[package]] name = "ruma-events-macros" version = "0.22.0-alpha.1" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#a6486e7a00183b4578650528d65e83318da53b23" dependencies = [ "proc-macro-crate", "proc-macro2", @@ -1653,6 +1661,7 @@ dependencies = [ [[package]] name = "ruma-federation-api" version = "0.0.3" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#a6486e7a00183b4578650528d65e83318da53b23" dependencies = [ "js_int", "ruma-api", @@ -1667,6 +1676,7 @@ dependencies = [ [[package]] name = "ruma-identifiers" version = "0.17.4" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#a6486e7a00183b4578650528d65e83318da53b23" dependencies = [ "rand", "ruma-identifiers-macros", @@ -1678,6 +1688,7 @@ dependencies = [ [[package]] name = "ruma-identifiers-macros" version = "0.17.4" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#a6486e7a00183b4578650528d65e83318da53b23" dependencies = [ "proc-macro2", "quote", @@ -1688,6 +1699,7 @@ dependencies = [ [[package]] name = "ruma-identifiers-validation" version = "0.1.1" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#a6486e7a00183b4578650528d65e83318da53b23" dependencies = [ "serde", "strum", @@ -1696,6 +1708,7 @@ dependencies = [ [[package]] name = "ruma-serde" version = "0.2.3" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#a6486e7a00183b4578650528d65e83318da53b23" dependencies = [ "form_urlencoded", "itoa", @@ -1707,6 +1720,7 @@ dependencies = [ [[package]] name = "ruma-signatures" version = "0.6.0-dev.1" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#a6486e7a00183b4578650528d65e83318da53b23" dependencies = [ "base64", "ring", @@ -1956,6 +1970,7 @@ checksum = "7345c971d1ef21ffdbd103a75990a15eb03604fc8b8852ca8cb418ee1a099028" [[package]] name = "state-res" version = "0.1.0" +source = "git+https://github.com/timokoesters/state-res?branch=spec-comp#d11a3feb5307715ab5d86af8f25d4bccfee6264b" dependencies = [ "itertools", "js_int", diff --git a/src/database/sending.rs b/src/database/sending.rs index a3f15742b..d3c7fc6a5 100644 --- a/src/database/sending.rs +++ b/src/database/sending.rs @@ -1,8 +1,11 @@ -use std::{convert::TryFrom, time::SystemTime}; +use std::{collections::HashSet, convert::TryFrom, time::SystemTime}; use crate::{server_server, utils, Error, Result}; +use federation::transactions::send_transaction_message; +use log::warn; use rocket::futures::stream::{FuturesUnordered, StreamExt}; use ruma::{api::federation, Raw, ServerName}; +use sled::IVec; use tokio::select; pub struct Sending { @@ -18,66 +21,49 @@ impl Sending { tokio::spawn(async move { let mut futures = FuturesUnordered::new(); + let mut waiting_servers = HashSet::new(); + let mut subscriber = serverpduids.watch_prefix(b""); loop { select! { - Some(_) = futures.next() => {}, - Some(event) = &mut subscriber => { - let serverpduid = if let sled::Event::Insert {key, ..} = event { - key - } else - { return Err::<(), Error>(Error::bad_database("")); }; - let mut parts = serverpduid.splitn(2, |&b| b == 0xff); - let server = Box::::try_from( - utils::string_from_bytes(parts.next().expect("splitn will always return 1 or more elements")) - .map_err(|_| Error::bad_database("ServerName in serverpduid bytes are invalid."))? - ).map_err(|_| Error::bad_database("ServerName in serverpduid is invalid."))?; - - let pdu_id = parts.next().ok_or_else(|| Error::bad_database("Invalid serverpduid in db."))?; - let mut pdu_json = rooms.get_pdu_json_from_id(&pdu_id.into())?.ok_or_else(|| Error::bad_database("Event in serverpduids not found in db."))?; - - if let Some(unsigned) = pdu_json - .as_object_mut() - .expect("json is object") - .get_mut("unsigned") { - unsigned.as_object_mut().expect("unsigned is object").remove("transaction_id"); - } - - pdu_json - .as_object_mut() - .expect("json is object") - .remove("event_id"); - - let raw_json = - serde_json::from_value::>(pdu_json).expect("Raw::from_value always works"); - - let globals = &globals; - - futures.push( - async move { - let pdus = vec![raw_json]; - let transaction_id = utils::random_string(16); - - server_server::send_request( - &globals, - server, - federation::transactions::send_transaction_message::v1::Request { - origin: globals.server_name(), - pdus: &pdus, - edus: &[], - origin_server_ts: SystemTime::now(), - transaction_id: &transaction_id, - }, - ).await + Some(server) = futures.next() => { + warn!("response: {:?}", &server); + match server { + Ok((server, _response)) => { + waiting_servers.remove(&server) } - ); + Err((server, _e)) => { + waiting_servers.remove(&server) + } + }; }, + Some(event) = &mut subscriber => { + if let sled::Event::Insert { key, .. } = event { + let serverpduid = key.clone(); + let mut parts = serverpduid.splitn(2, |&b| b == 0xff); + + if let Some((server, pdu_id)) = utils::string_from_bytes( + parts + .next() + .expect("splitn will always return 1 or more elements"), + ) + .map_err(|_| Error::bad_database("ServerName in serverpduid bytes are invalid.")) + .and_then(|server_str|Box::::try_from(server_str) + .map_err(|_| Error::bad_database("ServerName in serverpduid is invalid."))) + .ok() + .filter(|server| waiting_servers.insert(server.clone())) + .and_then(|server| parts + .next() + .ok_or_else(|| Error::bad_database("Invalid serverpduid in db.")).ok().map(|pdu_id| (server, pdu_id))) + { + futures.push(Self::handle_event(server, pdu_id.into(), &globals, &rooms)); + } + } + } } } }); } - /* - */ pub fn send_pdu(&self, server: Box, pdu_id: &[u8]) -> Result<()> { let mut key = server.as_bytes().to_vec(); @@ -87,4 +73,63 @@ impl Sending { Ok(()) } + + async fn handle_event( + server: Box, + pdu_id: IVec, + globals: &super::globals::Globals, + rooms: &super::rooms::Rooms, + ) -> std::result::Result< + (Box, send_transaction_message::v1::Response), + (Box, Error), + > { + let mut pdu_json = rooms + .get_pdu_json_from_id(&pdu_id) + .map_err(|e| (server.clone(), e))? + .ok_or_else(|| { + ( + server.clone(), + Error::bad_database("Event in serverpduids not found in db."), + ) + })?; + + if let Some(unsigned) = pdu_json + .as_object_mut() + .expect("json is object") + .get_mut("unsigned") + { + unsigned + .as_object_mut() + .expect("unsigned is object") + .remove("transaction_id"); + } + + pdu_json + .as_object_mut() + .expect("json is object") + .remove("event_id"); + + let raw_json = + serde_json::from_value::>(pdu_json).expect("Raw::from_value always works"); + + let globals = &globals; + + let pdus = vec![raw_json]; + let transaction_id = utils::random_string(16); + + server_server::send_request( + &globals, + server.clone(), + send_transaction_message::v1::Request { + origin: globals.server_name(), + pdus: &pdus, + edus: &[], + origin_server_ts: SystemTime::now(), + transaction_id: &transaction_id, + }, + ) + .await + .map(|response| (server.clone(), response)) + .map_err(|e| (server, e)) + } } diff --git a/src/server_server.rs b/src/server_server.rs index 3ebbeac46..d67b0b605 100644 --- a/src/server_server.rs +++ b/src/server_server.rs @@ -161,6 +161,7 @@ where *reqwest_request.timeout_mut() = Some(Duration::from_secs(30)); + let url = reqwest_request.url().clone(); let reqwest_response = globals.reqwest_client().execute(reqwest_request).await; // Because reqwest::Response -> http::Response is complicated: @@ -189,7 +190,10 @@ where .expect("reqwest body is valid http body"), ); response.map_err(|e| { - warn!("Server returned bad response: {:?}", e); + warn!( + "Server returned bad response {} ({}): {:?}", + destination, url, e + ); Error::BadServerResponse("Server returned bad response.") }) } From 0d6159c2dafc87685dc2e6fd79b034e4b3f21c7c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20K=C3=B6sters?= Date: Fri, 25 Sep 2020 12:26:29 +0200 Subject: [PATCH 46/51] improvement: get_missing_events route and cleanup --- Cargo.lock | 32 ++++++++++++------------- src/database/sending.rs | 52 ++++++++++++----------------------------- src/main.rs | 1 + src/pdu.rs | 43 ++++++++++++++-------------------- src/server_server.rs | 47 +++++++++++++++++++++++++++++++++++-- 5 files changed, 95 insertions(+), 80 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 5f6b21c5a..ed20ca37a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -627,9 +627,9 @@ dependencies = [ [[package]] name = "hermit-abi" -version = "0.1.15" +version = "0.1.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3deed196b6e7f9e44a2ae8d94225d80302d81208b1bb673fd21fe634645c85a9" +checksum = "4c30f6d0bc6b00693347368a67d41b58f2fb851215ff1da49e90fe2c5c667151" dependencies = [ "libc", ] @@ -1547,7 +1547,7 @@ dependencies = [ [[package]] name = "ruma" version = "0.0.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#a6486e7a00183b4578650528d65e83318da53b23" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#47fab87325b71b7f6c2fb3cd276d1f813e42abf7" dependencies = [ "ruma-api", "ruma-appservice-api", @@ -1563,7 +1563,7 @@ dependencies = [ [[package]] name = "ruma-api" version = "0.17.0-alpha.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#a6486e7a00183b4578650528d65e83318da53b23" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#47fab87325b71b7f6c2fb3cd276d1f813e42abf7" dependencies = [ "http", "percent-encoding", @@ -1578,7 +1578,7 @@ dependencies = [ [[package]] name = "ruma-api-macros" version = "0.17.0-alpha.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#a6486e7a00183b4578650528d65e83318da53b23" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#47fab87325b71b7f6c2fb3cd276d1f813e42abf7" dependencies = [ "proc-macro-crate", "proc-macro2", @@ -1589,7 +1589,7 @@ dependencies = [ [[package]] name = "ruma-appservice-api" version = "0.2.0-alpha.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#a6486e7a00183b4578650528d65e83318da53b23" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#47fab87325b71b7f6c2fb3cd276d1f813e42abf7" dependencies = [ "ruma-api", "ruma-common", @@ -1602,7 +1602,7 @@ dependencies = [ [[package]] name = "ruma-client-api" version = "0.10.0-alpha.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#a6486e7a00183b4578650528d65e83318da53b23" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#47fab87325b71b7f6c2fb3cd276d1f813e42abf7" dependencies = [ "assign", "http", @@ -1621,7 +1621,7 @@ dependencies = [ [[package]] name = "ruma-common" version = "0.2.0" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#a6486e7a00183b4578650528d65e83318da53b23" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#47fab87325b71b7f6c2fb3cd276d1f813e42abf7" dependencies = [ "js_int", "ruma-api", @@ -1635,7 +1635,7 @@ dependencies = [ [[package]] name = "ruma-events" version = "0.22.0-alpha.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#a6486e7a00183b4578650528d65e83318da53b23" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#47fab87325b71b7f6c2fb3cd276d1f813e42abf7" dependencies = [ "js_int", "ruma-common", @@ -1650,7 +1650,7 @@ dependencies = [ [[package]] name = "ruma-events-macros" version = "0.22.0-alpha.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#a6486e7a00183b4578650528d65e83318da53b23" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#47fab87325b71b7f6c2fb3cd276d1f813e42abf7" dependencies = [ "proc-macro-crate", "proc-macro2", @@ -1661,7 +1661,7 @@ dependencies = [ [[package]] name = "ruma-federation-api" version = "0.0.3" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#a6486e7a00183b4578650528d65e83318da53b23" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#47fab87325b71b7f6c2fb3cd276d1f813e42abf7" dependencies = [ "js_int", "ruma-api", @@ -1676,7 +1676,7 @@ dependencies = [ [[package]] name = "ruma-identifiers" version = "0.17.4" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#a6486e7a00183b4578650528d65e83318da53b23" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#47fab87325b71b7f6c2fb3cd276d1f813e42abf7" dependencies = [ "rand", "ruma-identifiers-macros", @@ -1688,7 +1688,7 @@ dependencies = [ [[package]] name = "ruma-identifiers-macros" version = "0.17.4" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#a6486e7a00183b4578650528d65e83318da53b23" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#47fab87325b71b7f6c2fb3cd276d1f813e42abf7" dependencies = [ "proc-macro2", "quote", @@ -1699,7 +1699,7 @@ dependencies = [ [[package]] name = "ruma-identifiers-validation" version = "0.1.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#a6486e7a00183b4578650528d65e83318da53b23" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#47fab87325b71b7f6c2fb3cd276d1f813e42abf7" dependencies = [ "serde", "strum", @@ -1708,7 +1708,7 @@ dependencies = [ [[package]] name = "ruma-serde" version = "0.2.3" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#a6486e7a00183b4578650528d65e83318da53b23" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#47fab87325b71b7f6c2fb3cd276d1f813e42abf7" dependencies = [ "form_urlencoded", "itoa", @@ -1720,7 +1720,7 @@ dependencies = [ [[package]] name = "ruma-signatures" version = "0.6.0-dev.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#a6486e7a00183b4578650528d65e83318da53b23" +source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#47fab87325b71b7f6c2fb3cd276d1f813e42abf7" dependencies = [ "base64", "ring", diff --git a/src/database/sending.rs b/src/database/sending.rs index d3c7fc6a5..1ed94cc2a 100644 --- a/src/database/sending.rs +++ b/src/database/sending.rs @@ -1,10 +1,10 @@ use std::{collections::HashSet, convert::TryFrom, time::SystemTime}; -use crate::{server_server, utils, Error, Result}; +use crate::{server_server, utils, Error, PduEvent, Result}; use federation::transactions::send_transaction_message; use log::warn; use rocket::futures::stream::{FuturesUnordered, StreamExt}; -use ruma::{api::federation, Raw, ServerName}; +use ruma::{api::federation, ServerName}; use sled::IVec; use tokio::select; @@ -83,49 +83,27 @@ impl Sending { (Box, send_transaction_message::v1::Response), (Box, Error), > { - let mut pdu_json = rooms - .get_pdu_json_from_id(&pdu_id) - .map_err(|e| (server.clone(), e))? - .ok_or_else(|| { - ( - server.clone(), - Error::bad_database("Event in serverpduids not found in db."), - ) - })?; - - if let Some(unsigned) = pdu_json - .as_object_mut() - .expect("json is object") - .get_mut("unsigned") - { - unsigned - .as_object_mut() - .expect("unsigned is object") - .remove("transaction_id"); - } - - pdu_json - .as_object_mut() - .expect("json is object") - .remove("event_id"); - - let raw_json = - serde_json::from_value::>(pdu_json).expect("Raw::from_value always works"); - - let globals = &globals; - - let pdus = vec![raw_json]; - let transaction_id = utils::random_string(16); + let pdu_json = PduEvent::to_outgoing_federation_event( + rooms + .get_pdu_json_from_id(&pdu_id) + .map_err(|e| (server.clone(), e))? + .ok_or_else(|| { + ( + server.clone(), + Error::bad_database("Event in serverpduids not found in db."), + ) + })?, + ); server_server::send_request( &globals, server.clone(), send_transaction_message::v1::Request { origin: globals.server_name(), - pdus: &pdus, + pdus: &[pdu_json], edus: &[], origin_server_ts: SystemTime::now(), - transaction_id: &transaction_id, + transaction_id: &utils::random_string(16), }, ) .await diff --git a/src/main.rs b/src/main.rs index 06fda59a0..fa1cc5cac 100644 --- a/src/main.rs +++ b/src/main.rs @@ -125,6 +125,7 @@ fn setup_rocket() -> rocket::Rocket { server_server::get_public_rooms_route, server_server::get_public_rooms_filtered_route, server_server::send_transaction_message_route, + server_server::get_missing_events_route, ], ) .attach(AdHoc::on_attach("Config", |mut rocket| async { diff --git a/src/pdu.rs b/src/pdu.rs index d5b5415c5..4b1df4b83 100644 --- a/src/pdu.rs +++ b/src/pdu.rs @@ -1,7 +1,6 @@ use crate::Error; use js_int::UInt; use ruma::{ - events::pdu::PduStub, events::{ pdu::EventHash, room::member::MemberEventContent, AnyEvent, AnyRoomEvent, AnyStateEvent, AnyStrippedStateEvent, AnySyncRoomEvent, AnySyncStateEvent, EventType, StateEvent, @@ -200,32 +199,26 @@ impl PduEvent { serde_json::from_value(json).expect("Raw::from_value always works") } - pub fn to_outgoing_federation_event(&self) -> Raw { - let mut unsigned = self.unsigned.clone(); - unsigned.remove("transaction_id"); - - let mut json = json!({ - "room_id": self.room_id, - "sender": self.sender, - "origin_server_ts": self.origin_server_ts, - "type": self.kind, - "content": self.content, - "prev_events": self.prev_events, - "depth": self.depth, - "auth_events": self.auth_events, - "unsigned": unsigned, - "hashes": self.hashes, - "signatures": self.signatures, - }); - - if let Some(state_key) = &self.state_key { - json["state_key"] = json!(state_key); - } - if let Some(redacts) = &self.redacts { - json["redacts"] = json!(redacts); + pub fn to_outgoing_federation_event( + mut pdu_json: serde_json::Value, + ) -> Raw { + if let Some(unsigned) = pdu_json + .as_object_mut() + .expect("json is object") + .get_mut("unsigned") + { + unsigned + .as_object_mut() + .expect("unsigned is object") + .remove("transaction_id"); } - serde_json::from_value(json).expect("Raw::from_value always works") + pdu_json + .as_object_mut() + .expect("json is object") + .remove("event_id"); + + serde_json::from_value::>(pdu_json).expect("Raw::from_value always works") } } diff --git a/src/server_server.rs b/src/server_server.rs index d67b0b605..2d52c4acc 100644 --- a/src/server_server.rs +++ b/src/server_server.rs @@ -3,13 +3,13 @@ use http::header::{HeaderValue, AUTHORIZATION, HOST}; use log::warn; use rocket::{get, post, put, response::content::Json, State}; use ruma::{ - api::federation::directory::get_public_rooms_filtered, api::{ federation::{ - directory::get_public_rooms, + directory::{get_public_rooms, get_public_rooms_filtered}, discovery::{ get_server_keys, get_server_version::v1 as get_server_version, ServerKey, VerifyKey, }, + event::get_missing_events, transactions::send_transaction_message, }, OutgoingRequest, @@ -373,3 +373,46 @@ pub fn send_transaction_message_route<'a>( } .into()) } + +#[cfg_attr( + feature = "conduit_bin", + post("/_matrix/federation/v1/get_missing_events/<_>", data = "") +)] +pub fn get_missing_events_route<'a>( + db: State<'a, Database>, + body: Ruma>, +) -> ConduitResult { + let mut queued_events = body.latest_events.clone(); + let mut events = Vec::new(); + + let mut i = 0; + while i < queued_events.len() && events.len() < u64::from(body.limit) as usize { + if let Some(pdu) = db.rooms.get_pdu_json(&queued_events[i])? { + if body.earliest_events.contains( + &serde_json::from_value( + pdu.get("event_id") + .cloned() + .ok_or_else(|| Error::bad_database("Event in db has no event_id field."))?, + ) + .map_err(|_| Error::bad_database("Invalid event_id field in pdu in db."))?, + ) { + i += 1; + continue; + } + queued_events.extend_from_slice( + &serde_json::from_value::>( + pdu.get("prev_events").cloned().ok_or_else(|| { + Error::bad_database("Invalid prev_events field of pdu in db.") + })?, + ) + .map_err(|_| Error::bad_database("Invalid prev_events content in pdu in db."))?, + ); + events.push(PduEvent::to_outgoing_federation_event(pdu)); + } + i += 1; + } + + dbg!(&events); + + Ok(get_missing_events::v1::Response { events }.into()) +} From bcd1fe18561cbda0f26f53464325fccd177e42ad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20K=C3=B6sters?= Date: Mon, 5 Oct 2020 22:19:22 +0200 Subject: [PATCH 47/51] feat: admin room --- Cargo.lock | 15 - src/client_server/account.rs | 301 +++++++++++++++-- src/client_server/membership.rs | 196 ++++++----- src/client_server/message.rs | 43 ++- src/client_server/profile.rs | 140 ++++---- src/client_server/redact.rs | 37 +-- src/client_server/room.rs | 556 +++++++++++++++----------------- src/client_server/state.rs | 31 +- src/database/rooms.rs | 49 ++- src/database/users.rs | 5 + src/main.rs | 1 + src/server_server.rs | 64 +++- 12 files changed, 864 insertions(+), 574 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ed20ca37a..6571f7cdb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1547,7 +1547,6 @@ dependencies = [ [[package]] name = "ruma" version = "0.0.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#47fab87325b71b7f6c2fb3cd276d1f813e42abf7" dependencies = [ "ruma-api", "ruma-appservice-api", @@ -1563,7 +1562,6 @@ dependencies = [ [[package]] name = "ruma-api" version = "0.17.0-alpha.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#47fab87325b71b7f6c2fb3cd276d1f813e42abf7" dependencies = [ "http", "percent-encoding", @@ -1578,7 +1576,6 @@ dependencies = [ [[package]] name = "ruma-api-macros" version = "0.17.0-alpha.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#47fab87325b71b7f6c2fb3cd276d1f813e42abf7" dependencies = [ "proc-macro-crate", "proc-macro2", @@ -1589,7 +1586,6 @@ dependencies = [ [[package]] name = "ruma-appservice-api" version = "0.2.0-alpha.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#47fab87325b71b7f6c2fb3cd276d1f813e42abf7" dependencies = [ "ruma-api", "ruma-common", @@ -1602,7 +1598,6 @@ dependencies = [ [[package]] name = "ruma-client-api" version = "0.10.0-alpha.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#47fab87325b71b7f6c2fb3cd276d1f813e42abf7" dependencies = [ "assign", "http", @@ -1621,7 +1616,6 @@ dependencies = [ [[package]] name = "ruma-common" version = "0.2.0" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#47fab87325b71b7f6c2fb3cd276d1f813e42abf7" dependencies = [ "js_int", "ruma-api", @@ -1635,7 +1629,6 @@ dependencies = [ [[package]] name = "ruma-events" version = "0.22.0-alpha.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#47fab87325b71b7f6c2fb3cd276d1f813e42abf7" dependencies = [ "js_int", "ruma-common", @@ -1650,7 +1643,6 @@ dependencies = [ [[package]] name = "ruma-events-macros" version = "0.22.0-alpha.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#47fab87325b71b7f6c2fb3cd276d1f813e42abf7" dependencies = [ "proc-macro-crate", "proc-macro2", @@ -1661,7 +1653,6 @@ dependencies = [ [[package]] name = "ruma-federation-api" version = "0.0.3" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#47fab87325b71b7f6c2fb3cd276d1f813e42abf7" dependencies = [ "js_int", "ruma-api", @@ -1676,7 +1667,6 @@ dependencies = [ [[package]] name = "ruma-identifiers" version = "0.17.4" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#47fab87325b71b7f6c2fb3cd276d1f813e42abf7" dependencies = [ "rand", "ruma-identifiers-macros", @@ -1688,7 +1678,6 @@ dependencies = [ [[package]] name = "ruma-identifiers-macros" version = "0.17.4" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#47fab87325b71b7f6c2fb3cd276d1f813e42abf7" dependencies = [ "proc-macro2", "quote", @@ -1699,7 +1688,6 @@ dependencies = [ [[package]] name = "ruma-identifiers-validation" version = "0.1.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#47fab87325b71b7f6c2fb3cd276d1f813e42abf7" dependencies = [ "serde", "strum", @@ -1708,7 +1696,6 @@ dependencies = [ [[package]] name = "ruma-serde" version = "0.2.3" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#47fab87325b71b7f6c2fb3cd276d1f813e42abf7" dependencies = [ "form_urlencoded", "itoa", @@ -1720,7 +1707,6 @@ dependencies = [ [[package]] name = "ruma-signatures" version = "0.6.0-dev.1" -source = "git+https://github.com/timokoesters/ruma?branch=timo-fed-fixes#47fab87325b71b7f6c2fb3cd276d1f813e42abf7" dependencies = [ "base64", "ring", @@ -1970,7 +1956,6 @@ checksum = "7345c971d1ef21ffdbd103a75990a15eb03604fc8b8852ca8cb418ee1a099028" [[package]] name = "state-res" version = "0.1.0" -source = "git+https://github.com/timokoesters/state-res?branch=spec-comp#d11a3feb5307715ab5d86af8f25d4bccfee6264b" dependencies = [ "itertools", "js_int", diff --git a/src/client_server/account.rs b/src/client_server/account.rs index 7e0f942e6..66b4a62a6 100644 --- a/src/client_server/account.rs +++ b/src/client_server/account.rs @@ -1,3 +1,5 @@ +use std::{collections::BTreeMap, convert::TryInto}; + use super::{State, DEVICE_ID_LENGTH, SESSION_ID_LENGTH, TOKEN_LENGTH}; use crate::{pdu::PduBuilder, utils, ConduitResult, Database, Error, Ruma}; use ruma::{ @@ -11,8 +13,11 @@ use ruma::{ uiaa::{AuthFlow, UiaaInfo}, }, }, - events::{room::member, EventType}, - UserId, + events::{ + room::canonical_alias, room::guest_access, room::history_visibility, room::join_rules, + room::member, room::name, room::topic, EventType, + }, + RoomAliasId, RoomId, RoomVersionId, UserId, }; use register::RegistrationKind; @@ -73,7 +78,7 @@ pub fn get_register_available_route( feature = "conduit_bin", post("/_matrix/client/r0/register", data = "") )] -pub fn register_route( +pub async fn register_route( db: State<'_, Database>, body: Ruma>, ) -> ConduitResult { @@ -202,6 +207,265 @@ pub fn register_route( body.initial_device_display_name.clone(), )?; + // If this is the first user on this server, create the admins room + if db.users.count() == 1 { + // Create a user for the server + let conduit_user = UserId::parse_with_server_name("conduit", db.globals.server_name()) + .expect("@conduit:server_name is valid"); + + db.users.create(&conduit_user, "")?; + + let room_id = RoomId::new(db.globals.server_name()); + + let mut content = ruma::events::room::create::CreateEventContent::new(conduit_user.clone()); + content.federate = true; + content.predecessor = None; + content.room_version = RoomVersionId::Version6; + + // 1. The room create event + db.rooms.build_and_append_pdu( + PduBuilder { + event_type: EventType::RoomCreate, + content: serde_json::to_value(content).expect("event is valid, we just created it"), + unsigned: None, + state_key: Some("".to_owned()), + redacts: None, + }, + &conduit_user, + &room_id, + &db.globals, + &db.sending, + &db.account_data, + )?; + + // 2. Make conduit bot join + db.rooms.build_and_append_pdu( + PduBuilder { + event_type: EventType::RoomMember, + content: serde_json::to_value(member::MemberEventContent { + membership: member::MembershipState::Join, + displayname: None, + avatar_url: None, + is_direct: None, + third_party_invite: None, + }) + .expect("event is valid, we just created it"), + unsigned: None, + state_key: Some(conduit_user.to_string()), + redacts: None, + }, + &conduit_user, + &room_id, + &db.globals, + &db.sending, + &db.account_data, + )?; + + // 3. Power levels + let mut users = BTreeMap::new(); + users.insert(conduit_user.clone(), 100.into()); + users.insert(user_id.clone(), 100.into()); + + db.rooms.build_and_append_pdu( + PduBuilder { + event_type: EventType::RoomPowerLevels, + content: serde_json::to_value( + ruma::events::room::power_levels::PowerLevelsEventContent { + ban: 50.into(), + events: BTreeMap::new(), + events_default: 0.into(), + invite: 50.into(), + kick: 50.into(), + redact: 50.into(), + state_default: 50.into(), + users, + users_default: 0.into(), + notifications: ruma::events::room::power_levels::NotificationPowerLevels { + room: 50.into(), + }, + }, + ) + .expect("event is valid, we just created it"), + unsigned: None, + state_key: Some("".to_owned()), + redacts: None, + }, + &conduit_user, + &room_id, + &db.globals, + &db.sending, + &db.account_data, + )?; + + // 4.1 Join Rules + db.rooms.build_and_append_pdu( + PduBuilder { + event_type: EventType::RoomJoinRules, + content: serde_json::to_value(join_rules::JoinRulesEventContent::new( + join_rules::JoinRule::Invite, + )) + .expect("event is valid, we just created it"), + unsigned: None, + state_key: Some("".to_owned()), + redacts: None, + }, + &conduit_user, + &room_id, + &db.globals, + &db.sending, + &db.account_data, + )?; + + // 4.2 History Visibility + db.rooms.build_and_append_pdu( + PduBuilder { + event_type: EventType::RoomHistoryVisibility, + content: serde_json::to_value( + history_visibility::HistoryVisibilityEventContent::new( + history_visibility::HistoryVisibility::Shared, + ), + ) + .expect("event is valid, we just created it"), + unsigned: None, + state_key: Some("".to_owned()), + redacts: None, + }, + &conduit_user, + &room_id, + &db.globals, + &db.sending, + &db.account_data, + )?; + + // 4.3 Guest Access + db.rooms.build_and_append_pdu( + PduBuilder { + event_type: EventType::RoomGuestAccess, + content: serde_json::to_value(guest_access::GuestAccessEventContent::new( + guest_access::GuestAccess::Forbidden, + )) + .expect("event is valid, we just created it"), + unsigned: None, + state_key: Some("".to_owned()), + redacts: None, + }, + &conduit_user, + &room_id, + &db.globals, + &db.sending, + &db.account_data, + )?; + + // 6. Events implied by name and topic + db.rooms.build_and_append_pdu( + PduBuilder { + event_type: EventType::RoomName, + content: serde_json::to_value( + name::NameEventContent::new("Admin Room".to_owned()).map_err(|_| { + Error::BadRequest(ErrorKind::InvalidParam, "Name is invalid.") + })?, + ) + .expect("event is valid, we just created it"), + unsigned: None, + state_key: Some("".to_owned()), + redacts: None, + }, + &conduit_user, + &room_id, + &db.globals, + &db.sending, + &db.account_data, + )?; + + db.rooms.build_and_append_pdu( + PduBuilder { + event_type: EventType::RoomTopic, + content: serde_json::to_value(topic::TopicEventContent { + topic: format!("Manage {}", db.globals.server_name()), + }) + .expect("event is valid, we just created it"), + unsigned: None, + state_key: Some("".to_owned()), + redacts: None, + }, + &conduit_user, + &room_id, + &db.globals, + &db.sending, + &db.account_data, + )?; + + // Room alias + let alias: RoomAliasId = format!("#admins:{}", db.globals.server_name()) + .try_into() + .expect("#admins:server_name is a valid alias name"); + + db.rooms.build_and_append_pdu( + PduBuilder { + event_type: EventType::RoomCanonicalAlias, + content: serde_json::to_value(canonical_alias::CanonicalAliasEventContent { + alias: Some(alias.clone()), + alt_aliases: Vec::new(), + }) + .expect("event is valid, we just created it"), + unsigned: None, + state_key: Some("".to_owned()), + redacts: None, + }, + &conduit_user, + &room_id, + &db.globals, + &db.sending, + &db.account_data, + )?; + + db.rooms.set_alias(&alias, Some(&room_id), &db.globals)?; + + // Invite and join the real user + db.rooms.build_and_append_pdu( + PduBuilder { + event_type: EventType::RoomMember, + content: serde_json::to_value(member::MemberEventContent { + membership: member::MembershipState::Invite, + displayname: None, + avatar_url: None, + is_direct: None, + third_party_invite: None, + }) + .expect("event is valid, we just created it"), + unsigned: None, + state_key: Some(user_id.to_string()), + redacts: None, + }, + &conduit_user, + &room_id, + &db.globals, + &db.sending, + &db.account_data, + )?; + db.rooms.build_and_append_pdu( + PduBuilder { + event_type: EventType::RoomMember, + content: serde_json::to_value(member::MemberEventContent { + membership: member::MembershipState::Join, + displayname: None, + avatar_url: None, + is_direct: None, + third_party_invite: None, + }) + .expect("event is valid, we just created it"), + unsigned: None, + state_key: Some(user_id.to_string()), + redacts: None, + }, + &user_id, + &room_id, + &db.globals, + &db.sending, + &db.account_data, + )?; + } + Ok(register::Response { access_token: Some(token), user_id, @@ -354,23 +618,20 @@ pub async fn deactivate_route( third_party_invite: None, }; - db.rooms - .build_and_append_pdu( - PduBuilder { - event_type: EventType::RoomMember, - content: serde_json::to_value(event) - .expect("event is valid, we just created it"), - unsigned: None, - state_key: Some(sender_id.to_string()), - redacts: None, - }, - &sender_id, - &room_id, - &db.globals, - &db.sending, - &db.account_data, - ) - .await?; + db.rooms.build_and_append_pdu( + PduBuilder { + event_type: EventType::RoomMember, + content: serde_json::to_value(event).expect("event is valid, we just created it"), + unsigned: None, + state_key: Some(sender_id.to_string()), + redacts: None, + }, + &sender_id, + &room_id, + &db.globals, + &db.sending, + &db.account_data, + )?; } // Remove devices and mark account as deactivated diff --git a/src/client_server/membership.rs b/src/client_server/membership.rs index 628045dee..526e82f1a 100644 --- a/src/client_server/membership.rs +++ b/src/client_server/membership.rs @@ -108,22 +108,20 @@ pub async fn leave_room_route( event.membership = member::MembershipState::Leave; - db.rooms - .build_and_append_pdu( - PduBuilder { - event_type: EventType::RoomMember, - content: serde_json::to_value(event).expect("event is valid, we just created it"), - unsigned: None, - state_key: Some(sender_id.to_string()), - redacts: None, - }, - &sender_id, - &body.room_id, - &db.globals, - &db.sending, - &db.account_data, - ) - .await?; + db.rooms.build_and_append_pdu( + PduBuilder { + event_type: EventType::RoomMember, + content: serde_json::to_value(event).expect("event is valid, we just created it"), + unsigned: None, + state_key: Some(sender_id.to_string()), + redacts: None, + }, + &sender_id, + &body.room_id, + &db.globals, + &db.sending, + &db.account_data, + )?; Ok(leave_room::Response::new().into()) } @@ -139,29 +137,27 @@ pub async fn invite_user_route( let sender_id = body.sender_id.as_ref().expect("user is authenticated"); if let invite_user::IncomingInvitationRecipient::UserId { user_id } = &body.recipient { - db.rooms - .build_and_append_pdu( - PduBuilder { - event_type: EventType::RoomMember, - content: serde_json::to_value(member::MemberEventContent { - membership: member::MembershipState::Invite, - displayname: db.users.displayname(&user_id)?, - avatar_url: db.users.avatar_url(&user_id)?, - is_direct: None, - third_party_invite: None, - }) - .expect("event is valid, we just created it"), - unsigned: None, - state_key: Some(user_id.to_string()), - redacts: None, - }, - &sender_id, - &body.room_id, - &db.globals, - &db.sending, - &db.account_data, - ) - .await?; + db.rooms.build_and_append_pdu( + PduBuilder { + event_type: EventType::RoomMember, + content: serde_json::to_value(member::MemberEventContent { + membership: member::MembershipState::Invite, + displayname: db.users.displayname(&user_id)?, + avatar_url: db.users.avatar_url(&user_id)?, + is_direct: None, + third_party_invite: None, + }) + .expect("event is valid, we just created it"), + unsigned: None, + state_key: Some(user_id.to_string()), + redacts: None, + }, + &sender_id, + &body.room_id, + &db.globals, + &db.sending, + &db.account_data, + )?; Ok(invite_user::Response.into()) } else { @@ -199,22 +195,20 @@ pub async fn kick_user_route( event.membership = ruma::events::room::member::MembershipState::Leave; // TODO: reason - db.rooms - .build_and_append_pdu( - PduBuilder { - event_type: EventType::RoomMember, - content: serde_json::to_value(event).expect("event is valid, we just created it"), - unsigned: None, - state_key: Some(body.user_id.to_string()), - redacts: None, - }, - &sender_id, - &body.room_id, - &db.globals, - &db.sending, - &db.account_data, - ) - .await?; + db.rooms.build_and_append_pdu( + PduBuilder { + event_type: EventType::RoomMember, + content: serde_json::to_value(event).expect("event is valid, we just created it"), + unsigned: None, + state_key: Some(body.user_id.to_string()), + redacts: None, + }, + &sender_id, + &body.room_id, + &db.globals, + &db.sending, + &db.account_data, + )?; Ok(kick_user::Response::new().into()) } @@ -257,22 +251,20 @@ pub async fn ban_user_route( }, )?; - db.rooms - .build_and_append_pdu( - PduBuilder { - event_type: EventType::RoomMember, - content: serde_json::to_value(event).expect("event is valid, we just created it"), - unsigned: None, - state_key: Some(body.user_id.to_string()), - redacts: None, - }, - &sender_id, - &body.room_id, - &db.globals, - &db.sending, - &db.account_data, - ) - .await?; + db.rooms.build_and_append_pdu( + PduBuilder { + event_type: EventType::RoomMember, + content: serde_json::to_value(event).expect("event is valid, we just created it"), + unsigned: None, + state_key: Some(body.user_id.to_string()), + redacts: None, + }, + &sender_id, + &body.room_id, + &db.globals, + &db.sending, + &db.account_data, + )?; Ok(ban_user::Response::new().into()) } @@ -306,22 +298,20 @@ pub async fn unban_user_route( event.membership = ruma::events::room::member::MembershipState::Leave; - db.rooms - .build_and_append_pdu( - PduBuilder { - event_type: EventType::RoomMember, - content: serde_json::to_value(event).expect("event is valid, we just created it"), - unsigned: None, - state_key: Some(body.user_id.to_string()), - redacts: None, - }, - &sender_id, - &body.room_id, - &db.globals, - &db.sending, - &db.account_data, - ) - .await?; + db.rooms.build_and_append_pdu( + PduBuilder { + event_type: EventType::RoomMember, + content: serde_json::to_value(event).expect("event is valid, we just created it"), + unsigned: None, + state_key: Some(body.user_id.to_string()), + redacts: None, + }, + &sender_id, + &body.room_id, + &db.globals, + &db.sending, + &db.account_data, + )?; Ok(unban_user::Response::new().into()) } @@ -640,6 +630,7 @@ async fn join_room_by_id_helper( &serde_json::to_value(&**pdu).expect("PDU is valid value"), &db.globals, &db.account_data, + &db.sending, )?; if state_events.contains(ev_id) { @@ -657,23 +648,20 @@ async fn join_room_by_id_helper( third_party_invite: None, }; - db.rooms - .build_and_append_pdu( - PduBuilder { - event_type: EventType::RoomMember, - content: serde_json::to_value(event) - .expect("event is valid, we just created it"), - unsigned: None, - state_key: Some(sender_id.to_string()), - redacts: None, - }, - &sender_id, - &room_id, - &db.globals, - &db.sending, - &db.account_data, - ) - .await?; + db.rooms.build_and_append_pdu( + PduBuilder { + event_type: EventType::RoomMember, + content: serde_json::to_value(event).expect("event is valid, we just created it"), + unsigned: None, + state_key: Some(sender_id.to_string()), + redacts: None, + }, + &sender_id, + &room_id, + &db.globals, + &db.sending, + &db.account_data, + )?; } Ok(join_room_by_id::Response::new(room_id.clone()).into()) diff --git a/src/client_server/message.rs b/src/client_server/message.rs index 5a4488fe5..c32bd687a 100644 --- a/src/client_server/message.rs +++ b/src/client_server/message.rs @@ -49,29 +49,26 @@ pub async fn send_message_event_route( let mut unsigned = serde_json::Map::new(); unsigned.insert("transaction_id".to_owned(), body.txn_id.clone().into()); - let event_id = db - .rooms - .build_and_append_pdu( - PduBuilder { - event_type: body.content.event_type().into(), - content: serde_json::from_str( - body.json_body - .as_ref() - .ok_or(Error::BadRequest(ErrorKind::BadJson, "Invalid JSON body."))? - .get(), - ) - .map_err(|_| Error::BadRequest(ErrorKind::BadJson, "Invalid JSON body."))?, - unsigned: Some(unsigned), - state_key: None, - redacts: None, - }, - &sender_id, - &body.room_id, - &db.globals, - &db.sending, - &db.account_data, - ) - .await?; + let event_id = db.rooms.build_and_append_pdu( + PduBuilder { + event_type: body.content.event_type().into(), + content: serde_json::from_str( + body.json_body + .as_ref() + .ok_or(Error::BadRequest(ErrorKind::BadJson, "Invalid JSON body."))? + .get(), + ) + .map_err(|_| Error::BadRequest(ErrorKind::BadJson, "Invalid JSON body."))?, + unsigned: Some(unsigned), + state_key: None, + redacts: None, + }, + &sender_id, + &body.room_id, + &db.globals, + &db.sending, + &db.account_data, + )?; db.transaction_ids .add_txnid(sender_id, device_id, &body.txn_id, event_id.as_bytes())?; diff --git a/src/client_server/profile.rs b/src/client_server/profile.rs index 686d4c3e3..9c6bd512e 100644 --- a/src/client_server/profile.rs +++ b/src/client_server/profile.rs @@ -31,43 +31,41 @@ pub async fn set_displayname_route( // Send a new membership event and presence update into all joined rooms for room_id in db.rooms.rooms_joined(&sender_id) { let room_id = room_id?; - db.rooms - .build_and_append_pdu( - PduBuilder { - event_type: EventType::RoomMember, - content: serde_json::to_value(ruma::events::room::member::MemberEventContent { - displayname: body.displayname.clone(), - ..serde_json::from_value::>( - db.rooms - .room_state_get( - &room_id, - &EventType::RoomMember, - &sender_id.to_string(), - )? - .ok_or_else(|| { - Error::bad_database( + db.rooms.build_and_append_pdu( + PduBuilder { + event_type: EventType::RoomMember, + content: serde_json::to_value(ruma::events::room::member::MemberEventContent { + displayname: body.displayname.clone(), + ..serde_json::from_value::>( + db.rooms + .room_state_get( + &room_id, + &EventType::RoomMember, + &sender_id.to_string(), + )? + .ok_or_else(|| { + Error::bad_database( "Tried to send displayname update for user not in the room.", ) - })? - .content - .clone(), - ) - .expect("from_value::> can never fail") - .deserialize() - .map_err(|_| Error::bad_database("Database contains invalid PDU."))? - }) - .expect("event is valid, we just created it"), - unsigned: None, - state_key: Some(sender_id.to_string()), - redacts: None, - }, - &sender_id, - &room_id, - &db.globals, - &db.sending, - &db.account_data, - ) - .await?; + })? + .content + .clone(), + ) + .expect("from_value::> can never fail") + .deserialize() + .map_err(|_| Error::bad_database("Database contains invalid PDU."))? + }) + .expect("event is valid, we just created it"), + unsigned: None, + state_key: Some(sender_id.to_string()), + redacts: None, + }, + &sender_id, + &room_id, + &db.globals, + &db.sending, + &db.account_data, + )?; // Presence update db.rooms.edus.update_presence( @@ -125,43 +123,41 @@ pub async fn set_avatar_url_route( // Send a new membership event and presence update into all joined rooms for room_id in db.rooms.rooms_joined(&sender_id) { let room_id = room_id?; - db.rooms - .build_and_append_pdu( - PduBuilder { - event_type: EventType::RoomMember, - content: serde_json::to_value(ruma::events::room::member::MemberEventContent { - avatar_url: body.avatar_url.clone(), - ..serde_json::from_value::>( - db.rooms - .room_state_get( - &room_id, - &EventType::RoomMember, - &sender_id.to_string(), - )? - .ok_or_else(|| { - Error::bad_database( - "Tried to send avatar url update for user not in the room.", - ) - })? - .content - .clone(), - ) - .expect("from_value::> can never fail") - .deserialize() - .map_err(|_| Error::bad_database("Database contains invalid PDU."))? - }) - .expect("event is valid, we just created it"), - unsigned: None, - state_key: Some(sender_id.to_string()), - redacts: None, - }, - &sender_id, - &room_id, - &db.globals, - &db.sending, - &db.account_data, - ) - .await?; + db.rooms.build_and_append_pdu( + PduBuilder { + event_type: EventType::RoomMember, + content: serde_json::to_value(ruma::events::room::member::MemberEventContent { + avatar_url: body.avatar_url.clone(), + ..serde_json::from_value::>( + db.rooms + .room_state_get( + &room_id, + &EventType::RoomMember, + &sender_id.to_string(), + )? + .ok_or_else(|| { + Error::bad_database( + "Tried to send avatar url update for user not in the room.", + ) + })? + .content + .clone(), + ) + .expect("from_value::> can never fail") + .deserialize() + .map_err(|_| Error::bad_database("Database contains invalid PDU."))? + }) + .expect("event is valid, we just created it"), + unsigned: None, + state_key: Some(sender_id.to_string()), + redacts: None, + }, + &sender_id, + &room_id, + &db.globals, + &db.sending, + &db.account_data, + )?; // Presence update db.rooms.edus.update_presence( diff --git a/src/client_server/redact.rs b/src/client_server/redact.rs index 24df8dd7d..b13cd802b 100644 --- a/src/client_server/redact.rs +++ b/src/client_server/redact.rs @@ -18,26 +18,23 @@ pub async fn redact_event_route( ) -> ConduitResult { let sender_id = body.sender_id.as_ref().expect("user is authenticated"); - let event_id = db - .rooms - .build_and_append_pdu( - PduBuilder { - event_type: EventType::RoomRedaction, - content: serde_json::to_value(redaction::RedactionEventContent { - reason: body.reason.clone(), - }) - .expect("event is valid, we just created it"), - unsigned: None, - state_key: None, - redacts: Some(body.event_id.clone()), - }, - &sender_id, - &body.room_id, - &db.globals, - &db.sending, - &db.account_data, - ) - .await?; + let event_id = db.rooms.build_and_append_pdu( + PduBuilder { + event_type: EventType::RoomRedaction, + content: serde_json::to_value(redaction::RedactionEventContent { + reason: body.reason.clone(), + }) + .expect("event is valid, we just created it"), + unsigned: None, + state_key: None, + redacts: Some(body.event_id.clone()), + }, + &sender_id, + &body.room_id, + &db.globals, + &db.sending, + &db.account_data, + )?; Ok(redact_event::Response { event_id }.into()) } diff --git a/src/client_server/room.rs b/src/client_server/room.rs index d21148bb7..28d30e29b 100644 --- a/src/client_server/room.rs +++ b/src/client_server/room.rs @@ -53,47 +53,43 @@ pub async fn create_room_route( content.room_version = RoomVersionId::Version6; // 1. The room create event - db.rooms - .build_and_append_pdu( - PduBuilder { - event_type: EventType::RoomCreate, - content: serde_json::to_value(content).expect("event is valid, we just created it"), - unsigned: None, - state_key: Some("".to_owned()), - redacts: None, - }, - &sender_id, - &room_id, - &db.globals, - &db.sending, - &db.account_data, - ) - .await?; + db.rooms.build_and_append_pdu( + PduBuilder { + event_type: EventType::RoomCreate, + content: serde_json::to_value(content).expect("event is valid, we just created it"), + unsigned: None, + state_key: Some("".to_owned()), + redacts: None, + }, + &sender_id, + &room_id, + &db.globals, + &db.sending, + &db.account_data, + )?; // 2. Let the room creator join - db.rooms - .build_and_append_pdu( - PduBuilder { - event_type: EventType::RoomMember, - content: serde_json::to_value(member::MemberEventContent { - membership: member::MembershipState::Join, - displayname: db.users.displayname(&sender_id)?, - avatar_url: db.users.avatar_url(&sender_id)?, - is_direct: Some(body.is_direct), - third_party_invite: None, - }) - .expect("event is valid, we just created it"), - unsigned: None, - state_key: Some(sender_id.to_string()), - redacts: None, - }, - &sender_id, - &room_id, - &db.globals, - &db.sending, - &db.account_data, - ) - .await?; + db.rooms.build_and_append_pdu( + PduBuilder { + event_type: EventType::RoomMember, + content: serde_json::to_value(member::MemberEventContent { + membership: member::MembershipState::Join, + displayname: db.users.displayname(&sender_id)?, + avatar_url: db.users.avatar_url(&sender_id)?, + is_direct: Some(body.is_direct), + third_party_invite: None, + }) + .expect("event is valid, we just created it"), + unsigned: None, + state_key: Some(sender_id.to_string()), + redacts: None, + }, + &sender_id, + &room_id, + &db.globals, + &db.sending, + &db.account_data, + )?; // 3. Power levels let mut users = BTreeMap::new(); @@ -123,22 +119,20 @@ pub async fn create_room_route( }) .expect("event is valid, we just created it") }; - db.rooms - .build_and_append_pdu( - PduBuilder { - event_type: EventType::RoomPowerLevels, - content: power_levels_content, - unsigned: None, - state_key: Some("".to_owned()), - redacts: None, - }, - &sender_id, - &room_id, - &db.globals, - &db.sending, - &db.account_data, - ) - .await?; + db.rooms.build_and_append_pdu( + PduBuilder { + event_type: EventType::RoomPowerLevels, + content: power_levels_content, + unsigned: None, + state_key: Some("".to_owned()), + redacts: None, + }, + &sender_id, + &room_id, + &db.globals, + &db.sending, + &db.account_data, + )?; // 4. Events set by preset @@ -149,84 +143,76 @@ pub async fn create_room_route( }); // 4.1 Join Rules - db.rooms - .build_and_append_pdu( - PduBuilder { - event_type: EventType::RoomJoinRules, - content: match preset { - create_room::RoomPreset::PublicChat => serde_json::to_value( - join_rules::JoinRulesEventContent::new(join_rules::JoinRule::Public), - ) - .expect("event is valid, we just created it"), - // according to spec "invite" is the default - _ => serde_json::to_value(join_rules::JoinRulesEventContent::new( - join_rules::JoinRule::Invite, - )) - .expect("event is valid, we just created it"), - }, - unsigned: None, - state_key: Some("".to_owned()), - redacts: None, - }, - &sender_id, - &room_id, - &db.globals, - &db.sending, - &db.account_data, - ) - .await?; - - // 4.2 History Visibility - db.rooms - .build_and_append_pdu( - PduBuilder { - event_type: EventType::RoomHistoryVisibility, - content: serde_json::to_value( - history_visibility::HistoryVisibilityEventContent::new( - history_visibility::HistoryVisibility::Shared, - ), + db.rooms.build_and_append_pdu( + PduBuilder { + event_type: EventType::RoomJoinRules, + content: match preset { + create_room::RoomPreset::PublicChat => serde_json::to_value( + join_rules::JoinRulesEventContent::new(join_rules::JoinRule::Public), ) .expect("event is valid, we just created it"), - unsigned: None, - state_key: Some("".to_owned()), - redacts: None, + // according to spec "invite" is the default + _ => serde_json::to_value(join_rules::JoinRulesEventContent::new( + join_rules::JoinRule::Invite, + )) + .expect("event is valid, we just created it"), }, - &sender_id, - &room_id, - &db.globals, - &db.sending, - &db.account_data, - ) - .await?; + unsigned: None, + state_key: Some("".to_owned()), + redacts: None, + }, + &sender_id, + &room_id, + &db.globals, + &db.sending, + &db.account_data, + )?; + + // 4.2 History Visibility + db.rooms.build_and_append_pdu( + PduBuilder { + event_type: EventType::RoomHistoryVisibility, + content: serde_json::to_value(history_visibility::HistoryVisibilityEventContent::new( + history_visibility::HistoryVisibility::Shared, + )) + .expect("event is valid, we just created it"), + unsigned: None, + state_key: Some("".to_owned()), + redacts: None, + }, + &sender_id, + &room_id, + &db.globals, + &db.sending, + &db.account_data, + )?; // 4.3 Guest Access - db.rooms - .build_and_append_pdu( - PduBuilder { - event_type: EventType::RoomGuestAccess, - content: match preset { - create_room::RoomPreset::PublicChat => { - serde_json::to_value(guest_access::GuestAccessEventContent::new( - guest_access::GuestAccess::Forbidden, - )) - .expect("event is valid, we just created it") - } - _ => serde_json::to_value(guest_access::GuestAccessEventContent::new( - guest_access::GuestAccess::CanJoin, + db.rooms.build_and_append_pdu( + PduBuilder { + event_type: EventType::RoomGuestAccess, + content: match preset { + create_room::RoomPreset::PublicChat => { + serde_json::to_value(guest_access::GuestAccessEventContent::new( + guest_access::GuestAccess::Forbidden, )) - .expect("event is valid, we just created it"), - }, - unsigned: None, - state_key: Some("".to_owned()), - redacts: None, + .expect("event is valid, we just created it") + } + _ => serde_json::to_value(guest_access::GuestAccessEventContent::new( + guest_access::GuestAccess::CanJoin, + )) + .expect("event is valid, we just created it"), }, - &sender_id, - &room_id, - &db.globals, - &db.sending, - &db.account_data, - ) - .await?; + unsigned: None, + state_key: Some("".to_owned()), + redacts: None, + }, + &sender_id, + &room_id, + &db.globals, + &db.sending, + &db.account_data, + )?; // 5. Events listed in initial_state for event in &body.initial_state { @@ -240,90 +226,82 @@ pub async fn create_room_route( continue; } - db.rooms - .build_and_append_pdu( - pdu_builder, - &sender_id, - &room_id, - &db.globals, - &db.sending, - &db.account_data, - ) - .await?; + db.rooms.build_and_append_pdu( + pdu_builder, + &sender_id, + &room_id, + &db.globals, + &db.sending, + &db.account_data, + )?; } // 6. Events implied by name and topic if let Some(name) = &body.name { - db.rooms - .build_and_append_pdu( - PduBuilder { - event_type: EventType::RoomName, - content: serde_json::to_value( - name::NameEventContent::new(name.clone()).map_err(|_| { - Error::BadRequest(ErrorKind::InvalidParam, "Name is invalid.") - })?, - ) - .expect("event is valid, we just created it"), - unsigned: None, - state_key: Some("".to_owned()), - redacts: None, - }, - &sender_id, - &room_id, - &db.globals, - &db.sending, - &db.account_data, - ) - .await?; + db.rooms.build_and_append_pdu( + PduBuilder { + event_type: EventType::RoomName, + content: serde_json::to_value( + name::NameEventContent::new(name.clone()).map_err(|_| { + Error::BadRequest(ErrorKind::InvalidParam, "Name is invalid.") + })?, + ) + .expect("event is valid, we just created it"), + unsigned: None, + state_key: Some("".to_owned()), + redacts: None, + }, + &sender_id, + &room_id, + &db.globals, + &db.sending, + &db.account_data, + )?; } if let Some(topic) = &body.topic { - db.rooms - .build_and_append_pdu( - PduBuilder { - event_type: EventType::RoomTopic, - content: serde_json::to_value(topic::TopicEventContent { - topic: topic.clone(), - }) - .expect("event is valid, we just created it"), - unsigned: None, - state_key: Some("".to_owned()), - redacts: None, - }, - &sender_id, - &room_id, - &db.globals, - &db.sending, - &db.account_data, - ) - .await?; + db.rooms.build_and_append_pdu( + PduBuilder { + event_type: EventType::RoomTopic, + content: serde_json::to_value(topic::TopicEventContent { + topic: topic.clone(), + }) + .expect("event is valid, we just created it"), + unsigned: None, + state_key: Some("".to_owned()), + redacts: None, + }, + &sender_id, + &room_id, + &db.globals, + &db.sending, + &db.account_data, + )?; } // 7. Events implied by invite (and TODO: invite_3pid) for user in &body.invite { - db.rooms - .build_and_append_pdu( - PduBuilder { - event_type: EventType::RoomMember, - content: serde_json::to_value(member::MemberEventContent { - membership: member::MembershipState::Invite, - displayname: db.users.displayname(&user)?, - avatar_url: db.users.avatar_url(&user)?, - is_direct: Some(body.is_direct), - third_party_invite: None, - }) - .expect("event is valid, we just created it"), - unsigned: None, - state_key: Some(user.to_string()), - redacts: None, - }, - &sender_id, - &room_id, - &db.globals, - &db.sending, - &db.account_data, - ) - .await?; + db.rooms.build_and_append_pdu( + PduBuilder { + event_type: EventType::RoomMember, + content: serde_json::to_value(member::MemberEventContent { + membership: member::MembershipState::Invite, + displayname: db.users.displayname(&user)?, + avatar_url: db.users.avatar_url(&user)?, + is_direct: Some(body.is_direct), + third_party_invite: None, + }) + .expect("event is valid, we just created it"), + unsigned: None, + state_key: Some(user.to_string()), + redacts: None, + }, + &sender_id, + &room_id, + &db.globals, + &db.sending, + &db.account_data, + )?; } // Homeserver specific stuff @@ -395,29 +373,24 @@ pub async fn upgrade_room_route( // Send a m.room.tombstone event to the old room to indicate that it is not intended to be used any further // Fail if the sender does not have the required permissions - let tombstone_event_id = db - .rooms - .build_and_append_pdu( - PduBuilder { - event_type: EventType::RoomTombstone, - content: serde_json::to_value( - ruma::events::room::tombstone::TombstoneEventContent { - body: "This room has been replaced".to_string(), - replacement_room: replacement_room.clone(), - }, - ) - .expect("event is valid, we just created it"), - unsigned: None, - state_key: Some("".to_owned()), - redacts: None, - }, - sender_id, - &body.room_id, - &db.globals, - &db.sending, - &db.account_data, - ) - .await?; + let tombstone_event_id = db.rooms.build_and_append_pdu( + PduBuilder { + event_type: EventType::RoomTombstone, + content: serde_json::to_value(ruma::events::room::tombstone::TombstoneEventContent { + body: "This room has been replaced".to_string(), + replacement_room: replacement_room.clone(), + }) + .expect("event is valid, we just created it"), + unsigned: None, + state_key: Some("".to_owned()), + redacts: None, + }, + sender_id, + &body.room_id, + &db.globals, + &db.sending, + &db.account_data, + )?; // Get the old room federations status let federate = serde_json::from_value::>( @@ -444,48 +417,44 @@ pub async fn upgrade_room_route( create_event_content.room_version = new_version; create_event_content.predecessor = predecessor; - db.rooms - .build_and_append_pdu( - PduBuilder { - event_type: EventType::RoomCreate, - content: serde_json::to_value(create_event_content) - .expect("event is valid, we just created it"), - unsigned: None, - state_key: Some("".to_owned()), - redacts: None, - }, - sender_id, - &replacement_room, - &db.globals, - &db.sending, - &db.account_data, - ) - .await?; + db.rooms.build_and_append_pdu( + PduBuilder { + event_type: EventType::RoomCreate, + content: serde_json::to_value(create_event_content) + .expect("event is valid, we just created it"), + unsigned: None, + state_key: Some("".to_owned()), + redacts: None, + }, + sender_id, + &replacement_room, + &db.globals, + &db.sending, + &db.account_data, + )?; // Join the new room - db.rooms - .build_and_append_pdu( - PduBuilder { - event_type: EventType::RoomMember, - content: serde_json::to_value(member::MemberEventContent { - membership: member::MembershipState::Join, - displayname: db.users.displayname(&sender_id)?, - avatar_url: db.users.avatar_url(&sender_id)?, - is_direct: None, - third_party_invite: None, - }) - .expect("event is valid, we just created it"), - unsigned: None, - state_key: Some(sender_id.to_string()), - redacts: None, - }, - sender_id, - &replacement_room, - &db.globals, - &db.sending, - &db.account_data, - ) - .await?; + db.rooms.build_and_append_pdu( + PduBuilder { + event_type: EventType::RoomMember, + content: serde_json::to_value(member::MemberEventContent { + membership: member::MembershipState::Join, + displayname: db.users.displayname(&sender_id)?, + avatar_url: db.users.avatar_url(&sender_id)?, + is_direct: None, + third_party_invite: None, + }) + .expect("event is valid, we just created it"), + unsigned: None, + state_key: Some(sender_id.to_string()), + redacts: None, + }, + sender_id, + &replacement_room, + &db.globals, + &db.sending, + &db.account_data, + )?; // Recommended transferable state events list from the specs let transferable_state_events = vec![ @@ -507,22 +476,20 @@ pub async fn upgrade_room_route( None => continue, // Skipping missing events. }; - db.rooms - .build_and_append_pdu( - PduBuilder { - event_type, - content: event_content, - unsigned: None, - state_key: Some("".to_owned()), - redacts: None, - }, - sender_id, - &replacement_room, - &db.globals, - &db.sending, - &db.account_data, - ) - .await?; + db.rooms.build_and_append_pdu( + PduBuilder { + event_type, + content: event_content, + unsigned: None, + state_key: Some("".to_owned()), + redacts: None, + }, + sender_id, + &replacement_room, + &db.globals, + &db.sending, + &db.account_data, + )?; } // Moves any local aliases to the new room @@ -552,24 +519,21 @@ pub async fn upgrade_room_route( power_levels_event_content.invite = new_level; // Modify the power levels in the old room to prevent sending of events and inviting new users - let _ = db - .rooms - .build_and_append_pdu( - PduBuilder { - event_type: EventType::RoomPowerLevels, - content: serde_json::to_value(power_levels_event_content) - .expect("event is valid, we just created it"), - unsigned: None, - state_key: Some("".to_owned()), - redacts: None, - }, - sender_id, - &body.room_id, - &db.globals, - &db.sending, - &db.account_data, - ) - .await; + let _ = db.rooms.build_and_append_pdu( + PduBuilder { + event_type: EventType::RoomPowerLevels, + content: serde_json::to_value(power_levels_event_content) + .expect("event is valid, we just created it"), + unsigned: None, + state_key: Some("".to_owned()), + redacts: None, + }, + sender_id, + &body.room_id, + &db.globals, + &db.sending, + &db.account_data, + )?; // Return the replacement room id Ok(upgrade_room::Response { replacement_room }.into()) diff --git a/src/client_server/state.rs b/src/client_server/state.rs index 46182a12e..1e13b42fe 100644 --- a/src/client_server/state.rs +++ b/src/client_server/state.rs @@ -213,23 +213,20 @@ pub async fn send_state_event_for_key_helper( } } - let event_id = db - .rooms - .build_and_append_pdu( - PduBuilder { - event_type: content.event_type().into(), - content: json, - unsigned: None, - state_key, - redacts: None, - }, - &sender_id, - &room_id, - &db.globals, - &db.sending, - &db.account_data, - ) - .await?; + let event_id = db.rooms.build_and_append_pdu( + PduBuilder { + event_type: content.event_type().into(), + content: json, + unsigned: None, + state_key, + redacts: None, + }, + &sender_id, + &room_id, + &db.globals, + &db.sending, + &db.account_data, + )?; Ok(event_id) } diff --git a/src/database/rooms.rs b/src/database/rooms.rs index 108edb5dd..ab05b390c 100644 --- a/src/database/rooms.rs +++ b/src/database/rooms.rs @@ -10,7 +10,7 @@ use ruma::{ events::{ ignored_user_list, room::{ - member, + member, message, power_levels::{self, PowerLevelsEventContent}, }, EventType, @@ -440,6 +440,7 @@ impl Rooms { pdu_json: &serde_json::Value, globals: &super::globals::Globals, account_data: &super::account_data::AccountData, + sending: &super::sending::Sending, ) -> Result> { self.replace_pdu_leaves(&pdu.room_id, &pdu.event_id)?; @@ -452,7 +453,8 @@ impl Rooms { self.edus .private_read_set(&pdu.room_id, &pdu.sender, index, &globals)?; - let mut pdu_id = pdu.room_id.as_bytes().to_vec(); + let room_id = pdu.room_id.clone(); + let mut pdu_id = room_id.as_bytes().to_vec(); pdu_id.push(0xff); pdu_id.extend_from_slice(&index.to_be_bytes()); @@ -503,6 +505,45 @@ impl Rooms { key.extend_from_slice(&pdu_id); self.tokenids.insert(key, &[])?; } + + if body.starts_with(&format!("@conduit:{}: ", globals.server_name())) + && self + .id_from_alias( + &format!("#admins:{}", globals.server_name()) + .try_into() + .expect("#admins:server_name is a valid room alias"), + )? + .as_ref() + == Some(&pdu.room_id) + { + let mut parts = body.split_whitespace().skip(1); + if let Some(command) = parts.next() { + let args = parts.collect::>(); + + self.build_and_append_pdu( + PduBuilder { + event_type: EventType::RoomMessage, + content: serde_json::to_value( + message::TextMessageEventContent { + body: format!("Command: {}, Args: {:?}", command, args), + formatted: None, + relates_to: None, + }, + ) + .expect("event is valid, we just created it"), + unsigned: None, + state_key: None, + redacts: None, + }, + &UserId::try_from(format!("@conduit:{}", globals.server_name())) + .expect("@conduit:server_name is valid"), + &room_id, + &globals, + &sending, + &account_data, + )?; + } + } } } _ => {} @@ -570,7 +611,7 @@ impl Rooms { } /// Creates a new persisted data unit and adds it to a room. - pub async fn build_and_append_pdu( + pub fn build_and_append_pdu( &self, pdu_builder: PduBuilder, sender: &UserId, @@ -793,7 +834,7 @@ impl Rooms { .expect("json is object") .insert("event_id".to_owned(), pdu.event_id.to_string().into()); - let pdu_id = self.append_pdu(&pdu, &pdu_json, globals, account_data)?; + let pdu_id = self.append_pdu(&pdu, &pdu_json, globals, account_data, sending)?; self.append_to_state(&pdu_id, &pdu)?; diff --git a/src/database/users.rs b/src/database/users.rs index 2e26c1ea4..0d35e3625 100644 --- a/src/database/users.rs +++ b/src/database/users.rs @@ -57,6 +57,11 @@ impl Users { Ok(()) } + /// Returns the number of users registered on this server. + pub fn count(&self) -> usize { + self.userid_password.iter().count() + } + /// Find out which user an access token belongs to. pub fn find_from_token(&self, token: &str) -> Result> { self.token_userdeviceid diff --git a/src/main.rs b/src/main.rs index fa1cc5cac..8fb5fda9f 100644 --- a/src/main.rs +++ b/src/main.rs @@ -126,6 +126,7 @@ fn setup_rocket() -> rocket::Rocket { server_server::get_public_rooms_filtered_route, server_server::send_transaction_message_route, server_server::get_missing_events_route, + server_server::get_profile_information_route, ], ) .attach(AdHoc::on_attach("Config", |mut rocket| async { diff --git a/src/server_server.rs b/src/server_server.rs index 2d52c4acc..0c175bfb2 100644 --- a/src/server_server.rs +++ b/src/server_server.rs @@ -1,4 +1,5 @@ use crate::{client_server, ConduitResult, Database, Error, PduEvent, Result, Ruma}; +use get_profile_information::v1::ProfileField; use http::header::{HeaderValue, AUTHORIZATION, HOST}; use log::warn; use rocket::{get, post, put, response::content::Json, State}; @@ -10,6 +11,7 @@ use ruma::{ get_server_keys, get_server_version::v1 as get_server_version, ServerKey, VerifyKey, }, event::get_missing_events, + query::get_profile_information, transactions::send_transaction_message, }, OutgoingRequest, @@ -362,9 +364,9 @@ pub fn send_transaction_message_route<'a>( let pdu = serde_json::from_value::(value.clone()) .expect("all ruma pdus are conduit pdus"); if db.rooms.exists(&pdu.room_id)? { - let pdu_id = db - .rooms - .append_pdu(&pdu, &value, &db.globals, &db.account_data)?; + let pdu_id = + db.rooms + .append_pdu(&pdu, &value, &db.globals, &db.account_data, &db.sending)?; db.rooms.append_to_state(&pdu_id, &pdu)?; } } @@ -416,3 +418,59 @@ pub fn get_missing_events_route<'a>( Ok(get_missing_events::v1::Response { events }.into()) } + +#[cfg_attr( + feature = "conduit_bin", + get("/_matrix/federation/v1/query/profile", data = "") +)] +pub fn get_profile_information_route<'a>( + db: State<'a, Database>, + body: Ruma>, +) -> ConduitResult { + let mut displayname = None; + let mut avatar_url = None; + + match body.field { + Some(ProfileField::DisplayName) => displayname = db.users.displayname(&body.user_id)?, + Some(ProfileField::AvatarUrl) => avatar_url = db.users.avatar_url(&body.user_id)?, + None => { + displayname = db.users.displayname(&body.user_id)?; + avatar_url = db.users.avatar_url(&body.user_id)?; + } + } + + Ok(get_profile_information::v1::Response { + displayname, + avatar_url, + } + .into()) +} + +/* +#[cfg_attr( + feature = "conduit_bin", + get("/_matrix/federation/v2/invite/<_>/<_>", data = "") +)] +pub fn get_user_devices_route<'a>( + db: State<'a, Database>, + body: Ruma>, +) -> ConduitResult { + let mut displayname = None; + let mut avatar_url = None; + + match body.field { + Some(ProfileField::DisplayName) => displayname = db.users.displayname(&body.user_id)?, + Some(ProfileField::AvatarUrl) => avatar_url = db.users.avatar_url(&body.user_id)?, + None => { + displayname = db.users.displayname(&body.user_id)?; + avatar_url = db.users.avatar_url(&body.user_id)?; + } + } + + Ok(get_profile_information::v1::Response { + displayname, + avatar_url, + } + .into()) +} +*/ From c15ae3c126020c3826f15e5f50ca70a669fc1402 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20K=C3=B6sters?= Date: Tue, 6 Oct 2020 20:43:35 +0200 Subject: [PATCH 48/51] fix: invalid typing bytes because of 0xff in numbers --- src/database/rooms/edus.rs | 9 ++++++--- src/database/sending.rs | 1 + 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/src/database/rooms/edus.rs b/src/database/rooms/edus.rs index a794c690a..29f5407b6 100644 --- a/src/database/rooms/edus.rs +++ b/src/database/rooms/edus.rs @@ -11,6 +11,7 @@ use ruma::{ use std::{ collections::HashMap, convert::{TryFrom, TryInto}, + mem, }; #[derive(Clone)] @@ -228,9 +229,11 @@ impl RoomEdus { let key = key?; Ok::<_, Error>(( key.clone(), - utils::u64_from_bytes(key.split(|&b| b == 0xff).nth(1).ok_or_else(|| { - Error::bad_database("RoomTyping has invalid timestamp or delimiters.") - })?) + utils::u64_from_bytes( + &key.splitn(2, |&b| b == 0xff).nth(1).ok_or_else(|| { + Error::bad_database("RoomTyping has invalid timestamp or delimiters.") + })?[0..mem::size_of::()], + ) .map_err(|_| Error::bad_database("RoomTyping has invalid timestamp bytes."))?, )) }) diff --git a/src/database/sending.rs b/src/database/sending.rs index 1ed94cc2a..c818cbfc8 100644 --- a/src/database/sending.rs +++ b/src/database/sending.rs @@ -28,6 +28,7 @@ impl Sending { select! { Some(server) = futures.next() => { warn!("response: {:?}", &server); + warn!("futures left: {}", &futures.len()); match server { Ok((server, _response)) => { waiting_servers.remove(&server) From 6afc4c9b3e066f2d071e8420c9e4111d0dc65d96 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20K=C3=B6sters?= Date: Tue, 6 Oct 2020 21:04:51 +0200 Subject: [PATCH 49/51] feat: federation disabled by default It can be enable in the Rocket.toml config or using ROCKET_FEDERATION_ENABLED=true --- DEPLOY_FROM_SOURCE.md | 3 +++ Rocket-example.toml | 2 ++ docker-compose.yml | 1 + src/database/globals.rs | 6 ++++++ src/server_server.rs | 39 ++++++++++++++++++++++++++++++++++++++- 5 files changed, 50 insertions(+), 1 deletion(-) diff --git a/DEPLOY_FROM_SOURCE.md b/DEPLOY_FROM_SOURCE.md index 4d685f6e0..456fe6ea0 100644 --- a/DEPLOY_FROM_SOURCE.md +++ b/DEPLOY_FROM_SOURCE.md @@ -27,7 +27,10 @@ Environment="ROCKET_SERVER_NAME=YOURSERVERNAME.HERE" # EDIT THIS Environment="ROCKET_PORT=14004" # Reverse proxy port +#Environment="ROCKET_MAX_REQUEST_SIZE=20000000" # in bytes #Environment="ROCKET_REGISTRATION_DISABLED=true" +#Environment="ROCKET_ENCRYPTION_DISABLED=true" +#Environment="ROCKET_FEDERATION_ENABLED=true" #Environment="ROCKET_LOG=normal" # Detailed logging Environment="ROCKET_ENV=production" diff --git a/Rocket-example.toml b/Rocket-example.toml index 41b36d3a9..8eb48e95f 100644 --- a/Rocket-example.toml +++ b/Rocket-example.toml @@ -16,6 +16,8 @@ port = 14004 # Note: existing rooms will continue to work #encryption_disabled = true +#federation_enabled = true + # Default path is in this user's data #database_path = "/home/timo/MyConduitServer" diff --git a/docker-compose.yml b/docker-compose.yml index f06eaca97..7d1976225 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -31,6 +31,7 @@ services: # ROCKET_PORT: 8000 # ROCKET_REGISTRATION_DISABLED: 'true' # ROCKET_ENCRYPTION_DISABLED: 'true' + # ROCKET_FEDERATION_ENABLED: 'true' # ROCKET_DATABASE_PATH: /srv/conduit/.local/share/conduit # ROCKET_WORKERS: 10 # ROCKET_MAX_REQUEST_SIZE: 20_000_000 # in bytes, ~20 MB diff --git a/src/database/globals.rs b/src/database/globals.rs index 8ce9c0113..37f10eec4 100644 --- a/src/database/globals.rs +++ b/src/database/globals.rs @@ -14,6 +14,7 @@ pub struct Globals { max_request_size: u32, registration_disabled: bool, encryption_disabled: bool, + federation_enabled: bool, } impl Globals { @@ -69,6 +70,7 @@ impl Globals { .map_err(|_| Error::BadConfig("Invalid max_request_size."))?, registration_disabled: config.get_bool("registration_disabled").unwrap_or(false), encryption_disabled: config.get_bool("encryption_disabled").unwrap_or(false), + federation_enabled: config.get_bool("federation_enabled").unwrap_or(false), }) } @@ -114,4 +116,8 @@ impl Globals { pub fn encryption_disabled(&self) -> bool { self.encryption_disabled } + + pub fn federation_enabled(&self) -> bool { + self.federation_enabled + } } diff --git a/src/server_server.rs b/src/server_server.rs index 0c175bfb2..79976c09d 100644 --- a/src/server_server.rs +++ b/src/server_server.rs @@ -57,6 +57,10 @@ pub async fn send_request( where T: Debug, { + if !globals.federation_enabled() { + return Err(Error::BadConfig("Federation is disabled.")); + } + let resolver = AsyncResolver::tokio_from_system_conf() .await .map_err(|_| Error::BadConfig("Failed to set up trust dns resolver with system config."))?; @@ -204,7 +208,11 @@ where } #[cfg_attr(feature = "conduit_bin", get("/_matrix/federation/v1/version"))] -pub fn get_server_version() -> ConduitResult { +pub fn get_server_version(db: State<'_, Database>) -> ConduitResult { + if !db.globals.federation_enabled() { + return Err(Error::BadConfig("Federation is disabled.")); + } + Ok(get_server_version::Response { server: Some(get_server_version::Server { name: Some("Conduit".to_owned()), @@ -216,6 +224,11 @@ pub fn get_server_version() -> ConduitResult { #[cfg_attr(feature = "conduit_bin", get("/_matrix/key/v2/server"))] pub fn get_server_keys(db: State<'_, Database>) -> Json { + if !db.globals.federation_enabled() { + // TODO: Use proper types + return Json("Federation is disabled.".to_owned()); + } + let mut verify_keys = BTreeMap::new(); verify_keys.insert( format!("ed25519:{}", db.globals.keypair().version()), @@ -259,6 +272,10 @@ pub async fn get_public_rooms_filtered_route( db: State<'_, Database>, body: Ruma>, ) -> ConduitResult { + if !db.globals.federation_enabled() { + return Err(Error::BadConfig("Federation is disabled.")); + } + let response = client_server::get_public_rooms_filtered_helper( &db, None, @@ -302,6 +319,10 @@ pub async fn get_public_rooms_route( db: State<'_, Database>, body: Ruma>, ) -> ConduitResult { + if !db.globals.federation_enabled() { + return Err(Error::BadConfig("Federation is disabled.")); + } + let response = client_server::get_public_rooms_filtered_helper( &db, None, @@ -345,6 +366,10 @@ pub fn send_transaction_message_route<'a>( db: State<'a, Database>, body: Ruma>, ) -> ConduitResult { + if !db.globals.federation_enabled() { + return Err(Error::BadConfig("Federation is disabled.")); + } + //dbg!(&*body); for pdu in &body.pdus { let mut value = serde_json::from_str(pdu.json().get()) @@ -384,6 +409,10 @@ pub fn get_missing_events_route<'a>( db: State<'a, Database>, body: Ruma>, ) -> ConduitResult { + if !db.globals.federation_enabled() { + return Err(Error::BadConfig("Federation is disabled.")); + } + let mut queued_events = body.latest_events.clone(); let mut events = Vec::new(); @@ -427,6 +456,10 @@ pub fn get_profile_information_route<'a>( db: State<'a, Database>, body: Ruma>, ) -> ConduitResult { + if !db.globals.federation_enabled() { + return Err(Error::BadConfig("Federation is disabled.")); + } + let mut displayname = None; let mut avatar_url = None; @@ -455,6 +488,10 @@ pub fn get_user_devices_route<'a>( db: State<'a, Database>, body: Ruma>, ) -> ConduitResult { + if !db.globals.federation_enabled() { + return Err(Error::BadConfig("Federation is disabled.")); + } + let mut displayname = None; let mut avatar_url = None; From 304c53c4f53397a251e567a891c54ece7835173f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20K=C3=B6sters?= Date: Wed, 7 Oct 2020 12:29:19 +0200 Subject: [PATCH 50/51] style: make clippy happier --- src/client_server/room.rs | 8 ++------ src/client_server/sync.rs | 2 +- src/database/sending.rs | 2 +- src/pdu.rs | 4 ++-- src/server_server.rs | 2 +- 5 files changed, 7 insertions(+), 11 deletions(-) diff --git a/src/client_server/room.rs b/src/client_server/room.rs index 28d30e29b..744d9490d 100644 --- a/src/client_server/room.rs +++ b/src/client_server/room.rs @@ -354,12 +354,8 @@ pub async fn upgrade_room_route( ) -> ConduitResult { let sender_id = body.sender_id.as_ref().expect("user is authenticated"); - // Validate the room version requested - let new_version = - RoomVersionId::try_from(body.new_version.clone()).expect("invalid room version id"); - if !matches!( - new_version, + body.new_version, RoomVersionId::Version5 | RoomVersionId::Version6 ) { return Err(Error::BadRequest( @@ -414,7 +410,7 @@ pub async fn upgrade_room_route( let mut create_event_content = ruma::events::room::create::CreateEventContent::new(sender_id.clone()); create_event_content.federate = federate; - create_event_content.room_version = new_version; + create_event_content.room_version = body.new_version.clone(); create_event_content.predecessor = predecessor; db.rooms.build_and_append_pdu( diff --git a/src/client_server/sync.rs b/src/client_server/sync.rs index aec03afae..6f4116020 100644 --- a/src/client_server/sync.rs +++ b/src/client_server/sync.rs @@ -477,7 +477,7 @@ pub async fn sync_events_route( }) .and_then(|pdu| { serde_json::from_value::>( - pdu.content.clone(), + pdu.content, ) .expect("Raw::from_value always works") .deserialize() diff --git a/src/database/sending.rs b/src/database/sending.rs index c818cbfc8..24a783b67 100644 --- a/src/database/sending.rs +++ b/src/database/sending.rs @@ -84,7 +84,7 @@ impl Sending { (Box, send_transaction_message::v1::Response), (Box, Error), > { - let pdu_json = PduEvent::to_outgoing_federation_event( + let pdu_json = PduEvent::convert_to_outgoing_federation_event( rooms .get_pdu_json_from_id(&pdu_id) .map_err(|e| (server.clone(), e))? diff --git a/src/pdu.rs b/src/pdu.rs index 4b1df4b83..7118bfc8e 100644 --- a/src/pdu.rs +++ b/src/pdu.rs @@ -199,7 +199,7 @@ impl PduEvent { serde_json::from_value(json).expect("Raw::from_value always works") } - pub fn to_outgoing_federation_event( + pub fn convert_to_outgoing_federation_event( mut pdu_json: serde_json::Value, ) -> Raw { if let Some(unsigned) = pdu_json @@ -239,7 +239,7 @@ impl From<&state_res::StateEvent> for PduEvent { content: pdu.content().clone(), state_key: Some(pdu.state_key()), prev_events: pdu.prev_event_ids(), - depth: pdu.depth().clone(), + depth: *pdu.depth(), auth_events: pdu.auth_events(), redacts: pdu.redacts().cloned(), unsigned: pdu.unsigned().clone().into_iter().collect(), diff --git a/src/server_server.rs b/src/server_server.rs index 79976c09d..462d636fb 100644 --- a/src/server_server.rs +++ b/src/server_server.rs @@ -438,7 +438,7 @@ pub fn get_missing_events_route<'a>( ) .map_err(|_| Error::bad_database("Invalid prev_events content in pdu in db."))?, ); - events.push(PduEvent::to_outgoing_federation_event(pdu)); + events.push(PduEvent::convert_to_outgoing_federation_event(pdu)); } i += 1; } From 9d1387954f0fcb59ddf805b972a7cbef489a7367 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20K=C3=B6sters?= Date: Fri, 16 Oct 2020 14:04:29 +0200 Subject: [PATCH 51/51] Update dependencies, remove dbgs --- Cargo.lock | 119 +++++++++++++++++++++++-------------------- rust-toolchain | 2 +- src/server_server.rs | 2 - 3 files changed, 64 insertions(+), 59 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 857635db4..a658ee2b8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -90,14 +90,14 @@ checksum = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a" [[package]] name = "backtrace" -version = "0.3.51" +version = "0.3.53" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec1931848a574faa8f7c71a12ea00453ff5effbb5f51afe7f77d7a48cace6ac1" +checksum = "707b586e0e2f247cbde68cdd2c3ce69ea7b7be43e1c5b426e37c9319c4b9838e" dependencies = [ "addr2line", - "cfg-if", + "cfg-if 1.0.0", "libc", - "miniz_oxide 0.4.2", + "miniz_oxide 0.4.3", "object", "rustc-demangle", ] @@ -163,9 +163,9 @@ checksum = "0e4cec68f03f32e44924783795810fa50a7035d8c8ebe78580ad7e6c703fba38" [[package]] name = "cc" -version = "1.0.60" +version = "1.0.61" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef611cc68ff783f18535d77ddd080185275713d852c4f5cbb6122c462a7a825c" +checksum = "ed67cbde08356238e75fc4656be4749481eeffb09e19f320a25237d5221c985d" [[package]] name = "cfg-if" @@ -173,6 +173,12 @@ version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822" +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + [[package]] name = "chrono" version = "0.4.19" @@ -197,9 +203,9 @@ dependencies = [ [[package]] name = "color_quant" -version = "1.0.1" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0dbbb57365263e881e805dc77d94697c9118fd94d8da011240555aa7b23445bd" +checksum = "3d7b894f5411737b7867f4827955924d7c254fc9f4d91a6aad6b097804b1018b" [[package]] name = "conduit" @@ -271,7 +277,7 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ba125de2af0df55319f41944744ad91c71113bf74a4646efff39afe1f6842db1" dependencies = [ - "cfg-if", + "cfg-if 0.1.10", ] [[package]] @@ -281,7 +287,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "058ed274caafc1f60c4997b5fc07bf7dc7cca454af7c6e81edffe5f33f70dace" dependencies = [ "autocfg", - "cfg-if", + "cfg-if 0.1.10", "crossbeam-utils", "lazy_static", "maybe-uninit", @@ -296,7 +302,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c3c7c73a2d1e9fc0886a08b93e98eb643461230d5f1925e4036204d5f2e261a8" dependencies = [ "autocfg", - "cfg-if", + "cfg-if 0.1.10", "lazy_static", ] @@ -384,7 +390,7 @@ version = "0.8.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a51b8cf747471cb9499b6d59e59b0444f4c90eba8968c4e44874e92b5b64ace2" dependencies = [ - "cfg-if", + "cfg-if 0.1.10", ] [[package]] @@ -566,7 +572,7 @@ version = "0.1.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fc587bc0ec293155d5bfa6b9891ec18a1e330c234f896ea47fbada4cadbe47e6" dependencies = [ - "cfg-if", + "cfg-if 0.1.10", "libc", "wasi 0.9.0+wasi-snapshot-preview1", ] @@ -766,7 +772,7 @@ version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "63312a18f7ea8760cdd0a7c5aac1a619752a246b833545e3e36d1f81f7cd9e66" dependencies = [ - "cfg-if", + "cfg-if 0.1.10", ] [[package]] @@ -881,7 +887,7 @@ version = "0.4.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4fabed175da42fed1fa0746b0ea71f412aa9d35e76e95e59b192c64b9dc2bf8b" dependencies = [ - "cfg-if", + "cfg-if 0.1.10", ] [[package]] @@ -968,9 +974,9 @@ dependencies = [ [[package]] name = "miniz_oxide" -version = "0.4.2" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c60c0dfe32c10b43a144bad8fc83538c52f58302c92300ea7ec7bf7b38d5a7b9" +checksum = "0f2d26ec3309788e423cfbf68ad1800f061638098d76a83681af979dc4eda19d" dependencies = [ "adler", "autocfg", @@ -982,7 +988,7 @@ version = "0.6.22" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fce347092656428bc8eaf6201042cb551b8d67855af7374542a92a0fbfcac430" dependencies = [ - "cfg-if", + "cfg-if 0.1.10", "fuchsia-zircon", "fuchsia-zircon-sys", "iovec", @@ -1042,7 +1048,7 @@ version = "0.2.35" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3ebc3ec692ed7c9a255596c67808dee269f64655d8baf7b4f0638e51ba1d6853" dependencies = [ - "cfg-if", + "cfg-if 0.1.10", "libc", "winapi 0.3.9", ] @@ -1100,9 +1106,9 @@ dependencies = [ [[package]] name = "object" -version = "0.20.0" +version = "0.21.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ab52be62400ca80aa00285d25253d7f7c437b7375c4de678f5405d3afe82ca5" +checksum = "37fd5004feb2ce328a52b0b3d01dbf4ffff72583493900ed15f22d4111c51693" [[package]] name = "once_cell" @@ -1117,7 +1123,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8d575eff3665419f9b83678ff2815858ad9d11567e082f5ac1814baba4e2bcb4" dependencies = [ "bitflags", - "cfg-if", + "cfg-if 0.1.10", "foreign-types", "lazy_static", "libc", @@ -1160,7 +1166,7 @@ version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c361aa727dd08437f2f1447be8b59a33b0edd15e0fcee698f935613d9efbca9b" dependencies = [ - "cfg-if", + "cfg-if 0.1.10", "cloudabi", "instant", "libc", @@ -1198,18 +1204,18 @@ checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e" [[package]] name = "pin-project" -version = "0.4.26" +version = "0.4.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13fbdfd6bdee3dc9be46452f86af4a4072975899cf8592466668620bebfbcc17" +checksum = "2ffbc8e94b38ea3d2d8ba92aea2983b503cd75d0888d75b86bb37970b5698e15" dependencies = [ "pin-project-internal", ] [[package]] name = "pin-project-internal" -version = "0.4.26" +version = "0.4.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c82fb1329f632c3552cf352d14427d57a511b1cf41db93b3a7d77906a82dcc8e" +checksum = "65ad2ae56b6abe3a1ee25f15ee605bacadb9a764edaba9c2bf4103800d4a1895" dependencies = [ "proc-macro2", "quote", @@ -1230,9 +1236,9 @@ checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" [[package]] name = "pkg-config" -version = "0.3.18" +version = "0.3.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d36492546b6af1463394d46f0c834346f31548646f6ba10849802c9c9a27ac33" +checksum = "3831453b3449ceb48b6d9c7ad7c96d5ea673e9b470a1dc578c2ce6521230884c" [[package]] name = "png" @@ -1389,9 +1395,9 @@ dependencies = [ [[package]] name = "regex" -version = "1.3.9" +version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c3780fcf44b193bc4d09f36d2a3c87b251da4a046c87795a0d35f4f927ad8e6" +checksum = "8963b85b8ce3074fecffde43b4b0dded83ce2f367dc8d363afc56679f3ee820b" dependencies = [ "regex-syntax", ] @@ -1408,9 +1414,9 @@ dependencies = [ [[package]] name = "regex-syntax" -version = "0.6.18" +version = "0.6.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26412eb97c6b088a6997e05f69403a802a92d520de2f8e63c2b65f9e0f47c4e8" +checksum = "8cab7a364d15cde1e505267766a2d3c4e22a843e1a601f0fa7564c0f82ced11c" [[package]] name = "remove_dir_all" @@ -1738,9 +1744,9 @@ dependencies = [ [[package]] name = "rustc-demangle" -version = "0.1.16" +version = "0.1.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c691c0e608126e00913e33f0ccf3727d5fc84573623b8d65b2df340b5201783" +checksum = "b2610b7f643d18c87dff3b489950269617e6601a51f1f05aa5daefee36f64f0b" [[package]] name = "rustc_version" @@ -1836,18 +1842,18 @@ checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" [[package]] name = "serde" -version = "1.0.116" +version = "1.0.117" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96fe57af81d28386a513cbc6858332abc6117cfdb5999647c6444b8f43a370a5" +checksum = "b88fa983de7720629c9387e9f517353ed404164b1e482c970a90c1a4aaf7dc1a" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.116" +version = "1.0.117" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f630a6370fd8e457873b4bd2ffdae75408bc291ba72be773772a4c2a065d9ae8" +checksum = "cbd1ae72adb44aab48f325a02444a5fc079349a8d804c1fc922aed3f7454c74e" dependencies = [ "proc-macro2", "quote", @@ -1856,9 +1862,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.58" +version = "1.0.59" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a230ea9107ca2220eea9d46de97eddcb04cd00e92d13dda78e478dd33fa82bd4" +checksum = "dcac07dbffa1c65e7f816ab9eba78eb142c6d44410f4eeba1e26e4f5dfa56b95" dependencies = [ "itoa", "ryu", @@ -1936,7 +1942,7 @@ version = "0.3.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b1fa70dc5c8104ec096f4fe7ede7a221d35ae13dcd19ba1ad9a81d2cab9a1c44" dependencies = [ - "cfg-if", + "cfg-if 0.1.10", "libc", "redox_syscall", "winapi 0.3.9", @@ -1950,9 +1956,9 @@ checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" [[package]] name = "standback" -version = "0.2.10" +version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33a71ea1ea5f8747d1af1979bfb7e65c3a025a70609f04ceb78425bc5adad8e6" +checksum = "f4e0831040d2cf2bdfd51b844be71885783d489898a192f254ae25d57cce725c" dependencies = [ "version_check", ] @@ -1966,7 +1972,7 @@ checksum = "7345c971d1ef21ffdbd103a75990a15eb03604fc8b8852ca8cb418ee1a099028" [[package]] name = "state-res" version = "0.1.0" -source = "git+https://github.com/timokoesters/state-res?branch=spec-comp#d11a3feb5307715ab5d86af8f25d4bccfee6264b" +source = "git+https://github.com/timokoesters/state-res?branch=spec-comp#a7d76935f12757aecfee305838069c9bcbe7d34a" dependencies = [ "itertools", "js_int", @@ -2051,9 +2057,9 @@ dependencies = [ [[package]] name = "syn" -version = "1.0.42" +version = "1.0.44" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c51d92969d209b54a98397e1b91c8ae82d8c87a7bb87df0b29aa2ad81454228" +checksum = "e03e57e4fcbfe7749842d53e24ccb9aa12b7252dbe5e91d2acad31834c8b8fdd" dependencies = [ "proc-macro2", "quote", @@ -2066,7 +2072,7 @@ version = "3.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7a6e24d9338a0a5be79593e2fa15a648add6138caa803e2d5bc782c371732ca9" dependencies = [ - "cfg-if", + "cfg-if 0.1.10", "libc", "rand", "redox_syscall", @@ -2230,9 +2236,9 @@ dependencies = [ [[package]] name = "toml" -version = "0.5.6" +version = "0.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ffc92d160b1eef40665be3a05630d003936a3bc7da7421277846c2613e92c71a" +checksum = "75cf45bb0bef80604d001caaec0d09da99611b3c0fd39d3080468875cdb65645" dependencies = [ "serde", ] @@ -2249,7 +2255,7 @@ version = "0.1.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b0987850db3733619253fe60e17cb59b82d37c7e6c0236bb81e4d6b87c879f27" dependencies = [ - "cfg-if", + "cfg-if 0.1.10", "log", "pin-project-lite", "tracing-attributes", @@ -2299,9 +2305,9 @@ dependencies = [ [[package]] name = "tracing-subscriber" -version = "0.2.12" +version = "0.2.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "82bb5079aa76438620837198db8a5c529fb9878c730bc2b28179b0241cf04c10" +checksum = "4ef0a5e15477aa303afbfac3a44cba9b6430fdaad52423b1e6c0dbbe28c3eedd" dependencies = [ "ansi_term", "chrono", @@ -2313,6 +2319,7 @@ dependencies = [ "sharded-slab", "smallvec", "thread_local", + "tracing", "tracing-core", "tracing-log", "tracing-serde", @@ -2345,7 +2352,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0f23cdfdc3d8300b3c50c9e84302d3bd6d860fb9529af84ace6cf9665f181b77" dependencies = [ "backtrace", - "cfg-if", + "cfg-if 0.1.10", "futures", "ipconfig", "lazy_static", @@ -2460,7 +2467,7 @@ version = "0.2.68" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1ac64ead5ea5f05873d7c12b545865ca2b8d28adfc50a49b84770a3a97265d42" dependencies = [ - "cfg-if", + "cfg-if 0.1.10", "serde", "serde_json", "wasm-bindgen-macro", @@ -2487,7 +2494,7 @@ version = "0.4.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b7866cab0aa01de1edf8b5d7936938a7e397ee50ce24119aef3e1eaa3b6171da" dependencies = [ - "cfg-if", + "cfg-if 0.1.10", "js-sys", "wasm-bindgen", "web-sys", diff --git a/rust-toolchain b/rust-toolchain index 50aceaa7b..21998d3c2 100644 --- a/rust-toolchain +++ b/rust-toolchain @@ -1 +1 @@ -1.45.0 +1.47.0 diff --git a/src/server_server.rs b/src/server_server.rs index 462d636fb..b8b575ebe 100644 --- a/src/server_server.rs +++ b/src/server_server.rs @@ -443,8 +443,6 @@ pub fn get_missing_events_route<'a>( i += 1; } - dbg!(&events); - Ok(get_missing_events::v1::Response { events }.into()) }