Compare commits

...
Author SHA1 Message Date
Jacob Taylor 3b55b750cb feat(service/sending): Sender Supervision 2026-11-17 19:32:48 -08:00
Jacob Taylor 0228de55bd fix: Toggle Experiments when toggled 2026-11-17 19:32:48 -08:00
Renovate Bot 57a0104ad0 chore(deps): update rust to v1.100.0 2026-11-17 19:32:48 -08:00
Jacob Taylor 79dd7737a8 chore: Clippy Take The Wheel 2026-11-17 19:32:48 -08:00
Jacob Taylor 95e75f8363 Chore: Revert "chore(deps): Upgrade rocksdb and jemalloc", and add cargo.lock changes clippy discovered
This reverts commit 1af92add88.
This reverts commit 779374bb54.

chore(deps): update rust crate syn to v3.0.10
2026-11-17 19:32:48 -08:00
Jacob Taylor 331c7c1b5d feat: Upgrade URL Previews to the moon 2026-11-17 19:32:48 -08:00
Jacob Taylor b4479456fc feat: Continuwuity notarizes the notary 2026-11-17 19:32:48 -08:00
Jacob Taylor c0e54218c8 feat: Enable converged 6g at the edge in continuwuity 2026-11-17 19:32:48 -08:00
Jacob Taylor a085212e71 chore: Commit basic QoDana config for local QA usage 2026-11-17 19:32:48 -08:00
Jacob Taylor 7bf59e1dc0 fix: Delete silly 2026-11-17 19:32:48 -08:00
Jacob Taylor 3d66198699 chore: Fix some clippy lints 2026-11-17 19:32:48 -08:00
Jacob Taylor 3082063026 bump the number of allowed immutable memtables by 1, to allow for greater flood protection
this should probably not be applied if you have rocksdb_atomic_flush = false (the default)
2026-11-17 19:32:48 -08:00
47 changed files with 1600 additions and 364 deletions

No files matched your search

+1 -1
View File
@@ -51,7 +51,7 @@ repos:
hooks:
- id: cargo-clippy
name: cargo clippy
entry: cargo clippy -- -D warnings
entry: cargo clippy --
language: system
pass_filenames: false
types: [ rust ]
Generated
+61 -27
View File
@@ -485,19 +485,20 @@ dependencies = [
[[package]]
name = "bindgen"
version = "0.73.2"
version = "0.72.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "787ef8ef523575546b106a58213d6e6b06198a05c2f757258c68a74273670cfa"
checksum = "993776b509cfb49c750f11b8f07a46fa23e0a1386ffc01fb1e7d343efc387895"
dependencies = [
"bitflags",
"cexpr",
"clang-sys",
"itertools 0.13.0",
"proc-macro2",
"quote",
"regex",
"rustc-hash",
"shlex",
"syn 3.0.10",
"shlex 1.3.0",
"syn 2.0.119",
]
[[package]]
@@ -677,7 +678,7 @@ dependencies = [
"find-msvc-tools",
"jobserver",
"libc",
"shlex",
"shlex 2.0.1",
]
[[package]]
@@ -867,9 +868,13 @@ dependencies = [
name = "conduwuit"
version = "26.11.0-alpha.2"
dependencies = [
"aws-lc-rs",
"clap",
"conduwuit_admin",
"conduwuit_api",
"conduwuit_build_metadata",
"conduwuit_core",
"conduwuit_database",
"conduwuit_macros",
"conduwuit_router",
"conduwuit_service",
@@ -877,6 +882,8 @@ dependencies = [
"const-str",
"ctor",
"dtor",
"hardened_malloc-rs",
"log",
"opentelemetry",
"opentelemetry-otlp",
"opentelemetry-prometheus",
@@ -884,10 +891,13 @@ dependencies = [
"opentelemetry_sdk",
"parking_lot",
"prometheus",
"reqwest 0.13.5",
"rustls",
"sentry",
"sentry-tower",
"sentry-tracing",
"tokio",
"tokio-metrics",
"tracing",
"tracing-flame",
"tracing-journald",
@@ -1127,7 +1137,6 @@ dependencies = [
"p256 0.14.0",
"rand 0.10.3",
"regex",
"reqwest 0.12.28",
"reqwest 0.13.5",
"resolvematrix",
"ruma",
@@ -2704,10 +2713,10 @@ dependencies = [
"hyper",
"hyper-util",
"rustls",
"rustls-native-certs",
"tokio",
"tokio-rustls",
"tower-service",
"webpki-roots",
]
[[package]]
@@ -2981,6 +2990,15 @@ dependencies = [
"serde",
]
[[package]]
name = "itertools"
version = "0.13.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186"
dependencies = [
"either",
]
[[package]]
name = "itertools"
version = "0.14.0"
@@ -3251,9 +3269,9 @@ checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981"
[[package]]
name = "libz-sys"
version = "1.1.32"
version = "1.1.29"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "69965d2409bff8c8cea3829009fff66a86f05a0570656552ef70f616e8e4bd85"
checksum = "85bc9657773828b90eeb625adff10eeac83cc21bbfd8e23a03eaa8a33c9e28d9"
dependencies = [
"cc",
"pkg-config",
@@ -4164,6 +4182,12 @@ dependencies = [
"phc",
]
[[package]]
name = "paste"
version = "1.0.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a"
[[package]]
name = "pear"
version = "0.2.9"
@@ -4827,8 +4851,8 @@ dependencies = [
"log",
"percent-encoding",
"pin-project-lite",
"quinn",
"rustls",
"rustls-native-certs",
"rustls-pki-types",
"serde",
"serde_json",
@@ -4843,6 +4867,7 @@ dependencies = [
"wasm-bindgen",
"wasm-bindgen-futures",
"web-sys",
"webpki-roots",
]
[[package]]
@@ -5180,8 +5205,8 @@ dependencies = [
[[package]]
name = "rust-librocksdb-sys"
version = "0.49.0+11.12.0"
source = "git+https://forgejo.ellis.link/continuwuation/rust-rocksdb-zaidoon1?rev=6021299f3e0e9455dc3f2e39c65e1dc217be8903#6021299f3e0e9455dc3f2e39c65e1dc217be8903"
version = "0.45.1+11.1.1"
source = "git+https://forgejo.ellis.link/continuwuation/rust-rocksdb-zaidoon1?rev=0a25ff92f7c09b55eec496b9c192c7d5136ab2b8#0a25ff92f7c09b55eec496b9c192c7d5136ab2b8"
dependencies = [
"bindgen",
"bzip2-sys",
@@ -5191,15 +5216,14 @@ dependencies = [
"libz-sys",
"lz4-sys",
"pkg-config",
"rustflags",
"tikv-jemalloc-sys",
"zstd-sys",
]
[[package]]
name = "rust-rocksdb"
version = "0.54.0"
source = "git+https://forgejo.ellis.link/continuwuation/rust-rocksdb-zaidoon1?rev=6021299f3e0e9455dc3f2e39c65e1dc217be8903#6021299f3e0e9455dc3f2e39c65e1dc217be8903"
version = "0.49.1"
source = "git+https://forgejo.ellis.link/continuwuation/rust-rocksdb-zaidoon1?rev=0a25ff92f7c09b55eec496b9c192c7d5136ab2b8#0a25ff92f7c09b55eec496b9c192c7d5136ab2b8"
dependencies = [
"libc",
"parking_lot",
@@ -5227,12 +5251,6 @@ dependencies = [
"semver",
]
[[package]]
name = "rustflags"
version = "0.1.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a39e0e9135d7a7208ee80aa4e3e4b88f0f5ad7be92153ed70686c38a03db2e63"
[[package]]
name = "rustix"
version = "1.1.6"
@@ -5861,6 +5879,12 @@ dependencies = [
"lazy_static",
]
[[package]]
name = "shlex"
version = "1.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64"
[[package]]
name = "shlex"
version = "2.0.1"
@@ -6220,17 +6244,18 @@ dependencies = [
[[package]]
name = "tikv-jemalloc-ctl"
version = "0.7.0"
source = "git+https://forgejo.ellis.link/continuwuation/jemallocator?rev=ac4d4861b335b199f812807996e8c9166961041a#ac4d4861b335b199f812807996e8c9166961041a"
version = "0.6.1"
source = "git+https://forgejo.ellis.link/continuwuation/jemallocator?rev=df86ff89d4b1e223b9f7d2dd2fbb7f202da7f554#df86ff89d4b1e223b9f7d2dd2fbb7f202da7f554"
dependencies = [
"libc",
"paste",
"tikv-jemalloc-sys",
]
[[package]]
name = "tikv-jemalloc-sys"
version = "0.7.0+5.4.0-0-g7a34f18502e7b222724097cdcd499b437d189acc"
source = "git+https://forgejo.ellis.link/continuwuation/jemallocator?rev=ac4d4861b335b199f812807996e8c9166961041a#ac4d4861b335b199f812807996e8c9166961041a"
version = "0.6.1+5.3.0-1-ge13ca993e8ccb9ba9847cc330696e02839f328f7"
source = "git+https://forgejo.ellis.link/continuwuation/jemallocator?rev=df86ff89d4b1e223b9f7d2dd2fbb7f202da7f554#df86ff89d4b1e223b9f7d2dd2fbb7f202da7f554"
dependencies = [
"cc",
"libc",
@@ -6238,8 +6263,8 @@ dependencies = [
[[package]]
name = "tikv-jemallocator"
version = "0.7.0"
source = "git+https://forgejo.ellis.link/continuwuation/jemallocator?rev=ac4d4861b335b199f812807996e8c9166961041a#ac4d4861b335b199f812807996e8c9166961041a"
version = "0.6.1"
source = "git+https://forgejo.ellis.link/continuwuation/jemallocator?rev=df86ff89d4b1e223b9f7d2dd2fbb7f202da7f554#df86ff89d4b1e223b9f7d2dd2fbb7f202da7f554"
dependencies = [
"libc",
"tikv-jemalloc-sys",
@@ -7097,6 +7122,15 @@ dependencies = [
"rustls-pki-types",
]
[[package]]
name = "webpki-roots"
version = "1.0.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f5f280bbb2b287bc8e7de7e3852945a0ff50d1bbcb5ed42b6c7a568c546df171"
dependencies = [
"rustls-pki-types",
]
[[package]]
name = "weezl"
version = "0.1.12"
+4 -15
View File
@@ -166,15 +166,6 @@ features = [
"form",
]
# Needed for as long as openidconnect is on an outdated reqwest
[workspace.dependencies.reqwest_0_12]
package = "reqwest"
version = "0.12"
default-features = false
features = [
"rustls-tls-native-roots-no-provider",
]
[workspace.dependencies.serde]
version = "1.0.229"
default-features = false
@@ -406,7 +397,7 @@ features = [
[workspace.dependencies.rust-rocksdb]
git = "https://forgejo.ellis.link/continuwuation/rust-rocksdb-zaidoon1"
rev = "6021299f3e0e9455dc3f2e39c65e1dc217be8903"
rev = "0a25ff92f7c09b55eec496b9c192c7d5136ab2b8"
default-features = false
features = [
"multi-threaded-cf",
@@ -426,8 +417,6 @@ default-features = false
[workspace.dependencies.openidconnect]
version = "4.1.0"
default-features = false
features = ["reqwest"]
# optional opentelemetry, performance measurements, flamegraphs, etc for performance measurements and monitoring
[workspace.dependencies.opentelemetry]
@@ -489,7 +478,7 @@ version = "0.49.5"
# jemalloc usage
[workspace.dependencies.tikv-jemalloc-sys]
git = "https://forgejo.ellis.link/continuwuation/jemallocator"
rev = "ac4d4861b335b199f812807996e8c9166961041a"
rev = "df86ff89d4b1e223b9f7d2dd2fbb7f202da7f554"
default-features = false
features = [
"background_threads_runtime_support",
@@ -497,7 +486,7 @@ features = [
]
[workspace.dependencies.tikv-jemallocator]
git = "https://forgejo.ellis.link/continuwuation/jemallocator"
rev = "ac4d4861b335b199f812807996e8c9166961041a"
rev = "df86ff89d4b1e223b9f7d2dd2fbb7f202da7f554"
default-features = false
features = [
"background_threads_runtime_support",
@@ -505,7 +494,7 @@ features = [
]
[workspace.dependencies.tikv-jemalloc-ctl]
git = "https://forgejo.ellis.link/continuwuation/jemallocator"
rev = "ac4d4861b335b199f812807996e8c9166961041a"
rev = "df86ff89d4b1e223b9f7d2dd2fbb7f202da7f554"
default-features = false
features = ["use_std"]
+1
View File
@@ -0,0 +1 @@
Added support for acting as a trusted server, meaning the `trusted_servers` list can consist of servers that are either Synapse OR continuwuity.
+1
View File
@@ -0,0 +1 @@
URL previews no longer completely fail if downloading auxiliary content (images/videos/audio) fails. Contributed by @nex.
+1
View File
@@ -0,0 +1 @@
Fixed the experiments implementation in #2517 not properly disabling some functionality when associated experiments weren't enabled.
+1
View File
@@ -0,0 +1 @@
Fix outgoing federation stalling after a sender worker panics. Contributed by @eleboucher.
+40 -5
View File
@@ -1025,10 +1025,7 @@
# (trusted key notary servers), as well as serve as trusted servers for
# other operations like backfill and event fetching.
#
# Currently, continuwuity doesn't support inbound key requests, so
# this list should only contain other Synapse servers.
#
# example: ["matrix.org", "starstruck.systems"]
# example: ["continuwuity.org", "matrix.org"]
#
# It is possible to restrict which signing keys trusted servers are
# allowed to sign responses with. Without configuring this, all responses
@@ -1066,7 +1063,7 @@
# is discouraged for performance and reliability reasons.
#
# env: CONTINUWUITY_TRUSTED_SERVERS
#trusted_servers = ["matrix.org"]
#trusted_servers = ["continuwuity.org", "continuwuity.rocks", "matrix.org"]
# Whether to prioritise trusted server lookups over origin server lookups.
#
@@ -2669,6 +2666,26 @@
# and consequently know why you should enable them.
# MSC3814: Dehydrated devices with SSSS
#
# Enables server-side support for "dehydrated devices", which are
# server-side devices (created by supporting clients) that can collect
# encryption keys when there are no online devices, which supporting
# clients can then "hydrate" to download any encryption keys that were
# sent while they were offline.
#
# Important: this is NOT required to receive encrypted messages while you
# are offline. Devices can still receive encrypted messages while they are
# offline, but to share those keys with new devices, they have to be
# online. Dehydrated devices work around this by providing a "shared"
# device new sessions can bootstrap from, to receive keys from before they
# logged in that haven't been uploaded to the server-side key backup yet.
#
# https://github.com/matrix-org/matrix-spec-proposals/pull/3814
#
# env: CONTINUWUITY_EXPERIMENTS__MSC3814
#msc3814 = false
# MSC4155: Invite filtering
#
# Provides advanced invite filtering rules for users, allowing complex
@@ -2693,6 +2710,24 @@
# env: CONTINUWUITY_EXPERIMENTS__MSC4174
#msc4174 = false
# MSC4406: `M_SENDER_IGNORED` error code
#
# Introduces a new error code that explicitly informs clients the reason
# they cannot access an event if it is sent by an ignored user. Without
# this proposal, clients will simply get a "not found" or "forbidden"
# error when requesting events sent by users that have been ignored, which
# leads to replies being shown as "this message could not be found".
# The new error code allows clients to distinguish unknown/forbidden
# events from ignored senders, allowing them to display an error message
# like "you have blocked this sender" instead.
#
# Requires a supporting client to be useful.
#
# https://github.com/matrix-org/matrix-spec-proposals/pull/4406
#
# env: CONTINUWUITY_EXPERIMENTS__MSC4406
#msc4406 = false
# MSC4484: Server Administration OAuth Scope
#
# Adds a new OAuth2 scope that explicitly grants a token temporary
+3
View File
@@ -71,6 +71,9 @@ Names below are as they appear in Prometheus. OTLP uses the dotted OpenTelemetry
| `continuwuity_sender_events_total` | counter | `destination`, `kind`, `outcome` | PDUs and EDUs sent in outgoing transactions |
| `continuwuity_sender_events_queued_total` | counter | `destination`, `kind` | PDUs and EDUs queued for sending |
| `continuwuity_sender_queue_depth` | gauge | | Messages waiting for the sender workers |
| `continuwuity_sender_requests` | gauge | `state` | Estimated outgoing requests stored as `queued` or `active` (in flight) |
| `continuwuity_sender_workers` | gauge | | Running sender workers; below the configured count means a worker is restarting |
| `continuwuity_sender_panics_total` | counter | `scope` | Panics caught in a single outgoing `transaction` or a whole sender `worker` |
| `continuwuity_users` | gauge | `origin` | Known `local` users, including deactivated, and `remote` users |
| `continuwuity_invites_total` | counter | `direction`, `destination`, `outcome` | Invites sent by local users or received over federation |
| `continuwuity_media_write_size_bytes` | histogram | `origin` | Media files written, from local uploads or cached remote media |
+46
View File
@@ -0,0 +1,46 @@
#-------------------------------------------------------------------------------#
# Qodana analysis is configured by qodana.yaml file #
# https://www.jetbrains.com/help/qodana/qodana-yaml.html #
#-------------------------------------------------------------------------------#
#################################################################################
# WARNING: Do not store sensitive information in this file, #
# as its contents will be included in the Qodana report. #
#################################################################################
version: "1.0"
#Specify inspection profile for code analysis
profile:
name: qodana.starter
#Enable inspections
#include:
# - name: <SomeEnabledInspectionId>
#Disable inspections
#exclude:
# - name: <SomeDisabledInspectionId>
# paths:
# - <path/where/not/run/inspection>
#Execute shell command before Qodana execution (Applied in CI/CD pipeline)
#bootstrap: sh ./prepare-qodana.sh
#Install IDE plugins before Qodana execution (Applied in CI/CD pipeline)
#plugins:
# - id: <plugin.id> #(plugin id can be found at https://plugins.jetbrains.com)
# Quality gate. Will fail the CI/CD pipeline if any condition is not met
# severityThresholds - configures maximum thresholds for different problem severities
# testCoverageThresholds - configures minimum code coverage on a whole project and newly added code
# Code Coverage is available in Ultimate and Ultimate Plus plans
#failureConditions:
# severityThresholds:
# any: 15
# critical: 5
# testCoverageThresholds:
# fresh: 70
# total: 50
#Specify Qodana linter for analysis (Applied in CI/CD pipeline)
linter: jetbrains/qodana-<linter>:2026.2
+1 -1
View File
@@ -10,7 +10,7 @@
[toolchain]
profile = "minimal"
channel = "1.99.1"
channel = "1.100.0"
components = [
# For rust-analyzer
"rust-src",
+10 -4
View File
@@ -1,7 +1,7 @@
use std::collections::{BTreeMap, BTreeSet};
use axum::extract::State;
use conduwuit::{Result, Server};
use conduwuit::{Result, Server, config::experiments::ToggledExperiment};
use ruma::{
RoomVersionId,
api::{
@@ -53,8 +53,11 @@ pub(crate) async fn get_capabilities_route(
capabilities.account_moderation.lock = true;
capabilities.account_moderation.suspend = true;
capabilities.admin =
AdminCapability::new(BTreeSet::from_iter([OAuthClientScope::ServerAdministration]));
if services.config.experiments.msc4484.is_enabled() {
capabilities.admin = AdminCapability::new(BTreeSet::from_iter([
OAuthClientScope::ServerAdministration,
]));
}
}
capabilities.profile_fields = Some(
@@ -63,7 +66,10 @@ pub(crate) async fn get_capabilities_route(
capabilities.set(
"org.matrix.msc4174.webpush",
json!({ "enabled": true, "vapid": services.pusher.webpush_vapid_public_key() }),
json!({
"enabled": services.config.experiments.msc4174.is_enabled(),
"vapid": services.pusher.webpush_vapid_public_key()
}),
)?;
Ok(get_capabilities::v3::Response::new(capabilities))
+6 -1
View File
@@ -2,7 +2,7 @@ use std::time::Duration;
use axum::extract::State;
use conduwuit::{
Err, Result, err,
Err, Result, err, info,
utils::{self, content_disposition::make_content_disposition, math::ruma_from_usize},
};
use conduwuit_core::error;
@@ -219,10 +219,15 @@ pub(crate) async fn get_media_preview_route(
)));
}
info!("Fetching a URL preview for {url} on behalf of {sender_user}");
let preview = services
.media
.get_url_preview(&url)
.await
.inspect(|_| info!(user=%sender_user, %url, "Finished fetching a URL preview"))
.inspect_err(
|e| info!(user=%sender_user, %url, error=?e, "Failed to fetch a URL preview"),
)
.map_err(|error| {
err!(Request(Unknown(
debug_error!(%sender_user, %url, "Failed to fetch URL preview: {error}")
+21 -11
View File
@@ -1,6 +1,8 @@
use axum::extract::State;
use conduwuit::{
Err, Error, Result, at, debug_warn,
Err, Error, Result, at,
config::experiments::ToggledExperiment,
debug_warn,
matrix::{
event::{Event, Matches},
pdu::PduCount,
@@ -315,21 +317,29 @@ where
if server_ignored {
// the sender's server is ignored, so ignore this event
return Err(Error::BadRequest(
ErrorKind::SenderIgnored(SenderIgnoredErrorData::new()),
"The sender's server is ignored by this server.",
));
return if services.config.experiments.msc4406.is_enabled() {
Err(Error::BadRequest(
ErrorKind::SenderIgnored(SenderIgnoredErrorData::new()),
"The sender's server is ignored by this server.",
))
} else {
Err!(Request(NotFound("This event's sender's homeserver is ignored by this server.")))
};
}
if user_ignored && !services.config.send_messages_from_ignored_users_to_client {
// the recipient of this PDU has the sender ignored, and we're not
// configured to send ignored messages to clients
return Err(Error::BadRequest(
ErrorKind::SenderIgnored(SenderIgnoredErrorData::with_sender(
event.sender().to_owned(),
)),
"You have ignored this sender.",
));
return if services.config.experiments.msc4406.is_enabled() {
Err(Error::BadRequest(
ErrorKind::SenderIgnored(SenderIgnoredErrorData::with_sender(
event.sender().to_owned(),
)),
"You have ignored this sender.",
))
} else {
Err!(Request(NotFound("You have ignored the sender of this event.")))
};
}
Ok(false)
+18 -8
View File
@@ -10,7 +10,7 @@ use axum::{
response::{IntoResponse, Redirect},
routing::{any, get},
};
use conduwuit::err;
use conduwuit::{config::experiments::ToggledExperiment, err};
pub(super) use conduwuit_service::state::State;
use http::{Uri, uri};
@@ -162,10 +162,6 @@ pub fn build(router: Router<State>, state: State) -> Router<State> {
.ruma_route(&client::update_device_route)
.ruma_route(&client::delete_device_route)
.ruma_route(&client::delete_devices_route)
.ruma_route(&client::put_dehydrated_device_route)
.ruma_route(&client::delete_dehydrated_device_route)
.ruma_route(&client::get_dehydrated_device_route)
.ruma_route(&client::get_dehydrated_events_route)
.ruma_route(&client::get_tags_route)
.ruma_route(&client::update_tag_route)
.ruma_route(&client::delete_tag_route)
@@ -184,7 +180,6 @@ pub fn build(router: Router<State>, state: State) -> Router<State> {
.ruma_route(&client::get_mutual_rooms_unstable_route)
.ruma_route(&client::get_mutual_rooms_route)
.ruma_route(&client::get_room_summary)
.ruma_route(&client::get_stripped_state_route)
.ruma_route(&client::get_suspended_status)
.ruma_route(&client::put_suspended_status)
.ruma_route(&client::get_lock_status)
@@ -199,6 +194,16 @@ pub fn build(router: Router<State>, state: State) -> Router<State> {
.route("/_continuwuity/server_version", get(client::continuwuity_server_version))
.ruma_route(&admin::rooms::ban::ban_room)
.ruma_route(&admin::rooms::list::list_rooms);
if config.experiments.msc3814.is_enabled() {
router = router
.ruma_route(&client::put_dehydrated_device_route)
.ruma_route(&client::delete_dehydrated_device_route)
.ruma_route(&client::get_dehydrated_device_route)
.ruma_route(&client::get_dehydrated_events_route);
}
if config.experiments.msc4560.is_enabled() {
router = router.ruma_route(&client::get_stripped_state_route);
}
if config.allow_prometheus_metrics {
router = router.route("/_continuwuity/metrics", get(client::continuwuity_metrics));
@@ -207,7 +212,10 @@ pub fn build(router: Router<State>, state: State) -> Router<State> {
if config.allow_federation {
router = router
.ruma_route(&server::get_server_version_route)
.route("/_matrix/key/v2/server", get(server::get_server_keys_route))
.ruma_route(&server::get_server_keys_route)
.ruma_route(&server::get_remote_server_keys_batch_route)
.ruma_route(&server::get_remote_server_keys_batch_v3_unstable_route)
.ruma_route(&server::get_remote_server_keys_route)
.ruma_route(&server::get_public_rooms_route)
.ruma_route(&server::get_public_rooms_filtered_route)
.ruma_route(&server::send_transaction_message_route)
@@ -230,11 +238,13 @@ pub fn build(router: Router<State>, state: State) -> Router<State> {
.ruma_route(&server::get_keys_route)
.ruma_route(&server::claim_keys_route)
.ruma_route(&server::get_hierarchy_route)
.ruma_route(&server::get_stripped_state_route)
.ruma_route(&server::well_known_server)
.ruma_route(&server::get_content_route)
.ruma_route(&server::get_content_thumbnail_route)
.route("/_continuwuity/local_user_count", get(client::continuwuity_local_user_count));
if config.experiments.msc4560.is_enabled() {
router = router.ruma_route(&server::get_stripped_state_route);
}
} else {
router = router
.route("/_matrix/federation/{*path}", any(federation_disabled))
+13 -9
View File
@@ -5,7 +5,9 @@ use std::{
};
use conduwuit::{
Err, Error, Result, err,
Err, Error, Result,
config::experiments::ToggledExperiment,
err,
utils::{IterStream, millis_since_unix_epoch},
};
use http::StatusCode;
@@ -338,17 +340,19 @@ async fn check_access_token(
.map(Box::new);
// Make sure the user has the right scopes to use the route
let user_scopes = if let Some(session_info) = &session_info {
let mut user_scopes = if let Some(session_info) = &session_info {
session_info.scopes()
} else {
let mut scopes = BTreeSet::from_iter([OAuthClientScope::ApiFullAccess]);
if services.admin.user_is_admin(&sender_user).await {
scopes.insert(OAuthClientScope::ServerAdministration);
}
scopes
BTreeSet::from_iter([OAuthClientScope::ApiFullAccess])
};
if (!services.config.experiments.msc4484.is_enabled() || session_info.is_none())
&& services.admin.user_is_admin(&sender_user).await
{
// This is required to make sure admin APIs still work when msc4484
// is disabled.
// https://forgejo.ellis.link/continuwuation/continuwuity/pulls/2533#issuecomment-40042
user_scopes.insert(OAuthClientScope::ServerAdministration);
}
if !required_scopes
.iter()
+360 -69
View File
@@ -1,89 +1,380 @@
use std::{collections::BTreeMap, mem::take, time::Duration};
use axum::{Json, extract::State, response::IntoResponse};
use conduwuit::{Result, utils::timepoint_from_now};
use futures::StreamExt;
use ruma::{
MilliSecondsSinceUnixEpoch,
api::{
OutgoingResponseExt,
federation::discovery::{OldVerifyKey, ServerSigningKeys, get_server_keys},
},
assign,
serde::Raw,
use std::{
collections::{BTreeMap, HashMap, HashSet},
ops::Index,
sync::Arc,
time::Instant,
};
use axum::extract::State;
use conduwuit::{
Err, Result, debug, debug_info, error,
utils::{
ReadyExt,
stream::{BroadbandExt, automatic_width},
to_canonical_object,
},
};
use futures::{StreamExt, stream::FuturesUnordered};
use ruma::{
MilliSecondsSinceUnixEpoch, OwnedServerName, OwnedServerSigningKeyId, ServerName,
api::federation::discovery::{
ServerSigningKeys, get_remote_server_keys, get_remote_server_keys_batch,
get_remote_server_keys_batch::v2::QueryCriteria, get_server_keys,
},
serde::Raw,
uint,
};
use serde_json::value::to_raw_value;
use service::{server_keys, server_keys::in_one_week};
use crate::router::Ruma;
/// # `GET /_matrix/key/v2/server`
///
/// Gets the public signing keys of this server.
pub(crate) async fn get_server_keys_route(
State(services): State<crate::State>,
) -> Result<impl IntoResponse> {
let server_name = services.globals.server_name();
let verify_keys = BTreeMap::from([services.server_keys.active_verify_key()]);
_body: Ruma<get_server_keys::v2::Request>,
) -> Result<get_server_keys::v2::Response> {
services
.server_keys
.build_server_keys_response()
.await
.map(get_server_keys::v2::Response::new)
}
let old_verify_keys = if let Some(k) = &services.config.old_verify_keys {
k.to_owned()
const MAX_KEYS_PER_QUERY: usize = 16384;
const MAX_SERVERS_PER_QUERY: usize = 4096;
/// # `POST /_matrix/key/v2/query`
///
/// Fetches remote server signing keys in bulk, seeding from the local server's
/// database.
///
/// This route applies a semaphore to slow down requests that query more signing
/// keys than is allowed by [MSC4556].
///
/// [MSC4556]: https://github.com/matrix-org/matrix-spec-proposals/pull/4556
pub(crate) async fn get_remote_server_keys_batch_route(
State(services): State<crate::State>,
body: Ruma<get_remote_server_keys_batch::v2::Request>,
) -> Result<get_remote_server_keys_batch::v2::Response> {
query_inner(&services, &body.server_keys, false)
.await
.map(get_remote_server_keys_batch::v2::Response::new)
}
/// # `POST /_matrix/key/unstable/org.continuwuity.msc4556/query`
/// ## `POST /_matrix/key/v3/query`
///
/// Fetches remote server signing keys in bulk, seeding from the local server's
/// database.
///
/// MSC4556: https://github.com/matrix-org/matrix-spec-proposals/pull/4556
pub(crate) async fn get_remote_server_keys_batch_v3_unstable_route(
State(services): State<crate::State>,
body: Ruma<ruminuwuity::federation::notary::unstable::Request>,
) -> Result<ruminuwuity::federation::notary::unstable::Response> {
query_inner(&services, &body.server_keys, false)
.await
.map(ruminuwuity::federation::notary::unstable::Response::new)
}
/// Handles querying the server keys for `/_matrix/key/*/query`.
///
/// `v2` should be `true` if the request path is `/_matrix/key/v2/query` - this
/// will potentially apply a rate-limit to the request.
async fn query_inner(
services: &crate::State,
body: &BTreeMap<OwnedServerName, BTreeMap<OwnedServerSigningKeyId, QueryCriteria>>,
v2: bool,
) -> Result<Vec<Raw<ServerSigningKeys>>> {
let start = (Instant::now(), MilliSecondsSinceUnixEpoch::now());
// If the remote server isn't asking for any keys, we can return an empty
// array immediately.
if body.is_empty() {
return Ok(Vec::new());
}
// Count how many keys are being requested.
// Each server incurs a cost of at least one, and then the count of each key
// being queried per-server.
let total_queried_servers = body.len();
let total_queried_keys = body
.values()
.fold(body.len(), |acc, q| acc.saturating_add(q.len()));
if !v2 {
// If we're on the v3 endpoint, we can apply these limits explicitly.
if total_queried_servers > MAX_SERVERS_PER_QUERY {
return Err!(Request(TooLarge(
"Too many server keys requested ({total_queried_servers} > \
{MAX_SERVERS_PER_QUERY}",
)));
}
if total_queried_keys > MAX_KEYS_PER_QUERY {
return Err!(Request(TooLarge(
"Too many keys requested ({total_queried_keys} > {MAX_KEYS_PER_QUERY})"
)));
}
}
let penalise =
total_queried_servers > MAX_SERVERS_PER_QUERY || total_queried_keys > MAX_KEYS_PER_QUERY;
debug_info!("Fetching {total_queried_keys} keys across {} servers", body.len());
let mut futs: FuturesUnordered<_> = FuturesUnordered::new();
// If we're on the v2 endpoint, we aren't allowed to reject the request.
// To avoid overloading the server, we'll apply a semaphore to limit how
// many futures can be active at a time.
let width = automatic_width();
let sem = if v2 && penalise {
Some(Arc::new(tokio::sync::Semaphore::new(width)))
} else {
// We need to figure out any old signing keys we've acquired from
// notaries.
let mut keys = services.server_keys.signing_keys_for(server_name);
let mut old_keys = BTreeMap::new();
while let Some(resp) = keys.next().await {
// We need to verify that this response is still trusted based on
// the current configuration. It's possible we got this response
// from a notary who we no longer trust.
if resp.valid_until_ts > in_one_week() {
continue;
}
let trusted = services.config.trusted_servers.iter().any(|notary| {
server_keys::Service::verify_server_keys_response(
&resp,
Some((notary.server_name(), notary.verify_keys())),
)
.is_ok()
});
if !trusted {
continue;
}
for (old_key_id, old_key) in resp.old_verify_keys {
if old_key.expired_ts > resp.valid_until_ts {
// This key expired in the future which is probably illegal
continue;
None
};
for (server_name, queries) in body.clone() {
futs.push(acquire_keys_as_notary(
services.server_keys.clone(),
services.globals.server_name().to_owned(),
server_name,
queries,
start.1,
sem.clone(),
));
}
let mut response = Vec::with_capacity(total_queried_keys);
while let Some(v) = futs.next().await {
response.extend(v);
}
debug_info!(
elapsed=?start.0.elapsed(),
%total_queried_servers,
%total_queried_keys,
"Fetched {} key responses",
response.len()
);
Ok(response)
}
async fn sign_ssk(
server_keys: &server_keys::Service,
ssk: ServerSigningKeys,
server_name: &ServerName,
our_name: &ServerName,
) -> Result<Raw<ServerSigningKeys>> {
if server_name == our_name {
// Should already be signed by us if we got this far.
debug_info!(
object=?ssk,
%server_name,
%our_name,
"Refusing to sign our own signing keys response (should already be signed)"
);
return to_raw_value(&ssk)
.map(Raw::<ServerSigningKeys>::from_json)
.map_err(Into::into);
}
let mut canonical = to_canonical_object(&ssk)?;
server_keys.sign_json(&mut canonical)?;
server_keys::strip_extraneous_signatures(&mut canonical, server_name, &[our_name]);
to_raw_value(&canonical)
.map(Raw::<ServerSigningKeys>::from_json)
.map_err(Into::into)
}
#[tracing::instrument(skip(server_keys, my_name, queries, start, semaphore))]
async fn acquire_keys_as_notary(
server_keys: Arc<server_keys::Service>,
my_name: OwnedServerName,
remote: OwnedServerName,
mut queries: BTreeMap<OwnedServerSigningKeyId, QueryCriteria>,
start: MilliSecondsSinceUnixEpoch,
semaphore: Option<Arc<tokio::sync::Semaphore>>,
) -> Vec<Raw<ServerSigningKeys>> {
let cost = queries
.len()
.saturating_add(1)
.min(automatic_width())
.try_into()
.unwrap_or(u32::MAX);
let _permit = if let Some(sem) = semaphore {
sem.acquire_many_owned(cost).await.ok()
} else {
None
};
let fetch_all = queries.is_empty();
let mut results = Vec::with_capacity(queries.len().max(1));
let mut keymap = HashMap::with_capacity(queries.len().max(1));
// First contact the origin (if we're allowed to)
if server_keys.notary_may_contact_origin(&remote) {
debug_info!("Asking remote directly for verify keys");
if let Ok(res) = server_keys.origin_request(remote.clone(), start).await
&& let Ok(ssk) = sign_ssk(&server_keys, res.clone(), &remote, &my_name).await
{
let index = results.len();
results.push(ssk);
for key_id in res.verify_keys.keys().chain(res.old_verify_keys.keys()) {
if fetch_all
|| queries.get(key_id).is_some_and(|c| {
c.minimum_valid_until_ts
.is_none_or(|m| res.valid_until_ts >= m)
}) {
queries.remove(key_id);
keymap.insert(key_id.to_owned(), index);
}
old_keys
.entry(old_key_id)
.and_modify(|current_old_key: &mut OldVerifyKey| {
// Use the lowest expiry timestamp
if current_old_key.expired_ts > old_key.expired_ts {
*current_old_key = old_key.clone();
}
})
.or_insert(old_key);
}
}
old_keys
};
} else {
debug!("Not asking remote for keys (already asked recently)");
}
debug!(keys=?keymap.keys(), "Live verify keys");
if fetch_all && keymap.is_empty() {
// If the server asked for all keys, AND we didn't get anything from the
// origin, just fetch any fresh responses we have.
server_keys
.signing_keys_for(&remote)
.broad_filter_map(|ssk| {
let server_name = &remote;
let my_name = &my_name;
let server_keys = &server_keys;
async move {
if ssk.valid_until_ts > in_one_week() || ssk.valid_until_ts < start {
return None;
}
let server_key = assign!(ServerSigningKeys::new(server_name.to_owned(), valid_until_ts()), {
verify_keys,
old_verify_keys,
});
sign_ssk(server_keys, ssk.clone(), server_name, my_name)
.await
.map(|signed_ssk| (ssk, signed_ssk))
.ok()
}
})
.ready_for_each(|(ssk, signed_ssk)| {
let idx = results.len();
results.push(signed_ssk);
for key_id in ssk.verify_keys.keys().chain(ssk.old_verify_keys.keys()) {
keymap.insert(key_id.to_owned(), idx);
}
})
.await;
}
let server_key = Raw::new(&server_key)?;
let mut response = get_server_keys::v2::Response::new(server_key)
.try_into_http_response::<Vec<u8>>()
.map(|mut response| take(response.body_mut()))
.and_then(|body| serde_json::from_slice(&body).map_err(Into::into))?;
// If we're still missing some keys, fetch them from the local cache
for (key_id, criteria) in queries {
debug!(%key_id, "Fetching verify key from local repository");
services.server_keys.sign_json(&mut response)?;
if keymap.contains_key(&key_id) {
debug!(%key_id, "Already found key");
continue;
}
Ok(Json(response))
let minimum_valid_until_ts = criteria
.clone()
.minimum_valid_until_ts
.unwrap_or_else(|| MilliSecondsSinceUnixEpoch(uint!(0)));
let Some(ssk) = server_keys.get_signing_key(&remote, &key_id).await else {
debug!(%remote, %key_id, "Could not find a matching signing key locally.");
continue;
};
if ssk.valid_until_ts < minimum_valid_until_ts || ssk.valid_until_ts > in_one_week() {
debug!(
%key_id,
valid_until_ts=?ssk.valid_until_ts,
?minimum_valid_until_ts,
"Stored verify key does not satisfy query criteria"
);
continue;
}
if let Some(old_verify_key) = ssk.old_verify_keys.get(&key_id) {
if old_verify_key.expired_ts < minimum_valid_until_ts {
debug!(
%key_id,
expired_ts=?old_verify_key.expired_ts,
?minimum_valid_until_ts,
"Stored old verify key does not satisfy query criteria"
);
continue;
}
}
debug!(%key_id, ?ssk, "Found key locally");
let rep_key_ids = ssk.verify_keys.keys().cloned().collect::<Vec<_>>();
let Ok(signed_ssk) = sign_ssk(&server_keys, ssk.clone(), &remote, &my_name)
.await
.inspect_err(
|e| error!(%key_id, %remote, "Failed to sign signing keys chunk: {e:?}"),
)
else {
continue;
};
let idx = results.len();
results.push(signed_ssk);
keymap.insert(key_id, idx);
for key_id in rep_key_ids {
keymap.insert(key_id, idx);
}
}
keymap
.into_values()
.collect::<HashSet<_>>()
.into_iter()
.map(|idx| results.index(idx).to_owned())
.collect()
}
fn valid_until_ts() -> MilliSecondsSinceUnixEpoch {
let dur = Duration::from_hours(12);
let timepoint = timepoint_from_now(dur).expect("SystemTime should not overflow");
MilliSecondsSinceUnixEpoch::from_system_time(timepoint).expect("UInt should not overflow")
/// `GET /_matrix/key/v2/query/{serverName}`
///
/// Fetches the latest keys for a specific server name.
pub(crate) async fn get_remote_server_keys_route(
State(services): State<crate::State>,
body: Ruma<get_remote_server_keys::v2::Request>,
) -> Result<get_remote_server_keys::v2::Response> {
let min_valid_ts = body.minimum_valid_until_ts;
if services
.server_keys
.notary_may_contact_origin(&body.server_name)
&& let Ok(response) = services
.server_keys
.origin_request(body.server_name.clone(), min_valid_ts)
.await
{
return sign_ssk(
&services.server_keys,
response,
&body.server_name,
services.globals.server_name(),
)
.await
.map(|r| Ok(get_remote_server_keys::v2::Response::new(vec![r])))?;
}
let response = services
.server_keys
.signing_keys_for(&body.server_name)
.broad_filter_map(|ssk| {
let server_name = body.server_name.clone();
async move {
if ssk.valid_until_ts > in_one_week() || ssk.valid_until_ts < min_valid_ts {
return None;
}
sign_ssk(
&services.server_keys,
ssk,
server_name.as_ref(),
services.globals.server_name(),
)
.await
.ok()
}
})
.collect::<Vec<_>>()
.await;
Ok(get_remote_server_keys::v2::Response::new(response))
}
+38
View File
@@ -24,6 +24,26 @@ pub struct Experiments {
/// display: nested
#[serde(default)]
pub msc2815: Option<MSC2815>,
/// MSC3814: Dehydrated devices with SSSS
///
/// Enables server-side support for "dehydrated devices", which are
/// server-side devices (created by supporting clients) that can collect
/// encryption keys when there are no online devices, which supporting
/// clients can then "hydrate" to download any encryption keys that were
/// sent while they were offline.
///
/// Important: this is NOT required to receive encrypted messages while you
/// are offline. Devices can still receive encrypted messages while they are
/// offline, but to share those keys with new devices, they have to be
/// online. Dehydrated devices work around this by providing a "shared"
/// device new sessions can bootstrap from, to receive keys from before they
/// logged in that haven't been uploaded to the server-side key backup yet.
///
/// https://github.com/matrix-org/matrix-spec-proposals/pull/3814
///
/// default: false
#[serde(default)]
pub msc3814: SimpleExperiment,
/// MSC4155: Invite filtering
///
/// Provides advanced invite filtering rules for users, allowing complex
@@ -48,6 +68,24 @@ pub struct Experiments {
/// default: false
#[serde(default)]
pub msc4174: SimpleExperiment,
/// MSC4406: `M_SENDER_IGNORED` error code
///
/// Introduces a new error code that explicitly informs clients the reason
/// they cannot access an event if it is sent by an ignored user. Without
/// this proposal, clients will simply get a "not found" or "forbidden"
/// error when requesting events sent by users that have been ignored, which
/// leads to replies being shown as "this message could not be found".
/// The new error code allows clients to distinguish unknown/forbidden
/// events from ignored senders, allowing them to display an error message
/// like "you have blocked this sender" instead.
///
/// Requires a supporting client to be useful.
///
/// https://github.com/matrix-org/matrix-spec-proposals/pull/4406
///
/// default: false
#[serde(default)]
pub msc4406: SimpleExperiment,
/// MSC4484: Server Administration OAuth Scope
///
/// Adds a new OAuth2 scope that explicitly grants a token temporary
+29 -28
View File
@@ -1133,10 +1133,7 @@ pub struct Config {
/// (trusted key notary servers), as well as serve as trusted servers for
/// other operations like backfill and event fetching.
///
/// Currently, continuwuity doesn't support inbound key requests, so
/// this list should only contain other Synapse servers.
///
/// example: ["matrix.org", "starstruck.systems"]
/// example: ["continuwuity.org", "matrix.org"]
///
/// It is possible to restrict which signing keys trusted servers are
/// allowed to sign responses with. Without configuring this, all responses
@@ -1173,7 +1170,7 @@ pub struct Config {
/// empty list (`[]`) to operate without trusted server assistance, but this
/// is discouraged for performance and reliability reasons.
///
/// default: ["matrix.org"]
/// default: ["continuwuity.org", "continuwuity.rocks", "matrix.org"]
#[serde(default = "default_trusted_servers")]
pub trusted_servers: Vec<TrustedServer>,
@@ -3118,41 +3115,41 @@ fn default_database_backups_to_keep() -> i16 { 1 }
fn default_db_write_buffer_capacity_mb() -> f64 { 48.0 + parallelism_scaled_f64(4.0) }
fn default_db_cache_capacity_mb() -> f64 { 128.0 + parallelism_scaled_f64(64.0) }
fn default_db_cache_capacity_mb() -> f64 { 512.0 + parallelism_scaled_f64(512.0) }
fn default_pdu_cache_capacity() -> u32 { parallelism_scaled_u32(10_000).saturating_add(100_000) }
fn default_pdu_cache_capacity() -> u32 { parallelism_scaled_u32(50_000).saturating_add(100_000) }
fn default_cache_capacity_modifier() -> f64 { 1.0 }
fn default_auth_chain_cache_capacity() -> u32 {
parallelism_scaled_u32(10_000).saturating_add(100_000)
}
fn default_shorteventid_cache_capacity() -> u32 {
parallelism_scaled_u32(50_000).saturating_add(100_000)
}
fn default_shorteventid_cache_capacity() -> u32 {
parallelism_scaled_u32(100_000).saturating_add(100_000)
}
fn default_eventidshort_cache_capacity() -> u32 {
parallelism_scaled_u32(25_000).saturating_add(100_000)
parallelism_scaled_u32(50_000).saturating_add(100_000)
}
fn default_eventid_pdu_cache_capacity() -> u32 {
parallelism_scaled_u32(25_000).saturating_add(100_000)
parallelism_scaled_u32(50_000).saturating_add(100_000)
}
fn default_shortstatekey_cache_capacity() -> u32 {
parallelism_scaled_u32(10_000).saturating_add(100_000)
parallelism_scaled_u32(100_000).saturating_add(100_000)
}
fn default_statekeyshort_cache_capacity() -> u32 {
parallelism_scaled_u32(10_000).saturating_add(100_000)
parallelism_scaled_u32(50_000).saturating_add(100_000)
}
fn default_servernameevent_data_cache_capacity() -> u32 {
parallelism_scaled_u32(100_000).saturating_add(500_000)
parallelism_scaled_u32(100_000).saturating_add(100_000)
}
fn default_stateinfo_cache_capacity() -> u32 { parallelism_scaled_u32(100) }
fn default_stateinfo_cache_capacity() -> u32 { parallelism_scaled_u32(500).clamp(100, 12000) }
fn default_dns_cache_entries() -> u32 { 32768 }
@@ -3239,12 +3236,11 @@ fn default_otlp_filter() -> String {
fn default_tracing_flame_output_path() -> String { "./tracing.folded".to_owned() }
fn default_trusted_servers() -> Vec<TrustedServer> {
// TODO(nex): Once we can be a notary, add maintainer(?) homeservers here.
// Rationale: Users are already running our code, arguably that's a higher
// level of trust than is assigned to notaries in the first place.
// We should still remind everyone that notaries are evil and out to get you
// and to replace this list with servers they actually trust.
let trusted_servers = vec![ruma::owned_server_name!("matrix.org")];
let trusted_servers = vec![
ruma::owned_server_name!("continuwuity.org"),
ruma::owned_server_name!("continuwuity.rocks"),
ruma::owned_server_name!("matrix.org"),
];
trusted_servers
.into_iter()
.map(TrustedServer::Name)
@@ -3375,15 +3371,20 @@ fn default_admin_log_capture() -> String {
fn default_admin_room_tag() -> String { "m.server_notice".to_owned() }
#[must_use]
#[allow(clippy::as_conversions, clippy::cast_precision_loss)]
fn parallelism_scaled_f64(val: f64) -> f64 { val * (sys::available_parallelism() as f64) }
pub fn parallelism_scaled_f64(val: f64) -> f64 { val * (sys::available_parallelism() as f64) }
fn parallelism_scaled_u32(val: u32) -> u32 {
let val = val.try_into().expect("failed to cast u32 to usize");
parallelism_scaled(val).try_into().unwrap_or(u32::MAX)
#[must_use]
#[allow(clippy::as_conversions, clippy::cast_possible_truncation)]
pub fn parallelism_scaled_u32(val: u32) -> u32 {
val.saturating_mul(sys::available_parallelism() as u32)
}
fn parallelism_scaled(val: usize) -> usize { val.saturating_mul(sys::available_parallelism()) }
#[must_use]
pub fn parallelism_scaled(val: usize) -> usize {
val.saturating_mul(sys::available_parallelism())
}
fn default_db_pool_workers() -> usize {
sys::available_parallelism()
+6
View File
@@ -21,6 +21,7 @@ pub struct Instruments {
pub sender_transaction_duration: Histogram<f64>,
pub sender_events: Counter<u64>,
pub sender_events_queued: Counter<u64>,
pub sender_panics: Counter<u64>,
pub media_write_size: Histogram<u64>,
pub media_read_size: Histogram<u64>,
@@ -106,6 +107,11 @@ impl Instruments {
.with_description("PDUs and EDUs queued for sending")
.with_unit("{event}")
.build(),
sender_panics: meter
.u64_counter("continuwuity.sender.panics")
.with_description("Panics caught in outgoing transactions and sender workers")
.with_unit("{panic}")
.build(),
media_write_size: size(
"continuwuity.media.write.size",
+3
View File
@@ -5,6 +5,7 @@ pub type DigestOut = [u8; 256 / 8];
/// Sha256 hash (input gather joined by 0xFF bytes)
#[must_use]
#[tracing::instrument(skip(inputs), level = "trace")]
#[allow(clippy::unnecessary_fallible_conversions)]
pub fn delimited<'a, T, I>(mut inputs: I) -> DigestOut
where
I: Iterator<Item = T> + 'a,
@@ -25,6 +26,7 @@ where
/// Sha256 hash (input gather)
#[must_use]
#[tracing::instrument(skip(inputs), level = "trace")]
#[allow(clippy::unnecessary_fallible_conversions)]
pub fn concat<'a, T, I>(inputs: I) -> DigestOut
where
I: Iterator<Item = T> + 'a,
@@ -43,6 +45,7 @@ where
#[inline]
#[must_use]
#[tracing::instrument(skip(input), level = "trace")]
#[allow(clippy::unnecessary_fallible_conversions)]
pub fn hash<T>(input: T) -> DigestOut
where
T: AsRef<[u8]>,
+1 -1
View File
@@ -29,7 +29,7 @@ fn descriptor_cf_options(
let mut opts = set_table_options(opts, &desc, cache)?;
opts.set_min_write_buffer_number(1);
opts.set_max_write_buffer_number(2);
opts.set_max_write_buffer_number(3);
opts.set_write_buffer_size(desc.write_size);
opts.set_target_file_size_base(desc.file_size);
+33 -8
View File
@@ -77,6 +77,7 @@ full = [
]
brotli_compression = [
"conduwuit-api/brotli_compression",
"conduwuit-core/brotli_compression",
"conduwuit-router/brotli_compression",
"conduwuit-service/brotli_compression",
@@ -88,10 +89,11 @@ direct_tls = [
"conduwuit-router/direct_tls"
]
element_hacks = [
"conduwuit-api/element_hacks",
"conduwuit-service/element_hacks",
"conduwuit-router/element_hacks",
]
gzip_compression = [
"conduwuit-api/gzip_compression",
"conduwuit-core/gzip_compression",
"conduwuit-router/gzip_compression",
"conduwuit-service/gzip_compression",
@@ -100,21 +102,21 @@ hardened_malloc = [
"conduwuit-core/hardened_malloc",
]
http3 = [
"conduwuit-service/http3"
"reqwest/http3"
]
ring = [
"rustls/ring",
"rustls/ring"
]
aws_lc_rs = [
"rustls/aws_lc_rs",
"dep:aws-lc-rs"
]
io_uring = [
"conduwuit-service/io_uring",
"conduwuit-database/io_uring",
]
jemalloc = [
"conduwuit-core/jemalloc",
"conduwuit-service/jemalloc",
"conduwuit-database/jemalloc",
]
jemalloc_prof = [
"conduwuit-core/jemalloc_prof",
@@ -143,13 +145,19 @@ perf_measurements = [
release_max_log_level = [
"tracing/max_level_trace",
"tracing/release_max_level_info",
"log/max_level_trace",
"log/release_max_level_info",
"conduwuit-admin/release_max_log_level",
"conduwuit-api/release_max_log_level",
"conduwuit-core/release_max_log_level",
"conduwuit-database/release_max_log_level",
"conduwuit-router/release_max_log_level",
"conduwuit-service/release_max_log_level",
]
sentry_telemetry = [
"dep:sentry",
"dep:sentry-tracing",
"dep:sentry-tower",
"conduwuit-core/sentry_telemetry",
"conduwuit-router/sentry_telemetry",
]
@@ -172,7 +180,9 @@ url_preview = [
"conduwuit-service/url_preview",
]
zstd_compression = [
"conduwuit-api/zstd_compression",
"conduwuit-core/zstd_compression",
"conduwuit-database/zstd_compression",
"conduwuit-router/zstd_compression",
"conduwuit-service/zstd_compression",
]
@@ -182,10 +192,10 @@ conduwuit_mods = [
bindgen-static = [
# "bindgen/static"
# "clang-sys/static"
"conduwuit-service/bindgen-static"
"conduwuit-database/bindgen-static"
]
bindgen-runtime = [
"conduwuit-service/bindgen-runtime"
"conduwuit-database/bindgen-runtime"
]
[build-dependencies]
@@ -193,7 +203,10 @@ bindgen-runtime = [
# clang-sys = {version = "1", default-features = false}
[dependencies]
conduwuit-admin.workspace = true
conduwuit-api.workspace = true
conduwuit-core.workspace = true
conduwuit-database.workspace = true
conduwuit-router.workspace = true
conduwuit-service.workspace = true
conduwuit-build-metadata.workspace = true
@@ -205,6 +218,7 @@ dtor.workspace = true
console-subscriber.optional = true
console-subscriber.workspace = true
const-str.workspace = true
log.workspace = true
opentelemetry.workspace = true
opentelemetry-otlp.optional = true
opentelemetry-otlp.workspace = true
@@ -212,10 +226,14 @@ opentelemetry-prometheus.workspace = true
opentelemetry-semantic-conventions.workspace = true
opentelemetry_sdk.workspace = true
prometheus.workspace = true
sentry-tower.optional = true
sentry-tower.workspace = true
sentry-tracing.optional = true
sentry-tracing.workspace = true
sentry.optional = true
sentry.workspace = true
tokio-metrics.optional = true
tokio-metrics.workspace = true
tokio.workspace = true
tracing-flame.optional = true
tracing-flame.workspace = true
@@ -225,7 +243,14 @@ tracing-subscriber.workspace = true
tracing.workspace = true
tracing-journald = { workspace = true, optional = true }
parking_lot.workspace = true
reqwest = { workspace = true, default-features = false }
rustls = { workspace = true, default-features = false }
aws-lc-rs = { version = "1.19.0", default-features = false, optional = true }
[target.'cfg(all(not(target_env = "msvc"), target_os = "linux"))'.dependencies]
hardened_malloc-rs.workspace = true
hardened_malloc-rs.optional = true
[lints]
workspace = true
+4 -4
View File
@@ -8,7 +8,7 @@ use axum::{
extract::State,
response::{IntoResponse, Response},
};
use conduwuit::{Result, debug, debug_error, debug_warn, err, error, trace};
use conduwuit::{Result, debug_warn, err, error, info, trace};
use conduwuit_service::Services;
use futures::FutureExt;
use http::{Method, StatusCode, Uri};
@@ -102,11 +102,11 @@ fn handle_result(method: &Method, uri: &Uri, result: Response) -> Result<Respons
let reason = status.canonical_reason().unwrap_or("Unknown Reason");
if status.is_server_error() {
error!(%method, %uri, "{code} {reason}");
info!(%method, %uri, "{code} {reason}");
} else if status.is_client_error() {
debug_error!(%method, %uri, "{code} {reason}");
info!(%method, %uri, "{code} {reason}");
} else if status.is_redirection() {
debug!(%method, %uri, "{code} {reason}");
trace!(%method, %uri, "{code} {reason}");
} else {
trace!(%method, %uri, "{code} {reason}");
}
+1
View File
@@ -0,0 +1 @@
pub mod notary;
+58
View File
@@ -0,0 +1,58 @@
//! `POST /_matrix/key/*/query`
//!
//! Endpoint to query signing keys through other (notary) servers.
pub mod unstable {
use std::collections::BTreeMap;
use ruma::{
OwnedServerName, OwnedServerSigningKeyId,
api::{
federation::{
authentication::ServerSignatures,
discovery::{ServerSigningKeys, get_remote_server_keys_batch::v2::QueryCriteria},
},
request, response,
},
metadata,
serde::Raw,
};
metadata! {
method: POST,
rate_limited: true,
authentication: ServerSignatures,
history: {
unstable("org.continuwuity.msc4553") => "/_matrix/key/unstable/org.continuwuity.msc4556/query",
}
}
#[request]
pub struct Request {
pub server_keys:
BTreeMap<OwnedServerName, BTreeMap<OwnedServerSigningKeyId, QueryCriteria>>,
}
#[response]
pub struct Response {
pub server_keys: Vec<Raw<ServerSigningKeys>>,
}
impl Request {
#[must_use]
pub fn new(
server_keys: BTreeMap<
OwnedServerName,
BTreeMap<OwnedServerSigningKeyId, QueryCriteria>,
>,
) -> Self {
Self { server_keys }
}
}
impl Response {
/// Creates a new `Response` with the given keys.
#[must_use]
pub fn new(server_keys: Vec<Raw<ServerSigningKeys>>) -> Self { Self { server_keys } }
}
}
+2
View File
@@ -6,3 +6,5 @@ pub mod invite_permission_config;
pub mod meowlnir_antispam;
pub mod policy;
pub mod pushers;
pub mod federation;
-12
View File
@@ -16,12 +16,6 @@ crate-type = [
]
[features]
bindgen-runtime = [
"conduwuit-database/bindgen-runtime"
]
bindgen-static = [
"conduwuit-database/bindgen-static"
]
brotli_compression = [
"conduwuit-core/brotli_compression",
"reqwest/brotli",
@@ -35,9 +29,6 @@ gzip_compression = [
"conduwuit-core/gzip_compression",
"reqwest/gzip",
]
http3 = [
"reqwest/http3"
]
io_uring = [
"conduwuit-database/io_uring",
]
@@ -131,9 +122,6 @@ lettre.workspace = true
resolvematrix.workspace = true
serde_urlencoded.workspace = true
openidconnect.workspace = true
# We use this to set TLS backend features on the outdated reqwest bundled by openidconnect,
# which is necessary to call `danger_accept_invalid_certs`.
reqwest_0_12.workspace = true
rand.workspace = true
[target.'cfg(all(unix, target_os = "linux"))'.dependencies]
+1 -1
View File
@@ -111,7 +111,7 @@ impl crate::Service for Service {
.local_address(url_preview_bind_addr)
.timeout(Duration::from_secs(config.url_preview_timeout))
.redirect(redirect::Policy::limited(3))
.user_agent(url_preview_user_agent)
.user_agent(url_preview_user_agent.replace("<version>", conduwuit::version_ua()))
.build()?,
external_resource: base(config, reqwest_dns_resolver.clone())?
+7
View File
@@ -248,6 +248,8 @@ impl Data {
x => x,
};*/
// TODO(nex): Oh my god this is awful when can we kill it
let title = match values
.next()
.and_then(|b| String::from_utf8(b.to_vec()).ok())
@@ -346,6 +348,11 @@ impl Data {
video_height,
audio,
audio_size,
..Default::default() /* TODO(nex,silversys): This weird manual field
* de/serialisation just needs replacing with JSON storage to
* avoid this mess. This is consequently intentionally
* dropping information added later (like image alt text).
* I'll do it later. */
})
}
}
+231 -46
View File
@@ -7,8 +7,9 @@
use std::time::SystemTime;
use conduwuit::{Err, Result, debug, err, utils::response::LimitReadExt};
use conduwuit::{Err, Result, debug, err, info, utils::response::LimitReadExt};
use ipaddress::IPAddress;
use lettre::message::header::ContentType;
use ruma::OwnedMxcUri;
use serde::Serialize;
use url::Url;
@@ -16,20 +17,54 @@ use url::Url;
use super::Service;
use crate::media::mxc::Mxc;
#[derive(Serialize, Default)]
#[derive(Serialize, Default, Debug)]
pub struct UrlPreviewData {
// From https://ogp.me/
/// The title of your object as it should appear within the graph
/// "The Rock".
#[serde(skip_serializing_if = "Option::is_none", rename(serialize = "og:title"))]
pub title: Option<String>,
/// The type of your object, e.g., "video.movie".
#[serde(skip_serializing_if = "String::is_empty", rename(serialize = "og:type"))]
pub site_type: String,
/// The canonical URL of your object that will be used as its permanent ID
/// in the graph.
#[serde(skip_serializing_if = "Option::is_none", rename(serialize = "og:url"))]
pub url: Option<String>,
/// A one to two sentence description of your object.
#[serde(skip_serializing_if = "Option::is_none", rename(serialize = "og:description"))]
pub description: Option<String>,
/// The word that appears before this object's title in a sentence.
#[serde(skip_serializing_if = "Option::is_none", rename(serialize = "og:determiner"))]
pub determiner: Option<String>,
/// The locale these tags are marked up in
#[serde(skip_serializing_if = "Option::is_none", rename(serialize = "og:locale"))]
pub locale: Option<String>,
/// If your object is part of a larger web site, the name which should be
/// displayed for the overall site.
#[serde(skip_serializing_if = "Option::is_none", rename(serialize = "og:site_name"))]
pub site_name: Option<String>,
/// An image URL which should represent your object within the graph.
#[serde(skip_serializing_if = "Option::is_none", rename(serialize = "og:image"))]
pub image: Option<String>,
/// A MIME type for this image.
#[serde(skip_serializing_if = "Option::is_none", rename(serialize = "og:image:type"))]
pub image_type: Option<ContentType>,
#[serde(skip_serializing_if = "Option::is_none", rename(serialize = "matrix:image:size"))]
pub image_size: Option<usize>,
/// The number of pixels wide.
#[serde(skip_serializing_if = "Option::is_none", rename(serialize = "og:image:width"))]
pub image_width: Option<u32>,
/// The number of pixels high.
#[serde(skip_serializing_if = "Option::is_none", rename(serialize = "og:image:height"))]
pub image_height: Option<u32>,
/// A description of what is in the image (not a caption).
#[serde(skip_serializing_if = "Option::is_none", rename(serialize = "og:image:alt"))]
pub image_alt: Option<String>,
/// A URL to a video file that complements this object.
#[serde(skip_serializing_if = "Option::is_none", rename(serialize = "og:video"))]
pub video: Option<String>,
#[serde(skip_serializing_if = "Option::is_none", rename(serialize = "matrix:video:size"))]
@@ -38,6 +73,11 @@ pub struct UrlPreviewData {
pub video_width: Option<u32>,
#[serde(skip_serializing_if = "Option::is_none", rename(serialize = "og:video:height"))]
pub video_height: Option<u32>,
#[serde(skip_serializing_if = "Option::is_none", rename(serialize = "og:video:type"))]
pub video_type: Option<ContentType>,
#[serde(skip_serializing_if = "Option::is_none", rename(serialize = "og:video:alt"))]
pub video_alt: Option<String>,
/// A URL to an audio file to accompany this object.
#[serde(skip_serializing_if = "Option::is_none", rename(serialize = "og:audio"))]
pub audio: Option<String>,
#[serde(skip_serializing_if = "Option::is_none", rename(serialize = "matrix:audio:size"))]
@@ -100,7 +140,9 @@ impl Service {
}
let Some(content_type) = response.headers().get(reqwest::header::CONTENT_TYPE) else {
return Err!(Request(Unknown("Unknown or invalid Content-Type header")));
return Err!(Request(Unknown(
"Response contains unknown, invalid, or missing Content-Type header"
)));
};
let content_type = content_type
@@ -109,12 +151,10 @@ impl Service {
let data = match content_type {
| html if html.starts_with("text/html") => self.download_html(url.as_str()).await?,
| img if img.starts_with("image/") => self.download_image(url.as_str(), None).await?,
| video if video.starts_with("video/") =>
self.download_video(url.as_str(), None).await?,
| audio if audio.starts_with("audio/") =>
self.download_audio(url.as_str(), None).await?,
| _ => return Err!(Request(Unknown("Unsupported Content-Type"))),
| img if img.starts_with("image/") => self.download_image(url.as_str()).await?,
| video if video.starts_with("video/") => self.download_video(url.as_str()).await?,
| audio if audio.starts_with("audio/") => self.download_audio(url.as_str()).await?,
| other => return Err!(Request(Unknown("Unsupported Content-Type: {other}"))),
};
self.set_url_preview(url.as_str(), &data).await?;
@@ -123,17 +163,13 @@ impl Service {
}
#[cfg(feature = "url_preview")]
pub async fn download_image(
&self,
url: &str,
preview_data: Option<UrlPreviewData>,
) -> Result<UrlPreviewData> {
pub async fn download_image(&self, url: &str) -> Result<UrlPreviewData> {
use conduwuit::utils::random_string;
use image::ImageReader;
use crate::media::mxc::Mxc;
let mut preview_data = preview_data.unwrap_or_default();
let mut preview_data = UrlPreviewData::default();
let response = self.services.client.url_preview.get(url).send().await?;
if !response.status().is_success() {
@@ -157,13 +193,12 @@ impl Service {
.ok_or_else(|| err!(Request(Unknown("Downloaded file is not a supported image"))))?
.to_mime_type();
if preview_data.image_height.is_none() || preview_data.image_width.is_none() {
let (width, height) = reader.into_dimensions().map_err(|error| {
err!(Request(Unknown("Failed to read preview image dimensions: {error}")))
})?;
preview_data.image_width.get_or_insert(width);
preview_data.image_height.get_or_insert(height);
}
let (width, height) = reader.into_dimensions().map_err(|error| {
err!(Request(Unknown("Failed to read preview image dimensions: {error}")))
})?;
preview_data.image_width.get_or_insert(width);
preview_data.image_height.get_or_insert(height);
preview_data.image_type = content_type.parse().ok();
let mxc = Mxc {
server_name: self.services.globals.server_name(),
@@ -179,12 +214,9 @@ impl Service {
}
#[cfg(feature = "url_preview")]
pub async fn download_video(
&self,
url: &str,
preview_data: Option<UrlPreviewData>,
) -> Result<UrlPreviewData> {
let mut preview_data = preview_data.unwrap_or_default();
#[tracing::instrument(skip(self))]
pub async fn download_video(&self, url: &str) -> Result<UrlPreviewData> {
let mut preview_data = UrlPreviewData::default();
if self.services.globals.url_preview_allow_audio_video() {
let (url, size) = self.download_media(url).await?;
@@ -196,12 +228,9 @@ impl Service {
}
#[cfg(feature = "url_preview")]
pub async fn download_audio(
&self,
url: &str,
preview_data: Option<UrlPreviewData>,
) -> Result<UrlPreviewData> {
let mut preview_data = preview_data.unwrap_or_default();
#[tracing::instrument(skip(self))]
pub async fn download_audio(&self, url: &str) -> Result<UrlPreviewData> {
let mut preview_data = UrlPreviewData::default();
if self.services.globals.url_preview_allow_audio_video() {
let (url, size) = self.download_media(url).await?;
@@ -212,6 +241,7 @@ impl Service {
Ok(preview_data)
}
#[tracing::instrument(skip(self))]
pub async fn download_media(&self, url: &str) -> Result<(OwnedMxcUri, usize)> {
use conduwuit::utils::random_string;
use http::header::CONTENT_TYPE;
@@ -269,6 +299,7 @@ impl Service {
}
#[cfg(feature = "url_preview")]
#[tracing::instrument(skip(self))]
async fn download_html(&self, url: &str) -> Result<UrlPreviewData> {
use webpage::HTML;
@@ -286,32 +317,75 @@ impl Service {
.expect("u64 should fit in usize"),
)
.await?;
let Ok(html) = HTML::from_string(body.clone(), Some(url.to_owned())) else {
let Ok(html) = HTML::from_string(body.clone(), Some(url.to_owned()))
.inspect_err(|e| debug!(error=?e, "Failed to parse HTML data"))
else {
return Err!(Request(Unknown("Failed to parse HTML")));
};
let mut preview_data = UrlPreviewData::default();
let mut preview_data = UrlPreviewData::from(&html.opengraph);
debug!(parsed=?preview_data, deserialised=?html.opengraph, "Parsed opengraph data");
if let Some(obj) = html.opengraph.images.first() {
preview_data = self.download_image(&obj.url, Some(preview_data)).await?;
info!(asset_url=%obj.url, "Downloading preview image");
if let Ok(image_data) = self.download_image(&obj.url).await.inspect_err(
|e| info!(error=?e, asset_url=%obj.url, "Could not download URL preview image"),
) {
preview_data.image = image_data.image;
preview_data.image_size = image_data.image_size;
// Prioritise what the top-level site specified, falling back to
// detected params.
preview_data.image_width = preview_data.image_width.or(image_data.image_width);
preview_data.image_height = preview_data.image_height.or(image_data.image_height);
preview_data.image_type = preview_data.image_type.or(image_data.image_type);
}
}
if let Some(obj) = html.opengraph.videos.first() {
preview_data = self.download_video(&obj.url, Some(preview_data)).await?;
preview_data.video_width = obj.properties.get("width").and_then(|v| v.parse().ok());
preview_data.video_height = obj.properties.get("height").and_then(|v| v.parse().ok());
info!(asset_url=%obj.url, "Downloading preview video");
if let Ok(video_data) = self.download_video(&obj.url).await.inspect_err(
|e| info!(error=?e, asset_url=%obj.url, "Could not download URL preview video"),
) {
preview_data.video = video_data.video;
preview_data.video_size = video_data.video_size;
}
}
if let Some(obj) = html.opengraph.audios.first() {
preview_data = self.download_audio(&obj.url, Some(preview_data)).await?;
info!(asset_url=%obj.url, "Downloading URL preview audio");
if let Ok(audio_data) = self.download_audio(&obj.url).await.inspect_err(
|e| info!(error=?e, asset_url=%obj.url, "Could not download URL preview audio"),
) {
preview_data.audio = audio_data.audio;
preview_data.audio_size = audio_data.audio_size;
}
}
let props = html.opengraph.properties;
// Provide defaults for some params
preview_data.title = preview_data.title.or(html.title);
preview_data.description = preview_data.description.or(html.description);
preview_data.url = preview_data.url.or_else(|| Some(url.to_owned()));
/* use OpenGraph title/description, but fall back to HTML if not
* available */
preview_data.title = props.get("title").cloned().or(html.title);
preview_data.description = props.get("description").cloned().or(html.description);
// We remove og:image:*, og:video:*, and og:audio:* if their respective
// prior-level properties are not present.
// We do this here instead of in the From<> impl because we can't know
// if we'll be able to fetch the media to convert it to an mxc that
// point.
if preview_data.image.is_none() {
preview_data.image_type = None;
preview_data.image_alt = None;
preview_data.image_height = None;
preview_data.image_width = None;
}
if preview_data.video.is_none() {
preview_data.video_type = None;
preview_data.video_alt = None;
preview_data.video_height = None;
preview_data.video_width = None;
}
if preview_data.audio.is_none() {
preview_data.audio_size = None;
}
Ok(preview_data)
}
@@ -443,3 +517,114 @@ impl Service {
false
}
}
impl From<&webpage::Opengraph> for UrlPreviewData {
fn from(og: &webpage::Opengraph) -> Self {
Self {
title: og.properties.get("title").cloned(),
description: og.properties.get("description").cloned(),
site_type: og.og_type.clone(),
url: og.properties.get("url").cloned(),
determiner: og.properties.get("determiner").cloned(),
locale: og.properties.get("locale").cloned(),
site_name: og.properties.get("site_name").cloned(),
image_type: og
.images
.first()
.and_then(|img| img.properties.get("type").and_then(|t| t.parse().ok())),
image_width: og
.images
.first()
.and_then(|img| img.properties.get("width").and_then(|w| w.parse().ok())),
image_height: og
.images
.first()
.and_then(|img| img.properties.get("height").and_then(|h| h.parse().ok())),
image_alt: og
.images
.first()
.and_then(|img| img.properties.get("alt").cloned()),
video_type: og
.videos
.first()
.and_then(|vid| vid.properties.get("type"))
.and_then(|t| t.parse().ok()),
video_width: og
.videos
.first()
.and_then(|vid| vid.properties.get("width"))
.and_then(|w| w.parse().ok()),
video_height: og
.videos
.first()
.and_then(|vid| vid.properties.get("height"))
.and_then(|h| h.parse().ok()),
video_alt: og
.videos
.first()
.and_then(|vid| vid.properties.get("alt").cloned()),
..Default::default()
}
}
}
#[cfg(test)]
mod tests {
use webpage::HTML;
use super::UrlPreviewData;
#[test]
fn test_urlpreviewdata_parses_correctly() {
let html = r#"
<html>
<head>
<meta property="og:title" content="Test Title" />
<meta property="og:description" content="Test Description" />
<meta property="og:image" content="https://example.com/image.png" />
<meta property="og:url" content="https://example.com/index.html" />
<meta property="og:type" content="org.continuwuity.other" />
<meta property="og:site_name" content="Example Site" />
<meta property="og:image:alt" content="An example image" />
<meta property="og:image:type" content="image/png" />
<meta property="og:image:height" content="69" />
<meta property="og:image:width" content="420" />
<meta property="og:video" content="https://example.com/video.mp4" />
<meta property="og:audio" content="https://example.com/audio.ogg" />
</head>
<body></body>
</html>
"#;
let parsed_html =
HTML::from_string(html.to_owned(), Some("https://example.com".to_owned()))
.expect("Failed to parse HTML");
let data = UrlPreviewData::from(&parsed_html.opengraph);
assert_eq!(data.title, Some("Test Title".to_owned()));
assert_eq!(data.description, Some("Test Description".to_owned()));
assert_eq!(
data.image, None,
"UrlPreviewData should not inherit HTTP/S URL from URL preview"
);
assert_eq!(
data.video, None,
"UrlPreviewData should not inherit HTTP/S URL from URL preview"
);
assert_eq!(
data.audio, None,
"UrlPreviewData should not inherit HTTP/S URL from URL preview"
);
assert_eq!(data.url, Some("https://example.com/index.html".to_owned()));
assert_eq!(data.site_type, "org.continuwuity.other".to_owned());
assert_eq!(data.site_name, Some("Example Site".to_owned()));
assert_eq!(data.image_alt, Some("An example image".to_owned()));
assert_eq!(data.image_type, Some("image/png".parse().unwrap()));
assert_eq!(data.image_height, Some(69));
assert_eq!(data.image_width, Some(420));
assert_eq!(data.image_size, None, "*_size must be None when pulling from opengraph");
assert_eq!(data.video_size, None, "*_size must be None when pulling from opengraph");
assert_eq!(data.audio_size, None, "*_size must be None when pulling from opengraph");
}
}
+22
View File
@@ -18,6 +18,28 @@ impl Services {
},
);
self.gauge(
"continuwuity.sender.requests",
"Estimated outgoing requests stored as queued or in flight",
"{request}",
|services, observer| {
for (state, count) in services.sending.db.request_counts() {
if let Some(count) = count {
observer.observe(count, &[KeyValue::new("state", state)]);
}
}
},
);
self.gauge(
"continuwuity.sender.workers",
"Running sender workers",
"{worker}",
|services, observer| {
observe(observer, services.sending.workers_alive(), &[]);
},
);
self.gauge(
"continuwuity.federation.incoming.transactions.active",
"Incoming federation transactions currently being processed",
+2
View File
@@ -12,6 +12,8 @@ mod migrations;
mod service;
pub mod services;
pub mod state;
#[cfg(test)]
mod test_utils;
pub mod account_data;
pub mod admin;
+4 -1
View File
@@ -6,7 +6,9 @@ use std::{
use base64::Engine;
use conduwuit::{
Result, SyncMutex, info,
Result, SyncMutex,
config::experiments::ToggledExperiment,
info,
utils::{self, hash::sha256},
};
use database::{Deserialized, Json, Map};
@@ -762,6 +764,7 @@ impl Service {
scopes: &BTreeSet<OAuthClientScope>,
) -> Result<(), OAuthError> {
if scopes.contains(&OAuthClientScope::ServerAdministration)
&& self.services.config.experiments.msc4484.is_enabled()
&& !self.services.admin.user_is_admin(authorizing_user).await
{
return Err(OAuthError::new(
+9 -5
View File
@@ -3,7 +3,7 @@ mod webpush;
use std::{fmt::Debug, mem, net::IpAddr, sync::Arc};
use bytes::BytesMut;
use conduwuit::utils::response::LimitReadExt;
use conduwuit::{config::experiments::ToggledExperiment, utils::response::LimitReadExt};
use conduwuit_core::{
Err, Event, Result, Server, debug_warn, err, trace,
utils::{stream::TryIgnore, string_from_bytes},
@@ -133,7 +133,9 @@ impl Service {
self.validate_push_url(&http.url, PUSH_GATEWAY_SCHEMES)
.await?;
},
| PusherKind::WebPush(data) => {
| PusherKind::WebPush(data)
if self.services.config.experiments.msc4174.is_enabled() =>
{
self.validate_push_url(&data.url, WEBPUSH_SCHEMES).await?;
webpush::decode_pushkey(pushkey)?;
webpush::decode_auth(&data.auth)?;
@@ -147,7 +149,7 @@ impl Service {
}
}
},
| PusherKind::Email(_) | PusherKind::Custom { .. } => (),
| _ => (),
}
let key = (sender, pushkey);
@@ -547,7 +549,9 @@ impl Service {
Ok(())
},
| PusherKind::WebPush(data) => {
| PusherKind::WebPush(data)
if self.services.config.experiments.msc4174.is_enabled() =>
{
let notify = self
.build_notification(
user,
@@ -562,7 +566,7 @@ impl Service {
self.send_webpush_notice(user, pusher, data, notify).await
},
| PusherKind::Email(_) | PusherKind::Custom { .. } => Ok(()),
| _ => Ok(()),
}
}
+14
View File
@@ -3,6 +3,7 @@ mod vapid;
use std::time::{Duration, SystemTime};
use conduwuit::config::experiments::ToggledExperiment;
use conduwuit_core::{
Err, Result, debug, debug_warn, err,
utils::{self, string::constant_time_eq},
@@ -85,6 +86,9 @@ impl Service {
/// Reports whether a Web Push pusher has completed its activation
/// handshake.
pub async fn is_webpush_activated(&self, sender: &UserId, pushkey: &str) -> bool {
if !self.services.config.experiments.msc4174.is_enabled() {
return false;
}
self.webpush_state(sender, pushkey).await.activated
}
@@ -109,6 +113,9 @@ impl Service {
pusher: &Pusher,
previous: Option<&Pusher>,
) -> bool {
if !self.services.config.experiments.msc4174.is_enabled() {
return false;
}
let Some(data) = pusher.kind.as_webpush() else {
return false;
};
@@ -126,6 +133,13 @@ impl Service {
}
pub(super) fn begin_webpush_activation(&self, sender: &UserId, pusher: &Pusher) {
debug_assert!(
self.services.config.experiments.msc4174.is_enabled(),
"begin_webpush_activation should not be called with MSC4174 disabled"
);
if !self.services.config.experiments.msc4174.is_enabled() {
return;
}
// Send a one-time encrypted token before allowing notification
// delivery.
let token = utils::rand::string(ACK_TOKEN_LENGTH);
@@ -1,14 +1,10 @@
use std::sync::Arc;
use conduwuit::{
Config, PduEvent, Server,
log::{Log, LogLevelReloadHandles},
metrics::Metrics,
};
use conduwuit::PduEvent;
use futures::StreamExt;
use ruma::{RoomId, RoomVersionId, events::StateEventType};
use serde_json::json;
use crate::test_utils::test_services;
#[tokio::test]
async fn admitted_joins_do_not_disappear_after_a_merge() {
use conduwuit::matrix::PartialPdu;
@@ -20,7 +16,7 @@ async fn admitted_joins_do_not_disappear_after_a_merge() {
},
};
let (services, path) = test_services().await;
let (services, path, _database) = test_services().await;
let alice = UserId::parse("@alice:test.example").unwrap();
let bob = UserId::parse("@bob:bob.example").unwrap();
let charlie = UserId::parse("@charlie:charlie.example").unwrap();
@@ -212,35 +208,6 @@ async fn admitted_joins_do_not_disappear_after_a_merge() {
assert_eq!(results, [true, true], "both joined users must be able to send messages");
}
async fn test_services() -> (Arc<crate::Services>, std::path::PathBuf) {
let _ = rustls::crypto::ring::default_provider().install_default();
let path = std::env::temp_dir()
.join(format!("continuwuity-shared-state-{:016x}", rand::random::<u64>()));
std::fs::create_dir(&path).unwrap();
let config: Config = serde_json::from_value(json!({
"server_name": "test.example",
"database_path": path,
"db_cache_capacity_mb": 16.0,
"rocksdb_write_buffer_capacity_mb": 4.0,
"allow_local_presence": false,
"dns_servers": ["127.0.0.1"],
"allow_invalid_tls_certificates_yes_i_know_what_the_fuck_i_am_doing_with_this_and_i_know_this_is_insecure": true
}))
.unwrap();
let runtime = tokio::runtime::Handle::current();
let server = Arc::new(Server::new(
config,
Some(runtime.clone()),
Log {
reload: LogLevelReloadHandles::default(),
capture: Arc::default(),
},
Metrics::new(Some(runtime), opentelemetry::global::meter("test"), None),
));
let services = crate::Services::build(server).await.unwrap();
(services, path)
}
fn cache_key(
services: &crate::Services,
server: &ruma::ServerName,
+28
View File
@@ -3,6 +3,7 @@ use std::{fmt::Debug, sync::Arc};
use conduwuit::{
Error, Result, at, utils,
utils::{ReadyExt, stream::TryIgnore},
warn,
};
use database::{Database, Deserialized, Map};
use futures::{Stream, StreamExt};
@@ -169,6 +170,33 @@ impl Data {
})
}
/// The oldest queued request of each destination, not yet in flight.
pub(super) async fn queued_heads(&self) -> Vec<(Destination, QueueItem)> {
let mut heads: Vec<(Destination, QueueItem)> = Vec::new();
self.servernameevent_data
.raw_stream()
.ignore_err()
.ready_for_each(|(key, val)| match parse_servercurrentevent(key, val) {
| Ok((dest, event)) =>
if heads.last().is_none_or(|(last, _)| *last != dest) {
heads.push((dest, (key.to_vec(), event)));
},
| Err(e) => warn!(?key, "invalid queued request key: {e}"),
})
.await;
heads
}
/// Estimated number of requests waiting to be sent and in flight.
pub(crate) fn request_counts(&self) -> [(&'static str, Option<u64>); 2] {
[
("queued", &self.servernameevent_data),
("active", &self.servercurrentevent_data),
]
.map(|(state, map)| (state, map.property_integer(c"rocksdb.estimate-num-keys").ok()))
}
pub(super) fn set_latest_educount(&self, server_name: &ServerName, last_count: u64) {
self.servername_educount.raw_put(server_name, last_count);
}
+106 -24
View File
@@ -3,12 +3,19 @@ mod appservice;
mod data;
mod dest;
mod sender;
#[cfg(test)]
mod tests;
use std::{
collections::HashMap,
fmt::Debug,
hash::{DefaultHasher, Hash, Hasher},
iter::once,
sync::Arc,
sync::{
Arc,
atomic::{AtomicUsize, Ordering},
},
time::{Duration, Instant},
};
use async_trait::async_trait;
@@ -19,7 +26,7 @@ use conduwuit::{
utils::{ReadyExt, TryReadyExt, available_parallelism, math::usize_from_u64_truncated},
warn,
};
use futures::{FutureExt, Stream, StreamExt};
use futures::{FutureExt, Stream, StreamExt, stream::FuturesUnordered};
use ruma::{
OwnedServerName, RoomId, ServerName, UserId,
api::{
@@ -27,7 +34,7 @@ use ruma::{
federation::authentication::ServerSignatures, path_builder::PathBuilder,
},
};
use tokio::{task, task::JoinSet};
use tokio::{task, task::JoinSet, time::sleep};
use self::data::Data;
pub use self::{
@@ -49,6 +56,7 @@ pub struct Service {
server: Arc<Server>,
services: Services,
channels: Vec<(loole::Sender<Msg>, loole::Receiver<Msg>)>,
workers_alive: AtomicUsize,
}
struct Services {
@@ -87,6 +95,8 @@ pub type EduVec = SmallVec<[EduBuf; EDU_VEC_CAP]>;
const EDU_BUF_CAP: usize = 128;
const EDU_VEC_CAP: usize = 1;
const WORKER_RESTART_DELAY: Duration = Duration::from_millis(2500);
const WORKER_RESTART_DELAY_MAX: Duration = Duration::from_secs(30);
#[async_trait]
impl crate::Service for Service {
@@ -111,36 +121,67 @@ impl crate::Service for Service {
federation: args.depend::<federation::Service>("federation"),
},
channels: (0..num_senders).map(|_| loole::unbounded()).collect(),
workers_alive: AtomicUsize::new(0),
}))
}
async fn worker(self: Arc<Self>) -> Result {
let mut senders =
self.channels
.iter()
.enumerate()
.fold(JoinSet::new(), |mut joinset, (id, _)| {
let self_ = self.clone();
let worker = self_.sender(id);
let worker = if self.unconstrained() {
task::unconstrained(worker).boxed()
} else {
worker.boxed()
if self.server.config.startup_netburst {
self.dispatch_queued(None).await;
}
let num_senders = self.channels.len();
let mut senders = JoinSet::new();
let mut shards = HashMap::new();
let mut started = vec![Instant::now(); num_senders];
let mut failures = vec![0_u32; num_senders];
let mut restarts = FuturesUnordered::new();
for id in 0..num_senders {
self.spawn_sender(&mut senders, &mut shards, id, false);
}
loop {
tokio::select! {
Some(ret) = senders.join_next_with_id() => {
self.workers_alive.fetch_sub(1, Ordering::Relaxed);
let (task_id, panicked) = match &ret {
| Ok((task_id, _)) => (*task_id, false),
| Err(error) => (error.id(), error.is_panic()),
};
let shard = shards.remove(&task_id).expect("sender worker has a shard");
let runtime = self.server.runtime();
let _abort = joinset.spawn_on(worker, runtime);
joinset
});
if panicked {
self.server
.metrics
.instruments
.sender_panics
.add(1, &[KeyValue::new("scope", "worker")]);
}
while let Some(ret) = senders.join_next_with_id().await {
match ret {
| Ok((id, error)) => {
debug!(?id, ?error, "sender worker finished");
if self.channels[shard].1.is_closed() || !self.server.running() {
debug!(shard, ?ret, "sender worker finished");
continue;
}
// A worker that ran a while before failing starts over from
// the shortest delay.
failures[shard] = if started[shard].elapsed() > WORKER_RESTART_DELAY_MAX {
1
} else {
failures[shard].saturating_add(1)
};
let delay = restart_delay(failures[shard]);
error!(shard, ?ret, ?delay, "sender worker unexpectedly finished, restarting");
restarts.push(sleep(delay).map(move |()| shard));
},
| Err(error) => {
error!(id = ?error.id(), ?error, "sender worker failed");
Some(shard) = restarts.next() => {
if !self.channels[shard].1.is_closed() {
started[shard] = Instant::now();
self.spawn_sender(&mut senders, &mut shards, shard, true);
}
},
() = self.server.until_shutdown(), if !restarts.is_empty() => restarts.clear(),
else => break,
}
}
@@ -161,6 +202,25 @@ impl crate::Service for Service {
}
impl Service {
fn spawn_sender(
self: &Arc<Self>,
senders: &mut JoinSet<Result>,
shards: &mut HashMap<task::Id, usize>,
id: usize,
restarted: bool,
) {
let worker = self.clone().sender(id, restarted);
let worker = if crate::Service::unconstrained(&**self) {
task::unconstrained(worker).boxed()
} else {
worker.boxed()
};
let handle = senders.spawn_on(worker, self.server.runtime());
shards.insert(handle.id(), id);
self.workers_alive.fetch_add(1, Ordering::Relaxed);
}
#[tracing::instrument(skip(self, pdu_id, user, pushkey), level = "debug")]
pub fn send_pdu_push(&self, pdu_id: &RawPduId, user: &UserId, pushkey: String) -> Result {
let dest = Destination::Push(user.to_owned(), pushkey);
@@ -480,10 +540,24 @@ impl Service {
sender.send(msg).map_err(|e| err!("{e}"))
}
/// Hands the oldest unsent request of each destination to its worker, so
/// a backlog with nothing in flight is sent without waiting for a new
/// event. Bypasses `dispatch` as these requests were already counted.
async fn dispatch_queued(&self, shard: Option<usize>) {
for (dest, (queue_id, event)) in self.db.queued_heads().await {
let id = self.shard_id(&dest);
if shard.is_none_or(|shard| shard == id) {
self.channels[id].0.send(Msg { dest, event, queue_id }).ok();
}
}
}
pub fn queue_depth(&self) -> usize {
self.channels.iter().map(|(sender, _)| sender.len()).sum()
}
pub fn workers_alive(&self) -> usize { self.workers_alive.load(Ordering::Relaxed) }
pub(super) fn shard_id(&self, dest: &Destination) -> usize {
if self.channels.len() <= 1 {
return 0;
@@ -519,3 +593,11 @@ fn num_senders(args: &crate::Args<'_>) -> usize {
worker_count.clamp(MIN_SENDERS, max_senders)
}
}
/// Delay before restarting a worker after its `failures`-th consecutive
/// failure, doubling each time.
fn restart_delay(failures: u32) -> Duration {
WORKER_RESTART_DELAY
.saturating_mul(2_u32.saturating_pow(failures.saturating_sub(1)))
.min(WORKER_RESTART_DELAY_MAX)
}
+56 -18
View File
@@ -1,6 +1,7 @@
use std::{
collections::{BTreeMap, HashMap, HashSet},
fmt::Debug,
panic::AssertUnwindSafe,
sync::{
Arc,
atomic::{AtomicU64, AtomicUsize, Ordering},
@@ -55,6 +56,9 @@ use serde_json::value::{RawValue as RawJsonValue, to_raw_value};
use super::{Destination, EduBuf, EduVec, Msg, SendingEvent, Service, data::QueueItem};
#[cfg(test)]
mod tests;
#[derive(Debug)]
enum TransactionStatus {
Running,
@@ -78,12 +82,18 @@ pub const EDU_LIMIT: usize = 100;
impl Service {
#[tracing::instrument(skip(self), level = "debug")]
pub(super) async fn sender(self: Arc<Self>, id: usize) -> Result {
pub(super) async fn sender(self: Arc<Self>, id: usize, restarted: bool) -> Result {
let mut statuses: CurTransactionStatus = CurTransactionStatus::new();
let mut futures: SendingFutures<'_> = FuturesUnordered::new();
// The previous worker of this shard may have dropped requests it had
// taken from the channel.
if restarted {
self.dispatch_queued(Some(id)).await;
}
info!(shard = id, "Running startup netburst");
self.startup_netburst(id, &mut futures, &mut statuses)
self.startup_netburst(id, restarted, &mut futures, &mut statuses)
.boxed()
.await;
info!(shard = id, "Finished startup netburst");
@@ -142,6 +152,9 @@ impl Service {
) {
match response {
| Ok(dest) => self.handle_response_ok(&dest, futures, statuses).await,
// A bug on our side, not the remote's fault: retry with backoff
// without marking the remote unhealthy.
| Err((dest, e)) if e.is_panic() => Self::handle_response_err(dest, statuses, &e),
| Err((dest, e)) => {
if let Destination::Federation(dest) = &dest {
if self.services.federation.should_mark_stale(&e)
@@ -274,7 +287,8 @@ impl Service {
}
/// Performs startup netburst, which sends any previously queued up but not
/// yet sent requests immediately.
/// yet sent requests immediately. A restarted worker always resends its
/// shard's requests in flight.
#[tracing::instrument(
name = "netburst",
level = "debug",
@@ -285,6 +299,7 @@ impl Service {
async fn startup_netburst<'a>(
&'a self,
id: usize,
restarted: bool,
futures: &mut SendingFutures<'a>,
statuses: &mut CurTransactionStatus,
) {
@@ -315,8 +330,12 @@ impl Service {
}
}
if !restarted && !self.server.config.startup_netburst {
return;
}
for (dest, events) in txns {
if self.server.config.startup_netburst && !events.is_empty() {
if !events.is_empty() {
statuses.insert(dest.clone(), TransactionStatus::Running);
futures.push(self.send_events(dest, events));
}
@@ -772,6 +791,7 @@ impl Service {
});
let start = Instant::now();
let panic_dest = dest.clone();
let send = match dest {
| Destination::Federation(server) =>
self.send_events_dest_federation(server, events).boxed(),
@@ -780,24 +800,42 @@ impl Service {
self.send_events_dest_push(user_id, pushkey, events).boxed(),
};
send.inspect(move |result| {
let instruments = &self.server.metrics.instruments;
let outcome =
KeyValue::new("outcome", if result.is_ok() { "success" } else { "failure" });
instruments
.sender_transaction_duration
.record(start.elapsed().as_secs_f64(), &[destination.clone(), outcome.clone()]);
for (kind, count) in [("pdu", pdus), ("edu", edus)] {
if count > 0 {
instruments.sender_events.add(count, &[
AssertUnwindSafe(send)
.catch_unwind()
.map(move |result| {
result.unwrap_or_else(|panic| {
let error = Error::from_panic(panic);
error!(dest = ?panic_dest, ?error, "Panicked while sending transaction");
self.server
.metrics
.instruments
.sender_panics
.add(1, &[KeyValue::new("scope", "transaction")]);
Err((panic_dest, Box::new(error)))
})
})
.inspect(move |result| {
let instruments = &self.server.metrics.instruments;
let outcome =
KeyValue::new("outcome", if result.is_ok() { "success" } else { "failure" });
instruments
.sender_transaction_duration
.record(start.elapsed().as_secs_f64(), &[
destination.clone(),
KeyValue::new("kind", kind),
outcome.clone(),
]);
for (kind, count) in [("pdu", pdus), ("edu", edus)] {
if count > 0 {
instruments.sender_events.add(count, &[
destination.clone(),
KeyValue::new("kind", kind),
outcome.clone(),
]);
}
}
}
})
.boxed()
})
.boxed()
}
#[tracing::instrument(
+86
View File
@@ -0,0 +1,86 @@
use std::iter::once;
use conduwuit::Error;
use futures::{StreamExt, stream::FuturesUnordered};
use ruma::server_name;
use serde_json::json;
use super::{CurTransactionStatus, TransactionStatus};
use crate::{
sending::{Destination, EduBuf, SendingEvent},
test_utils::{test_services, test_services_with},
};
#[tokio::test]
async fn panicked_transaction_is_kept_for_retry_without_blaming_the_remote() {
let (services, path, _database) = test_services().await;
let sending = &services.sending;
let server = server_name!("remote.example");
let dest = Destination::Federation(server.to_owned());
let event = SendingEvent::Edu(EduBuf::from_slice(br#"{"edu_type":"m.typing"}"#));
sending.db.queue_requests(once((&event, &dest)));
let in_flight: Vec<_> = sending.db.queued_requests(&dest).collect().await;
sending.db.mark_as_active(in_flight.iter());
let mut futures = FuturesUnordered::new();
let mut statuses = CurTransactionStatus::from([(dest.clone(), TransactionStatus::Running)]);
let panic = Error::from_panic(Box::new("boom"));
sending
.handle_response(Err((dest.clone(), Box::new(panic))), &mut futures, &mut statuses)
.await;
let active: Vec<_> = sending
.db
.active_requests_for(&dest)
.map(|(key, _)| key)
.collect()
.await;
let retry_after = sending.services.federation.retry_after(server);
let sending_next = futures.len();
let failed = matches!(statuses.get(&dest), Some(TransactionStatus::Failed(1, _)));
drop(futures);
drop(services);
std::fs::remove_dir_all(path).unwrap();
assert_eq!(
active,
in_flight
.into_iter()
.map(|(key, _)| key)
.collect::<Vec<_>>()
);
assert_eq!(sending_next, 0);
assert!(failed);
assert_eq!(retry_after, None);
}
#[tokio::test]
async fn restarted_worker_resends_trimmed_in_flight_requests() {
let (services, path, _database) =
test_services_with(json!({ "startup_netburst_keep": 1 })).await;
let sending = &services.sending;
let dest = Destination::Federation(server_name!("remote.example").to_owned());
let event = SendingEvent::Edu(EduBuf::from_slice(br#"{"edu_type":"m.typing"}"#));
for _ in 0..3 {
sending.db.queue_requests(once((&event, &dest)));
}
let queued: Vec<_> = sending.db.queued_requests(&dest).collect().await;
sending.db.mark_as_active(queued.iter());
let id = sending.shard_id(&dest);
let mut futures = FuturesUnordered::new();
let mut statuses = CurTransactionStatus::new();
sending
.startup_netburst(id, true, &mut futures, &mut statuses)
.await;
let active = sending.db.active_requests_for(&dest).count().await;
let sending_now = futures.len();
drop(futures);
drop(services);
std::fs::remove_dir_all(path).unwrap();
assert_eq!(active, 1);
assert_eq!(sending_now, 1);
}
+42
View File
@@ -0,0 +1,42 @@
use std::{iter::once, time::Duration};
use futures::StreamExt;
use ruma::server_name;
use super::{Destination, EduBuf, SendingEvent, restart_delay};
use crate::test_utils::test_services;
#[tokio::test]
async fn queued_heads_lists_the_oldest_unsent_request_per_destination() {
let (services, path, _database) = test_services().await;
let db = &services.sending.db;
let queued = Destination::Federation(server_name!("queued.example").to_owned());
let active = Destination::Federation(server_name!("active.example").to_owned());
let event = SendingEvent::Edu(EduBuf::from_slice(br#"{"edu_type":"m.typing"}"#));
let first = db.queue_requests(once((&event, &queued)));
db.queue_requests(once((&event, &queued)));
db.queue_requests(once((&event, &active)));
let in_flight: Vec<_> = db.queued_requests(&active).collect().await;
db.mark_as_active(in_flight.iter());
let heads: Vec<_> = db
.queued_heads()
.await
.into_iter()
.map(|(dest, (key, _))| (dest, key))
.collect();
drop(services);
std::fs::remove_dir_all(path).unwrap();
assert_eq!(heads, [(queued, first[0].clone())]);
}
#[test]
fn restart_delay_doubles_up_to_the_cap() {
assert_eq!(restart_delay(1), Duration::from_millis(2500));
assert_eq!(restart_delay(2), Duration::from_secs(5));
assert_eq!(restart_delay(4), Duration::from_secs(20));
assert_eq!(restart_delay(5), Duration::from_secs(30));
assert_eq!(restart_delay(u32::MAX), Duration::from_secs(30));
}
+111 -6
View File
@@ -5,30 +5,42 @@ mod sign;
mod util;
mod verify;
use std::{collections::BTreeMap, sync::Arc};
use std::{
collections::{BTreeMap, HashMap},
sync::Arc,
time::{Duration, Instant},
};
use assign::assign;
use async_trait::async_trait;
use conduwuit::{
Result, Server,
utils::{IterStream, ReadyExt, stream::TryIgnore},
Result, Server, SyncRwLock,
utils::{IterStream, ReadyExt, stream::TryIgnore, timepoint_from_now, to_canonical_object},
};
use database::{Deserialized, Ignore, Interfix, Json, Map};
use futures::{Stream, StreamExt};
pub use request::in_one_week;
use ruma::{
CanonicalJsonObject, MilliSecondsSinceUnixEpoch, OwnedServerSigningKeyId, ServerName,
ServerSigningKeyId,
api::federation::discovery::{ServerSigningKeys, VerifyKey},
CanonicalJsonObject, MilliSecondsSinceUnixEpoch, OwnedServerName, OwnedServerSigningKeyId,
ServerName, ServerSigningKeyId,
api::federation::discovery::{OldVerifyKey, ServerSigningKeys, VerifyKey},
room_version_rules::RoomVersionRules,
serde::Raw,
signatures::{Ed25519KeyPair, PublicKeyMap, PublicKeySet},
};
use serde_json::value::to_raw_value;
pub use verify::strip_extraneous_signatures;
use crate::{Dep, globals, sending, server_keys::util::required_keys};
const VALID_UNTIL_TS_DURATION: Duration = Duration::from_hours(12);
pub struct Service {
keypair: Box<Ed25519KeyPair>,
verify_keys: VerifyKeys,
services: Services,
db: Data,
last_lookup: SyncRwLock<HashMap<OwnedServerName, Instant>>,
}
struct Services {
@@ -45,6 +57,7 @@ pub type VerifyKeys = BTreeMap<OwnedServerSigningKeyId, VerifyKey>;
pub type PubKeyMap = PublicKeyMap;
pub type PubKeys = PublicKeySet;
#[async_trait]
impl crate::Service for Service {
fn build(args: crate::Args<'_>) -> Result<Arc<Self>> {
let (keypair, verify_keys) = keypair::init(args.db)?;
@@ -61,9 +74,12 @@ impl crate::Service for Service {
db: Data {
servernamekeyid_response: args.db["servernamekeyid_response"].clone(),
},
last_lookup: SyncRwLock::new(HashMap::new()),
}))
}
async fn clear_cache(&self) { self.last_lookup.write().clear(); }
fn name(&self) -> &str { crate::service::make_name(std::module_path!()) }
}
@@ -183,6 +199,8 @@ impl Service {
}
/// Returns the stored server signing keys responses for the origin.
///
/// Does not imply that the stored responses are still in-date.
pub fn signing_keys_for<'a>(
&'a self,
origin: &'a ServerName,
@@ -193,4 +211,91 @@ impl Service {
.ignore_err()
.map(|(_, v): (Ignore, ServerSigningKeys)| v)
}
/// Determines if the server may contact the origin server to fetch keys
/// when acting as a notary server. This limits origin lookups to once per
/// minute, which prevents amplification attacks.
#[must_use]
pub fn notary_may_contact_origin(&self, server_name: &ServerName) -> bool {
if server_name == self.services.server.name {
// We always intercept origin requests to us with
// build_server_keys_response, so this is equivalent to
// calling `/_matrix/key/v2/server`.
return true;
}
self.last_lookup
.read()
.get(server_name)
.is_none_or(|last| last.elapsed() >= Duration::from_mins(1))
}
/// Generates a signing key response to serve to `/_matrix/key/v2/server`.
pub async fn build_server_keys_response(&self) -> Result<Raw<ServerSigningKeys>> {
let verify_keys = BTreeMap::from([self.active_verify_key()]);
let old_verify_keys = match &self.services.server.config.old_verify_keys {
| Some(old_verify_keys) => old_verify_keys.to_owned(),
| None => self.discover_old_signing_keys().await,
};
let valid_until_ts = timepoint_from_now(VALID_UNTIL_TS_DURATION)
.map(|tp| {
MilliSecondsSinceUnixEpoch::from_system_time(tp)
.expect("key validity period must be before the heat death of the universe")
})
.expect("key validity period must be before the heat death of the universe");
let server_keys = assign!(
ServerSigningKeys::new(self.services.server.name.clone(), valid_until_ts),
{verify_keys, old_verify_keys}
);
let mut canonical_obj = to_canonical_object(&server_keys)?;
self.sign_json(&mut canonical_obj)
.and_then(|()| to_raw_value(&canonical_obj).map_err(Into::into))
.map(Raw::from_json)
}
async fn discover_old_signing_keys(&self) -> BTreeMap<OwnedServerSigningKeyId, OldVerifyKey> {
let now = MilliSecondsSinceUnixEpoch::now();
let mut keys = self.signing_keys_for(&self.services.server.name);
let mut old_keys = BTreeMap::new();
while let Some(resp) = keys.next().await {
// We need to verify that this response is still trusted based on
// the current configuration. It's possible we got this response
// from a notary who we no longer trust.
if resp.valid_until_ts > in_one_week() || resp.valid_until_ts < now {
continue;
}
let trusted = self
.services
.server
.config
.trusted_servers
.iter()
.any(|notary| {
Self::verify_server_keys_response(
&resp,
Some((notary.server_name(), notary.verify_keys())),
)
.is_ok()
});
if !trusted {
continue;
}
for (old_key_id, old_key) in resp.old_verify_keys {
if old_key.expired_ts > resp.valid_until_ts {
// This key expired in the future which is probably illegal
continue;
}
old_keys
.entry(old_key_id)
.and_modify(|current_old_key: &mut OldVerifyKey| {
// Use the lowest expiry timestamp
if current_old_key.expired_ts < old_key.expired_ts {
*current_old_key = old_key.clone();
}
})
.or_insert(old_key);
}
}
old_keys
}
}
+18 -1
View File
@@ -1,4 +1,4 @@
use std::collections::BTreeMap;
use std::{collections::BTreeMap, time::Instant};
use conduwuit::{Err, Result, trace, utils::millis_since_unix_epoch, warn};
use ruma::{
@@ -30,6 +30,23 @@ impl super::Service {
) -> Result<ServerSigningKeys> {
use get_server_keys::v2::Request;
if target == self.services.server.name {
return self
.build_server_keys_response()
.await
.and_then(|r| r.deserialize().map_err(Into::into));
}
// N.B. The "last lookup" is written before the request is actually made
// to prevent concurrent notary requests from spawning... concurrent
// origin requests, especially if the origin is slow.
// This has the downside that, if the origin is temporarily unreachable
// (including if we're backing off from it), the notary might take an
// additional minute to recover compared to the rest of the server. This
// is deemed acceptable.
self.last_lookup
.write()
.insert(target.clone(), Instant::now());
let server_signing_key = self
.services
.sending
+18 -19
View File
@@ -78,7 +78,7 @@ impl super::Service {
pubkey_map.insert(server_keys.server_name.to_string(), set);
let mut canonical = to_canonical_object(server_keys)?;
Self::strip_extraneous_signatures(&mut canonical, &server_keys.server_name, &[]);
strip_extraneous_signatures(&mut canonical, &server_keys.server_name, &[]);
ruma::signatures::verify_json(&pubkey_map, &canonical).map_err(Into::into)
}
@@ -100,13 +100,12 @@ impl super::Service {
)));
};
let mut canonical_object = to_canonical_object(notary_signatures)?;
Self::strip_extraneous_signatures(&mut canonical_object, &server_keys.server_name, &[
strip_extraneous_signatures(&mut canonical_object, &server_keys.server_name, &[
notary_name,
]);
let for_verify = ruma::signatures::to_canonical_json_string_for_signing(
&to_canonical_object(server_keys).expect("server keys object must be canonical json"),
)
.expect("canonical JSON must be stringable");
let for_verify =
ruma::signatures::to_canonical_json_string_for_signing(&canonical_object)
.expect("canonical JSON must be stringable");
let raw_notary_keys = notary_keys.iter().map(Base64::as_bytes).collect::<Vec<_>>();
let raw_notary_signatures = notary_signatures
@@ -137,22 +136,22 @@ impl super::Service {
"No valid signature from {notary_name} present on signing keys response"
)))
}
}
fn strip_extraneous_signatures(
canonical: &mut CanonicalJsonObject,
origin: &ServerName,
notaries: &[&ServerName],
) {
canonical.entry("signatures".to_owned()).and_modify(|sigs| {
sigs.as_object_mut().map(|s| {
s.retain(|server_name, _| {
let Ok(server_name) = ServerName::parse(server_name) else { return false };
origin == server_name || notaries.iter().any(|ns| *ns == server_name)
});
Some(s)
pub fn strip_extraneous_signatures(
canonical: &mut CanonicalJsonObject,
origin: &ServerName,
notaries: &[&ServerName],
) {
canonical.entry("signatures".to_owned()).and_modify(|sigs| {
sigs.as_object_mut().map(|s| {
s.retain(|server_name, _| {
let Ok(server_name) = ServerName::parse(server_name) else { return false };
origin == server_name || notaries.iter().any(|ns| *ns == server_name)
});
Some(s)
});
}
});
}
#[cfg(test)]
+52
View File
@@ -0,0 +1,52 @@
use std::{path::PathBuf, sync::Arc};
use conduwuit::{
Config, Server,
log::{Log, LogLevelReloadHandles},
metrics::Metrics,
};
use serde_json::json;
use tokio::sync::{Mutex, MutexGuard};
/// RocksDB's default `Env` is shared by the whole process and closing a
/// database stops its background threads, so tests must not keep databases
/// open in parallel. Hold the returned guard until the services are dropped.
static DATABASE: Mutex<()> = Mutex::const_new(());
type TestServices = (Arc<crate::Services>, PathBuf, MutexGuard<'static, ()>);
pub(crate) async fn test_services() -> TestServices { test_services_with(json!({})).await }
pub(crate) async fn test_services_with(overrides: serde_json::Value) -> TestServices {
let guard = DATABASE.lock().await;
let _ = rustls::crypto::ring::default_provider().install_default();
let path = std::env::temp_dir()
.join(format!("continuwuity-shared-state-{:016x}", rand::random::<u64>()));
std::fs::create_dir(&path).unwrap();
let mut config = json!({
"server_name": "test.example",
"database_path": path,
"db_cache_capacity_mb": 16.0,
"rocksdb_write_buffer_capacity_mb": 4.0,
"allow_local_presence": false,
"dns_servers": ["127.0.0.1"],
"allow_invalid_tls_certificates_yes_i_know_what_the_fuck_i_am_doing_with_this_and_i_know_this_is_insecure": true
});
config
.as_object_mut()
.unwrap()
.extend(overrides.as_object().unwrap().clone());
let config: Config = serde_json::from_value(config).unwrap();
let runtime = tokio::runtime::Handle::current();
let server = Arc::new(Server::new(
config,
Some(runtime.clone()),
Log {
reload: LogLevelReloadHandles::default(),
capture: Arc::default(),
},
Metrics::new(Some(runtime), opentelemetry::global::meter("test"), None),
));
let services = crate::Services::build(server).await.unwrap();
(services, path, guard)
}
+26 -2
View File
@@ -4,7 +4,9 @@ use std::{
};
use conduwuit::{
Err, Result, debug, debug_error, debug_warn, err, error, info, trace,
Err, Result,
config::experiments::ToggledExperiment,
debug, debug_error, debug_warn, err, error, info, trace,
utils::{self, stream::TryIgnore},
warn,
};
@@ -107,6 +109,27 @@ impl super::Service {
if stable.is_err() && unstable.is_err() {
return FilterLevel::Default;
}
let msc4494_enabled = self.services.config.experiments.msc4494.is_enabled();
if !self.services.config.experiments.msc4155.is_enabled() {
// Only MSC4380 should be checked here, since it's in spec.
let Some(action) = stable.ok().and_then(|c| c.content.default_action) else {
return FilterLevel::Default;
};
return match action {
| InvitePermissionAction::Block => FilterLevel::Block,
| InvitePermissionAction::DenyPublic if msc4494_enabled => {
if !self
.check_msc4494_mutual_room(sender_user, recipient_user)
.await
{
FilterLevel::Block
} else {
FilterLevel::Default
}
},
| _ => FilterLevel::Allow,
};
}
let content = stable.unwrap_or_else(|_| unstable.unwrap()).content;
let verdict = content.user_filter_level(sender_user);
if verdict != FilterLevel::Default {
@@ -114,7 +137,8 @@ impl super::Service {
}
// Apply other layers of invite blocking first, so that user/server
// denies take priority over implicit allows.
if content.default_action == Some(InvitePermissionAction::DenyPublic)
if msc4494_enabled
&& content.default_action == Some(InvitePermissionAction::DenyPublic)
&& !self
.check_msc4494_mutual_room(sender_user, recipient_user)
.await