Compare commits

...
Author SHA1 Message Date
Jade Ellis b6bc7dfc16 feat: Debounce extremity squashing
Additionaly circuit-breaks it if the squash would have only
b triggered by dummy events / other squashes
2026-06-28 01:29:52 +01:00
Jade Ellis 945ea5a78a chore: Box large futures 2026-06-28 01:29:52 +01:00
timedout d719fe2048 chore: Merge origin/main into nex/perf/get-missing-events 2026-06-28 01:29:23 +01:00
timedout 458e0b53ac style: Reformat 2026-06-27 18:41:43 +01:00
timedout 2d65d47498 fix: Load bearing mut refs 2026-06-27 18:37:33 +01:00
timedout e4922814b9 refactor: Add better documentation to sender service, add additional logging traces
This is a surprise tool that will help us later!
2026-06-27 18:37:33 +01:00
timedout 064028689c style: Add documentation to execute module of sender service 2026-06-27 18:37:33 +01:00
timedout 124241238e refactor: Rename "synapse"-related federation methods to "slow"
I won't even comment on how stupid that naming was.
2026-06-27 18:37:33 +01:00
timedout 967e2cc54f refactor: Split the users service into several modules
Also resolvers several lints, including needless async on create_user
2026-06-27 18:37:33 +01:00
timedout 6fdeadc356 style: Refactor admin crate, drop admin_command macro
`admin_command` depended on `implement`, which is now gone.
2026-06-27 18:37:26 +01:00
timedout 478560daae style: Remove now dead code 2026-06-27 18:36:44 +01:00
timedout 0310b4b2c5 style: Remove implement entirely 2026-06-27 18:36:44 +01:00
timedout 3eb4257ae8 style: Remove remaining implement imports 2026-06-27 18:36:44 +01:00
timedout f1d3db0ea3 style: Eliminate implement 2026-06-27 18:36:44 +01:00
timedout e7b302955f style: Eliminate implement database 2026-06-27 18:36:44 +01:00
timedout 85122c62cd style: Eliminate implement in database map keys 2026-06-27 18:36:44 +01:00
timedout ba0ffee7cf fix: Style errors and compile problems 2026-06-27 18:36:44 +01:00
timedout 86e1101b40 style: Remove implement for database pool 2026-06-27 18:36:44 +01:00
timedout fdcb7e3957 style: Remove implement for config Manager 2026-06-27 18:36:44 +01:00
timedout c54e56a73e style: Remove implement for AccessCheck, rename check to assert 2026-06-27 18:36:44 +01:00
timedout fed9a599d9 style: Refactor remaining parts of admin service 2026-06-27 18:36:44 +01:00
timedout 0bb97a53c5 style: Refactor account_data service 2026-06-27 18:36:44 +01:00
timedout 9b76c9a085 feat: Support arbitrary room versions while creating the admin room
Also changes the default power levels so people stop shooting their feet off
2026-06-27 18:36:44 +01:00
timedout 6f20b7bc3f style: Refactor config service 2026-06-27 18:36:44 +01:00
timedout 580d9b0318 style: Refactor federation service 2026-06-27 18:36:44 +01:00
timedout ae572ec108 style: Refactor key_backups service 2026-06-27 18:36:44 +01:00
timedout b1c5b3d000 style: Refactor media service 2026-06-27 18:36:44 +01:00
timedout 0c9230f35e style: Document & refactor resolver service
good riddance to this thing
2026-06-27 18:36:44 +01:00
timedout 77dab8fd20 style: Document & refactor rooms/alias service 2026-06-27 18:36:44 +01:00
timedout 10b5ca8e3a style: Document & refactor rooms/directory service 2026-06-27 18:36:44 +01:00
timedout f9eed42cb7 style: Document & refactor rooms/directory service 2026-06-27 18:36:44 +01:00
timedout 19f4c24c00 style: Document & refactor rooms/event_handler service 2026-06-27 18:36:44 +01:00
timedout 0a162b7bc4 style: Document & refactor rooms/lazy_loading service 2026-06-27 18:36:44 +01:00
timedout 0d1b130127 style: Document & refactor rooms/metadata service 2026-06-27 18:36:44 +01:00
timedout 5e0e82e7e8 style: Document & refactor rooms/outlier service 2026-06-27 18:36:44 +01:00
timedout 4bf0816bcc style: Document & refactor rooms/search service 2026-06-27 18:36:44 +01:00
timedout ecbe035b36 style: Document & refactor rooms/short service 2026-06-27 18:36:44 +01:00
timedout 4d30ee5d33 style: Document & refactor rooms/state_accessor service 2026-06-27 18:36:44 +01:00
timedout 4844c6ae3e style: Document & refactor rooms/state_cache service 2026-06-27 18:36:44 +01:00
timedout 6603798d60 style: Document & refactor rooms/state_compressor service 2026-06-27 18:36:44 +01:00
timedout 199f6eb67b style: Document & refactor rooms/user and rooms/timeline services 2026-06-27 18:36:44 +01:00
timedout aa2f506996 style: Document & refactor sending service 2026-06-27 18:36:44 +01:00
timedout a3994fe1a3 style: Document & refactor server_keys service 2026-06-27 18:36:44 +01:00
timedout 6312ff8088 style: Document & refactor sync service 2026-06-27 18:36:44 +01:00
timedout 17cff1fb36 style: Document & refactor users service 2026-06-27 18:36:44 +01:00
timedout aff6315ae1 style: Document moderation service 2026-06-27 18:36:44 +01:00
timedout 574d3e9164 style: Use impl instead of implement in moderation service 2026-06-27 18:36:44 +01:00
31a05b9c 21e93f8feb chore: changelog 2026-06-27 01:05:53 +01:00
31a05b9c 4b2f032ae6 feat: get-state-at admin command 2026-06-27 01:05:48 +01:00
Jade Ellis 8a495a7d7f fix(ci): Correct remote ref fetch 2026-06-27 01:00:33 +01:00
Jade Ellis cee51d5717 fix(ci): Correct remote ref fetch 2026-06-27 00:57:45 +01:00
Jade Ellis 0483d3e155 ci: Don't checkout repo in pull_request_target 2026-06-27 00:54:31 +01:00
timedout 2a4b9f9323 fix: Check are_all_blocked in server_filter_level 2026-06-24 20:13:17 +01:00
timedout 63a1a3d9ab feat: Advertise stable prefix in unstable features 2026-06-24 20:05:34 +01:00
timedout f19f5c71a5 chore(1875): Remove extra changelog entry 2026-06-24 19:13:35 +01:00
timedout b009ffd31c chore(1875): Add changelogs 2026-06-24 19:03:38 +01:00
timedout b6baf66399 feat: Support MSC4380 invite blocking
fix: Re-add silently dropped support for MSC4155
2026-06-24 18:59:49 +01:00
ginger 6dac46e1ad Update SECURITY.md 2026-06-24 13:00:27 +00:00
ginger 4d40e20fdd docs: Fix typo 2026-06-23 14:56:14 +00:00
timedout 3b6858e936 perf: Throttle dummy events to prevent stampeding 2026-06-23 15:26:28 +01:00
Renovate Bot 3b455218f5 chore(deps): update rust crate itertools to 0.15.0 2026-06-23 13:53:25 +00:00
timedout 7ca00e4ab9 chore: Drop unused param from handle_outlier_pdu 2026-06-20 17:08:00 +01:00
timedout b8ca06029f perf: Use a hashmap for full-state filtering 2026-06-20 17:08:00 +01:00
timedout 2efe8f2ec0 fix: Inverted power level check in extremity squash 2026-06-20 17:08:00 +01:00
timedout d1aa911739 fix: Rename variables in auth event fetcher
Also fixes a couple bugs where events were being misattributed
2026-06-20 17:08:00 +01:00
timedout 4673282ca1 perf: Don't try to fetch prevs we already fetched
graphs are hard
2026-06-20 17:08:00 +01:00
timedout 0c03195aec fix: Fall back to legacy behaviour when prev events are missed from get_missing_events 2026-06-20 17:08:00 +01:00
timedout 439bc2784d style: Use more explicit variable names 2026-06-20 17:08:00 +01:00
timedout 689a1ce59b style: Use user_can_send_message 2026-06-20 17:08:00 +01:00
timedout c141503ccb style: Re-use GET_MISSING_EVENTS_MAX_BATCH_SIZE 2026-06-20 17:08:00 +01:00
timedout bcadecdc3b feat: Add !admin debug rooms-by-extremity-count command 2026-06-20 17:08:00 +01:00
timedout ab3be337cb chore: Reformat 2026-06-20 17:08:00 +01:00
timedout aa1281a9e0 fix: Prevent arbitrary state injection attack 2026-06-20 17:08:00 +01:00
timedout 115a2e802e style: Check power levels before attempting to send extremity squashes
Solves a problem where the console screams in agony when local users can't send dummy events
2026-06-20 17:08:00 +01:00
timedout 968328d788 perf: Squash weird mutable variable 2026-06-20 17:08:00 +01:00
timedout f1cde5f323 style: Fix up some TODOs 2026-06-20 17:08:00 +01:00
timedout 832ee8650b style: Adjust docstrings and dodgy comment 2026-06-20 17:08:00 +01:00
timedout c0666a1793 fix: Default PDU content to empty object instead of literal NULL 2026-06-20 17:08:00 +01:00
timedout ddb4ef539f fix: un-forget how streams work 2026-06-20 17:08:00 +01:00
timedout 3faedc4581 perf: Remove huge clone and tackle TODOs 2026-06-20 17:08:00 +01:00
timedout a3544353ba feat: Automatically squash extremities when they exceed a threshold
Attempts to tackle #1844
2026-06-20 17:08:00 +01:00
timedout b0612397d3 style: Tidy up 2026-06-20 17:08:00 +01:00
timedout 31737e127e fix: Make fetch_state_ids_from_backfill_servers candidate-free safe 2026-06-20 17:08:00 +01:00
timedout 486dcd208c style: Resolve lint complaints 2026-06-20 17:08:00 +01:00
timedout a945a4b2ad fix: Correctly handle still-missing state, always fetch full state atomically if regular fetch fails 2026-06-20 17:08:00 +01:00
timedout c28ea44e11 fix: Correct inverted boolean condition, add explicit timeout on /state fetch 2026-06-20 17:08:00 +01:00
timedout 3e4d6b2565 perf: Always fetch at least N events per GME 2026-06-20 17:08:00 +01:00
timedout 29ce21cd2e fix: Correctly pre-populate state events vec with known events 2026-06-20 17:08:00 +01:00
timedout d461c6977a fix: Friendly assertations 2026-06-20 17:08:00 +01:00
timedout 5a0d6461d1 perf: Don't try to re-persist non-outliers we already have 2026-06-20 17:08:00 +01:00
timedout 2ec7394785 perf: Don't add trees we already have to latest boundary 2026-06-20 17:08:00 +01:00
timedout 8a5708b9f9 fix: Be noisy when there's no incoming state 2026-06-20 17:08:00 +01:00
timedout cd46070bd3 fix: Elide auth chain from fetch_and_handle_outliers 2026-06-20 17:08:00 +01:00
timedout 9930e549a0 fix: Progress log in fetch_prev 2026-06-20 17:08:00 +01:00
timedout 36ed20cb04 fix: Downgrade safe assert to debug assert 2026-06-20 17:08:00 +01:00
timedout 3907589b6c fix: Don't download the world 2026-06-20 17:08:00 +01:00
timedout 2a598af888 feat: Make logging more verbose to diagnose the aranjesplosion 2026-06-20 17:08:00 +01:00
timedout f34f84832b feat: Include timing information in debug logs 2026-06-20 17:08:00 +01:00
timedout 0efa6ed1f2 fix: Don't treat prev outlier upgrades as fetch failures 2026-06-20 17:08:00 +01:00
timedout 2bf5876778 fix: Ask more servers for state_ids when origin fails to provide
Some servers reference events in prev_events that they might not yet have finished processing, so this allows us to at least attempt to get the state from another trustworthy server in the room that might be faster. I don't think this is too effective, however it's more effective than giving up immediately.
2026-06-20 17:08:00 +01:00
timedout 31982e84de fix: Remove redundant check 2
This may look scary, but this is safe because event auth performs the same check, and will reject the event if it doesn't reference the create event correctly.
2026-06-20 17:08:00 +01:00
timedout 970958652a fix: Remove redundant check that accidentally banned everyone 2026-06-20 17:08:00 +01:00
timedout c94a395bf0 fix: Make PDU handle errors noisier & correct error types 2026-06-20 17:08:00 +01:00
timedout 43adff926f fix: Make dedupe noisy, don't allow non-create event as create event 2026-06-20 17:07:59 +01:00
timedout 04fce56381 fix: Don't silence PDU handle logs 2026-06-20 17:07:59 +01:00
timedout 4df2097e6c style: Rename gapfill helpers instruments 2026-06-20 17:07:59 +01:00
timedout 8b85b04d10 fix: Properly remove event_id from the PDU JSON before upgrading it 2026-06-20 17:07:59 +01:00
timedout 9e0bcd3be8 fix: Hold a federation room lock while remotely joining a room 2026-06-20 17:07:59 +01:00
timedout 9509080e0d fix: Replace our local extremity tracking when joining a disconnected room remotely 2026-06-20 17:07:59 +01:00
timedout 61066bb0c6 fix: Don't try and fetch zero events 2026-06-20 17:07:59 +01:00
timedout 573d5bc50e fix: Fall back to atomic fetch when full-state fetch fails 2026-06-20 17:07:59 +01:00
timedout 162e6eb92f fix: Remove short-term memory loss
I keep writing forgetful code, it's a problem
2026-06-20 17:07:59 +01:00
timedout cef4ebe38e fix: Don't try to fetch the same event endlessly 2026-06-20 17:07:59 +01:00
timedout 316a0b7d58 fix: Don't repeat already-included metadata in fetch_state instrument 2026-06-20 17:07:59 +01:00
timedout 2bcc56704b feat: Enhance reliability by fetching full state when we're missing a lot of auth events 2026-06-20 17:07:59 +01:00
timedout 7f64de9727 fix: Calculate max iterations dynamically, and bump max prevs 2026-06-20 17:07:59 +01:00
timedout aea03f2f99 perf(wip): Improve individual events fetcher 2026-06-20 17:07:59 +01:00
timedout 8edf9552b8 fix: Don't lie about using already-known content 2026-06-20 17:07:59 +01:00
timedout d5f69c8a31 fix: Be smarter when re-receiving already-seen PDUs 2026-06-20 17:07:59 +01:00
timedout 9ea9b0e04c perf: Don't re-process events as outliers 2026-06-20 17:07:59 +01:00
timedout e8db01fc8d style: Improve logging 2026-06-20 17:07:59 +01:00
timedout f80e1e89a5 fix: Lower floor for min depth 2026-06-20 17:07:59 +01:00
timedout b9dca84acf fix: Only increment mindepth on state events 2026-06-20 17:07:59 +01:00
timedout e3ec1066c4 chore: Add newsfrag 2026-06-20 17:07:59 +01:00
timedout 1445a8d446 feat: Keep track of a min_depth value
Should prevent weird situations where we accidentally gapfill into backfill territory
2026-06-20 17:07:59 +01:00
timedout 9547c438d6 perf: Increase default max_fetch_prev_events to 256 2026-06-20 17:07:59 +01:00
timedout 51d0e615f5 perf: Make max gap depth fetch configurable 2026-06-20 17:07:59 +01:00
timedout eeb937416c perf: Improve gap filling, handle missing auth events better 2026-06-20 17:07:59 +01:00
timedout 0d5aa7ede1 fix: This is some bullshit I tell you 2026-06-20 17:07:59 +01:00
timedout ba9dc27773 feat: Better prev event fetching
fix: Don't panic in debug mode when making an empty notary query
2026-06-20 17:07:59 +01:00
timedout abf5a155ba feat: Add backfill_missing_events helper 2026-06-20 17:07:59 +01:00
151 changed files with 17365 additions and 16041 deletions

No files matched your search

+7 -11
View File
@@ -14,23 +14,19 @@ jobs:
name: Check changelog is added
runs-on: ubuntu-latest
steps:
- name: Checkout repository
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
with:
ref: ${{ github.event.pull_request.head.sha }}
fetch-depth: 0
persist-credentials: false
sparse-checkout: .
- name: Check for changelog entry
id: check_files
run: |
git fetch origin ${GITHUB_BASE_REF}
AUTH=$(echo -n "x-access-token:${{ secrets.GITHUB_TOKEN }}" | base64 -w 0)
git config --global http.${{ github.server_url }}/.extraheader "Authorization: basic $AUTH"
git clone "${{ github.event.repository.clone_url }}" repo.git --bare
git -C repo.git fetch origin pull/${{ github.event.pull_request.number }}/head
# Check for Added (A) or Modified (M) files in changelog.d
CHANGELOG_CHANGES=$(git diff --name-status origin/${GITHUB_BASE_REF}...HEAD -- changelog.d/)
CHANGELOG_CHANGES=$(git -C repo.git diff --name-status ${{ github.event.pull_request.base.sha }}...${{ github.event.pull_request.head.sha }} -- changelog.d/)
SRC_CHANGES=$(git diff --name-status origin/${GITHUB_BASE_REF}...HEAD -- src/)
SRC_CHANGES=$(git -C repo.git diff --name-status ${{ github.event.pull_request.base.sha }}...${{ github.event.pull_request.head.sha }} -- src/)
echo "Changes in changelog.d/:"
echo "$CHANGELOG_CHANGES"
+1
View File
@@ -23,6 +23,7 @@ We appreciate the efforts of security researchers and the community in identifyi
1. **Contact members of the team directly** over E2EE private message.
- [@jade:ellis.link](https://matrix.to/#/@jade:ellis.link)
- [@nex:nexy7574.co.uk](https://matrix.to/#/@nex:nexy7574.co.uk)
- [@ginger:gingershaped.computer](https://matrix.to/#/@ginger:gingershaped.computer)
2. **Email the security team** at [security@continuwuity.org](mailto:security@continuwuity.org). This is not E2EE, so don't include sensitive details.
3. **Do not disclose the vulnerability publicly** until it has been addressed
4. **Provide detailed information** about the vulnerability, including:
+2
View File
@@ -0,0 +1,2 @@
Improved the performance and reliability of fetching missing events, improving network partition recovery. Contributed
by @nex.
+1
View File
@@ -0,0 +1 @@
Added support for MSC4380 invite blocking, which has become part of the Matrix specification in v1.18. Contributed by @nex.
+1
View File
@@ -0,0 +1 @@
Added `!admin debug get-state-at` command
+9 -1
View File
@@ -297,7 +297,7 @@
# This item is undocumented. Please contribute documentation for it.
#
#max_fetch_prev_events = 192
#max_fetch_prev_events = 1024
# How many incoming federation transactions the server is willing to be
# processing at any given time before it becomes overloaded and starts
@@ -645,6 +645,14 @@
#
#default_room_acl_deny =
# The number of forward extremities to tolerate in a room before
# attempting to manually squash them with a "dummy event". Setting this
# above 20 will hinder its efficacy, and setting it below 5 will cause
# more dummy events to be sent than necessary (which increases federation
# traffic).
#
#dummy_event_threshold = 10
# Enable OpenTelemetry OTLP tracing export. This replaces the deprecated
# Jaeger exporter. Traces will be sent via OTLP to a collector (such as
# Jaeger) that supports the OpenTelemetry Protocol.
+1 -1
View File
@@ -10,7 +10,7 @@ Please check that your issues are not due to problems with your Docker setup.
### Slow joins to rooms
Some slowness is to be expected if you're the first person on your homserver to join a room (which will
Some slowness is to be expected if you're the first person on your homeserver to join a room (which will
always be the case for single-user homeservers). In this situation, your homeserver has to verify the signatures of
all of the state events sent by other servers before your join. To make this process as fast as possible, make sure you have
multiple fast, trusted servers listed in `trusted_servers` in your configuration, and ensure
+59 -63
View File
@@ -1,80 +1,76 @@
use conduwuit::{Err, Result, checked};
use futures::{FutureExt, StreamExt, TryFutureExt};
use crate::admin_command;
impl crate::Context<'_> {
pub(super) async fn register(&self) -> Result {
let body = &self.body;
let body_len = self.body.len();
if body_len < 2
|| !body[0].trim().starts_with("```")
|| body.last().unwrap_or(&"").trim() != "```"
{
return Err!("Expected code block in command body. Add --help for details.");
}
#[admin_command]
pub(super) async fn register(&self) -> Result {
let body = &self.body;
let body_len = self.body.len();
if body_len < 2
|| !body[0].trim().starts_with("```")
|| body.last().unwrap_or(&"").trim() != "```"
{
return Err!("Expected code block in command body. Add --help for details.");
let range = 1..checked!(body_len - 1)?;
let appservice_config_body = body[range].join("\n");
let parsed_config = serde_saphyr::from_str(&appservice_config_body);
match parsed_config {
| Err(e) => return Err!("Could not parse appservice config as YAML: {e}"),
| Ok(registration) => match self
.services
.appservice
.register_appservice(&registration, &appservice_config_body)
.await
.map(|()| registration.id)
{
| Err(e) => return Err!("Failed to register appservice: {e}"),
| Ok(id) => write!(self, "Appservice registered with ID: {id}"),
},
}
.await
}
let range = 1..checked!(body_len - 1)?;
let appservice_config_body = body[range].join("\n");
let parsed_config = serde_saphyr::from_str(&appservice_config_body);
match parsed_config {
| Err(e) => return Err!("Could not parse appservice config as YAML: {e}"),
| Ok(registration) => match self
pub(super) async fn unregister(&self, appservice_identifier: String) -> Result {
match self
.services
.appservice
.register_appservice(&registration, &appservice_config_body)
.unregister_appservice(&appservice_identifier)
.await
.map(|()| registration.id)
{
| Err(e) => return Err!("Failed to register appservice: {e}"),
| Ok(id) => write!(self, "Appservice registered with ID: {id}"),
},
}
.await
}
#[admin_command]
pub(super) async fn unregister(&self, appservice_identifier: String) -> Result {
match self
.services
.appservice
.unregister_appservice(&appservice_identifier)
| Err(e) => return Err!("Failed to unregister appservice: {e}"),
| Ok(()) => write!(self, "Appservice unregistered."),
}
.await
{
| Err(e) => return Err!("Failed to unregister appservice: {e}"),
| Ok(()) => write!(self, "Appservice unregistered."),
}
.await
}
#[admin_command]
pub(super) async fn show_appservice_config(&self, appservice_identifier: String) -> Result {
match self
.services
.appservice
.get_registration(&appservice_identifier)
pub(super) async fn show_appservice_config(&self, appservice_identifier: String) -> Result {
match self
.services
.appservice
.get_registration(&appservice_identifier)
.await
{
| None => return Err!("Appservice does not exist."),
| Some(config) => {
let config_str = serde_saphyr::to_string(&config)?;
write!(self, "Config for {appservice_identifier}:\n\n```yaml\n{config_str}\n```")
},
}
.await
{
| None => return Err!("Appservice does not exist."),
| Some(config) => {
let config_str = serde_saphyr::to_string(&config)?;
write!(self, "Config for {appservice_identifier}:\n\n```yaml\n{config_str}\n```")
},
}
.await
}
#[admin_command]
pub(super) async fn list_registered(&self) -> Result {
self.services
.appservice
.iter_ids()
.collect()
.map(Ok)
.and_then(|appservices: Vec<_>| {
let len = appservices.len();
let list = appservices.join(", ");
write!(self, "Appservices ({len}): {list}")
})
.await
pub(super) async fn list_registered(&self) -> Result {
self.services
.appservice
.iter_ids()
.collect()
.map(Ok)
.and_then(|appservices: Vec<_>| {
let len = appservices.len();
let list = appservices.join(", ");
write!(self, "Appservices ({len}): {list}")
})
.await
}
}
+15 -15
View File
@@ -1,23 +1,23 @@
use conduwuit::Result;
use conduwuit_macros::implement;
use futures::StreamExt;
use crate::Context;
#[implement(Context, params = "<'_>")]
pub(super) async fn check_all_users(&self) -> Result {
let timer = tokio::time::Instant::now();
let users = self.services.users.stream().collect::<Vec<_>>().await;
let query_time = timer.elapsed();
impl Context<'_> {
pub(super) async fn check_all_users(&self) -> Result {
let timer = tokio::time::Instant::now();
let users = self.services.users.stream().collect::<Vec<_>>().await;
let query_time = timer.elapsed();
let total = users.len();
let err_count = users.iter().filter(|_user| false).count();
let ok_count = users.iter().filter(|_user| true).count();
let total = users.len();
let err_count = users.iter().filter(|_user| false).count();
let ok_count = users.iter().filter(|_user| true).count();
self.write_str(&format!(
"Database query completed in {query_time:?}:\n\n```\nTotal entries: \
{total:?}\nFailure/Invalid user count: {err_count:?}\nSuccess/Valid user count: \
{ok_count:?}\n```"
))
.await
self.write_str(&format!(
"Database query completed in {query_time:?}:\n\n```\nTotal entries: \
{total:?}\nFailure/Invalid user count: {err_count:?}\nSuccess/Valid user count: \
{ok_count:?}\n```"
))
.await
}
}
+1041 -958
View File
File diff suppressed because it is too large. Load diff
+13
View File
@@ -95,6 +95,14 @@ pub enum DebugCommand {
room_id: OwnedRoomOrAliasId,
},
/// Gets all the room state events at the specified event.
///
/// State at event might not be available for some PDUs, such as rejected
/// ones.
GetStateAt {
event_id: OwnedEventId,
},
/// Get and display signing keys from local cache or remote server.
GetSigningKeys {
server_name: Option<OwnedServerName>,
@@ -237,6 +245,11 @@ pub enum DebugCommand {
/// Send a test email to the invoking admin's email address
SendTestEmail,
/// Lists room IDs by forward extremity count in descending order
RoomsByExtremityCount {
page: Option<usize>,
},
/// Developer test stubs
#[command(subcommand)]
#[allow(non_snake_case)]
+23 -28
View File
@@ -1,6 +1,6 @@
use conduwuit::{Err, Result};
use crate::{admin_command, admin_command_dispatch};
use crate::admin_command_dispatch;
#[admin_command_dispatch]
#[derive(Debug, clap::Subcommand)]
@@ -11,37 +11,32 @@ pub enum TesterCommand {
Timer,
}
#[rustfmt::skip]
#[admin_command]
async fn panic(&self) -> Result {
impl crate::Context<'_> {
#[rustfmt::skip]
async fn panic(&self) -> Result {
panic!("panicked")
}
panic!("panicked")
}
#[rustfmt::skip]
async fn failure(&self) -> Result {
Err!("failed")
}
#[rustfmt::skip]
#[admin_command]
async fn failure(&self) -> Result {
#[inline(never)]
#[rustfmt::skip]
async fn tester(&self) -> Result {
self.write_str("Ok").await
}
Err!("failed")
}
#[inline(never)]
#[rustfmt::skip]
async fn timer(&self) -> Result {
let started = std::time::Instant::now();
timed(self.body);
#[inline(never)]
#[rustfmt::skip]
#[admin_command]
async fn tester(&self) -> Result {
self.write_str("Ok").await
}
#[inline(never)]
#[rustfmt::skip]
#[admin_command]
async fn timer(&self) -> Result {
let started = std::time::Instant::now();
timed(self.body);
let elapsed = started.elapsed();
self.write_str(&format!("completed in {elapsed:#?}")).await
let elapsed = started.elapsed();
self.write_str(&format!("completed in {elapsed:#?}")).await
}
}
#[inline(never)]
+116 -112
View File
@@ -4,127 +4,131 @@ use conduwuit::{Err, Result, utils::response::LimitReadExt};
use futures::StreamExt;
use ruma::{OwnedRoomId, OwnedServerName, OwnedUserId};
use crate::{admin_command, get_room_info};
use crate::get_room_info;
#[admin_command]
pub(super) async fn disable_room(&self, room_id: OwnedRoomId) -> Result {
self.bail_restricted()?;
self.services.rooms.metadata.disable_room(&room_id, true);
self.write_str("Room disabled.").await
}
impl crate::Context<'_> {
pub(super) async fn disable_room(&self, room_id: OwnedRoomId) -> Result {
self.bail_restricted()?;
self.services.rooms.metadata.disable_room(&room_id, true);
self.write_str("Room disabled.").await
}
#[admin_command]
pub(super) async fn enable_room(&self, room_id: OwnedRoomId) -> Result {
self.bail_restricted()?;
self.services.rooms.metadata.disable_room(&room_id, false);
self.write_str("Room enabled.").await
}
pub(super) async fn enable_room(&self, room_id: OwnedRoomId) -> Result {
self.bail_restricted()?;
self.services.rooms.metadata.disable_room(&room_id, false);
self.write_str("Room enabled.").await
}
#[admin_command]
pub(super) async fn incoming_federation(&self) -> Result {
let msg = {
let map = self
pub(super) async fn incoming_federation(&self) -> Result {
let msg = {
let map = self
.services
.rooms
.event_handler
.federation_handletime
.read();
let mut msg = format!(
"Handling {} incoming PDUs across {} active transactions:\n",
map.len(),
self.services.transactions.txn_active_handle_count()
);
for (r, (e, i)) in map.iter() {
let elapsed = i.elapsed();
writeln!(
msg,
"{} {}: {}m{}s",
r,
e,
elapsed.as_secs() / 60,
elapsed.as_secs() % 60
)?;
}
msg
};
self.write_str(&msg).await
}
pub(super) async fn fetch_support_well_known(&self, server_name: OwnedServerName) -> Result {
let response = self
.services
.client
.default
.get(format!("https://{server_name}/.well-known/matrix/support"))
.send()
.await?;
let text = response
.limit_read_text(
self.services
.config
.max_request_size
.try_into()
.expect("u64 fits into usize"),
)
.await?;
if text.is_empty() {
return Err!("Response text/body is empty.");
}
if text.len() > 1500 {
return Err!(
"Response text/body is over 1500 characters, assuming no support well-known.",
);
}
let json: serde_json::Value = match serde_json::from_str(&text) {
| Ok(json) => json,
| Err(_) => {
return Err!("Response text/body is not valid JSON.",);
},
};
let pretty_json: String = match serde_json::to_string_pretty(&json) {
| Ok(json) => json,
| Err(_) => {
return Err!("Response text/body is not valid JSON.",);
},
};
self.write_str(&format!("Got JSON response:\n\n```json\n{pretty_json}\n```"))
.await
}
pub(super) async fn remote_user_in_rooms(&self, user_id: OwnedUserId) -> Result {
if user_id.server_name() == self.services.server.name {
return Err!(
"User belongs to our server, please use `list-joined-rooms` user admin command \
instead.",
);
}
let mut rooms: Vec<(OwnedRoomId, u64, String)> = self
.services
.rooms
.event_handler
.federation_handletime
.read();
.state_cache
.rooms_joined(&user_id)
.then(async |room_id| get_room_info(self.services, &room_id).await)
.collect()
.await;
let mut msg = format!(
"Handling {} incoming PDUs across {} active transactions:\n",
map.len(),
self.services.transactions.txn_active_handle_count()
);
for (r, (e, i)) in map.iter() {
let elapsed = i.elapsed();
writeln!(msg, "{} {}: {}m{}s", r, e, elapsed.as_secs() / 60, elapsed.as_secs() % 60)?;
if rooms.is_empty() {
return Err!("User is not in any rooms.");
}
msg
};
self.write_str(&msg).await
}
rooms.sort_by_key(|r| r.1);
rooms.reverse();
#[admin_command]
pub(super) async fn fetch_support_well_known(&self, server_name: OwnedServerName) -> Result {
let response = self
.services
.client
.default
.get(format!("https://{server_name}/.well-known/matrix/support"))
.send()
.await?;
let num = rooms.len();
let body = rooms
.iter()
.map(|(id, members, name)| format!("{id} | Members: {members} | Name: {name}"))
.collect::<Vec<_>>()
.join("\n");
let text = response
.limit_read_text(
self.services
.config
.max_request_size
.try_into()
.expect("u64 fits into usize"),
)
.await?;
if text.is_empty() {
return Err!("Response text/body is empty.");
}
if text.len() > 1500 {
return Err!(
"Response text/body is over 1500 characters, assuming no support well-known.",
);
}
let json: serde_json::Value = match serde_json::from_str(&text) {
| Ok(json) => json,
| Err(_) => {
return Err!("Response text/body is not valid JSON.",);
},
};
let pretty_json: String = match serde_json::to_string_pretty(&json) {
| Ok(json) => json,
| Err(_) => {
return Err!("Response text/body is not valid JSON.",);
},
};
self.write_str(&format!("Got JSON response:\n\n```json\n{pretty_json}\n```"))
.await
}
#[admin_command]
pub(super) async fn remote_user_in_rooms(&self, user_id: OwnedUserId) -> Result {
if user_id.server_name() == self.services.server.name {
return Err!(
"User belongs to our server, please use `list-joined-rooms` user admin command \
instead.",
);
}
let mut rooms: Vec<(OwnedRoomId, u64, String)> = self
.services
.rooms
.state_cache
.rooms_joined(&user_id)
.then(async |room_id| get_room_info(self.services, &room_id).await)
.collect()
.await;
if rooms.is_empty() {
return Err!("User is not in any rooms.");
self.write_str(&format!("Rooms {user_id} shares with us ({num}):\n```\n{body}\n```"))
.await
}
rooms.sort_by_key(|r| r.1);
rooms.reverse();
let num = rooms.len();
let body = rooms
.iter()
.map(|(id, members, name)| format!("{id} | Members: {members} | Name: {name}"))
.collect::<Vec<_>>()
.join("\n");
self.write_str(&format!("Rooms {user_id} shares with us ({num}):\n```\n{body}\n```"))
.await
}
+336 -334
View File
@@ -9,401 +9,403 @@ use conduwuit_service::media::Dim;
use ruma::{OwnedEventId, OwnedMxcUri, OwnedServerName};
use service::media::mxc::Mxc;
use crate::{admin_command, utils::parse_local_user_id};
use crate::utils::parse_local_user_id;
#[admin_command]
pub(super) async fn delete(
&self,
mxc: Option<OwnedMxcUri>,
event_id: Option<OwnedEventId>,
) -> Result {
self.bail_restricted()?;
impl crate::Context<'_> {
pub(super) async fn delete(
&self,
mxc: Option<OwnedMxcUri>,
event_id: Option<OwnedEventId>,
) -> Result {
self.bail_restricted()?;
if event_id.is_some() && mxc.is_some() {
return Err!("Please specify either an MXC or an event ID, not both.",);
}
if event_id.is_some() && mxc.is_some() {
return Err!("Please specify either an MXC or an event ID, not both.",);
}
if let Some(mxc) = mxc {
trace!("Got MXC URL: {mxc}");
self.services
.media
.delete(&mxc.as_str().try_into()?)
.await?;
if let Some(mxc) = mxc {
trace!("Got MXC URL: {mxc}");
self.services
.media
.delete(&mxc.as_str().try_into()?)
.await?;
return self
.write_str("Deleted the MXC from our database and on our filesystem.")
.await;
}
return self
.write_str("Deleted the MXC from our database and on our filesystem.")
.await;
}
if let Some(event_id) = event_id {
trace!("Got event ID to delete media from: {event_id}");
if let Some(event_id) = event_id {
trace!("Got event ID to delete media from: {event_id}");
let mut mxc_urls = Vec::with_capacity(4);
let mut mxc_urls = Vec::with_capacity(4);
// parsing the PDU for any MXC URLs begins here
match self.services.rooms.timeline.get_pdu_json(&event_id).await {
| Ok(event_json) => {
if let Some(content_key) = event_json.get("content") {
debug!("Event ID has \"content\".");
let content_obj = content_key.as_object();
// parsing the PDU for any MXC URLs begins here
match self.services.rooms.timeline.get_pdu_json(&event_id).await {
| Ok(event_json) => {
if let Some(content_key) = event_json.get("content") {
debug!("Event ID has \"content\".");
let content_obj = content_key.as_object();
if let Some(content) = content_obj {
// 1. attempts to parse the "url" key
debug!("Attempting to go into \"url\" key for main media file");
if let Some(url) = content.get("url") {
debug!("Got a URL in the event ID {event_id}: {url}");
if let Some(content) = content_obj {
// 1. attempts to parse the "url" key
debug!("Attempting to go into \"url\" key for main media file");
if let Some(url) = content.get("url") {
debug!("Got a URL in the event ID {event_id}: {url}");
if url.to_string().starts_with("\"mxc://") {
debug!("Pushing URL {url} to list of MXCs to delete");
let final_url = url.to_string().replace('"', "");
mxc_urls.push(final_url);
} else {
info!(
"Found a URL in the event ID {event_id} but did not start \
with mxc://, ignoring"
);
}
}
// 2. attempts to parse the "info" key
debug!("Attempting to go into \"info\" key for thumbnails");
if let Some(info_key) = content.get("info") {
debug!("Event ID has \"info\".");
let info_obj = info_key.as_object();
if let Some(info) = info_obj {
if let Some(thumbnail_url) = info.get("thumbnail_url") {
debug!("Found a thumbnail_url in info key: {thumbnail_url}");
if thumbnail_url.to_string().starts_with("\"mxc://") {
debug!(
"Pushing thumbnail URL {thumbnail_url} to list of \
MXCs to delete"
);
let final_thumbnail_url =
thumbnail_url.to_string().replace('"', "");
mxc_urls.push(final_thumbnail_url);
} else {
info!(
"Found a thumbnail URL in the event ID {event_id} \
but did not start with mxc://, ignoring"
);
}
if url.to_string().starts_with("\"mxc://") {
debug!("Pushing URL {url} to list of MXCs to delete");
let final_url = url.to_string().replace('"', "");
mxc_urls.push(final_url);
} else {
info!(
"No \"thumbnail_url\" key in \"info\" key, assuming no \
thumbnails."
"Found a URL in the event ID {event_id} but did not \
start with mxc://, ignoring"
);
}
}
}
// 3. attempts to parse the "file" key
debug!("Attempting to go into \"file\" key");
if let Some(file_key) = content.get("file") {
debug!("Event ID has \"file\".");
let file_obj = file_key.as_object();
// 2. attempts to parse the "info" key
debug!("Attempting to go into \"info\" key for thumbnails");
if let Some(info_key) = content.get("info") {
debug!("Event ID has \"info\".");
let info_obj = info_key.as_object();
if let Some(file) = file_obj {
if let Some(url) = file.get("url") {
debug!("Found url in file key: {url}");
if let Some(info) = info_obj {
if let Some(thumbnail_url) = info.get("thumbnail_url") {
debug!(
"Found a thumbnail_url in info key: {thumbnail_url}"
);
if url.to_string().starts_with("\"mxc://") {
debug!("Pushing URL {url} to list of MXCs to delete");
let final_url = url.to_string().replace('"', "");
mxc_urls.push(final_url);
if thumbnail_url.to_string().starts_with("\"mxc://") {
debug!(
"Pushing thumbnail URL {thumbnail_url} to list \
of MXCs to delete"
);
let final_thumbnail_url =
thumbnail_url.to_string().replace('"', "");
mxc_urls.push(final_thumbnail_url);
} else {
info!(
"Found a thumbnail URL in the event ID \
{event_id} but did not start with mxc://, \
ignoring"
);
}
} else {
warn!(
"Found a URL in the event ID {event_id} but did not \
start with mxc://, ignoring"
info!(
"No \"thumbnail_url\" key in \"info\" key, assuming \
no thumbnails."
);
}
} else {
error!("No \"url\" key in \"file\" key.");
}
}
// 3. attempts to parse the "file" key
debug!("Attempting to go into \"file\" key");
if let Some(file_key) = content.get("file") {
debug!("Event ID has \"file\".");
let file_obj = file_key.as_object();
if let Some(file) = file_obj {
if let Some(url) = file.get("url") {
debug!("Found url in file key: {url}");
if url.to_string().starts_with("\"mxc://") {
debug!("Pushing URL {url} to list of MXCs to delete");
let final_url = url.to_string().replace('"', "");
mxc_urls.push(final_url);
} else {
warn!(
"Found a URL in the event ID {event_id} but did \
not start with mxc://, ignoring"
);
}
} else {
error!("No \"url\" key in \"file\" key.");
}
}
}
} else {
return Err!(
"Event ID does not have a \"content\" key or failed parsing the \
event ID JSON.",
);
}
} else {
return Err!(
"Event ID does not have a \"content\" key or failed parsing the \
event ID JSON.",
"Event ID does not have a \"content\" key, this is not a message or \
an event type that contains media.",
);
}
} else {
return Err!(
"Event ID does not have a \"content\" key, this is not a message or an \
event type that contains media.",
);
},
| _ => {
return Err!("Event ID does not exist or is not known to us.",);
},
}
if mxc_urls.is_empty() {
return Err!("Parsed event ID but found no MXC URLs.",);
}
let mut mxc_deletion_count: usize = 0;
for mxc_url in mxc_urls {
match self
.services
.media
.delete(&mxc_url.as_str().try_into()?)
.await
{
| Ok(()) => {
debug_info!(
"Successfully deleted {mxc_url} from filesystem and database"
);
mxc_deletion_count = mxc_deletion_count.saturating_add(1);
},
| Err(e) => {
debug_warn!(
"Failed to delete {mxc_url}, ignoring error and skipping: {e}"
);
continue;
},
}
},
| _ => {
return Err!("Event ID does not exist or is not known to us.",);
},
}
return self
.write_str(&format!(
"Deleted {mxc_deletion_count} total MXCs from our database and the \
filesystem from event ID {event_id}."
))
.await;
}
if mxc_urls.is_empty() {
return Err!("Parsed event ID but found no MXC URLs.",);
Err!(
"Please specify either an MXC using --mxc or an event ID using --event-id of the \
message containing an image. See --help for details."
)
}
pub(super) async fn delete_list(&self) -> Result {
self.bail_restricted()?;
if self.body.len() < 2
|| !self.body[0].trim().starts_with("```")
|| self.body.last().unwrap_or(&"").trim() != "```"
{
return Err!("Expected code block in command body. Add --help for details.",);
}
let mut failed_parsed_mxcs: usize = 0;
let mxc_list = self
.body
.to_vec()
.drain(1..self.body.len().checked_sub(1).unwrap())
.filter_map(|mxc_s| {
mxc_s
.try_into()
.inspect_err(|e| {
debug_warn!("Failed to parse user-provided MXC URI: {e}");
failed_parsed_mxcs = failed_parsed_mxcs.saturating_add(1);
})
.ok()
})
.collect::<Vec<Mxc<'_>>>();
let mut mxc_deletion_count: usize = 0;
for mxc_url in mxc_urls {
match self
.services
.media
.delete(&mxc_url.as_str().try_into()?)
.await
{
for mxc in &mxc_list {
trace!(%failed_parsed_mxcs, %mxc_deletion_count, "Deleting MXC {mxc} in bulk");
match self.services.media.delete(mxc).await {
| Ok(()) => {
debug_info!("Successfully deleted {mxc_url} from filesystem and database");
debug_info!("Successfully deleted {mxc} from filesystem and database");
mxc_deletion_count = mxc_deletion_count.saturating_add(1);
},
| Err(e) => {
debug_warn!("Failed to delete {mxc_url}, ignoring error and skipping: {e}");
debug_warn!("Failed to delete {mxc}, ignoring error and skipping: {e}");
continue;
},
}
}
return self
.write_str(&format!(
"Deleted {mxc_deletion_count} total MXCs from our database and the filesystem \
from event ID {event_id}."
))
.await;
self.write_str(&format!(
"Finished bulk MXC deletion, deleted {mxc_deletion_count} total MXCs from our \
database and the filesystem. {failed_parsed_mxcs} MXCs failed to be parsed from \
the database.",
))
.await
}
Err!(
"Please specify either an MXC using --mxc or an event ID using --event-id of the \
message containing an image. See --help for details."
)
}
pub(super) async fn delete_past_remote_media(
&self,
duration: String,
before: bool,
after: bool,
yes_i_want_to_delete_local_media: bool,
) -> Result {
self.bail_restricted()?;
#[admin_command]
pub(super) async fn delete_list(&self) -> Result {
self.bail_restricted()?;
if self.body.len() < 2
|| !self.body[0].trim().starts_with("```")
|| self.body.last().unwrap_or(&"").trim() != "```"
{
return Err!("Expected code block in command body. Add --help for details.",);
}
let mut failed_parsed_mxcs: usize = 0;
let mxc_list = self
.body
.to_vec()
.drain(1..self.body.len().checked_sub(1).unwrap())
.filter_map(|mxc_s| {
mxc_s
.try_into()
.inspect_err(|e| {
debug_warn!("Failed to parse user-provided MXC URI: {e}");
failed_parsed_mxcs = failed_parsed_mxcs.saturating_add(1);
})
.ok()
})
.collect::<Vec<Mxc<'_>>>();
let mut mxc_deletion_count: usize = 0;
for mxc in &mxc_list {
trace!(%failed_parsed_mxcs, %mxc_deletion_count, "Deleting MXC {mxc} in bulk");
match self.services.media.delete(mxc).await {
| Ok(()) => {
debug_info!("Successfully deleted {mxc} from filesystem and database");
mxc_deletion_count = mxc_deletion_count.saturating_add(1);
},
| Err(e) => {
debug_warn!("Failed to delete {mxc}, ignoring error and skipping: {e}");
continue;
},
if before && after {
return Err!("Please only pick one argument, --before or --after.",);
}
}
assert!(!(before && after), "--before and --after should not be specified together");
self.write_str(&format!(
"Finished bulk MXC deletion, deleted {mxc_deletion_count} total MXCs from our database \
and the filesystem. {failed_parsed_mxcs} MXCs failed to be parsed from the database.",
))
.await
}
#[admin_command]
pub(super) async fn delete_past_remote_media(
&self,
duration: String,
before: bool,
after: bool,
yes_i_want_to_delete_local_media: bool,
) -> Result {
self.bail_restricted()?;
if before && after {
return Err!("Please only pick one argument, --before or --after.",);
}
assert!(!(before && after), "--before and --after should not be specified together");
let direction = if after {
TimeDirection::After
} else {
TimeDirection::Before
};
let time_boundary = parse_timepoint_ago(&duration)?;
let deleted_count = self
.services
.media
.delete_all_media_within_timeframe(
time_boundary,
direction,
yes_i_want_to_delete_local_media,
)
.await?;
self.write_str(&format!("Deleted {deleted_count} total files."))
.await
}
#[admin_command]
pub(super) async fn delete_all_from_user(&self, username: String) -> Result {
let user_id = parse_local_user_id(self.services, &username)?;
let deleted_count = self.services.media.delete_from_user(&user_id).await?;
self.write_str(&format!("Deleted {deleted_count} total files."))
.await
}
#[admin_command]
pub(super) async fn delete_all_from_server(
&self,
server_name: OwnedServerName,
yes_i_want_to_delete_local_media: bool,
) -> Result {
self.bail_restricted()?;
if server_name == self.services.globals.server_name() && !yes_i_want_to_delete_local_media {
return Err!("This command only works for remote media by default.",);
}
let Ok(all_mxcs) = self
.services
.media
.get_all_mxcs()
.await
.inspect_err(|e| error!("Failed to get MXC URIs from our database: {e}"))
else {
return Err!("Failed to get MXC URIs from our database",);
};
let mut deleted_count: usize = 0;
for mxc in all_mxcs {
let Ok(mxc_server_name) = mxc.server_name().inspect_err(|e| {
debug_warn!(
"Failed to parse MXC {mxc} server name from database, ignoring error and \
skipping: {e}"
);
}) else {
continue;
let direction = if after {
TimeDirection::After
} else {
TimeDirection::Before
};
if mxc_server_name != server_name
|| (self.services.globals.server_is_ours(mxc_server_name)
&& !yes_i_want_to_delete_local_media)
let time_boundary = parse_timepoint_ago(&duration)?;
let deleted_count = self
.services
.media
.delete_all_media_within_timeframe(
time_boundary,
direction,
yes_i_want_to_delete_local_media,
)
.await?;
self.write_str(&format!("Deleted {deleted_count} total files."))
.await
}
pub(super) async fn delete_all_from_user(&self, username: String) -> Result {
let user_id = parse_local_user_id(self.services, &username)?;
let deleted_count = self.services.media.delete_from_user(&user_id).await?;
self.write_str(&format!("Deleted {deleted_count} total files."))
.await
}
pub(super) async fn delete_all_from_server(
&self,
server_name: OwnedServerName,
yes_i_want_to_delete_local_media: bool,
) -> Result {
self.bail_restricted()?;
if server_name == self.services.globals.server_name() && !yes_i_want_to_delete_local_media
{
trace!("skipping MXC URI {mxc}");
continue;
return Err!("This command only works for remote media by default.",);
}
let mxc: Mxc<'_> = mxc.as_str().try_into()?;
let Ok(all_mxcs) = self
.services
.media
.get_all_mxcs()
.await
.inspect_err(|e| error!("Failed to get MXC URIs from our database: {e}"))
else {
return Err!("Failed to get MXC URIs from our database",);
};
match self.services.media.delete(&mxc).await {
| Ok(()) => {
deleted_count = deleted_count.saturating_add(1);
},
| Err(e) => {
debug_warn!("Failed to delete {mxc}, ignoring error and skipping: {e}");
let mut deleted_count: usize = 0;
for mxc in all_mxcs {
let Ok(mxc_server_name) = mxc.server_name().inspect_err(|e| {
debug_warn!(
"Failed to parse MXC {mxc} server name from database, ignoring error and \
skipping: {e}"
);
}) else {
continue;
},
};
if mxc_server_name != server_name
|| (self.services.globals.server_is_ours(mxc_server_name)
&& !yes_i_want_to_delete_local_media)
{
trace!("skipping MXC URI {mxc}");
continue;
}
let mxc: Mxc<'_> = mxc.as_str().try_into()?;
match self.services.media.delete(&mxc).await {
| Ok(()) => {
deleted_count = deleted_count.saturating_add(1);
},
| Err(e) => {
debug_warn!("Failed to delete {mxc}, ignoring error and skipping: {e}");
continue;
},
}
}
self.write_str(&format!("Deleted {deleted_count} total files."))
.await
}
self.write_str(&format!("Deleted {deleted_count} total files."))
.await
}
pub(super) async fn get_file_info(&self, mxc: OwnedMxcUri) -> Result {
let mxc: Mxc<'_> = mxc.as_str().try_into()?;
let metadata = self.services.media.get_metadata(&mxc).await;
#[admin_command]
pub(super) async fn get_file_info(&self, mxc: OwnedMxcUri) -> Result {
let mxc: Mxc<'_> = mxc.as_str().try_into()?;
let metadata = self.services.media.get_metadata(&mxc).await;
self.write_str(&format!("```\n{metadata:#?}\n```")).await
}
#[admin_command]
pub(super) async fn get_remote_file(
&self,
mxc: OwnedMxcUri,
server: Option<OwnedServerName>,
timeout: u32,
) -> Result {
let mxc: Mxc<'_> = mxc.as_str().try_into()?;
let timeout = Duration::from_millis(timeout.into());
let mut result = self
.services
.media
.fetch_remote_content(&mxc, None, server.as_deref(), timeout)
.await?;
// Grab the length of the content before clearing it to not flood the output
let len = result.content.as_ref().expect("content").len();
result.content.as_mut().expect("content").clear();
self.write_str(&format!("```\n{result:#?}\nreceived {len} bytes for file content.\n```"))
.await
}
#[admin_command]
pub(super) async fn get_remote_thumbnail(
&self,
mxc: OwnedMxcUri,
server: Option<OwnedServerName>,
timeout: u32,
width: u32,
height: u32,
) -> Result {
let mxc: Mxc<'_> = mxc.as_str().try_into()?;
let timeout = Duration::from_millis(timeout.into());
let dim = Dim::new(width, height, None);
let mut result = self
.services
.media
.fetch_remote_thumbnail(&mxc, None, server.as_deref(), timeout, &dim)
.await?;
// Grab the length of the content before clearing it to not flood the output
let len = result.content.as_ref().expect("content").len();
result.content.as_mut().expect("content").clear();
self.write_str(&format!("```\n{result:#?}\nreceived {len} bytes for file content.\n```"))
.await
}
#[admin_command]
pub(super) async fn delete_url_preview(&self, url: Option<String>, all: bool) -> Result {
if all {
self.services.media.clear_url_previews().await;
return self.write_str("Deleted all cached URL previews.").await;
self.write_str(&format!("```\n{metadata:#?}\n```")).await
}
let url = url.expect("clap enforces url is required unless --all");
pub(super) async fn get_remote_file(
&self,
mxc: OwnedMxcUri,
server: Option<OwnedServerName>,
timeout: u32,
) -> Result {
let mxc: Mxc<'_> = mxc.as_str().try_into()?;
let timeout = Duration::from_millis(timeout.into());
let mut result = self
.services
.media
.fetch_remote_content(&mxc, None, server.as_deref(), timeout)
.await?;
self.services.media.remove_url_preview(&url).await?;
// Grab the length of the content before clearing it to not flood the output
let len = result.content.as_ref().expect("content").len();
result.content.as_mut().expect("content").clear();
self.write_str(&format!("Deleted cached URL preview for: {url}"))
.await
self.write_str(&format!("```\n{result:#?}\nreceived {len} bytes for file content.\n```"))
.await
}
pub(super) async fn get_remote_thumbnail(
&self,
mxc: OwnedMxcUri,
server: Option<OwnedServerName>,
timeout: u32,
width: u32,
height: u32,
) -> Result {
let mxc: Mxc<'_> = mxc.as_str().try_into()?;
let timeout = Duration::from_millis(timeout.into());
let dim = Dim::new(width, height, None);
let mut result = self
.services
.media
.fetch_remote_thumbnail(&mxc, None, server.as_deref(), timeout, &dim)
.await?;
// Grab the length of the content before clearing it to not flood the output
let len = result.content.as_ref().expect("content").len();
result.content.as_mut().expect("content").clear();
self.write_str(&format!("```\n{result:#?}\nreceived {len} bytes for file content.\n```"))
.await
}
pub(super) async fn delete_url_preview(&self, url: Option<String>, all: bool) -> Result {
if all {
self.services.media.clear_url_previews().await;
return self.write_str("Deleted all cached URL previews.").await;
}
let url = url.expect("clap enforces url is required unless --all");
self.services.media.remove_url_preview(&url).await?;
self.write_str(&format!("Deleted cached URL preview for: {url}"))
.await
}
}
+1 -1
View File
@@ -26,7 +26,7 @@ extern crate conduwuit_api as api;
extern crate conduwuit_core as conduwuit;
extern crate conduwuit_service as service;
pub(crate) use conduwuit_macros::{admin_command, admin_command_dispatch};
pub(crate) use conduwuit_macros::admin_command_dispatch;
pub(crate) use crate::{context::Context, utils::get_room_info};
+43 -43
View File
@@ -4,7 +4,7 @@ use conduwuit_database::Deserialized as _;
use futures::StreamExt;
use ruma::{OwnedRoomId, OwnedUserId, exports::serde::Serialize};
use crate::{admin_command, admin_command_dispatch};
use crate::admin_command_dispatch;
#[admin_command_dispatch]
#[derive(Debug, Subcommand)]
@@ -31,50 +31,50 @@ pub enum AccountDataCommand {
},
}
#[admin_command]
async fn changes_since(
&self,
user_id: OwnedUserId,
since: u64,
room_id: Option<OwnedRoomId>,
) -> Result {
let timer = tokio::time::Instant::now();
let results: Vec<_> = self
.services
.account_data
.changes_since(room_id.as_deref(), &user_id, Some(since), None)
.collect()
.await;
let query_time = timer.elapsed();
impl crate::Context<'_> {
async fn changes_since(
&self,
user_id: OwnedUserId,
since: u64,
room_id: Option<OwnedRoomId>,
) -> Result {
let timer = tokio::time::Instant::now();
let results: Vec<_> = self
.services
.account_data
.changes_since(room_id.as_deref(), &user_id, Some(since), None)
.collect()
.await;
let query_time = timer.elapsed();
self.write_str(&format!("Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```"))
.await
}
self.write_str(&format!("Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```"))
.await
}
#[admin_command]
async fn account_data_get(
&self,
user_id: OwnedUserId,
kind: String,
room_id: Option<OwnedRoomId>,
) -> Result {
let timer = tokio::time::Instant::now();
let result = self
.services
.account_data
.get_raw(room_id.as_deref(), &user_id, &kind)
.await;
let query_time = timer.elapsed();
async fn account_data_get(
&self,
user_id: OwnedUserId,
kind: String,
room_id: Option<OwnedRoomId>,
) -> Result {
let timer = tokio::time::Instant::now();
let result = self
.services
.account_data
.get_raw(room_id.as_deref(), &user_id, &kind)
.await;
let query_time = timer.elapsed();
let json = serde_json::to_string_pretty(&match room_id {
| None => result
.deserialized::<ruma::serde::Raw<ruma::events::AnyGlobalAccountDataEvent>>()?
.serialize(serde_json::value::Serializer)?,
| Some(_) => result
.deserialized::<ruma::serde::Raw<ruma::events::AnyRoomAccountDataEvent>>()?
.serialize(serde_json::value::Serializer)?,
})?;
let json = serde_json::to_string_pretty(&match room_id {
| None => result
.deserialized::<ruma::serde::Raw<ruma::events::AnyGlobalAccountDataEvent>>()?
.serialize(serde_json::value::Serializer)?,
| Some(_) => result
.deserialized::<ruma::serde::Raw<ruma::events::AnyRoomAccountDataEvent>>()?
.serialize(serde_json::value::Serializer)?,
})?;
self.write_str(&format!("Query completed in {query_time:?}:\n\n```rs\n{json}\n```"))
.await
self.write_str(&format!("Query completed in {query_time:?}:\n\n```rs\n{json}\n```"))
.await
}
}
+246 -238
View File
@@ -13,7 +13,7 @@ use conduwuit_service::Services;
use futures::{FutureExt, Stream, StreamExt, TryStreamExt};
use tokio::time::Instant;
use crate::{admin_command, admin_command_dispatch};
use crate::admin_command_dispatch;
#[admin_command_dispatch]
#[derive(Debug, Subcommand)]
@@ -159,279 +159,287 @@ pub enum RawCommand {
},
}
#[admin_command]
pub(super) async fn compact(
&self,
map: Option<Vec<String>>,
start: Option<String>,
stop: Option<String>,
from: Option<usize>,
into: Option<usize>,
parallelism: Option<usize>,
exhaustive: bool,
) -> Result {
use conduwuit_database::compact::Options;
impl crate::Context<'_> {
pub(super) async fn compact(
&self,
map: Option<Vec<String>>,
start: Option<String>,
stop: Option<String>,
from: Option<usize>,
into: Option<usize>,
parallelism: Option<usize>,
exhaustive: bool,
) -> Result {
use conduwuit_database::compact::Options;
let default_all_maps: Option<_> = map.is_none().then(|| {
self.services
.db
.keys()
.map(Deref::deref)
.map(ToOwned::to_owned)
});
let default_all_maps: Option<_> = map.is_none().then(|| {
self.services
.db
.keys()
.map(Deref::deref)
.map(ToOwned::to_owned)
});
let maps: Vec<_> = map
.unwrap_or_default()
.into_iter()
.chain(default_all_maps.into_iter().flatten())
.map(|map| self.services.db.get(&map))
.filter_map(Result::ok)
.cloned()
.collect();
let maps: Vec<_> = map
.unwrap_or_default()
.into_iter()
.chain(default_all_maps.into_iter().flatten())
.map(|map| self.services.db.get(&map))
.filter_map(Result::ok)
.cloned()
.collect();
if maps.is_empty() {
return Err!("--map argument invalid. not found in database");
if maps.is_empty() {
return Err!("--map argument invalid. not found in database");
}
let range = (
start.as_ref().map(String::as_bytes).map(Into::into),
stop.as_ref().map(String::as_bytes).map(Into::into),
);
let options = Options {
range,
level: (from, into),
exclusive: parallelism.is_some_and(is_zero!()),
exhaustive,
};
let runtime = self.services.server.runtime().clone();
let parallelism = parallelism.unwrap_or(1);
let results = maps
.into_iter()
.try_stream::<conduwuit::Error>()
.paralleln_and_then(runtime, parallelism, move |map| {
map.compact_blocking(options.clone())?;
Ok(map.name().to_owned())
})
.collect::<Vec<_>>();
let timer = Instant::now();
let results = results.await;
let query_time = timer.elapsed();
self.write_str(&format!("Jobs completed in {query_time:?}:\n\n```rs\n{results:#?}\n```"))
.await
}
let range = (
start.as_ref().map(String::as_bytes).map(Into::into),
stop.as_ref().map(String::as_bytes).map(Into::into),
);
pub(super) async fn raw_count(&self, map: Option<String>, prefix: Option<String>) -> Result {
let prefix = prefix.as_deref().unwrap_or(EMPTY);
let options = Options {
range,
level: (from, into),
exclusive: parallelism.is_some_and(is_zero!()),
exhaustive,
};
let timer = Instant::now();
let count = with_maps_or(map.as_deref(), self.services)
.then(|map| map.raw_count_prefix(&prefix))
.ready_fold(0_usize, usize::saturating_add)
.await;
let runtime = self.services.server.runtime().clone();
let parallelism = parallelism.unwrap_or(1);
let results = maps
.into_iter()
.try_stream::<conduwuit::Error>()
.paralleln_and_then(runtime, parallelism, move |map| {
map.compact_blocking(options.clone())?;
Ok(map.name().to_owned())
})
.collect::<Vec<_>>();
let query_time = timer.elapsed();
self.write_str(&format!("Query completed in {query_time:?}:\n\n```rs\n{count:#?}\n```"))
.await
}
let timer = Instant::now();
let results = results.await;
let query_time = timer.elapsed();
self.write_str(&format!("Jobs completed in {query_time:?}:\n\n```rs\n{results:#?}\n```"))
.await
}
pub(super) async fn raw_keys(&self, map: String, prefix: Option<String>) -> Result {
writeln!(self, "```").boxed().await?;
#[admin_command]
pub(super) async fn raw_count(&self, map: Option<String>, prefix: Option<String>) -> Result {
let prefix = prefix.as_deref().unwrap_or(EMPTY);
let map = self.services.db.get(map.as_str())?;
let timer = Instant::now();
prefix
.as_deref()
.map_or_else(|| map.raw_keys().boxed(), |prefix| map.raw_keys_prefix(prefix).boxed())
.map_ok(String::from_utf8_lossy)
.try_for_each(|str| writeln!(self, "{str:?}"))
.boxed()
.await?;
let timer = Instant::now();
let count = with_maps_or(map.as_deref(), self.services)
.then(|map| map.raw_count_prefix(&prefix))
.ready_fold(0_usize, usize::saturating_add)
.await;
let query_time = timer.elapsed();
self.write_str(&format!("\n```\n\nQuery completed in {query_time:?}"))
.await
}
let query_time = timer.elapsed();
self.write_str(&format!("Query completed in {query_time:?}:\n\n```rs\n{count:#?}\n```"))
.await
}
pub(super) async fn raw_keys_sizes(
&self,
map: Option<String>,
prefix: Option<String>,
) -> Result {
let prefix = prefix.as_deref().unwrap_or(EMPTY);
#[admin_command]
pub(super) async fn raw_keys(&self, map: String, prefix: Option<String>) -> Result {
writeln!(self, "```").boxed().await?;
let timer = Instant::now();
let result = with_maps_or(map.as_deref(), self.services)
.map(|map| map.raw_keys_prefix(&prefix))
.flatten()
.ignore_err()
.map(<[u8]>::len)
.ready_fold_default(|mut map: BTreeMap<_, usize>, len| {
let entry = map.entry(len).or_default();
*entry = entry.saturating_add(1);
map
})
.await;
let map = self.services.db.get(map.as_str())?;
let timer = Instant::now();
prefix
.as_deref()
.map_or_else(|| map.raw_keys().boxed(), |prefix| map.raw_keys_prefix(prefix).boxed())
.map_ok(String::from_utf8_lossy)
.try_for_each(|str| writeln!(self, "{str:?}"))
.boxed()
.await?;
let query_time = timer.elapsed();
self.write_str(&format!("```\n{result:#?}\n```\n\nQuery completed in {query_time:?}"))
.await
}
let query_time = timer.elapsed();
self.write_str(&format!("\n```\n\nQuery completed in {query_time:?}"))
.await
}
pub(super) async fn raw_keys_total(
&self,
map: Option<String>,
prefix: Option<String>,
) -> Result {
let prefix = prefix.as_deref().unwrap_or(EMPTY);
#[admin_command]
pub(super) async fn raw_keys_sizes(&self, map: Option<String>, prefix: Option<String>) -> Result {
let prefix = prefix.as_deref().unwrap_or(EMPTY);
let timer = Instant::now();
let result = with_maps_or(map.as_deref(), self.services)
.map(|map| map.raw_keys_prefix(&prefix))
.flatten()
.ignore_err()
.map(<[u8]>::len)
.ready_fold_default(|acc: usize, len| acc.saturating_add(len))
.await;
let timer = Instant::now();
let result = with_maps_or(map.as_deref(), self.services)
.map(|map| map.raw_keys_prefix(&prefix))
.flatten()
.ignore_err()
.map(<[u8]>::len)
.ready_fold_default(|mut map: BTreeMap<_, usize>, len| {
let entry = map.entry(len).or_default();
*entry = entry.saturating_add(1);
map
})
.await;
let query_time = timer.elapsed();
self.write_str(&format!("```\n{result:#?}\n\n```\n\nQuery completed in {query_time:?}"))
.await
}
let query_time = timer.elapsed();
self.write_str(&format!("```\n{result:#?}\n```\n\nQuery completed in {query_time:?}"))
.await
}
pub(super) async fn raw_vals_sizes(
&self,
map: Option<String>,
prefix: Option<String>,
) -> Result {
let prefix = prefix.as_deref().unwrap_or(EMPTY);
#[admin_command]
pub(super) async fn raw_keys_total(&self, map: Option<String>, prefix: Option<String>) -> Result {
let prefix = prefix.as_deref().unwrap_or(EMPTY);
let timer = Instant::now();
let result = with_maps_or(map.as_deref(), self.services)
.map(|map| map.raw_stream_prefix(&prefix))
.flatten()
.ignore_err()
.map(at!(1))
.map(<[u8]>::len)
.ready_fold_default(|mut map: BTreeMap<_, usize>, len| {
let entry = map.entry(len).or_default();
*entry = entry.saturating_add(1);
map
})
.await;
let timer = Instant::now();
let result = with_maps_or(map.as_deref(), self.services)
.map(|map| map.raw_keys_prefix(&prefix))
.flatten()
.ignore_err()
.map(<[u8]>::len)
.ready_fold_default(|acc: usize, len| acc.saturating_add(len))
.await;
let query_time = timer.elapsed();
self.write_str(&format!("```\n{result:#?}\n```\n\nQuery completed in {query_time:?}"))
.await
}
let query_time = timer.elapsed();
self.write_str(&format!("```\n{result:#?}\n\n```\n\nQuery completed in {query_time:?}"))
.await
}
pub(super) async fn raw_vals_total(
&self,
map: Option<String>,
prefix: Option<String>,
) -> Result {
let prefix = prefix.as_deref().unwrap_or(EMPTY);
#[admin_command]
pub(super) async fn raw_vals_sizes(&self, map: Option<String>, prefix: Option<String>) -> Result {
let prefix = prefix.as_deref().unwrap_or(EMPTY);
let timer = Instant::now();
let result = with_maps_or(map.as_deref(), self.services)
.map(|map| map.raw_stream_prefix(&prefix))
.flatten()
.ignore_err()
.map(at!(1))
.map(<[u8]>::len)
.ready_fold_default(|acc: usize, len| acc.saturating_add(len))
.await;
let timer = Instant::now();
let result = with_maps_or(map.as_deref(), self.services)
.map(|map| map.raw_stream_prefix(&prefix))
.flatten()
.ignore_err()
.map(at!(1))
.map(<[u8]>::len)
.ready_fold_default(|mut map: BTreeMap<_, usize>, len| {
let entry = map.entry(len).or_default();
*entry = entry.saturating_add(1);
map
})
.await;
let query_time = timer.elapsed();
self.write_str(&format!("```\n{result:#?}\n\n```\n\nQuery completed in {query_time:?}"))
.await
}
let query_time = timer.elapsed();
self.write_str(&format!("```\n{result:#?}\n```\n\nQuery completed in {query_time:?}"))
.await
}
pub(super) async fn raw_iter(&self, map: String, prefix: Option<String>) -> Result {
writeln!(self, "```").await?;
#[admin_command]
pub(super) async fn raw_vals_total(&self, map: Option<String>, prefix: Option<String>) -> Result {
let prefix = prefix.as_deref().unwrap_or(EMPTY);
let map = self.services.db.get(&map)?;
let timer = Instant::now();
prefix
.as_deref()
.map_or_else(
|| map.raw_stream().boxed(),
|prefix| map.raw_stream_prefix(prefix).boxed(),
)
.map_ok(apply!(2, String::from_utf8_lossy))
.map_ok(apply!(2, Cow::into_owned))
.try_for_each(|keyval| writeln!(self, "{keyval:?}"))
.boxed()
.await?;
let timer = Instant::now();
let result = with_maps_or(map.as_deref(), self.services)
.map(|map| map.raw_stream_prefix(&prefix))
.flatten()
.ignore_err()
.map(at!(1))
.map(<[u8]>::len)
.ready_fold_default(|acc: usize, len| acc.saturating_add(len))
.await;
let query_time = timer.elapsed();
self.write_str(&format!("\n```\n\nQuery completed in {query_time:?}"))
.await
}
let query_time = timer.elapsed();
self.write_str(&format!("```\n{result:#?}\n\n```\n\nQuery completed in {query_time:?}"))
.await
}
pub(super) async fn raw_keys_from(
&self,
map: String,
start: String,
limit: Option<usize>,
) -> Result {
writeln!(self, "```").await?;
#[admin_command]
pub(super) async fn raw_iter(&self, map: String, prefix: Option<String>) -> Result {
writeln!(self, "```").await?;
let map = self.services.db.get(&map)?;
let timer = Instant::now();
map.raw_keys_from(&start)
.map_ok(String::from_utf8_lossy)
.take(limit.unwrap_or(usize::MAX))
.try_for_each(|str| writeln!(self, "{str:?}"))
.boxed()
.await?;
let map = self.services.db.get(&map)?;
let timer = Instant::now();
prefix
.as_deref()
.map_or_else(|| map.raw_stream().boxed(), |prefix| map.raw_stream_prefix(prefix).boxed())
.map_ok(apply!(2, String::from_utf8_lossy))
.map_ok(apply!(2, Cow::into_owned))
.try_for_each(|keyval| writeln!(self, "{keyval:?}"))
.boxed()
.await?;
let query_time = timer.elapsed();
self.write_str(&format!("\n```\n\nQuery completed in {query_time:?}"))
.await
}
let query_time = timer.elapsed();
self.write_str(&format!("\n```\n\nQuery completed in {query_time:?}"))
.await
}
pub(super) async fn raw_iter_from(
&self,
map: String,
start: String,
limit: Option<usize>,
) -> Result {
let map = self.services.db.get(&map)?;
let timer = Instant::now();
let result = map
.raw_stream_from(&start)
.map_ok(apply!(2, String::from_utf8_lossy))
.map_ok(apply!(2, Cow::into_owned))
.take(limit.unwrap_or(usize::MAX))
.try_collect::<Vec<(String, String)>>()
.await?;
#[admin_command]
pub(super) async fn raw_keys_from(
&self,
map: String,
start: String,
limit: Option<usize>,
) -> Result {
writeln!(self, "```").await?;
let query_time = timer.elapsed();
self.write_str(&format!("Query completed in {query_time:?}:\n\n```rs\n{result:#?}\n```"))
.await
}
let map = self.services.db.get(&map)?;
let timer = Instant::now();
map.raw_keys_from(&start)
.map_ok(String::from_utf8_lossy)
.take(limit.unwrap_or(usize::MAX))
.try_for_each(|str| writeln!(self, "{str:?}"))
.boxed()
.await?;
pub(super) async fn raw_del(&self, map: String, key: String) -> Result {
let map = self.services.db.get(&map)?;
let timer = Instant::now();
map.remove(&key);
let query_time = timer.elapsed();
self.write_str(&format!("\n```\n\nQuery completed in {query_time:?}"))
.await
}
let query_time = timer.elapsed();
self.write_str(&format!("Operation completed in {query_time:?}"))
.await
}
#[admin_command]
pub(super) async fn raw_iter_from(
&self,
map: String,
start: String,
limit: Option<usize>,
) -> Result {
let map = self.services.db.get(&map)?;
let timer = Instant::now();
let result = map
.raw_stream_from(&start)
.map_ok(apply!(2, String::from_utf8_lossy))
.map_ok(apply!(2, Cow::into_owned))
.take(limit.unwrap_or(usize::MAX))
.try_collect::<Vec<(String, String)>>()
.await?;
pub(super) async fn raw_get(&self, map: String, key: String) -> Result {
let map = self.services.db.get(&map)?;
let timer = Instant::now();
let handle = map.get(&key).await?;
let query_time = timer.elapsed();
self.write_str(&format!("Query completed in {query_time:?}:\n\n```rs\n{result:#?}\n```"))
.await
}
let query_time = timer.elapsed();
let result = String::from_utf8_lossy(&handle);
self.write_str(&format!("Query completed in {query_time:?}:\n\n```rs\n{result:?}\n```"))
.await
}
#[admin_command]
pub(super) async fn raw_del(&self, map: String, key: String) -> Result {
let map = self.services.db.get(&map)?;
let timer = Instant::now();
map.remove(&key);
pub(super) async fn raw_maps(&self) -> Result {
let list: Vec<_> = self.services.db.iter().map(at!(0)).copied().collect();
let query_time = timer.elapsed();
self.write_str(&format!("Operation completed in {query_time:?}"))
.await
}
#[admin_command]
pub(super) async fn raw_get(&self, map: String, key: String) -> Result {
let map = self.services.db.get(&map)?;
let timer = Instant::now();
let handle = map.get(&key).await?;
let query_time = timer.elapsed();
let result = String::from_utf8_lossy(&handle);
self.write_str(&format!("Query completed in {query_time:?}:\n\n```rs\n{result:?}\n```"))
.await
}
#[admin_command]
pub(super) async fn raw_maps(&self) -> Result {
let list: Vec<_> = self.services.db.iter().map(at!(0)).copied().collect();
self.write_str(&format!("{list:#?}")).await
self.write_str(&format!("{list:#?}")).await
}
}
fn with_maps_or<'a>(
+48 -47
View File
@@ -3,7 +3,7 @@ use conduwuit::{Err, Result, utils::time};
use futures::StreamExt;
use ruma::OwnedServerName;
use crate::{admin_command, admin_command_dispatch};
use crate::admin_command_dispatch;
#[admin_command_dispatch]
#[derive(Debug, Subcommand)]
@@ -39,67 +39,68 @@ pub enum ResolverCommand {
},
}
#[admin_command]
async fn destinations_cache(&self, server_name: Option<OwnedServerName>) -> Result {
use service::resolver::cache::CachedDest;
impl crate::Context<'_> {
async fn destinations_cache(&self, server_name: Option<OwnedServerName>) -> Result {
use service::resolver::cache::CachedDest;
writeln!(self, "| Server Name | Destination | Hostname | Expires |").await?;
writeln!(self, "| ----------- | ----------- | -------- | ------- |").await?;
writeln!(self, "| Server Name | Destination | Hostname | Expires |").await?;
writeln!(self, "| ----------- | ----------- | -------- | ------- |").await?;
let mut destinations = self.services.resolver.cache.destinations().boxed();
let mut destinations = self.services.resolver.cache.destinations().boxed();
while let Some((name, CachedDest { dest, host, expire })) = destinations.next().await {
if let Some(server_name) = server_name.as_ref() {
if name != *server_name {
continue;
while let Some((name, CachedDest { dest, host, expire })) = destinations.next().await {
if let Some(server_name) = server_name.as_ref() {
if name != *server_name {
continue;
}
}
let expire = time::format(expire, "%+");
self.write_str(&format!("| {name} | {dest} | {host} | {expire} |\n"))
.await?;
}
let expire = time::format(expire, "%+");
self.write_str(&format!("| {name} | {dest} | {host} | {expire} |\n"))
.await?;
Ok(())
}
Ok(())
}
async fn overrides_cache(&self, server_name: Option<String>) -> Result {
use service::resolver::cache::CachedOverride;
#[admin_command]
async fn overrides_cache(&self, server_name: Option<String>) -> Result {
use service::resolver::cache::CachedOverride;
writeln!(self, "| Server Name | IP | Port | Expires | Overriding |").await?;
writeln!(self, "| ----------- | --- | ----:| ------- | ---------- |").await?;
writeln!(self, "| Server Name | IP | Port | Expires | Overriding |").await?;
writeln!(self, "| ----------- | --- | ----:| ------- | ---------- |").await?;
let mut overrides = self.services.resolver.cache.overrides().boxed();
let mut overrides = self.services.resolver.cache.overrides().boxed();
while let Some((name, CachedOverride { ips, port, expire, overriding })) =
overrides.next().await
{
if let Some(server_name) = server_name.as_ref() {
if name != *server_name {
continue;
while let Some((name, CachedOverride { ips, port, expire, overriding })) =
overrides.next().await
{
if let Some(server_name) = server_name.as_ref() {
if name != *server_name {
continue;
}
}
let expire = time::format(expire, "%+");
self.write_str(&format!(
"| {name} | {ips:?} | {port} | {expire} | {overriding:?} |\n"
))
.await?;
}
let expire = time::format(expire, "%+");
self.write_str(&format!("| {name} | {ips:?} | {port} | {expire} | {overriding:?} |\n"))
.await?;
Ok(())
}
Ok(())
}
#[admin_command]
async fn flush_cache(&self, name: Option<OwnedServerName>, all: bool) -> Result {
if all {
self.services.resolver.cache.clear().await;
writeln!(self, "Resolver caches cleared!").await
} else if let Some(name) = name {
self.services.resolver.cache.del_destination(&name);
self.services.resolver.cache.del_override(&name);
self.write_str(&format!("Cleared {name} from resolver caches!"))
.await
} else {
Err!("Missing name. Supply a name or use --all to flush the whole cache.")
async fn flush_cache(&self, name: Option<OwnedServerName>, all: bool) -> Result {
if all {
self.services.resolver.cache.clear().await;
writeln!(self, "Resolver caches cleared!").await
} else if let Some(name) = name {
self.services.resolver.cache.del_destination(&name);
self.services.resolver.cache.del_override(&name);
self.write_str(&format!("Cleared {name} from resolver caches!"))
.await
} else {
Err!("Missing name. Supply a name or use --all to flush the whole cache.")
}
}
}
+30 -30
View File
@@ -3,7 +3,7 @@ use conduwuit::{PduCount, Result, utils::stream::TryTools};
use futures::TryStreamExt;
use ruma::OwnedRoomOrAliasId;
use crate::{admin_command, admin_command_dispatch};
use crate::admin_command_dispatch;
#[admin_command_dispatch]
#[derive(Debug, Subcommand)]
@@ -23,39 +23,39 @@ pub enum RoomTimelineCommand {
},
}
#[admin_command]
pub(super) async fn last(&self, room_id: OwnedRoomOrAliasId) -> Result {
let room_id = self.services.rooms.alias.resolve(&room_id).await?;
impl crate::Context<'_> {
pub(super) async fn last(&self, room_id: OwnedRoomOrAliasId) -> Result {
let room_id = self.services.rooms.alias.resolve(&room_id).await?;
let result = self
.services
.rooms
.timeline
.last_timeline_count(&room_id)
.await?;
let result = self
.services
.rooms
.timeline
.last_timeline_count(&room_id)
.await?;
self.write_str(&format!("{result:#?}")).await
}
self.write_str(&format!("{result:#?}")).await
}
#[admin_command]
pub(super) async fn pdus(
&self,
room_id: OwnedRoomOrAliasId,
from: Option<String>,
limit: Option<usize>,
) -> Result {
let room_id = self.services.rooms.alias.resolve(&room_id).await?;
pub(super) async fn pdus(
&self,
room_id: OwnedRoomOrAliasId,
from: Option<String>,
limit: Option<usize>,
) -> Result {
let room_id = self.services.rooms.alias.resolve(&room_id).await?;
let from: Option<PduCount> = from.as_deref().map(str::parse).transpose()?;
let from: Option<PduCount> = from.as_deref().map(str::parse).transpose()?;
let result: Vec<_> = self
.services
.rooms
.timeline
.pdus_rev(&room_id, from)
.try_take(limit.unwrap_or(3))
.try_collect()
.await?;
let result: Vec<_> = self
.services
.rooms
.timeline
.pdus_rev(&room_id, from)
.try_take(limit.unwrap_or(3))
.try_collect()
.await?;
self.write_str(&format!("```\n{result:#?}\n```")).await
self.write_str(&format!("```\n{result:#?}\n```")).await
}
}
+16 -16
View File
@@ -2,7 +2,7 @@ use clap::Subcommand;
use conduwuit::Result;
use ruma::{OwnedEventId, OwnedRoomOrAliasId};
use crate::{admin_command, admin_command_dispatch};
use crate::admin_command_dispatch;
#[admin_command_dispatch]
#[derive(Debug, Subcommand)]
@@ -17,23 +17,23 @@ pub enum ShortCommand {
},
}
#[admin_command]
pub(super) async fn short_event_id(&self, event_id: OwnedEventId) -> Result {
let shortid = self
.services
.rooms
.short
.get_shorteventid(&event_id)
.await?;
impl crate::Context<'_> {
pub(super) async fn short_event_id(&self, event_id: OwnedEventId) -> Result {
let shortid = self
.services
.rooms
.short
.get_shorteventid(&event_id)
.await?;
self.write_str(&format!("{shortid:#?}")).await
}
self.write_str(&format!("{shortid:#?}")).await
}
#[admin_command]
pub(super) async fn short_room_id(&self, room_id: OwnedRoomOrAliasId) -> Result {
let room_id = self.services.rooms.alias.resolve(&room_id).await?;
pub(super) async fn short_room_id(&self, room_id: OwnedRoomOrAliasId) -> Result {
let room_id = self.services.rooms.alias.resolve(&room_id).await?;
let shortid = self.services.rooms.short.get_shortroomid(&room_id).await?;
let shortid = self.services.rooms.short.get_shortroomid(&room_id).await?;
self.write_str(&format!("{shortid:#?}")).await
self.write_str(&format!("{shortid:#?}")).await
}
}
+255 -259
View File
@@ -3,7 +3,7 @@ use conduwuit::Result;
use futures::stream::StreamExt;
use ruma::{OwnedDeviceId, OwnedRoomId, OwnedUserId};
use crate::{admin_command, admin_command_dispatch};
use crate::admin_command_dispatch;
#[admin_command_dispatch]
#[derive(Debug, Subcommand)]
@@ -92,263 +92,259 @@ pub enum UsersCommand {
},
}
#[admin_command]
async fn get_shared_rooms(&self, user_a: OwnedUserId, user_b: OwnedUserId) -> Result {
let timer = tokio::time::Instant::now();
let result: Vec<_> = self
.services
.rooms
.state_cache
.get_shared_rooms(&user_a, &user_b)
.collect()
.await;
let query_time = timer.elapsed();
impl crate::Context<'_> {
async fn get_shared_rooms(&self, user_a: OwnedUserId, user_b: OwnedUserId) -> Result {
let timer = tokio::time::Instant::now();
let result: Vec<_> = self
.services
.rooms
.state_cache
.get_shared_rooms(&user_a, &user_b)
.collect()
.await;
let query_time = timer.elapsed();
self.write_str(&format!("Query completed in {query_time:?}:\n\n```rs\n{result:#?}\n```"))
.await
}
#[admin_command]
async fn get_backup_session(
&self,
user_id: OwnedUserId,
version: String,
room_id: OwnedRoomId,
session_id: String,
) -> Result {
let timer = tokio::time::Instant::now();
let result = self
.services
.key_backups
.get_session(&user_id, &version, &room_id, &session_id)
.await;
let query_time = timer.elapsed();
self.write_str(&format!("Query completed in {query_time:?}:\n\n```rs\n{result:#?}\n```"))
.await
}
#[admin_command]
async fn get_room_backups(
&self,
user_id: OwnedUserId,
version: String,
room_id: OwnedRoomId,
) -> Result {
let timer = tokio::time::Instant::now();
let result = self
.services
.key_backups
.get_room(&user_id, &version, &room_id)
.await;
let query_time = timer.elapsed();
self.write_str(&format!("Query completed in {query_time:?}:\n\n```rs\n{result:#?}\n```"))
.await
}
#[admin_command]
async fn get_all_backups(&self, user_id: OwnedUserId, version: String) -> Result {
let timer = tokio::time::Instant::now();
let result = self.services.key_backups.get_all(&user_id, &version).await;
let query_time = timer.elapsed();
self.write_str(&format!("Query completed in {query_time:?}:\n\n```rs\n{result:#?}\n```"))
.await
}
#[admin_command]
async fn get_backup_algorithm(&self, user_id: OwnedUserId, version: String) -> Result {
let timer = tokio::time::Instant::now();
let result = self
.services
.key_backups
.get_backup(&user_id, &version)
.await;
let query_time = timer.elapsed();
self.write_str(&format!("Query completed in {query_time:?}:\n\n```rs\n{result:#?}\n```"))
.await
}
#[admin_command]
async fn get_latest_backup_version(&self, user_id: OwnedUserId) -> Result {
let timer = tokio::time::Instant::now();
let result = self
.services
.key_backups
.get_latest_backup_version(&user_id)
.await;
let query_time = timer.elapsed();
self.write_str(&format!("Query completed in {query_time:?}:\n\n```rs\n{result:#?}\n```"))
.await
}
#[admin_command]
async fn get_latest_backup(&self, user_id: OwnedUserId) -> Result {
let timer = tokio::time::Instant::now();
let result = self.services.key_backups.get_latest_backup(&user_id).await;
let query_time = timer.elapsed();
self.write_str(&format!("Query completed in {query_time:?}:\n\n```rs\n{result:#?}\n```"))
.await
}
#[admin_command]
async fn iter_users(&self) -> Result {
let timer = tokio::time::Instant::now();
let result: Vec<OwnedUserId> = self.services.users.stream().map(Into::into).collect().await;
let query_time = timer.elapsed();
self.write_str(&format!("Query completed in {query_time:?}:\n\n```rs\n{result:#?}\n```"))
.await
}
#[admin_command]
async fn iter_users2(&self) -> Result {
let timer = tokio::time::Instant::now();
let result: Vec<_> = self.services.users.stream().collect().await;
let result: Vec<_> = result
.into_iter()
.map(|user_id| String::from_utf8_lossy(user_id.as_bytes()).into_owned())
.collect();
let query_time = timer.elapsed();
self.write_str(&format!("Query completed in {query_time:?}:\n\n```rs\n{result:?}\n```"))
.await
}
#[admin_command]
async fn count_users(&self) -> Result {
let timer = tokio::time::Instant::now();
let result = self.services.users.count().await;
let query_time = timer.elapsed();
self.write_str(&format!("Query completed in {query_time:?}:\n\n```rs\n{result:#?}\n```"))
.await
}
#[admin_command]
async fn list_devices(&self, user_id: OwnedUserId) -> Result {
let timer = tokio::time::Instant::now();
let devices = self
.services
.users
.all_device_ids(&user_id)
.collect::<Vec<_>>()
.await;
let query_time = timer.elapsed();
self.write_str(&format!("Query completed in {query_time:?}:\n\n```rs\n{devices:#?}\n```"))
.await
}
#[admin_command]
async fn list_devices_metadata(&self, user_id: OwnedUserId) -> Result {
let timer = tokio::time::Instant::now();
let devices = self
.services
.users
.all_devices_metadata(&user_id)
.collect::<Vec<_>>()
.await;
let query_time = timer.elapsed();
self.write_str(&format!("Query completed in {query_time:?}:\n\n```rs\n{devices:#?}\n```"))
.await
}
#[admin_command]
async fn get_device_metadata(&self, user_id: OwnedUserId, device_id: OwnedDeviceId) -> Result {
let timer = tokio::time::Instant::now();
let device = self
.services
.users
.get_device_metadata(&user_id, &device_id)
.await;
let query_time = timer.elapsed();
self.write_str(&format!("Query completed in {query_time:?}:\n\n```rs\n{device:#?}\n```"))
.await
}
#[admin_command]
async fn get_devices_version(&self, user_id: OwnedUserId) -> Result {
let timer = tokio::time::Instant::now();
let device = self.services.users.get_devicelist_version(&user_id).await;
let query_time = timer.elapsed();
self.write_str(&format!("Query completed in {query_time:?}:\n\n```rs\n{device:#?}\n```"))
.await
}
#[admin_command]
async fn count_one_time_keys(&self, user_id: OwnedUserId, device_id: OwnedDeviceId) -> Result {
let timer = tokio::time::Instant::now();
let result = self
.services
.users
.count_one_time_keys(&user_id, &device_id)
.await;
let query_time = timer.elapsed();
self.write_str(&format!("Query completed in {query_time:?}:\n\n```rs\n{result:#?}\n```"))
.await
}
#[admin_command]
async fn get_device_keys(&self, user_id: OwnedUserId, device_id: OwnedDeviceId) -> Result {
let timer = tokio::time::Instant::now();
let result = self
.services
.users
.get_device_keys(&user_id, &device_id)
.await;
let query_time = timer.elapsed();
self.write_str(&format!("Query completed in {query_time:?}:\n\n```rs\n{result:#?}\n```"))
.await
}
#[admin_command]
async fn get_user_signing_key(&self, user_id: OwnedUserId) -> Result {
let timer = tokio::time::Instant::now();
let result = self.services.users.get_user_signing_key(&user_id).await;
let query_time = timer.elapsed();
self.write_str(&format!("Query completed in {query_time:?}:\n\n```rs\n{result:#?}\n```"))
.await
}
#[admin_command]
async fn get_master_key(&self, user_id: OwnedUserId) -> Result {
let timer = tokio::time::Instant::now();
let result = self
.services
.users
.get_master_key(None, &user_id, &|_| true)
.await;
let query_time = timer.elapsed();
self.write_str(&format!("Query completed in {query_time:?}:\n\n```rs\n{result:#?}\n```"))
.await
}
#[admin_command]
async fn get_to_device_events(&self, user_id: OwnedUserId, device_id: OwnedDeviceId) -> Result {
let timer = tokio::time::Instant::now();
let result = self
.services
.users
.get_to_device_events(&user_id, &device_id, None, None)
.collect::<Vec<_>>()
.await;
let query_time = timer.elapsed();
self.write_str(&format!("Query completed in {query_time:?}:\n\n```rs\n{result:#?}\n```"))
.await
self.write_str(&format!("Query completed in {query_time:?}:\n\n```rs\n{result:#?}\n```"))
.await
}
async fn get_backup_session(
&self,
user_id: OwnedUserId,
version: String,
room_id: OwnedRoomId,
session_id: String,
) -> Result {
let timer = tokio::time::Instant::now();
let result = self
.services
.key_backups
.get_session(&user_id, &version, &room_id, &session_id)
.await;
let query_time = timer.elapsed();
self.write_str(&format!("Query completed in {query_time:?}:\n\n```rs\n{result:#?}\n```"))
.await
}
async fn get_room_backups(
&self,
user_id: OwnedUserId,
version: String,
room_id: OwnedRoomId,
) -> Result {
let timer = tokio::time::Instant::now();
let result = self
.services
.key_backups
.get_room(&user_id, &version, &room_id)
.await;
let query_time = timer.elapsed();
self.write_str(&format!("Query completed in {query_time:?}:\n\n```rs\n{result:#?}\n```"))
.await
}
async fn get_all_backups(&self, user_id: OwnedUserId, version: String) -> Result {
let timer = tokio::time::Instant::now();
let result = self.services.key_backups.get_all(&user_id, &version).await;
let query_time = timer.elapsed();
self.write_str(&format!("Query completed in {query_time:?}:\n\n```rs\n{result:#?}\n```"))
.await
}
async fn get_backup_algorithm(&self, user_id: OwnedUserId, version: String) -> Result {
let timer = tokio::time::Instant::now();
let result = self
.services
.key_backups
.get_backup(&user_id, &version)
.await;
let query_time = timer.elapsed();
self.write_str(&format!("Query completed in {query_time:?}:\n\n```rs\n{result:#?}\n```"))
.await
}
async fn get_latest_backup_version(&self, user_id: OwnedUserId) -> Result {
let timer = tokio::time::Instant::now();
let result = self
.services
.key_backups
.get_latest_backup_version(&user_id)
.await;
let query_time = timer.elapsed();
self.write_str(&format!("Query completed in {query_time:?}:\n\n```rs\n{result:#?}\n```"))
.await
}
async fn get_latest_backup(&self, user_id: OwnedUserId) -> Result {
let timer = tokio::time::Instant::now();
let result = self.services.key_backups.get_latest_backup(&user_id).await;
let query_time = timer.elapsed();
self.write_str(&format!("Query completed in {query_time:?}:\n\n```rs\n{result:#?}\n```"))
.await
}
async fn iter_users(&self) -> Result {
let timer = tokio::time::Instant::now();
let result: Vec<OwnedUserId> =
self.services.users.stream().map(Into::into).collect().await;
let query_time = timer.elapsed();
self.write_str(&format!("Query completed in {query_time:?}:\n\n```rs\n{result:#?}\n```"))
.await
}
async fn iter_users2(&self) -> Result {
let timer = tokio::time::Instant::now();
let result: Vec<_> = self.services.users.stream().collect().await;
let result: Vec<_> = result
.into_iter()
.map(|user_id| String::from_utf8_lossy(user_id.as_bytes()).into_owned())
.collect();
let query_time = timer.elapsed();
self.write_str(&format!("Query completed in {query_time:?}:\n\n```rs\n{result:?}\n```"))
.await
}
async fn count_users(&self) -> Result {
let timer = tokio::time::Instant::now();
let result = self.services.users.count().await;
let query_time = timer.elapsed();
self.write_str(&format!("Query completed in {query_time:?}:\n\n```rs\n{result:#?}\n```"))
.await
}
async fn list_devices(&self, user_id: OwnedUserId) -> Result {
let timer = tokio::time::Instant::now();
let devices = self
.services
.users
.all_device_ids(&user_id)
.collect::<Vec<_>>()
.await;
let query_time = timer.elapsed();
self.write_str(&format!("Query completed in {query_time:?}:\n\n```rs\n{devices:#?}\n```"))
.await
}
async fn list_devices_metadata(&self, user_id: OwnedUserId) -> Result {
let timer = tokio::time::Instant::now();
let devices = self
.services
.users
.all_devices_metadata(&user_id)
.collect::<Vec<_>>()
.await;
let query_time = timer.elapsed();
self.write_str(&format!("Query completed in {query_time:?}:\n\n```rs\n{devices:#?}\n```"))
.await
}
async fn get_device_metadata(
&self,
user_id: OwnedUserId,
device_id: OwnedDeviceId,
) -> Result {
let timer = tokio::time::Instant::now();
let device = self
.services
.users
.get_device_metadata(&user_id, &device_id)
.await;
let query_time = timer.elapsed();
self.write_str(&format!("Query completed in {query_time:?}:\n\n```rs\n{device:#?}\n```"))
.await
}
async fn get_devices_version(&self, user_id: OwnedUserId) -> Result {
let timer = tokio::time::Instant::now();
let device = self.services.users.get_devicelist_version(&user_id).await;
let query_time = timer.elapsed();
self.write_str(&format!("Query completed in {query_time:?}:\n\n```rs\n{device:#?}\n```"))
.await
}
async fn count_one_time_keys(
&self,
user_id: OwnedUserId,
device_id: OwnedDeviceId,
) -> Result {
let timer = tokio::time::Instant::now();
let result = self
.services
.users
.count_one_time_keys(&user_id, &device_id)
.await;
let query_time = timer.elapsed();
self.write_str(&format!("Query completed in {query_time:?}:\n\n```rs\n{result:#?}\n```"))
.await
}
async fn get_device_keys(&self, user_id: OwnedUserId, device_id: OwnedDeviceId) -> Result {
let timer = tokio::time::Instant::now();
let result = self
.services
.users
.get_device_keys(&user_id, &device_id)
.await;
let query_time = timer.elapsed();
self.write_str(&format!("Query completed in {query_time:?}:\n\n```rs\n{result:#?}\n```"))
.await
}
async fn get_user_signing_key(&self, user_id: OwnedUserId) -> Result {
let timer = tokio::time::Instant::now();
let result = self.services.users.get_user_signing_key(&user_id).await;
let query_time = timer.elapsed();
self.write_str(&format!("Query completed in {query_time:?}:\n\n```rs\n{result:#?}\n```"))
.await
}
async fn get_master_key(&self, user_id: OwnedUserId) -> Result {
let timer = tokio::time::Instant::now();
let result = self
.services
.users
.get_master_key(None, &user_id, &|_| true)
.await;
let query_time = timer.elapsed();
self.write_str(&format!("Query completed in {query_time:?}:\n\n```rs\n{result:#?}\n```"))
.await
}
async fn get_to_device_events(
&self,
user_id: OwnedUserId,
device_id: OwnedDeviceId,
) -> Result {
let timer = tokio::time::Instant::now();
let result = self
.services
.users
.get_to_device_events(&user_id, &device_id, None, None)
.collect::<Vec<_>>()
.await;
let query_time = timer.elapsed();
self.write_str(&format!("Query completed in {query_time:?}:\n\n```rs\n{result:#?}\n```"))
.await
}
}
+72 -72
View File
@@ -2,83 +2,83 @@ use conduwuit::{Err, Result};
use futures::StreamExt;
use ruma::OwnedRoomId;
use crate::{PAGE_SIZE, admin_command, get_room_info};
use crate::{PAGE_SIZE, get_room_info};
#[allow(clippy::fn_params_excessive_bools)]
#[admin_command]
pub(super) async fn list_rooms(
&self,
page: Option<usize>,
exclude_disabled: bool,
exclude_banned: bool,
include_empty: bool,
no_details: bool,
) -> Result {
// TODO: i know there's a way to do this with clap, but i can't seem to find it
let page = page.unwrap_or(1);
let mut rooms = self
.services
.rooms
.metadata
.iter_ids()
.filter_map(|room_id| async move {
(!exclude_disabled || !self.services.rooms.metadata.is_disabled(&room_id).await)
.then_some(room_id)
})
.filter_map(|room_id| async move {
(!exclude_banned || !self.services.rooms.metadata.is_banned(&room_id).await)
.then_some(room_id)
})
.then(async |room_id| get_room_info(self.services, &room_id).await)
.then(|(room_id, total_members, name)| async move {
let local_members: Vec<_> = self
.services
.rooms
.state_cache
.active_local_users_in_room(&room_id)
.collect()
.await;
let local_members = local_members.len();
(room_id, total_members, local_members, name)
})
.filter_map(|(room_id, total_members, local_members, name)| async move {
(include_empty || local_members > 0).then_some((room_id, total_members, name))
})
.collect::<Vec<_>>()
.await;
impl crate::Context<'_> {
#[allow(clippy::fn_params_excessive_bools)]
pub(super) async fn list_rooms(
&self,
page: Option<usize>,
exclude_disabled: bool,
exclude_banned: bool,
include_empty: bool,
no_details: bool,
) -> Result {
// TODO: i know there's a way to do this with clap, but i can't seem to find it
let page = page.unwrap_or(1);
let mut rooms = self
.services
.rooms
.metadata
.iter_ids()
.filter_map(|room_id| async move {
(!exclude_disabled || !self.services.rooms.metadata.is_disabled(&room_id).await)
.then_some(room_id)
})
.filter_map(|room_id| async move {
(!exclude_banned || !self.services.rooms.metadata.is_banned(&room_id).await)
.then_some(room_id)
})
.then(async |room_id| get_room_info(self.services, &room_id).await)
.then(|(room_id, total_members, name)| async move {
let local_members: Vec<_> = self
.services
.rooms
.state_cache
.active_local_users_in_room(&room_id)
.collect()
.await;
let local_members = local_members.len();
(room_id, total_members, local_members, name)
})
.filter_map(|(room_id, total_members, local_members, name)| async move {
(include_empty || local_members > 0).then_some((room_id, total_members, name))
})
.collect::<Vec<_>>()
.await;
rooms.sort_by_key(|r| r.1);
rooms.reverse();
rooms.sort_by_key(|r| r.1);
rooms.reverse();
let rooms = rooms
.into_iter()
.skip(page.saturating_sub(1).saturating_mul(PAGE_SIZE))
.take(PAGE_SIZE)
.collect::<Vec<_>>();
let rooms = rooms
.into_iter()
.skip(page.saturating_sub(1).saturating_mul(PAGE_SIZE))
.take(PAGE_SIZE)
.collect::<Vec<_>>();
if rooms.is_empty() {
return Err!("No more rooms.");
if rooms.is_empty() {
return Err!("No more rooms.");
}
let body = rooms
.iter()
.map(|(id, members, name)| {
if no_details {
format!("{id}")
} else {
format!("{id}\tMembers: {members}\tName: {name}")
}
})
.collect::<Vec<_>>()
.join("\n");
self.write_str(&format!("Rooms ({}):\n```\n{body}\n```", rooms.len()))
.await
}
let body = rooms
.iter()
.map(|(id, members, name)| {
if no_details {
format!("{id}")
} else {
format!("{id}\tMembers: {members}\tName: {name}")
}
})
.collect::<Vec<_>>()
.join("\n");
pub(super) async fn exists(&self, room_id: OwnedRoomId) -> Result {
let result = self.services.rooms.metadata.exists(&room_id).await;
self.write_str(&format!("Rooms ({}):\n```\n{body}\n```", rooms.len()))
.await
}
#[admin_command]
pub(super) async fn exists(&self, room_id: OwnedRoomId) -> Result {
let result = self.services.rooms.metadata.exists(&room_id).await;
self.write_str(&format!("{result}")).await
self.write_str(&format!("{result}")).await
}
}
+54 -54
View File
@@ -3,7 +3,7 @@ use conduwuit::{Err, Result, utils::ReadyExt};
use futures::StreamExt;
use ruma::OwnedRoomId;
use crate::{admin_command, admin_command_dispatch};
use crate::admin_command_dispatch;
#[admin_command_dispatch]
#[derive(Debug, Subcommand)]
@@ -26,62 +26,62 @@ pub enum RoomInfoCommand {
},
}
#[admin_command]
async fn list_joined_members(&self, room_id: OwnedRoomId, local_only: bool) -> Result {
let room_name = self
.services
.rooms
.state_accessor
.get_name(&room_id)
.await
.unwrap_or_else(|_| room_id.to_string());
impl crate::Context<'_> {
async fn list_joined_members(&self, room_id: OwnedRoomId, local_only: bool) -> Result {
let room_name = self
.services
.rooms
.state_accessor
.get_name(&room_id)
.await
.unwrap_or_else(|_| room_id.to_string());
let member_info: Vec<_> = self
.services
.rooms
.state_cache
.room_members(&room_id)
.ready_filter(|user_id| {
local_only
.then(|| self.services.globals.user_is_local(user_id))
.unwrap_or(true)
})
.filter_map(|user_id| async move {
Some((
self.services
.users
.displayname(&user_id)
.await
.unwrap_or_else(|_| user_id.to_string()),
user_id,
))
})
.collect()
.await;
let member_info: Vec<_> = self
.services
.rooms
.state_cache
.room_members(&room_id)
.ready_filter(|user_id| {
local_only
.then(|| self.services.globals.user_is_local(user_id))
.unwrap_or(true)
})
.filter_map(|user_id| async move {
Some((
self.services
.users
.displayname(&user_id)
.await
.unwrap_or_else(|_| user_id.to_string()),
user_id,
))
})
.collect()
.await;
let num = member_info.len();
let body = member_info
.into_iter()
.map(|(displayname, mxid)| format!("{mxid} | {displayname}"))
.collect::<Vec<_>>()
.join("\n");
let num = member_info.len();
let body = member_info
.into_iter()
.map(|(displayname, mxid)| format!("{mxid} | {displayname}"))
.collect::<Vec<_>>()
.join("\n");
self.write_str(&format!("{num} Members in Room \"{room_name}\":\n```\n{body}\n```"))
.await
}
self.write_str(&format!("{num} Members in Room \"{room_name}\":\n```\n{body}\n```"))
.await
}
#[admin_command]
async fn view_room_topic(&self, room_id: OwnedRoomId) -> Result {
let Ok(room_topic) = self
.services
.rooms
.state_accessor
.get_room_topic(&room_id)
.await
else {
return Err!("Room does not have a room topic set.");
};
async fn view_room_topic(&self, room_id: OwnedRoomId) -> Result {
let Ok(room_topic) = self
.services
.rooms
.state_accessor
.get_room_topic(&room_id)
.await
else {
return Err!("Room does not have a room topic set.");
};
self.write_str(&format!("Room topic:\n```\n{room_topic}\n```"))
.await
self.write_str(&format!("Room topic:\n```\n{room_topic}\n```"))
.await
}
}
+352 -351
View File
@@ -8,7 +8,7 @@ use conduwuit::{
use futures::{FutureExt, StreamExt};
use ruma::{OwnedRoomId, OwnedRoomOrAliasId, RoomAliasId, RoomId, RoomOrAliasId};
use crate::{admin_command, admin_command_dispatch, get_room_info};
use crate::{admin_command_dispatch, get_room_info};
#[admin_command_dispatch]
#[derive(Debug, Subcommand)]
@@ -45,238 +45,72 @@ pub enum RoomModerationCommand {
},
}
#[admin_command]
async fn ban_room(&self, room: OwnedRoomOrAliasId) -> Result {
debug!("Got room alias or ID: {}", room);
impl crate::Context<'_> {
async fn ban_room(&self, room: OwnedRoomOrAliasId) -> Result {
debug!("Got room alias or ID: {}", room);
let admin_room_alias = &self.services.globals.admin_alias;
let admin_room_alias = &self.services.globals.admin_alias;
if let Ok(admin_room_id) = self.services.admin.get_admin_room().await {
if room.to_string().eq(&admin_room_id) || room.to_string().eq(admin_room_alias) {
return Err!("Not allowed to ban the admin room.");
if let Ok(admin_room_id) = self.services.admin.get_admin_room().await {
if room.to_string().eq(&admin_room_id) || room.to_string().eq(admin_room_alias) {
return Err!("Not allowed to ban the admin room.");
}
}
}
let room_id = if room.is_room_id() {
let room_id = match RoomId::parse(&room) {
| Ok(room_id) => room_id,
| Err(e) => {
return Err!(
"Failed to parse room ID {room}. Please note that this requires a full room \
ID (`!awIh6gGInaS5wLQJwa:example.com`) or a room alias \
(`#roomalias:example.com`): {e}"
);
},
let room_id = if room.is_room_id() {
let room_id = match RoomId::parse(&room) {
| Ok(room_id) => room_id,
| Err(e) => {
return Err!(
"Failed to parse room ID {room}. Please note that this requires a full \
room ID (`!awIh6gGInaS5wLQJwa:example.com`) or a room alias \
(`#roomalias:example.com`): {e}"
);
},
};
debug!("Room specified is a room ID, banning room ID");
room_id.clone()
} else if room.is_room_alias_id() {
let room_alias = match RoomAliasId::parse(&room) {
| Ok(room_alias) => room_alias,
| Err(e) => {
return Err!(
"Failed to parse room ID {room}. Please note that this requires a full \
room ID (`!awIh6gGInaS5wLQJwa:example.com`) or a room alias \
(`#roomalias:example.com`): {e}"
);
},
};
debug!(
"Room specified is not a room ID, attempting to resolve room alias to a room ID \
locally, if not using get_alias_helper to fetch room ID remotely"
);
match self.services.rooms.alias.resolve_alias(&room_alias).await {
| Ok((room_id, servers)) => {
debug!(
%room_id,
?servers,
"Got federation response fetching room ID for room {room}"
);
room_id
},
| Err(e) => {
return Err!("Failed to resolve room alias {room} to a room ID: {e}");
},
}
} else {
return Err!(
"Room specified is not a room ID or room alias. Please note that this requires \
a full room ID (`!awIh6gGInaS5wLQJwa:example.com`) or a room alias \
(`#roomalias:example.com`)",
);
};
debug!("Room specified is a room ID, banning room ID");
room_id.clone()
} else if room.is_room_alias_id() {
let room_alias = match RoomAliasId::parse(&room) {
| Ok(room_alias) => room_alias,
| Err(e) => {
return Err!(
"Failed to parse room ID {room}. Please note that this requires a full room \
ID (`!awIh6gGInaS5wLQJwa:example.com`) or a room alias \
(`#roomalias:example.com`): {e}"
);
},
};
debug!(
"Room specified is not a room ID, attempting to resolve room alias to a room ID \
locally, if not using get_alias_helper to fetch room ID remotely"
);
match self.services.rooms.alias.resolve_alias(&room_alias).await {
| Ok((room_id, servers)) => {
debug!(
%room_id,
?servers,
"Got federation response fetching room ID for room {room}"
);
room_id
},
| Err(e) => {
return Err!("Failed to resolve room alias {room} to a room ID: {e}");
},
}
} else {
return Err!(
"Room specified is not a room ID or room alias. Please note that this requires a \
full room ID (`!awIh6gGInaS5wLQJwa:example.com`) or a room alias \
(`#roomalias:example.com`)",
);
};
info!("Making all users leave the room {room_id} and forgetting it");
let mut users = self
.services
.rooms
.state_cache
.room_members(&room_id)
.ready_filter(|user| self.services.globals.user_is_local(user))
.boxed();
while let Some(ref user_id) = users.next().await {
info!(
"Attempting leave for user {user_id} in room {room_id} (ignoring all errors, \
evicting admins too)",
);
if let Err(e) = leave_room(self.services, user_id, &room_id, None)
.boxed()
.await
{
warn!("Failed to leave room: {e}");
}
self.services.rooms.state_cache.forget(&room_id, user_id);
}
self.services
.rooms
.alias
.local_aliases_for_room(&room_id)
.for_each(|local_alias| async move {
self.services
.rooms
.alias
.remove_alias(&local_alias, &self.services.globals.server_user)
.await
.ok();
})
.await;
self.services.rooms.directory.set_not_public(&room_id); // remove from the room directory
self.services.rooms.metadata.ban_room(&room_id, true); // prevent further joins
self.services.rooms.metadata.disable_room(&room_id, true); // disable federation
self.write_str(
"Room banned, removed all our local users, and disabled incoming federation with room.",
)
.await
}
#[admin_command]
async fn ban_list_of_rooms(&self) -> Result {
if self.body.len() < 2
|| !self.body[0].trim().starts_with("```")
|| self.body.last().unwrap_or(&"").trim() != "```"
{
return Err!("Expected code block in command body. Add --help for details.",);
}
let rooms_s = self
.body
.to_vec()
.drain(1..self.body.len().saturating_sub(1))
.collect::<Vec<_>>();
let admin_room_alias = &self.services.globals.admin_alias;
let mut room_ban_count: usize = 0;
let mut room_ids: Vec<OwnedRoomId> = Vec::new();
for &room in &rooms_s {
match <&RoomOrAliasId>::try_from(room) {
| Ok(room_alias_or_id) => {
if let Ok(admin_room_id) = self.services.admin.get_admin_room().await {
if room.to_owned().eq(&admin_room_id) || room.to_owned().eq(admin_room_alias)
{
warn!("User specified admin room in bulk ban list, ignoring");
continue;
}
}
if room_alias_or_id.is_room_id() {
let room_id = match RoomId::parse(room_alias_or_id) {
| Ok(room_id) => room_id,
| Err(e) => {
// ignore rooms we failed to parse
warn!(
"Error parsing room \"{room}\" during bulk room banning, \
ignoring error and logging here: {e}"
);
continue;
},
};
room_ids.push(room_id.clone());
}
if room_alias_or_id.is_room_alias_id() {
match RoomAliasId::parse(room_alias_or_id) {
| Ok(room_alias) => {
let room_id = match self
.services
.rooms
.alias
.resolve_local_alias(&room_alias)
.await
{
| Ok(room_id) => room_id,
| _ => {
debug!(
"We don't have this room alias to a room ID locally, \
attempting to fetch room ID over federation"
);
match self
.services
.rooms
.alias
.resolve_alias(&room_alias)
.await
{
| Ok((room_id, servers)) => {
debug!(
%room_id,
?servers,
"Got federation response fetching room ID for \
{room}",
);
room_id
},
| Err(e) => {
warn!(
"Failed to resolve room alias {room} to a room \
ID: {e}"
);
continue;
},
}
},
};
room_ids.push(room_id);
},
| Err(e) => {
warn!(
"Error parsing room \"{room}\" during bulk room banning, \
ignoring error and logging here: {e}"
);
continue;
},
}
}
},
| Err(e) => {
warn!(
"Error parsing room \"{room}\" during bulk room banning, ignoring error and \
logging here: {e}"
);
continue;
},
}
}
for room_id in room_ids {
debug!("Banned {room_id} successfully");
room_ban_count = room_ban_count.saturating_add(1);
debug!("Making all users leave the room {room_id} and forgetting it");
info!("Making all users leave the room {room_id} and forgetting it");
let mut users = self
.services
.rooms
@@ -286,7 +120,7 @@ async fn ban_list_of_rooms(&self) -> Result {
.boxed();
while let Some(ref user_id) = users.next().await {
debug!(
info!(
"Attempting leave for user {user_id} in room {room_id} (ignoring all errors, \
evicting admins too)",
);
@@ -301,7 +135,6 @@ async fn ban_list_of_rooms(&self) -> Result {
self.services.rooms.state_cache.forget(&room_id, user_id);
}
// remove any local aliases, ignore errors
self.services
.rooms
.alias
@@ -316,139 +149,307 @@ async fn ban_list_of_rooms(&self) -> Result {
})
.await;
self.services.rooms.metadata.ban_room(&room_id, true);
// unpublish from room directory, ignore errors
self.services.rooms.directory.set_not_public(&room_id);
self.services.rooms.metadata.disable_room(&room_id, true);
self.services.rooms.directory.set_not_public(&room_id); // remove from the room directory
self.services.rooms.metadata.ban_room(&room_id, true); // prevent further joins
self.services.rooms.metadata.disable_room(&room_id, true); // disable federation
self.write_str(
"Room banned, removed all our local users, and disabled incoming federation with \
room.",
)
.await
}
self.write_str(&format!(
"Finished bulk room ban, banned {room_ban_count} total rooms, evicted all users, and \
disabled incoming federation with the room."
))
.await
}
async fn ban_list_of_rooms(&self) -> Result {
if self.body.len() < 2
|| !self.body[0].trim().starts_with("```")
|| self.body.last().unwrap_or(&"").trim() != "```"
{
return Err!("Expected code block in command body. Add --help for details.",);
}
#[admin_command]
async fn unban_room(&self, room: OwnedRoomOrAliasId) -> Result {
let room_id = if room.is_room_id() {
let room_id = match RoomId::parse(&room) {
| Ok(room_id) => room_id,
| Err(e) => {
return Err!(
"Failed to parse room ID {room}. Please note that this requires a full room \
ID (`!awIh6gGInaS5wLQJwa:example.com`) or a room alias \
(`#roomalias:example.com`): {e}"
let rooms_s = self
.body
.to_vec()
.drain(1..self.body.len().saturating_sub(1))
.collect::<Vec<_>>();
let admin_room_alias = &self.services.globals.admin_alias;
let mut room_ban_count: usize = 0;
let mut room_ids: Vec<OwnedRoomId> = Vec::new();
for &room in &rooms_s {
match <&RoomOrAliasId>::try_from(room) {
| Ok(room_alias_or_id) => {
if let Ok(admin_room_id) = self.services.admin.get_admin_room().await {
if room.to_owned().eq(&admin_room_id)
|| room.to_owned().eq(admin_room_alias)
{
warn!("User specified admin room in bulk ban list, ignoring");
continue;
}
}
if room_alias_or_id.is_room_id() {
let room_id = match RoomId::parse(room_alias_or_id) {
| Ok(room_id) => room_id,
| Err(e) => {
// ignore rooms we failed to parse
warn!(
"Error parsing room \"{room}\" during bulk room banning, \
ignoring error and logging here: {e}"
);
continue;
},
};
room_ids.push(room_id.clone());
}
if room_alias_or_id.is_room_alias_id() {
match RoomAliasId::parse(room_alias_or_id) {
| Ok(room_alias) => {
let room_id = match self
.services
.rooms
.alias
.resolve_local_alias(&room_alias)
.await
{
| Ok(room_id) => room_id,
| _ => {
debug!(
"We don't have this room alias to a room ID \
locally, attempting to fetch room ID over \
federation"
);
match self
.services
.rooms
.alias
.resolve_alias(&room_alias)
.await
{
| Ok((room_id, servers)) => {
debug!(
%room_id,
?servers,
"Got federation response fetching room ID for \
{room}",
);
room_id
},
| Err(e) => {
warn!(
"Failed to resolve room alias {room} to a \
room ID: {e}"
);
continue;
},
}
},
};
room_ids.push(room_id);
},
| Err(e) => {
warn!(
"Error parsing room \"{room}\" during bulk room banning, \
ignoring error and logging here: {e}"
);
continue;
},
}
}
},
| Err(e) => {
warn!(
"Error parsing room \"{room}\" during bulk room banning, ignoring error \
and logging here: {e}"
);
continue;
},
}
}
for room_id in room_ids {
debug!("Banned {room_id} successfully");
room_ban_count = room_ban_count.saturating_add(1);
debug!("Making all users leave the room {room_id} and forgetting it");
let mut users = self
.services
.rooms
.state_cache
.room_members(&room_id)
.ready_filter(|user| self.services.globals.user_is_local(user))
.boxed();
while let Some(ref user_id) = users.next().await {
debug!(
"Attempting leave for user {user_id} in room {room_id} (ignoring all \
errors, evicting admins too)",
);
},
if let Err(e) = leave_room(self.services, user_id, &room_id, None)
.boxed()
.await
{
warn!("Failed to leave room: {e}");
}
self.services.rooms.state_cache.forget(&room_id, user_id);
}
// remove any local aliases, ignore errors
self.services
.rooms
.alias
.local_aliases_for_room(&room_id)
.for_each(|local_alias| async move {
self.services
.rooms
.alias
.remove_alias(&local_alias, &self.services.globals.server_user)
.await
.ok();
})
.await;
self.services.rooms.metadata.ban_room(&room_id, true);
// unpublish from room directory, ignore errors
self.services.rooms.directory.set_not_public(&room_id);
self.services.rooms.metadata.disable_room(&room_id, true);
}
self.write_str(&format!(
"Finished bulk room ban, banned {room_ban_count} total rooms, evicted all users, \
and disabled incoming federation with the room."
))
.await
}
async fn unban_room(&self, room: OwnedRoomOrAliasId) -> Result {
let room_id = if room.is_room_id() {
let room_id = match RoomId::parse(&room) {
| Ok(room_id) => room_id,
| Err(e) => {
return Err!(
"Failed to parse room ID {room}. Please note that this requires a full \
room ID (`!awIh6gGInaS5wLQJwa:example.com`) or a room alias \
(`#roomalias:example.com`): {e}"
);
},
};
debug!("Room specified is a room ID, unbanning room ID");
self.services.rooms.metadata.ban_room(&room_id, false);
room_id.clone()
} else if room.is_room_alias_id() {
let room_alias = match RoomAliasId::parse(&room) {
| Ok(room_alias) => room_alias,
| Err(e) => {
return Err!(
"Failed to parse room ID {room}. Please note that this requires a full \
room ID (`!awIh6gGInaS5wLQJwa:example.com`) or a room alias \
(`#roomalias:example.com`): {e}"
);
},
};
debug!(
"Room specified is not a room ID, attempting to resolve room alias to a room ID \
locally, if not using get_alias_helper to fetch room ID remotely"
);
let room_id = match self
.services
.rooms
.alias
.resolve_local_alias(&room_alias)
.await
{
| Ok(room_id) => room_id,
| _ => {
debug!(
"We don't have this room alias to a room ID locally, attempting to \
fetch room ID over federation"
);
match self.services.rooms.alias.resolve_alias(&room_alias).await {
| Ok((room_id, servers)) => {
debug!(
%room_id,
?servers,
"Got federation response fetching room ID for room {room}"
);
room_id
},
| Err(e) => {
return Err!("Failed to resolve room alias {room} to a room ID: {e}");
},
}
},
};
self.services.rooms.metadata.ban_room(&room_id, false);
room_id
} else {
return Err!(
"Room specified is not a room ID or room alias. Please note that this requires \
a full room ID (`!awIh6gGInaS5wLQJwa:example.com`) or a room alias \
(`#roomalias:example.com`)",
);
};
debug!("Room specified is a room ID, unbanning room ID");
self.services.rooms.metadata.ban_room(&room_id, false);
self.services.rooms.metadata.disable_room(&room_id, false);
self.write_str("Room unbanned and federation re-enabled.")
.await
}
room_id.clone()
} else if room.is_room_alias_id() {
let room_alias = match RoomAliasId::parse(&room) {
| Ok(room_alias) => room_alias,
| Err(e) => {
return Err!(
"Failed to parse room ID {room}. Please note that this requires a full room \
ID (`!awIh6gGInaS5wLQJwa:example.com`) or a room alias \
(`#roomalias:example.com`): {e}"
);
},
};
debug!(
"Room specified is not a room ID, attempting to resolve room alias to a room ID \
locally, if not using get_alias_helper to fetch room ID remotely"
);
let room_id = match self
async fn list_banned_rooms(&self, no_details: bool) -> Result {
let room_ids: Vec<OwnedRoomId> = self
.services
.rooms
.alias
.resolve_local_alias(&room_alias)
.await
{
| Ok(room_id) => room_id,
| _ => {
debug!(
"We don't have this room alias to a room ID locally, attempting to fetch \
room ID over federation"
);
.metadata
.list_banned_rooms()
.map(Into::into)
.collect()
.await;
match self.services.rooms.alias.resolve_alias(&room_alias).await {
| Ok((room_id, servers)) => {
debug!(
%room_id,
?servers,
"Got federation response fetching room ID for room {room}"
);
room_id
},
| Err(e) => {
return Err!("Failed to resolve room alias {room} to a room ID: {e}");
},
if room_ids.is_empty() {
return Err!("No rooms are banned.");
}
let mut rooms = room_ids
.iter()
.stream()
.then(|room_id| get_room_info(self.services, room_id))
.collect::<Vec<_>>()
.await;
rooms.sort_by_key(|r| r.1);
rooms.reverse();
let num = rooms.len();
let body = rooms
.iter()
.map(|(id, members, name)| {
if no_details {
format!("{id}")
} else {
format!("{id}\tMembers: {members}\tName: {name}")
}
},
};
})
.collect::<Vec<_>>()
.join("\n");
self.services.rooms.metadata.ban_room(&room_id, false);
room_id
} else {
return Err!(
"Room specified is not a room ID or room alias. Please note that this requires a \
full room ID (`!awIh6gGInaS5wLQJwa:example.com`) or a room alias \
(`#roomalias:example.com`)",
);
};
self.services.rooms.metadata.disable_room(&room_id, false);
self.write_str("Room unbanned and federation re-enabled.")
.await
}
#[admin_command]
async fn list_banned_rooms(&self, no_details: bool) -> Result {
let room_ids: Vec<OwnedRoomId> = self
.services
.rooms
.metadata
.list_banned_rooms()
.map(Into::into)
.collect()
.await;
if room_ids.is_empty() {
return Err!("No rooms are banned.");
self.write_str(&format!("Rooms Banned ({num}):\n```\n{body}\n```"))
.await
}
let mut rooms = room_ids
.iter()
.stream()
.then(|room_id| get_room_info(self.services, room_id))
.collect::<Vec<_>>()
.await;
rooms.sort_by_key(|r| r.1);
rooms.reverse();
let num = rooms.len();
let body = rooms
.iter()
.map(|(id, members, name)| {
if no_details {
format!("{id}")
} else {
format!("{id}\tMembers: {members}\tName: {name}")
}
})
.collect::<Vec<_>>()
.join("\n");
self.write_str(&format!("Rooms Banned ({num}):\n```\n{body}\n```"))
.await
}
+197 -210
View File
@@ -7,243 +7,230 @@ use conduwuit::{
};
use futures::TryStreamExt;
use crate::admin_command;
impl crate::Context<'_> {
pub(super) async fn uptime(&self) -> Result {
let elapsed = self
.services
.server
.started
.elapsed()
.expect("standard duration");
#[admin_command]
pub(super) async fn uptime(&self) -> Result {
let elapsed = self
.services
.server
.started
.elapsed()
.expect("standard duration");
let result = time::pretty(elapsed);
self.write_str(&format!("{result}.")).await
}
let result = time::pretty(elapsed);
self.write_str(&format!("{result}.")).await
}
pub(super) async fn show_config(&self) -> Result {
self.bail_restricted()?;
#[admin_command]
pub(super) async fn show_config(&self) -> Result {
self.bail_restricted()?;
self.write_str(&format!("{}", *self.services.server.config))
.await
}
self.write_str(&format!("{}", *self.services.server.config))
.await
}
pub(super) async fn reload_config(&self, path: Option<PathBuf>) -> Result {
// The path argument is only what's optionally passed via the admin command,
// so we need to merge it with the existing paths if any were given at startup.
let mut paths = Vec::new();
#[admin_command]
pub(super) async fn reload_config(&self, path: Option<PathBuf>) -> Result {
// The path argument is only what's optionally passed via the admin command,
// so we need to merge it with the existing paths if any were given at startup.
let mut paths = Vec::new();
// Add previously saved paths to the argument list
self.services
.config
.config_paths
.clone()
.unwrap_or_default()
.iter()
.for_each(|p| paths.push(p.to_owned()));
// Add previously saved paths to the argument list
self.services
.config
.config_paths
.clone()
.unwrap_or_default()
.iter()
.for_each(|p| paths.push(p.to_owned()));
// If a path is given, and it's not already in the list,
// add it last, so that it overrides earlier files
if let Some(p) = path {
if !paths.contains(&p) {
paths.push(p);
// If a path is given, and it's not already in the list,
// add it last, so that it overrides earlier files
if let Some(p) = path {
if !paths.contains(&p) {
paths.push(p);
}
}
self.services.config.reload(&paths)?;
self.write_str(&format!("Successfully reconfigured from paths: {paths:?}"))
.await
}
self.services.config.reload(&paths)?;
pub(super) async fn memory_usage(&self) -> Result {
let services_usage = self.services.memory_usage().await?;
let database_usage = self.services.db.db.memory_usage()?;
let allocator_usage = conduwuit::alloc::memory_usage()
.map_or(String::new(), |s| format!("\nAllocator:\n{s}"));
self.write_str(&format!("Successfully reconfigured from paths: {paths:?}"))
self.write_str(&format!(
"Services:\n{services_usage}\nDatabase:\n{database_usage}{allocator_usage}",
))
.await
}
#[admin_command]
pub(super) async fn memory_usage(&self) -> Result {
let services_usage = self.services.memory_usage().await?;
let database_usage = self.services.db.db.memory_usage()?;
let allocator_usage =
conduwuit::alloc::memory_usage().map_or(String::new(), |s| format!("\nAllocator:\n{s}"));
self.write_str(&format!(
"Services:\n{services_usage}\nDatabase:\n{database_usage}{allocator_usage}",
))
.await
}
#[admin_command]
pub(super) async fn clear_caches(&self) -> Result {
self.services.clear_cache().await;
self.write_str("Done.").await
}
#[admin_command]
pub(super) async fn list_backups(&self) -> Result {
self.services
.db
.db
.backup_list()?
.try_stream()
.try_for_each(|result| writeln!(self, "{result}"))
.await
}
#[admin_command]
pub(super) async fn backup_database(&self) -> Result {
self.bail_restricted()?;
let db = Arc::clone(&self.services.db);
let result = self
.services
.server
.runtime()
.spawn_blocking(move || match db.db.backup() {
| Ok(()) => "Done".to_owned(),
| Err(e) => format!("Failed: {e}"),
})
.await?;
let count = self.services.db.db.backup_count()?;
self.write_str(&format!("{result}. Currently have {count} backups."))
.await
}
#[admin_command]
pub(super) async fn admin_notice(&self, message: Vec<String>) -> Result {
let message = message.join(" ");
self.services.admin.send_text(&message).await;
self.write_str("Notice was sent to #admins").await
}
#[admin_command]
pub(super) async fn reload_mods(&self) -> Result {
self.bail_restricted()?;
self.services.server.reload()?;
self.write_str("Reloading server...").await
}
#[admin_command]
#[cfg(unix)]
pub(super) async fn restart(&self, force: bool) -> Result {
use conduwuit::utils::sys::current_exe_deleted;
if !force && current_exe_deleted() {
return Err!(
"The server cannot be restarted because the executable changed. If this is expected \
use --force to override."
);
}
self.services.server.restart()?;
pub(super) async fn clear_caches(&self) -> Result {
self.services.clear_cache().await;
self.write_str("Restarting server...").await
}
#[admin_command]
pub(super) async fn shutdown(&self) -> Result {
self.bail_restricted()?;
warn!("shutdown command");
self.services.server.shutdown()?;
self.write_str("Shutting down server...").await
}
#[admin_command]
pub(super) async fn list_features(&self) -> Result {
let mut enabled_features = conduwuit::info::introspection::ENABLED_FEATURES
.lock()
.expect("locked")
.values()
.flat_map(|f| f.iter())
.collect::<Vec<_>>();
enabled_features.sort_unstable();
enabled_features.dedup();
let mut available_features = conduwuit::build_metadata::WORKSPACE_FEATURES
.iter()
.flat_map(|(_, f)| f.iter())
.collect::<Vec<_>>();
available_features.sort_unstable();
available_features.dedup();
let mut features = String::new();
for feature in available_features {
let active = enabled_features.contains(&feature);
let emoji = if active { "✅" } else { "❌" };
let remark = if active { "[enabled]" } else { "" };
writeln!(features, "{emoji} {feature} {remark}")?;
self.write_str("Done.").await
}
self.write_str(&features).await
}
#[admin_command]
pub(super) async fn build_info(&self) -> Result {
use conduwuit::build_metadata::built;
pub(super) async fn list_backups(&self) -> Result {
self.services
.db
.db
.backup_list()?
.try_stream()
.try_for_each(|result| writeln!(self, "{result}"))
.await
}
let mut info = String::new();
pub(super) async fn backup_database(&self) -> Result {
self.bail_restricted()?;
let db = Arc::clone(&self.services.db);
let result = self
.services
.server
.runtime()
.spawn_blocking(move || match db.db.backup() {
| Ok(()) => "Done".to_owned(),
| Err(e) => format!("Failed: {e}"),
})
.await?;
let count = self.services.db.db.backup_count()?;
self.write_str(&format!("{result}. Currently have {count} backups."))
.await
}
// Version information
writeln!(info, "# Build Information\n")?;
writeln!(info, "**Version:** {}", built::PKG_VERSION)?;
writeln!(info, "**Package:** {}", built::PKG_NAME)?;
writeln!(info, "**Description:** {}", built::PKG_DESCRIPTION)?;
pub(super) async fn admin_notice(&self, message: Vec<String>) -> Result {
let message = message.join(" ");
self.services.admin.send_text(&message).await;
// Git information
writeln!(info, "\n## Git Information\n")?;
if let Some(hash) = conduwuit::build_metadata::GIT_COMMIT_HASH {
writeln!(info, "**Commit Hash:** {hash}")?;
}
if let Some(hash) = conduwuit::build_metadata::GIT_COMMIT_HASH_SHORT {
writeln!(info, "**Commit Hash (short):** {hash}")?;
}
if let Some(url) = conduwuit::build_metadata::GIT_REMOTE_WEB_URL {
writeln!(info, "**Repository:** {url}")?;
self.write_str("Notice was sent to #admins").await
}
if let Some(url) = conduwuit::build_metadata::GIT_REMOTE_COMMIT_URL {
writeln!(info, "**Commit URL:** {url}")?;
pub(super) async fn reload_mods(&self) -> Result {
self.bail_restricted()?;
self.services.server.reload()?;
self.write_str("Reloading server...").await
}
// Build environment
writeln!(info, "\n## Build Environment\n")?;
writeln!(info, "**Profile:** {}", built::PROFILE)?;
writeln!(info, "**Optimization Level:** {}", built::OPT_LEVEL)?;
writeln!(info, "**Debug:** {}", built::DEBUG)?;
writeln!(info, "**Target:** {}", built::TARGET)?;
writeln!(info, "**Host:** {}", built::HOST)?;
#[cfg(unix)]
pub(super) async fn restart(&self, force: bool) -> Result {
use conduwuit::utils::sys::current_exe_deleted;
// Rust compiler information
writeln!(info, "\n## Compiler Information\n")?;
writeln!(info, "**Rustc Version:** {}", built::RUSTC_VERSION)?;
if !built::RUSTDOC_VERSION.is_empty() {
writeln!(info, "**Rustdoc Version:** {}", built::RUSTDOC_VERSION)?;
if !force && current_exe_deleted() {
return Err!(
"The server cannot be restarted because the executable changed. If this is \
expected use --force to override."
);
}
self.services.server.restart()?;
self.write_str("Restarting server...").await
}
// Target configuration
writeln!(info, "\n## Target Configuration\n")?;
writeln!(info, "**Architecture:** {}", built::CFG_TARGET_ARCH)?;
writeln!(info, "**OS:** {}", built::CFG_OS)?;
writeln!(info, "**Family:** {}", built::CFG_FAMILY)?;
writeln!(info, "**Endianness:** {}", built::CFG_ENDIAN)?;
writeln!(info, "**Pointer Width:** {} bits", built::CFG_POINTER_WIDTH)?;
if !built::CFG_ENV.is_empty() {
writeln!(info, "**Environment:** {}", built::CFG_ENV)?;
pub(super) async fn shutdown(&self) -> Result {
self.bail_restricted()?;
warn!("shutdown command");
self.services.server.shutdown()?;
self.write_str("Shutting down server...").await
}
// CI information
if let Some(ci) = built::CI_PLATFORM {
writeln!(info, "\n## CI Platform\n")?;
writeln!(info, "**Platform:** {ci}")?;
pub(super) async fn list_features(&self) -> Result {
let mut enabled_features = conduwuit::info::introspection::ENABLED_FEATURES
.lock()
.expect("locked")
.values()
.flat_map(|f| f.iter())
.collect::<Vec<_>>();
enabled_features.sort_unstable();
enabled_features.dedup();
let mut available_features = conduwuit::build_metadata::WORKSPACE_FEATURES
.iter()
.flat_map(|(_, f)| f.iter())
.collect::<Vec<_>>();
available_features.sort_unstable();
available_features.dedup();
let mut features = String::new();
for feature in available_features {
let active = enabled_features.contains(&feature);
let emoji = if active { "✅" } else { "❌" };
let remark = if active { "[enabled]" } else { "" };
writeln!(features, "{emoji} {feature} {remark}")?;
}
self.write_str(&features).await
}
self.write_str(&info).await
pub(super) async fn build_info(&self) -> Result {
use conduwuit::build_metadata::built;
let mut info = String::new();
// Version information
writeln!(info, "# Build Information\n")?;
writeln!(info, "**Version:** {}", built::PKG_VERSION)?;
writeln!(info, "**Package:** {}", built::PKG_NAME)?;
writeln!(info, "**Description:** {}", built::PKG_DESCRIPTION)?;
// Git information
writeln!(info, "\n## Git Information\n")?;
if let Some(hash) = conduwuit::build_metadata::GIT_COMMIT_HASH {
writeln!(info, "**Commit Hash:** {hash}")?;
}
if let Some(hash) = conduwuit::build_metadata::GIT_COMMIT_HASH_SHORT {
writeln!(info, "**Commit Hash (short):** {hash}")?;
}
if let Some(url) = conduwuit::build_metadata::GIT_REMOTE_WEB_URL {
writeln!(info, "**Repository:** {url}")?;
}
if let Some(url) = conduwuit::build_metadata::GIT_REMOTE_COMMIT_URL {
writeln!(info, "**Commit URL:** {url}")?;
}
// Build environment
writeln!(info, "\n## Build Environment\n")?;
writeln!(info, "**Profile:** {}", built::PROFILE)?;
writeln!(info, "**Optimization Level:** {}", built::OPT_LEVEL)?;
writeln!(info, "**Debug:** {}", built::DEBUG)?;
writeln!(info, "**Target:** {}", built::TARGET)?;
writeln!(info, "**Host:** {}", built::HOST)?;
// Rust compiler information
writeln!(info, "\n## Compiler Information\n")?;
writeln!(info, "**Rustc Version:** {}", built::RUSTC_VERSION)?;
if !built::RUSTDOC_VERSION.is_empty() {
writeln!(info, "**Rustdoc Version:** {}", built::RUSTDOC_VERSION)?;
}
// Target configuration
writeln!(info, "\n## Target Configuration\n")?;
writeln!(info, "**Architecture:** {}", built::CFG_TARGET_ARCH)?;
writeln!(info, "**OS:** {}", built::CFG_OS)?;
writeln!(info, "**Family:** {}", built::CFG_FAMILY)?;
writeln!(info, "**Endianness:** {}", built::CFG_ENDIAN)?;
writeln!(info, "**Pointer Width:** {} bits", built::CFG_POINTER_WIDTH)?;
if !built::CFG_ENV.is_empty() {
writeln!(info, "**Environment:** {}", built::CFG_ENV)?;
}
// CI information
if let Some(ci) = built::CI_PLATFORM {
writeln!(info, "\n## CI Platform\n")?;
writeln!(info, "**Platform:** {ci}")?;
}
self.write_str(&info).await
}
}
+66 -66
View File
@@ -1,76 +1,76 @@
use conduwuit::{Err, Result, utils};
use conduwuit_macros::admin_command;
use futures::StreamExt;
use service::registration_tokens::TokenExpires;
#[admin_command]
pub(super) async fn issue_token(&self, expires: super::TokenExpires) -> Result {
let expires = {
if expires.immortal {
None
} else if let Some(max_uses) = expires.max_uses {
Some(TokenExpires::AfterUses(max_uses))
} else if expires.once {
Some(TokenExpires::AfterUses(1))
} else if let Some(max_age) = expires
.max_age
.as_deref()
.map(|max_age| utils::time::timepoint_from_now(utils::time::parse_duration(max_age)?))
.transpose()?
{
Some(TokenExpires::AfterTime(max_age))
} else {
unreachable!();
}
};
impl crate::Context<'_> {
pub(super) async fn issue_token(&self, expires: super::TokenExpires) -> Result {
let expires = {
if expires.immortal {
None
} else if let Some(max_uses) = expires.max_uses {
Some(TokenExpires::AfterUses(max_uses))
} else if expires.once {
Some(TokenExpires::AfterUses(1))
} else if let Some(max_age) = expires
.max_age
.as_deref()
.map(|max_age| {
utils::time::timepoint_from_now(utils::time::parse_duration(max_age)?)
})
.transpose()?
{
Some(TokenExpires::AfterTime(max_age))
} else {
unreachable!();
}
};
let (token, info) = self
.services
.registration_tokens
.issue_token(self.sender_or_service_user().into(), expires);
let (token, info) = self
.services
.registration_tokens
.issue_token(self.sender_or_service_user().into(), expires);
self.write_str(&format!(
"New registration token issued: `{token}`. {}.",
if let Some(expires) = info.expires {
format!("{expires}")
} else {
"Never expires".to_owned()
}
))
.await
}
#[admin_command]
pub(super) async fn revoke_token(&self, token: String) -> Result {
let Some(token) = self
.services
.registration_tokens
.validate_token(token)
self.write_str(&format!(
"New registration token issued: `{token}`. {}.",
if let Some(expires) = info.expires {
format!("{expires}")
} else {
"Never expires".to_owned()
}
))
.await
else {
return Err!("This token does not exist or has already expired.");
};
self.services.registration_tokens.revoke_token(token)?;
self.write_str("Token revoked successfully.").await
}
#[admin_command]
pub(super) async fn list_tokens(&self) -> Result {
let tokens: Vec<_> = self
.services
.registration_tokens
.iterate_tokens()
.collect()
.await;
self.write_str(&format!("Found {} registration tokens:\n", tokens.len()))
.await?;
for token in tokens {
self.write_str(&format!("- {token}\n")).await?;
}
Ok(())
pub(super) async fn revoke_token(&self, token: String) -> Result {
let Some(token) = self
.services
.registration_tokens
.validate_token(token)
.await
else {
return Err!("This token does not exist or has already expired.");
};
self.services.registration_tokens.revoke_token(token)?;
self.write_str("Token revoked successfully.").await
}
pub(super) async fn list_tokens(&self) -> Result {
let tokens: Vec<_> = self
.services
.registration_tokens
.iterate_tokens()
.collect()
.await;
self.write_str(&format!("Found {} registration tokens:\n", tokens.len()))
.await?;
for token in tokens {
self.write_str(&format!("- {token}\n")).await?;
}
Ok(())
}
}
+1154 -1158
View File
File diff suppressed because it is too large. Load diff
+1 -1
View File
@@ -134,7 +134,7 @@ pub(crate) async fn register_route(
};
// Create user
services.users.create(&user_id, password).await?;
services.users.create(&user_id, password)?;
// Set an initial display name
let mut displayname = user_id.localpart().to_owned();
+1 -1
View File
@@ -32,7 +32,7 @@ pub(crate) async fn get_backfill_route(
room_id: &body.room_id,
event_id: None,
}
.check()
.assert()
.await?;
if !services
.rooms
+1 -1
View File
@@ -44,7 +44,7 @@ pub(crate) async fn get_event_route(
room_id,
event_id: Some(&body.event_id),
}
.check()
.assert()
.await?;
if !services
+1 -1
View File
@@ -23,7 +23,7 @@ pub(crate) async fn get_event_authorization_route(
room_id: &body.room_id,
event_id: None,
}
.check()
.assert()
.await?;
if services
+4 -8
View File
@@ -4,15 +4,11 @@ use axum::extract::State;
use conduwuit::{Err, Event, Result, debug, info, trace, utils::to_canonical_object, warn};
use ruma::{OwnedEventId, api::federation::event::get_missing_events};
use serde_json::{json, value::RawValue};
use service::rooms::event_handler::GET_MISSING_EVENTS_MAX_BATCH_SIZE;
use super::AccessCheck;
use crate::Ruma;
/// arbitrary number but synapse's is 20 and we can handle lots of these anyways
const LIMIT_MAX: usize = 50;
/// spec says default is 10
const LIMIT_DEFAULT: usize = 10;
/// # `POST /_matrix/federation/v1/get_missing_events/{roomId}`
///
/// Retrieves events that the sender is missing.
@@ -26,7 +22,7 @@ pub(crate) async fn get_missing_events_route(
room_id: &body.room_id,
event_id: None,
}
.check()
.assert()
.await?;
if !services
@@ -45,8 +41,8 @@ pub(crate) async fn get_missing_events_route(
let limit = body
.limit
.try_into()
.unwrap_or(LIMIT_DEFAULT)
.min(LIMIT_MAX);
.unwrap_or(10)
.min(GET_MISSING_EVENTS_MAX_BATCH_SIZE);
let room_version = services.rooms.state.get_room_version(&body.room_id).await?;
+11 -3
View File
@@ -7,7 +7,7 @@ use std::{
use axum::extract::State;
use axum_client_ip::ClientIp;
use conduwuit::{
Err, Error, Result, debug, debug_warn, err, error,
Err, Error, Result, debug, debug_error, debug_warn, err, error,
result::LogErr,
state_res::lexicographical_topological_sort,
trace,
@@ -133,6 +133,7 @@ async fn wait_for_result(
}
#[instrument(
name="transaction"
skip_all,
fields(
id = ?body.transaction_id.as_str(),
@@ -174,8 +175,14 @@ async fn process_inbound_transaction(
for (id, result) in &results {
if let Err(e) = result {
if matches!(e, Error::BadRequest(ErrorKind::NotFound, _)) {
debug_warn!("Incoming PDU failed {id}: {e:?}");
match e {
| Error::BadRequest(
ErrorKind::Forbidden | ErrorKind::InvalidParam | ErrorKind::BadJson,
..,
) => {
debug_warn!("Incoming PDU {id} failed: {e:?}");
},
| _ => debug_error!("Incoming PDU {id} failed: {e:?}"),
}
}
}
@@ -381,6 +388,7 @@ async fn handle_room(
.rooms
.event_handler
.handle_incoming_pdu(origin, room_id, &event_id, value, true)
.boxed()
.await
.map(|_| ());
results.push((event_id, result));
+1 -1
View File
@@ -21,7 +21,7 @@ pub(crate) async fn get_room_state_route(
room_id: &body.room_id,
event_id: None,
}
.check()
.assert()
.await?;
if services
+1 -1
View File
@@ -22,7 +22,7 @@ pub(crate) async fn get_room_state_ids_route(
room_id: &body.room_id,
event_id: None,
}
.check()
.assert()
.await?;
if services
+47 -43
View File
@@ -1,4 +1,4 @@
use conduwuit::{Err, Result, implement, is_false};
use conduwuit::{Err, Result, is_false};
use conduwuit_service::Services;
use futures::{FutureExt, future::OptionFuture, join};
use ruma::{EventId, RoomId, ServerName};
@@ -10,52 +10,56 @@ pub(super) struct AccessCheck<'a> {
pub(super) event_id: Option<&'a EventId>,
}
#[implement(AccessCheck, params = "<'_>")]
pub(super) async fn check(&self) -> Result {
let acl_check = self
.services
.rooms
.event_handler
.acl_check(self.origin, self.room_id)
.map(|result| result.is_ok());
impl AccessCheck<'_> {
/// Asserts that the server has access to the room and event (if any).
/// If the server is permitted, `Ok(())` is returned. Otherwise, a Forbidden
/// error is returned.
pub(super) async fn assert(&self) -> Result {
let acl_check = self
.services
.rooms
.event_handler
.acl_check(self.origin, self.room_id)
.map(|result| result.is_ok());
let world_readable = self
.services
.rooms
.state_accessor
.is_world_readable(self.room_id);
let world_readable = self
.services
.rooms
.state_accessor
.is_world_readable(self.room_id);
let server_in_room = self
.services
.rooms
.state_cache
.server_in_room(self.origin, self.room_id);
let server_in_room = self
.services
.rooms
.state_cache
.server_in_room(self.origin, self.room_id);
let server_can_see: OptionFuture<_> = self
.event_id
.map(|event_id| {
self.services.rooms.state_accessor.server_can_see_event(
self.origin,
self.room_id,
event_id,
)
})
.into();
let server_can_see: OptionFuture<_> = self
.event_id
.map(|event_id| {
self.services.rooms.state_accessor.server_can_see_event(
self.origin,
self.room_id,
event_id,
)
})
.into();
let (world_readable, server_in_room, server_can_see, acl_check) =
join!(world_readable, server_in_room, server_can_see, acl_check);
let (world_readable, server_in_room, server_can_see, acl_check) =
join!(world_readable, server_in_room, server_can_see, acl_check);
if !acl_check {
return Err!(Request(Forbidden("Server access denied.")));
if !acl_check {
return Err!(Request(Forbidden("Server access denied.")));
}
if !world_readable && !server_in_room {
return Err!(Request(Forbidden("Server is not in room.")));
}
if server_can_see.is_some_and(is_false!()) {
return Err!(Request(Forbidden("Server is not allowed to see event.")));
}
Ok(())
}
if !world_readable && !server_in_room {
return Err!(Request(Forbidden("Server is not in room.")));
}
if server_can_see.is_some_and(is_false!()) {
return Err!(Request(Forbidden("Server is not allowed to see event.")));
}
Ok(())
}
+43 -45
View File
@@ -10,7 +10,7 @@ use std::{
};
use super::Config;
use crate::{Result, implement};
use crate::Result;
/// The configuration manager is an indirection to reload the configuration for
/// the server while it is running. In order to not burden or clutter the many
@@ -39,6 +39,48 @@ impl Manager {
active: AtomicPtr::new(Arc::into_raw(config).cast_mut()),
}
}
/// Update the active configuration, returning prior configuration.
#[tracing::instrument(skip_all, level = "info")]
pub fn update(&self, config: Config) -> Result<Arc<Config>> {
let config = Arc::new(config);
let new = Arc::into_raw(config);
let old = self.active.swap(new.cast_mut(), Ordering::AcqRel);
// SAFETY: The old active pointer was set using an Arc::into_raw(). We're
// obliged to reconstitute that into Arc otherwise it will leak.
Ok(unsafe { Arc::from_raw(old) })
}
fn load(&self, handle: &mut [Option<Arc<Config>>]) -> &'static Arc<Config> {
let config = self.active.load(Ordering::Acquire);
// Branch taken after config reload or first access by this thread.
if handle[INDEX.get()]
.as_ref()
.is_none_or(|handle| !ptr::eq(config, Arc::as_ptr(handle)))
{
INDEX.set(INDEX.get().wrapping_add(1).wrapping_rem(HISTORY));
return load_miss(handle, INDEX.get(), config);
}
let config: &Arc<Config> = handle[INDEX.get()]
.as_ref()
.expect("handle was already cached for this thread");
// SAFETY: The caller should not hold multiple references at a time directly
// into Config, as a subsequent reference might invalidate the thread's cache
// causing another reference to dangle.
//
// This is a highly unusual pattern as most config values are copied by value or
// used immediately without running overlap with another value. Even if it does
// actually occur somewhere, the window of danger is limited to the config being
// reloaded while the reference is held and another access is made by the same
// thread into a different config value. This is mitigated by creating a buffer
// of old configs rather than discarding at the earliest opportunity; the odds
// of this scenario are thus astronomical.
unsafe { std::mem::transmute(config) }
}
}
impl Drop for Manager {
@@ -57,50 +99,6 @@ impl Deref for Manager {
fn deref(&self) -> &Self::Target { HANDLE.with_borrow_mut(|handle| self.load(handle)) }
}
/// Update the active configuration, returning prior configuration.
#[implement(Manager)]
#[tracing::instrument(skip_all, level = "info")]
pub fn update(&self, config: Config) -> Result<Arc<Config>> {
let config = Arc::new(config);
let new = Arc::into_raw(config);
let old = self.active.swap(new.cast_mut(), Ordering::AcqRel);
// SAFETY: The old active pointer was set using an Arc::into_raw(). We're
// obliged to reconstitute that into Arc otherwise it will leak.
Ok(unsafe { Arc::from_raw(old) })
}
#[implement(Manager)]
fn load(&self, handle: &mut [Option<Arc<Config>>]) -> &'static Arc<Config> {
let config = self.active.load(Ordering::Acquire);
// Branch taken after config reload or first access by this thread.
if handle[INDEX.get()]
.as_ref()
.is_none_or(|handle| !ptr::eq(config, Arc::as_ptr(handle)))
{
INDEX.set(INDEX.get().wrapping_add(1).wrapping_rem(HISTORY));
return load_miss(handle, INDEX.get(), config);
}
let config: &Arc<Config> = handle[INDEX.get()]
.as_ref()
.expect("handle was already cached for this thread");
// SAFETY: The caller should not hold multiple references at a time directly
// into Config, as a subsequent reference might invalidate the thread's cache
// causing another reference to dangle.
//
// This is a highly unusual pattern as most config values are copied by value or
// used immediately without running overlap with another value. Even if it does
// actually occur somewhere, the window of danger is limited to the config being
// reloaded while the reference is held and another access is made by the same
// thread into a different config value. This is mitigated by creating a buffer
// of old configs rather than discarding at the earliest opportunity; the odds
// of this scenario are thus astronomical.
unsafe { std::mem::transmute(config) }
}
#[tracing::instrument(
name = "miss",
level = "trace",
+14 -2
View File
@@ -375,7 +375,7 @@ pub struct Config {
#[serde(default = "default_max_request_size")]
pub max_request_size: usize,
/// default: 192
/// default: 1024
#[serde(default = "default_max_fetch_prev_events")]
pub max_fetch_prev_events: u16,
@@ -781,6 +781,16 @@ pub struct Config {
/// a substitute for moderation bots.
pub default_room_acl_deny: Option<Vec<String>>,
/// The number of forward extremities to tolerate in a room before
/// attempting to manually squash them with a "dummy event". Setting this
/// above 20 will hinder its efficacy, and setting it below 5 will cause
/// more dummy events to be sent than necessary (which increases federation
/// traffic).
///
/// default: 10
#[serde(default = "default_extremity_threshold")]
pub dummy_event_threshold: u8,
/// display: nested
#[serde(default)]
pub well_known: WellKnownConfig,
@@ -2549,7 +2559,7 @@ fn default_pusher_timeout() -> u64 { 60 }
fn default_pusher_idle_timeout() -> u64 { 15 }
fn default_max_fetch_prev_events() -> u16 { 192_u16 }
fn default_max_fetch_prev_events() -> u16 { 1024 }
fn default_max_concurrent_inbound_transactions() -> usize { 150 }
@@ -2652,6 +2662,8 @@ fn default_rocksdb_stats_level() -> u8 { 1 }
#[inline]
pub fn default_default_room_version() -> RoomVersionId { RoomVersionId::V12 }
fn default_extremity_threshold() -> u8 { 10 }
fn default_ip_range_denylist() -> Vec<String> {
vec![
"127.0.0.0/8".to_owned(),
+1 -1
View File
@@ -62,7 +62,7 @@ impl Default for PartialPdu {
fn default() -> Self {
Self {
event_type: "m.room.message".into(),
content: Box::<RawJsonValue>::default(),
content: to_raw_value("{}").unwrap(),
unsigned: None,
state_key: None,
redacts: None,
+21 -19
View File
@@ -1,32 +1,34 @@
use ruma::{RoomVersionId, canonical_json::redact_content_in_place};
use serde_json::{Value as JsonValue, json, value::to_raw_value};
use crate::{Err, Result, err, implement};
use crate::{Err, Result, err};
#[implement(super::Pdu)]
pub fn redact(&mut self, room_version_id: &RoomVersionId, reason: JsonValue) -> Result {
let Some(rules) = room_version_id.rules() else {
return Err!("Cannot redact event for unknown room version {room_version_id}");
};
impl super::Pdu {
pub fn redact(&mut self, room_version_id: &RoomVersionId, reason: JsonValue) -> Result {
let Some(rules) = room_version_id.rules() else {
return Err!("Cannot redact event for unknown room version {room_version_id}");
};
self.unsigned = None;
self.unsigned = None;
let mut content = serde_json::from_str(self.content.get())
.map_err(|e| err!(Request(BadJson("Failed to deserialize content into type: {e}"))))?;
let mut content = serde_json::from_str(self.content.get()).map_err(|e| {
err!(Request(BadJson("Failed to deserialize content into type: {e}")))
})?;
redact_content_in_place(&mut content, &rules.redaction, self.kind.to_string());
redact_content_in_place(&mut content, &rules.redaction, self.kind.to_string());
let reason = serde_json::to_value(reason).expect("Failed to preserialize reason");
let reason = serde_json::to_value(reason).expect("Failed to preserialize reason");
let redacted_because = json!({
"redacted_because": reason,
});
let redacted_because = json!({
"redacted_because": reason,
});
self.unsigned = to_raw_value(&redacted_because)
.expect("Failed to serialize unsigned")
.into();
self.unsigned = to_raw_value(&redacted_because)
.expect("Failed to serialize unsigned")
.into();
self.content = to_raw_value(&content).expect("Failed to serialize content");
self.content = to_raw_value(&content).expect("Failed to serialize content");
Ok(())
Ok(())
}
}
+77 -79
View File
@@ -4,85 +4,83 @@ use ruma::MilliSecondsSinceUnixEpoch;
use serde_json::value::{RawValue as RawJsonValue, Value as JsonValue, to_raw_value};
use super::Pdu;
use crate::{Result, err, implement, result::LogErr};
use crate::{Result, err, result::LogErr};
/// Set the `unsigned` field of the PDU using only information in the PDU.
/// Some unsigned data is already set within the database (eg. prev events,
/// threads). Once this is done, other data must be calculated from the database
/// (eg. relations) This is for server-to-client events.
/// Backfill handles this itself.
#[implement(Pdu)]
pub fn set_unsigned(&mut self, user_id: Option<&ruma::UserId>) {
if Some(self.sender.borrow()) != user_id {
self.remove_transaction_id().log_err().ok();
impl Pdu {
/// Set the `unsigned` field of the PDU using only information in the PDU.
/// Some unsigned data is already set within the database (eg. prev events,
/// threads). Once this is done, other data must be calculated from the
/// database (eg. relations) This is for server-to-client events.
/// Backfill handles this itself.
pub fn set_unsigned(&mut self, user_id: Option<&ruma::UserId>) {
if Some(self.sender.borrow()) != user_id {
self.remove_transaction_id().log_err().ok();
}
self.add_age().log_err().ok();
}
pub fn remove_transaction_id(&mut self) -> Result {
use BTreeMap as Map;
let Some(unsigned) = &self.unsigned else {
return Ok(());
};
let mut unsigned: Map<&str, Box<RawJsonValue>> = serde_json::from_str(unsigned.get())
.map_err(|e| err!(Database("Invalid unsigned in pdu event: {e}")))?;
unsigned.remove("transaction_id");
self.unsigned = to_raw_value(&unsigned)
.map(Some)
.expect("unsigned is valid");
Ok(())
}
pub fn add_age(&mut self) -> Result {
use BTreeMap as Map;
let mut unsigned: Map<&str, Box<RawJsonValue>> = self
.unsigned
.as_deref()
.map(RawJsonValue::get)
.map_or_else(|| Ok(Map::new()), serde_json::from_str)
.map_err(|e| err!(Database("Invalid unsigned in pdu event: {e}")))?;
// deliberately allowing for the possibility of negative age
let now: i128 = MilliSecondsSinceUnixEpoch::now().get().into();
let then: i128 = self.origin_server_ts.into();
let this_age = now.saturating_sub(then);
unsigned.insert("age", to_raw_value(&this_age)?);
self.unsigned = Some(to_raw_value(&unsigned)?);
Ok(())
}
pub fn add_relation(&mut self, name: &str, pdu: Option<&Self>) -> Result {
use serde_json::Map;
let mut unsigned: Map<String, JsonValue> = self
.unsigned
.as_deref()
.map(RawJsonValue::get)
.map_or_else(|| Ok(Map::new()), serde_json::from_str)
.map_err(|e| err!(Database("Invalid unsigned in pdu event: {e}")))?;
let pdu = pdu
.map(serde_json::to_value)
.transpose()?
.unwrap_or_else(|| JsonValue::Object(Map::new()));
unsigned
.entry("m.relations")
.or_insert(JsonValue::Object(Map::new()))
.as_object_mut()
.map(|object| object.insert(name.to_owned(), pdu));
self.unsigned = Some(to_raw_value(&unsigned)?);
Ok(())
}
self.add_age().log_err().ok();
}
#[implement(Pdu)]
pub fn remove_transaction_id(&mut self) -> Result {
use BTreeMap as Map;
let Some(unsigned) = &self.unsigned else {
return Ok(());
};
let mut unsigned: Map<&str, Box<RawJsonValue>> = serde_json::from_str(unsigned.get())
.map_err(|e| err!(Database("Invalid unsigned in pdu event: {e}")))?;
unsigned.remove("transaction_id");
self.unsigned = to_raw_value(&unsigned)
.map(Some)
.expect("unsigned is valid");
Ok(())
}
#[implement(Pdu)]
pub fn add_age(&mut self) -> Result {
use BTreeMap as Map;
let mut unsigned: Map<&str, Box<RawJsonValue>> = self
.unsigned
.as_deref()
.map(RawJsonValue::get)
.map_or_else(|| Ok(Map::new()), serde_json::from_str)
.map_err(|e| err!(Database("Invalid unsigned in pdu event: {e}")))?;
// deliberately allowing for the possibility of negative age
let now: i128 = MilliSecondsSinceUnixEpoch::now().get().into();
let then: i128 = self.origin_server_ts.into();
let this_age = now.saturating_sub(then);
unsigned.insert("age", to_raw_value(&this_age)?);
self.unsigned = Some(to_raw_value(&unsigned)?);
Ok(())
}
#[implement(Pdu)]
pub fn add_relation(&mut self, name: &str, pdu: Option<&Pdu>) -> Result {
use serde_json::Map;
let mut unsigned: Map<String, JsonValue> = self
.unsigned
.as_deref()
.map(RawJsonValue::get)
.map_or_else(|| Ok(Map::new()), serde_json::from_str)
.map_err(|e| err!(Database("Invalid unsigned in pdu event: {e}")))?;
let pdu = pdu
.map(serde_json::to_value)
.transpose()?
.unwrap_or_else(|| JsonValue::Object(Map::new()));
unsigned
.entry("m.relations")
.or_insert(JsonValue::Object(Map::new()))
.as_object_mut()
.map(|object| object.insert(name.to_owned(), pdu));
self.unsigned = Some(to_raw_value(&unsigned)?);
Ok(())
}
+1
View File
@@ -44,5 +44,6 @@ pub fn unstable_features() -> BTreeMap<String, bool> {
("uk.timedout.msc4323".to_owned(), true), /* agnostic suspend (https://github.com/matrix-org/matrix-spec-proposals/pull/4323) */
("org.matrix.msc4155".to_owned(), true), /* invite filtering (https://github.com/matrix-org/matrix-spec-proposals/pull/4155) */
("computer.gingershaped.msc4466".to_owned(), true), /* profile change propagation (https://github.com/matrix-org/matrix-spec-proposals/pull/4466) */
("org.matrix.msc4380.stable".to_owned(), true),
])
}
+1 -1
View File
@@ -41,7 +41,7 @@ pub use info::{
pub use matrix::{Event, EventTypeExt, Pdu, PduCount, PduEvent, PduId, pdu, state_res};
pub use parking_lot::{Mutex as SyncMutex, RwLock as SyncRwLock};
pub use server::Server;
pub use utils::{implement, result, result::Result};
pub use utils::{result, result::Result};
pub use crate as conduwuit_core;
-2
View File
@@ -22,8 +22,6 @@ mod tests;
pub mod time;
pub mod with_lock;
pub use ::conduwuit_macros::implement;
pub use self::{
arrayvec::ArrayVecExt,
bool::BoolExt,
@@ -27,3 +27,12 @@ pub fn continue_exponential_backoff(
let min = cmp::min(min, max);
elapsed < min
}
/// Determines the minimum number of backoff seconds
#[must_use]
pub fn min_exp_backoff_duration(min: u64, max: u64, retries: u32) -> Duration {
let min = Duration::from_secs(min)
.saturating_mul(retries)
.saturating_mul(retries);
Duration::from_secs(max).min(min)
}
+70 -74
View File
@@ -1,92 +1,88 @@
use std::{ffi::OsString, path::PathBuf};
use conduwuit::{Err, Result, error, implement, info, utils::time::rfc2822_from_seconds, warn};
use conduwuit::{Err, Result, error, info, utils::time::rfc2822_from_seconds, warn};
use rocksdb::backup::{BackupEngine, BackupEngineOptions};
use super::Engine;
use crate::util::map_err;
#[implement(Engine)]
#[tracing::instrument(skip(self), level = "info")]
pub fn backup(&self) -> Result {
let mut engine = self.backup_engine()?;
let config = &self.ctx.server.config;
if config.database_backups_to_keep > 0 {
engine
.create_new_backup_flush(&self.db, true)
.map_err(map_err)?;
impl super::Engine {
#[tracing::instrument(skip(self), level = "info")]
pub fn backup(&self) -> Result {
let mut engine = self.backup_engine()?;
let config = &self.ctx.server.config;
if config.database_backups_to_keep > 0 {
engine
.create_new_backup_flush(&self.db, true)
.map_err(map_err)?;
let engine_info = engine.get_backup_info();
let info = &engine_info.last().expect("backup engine info is not empty");
info!(
"Created database backup #{} using {} bytes in {} files",
info.backup_id, info.size, info.num_files,
);
}
if config.database_backups_to_keep >= 0 {
let keep = u32::try_from(config.database_backups_to_keep)?;
if let Err(e) = engine.purge_old_backups(keep.try_into()?) {
error!("Failed to purge old backup: {e:?}");
let engine_info = engine.get_backup_info();
let info = &engine_info.last().expect("backup engine info is not empty");
info!(
"Created database backup #{} using {} bytes in {} files",
info.backup_id, info.size, info.num_files,
);
}
if config.database_backups_to_keep >= 0 {
let keep = u32::try_from(config.database_backups_to_keep)?;
if let Err(e) = engine.purge_old_backups(keep.try_into()?) {
error!("Failed to purge old backup: {e:?}");
}
}
if config.database_backups_to_keep == 0 {
warn!("Configuration item `database_backups_to_keep` is set to 0.");
}
Ok(())
}
if config.database_backups_to_keep == 0 {
warn!("Configuration item `database_backups_to_keep` is set to 0.");
pub fn backup_list(&self) -> Result<impl Iterator<Item = String> + Send> {
let info = self.backup_engine()?.get_backup_info();
if info.is_empty() {
return Err!("No backups found.");
}
let list = info.into_iter().map(|info| {
format!(
"#{} {}: {} bytes, {} files",
info.backup_id,
rfc2822_from_seconds(info.timestamp),
info.size,
info.num_files,
)
});
Ok(list)
}
Ok(())
}
pub fn backup_count(&self) -> Result<usize> {
let info = self.backup_engine()?.get_backup_info();
#[implement(Engine)]
pub fn backup_list(&self) -> Result<impl Iterator<Item = String> + Send> {
let info = self.backup_engine()?.get_backup_info();
if info.is_empty() {
return Err!("No backups found.");
Ok(info.len())
}
let list = info.into_iter().map(|info| {
format!(
"#{} {}: {} bytes, {} files",
info.backup_id,
rfc2822_from_seconds(info.timestamp),
info.size,
info.num_files,
)
});
Ok(list)
}
#[implement(Engine)]
pub fn backup_count(&self) -> Result<usize> {
let info = self.backup_engine()?.get_backup_info();
Ok(info.len())
}
#[implement(Engine)]
fn backup_engine(&self) -> Result<BackupEngine> {
let path = self.backup_path()?;
let options = BackupEngineOptions::new(path).map_err(map_err)?;
BackupEngine::open(&options, &self.ctx.env.lock()).map_err(map_err)
}
#[implement(Engine)]
fn backup_path(&self) -> Result<OsString> {
let path = self
.ctx
.server
.config
.database_backup_path
.clone()
.map(PathBuf::into_os_string)
.unwrap_or_default();
if path.is_empty() {
return Err!(Config("database_backup_path", "Configure path to enable backups"));
fn backup_engine(&self) -> Result<BackupEngine> {
let path = self.backup_path()?;
let options = BackupEngineOptions::new(path).map_err(map_err)?;
BackupEngine::open(&options, &self.ctx.env.lock()).map_err(map_err)
}
Ok(path)
fn backup_path(&self) -> Result<OsString> {
let path = self
.ctx
.server
.config
.database_backup_path
.clone()
.map(PathBuf::into_os_string)
.unwrap_or_default();
if path.is_empty() {
return Err!(Config("database_backup_path", "Configure path to enable backups"));
}
Ok(path)
}
}
+10 -10
View File
@@ -1,15 +1,15 @@
use conduwuit::{Result, implement};
use conduwuit::Result;
use rocksdb::LiveFile as SstFile;
use super::Engine;
use crate::util::map_err;
#[implement(Engine)]
pub fn file_list(&self) -> impl Iterator<Item = Result<SstFile>> + Send + use<> {
self.db
.live_files()
.map_err(map_err)
.into_iter()
.flat_map(Vec::into_iter)
.map(Ok)
impl super::Engine {
pub fn file_list(&self) -> impl Iterator<Item = Result<SstFile>> + Send + use<> {
self.db
.live_files()
.map_err(map_err)
.into_iter()
.flat_map(Vec::into_iter)
.map(Ok)
}
}
+22 -21
View File
@@ -1,30 +1,31 @@
use std::fmt::Write;
use conduwuit::{Result, implement};
use conduwuit::Result;
use rocksdb::perf::get_memory_usage_stats;
use super::Engine;
use crate::or_else;
#[implement(Engine)]
pub fn memory_usage(&self) -> Result<String> {
let mut res = String::new();
let stats = get_memory_usage_stats(Some(&[&self.db]), Some(&[&*self.ctx.row_cache.lock()]))
.or_else(or_else)?;
let mibs = |input| f64::from(u32::try_from(input / 1024).unwrap_or(0)) / 1024.0;
writeln!(
res,
"Memory buffers: {:.2} MiB\nPending write: {:.2} MiB\nTable readers: {:.2} MiB\nRow \
cache: {:.2} MiB",
mibs(stats.mem_table_total),
mibs(stats.mem_table_unflushed),
mibs(stats.mem_table_readers_total),
mibs(u64::try_from(self.ctx.row_cache.lock().get_usage())?),
)?;
impl super::Engine {
pub fn memory_usage(&self) -> Result<String> {
let mut res = String::new();
let stats =
get_memory_usage_stats(Some(&[&self.db]), Some(&[&*self.ctx.row_cache.lock()]))
.or_else(or_else)?;
let mibs = |input| f64::from(u32::try_from(input / 1024).unwrap_or(0)) / 1024.0;
writeln!(
res,
"Memory buffers: {:.2} MiB\nPending write: {:.2} MiB\nTable readers: {:.2} MiB\nRow \
cache: {:.2} MiB",
mibs(stats.mem_table_total),
mibs(stats.mem_table_unflushed),
mibs(stats.mem_table_readers_total),
mibs(u64::try_from(self.ctx.row_cache.lock().get_usage())?),
)?;
for (name, cache) in &*self.ctx.col_cache.lock() {
writeln!(res, "{name} cache: {:.2} MiB", mibs(u64::try_from(cache.get_usage())?))?;
for (name, cache) in &*self.ctx.col_cache.lock() {
writeln!(res, "{name} cache: {:.2} MiB", mibs(u64::try_from(cache.get_usage())?))?;
}
Ok(res)
}
Ok(res)
}
+94 -95
View File
@@ -4,11 +4,11 @@ use std::{
sync::{Arc, atomic::AtomicU32},
};
use conduwuit::{Result, debug, implement, info, warn};
use conduwuit::{Result, debug, info, warn};
use rocksdb::{ColumnFamilyDescriptor, Options};
use super::{
Db, Engine,
Db,
cf_opts::cf_options,
db_opts::db_options,
descriptor::{self, Descriptor},
@@ -16,104 +16,103 @@ use super::{
};
use crate::{Context, or_else};
#[implement(Engine)]
#[tracing::instrument(skip_all, level = "info")]
pub(crate) async fn open(ctx: Arc<Context>, desc: &[Descriptor]) -> Result<Arc<Self>> {
let server = &ctx.server;
let config = &server.config;
let path = &config.database_path;
impl super::Engine {
#[tracing::instrument(skip_all, level = "info")]
pub(crate) async fn open(ctx: Arc<Context>, desc: &[Descriptor]) -> Result<Arc<Self>> {
let server = &ctx.server;
let config = &server.config;
let path = &config.database_path;
let db_opts = db_options(config, &ctx.env.lock(), &ctx.row_cache.lock())?;
let db_opts = db_options(config, &ctx.env.lock(), &ctx.row_cache.lock())?;
let cfds = Self::configure_cfds(&ctx, &db_opts, desc)?;
let num_cfds = cfds.len();
debug!("Configured {num_cfds} column descriptors...");
let cfds = Self::configure_cfds(&ctx, &db_opts, desc)?;
let num_cfds = cfds.len();
debug!("Configured {num_cfds} column descriptors...");
let load_time = std::time::Instant::now();
if config.rocksdb_repair {
repair(&db_opts, &config.database_path)?;
let load_time = std::time::Instant::now();
if config.rocksdb_repair {
repair(&db_opts, &config.database_path)?;
}
debug!("Opening database...");
let db = Db::open_cf_descriptors(&db_opts, path, cfds).or_else(or_else)?;
info!(
columns = num_cfds,
sequence = %db.latest_sequence_number(),
time = ?load_time.elapsed(),
"Opened database."
);
Ok(Arc::new(Self {
db,
pool: ctx.pool.clone(),
ctx: ctx.clone(),
checksums: config.rocksdb_checksums,
corks: AtomicU32::new(0),
}))
}
debug!("Opening database...");
let db = Db::open_cf_descriptors(&db_opts, path, cfds).or_else(or_else)?;
#[tracing::instrument(name = "configure", skip_all, level = "debug")]
fn configure_cfds(
ctx: &Arc<Context>,
db_opts: &Options,
desc: &[Descriptor],
) -> Result<Vec<ColumnFamilyDescriptor>> {
let server = &ctx.server;
let config = &server.config;
let path = &config.database_path;
let existing = Self::discover_cfs(path, db_opts);
info!(
columns = num_cfds,
sequence = %db.latest_sequence_number(),
time = ?load_time.elapsed(),
"Opened database."
);
let creating = desc.iter().filter(|desc| !existing.contains(desc.name));
Ok(Arc::new(Self {
db,
pool: ctx.pool.clone(),
ctx: ctx.clone(),
checksums: config.rocksdb_checksums,
corks: AtomicU32::new(0),
}))
}
#[implement(Engine)]
#[tracing::instrument(name = "configure", skip_all, level = "debug")]
fn configure_cfds(
ctx: &Arc<Context>,
db_opts: &Options,
desc: &[Descriptor],
) -> Result<Vec<ColumnFamilyDescriptor>> {
let server = &ctx.server;
let config = &server.config;
let path = &config.database_path;
let existing = Self::discover_cfs(path, db_opts);
let creating = desc.iter().filter(|desc| !existing.contains(desc.name));
let missing = existing
.iter()
.filter(|&name| name != "default")
.filter(|&name| !desc.iter().any(|desc| desc.name == name));
debug!(
existing = existing.len(),
described = desc.len(),
missing = missing.clone().count(),
creating = creating.clone().count(),
"Discovered database columns"
);
missing.clone().for_each(|name| {
debug!("Found unrecognized column {name:?} in existing database.");
});
creating.map(|desc| desc.name).for_each(|name| {
debug!("Creating new column {name:?} not previously found in existing database.");
});
let missing_descriptors = missing.clone().map(|_| descriptor::DROPPED);
let cfopts: Vec<_> = desc
.iter()
.copied()
.chain(missing_descriptors)
.map(|ref desc| cf_options(ctx, db_opts.clone(), desc))
.collect::<Result<_>>()?;
let cfds: Vec<_> = desc
.iter()
.map(|desc| desc.name)
.map(ToOwned::to_owned)
.chain(missing.cloned())
.zip(cfopts.into_iter())
.map(|(name, opts)| ColumnFamilyDescriptor::new(name, opts))
.collect();
Ok(cfds)
}
#[implement(Engine)]
#[tracing::instrument(name = "discover", skip_all, level = "debug")]
fn discover_cfs(path: &Path, opts: &Options) -> BTreeSet<String> {
Db::list_cf(opts, path)
.unwrap_or_default()
.into_iter()
.collect::<BTreeSet<_>>()
let missing = existing
.iter()
.filter(|&name| name != "default")
.filter(|&name| !desc.iter().any(|desc| desc.name == name));
debug!(
existing = existing.len(),
described = desc.len(),
missing = missing.clone().count(),
creating = creating.clone().count(),
"Discovered database columns"
);
missing.clone().for_each(|name| {
debug!("Found unrecognized column {name:?} in existing database.");
});
creating.map(|desc| desc.name).for_each(|name| {
debug!("Creating new column {name:?} not previously found in existing database.");
});
let missing_descriptors = missing.clone().map(|_| descriptor::DROPPED);
let cfopts: Vec<_> = desc
.iter()
.copied()
.chain(missing_descriptors)
.map(|ref desc| cf_options(ctx, db_opts.clone(), desc))
.collect::<Result<_>>()?;
let cfds: Vec<_> = desc
.iter()
.map(|desc| desc.name)
.map(ToOwned::to_owned)
.chain(missing.cloned())
.zip(cfopts.into_iter())
.map(|(name, opts)| ColumnFamilyDescriptor::new(name, opts))
.collect();
Ok(cfds)
}
#[tracing::instrument(name = "discover", skip_all, level = "debug")]
fn discover_cfs(path: &Path, opts: &Options) -> BTreeSet<String> {
Db::list_cf(opts, path)
.unwrap_or_default()
.into_iter()
.collect::<BTreeSet<_>>()
}
}
+20 -19
View File
@@ -1,30 +1,31 @@
use std::sync::Arc;
use conduwuit::{
Result, implement,
Result,
utils::stream::{ReadyExt, TryIgnore},
};
use futures::{Stream, TryStreamExt};
use crate::keyval::Key;
/// Delete all data stored in this map. !!! USE WITH CAUTION !!!
///
/// See for_clear() with additional details.
#[implement(super::Map)]
#[tracing::instrument(level = "trace")]
pub async fn clear(self: &Arc<Self>) {
self.for_clear().ignore_err().ready_for_each(|_| ()).await;
}
impl super::Map {
/// Delete all data stored in this map. !!! USE WITH CAUTION !!!
///
/// See for_clear() with additional details.
#[tracing::instrument(level = "trace")]
pub async fn clear(self: &Arc<Self>) {
// drops your entire database cutely :3
self.for_clear().ignore_err().ready_for_each(|_| ()).await;
}
/// Delete all data stored in this map. !!! USE WITH CAUTION !!!
///
/// Provides stream of keys undergoing deletion along with any errors.
///
/// Note this operation applies to a snapshot of the data when invoked.
/// Additional data written during or after this call may be missed.
#[implement(super::Map)]
#[tracing::instrument(level = "trace")]
pub fn for_clear(self: &Arc<Self>) -> impl Stream<Item = Result<Key<'_>>> + Send {
self.raw_keys().inspect_ok(|key| self.remove(key))
/// Delete all data stored in this map. !!! USE WITH CAUTION !!!
///
/// Provides stream of keys undergoing deletion along with any errors.
///
/// Note this operation applies to a snapshot of the data when invoked.
/// Additional data written during or after this call may be missed.
#[tracing::instrument(level = "trace")]
pub fn for_clear(self: &Arc<Self>) -> impl Stream<Item = Result<Key<'_>>> + Send {
self.raw_keys().inspect_ok(|key| self.remove(key))
}
}
+36 -35
View File
@@ -1,4 +1,4 @@
use conduwuit::{Err, Result, implement};
use conduwuit::{Err, Result};
use rocksdb::{BottommostLevelCompaction, CompactOptions};
use crate::keyval::KeyBuf;
@@ -23,40 +23,41 @@ pub struct Options {
pub exclusive: bool,
}
#[implement(super::Map)]
#[tracing::instrument(
name = "compact",
level = "info",
skip(self),
fields(%self),
)]
pub fn compact_blocking(&self, opts: Options) -> Result {
let mut co = CompactOptions::default();
co.set_exclusive_manual_compaction(opts.exclusive);
co.set_bottommost_level_compaction(match opts.exhaustive {
| true => BottommostLevelCompaction::Force,
| false => BottommostLevelCompaction::ForceOptimized,
});
impl super::Map {
#[tracing::instrument(
name = "compact",
skip(self),
fields(%self),
)]
pub fn compact_blocking(&self, opts: Options) -> Result {
let mut co = CompactOptions::default();
co.set_exclusive_manual_compaction(opts.exclusive);
co.set_bottommost_level_compaction(match opts.exhaustive {
| true => BottommostLevelCompaction::Force,
| false => BottommostLevelCompaction::ForceOptimized,
});
match opts.level {
| (None, None) => {
co.set_change_level(true);
co.set_target_level(-1);
},
| (None, Some(level)) => {
co.set_change_level(true);
co.set_target_level(level.try_into()?);
},
| (Some(level), None) => {
co.set_change_level(false);
co.set_target_level(level.try_into()?);
},
| (Some(_), Some(_)) => return Err!("compacting between specific levels not supported"),
match opts.level {
| (None, None) => {
co.set_change_level(true);
co.set_target_level(-1);
},
| (None, Some(level)) => {
co.set_change_level(true);
co.set_target_level(level.try_into()?);
},
| (Some(level), None) => {
co.set_change_level(false);
co.set_target_level(level.try_into()?);
},
| (Some(_), Some(_)) =>
return Err!("compacting between specific levels not supported"),
}
self.db
.db
.compact_range_cf_opt(&self.cf(), opts.range.0, opts.range.1, &co);
Ok(())
}
self.db
.db
.compact_range_cf_opt(&self.cf(), opts.range.0, opts.range.1, &co);
Ok(())
}
+82 -85
View File
@@ -3,7 +3,7 @@ use std::{convert::AsRef, fmt::Debug, future::Future, io::Write, sync::Arc};
use conduwuit::{
Result,
arrayvec::ArrayVec,
err, implement,
err,
utils::{future::TryExtExt, result::FlatOk},
};
use futures::FutureExt;
@@ -11,93 +11,90 @@ use serde::Serialize;
use crate::{keyval::KeyBuf, ser};
/// Returns true if the map contains the key.
/// - key is serialized into allocated buffer
/// - harder errors may not be reported
#[inline]
#[implement(super::Map)]
pub fn contains<K>(
self: &Arc<Self>,
key: &K,
) -> impl Future<Output = bool> + Send + '_ + use<'_, K>
where
K: Serialize + ?Sized + Debug,
{
let mut buf = KeyBuf::new();
self.bcontains(key, &mut buf)
}
impl super::Map {
/// Returns true if the map contains the key.
/// - key is serialized into allocated buffer
/// - harder errors may not be reported
#[inline]
pub fn contains<K>(
self: &Arc<Self>,
key: &K,
) -> impl Future<Output = bool> + Send + '_ + use<'_, K>
where
K: Serialize + ?Sized + Debug,
{
let mut buf = KeyBuf::new();
self.bcontains(key, &mut buf)
}
/// Returns true if the map contains the key.
/// - key is serialized into stack-buffer
/// - harder errors will panic
#[inline]
#[implement(super::Map)]
pub fn acontains<const MAX: usize, K>(
self: &Arc<Self>,
key: &K,
) -> impl Future<Output = bool> + Send + '_ + use<'_, MAX, K>
where
K: Serialize + ?Sized + Debug,
{
let mut buf = ArrayVec::<u8, MAX>::new();
self.bcontains(key, &mut buf)
}
/// Returns true if the map contains the key.
/// - key is serialized into stack-buffer
/// - harder errors will panic
#[inline]
pub fn acontains<const MAX: usize, K>(
self: &Arc<Self>,
key: &K,
) -> impl Future<Output = bool> + Send + '_ + use<'_, MAX, K>
where
K: Serialize + ?Sized + Debug,
{
let mut buf = ArrayVec::<u8, MAX>::new();
self.bcontains(key, &mut buf)
}
/// Returns true if the map contains the key.
/// - key is serialized into provided buffer
/// - harder errors will panic
#[implement(super::Map)]
#[tracing::instrument(skip(self, buf), fields(%self), level = "trace")]
pub fn bcontains<K, B>(
self: &Arc<Self>,
key: &K,
buf: &mut B,
) -> impl Future<Output = bool> + Send + '_ + use<'_, K, B>
where
K: Serialize + ?Sized + Debug,
B: Write + AsRef<[u8]>,
{
let key = ser::serialize(buf, key).expect("failed to serialize query key");
self.exists(key).is_ok()
}
/// Returns true if the map contains the key.
/// - key is serialized into provided buffer
/// - harder errors will panic
#[tracing::instrument(skip(self, buf), fields(%self), level = "trace")]
pub fn bcontains<K, B>(
self: &Arc<Self>,
key: &K,
buf: &mut B,
) -> impl Future<Output = bool> + Send + '_ + use<'_, K, B>
where
K: Serialize + ?Sized + Debug,
B: Write + AsRef<[u8]>,
{
let key = ser::serialize(buf, key).expect("failed to serialize query key");
self.exists(key).is_ok()
}
/// Returns Ok if the map contains the key.
/// - key is raw
#[inline]
#[implement(super::Map)]
pub fn exists<'a, K>(
self: &'a Arc<Self>,
key: &K,
) -> impl Future<Output = Result> + Send + 'a + use<'a, K>
where
K: AsRef<[u8]> + ?Sized + Debug + 'a,
{
self.get(key).map(|res| res.map(|_| ()))
}
/// Returns Ok if the map contains the key.
/// - key is raw
#[inline]
pub fn exists<'a, K>(
self: &'a Arc<Self>,
key: &K,
) -> impl Future<Output = Result> + Send + 'a + use<'a, K>
where
K: AsRef<[u8]> + ?Sized + Debug + 'a,
{
self.get(key).map(|res| res.map(|_| ()))
}
/// Returns Ok if the map contains the key; NotFound otherwise. Harder errors
/// may not always be reported properly.
#[implement(super::Map)]
#[tracing::instrument(skip(self, key), fields(%self), level = "trace")]
pub fn exists_blocking<K>(&self, key: &K) -> Result
where
K: AsRef<[u8]> + ?Sized + Debug,
{
self.maybe_exists(key)
.then(|| self.get_blocking(key))
.flat_ok()
.map(|_| ())
.ok_or_else(|| err!(Request(NotFound("Not found in database"))))
}
/// Returns Ok if the map contains the key; NotFound otherwise. Harder
/// errors may not always be reported properly.
#[tracing::instrument(skip(self, key), fields(%self), level = "trace")]
pub fn exists_blocking<K>(&self, key: &K) -> Result
where
K: AsRef<[u8]> + ?Sized + Debug,
{
self.maybe_exists(key)
.then(|| self.get_blocking(key))
.flat_ok()
.map(|_| ())
.ok_or_else(|| err!(Request(NotFound("Not found in database"))))
}
/// Rocksdb limits this to kBlockCacheTier internally so this is not actually a
/// blocking call; in case that changes we set this as well in our read_options.
#[implement(super::Map)]
pub(crate) fn maybe_exists<K>(&self, key: &K) -> bool
where
K: AsRef<[u8]> + ?Sized,
{
self.db
.db
.key_may_exist_cf_opt(&self.cf(), key, &self.cache_read_options)
/// Rocksdb limits this to kBlockCacheTier internally so this is not
/// actually a blocking call; in case that changes we set this as well in
/// our read_options.
pub(crate) fn maybe_exists<K>(&self, key: &K) -> bool
where
K: AsRef<[u8]> + ?Sized,
{
self.db
.db
.key_may_exist_cf_opt(&self.cf(), key, &self.cache_read_options)
}
}
+58 -62
View File
@@ -1,72 +1,68 @@
use std::{fmt::Debug, future::Future, sync::Arc};
use conduwuit::implement;
use futures::stream::StreamExt;
use serde::Serialize;
/// Count the total number of entries in the map.
#[implement(super::Map)]
#[inline]
pub fn count(self: &Arc<Self>) -> impl Future<Output = usize> + Send + '_ {
self.raw_keys().count()
}
impl super::Map {
/// Count the total number of entries in the map.
#[inline]
pub fn count(self: &Arc<Self>) -> impl Future<Output = usize> + Send + '_ {
self.raw_keys().count()
}
/// Count the number of entries in the map starting from a lower-bound.
///
/// - From is a structured key
#[implement(super::Map)]
#[inline]
pub fn count_from<'a, P>(
self: &'a Arc<Self>,
from: &P,
) -> impl Future<Output = usize> + Send + 'a + use<'a, P>
where
P: Serialize + ?Sized + Debug + 'a,
{
self.keys_from_raw(from).count()
}
/// Count the number of entries in the map starting from a lower-bound.
///
/// - From is a structured key
#[inline]
pub fn count_from<'a, P>(
self: &'a Arc<Self>,
from: &P,
) -> impl Future<Output = usize> + Send + 'a + use<'a, P>
where
P: Serialize + ?Sized + Debug + 'a,
{
self.keys_from_raw(from).count()
}
/// Count the number of entries in the map starting from a lower-bound.
///
/// - From is a raw
#[implement(super::Map)]
#[inline]
pub fn raw_count_from<'a, P>(
self: &'a Arc<Self>,
from: &'a P,
) -> impl Future<Output = usize> + Send + 'a
where
P: AsRef<[u8]> + ?Sized + Debug + Sync + 'a,
{
self.raw_keys_from(from).count()
}
/// Count the number of entries in the map starting from a lower-bound.
///
/// - From is a raw
#[inline]
pub fn raw_count_from<'a, P>(
self: &'a Arc<Self>,
from: &'a P,
) -> impl Future<Output = usize> + Send + 'a
where
P: AsRef<[u8]> + ?Sized + Debug + Sync + 'a,
{
self.raw_keys_from(from).count()
}
/// Count the number of entries in the map matching a prefix.
///
/// - Prefix is structured key
#[implement(super::Map)]
#[inline]
pub fn count_prefix<'a, P>(
self: &'a Arc<Self>,
prefix: &P,
) -> impl Future<Output = usize> + Send + 'a + use<'a, P>
where
P: Serialize + ?Sized + Debug + 'a,
{
self.keys_prefix_raw(prefix).count()
}
/// Count the number of entries in the map matching a prefix.
///
/// - Prefix is structured key
#[inline]
pub fn count_prefix<'a, P>(
self: &'a Arc<Self>,
prefix: &P,
) -> impl Future<Output = usize> + Send + 'a + use<'a, P>
where
P: Serialize + ?Sized + Debug + 'a,
{
self.keys_prefix_raw(prefix).count()
}
/// Count the number of entries in the map matching a prefix.
///
/// - Prefix is raw
#[implement(super::Map)]
#[inline]
pub fn raw_count_prefix<'a, P>(
self: &'a Arc<Self>,
prefix: &'a P,
) -> impl Future<Output = usize> + Send + 'a
where
P: AsRef<[u8]> + ?Sized + Debug + Sync + 'a,
{
self.raw_keys_prefix(prefix).count()
/// Count the number of entries in the map matching a prefix.
///
/// - Prefix is raw
#[inline]
pub fn raw_count_prefix<'a, P>(
self: &'a Arc<Self>,
prefix: &'a P,
) -> impl Future<Output = usize> + Send + 'a
where
P: AsRef<[u8]> + ?Sized + Debug + Sync + 'a,
{
self.raw_keys_prefix(prefix).count()
}
}
+62 -64
View File
@@ -1,6 +1,6 @@
use std::{convert::AsRef, fmt::Debug, sync::Arc};
use conduwuit::{Err, Result, err, implement, utils::result::MapExpect};
use conduwuit::{Err, Result, err, utils::result::MapExpect};
use futures::{Future, FutureExt, TryFutureExt, future::ready};
use rocksdb::{DBPinnableSlice, ReadOptions};
use tokio::task;
@@ -10,74 +10,72 @@ use crate::{
util::{is_incomplete, map_err, or_else},
};
/// Fetch a value from the database into cache, returning a reference-handle
/// asynchronously. The key is referenced directly to perform the query.
#[implement(super::Map)]
#[tracing::instrument(skip(self, key), fields(%self), level = "trace")]
pub fn get<K>(
self: &Arc<Self>,
key: &K,
) -> impl Future<Output = Result<Handle<'_>>> + Send + use<'_, K>
where
K: AsRef<[u8]> + Debug + ?Sized,
{
use crate::pool::Get;
impl super::Map {
/// Fetch a value from the database into cache, returning a reference-handle
/// asynchronously. The key is referenced directly to perform the query.
#[tracing::instrument(skip(self, key), fields(%self), level = "trace")]
pub fn get<K>(
self: &Arc<Self>,
key: &K,
) -> impl Future<Output = Result<Handle<'_>>> + Send + use<'_, K>
where
K: AsRef<[u8]> + Debug + ?Sized,
{
use crate::pool::Get;
let cached = self.get_cached(key);
if matches!(cached, Err(_) | Ok(Some(_))) {
return task::consume_budget()
.map(move |()| cached.map_expect("data found in cache"))
.boxed();
let cached = self.get_cached(key);
if matches!(cached, Err(_) | Ok(Some(_))) {
return task::consume_budget()
.map(move |()| cached.map_expect("data found in cache"))
.boxed();
}
debug_assert!(matches!(cached, Ok(None)), "expected status Incomplete");
let cmd = Get {
map: self.clone(),
key: [key.as_ref().into()].into(),
res: None,
};
self.db
.pool
.execute_get(cmd)
.and_then(|mut res| ready(res.remove(0)))
.boxed()
}
debug_assert!(matches!(cached, Ok(None)), "expected status Incomplete");
let cmd = Get {
map: self.clone(),
key: [key.as_ref().into()].into(),
res: None,
};
/// Fetch a value from the cache without I/O.
#[tracing::instrument(skip(self, key), name = "cache", level = "trace")]
pub(crate) fn get_cached<K>(&self, key: &K) -> Result<Option<Handle<'_>>>
where
K: AsRef<[u8]> + Debug + ?Sized,
{
let res = self.get_blocking_opts(key, &self.cache_read_options);
cached_handle_from(res)
}
self.db
.pool
.execute_get(cmd)
.and_then(|mut res| ready(res.remove(0)))
.boxed()
}
/// Fetch a value from the database into cache, returning a
/// reference-handle. The key is referenced directly to perform the query.
/// This is a thread- blocking call.
#[tracing::instrument(skip(self, key), name = "blocking", level = "trace")]
pub fn get_blocking<K>(&self, key: &K) -> Result<Handle<'_>>
where
K: AsRef<[u8]> + ?Sized,
{
let res = self.get_blocking_opts(key, &self.read_options);
handle_from(res)
}
/// Fetch a value from the cache without I/O.
#[implement(super::Map)]
#[tracing::instrument(skip(self, key), name = "cache", level = "trace")]
pub(crate) fn get_cached<K>(&self, key: &K) -> Result<Option<Handle<'_>>>
where
K: AsRef<[u8]> + Debug + ?Sized,
{
let res = self.get_blocking_opts(key, &self.cache_read_options);
cached_handle_from(res)
}
/// Fetch a value from the database into cache, returning a reference-handle.
/// The key is referenced directly to perform the query. This is a thread-
/// blocking call.
#[implement(super::Map)]
#[tracing::instrument(skip(self, key), name = "blocking", level = "trace")]
pub fn get_blocking<K>(&self, key: &K) -> Result<Handle<'_>>
where
K: AsRef<[u8]> + ?Sized,
{
let res = self.get_blocking_opts(key, &self.read_options);
handle_from(res)
}
#[implement(super::Map)]
fn get_blocking_opts<K>(
&self,
key: &K,
read_options: &ReadOptions,
) -> Result<Option<DBPinnableSlice<'_>>, rocksdb::Error>
where
K: AsRef<[u8]> + ?Sized,
{
self.db.db.get_pinned_cf_opt(&self.cf(), key, read_options)
fn get_blocking_opts<K>(
&self,
key: &K,
read_options: &ReadOptions,
) -> Result<Option<DBPinnableSlice<'_>>, rocksdb::Error>
where
K: AsRef<[u8]> + ?Sized,
{
self.db.db.get_pinned_cf_opt(&self.cf(), key, read_options)
}
}
#[inline]
+51 -52
View File
@@ -1,7 +1,7 @@
use std::{convert::AsRef, sync::Arc};
use conduwuit::{
Result, implement,
Result,
utils::{
IterStream,
stream::{WidebandExt, automatic_amplification, automatic_width},
@@ -34,60 +34,59 @@ where
}
}
#[implement(super::Map)]
#[tracing::instrument(skip(self, keys), level = "trace")]
pub(crate) fn get_batch<'a, S, K>(
self: &'a Arc<Self>,
keys: S,
) -> impl Stream<Item = Result<Handle<'a>>> + Send + 'a
where
S: Stream<Item = K> + Send + 'a,
K: AsRef<[u8]> + Send + Sync + 'a,
{
use crate::pool::Get;
impl super::Map {
#[tracing::instrument(skip(self, keys), level = "trace")]
pub(crate) fn get_batch<'a, S, K>(
self: &'a Arc<Self>,
keys: S,
) -> impl Stream<Item = Result<Handle<'a>>> + Send + 'a
where
S: Stream<Item = K> + Send + 'a,
K: AsRef<[u8]> + Send + Sync + 'a,
{
use crate::pool::Get;
keys.ready_chunks(automatic_amplification())
.widen_then(automatic_width(), |chunk| {
self.db.pool.execute_get(Get {
map: self.clone(),
key: chunk.iter().map(AsRef::as_ref).map(Into::into).collect(),
res: None,
keys.ready_chunks(automatic_amplification())
.widen_then(automatic_width(), |chunk| {
self.db.pool.execute_get(Get {
map: self.clone(),
key: chunk.iter().map(AsRef::as_ref).map(Into::into).collect(),
res: None,
})
})
})
.map_ok(|results| results.into_iter().stream())
.try_flatten()
}
.map_ok(|results| results.into_iter().stream())
.try_flatten()
}
#[implement(super::Map)]
#[tracing::instrument(name = "batch_blocking", level = "trace", skip_all)]
pub(crate) fn get_batch_blocking<'a, I, K>(
&self,
keys: I,
) -> impl Iterator<Item = Result<Handle<'_>>> + Send + use<'_, I, K>
where
I: Iterator<Item = &'a K> + ExactSizeIterator + Send,
K: AsRef<[u8]> + Send + ?Sized + Sync + 'a,
{
self.get_batch_blocking_opts(keys, &self.read_options)
.map(handle_from)
}
#[tracing::instrument(name = "batch_blocking", level = "trace", skip_all)]
pub(crate) fn get_batch_blocking<'a, I, K>(
&self,
keys: I,
) -> impl Iterator<Item = Result<Handle<'_>>> + Send + use<'_, I, K>
where
I: Iterator<Item = &'a K> + ExactSizeIterator + Send,
K: AsRef<[u8]> + Send + ?Sized + Sync + 'a,
{
self.get_batch_blocking_opts(keys, &self.read_options)
.map(handle_from)
}
#[implement(super::Map)]
fn get_batch_blocking_opts<'a, I, K>(
&self,
keys: I,
read_options: &ReadOptions,
) -> impl Iterator<Item = Result<Option<DBPinnableSlice<'_>>, rocksdb::Error>> + Send + use<'_, I, K>
where
I: Iterator<Item = &'a K> + ExactSizeIterator + Send,
K: AsRef<[u8]> + Send + ?Sized + Sync + 'a,
{
// Optimization can be `true` if key vector is pre-sorted **by the column
// comparator**.
const SORTED: bool = false;
fn get_batch_blocking_opts<'a, I, K>(
&self,
keys: I,
read_options: &ReadOptions,
) -> impl Iterator<Item = Result<Option<DBPinnableSlice<'_>>, rocksdb::Error>> + Send + use<'_, I, K>
where
I: Iterator<Item = &'a K> + ExactSizeIterator + Send,
K: AsRef<[u8]> + Send + ?Sized + Sync + 'a,
{
// Optimization can be `true` if key vector is pre-sorted **by the column
// comparator**.
const SORTED: bool = false;
self.db
.db
.batched_multi_get_cf_opt(&self.cf(), keys, SORTED, read_options)
.into_iter()
self.db
.db
.batched_multi_get_cf_opt(&self.cf(), keys, SORTED, read_options)
.into_iter()
}
}
+203 -214
View File
@@ -5,7 +5,7 @@
use std::{convert::AsRef, fmt::Debug, io::Write};
use conduwuit::{arrayvec::ArrayVec, implement};
use conduwuit::arrayvec::ArrayVec;
use rocksdb::WriteBatchWithTransaction;
use serde::Serialize;
@@ -15,223 +15,212 @@ use crate::{
util::or_else,
};
/// Insert Key/Value
///
/// - Key is serialized
/// - Val is serialized
#[implement(super::Map)]
#[inline]
pub fn put<K, V>(&self, key: K, val: V)
where
K: Serialize + Debug,
V: Serialize,
{
let mut key_buf = KeyBuf::new();
let mut val_buf = ValBuf::new();
self.bput(key, val, (&mut key_buf, &mut val_buf));
}
/// Insert Key/Value
///
/// - Key is serialized
/// - Val is raw
#[implement(super::Map)]
#[inline]
pub fn put_raw<K, V>(&self, key: K, val: V)
where
K: Serialize + Debug,
V: AsRef<[u8]>,
{
let mut key_buf = KeyBuf::new();
self.bput_raw(key, val, &mut key_buf);
}
/// Insert Key/Value
///
/// - Key is raw
/// - Val is serialized
#[implement(super::Map)]
#[inline]
pub fn raw_put<K, V>(&self, key: K, val: V)
where
K: AsRef<[u8]>,
V: Serialize,
{
let mut val_buf = ValBuf::new();
self.raw_bput(key, val, &mut val_buf);
}
/// Insert Key/Value
///
/// - Key is serialized
/// - Val is serialized to stack-buffer
#[implement(super::Map)]
#[inline]
pub fn put_aput<const VMAX: usize, K, V>(&self, key: K, val: V)
where
K: Serialize + Debug,
V: Serialize,
{
let mut key_buf = KeyBuf::new();
let mut val_buf = ArrayVec::<u8, VMAX>::new();
self.bput(key, val, (&mut key_buf, &mut val_buf));
}
/// Insert Key/Value
///
/// - Key is serialized to stack-buffer
/// - Val is serialized
#[implement(super::Map)]
#[inline]
pub fn aput_put<const KMAX: usize, K, V>(&self, key: K, val: V)
where
K: Serialize + Debug,
V: Serialize,
{
let mut key_buf = ArrayVec::<u8, KMAX>::new();
let mut val_buf = ValBuf::new();
self.bput(key, val, (&mut key_buf, &mut val_buf));
}
/// Insert Key/Value
///
/// - Key is serialized to stack-buffer
/// - Val is serialized to stack-buffer
#[implement(super::Map)]
#[inline]
pub fn aput<const KMAX: usize, const VMAX: usize, K, V>(&self, key: K, val: V)
where
K: Serialize + Debug,
V: Serialize,
{
let mut key_buf = ArrayVec::<u8, KMAX>::new();
let mut val_buf = ArrayVec::<u8, VMAX>::new();
self.bput(key, val, (&mut key_buf, &mut val_buf));
}
/// Insert Key/Value
///
/// - Key is serialized to stack-buffer
/// - Val is raw
#[implement(super::Map)]
#[inline]
pub fn aput_raw<const KMAX: usize, K, V>(&self, key: K, val: V)
where
K: Serialize + Debug,
V: AsRef<[u8]>,
{
let mut key_buf = ArrayVec::<u8, KMAX>::new();
self.bput_raw(key, val, &mut key_buf);
}
/// Insert Key/Value
///
/// - Key is raw
/// - Val is serialized to stack-buffer
#[implement(super::Map)]
#[inline]
pub fn raw_aput<const VMAX: usize, K, V>(&self, key: K, val: V)
where
K: AsRef<[u8]>,
V: Serialize,
{
let mut val_buf = ArrayVec::<u8, VMAX>::new();
self.raw_bput(key, val, &mut val_buf);
}
/// Insert Key/Value
///
/// - Key is serialized to supplied buffer
/// - Val is serialized to supplied buffer
#[implement(super::Map)]
pub fn bput<K, V, Bk, Bv>(&self, key: K, val: V, mut buf: (Bk, Bv))
where
K: Serialize + Debug,
V: Serialize,
Bk: Write + AsRef<[u8]>,
Bv: Write + AsRef<[u8]>,
{
let val = ser::serialize(&mut buf.1, val).expect("failed to serialize insertion val");
self.bput_raw(key, val, &mut buf.0);
}
/// Insert Key/Value
///
/// - Key is serialized to supplied buffer
/// - Val is raw
#[implement(super::Map)]
#[tracing::instrument(skip(self, val, buf), level = "trace")]
pub fn bput_raw<K, V, Bk>(&self, key: K, val: V, mut buf: Bk)
where
K: Serialize + Debug,
V: AsRef<[u8]>,
Bk: Write + AsRef<[u8]>,
{
let key = ser::serialize(&mut buf, key).expect("failed to serialize insertion key");
self.insert(&key, val);
}
/// Insert Key/Value
///
/// - Key is raw
/// - Val is serialized to supplied buffer
#[implement(super::Map)]
pub fn raw_bput<K, V, Bv>(&self, key: K, val: V, mut buf: Bv)
where
K: AsRef<[u8]>,
V: Serialize,
Bv: Write + AsRef<[u8]>,
{
let val = ser::serialize(&mut buf, val).expect("failed to serialize insertion val");
self.insert(&key, val);
}
/// Insert Key/Value
///
/// - Key is raw
/// - Val is raw
#[implement(super::Map)]
#[tracing::instrument(skip_all, fields(%self), level = "trace")]
pub fn insert<K, V>(&self, key: &K, val: V)
where
K: AsRef<[u8]> + ?Sized,
V: AsRef<[u8]>,
{
let write_options = &self.write_options;
self.db
.db
.put_cf_opt(&self.cf(), key, val, write_options)
.or_else(or_else)
.expect("database insert error");
if !self.db.corked() {
self.db.flush().expect("database flush error");
impl super::Map {
/// Insert Key/Value
///
/// - Key is serialized
/// - Val is serialized
#[inline]
pub fn put<K, V>(&self, key: K, val: V)
where
K: Serialize + Debug,
V: Serialize,
{
let mut key_buf = KeyBuf::new();
let mut val_buf = ValBuf::new();
self.bput(key, val, (&mut key_buf, &mut val_buf));
}
self.watchers.wake(key.as_ref());
}
#[implement(super::Map)]
#[tracing::instrument(skip(self, iter), fields(%self), level = "trace")]
pub fn insert_batch<'a, I, K, V>(&'a self, iter: I)
where
I: Iterator<Item = (K, V)> + Send + Debug,
K: AsRef<[u8]> + Sized + Debug + 'a,
V: AsRef<[u8]> + Sized + 'a,
{
let mut batch = WriteBatchWithTransaction::<false>::default();
for (key, val) in iter {
batch.put_cf(&self.cf(), key.as_ref(), val.as_ref());
/// Insert Key/Value
///
/// - Key is serialized
/// - Val is raw
#[inline]
pub fn put_raw<K, V>(&self, key: K, val: V)
where
K: Serialize + Debug,
V: AsRef<[u8]>,
{
let mut key_buf = KeyBuf::new();
self.bput_raw(key, val, &mut key_buf);
}
let write_options = &self.write_options;
self.db
.db
.write_opt(&batch, write_options)
.or_else(or_else)
.expect("database insert batch error");
/// Insert Key/Value
///
/// - Key is raw
/// - Val is serialized
#[inline]
pub fn raw_put<K, V>(&self, key: K, val: V)
where
K: AsRef<[u8]>,
V: Serialize,
{
let mut val_buf = ValBuf::new();
self.raw_bput(key, val, &mut val_buf);
}
if !self.db.corked() {
self.db.flush().expect("database flush error");
/// Insert Key/Value
///
/// - Key is serialized
/// - Val is serialized to stack-buffer
#[inline]
pub fn put_aput<const VMAX: usize, K, V>(&self, key: K, val: V)
where
K: Serialize + Debug,
V: Serialize,
{
let mut key_buf = KeyBuf::new();
let mut val_buf = ArrayVec::<u8, VMAX>::new();
self.bput(key, val, (&mut key_buf, &mut val_buf));
}
/// Insert Key/Value
///
/// - Key is serialized to stack-buffer
/// - Val is serialized
#[inline]
pub fn aput_put<const KMAX: usize, K, V>(&self, key: K, val: V)
where
K: Serialize + Debug,
V: Serialize,
{
let mut key_buf = ArrayVec::<u8, KMAX>::new();
let mut val_buf = ValBuf::new();
self.bput(key, val, (&mut key_buf, &mut val_buf));
}
/// Insert Key/Value
///
/// - Key is serialized to stack-buffer
/// - Val is serialized to stack-buffer
#[inline]
pub fn aput<const KMAX: usize, const VMAX: usize, K, V>(&self, key: K, val: V)
where
K: Serialize + Debug,
V: Serialize,
{
let mut key_buf = ArrayVec::<u8, KMAX>::new();
let mut val_buf = ArrayVec::<u8, VMAX>::new();
self.bput(key, val, (&mut key_buf, &mut val_buf));
}
/// Insert Key/Value
///
/// - Key is serialized to stack-buffer
/// - Val is raw
#[inline]
pub fn aput_raw<const KMAX: usize, K, V>(&self, key: K, val: V)
where
K: Serialize + Debug,
V: AsRef<[u8]>,
{
let mut key_buf = ArrayVec::<u8, KMAX>::new();
self.bput_raw(key, val, &mut key_buf);
}
/// Insert Key/Value
///
/// - Key is raw
/// - Val is serialized to stack-buffer
#[inline]
pub fn raw_aput<const VMAX: usize, K, V>(&self, key: K, val: V)
where
K: AsRef<[u8]>,
V: Serialize,
{
let mut val_buf = ArrayVec::<u8, VMAX>::new();
self.raw_bput(key, val, &mut val_buf);
}
/// Insert Key/Value
///
/// - Key is serialized to supplied buffer
/// - Val is serialized to supplied buffer
pub fn bput<K, V, Bk, Bv>(&self, key: K, val: V, mut buf: (Bk, Bv))
where
K: Serialize + Debug,
V: Serialize,
Bk: Write + AsRef<[u8]>,
Bv: Write + AsRef<[u8]>,
{
let val = ser::serialize(&mut buf.1, val).expect("failed to serialize insertion val");
self.bput_raw(key, val, &mut buf.0);
}
/// Insert Key/Value
///
/// - Key is serialized to supplied buffer
/// - Val is raw
#[tracing::instrument(skip(self, val, buf), level = "trace")]
pub fn bput_raw<K, V, Bk>(&self, key: K, val: V, mut buf: Bk)
where
K: Serialize + Debug,
V: AsRef<[u8]>,
Bk: Write + AsRef<[u8]>,
{
let key = ser::serialize(&mut buf, key).expect("failed to serialize insertion key");
self.insert(&key, val);
}
/// Insert Key/Value
///
/// - Key is raw
/// - Val is serialized to supplied buffer
pub fn raw_bput<K, V, Bv>(&self, key: K, val: V, mut buf: Bv)
where
K: AsRef<[u8]>,
V: Serialize,
Bv: Write + AsRef<[u8]>,
{
let val = ser::serialize(&mut buf, val).expect("failed to serialize insertion val");
self.insert(&key, val);
}
/// Insert Key/Value
///
/// - Key is raw
/// - Val is raw
#[tracing::instrument(skip_all, fields(%self), level = "trace")]
pub fn insert<K, V>(&self, key: &K, val: V)
where
K: AsRef<[u8]> + ?Sized,
V: AsRef<[u8]>,
{
let write_options = &self.write_options;
self.db
.db
.put_cf_opt(&self.cf(), key, val, write_options)
.or_else(or_else)
.expect("database insert error");
if !self.db.corked() {
self.db.flush().expect("database flush error");
}
self.watchers.wake(key.as_ref());
}
#[tracing::instrument(skip(self, iter), fields(%self), level = "trace")]
pub fn insert_batch<'a, I, K, V>(&'a self, iter: I)
where
I: Iterator<Item = (K, V)> + Send + Debug,
K: AsRef<[u8]> + Sized + Debug + 'a,
V: AsRef<[u8]> + Sized + 'a,
{
let mut batch = WriteBatchWithTransaction::<false>::default();
for (key, val) in iter {
batch.put_cf(&self.cf(), key.as_ref(), val.as_ref());
}
let write_options = &self.write_options;
self.db
.db
.write_opt(&batch, write_options)
.or_else(or_else)
.expect("database insert batch error");
if !self.db.corked() {
self.db.flush().expect("database flush error");
}
}
}
+36 -37
View File
@@ -1,6 +1,6 @@
use std::sync::Arc;
use conduwuit::{Result, implement};
use conduwuit::Result;
use futures::{FutureExt, Stream, StreamExt, TryFutureExt, TryStreamExt};
use rocksdb::Direction;
use serde::Deserialize;
@@ -9,43 +9,42 @@ use tokio::task;
use super::stream::is_cached;
use crate::{keyval, keyval::Key, stream};
#[implement(super::Map)]
pub fn keys<'a, K>(self: &'a Arc<Self>) -> impl Stream<Item = Result<Key<'a, K>>> + Send
where
K: Deserialize<'a> + Send,
{
self.raw_keys().map(keyval::result_deserialize_key::<K>)
}
#[implement(super::Map)]
#[tracing::instrument(skip(self), fields(%self), level = "trace")]
pub fn raw_keys(self: &Arc<Self>) -> impl Stream<Item = Result<Key<'_>>> + Send {
use crate::pool::Seek;
let opts = super::iter_options_default(&self.db);
let state = stream::State::new(self, opts);
if is_cached(self) {
let state = state.init_fwd(None);
return task::consume_budget()
.map(move |()| stream::Keys::<'_>::from(state))
.into_stream()
.flatten()
.boxed();
impl super::Map {
pub fn keys<'a, K>(self: &'a Arc<Self>) -> impl Stream<Item = Result<Key<'a, K>>> + Send
where
K: Deserialize<'a> + Send,
{
self.raw_keys().map(keyval::result_deserialize_key::<K>)
}
let seek = Seek {
map: self.clone(),
dir: Direction::Forward,
state: crate::pool::into_send_seek(state),
key: None,
res: None,
};
pub fn raw_keys(self: &Arc<Self>) -> impl Stream<Item = Result<Key<'_>>> + Send {
use crate::pool::Seek;
self.db
.pool
.execute_iter(seek)
.ok_into::<stream::Keys<'_>>()
.into_stream()
.try_flatten()
.boxed()
let opts = super::iter_options_default(&self.db);
let state = stream::State::new(self, opts);
if is_cached(self) {
let state = state.init_fwd(None);
return task::consume_budget()
.map(move |()| stream::Keys::<'_>::from(state))
.into_stream()
.flatten()
.boxed();
}
let seek = Seek {
map: self.clone(),
dir: Direction::Forward,
state: crate::pool::into_send_seek(state),
key: None,
res: None,
};
self.db
.pool
.execute_iter(seek)
.ok_into::<stream::Keys<'_>>()
.into_stream()
.try_flatten()
.boxed()
}
}
+65 -67
View File
@@ -1,6 +1,6 @@
use std::{convert::AsRef, fmt::Debug, sync::Arc};
use conduwuit::{Result, implement};
use conduwuit::Result;
use futures::{FutureExt, Stream, StreamExt, TryFutureExt, TryStreamExt};
use rocksdb::Direction;
use serde::{Deserialize, Serialize};
@@ -11,73 +11,71 @@ use crate::{
stream,
};
#[implement(super::Map)]
pub fn keys_from<'a, K, P>(
self: &'a Arc<Self>,
from: &P,
) -> impl Stream<Item = Result<Key<'a, K>>> + Send + use<'a, K, P>
where
P: Serialize + ?Sized + Debug,
K: Deserialize<'a> + Send,
{
self.keys_from_raw(from).map(result_deserialize_key::<K>)
}
#[implement(super::Map)]
#[tracing::instrument(skip(self), level = "trace")]
pub fn keys_from_raw<P>(
self: &Arc<Self>,
from: &P,
) -> impl Stream<Item = Result<Key<'_>>> + Send + use<'_, P>
where
P: Serialize + ?Sized + Debug,
{
let key = serialize_key(from).expect("failed to serialize query key");
self.raw_keys_from(&key)
}
#[implement(super::Map)]
pub fn keys_raw_from<'a, K, P>(
self: &'a Arc<Self>,
from: &P,
) -> impl Stream<Item = Result<Key<'a, K>>> + Send + use<'a, K, P>
where
P: AsRef<[u8]> + ?Sized + Debug + Sync,
K: Deserialize<'a> + Send,
{
self.raw_keys_from(from).map(result_deserialize_key::<K>)
}
#[implement(super::Map)]
#[tracing::instrument(skip(self, from), fields(%self), level = "trace")]
pub fn raw_keys_from<P>(
self: &Arc<Self>,
from: &P,
) -> impl Stream<Item = Result<Key<'_>>> + Send + use<'_, P>
where
P: AsRef<[u8]> + ?Sized + Debug,
{
use crate::pool::Seek;
let opts = super::iter_options_default(&self.db);
let state = stream::State::new(self, opts);
if is_cached(self, from) {
return stream::Keys::<'_>::from(state.init_fwd(from.as_ref().into())).boxed();
impl super::Map {
pub fn keys_from<'a, K, P>(
self: &'a Arc<Self>,
from: &P,
) -> impl Stream<Item = Result<Key<'a, K>>> + Send + use<'a, K, P>
where
P: Serialize + ?Sized + Debug,
K: Deserialize<'a> + Send,
{
self.keys_from_raw(from).map(result_deserialize_key::<K>)
}
let seek = Seek {
map: self.clone(),
dir: Direction::Forward,
key: Some(from.as_ref().into()),
state: crate::pool::into_send_seek(state),
res: None,
};
#[tracing::instrument(skip(self), level = "trace")]
pub fn keys_from_raw<P>(
self: &Arc<Self>,
from: &P,
) -> impl Stream<Item = Result<Key<'_>>> + Send + use<'_, P>
where
P: Serialize + ?Sized + Debug,
{
let key = serialize_key(from).expect("failed to serialize query key");
self.raw_keys_from(&key)
}
self.db
.pool
.execute_iter(seek)
.ok_into::<stream::Keys<'_>>()
.into_stream()
.try_flatten()
.boxed()
pub fn keys_raw_from<'a, K, P>(
self: &'a Arc<Self>,
from: &P,
) -> impl Stream<Item = Result<Key<'a, K>>> + Send + use<'a, K, P>
where
P: AsRef<[u8]> + ?Sized + Debug + Sync,
K: Deserialize<'a> + Send,
{
self.raw_keys_from(from).map(result_deserialize_key::<K>)
}
#[tracing::instrument(skip(self, from), fields(%self), level = "trace")]
pub fn raw_keys_from<P>(
self: &Arc<Self>,
from: &P,
) -> impl Stream<Item = Result<Key<'_>>> + Send + use<'_, P>
where
P: AsRef<[u8]> + ?Sized + Debug,
{
use crate::pool::Seek;
let opts = super::iter_options_default(&self.db);
let state = stream::State::new(self, opts);
if is_cached(self, from) {
return stream::Keys::<'_>::from(state.init_fwd(from.as_ref().into())).boxed();
}
let seek = Seek {
map: self.clone(),
dir: Direction::Forward,
key: Some(from.as_ref().into()),
state: crate::pool::into_send_seek(state),
res: None,
};
self.db
.pool
.execute_iter(seek)
.ok_into::<stream::Keys<'_>>()
.into_stream()
.try_flatten()
.boxed()
}
}
+46 -48
View File
@@ -1,59 +1,57 @@
use std::{convert::AsRef, fmt::Debug, sync::Arc};
use conduwuit::{Result, implement};
use conduwuit::Result;
use futures::{Stream, StreamExt, TryStreamExt, future};
use serde::{Deserialize, Serialize};
use crate::keyval::{Key, result_deserialize_key, serialize_key};
#[implement(super::Map)]
pub fn keys_prefix<'a, K, P>(
self: &'a Arc<Self>,
prefix: &P,
) -> impl Stream<Item = Result<Key<'a, K>>> + Send + use<'a, K, P>
where
P: Serialize + ?Sized + Debug,
K: Deserialize<'a> + Send,
{
self.keys_prefix_raw(prefix)
.map(result_deserialize_key::<K>)
}
impl super::Map {
pub fn keys_prefix<'a, K, P>(
self: &'a Arc<Self>,
prefix: &P,
) -> impl Stream<Item = Result<Key<'a, K>>> + Send + use<'a, K, P>
where
P: Serialize + ?Sized + Debug,
K: Deserialize<'a> + Send,
{
self.keys_prefix_raw(prefix)
.map(result_deserialize_key::<K>)
}
#[implement(super::Map)]
#[tracing::instrument(skip(self), level = "trace")]
pub fn keys_prefix_raw<P>(
self: &Arc<Self>,
prefix: &P,
) -> impl Stream<Item = Result<Key<'_>>> + Send + use<'_, P>
where
P: Serialize + ?Sized + Debug,
{
let key = serialize_key(prefix).expect("failed to serialize query key");
self.raw_keys_from(&key)
.try_take_while(move |k: &Key<'_>| future::ok(k.starts_with(&key)))
}
#[tracing::instrument(skip(self), level = "trace")]
pub fn keys_prefix_raw<P>(
self: &Arc<Self>,
prefix: &P,
) -> impl Stream<Item = Result<Key<'_>>> + Send + use<'_, P>
where
P: Serialize + ?Sized + Debug,
{
let key = serialize_key(prefix).expect("failed to serialize query key");
self.raw_keys_from(&key)
.try_take_while(move |k: &Key<'_>| future::ok(k.starts_with(&key)))
}
#[implement(super::Map)]
pub fn keys_raw_prefix<'a, K, P>(
self: &'a Arc<Self>,
prefix: &'a P,
) -> impl Stream<Item = Result<Key<'a, K>>> + Send + 'a
where
P: AsRef<[u8]> + ?Sized + Debug + Sync + 'a,
K: Deserialize<'a> + Send + 'a,
{
self.raw_keys_prefix(prefix)
.map(result_deserialize_key::<K>)
}
pub fn keys_raw_prefix<'a, K, P>(
self: &'a Arc<Self>,
prefix: &'a P,
) -> impl Stream<Item = Result<Key<'a, K>>> + Send + 'a
where
P: AsRef<[u8]> + ?Sized + Debug + Sync + 'a,
K: Deserialize<'a> + Send + 'a,
{
self.raw_keys_prefix(prefix)
.map(result_deserialize_key::<K>)
}
#[implement(super::Map)]
pub fn raw_keys_prefix<'a, P>(
self: &'a Arc<Self>,
prefix: &'a P,
) -> impl Stream<Item = Result<Key<'a>>> + Send + 'a
where
P: AsRef<[u8]> + ?Sized + Debug + Sync + 'a,
{
self.raw_keys_from(prefix)
.try_take_while(|k: &Key<'_>| future::ok(k.starts_with(prefix.as_ref())))
pub fn raw_keys_prefix<'a, P>(
self: &'a Arc<Self>,
prefix: &'a P,
) -> impl Stream<Item = Result<Key<'a>>> + Send + 'a
where
P: AsRef<[u8]> + ?Sized + Debug + Sync + 'a,
{
self.raw_keys_from(prefix)
.try_take_while(|k: &Key<'_>| future::ok(k.starts_with(prefix.as_ref())))
}
}
+46 -46
View File
@@ -1,56 +1,56 @@
use std::{convert::AsRef, fmt::Debug, io::Write, sync::Arc};
use conduwuit::{Result, arrayvec::ArrayVec, implement};
use conduwuit::{Result, arrayvec::ArrayVec};
use futures::Future;
use serde::Serialize;
use crate::{Handle, keyval::KeyBuf, ser};
/// Fetch a value from the database into cache, returning a reference-handle
/// asynchronously. The key is serialized into an allocated buffer to perform
/// the query.
#[implement(super::Map)]
#[inline]
pub fn qry<K>(
self: &Arc<Self>,
key: &K,
) -> impl Future<Output = Result<Handle<'_>>> + Send + use<'_, K>
where
K: Serialize + ?Sized + Debug,
{
let mut buf = KeyBuf::new();
self.bqry(key, &mut buf)
}
impl super::Map {
/// Fetch a value from the database into cache, returning a reference-handle
/// asynchronously. The key is serialized into an allocated buffer to
/// perform the query.
#[inline]
pub fn qry<K>(
self: &Arc<Self>,
key: &K,
) -> impl Future<Output = Result<Handle<'_>>> + Send + use<'_, K>
where
K: Serialize + ?Sized + Debug,
{
let mut buf = KeyBuf::new();
self.bqry(key, &mut buf)
}
/// Fetch a value from the database into cache, returning a reference-handle
/// asynchronously. The key is serialized into a fixed-sized buffer to perform
/// the query. The maximum size is supplied as const generic parameter.
#[implement(super::Map)]
#[inline]
pub fn aqry<const MAX: usize, K>(
self: &Arc<Self>,
key: &K,
) -> impl Future<Output = Result<Handle<'_>>> + Send + use<'_, MAX, K>
where
K: Serialize + ?Sized + Debug,
{
let mut buf = ArrayVec::<u8, MAX>::new();
self.bqry(key, &mut buf)
}
/// Fetch a value from the database into cache, returning a reference-handle
/// asynchronously. The key is serialized into a fixed-sized buffer to
/// perform the query. The maximum size is supplied as const generic
/// parameter.
#[inline]
pub fn aqry<const MAX: usize, K>(
self: &Arc<Self>,
key: &K,
) -> impl Future<Output = Result<Handle<'_>>> + Send + use<'_, MAX, K>
where
K: Serialize + ?Sized + Debug,
{
let mut buf = ArrayVec::<u8, MAX>::new();
self.bqry(key, &mut buf)
}
/// Fetch a value from the database into cache, returning a reference-handle
/// asynchronously. The key is serialized into a user-supplied Writer.
#[implement(super::Map)]
#[tracing::instrument(skip(self, buf), level = "trace")]
pub fn bqry<K, B>(
self: &Arc<Self>,
key: &K,
buf: &mut B,
) -> impl Future<Output = Result<Handle<'_>>> + Send + use<'_, K, B>
where
K: Serialize + ?Sized + Debug,
B: Write + AsRef<[u8]>,
{
let key = ser::serialize(buf, key).expect("failed to serialize query key");
self.get(key)
/// Fetch a value from the database into cache, returning a reference-handle
/// asynchronously. The key is serialized into a user-supplied Writer.
#[tracing::instrument(skip(self, buf), level = "trace")]
pub fn bqry<K, B>(
self: &Arc<Self>,
key: &K,
buf: &mut B,
) -> impl Future<Output = Result<Handle<'_>>> + Send + use<'_, K, B>
where
K: Serialize + ?Sized + Debug,
B: Write + AsRef<[u8]>,
{
let key = ser::serialize(buf, key).expect("failed to serialize query key");
self.get(key)
}
}
+26 -25
View File
@@ -1,7 +1,7 @@
use std::{fmt::Debug, sync::Arc};
use conduwuit::{
Result, implement,
Result,
utils::{
IterStream,
stream::{WidebandExt, automatic_amplification, automatic_width},
@@ -32,30 +32,31 @@ where
}
}
#[implement(super::Map)]
#[tracing::instrument(skip(self, keys), level = "trace")]
pub(crate) fn qry_batch<'a, S, K>(
self: &'a Arc<Self>,
keys: S,
) -> impl Stream<Item = Result<Handle<'a>>> + Send + 'a
where
S: Stream<Item = K> + Send + 'a,
K: Serialize + Debug + 'a,
{
use crate::pool::Get;
impl super::Map {
#[tracing::instrument(skip(self, keys), level = "trace")]
pub(crate) fn qry_batch<'a, S, K>(
self: &'a Arc<Self>,
keys: S,
) -> impl Stream<Item = Result<Handle<'a>>> + Send + 'a
where
S: Stream<Item = K> + Send + 'a,
K: Serialize + Debug + 'a,
{
use crate::pool::Get;
keys.ready_chunks(automatic_amplification())
.widen_then(automatic_width(), |chunk| {
let keys = chunk
.iter()
.map(ser::serialize_to::<KeyBuf, _>)
.map(|result| result.expect("failed to serialize query key"))
.collect();
keys.ready_chunks(automatic_amplification())
.widen_then(automatic_width(), |chunk| {
let keys = chunk
.iter()
.map(ser::serialize_to::<KeyBuf, _>)
.map(|result| result.expect("failed to serialize query key"))
.collect();
self.db
.pool
.execute_get(Get { map: self.clone(), key: keys, res: None })
})
.map_ok(|results| results.into_iter().stream())
.try_flatten()
self.db
.pool
.execute_get(Get { map: self.clone(), key: keys, res: None })
})
.map_ok(|results| results.into_iter().stream())
.try_flatten()
}
}
+41 -43
View File
@@ -1,55 +1,53 @@
use std::{convert::AsRef, fmt::Debug, io::Write};
use conduwuit::{arrayvec::ArrayVec, implement};
use conduwuit::arrayvec::ArrayVec;
use serde::Serialize;
use crate::{keyval::KeyBuf, ser, util::or_else};
#[implement(super::Map)]
#[inline]
pub fn del<K>(&self, key: K)
where
K: Serialize + Debug,
{
let mut buf = KeyBuf::new();
self.bdel(key, &mut buf);
}
impl super::Map {
#[inline]
pub fn del<K>(&self, key: K)
where
K: Serialize + Debug,
{
let mut buf = KeyBuf::new();
self.bdel(key, &mut buf);
}
#[implement(super::Map)]
#[inline]
pub fn adel<const MAX: usize, K>(&self, key: K)
where
K: Serialize + Debug,
{
let mut buf = ArrayVec::<u8, MAX>::new();
self.bdel(key, &mut buf);
}
#[inline]
pub fn adel<const MAX: usize, K>(&self, key: K)
where
K: Serialize + Debug,
{
let mut buf = ArrayVec::<u8, MAX>::new();
self.bdel(key, &mut buf);
}
#[implement(super::Map)]
#[tracing::instrument(skip(self, buf), level = "trace")]
pub fn bdel<K, B>(&self, key: K, buf: &mut B)
where
K: Serialize + Debug,
B: Write + AsRef<[u8]>,
{
let key = ser::serialize(buf, key).expect("failed to serialize deletion key");
self.remove(key);
}
#[tracing::instrument(skip(self, buf), level = "trace")]
pub fn bdel<K, B>(&self, key: K, buf: &mut B)
where
K: Serialize + Debug,
B: Write + AsRef<[u8]>,
{
let key = ser::serialize(buf, key).expect("failed to serialize deletion key");
self.remove(key);
}
#[implement(super::Map)]
#[tracing::instrument(skip(self, key), fields(%self), level = "trace")]
pub fn remove<K>(&self, key: &K)
where
K: AsRef<[u8]> + ?Sized + Debug,
{
let write_options = &self.write_options;
self.db
.db
.delete_cf_opt(&self.cf(), key, write_options)
.or_else(or_else)
.expect("database remove error");
#[tracing::instrument(skip(self, key), fields(%self), level = "trace")]
pub fn remove<K>(&self, key: &K)
where
K: AsRef<[u8]> + ?Sized + Debug,
{
let write_options = &self.write_options;
self.db
.db
.delete_cf_opt(&self.cf(), key, write_options)
.or_else(or_else)
.expect("database remove error");
if !self.db.corked() {
self.db.flush().expect("database flush error");
if !self.db.corked() {
self.db.flush().expect("database flush error");
}
}
}
+37 -37
View File
@@ -1,6 +1,6 @@
use std::sync::Arc;
use conduwuit::{Result, implement};
use conduwuit::Result;
use futures::{FutureExt, Stream, StreamExt, TryFutureExt, TryStreamExt};
use rocksdb::Direction;
use serde::Deserialize;
@@ -9,43 +9,43 @@ use tokio::task;
use super::rev_stream::is_cached;
use crate::{keyval, keyval::Key, stream};
#[implement(super::Map)]
pub fn rev_keys<'a, K>(self: &'a Arc<Self>) -> impl Stream<Item = Result<Key<'a, K>>> + Send
where
K: Deserialize<'a> + Send,
{
self.rev_raw_keys().map(keyval::result_deserialize_key::<K>)
}
#[implement(super::Map)]
#[tracing::instrument(skip(self), fields(%self), level = "trace")]
pub fn rev_raw_keys(self: &Arc<Self>) -> impl Stream<Item = Result<Key<'_>>> + Send {
use crate::pool::Seek;
let opts = super::iter_options_default(&self.db);
let state = stream::State::new(self, opts);
if is_cached(self) {
let state = state.init_rev(None);
return task::consume_budget()
.map(move |()| stream::KeysRev::<'_>::from(state))
.into_stream()
.flatten()
.boxed();
impl super::Map {
pub fn rev_keys<'a, K>(self: &'a Arc<Self>) -> impl Stream<Item = Result<Key<'a, K>>> + Send
where
K: Deserialize<'a> + Send,
{
self.rev_raw_keys().map(keyval::result_deserialize_key::<K>)
}
let seek = Seek {
map: self.clone(),
dir: Direction::Reverse,
state: crate::pool::into_send_seek(state),
key: None,
res: None,
};
#[tracing::instrument(skip(self), fields(%self), level = "trace")]
pub fn rev_raw_keys(self: &Arc<Self>) -> impl Stream<Item = Result<Key<'_>>> + Send {
use crate::pool::Seek;
self.db
.pool
.execute_iter(seek)
.ok_into::<stream::KeysRev<'_>>()
.into_stream()
.try_flatten()
.boxed()
let opts = super::iter_options_default(&self.db);
let state = stream::State::new(self, opts);
if is_cached(self) {
let state = state.init_rev(None);
return task::consume_budget()
.map(move |()| stream::KeysRev::<'_>::from(state))
.into_stream()
.flatten()
.boxed();
}
let seek = Seek {
map: self.clone(),
dir: Direction::Reverse,
state: crate::pool::into_send_seek(state),
key: None,
res: None,
};
self.db
.pool
.execute_iter(seek)
.ok_into::<stream::KeysRev<'_>>()
.into_stream()
.try_flatten()
.boxed()
}
}
+67 -69
View File
@@ -1,6 +1,6 @@
use std::{convert::AsRef, fmt::Debug, sync::Arc};
use conduwuit::{Result, implement};
use conduwuit::Result;
use futures::{FutureExt, Stream, StreamExt, TryFutureExt, TryStreamExt};
use rocksdb::Direction;
use serde::{Deserialize, Serialize};
@@ -11,75 +11,73 @@ use crate::{
stream,
};
#[implement(super::Map)]
pub fn rev_keys_from<'a, K, P>(
self: &'a Arc<Self>,
from: &P,
) -> impl Stream<Item = Result<Key<'a, K>>> + Send + use<'a, K, P>
where
P: Serialize + ?Sized + Debug,
K: Deserialize<'a> + Send,
{
self.rev_keys_from_raw(from)
.map(result_deserialize_key::<K>)
}
#[implement(super::Map)]
#[tracing::instrument(skip(self), level = "trace")]
pub fn rev_keys_from_raw<P>(
self: &Arc<Self>,
from: &P,
) -> impl Stream<Item = Result<Key<'_>>> + Send + use<'_, P>
where
P: Serialize + ?Sized + Debug,
{
let key = serialize_key(from).expect("failed to serialize query key");
self.rev_raw_keys_from(&key)
}
#[implement(super::Map)]
pub fn rev_keys_raw_from<'a, K, P>(
self: &'a Arc<Self>,
from: &P,
) -> impl Stream<Item = Result<Key<'a, K>>> + Send + use<'a, K, P>
where
P: AsRef<[u8]> + ?Sized + Debug + Sync,
K: Deserialize<'a> + Send,
{
self.rev_raw_keys_from(from)
.map(result_deserialize_key::<K>)
}
#[implement(super::Map)]
#[tracing::instrument(skip(self, from), fields(%self), level = "trace")]
pub fn rev_raw_keys_from<P>(
self: &Arc<Self>,
from: &P,
) -> impl Stream<Item = Result<Key<'_>>> + Send + use<'_, P>
where
P: AsRef<[u8]> + ?Sized + Debug,
{
use crate::pool::Seek;
let opts = super::iter_options_default(&self.db);
let state = stream::State::new(self, opts);
if is_cached(self, from) {
return stream::KeysRev::<'_>::from(state.init_rev(from.as_ref().into())).boxed();
impl super::Map {
pub fn rev_keys_from<'a, K, P>(
self: &'a Arc<Self>,
from: &P,
) -> impl Stream<Item = Result<Key<'a, K>>> + Send + use<'a, K, P>
where
P: Serialize + ?Sized + Debug,
K: Deserialize<'a> + Send,
{
self.rev_keys_from_raw(from)
.map(result_deserialize_key::<K>)
}
let seek = Seek {
map: self.clone(),
dir: Direction::Reverse,
key: Some(from.as_ref().into()),
state: crate::pool::into_send_seek(state),
res: None,
};
#[tracing::instrument(skip(self), level = "trace")]
pub fn rev_keys_from_raw<P>(
self: &Arc<Self>,
from: &P,
) -> impl Stream<Item = Result<Key<'_>>> + Send + use<'_, P>
where
P: Serialize + ?Sized + Debug,
{
let key = serialize_key(from).expect("failed to serialize query key");
self.rev_raw_keys_from(&key)
}
self.db
.pool
.execute_iter(seek)
.ok_into::<stream::KeysRev<'_>>()
.into_stream()
.try_flatten()
.boxed()
pub fn rev_keys_raw_from<'a, K, P>(
self: &'a Arc<Self>,
from: &P,
) -> impl Stream<Item = Result<Key<'a, K>>> + Send + use<'a, K, P>
where
P: AsRef<[u8]> + ?Sized + Debug + Sync,
K: Deserialize<'a> + Send,
{
self.rev_raw_keys_from(from)
.map(result_deserialize_key::<K>)
}
#[tracing::instrument(skip(self, from), fields(%self), level = "trace")]
pub fn rev_raw_keys_from<P>(
self: &Arc<Self>,
from: &P,
) -> impl Stream<Item = Result<Key<'_>>> + Send + use<'_, P>
where
P: AsRef<[u8]> + ?Sized + Debug,
{
use crate::pool::Seek;
let opts = super::iter_options_default(&self.db);
let state = stream::State::new(self, opts);
if is_cached(self, from) {
return stream::KeysRev::<'_>::from(state.init_rev(from.as_ref().into())).boxed();
}
let seek = Seek {
map: self.clone(),
dir: Direction::Reverse,
key: Some(from.as_ref().into()),
state: crate::pool::into_send_seek(state),
res: None,
};
self.db
.pool
.execute_iter(seek)
.ok_into::<stream::KeysRev<'_>>()
.into_stream()
.try_flatten()
.boxed()
}
}
+46 -48
View File
@@ -1,59 +1,57 @@
use std::{convert::AsRef, fmt::Debug, sync::Arc};
use conduwuit::{Result, implement};
use conduwuit::Result;
use futures::{Stream, StreamExt, TryStreamExt, future};
use serde::{Deserialize, Serialize};
use crate::keyval::{Key, result_deserialize_key, serialize_key};
#[implement(super::Map)]
pub fn rev_keys_prefix<'a, K, P>(
self: &'a Arc<Self>,
prefix: &P,
) -> impl Stream<Item = Result<Key<'a, K>>> + Send + use<'a, K, P>
where
P: Serialize + ?Sized + Debug,
K: Deserialize<'a> + Send,
{
self.rev_keys_prefix_raw(prefix)
.map(result_deserialize_key::<K>)
}
impl super::Map {
pub fn rev_keys_prefix<'a, K, P>(
self: &'a Arc<Self>,
prefix: &P,
) -> impl Stream<Item = Result<Key<'a, K>>> + Send + use<'a, K, P>
where
P: Serialize + ?Sized + Debug,
K: Deserialize<'a> + Send,
{
self.rev_keys_prefix_raw(prefix)
.map(result_deserialize_key::<K>)
}
#[implement(super::Map)]
#[tracing::instrument(skip(self), level = "trace")]
pub fn rev_keys_prefix_raw<P>(
self: &Arc<Self>,
prefix: &P,
) -> impl Stream<Item = Result<Key<'_>>> + Send + use<'_, P>
where
P: Serialize + ?Sized + Debug,
{
let key = serialize_key(prefix).expect("failed to serialize query key");
self.rev_raw_keys_from(&key)
.try_take_while(move |k: &Key<'_>| future::ok(k.starts_with(&key)))
}
#[tracing::instrument(skip(self), level = "trace")]
pub fn rev_keys_prefix_raw<P>(
self: &Arc<Self>,
prefix: &P,
) -> impl Stream<Item = Result<Key<'_>>> + Send + use<'_, P>
where
P: Serialize + ?Sized + Debug,
{
let key = serialize_key(prefix).expect("failed to serialize query key");
self.rev_raw_keys_from(&key)
.try_take_while(move |k: &Key<'_>| future::ok(k.starts_with(&key)))
}
#[implement(super::Map)]
pub fn rev_keys_raw_prefix<'a, K, P>(
self: &'a Arc<Self>,
prefix: &'a P,
) -> impl Stream<Item = Result<Key<'a, K>>> + Send + 'a
where
P: AsRef<[u8]> + ?Sized + Debug + Sync + 'a,
K: Deserialize<'a> + Send + 'a,
{
self.rev_raw_keys_prefix(prefix)
.map(result_deserialize_key::<K>)
}
pub fn rev_keys_raw_prefix<'a, K, P>(
self: &'a Arc<Self>,
prefix: &'a P,
) -> impl Stream<Item = Result<Key<'a, K>>> + Send + 'a
where
P: AsRef<[u8]> + ?Sized + Debug + Sync + 'a,
K: Deserialize<'a> + Send + 'a,
{
self.rev_raw_keys_prefix(prefix)
.map(result_deserialize_key::<K>)
}
#[implement(super::Map)]
pub fn rev_raw_keys_prefix<'a, P>(
self: &'a Arc<Self>,
prefix: &'a P,
) -> impl Stream<Item = Result<Key<'a>>> + Send + 'a
where
P: AsRef<[u8]> + ?Sized + Debug + Sync + 'a,
{
self.rev_raw_keys_from(prefix)
.try_take_while(|k: &Key<'_>| future::ok(k.starts_with(prefix.as_ref())))
pub fn rev_raw_keys_prefix<'a, P>(
self: &'a Arc<Self>,
prefix: &'a P,
) -> impl Stream<Item = Result<Key<'a>>> + Send + 'a
where
P: AsRef<[u8]> + ?Sized + Debug + Sync + 'a,
{
self.rev_raw_keys_from(prefix)
.try_take_while(|k: &Key<'_>| future::ok(k.starts_with(prefix.as_ref())))
}
}
+47 -47
View File
@@ -1,6 +1,6 @@
use std::sync::Arc;
use conduwuit::{Result, implement};
use conduwuit::Result;
use futures::{FutureExt, Stream, StreamExt, TryFutureExt, TryStreamExt};
use rocksdb::Direction;
use serde::Deserialize;
@@ -8,55 +8,55 @@ use tokio::task;
use crate::{keyval, keyval::KeyVal, stream};
/// Iterate key-value entries in the map from the end.
///
/// - Result is deserialized
#[implement(super::Map)]
pub fn rev_stream<'a, K, V>(
self: &'a Arc<Self>,
) -> impl Stream<Item = Result<KeyVal<'a, K, V>>> + Send
where
K: Deserialize<'a> + Send,
V: Deserialize<'a> + Send,
{
self.rev_raw_stream()
.map(keyval::result_deserialize::<K, V>)
}
/// Iterate key-value entries in the map from the end.
///
/// - Result is raw
#[implement(super::Map)]
#[tracing::instrument(skip(self), fields(%self), level = "trace")]
pub fn rev_raw_stream(self: &Arc<Self>) -> impl Stream<Item = Result<KeyVal<'_>>> + Send {
use crate::pool::Seek;
let opts = super::iter_options_default(&self.db);
let state = stream::State::new(self, opts);
if is_cached(self) {
let state = state.init_rev(None);
return task::consume_budget()
.map(move |()| stream::ItemsRev::<'_>::from(state))
.into_stream()
.flatten()
.boxed();
impl super::Map {
/// Iterate key-value entries in the map from the end.
///
/// - Result is deserialized
pub fn rev_stream<'a, K, V>(
self: &'a Arc<Self>,
) -> impl Stream<Item = Result<KeyVal<'a, K, V>>> + Send
where
K: Deserialize<'a> + Send,
V: Deserialize<'a> + Send,
{
self.rev_raw_stream()
.map(keyval::result_deserialize::<K, V>)
}
let seek = Seek {
map: self.clone(),
dir: Direction::Reverse,
state: crate::pool::into_send_seek(state),
key: None,
res: None,
};
/// Iterate key-value entries in the map from the end.
///
/// - Result is raw
#[tracing::instrument(skip(self), fields(%self), level = "trace")]
pub fn rev_raw_stream(self: &Arc<Self>) -> impl Stream<Item = Result<KeyVal<'_>>> + Send {
use crate::pool::Seek;
self.db
.pool
.execute_iter(seek)
.ok_into::<stream::ItemsRev<'_>>()
.into_stream()
.try_flatten()
.boxed()
let opts = super::iter_options_default(&self.db);
let state = stream::State::new(self, opts);
if is_cached(self) {
let state = state.init_rev(None);
return task::consume_budget()
.map(move |()| stream::ItemsRev::<'_>::from(state))
.into_stream()
.flatten()
.boxed();
}
let seek = Seek {
map: self.clone(),
dir: Direction::Reverse,
state: crate::pool::into_send_seek(state),
key: None,
res: None,
};
self.db
.pool
.execute_iter(seek)
.ok_into::<stream::ItemsRev<'_>>()
.into_stream()
.try_flatten()
.boxed()
}
}
#[tracing::instrument(
+90 -92
View File
@@ -1,6 +1,6 @@
use std::{convert::AsRef, fmt::Debug, sync::Arc};
use conduwuit::{Result, implement};
use conduwuit::Result;
use futures::{FutureExt, Stream, StreamExt, TryFutureExt, TryStreamExt};
use rocksdb::Direction;
use serde::{Deserialize, Serialize};
@@ -12,100 +12,98 @@ use crate::{
util::is_incomplete,
};
/// Iterate key-value entries in the map starting from upper-bound.
///
/// - Query is serialized
/// - Result is deserialized
#[implement(super::Map)]
pub fn rev_stream_from<'a, K, V, P>(
self: &'a Arc<Self>,
from: &P,
) -> impl Stream<Item = Result<KeyVal<'a, K, V>>> + Send + use<'a, K, V, P>
where
P: Serialize + ?Sized + Debug,
K: Deserialize<'a> + Send,
V: Deserialize<'a> + Send,
{
self.rev_stream_from_raw(from)
.map(result_deserialize::<K, V>)
}
/// Iterate key-value entries in the map starting from upper-bound.
///
/// - Query is serialized
/// - Result is raw
#[implement(super::Map)]
#[tracing::instrument(skip(self), level = "trace")]
pub fn rev_stream_from_raw<P>(
self: &Arc<Self>,
from: &P,
) -> impl Stream<Item = Result<KeyVal<'_>>> + Send + use<'_, P>
where
P: Serialize + ?Sized + Debug,
{
let key = serialize_key(from).expect("failed to serialize query key");
self.rev_raw_stream_from(&key)
}
/// Iterate key-value entries in the map starting from upper-bound.
///
/// - Query is raw
/// - Result is deserialized
#[implement(super::Map)]
pub fn rev_stream_raw_from<'a, K, V, P>(
self: &'a Arc<Self>,
from: &P,
) -> impl Stream<Item = Result<KeyVal<'a, K, V>>> + Send + use<'a, K, V, P>
where
P: AsRef<[u8]> + ?Sized + Debug + Sync,
K: Deserialize<'a> + Send,
V: Deserialize<'a> + Send,
{
self.rev_raw_stream_from(from)
.map(result_deserialize::<K, V>)
}
/// Iterate key-value entries in the map starting from upper-bound.
///
/// - Query is raw
/// - Result is raw
#[implement(super::Map)]
#[tracing::instrument(skip(self, from), fields(%self), level = "trace")]
pub fn rev_raw_stream_from<P>(
self: &Arc<Self>,
from: &P,
) -> impl Stream<Item = Result<KeyVal<'_>>> + Send + use<'_, P>
where
P: AsRef<[u8]> + ?Sized + Debug,
{
use crate::pool::Seek;
let opts = super::iter_options_default(&self.db);
let state = stream::State::new(self, opts);
if is_cached(self, from) {
let state = state.init_rev(from.as_ref().into());
return task::consume_budget()
.map(move |()| stream::ItemsRev::<'_>::from(state))
.into_stream()
.flatten()
.boxed();
impl super::Map {
/// Iterate key-value entries in the map starting from upper-bound.
///
/// - Query is serialized
/// - Result is deserialized
pub fn rev_stream_from<'a, K, V, P>(
self: &'a Arc<Self>,
from: &P,
) -> impl Stream<Item = Result<KeyVal<'a, K, V>>> + Send + use<'a, K, V, P>
where
P: Serialize + ?Sized + Debug,
K: Deserialize<'a> + Send,
V: Deserialize<'a> + Send,
{
self.rev_stream_from_raw(from)
.map(result_deserialize::<K, V>)
}
let seek = Seek {
map: self.clone(),
dir: Direction::Reverse,
key: Some(from.as_ref().into()),
state: crate::pool::into_send_seek(state),
res: None,
};
/// Iterate key-value entries in the map starting from upper-bound.
///
/// - Query is serialized
/// - Result is raw
#[tracing::instrument(skip(self), level = "trace")]
pub fn rev_stream_from_raw<P>(
self: &Arc<Self>,
from: &P,
) -> impl Stream<Item = Result<KeyVal<'_>>> + Send + use<'_, P>
where
P: Serialize + ?Sized + Debug,
{
let key = serialize_key(from).expect("failed to serialize query key");
self.rev_raw_stream_from(&key)
}
self.db
.pool
.execute_iter(seek)
.ok_into::<stream::ItemsRev<'_>>()
.into_stream()
.try_flatten()
.boxed()
/// Iterate key-value entries in the map starting from upper-bound.
///
/// - Query is raw
/// - Result is deserialized
pub fn rev_stream_raw_from<'a, K, V, P>(
self: &'a Arc<Self>,
from: &P,
) -> impl Stream<Item = Result<KeyVal<'a, K, V>>> + Send + use<'a, K, V, P>
where
P: AsRef<[u8]> + ?Sized + Debug + Sync,
K: Deserialize<'a> + Send,
V: Deserialize<'a> + Send,
{
self.rev_raw_stream_from(from)
.map(result_deserialize::<K, V>)
}
/// Iterate key-value entries in the map starting from upper-bound.
///
/// - Query is raw
/// - Result is raw
#[tracing::instrument(skip(self, from), fields(%self), level = "trace")]
pub fn rev_raw_stream_from<P>(
self: &Arc<Self>,
from: &P,
) -> impl Stream<Item = Result<KeyVal<'_>>> + Send + use<'_, P>
where
P: AsRef<[u8]> + ?Sized + Debug,
{
use crate::pool::Seek;
let opts = super::iter_options_default(&self.db);
let state = stream::State::new(self, opts);
if is_cached(self, from) {
let state = state.init_rev(from.as_ref().into());
return task::consume_budget()
.map(move |()| stream::ItemsRev::<'_>::from(state))
.into_stream()
.flatten()
.boxed();
}
let seek = Seek {
map: self.clone(),
dir: Direction::Reverse,
key: Some(from.as_ref().into()),
state: crate::pool::into_send_seek(state),
res: None,
};
self.db
.pool
.execute_iter(seek)
.ok_into::<stream::ItemsRev<'_>>()
.into_stream()
.try_flatten()
.boxed()
}
}
#[tracing::instrument(
+64 -66
View File
@@ -1,77 +1,75 @@
use std::{convert::AsRef, fmt::Debug, sync::Arc};
use conduwuit::{Result, implement};
use conduwuit::Result;
use futures::{Stream, StreamExt, TryStreamExt, future};
use serde::{Deserialize, Serialize};
use crate::keyval::{KeyVal, result_deserialize, serialize_key};
/// Iterate key-value entries in the map where the key matches a prefix.
///
/// - Query is serialized
/// - Result is deserialized
#[implement(super::Map)]
pub fn rev_stream_prefix<'a, K, V, P>(
self: &'a Arc<Self>,
prefix: &P,
) -> impl Stream<Item = Result<KeyVal<'a, K, V>>> + Send + use<'a, K, V, P>
where
P: Serialize + ?Sized + Debug,
K: Deserialize<'a> + Send,
V: Deserialize<'a> + Send,
{
self.rev_stream_prefix_raw(prefix)
.map(result_deserialize::<K, V>)
}
impl super::Map {
/// Iterate key-value entries in the map where the key matches a prefix.
///
/// - Query is serialized
/// - Result is deserialized
pub fn rev_stream_prefix<'a, K, V, P>(
self: &'a Arc<Self>,
prefix: &P,
) -> impl Stream<Item = Result<KeyVal<'a, K, V>>> + Send + use<'a, K, V, P>
where
P: Serialize + ?Sized + Debug,
K: Deserialize<'a> + Send,
V: Deserialize<'a> + Send,
{
self.rev_stream_prefix_raw(prefix)
.map(result_deserialize::<K, V>)
}
/// Iterate key-value entries in the map where the key matches a prefix.
///
/// - Query is serialized
/// - Result is raw
#[implement(super::Map)]
#[tracing::instrument(skip(self), level = "trace")]
pub fn rev_stream_prefix_raw<P>(
self: &Arc<Self>,
prefix: &P,
) -> impl Stream<Item = Result<KeyVal<'_>>> + Send + use<'_, P>
where
P: Serialize + ?Sized + Debug,
{
let key = serialize_key(prefix).expect("failed to serialize query key");
self.rev_raw_stream_from(&key)
.try_take_while(move |(k, _): &KeyVal<'_>| future::ok(k.starts_with(&key)))
}
/// Iterate key-value entries in the map where the key matches a prefix.
///
/// - Query is serialized
/// - Result is raw
#[tracing::instrument(skip(self), level = "trace")]
pub fn rev_stream_prefix_raw<P>(
self: &Arc<Self>,
prefix: &P,
) -> impl Stream<Item = Result<KeyVal<'_>>> + Send + use<'_, P>
where
P: Serialize + ?Sized + Debug,
{
let key = serialize_key(prefix).expect("failed to serialize query key");
self.rev_raw_stream_from(&key)
.try_take_while(move |(k, _): &KeyVal<'_>| future::ok(k.starts_with(&key)))
}
/// Iterate key-value entries in the map where the key matches a prefix.
///
/// - Query is raw
/// - Result is deserialized
#[implement(super::Map)]
pub fn rev_stream_raw_prefix<'a, K, V, P>(
self: &'a Arc<Self>,
prefix: &'a P,
) -> impl Stream<Item = Result<KeyVal<'a, K, V>>> + Send + 'a
where
P: AsRef<[u8]> + ?Sized + Debug + Sync + 'a,
K: Deserialize<'a> + Send + 'a,
V: Deserialize<'a> + Send + 'a,
{
self.rev_raw_stream_prefix(prefix)
.map(result_deserialize::<K, V>)
}
/// Iterate key-value entries in the map where the key matches a prefix.
///
/// - Query is raw
/// - Result is deserialized
pub fn rev_stream_raw_prefix<'a, K, V, P>(
self: &'a Arc<Self>,
prefix: &'a P,
) -> impl Stream<Item = Result<KeyVal<'a, K, V>>> + Send + 'a
where
P: AsRef<[u8]> + ?Sized + Debug + Sync + 'a,
K: Deserialize<'a> + Send + 'a,
V: Deserialize<'a> + Send + 'a,
{
self.rev_raw_stream_prefix(prefix)
.map(result_deserialize::<K, V>)
}
/// Iterate key-value entries in the map where the key matches a prefix.
///
/// - Query is raw
/// - Result is raw
#[implement(super::Map)]
pub fn rev_raw_stream_prefix<'a, P>(
self: &'a Arc<Self>,
prefix: &'a P,
) -> impl Stream<Item = Result<KeyVal<'a>>> + Send + 'a
where
P: AsRef<[u8]> + ?Sized + Debug + Sync + 'a,
{
self.rev_raw_stream_from(prefix)
.try_take_while(|(k, _): &KeyVal<'_>| future::ok(k.starts_with(prefix.as_ref())))
/// Iterate key-value entries in the map where the key matches a prefix.
///
/// - Query is raw
/// - Result is raw
pub fn rev_raw_stream_prefix<'a, P>(
self: &'a Arc<Self>,
prefix: &'a P,
) -> impl Stream<Item = Result<KeyVal<'a>>> + Send + 'a
where
P: AsRef<[u8]> + ?Sized + Debug + Sync + 'a,
{
self.rev_raw_stream_from(prefix)
.try_take_while(|(k, _): &KeyVal<'_>| future::ok(k.starts_with(prefix.as_ref())))
}
}
+46 -46
View File
@@ -1,6 +1,6 @@
use std::sync::Arc;
use conduwuit::{Result, implement};
use conduwuit::Result;
use futures::{FutureExt, Stream, StreamExt, TryFutureExt, TryStreamExt};
use rocksdb::Direction;
use serde::Deserialize;
@@ -8,54 +8,54 @@ use tokio::task;
use crate::{keyval, keyval::KeyVal, stream};
/// Iterate key-value entries in the map from the beginning.
///
/// - Result is deserialized
#[implement(super::Map)]
pub fn stream<'a, K, V>(
self: &'a Arc<Self>,
) -> impl Stream<Item = Result<KeyVal<'a, K, V>>> + Send
where
K: Deserialize<'a> + Send,
V: Deserialize<'a> + Send,
{
self.raw_stream().map(keyval::result_deserialize::<K, V>)
}
/// Iterate key-value entries in the map from the beginning.
///
/// - Result is raw
#[implement(super::Map)]
#[tracing::instrument(skip(self), fields(%self), level = "trace")]
pub fn raw_stream(self: &Arc<Self>) -> impl Stream<Item = Result<KeyVal<'_>>> + Send {
use crate::pool::Seek;
let opts = super::iter_options_default(&self.db);
let state = stream::State::new(self, opts);
if is_cached(self) {
let state = state.init_fwd(None);
return task::consume_budget()
.map(move |()| stream::Items::<'_>::from(state))
.into_stream()
.flatten()
.boxed();
impl super::Map {
/// Iterate key-value entries in the map from the beginning.
///
/// - Result is deserialized
pub fn stream<'a, K, V>(
self: &'a Arc<Self>,
) -> impl Stream<Item = Result<KeyVal<'a, K, V>>> + Send
where
K: Deserialize<'a> + Send,
V: Deserialize<'a> + Send,
{
self.raw_stream().map(keyval::result_deserialize::<K, V>)
}
let seek = Seek {
map: self.clone(),
dir: Direction::Forward,
state: crate::pool::into_send_seek(state),
key: None,
res: None,
};
/// Iterate key-value entries in the map from the beginning.
///
/// - Result is raw
#[tracing::instrument(skip(self), fields(%self), level = "trace")]
pub fn raw_stream(self: &Arc<Self>) -> impl Stream<Item = Result<KeyVal<'_>>> + Send {
use crate::pool::Seek;
self.db
.pool
.execute_iter(seek)
.ok_into::<stream::Items<'_>>()
.into_stream()
.try_flatten()
.boxed()
let opts = super::iter_options_default(&self.db);
let state = stream::State::new(self, opts);
if is_cached(self) {
let state = state.init_fwd(None);
return task::consume_budget()
.map(move |()| stream::Items::<'_>::from(state))
.into_stream()
.flatten()
.boxed();
}
let seek = Seek {
map: self.clone(),
dir: Direction::Forward,
state: crate::pool::into_send_seek(state),
key: None,
res: None,
};
self.db
.pool
.execute_iter(seek)
.ok_into::<stream::Items<'_>>()
.into_stream()
.try_flatten()
.boxed()
}
}
#[tracing::instrument(
+92 -94
View File
@@ -1,6 +1,6 @@
use std::{convert::AsRef, fmt::Debug, sync::Arc};
use conduwuit::{Result, implement};
use conduwuit::Result;
use futures::{FutureExt, Stream, StreamExt, TryFutureExt, TryStreamExt};
use rocksdb::Direction;
use serde::{Deserialize, Serialize};
@@ -11,105 +11,103 @@ use crate::{
stream,
};
/// Iterate key-value entries in the map starting from lower-bound.
///
/// - Query is serialized
/// - Result is deserialized
#[implement(super::Map)]
pub fn stream_from<'a, K, V, P>(
self: &'a Arc<Self>,
from: &P,
) -> impl Stream<Item = Result<KeyVal<'a, K, V>>> + Send + use<'a, K, V, P>
where
P: Serialize + ?Sized + Debug,
K: Deserialize<'a> + Send,
V: Deserialize<'a> + Send,
{
self.stream_from_raw(from).map(result_deserialize::<K, V>)
}
/// Iterate key-value entries in the map starting from lower-bound.
///
/// - Query is serialized
/// - Result is raw
#[implement(super::Map)]
#[tracing::instrument(skip(self), level = "trace")]
pub fn stream_from_raw<P>(
self: &Arc<Self>,
from: &P,
) -> impl Stream<Item = Result<KeyVal<'_>>> + Send + use<'_, P>
where
P: Serialize + ?Sized + Debug,
{
let key = serialize_key(from).expect("failed to serialize query key");
self.raw_stream_from(&key)
}
/// Iterate key-value entries in the map starting from lower-bound.
///
/// - Query is raw
/// - Result is deserialized
#[implement(super::Map)]
pub fn stream_raw_from<'a, K, V, P>(
self: &'a Arc<Self>,
from: &P,
) -> impl Stream<Item = Result<KeyVal<'a, K, V>>> + Send + use<'a, K, V, P>
where
P: AsRef<[u8]> + ?Sized + Debug + Sync,
K: Deserialize<'a> + Send,
V: Deserialize<'a> + Send,
{
self.raw_stream_from(from).map(result_deserialize::<K, V>)
}
/// Iterate key-value entries in the map starting from lower-bound.
///
/// - Query is raw
/// - Result is raw
#[implement(super::Map)]
#[tracing::instrument(skip(self, from), fields(%self), level = "trace")]
pub fn raw_stream_from<P>(
self: &Arc<Self>,
from: &P,
) -> impl Stream<Item = Result<KeyVal<'_>>> + Send + use<'_, P>
where
P: AsRef<[u8]> + ?Sized + Debug,
{
use crate::pool::Seek;
let opts = super::iter_options_default(&self.db);
let state = stream::State::new(self, opts);
if is_cached(self, from) {
let state = state.init_fwd(from.as_ref().into());
return task::consume_budget()
.map(move |()| stream::Items::<'_>::from(state))
.into_stream()
.flatten()
.boxed();
impl super::Map {
/// Iterate key-value entries in the map starting from lower-bound.
///
/// - Query is serialized
/// - Result is deserialized
pub fn stream_from<'a, K, V, P>(
self: &'a Arc<Self>,
from: &P,
) -> impl Stream<Item = Result<KeyVal<'a, K, V>>> + Send + use<'a, K, V, P>
where
P: Serialize + ?Sized + Debug,
K: Deserialize<'a> + Send,
V: Deserialize<'a> + Send,
{
self.stream_from_raw(from).map(result_deserialize::<K, V>)
}
let seek = Seek {
map: self.clone(),
dir: Direction::Forward,
key: Some(from.as_ref().into()),
state: crate::pool::into_send_seek(state),
res: None,
};
/// Iterate key-value entries in the map starting from lower-bound.
///
/// - Query is serialized
/// - Result is raw
#[tracing::instrument(skip(self), level = "trace")]
pub fn stream_from_raw<P>(
self: &Arc<Self>,
from: &P,
) -> impl Stream<Item = Result<KeyVal<'_>>> + Send + use<'_, P>
where
P: Serialize + ?Sized + Debug,
{
let key = serialize_key(from).expect("failed to serialize query key");
self.raw_stream_from(&key)
}
self.db
.pool
.execute_iter(seek)
.ok_into::<stream::Items<'_>>()
.into_stream()
.try_flatten()
.boxed()
/// Iterate key-value entries in the map starting from lower-bound.
///
/// - Query is raw
/// - Result is deserialized
pub fn stream_raw_from<'a, K, V, P>(
self: &'a Arc<Self>,
from: &P,
) -> impl Stream<Item = Result<KeyVal<'a, K, V>>> + Send + use<'a, K, V, P>
where
P: AsRef<[u8]> + ?Sized + Debug + Sync,
K: Deserialize<'a> + Send,
V: Deserialize<'a> + Send,
{
self.raw_stream_from(from).map(result_deserialize::<K, V>)
}
/// Iterate key-value entries in the map starting from lower-bound.
///
/// - Query is raw
/// - Result is raw
#[tracing::instrument(skip(self, from), fields(%self), level = "trace")]
pub fn raw_stream_from<P>(
self: &Arc<Self>,
from: &P,
) -> impl Stream<Item = Result<KeyVal<'_>>> + Send + use<'_, P>
where
P: AsRef<[u8]> + ?Sized + Debug,
{
use crate::pool::Seek;
let opts = super::iter_options_default(&self.db);
let state = stream::State::new(self, opts);
if is_cached(self, from) {
let state = state.init_fwd(from.as_ref().into());
return task::consume_budget()
.map(move |()| stream::Items::<'_>::from(state))
.into_stream()
.flatten()
.boxed();
}
let seek = Seek {
map: self.clone(),
dir: Direction::Forward,
key: Some(from.as_ref().into()),
state: crate::pool::into_send_seek(state),
res: None,
};
self.db
.pool
.execute_iter(seek)
.ok_into::<stream::Items<'_>>()
.into_stream()
.try_flatten()
.boxed()
}
}
#[tracing::instrument(
name = "cached",
level = "trace",
skip(map, from),
fields(%map),
name = "cached",
level = "trace",
skip(map, from),
fields(%map),
)]
pub(super) fn is_cached<P>(map: &Arc<super::Map>, from: &P) -> bool
where
+64 -66
View File
@@ -1,77 +1,75 @@
use std::{convert::AsRef, fmt::Debug, sync::Arc};
use conduwuit::{Result, implement};
use conduwuit::Result;
use futures::{Stream, StreamExt, TryStreamExt, future};
use serde::{Deserialize, Serialize};
use crate::keyval::{KeyVal, result_deserialize, serialize_key};
/// Iterate key-value entries in the map where the key matches a prefix.
///
/// - Query is serialized
/// - Result is deserialized
#[implement(super::Map)]
pub fn stream_prefix<'a, K, V, P>(
self: &'a Arc<Self>,
prefix: &P,
) -> impl Stream<Item = Result<KeyVal<'a, K, V>>> + Send + use<'a, K, V, P>
where
P: Serialize + ?Sized + Debug,
K: Deserialize<'a> + Send,
V: Deserialize<'a> + Send,
{
self.stream_prefix_raw(prefix)
.map(result_deserialize::<K, V>)
}
impl super::Map {
/// Iterate key-value entries in the map where the key matches a prefix.
///
/// - Query is serialized
/// - Result is deserialized
pub fn stream_prefix<'a, K, V, P>(
self: &'a Arc<Self>,
prefix: &P,
) -> impl Stream<Item = Result<KeyVal<'a, K, V>>> + Send + use<'a, K, V, P>
where
P: Serialize + ?Sized + Debug,
K: Deserialize<'a> + Send,
V: Deserialize<'a> + Send,
{
self.stream_prefix_raw(prefix)
.map(result_deserialize::<K, V>)
}
/// Iterate key-value entries in the map where the key matches a prefix.
///
/// - Query is serialized
/// - Result is raw
#[implement(super::Map)]
#[tracing::instrument(skip(self), level = "trace")]
pub fn stream_prefix_raw<P>(
self: &Arc<Self>,
prefix: &P,
) -> impl Stream<Item = Result<KeyVal<'_>>> + Send + use<'_, P>
where
P: Serialize + ?Sized + Debug,
{
let key = serialize_key(prefix).expect("failed to serialize query key");
self.raw_stream_from(&key)
.try_take_while(move |(k, _): &KeyVal<'_>| future::ok(k.starts_with(&key)))
}
/// Iterate key-value entries in the map where the key matches a prefix.
///
/// - Query is serialized
/// - Result is raw
#[tracing::instrument(skip(self), level = "trace")]
pub fn stream_prefix_raw<P>(
self: &Arc<Self>,
prefix: &P,
) -> impl Stream<Item = Result<KeyVal<'_>>> + Send + use<'_, P>
where
P: Serialize + ?Sized + Debug,
{
let key = serialize_key(prefix).expect("failed to serialize query key");
self.raw_stream_from(&key)
.try_take_while(move |(k, _): &KeyVal<'_>| future::ok(k.starts_with(&key)))
}
/// Iterate key-value entries in the map where the key matches a prefix.
///
/// - Query is raw
/// - Result is deserialized
#[implement(super::Map)]
pub fn stream_raw_prefix<'a, K, V, P>(
self: &'a Arc<Self>,
prefix: &'a P,
) -> impl Stream<Item = Result<KeyVal<'a, K, V>>> + Send + 'a
where
P: AsRef<[u8]> + ?Sized + Debug + Sync + 'a,
K: Deserialize<'a> + Send + 'a,
V: Deserialize<'a> + Send + 'a,
{
self.raw_stream_prefix(prefix)
.map(result_deserialize::<K, V>)
}
/// Iterate key-value entries in the map where the key matches a prefix.
///
/// - Query is raw
/// - Result is deserialized
pub fn stream_raw_prefix<'a, K, V, P>(
self: &'a Arc<Self>,
prefix: &'a P,
) -> impl Stream<Item = Result<KeyVal<'a, K, V>>> + Send + 'a
where
P: AsRef<[u8]> + ?Sized + Debug + Sync + 'a,
K: Deserialize<'a> + Send + 'a,
V: Deserialize<'a> + Send + 'a,
{
self.raw_stream_prefix(prefix)
.map(result_deserialize::<K, V>)
}
/// Iterate key-value entries in the map where the key matches a prefix.
///
/// - Query is raw
/// - Result is raw
#[implement(super::Map)]
pub fn raw_stream_prefix<'a, P>(
self: &'a Arc<Self>,
prefix: &'a P,
) -> impl Stream<Item = Result<KeyVal<'a>>> + Send + 'a
where
P: AsRef<[u8]> + ?Sized + Debug + Sync + 'a,
{
self.raw_stream_from(prefix)
.try_take_while(|(k, _): &KeyVal<'_>| future::ok(k.starts_with(prefix.as_ref())))
/// Iterate key-value entries in the map where the key matches a prefix.
///
/// - Query is raw
/// - Result is raw
pub fn raw_stream_prefix<'a, P>(
self: &'a Arc<Self>,
prefix: &'a P,
) -> impl Stream<Item = Result<KeyVal<'a>>> + Send + 'a
where
P: AsRef<[u8]> + ?Sized + Debug + Sync + 'a,
{
self.raw_stream_from(prefix)
.try_take_while(|(k, _): &KeyVal<'_>| future::ok(k.starts_with(prefix.as_ref())))
}
}
+4
View File
@@ -187,6 +187,10 @@ pub(super) static MAPS: &[Descriptor] = &[
val_size_hint: Some(8),
..descriptor::RANDOM_SMALL
},
Descriptor {
name: "roomid_mindepth",
..descriptor::RANDOM_SMALL
},
Descriptor {
name: "roomserverids",
..descriptor::RANDOM_SMALL
+322 -332
View File
@@ -12,7 +12,7 @@ use std::{
use async_channel::{QueueStrategy, Receiver, RecvError, Sender};
use conduwuit::{
Error, Result, Server, SyncMutex, debug, err, error, implement,
Error, Result, Server, SyncMutex, debug, err, error,
result::DebugInspect,
smallvec::SmallVec,
trace,
@@ -72,29 +72,332 @@ const BATCH_INLINE: usize = 1;
const WORKER_STACK_SIZE: usize = 1_048_576;
const WORKER_NAME: &str = "conduwuit:db";
#[implement(Pool)]
pub(crate) fn new(server: &Arc<Server>) -> Result<Arc<Self>> {
const CHAN_SCHED: (QueueStrategy, QueueStrategy) = (QueueStrategy::Fifo, QueueStrategy::Lifo);
impl Pool {
pub(crate) fn new(server: &Arc<Server>) -> Result<Arc<Self>> {
const CHAN_SCHED: (QueueStrategy, QueueStrategy) =
(QueueStrategy::Fifo, QueueStrategy::Lifo);
let (total_workers, queue_sizes, topology) = configure(server);
let (total_workers, queue_sizes, topology) = configure(server);
let (senders, receivers): (Vec<_>, Vec<_>) = queue_sizes
.into_iter()
.map(|cap| async_channel::bounded_with_queue_strategy(cap, CHAN_SCHED))
.unzip();
let (senders, receivers): (Vec<_>, Vec<_>) = queue_sizes
.into_iter()
.map(|cap| async_channel::bounded_with_queue_strategy(cap, CHAN_SCHED))
.unzip();
let pool = Arc::new(Self {
server: server.clone(),
queues: senders,
workers: Vec::new().into(),
topology,
busy: AtomicUsize::default(),
queued_max: AtomicUsize::default(),
});
let pool = Arc::new(Self {
server: server.clone(),
queues: senders,
workers: Vec::new().into(),
topology,
busy: AtomicUsize::default(),
queued_max: AtomicUsize::default(),
});
pool.spawn_until(&receivers, total_workers)?;
pool.spawn_until(&receivers, total_workers)?;
Ok(pool)
Ok(pool)
}
#[tracing::instrument(skip_all, level = "debug")]
pub(crate) fn close(&self) {
let workers = take(&mut *self.workers.lock());
let senders = self.queues.iter().map(Sender::sender_count).sum::<usize>();
let receivers = self
.queues
.iter()
.map(Sender::receiver_count)
.sum::<usize>();
for queue in &self.queues {
queue.close();
}
if workers.is_empty() {
return;
}
debug!(
senders,
receivers,
queues = self.queues.len(),
workers = workers.len(),
"Closing pool. Waiting for workers to join..."
);
workers
.into_iter()
.map(JoinHandle::join)
.map(|result| result.map_err(Error::from_panic))
.enumerate()
.for_each(|(id, result)| match result {
| Ok(()) => trace!(%id, "worker joined"),
| Err(error) => error!(%id, "worker joined with error: {error}"),
});
}
fn spawn_until(self: &Arc<Self>, recv: &[Receiver<Cmd>], count: usize) -> Result {
let mut workers = self.workers.lock();
while workers.len() < count {
self.clone().spawn_one(&mut workers, recv)?;
}
Ok(())
}
#[tracing::instrument(
name = "spawn",
level = "trace",
skip_all,
fields(id = %workers.len())
)]
fn spawn_one(
self: Arc<Self>,
workers: &mut Vec<JoinHandle<()>>,
recv: &[Receiver<Cmd>],
) -> Result {
debug_assert!(!self.queues.is_empty(), "Must have at least one queue");
debug_assert!(!recv.is_empty(), "Must have at least one receiver");
let id = workers.len();
let group = id.overflowing_rem(self.queues.len()).0;
let recv = recv[group].clone();
let handle = thread::Builder::new()
.name(WORKER_NAME.into())
.stack_size(WORKER_STACK_SIZE)
.spawn(move || self.worker(id, &recv))?;
workers.push(handle);
Ok(())
}
#[tracing::instrument(level = "trace", name = "get", skip(self, cmd))]
pub(crate) async fn execute_get(self: &Arc<Self>, mut cmd: Get) -> Result<BatchResult<'_>> {
let (send, recv) = oneshot::channel();
_ = cmd.res.insert(send);
let queue = self.select_queue();
self.execute(queue, Cmd::Get(cmd))
.and_then(move |()| {
recv.map_ok(into_recv_get)
.map_err(|e| err!(error!("recv failed {e:?}")))
})
.await
}
#[tracing::instrument(level = "trace", name = "iter", skip(self, cmd))]
pub(crate) async fn execute_iter(
self: &Arc<Self>,
mut cmd: Seek,
) -> Result<stream::State<'_>> {
let (send, recv) = oneshot::channel();
_ = cmd.res.insert(send);
let queue = self.select_queue();
self.execute(queue, Cmd::Iter(cmd))
.and_then(|()| {
recv.map_ok(into_recv_seek)
.map_err(|e| err!(error!("recv failed {e:?}")))
})
.await
}
fn select_queue(&self) -> &Sender<Cmd> {
let core_id = get_affinity().next().unwrap_or(0);
let chan_id = self.topology[core_id];
self.queues.get(chan_id).unwrap_or_else(|| &self.queues[0])
}
#[tracing::instrument(
level = "trace",
name = "execute",
skip(self, cmd),
fields(
task = ?tokio::task::try_id(),
receivers = queue.receiver_count(),
queued = queue.len(),
queued_max = self.queued_max.load(Ordering::Relaxed),
),
)]
async fn execute(&self, queue: &Sender<Cmd>, cmd: Cmd) -> Result {
if cfg!(debug_assertions) {
self.queued_max.fetch_max(queue.len(), Ordering::Relaxed);
}
queue
.send(cmd)
.await
.map_err(|e| err!(error!("send failed {e:?}")))
}
#[tracing::instrument(
parent = None,
level = "debug",
skip(self, recv),
fields(
tid = ?thread::current().id(),
),
)]
fn worker(self: Arc<Self>, id: usize, recv: &Receiver<Cmd>) {
self.worker_init(id);
self.worker_loop(recv);
}
fn worker_init(&self, id: usize) {
let group = id.overflowing_rem(self.queues.len()).0;
let affinity = self
.topology
.iter()
.enumerate()
.filter(|_| self.queues.len() > 1)
.filter(|_| self.server.config.db_pool_affinity)
.filter_map(|(core_id, &queue_id)| (group == queue_id).then_some(core_id))
.filter_map(nth_core_available);
// affinity is empty (no-op) if there's only one queue
set_affinity(affinity.clone());
#[cfg(all(not(target_env = "msvc"), feature = "jemalloc"))]
if affinity.clone().count() == 1 && conduwuit::alloc::je::is_affine_arena() {
use conduwuit::{
alloc::je::this_thread::{arena_id, set_arena},
result::LogDebugErr,
};
let id = affinity.clone().next().expect("at least one id");
if let Ok(arena) = arena_id() {
if arena != id {
set_arena(id).log_debug_err().ok();
}
}
}
debug!(
%group,
affinity = ?affinity.collect::<Vec<_>>(),
"worker ready"
);
}
fn worker_loop(self: &Arc<Self>, recv: &Receiver<Cmd>) {
// initial +1 needed prior to entering wait
self.busy.fetch_add(1, Ordering::Relaxed);
while let Ok(cmd) = self.worker_wait(recv) {
Self::worker_handle(cmd);
}
}
#[tracing::instrument(
name = "wait",
level = "trace",
skip_all,
fields(
receivers = recv.receiver_count(),
queued = recv.len(),
busy = self.busy.fetch_sub(1, Ordering::Relaxed) - 1,
),
)]
fn worker_wait(self: &Arc<Self>, recv: &Receiver<Cmd>) -> Result<Cmd, RecvError> {
recv.recv_blocking().debug_inspect(|_| {
self.busy.fetch_add(1, Ordering::Relaxed);
})
}
fn worker_handle(cmd: Cmd) {
match cmd {
| Cmd::Get(cmd) if cmd.key.len() == 1 => Self::handle_get(cmd),
| Cmd::Get(cmd) => Self::handle_batch(cmd),
| Cmd::Iter(cmd) => Self::handle_iter(cmd),
}
}
#[tracing::instrument(
name = "iter",
level = "trace",
skip_all,
fields(%cmd.map),
)]
fn handle_iter(mut cmd: Seek) {
let chan = cmd.res.take().expect("missing result channel");
if chan.is_canceled() {
return;
}
let from = cmd.key.as_deref();
let result = match cmd.dir {
| Direction::Forward => cmd.state.init_fwd(from),
| Direction::Reverse => cmd.state.init_rev(from),
};
let chan_result = chan.send(into_send_seek(result));
let _chan_sent = chan_result.is_ok();
}
#[tracing::instrument(
name = "batch",
level = "trace",
skip_all,
fields(
%cmd.map,
keys = %cmd.key.len(),
),
)]
fn handle_batch(mut cmd: Get) {
debug_assert!(cmd.key.len() > 1, "should have more than one key");
debug_assert!(!cmd.key.iter().any(SmallVec::is_empty), "querying for empty key");
let chan = cmd.res.take().expect("missing result channel");
if chan.is_canceled() {
return;
}
let keys = cmd.key.iter();
let result: SmallVec<_> = cmd.map.get_batch_blocking(keys).collect();
let chan_result = chan.send(into_send_get(result));
let _chan_sent = chan_result.is_ok();
}
#[tracing::instrument(
name = "get",
level = "trace",
skip_all,
fields(%cmd.map),
)]
fn handle_get(mut cmd: Get) {
debug_assert!(!cmd.key[0].is_empty(), "querying for empty key");
// Obtain the result channel.
let chan = cmd.res.take().expect("missing result channel");
// It is worth checking if the future was dropped while the command was queued
// so we can bail without paying for any query.
if chan.is_canceled() {
return;
}
// Perform the actual database query. We reuse our database::Map interface but
// limited to the blocking calls, rather than creating another surface directly
// with rocksdb here.
let result = cmd.map.get_blocking(&cmd.key[0]);
// Send the result back to the submitter.
let chan_result = chan.send(into_send_get([result].into()));
// If the future was dropped during the query this will fail acceptably.
let _chan_sent = chan_result.is_ok();
}
}
impl Drop for Pool {
@@ -112,319 +415,6 @@ impl Drop for Pool {
}
}
#[implement(Pool)]
#[tracing::instrument(skip_all, level = "debug")]
pub(crate) fn close(&self) {
let workers = take(&mut *self.workers.lock());
let senders = self.queues.iter().map(Sender::sender_count).sum::<usize>();
let receivers = self
.queues
.iter()
.map(Sender::receiver_count)
.sum::<usize>();
for queue in &self.queues {
queue.close();
}
if workers.is_empty() {
return;
}
debug!(
senders,
receivers,
queues = self.queues.len(),
workers = workers.len(),
"Closing pool. Waiting for workers to join..."
);
workers
.into_iter()
.map(JoinHandle::join)
.map(|result| result.map_err(Error::from_panic))
.enumerate()
.for_each(|(id, result)| match result {
| Ok(()) => trace!(%id, "worker joined"),
| Err(error) => error!(%id, "worker joined with error: {error}"),
});
}
#[implement(Pool)]
fn spawn_until(self: &Arc<Self>, recv: &[Receiver<Cmd>], count: usize) -> Result {
let mut workers = self.workers.lock();
while workers.len() < count {
self.clone().spawn_one(&mut workers, recv)?;
}
Ok(())
}
#[implement(Pool)]
#[tracing::instrument(
name = "spawn",
level = "trace",
skip_all,
fields(id = %workers.len())
)]
fn spawn_one(
self: Arc<Self>,
workers: &mut Vec<JoinHandle<()>>,
recv: &[Receiver<Cmd>],
) -> Result {
debug_assert!(!self.queues.is_empty(), "Must have at least one queue");
debug_assert!(!recv.is_empty(), "Must have at least one receiver");
let id = workers.len();
let group = id.overflowing_rem(self.queues.len()).0;
let recv = recv[group].clone();
let handle = thread::Builder::new()
.name(WORKER_NAME.into())
.stack_size(WORKER_STACK_SIZE)
.spawn(move || self.worker(id, &recv))?;
workers.push(handle);
Ok(())
}
#[implement(Pool)]
#[tracing::instrument(level = "trace", name = "get", skip(self, cmd))]
pub(crate) async fn execute_get(self: &Arc<Self>, mut cmd: Get) -> Result<BatchResult<'_>> {
let (send, recv) = oneshot::channel();
_ = cmd.res.insert(send);
let queue = self.select_queue();
self.execute(queue, Cmd::Get(cmd))
.and_then(move |()| {
recv.map_ok(into_recv_get)
.map_err(|e| err!(error!("recv failed {e:?}")))
})
.await
}
#[implement(Pool)]
#[tracing::instrument(level = "trace", name = "iter", skip(self, cmd))]
pub(crate) async fn execute_iter(self: &Arc<Self>, mut cmd: Seek) -> Result<stream::State<'_>> {
let (send, recv) = oneshot::channel();
_ = cmd.res.insert(send);
let queue = self.select_queue();
self.execute(queue, Cmd::Iter(cmd))
.and_then(|()| {
recv.map_ok(into_recv_seek)
.map_err(|e| err!(error!("recv failed {e:?}")))
})
.await
}
#[implement(Pool)]
fn select_queue(&self) -> &Sender<Cmd> {
let core_id = get_affinity().next().unwrap_or(0);
let chan_id = self.topology[core_id];
self.queues.get(chan_id).unwrap_or_else(|| &self.queues[0])
}
#[implement(Pool)]
#[tracing::instrument(
level = "trace",
name = "execute",
skip(self, cmd),
fields(
task = ?tokio::task::try_id(),
receivers = queue.receiver_count(),
queued = queue.len(),
queued_max = self.queued_max.load(Ordering::Relaxed),
),
)]
async fn execute(&self, queue: &Sender<Cmd>, cmd: Cmd) -> Result {
if cfg!(debug_assertions) {
self.queued_max.fetch_max(queue.len(), Ordering::Relaxed);
}
queue
.send(cmd)
.await
.map_err(|e| err!(error!("send failed {e:?}")))
}
#[implement(Pool)]
#[tracing::instrument(
parent = None,
level = "debug",
skip(self, recv),
fields(
tid = ?thread::current().id(),
),
)]
fn worker(self: Arc<Self>, id: usize, recv: &Receiver<Cmd>) {
self.worker_init(id);
self.worker_loop(recv);
}
#[implement(Pool)]
fn worker_init(&self, id: usize) {
let group = id.overflowing_rem(self.queues.len()).0;
let affinity = self
.topology
.iter()
.enumerate()
.filter(|_| self.queues.len() > 1)
.filter(|_| self.server.config.db_pool_affinity)
.filter_map(|(core_id, &queue_id)| (group == queue_id).then_some(core_id))
.filter_map(nth_core_available);
// affinity is empty (no-op) if there's only one queue
set_affinity(affinity.clone());
#[cfg(all(not(target_env = "msvc"), feature = "jemalloc"))]
if affinity.clone().count() == 1 && conduwuit::alloc::je::is_affine_arena() {
use conduwuit::{
alloc::je::this_thread::{arena_id, set_arena},
result::LogDebugErr,
};
let id = affinity.clone().next().expect("at least one id");
if let Ok(arena) = arena_id() {
if arena != id {
set_arena(id).log_debug_err().ok();
}
}
}
debug!(
%group,
affinity = ?affinity.collect::<Vec<_>>(),
"worker ready"
);
}
#[implement(Pool)]
fn worker_loop(self: &Arc<Self>, recv: &Receiver<Cmd>) {
// initial +1 needed prior to entering wait
self.busy.fetch_add(1, Ordering::Relaxed);
while let Ok(cmd) = self.worker_wait(recv) {
Pool::worker_handle(cmd);
}
}
#[implement(Pool)]
#[tracing::instrument(
name = "wait",
level = "trace",
skip_all,
fields(
receivers = recv.receiver_count(),
queued = recv.len(),
busy = self.busy.fetch_sub(1, Ordering::Relaxed) - 1,
),
)]
fn worker_wait(self: &Arc<Self>, recv: &Receiver<Cmd>) -> Result<Cmd, RecvError> {
recv.recv_blocking().debug_inspect(|_| {
self.busy.fetch_add(1, Ordering::Relaxed);
})
}
#[implement(Pool)]
fn worker_handle(cmd: Cmd) {
match cmd {
| Cmd::Get(cmd) if cmd.key.len() == 1 => Pool::handle_get(cmd),
| Cmd::Get(cmd) => Pool::handle_batch(cmd),
| Cmd::Iter(cmd) => Pool::handle_iter(cmd),
}
}
#[implement(Pool)]
#[tracing::instrument(
name = "iter",
level = "trace",
skip_all,
fields(%cmd.map),
)]
fn handle_iter(mut cmd: Seek) {
let chan = cmd.res.take().expect("missing result channel");
if chan.is_canceled() {
return;
}
let from = cmd.key.as_deref();
let result = match cmd.dir {
| Direction::Forward => cmd.state.init_fwd(from),
| Direction::Reverse => cmd.state.init_rev(from),
};
let chan_result = chan.send(into_send_seek(result));
let _chan_sent = chan_result.is_ok();
}
#[implement(Pool)]
#[tracing::instrument(
name = "batch",
level = "trace",
skip_all,
fields(
%cmd.map,
keys = %cmd.key.len(),
),
)]
fn handle_batch(mut cmd: Get) {
debug_assert!(cmd.key.len() > 1, "should have more than one key");
debug_assert!(!cmd.key.iter().any(SmallVec::is_empty), "querying for empty key");
let chan = cmd.res.take().expect("missing result channel");
if chan.is_canceled() {
return;
}
let keys = cmd.key.iter();
let result: SmallVec<_> = cmd.map.get_batch_blocking(keys).collect();
let chan_result = chan.send(into_send_get(result));
let _chan_sent = chan_result.is_ok();
}
#[implement(Pool)]
#[tracing::instrument(
name = "get",
level = "trace",
skip_all,
fields(%cmd.map),
)]
fn handle_get(mut cmd: Get) {
debug_assert!(!cmd.key[0].is_empty(), "querying for empty key");
// Obtain the result channel.
let chan = cmd.res.take().expect("missing result channel");
// It is worth checking if the future was dropped while the command was queued
// so we can bail without paying for any query.
if chan.is_canceled() {
return;
}
// Perform the actual database query. We reuse our database::Map interface but
// limited to the blocking calls, rather than creating another surface directly
// with rocksdb here.
let result = cmd.map.get_blocking(&cmd.key[0]);
// Send the result back to the submitter.
let chan_result = chan.send(into_send_get([result].into()));
// If the future was dropped during the query this will fail acceptably.
let _chan_sent = chan_result.is_ok();
}
fn into_send_get(result: BatchResult<'_>) -> BatchResult<'static> {
// SAFETY: Necessary to send the Handle (rust_rocksdb::PinnableSlice) through
// the channel. The lifetime on the handle is a device by rust-rocksdb to
+1 -10
View File
@@ -2,19 +2,10 @@ use itertools::Itertools;
use proc_macro::{Span, TokenStream};
use proc_macro2::TokenStream as TokenStream2;
use quote::{ToTokens, quote};
use syn::{Attribute, Error, Fields, Ident, ItemEnum, ItemFn, Meta, Variant, parse_quote};
use syn::{Error, Fields, Ident, ItemEnum, Meta, Variant};
use crate::{Result, utils::camel_to_snake_string};
pub(super) fn command(mut item: ItemFn, _args: &[Meta]) -> Result<TokenStream> {
let attr: Attribute = parse_quote! {
#[conduwuit_macros::implement(crate::Context, params = "<'_>")]
};
item.attrs.push(attr);
Ok(item.into_token_stream().into())
}
pub(super) fn command_dispatch(item: ItemEnum, _args: &[Meta]) -> Result<TokenStream> {
let name = &item.ident;
let arm: Vec<TokenStream2> = item.variants.iter().map(dispatch_arm).try_collect()?;
-35
View File
@@ -1,35 +0,0 @@
use proc_macro::{Span, TokenStream};
use quote::quote;
use syn::{Error, ItemFn, Meta, Path};
use utils::get_named_generics;
use crate::{Result, utils};
pub(super) fn implement(item: ItemFn, args: &[Meta]) -> Result<TokenStream> {
let generics = get_named_generics(args, "generics")?;
let receiver = get_receiver(args)?;
let params = get_named_generics(args, "params")?;
let input = item;
let out = quote! {
impl #generics #receiver #params {
#input
}
};
Ok(out.into())
}
fn get_receiver(args: &[Meta]) -> Result<Path> {
let receiver = &args.first().ok_or_else(|| {
Error::new(Span::call_site().into(), "Missing required argument to receiver")
})?;
let Meta::Path(receiver) = receiver else {
return Err(Error::new(
Span::call_site().into(),
"First argument is not path to receiver",
));
};
Ok(receiver.clone())
}
-11
View File
@@ -2,7 +2,6 @@ mod admin;
mod build_info;
mod config;
mod debug;
mod implement;
mod refutable;
mod utils;
@@ -15,11 +14,6 @@ use syn::{
pub(crate) type Result<T> = std::result::Result<T, Error>;
#[proc_macro_attribute]
pub fn admin_command(args: TokenStream, input: TokenStream) -> TokenStream {
attribute_macro::<ItemFn, _>(args, input, admin::command)
}
#[proc_macro_attribute]
pub fn admin_command_dispatch(args: TokenStream, input: TokenStream) -> TokenStream {
attribute_macro::<ItemEnum, _>(args, input, admin::command_dispatch)
@@ -35,11 +29,6 @@ pub fn refutable(args: TokenStream, input: TokenStream) -> TokenStream {
attribute_macro::<ItemFn, _>(args, input, refutable::refutable)
}
#[proc_macro_attribute]
pub fn implement(args: TokenStream, input: TokenStream) -> TokenStream {
attribute_macro::<ItemFn, _>(args, input, implement::implement)
}
#[proc_macro_attribute]
pub fn config_example_generator(args: TokenStream, input: TokenStream) -> TokenStream {
attribute_macro::<ItemStruct, _>(args, input, config::example_generator)
+1 -22
View File
@@ -1,8 +1,6 @@
use std::collections::HashMap;
use syn::{Expr, ExprLit, Generics, Lit, Meta, MetaNameValue, parse_str};
use crate::Result;
use syn::{Expr, ExprLit, Lit, Meta, MetaNameValue};
pub(crate) fn get_simple_settings(args: &[Meta]) -> HashMap<String, String> {
args.iter().fold(HashMap::new(), |mut map, arg| {
@@ -44,25 +42,6 @@ pub(crate) fn legacy_is_cargo_build() -> bool {
pub(crate) fn is_cargo_test() -> bool { std::env::args().any(|flag| flag == "--test") }
pub(crate) fn get_named_generics(args: &[Meta], name: &str) -> Result<Generics> {
const DEFAULT: &str = "<>";
parse_str::<Generics>(&get_named_string(args, name).unwrap_or_else(|| DEFAULT.to_owned()))
}
pub(crate) fn get_named_string(args: &[Meta], name: &str) -> Option<String> {
args.iter().find_map(|arg| {
let value = arg.require_name_value().ok()?;
let Expr::Lit(ref lit) = value.value else {
return None;
};
let Lit::Str(ref str) = lit.lit else {
return None;
};
value.path.is_ident(name).then_some(str.value())
})
}
#[must_use]
pub(crate) fn camel_to_snake_string(s: &str) -> String {
let mut output = String::with_capacity(
+96 -5
View File
@@ -1,8 +1,13 @@
//! Types for invite filtering ([MSC4155]).
//! Types for invite filtering ([MSC4155]) and
//! invite blocking ([invite-blocking]).
//!
//! MSC4155: https://github.com/matrix-org/matrix-spec-proposals/pull/4155
//! invite-blocking: https://spec.matrix.org/v1.18/client-server-api/#invite-permission
use ruma::{ServerName, UserId, exports::ruma_macros::EventContent};
use ruma::{
ServerName, UserId, events::invite_permission_config::InvitePermissionAction,
exports::ruma_macros::EventContent,
};
use serde::{Deserialize, Serialize};
use wildmatch::WildMatch;
@@ -25,6 +30,21 @@ pub struct InvitePermissionConfigEventContent {
#[serde(default = "ruma::serde::default_true")]
pub enabled: bool,
/// The default action chosen by the user that the homeserver should perform
/// automatically when receiving an invitation for this account.
///
/// A missing, invalid or unsupported value means that the user wants to
/// receive invites as normal. Other parts of the specification might still
/// have effects on invites, like [ignoring users].
///
/// [ignoring users]: https://spec.matrix.org/v1.18/client-server-api/#ignoring-users
#[serde(
default,
deserialize_with = "ruma::serde::default_on_error",
skip_serializing_if = "Option::is_none"
)]
pub default_action: Option<InvitePermissionAction>,
/// A list of globs matching users which are allowed to send an invite.
/// Entries in this list supersede entries in the ignored and blocked lists.
#[serde(default, skip_serializing_if = "Vec::is_empty")]
@@ -77,6 +97,7 @@ impl InvitePermissionConfigEventContent {
) -> Self {
Self {
enabled,
default_action: None,
allowed_users,
ignored_users,
blocked_users,
@@ -86,12 +107,21 @@ impl InvitePermissionConfigEventContent {
}
}
/// Returns true if the default action is configured and set to block.
fn are_all_blocked(&self) -> bool {
self.default_action
.clone()
.is_some_and(|action| action == InvitePermissionAction::Block)
}
/// Test the filters against a user id. This function will check both the
/// user rules _and_ the server rules.
#[must_use]
#[allow(clippy::if_same_then_else)]
pub fn user_filter_level(&self, user: &UserId) -> FilterLevel {
if !self.enabled {
if self.are_all_blocked() {
FilterLevel::Block
} else if !self.enabled {
FilterLevel::Allow
} else if Self::matches(&self.allowed_users, user.as_str()) {
FilterLevel::Allow
@@ -107,7 +137,9 @@ impl InvitePermissionConfigEventContent {
/// Test the filters against a server name. Port numbers are ignored.
#[must_use]
pub fn server_filter_level(&self, server: &ServerName) -> FilterLevel {
if !self.enabled {
if self.are_all_blocked() {
FilterLevel::Block
} else if !self.enabled {
FilterLevel::Allow
} else {
let server = server.host();
@@ -132,7 +164,12 @@ impl InvitePermissionConfigEventContent {
#[cfg(test)]
mod tests {
use ruma::{ServerName, UserId, events::GlobalAccountDataEvent};
use assign::assign;
use ruma::{
ServerName, UserId,
events::{GlobalAccountDataEvent, invite_permission_config::InvitePermissionAction},
user_id,
};
use serde_json::{from_value as from_json_value, json};
use crate::invite_permission_config::{FilterLevel, InvitePermissionConfigEventContent};
@@ -268,4 +305,58 @@ mod tests {
FilterLevel::Ignore
);
}
#[test]
fn v118_blocking_enabled_blocks_all() {
let event = assign!(
InvitePermissionConfigEventContent::default(),
{ default_action: Some(InvitePermissionAction::Block) }
);
let fixtures = [
user_id!("@alice:goodguys.org"),
user_id!("@alice:goodguys.org:8080"),
user_id!("@bob:bar.com"),
user_id!("@bob:Bar.com"),
user_id!("@kevin:reallybadguys.org"),
];
assert!(
fixtures
.iter()
.all(|f| event.user_filter_level(f) == FilterLevel::Block)
);
}
#[test]
fn v118_blocking_enabled_is_uninfluenced_by_msc4155() {
let toggled = InvitePermissionConfigEventContent {
enabled: false,
default_action: Some(InvitePermissionAction::Block),
..Default::default()
};
let only_goodguys = InvitePermissionConfigEventContent {
enabled: true,
default_action: Some(InvitePermissionAction::Block),
allowed_servers: vec!["goodguys.org".to_owned()],
ignored_servers: vec!["reallybadguys.org".to_owned()],
blocked_servers: vec!["*".to_owned()],
..Default::default()
};
let fixtures = [
user_id!("@alice:goodguys.org"),
user_id!("@alice:goodguys.org:8080"),
user_id!("@bob:bar.com"),
user_id!("@bob:Bar.com"),
user_id!("@kevin:reallybadguys.org"),
];
assert!(
fixtures
.iter()
.all(|f| toggled.user_filter_level(f) == FilterLevel::Block)
);
assert!(
fixtures
.iter()
.all(|f| only_goodguys.user_filter_level(f) == FilterLevel::Block)
);
}
}
+104 -103
View File
@@ -1,7 +1,7 @@
use std::sync::Arc;
use conduwuit::{
Err, Result, err, implement,
Err, Result, err,
utils::{ReadyExt, result::LogErr, stream::TryIgnore},
};
use database::{Deserialized, Handle, Ignore, Json, Map};
@@ -54,114 +54,115 @@ impl crate::Service for Service {
fn name(&self) -> &str { crate::service::make_name(std::module_path!()) }
}
/// Places one event in the account data of the user and removes the
/// previous entry.
#[allow(clippy::needless_pass_by_value)]
#[implement(Service)]
pub async fn update(
&self,
room_id: Option<&RoomId>,
user_id: &UserId,
event_type: RoomAccountDataEventType,
data: &serde_json::Value,
) -> Result<()> {
if data.get("type").is_none() || data.get("content").is_none() {
return Err!(Request(InvalidParam("Account data doesn't have all required fields.")));
impl Service {
/// Places one event in the account data of the user and removes the
/// previous entry.
#[allow(clippy::needless_pass_by_value)]
pub async fn update(
&self,
room_id: Option<&RoomId>,
user_id: &UserId,
event_type: RoomAccountDataEventType,
data: &serde_json::Value,
) -> Result<()> {
if data.get("type").is_none() || data.get("content").is_none() {
return Err!(Request(InvalidParam("Account data doesn't have all required fields.")));
}
let count = self.services.globals.next_count().unwrap();
let roomuserdataid = (room_id, user_id, count, &event_type);
self.db
.roomuserdataid_accountdata
.put(roomuserdataid, Json(data));
let key = (room_id, user_id, &event_type);
let prev = self.db.roomusertype_roomuserdataid.qry(&key).await;
self.db.roomusertype_roomuserdataid.put(key, roomuserdataid);
// Remove old entry
if let Ok(prev) = prev {
self.db.roomuserdataid_accountdata.remove(&prev);
}
Ok(())
}
let count = self.services.globals.next_count().unwrap();
let roomuserdataid = (room_id, user_id, count, &event_type);
self.db
.roomuserdataid_accountdata
.put(roomuserdataid, Json(data));
let key = (room_id, user_id, &event_type);
let prev = self.db.roomusertype_roomuserdataid.qry(&key).await;
self.db.roomusertype_roomuserdataid.put(key, roomuserdataid);
// Remove old entry
if let Ok(prev) = prev {
self.db.roomuserdataid_accountdata.remove(&prev);
/// Searches the room account data for a specific kind.
pub async fn get_global<T>(
&self,
user_id: &UserId,
kind: GlobalAccountDataEventType,
) -> Result<T>
where
T: for<'de> Deserialize<'de>,
{
self.get_raw(None, user_id, &kind.to_string())
.await
.deserialized()
}
Ok(())
}
/// Searches the global account data for a specific kind.
pub async fn get_room<T>(
&self,
room_id: &RoomId,
user_id: &UserId,
kind: RoomAccountDataEventType,
) -> Result<T>
where
T: for<'de> Deserialize<'de>,
{
self.get_raw(Some(room_id), user_id, &kind.to_string())
.await
.deserialized()
}
/// Searches the room account data for a specific kind.
#[implement(Service)]
pub async fn get_global<T>(&self, user_id: &UserId, kind: GlobalAccountDataEventType) -> Result<T>
where
T: for<'de> Deserialize<'de>,
{
self.get_raw(None, user_id, &kind.to_string())
.await
.deserialized()
}
pub async fn get_raw(
&self,
room_id: Option<&RoomId>,
user_id: &UserId,
kind: &str,
) -> Result<Handle<'_>> {
let key = (room_id, user_id, kind.to_owned());
self.db
.roomusertype_roomuserdataid
.qry(&key)
.and_then(|roomuserdataid| self.db.roomuserdataid_accountdata.get(&roomuserdataid))
.await
}
/// Searches the global account data for a specific kind.
#[implement(Service)]
pub async fn get_room<T>(
&self,
room_id: &RoomId,
user_id: &UserId,
kind: RoomAccountDataEventType,
) -> Result<T>
where
T: for<'de> Deserialize<'de>,
{
self.get_raw(Some(room_id), user_id, &kind.to_string())
.await
.deserialized()
}
/// Returns all changes to the account data that happened after `since`.
pub fn changes_since<'a>(
&'a self,
room_id: Option<&'a RoomId>,
user_id: &'a UserId,
since: Option<u64>,
to: Option<u64>,
) -> impl Stream<Item = AnyRawAccountDataEvent> + Send + 'a {
type Key = (Option<OwnedRoomId>, OwnedUserId, u64, Ignore);
#[implement(Service)]
pub async fn get_raw(
&self,
room_id: Option<&RoomId>,
user_id: &UserId,
kind: &str,
) -> Result<Handle<'_>> {
let key = (room_id, user_id, kind.to_owned());
self.db
.roomusertype_roomuserdataid
.qry(&key)
.and_then(|roomuserdataid| self.db.roomuserdataid_accountdata.get(&roomuserdataid))
.await
}
// Skip the data that's exactly at since, because we sent that last time
// ...unless this is an initial sync, in which case send everything
let first_possible = (room_id, user_id, since.map_or(0, |since| since.saturating_add(1)));
/// Returns all changes to the account data that happened after `since`.
#[implement(Service)]
pub fn changes_since<'a>(
&'a self,
room_id: Option<&'a RoomId>,
user_id: &'a UserId,
since: Option<u64>,
to: Option<u64>,
) -> impl Stream<Item = AnyRawAccountDataEvent> + Send + 'a {
type Key = (Option<OwnedRoomId>, OwnedUserId, u64, Ignore);
// Skip the data that's exactly at since, because we sent that last time
// ...unless this is an initial sync, in which case send everything
let first_possible = (room_id, user_id, since.map_or(0, |since| since.saturating_add(1)));
self.db
.roomuserdataid_accountdata
.stream_from(&first_possible)
.ignore_err()
.ready_take_while(move |((room_id_, user_id_, count, _), _): &(Key, _)| {
room_id == room_id_.as_deref()
&& user_id == user_id_
&& to.is_none_or(|to| *count <= to)
})
.map(move |(_, v)| {
match room_id {
| Some(_) => serde_json::from_slice::<Raw<AnyRoomAccountDataEvent>>(v)
.map(AnyRawAccountDataEvent::Room),
| None => serde_json::from_slice::<Raw<AnyGlobalAccountDataEvent>>(v)
.map(AnyRawAccountDataEvent::Global),
}
.map_err(|e| err!(Database("Database contains invalid account data: {e}")))
.log_err()
})
.ignore_err()
self.db
.roomuserdataid_accountdata
.stream_from(&first_possible)
.ignore_err()
.ready_take_while(move |((room_id_, user_id_, count, _), _): &(Key, _)| {
room_id == room_id_.as_deref()
&& user_id == user_id_
&& to.is_none_or(|to| *count <= to)
})
.map(move |(_, v)| {
match room_id {
| Some(_) => serde_json::from_slice::<Raw<AnyRoomAccountDataEvent>>(v)
.map(AnyRawAccountDataEvent::Room),
| None => serde_json::from_slice::<Raw<AnyGlobalAccountDataEvent>>(v)
.map(AnyRawAccountDataEvent::Global),
}
.map_err(|e| err!(Database("Database contains invalid account data: {e}")))
.log_err()
})
.ignore_err()
}
}
+76 -37
View File
@@ -1,20 +1,22 @@
use std::collections::BTreeMap;
use conduwuit::{Result, info, pdu::PartialPdu};
use futures::FutureExt;
use ruma::{
RoomId, RoomVersionId,
events::room::{
canonical_alias::RoomCanonicalAliasEventContent,
create::RoomCreateEventContent,
guest_access::{GuestAccess, RoomGuestAccessEventContent},
history_visibility::{HistoryVisibility, RoomHistoryVisibilityEventContent},
join_rules::{JoinRule, RoomJoinRulesEventContent},
member::{MembershipState, RoomMemberEventContent},
name::RoomNameEventContent,
power_levels::RoomPowerLevelsEventContent,
topic::RoomTopicEventContent,
Int, RoomId,
events::{
TimelineEventType,
room::{
canonical_alias::RoomCanonicalAliasEventContent,
create::RoomCreateEventContent,
guest_access::{GuestAccess, RoomGuestAccessEventContent},
history_visibility::{HistoryVisibility, RoomHistoryVisibilityEventContent},
join_rules::{JoinRule, RoomJoinRulesEventContent},
member::{MembershipState, RoomMemberEventContent},
name::RoomNameEventContent,
power_levels::RoomPowerLevelsEventContent,
topic::RoomTopicEventContent,
},
},
room_version_rules::RoomIdFormatVersion,
};
use crate::Services;
@@ -24,47 +26,68 @@ use crate::Services;
/// Users in this room are considered admins by conduwuit, and the room can be
/// used to issue admin commands by talking to the server user inside it.
pub async fn create_admin_room(services: &Services) -> Result {
let room_id = RoomId::new_v1(services.globals.server_name());
let room_version = &RoomVersionId::V11;
let room_version = services.config.default_room_version.clone();
let room_version_rules = room_version
.rules()
.expect("default_room_version must be supported");
let room_id = match room_version_rules.room_id_format {
| RoomIdFormatVersion::V1 => {
let room_id = RoomId::new_v1(services.globals.server_name());
services
.rooms
.short
.get_or_create_shortroomid(&room_id)
.await;
Some(room_id)
},
| RoomIdFormatVersion::V2 => None,
| _ => panic!("Unknown room version format"),
};
let _short_id = services
.rooms
.short
.get_or_create_shortroomid(&room_id)
.await;
let state_lock = services.rooms.state.mutex.lock(room_id.as_str()).await;
let state_lock = services.rooms.state.mutex.lock("!new-room").await;
// Create a user for the server
let server_user = services.globals.server_user.as_ref();
services.users.create(server_user, None).await?;
services.users.create(server_user, None)?;
let mut create_content = {
use RoomVersionId::*;
match room_version {
| V1 | V2 | V3 | V4 | V5 | V6 | V7 | V8 | V9 | V10 =>
RoomCreateEventContent::new_v1(server_user.into()),
| _ => RoomCreateEventContent::new_v11(),
}
let mut create_content = if room_version_rules.authorization.use_room_create_sender {
RoomCreateEventContent::new_v1(server_user.into())
} else {
RoomCreateEventContent::new_v11()
};
create_content.federate = true;
create_content.room_version = room_version.clone();
info!("Creating admin room {} with version {}", room_id, room_version);
info!(
"Creating admin room {} with version {}",
room_id
.clone()
.map_or_else(|| "<not known ahead of time>".to_owned(), |id| id.as_str().to_owned()),
room_version
);
// 1. The room create event
services
let create_event_id = services
.rooms
.timeline
.build_and_append_pdu(
PartialPdu::state(String::new(), &create_content),
server_user,
Some(&room_id),
room_id.as_deref(),
&state_lock,
)
.boxed()
.await?;
let room_id = room_id.unwrap_or_else(|| {
RoomId::new_v2(
create_event_id
.as_str()
.strip_prefix("$")
.expect("event ID must start with a $ sigil"),
)
.expect("event ID without sigil must be a valid room ID")
});
// 2. Make server user/bot join
services
@@ -83,11 +106,27 @@ pub async fn create_admin_room(services: &Services) -> Result {
.await?;
// 3. Power levels
let users = BTreeMap::from_iter([(server_user.into(), 69420.into())]);
let mut power_levels_content =
RoomPowerLevelsEventContent::new(&room_version.rules().unwrap().authorization);
power_levels_content.users = users;
RoomPowerLevelsEventContent::new(&room_version_rules.authorization);
if !room_version_rules
.authorization
.explicitly_privilege_room_creators
{
power_levels_content
.users
.insert(server_user.into(), Int::MAX);
}
// Prevent common foot-shotguns
power_levels_content
.events
.insert(TimelineEventType::RoomTombstone, Int::MAX);
power_levels_content
.events
.insert(TimelineEventType::RoomEncryption, Int::MAX);
power_levels_content
.events
.insert(TimelineEventType::RoomCanonicalAlias, Int::MAX);
power_levels_content.invite = power_levels_content.state_default;
services
.rooms
+100 -107
View File
@@ -1,4 +1,4 @@
use conduwuit::{Err, Result, debug, debug_info, error, implement, info};
use conduwuit::{Err, Result, debug, debug_info, error, info};
use ruma::events::room::message::RoomMessageEventContent;
use tokio::time::{Duration, sleep};
@@ -6,127 +6,120 @@ use crate::admin::InvocationSource;
pub(super) const SIGNAL: &str = "SIGUSR2";
/// Possibly spawn the terminal console at startup if configured.
#[implement(super::Service)]
pub(super) async fn console_auto_start(&self) {
#[cfg(feature = "console")]
if self.services.server.config.admin_console_automatic {
// Allow more of the startup sequence to execute before spawning
tokio::task::yield_now().await;
self.console.start().await;
impl super::Service {
/// Possibly spawn the terminal console at startup if configured.
pub(super) async fn console_auto_start(&self) {
#[cfg(feature = "console")]
if self.services.server.config.admin_console_automatic {
// Allow more of the startup sequence to execute before spawning
tokio::task::yield_now().await;
self.console.start().await;
}
}
}
/// Shutdown the console when the admin worker terminates.
#[implement(super::Service)]
pub(super) async fn console_auto_stop(&self) {
#[cfg(feature = "console")]
self.console.close().await;
}
/// Shutdown the console when the admin worker terminates.
pub(super) async fn console_auto_stop(&self) {
#[cfg(feature = "console")]
self.console.close().await;
}
/// Execute admin commands after startup
#[implement(super::Service)]
pub async fn startup_execute(&self) -> Result {
// List of commands to execute
let commands = &self.services.server.config.admin_execute;
/// Execute admin commands after startup
pub async fn startup_execute(&self) -> Result {
// List of commands to execute
let commands = &self.services.server.config.admin_execute;
// Determine if we're running in smoketest-mode which will change some behaviors
let smoketest = self.services.server.config.test.contains("smoke");
// Determine if we're running in smoketest-mode which will change some behaviors
let smoketest = self.services.server.config.test.contains("smoke");
// When true, errors are ignored and startup continues.
let errors = !smoketest && self.services.server.config.admin_execute_errors_ignore;
// When true, errors are ignored and startup continues.
let errors = !smoketest && self.services.server.config.admin_execute_errors_ignore;
//TODO: remove this after run-states are broadcast
sleep(Duration::from_millis(500)).await;
//TODO: remove this after run-states are broadcast
sleep(Duration::from_millis(500)).await;
for (i, command) in commands.iter().enumerate() {
if let Err(e) = self.execute_command(i, command.clone()).await {
if !errors {
return Err(e);
for (i, command) in commands.iter().enumerate() {
if let Err(e) = self.execute_command(i, command.clone()).await {
if !errors {
return Err(e);
}
}
tokio::task::yield_now().await;
}
tokio::task::yield_now().await;
}
// The smoketest functionality is placed here for now and simply initiates
// shutdown after all commands have executed.
if smoketest {
debug_info!("Smoketest mode. All commands complete. Shutting down now...");
self.services
.server
.shutdown()
.inspect_err(error::inspect_log)
.expect("Error shutting down from smoketest");
}
Ok(())
}
/// Execute admin commands after signal
#[implement(super::Service)]
pub(super) async fn signal_execute(&self) -> Result {
// List of commands to execute
let commands = self.services.server.config.admin_signal_execute.clone();
// When true, errors are ignored and execution continues.
let ignore_errors = self.services.server.config.admin_execute_errors_ignore;
for (i, command) in commands.iter().enumerate() {
if let Err(e) = self.execute_command(i, command.clone()).await {
if !ignore_errors {
return Err(e);
}
// The smoketest functionality is placed here for now and simply initiates
// shutdown after all commands have executed.
if smoketest {
debug_info!("Smoketest mode. All commands complete. Shutting down now...");
self.services
.server
.shutdown()
.inspect_err(error::inspect_log)
.expect("Error shutting down from smoketest");
}
tokio::task::yield_now().await;
Ok(())
}
Ok(())
}
/// Execute admin commands after signal
pub(super) async fn signal_execute(&self) -> Result {
// List of commands to execute
let commands = self.services.server.config.admin_signal_execute.clone();
/// Execute one admin command after startup or signal
#[implement(super::Service)]
async fn execute_command(&self, i: usize, command: String) -> Result {
debug!("Execute command #{i}: executing {command:?}");
// When true, errors are ignored and execution continues.
let ignore_errors = self.services.server.config.admin_execute_errors_ignore;
match self
.command_in_place(command, None, InvocationSource::Console)
.await
{
| Ok(Some(output)) => Self::execute_command_output(i, &output),
| Err(output) => Self::execute_command_error(i, &output),
| Ok(None) => {
info!("Execute command #{i} completed (no output).");
Ok(())
},
for (i, command) in commands.iter().enumerate() {
if let Err(e) = self.execute_command(i, command.clone()).await {
if !ignore_errors {
return Err(e);
}
}
tokio::task::yield_now().await;
}
Ok(())
}
/// Execute one admin command after startup or signal
async fn execute_command(&self, i: usize, command: String) -> Result {
debug!("Execute command #{i}: executing {command:?}");
match self
.command_in_place(command, None, InvocationSource::Console)
.await
{
| Ok(Some(output)) => Self::execute_command_output(i, &output),
| Err(output) => Self::execute_command_error(i, &output),
| Ok(None) => {
info!("Execute command #{i} completed (no output).");
Ok(())
},
}
}
#[cfg(feature = "console")]
fn execute_command_output(i: usize, content: &RoomMessageEventContent) -> Result {
debug_info!("Execute command #{i} completed:");
super::console::print(content.body());
Ok(())
}
#[cfg(feature = "console")]
fn execute_command_error(i: usize, content: &RoomMessageEventContent) -> Result {
super::console::print_err(content.body());
Err!(debug_error!("Execute command #{i} failed."))
}
#[cfg(not(feature = "console"))]
fn execute_command_output(i: usize, content: &RoomMessageEventContent) -> Result {
info!("Execute command #{i} completed:\n{:#}", content.body());
Ok(())
}
#[cfg(not(feature = "console"))]
fn execute_command_error(i: usize, content: &RoomMessageEventContent) -> Result {
Err!(error!("Execute command #{i} failed:\n{:#}", content.body()))
}
}
#[cfg(feature = "console")]
#[implement(super::Service)]
fn execute_command_output(i: usize, content: &RoomMessageEventContent) -> Result {
debug_info!("Execute command #{i} completed:");
super::console::print(content.body());
Ok(())
}
#[cfg(feature = "console")]
#[implement(super::Service)]
fn execute_command_error(i: usize, content: &RoomMessageEventContent) -> Result {
super::console::print_err(content.body());
Err!(debug_error!("Execute command #{i} failed."))
}
#[cfg(not(feature = "console"))]
#[implement(super::Service)]
fn execute_command_output(i: usize, content: &RoomMessageEventContent) -> Result {
info!("Execute command #{i} completed:\n{:#}", content.body());
Ok(())
}
#[cfg(not(feature = "console"))]
#[implement(super::Service)]
fn execute_command_error(i: usize, content: &RoomMessageEventContent) -> Result {
Err!(error!("Execute command #{i} failed:\n{:#}", content.body()))
}
+233 -199
View File
@@ -1,10 +1,8 @@
use std::collections::BTreeMap;
use conduwuit::{
Err, Result, debug_info, debug_warn, error, implement, matrix::pdu::PartialPdu, warn,
};
use conduwuit::{Err, Result, debug_info, debug_warn, error, matrix::pdu::PartialPdu, warn};
use ruma::{
RoomId, UserId,
Int, RoomId, UserId,
events::{
RoomAccountDataEventType, StateEventType,
room::{
@@ -13,205 +11,241 @@ use ruma::{
},
tag::{TagEvent, TagEventContent, TagInfo},
},
int,
};
/// Invite the user to the conduwuit admin room.
///
/// This is equivalent to granting server admin privileges.
#[implement(super::Service)]
pub async fn make_user_admin(&self, user_id: &UserId) -> Result {
let Ok(room_id) = self.get_admin_room().await else {
debug_warn!(
"make_user_admin was called without an admin room being available or created"
);
return Ok(());
};
let state_lock = self.services.state.mutex.lock(room_id.as_str()).await;
if self.services.state_cache.is_joined(user_id, &room_id).await {
return Err!(debug_warn!("User is already joined in the admin room"));
}
if self
.services
.state_cache
.is_invited(user_id, &room_id)
.await
{
return Err!(debug_warn!("User is already pending an invitation to the admin room"));
}
// Use the server user to grant the new admin's power level
let server_user = self.services.globals.server_user.as_ref();
// if this is our local user, just forcefully join them in the room. otherwise,
// invite the remote user.
if self.services.globals.user_is_local(user_id) {
debug_info!("Inviting local user {user_id} to admin room {room_id}");
self.services
.timeline
.build_and_append_pdu(
PartialPdu::state(
String::from(user_id),
&RoomMemberEventContent::new(MembershipState::Invite),
),
server_user,
Some(&room_id),
&state_lock,
)
.await?;
debug_info!("Force joining local user {user_id} to admin room {room_id}");
self.services
.timeline
.build_and_append_pdu(
PartialPdu::state(
String::from(user_id),
&RoomMemberEventContent::new(MembershipState::Join),
),
user_id,
Some(&room_id),
&state_lock,
)
.await?;
} else {
debug_info!("Inviting remote user {user_id} to admin room {room_id}");
self.services
.timeline
.build_and_append_pdu(
PartialPdu::state(
user_id.to_string(),
&RoomMemberEventContent::new(MembershipState::Invite),
),
server_user,
Some(&room_id),
&state_lock,
)
.await?;
}
// Set power levels
let mut room_power_levels = self
.services
.state_accessor
.room_state_get_content::<RoomPowerLevelsEventContent>(
&room_id,
&StateEventType::RoomPowerLevels,
"",
)
.await
.expect("admin room should have power levels");
room_power_levels
.users
.insert(server_user.into(), 69420.into());
room_power_levels.users.insert(user_id.into(), 100.into());
self.services
.timeline
.build_and_append_pdu(
PartialPdu::state(String::new(), &room_power_levels),
server_user,
Some(&room_id),
&state_lock,
)
.await?;
// Set room tag
let room_tag = self.services.server.config.admin_room_tag.as_str();
if !room_tag.is_empty() {
if let Err(e) = self.set_room_tag(&room_id, user_id, room_tag).await {
error!(%room_id, %user_id, %room_tag, "Failed to set tag for admin grant: {e}");
}
}
Ok(())
}
#[implement(super::Service)]
async fn set_room_tag(&self, room_id: &RoomId, user_id: &UserId, tag: &str) -> Result {
let mut event = self
.services
.account_data
.get_room(room_id, user_id, RoomAccountDataEventType::Tag)
.await
.unwrap_or_else(|_| TagEvent::new(TagEventContent::new(BTreeMap::new())));
event
.content
.tags
.insert(tag.to_owned().into(), TagInfo::new());
self.services
.account_data
.update(
Some(room_id),
user_id,
RoomAccountDataEventType::Tag,
&serde_json::to_value(event)?,
)
.await
}
/// Demote an admin, removing its rights.
#[implement(super::Service)]
pub async fn revoke_admin(&self, user_id: &UserId) -> Result {
use MembershipState::{Invite, Join, Knock, Leave};
if self
.services
.server
.config
.admins_list
.contains(&user_id.to_owned())
{
warn!(
"Revoking the admin status of {user_id} will not work correctly as they are within \
the admins_list config."
);
}
let Ok(room_id) = self.get_admin_room().await else {
return Err!(error!("No admin room available or created."));
};
let state_lock = self.services.state.mutex.lock(room_id.as_str()).await;
let mut member_content = match self
.services
.state_accessor
.get_member(&room_id, user_id)
.await
{
| Err(e) if e.is_not_found() => return Err!("{user_id} was never an admin."),
| Err(e) => return Err!(error!(?e, "Failure occurred while attempting revoke.")),
| Ok(event) if !matches!(event.membership, Invite | Knock | Join) => {
return Err!("Cannot revoke {user_id} in membership state {:?}.", event.membership);
},
| Ok(event) => {
assert!(
matches!(event.membership, Invite | Knock | Join),
"Incorrect membership state to remove user."
impl super::Service {
/// Invite the user to the conduwuit admin room.
///
/// This is equivalent to granting server admin privileges.
pub async fn make_user_admin(&self, user_id: &UserId) -> Result {
let Ok(room_id) = self.get_admin_room().await else {
debug_warn!(
"make_user_admin was called without an admin room being available or created"
);
return Ok(());
};
event
},
};
let state_lock = self.services.state.mutex.lock(room_id.as_str()).await;
member_content.membership = Leave;
member_content.reason = Some("Admin Revoked".to_owned());
if self.services.state_cache.is_joined(user_id, &room_id).await {
return Err!(debug_warn!("User is already joined in the admin room"));
}
if self
.services
.state_cache
.is_invited(user_id, &room_id)
.await
{
return Err!(debug_warn!("User is already pending an invitation to the admin room"));
}
self.services
.timeline
.build_and_append_pdu(
PartialPdu::state(user_id.to_string(), &member_content),
self.services.globals.server_user.as_ref(),
Some(&room_id),
&state_lock,
)
.await
.map(|_| ())
// Use the server user to grant the new admin's power level
let server_user = self.services.globals.server_user.as_ref();
// if this is our local user, just forcefully join them in the room. otherwise,
// invite the remote user.
if self.services.globals.user_is_local(user_id) {
debug_info!("Inviting local user {user_id} to admin room {room_id}");
self.services
.timeline
.build_and_append_pdu(
PartialPdu::state(
String::from(user_id),
&RoomMemberEventContent::new(MembershipState::Invite),
),
server_user,
Some(&room_id),
&state_lock,
)
.await?;
debug_info!("Force joining local user {user_id} to admin room {room_id}");
self.services
.timeline
.build_and_append_pdu(
PartialPdu::state(
String::from(user_id),
&RoomMemberEventContent::new(MembershipState::Join),
),
user_id,
Some(&room_id),
&state_lock,
)
.await?;
} else {
debug_info!("Inviting remote user {user_id} to admin room {room_id}");
self.services
.timeline
.build_and_append_pdu(
PartialPdu::state(
user_id.to_string(),
&RoomMemberEventContent::new(MembershipState::Invite),
),
server_user,
Some(&room_id),
&state_lock,
)
.await?;
}
// Set power levels
let mut room_power_levels = self
.services
.state_accessor
.room_state_get_content::<RoomPowerLevelsEventContent>(
&room_id,
&StateEventType::RoomPowerLevels,
"",
)
.await
.expect("admin room should have power levels");
let server_user_power_level = room_power_levels
.users
.get(server_user)
.copied()
.unwrap_or(Int::MAX);
room_power_levels
.users
.insert(user_id.into(), server_user_power_level.saturating_sub(int!(1)));
self.services
.timeline
.build_and_append_pdu(
PartialPdu::state(String::new(), &room_power_levels),
server_user,
Some(&room_id),
&state_lock,
)
.await?;
// Set room tag
let room_tag = self.services.server.config.admin_room_tag.as_str();
if !room_tag.is_empty() {
if let Err(e) = self.set_room_tag(&room_id, user_id, room_tag).await {
error!(%room_id, %user_id, %room_tag, "Failed to set tag for admin grant: {e}");
}
}
Ok(())
}
async fn set_room_tag(&self, room_id: &RoomId, user_id: &UserId, tag: &str) -> Result {
let mut event = self
.services
.account_data
.get_room(room_id, user_id, RoomAccountDataEventType::Tag)
.await
.unwrap_or_else(|_| TagEvent::new(TagEventContent::new(BTreeMap::new())));
event
.content
.tags
.insert(tag.to_owned().into(), TagInfo::new());
self.services
.account_data
.update(
Some(room_id),
user_id,
RoomAccountDataEventType::Tag,
&serde_json::to_value(event)?,
)
.await
}
/// Demote an admin, removing its rights.
pub async fn revoke_admin(&self, user_id: &UserId) -> Result {
use MembershipState::{Invite, Join, Knock, Leave};
if self
.services
.server
.config
.admins_list
.contains(&user_id.to_owned())
{
warn!(
"Revoking the admin status of {user_id} will not work correctly as they are \
within the admins_list config."
);
}
let Ok(room_id) = self.get_admin_room().await else {
return Err!(error!("No admin room available or created."));
};
let state_lock = self.services.state.mutex.lock(room_id.as_str()).await;
let mut room_power_levels = self
.services
.state_accessor
.room_state_get_content::<RoomPowerLevelsEventContent>(
&room_id,
&StateEventType::RoomPowerLevels,
"",
)
.await
.expect("admin room should have power levels");
if room_power_levels.users.remove(user_id).is_some() {
// drop the target's power level
self.services
.timeline
.build_and_append_pdu(
PartialPdu::state(String::new(), &room_power_levels),
self.services.globals.server_user.as_ref(),
Some(&room_id),
&state_lock,
)
.await
.inspect_err(|e| {
error!(?e, "Failed to update power levels while revoking admin.");
})
.ok();
// Don't treat this as fatal. removing them from the room is more
// important.
}
let mut member_content = match self
.services
.state_accessor
.get_member(&room_id, user_id)
.await
{
| Err(e) if e.is_not_found() => return Err!("{user_id} was never an admin."),
| Err(e) => return Err!(error!(?e, "Failure occurred while attempting revoke.")),
| Ok(event) if !matches!(event.membership, Invite | Knock | Join) => {
return Err!(
"Cannot revoke {user_id} in membership state {:?}.",
event.membership
);
},
| Ok(event) => {
assert!(
matches!(event.membership, Invite | Knock | Join),
"Incorrect membership state to remove user."
);
event
},
};
member_content.membership = Leave;
member_content.reason = Some("Admin Revoked".to_owned());
self.services
.timeline
.build_and_append_pdu(
PartialPdu::state(user_id.to_string(), &member_content),
self.services.globals.server_user.as_ref(),
Some(&room_id),
&state_lock,
)
.await
.map(|_| ())
}
}
+1 -4
View File
@@ -109,10 +109,7 @@ impl Service {
)?;
if !self.services.users.exists(&appservice_user_id).await {
self.services
.users
.create(&appservice_user_id, None)
.await?;
self.services.users.create(&appservice_user_id, None)?;
} else if self
.services
.users
+13 -12
View File
@@ -1,6 +1,6 @@
use std::{sync::Arc, time::Duration};
use conduwuit::{Config, Result, err, implement, trace};
use conduwuit::{Config, Result, err, trace};
use either::Either;
use ipaddress::IPAddress;
use reqwest::redirect;
@@ -13,7 +13,7 @@ pub struct Service {
pub extern_media: reqwest::Client,
pub well_known: reqwest::Client,
pub federation: reqwest::Client,
pub synapse: reqwest::Client,
pub federation_slow: reqwest::Client,
pub sender: reqwest::Client,
pub appservice: reqwest::Client,
pub pusher: reqwest::Client,
@@ -85,7 +85,7 @@ impl crate::Service for Service {
.redirect(redirect::Policy::limited(3))
.build()?,
synapse: base(config)?
federation_slow: base(config)?
.dns_resolver(resolver.resolver.hooked.clone())
.connect_timeout(Duration::from_secs(config.federation_conn_timeout))
.read_timeout(Duration::from_secs(config.federation_timeout.saturating_mul(6)))
@@ -141,6 +141,16 @@ impl crate::Service for Service {
fn name(&self) -> &str { service::make_name(std::module_path!()) }
}
impl Service {
#[inline]
#[must_use]
pub fn valid_cidr_range(&self, ip: &IPAddress) -> bool {
self.cidr_range_denylist
.iter()
.all(|cidr| !cidr.includes(ip))
}
}
fn base(config: &Config) -> Result<reqwest::ClientBuilder> {
let mut builder = reqwest::Client::builder()
.hickory_dns(true)
@@ -227,12 +237,3 @@ fn builder_interface(
Ok(builder)
}
}
#[inline]
#[must_use]
#[implement(Service)]
pub fn valid_cidr_range(&self, ip: &IPAddress) -> bool {
self.cidr_range_denylist
.iter()
.all(|cidr| !cidr.includes(ip))
}
+46 -47
View File
@@ -4,7 +4,7 @@ use async_trait::async_trait;
use conduwuit::{
Result, Server,
config::{Config, check},
error, implement,
error,
};
use url::Url;
@@ -16,28 +16,6 @@ pub struct Service {
const SIGNAL: &str = "SIGUSR1";
impl Service {
/// Get the registration token set in the config file, if it exists.
#[must_use]
pub fn get_config_file_token(&self) -> Option<ValidToken> {
self.registration_token
.clone()
.map(|token| ValidToken { token, source: ValidTokenSource::Config })
}
/// Get the base domain to use for user-facing URLs.
#[must_use]
pub fn get_client_domain(&self) -> Url {
self.well_known.client.clone().unwrap_or_else(|| {
let host = self.server_name.host();
format!("https://{host}")
.as_str()
.try_into()
.expect("server name should be a valid host")
})
}
}
#[async_trait]
impl crate::Service for Service {
fn build(args: crate::Args<'_>) -> Result<Arc<Self>> {
@@ -66,32 +44,53 @@ impl Deref for Service {
fn deref(&self) -> &Self::Target { &self.server.config }
}
#[implement(Service)]
fn handle_reload(&self) -> Result {
if self.server.config.config_reload_signal {
#[cfg(all(feature = "systemd", target_os = "linux"))]
sd_notify::notify(&[
sd_notify::NotifyState::Reloading,
sd_notify::NotifyState::monotonic_usec_now().expect("Failed to read monotonic time"),
])
.expect("failed to notify systemd of reloading state");
let config_paths = self.server.config.config_paths.clone().unwrap_or_default();
self.reload(&config_paths)?;
#[cfg(all(feature = "systemd", target_os = "linux"))]
sd_notify::notify(&[sd_notify::NotifyState::Ready])
.expect("failed to notify systemd of ready state");
impl Service {
/// Get the registration token set in the config file, if it exists.
#[must_use]
pub fn get_config_file_token(&self) -> Option<ValidToken> {
self.registration_token
.clone()
.map(|token| ValidToken { token, source: ValidTokenSource::Config })
}
Ok(())
}
/// Get the base domain to use for user-facing URLs.
#[must_use]
pub fn get_client_domain(&self) -> Url {
self.well_known.client.clone().unwrap_or_else(|| {
let host = self.server_name.host();
format!("https://{host}")
.as_str()
.try_into()
.expect("server name should be a valid host")
})
}
#[implement(Service)]
pub fn reload(&self, paths: &[PathBuf]) -> Result<Arc<Config>> {
let old = self.server.config.clone();
let new = Config::load(paths).and_then(|raw| Config::new(&raw))?;
fn handle_reload(&self) -> Result {
if self.server.config.config_reload_signal {
#[cfg(all(feature = "systemd", target_os = "linux"))]
sd_notify::notify(&[
sd_notify::NotifyState::Reloading,
sd_notify::NotifyState::monotonic_usec_now()
.expect("Failed to read monotonic time"),
])
.expect("failed to notify systemd of reloading state");
check::reload(&old, &new)?;
self.server.config.update(new)
let config_paths = self.server.config.config_paths.clone().unwrap_or_default();
self.reload(&config_paths)?;
#[cfg(all(feature = "systemd", target_os = "linux"))]
sd_notify::notify(&[sd_notify::NotifyState::Ready])
.expect("failed to notify systemd of ready state");
}
Ok(())
}
pub fn reload(&self, paths: &[PathBuf]) -> Result<Arc<Config>> {
let old = self.server.config.clone();
let new = Config::load(paths).and_then(|raw| Config::new(&raw))?;
check::reload(&old, &new)?;
self.server.config.update(new)
}
}
+196 -164
View File
@@ -1,8 +1,8 @@
use std::{borrow::Cow, fmt::Debug, mem};
use std::{borrow::Cow, fmt::Debug, mem, time::Instant};
use bytes::Bytes;
use conduwuit::{
Err, Error, Result, debug, debug_error, debug_warn, err, implement, trace,
Err, Error, Result, debug, debug_error, debug_info, debug_warn, err, trace,
utils::response::LimitReadExt,
};
use ipaddress::IPAddress;
@@ -20,181 +20,213 @@ use ruma::{
use crate::{SUPPORTED_VERSIONS, resolver::actual::ActualDest};
/// Sends a request to a federation server
#[implement(super::Service)]
#[tracing::instrument(skip_all, name = "request", level = "debug")]
pub async fn execute<'i, T>(&self, dest: &ServerName, request: T) -> Result<T::IncomingResponse>
where
T: OutgoingRequest<
Authentication = ServerSignatures,
PathBuilder: PathBuilder<Input<'i>: FederationPathBuilderInput>,
> + Debug
+ Send,
{
let client = &self.services.client.federation;
self.execute_signed(client, dest, request).await
}
/// Like execute() but with a very large timeout
#[implement(super::Service)]
#[tracing::instrument(skip_all, name = "synapse", level = "debug")]
pub async fn execute_synapse<'i, T>(
&self,
dest: &ServerName,
request: T,
) -> Result<T::IncomingResponse>
where
T: OutgoingRequest<
Authentication = ServerSignatures,
PathBuilder: PathBuilder<Input<'i>: FederationPathBuilderInput>,
> + Debug
+ Send,
{
let client = &self.services.client.synapse;
self.execute_signed(client, dest, request).await
}
#[implement(super::Service)]
pub async fn execute_unauthenticated<'i, T>(
&self,
dest: &ServerName,
request: T,
) -> Result<T::IncomingResponse>
where
T: OutgoingRequest<
Authentication = NoAuthentication,
PathBuilder: PathBuilder<Input<'i>: FederationPathBuilderInput>,
> + Debug
+ Send,
{
let client = &self.services.client.federation;
self.execute_on(client, dest, request, ()).await
}
#[implement(super::Service)]
pub async fn execute_signed<'i, T>(
&self,
client: &Client,
dest: &ServerName,
request: T,
) -> Result<T::IncomingResponse>
where
T: OutgoingRequest<
Authentication = ServerSignatures,
PathBuilder: PathBuilder<Input<'i>: FederationPathBuilderInput>,
> + Send,
{
let authentication = ServerSignaturesInput::new(
self.services.server.name.clone(),
dest.to_owned(),
self.services.server_keys.keypair(),
);
self.execute_on(client, dest, request, authentication).await
}
#[implement(super::Service)]
#[tracing::instrument(name = "fed", level = "info", skip(self, client, request, authentication))]
pub async fn execute_on<'i, T, PathBuilderInput>(
&self,
client: &Client,
dest: &ServerName,
request: T,
authentication: <T::Authentication as AuthScheme>::Input<'_>,
) -> Result<T::IncomingResponse>
where
T: OutgoingRequest<PathBuilder: PathBuilder<Input<'i> = PathBuilderInput>> + Send,
PathBuilderInput: FederationPathBuilderInput,
{
if !self.services.server.config.allow_federation {
return Err!(Config("allow_federation", "Federation is disabled."));
impl super::Service {
/// Sends a signed request to a remote server over federation.
#[tracing::instrument(skip_all, name = "request", level = "debug")]
pub async fn execute<'i, T>(
&self,
dest: &ServerName,
request: T,
) -> Result<T::IncomingResponse>
where
T: OutgoingRequest<
Authentication = ServerSignatures,
PathBuilder: PathBuilder<Input<'i>: FederationPathBuilderInput>,
> + Debug
+ Send,
{
let client = &self.services.client.federation;
self.execute_signed(client, dest, request).await
}
if self.services.moderation.is_remote_server_forbidden(dest) {
return Err!(Request(Forbidden(debug_warn!("Federation with {dest} is not allowed."))));
/// Sends a signed request to a remote server over federation, with a
/// significantly higher timeout to account for slow endpoints.
#[tracing::instrument(skip_all, name = "request_slow", level = "debug")]
pub async fn execute_slow<'i, T>(
&self,
dest: &ServerName,
request: T,
) -> Result<T::IncomingResponse>
where
T: OutgoingRequest<
Authentication = ServerSignatures,
PathBuilder: PathBuilder<Input<'i>: FederationPathBuilderInput>,
> + Debug
+ Send,
{
let client = &self.services.client.federation_slow;
self.execute_signed(client, dest, request).await
}
let actual = self.services.resolver.get_actual_dest(dest).await?;
/// Sends an unsigned (unauthenticated) request to a remote server over
/// federation.
pub async fn execute_unauthenticated<'i, T>(
&self,
dest: &ServerName,
request: T,
) -> Result<T::IncomingResponse>
where
T: OutgoingRequest<
Authentication = NoAuthentication,
PathBuilder: PathBuilder<Input<'i>: FederationPathBuilderInput>,
> + Debug
+ Send,
{
let client = &self.services.client.federation;
let request = Request::try_from(request.try_into_http_request::<Vec<u8>>(
actual.string().as_str(),
authentication,
PathBuilderInput::create(),
)?)?;
self.validate_url(request.url())?;
self.services.server.check_running()?;
self.perform::<T>(dest, &actual, request, client).await
}
#[implement(super::Service)]
async fn perform<T>(
&self,
dest: &ServerName,
actual: &ActualDest,
request: Request,
client: &Client,
) -> Result<T::IncomingResponse>
where
T: OutgoingRequest + Send,
{
let url = request.url().clone();
let method = request.method().clone();
debug!(%method, %url, "Sending request");
match client.execute(request).await {
| Ok(response) =>
self.handle_response::<T>(dest, actual, &method, &url, response)
.await,
| Err(error) =>
Err(handle_error(actual, &method, &url, error).expect_err("always returns error")),
self.execute_on(client, dest, request, ()).await
}
}
#[implement(super::Service)]
fn validate_url(&self, url: &Url) -> Result<()> {
if let Some(url_host) = url.host_str() {
if let Ok(ip) = IPAddress::parse(url_host) {
trace!("Checking request URL IP {ip:?}");
self.services.resolver.validate_ip(&ip)?;
/// Sends a signed request to a remote server over federation, given the
/// specific client.
pub async fn execute_signed<'i, T>(
&self,
client: &Client,
dest: &ServerName,
request: T,
) -> Result<T::IncomingResponse>
where
T: OutgoingRequest<
Authentication = ServerSignatures,
PathBuilder: PathBuilder<Input<'i>: FederationPathBuilderInput>,
> + Send,
{
let authentication = ServerSignaturesInput::new(
self.services.server.name.clone(),
dest.to_owned(),
self.services.server_keys.keypair(),
);
self.execute_on(client, dest, request, authentication).await
}
/// Prepares and sends a federation request with the given client.
/// If federation is disabled, an error is returned here. Likewise, if the
/// destination is forbidden.
/// Preparation includes resolving the server before sending the request.
#[tracing::instrument(
name = "fed",
level = "info",
skip(self, client, request, authentication)
)]
pub async fn execute_on<'i, T, PathBuilderInput>(
&self,
client: &Client,
dest: &ServerName,
request: T,
authentication: <T::Authentication as AuthScheme>::Input<'_>,
) -> Result<T::IncomingResponse>
where
T: OutgoingRequest<PathBuilder: PathBuilder<Input<'i> = PathBuilderInput>> + Send,
PathBuilderInput: FederationPathBuilderInput,
{
if !self.services.server.config.allow_federation {
return Err!(Config("allow_federation", "Federation is disabled."));
}
if self.services.moderation.is_remote_server_forbidden(dest) {
return Err!(Request(Forbidden(debug_warn!(
"Federation with {dest} is not allowed."
))));
}
let actual = self.services.resolver.get_actual_dest(dest).await?;
let request = Request::try_from(request.try_into_http_request::<Vec<u8>>(
actual.string().as_str(),
authentication,
PathBuilderInput::create(),
)?)?;
self.validate_url(request.url())?;
self.services.server.check_running()?;
self.perform::<T>(dest, &actual, request, client).await
}
/// Actually sends the federation request, handling the response before
/// returning.
///
/// Timing info and request logs are available when debug logs are enabled.
async fn perform<T>(
&self,
dest: &ServerName,
actual: &ActualDest,
request: Request,
client: &Client,
) -> Result<T::IncomingResponse>
where
T: OutgoingRequest + Send,
{
let url = request.url().clone();
let method = request.method().clone();
debug_info!(
"Sending request: {} matrix-federation://{dest}{}",
method.as_str(),
url.path(),
);
let start = Instant::now();
let response = client.execute(request).await;
debug_info!(
"Request finished in {:?}: {} matrix-federation://{dest}{}",
start.elapsed(),
method.as_str(),
url.path(),
);
match response {
| Ok(response) =>
self.handle_response::<T>(dest, actual, &method, &url, response)
.await,
| Err(error) =>
Err(handle_error(actual, &method, &url, error).expect_err("always returns error")),
}
}
Ok(())
}
fn validate_url(&self, url: &Url) -> Result<()> {
if let Some(url_host) = url.host_str() {
if let Ok(ip) = IPAddress::parse(url_host) {
trace!("Checking request URL IP {ip:?}");
self.services.resolver.validate_ip(&ip)?;
}
}
#[implement(super::Service)]
async fn handle_response<T>(
&self,
dest: &ServerName,
actual: &ActualDest,
method: &Method,
url: &Url,
response: Response,
) -> Result<T::IncomingResponse>
where
T: OutgoingRequest + Send,
{
const HUGE_ENDPOINTS: [&str; 2] =
["/_matrix/federation/v2/send_join/", "/_matrix/federation/v2/state/"];
let size_limit: u64 = if HUGE_ENDPOINTS.iter().any(|e| url.path().starts_with(e)) {
// Some federation endpoints can return huge response bodies, so we'll bump the
// limit for those endpoints specifically.
self.services
.server
.config
.max_request_size
.saturating_mul(10)
} else {
self.services.server.config.max_request_size
Ok(())
}
.try_into()
.expect("size_limit (usize) should fit within a u64");
let response = into_http_response(dest, actual, method, url, response, size_limit).await?;
T::IncomingResponse::try_from_http_response(response)
.map_err(|e| err!(BadServerResponse("Server returned bad 200 response: {e:?}")))
/// Handles a successful response from a remote server, reading and
/// deserializing the response body (if any).
async fn handle_response<T>(
&self,
dest: &ServerName,
actual: &ActualDest,
method: &Method,
url: &Url,
response: Response,
) -> Result<T::IncomingResponse>
where
T: OutgoingRequest + Send,
{
const HUGE_ENDPOINTS: [&str; 2] =
["/_matrix/federation/v2/send_join/", "/_matrix/federation/v2/state/"];
let size_limit: u64 = if HUGE_ENDPOINTS.iter().any(|e| url.path().starts_with(e)) {
// Some federation endpoints can return huge response bodies, so we'll bump the
// limit for those endpoints specifically.
self.services
.server
.config
.max_request_size
.saturating_mul(10)
} else {
self.services.server.config.max_request_size
}
.try_into()
.expect("size_limit (usize) should fit within a u64");
let response =
into_http_response(dest, actual, method, url, response, size_limit).await?;
T::IncomingResponse::try_from_http_response(response)
.map_err(|e| err!(BadServerResponse("Server returned bad 200 response: {e:?}")))
}
}
async fn into_http_response(
+231 -240
View File
@@ -1,7 +1,7 @@
use std::{collections::BTreeMap, sync::Arc};
use conduwuit::{
Err, Result, err, implement,
Err, Result, err,
utils::stream::{ReadyExt, TryIgnore},
};
use database::{Deserialized, Ignore, Interfix, Json, Map};
@@ -46,253 +46,244 @@ impl crate::Service for Service {
fn name(&self) -> &str { crate::service::make_name(std::module_path!()) }
}
#[implement(Service)]
pub fn create_backup(
&self,
user_id: &UserId,
backup_metadata: &Raw<BackupAlgorithm>,
) -> Result<String> {
let version = self.services.globals.next_count()?.to_string();
let count = self.services.globals.next_count()?;
impl Service {
pub fn create_backup(
&self,
user_id: &UserId,
backup_metadata: &Raw<BackupAlgorithm>,
) -> Result<String> {
let version = self.services.globals.next_count()?.to_string();
let count = self.services.globals.next_count()?;
let key = (user_id, &version);
self.db.backupid_algorithm.put(key, Json(backup_metadata));
let key = (user_id, &version);
self.db.backupid_algorithm.put(key, Json(backup_metadata));
self.db.backupid_etag.put(key, count);
self.db.backupid_etag.put(key, count);
Ok(version)
}
#[implement(Service)]
pub async fn delete_backup(&self, user_id: &UserId, version: &str) {
let key = (user_id, version);
self.db.backupid_algorithm.del(key);
self.db.backupid_etag.del(key);
let key = (user_id, version, Interfix);
self.db
.backupkeyid_backup
.keys_prefix_raw(&key)
.ignore_err()
.ready_for_each(|outdated_key| {
self.db.backupkeyid_backup.remove(outdated_key);
})
.await;
}
#[implement(Service)]
pub async fn update_backup<'a>(
&self,
user_id: &UserId,
version: &'a str,
backup_metadata: &Raw<BackupAlgorithm>,
) -> Result<&'a str> {
let key = (user_id, version);
if self.db.backupid_algorithm.qry(&key).await.is_err() {
return Err!(Request(NotFound("Tried to update nonexistent backup.")));
Ok(version)
}
let count = self.services.globals.next_count().unwrap();
self.db.backupid_etag.put(key, count);
self.db
.backupid_algorithm
.put_raw(key, backup_metadata.json().get());
pub async fn delete_backup(&self, user_id: &UserId, version: &str) {
let key = (user_id, version);
self.db.backupid_algorithm.del(key);
self.db.backupid_etag.del(key);
Ok(version)
}
#[implement(Service)]
pub async fn get_latest_backup_version(&self, user_id: &UserId) -> Result<String> {
type Key<'a> = (OwnedUserId, &'a str);
let last_possible_key = (user_id, u64::MAX);
self.db
.backupid_algorithm
.rev_keys_from(&last_possible_key)
.ignore_err()
.ready_take_while(|(user_id_, _): &Key<'_>| *user_id_ == user_id)
.map(|(_, version): Key<'_>| version.to_owned())
.next()
.await
.ok_or_else(|| err!(Request(NotFound("No backup versions found"))))
}
#[implement(Service)]
pub async fn get_latest_backup(
&self,
user_id: &UserId,
) -> Result<(String, Raw<BackupAlgorithm>)> {
type Key<'a> = (OwnedUserId, &'a str);
type KeyVal<'a> = (Key<'a>, Raw<BackupAlgorithm>);
let last_possible_key = (user_id, u64::MAX);
self.db
.backupid_algorithm
.rev_stream_from(&last_possible_key)
.ignore_err()
.ready_take_while(|((user_id_, _), _): &KeyVal<'_>| *user_id_ == user_id)
.map(|((_, version), algorithm): KeyVal<'_>| (version.to_owned(), algorithm))
.next()
.await
.ok_or_else(|| err!(Request(NotFound("No backup found"))))
}
#[implement(Service)]
pub async fn get_backup(&self, user_id: &UserId, version: &str) -> Result<Raw<BackupAlgorithm>> {
let key = (user_id, version);
self.db.backupid_algorithm.qry(&key).await.deserialized()
}
#[implement(Service)]
pub async fn add_key(
&self,
user_id: &UserId,
version: &str,
room_id: &RoomId,
session_id: &str,
key_data: &Raw<KeyBackupData>,
) -> Result<()> {
let key = (user_id, version);
if self.db.backupid_algorithm.qry(&key).await.is_err() {
return Err!(Request(NotFound("Tried to update nonexistent backup.")));
let key = (user_id, version, Interfix);
self.db
.backupkeyid_backup
.keys_prefix_raw(&key)
.ignore_err()
.ready_for_each(|outdated_key| {
self.db.backupkeyid_backup.remove(outdated_key);
})
.await;
}
let count = self.services.globals.next_count().unwrap();
self.db.backupid_etag.put(key, count);
pub async fn update_backup<'a>(
&self,
user_id: &UserId,
version: &'a str,
backup_metadata: &Raw<BackupAlgorithm>,
) -> Result<&'a str> {
let key = (user_id, version);
if self.db.backupid_algorithm.qry(&key).await.is_err() {
return Err!(Request(NotFound("Tried to update nonexistent backup.")));
}
let key = (user_id, version, room_id, session_id);
self.db
.backupkeyid_backup
.put_raw(key, key_data.json().get());
let count = self.services.globals.next_count().unwrap();
self.db.backupid_etag.put(key, count);
self.db
.backupid_algorithm
.put_raw(key, backup_metadata.json().get());
Ok(())
}
#[implement(Service)]
pub async fn count_keys(&self, user_id: &UserId, version: &str) -> usize {
let prefix = (user_id, version);
self.db
.backupkeyid_backup
.keys_prefix_raw(&prefix)
.count()
.await
}
#[implement(Service)]
pub async fn get_etag(&self, user_id: &UserId, version: &str) -> String {
let key = (user_id, version);
self.db
.backupid_etag
.qry(&key)
.await
.deserialized::<u64>()
.as_ref()
.map(ToString::to_string)
.expect("Backup has no etag.")
}
#[implement(Service)]
pub async fn get_all(
&self,
user_id: &UserId,
version: &str,
) -> BTreeMap<OwnedRoomId, RoomKeyBackup> {
type Key<'a> = (Ignore, Ignore, OwnedRoomId, &'a str);
type KeyVal<'a> = (Key<'a>, Raw<KeyBackupData>);
let mut rooms = BTreeMap::<OwnedRoomId, RoomKeyBackup>::new();
let default = || RoomKeyBackup::new(BTreeMap::new());
let prefix = (user_id, version, Interfix);
self.db
.backupkeyid_backup
.stream_prefix(&prefix)
.ignore_err()
.ready_for_each(|((_, _, room_id, session_id), key_backup_data): KeyVal<'_>| {
rooms
.entry(room_id)
.or_insert_with(default)
.sessions
.insert(session_id.into(), key_backup_data);
})
.await;
rooms
}
#[implement(Service)]
pub async fn get_room(
&self,
user_id: &UserId,
version: &str,
room_id: &RoomId,
) -> BTreeMap<String, Raw<KeyBackupData>> {
type KeyVal<'a> = ((Ignore, Ignore, Ignore, &'a str), Raw<KeyBackupData>);
let prefix = (user_id, version, room_id, Interfix);
self.db
.backupkeyid_backup
.stream_prefix(&prefix)
.ignore_err()
.map(|((.., session_id), key_backup_data): KeyVal<'_>| {
(session_id.to_owned(), key_backup_data)
})
.collect()
.await
}
#[implement(Service)]
pub async fn get_session(
&self,
user_id: &UserId,
version: &str,
room_id: &RoomId,
session_id: &str,
) -> Result<Raw<KeyBackupData>> {
let key = (user_id, version, room_id, session_id);
self.db.backupkeyid_backup.qry(&key).await.deserialized()
}
#[implement(Service)]
pub async fn delete_all_keys(&self, user_id: &UserId, version: &str) {
let key = (user_id, version, Interfix);
self.db
.backupkeyid_backup
.keys_prefix_raw(&key)
.ignore_err()
.ready_for_each(|outdated_key| self.db.backupkeyid_backup.remove(outdated_key))
.await;
}
#[implement(Service)]
pub async fn delete_room_keys(&self, user_id: &UserId, version: &str, room_id: &RoomId) {
let key = (user_id, version, room_id, Interfix);
self.db
.backupkeyid_backup
.keys_prefix_raw(&key)
.ignore_err()
.ready_for_each(|outdated_key| {
self.db.backupkeyid_backup.remove(outdated_key);
})
.await;
}
#[implement(Service)]
pub async fn delete_room_key(
&self,
user_id: &UserId,
version: &str,
room_id: &RoomId,
session_id: &str,
) {
let key = (user_id, version, room_id, session_id);
self.db
.backupkeyid_backup
.keys_prefix_raw(&key)
.ignore_err()
.ready_for_each(|outdated_key| {
self.db.backupkeyid_backup.remove(outdated_key);
})
.await;
Ok(version)
}
pub async fn get_latest_backup_version(&self, user_id: &UserId) -> Result<String> {
type Key<'a> = (OwnedUserId, &'a str);
let last_possible_key = (user_id, u64::MAX);
self.db
.backupid_algorithm
.rev_keys_from(&last_possible_key)
.ignore_err()
.ready_take_while(|(user_id_, _): &Key<'_>| *user_id_ == user_id)
.map(|(_, version): Key<'_>| version.to_owned())
.next()
.await
.ok_or_else(|| err!(Request(NotFound("No backup versions found"))))
}
pub async fn get_latest_backup(
&self,
user_id: &UserId,
) -> Result<(String, Raw<BackupAlgorithm>)> {
type Key<'a> = (OwnedUserId, &'a str);
type KeyVal<'a> = (Key<'a>, Raw<BackupAlgorithm>);
let last_possible_key = (user_id, u64::MAX);
self.db
.backupid_algorithm
.rev_stream_from(&last_possible_key)
.ignore_err()
.ready_take_while(|((user_id_, _), _): &KeyVal<'_>| *user_id_ == user_id)
.map(|((_, version), algorithm): KeyVal<'_>| (version.to_owned(), algorithm))
.next()
.await
.ok_or_else(|| err!(Request(NotFound("No backup found"))))
}
pub async fn get_backup(
&self,
user_id: &UserId,
version: &str,
) -> Result<Raw<BackupAlgorithm>> {
let key = (user_id, version);
self.db.backupid_algorithm.qry(&key).await.deserialized()
}
pub async fn add_key(
&self,
user_id: &UserId,
version: &str,
room_id: &RoomId,
session_id: &str,
key_data: &Raw<KeyBackupData>,
) -> Result<()> {
let key = (user_id, version);
if self.db.backupid_algorithm.qry(&key).await.is_err() {
return Err!(Request(NotFound("Tried to update nonexistent backup.")));
}
let count = self.services.globals.next_count().unwrap();
self.db.backupid_etag.put(key, count);
let key = (user_id, version, room_id, session_id);
self.db
.backupkeyid_backup
.put_raw(key, key_data.json().get());
Ok(())
}
pub async fn count_keys(&self, user_id: &UserId, version: &str) -> usize {
let prefix = (user_id, version);
self.db
.backupkeyid_backup
.keys_prefix_raw(&prefix)
.count()
.await
}
pub async fn get_etag(&self, user_id: &UserId, version: &str) -> String {
let key = (user_id, version);
self.db
.backupid_etag
.qry(&key)
.await
.deserialized::<u64>()
.as_ref()
.map(ToString::to_string)
.expect("Backup has no etag.")
}
pub async fn get_all(
&self,
user_id: &UserId,
version: &str,
) -> BTreeMap<OwnedRoomId, RoomKeyBackup> {
type Key<'a> = (Ignore, Ignore, OwnedRoomId, &'a str);
type KeyVal<'a> = (Key<'a>, Raw<KeyBackupData>);
let mut rooms = BTreeMap::<OwnedRoomId, RoomKeyBackup>::new();
let default = || RoomKeyBackup::new(BTreeMap::new());
let prefix = (user_id, version, Interfix);
self.db
.backupkeyid_backup
.stream_prefix(&prefix)
.ignore_err()
.ready_for_each(|((_, _, room_id, session_id), key_backup_data): KeyVal<'_>| {
rooms
.entry(room_id)
.or_insert_with(default)
.sessions
.insert(session_id.into(), key_backup_data);
})
.await;
rooms
}
pub async fn get_room(
&self,
user_id: &UserId,
version: &str,
room_id: &RoomId,
) -> BTreeMap<String, Raw<KeyBackupData>> {
type KeyVal<'a> = ((Ignore, Ignore, Ignore, &'a str), Raw<KeyBackupData>);
let prefix = (user_id, version, room_id, Interfix);
self.db
.backupkeyid_backup
.stream_prefix(&prefix)
.ignore_err()
.map(|((.., session_id), key_backup_data): KeyVal<'_>| {
(session_id.to_owned(), key_backup_data)
})
.collect()
.await
}
pub async fn get_session(
&self,
user_id: &UserId,
version: &str,
room_id: &RoomId,
session_id: &str,
) -> Result<Raw<KeyBackupData>> {
let key = (user_id, version, room_id, session_id);
self.db.backupkeyid_backup.qry(&key).await.deserialized()
}
pub async fn delete_all_keys(&self, user_id: &UserId, version: &str) {
let key = (user_id, version, Interfix);
self.db
.backupkeyid_backup
.keys_prefix_raw(&key)
.ignore_err()
.ready_for_each(|outdated_key| self.db.backupkeyid_backup.remove(outdated_key))
.await;
}
pub async fn delete_room_keys(&self, user_id: &UserId, version: &str, room_id: &RoomId) {
let key = (user_id, version, room_id, Interfix);
self.db
.backupkeyid_backup
.keys_prefix_raw(&key)
.ignore_err()
.ready_for_each(|outdated_key| {
self.db.backupkeyid_backup.remove(outdated_key);
})
.await;
}
pub async fn delete_room_key(
&self,
user_id: &UserId,
version: &str,
room_id: &RoomId,
session_id: &str,
) {
let key = (user_id, version, room_id, session_id);
self.db
.backupkeyid_backup
.keys_prefix_raw(&key)
.ignore_err()
.ready_for_each(|outdated_key| {
self.db.backupkeyid_backup.remove(outdated_key);
})
.await;
}
}
+378 -384
View File
@@ -10,7 +10,6 @@ use std::time::SystemTime;
#[cfg(feature = "url_preview")]
use conduwuit::utils::response::LimitReadExt;
use conduwuit::{Err, Result, debug, err};
use conduwuit_core::implement;
use ipaddress::IPAddress;
#[cfg(feature = "url_preview")]
use ruma::OwnedMxcUri;
@@ -49,407 +48,402 @@ pub struct UrlPreviewData {
pub audio_size: Option<usize>,
}
#[implement(Service)]
pub async fn remove_url_preview(&self, url: &str) -> Result<()> {
// TODO: also remove the downloaded image
self.db.remove_url_preview(url)
}
#[implement(Service)]
pub async fn clear_url_previews(&self) { self.db.clear_url_previews().await; }
#[implement(Service)]
pub async fn set_url_preview(&self, url: &str, data: &UrlPreviewData) -> Result<()> {
let now = SystemTime::now()
.duration_since(SystemTime::UNIX_EPOCH)
.expect("valid system time");
self.db.set_url_preview(url, data, now)
}
#[implement(Service)]
pub async fn get_url_preview(&self, url: &Url) -> Result<UrlPreviewData> {
if let Ok(preview) = self.db.get_url_preview(url.as_str()).await {
return Ok(preview);
impl Service {
pub async fn remove_url_preview(&self, url: &str) -> Result<()> {
// TODO: also remove the downloaded image
self.db.remove_url_preview(url)
}
// ensure that only one request is made per URL
let _request_lock = self.url_preview_mutex.lock(url.as_str()).await;
pub async fn clear_url_previews(&self) { self.db.clear_url_previews().await; }
match self.db.get_url_preview(url.as_str()).await {
| Ok(preview) => Ok(preview),
| Err(_) => self.request_url_preview(url).await,
pub async fn set_url_preview(&self, url: &str, data: &UrlPreviewData) -> Result<()> {
let now = SystemTime::now()
.duration_since(SystemTime::UNIX_EPOCH)
.expect("valid system time");
self.db.set_url_preview(url, data, now)
}
}
#[implement(Service)]
async fn request_url_preview(&self, url: &Url) -> Result<UrlPreviewData> {
if let Ok(ip) = IPAddress::parse(url.host_str().expect("URL previously validated")) {
if !self.services.client.valid_cidr_range(&ip) {
return Err!(Request(Forbidden("Requesting from this address is forbidden")));
pub async fn get_url_preview(&self, url: &Url) -> Result<UrlPreviewData> {
if let Ok(preview) = self.db.get_url_preview(url.as_str()).await {
return Ok(preview);
}
// ensure that only one request is made per URL
let _request_lock = self.url_preview_mutex.lock(url.as_str()).await;
match self.db.get_url_preview(url.as_str()).await {
| Ok(preview) => Ok(preview),
| Err(_) => self.request_url_preview(url).await,
}
}
let client = &self.services.client.url_preview;
let response = client.head(url.as_str()).send().await?;
debug!(%url, "URL preview response headers: {:?}", response.headers());
if let Some(remote_addr) = response.remote_addr() {
debug!(%url, "URL preview response remote address: {:?}", remote_addr);
if let Ok(ip) = IPAddress::parse(remote_addr.ip().to_string()) {
async fn request_url_preview(&self, url: &Url) -> Result<UrlPreviewData> {
if let Ok(ip) = IPAddress::parse(url.host_str().expect("URL previously validated")) {
if !self.services.client.valid_cidr_range(&ip) {
return Err!(Request(Forbidden("Requesting from this address is forbidden")));
}
}
}
let Some(content_type) = response.headers().get(reqwest::header::CONTENT_TYPE) else {
return Err!(Request(Unknown("Unknown or invalid Content-Type header")));
};
let client = &self.services.client.url_preview;
let response = client.head(url.as_str()).send().await?;
let content_type = content_type
.to_str()
.map_err(|e| err!(Request(Unknown("Unknown or invalid Content-Type header: {e}"))))?;
debug!(%url, "URL preview response headers: {:?}", response.headers());
let data = match content_type {
| html if html.starts_with("text/html") => self.download_html(url.as_str()).await?,
| img if img.starts_with("image/") => self.download_image(url.as_str(), None).await?,
| video if video.starts_with("video/") => self.download_video(url.as_str(), None).await?,
| audio if audio.starts_with("audio/") => self.download_audio(url.as_str(), None).await?,
| _ => return Err!(Request(Unknown("Unsupported Content-Type"))),
};
if let Some(remote_addr) = response.remote_addr() {
debug!(%url, "URL preview response remote address: {:?}", remote_addr);
self.set_url_preview(url.as_str(), &data).await?;
Ok(data)
}
#[cfg(feature = "url_preview")]
#[implement(Service)]
pub async fn download_image(
&self,
url: &str,
preview_data: Option<UrlPreviewData>,
) -> Result<UrlPreviewData> {
use conduwuit::utils::random_string;
use image::ImageReader;
use crate::media::mxc::Mxc;
let mut preview_data = preview_data.unwrap_or_default();
let image = self
.services
.client
.url_preview
.get(url)
.send()
.await?
.limit_read(
self.services
.server
.config
.max_request_size
.try_into()
.expect("u64 should fit in usize"),
)
.await?;
let mxc = Mxc {
server_name: self.services.globals.server_name(),
media_id: &random_string(super::MXC_LENGTH),
};
self.create(&mxc, None, None, None, &image).await?;
preview_data.image = Some(mxc.to_string());
if preview_data.image_height.is_none() || preview_data.image_width.is_none() {
let cursor = std::io::Cursor::new(&image);
let (width, height) = match ImageReader::new(cursor).with_guessed_format() {
| Err(_) => (None, None),
| Ok(reader) => match reader.into_dimensions() {
| Err(_) => (None, None),
| Ok((width, height)) => (Some(width), Some(height)),
},
};
preview_data.image_width = width;
preview_data.image_height = height;
}
Ok(preview_data)
}
#[cfg(feature = "url_preview")]
#[implement(Service)]
pub async fn download_video(
&self,
url: &str,
preview_data: Option<UrlPreviewData>,
) -> Result<UrlPreviewData> {
let mut preview_data = preview_data.unwrap_or_default();
if self.services.globals.url_preview_allow_audio_video() {
let (url, size) = self.download_media(url).await?;
preview_data.video = Some(url.to_string());
preview_data.video_size = Some(size);
}
Ok(preview_data)
}
#[cfg(feature = "url_preview")]
#[implement(Service)]
pub async fn download_audio(
&self,
url: &str,
preview_data: Option<UrlPreviewData>,
) -> Result<UrlPreviewData> {
let mut preview_data = preview_data.unwrap_or_default();
if self.services.globals.url_preview_allow_audio_video() {
let (url, size) = self.download_media(url).await?;
preview_data.audio = Some(url.to_string());
preview_data.audio_size = Some(size);
}
Ok(preview_data)
}
#[cfg(feature = "url_preview")]
#[implement(Service)]
pub async fn download_media(&self, url: &str) -> Result<(OwnedMxcUri, usize)> {
use conduwuit::utils::random_string;
use http::header::CONTENT_TYPE;
let response = self.services.client.url_preview.get(url).send().await?;
let content_type = response.headers().get(CONTENT_TYPE).cloned();
let media = response
.limit_read(
self.services
.server
.config
.max_request_size
.try_into()
.expect("u64 should fit in usize"),
)
.await?;
let mxc = Mxc {
server_name: self.services.globals.server_name(),
media_id: &random_string(super::MXC_LENGTH),
};
let content_type = content_type.and_then(|v| v.to_str().map(ToOwned::to_owned).ok());
self.create(&mxc, None, None, content_type.as_deref(), &media)
.await?;
Ok((OwnedMxcUri::from(mxc.to_string()), media.len()))
}
#[cfg(not(feature = "url_preview"))]
#[implement(Service)]
pub async fn download_image(
&self,
_url: &str,
_preview_data: Option<UrlPreviewData>,
) -> Result<UrlPreviewData> {
Err!(FeatureDisabled("url_preview"))
}
#[cfg(not(feature = "url_preview"))]
#[implement(Service)]
pub async fn download_video(
&self,
_url: &str,
_preview_data: Option<UrlPreviewData>,
) -> Result<UrlPreviewData> {
Err!(FeatureDisabled("url_preview"))
}
#[cfg(not(feature = "url_preview"))]
#[implement(Service)]
pub async fn download_audio(
&self,
_url: &str,
_preview_data: Option<UrlPreviewData>,
) -> Result<UrlPreviewData> {
Err!(FeatureDisabled("url_preview"))
}
#[cfg(not(feature = "url_preview"))]
#[implement(Service)]
pub async fn download_media(&self, _url: &str) -> Result<UrlPreviewData> {
Err!(FeatureDisabled("url_preview"))
}
#[cfg(feature = "url_preview")]
#[implement(Service)]
async fn download_html(&self, url: &str) -> Result<UrlPreviewData> {
use webpage::HTML;
let client = &self.services.client.url_preview;
let body = client
.get(url)
.send()
.await?
.limit_read_text(
self.services
.server
.config
.max_request_size
.try_into()
.expect("u64 should fit in usize"),
)
.await?;
let Ok(html) = HTML::from_string(body.clone(), Some(url.to_owned())) else {
return Err!(Request(Unknown("Failed to parse HTML")));
};
let mut preview_data = UrlPreviewData::default();
if let Some(obj) = html.opengraph.images.first() {
preview_data = self.download_image(&obj.url, Some(preview_data)).await?;
}
if let Some(obj) = html.opengraph.videos.first() {
preview_data = self.download_video(&obj.url, Some(preview_data)).await?;
preview_data.video_width = obj.properties.get("width").and_then(|v| v.parse().ok());
preview_data.video_height = obj.properties.get("height").and_then(|v| v.parse().ok());
}
if let Some(obj) = html.opengraph.audios.first() {
preview_data = self.download_audio(&obj.url, Some(preview_data)).await?;
}
let props = html.opengraph.properties;
/* use OpenGraph title/description, but fall back to HTML if not available */
preview_data.title = props.get("title").cloned().or(html.title);
preview_data.description = props.get("description").cloned().or(html.description);
Ok(preview_data)
}
#[cfg(not(feature = "url_preview"))]
#[implement(Service)]
async fn download_html(&self, _url: &str) -> Result<UrlPreviewData> {
Err!(FeatureDisabled("url_preview"))
}
#[implement(Service)]
pub fn url_preview_allowed(&self, url: &Url) -> bool {
if ["http", "https"]
.iter()
.all(|&scheme| scheme != url.scheme().to_lowercase())
{
debug!("Ignoring non-HTTP/HTTPS URL to preview: {}", url);
return false;
}
let host = match url.host_str() {
| None => {
debug!("Ignoring URL preview for a URL that does not have a host (?): {}", url);
return false;
},
| Some(h) => h.to_owned(),
};
let allowlist_domain_contains = self
.services
.globals
.url_preview_domain_contains_allowlist();
let allowlist_domain_explicit = self
.services
.globals
.url_preview_domain_explicit_allowlist();
let denylist_domain_explicit = self.services.globals.url_preview_domain_explicit_denylist();
let allowlist_url_contains = self.services.globals.url_preview_url_contains_allowlist();
if allowlist_domain_contains.contains(&"*".to_owned())
|| allowlist_domain_explicit.contains(&"*".to_owned())
|| allowlist_url_contains.contains(&"*".to_owned())
{
debug!("Config key contains * which is allowing all URL previews. Allowing URL {}", url);
return true;
}
if !host.is_empty() {
if denylist_domain_explicit.contains(&host) {
debug!(
"Host {} is not allowed by url_preview_domain_explicit_denylist (check 1/4)",
&host
);
return false;
}
if allowlist_domain_explicit.contains(&host) {
debug!(
"Host {} is allowed by url_preview_domain_explicit_allowlist (check 2/4)",
&host
);
return true;
}
if allowlist_domain_contains
.iter()
.any(|domain_s| domain_s.contains(&host.clone()))
{
debug!(
"Host {} is allowed by url_preview_domain_contains_allowlist (check 3/4)",
&host
);
return true;
}
if allowlist_url_contains
.iter()
.any(|url_s| url.to_string().contains(url_s))
{
debug!("URL {} is allowed by url_preview_url_contains_allowlist (check 4/4)", &host);
return true;
}
// check root domain if available and if user has root domain checks
if self.services.globals.url_preview_check_root_domain() {
debug!("Checking root domain");
match host.split_once('.') {
| None => return false,
| Some((_, root_domain)) => {
if denylist_domain_explicit.contains(&root_domain.to_owned()) {
debug!(
"Root domain {} is not allowed by \
url_preview_domain_explicit_denylist (check 1/3)",
&root_domain
);
return false;
}
if allowlist_domain_explicit.contains(&root_domain.to_owned()) {
debug!(
"Root domain {} is allowed by url_preview_domain_explicit_allowlist \
(check 2/3)",
&root_domain
);
return true;
}
if allowlist_domain_contains
.iter()
.any(|domain_s| domain_s.contains(&root_domain.to_owned()))
{
debug!(
"Root domain {} is allowed by url_preview_domain_contains_allowlist \
(check 3/3)",
&root_domain
);
return true;
}
},
if let Ok(ip) = IPAddress::parse(remote_addr.ip().to_string()) {
if !self.services.client.valid_cidr_range(&ip) {
return Err!(Request(Forbidden("Requesting from this address is forbidden")));
}
}
}
let Some(content_type) = response.headers().get(reqwest::header::CONTENT_TYPE) else {
return Err!(Request(Unknown("Unknown or invalid Content-Type header")));
};
let content_type = content_type
.to_str()
.map_err(|e| err!(Request(Unknown("Unknown or invalid Content-Type header: {e}"))))?;
let data = match content_type {
| html if html.starts_with("text/html") => self.download_html(url.as_str()).await?,
| img if img.starts_with("image/") => self.download_image(url.as_str(), None).await?,
| video if video.starts_with("video/") =>
self.download_video(url.as_str(), None).await?,
| audio if audio.starts_with("audio/") =>
self.download_audio(url.as_str(), None).await?,
| _ => return Err!(Request(Unknown("Unsupported Content-Type"))),
};
self.set_url_preview(url.as_str(), &data).await?;
Ok(data)
}
false
#[cfg(feature = "url_preview")]
pub async fn download_image(
&self,
url: &str,
preview_data: Option<UrlPreviewData>,
) -> Result<UrlPreviewData> {
use conduwuit::utils::random_string;
use image::ImageReader;
use crate::media::mxc::Mxc;
let mut preview_data = preview_data.unwrap_or_default();
let image = self
.services
.client
.url_preview
.get(url)
.send()
.await?
.limit_read(
self.services
.server
.config
.max_request_size
.try_into()
.expect("u64 should fit in usize"),
)
.await?;
let mxc = Mxc {
server_name: self.services.globals.server_name(),
media_id: &random_string(super::MXC_LENGTH),
};
self.create(&mxc, None, None, None, &image).await?;
preview_data.image = Some(mxc.to_string());
if preview_data.image_height.is_none() || preview_data.image_width.is_none() {
let cursor = std::io::Cursor::new(&image);
let (width, height) = match ImageReader::new(cursor).with_guessed_format() {
| Err(_) => (None, None),
| Ok(reader) => match reader.into_dimensions() {
| Err(_) => (None, None),
| Ok((width, height)) => (Some(width), Some(height)),
},
};
preview_data.image_width = width;
preview_data.image_height = height;
}
Ok(preview_data)
}
#[cfg(feature = "url_preview")]
pub async fn download_video(
&self,
url: &str,
preview_data: Option<UrlPreviewData>,
) -> Result<UrlPreviewData> {
let mut preview_data = preview_data.unwrap_or_default();
if self.services.globals.url_preview_allow_audio_video() {
let (url, size) = self.download_media(url).await?;
preview_data.video = Some(url.to_string());
preview_data.video_size = Some(size);
}
Ok(preview_data)
}
#[cfg(feature = "url_preview")]
pub async fn download_audio(
&self,
url: &str,
preview_data: Option<UrlPreviewData>,
) -> Result<UrlPreviewData> {
let mut preview_data = preview_data.unwrap_or_default();
if self.services.globals.url_preview_allow_audio_video() {
let (url, size) = self.download_media(url).await?;
preview_data.audio = Some(url.to_string());
preview_data.audio_size = Some(size);
}
Ok(preview_data)
}
#[cfg(feature = "url_preview")]
pub async fn download_media(&self, url: &str) -> Result<(OwnedMxcUri, usize)> {
use conduwuit::utils::random_string;
use http::header::CONTENT_TYPE;
let response = self.services.client.url_preview.get(url).send().await?;
let content_type = response.headers().get(CONTENT_TYPE).cloned();
let media = response
.limit_read(
self.services
.server
.config
.max_request_size
.try_into()
.expect("u64 should fit in usize"),
)
.await?;
let mxc = Mxc {
server_name: self.services.globals.server_name(),
media_id: &random_string(super::MXC_LENGTH),
};
let content_type = content_type.and_then(|v| v.to_str().map(ToOwned::to_owned).ok());
self.create(&mxc, None, None, content_type.as_deref(), &media)
.await?;
Ok((OwnedMxcUri::from(mxc.to_string()), media.len()))
}
#[cfg(not(feature = "url_preview"))]
pub async fn download_image(
&self,
_url: &str,
_preview_data: Option<UrlPreviewData>,
) -> Result<UrlPreviewData> {
Err!(FeatureDisabled("url_preview"))
}
#[cfg(not(feature = "url_preview"))]
pub async fn download_video(
&self,
_url: &str,
_preview_data: Option<UrlPreviewData>,
) -> Result<UrlPreviewData> {
Err!(FeatureDisabled("url_preview"))
}
#[cfg(not(feature = "url_preview"))]
pub async fn download_audio(
&self,
_url: &str,
_preview_data: Option<UrlPreviewData>,
) -> Result<UrlPreviewData> {
Err!(FeatureDisabled("url_preview"))
}
#[cfg(not(feature = "url_preview"))]
pub async fn download_media(&self, _url: &str) -> Result<UrlPreviewData> {
Err!(FeatureDisabled("url_preview"))
}
#[cfg(feature = "url_preview")]
async fn download_html(&self, url: &str) -> Result<UrlPreviewData> {
use webpage::HTML;
let client = &self.services.client.url_preview;
let body = client
.get(url)
.send()
.await?
.limit_read_text(
self.services
.server
.config
.max_request_size
.try_into()
.expect("u64 should fit in usize"),
)
.await?;
let Ok(html) = HTML::from_string(body.clone(), Some(url.to_owned())) else {
return Err!(Request(Unknown("Failed to parse HTML")));
};
let mut preview_data = UrlPreviewData::default();
if let Some(obj) = html.opengraph.images.first() {
preview_data = self.download_image(&obj.url, Some(preview_data)).await?;
}
if let Some(obj) = html.opengraph.videos.first() {
preview_data = self.download_video(&obj.url, Some(preview_data)).await?;
preview_data.video_width = obj.properties.get("width").and_then(|v| v.parse().ok());
preview_data.video_height = obj.properties.get("height").and_then(|v| v.parse().ok());
}
if let Some(obj) = html.opengraph.audios.first() {
preview_data = self.download_audio(&obj.url, Some(preview_data)).await?;
}
let props = html.opengraph.properties;
/* use OpenGraph title/description, but fall back to HTML if not available */
preview_data.title = props.get("title").cloned().or(html.title);
preview_data.description = props.get("description").cloned().or(html.description);
Ok(preview_data)
}
#[cfg(not(feature = "url_preview"))]
async fn download_html(&self, _url: &str) -> Result<UrlPreviewData> {
Err!(FeatureDisabled("url_preview"))
}
pub fn url_preview_allowed(&self, url: &Url) -> bool {
if ["http", "https"]
.iter()
.all(|&scheme| scheme != url.scheme().to_lowercase())
{
debug!("Ignoring non-HTTP/HTTPS URL to preview: {}", url);
return false;
}
let host = match url.host_str() {
| None => {
debug!("Ignoring URL preview for a URL that does not have a host (?): {}", url);
return false;
},
| Some(h) => h.to_owned(),
};
let allowlist_domain_contains = self
.services
.globals
.url_preview_domain_contains_allowlist();
let allowlist_domain_explicit = self
.services
.globals
.url_preview_domain_explicit_allowlist();
let denylist_domain_explicit =
self.services.globals.url_preview_domain_explicit_denylist();
let allowlist_url_contains = self.services.globals.url_preview_url_contains_allowlist();
if allowlist_domain_contains.contains(&"*".to_owned())
|| allowlist_domain_explicit.contains(&"*".to_owned())
|| allowlist_url_contains.contains(&"*".to_owned())
{
debug!(
"Config key contains * which is allowing all URL previews. Allowing URL {}",
url
);
return true;
}
if !host.is_empty() {
if denylist_domain_explicit.contains(&host) {
debug!(
"Host {} is not allowed by url_preview_domain_explicit_denylist (check 1/4)",
&host
);
return false;
}
if allowlist_domain_explicit.contains(&host) {
debug!(
"Host {} is allowed by url_preview_domain_explicit_allowlist (check 2/4)",
&host
);
return true;
}
if allowlist_domain_contains
.iter()
.any(|domain_s| domain_s.contains(&host.clone()))
{
debug!(
"Host {} is allowed by url_preview_domain_contains_allowlist (check 3/4)",
&host
);
return true;
}
if allowlist_url_contains
.iter()
.any(|url_s| url.to_string().contains(url_s))
{
debug!(
"URL {} is allowed by url_preview_url_contains_allowlist (check 4/4)",
&host
);
return true;
}
// check root domain if available and if user has root domain checks
if self.services.globals.url_preview_check_root_domain() {
debug!("Checking root domain");
match host.split_once('.') {
| None => return false,
| Some((_, root_domain)) => {
if denylist_domain_explicit.contains(&root_domain.to_owned()) {
debug!(
"Root domain {} is not allowed by \
url_preview_domain_explicit_denylist (check 1/3)",
&root_domain
);
return false;
}
if allowlist_domain_explicit.contains(&root_domain.to_owned()) {
debug!(
"Root domain {} is allowed by \
url_preview_domain_explicit_allowlist (check 2/3)",
&root_domain
);
return true;
}
if allowlist_domain_contains
.iter()
.any(|domain_s| domain_s.contains(&root_domain.to_owned()))
{
debug!(
"Root domain {} is allowed by \
url_preview_domain_contains_allowlist (check 3/3)",
&root_domain
);
return true;
}
},
}
}
}
false
}
}
+385 -397
View File
@@ -1,7 +1,7 @@
use std::{fmt::Debug, time::Duration};
use conduwuit::{
Err, Error, Result, debug_warn, err, implement,
Err, Error, Result, debug_warn, err,
utils::{content_disposition::make_content_disposition, response::LimitReadExt},
};
use http::header::{CONTENT_DISPOSITION, CONTENT_TYPE, HeaderValue};
@@ -24,436 +24,424 @@ use ruma::{
use super::{Dim, FileMeta};
use crate::{federation::FederationPathBuilderInput, media::mxc::Mxc};
#[implement(super::Service)]
pub async fn fetch_remote_thumbnail(
&self,
mxc: &Mxc<'_>,
user: Option<&UserId>,
server: Option<&ServerName>,
timeout_ms: Duration,
dim: &Dim,
) -> Result<FileMeta> {
self.check_fetch_authorized(mxc)?;
impl super::Service {
pub async fn fetch_remote_thumbnail(
&self,
mxc: &Mxc<'_>,
user: Option<&UserId>,
server: Option<&ServerName>,
timeout_ms: Duration,
dim: &Dim,
) -> Result<FileMeta> {
self.check_fetch_authorized(mxc)?;
let result = self
.fetch_thumbnail_authenticated(mxc, user, server, timeout_ms, dim)
.await;
if let Err(Error::Request(NotFound, ..)) = &result {
return self
.fetch_thumbnail_unauthenticated(mxc, user, server, timeout_ms, dim)
let result = self
.fetch_thumbnail_authenticated(mxc, user, server, timeout_ms, dim)
.await;
if let Err(Error::Request(NotFound, ..)) = &result {
return self
.fetch_thumbnail_unauthenticated(mxc, user, server, timeout_ms, dim)
.await;
}
result
}
result
}
pub async fn fetch_remote_content(
&self,
mxc: &Mxc<'_>,
user: Option<&UserId>,
server: Option<&ServerName>,
timeout_ms: Duration,
) -> Result<FileMeta> {
self.check_fetch_authorized(mxc)?;
#[implement(super::Service)]
pub async fn fetch_remote_content(
&self,
mxc: &Mxc<'_>,
user: Option<&UserId>,
server: Option<&ServerName>,
timeout_ms: Duration,
) -> Result<FileMeta> {
self.check_fetch_authorized(mxc)?;
let result = self
.fetch_content_authenticated(mxc, user, server, timeout_ms)
.await
.inspect_err(|error| {
debug_warn!(
%mxc,
?user,
?server,
?error,
"Authenticated fetch of remote content failed"
);
});
let result = self
.fetch_content_authenticated(mxc, user, server, timeout_ms)
.await
.inspect_err(|error| {
debug_warn!(
%mxc,
?user,
?server,
?error,
"Authenticated fetch of remote content failed"
);
});
if let Err(Error::Request(Unrecognized, ..)) = &result {
return self
.fetch_content_unauthenticated(mxc, user, server, timeout_ms)
.await;
}
if let Err(Error::Request(Unrecognized, ..)) = &result {
return self
.fetch_content_unauthenticated(mxc, user, server, timeout_ms)
.await;
result
}
result
}
async fn fetch_thumbnail_authenticated(
&self,
mxc: &Mxc<'_>,
user: Option<&UserId>,
server: Option<&ServerName>,
timeout_ms: Duration,
dim: &Dim,
) -> Result<FileMeta> {
use federation::authenticated_media::get_content_thumbnail::v1::{Request, Response};
#[implement(super::Service)]
async fn fetch_thumbnail_authenticated(
&self,
mxc: &Mxc<'_>,
user: Option<&UserId>,
server: Option<&ServerName>,
timeout_ms: Duration,
dim: &Dim,
) -> Result<FileMeta> {
use federation::authenticated_media::get_content_thumbnail::v1::{Request, Response};
let mut request = Request::new(mxc.media_id.into(), dim.width.into(), dim.height.into());
request.method = Some(dim.method.clone());
request.animated = Some(true);
request.timeout_ms = timeout_ms;
let mut request = Request::new(mxc.media_id.into(), dim.width.into(), dim.height.into());
request.method = Some(dim.method.clone());
request.animated = Some(true);
request.timeout_ms = timeout_ms;
let Response { content, .. } = self.federation_request(mxc, server, request).await?;
let Response { content, .. } = self.federation_request(mxc, server, request).await?;
match content {
| FileOrLocation::File(content) =>
self.handle_thumbnail_file(mxc, user, dim, content).await,
| FileOrLocation::Location(location) => self.handle_location(mxc, user, &location).await,
| _ => Err!("Unknown content in response"),
match content {
| FileOrLocation::File(content) =>
self.handle_thumbnail_file(mxc, user, dim, content).await,
| FileOrLocation::Location(location) =>
self.handle_location(mxc, user, &location).await,
| _ => Err!("Unknown content in response"),
}
}
}
#[implement(super::Service)]
async fn fetch_content_authenticated(
&self,
mxc: &Mxc<'_>,
user: Option<&UserId>,
server: Option<&ServerName>,
timeout_ms: Duration,
) -> Result<FileMeta> {
use federation::authenticated_media::get_content::v1::{Request, Response};
async fn fetch_content_authenticated(
&self,
mxc: &Mxc<'_>,
user: Option<&UserId>,
server: Option<&ServerName>,
timeout_ms: Duration,
) -> Result<FileMeta> {
use federation::authenticated_media::get_content::v1::{Request, Response};
let mut request = Request::new(mxc.media_id.into());
request.timeout_ms = timeout_ms;
let mut request = Request::new(mxc.media_id.into());
request.timeout_ms = timeout_ms;
let Response { content, .. } = self.federation_request(mxc, server, request).await?;
let Response { content, .. } = self.federation_request(mxc, server, request).await?;
match content {
| FileOrLocation::File(content) => self.handle_content_file(mxc, user, content).await,
| FileOrLocation::Location(location) => self.handle_location(mxc, user, &location).await,
| _ => Err!("Unknown content in response"),
match content {
| FileOrLocation::File(content) => self.handle_content_file(mxc, user, content).await,
| FileOrLocation::Location(location) =>
self.handle_location(mxc, user, &location).await,
| _ => Err!("Unknown content in response"),
}
}
}
#[allow(deprecated)]
#[implement(super::Service)]
async fn fetch_thumbnail_unauthenticated(
&self,
mxc: &Mxc<'_>,
user: Option<&UserId>,
server: Option<&ServerName>,
timeout_ms: Duration,
dim: &Dim,
) -> Result<FileMeta> {
use media::get_content_thumbnail::v3::{Request, Response};
#[allow(deprecated)]
async fn fetch_thumbnail_unauthenticated(
&self,
mxc: &Mxc<'_>,
user: Option<&UserId>,
server: Option<&ServerName>,
timeout_ms: Duration,
dim: &Dim,
) -> Result<FileMeta> {
use media::get_content_thumbnail::v3::{Request, Response};
let mut request = Request::new(
mxc.media_id.into(),
mxc.server_name.into(),
dim.width.into(),
dim.height.into(),
);
request.allow_redirect = true;
request.allow_remote = true;
request.animated = Some(true);
request.method = Some(dim.method.clone());
request.timeout_ms = timeout_ms;
let mut request = Request::new(
mxc.media_id.into(),
mxc.server_name.into(),
dim.width.into(),
dim.height.into(),
);
request.allow_redirect = true;
request.allow_remote = true;
request.animated = Some(true);
request.method = Some(dim.method.clone());
request.timeout_ms = timeout_ms;
let Response {
file, content_type, content_disposition, ..
} = self
.federation_request_legacy_media(mxc, server, request)
.await?;
let Response {
file, content_type, content_disposition, ..
} = self
.federation_request_legacy_media(mxc, server, request)
.await?;
let content = Content::new(file, content_type.unwrap(), content_disposition.unwrap());
let content = Content::new(file, content_type.unwrap(), content_disposition.unwrap());
self.handle_thumbnail_file(mxc, user, dim, content).await
}
self.handle_thumbnail_file(mxc, user, dim, content).await
}
#[allow(deprecated)]
#[implement(super::Service)]
async fn fetch_content_unauthenticated(
&self,
mxc: &Mxc<'_>,
user: Option<&UserId>,
server: Option<&ServerName>,
timeout_ms: Duration,
) -> Result<FileMeta> {
use media::get_content::v3::{Request, Response};
#[allow(deprecated)]
async fn fetch_content_unauthenticated(
&self,
mxc: &Mxc<'_>,
user: Option<&UserId>,
server: Option<&ServerName>,
timeout_ms: Duration,
) -> Result<FileMeta> {
use media::get_content::v3::{Request, Response};
let mut request = Request::new(mxc.media_id.into(), mxc.server_name.into());
request.allow_remote = true;
request.allow_redirect = true;
request.timeout_ms = timeout_ms;
let mut request = Request::new(mxc.media_id.into(), mxc.server_name.into());
request.allow_remote = true;
request.allow_redirect = true;
request.timeout_ms = timeout_ms;
let Response {
file, content_type, content_disposition, ..
} = self
.federation_request_legacy_media(mxc, server, request)
.await?;
let Response {
file, content_type, content_disposition, ..
} = self
.federation_request_legacy_media(mxc, server, request)
.await?;
let content = Content::new(file, content_type.unwrap(), content_disposition.unwrap());
let content = Content::new(file, content_type.unwrap(), content_disposition.unwrap());
self.handle_content_file(mxc, user, content).await
}
self.handle_content_file(mxc, user, content).await
}
#[implement(super::Service)]
async fn handle_thumbnail_file(
&self,
mxc: &Mxc<'_>,
user: Option<&UserId>,
dim: &Dim,
content: Content,
) -> Result<FileMeta> {
let content_disposition = make_content_disposition(
content.content_disposition.as_ref(),
content.content_type.as_deref(),
None,
);
async fn handle_thumbnail_file(
&self,
mxc: &Mxc<'_>,
user: Option<&UserId>,
dim: &Dim,
content: Content,
) -> Result<FileMeta> {
let content_disposition = make_content_disposition(
content.content_disposition.as_ref(),
content.content_type.as_deref(),
None,
);
self.upload_thumbnail(
mxc,
user,
Some(&content_disposition),
content.content_type.as_deref(),
dim,
&content.file,
)
.await
.map(|()| FileMeta {
content: Some(content.file),
content_type: content.content_type,
content_disposition: Some(content_disposition),
})
}
#[implement(super::Service)]
async fn handle_content_file(
&self,
mxc: &Mxc<'_>,
user: Option<&UserId>,
content: Content,
) -> Result<FileMeta> {
let content_disposition = make_content_disposition(
content.content_disposition.as_ref(),
content.content_type.as_deref(),
None,
);
self.create(
mxc,
user,
Some(&content_disposition),
content.content_type.as_deref(),
&content.file,
)
.await
.map(|()| FileMeta {
content: Some(content.file),
content_type: content.content_type,
content_disposition: Some(content_disposition),
})
}
#[implement(super::Service)]
async fn handle_location(
&self,
mxc: &Mxc<'_>,
user: Option<&UserId>,
location: &str,
) -> Result<FileMeta> {
self.location_request(location).await.map_err(|error| {
err!(Request(NotFound(
debug_warn!(%mxc, user = user.map(tracing::field::display), ?location, ?error, "Fetching media from location failed")
)))
})
}
#[implement(super::Service)]
async fn location_request(&self, location: &str) -> Result<FileMeta> {
let response = self
.services
.client
.extern_media
.get(location)
.send()
.await?;
let content_type = response
.headers()
.get(CONTENT_TYPE)
.map(HeaderValue::to_str)
.and_then(Result::ok)
.map(str::to_owned);
let content_disposition = response
.headers()
.get(CONTENT_DISPOSITION)
.map(HeaderValue::as_bytes)
.map(TryFrom::try_from)
.and_then(Result::ok);
response
.limit_read(
self.services
.server
.config
.max_request_size
.try_into()
.expect("u64 should fit in usize"),
self.upload_thumbnail(
mxc,
user,
Some(&content_disposition),
content.content_type.as_deref(),
dim,
&content.file,
)
.await
.map(|content| FileMeta {
content: Some(content),
content_type: content_type.clone(),
content_disposition: Some(make_content_disposition(
content_disposition.as_ref(),
content_type.as_deref(),
None,
)),
.map(|()| FileMeta {
content: Some(content.file),
content_type: content.content_type,
content_disposition: Some(content_disposition),
})
}
#[implement(super::Service)]
async fn federation_request<'i, Request>(
&self,
mxc: &Mxc<'_>,
server: Option<&ServerName>,
request: Request,
) -> Result<Request::IncomingResponse>
where
Request: OutgoingRequest<
Authentication = ServerSignatures,
PathBuilder: PathBuilder<Input<'i>: FederationPathBuilderInput>,
> + Debug
+ Send,
{
self.services
.sending
.send_federation_request(server.unwrap_or(mxc.server_name), request)
.await
}
#[implement(super::Service)]
async fn federation_request_legacy_media<'i, Request>(
&self,
mxc: &Mxc<'_>,
server: Option<&ServerName>,
request: Request,
) -> Result<Request::IncomingResponse>
where
Request: OutgoingRequest<
Authentication = NoAccessToken,
PathBuilder: PathBuilder<Input<'i>: FederationPathBuilderInput>,
> + Debug
+ Send,
{
self.services
.sending
.send_legacy_media_request(server.unwrap_or(mxc.server_name), request)
.await
}
#[implement(super::Service)]
#[allow(deprecated)]
pub async fn fetch_remote_thumbnail_legacy(
&self,
body: &media::get_content_thumbnail::v3::Request,
) -> Result<media::get_content_thumbnail::v3::Response> {
let mxc = Mxc {
server_name: &body.server_name,
media_id: &body.media_id,
};
let mut request = media::get_content_thumbnail::v3::Request::new(
body.media_id.clone(),
body.server_name.clone(),
body.width,
body.height,
);
request.method.clone_from(&body.method);
request.allow_remote = body.allow_remote;
request.allow_redirect = body.allow_redirect;
request.animated = body.animated;
request.timeout_ms = body.timeout_ms;
self.check_legacy_freeze()?;
self.check_fetch_authorized(&mxc)?;
let response = self
.services
.sending
.send_legacy_media_request(mxc.server_name, request)
.await?;
let dim = Dim::from_ruma(body.width, body.height, body.method.clone())?;
self.upload_thumbnail(
&mxc,
None,
None,
response.content_type.as_deref(),
&dim,
&response.file,
)
.await?;
Ok(response)
}
#[implement(super::Service)]
#[allow(deprecated)]
pub async fn fetch_remote_content_legacy(
&self,
mxc: &Mxc<'_>,
allow_redirect: bool,
timeout_ms: Duration,
) -> Result<media::get_content::v3::Response, Error> {
let mut request =
media::get_content::v3::Request::new(mxc.media_id.into(), mxc.server_name.into());
request.allow_remote = true;
request.allow_redirect = allow_redirect;
request.timeout_ms = timeout_ms;
self.check_legacy_freeze()?;
self.check_fetch_authorized(mxc)?;
let response = self
.services
.sending
.send_legacy_media_request(mxc.server_name, request)
.await?;
let content_disposition = make_content_disposition(
response.content_disposition.as_ref(),
response.content_type.as_deref(),
None,
);
self.create(
mxc,
None,
Some(&content_disposition),
response.content_type.as_deref(),
&response.file,
)
.await?;
Ok(response)
}
#[implement(super::Service)]
fn check_fetch_authorized(&self, mxc: &Mxc<'_>) -> Result<()> {
if self
.services
.moderation
.is_remote_server_media_downloads_forbidden(mxc.server_name)
{
// we'll lie to the client and say the blocked server's media was not found and
// log. the client has no way of telling anyways so this is a security bonus.
debug_warn!(%mxc, "Received request for media on blocklisted server");
return Err!(Request(NotFound("Media not found.")));
}
Ok(())
}
async fn handle_content_file(
&self,
mxc: &Mxc<'_>,
user: Option<&UserId>,
content: Content,
) -> Result<FileMeta> {
let content_disposition = make_content_disposition(
content.content_disposition.as_ref(),
content.content_type.as_deref(),
None,
);
#[implement(super::Service)]
fn check_legacy_freeze(&self) -> Result<()> {
self.services
.server
.config
.freeze_legacy_media
.then_some(())
.ok_or(err!(Request(NotFound("Remote media is frozen."))))
self.create(
mxc,
user,
Some(&content_disposition),
content.content_type.as_deref(),
&content.file,
)
.await
.map(|()| FileMeta {
content: Some(content.file),
content_type: content.content_type,
content_disposition: Some(content_disposition),
})
}
async fn handle_location(
&self,
mxc: &Mxc<'_>,
user: Option<&UserId>,
location: &str,
) -> Result<FileMeta> {
self.location_request(location).await.map_err(|error| {
err!(Request(NotFound(
debug_warn!(%mxc, user = user.map(tracing::field::display), ?location, ?error, "Fetching media from location failed")
)))
})
}
async fn location_request(&self, location: &str) -> Result<FileMeta> {
let response = self
.services
.client
.extern_media
.get(location)
.send()
.await?;
let content_type = response
.headers()
.get(CONTENT_TYPE)
.map(HeaderValue::to_str)
.and_then(Result::ok)
.map(str::to_owned);
let content_disposition = response
.headers()
.get(CONTENT_DISPOSITION)
.map(HeaderValue::as_bytes)
.map(TryFrom::try_from)
.and_then(Result::ok);
response
.limit_read(
self.services
.server
.config
.max_request_size
.try_into()
.expect("u64 should fit in usize"),
)
.await
.map(|content| FileMeta {
content: Some(content),
content_type: content_type.clone(),
content_disposition: Some(make_content_disposition(
content_disposition.as_ref(),
content_type.as_deref(),
None,
)),
})
}
async fn federation_request<'i, Request>(
&self,
mxc: &Mxc<'_>,
server: Option<&ServerName>,
request: Request,
) -> Result<Request::IncomingResponse>
where
Request: OutgoingRequest<
Authentication = ServerSignatures,
PathBuilder: PathBuilder<Input<'i>: FederationPathBuilderInput>,
> + Debug
+ Send,
{
self.services
.sending
.send_federation_request(server.unwrap_or(mxc.server_name), request)
.await
}
async fn federation_request_legacy_media<'i, Request>(
&self,
mxc: &Mxc<'_>,
server: Option<&ServerName>,
request: Request,
) -> Result<Request::IncomingResponse>
where
Request: OutgoingRequest<
Authentication = NoAccessToken,
PathBuilder: PathBuilder<Input<'i>: FederationPathBuilderInput>,
> + Debug
+ Send,
{
self.services
.sending
.send_legacy_media_request(server.unwrap_or(mxc.server_name), request)
.await
}
#[allow(deprecated)]
pub async fn fetch_remote_thumbnail_legacy(
&self,
body: &media::get_content_thumbnail::v3::Request,
) -> Result<media::get_content_thumbnail::v3::Response> {
let mxc = Mxc {
server_name: &body.server_name,
media_id: &body.media_id,
};
let mut request = media::get_content_thumbnail::v3::Request::new(
body.media_id.clone(),
body.server_name.clone(),
body.width,
body.height,
);
request.method.clone_from(&body.method);
request.allow_remote = body.allow_remote;
request.allow_redirect = body.allow_redirect;
request.animated = body.animated;
request.timeout_ms = body.timeout_ms;
self.check_legacy_freeze()?;
self.check_fetch_authorized(&mxc)?;
let response = self
.services
.sending
.send_legacy_media_request(mxc.server_name, request)
.await?;
let dim = Dim::from_ruma(body.width, body.height, body.method.clone())?;
self.upload_thumbnail(
&mxc,
None,
None,
response.content_type.as_deref(),
&dim,
&response.file,
)
.await?;
Ok(response)
}
#[allow(deprecated)]
pub async fn fetch_remote_content_legacy(
&self,
mxc: &Mxc<'_>,
allow_redirect: bool,
timeout_ms: Duration,
) -> Result<media::get_content::v3::Response, Error> {
let mut request =
media::get_content::v3::Request::new(mxc.media_id.into(), mxc.server_name.into());
request.allow_remote = true;
request.allow_redirect = allow_redirect;
request.timeout_ms = timeout_ms;
self.check_legacy_freeze()?;
self.check_fetch_authorized(mxc)?;
let response = self
.services
.sending
.send_legacy_media_request(mxc.server_name, request)
.await?;
let content_disposition = make_content_disposition(
response.content_disposition.as_ref(),
response.content_type.as_deref(),
None,
);
self.create(
mxc,
None,
Some(&content_disposition),
response.content_type.as_deref(),
&response.file,
)
.await?;
Ok(response)
}
fn check_fetch_authorized(&self, mxc: &Mxc<'_>) -> Result<()> {
if self
.services
.moderation
.is_remote_server_media_downloads_forbidden(mxc.server_name)
{
// we'll lie to the client and say the blocked server's media was not found and
// log. the client has no way of telling anyways so this is a security bonus.
debug_warn!(%mxc, "Received request for media on blocklisted server");
return Err!(Request(NotFound("Media not found.")));
}
Ok(())
}
fn check_legacy_freeze(&self) -> Result<()> {
self.services
.server
.config
.freeze_legacy_media
.then_some(())
.ok_or(err!(Request(NotFound("Remote media is frozen."))))
}
}
+64 -67
View File
@@ -7,7 +7,7 @@
use std::{cmp, num::Saturating as Sat};
use conduwuit::{Result, checked, err, implement};
use conduwuit::{Result, checked, err};
use ruma::{UInt, UserId, http_headers::ContentDisposition, media::Method};
use tokio::{
fs,
@@ -74,80 +74,77 @@ impl super::Service {
},
}
}
}
/// Using saved thumbnail
#[implement(super::Service)]
#[tracing::instrument(name = "saved", level = "debug", skip(self, data))]
async fn get_thumbnail_saved(&self, data: Metadata) -> Result<Option<FileMeta>> {
let mut content = Vec::new();
let path = self.get_media_file(&data.key);
fs::File::open(path)
.await?
.read_to_end(&mut content)
.await?;
/// Using saved thumbnail
#[tracing::instrument(name = "saved", level = "debug", skip(self, data))]
async fn get_thumbnail_saved(&self, data: Metadata) -> Result<Option<FileMeta>> {
let mut content = Vec::new();
let path = self.get_media_file(&data.key);
fs::File::open(path)
.await?
.read_to_end(&mut content)
.await?;
Ok(Some(into_filemeta(data, content)))
}
/// Generate a thumbnail
#[cfg(feature = "media_thumbnail")]
#[implement(super::Service)]
#[tracing::instrument(name = "generate", level = "debug", skip(self, data))]
async fn get_thumbnail_generate(
&self,
mxc: &Mxc<'_>,
dim: &Dim,
data: Metadata,
) -> Result<Option<FileMeta>> {
let mut content = Vec::new();
let path = self.get_media_file(&data.key);
fs::File::open(path)
.await?
.read_to_end(&mut content)
.await?;
let Ok(image) = image::load_from_memory(&content) else {
// Couldn't parse file to generate thumbnail, send original
return Ok(Some(into_filemeta(data, content)));
};
if dim.width > image.width() || dim.height > image.height() {
return Ok(Some(into_filemeta(data, content)));
Ok(Some(into_filemeta(data, content)))
}
let mut thumbnail_bytes = Vec::new();
let thumbnail = thumbnail_generate(&image, dim)?;
let mut cursor = std::io::Cursor::new(&mut thumbnail_bytes);
thumbnail
.write_to(&mut cursor, image::ImageFormat::Png)
.map_err(|error| err!(error!(%error, "Error writing PNG thumbnail.")))?;
/// Generate a thumbnail
#[cfg(feature = "media_thumbnail")]
#[tracing::instrument(name = "generate", level = "debug", skip(self, data))]
async fn get_thumbnail_generate(
&self,
mxc: &Mxc<'_>,
dim: &Dim,
data: Metadata,
) -> Result<Option<FileMeta>> {
let mut content = Vec::new();
let path = self.get_media_file(&data.key);
fs::File::open(path)
.await?
.read_to_end(&mut content)
.await?;
// Save thumbnail in database so we don't have to generate it again next time
let thumbnail_key = self.db.create_file_metadata(
mxc,
None,
dim,
data.content_disposition.as_ref(),
data.content_type.as_deref(),
)?;
let Ok(image) = image::load_from_memory(&content) else {
// Couldn't parse file to generate thumbnail, send original
return Ok(Some(into_filemeta(data, content)));
};
let mut f = self.create_media_file(&thumbnail_key).await?;
f.write_all(&thumbnail_bytes).await?;
if dim.width > image.width() || dim.height > image.height() {
return Ok(Some(into_filemeta(data, content)));
}
Ok(Some(into_filemeta(data, thumbnail_bytes)))
}
let mut thumbnail_bytes = Vec::new();
let thumbnail = thumbnail_generate(&image, dim)?;
let mut cursor = std::io::Cursor::new(&mut thumbnail_bytes);
thumbnail
.write_to(&mut cursor, image::ImageFormat::Png)
.map_err(|error| err!(error!(%error, "Error writing PNG thumbnail.")))?;
#[cfg(not(feature = "media_thumbnail"))]
#[implement(super::Service)]
#[tracing::instrument(name = "fallback", level = "debug", skip_all)]
async fn get_thumbnail_generate(
&self,
_mxc: &Mxc<'_>,
_dim: &Dim,
data: Metadata,
) -> Result<Option<FileMeta>> {
self.get_thumbnail_saved(data).await
// Save thumbnail in database so we don't have to generate it again next time
let thumbnail_key = self.db.create_file_metadata(
mxc,
None,
dim,
data.content_disposition.as_ref(),
data.content_type.as_deref(),
)?;
let mut f = self.create_media_file(&thumbnail_key).await?;
f.write_all(&thumbnail_bytes).await?;
Ok(Some(into_filemeta(data, thumbnail_bytes)))
}
#[cfg(not(feature = "media_thumbnail"))]
#[tracing::instrument(name = "fallback", level = "debug", skip_all)]
async fn get_thumbnail_generate(
&self,
_mxc: &Mxc<'_>,
_dim: &Dim,
data: Metadata,
) -> Result<Option<FileMeta>> {
self.get_thumbnail_saved(data).await
}
}
#[cfg(feature = "media_thumbnail")]
+66 -55
View File
@@ -1,6 +1,6 @@
use std::sync::Arc;
use conduwuit::{Result, implement};
use conduwuit::Result;
use ruma::ServerName;
use crate::{Dep, config};
@@ -27,67 +27,78 @@ impl crate::Service for Service {
fn name(&self) -> &str { crate::service::make_name(std::module_path!()) }
}
#[implement(Service)]
#[must_use]
pub fn is_remote_server_ignored(&self, server_name: &ServerName) -> bool {
// We must never block federating with ourselves
if server_name == self.services.config.server_name {
return false;
impl Service {
/// Returns true if the remote server is *ignored*. Ignored server names are
/// still permitted in federation, but should not have their events sent to
/// local users.
#[must_use]
pub fn is_remote_server_ignored(&self, server_name: &ServerName) -> bool {
// We must never block federating with ourselves
if server_name == self.services.config.server_name {
return false;
}
self.services
.config
.ignore_messages_from_server_names
.is_match(server_name.host())
}
self.services
.config
.ignore_messages_from_server_names
.is_match(server_name.host())
}
/// Returns true if the remote server is *forbidden*. Forbidden server names
/// must be blocked in federation in both directions - no inbound nor
/// outbound requests from the remote are permitted.
#[must_use]
pub fn is_remote_server_forbidden(&self, server_name: &ServerName) -> bool {
// We must never block federating with ourselves
if server_name == self.services.config.server_name {
return false;
}
#[implement(Service)]
#[must_use]
pub fn is_remote_server_forbidden(&self, server_name: &ServerName) -> bool {
// We must never block federating with ourselves
if server_name == self.services.config.server_name {
return false;
}
// Check if server is explicitly allowed
if self
.services
.config
.allowed_remote_server_names
.is_match(server_name.host())
{
return false;
}
// Check if server is explicitly forbidden
self.services
.config
.forbidden_remote_server_names
.is_match(server_name.host())
}
#[implement(Service)]
#[must_use]
pub fn is_remote_server_room_directory_forbidden(&self, server_name: &ServerName) -> bool {
// Forbidden if NOT (allowed is empty OR allowed contains server OR is self)
// OR forbidden contains server
self.is_remote_server_forbidden(server_name)
|| self
// Check if server is explicitly allowed
if self
.services
.config
.forbidden_remote_room_directory_server_names
.allowed_remote_server_names
.is_match(server_name.host())
}
{
return false;
}
#[implement(Service)]
#[must_use]
pub fn is_remote_server_media_downloads_forbidden(&self, server_name: &ServerName) -> bool {
// Forbidden if NOT (allowed is empty OR allowed contains server OR is self)
// OR forbidden contains server
self.is_remote_server_forbidden(server_name)
|| self
.services
// Check if server is explicitly forbidden
self.services
.config
.prevent_media_downloads_from
.forbidden_remote_server_names
.is_match(server_name.host())
}
/// Returns true if the remote server is *forbidden*, or if it is only
/// forbidden from having its room directory checked. Does not prevent the
/// remote from reading our room directory.
#[must_use]
pub fn is_remote_server_room_directory_forbidden(&self, server_name: &ServerName) -> bool {
// Forbidden if NOT (allowed is empty OR allowed contains server OR is self)
// OR forbidden contains server
self.is_remote_server_forbidden(server_name)
|| self
.services
.config
.forbidden_remote_room_directory_server_names
.is_match(server_name.host())
}
/// Returns true if the remote server is *forbidden*, or if it is only
/// forbidden from serving media to us (i.e. we aren't allowed to download
/// media from the remote). Does not prevent the remote downloading media
/// from us.
#[must_use]
pub fn is_remote_server_media_downloads_forbidden(&self, server_name: &ServerName) -> bool {
// Forbidden if NOT (allowed is empty OR allowed contains server OR is self)
// OR forbidden contains server
self.is_remote_server_forbidden(server_name)
|| self
.services
.config
.prevent_media_downloads_from
.is_match(server_name.host())
}
}
+52 -65
View File
@@ -3,7 +3,7 @@ use std::{net::IpAddr, sync::Arc, time::SystemTime};
use conduwuit::{
Result,
arrayvec::ArrayVec,
at, err, implement,
at, err,
utils::{math::Expected, rand, stream::TryIgnore},
};
use database::{Cbor, Deserialized, Map};
@@ -43,83 +43,70 @@ impl Cache {
overrides: args.db["servername_override"].clone(),
})
}
}
#[implement(Cache)]
pub async fn clear(&self) { join(self.clear_destinations(), self.clear_overrides()).await; }
pub async fn clear(&self) { join(self.clear_destinations(), self.clear_overrides()).await; }
#[implement(Cache)]
pub async fn clear_destinations(&self) { self.destinations.clear().await; }
pub async fn clear_destinations(&self) { self.destinations.clear().await; }
#[implement(Cache)]
pub async fn clear_overrides(&self) { self.overrides.clear().await; }
pub async fn clear_overrides(&self) { self.overrides.clear().await; }
#[implement(Cache)]
pub fn del_destination(&self, name: &ServerName) { self.destinations.remove(name); }
pub fn del_destination(&self, name: &ServerName) { self.destinations.remove(name); }
#[implement(Cache)]
pub fn del_override(&self, name: &ServerName) { self.overrides.remove(name); }
pub fn del_override(&self, name: &ServerName) { self.overrides.remove(name); }
#[implement(Cache)]
pub fn set_destination(&self, name: &ServerName, dest: &CachedDest) {
self.destinations.raw_put(name, Cbor(dest));
}
pub fn set_destination(&self, name: &ServerName, dest: &CachedDest) {
self.destinations.raw_put(name, Cbor(dest));
}
#[implement(Cache)]
pub fn set_override(&self, name: &str, over: &CachedOverride) {
self.overrides.raw_put(name, Cbor(over));
}
pub fn set_override(&self, name: &str, over: &CachedOverride) {
self.overrides.raw_put(name, Cbor(over));
}
#[implement(Cache)]
#[must_use]
pub async fn has_destination(&self, destination: &ServerName) -> bool {
self.get_destination(destination).await.is_ok()
}
#[must_use]
pub async fn has_destination(&self, destination: &ServerName) -> bool {
self.get_destination(destination).await.is_ok()
}
#[implement(Cache)]
#[must_use]
pub async fn has_override(&self, destination: &str) -> bool {
self.get_override(destination)
.await
.iter()
.any(CachedOverride::valid)
}
#[must_use]
pub async fn has_override(&self, destination: &str) -> bool {
self.get_override(destination)
.await
.iter()
.any(CachedOverride::valid)
}
#[implement(Cache)]
pub async fn get_destination(&self, name: &ServerName) -> Result<CachedDest> {
self.destinations
.get(name)
.await
.deserialized::<Cbor<_>>()
.map(at!(0))
.into_iter()
.find(CachedDest::valid)
.ok_or(err!(Request(NotFound("Expired from cache"))))
}
pub async fn get_destination(&self, name: &ServerName) -> Result<CachedDest> {
self.destinations
.get(name)
.await
.deserialized::<Cbor<_>>()
.map(at!(0))
.into_iter()
.find(CachedDest::valid)
.ok_or(err!(Request(NotFound("Expired from cache"))))
}
#[implement(Cache)]
pub async fn get_override(&self, name: &str) -> Result<CachedOverride> {
self.overrides
.get(name)
.await
.deserialized::<Cbor<_>>()
.map(at!(0))
}
pub async fn get_override(&self, name: &str) -> Result<CachedOverride> {
self.overrides
.get(name)
.await
.deserialized::<Cbor<_>>()
.map(at!(0))
}
#[implement(Cache)]
pub fn destinations(&self) -> impl Stream<Item = (OwnedServerName, CachedDest)> + Send + '_ {
self.destinations
.stream()
.ignore_err()
.map(|item: (OwnedServerName, Cbor<_>)| (item.0, item.1.0))
}
pub fn destinations(&self) -> impl Stream<Item = (OwnedServerName, CachedDest)> + Send + '_ {
self.destinations
.stream()
.ignore_err()
.map(|item: (OwnedServerName, Cbor<_>)| (item.0, item.1.0))
}
#[implement(Cache)]
pub fn overrides(&self) -> impl Stream<Item = (OwnedServerName, CachedOverride)> + Send + '_ {
self.overrides
.stream()
.ignore_err()
.map(|item: (OwnedServerName, Cbor<_>)| (item.0, item.1.0))
pub fn overrides(&self) -> impl Stream<Item = (OwnedServerName, CachedOverride)> + Send + '_ {
self.overrides
.stream()
.ignore_err()
.map(|item: (OwnedServerName, Cbor<_>)| (item.0, item.1.0))
}
}
impl CachedDest {
+45 -46
View File
@@ -1,51 +1,50 @@
use conduwuit::{
Result, debug, debug_error, debug_info, implement, trace, utils::response::LimitReadExt,
};
use conduwuit::{Result, debug, debug_error, debug_info, trace, utils::response::LimitReadExt};
use ruma::ServerName;
#[implement(super::Service)]
#[tracing::instrument(name = "well-known", level = "debug", skip(self, dest))]
pub(super) async fn request_well_known(&self, dest: &str) -> Result<Option<String>> {
trace!("Requesting well known for {dest}");
let response = self
.services
.client
.well_known
.get(format!("https://{dest}/.well-known/matrix/server"))
.send()
.await;
impl super::Service {
#[tracing::instrument(name = "well-known", level = "debug", skip(self, dest))]
pub(super) async fn request_well_known(&self, dest: &str) -> Result<Option<String>> {
trace!("Requesting well known for {dest}");
let response = self
.services
.client
.well_known
.get(format!("https://{dest}/.well-known/matrix/server"))
.send()
.await;
trace!("response: {response:?}");
if let Err(e) = &response {
debug!("error: {e:?}");
return Ok(None);
trace!("response: {response:?}");
if let Err(e) = &response {
debug!("error: {e:?}");
return Ok(None);
}
let response = response?;
if !response.status().is_success() {
debug!("response not 2XX");
return Ok(None);
}
let Ok(text) = response.limit_read_text(8192).await else {
debug!("failed to read well-known response (too large or non-text content)");
return Ok(None);
};
trace!("response text: {text:?}");
let body: serde_json::Value = serde_json::from_str(&text).unwrap_or_default();
let m_server = body
.get("m.server")
.unwrap_or(&serde_json::Value::Null)
.as_str()
.unwrap_or_default();
if ServerName::parse(m_server).is_err() {
debug_error!("response content missing or invalid");
return Ok(None);
}
debug_info!("{dest:?} found at {m_server:?}");
Ok(Some(m_server.to_owned()))
}
let response = response?;
if !response.status().is_success() {
debug!("response not 2XX");
return Ok(None);
}
let Ok(text) = response.limit_read_text(8192).await else {
debug!("failed to read well-known response (too large or non-text content)");
return Ok(None);
};
trace!("response text: {text:?}");
let body: serde_json::Value = serde_json::from_str(&text).unwrap_or_default();
let m_server = body
.get("m.server")
.unwrap_or(&serde_json::Value::Null)
.as_str()
.unwrap_or_default();
if ServerName::parse(m_server).is_err() {
debug_error!("response content missing or invalid");
return Ok(None);
}
debug_info!("{dest:?} found at {m_server:?}");
Ok(Some(m_server.to_owned()))
}
+37 -35
View File
@@ -1,40 +1,42 @@
use conduwuit::{Result, debug, error, implement};
use conduwuit::{Result, debug, error};
use federation::query::get_room_information::v1::Response;
use ruma::{OwnedRoomId, OwnedServerName, RoomAliasId, ServerName, api::federation};
#[implement(super::Service)]
pub(super) async fn remote_resolve(
&self,
room_alias: &RoomAliasId,
) -> Result<(OwnedRoomId, Vec<OwnedServerName>)> {
debug!("Asking {} to resolve {room_alias:?}", room_alias.server_name());
match self
.remote_request(room_alias, room_alias.server_name())
.await
{
| Err(e) => {
error!("Unable to resolve remote room alias {}: {e}", room_alias);
Err(e)
},
| Ok(Response { room_id, servers, .. }) => {
debug!("Remote resolved {room_alias:?} to {room_id:?} with servers {servers:?}");
Ok((room_id, servers))
},
impl super::Service {
/// Resolves the given room alias to a room ID and a list of servers that
/// are in the room by asking the remote server which owns it.
pub(super) async fn remote_resolve(
&self,
room_alias: &RoomAliasId,
) -> Result<(OwnedRoomId, Vec<OwnedServerName>)> {
debug!("Asking {} to resolve {room_alias:?}", room_alias.server_name());
match self
.remote_request(room_alias, room_alias.server_name())
.await
{
| Err(e) => {
error!("Unable to resolve remote room alias {}: {e}", room_alias);
Err(e)
},
| Ok(Response { room_id, servers, .. }) => {
debug!("Remote resolved {room_alias:?} to {room_id:?} with servers {servers:?}");
Ok((room_id, servers))
},
}
}
async fn remote_request(
&self,
room_alias: &RoomAliasId,
server: &ServerName,
) -> Result<Response> {
use federation::query::get_room_information::v1::Request;
let request = Request::new(room_alias.to_owned());
self.services
.sending
.send_federation_request(server, request)
.await
}
}
#[implement(super::Service)]
async fn remote_request(
&self,
room_alias: &RoomAliasId,
server: &ServerName,
) -> Result<Response> {
use federation::query::get_room_information::v1::Request;
let request = Request::new(room_alias.to_owned());
self.services
.sending
.send_federation_request(server, request)
.await
}
+195 -204
View File
@@ -8,12 +8,12 @@ use std::{
};
use conduwuit::{
Err, Result, at, debug, debug_error, implement, trace,
Err, Result, at, debug, debug_error, trace,
utils::{
IterStream,
stream::{ReadyExt, TryBroadbandExt},
},
validated, warn,
validated,
};
use futures::{FutureExt, Stream, StreamExt, TryFutureExt, TryStreamExt};
use ruma::{EventId, OwnedEventId, RoomId};
@@ -47,213 +47,204 @@ impl crate::Service for Service {
fn name(&self) -> &str { crate::service::make_name(std::module_path!()) }
}
#[implement(Service)]
pub fn event_ids_iter<'a, I>(
&'a self,
room_id: &'a RoomId,
starting_events: I,
) -> impl Stream<Item = Result<OwnedEventId>> + Send + 'a
where
I: Iterator<Item = &'a EventId> + Clone + Debug + ExactSizeIterator + Send + 'a,
{
self.get_auth_chain(room_id, starting_events)
.map_ok(|chain| {
self.services
.short
.multi_get_eventid_from_short(chain.into_iter().stream())
.ready_filter(Result::is_ok)
})
.try_flatten_stream()
}
#[implement(Service)]
#[tracing::instrument(name = "auth_chain", level = "debug", skip_all)]
pub async fn get_auth_chain<'a, I>(
&'a self,
room_id: &RoomId,
starting_events: I,
) -> Result<Vec<ShortEventId>>
where
I: Iterator<Item = &'a EventId> + Clone + Debug + ExactSizeIterator + Send + 'a,
{
const NUM_BUCKETS: usize = 50; //TODO: change possible w/o disrupting db?
const BUCKET: Bucket<'_> = BTreeSet::new();
let started = Instant::now();
let mut starting_ids = self
.services
.short
.multi_get_or_create_shorteventid(starting_events.clone())
.zip(starting_events.clone().stream())
.boxed();
let mut buckets = [BUCKET; NUM_BUCKETS];
while let Some((short, starting_event)) = starting_ids.next().await {
let bucket: usize = short.try_into()?;
let bucket: usize = validated!(bucket % NUM_BUCKETS);
buckets[bucket].insert((short, starting_event));
impl Service {
pub fn event_ids_iter<'a, I>(
&'a self,
room_id: &'a RoomId,
starting_events: I,
) -> impl Stream<Item = Result<OwnedEventId>> + Send + 'a
where
I: Iterator<Item = &'a EventId> + Clone + Debug + ExactSizeIterator + Send + 'a,
{
self.get_auth_chain(room_id, starting_events)
.map_ok(|chain| {
self.services
.short
.multi_get_eventid_from_short(chain.into_iter().stream())
.ready_filter(Result::is_ok)
})
.try_flatten_stream()
}
debug!(
starting_events = ?starting_events.count(),
elapsed = ?started.elapsed(),
"start",
);
#[tracing::instrument(name = "auth_chain", skip_all)]
pub async fn get_auth_chain<'a, I>(
&'a self,
room_id: &RoomId,
starting_events: I,
) -> Result<Vec<ShortEventId>>
where
I: Iterator<Item = &'a EventId> + Clone + Debug + ExactSizeIterator + Send + 'a,
{
const NUM_BUCKETS: usize = 50; //TODO: change possible w/o disrupting db?
const BUCKET: Bucket<'_> = BTreeSet::new();
let full_auth_chain: Vec<ShortEventId> = buckets
.into_iter()
.try_stream()
.broad_and_then(|chunk| self.get_auth_chain_outer(room_id, started, chunk))
.try_collect()
.map_ok(|auth_chain: Vec<_>| auth_chain.into_iter().flatten().collect())
.map_ok(|mut full_auth_chain: Vec<_>| {
full_auth_chain.sort_unstable();
full_auth_chain.dedup();
full_auth_chain
})
.boxed()
.await?;
let started = Instant::now();
let mut starting_ids = self
.services
.short
.multi_get_or_create_shorteventid(starting_events.clone())
.zip(starting_events.clone().stream())
.boxed();
debug!(
chain_length = ?full_auth_chain.len(),
elapsed = ?started.elapsed(),
"done",
);
Ok(full_auth_chain)
}
#[implement(Service)]
async fn get_auth_chain_outer(
&self,
room_id: &RoomId,
started: Instant,
chunk: Bucket<'_>,
) -> Result<Vec<ShortEventId>> {
let chunk_key: Vec<ShortEventId> = chunk.iter().map(at!(0)).collect();
if chunk_key.is_empty() {
return Ok(Vec::new());
}
if let Ok(cached) = self.get_cached_eventid_authchain(&chunk_key).await {
return Ok(cached.to_vec());
}
let chunk_cache: Vec<_> = chunk
.into_iter()
.try_stream::<conduwuit::Error>()
.broad_and_then(|(shortid, event_id)| async move {
if let Ok(cached) = self.get_cached_eventid_authchain(&[shortid]).await {
return Ok(cached.to_vec());
}
let auth_chain = self.get_auth_chain_inner(room_id, event_id).await?;
self.cache_auth_chain_vec(vec![shortid], auth_chain.as_slice());
debug!(
%event_id,
elapsed = ?started.elapsed(),
"Cache missed event"
);
Ok(auth_chain)
})
.try_collect()
.map_ok(|chunk_cache: Vec<_>| chunk_cache.into_iter().flatten().collect())
.map_ok(|mut chunk_cache: Vec<_>| {
chunk_cache.sort_unstable();
chunk_cache.dedup();
chunk_cache
})
.await?;
self.cache_auth_chain_vec(chunk_key, chunk_cache.as_slice());
debug!(
chunk_cache_length = ?chunk_cache.len(),
elapsed = ?started.elapsed(),
"Cache missed chunk",
);
Ok(chunk_cache)
}
#[implement(Service)]
#[tracing::instrument(name = "inner", level = "trace", skip(self, room_id))]
async fn get_auth_chain_inner(
&self,
room_id: &RoomId,
event_id: &EventId,
) -> Result<Vec<ShortEventId>> {
let mut todo: VecDeque<_> = [event_id.to_owned()].into();
let mut found = HashSet::new();
while let Some(event_id) = todo.pop_front() {
trace!(%event_id, "processing auth event");
match self.services.timeline.get_pdu(&event_id).await {
| Err(e) => {
debug_error!(%event_id, ?e, "Could not find pdu mentioned in auth events");
},
| Ok(pdu) => {
if let Some(claimed_room_id) = pdu.room_id.clone() {
if claimed_room_id != *room_id {
return Err!(Request(Forbidden(error!(
%event_id,
%room_id,
wrong_room_id = ?pdu.room_id.unwrap(),
"auth event for incorrect room"
))));
}
}
for auth_event in &pdu.auth_events {
let sauthevent = self
.services
.short
.get_or_create_shorteventid(auth_event)
.await;
if found.insert(sauthevent) {
trace!(%event_id, ?auth_event, "adding auth event to processing queue");
todo.push_back(auth_event.clone());
}
}
},
let mut buckets = [BUCKET; NUM_BUCKETS];
while let Some((short, starting_event)) = starting_ids.next().await {
let bucket: usize = short.try_into()?;
let bucket: usize = validated!(bucket % NUM_BUCKETS);
buckets[bucket].insert((short, starting_event));
}
trace!(
starting_events = ?starting_events.count(),
elapsed = ?started.elapsed(),
"start",
);
let full_auth_chain: Vec<ShortEventId> = buckets
.into_iter()
.try_stream()
.broad_and_then(|chunk| self.get_auth_chain_outer(room_id, started, chunk))
.try_collect()
.map_ok(|auth_chain: Vec<_>| auth_chain.into_iter().flatten().collect())
.map_ok(|mut full_auth_chain: Vec<_>| {
full_auth_chain.sort_unstable();
full_auth_chain.dedup();
full_auth_chain
})
.boxed()
.await?;
trace!(
chain_length = ?full_auth_chain.len(),
elapsed = ?started.elapsed(),
"done",
);
Ok(full_auth_chain)
}
Ok(found.into_iter().collect())
async fn get_auth_chain_outer(
&self,
room_id: &RoomId,
started: Instant,
chunk: Bucket<'_>,
) -> Result<Vec<ShortEventId>> {
let chunk_key: Vec<ShortEventId> = chunk.iter().map(at!(0)).collect();
if chunk_key.is_empty() {
return Ok(Vec::new());
}
if let Ok(cached) = self.get_cached_eventid_authchain(&chunk_key).await {
return Ok(cached.to_vec());
}
let chunk_cache: Vec<_> = chunk
.into_iter()
.try_stream::<conduwuit::Error>()
.broad_and_then(|(shortid, event_id)| async move {
if let Ok(cached) = self.get_cached_eventid_authchain(&[shortid]).await {
return Ok(cached.to_vec());
}
let auth_chain = self.get_auth_chain_inner(room_id, event_id).await?;
self.cache_auth_chain_vec(vec![shortid], auth_chain.as_slice());
debug!(
%event_id,
elapsed = ?started.elapsed(),
"Cache missed event"
);
Ok(auth_chain)
})
.try_collect()
.map_ok(|chunk_cache: Vec<_>| chunk_cache.into_iter().flatten().collect())
.map_ok(|mut chunk_cache: Vec<_>| {
chunk_cache.sort_unstable();
chunk_cache.dedup();
chunk_cache
})
.await?;
self.cache_auth_chain_vec(chunk_key, chunk_cache.as_slice());
debug!(
chunk_cache_length = ?chunk_cache.len(),
elapsed = ?started.elapsed(),
"Cache missed chunk",
);
Ok(chunk_cache)
}
#[tracing::instrument(name = "inner", level = "trace", skip(self, room_id))]
async fn get_auth_chain_inner(
&self,
room_id: &RoomId,
event_id: &EventId,
) -> Result<Vec<ShortEventId>> {
let mut todo: VecDeque<_> = [event_id.to_owned()].into();
let mut found = HashSet::new();
while let Some(event_id) = todo.pop_front() {
trace!(%event_id, "processing auth event");
match self.services.timeline.get_pdu(&event_id).await {
| Err(e) => {
debug_error!(%event_id, ?e, "Could not find pdu mentioned in auth events");
},
| Ok(pdu) => {
if let Some(claimed_room_id) = pdu.room_id.clone() {
if claimed_room_id != *room_id {
return Err!(Request(Forbidden(error!(
%event_id,
%room_id,
wrong_room_id = ?pdu.room_id.unwrap(),
"auth event for incorrect room"
))));
}
}
for auth_event in &pdu.auth_events {
let sauthevent = self
.services
.short
.get_or_create_shorteventid(auth_event)
.await;
if found.insert(sauthevent) {
trace!(%event_id, ?auth_event, "adding auth event to processing queue");
todo.push_back(auth_event.clone());
}
}
},
}
}
Ok(found.into_iter().collect())
}
#[inline]
pub async fn get_cached_eventid_authchain(&self, key: &[u64]) -> Result<Arc<[ShortEventId]>> {
self.db.get_cached_eventid_authchain(key).await
}
pub fn cache_auth_chain(&self, key: Vec<u64>, auth_chain: &HashSet<ShortEventId>) {
let val: Arc<[ShortEventId]> = auth_chain.iter().copied().collect();
self.db.cache_auth_chain(key, val);
}
pub fn cache_auth_chain_vec(&self, key: Vec<u64>, auth_chain: &[ShortEventId]) {
let val: Arc<[ShortEventId]> = auth_chain.iter().copied().collect();
self.db.cache_auth_chain(key, val);
}
pub fn get_cache_usage(&self) -> (usize, usize) {
let cache = self.db.auth_chain_cache.lock();
(cache.len(), cache.capacity())
}
pub fn clear_cache(&self) { self.db.auth_chain_cache.lock().clear(); }
}
#[implement(Service)]
#[inline]
pub async fn get_cached_eventid_authchain(&self, key: &[u64]) -> Result<Arc<[ShortEventId]>> {
self.db.get_cached_eventid_authchain(key).await
}
#[implement(Service)]
#[tracing::instrument(skip_all, level = "debug")]
pub fn cache_auth_chain(&self, key: Vec<u64>, auth_chain: &HashSet<ShortEventId>) {
let val: Arc<[ShortEventId]> = auth_chain.iter().copied().collect();
self.db.cache_auth_chain(key, val);
}
#[implement(Service)]
#[tracing::instrument(skip_all, level = "debug")]
pub fn cache_auth_chain_vec(&self, key: Vec<u64>, auth_chain: &[ShortEventId]) {
let val: Arc<[ShortEventId]> = auth_chain.iter().copied().collect();
self.db.cache_auth_chain(key, val);
}
#[implement(Service)]
pub fn get_cache_usage(&self) -> (usize, usize) {
let cache = self.db.auth_chain_cache.lock();
(cache.len(), cache.capacity())
}
#[implement(Service)]
pub fn clear_cache(&self) { self.db.auth_chain_cache.lock().clear(); }
+22 -19
View File
@@ -1,6 +1,6 @@
use std::sync::Arc;
use conduwuit::{Result, implement, utils::stream::TryIgnore};
use conduwuit::{Result, utils::stream::TryIgnore};
use database::Map;
use futures::Stream;
use ruma::{OwnedRoomId, RoomId, api::client::room::Visibility};
@@ -25,27 +25,30 @@ impl crate::Service for Service {
fn name(&self) -> &str { crate::service::make_name(std::module_path!()) }
}
#[implement(Service)]
pub fn set_public(&self, room_id: &RoomId) { self.db.publicroomids.insert(room_id, []); }
impl Service {
/// Marks a room as public, adding it to the room directory. Has no relation
/// to the join rule.
pub fn set_public(&self, room_id: &RoomId) { self.db.publicroomids.insert(room_id, []); }
#[implement(Service)]
pub fn set_not_public(&self, room_id: &RoomId) { self.db.publicroomids.remove(room_id); }
/// Removes a room from the public room directory.
pub fn set_not_public(&self, room_id: &RoomId) { self.db.publicroomids.remove(room_id); }
#[implement(Service)]
pub fn public_rooms(&self) -> impl Stream<Item = OwnedRoomId> + Send {
self.db.publicroomids.keys().ignore_err()
}
/// Lists all public rooms in the directory.
pub fn public_rooms(&self) -> impl Stream<Item = OwnedRoomId> + Send {
self.db.publicroomids.keys().ignore_err()
}
#[implement(Service)]
pub async fn is_public_room(&self, room_id: &RoomId) -> bool {
self.visibility(room_id).await == Visibility::Public
}
/// Checks if a room is public (per visibility, not join rule).
pub async fn is_public_room(&self, room_id: &RoomId) -> bool {
self.visibility(room_id).await == Visibility::Public
}
#[implement(Service)]
pub async fn visibility(&self, room_id: &RoomId) -> Visibility {
if self.db.publicroomids.get(room_id).await.is_ok() {
Visibility::Public
} else {
Visibility::Private
/// Fetches the visibility of a specific room ID.
pub async fn visibility(&self, room_id: &RoomId) -> Visibility {
if self.db.publicroomids.get(room_id).await.is_ok() {
Visibility::Public
} else {
Visibility::Private
}
}
}
+30 -32
View File
@@ -1,42 +1,40 @@
use conduwuit::{Err, Result, debug, implement, trace, warn};
use conduwuit::{Err, Result, debug, debug_warn, trace};
use ruma::{
RoomId, ServerName,
events::{StateEventType, room::server_acl::RoomServerAclEventContent},
};
/// Returns Ok if the acl allows the server
#[implement(super::Service)]
#[tracing::instrument(skip_all, level = "debug")]
pub async fn acl_check(&self, server_name: &ServerName, room_id: &RoomId) -> Result {
let Ok(acl_event_content) = self
.services
.state_accessor
.room_state_get_content(room_id, &StateEventType::RoomServerAcl, "")
.await
.map(|c: RoomServerAclEventContent| c)
.inspect(|acl| trace!(%room_id, "ACL content found: {acl:?}"))
.inspect_err(|e| trace!(%room_id, "No ACL content found: {e:?}"))
else {
return Ok(());
};
impl super::Service {
/// Returns Ok if the acl allows the server, otherwise Forbidden.
pub async fn acl_check(&self, server_name: &ServerName, room_id: &RoomId) -> Result {
let Ok(acl_event_content) = self
.services
.state_accessor
.room_state_get_content(room_id, &StateEventType::RoomServerAcl, "")
.await
.map(|c: RoomServerAclEventContent| c)
else {
return Ok(());
};
if acl_event_content.allow.is_empty() {
warn!(%room_id, "Ignoring broken ACL event (allow key is empty)");
return Ok(());
}
if acl_event_content.allow.is_empty() {
debug_warn!(%room_id, "Ignoring broken ACL event (allow key is empty)");
return Ok(());
}
if acl_event_content.deny.contains(&String::from("*"))
&& acl_event_content.allow.contains(&String::from("*"))
{
warn!(%room_id, "Ignoring broken ACL event (allow key and deny key both contain wildcard \"*\"");
return Ok(());
}
if acl_event_content.deny.contains(&String::from("*"))
&& acl_event_content.allow.contains(&String::from("*"))
{
debug_warn!(%room_id, "Ignoring broken ACL event (allow key and deny key both contain wildcard \"*\"");
return Ok(());
}
if acl_event_content.is_allowed(server_name) {
trace!("server {server_name} is allowed by ACL");
Ok(())
} else {
debug!("Server {server_name} was denied by room ACL in {room_id}");
Err!(Request(Forbidden("Server was denied by room ACL")))
if acl_event_content.is_allowed(server_name) {
trace!("server {server_name} is allowed by ACL");
Ok(())
} else {
debug!("Server {server_name} was denied by room ACL in {room_id}");
Err!(Request(Forbidden("Server was denied by room ACL")))
}
}
}
@@ -1,233 +1,667 @@
use std::{
collections::{BTreeMap, HashSet, VecDeque, hash_map},
collections::{HashMap, HashSet, VecDeque},
time::Instant,
};
use assign::assign;
#[cfg(debug_assertions)]
use conduwuit::error;
use conduwuit::{
Event, PduEvent, debug, debug_warn, implement, matrix::event::gen_event_id_canonical_json,
trace, utils::continue_exponential_backoff_secs, warn,
Err, Event, PduEvent, debug, debug_error, debug_info, debug_warn, err,
state_res::lexicographical_topological_sort,
trace,
utils::{IterStream, math::Expected, stream::BroadbandExt},
warn,
};
use futures::{StreamExt, future::select_ok};
use ruma::{
CanonicalJsonValue, EventId, OwnedEventId, RoomId, ServerName,
api::federation::event::get_event,
CanonicalJsonObject, CanonicalJsonValue, EventId, MilliSecondsSinceUnixEpoch, OwnedEventId,
OwnedServerName, RoomId, ServerName, UInt,
api::federation::event::{get_event, get_missing_events},
int,
room_version_rules::RoomVersionRules,
};
use super::get_room_version_rules;
use crate::rooms::event_handler::parse_incoming_pdu::expect_event_id_array;
/// Find the event and auth it. Once the event is validated (steps 1 - 8)
/// it is appended to the outliers Tree.
pub const GET_MISSING_EVENTS_MAX_BATCH_SIZE: usize = 50;
/// Attempts to build a localised directed acyclic graph out of the given PDUs,
/// returning them in a topologically sorted order.
///
/// Returns pdu and if we fetched it over federation the raw json.
///
/// a. Look in the main timeline (pduid_pdu tree)
/// b. Look at outlier pdu tree
/// c. Ask origin server over federation
/// d. TODO: Ask other servers over federation?
#[implement(super::Service)]
pub(super) async fn fetch_and_handle_outliers<'a, Pdu, Events>(
&self,
origin: &'a ServerName,
events: Events,
create_event: &'a Pdu,
room_id: &'a RoomId,
) -> Vec<(PduEvent, Option<BTreeMap<String, CanonicalJsonValue>>)>
where
Pdu: Event + Send + Sync,
Events: Iterator<Item = &'a EventId> + Clone + Send,
{
let back_off = |id| match self
.services
.globals
.bad_event_ratelimiter
.write()
.entry(id)
{
| hash_map::Entry::Vacant(e) => {
e.insert((Instant::now(), 1));
},
| hash_map::Entry::Occupied(mut e) => {
*e.get_mut() = (Instant::now(), e.get().1.saturating_add(1));
},
};
/// This is used to attempt to process PDUs in an order that respects their
/// dependencies, however it is ultimately the sender's responsibility to send
/// them in a processable order, so this is just a best effort attempt. It does
/// not account for power levels or other tie breaks.
pub async fn build_local_dag<S: std::hash::BuildHasher + Send + Sync>(
pdu_map: &HashMap<OwnedEventId, &CanonicalJsonObject, S>,
) -> conduwuit::Result<Vec<OwnedEventId>> {
debug_assert!(pdu_map.len() >= 2, "needless call to build_local_dag with less than 2 PDUs");
let mut dag: HashMap<OwnedEventId, HashSet<OwnedEventId>> =
HashMap::with_capacity(pdu_map.len());
let mut id_origin_ts: HashMap<OwnedEventId, _> = HashMap::with_capacity(pdu_map.len());
let mut events_with_auth_events = Vec::with_capacity(events.clone().count());
trace!("Fetching {} outlier pdus", events.clone().count());
for (event_id, value) in pdu_map {
// We already checked that these properties are correct in parse_incoming_pdu,
// so it's safe to unwrap here.
// We also filter to remove any prev_events that are not in this pdu_map, as we
// need to have at least one event with zero out degrees for the lexico-topo
// sort below. If there are multiple events with omitted prevs, they will be
// ordered by timestamp, then event ID. At that point though, it's unlikely to
// matter.
let prev_events = value
.get("prev_events")
.unwrap()
.as_array()
.unwrap()
.iter()
.map(|v| EventId::parse(v.as_str().unwrap()).unwrap())
.filter(|id| pdu_map.contains_key(id))
.collect();
for id in events {
// a. Look in the main timeline (pduid_pdu tree)
// b. Look at outlier pdu tree
// (get_pdu_json checks both)
if let Ok(local_pdu) = self.services.timeline.get_pdu(id).await {
trace!("Found {id} in main timeline or outlier tree");
events_with_auth_events.push((id.to_owned(), Some(local_pdu), vec![]));
continue;
}
dag.insert(event_id.clone(), prev_events);
let origin_server_ts = value
.get("origin_server_ts")
.and_then(CanonicalJsonValue::as_integer)
.unwrap_or_default();
id_origin_ts.insert(event_id.clone(), origin_server_ts);
}
// c. Ask origin server over federation
// We also handle its auth chain here so we don't get a stack overflow in
// handle_outlier_pdu.
let mut todo_auth_events: VecDeque<_> = [id.to_owned()].into();
let mut events_in_reverse_order = Vec::with_capacity(todo_auth_events.len());
debug!(count = dag.len(), "Sorting incoming events with partial graph");
lexicographical_topological_sort(&dag, &async |node_id| {
// Note: we don't bother fetching power levels because that would massively slow
// this function down. This is a best-effort attempt to order events correctly
// for processing, however ultimately that should be the sender's job.
let ts = id_origin_ts
.get(&node_id)
.copied()
.unwrap_or_else(|| int!(0))
.to_string()
.parse::<u64>()
.ok()
.and_then(UInt::new)
.unwrap_or_default();
Ok((int!(0), MilliSecondsSinceUnixEpoch(ts)))
})
.await
.inspect(|sorted| {
debug_assert_eq!(
sorted.len(),
pdu_map.len(),
"Sorted graph was not the same size as the input graph"
);
})
.map_err(|e| err!("failed to resolve local graph: {e}"))
}
let mut events_all = HashSet::with_capacity(todo_auth_events.len());
while let Some(next_id) = todo_auth_events.pop_front() {
if let Some((time, tries)) = self
.services
.globals
.bad_event_ratelimiter
.read()
.get(&*next_id)
{
// Exponential backoff
const MIN_DURATION: u64 = 60 * 2;
const MAX_DURATION: u64 = 60 * 60 * 8;
if continue_exponential_backoff_secs(
MIN_DURATION,
MAX_DURATION,
time.elapsed(),
*tries,
) {
debug_warn!(
tried = ?*tries,
elapsed = ?time.elapsed(),
"Backing off from {next_id}",
);
continue;
}
}
impl super::Service {
/// Uses `POST /_matrix/federation/v1/get_missing_events/{room_id}` to fill
/// gaps in the DAG.
///
/// This function walks backwards from `head`, fetching incrementally (by a
/// factor of 10) more events until the remote we're fetching from either
/// stops returning new events, or the min_depth is reached.
///
/// This function does not persist the events, but does validate them. The
/// caller is responsible for passing them through handle_incoming_pdu or
/// related functions.
///
/// Only the one `via` is asked for missing events, as multiplexing remotes
/// may result in the event tree being walked in a gappy or disordered
/// manner.
///
/// ## Parameters
///
/// - `room_id`: The room's ID.
/// - `head`: The event we are potentially missing prev_events for.
/// - `tail`: The most recently known events in the graph (typically forward
/// extremities).
/// - `via`: The server to ask for missing events.
/// - `min_depth`: Don't process events with a `depth` lower than this
/// value. Not massively useful, but can help short-circuit infinite loops
/// and weird edge paths.
#[tracing::instrument(name = "get_missing_events_bulk", skip_all)]
pub async fn get_missing_events(
&self,
room_id: &RoomId,
head: &PduEvent,
tail: Vec<OwnedEventId>,
via: &ServerName,
min_depth: UInt,
) -> conduwuit::Result<HashMap<OwnedEventId, PduEvent>> {
let start = Instant::now();
#[cfg(debug_assertions)]
{
let missing_count = head
.prev_events()
.stream()
.fold(0_u8, |i, event_id| async move {
if self.services.timeline.pdu_exists(event_id).await {
i.expected_add(1)
} else {
i
}
})
.await;
debug_assert_ne!(
missing_count, 0,
"event passed to get_missing_events is not missing any events (wasteful call)"
);
};
assert!(!tail.is_empty(), "empty tail");
assert_ne!(via, self.services.globals.server_name(), "cannot ask ourselves for events");
if events_all.contains(&next_id) {
continue;
}
// The iteration limit is in place to ensure that if the remote server leaves us
// in a state of infinite recursion (as old versions of continuwuity and
// predecessors would), we give up. However, get_missing_events doesn't return
// that many events per-request. Synapse returns 20, and conduwuit+ return 50.
// This means with a hard iteration limit, we might give up too early, before
// we get a chance to even come close to max_fetch_prev_events. As such, we'll
// calculate the limit based on that config option and the aforementioned
// averages.
let max_fetch = self.services.server.config.max_fetch_prev_events;
let iteration_limit = max_fetch.saturating_div(20).max(10);
if self.services.timeline.pdu_exists(&next_id).await {
trace!("Found {next_id} in db");
continue;
}
debug!("Fetching {next_id} over federation from {origin}.");
match self
let mut discovered = HashMap::with_capacity(head.prev_events.len());
let mut latest_events: Vec<OwnedEventId> = vec![head.event_id().to_owned()];
debug!(elapsed=?start.elapsed(),
%room_id,
event_id=%head.event_id(),
%iteration_limit,
"Fetching any missing events for head event",
);
for iteration in 0..iteration_limit {
let limit = iteration
.expected_add(1)
.saturating_mul(10)
.min(GET_MISSING_EVENTS_MAX_BATCH_SIZE.try_into().expect(
"GET_MISSING_EVENTS_MAX_BATCH_SIZE (usize) should fit in u16 (<=65536)",
))
.max(
// This max call ensures we fetch *at least* all the prev events the
// head has.
u16::try_from(head.prev_events.len())
.expect("cannot have more than 20 prev events, which fits in u16"),
);
debug_info!(elapsed=?start.elapsed(),
%limit,
%via,
%iteration,
%iteration_limit,
discovered=discovered.len(),
%min_depth,
"Attempting to gap fill missing events"
);
let response: get_missing_events::v1::Response = self
.services
.sending
.send_federation_request(
origin,
get_event::v1::Request::new((*next_id).to_owned()),
via,
assign!(
get_missing_events::v1::Request::new(
room_id.to_owned(),
tail.clone(),
latest_events.clone()
),
{limit: limit.into(), min_depth}
),
)
.await
{
| Ok(res) => {
debug!("Got {next_id} over federation from {origin}");
let Ok(room_version_rules) = get_room_version_rules(create_event) else {
back_off((*next_id).to_owned());
continue;
};
.await?;
let Ok((calculated_event_id, value)) =
gen_event_id_canonical_json(&res.pdu, &room_version_rules)
else {
back_off((*next_id).to_owned());
continue;
};
if calculated_event_id != *next_id {
warn!(
"Server didn't return event id we requested: requested: {next_id}, \
we got {calculated_event_id}. Event: {:?}",
&res.pdu
);
}
if let Some(auth_events) = value
.get("auth_events")
.and_then(CanonicalJsonValue::as_array)
{
for auth_event in auth_events {
match serde_json::from_value::<OwnedEventId>(
auth_event.clone().into(),
) {
| Ok(auth_event) => {
trace!(
"Found auth event id {auth_event} for event {next_id}"
);
todo_auth_events.push_back(auth_event);
},
| _ => {
warn!("Auth event id is not valid");
},
}
}
} else {
warn!("Auth event list invalid");
}
events_in_reverse_order.push((next_id.clone(), value));
events_all.insert(next_id);
},
| Err(e) => {
warn!("Failed to fetch auth event {next_id} from {origin}: {e}");
back_off((*next_id).to_owned());
},
if response.events.is_empty() {
debug_info!(
elapsed=?start.elapsed(),
%via,
"Finished gap filling missing events (remote returned no more events)."
);
break;
}
}
debug_info!(
elapsed=?start.elapsed(),
"Got {} events back from remote",
response.events.len()
);
events_with_auth_events.push((id.to_owned(), None, events_in_reverse_order));
}
let mut pdus = Vec::with_capacity(events_with_auth_events.len());
for (id, local_pdu, events_in_reverse_order) in events_with_auth_events {
// a. Look in the main timeline (pduid_pdu tree)
// b. Look at outlier pdu tree
// (get_pdu_json checks both)
if let Some(local_pdu) = local_pdu {
trace!("Found {id} in main timeline or outlier tree");
pdus.push((local_pdu.clone(), None));
}
for (next_id, value) in events_in_reverse_order.into_iter().rev() {
if let Some((time, tries)) = self
.services
.globals
.bad_event_ratelimiter
.read()
.get(&*next_id)
{
// Exponential backoff
const MIN_DURATION: u64 = 5 * 60;
const MAX_DURATION: u64 = 60 * 60 * 24;
if continue_exponential_backoff_secs(
MIN_DURATION,
MAX_DURATION,
time.elapsed(),
*tries,
) {
debug!("Backing off from {next_id}");
latest_events.clear();
for raw_event in response.events {
let (_, event_id, pdu_json) = self.parse_incoming_pdu(&raw_event).await?;
let pdu = PduEvent::from_id_val(&event_id, pdu_json).map_err(|e| {
err!(Request(BadJson("Failed to parse gapfilled event {event_id}: {e}")))
})?;
if discovered.contains_key(&event_id) {
// We already received this event.
trace!("Already received {event_id}");
continue;
}
if self
.services
.timeline
.non_outlier_pdu_exists(&event_id)
.await
{
// NOTE: we explicitly check for *non*-outlier events here, as if we end
// up discovering outlier events, we will be able to upgrade them
// immediately.
trace!("Already have {event_id} as a timeline PDU");
continue;
}
if pdu.depth < min_depth {
debug_warn!(
elapsed=?start.elapsed(),
"Received PDU with depth {} below min_depth {}",
pdu.depth,
min_depth
);
discovered.insert(event_id.clone(), pdu);
continue;
}
for prev_event_id in pdu.prev_events() {
if discovered.contains_key(prev_event_id) {
// We already received this event.
trace!("Already received prev event {prev_event_id}");
continue;
}
if self
.services
.timeline
.non_outlier_pdu_exists(prev_event_id)
.await
{
// NOTE: we explicitly check for *non*-outlier events here, as if we end
// up discovering outlier events, we will be able to upgrade them
// immediately.
trace!("Already have prev event {prev_event_id} as a timeline PDU");
continue;
}
if let Ok(outlier) = self.services.timeline.get_pdu(prev_event_id).await {
// We already have this PDU as an outlier, don't ask for
// it. However, if we are missing any prev events for it, add it to the
// latest events anyway.
let outlier_missing_prevs = outlier
.prev_events()
.stream()
.fold(0_u8, |i, event_id| async move {
if self.services.timeline.pdu_exists(event_id).await {
i.expected_add(1)
} else {
i
}
})
.await;
if outlier_missing_prevs > 0 {
trace!("Missing {outlier_missing_prevs} PDU(s) for prev event");
latest_events.push(prev_event_id.to_owned());
}
trace!("Had {prev_event_id} as an outlier already, skipping discovery");
discovered.insert(prev_event_id.to_owned(), outlier);
continue;
}
trace!("Missing prev {prev_event_id} of {event_id}");
latest_events.push(prev_event_id.to_owned());
}
trace!("Discovered {event_id}");
discovered.insert(event_id.clone(), pdu);
}
trace!("Handling outlier {next_id}");
if latest_events.is_empty() {
debug!(elapsed=?start.elapsed(),
%limit,
%via,
%iteration,
discovered=discovered.len(),
"No more events to fetch."
);
break;
}
if discovered.len() >= self.services.server.config.max_fetch_prev_events.into() {
// Stupid hack, debug_error!() drops the log to a DEBUG when not in debug mode,
// which is bad because this should at least produce a warning. It's an error in
// debug mode because this can be important, but typically not much can be done
// about it as a user.
#[cfg(debug_assertions)]
error!(elapsed=?start.elapsed(),
discovered=discovered.len(),
max_fetch_prev_events=self.services.server.config.max_fetch_prev_events,
%iteration,
%iteration_limit,
%via,
event_id=%head.event_id(),
%room_id,
"Encountered a gap too large to fill, giving up"
);
#[cfg(not(debug_assertions))]
warn!(elapsed=?start.elapsed(),
discovered=discovered.len(),
max_fetch_prev_events=self.services.server.config.max_fetch_prev_events,
%iteration,
%iteration_limit,
%via,
event_id=%head.event_id(),
%room_id,
"Encountered a gap too large to fill"
);
break;
}
}
trace!(elapsed=?start.elapsed(), "Finished get_missing_events");
Ok(discovered)
}
/// Sends a `GET /_matrix/federation/v1/event/{event_id}` request to the
/// target `remote`, parses the resulting PDU, and ensures the remote
/// returned the correct event.
/// Allows `fetch_and_handle_missing_events` to atomically fetch events from
/// multiple remotes in parallel.
async fn fetch_event_via(
&self,
remote: OwnedServerName,
event_id: OwnedEventId,
room_version_rules: &RoomVersionRules,
) -> conduwuit::Result<(OwnedEventId, CanonicalJsonObject)> {
let res = self
.services
.sending
.send_federation_request(&remote, get_event::v1::Request::new(event_id.clone()))
.await?;
let (calculated_event_id, value) = self
.parse_incoming_pdu_with_known_room(&res.pdu, room_version_rules)
.await?;
if calculated_event_id != event_id {
Err!(Request(BadJson(warn!(
expected=%event_id,
received=%calculated_event_id,
"Server didn't return event id we requested",
))))
} else {
Ok((event_id, value))
}
}
async fn fetch_event_vias(
&self,
candidates: impl Iterator<Item = &OwnedServerName>,
event_id: &EventId,
room_version_rules: &RoomVersionRules,
) -> conduwuit::Result<(OwnedEventId, CanonicalJsonObject)> {
if let Ok(pdu_json) = self.services.timeline.get_pdu_json(event_id).await {
return Ok((event_id.to_owned(), pdu_json));
}
let futures = candidates
.map(|remote| {
Box::pin(self.fetch_event_via(
remote.to_owned(),
event_id.to_owned(),
room_version_rules,
))
})
.collect::<Vec<_>>();
select_ok(futures).await.map(|(res, _)| res)
}
/// Asks remote servers for any individual events that are missing, also
/// known as "atomic fetch". Should only be used for fetching missing auth
/// events or resolving missing events from state_ids. For all other uses,
/// use get_missing_events.
///
/// This function manually walks auth_events trees in a breadth-first
/// search, and persists all fetched events as outliers when all the
/// backwards extremities have been resolved.
#[tracing::instrument(name = "get_missing_auth_events_atomic", skip_all)]
pub(super) async fn fetch_and_handle_auth_events<Pdu>(
&self,
origin: &ServerName,
events: Vec<OwnedEventId>,
create_event: &Pdu,
room_id: &RoomId,
) -> HashMap<OwnedEventId, PduEvent>
where
Pdu: Event + Send + Sync,
{
let start = Instant::now();
let room_version_rules =
&get_room_version_rules(create_event).unwrap_or(RoomVersionRules::V1);
let mut candidates = self
.services
.timeline
.candidate_backfill_servers(room_id)
.await;
candidates.insert(origin.to_owned());
assert!(!candidates.is_empty(), "no candidates to fetch missing events from");
let mut discovered_events =
HashMap::with_capacity(events.len().saturating_add(events.len().saturating_mul(3)));
trace!(
elapsed=?start.elapsed(),
"Fetching {} unknown PDUs on demand from {} candidates",
events.len(),
candidates.len()
);
let mut seen: HashMap<OwnedEventId, u8> = HashMap::new();
for apex_event_id in &events {
let mut todo: VecDeque<OwnedEventId> = [apex_event_id.to_owned()].into();
while let Some(target_id) = todo.pop_front() {
if discovered_events.contains_key(&target_id) {
continue;
}
if let Ok(local_pdu) = self.services.timeline.get_pdu(&target_id).await {
trace!(elapsed=?start.elapsed(), "Found {target_id} in db");
let mut obj = local_pdu.into_canonical_object();
obj.remove("event_id");
discovered_events.insert(target_id.clone(), obj);
continue;
}
let attempts = seen.get(&*target_id).copied().unwrap_or_default();
if attempts >= 5 {
debug_error!(
elapsed=?start.elapsed(),
%attempts,
%target_id,
"Could not fetch missing event after 5 attempts, giving up"
);
continue;
}
debug!(elapsed=?start.elapsed(),"Fetching {target_id} over federation");
let value = match self
.fetch_event_vias(candidates.iter(), &target_id, room_version_rules)
.await
{
| Ok((_, x)) => x,
| Err(e) => {
warn!(elapsed=?start.elapsed(),"failed to fetch missing event {target_id} from any candidate: {e}");
continue;
},
};
let auth_events =
match expect_event_id_array(&value, "auth_events").map_err(|e| {
err!(Request(BadJson(warn!(
elapsed=?start.elapsed(),
event_id=%target_id,
"Failed to parse event fetched from remote: {e}"
))))
}) {
| Ok(auth_events) => auth_events,
| Err(e) => {
warn!(
elapsed=?start.elapsed(),
?e,
"event {target_id} is malformed (bad auth_events), skipping"
);
continue;
},
};
let mut have_all_auth = true;
for auth_event_id in auth_events {
if let Ok(local_pdu) = self.services.timeline.get_pdu(&auth_event_id).await {
trace!(elapsed=?start.elapsed(),"Found auth event {auth_event_id} in db");
let mut obj = local_pdu.into_canonical_object();
obj.remove("event_id");
discovered_events.insert(auth_event_id.clone(), obj);
continue;
}
if discovered_events.contains_key(&auth_event_id) {
trace!(elapsed=?start.elapsed(),%auth_event_id, "Already found auth event");
continue;
}
debug!(elapsed=?start.elapsed(),"Missing auth event {auth_event_id} for event {target_id}");
seen.insert(
auth_event_id.clone(),
seen.get(&auth_event_id)
.copied()
.unwrap_or_default()
.saturating_add(1),
);
todo.push_back(auth_event_id);
have_all_auth = false;
}
// Insert this PDU back at the end of the queue so that it will be resolved once
// all of its auth events have been fetched.
if have_all_auth {
debug!(elapsed=?start.elapsed(),%target_id, "Have all auth events");
discovered_events.insert(target_id, value);
} else {
debug_warn!(elapsed=?start.elapsed(),
"Fetched {target_id} but missing some auth events, will have to re-fetch."
);
seen.insert(target_id.clone(), attempts.saturating_add(1));
todo.push_back(target_id);
}
}
}
let refmap: HashMap<OwnedEventId, &CanonicalJsonObject> = discovered_events
.iter()
.map(|(id, data)| (id.clone(), data))
.collect();
let seeded_ordered = build_local_dag(&refmap)
.await
.expect("failed to build local DAG");
let mut pdus = HashMap::with_capacity(seeded_ordered.len());
for discovered_event_id in seeded_ordered {
let pdu_json = discovered_events.remove(&discovered_event_id).unwrap();
debug_info!(
elapsed=?start.elapsed(),
"Handling missing event {discovered_event_id} as outlier"
);
assert_eq!(pdu_json.get("event_id"), None, "pdu_json had event_id");
match Box::pin(self.handle_outlier_pdu(
origin,
create_event,
&next_id,
&discovered_event_id,
room_id,
value.clone(),
true,
pdu_json,
))
.await
{
| Ok((pdu, json)) =>
if next_id == *id {
trace!("Handled outlier {next_id} (original request)");
pdus.push((pdu, Some(json)));
},
| Err(e) => {
warn!("Authentication of event {next_id} failed: {e:?}");
back_off(next_id);
| Ok((pdu, _)) => {
trace!(elapsed=?start.elapsed(), "Persisted {discovered_event_id}");
let _ = pdus.insert(discovered_event_id, pdu);
},
| Err(e) => warn!(
elapsed=?start.elapsed(),
"Authentication of event {discovered_event_id} failed: {e:?}"
),
}
}
trace!(
elapsed=?start.elapsed(),
"Finished fetch_and_handle_missing_events: fetched and handled {} missing PDUs",
pdus.len()
);
pdus.retain(|id, _| events.contains(id)); // Only return state events
trace!(elapsed=?start.elapsed(), "Filtered return value down to {} PDUs", pdus.len());
pdus
}
/// Similar to `fetch_and_handle_missing_events`, but simply walks the
/// prev events tree instead of the auth events tree. Additionally, it does
/// not *handle* fetched PDUs in any capacity.
#[tracing::instrument(name = "get_missing_prev_events_atomic", skip_all)]
pub(super) async fn fetch_prev_events<Pdu>(
&self,
origin: &ServerName,
events: Vec<OwnedEventId>,
create_event: &Pdu,
room_id: &RoomId,
) -> HashMap<OwnedEventId, PduEvent>
where
Pdu: Event + Send + Sync,
{
let room_version_rules =
&get_room_version_rules(create_event).unwrap_or(RoomVersionRules::V1);
let mut candidates = self
.services
.timeline
.candidate_backfill_servers(room_id)
.await;
candidates.insert(origin.to_owned());
let mut todo: VecDeque<OwnedEventId> = VecDeque::from(events);
let mut discovered_events = HashMap::new();
while let Some(next_id) = todo.pop_front() {
if discovered_events.len() >= self.services.server.config.max_fetch_prev_events.into()
{
debug_warn!(
"Encountered a gap too large to fill, giving up (fetched {} events)",
discovered_events.len()
);
break;
}
if discovered_events.contains_key(&next_id) {
continue;
}
let pdu = match self
.fetch_event_vias(candidates.iter(), &next_id, room_version_rules)
.await
{
| Ok((_, data)) => data,
| Err(e) => {
warn!("Failed to fetch prev event {next_id} from any candidate: {e}");
continue;
},
};
let prev_events = match expect_event_id_array(&pdu, "prev_events").map_err(|e| {
err!(Request(BadJson(warn!(
event_id=%next_id,
"Failed to parse event fetched from remote: {e}"
))))
}) {
| Ok(auth_events) => auth_events,
| Err(e) => {
warn!(?e, "event {next_id} is malformed (bad prev_events), skipping");
continue;
},
};
let missing_prev = prev_events
.iter()
.stream()
.broad_filter_map(|event_id| async {
if discovered_events.contains_key(event_id)
|| self.services.timeline.pdu_exists(event_id).await
{
None
} else {
Some(event_id.to_owned())
}
})
.collect::<Vec<_>>()
.await;
todo.extend(missing_prev);
discovered_events.insert(
next_id.clone(),
PduEvent::from_id_val(&next_id, pdu).expect("fetched PDU was already validated"),
);
}
discovered_events
}
trace!("Fetched and handled {} outlier pdus", pdus.len());
pdus
}
+147 -120
View File
@@ -1,128 +1,155 @@
use std::{
collections::{BTreeMap, HashMap, HashSet, VecDeque},
iter::once,
};
use std::{collections::HashMap, time::Instant};
use conduwuit::{
Event, PduEvent, Result, debug_warn, err, implement,
state_res::{self},
};
use futures::{FutureExt, future};
use ruma::{
CanonicalJsonValue, EventId, MilliSecondsSinceUnixEpoch, OwnedEventId, RoomId, ServerName,
int, uint,
Event, PduEvent, debug, debug_info, debug_warn, trace,
utils::{BoolExt, IterStream, stream::BroadbandExt},
};
use futures::StreamExt;
use ruma::{CanonicalJsonObject, MilliSecondsSinceUnixEpoch, OwnedEventId, RoomId, ServerName};
use super::check_room_id;
use crate::rooms::event_handler::build_local_dag;
#[implement(super::Service)]
#[tracing::instrument(
level = "debug",
skip_all,
fields(%origin),
)]
#[allow(clippy::type_complexity)]
pub(super) async fn fetch_prev<'a, Pdu, Events>(
&self,
origin: &ServerName,
create_event: &Pdu,
room_id: &RoomId,
first_ts_in_room: MilliSecondsSinceUnixEpoch,
initial_set: Events,
) -> Result<(
Vec<OwnedEventId>,
HashMap<OwnedEventId, (PduEvent, BTreeMap<String, CanonicalJsonValue>)>,
)>
where
Pdu: Event + Send + Sync,
Events: Iterator<Item = &'a EventId> + Clone + Send,
{
let num_ids = initial_set.clone().count();
let mut eventid_info = HashMap::new();
let mut graph: HashMap<OwnedEventId, _> = HashMap::with_capacity(num_ids);
let mut todo_outlier_stack: VecDeque<OwnedEventId> =
initial_set.map(ToOwned::to_owned).collect();
let mut amount = 0;
while let Some(prev_event_id) = todo_outlier_stack.pop_front() {
self.services.server.check_running()?;
match self
.fetch_and_handle_outliers(
origin,
once(prev_event_id.as_ref()),
create_event,
room_id,
)
.boxed()
.await
.pop()
{
| Some((pdu, mut json_opt)) => {
check_room_id(room_id, &pdu)?;
let limit = self.services.server.config.max_fetch_prev_events;
if amount > limit {
debug_warn!("Max prev event limit reached! Limit: {limit}");
graph.insert(prev_event_id.clone(), HashSet::new());
continue;
}
if json_opt.is_none() {
json_opt = self
.services
.outlier
.get_outlier_pdu_json(&prev_event_id)
.await
.ok();
}
if let Some(json) = json_opt {
if pdu.origin_server_ts() > first_ts_in_room {
amount = amount.saturating_add(1);
for prev_prev in pdu.prev_events() {
if !graph.contains_key(prev_prev) {
todo_outlier_stack.push_back(prev_prev.to_owned());
}
}
graph.insert(
prev_event_id.clone(),
pdu.prev_events().map(ToOwned::to_owned).collect(),
);
} else {
// Time based check failed
graph.insert(prev_event_id.clone(), HashSet::new());
}
eventid_info.insert(prev_event_id.clone(), (pdu, json));
} else {
// Get json failed, so this was not fetched over federation
graph.insert(prev_event_id.clone(), HashSet::new());
}
},
| _ => {
// Fetch and handle failed
graph.insert(prev_event_id.clone(), HashSet::new());
},
impl super::Service {
/// Fetches any missing prev_events for this event and persists them before
/// returning.
pub(super) async fn fetch_prevs(
&self,
room_id: &RoomId,
create_event: &PduEvent,
incoming_pdu: &PduEvent,
origin: &ServerName,
first_ts_in_room: MilliSecondsSinceUnixEpoch,
) -> conduwuit::Result<()> {
let start = Instant::now();
let mut missing = incoming_pdu
.prev_events()
.stream()
.broad_filter_map(|event_id| async move {
self.services
.timeline
.get_non_outlier_pdu_json(event_id)
.await
.is_ok()
.or(|| event_id.to_owned())
})
.collect::<Vec<_>>()
.await;
if missing.is_empty() {
debug!(elapsed=?start.elapsed(), event_id=%incoming_pdu.event_id(), "No missing prev events.");
return Ok(());
}
debug!(elapsed=?start.elapsed(), %room_id, event_id=%incoming_pdu.event_id(), ?missing, "Fetching previous events");
let tail = self
.services
.state
.get_forward_extremities(room_id)
.collect::<Vec<_>>()
.await;
let mut gapfilled = self
.get_missing_events(
room_id,
incoming_pdu,
tail,
origin,
self.services
.metadata
.get_mindepth(room_id)
.await
.saturating_sub(
u8::try_from(incoming_pdu.prev_events.len())
.unwrap()
.saturating_mul(2)
.into(),
),
)
.await?;
debug_info!(elapsed=?start.elapsed(), "Fetched {} missing events", gapfilled.len());
missing.retain(|eid| !gapfilled.contains_key(eid));
if !missing.is_empty() {
debug_warn!(elapsed=?start.elapsed(), "Still missing {} events, falling back to atomic fetch.", missing.len());
gapfilled.extend(
self.fetch_prev_events(origin, missing, create_event, room_id)
.await,
);
}
// Persist all fetched events
let mapped = gapfilled
.iter()
.map(|(eid, evt)| {
let mut obj = evt.to_canonical_object();
obj.remove("event_id"); // event_id is inserted by backfill_missing_events
(eid.clone(), obj)
})
.collect::<HashMap<_, _>>();
let to_persist = if mapped.len() <= 1 {
mapped.keys().map(ToOwned::to_owned).collect()
} else {
let refmap: HashMap<OwnedEventId, &CanonicalJsonObject> =
mapped.iter().map(|(id, data)| (id.clone(), data)).collect();
build_local_dag(&refmap).await?
};
let job_start = Instant::now();
trace!("Starting to persist {} prev events", to_persist.len());
for (i, event_id) in to_persist.iter().enumerate() {
debug!(
elapsed=?start.elapsed(),
"Persisting fetched prev event: {event_id} ({}/{})",
i.saturating_add(1),
to_persist.len(),
);
let obj = mapped.get(event_id).cloned().unwrap();
let persist_start = Instant::now();
match self
.handle_outlier_pdu(origin, create_event, event_id, room_id, obj)
.await
{
| Ok((pdu, val)) if pdu.origin_server_ts() >= first_ts_in_room => {
Box::pin(self.upgrade_outlier_to_timeline_pdu(
pdu,
val,
create_event,
origin,
room_id,
))
.await
.inspect_err(|e| {
debug_warn!(
total_elapsed=?start.elapsed(),
job_elapsed=?job_start.elapsed(),
task_elapsed=?persist_start.elapsed(),
"Failed to upgrade prev event {event_id}: {e}",
);
})
.inspect(|_| {
debug_info!(
total_elapsed=?start.elapsed(),
job_elapsed=?job_start.elapsed(),
task_elapsed=?persist_start.elapsed(),
"Upgraded prev event {event_id}",
);
})
.ok();
},
| Err(e) => debug_warn!(
total_elapsed=?start.elapsed(),
job_elapsed=?job_start.elapsed(),
task_elapsed=?persist_start.elapsed(),
"Failed to persist prev event {event_id}: {e}",
),
| _ => {},
}
}
// NOTE because i keep forgetting: the caller persists incoming_pdu.
// we only care about its prev events
trace!(
total_elapsed=?start.elapsed(),
persist_elapsed=?job_start.elapsed(),
);
Ok(())
}
let event_fetch = |event_id| {
let origin_server_ts = eventid_info
.get(&event_id)
.map_or_else(|| uint!(0), |info| info.0.origin_server_ts().get());
// This return value is the key used for sorting events,
// events are then sorted by power level, time,
// and lexically by event_id.
future::ok((int!(0), MilliSecondsSinceUnixEpoch(origin_server_ts)))
};
let sorted = state_res::lexicographical_topological_sort(&graph, &event_fetch)
.await
.map_err(|e| err!(Database(error!("Error sorting prev events: {e}"))))?;
Ok((sorted, eventid_info))
}
+379 -73
View File
@@ -1,86 +1,392 @@
use std::collections::{HashMap, hash_map};
use conduwuit::{Err, Event, Result, debug, debug_warn, err, implement};
use futures::FutureExt;
use ruma::{
EventId, OwnedEventId, RoomId, ServerName, api::federation::event::get_room_state_ids,
events::StateEventType,
use std::{
cmp::max,
collections::{HashMap, HashSet, hash_map},
hash::{BuildHasherDefault, DefaultHasher},
time::{Duration, Instant},
};
use crate::rooms::short::ShortStateKey;
use conduwuit::{
Err, Event, PduEvent, Result, debug, debug_warn, err, info, trace,
utils::{BoolExt, IterStream},
warn,
};
use futures::{StreamExt, TryFutureExt, future::select_ok};
use ruma::{
EventId, OwnedEventId, OwnedRoomId, RoomId, ServerName,
api::federation::event::{get_room_state, get_room_state_ids},
};
/// Call /state_ids to find out what the state at this pdu is. We trust the
/// server's response to some extend (sic), but we still do a lot of checks
/// on the events
#[implement(super::Service)]
#[tracing::instrument(
level = "debug",
skip_all,
fields(%origin),
)]
pub(super) async fn fetch_state<Pdu>(
&self,
origin: &ServerName,
create_event: &Pdu,
room_id: &RoomId,
event_id: &EventId,
) -> Result<Option<HashMap<u64, OwnedEventId>>>
where
Pdu: Event + Send + Sync,
{
let res = self
.services
.sending
.send_federation_request(
origin,
get_room_state_ids::v1::Request::new(event_id.to_owned(), room_id.to_owned()),
)
.await
.inspect_err(|e| debug_warn!("Fetching state for event failed: {e}"))?;
use crate::{conduwuit::utils::stream::BroadbandExt, rooms::short::ShortStateKey};
debug!("Fetching state events");
let state_ids = res.pdu_ids.iter().map(AsRef::as_ref);
let state_vec = self
.fetch_and_handle_outliers(origin, state_ids, create_event, room_id)
.boxed()
.await;
let mut state: HashMap<ShortStateKey, OwnedEventId> = HashMap::with_capacity(state_vec.len());
for (pdu, _) in state_vec {
let state_key = pdu
.state_key()
.ok_or_else(|| err!(Database("Found non-state pdu in state events.")))?;
let shortstatekey = self
impl super::Service {
/// Asks a remote server what the state at this event is.
/// It first attempts to call `GET /_matrix/federation/v1/state_ids` (fast).
/// If any events are missing, they are fetched from the remote, and
/// persisted as outliers, before being returned back to this function. If
/// we are missing a lot of events locally (>=50), this function falls back
/// to requesting the full state in PDU format from the remote (`GET
/// /_matrix/federation/v1/state, very slow in large rooms), and persists
/// them directly.
#[tracing::instrument(skip_all)]
pub(super) async fn fetch_state(
&self,
origin: &ServerName,
create_event: &PduEvent,
room_id: &RoomId,
event_id: &EventId,
) -> Result<HashMap<u64, OwnedEventId>> {
let start = Instant::now();
trace!(%origin, "Asking remote for state_ids");
let res: get_room_state_ids::v1::Response = match self
.services
.short
.get_or_create_shortstatekey(&pdu.kind().to_string().into(), state_key)
.sending
.send_federation_request(
origin,
get_room_state_ids::v1::Request::new(event_id.to_owned(), room_id.to_owned()),
)
.await
.inspect_err(
|e| debug_warn!(elapsed=?start.elapsed(), "Fetching state for event failed: {e}"),
) {
| Ok(resp) => Ok(resp),
| Err(e) =>
if e.is_not_found() {
self.fetch_state_ids_from_backfill_servers(
event_id.to_owned(),
room_id.to_owned(),
)
.await
} else {
Err(e)
},
}?;
debug!(elapsed=?start.elapsed(), events = res.pdu_ids.len(), "Fetching state events");
let mut state_events: HashMap<OwnedEventId, PduEvent> =
HashMap::with_capacity(res.pdu_ids.len());
let to_fetch: Vec<OwnedEventId> = res
.pdu_ids
.clone()
.into_iter()
.stream()
.broad_filter_map(|event_id| async move {
self.services
.timeline
.pdu_exists(&event_id)
.await
.or_some(event_id)
})
.collect()
.await;
if to_fetch.is_empty() {
debug!(elapsed=?start.elapsed(), "All required state events are already known.");
state_events = res
.pdu_ids
.iter()
.stream()
.broad_filter_map(|event_id| async move {
Some((
event_id.clone(),
self.services
.timeline
.get_pdu(event_id)
.await
.expect("Event disappeared between filtering and fetching"),
))
})
.collect()
.await;
assert_eq!(
state_events.len(),
res.pdu_ids.len(),
"Failed to load all required state events despite allegedly knowing all of them \
already",
);
} else {
let total_count = res.pdu_ids.len();
let missing_count = to_fetch.len();
let missing_threshold = max(50, total_count >> 2);
if missing_count >= missing_threshold {
// If there's more than 50 events to fetch, or we're missing 25% or more of the
// state, we would need to make a lot of atomic requests, so we'll just try
// to fetch the full state from the remote instead.
// Since this endpoint might fail in huge rooms, we fall back to atomic fetch
// anyway.
warn!(
elapsed=?start.elapsed(),
%missing_count,
%total_count,
%missing_threshold,
"Fetching full state from remote server for event"
);
let state_response = tokio::time::timeout(
Duration::from_secs(30),
self.fetch_full_state(origin, create_event, room_id, event_id),
)
.await;
info!(
elapsed=?start.elapsed(),
%missing_count,
%total_count,
%missing_threshold,
"Fetched full state from remote server for event"
);
let fetched_state = match state_response {
| Ok(Ok(state)) => {
// Filter to ensure we only use the PDUs we were expecting, preventing
// arbitrary state injection.
// Atomic fetch does not have this problem as each PDU is evaluated
// individually.
let expected: &HashSet<OwnedEventId, BuildHasherDefault<DefaultHasher>> =
&HashSet::from_iter(res.pdu_ids.clone());
state
.into_iter()
.stream()
.broad_filter_map(|(event_id, pdu)| async move {
expected.contains(&event_id).then_some((event_id, pdu))
})
.collect()
.await
},
| Ok(Err(e)) => {
warn!(
elapsed=?start.elapsed(),
error=?e,
%origin,
"Failed to fetch full state from remote, falling back to atomic fetch"
);
self.fetch_and_handle_auth_events(
origin,
res.pdu_ids.clone(),
create_event,
room_id,
)
.await
},
| Err(e) => {
warn!(
elapsed=?start.elapsed(),
error=?e,
%origin,
"Remote did not return room state in an acceptable timeframe, falling back to atomic fetch"
);
self.fetch_and_handle_auth_events(
origin,
res.pdu_ids.clone(),
create_event,
room_id,
)
.await
},
};
match state.entry(shortstatekey) {
| hash_map::Entry::Vacant(v) => {
v.insert(pdu.event_id().to_owned());
},
| hash_map::Entry::Occupied(_) => {
return Err!(Database(
"State event's type and state_key combination exists multiple times: {}, {}",
pdu.kind(),
state_key
));
},
assert!(
!fetched_state.is_empty(),
"fetch_full_state or fetch_and_handle_missing_events returned empty state \
map"
);
state_events.extend(fetched_state);
} else {
state_events = res
.pdu_ids
.iter()
.stream()
.broad_filter_map(|event_id| async move {
self.services
.timeline
.get_pdu(event_id)
.await
.map(|p| (event_id.to_owned(), p))
.ok()
})
.collect()
.await;
assert!(
!state_events.is_empty(),
"Only missing {} events but read-ahead state vec was empty",
to_fetch.len()
);
debug!(
elapsed=?start.elapsed(),
to_fetch = to_fetch.len(),
"Fetching missing events for state from remote"
);
let fetched_state = self
.fetch_and_handle_auth_events(origin, to_fetch, create_event, room_id)
.await;
state_events.extend(fetched_state);
}
}
if state_events.is_empty() {
return Ok(HashMap::new());
}
let mut state: HashMap<ShortStateKey, OwnedEventId> =
HashMap::with_capacity(state_events.len());
debug!(elapsed=?start.elapsed(), events = state_events.len(), "Processing state events");
for (event_id, pdu) in state_events {
let state_key = pdu.state_key().ok_or_else(|| {
err!(Request(BadJson("Found non-state pdu in state events: {event_id}")))
})?;
let shortstatekey = self
.services
.short
.get_or_create_shortstatekey(&pdu.kind().to_string().into(), state_key)
.await;
match state.entry(shortstatekey) {
| hash_map::Entry::Vacant(v) => {
v.insert(pdu.event_id().to_owned());
},
| hash_map::Entry::Occupied(existing) => {
return Err!(Request(Forbidden(
"State event's type and state_key combination exists multiple times \
({event_id} + {}): ({}, \"{}\")",
existing.get(),
pdu.kind(),
state_key,
)));
},
}
}
trace!(elapsed=?start.elapsed(), "fetch_state finished");
Ok(state)
}
// The original create event must still be in the state
let create_shortstatekey = self
.services
.short
.get_shortstatekey(&StateEventType::RoomCreate, "")
.await?;
if state.get(&create_shortstatekey).map(AsRef::as_ref) != Some(create_event.event_id()) {
return Err!(Database("Incoming event refers to wrong create event."));
async fn fetch_state_ids_from_backfill_servers(
&self,
event_id: OwnedEventId,
room_id: OwnedRoomId,
) -> Result<get_room_state_ids::v1::Response> {
let candidates = self
.services
.timeline
.candidate_backfill_servers(&room_id)
.await;
if candidates.is_empty() {
return Err!(Request(NotFound(
"Cannot ask any other servers for the state at this event"
)));
}
debug!(%room_id, ?candidates, "Asking backfill servers for state_ids");
let futures = candidates.iter().map(|server_name| {
Box::pin(
self.services
.sending
.send_federation_request(
server_name,
get_room_state_ids::v1::Request::new(event_id.clone(), room_id.clone()),
)
.inspect_err(|e| {
debug_warn!("Fallback fetching state for event failed: {e}");
}),
)
});
Ok(select_ok(futures).await?.0)
}
Ok(Some(state))
/// Fetches the full state via `GET /_matrix/federation/v1/state` from a
/// remote server, and persists all the incoming auth chain events and
/// state events as outliers, for use later.
///
/// Any events that cannot be persisted are dropped with a warning.
pub(super) async fn fetch_full_state(
&self,
origin: &ServerName,
create_event: &PduEvent,
room_id: &RoomId,
event_id: &EventId,
) -> Result<HashMap<OwnedEventId, PduEvent>> {
let start = Instant::now();
trace!("Fetching full state from remote server");
let res: get_room_state::v1::Response = self
.services
.sending
.send_federation_request(
origin,
get_room_state::v1::Request::new(event_id.to_owned(), room_id.to_owned()),
)
.await
.inspect_err(|e| debug_warn!("Fetching state for event failed: {e}"))?;
debug!(elapsed=?start.elapsed(), count = res.auth_chain.len(), "Handling incoming auth chain...");
res.auth_chain
.iter()
.stream()
.broad_filter_map(|raw_event_json| async {
if let Some(parsed) = self.parse_incoming_pdu(raw_event_json).await.ok()
&& parsed.0 == room_id
{
Some(parsed)
} else {
None
}
})
.for_each_concurrent(
None,
|(incoming_room_id, incoming_event_id, incoming_event_json)| async move {
self.handle_outlier_pdu(
origin,
create_event,
&incoming_event_id,
&incoming_room_id,
incoming_event_json,
)
.await
.inspect_err(|e| {
warn!(
%incoming_room_id,
%incoming_event_id,
?e,
"Failed to handle auth chain event from state fetch"
);
})
.ok();
},
)
.await;
debug!(elapsed=?start.elapsed(), count = res.pdus.len(), "Handling incoming state PDUs...");
let r = res
.pdus
.iter()
.stream()
.broad_filter_map(|raw_event_json| async {
if let Some(parsed) = self.parse_incoming_pdu(raw_event_json).await.ok()
&& parsed.0 == room_id
{
Some(parsed)
} else {
None
}
})
.broad_filter_map(
|(incoming_room_id, incoming_event_id, incoming_event_json)| async move {
self.handle_outlier_pdu(
origin,
create_event,
&incoming_event_id,
&incoming_room_id,
incoming_event_json,
)
.await
.inspect_err(|e| {
warn!(
elapsed=?start.elapsed(),
%incoming_room_id,
%incoming_event_id,
?e,
"Failed to handle state event from state fetch"
);
})
.ok()
},
)
.fold(HashMap::new(), |mut acc, (event, _)| async move {
acc.insert(event.event_id().to_owned(), event);
acc
})
.await;
trace!(elapsed=?start.elapsed(), "fetch_full_state finished");
Ok(r)
}
}
Loaded 100 of 151 files, more files were not shown because too many files have changed in this diff. Show more