mirror of
https://forgejo.ellis.link/continuwuation/continuwuity.git
synced 2026-11-20 07:27:34 +00:00
Compare commits
131
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
b6bc7dfc16 | ||
|
|
945ea5a78a | ||
|
|
d719fe2048 | ||
|
|
458e0b53ac | ||
|
|
2d65d47498 | ||
|
|
e4922814b9 | ||
|
|
064028689c | ||
|
|
124241238e | ||
|
|
967e2cc54f | ||
|
|
6fdeadc356 | ||
|
|
478560daae | ||
|
|
0310b4b2c5 | ||
|
|
3eb4257ae8 | ||
|
|
f1d3db0ea3 | ||
|
|
e7b302955f | ||
|
|
85122c62cd | ||
|
|
ba0ffee7cf | ||
|
|
86e1101b40 | ||
|
|
fdcb7e3957 | ||
|
|
c54e56a73e | ||
|
|
fed9a599d9 | ||
|
|
0bb97a53c5 | ||
|
|
9b76c9a085 | ||
|
|
6f20b7bc3f | ||
|
|
580d9b0318 | ||
|
|
ae572ec108 | ||
|
|
b1c5b3d000 | ||
|
|
0c9230f35e | ||
|
|
77dab8fd20 | ||
|
|
10b5ca8e3a | ||
|
|
f9eed42cb7 | ||
|
|
19f4c24c00 | ||
|
|
0a162b7bc4 | ||
|
|
0d1b130127 | ||
|
|
5e0e82e7e8 | ||
|
|
4bf0816bcc | ||
|
|
ecbe035b36 | ||
|
|
4d30ee5d33 | ||
|
|
4844c6ae3e | ||
|
|
6603798d60 | ||
|
|
199f6eb67b | ||
|
|
aa2f506996 | ||
|
|
a3994fe1a3 | ||
|
|
6312ff8088 | ||
|
|
17cff1fb36 | ||
|
|
aff6315ae1 | ||
|
|
574d3e9164 | ||
|
|
21e93f8feb | ||
|
|
4b2f032ae6 | ||
|
|
8a495a7d7f | ||
|
|
cee51d5717 | ||
|
|
0483d3e155 | ||
|
|
2a4b9f9323 | ||
|
|
63a1a3d9ab | ||
|
|
f19f5c71a5 | ||
|
|
b009ffd31c | ||
|
|
b6baf66399 | ||
|
|
6dac46e1ad | ||
|
|
4d40e20fdd | ||
|
|
3b6858e936 | ||
|
|
3b455218f5 | ||
|
|
7ca00e4ab9 | ||
|
|
b8ca06029f | ||
|
|
2efe8f2ec0 | ||
|
|
d1aa911739 | ||
|
|
4673282ca1 | ||
|
|
0c03195aec | ||
|
|
439bc2784d | ||
|
|
689a1ce59b | ||
|
|
c141503ccb | ||
|
|
bcadecdc3b | ||
|
|
ab3be337cb | ||
|
|
aa1281a9e0 | ||
|
|
115a2e802e | ||
|
|
968328d788 | ||
|
|
f1cde5f323 | ||
|
|
832ee8650b | ||
|
|
c0666a1793 | ||
|
|
ddb4ef539f | ||
|
|
3faedc4581 | ||
|
|
a3544353ba | ||
|
|
b0612397d3 | ||
|
|
31737e127e | ||
|
|
486dcd208c | ||
|
|
a945a4b2ad | ||
|
|
c28ea44e11 | ||
|
|
3e4d6b2565 | ||
|
|
29ce21cd2e | ||
|
|
d461c6977a | ||
|
|
5a0d6461d1 | ||
|
|
2ec7394785 | ||
|
|
8a5708b9f9 | ||
|
|
cd46070bd3 | ||
|
|
9930e549a0 | ||
|
|
36ed20cb04 | ||
|
|
3907589b6c | ||
|
|
2a598af888 | ||
|
|
f34f84832b | ||
|
|
0efa6ed1f2 | ||
|
|
2bf5876778 | ||
|
|
31982e84de | ||
|
|
970958652a | ||
|
|
c94a395bf0 | ||
|
|
43adff926f | ||
|
|
04fce56381 | ||
|
|
4df2097e6c | ||
|
|
8b85b04d10 | ||
|
|
9e0bcd3be8 | ||
|
|
9509080e0d | ||
|
|
61066bb0c6 | ||
|
|
573d5bc50e | ||
|
|
162e6eb92f | ||
|
|
cef4ebe38e | ||
|
|
316a0b7d58 | ||
|
|
2bcc56704b | ||
|
|
7f64de9727 | ||
|
|
aea03f2f99 | ||
|
|
8edf9552b8 | ||
|
|
d5f69c8a31 | ||
|
|
9ea9b0e04c | ||
|
|
e8db01fc8d | ||
|
|
f80e1e89a5 | ||
|
|
b9dca84acf | ||
|
|
e3ec1066c4 | ||
|
|
1445a8d446 | ||
|
|
9547c438d6 | ||
|
|
51d0e615f5 | ||
|
|
eeb937416c | ||
|
|
0d5aa7ede1 | ||
|
|
ba9dc27773 | ||
|
|
abf5a155ba |
No files matched your search
@@ -14,23 +14,19 @@ jobs:
|
||||
name: Check changelog is added
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
||||
with:
|
||||
ref: ${{ github.event.pull_request.head.sha }}
|
||||
fetch-depth: 0
|
||||
persist-credentials: false
|
||||
sparse-checkout: .
|
||||
|
||||
- name: Check for changelog entry
|
||||
id: check_files
|
||||
run: |
|
||||
git fetch origin ${GITHUB_BASE_REF}
|
||||
AUTH=$(echo -n "x-access-token:${{ secrets.GITHUB_TOKEN }}" | base64 -w 0)
|
||||
git config --global http.${{ github.server_url }}/.extraheader "Authorization: basic $AUTH"
|
||||
git clone "${{ github.event.repository.clone_url }}" repo.git --bare
|
||||
|
||||
git -C repo.git fetch origin pull/${{ github.event.pull_request.number }}/head
|
||||
|
||||
# Check for Added (A) or Modified (M) files in changelog.d
|
||||
CHANGELOG_CHANGES=$(git diff --name-status origin/${GITHUB_BASE_REF}...HEAD -- changelog.d/)
|
||||
CHANGELOG_CHANGES=$(git -C repo.git diff --name-status ${{ github.event.pull_request.base.sha }}...${{ github.event.pull_request.head.sha }} -- changelog.d/)
|
||||
|
||||
SRC_CHANGES=$(git diff --name-status origin/${GITHUB_BASE_REF}...HEAD -- src/)
|
||||
SRC_CHANGES=$(git -C repo.git diff --name-status ${{ github.event.pull_request.base.sha }}...${{ github.event.pull_request.head.sha }} -- src/)
|
||||
|
||||
echo "Changes in changelog.d/:"
|
||||
echo "$CHANGELOG_CHANGES"
|
||||
|
||||
@@ -23,6 +23,7 @@ We appreciate the efforts of security researchers and the community in identifyi
|
||||
1. **Contact members of the team directly** over E2EE private message.
|
||||
- [@jade:ellis.link](https://matrix.to/#/@jade:ellis.link)
|
||||
- [@nex:nexy7574.co.uk](https://matrix.to/#/@nex:nexy7574.co.uk)
|
||||
- [@ginger:gingershaped.computer](https://matrix.to/#/@ginger:gingershaped.computer)
|
||||
2. **Email the security team** at [security@continuwuity.org](mailto:security@continuwuity.org). This is not E2EE, so don't include sensitive details.
|
||||
3. **Do not disclose the vulnerability publicly** until it has been addressed
|
||||
4. **Provide detailed information** about the vulnerability, including:
|
||||
|
||||
@@ -0,0 +1,2 @@
|
||||
Improved the performance and reliability of fetching missing events, improving network partition recovery. Contributed
|
||||
by @nex.
|
||||
@@ -0,0 +1 @@
|
||||
Added support for MSC4380 invite blocking, which has become part of the Matrix specification in v1.18. Contributed by @nex.
|
||||
@@ -0,0 +1 @@
|
||||
Added `!admin debug get-state-at` command
|
||||
@@ -297,7 +297,7 @@
|
||||
|
||||
# This item is undocumented. Please contribute documentation for it.
|
||||
#
|
||||
#max_fetch_prev_events = 192
|
||||
#max_fetch_prev_events = 1024
|
||||
|
||||
# How many incoming federation transactions the server is willing to be
|
||||
# processing at any given time before it becomes overloaded and starts
|
||||
@@ -645,6 +645,14 @@
|
||||
#
|
||||
#default_room_acl_deny =
|
||||
|
||||
# The number of forward extremities to tolerate in a room before
|
||||
# attempting to manually squash them with a "dummy event". Setting this
|
||||
# above 20 will hinder its efficacy, and setting it below 5 will cause
|
||||
# more dummy events to be sent than necessary (which increases federation
|
||||
# traffic).
|
||||
#
|
||||
#dummy_event_threshold = 10
|
||||
|
||||
# Enable OpenTelemetry OTLP tracing export. This replaces the deprecated
|
||||
# Jaeger exporter. Traces will be sent via OTLP to a collector (such as
|
||||
# Jaeger) that supports the OpenTelemetry Protocol.
|
||||
|
||||
@@ -10,7 +10,7 @@ Please check that your issues are not due to problems with your Docker setup.
|
||||
|
||||
### Slow joins to rooms
|
||||
|
||||
Some slowness is to be expected if you're the first person on your homserver to join a room (which will
|
||||
Some slowness is to be expected if you're the first person on your homeserver to join a room (which will
|
||||
always be the case for single-user homeservers). In this situation, your homeserver has to verify the signatures of
|
||||
all of the state events sent by other servers before your join. To make this process as fast as possible, make sure you have
|
||||
multiple fast, trusted servers listed in `trusted_servers` in your configuration, and ensure
|
||||
|
||||
@@ -1,80 +1,76 @@
|
||||
use conduwuit::{Err, Result, checked};
|
||||
use futures::{FutureExt, StreamExt, TryFutureExt};
|
||||
|
||||
use crate::admin_command;
|
||||
impl crate::Context<'_> {
|
||||
pub(super) async fn register(&self) -> Result {
|
||||
let body = &self.body;
|
||||
let body_len = self.body.len();
|
||||
if body_len < 2
|
||||
|| !body[0].trim().starts_with("```")
|
||||
|| body.last().unwrap_or(&"").trim() != "```"
|
||||
{
|
||||
return Err!("Expected code block in command body. Add --help for details.");
|
||||
}
|
||||
|
||||
#[admin_command]
|
||||
pub(super) async fn register(&self) -> Result {
|
||||
let body = &self.body;
|
||||
let body_len = self.body.len();
|
||||
if body_len < 2
|
||||
|| !body[0].trim().starts_with("```")
|
||||
|| body.last().unwrap_or(&"").trim() != "```"
|
||||
{
|
||||
return Err!("Expected code block in command body. Add --help for details.");
|
||||
let range = 1..checked!(body_len - 1)?;
|
||||
let appservice_config_body = body[range].join("\n");
|
||||
let parsed_config = serde_saphyr::from_str(&appservice_config_body);
|
||||
match parsed_config {
|
||||
| Err(e) => return Err!("Could not parse appservice config as YAML: {e}"),
|
||||
| Ok(registration) => match self
|
||||
.services
|
||||
.appservice
|
||||
.register_appservice(®istration, &appservice_config_body)
|
||||
.await
|
||||
.map(|()| registration.id)
|
||||
{
|
||||
| Err(e) => return Err!("Failed to register appservice: {e}"),
|
||||
| Ok(id) => write!(self, "Appservice registered with ID: {id}"),
|
||||
},
|
||||
}
|
||||
.await
|
||||
}
|
||||
|
||||
let range = 1..checked!(body_len - 1)?;
|
||||
let appservice_config_body = body[range].join("\n");
|
||||
let parsed_config = serde_saphyr::from_str(&appservice_config_body);
|
||||
match parsed_config {
|
||||
| Err(e) => return Err!("Could not parse appservice config as YAML: {e}"),
|
||||
| Ok(registration) => match self
|
||||
pub(super) async fn unregister(&self, appservice_identifier: String) -> Result {
|
||||
match self
|
||||
.services
|
||||
.appservice
|
||||
.register_appservice(®istration, &appservice_config_body)
|
||||
.unregister_appservice(&appservice_identifier)
|
||||
.await
|
||||
.map(|()| registration.id)
|
||||
{
|
||||
| Err(e) => return Err!("Failed to register appservice: {e}"),
|
||||
| Ok(id) => write!(self, "Appservice registered with ID: {id}"),
|
||||
},
|
||||
}
|
||||
.await
|
||||
}
|
||||
|
||||
#[admin_command]
|
||||
pub(super) async fn unregister(&self, appservice_identifier: String) -> Result {
|
||||
match self
|
||||
.services
|
||||
.appservice
|
||||
.unregister_appservice(&appservice_identifier)
|
||||
| Err(e) => return Err!("Failed to unregister appservice: {e}"),
|
||||
| Ok(()) => write!(self, "Appservice unregistered."),
|
||||
}
|
||||
.await
|
||||
{
|
||||
| Err(e) => return Err!("Failed to unregister appservice: {e}"),
|
||||
| Ok(()) => write!(self, "Appservice unregistered."),
|
||||
}
|
||||
.await
|
||||
}
|
||||
|
||||
#[admin_command]
|
||||
pub(super) async fn show_appservice_config(&self, appservice_identifier: String) -> Result {
|
||||
match self
|
||||
.services
|
||||
.appservice
|
||||
.get_registration(&appservice_identifier)
|
||||
pub(super) async fn show_appservice_config(&self, appservice_identifier: String) -> Result {
|
||||
match self
|
||||
.services
|
||||
.appservice
|
||||
.get_registration(&appservice_identifier)
|
||||
.await
|
||||
{
|
||||
| None => return Err!("Appservice does not exist."),
|
||||
| Some(config) => {
|
||||
let config_str = serde_saphyr::to_string(&config)?;
|
||||
write!(self, "Config for {appservice_identifier}:\n\n```yaml\n{config_str}\n```")
|
||||
},
|
||||
}
|
||||
.await
|
||||
{
|
||||
| None => return Err!("Appservice does not exist."),
|
||||
| Some(config) => {
|
||||
let config_str = serde_saphyr::to_string(&config)?;
|
||||
write!(self, "Config for {appservice_identifier}:\n\n```yaml\n{config_str}\n```")
|
||||
},
|
||||
}
|
||||
.await
|
||||
}
|
||||
|
||||
#[admin_command]
|
||||
pub(super) async fn list_registered(&self) -> Result {
|
||||
self.services
|
||||
.appservice
|
||||
.iter_ids()
|
||||
.collect()
|
||||
.map(Ok)
|
||||
.and_then(|appservices: Vec<_>| {
|
||||
let len = appservices.len();
|
||||
let list = appservices.join(", ");
|
||||
write!(self, "Appservices ({len}): {list}")
|
||||
})
|
||||
.await
|
||||
pub(super) async fn list_registered(&self) -> Result {
|
||||
self.services
|
||||
.appservice
|
||||
.iter_ids()
|
||||
.collect()
|
||||
.map(Ok)
|
||||
.and_then(|appservices: Vec<_>| {
|
||||
let len = appservices.len();
|
||||
let list = appservices.join(", ");
|
||||
write!(self, "Appservices ({len}): {list}")
|
||||
})
|
||||
.await
|
||||
}
|
||||
}
|
||||
+15
-15
@@ -1,23 +1,23 @@
|
||||
use conduwuit::Result;
|
||||
use conduwuit_macros::implement;
|
||||
use futures::StreamExt;
|
||||
|
||||
use crate::Context;
|
||||
|
||||
#[implement(Context, params = "<'_>")]
|
||||
pub(super) async fn check_all_users(&self) -> Result {
|
||||
let timer = tokio::time::Instant::now();
|
||||
let users = self.services.users.stream().collect::<Vec<_>>().await;
|
||||
let query_time = timer.elapsed();
|
||||
impl Context<'_> {
|
||||
pub(super) async fn check_all_users(&self) -> Result {
|
||||
let timer = tokio::time::Instant::now();
|
||||
let users = self.services.users.stream().collect::<Vec<_>>().await;
|
||||
let query_time = timer.elapsed();
|
||||
|
||||
let total = users.len();
|
||||
let err_count = users.iter().filter(|_user| false).count();
|
||||
let ok_count = users.iter().filter(|_user| true).count();
|
||||
let total = users.len();
|
||||
let err_count = users.iter().filter(|_user| false).count();
|
||||
let ok_count = users.iter().filter(|_user| true).count();
|
||||
|
||||
self.write_str(&format!(
|
||||
"Database query completed in {query_time:?}:\n\n```\nTotal entries: \
|
||||
{total:?}\nFailure/Invalid user count: {err_count:?}\nSuccess/Valid user count: \
|
||||
{ok_count:?}\n```"
|
||||
))
|
||||
.await
|
||||
self.write_str(&format!(
|
||||
"Database query completed in {query_time:?}:\n\n```\nTotal entries: \
|
||||
{total:?}\nFailure/Invalid user count: {err_count:?}\nSuccess/Valid user count: \
|
||||
{ok_count:?}\n```"
|
||||
))
|
||||
.await
|
||||
}
|
||||
}
|
||||
+1041
-958
File diff suppressed because it is too large.
Load diff
@@ -95,6 +95,14 @@ pub enum DebugCommand {
|
||||
room_id: OwnedRoomOrAliasId,
|
||||
},
|
||||
|
||||
/// Gets all the room state events at the specified event.
|
||||
///
|
||||
/// State at event might not be available for some PDUs, such as rejected
|
||||
/// ones.
|
||||
GetStateAt {
|
||||
event_id: OwnedEventId,
|
||||
},
|
||||
|
||||
/// Get and display signing keys from local cache or remote server.
|
||||
GetSigningKeys {
|
||||
server_name: Option<OwnedServerName>,
|
||||
@@ -237,6 +245,11 @@ pub enum DebugCommand {
|
||||
/// Send a test email to the invoking admin's email address
|
||||
SendTestEmail,
|
||||
|
||||
/// Lists room IDs by forward extremity count in descending order
|
||||
RoomsByExtremityCount {
|
||||
page: Option<usize>,
|
||||
},
|
||||
|
||||
/// Developer test stubs
|
||||
#[command(subcommand)]
|
||||
#[allow(non_snake_case)]
|
||||
|
||||
+23
-28
@@ -1,6 +1,6 @@
|
||||
use conduwuit::{Err, Result};
|
||||
|
||||
use crate::{admin_command, admin_command_dispatch};
|
||||
use crate::admin_command_dispatch;
|
||||
|
||||
#[admin_command_dispatch]
|
||||
#[derive(Debug, clap::Subcommand)]
|
||||
@@ -11,37 +11,32 @@ pub enum TesterCommand {
|
||||
Timer,
|
||||
}
|
||||
|
||||
#[rustfmt::skip]
|
||||
#[admin_command]
|
||||
async fn panic(&self) -> Result {
|
||||
impl crate::Context<'_> {
|
||||
#[rustfmt::skip]
|
||||
async fn panic(&self) -> Result {
|
||||
panic!("panicked")
|
||||
}
|
||||
|
||||
panic!("panicked")
|
||||
}
|
||||
#[rustfmt::skip]
|
||||
async fn failure(&self) -> Result {
|
||||
Err!("failed")
|
||||
}
|
||||
|
||||
#[rustfmt::skip]
|
||||
#[admin_command]
|
||||
async fn failure(&self) -> Result {
|
||||
#[inline(never)]
|
||||
#[rustfmt::skip]
|
||||
async fn tester(&self) -> Result {
|
||||
self.write_str("Ok").await
|
||||
}
|
||||
|
||||
Err!("failed")
|
||||
}
|
||||
#[inline(never)]
|
||||
#[rustfmt::skip]
|
||||
async fn timer(&self) -> Result {
|
||||
let started = std::time::Instant::now();
|
||||
timed(self.body);
|
||||
|
||||
#[inline(never)]
|
||||
#[rustfmt::skip]
|
||||
#[admin_command]
|
||||
async fn tester(&self) -> Result {
|
||||
|
||||
self.write_str("Ok").await
|
||||
}
|
||||
|
||||
#[inline(never)]
|
||||
#[rustfmt::skip]
|
||||
#[admin_command]
|
||||
async fn timer(&self) -> Result {
|
||||
let started = std::time::Instant::now();
|
||||
timed(self.body);
|
||||
|
||||
let elapsed = started.elapsed();
|
||||
self.write_str(&format!("completed in {elapsed:#?}")).await
|
||||
let elapsed = started.elapsed();
|
||||
self.write_str(&format!("completed in {elapsed:#?}")).await
|
||||
}
|
||||
}
|
||||
|
||||
#[inline(never)]
|
||||
|
||||
+116
-112
@@ -4,127 +4,131 @@ use conduwuit::{Err, Result, utils::response::LimitReadExt};
|
||||
use futures::StreamExt;
|
||||
use ruma::{OwnedRoomId, OwnedServerName, OwnedUserId};
|
||||
|
||||
use crate::{admin_command, get_room_info};
|
||||
use crate::get_room_info;
|
||||
|
||||
#[admin_command]
|
||||
pub(super) async fn disable_room(&self, room_id: OwnedRoomId) -> Result {
|
||||
self.bail_restricted()?;
|
||||
self.services.rooms.metadata.disable_room(&room_id, true);
|
||||
self.write_str("Room disabled.").await
|
||||
}
|
||||
impl crate::Context<'_> {
|
||||
pub(super) async fn disable_room(&self, room_id: OwnedRoomId) -> Result {
|
||||
self.bail_restricted()?;
|
||||
self.services.rooms.metadata.disable_room(&room_id, true);
|
||||
self.write_str("Room disabled.").await
|
||||
}
|
||||
|
||||
#[admin_command]
|
||||
pub(super) async fn enable_room(&self, room_id: OwnedRoomId) -> Result {
|
||||
self.bail_restricted()?;
|
||||
self.services.rooms.metadata.disable_room(&room_id, false);
|
||||
self.write_str("Room enabled.").await
|
||||
}
|
||||
pub(super) async fn enable_room(&self, room_id: OwnedRoomId) -> Result {
|
||||
self.bail_restricted()?;
|
||||
self.services.rooms.metadata.disable_room(&room_id, false);
|
||||
self.write_str("Room enabled.").await
|
||||
}
|
||||
|
||||
#[admin_command]
|
||||
pub(super) async fn incoming_federation(&self) -> Result {
|
||||
let msg = {
|
||||
let map = self
|
||||
pub(super) async fn incoming_federation(&self) -> Result {
|
||||
let msg = {
|
||||
let map = self
|
||||
.services
|
||||
.rooms
|
||||
.event_handler
|
||||
.federation_handletime
|
||||
.read();
|
||||
|
||||
let mut msg = format!(
|
||||
"Handling {} incoming PDUs across {} active transactions:\n",
|
||||
map.len(),
|
||||
self.services.transactions.txn_active_handle_count()
|
||||
);
|
||||
for (r, (e, i)) in map.iter() {
|
||||
let elapsed = i.elapsed();
|
||||
writeln!(
|
||||
msg,
|
||||
"{} {}: {}m{}s",
|
||||
r,
|
||||
e,
|
||||
elapsed.as_secs() / 60,
|
||||
elapsed.as_secs() % 60
|
||||
)?;
|
||||
}
|
||||
msg
|
||||
};
|
||||
|
||||
self.write_str(&msg).await
|
||||
}
|
||||
|
||||
pub(super) async fn fetch_support_well_known(&self, server_name: OwnedServerName) -> Result {
|
||||
let response = self
|
||||
.services
|
||||
.client
|
||||
.default
|
||||
.get(format!("https://{server_name}/.well-known/matrix/support"))
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
let text = response
|
||||
.limit_read_text(
|
||||
self.services
|
||||
.config
|
||||
.max_request_size
|
||||
.try_into()
|
||||
.expect("u64 fits into usize"),
|
||||
)
|
||||
.await?;
|
||||
|
||||
if text.is_empty() {
|
||||
return Err!("Response text/body is empty.");
|
||||
}
|
||||
|
||||
if text.len() > 1500 {
|
||||
return Err!(
|
||||
"Response text/body is over 1500 characters, assuming no support well-known.",
|
||||
);
|
||||
}
|
||||
|
||||
let json: serde_json::Value = match serde_json::from_str(&text) {
|
||||
| Ok(json) => json,
|
||||
| Err(_) => {
|
||||
return Err!("Response text/body is not valid JSON.",);
|
||||
},
|
||||
};
|
||||
|
||||
let pretty_json: String = match serde_json::to_string_pretty(&json) {
|
||||
| Ok(json) => json,
|
||||
| Err(_) => {
|
||||
return Err!("Response text/body is not valid JSON.",);
|
||||
},
|
||||
};
|
||||
|
||||
self.write_str(&format!("Got JSON response:\n\n```json\n{pretty_json}\n```"))
|
||||
.await
|
||||
}
|
||||
|
||||
pub(super) async fn remote_user_in_rooms(&self, user_id: OwnedUserId) -> Result {
|
||||
if user_id.server_name() == self.services.server.name {
|
||||
return Err!(
|
||||
"User belongs to our server, please use `list-joined-rooms` user admin command \
|
||||
instead.",
|
||||
);
|
||||
}
|
||||
|
||||
let mut rooms: Vec<(OwnedRoomId, u64, String)> = self
|
||||
.services
|
||||
.rooms
|
||||
.event_handler
|
||||
.federation_handletime
|
||||
.read();
|
||||
.state_cache
|
||||
.rooms_joined(&user_id)
|
||||
.then(async |room_id| get_room_info(self.services, &room_id).await)
|
||||
.collect()
|
||||
.await;
|
||||
|
||||
let mut msg = format!(
|
||||
"Handling {} incoming PDUs across {} active transactions:\n",
|
||||
map.len(),
|
||||
self.services.transactions.txn_active_handle_count()
|
||||
);
|
||||
for (r, (e, i)) in map.iter() {
|
||||
let elapsed = i.elapsed();
|
||||
writeln!(msg, "{} {}: {}m{}s", r, e, elapsed.as_secs() / 60, elapsed.as_secs() % 60)?;
|
||||
if rooms.is_empty() {
|
||||
return Err!("User is not in any rooms.");
|
||||
}
|
||||
msg
|
||||
};
|
||||
|
||||
self.write_str(&msg).await
|
||||
}
|
||||
rooms.sort_by_key(|r| r.1);
|
||||
rooms.reverse();
|
||||
|
||||
#[admin_command]
|
||||
pub(super) async fn fetch_support_well_known(&self, server_name: OwnedServerName) -> Result {
|
||||
let response = self
|
||||
.services
|
||||
.client
|
||||
.default
|
||||
.get(format!("https://{server_name}/.well-known/matrix/support"))
|
||||
.send()
|
||||
.await?;
|
||||
let num = rooms.len();
|
||||
let body = rooms
|
||||
.iter()
|
||||
.map(|(id, members, name)| format!("{id} | Members: {members} | Name: {name}"))
|
||||
.collect::<Vec<_>>()
|
||||
.join("\n");
|
||||
|
||||
let text = response
|
||||
.limit_read_text(
|
||||
self.services
|
||||
.config
|
||||
.max_request_size
|
||||
.try_into()
|
||||
.expect("u64 fits into usize"),
|
||||
)
|
||||
.await?;
|
||||
|
||||
if text.is_empty() {
|
||||
return Err!("Response text/body is empty.");
|
||||
}
|
||||
|
||||
if text.len() > 1500 {
|
||||
return Err!(
|
||||
"Response text/body is over 1500 characters, assuming no support well-known.",
|
||||
);
|
||||
}
|
||||
|
||||
let json: serde_json::Value = match serde_json::from_str(&text) {
|
||||
| Ok(json) => json,
|
||||
| Err(_) => {
|
||||
return Err!("Response text/body is not valid JSON.",);
|
||||
},
|
||||
};
|
||||
|
||||
let pretty_json: String = match serde_json::to_string_pretty(&json) {
|
||||
| Ok(json) => json,
|
||||
| Err(_) => {
|
||||
return Err!("Response text/body is not valid JSON.",);
|
||||
},
|
||||
};
|
||||
|
||||
self.write_str(&format!("Got JSON response:\n\n```json\n{pretty_json}\n```"))
|
||||
.await
|
||||
}
|
||||
|
||||
#[admin_command]
|
||||
pub(super) async fn remote_user_in_rooms(&self, user_id: OwnedUserId) -> Result {
|
||||
if user_id.server_name() == self.services.server.name {
|
||||
return Err!(
|
||||
"User belongs to our server, please use `list-joined-rooms` user admin command \
|
||||
instead.",
|
||||
);
|
||||
}
|
||||
|
||||
let mut rooms: Vec<(OwnedRoomId, u64, String)> = self
|
||||
.services
|
||||
.rooms
|
||||
.state_cache
|
||||
.rooms_joined(&user_id)
|
||||
.then(async |room_id| get_room_info(self.services, &room_id).await)
|
||||
.collect()
|
||||
.await;
|
||||
|
||||
if rooms.is_empty() {
|
||||
return Err!("User is not in any rooms.");
|
||||
self.write_str(&format!("Rooms {user_id} shares with us ({num}):\n```\n{body}\n```"))
|
||||
.await
|
||||
}
|
||||
|
||||
rooms.sort_by_key(|r| r.1);
|
||||
rooms.reverse();
|
||||
|
||||
let num = rooms.len();
|
||||
let body = rooms
|
||||
.iter()
|
||||
.map(|(id, members, name)| format!("{id} | Members: {members} | Name: {name}"))
|
||||
.collect::<Vec<_>>()
|
||||
.join("\n");
|
||||
|
||||
self.write_str(&format!("Rooms {user_id} shares with us ({num}):\n```\n{body}\n```"))
|
||||
.await
|
||||
}
|
||||
+336
-334
@@ -9,401 +9,403 @@ use conduwuit_service::media::Dim;
|
||||
use ruma::{OwnedEventId, OwnedMxcUri, OwnedServerName};
|
||||
use service::media::mxc::Mxc;
|
||||
|
||||
use crate::{admin_command, utils::parse_local_user_id};
|
||||
use crate::utils::parse_local_user_id;
|
||||
|
||||
#[admin_command]
|
||||
pub(super) async fn delete(
|
||||
&self,
|
||||
mxc: Option<OwnedMxcUri>,
|
||||
event_id: Option<OwnedEventId>,
|
||||
) -> Result {
|
||||
self.bail_restricted()?;
|
||||
impl crate::Context<'_> {
|
||||
pub(super) async fn delete(
|
||||
&self,
|
||||
mxc: Option<OwnedMxcUri>,
|
||||
event_id: Option<OwnedEventId>,
|
||||
) -> Result {
|
||||
self.bail_restricted()?;
|
||||
|
||||
if event_id.is_some() && mxc.is_some() {
|
||||
return Err!("Please specify either an MXC or an event ID, not both.",);
|
||||
}
|
||||
if event_id.is_some() && mxc.is_some() {
|
||||
return Err!("Please specify either an MXC or an event ID, not both.",);
|
||||
}
|
||||
|
||||
if let Some(mxc) = mxc {
|
||||
trace!("Got MXC URL: {mxc}");
|
||||
self.services
|
||||
.media
|
||||
.delete(&mxc.as_str().try_into()?)
|
||||
.await?;
|
||||
if let Some(mxc) = mxc {
|
||||
trace!("Got MXC URL: {mxc}");
|
||||
self.services
|
||||
.media
|
||||
.delete(&mxc.as_str().try_into()?)
|
||||
.await?;
|
||||
|
||||
return self
|
||||
.write_str("Deleted the MXC from our database and on our filesystem.")
|
||||
.await;
|
||||
}
|
||||
return self
|
||||
.write_str("Deleted the MXC from our database and on our filesystem.")
|
||||
.await;
|
||||
}
|
||||
|
||||
if let Some(event_id) = event_id {
|
||||
trace!("Got event ID to delete media from: {event_id}");
|
||||
if let Some(event_id) = event_id {
|
||||
trace!("Got event ID to delete media from: {event_id}");
|
||||
|
||||
let mut mxc_urls = Vec::with_capacity(4);
|
||||
let mut mxc_urls = Vec::with_capacity(4);
|
||||
|
||||
// parsing the PDU for any MXC URLs begins here
|
||||
match self.services.rooms.timeline.get_pdu_json(&event_id).await {
|
||||
| Ok(event_json) => {
|
||||
if let Some(content_key) = event_json.get("content") {
|
||||
debug!("Event ID has \"content\".");
|
||||
let content_obj = content_key.as_object();
|
||||
// parsing the PDU for any MXC URLs begins here
|
||||
match self.services.rooms.timeline.get_pdu_json(&event_id).await {
|
||||
| Ok(event_json) => {
|
||||
if let Some(content_key) = event_json.get("content") {
|
||||
debug!("Event ID has \"content\".");
|
||||
let content_obj = content_key.as_object();
|
||||
|
||||
if let Some(content) = content_obj {
|
||||
// 1. attempts to parse the "url" key
|
||||
debug!("Attempting to go into \"url\" key for main media file");
|
||||
if let Some(url) = content.get("url") {
|
||||
debug!("Got a URL in the event ID {event_id}: {url}");
|
||||
if let Some(content) = content_obj {
|
||||
// 1. attempts to parse the "url" key
|
||||
debug!("Attempting to go into \"url\" key for main media file");
|
||||
if let Some(url) = content.get("url") {
|
||||
debug!("Got a URL in the event ID {event_id}: {url}");
|
||||
|
||||
if url.to_string().starts_with("\"mxc://") {
|
||||
debug!("Pushing URL {url} to list of MXCs to delete");
|
||||
let final_url = url.to_string().replace('"', "");
|
||||
mxc_urls.push(final_url);
|
||||
} else {
|
||||
info!(
|
||||
"Found a URL in the event ID {event_id} but did not start \
|
||||
with mxc://, ignoring"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// 2. attempts to parse the "info" key
|
||||
debug!("Attempting to go into \"info\" key for thumbnails");
|
||||
if let Some(info_key) = content.get("info") {
|
||||
debug!("Event ID has \"info\".");
|
||||
let info_obj = info_key.as_object();
|
||||
|
||||
if let Some(info) = info_obj {
|
||||
if let Some(thumbnail_url) = info.get("thumbnail_url") {
|
||||
debug!("Found a thumbnail_url in info key: {thumbnail_url}");
|
||||
|
||||
if thumbnail_url.to_string().starts_with("\"mxc://") {
|
||||
debug!(
|
||||
"Pushing thumbnail URL {thumbnail_url} to list of \
|
||||
MXCs to delete"
|
||||
);
|
||||
let final_thumbnail_url =
|
||||
thumbnail_url.to_string().replace('"', "");
|
||||
mxc_urls.push(final_thumbnail_url);
|
||||
} else {
|
||||
info!(
|
||||
"Found a thumbnail URL in the event ID {event_id} \
|
||||
but did not start with mxc://, ignoring"
|
||||
);
|
||||
}
|
||||
if url.to_string().starts_with("\"mxc://") {
|
||||
debug!("Pushing URL {url} to list of MXCs to delete");
|
||||
let final_url = url.to_string().replace('"', "");
|
||||
mxc_urls.push(final_url);
|
||||
} else {
|
||||
info!(
|
||||
"No \"thumbnail_url\" key in \"info\" key, assuming no \
|
||||
thumbnails."
|
||||
"Found a URL in the event ID {event_id} but did not \
|
||||
start with mxc://, ignoring"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 3. attempts to parse the "file" key
|
||||
debug!("Attempting to go into \"file\" key");
|
||||
if let Some(file_key) = content.get("file") {
|
||||
debug!("Event ID has \"file\".");
|
||||
let file_obj = file_key.as_object();
|
||||
// 2. attempts to parse the "info" key
|
||||
debug!("Attempting to go into \"info\" key for thumbnails");
|
||||
if let Some(info_key) = content.get("info") {
|
||||
debug!("Event ID has \"info\".");
|
||||
let info_obj = info_key.as_object();
|
||||
|
||||
if let Some(file) = file_obj {
|
||||
if let Some(url) = file.get("url") {
|
||||
debug!("Found url in file key: {url}");
|
||||
if let Some(info) = info_obj {
|
||||
if let Some(thumbnail_url) = info.get("thumbnail_url") {
|
||||
debug!(
|
||||
"Found a thumbnail_url in info key: {thumbnail_url}"
|
||||
);
|
||||
|
||||
if url.to_string().starts_with("\"mxc://") {
|
||||
debug!("Pushing URL {url} to list of MXCs to delete");
|
||||
let final_url = url.to_string().replace('"', "");
|
||||
mxc_urls.push(final_url);
|
||||
if thumbnail_url.to_string().starts_with("\"mxc://") {
|
||||
debug!(
|
||||
"Pushing thumbnail URL {thumbnail_url} to list \
|
||||
of MXCs to delete"
|
||||
);
|
||||
let final_thumbnail_url =
|
||||
thumbnail_url.to_string().replace('"', "");
|
||||
mxc_urls.push(final_thumbnail_url);
|
||||
} else {
|
||||
info!(
|
||||
"Found a thumbnail URL in the event ID \
|
||||
{event_id} but did not start with mxc://, \
|
||||
ignoring"
|
||||
);
|
||||
}
|
||||
} else {
|
||||
warn!(
|
||||
"Found a URL in the event ID {event_id} but did not \
|
||||
start with mxc://, ignoring"
|
||||
info!(
|
||||
"No \"thumbnail_url\" key in \"info\" key, assuming \
|
||||
no thumbnails."
|
||||
);
|
||||
}
|
||||
} else {
|
||||
error!("No \"url\" key in \"file\" key.");
|
||||
}
|
||||
}
|
||||
|
||||
// 3. attempts to parse the "file" key
|
||||
debug!("Attempting to go into \"file\" key");
|
||||
if let Some(file_key) = content.get("file") {
|
||||
debug!("Event ID has \"file\".");
|
||||
let file_obj = file_key.as_object();
|
||||
|
||||
if let Some(file) = file_obj {
|
||||
if let Some(url) = file.get("url") {
|
||||
debug!("Found url in file key: {url}");
|
||||
|
||||
if url.to_string().starts_with("\"mxc://") {
|
||||
debug!("Pushing URL {url} to list of MXCs to delete");
|
||||
let final_url = url.to_string().replace('"', "");
|
||||
mxc_urls.push(final_url);
|
||||
} else {
|
||||
warn!(
|
||||
"Found a URL in the event ID {event_id} but did \
|
||||
not start with mxc://, ignoring"
|
||||
);
|
||||
}
|
||||
} else {
|
||||
error!("No \"url\" key in \"file\" key.");
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
return Err!(
|
||||
"Event ID does not have a \"content\" key or failed parsing the \
|
||||
event ID JSON.",
|
||||
);
|
||||
}
|
||||
} else {
|
||||
return Err!(
|
||||
"Event ID does not have a \"content\" key or failed parsing the \
|
||||
event ID JSON.",
|
||||
"Event ID does not have a \"content\" key, this is not a message or \
|
||||
an event type that contains media.",
|
||||
);
|
||||
}
|
||||
} else {
|
||||
return Err!(
|
||||
"Event ID does not have a \"content\" key, this is not a message or an \
|
||||
event type that contains media.",
|
||||
);
|
||||
},
|
||||
| _ => {
|
||||
return Err!("Event ID does not exist or is not known to us.",);
|
||||
},
|
||||
}
|
||||
|
||||
if mxc_urls.is_empty() {
|
||||
return Err!("Parsed event ID but found no MXC URLs.",);
|
||||
}
|
||||
|
||||
let mut mxc_deletion_count: usize = 0;
|
||||
|
||||
for mxc_url in mxc_urls {
|
||||
match self
|
||||
.services
|
||||
.media
|
||||
.delete(&mxc_url.as_str().try_into()?)
|
||||
.await
|
||||
{
|
||||
| Ok(()) => {
|
||||
debug_info!(
|
||||
"Successfully deleted {mxc_url} from filesystem and database"
|
||||
);
|
||||
mxc_deletion_count = mxc_deletion_count.saturating_add(1);
|
||||
},
|
||||
| Err(e) => {
|
||||
debug_warn!(
|
||||
"Failed to delete {mxc_url}, ignoring error and skipping: {e}"
|
||||
);
|
||||
continue;
|
||||
},
|
||||
}
|
||||
},
|
||||
| _ => {
|
||||
return Err!("Event ID does not exist or is not known to us.",);
|
||||
},
|
||||
}
|
||||
|
||||
return self
|
||||
.write_str(&format!(
|
||||
"Deleted {mxc_deletion_count} total MXCs from our database and the \
|
||||
filesystem from event ID {event_id}."
|
||||
))
|
||||
.await;
|
||||
}
|
||||
|
||||
if mxc_urls.is_empty() {
|
||||
return Err!("Parsed event ID but found no MXC URLs.",);
|
||||
Err!(
|
||||
"Please specify either an MXC using --mxc or an event ID using --event-id of the \
|
||||
message containing an image. See --help for details."
|
||||
)
|
||||
}
|
||||
|
||||
pub(super) async fn delete_list(&self) -> Result {
|
||||
self.bail_restricted()?;
|
||||
|
||||
if self.body.len() < 2
|
||||
|| !self.body[0].trim().starts_with("```")
|
||||
|| self.body.last().unwrap_or(&"").trim() != "```"
|
||||
{
|
||||
return Err!("Expected code block in command body. Add --help for details.",);
|
||||
}
|
||||
|
||||
let mut failed_parsed_mxcs: usize = 0;
|
||||
|
||||
let mxc_list = self
|
||||
.body
|
||||
.to_vec()
|
||||
.drain(1..self.body.len().checked_sub(1).unwrap())
|
||||
.filter_map(|mxc_s| {
|
||||
mxc_s
|
||||
.try_into()
|
||||
.inspect_err(|e| {
|
||||
debug_warn!("Failed to parse user-provided MXC URI: {e}");
|
||||
failed_parsed_mxcs = failed_parsed_mxcs.saturating_add(1);
|
||||
})
|
||||
.ok()
|
||||
})
|
||||
.collect::<Vec<Mxc<'_>>>();
|
||||
|
||||
let mut mxc_deletion_count: usize = 0;
|
||||
|
||||
for mxc_url in mxc_urls {
|
||||
match self
|
||||
.services
|
||||
.media
|
||||
.delete(&mxc_url.as_str().try_into()?)
|
||||
.await
|
||||
{
|
||||
for mxc in &mxc_list {
|
||||
trace!(%failed_parsed_mxcs, %mxc_deletion_count, "Deleting MXC {mxc} in bulk");
|
||||
match self.services.media.delete(mxc).await {
|
||||
| Ok(()) => {
|
||||
debug_info!("Successfully deleted {mxc_url} from filesystem and database");
|
||||
debug_info!("Successfully deleted {mxc} from filesystem and database");
|
||||
mxc_deletion_count = mxc_deletion_count.saturating_add(1);
|
||||
},
|
||||
| Err(e) => {
|
||||
debug_warn!("Failed to delete {mxc_url}, ignoring error and skipping: {e}");
|
||||
debug_warn!("Failed to delete {mxc}, ignoring error and skipping: {e}");
|
||||
continue;
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
return self
|
||||
.write_str(&format!(
|
||||
"Deleted {mxc_deletion_count} total MXCs from our database and the filesystem \
|
||||
from event ID {event_id}."
|
||||
))
|
||||
.await;
|
||||
self.write_str(&format!(
|
||||
"Finished bulk MXC deletion, deleted {mxc_deletion_count} total MXCs from our \
|
||||
database and the filesystem. {failed_parsed_mxcs} MXCs failed to be parsed from \
|
||||
the database.",
|
||||
))
|
||||
.await
|
||||
}
|
||||
|
||||
Err!(
|
||||
"Please specify either an MXC using --mxc or an event ID using --event-id of the \
|
||||
message containing an image. See --help for details."
|
||||
)
|
||||
}
|
||||
pub(super) async fn delete_past_remote_media(
|
||||
&self,
|
||||
duration: String,
|
||||
before: bool,
|
||||
after: bool,
|
||||
yes_i_want_to_delete_local_media: bool,
|
||||
) -> Result {
|
||||
self.bail_restricted()?;
|
||||
|
||||
#[admin_command]
|
||||
pub(super) async fn delete_list(&self) -> Result {
|
||||
self.bail_restricted()?;
|
||||
|
||||
if self.body.len() < 2
|
||||
|| !self.body[0].trim().starts_with("```")
|
||||
|| self.body.last().unwrap_or(&"").trim() != "```"
|
||||
{
|
||||
return Err!("Expected code block in command body. Add --help for details.",);
|
||||
}
|
||||
|
||||
let mut failed_parsed_mxcs: usize = 0;
|
||||
|
||||
let mxc_list = self
|
||||
.body
|
||||
.to_vec()
|
||||
.drain(1..self.body.len().checked_sub(1).unwrap())
|
||||
.filter_map(|mxc_s| {
|
||||
mxc_s
|
||||
.try_into()
|
||||
.inspect_err(|e| {
|
||||
debug_warn!("Failed to parse user-provided MXC URI: {e}");
|
||||
failed_parsed_mxcs = failed_parsed_mxcs.saturating_add(1);
|
||||
})
|
||||
.ok()
|
||||
})
|
||||
.collect::<Vec<Mxc<'_>>>();
|
||||
|
||||
let mut mxc_deletion_count: usize = 0;
|
||||
|
||||
for mxc in &mxc_list {
|
||||
trace!(%failed_parsed_mxcs, %mxc_deletion_count, "Deleting MXC {mxc} in bulk");
|
||||
match self.services.media.delete(mxc).await {
|
||||
| Ok(()) => {
|
||||
debug_info!("Successfully deleted {mxc} from filesystem and database");
|
||||
mxc_deletion_count = mxc_deletion_count.saturating_add(1);
|
||||
},
|
||||
| Err(e) => {
|
||||
debug_warn!("Failed to delete {mxc}, ignoring error and skipping: {e}");
|
||||
continue;
|
||||
},
|
||||
if before && after {
|
||||
return Err!("Please only pick one argument, --before or --after.",);
|
||||
}
|
||||
}
|
||||
assert!(!(before && after), "--before and --after should not be specified together");
|
||||
|
||||
self.write_str(&format!(
|
||||
"Finished bulk MXC deletion, deleted {mxc_deletion_count} total MXCs from our database \
|
||||
and the filesystem. {failed_parsed_mxcs} MXCs failed to be parsed from the database.",
|
||||
))
|
||||
.await
|
||||
}
|
||||
|
||||
#[admin_command]
|
||||
pub(super) async fn delete_past_remote_media(
|
||||
&self,
|
||||
duration: String,
|
||||
before: bool,
|
||||
after: bool,
|
||||
yes_i_want_to_delete_local_media: bool,
|
||||
) -> Result {
|
||||
self.bail_restricted()?;
|
||||
|
||||
if before && after {
|
||||
return Err!("Please only pick one argument, --before or --after.",);
|
||||
}
|
||||
assert!(!(before && after), "--before and --after should not be specified together");
|
||||
|
||||
let direction = if after {
|
||||
TimeDirection::After
|
||||
} else {
|
||||
TimeDirection::Before
|
||||
};
|
||||
|
||||
let time_boundary = parse_timepoint_ago(&duration)?;
|
||||
let deleted_count = self
|
||||
.services
|
||||
.media
|
||||
.delete_all_media_within_timeframe(
|
||||
time_boundary,
|
||||
direction,
|
||||
yes_i_want_to_delete_local_media,
|
||||
)
|
||||
.await?;
|
||||
|
||||
self.write_str(&format!("Deleted {deleted_count} total files."))
|
||||
.await
|
||||
}
|
||||
|
||||
#[admin_command]
|
||||
pub(super) async fn delete_all_from_user(&self, username: String) -> Result {
|
||||
let user_id = parse_local_user_id(self.services, &username)?;
|
||||
|
||||
let deleted_count = self.services.media.delete_from_user(&user_id).await?;
|
||||
|
||||
self.write_str(&format!("Deleted {deleted_count} total files."))
|
||||
.await
|
||||
}
|
||||
|
||||
#[admin_command]
|
||||
pub(super) async fn delete_all_from_server(
|
||||
&self,
|
||||
server_name: OwnedServerName,
|
||||
yes_i_want_to_delete_local_media: bool,
|
||||
) -> Result {
|
||||
self.bail_restricted()?;
|
||||
|
||||
if server_name == self.services.globals.server_name() && !yes_i_want_to_delete_local_media {
|
||||
return Err!("This command only works for remote media by default.",);
|
||||
}
|
||||
|
||||
let Ok(all_mxcs) = self
|
||||
.services
|
||||
.media
|
||||
.get_all_mxcs()
|
||||
.await
|
||||
.inspect_err(|e| error!("Failed to get MXC URIs from our database: {e}"))
|
||||
else {
|
||||
return Err!("Failed to get MXC URIs from our database",);
|
||||
};
|
||||
|
||||
let mut deleted_count: usize = 0;
|
||||
|
||||
for mxc in all_mxcs {
|
||||
let Ok(mxc_server_name) = mxc.server_name().inspect_err(|e| {
|
||||
debug_warn!(
|
||||
"Failed to parse MXC {mxc} server name from database, ignoring error and \
|
||||
skipping: {e}"
|
||||
);
|
||||
}) else {
|
||||
continue;
|
||||
let direction = if after {
|
||||
TimeDirection::After
|
||||
} else {
|
||||
TimeDirection::Before
|
||||
};
|
||||
|
||||
if mxc_server_name != server_name
|
||||
|| (self.services.globals.server_is_ours(mxc_server_name)
|
||||
&& !yes_i_want_to_delete_local_media)
|
||||
let time_boundary = parse_timepoint_ago(&duration)?;
|
||||
let deleted_count = self
|
||||
.services
|
||||
.media
|
||||
.delete_all_media_within_timeframe(
|
||||
time_boundary,
|
||||
direction,
|
||||
yes_i_want_to_delete_local_media,
|
||||
)
|
||||
.await?;
|
||||
|
||||
self.write_str(&format!("Deleted {deleted_count} total files."))
|
||||
.await
|
||||
}
|
||||
|
||||
pub(super) async fn delete_all_from_user(&self, username: String) -> Result {
|
||||
let user_id = parse_local_user_id(self.services, &username)?;
|
||||
|
||||
let deleted_count = self.services.media.delete_from_user(&user_id).await?;
|
||||
|
||||
self.write_str(&format!("Deleted {deleted_count} total files."))
|
||||
.await
|
||||
}
|
||||
|
||||
pub(super) async fn delete_all_from_server(
|
||||
&self,
|
||||
server_name: OwnedServerName,
|
||||
yes_i_want_to_delete_local_media: bool,
|
||||
) -> Result {
|
||||
self.bail_restricted()?;
|
||||
|
||||
if server_name == self.services.globals.server_name() && !yes_i_want_to_delete_local_media
|
||||
{
|
||||
trace!("skipping MXC URI {mxc}");
|
||||
continue;
|
||||
return Err!("This command only works for remote media by default.",);
|
||||
}
|
||||
|
||||
let mxc: Mxc<'_> = mxc.as_str().try_into()?;
|
||||
let Ok(all_mxcs) = self
|
||||
.services
|
||||
.media
|
||||
.get_all_mxcs()
|
||||
.await
|
||||
.inspect_err(|e| error!("Failed to get MXC URIs from our database: {e}"))
|
||||
else {
|
||||
return Err!("Failed to get MXC URIs from our database",);
|
||||
};
|
||||
|
||||
match self.services.media.delete(&mxc).await {
|
||||
| Ok(()) => {
|
||||
deleted_count = deleted_count.saturating_add(1);
|
||||
},
|
||||
| Err(e) => {
|
||||
debug_warn!("Failed to delete {mxc}, ignoring error and skipping: {e}");
|
||||
let mut deleted_count: usize = 0;
|
||||
|
||||
for mxc in all_mxcs {
|
||||
let Ok(mxc_server_name) = mxc.server_name().inspect_err(|e| {
|
||||
debug_warn!(
|
||||
"Failed to parse MXC {mxc} server name from database, ignoring error and \
|
||||
skipping: {e}"
|
||||
);
|
||||
}) else {
|
||||
continue;
|
||||
},
|
||||
};
|
||||
|
||||
if mxc_server_name != server_name
|
||||
|| (self.services.globals.server_is_ours(mxc_server_name)
|
||||
&& !yes_i_want_to_delete_local_media)
|
||||
{
|
||||
trace!("skipping MXC URI {mxc}");
|
||||
continue;
|
||||
}
|
||||
|
||||
let mxc: Mxc<'_> = mxc.as_str().try_into()?;
|
||||
|
||||
match self.services.media.delete(&mxc).await {
|
||||
| Ok(()) => {
|
||||
deleted_count = deleted_count.saturating_add(1);
|
||||
},
|
||||
| Err(e) => {
|
||||
debug_warn!("Failed to delete {mxc}, ignoring error and skipping: {e}");
|
||||
continue;
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
self.write_str(&format!("Deleted {deleted_count} total files."))
|
||||
.await
|
||||
}
|
||||
|
||||
self.write_str(&format!("Deleted {deleted_count} total files."))
|
||||
.await
|
||||
}
|
||||
pub(super) async fn get_file_info(&self, mxc: OwnedMxcUri) -> Result {
|
||||
let mxc: Mxc<'_> = mxc.as_str().try_into()?;
|
||||
let metadata = self.services.media.get_metadata(&mxc).await;
|
||||
|
||||
#[admin_command]
|
||||
pub(super) async fn get_file_info(&self, mxc: OwnedMxcUri) -> Result {
|
||||
let mxc: Mxc<'_> = mxc.as_str().try_into()?;
|
||||
let metadata = self.services.media.get_metadata(&mxc).await;
|
||||
|
||||
self.write_str(&format!("```\n{metadata:#?}\n```")).await
|
||||
}
|
||||
|
||||
#[admin_command]
|
||||
pub(super) async fn get_remote_file(
|
||||
&self,
|
||||
mxc: OwnedMxcUri,
|
||||
server: Option<OwnedServerName>,
|
||||
timeout: u32,
|
||||
) -> Result {
|
||||
let mxc: Mxc<'_> = mxc.as_str().try_into()?;
|
||||
let timeout = Duration::from_millis(timeout.into());
|
||||
let mut result = self
|
||||
.services
|
||||
.media
|
||||
.fetch_remote_content(&mxc, None, server.as_deref(), timeout)
|
||||
.await?;
|
||||
|
||||
// Grab the length of the content before clearing it to not flood the output
|
||||
let len = result.content.as_ref().expect("content").len();
|
||||
result.content.as_mut().expect("content").clear();
|
||||
|
||||
self.write_str(&format!("```\n{result:#?}\nreceived {len} bytes for file content.\n```"))
|
||||
.await
|
||||
}
|
||||
|
||||
#[admin_command]
|
||||
pub(super) async fn get_remote_thumbnail(
|
||||
&self,
|
||||
mxc: OwnedMxcUri,
|
||||
server: Option<OwnedServerName>,
|
||||
timeout: u32,
|
||||
width: u32,
|
||||
height: u32,
|
||||
) -> Result {
|
||||
let mxc: Mxc<'_> = mxc.as_str().try_into()?;
|
||||
let timeout = Duration::from_millis(timeout.into());
|
||||
let dim = Dim::new(width, height, None);
|
||||
let mut result = self
|
||||
.services
|
||||
.media
|
||||
.fetch_remote_thumbnail(&mxc, None, server.as_deref(), timeout, &dim)
|
||||
.await?;
|
||||
|
||||
// Grab the length of the content before clearing it to not flood the output
|
||||
let len = result.content.as_ref().expect("content").len();
|
||||
result.content.as_mut().expect("content").clear();
|
||||
|
||||
self.write_str(&format!("```\n{result:#?}\nreceived {len} bytes for file content.\n```"))
|
||||
.await
|
||||
}
|
||||
|
||||
#[admin_command]
|
||||
pub(super) async fn delete_url_preview(&self, url: Option<String>, all: bool) -> Result {
|
||||
if all {
|
||||
self.services.media.clear_url_previews().await;
|
||||
|
||||
return self.write_str("Deleted all cached URL previews.").await;
|
||||
self.write_str(&format!("```\n{metadata:#?}\n```")).await
|
||||
}
|
||||
|
||||
let url = url.expect("clap enforces url is required unless --all");
|
||||
pub(super) async fn get_remote_file(
|
||||
&self,
|
||||
mxc: OwnedMxcUri,
|
||||
server: Option<OwnedServerName>,
|
||||
timeout: u32,
|
||||
) -> Result {
|
||||
let mxc: Mxc<'_> = mxc.as_str().try_into()?;
|
||||
let timeout = Duration::from_millis(timeout.into());
|
||||
let mut result = self
|
||||
.services
|
||||
.media
|
||||
.fetch_remote_content(&mxc, None, server.as_deref(), timeout)
|
||||
.await?;
|
||||
|
||||
self.services.media.remove_url_preview(&url).await?;
|
||||
// Grab the length of the content before clearing it to not flood the output
|
||||
let len = result.content.as_ref().expect("content").len();
|
||||
result.content.as_mut().expect("content").clear();
|
||||
|
||||
self.write_str(&format!("Deleted cached URL preview for: {url}"))
|
||||
.await
|
||||
self.write_str(&format!("```\n{result:#?}\nreceived {len} bytes for file content.\n```"))
|
||||
.await
|
||||
}
|
||||
|
||||
pub(super) async fn get_remote_thumbnail(
|
||||
&self,
|
||||
mxc: OwnedMxcUri,
|
||||
server: Option<OwnedServerName>,
|
||||
timeout: u32,
|
||||
width: u32,
|
||||
height: u32,
|
||||
) -> Result {
|
||||
let mxc: Mxc<'_> = mxc.as_str().try_into()?;
|
||||
let timeout = Duration::from_millis(timeout.into());
|
||||
let dim = Dim::new(width, height, None);
|
||||
let mut result = self
|
||||
.services
|
||||
.media
|
||||
.fetch_remote_thumbnail(&mxc, None, server.as_deref(), timeout, &dim)
|
||||
.await?;
|
||||
|
||||
// Grab the length of the content before clearing it to not flood the output
|
||||
let len = result.content.as_ref().expect("content").len();
|
||||
result.content.as_mut().expect("content").clear();
|
||||
|
||||
self.write_str(&format!("```\n{result:#?}\nreceived {len} bytes for file content.\n```"))
|
||||
.await
|
||||
}
|
||||
|
||||
pub(super) async fn delete_url_preview(&self, url: Option<String>, all: bool) -> Result {
|
||||
if all {
|
||||
self.services.media.clear_url_previews().await;
|
||||
|
||||
return self.write_str("Deleted all cached URL previews.").await;
|
||||
}
|
||||
|
||||
let url = url.expect("clap enforces url is required unless --all");
|
||||
|
||||
self.services.media.remove_url_preview(&url).await?;
|
||||
|
||||
self.write_str(&format!("Deleted cached URL preview for: {url}"))
|
||||
.await
|
||||
}
|
||||
}
|
||||
+1
-1
@@ -26,7 +26,7 @@ extern crate conduwuit_api as api;
|
||||
extern crate conduwuit_core as conduwuit;
|
||||
extern crate conduwuit_service as service;
|
||||
|
||||
pub(crate) use conduwuit_macros::{admin_command, admin_command_dispatch};
|
||||
pub(crate) use conduwuit_macros::admin_command_dispatch;
|
||||
|
||||
pub(crate) use crate::{context::Context, utils::get_room_info};
|
||||
|
||||
|
||||
@@ -4,7 +4,7 @@ use conduwuit_database::Deserialized as _;
|
||||
use futures::StreamExt;
|
||||
use ruma::{OwnedRoomId, OwnedUserId, exports::serde::Serialize};
|
||||
|
||||
use crate::{admin_command, admin_command_dispatch};
|
||||
use crate::admin_command_dispatch;
|
||||
|
||||
#[admin_command_dispatch]
|
||||
#[derive(Debug, Subcommand)]
|
||||
@@ -31,50 +31,50 @@ pub enum AccountDataCommand {
|
||||
},
|
||||
}
|
||||
|
||||
#[admin_command]
|
||||
async fn changes_since(
|
||||
&self,
|
||||
user_id: OwnedUserId,
|
||||
since: u64,
|
||||
room_id: Option<OwnedRoomId>,
|
||||
) -> Result {
|
||||
let timer = tokio::time::Instant::now();
|
||||
let results: Vec<_> = self
|
||||
.services
|
||||
.account_data
|
||||
.changes_since(room_id.as_deref(), &user_id, Some(since), None)
|
||||
.collect()
|
||||
.await;
|
||||
let query_time = timer.elapsed();
|
||||
impl crate::Context<'_> {
|
||||
async fn changes_since(
|
||||
&self,
|
||||
user_id: OwnedUserId,
|
||||
since: u64,
|
||||
room_id: Option<OwnedRoomId>,
|
||||
) -> Result {
|
||||
let timer = tokio::time::Instant::now();
|
||||
let results: Vec<_> = self
|
||||
.services
|
||||
.account_data
|
||||
.changes_since(room_id.as_deref(), &user_id, Some(since), None)
|
||||
.collect()
|
||||
.await;
|
||||
let query_time = timer.elapsed();
|
||||
|
||||
self.write_str(&format!("Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```"))
|
||||
.await
|
||||
}
|
||||
self.write_str(&format!("Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```"))
|
||||
.await
|
||||
}
|
||||
|
||||
#[admin_command]
|
||||
async fn account_data_get(
|
||||
&self,
|
||||
user_id: OwnedUserId,
|
||||
kind: String,
|
||||
room_id: Option<OwnedRoomId>,
|
||||
) -> Result {
|
||||
let timer = tokio::time::Instant::now();
|
||||
let result = self
|
||||
.services
|
||||
.account_data
|
||||
.get_raw(room_id.as_deref(), &user_id, &kind)
|
||||
.await;
|
||||
let query_time = timer.elapsed();
|
||||
async fn account_data_get(
|
||||
&self,
|
||||
user_id: OwnedUserId,
|
||||
kind: String,
|
||||
room_id: Option<OwnedRoomId>,
|
||||
) -> Result {
|
||||
let timer = tokio::time::Instant::now();
|
||||
let result = self
|
||||
.services
|
||||
.account_data
|
||||
.get_raw(room_id.as_deref(), &user_id, &kind)
|
||||
.await;
|
||||
let query_time = timer.elapsed();
|
||||
|
||||
let json = serde_json::to_string_pretty(&match room_id {
|
||||
| None => result
|
||||
.deserialized::<ruma::serde::Raw<ruma::events::AnyGlobalAccountDataEvent>>()?
|
||||
.serialize(serde_json::value::Serializer)?,
|
||||
| Some(_) => result
|
||||
.deserialized::<ruma::serde::Raw<ruma::events::AnyRoomAccountDataEvent>>()?
|
||||
.serialize(serde_json::value::Serializer)?,
|
||||
})?;
|
||||
let json = serde_json::to_string_pretty(&match room_id {
|
||||
| None => result
|
||||
.deserialized::<ruma::serde::Raw<ruma::events::AnyGlobalAccountDataEvent>>()?
|
||||
.serialize(serde_json::value::Serializer)?,
|
||||
| Some(_) => result
|
||||
.deserialized::<ruma::serde::Raw<ruma::events::AnyRoomAccountDataEvent>>()?
|
||||
.serialize(serde_json::value::Serializer)?,
|
||||
})?;
|
||||
|
||||
self.write_str(&format!("Query completed in {query_time:?}:\n\n```rs\n{json}\n```"))
|
||||
.await
|
||||
self.write_str(&format!("Query completed in {query_time:?}:\n\n```rs\n{json}\n```"))
|
||||
.await
|
||||
}
|
||||
}
|
||||
+246
-238
@@ -13,7 +13,7 @@ use conduwuit_service::Services;
|
||||
use futures::{FutureExt, Stream, StreamExt, TryStreamExt};
|
||||
use tokio::time::Instant;
|
||||
|
||||
use crate::{admin_command, admin_command_dispatch};
|
||||
use crate::admin_command_dispatch;
|
||||
|
||||
#[admin_command_dispatch]
|
||||
#[derive(Debug, Subcommand)]
|
||||
@@ -159,279 +159,287 @@ pub enum RawCommand {
|
||||
},
|
||||
}
|
||||
|
||||
#[admin_command]
|
||||
pub(super) async fn compact(
|
||||
&self,
|
||||
map: Option<Vec<String>>,
|
||||
start: Option<String>,
|
||||
stop: Option<String>,
|
||||
from: Option<usize>,
|
||||
into: Option<usize>,
|
||||
parallelism: Option<usize>,
|
||||
exhaustive: bool,
|
||||
) -> Result {
|
||||
use conduwuit_database::compact::Options;
|
||||
impl crate::Context<'_> {
|
||||
pub(super) async fn compact(
|
||||
&self,
|
||||
map: Option<Vec<String>>,
|
||||
start: Option<String>,
|
||||
stop: Option<String>,
|
||||
from: Option<usize>,
|
||||
into: Option<usize>,
|
||||
parallelism: Option<usize>,
|
||||
exhaustive: bool,
|
||||
) -> Result {
|
||||
use conduwuit_database::compact::Options;
|
||||
|
||||
let default_all_maps: Option<_> = map.is_none().then(|| {
|
||||
self.services
|
||||
.db
|
||||
.keys()
|
||||
.map(Deref::deref)
|
||||
.map(ToOwned::to_owned)
|
||||
});
|
||||
let default_all_maps: Option<_> = map.is_none().then(|| {
|
||||
self.services
|
||||
.db
|
||||
.keys()
|
||||
.map(Deref::deref)
|
||||
.map(ToOwned::to_owned)
|
||||
});
|
||||
|
||||
let maps: Vec<_> = map
|
||||
.unwrap_or_default()
|
||||
.into_iter()
|
||||
.chain(default_all_maps.into_iter().flatten())
|
||||
.map(|map| self.services.db.get(&map))
|
||||
.filter_map(Result::ok)
|
||||
.cloned()
|
||||
.collect();
|
||||
let maps: Vec<_> = map
|
||||
.unwrap_or_default()
|
||||
.into_iter()
|
||||
.chain(default_all_maps.into_iter().flatten())
|
||||
.map(|map| self.services.db.get(&map))
|
||||
.filter_map(Result::ok)
|
||||
.cloned()
|
||||
.collect();
|
||||
|
||||
if maps.is_empty() {
|
||||
return Err!("--map argument invalid. not found in database");
|
||||
if maps.is_empty() {
|
||||
return Err!("--map argument invalid. not found in database");
|
||||
}
|
||||
|
||||
let range = (
|
||||
start.as_ref().map(String::as_bytes).map(Into::into),
|
||||
stop.as_ref().map(String::as_bytes).map(Into::into),
|
||||
);
|
||||
|
||||
let options = Options {
|
||||
range,
|
||||
level: (from, into),
|
||||
exclusive: parallelism.is_some_and(is_zero!()),
|
||||
exhaustive,
|
||||
};
|
||||
|
||||
let runtime = self.services.server.runtime().clone();
|
||||
let parallelism = parallelism.unwrap_or(1);
|
||||
let results = maps
|
||||
.into_iter()
|
||||
.try_stream::<conduwuit::Error>()
|
||||
.paralleln_and_then(runtime, parallelism, move |map| {
|
||||
map.compact_blocking(options.clone())?;
|
||||
Ok(map.name().to_owned())
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
|
||||
let timer = Instant::now();
|
||||
let results = results.await;
|
||||
let query_time = timer.elapsed();
|
||||
self.write_str(&format!("Jobs completed in {query_time:?}:\n\n```rs\n{results:#?}\n```"))
|
||||
.await
|
||||
}
|
||||
|
||||
let range = (
|
||||
start.as_ref().map(String::as_bytes).map(Into::into),
|
||||
stop.as_ref().map(String::as_bytes).map(Into::into),
|
||||
);
|
||||
pub(super) async fn raw_count(&self, map: Option<String>, prefix: Option<String>) -> Result {
|
||||
let prefix = prefix.as_deref().unwrap_or(EMPTY);
|
||||
|
||||
let options = Options {
|
||||
range,
|
||||
level: (from, into),
|
||||
exclusive: parallelism.is_some_and(is_zero!()),
|
||||
exhaustive,
|
||||
};
|
||||
let timer = Instant::now();
|
||||
let count = with_maps_or(map.as_deref(), self.services)
|
||||
.then(|map| map.raw_count_prefix(&prefix))
|
||||
.ready_fold(0_usize, usize::saturating_add)
|
||||
.await;
|
||||
|
||||
let runtime = self.services.server.runtime().clone();
|
||||
let parallelism = parallelism.unwrap_or(1);
|
||||
let results = maps
|
||||
.into_iter()
|
||||
.try_stream::<conduwuit::Error>()
|
||||
.paralleln_and_then(runtime, parallelism, move |map| {
|
||||
map.compact_blocking(options.clone())?;
|
||||
Ok(map.name().to_owned())
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
let query_time = timer.elapsed();
|
||||
self.write_str(&format!("Query completed in {query_time:?}:\n\n```rs\n{count:#?}\n```"))
|
||||
.await
|
||||
}
|
||||
|
||||
let timer = Instant::now();
|
||||
let results = results.await;
|
||||
let query_time = timer.elapsed();
|
||||
self.write_str(&format!("Jobs completed in {query_time:?}:\n\n```rs\n{results:#?}\n```"))
|
||||
.await
|
||||
}
|
||||
pub(super) async fn raw_keys(&self, map: String, prefix: Option<String>) -> Result {
|
||||
writeln!(self, "```").boxed().await?;
|
||||
|
||||
#[admin_command]
|
||||
pub(super) async fn raw_count(&self, map: Option<String>, prefix: Option<String>) -> Result {
|
||||
let prefix = prefix.as_deref().unwrap_or(EMPTY);
|
||||
let map = self.services.db.get(map.as_str())?;
|
||||
let timer = Instant::now();
|
||||
prefix
|
||||
.as_deref()
|
||||
.map_or_else(|| map.raw_keys().boxed(), |prefix| map.raw_keys_prefix(prefix).boxed())
|
||||
.map_ok(String::from_utf8_lossy)
|
||||
.try_for_each(|str| writeln!(self, "{str:?}"))
|
||||
.boxed()
|
||||
.await?;
|
||||
|
||||
let timer = Instant::now();
|
||||
let count = with_maps_or(map.as_deref(), self.services)
|
||||
.then(|map| map.raw_count_prefix(&prefix))
|
||||
.ready_fold(0_usize, usize::saturating_add)
|
||||
.await;
|
||||
let query_time = timer.elapsed();
|
||||
self.write_str(&format!("\n```\n\nQuery completed in {query_time:?}"))
|
||||
.await
|
||||
}
|
||||
|
||||
let query_time = timer.elapsed();
|
||||
self.write_str(&format!("Query completed in {query_time:?}:\n\n```rs\n{count:#?}\n```"))
|
||||
.await
|
||||
}
|
||||
pub(super) async fn raw_keys_sizes(
|
||||
&self,
|
||||
map: Option<String>,
|
||||
prefix: Option<String>,
|
||||
) -> Result {
|
||||
let prefix = prefix.as_deref().unwrap_or(EMPTY);
|
||||
|
||||
#[admin_command]
|
||||
pub(super) async fn raw_keys(&self, map: String, prefix: Option<String>) -> Result {
|
||||
writeln!(self, "```").boxed().await?;
|
||||
let timer = Instant::now();
|
||||
let result = with_maps_or(map.as_deref(), self.services)
|
||||
.map(|map| map.raw_keys_prefix(&prefix))
|
||||
.flatten()
|
||||
.ignore_err()
|
||||
.map(<[u8]>::len)
|
||||
.ready_fold_default(|mut map: BTreeMap<_, usize>, len| {
|
||||
let entry = map.entry(len).or_default();
|
||||
*entry = entry.saturating_add(1);
|
||||
map
|
||||
})
|
||||
.await;
|
||||
|
||||
let map = self.services.db.get(map.as_str())?;
|
||||
let timer = Instant::now();
|
||||
prefix
|
||||
.as_deref()
|
||||
.map_or_else(|| map.raw_keys().boxed(), |prefix| map.raw_keys_prefix(prefix).boxed())
|
||||
.map_ok(String::from_utf8_lossy)
|
||||
.try_for_each(|str| writeln!(self, "{str:?}"))
|
||||
.boxed()
|
||||
.await?;
|
||||
let query_time = timer.elapsed();
|
||||
self.write_str(&format!("```\n{result:#?}\n```\n\nQuery completed in {query_time:?}"))
|
||||
.await
|
||||
}
|
||||
|
||||
let query_time = timer.elapsed();
|
||||
self.write_str(&format!("\n```\n\nQuery completed in {query_time:?}"))
|
||||
.await
|
||||
}
|
||||
pub(super) async fn raw_keys_total(
|
||||
&self,
|
||||
map: Option<String>,
|
||||
prefix: Option<String>,
|
||||
) -> Result {
|
||||
let prefix = prefix.as_deref().unwrap_or(EMPTY);
|
||||
|
||||
#[admin_command]
|
||||
pub(super) async fn raw_keys_sizes(&self, map: Option<String>, prefix: Option<String>) -> Result {
|
||||
let prefix = prefix.as_deref().unwrap_or(EMPTY);
|
||||
let timer = Instant::now();
|
||||
let result = with_maps_or(map.as_deref(), self.services)
|
||||
.map(|map| map.raw_keys_prefix(&prefix))
|
||||
.flatten()
|
||||
.ignore_err()
|
||||
.map(<[u8]>::len)
|
||||
.ready_fold_default(|acc: usize, len| acc.saturating_add(len))
|
||||
.await;
|
||||
|
||||
let timer = Instant::now();
|
||||
let result = with_maps_or(map.as_deref(), self.services)
|
||||
.map(|map| map.raw_keys_prefix(&prefix))
|
||||
.flatten()
|
||||
.ignore_err()
|
||||
.map(<[u8]>::len)
|
||||
.ready_fold_default(|mut map: BTreeMap<_, usize>, len| {
|
||||
let entry = map.entry(len).or_default();
|
||||
*entry = entry.saturating_add(1);
|
||||
map
|
||||
})
|
||||
.await;
|
||||
let query_time = timer.elapsed();
|
||||
self.write_str(&format!("```\n{result:#?}\n\n```\n\nQuery completed in {query_time:?}"))
|
||||
.await
|
||||
}
|
||||
|
||||
let query_time = timer.elapsed();
|
||||
self.write_str(&format!("```\n{result:#?}\n```\n\nQuery completed in {query_time:?}"))
|
||||
.await
|
||||
}
|
||||
pub(super) async fn raw_vals_sizes(
|
||||
&self,
|
||||
map: Option<String>,
|
||||
prefix: Option<String>,
|
||||
) -> Result {
|
||||
let prefix = prefix.as_deref().unwrap_or(EMPTY);
|
||||
|
||||
#[admin_command]
|
||||
pub(super) async fn raw_keys_total(&self, map: Option<String>, prefix: Option<String>) -> Result {
|
||||
let prefix = prefix.as_deref().unwrap_or(EMPTY);
|
||||
let timer = Instant::now();
|
||||
let result = with_maps_or(map.as_deref(), self.services)
|
||||
.map(|map| map.raw_stream_prefix(&prefix))
|
||||
.flatten()
|
||||
.ignore_err()
|
||||
.map(at!(1))
|
||||
.map(<[u8]>::len)
|
||||
.ready_fold_default(|mut map: BTreeMap<_, usize>, len| {
|
||||
let entry = map.entry(len).or_default();
|
||||
*entry = entry.saturating_add(1);
|
||||
map
|
||||
})
|
||||
.await;
|
||||
|
||||
let timer = Instant::now();
|
||||
let result = with_maps_or(map.as_deref(), self.services)
|
||||
.map(|map| map.raw_keys_prefix(&prefix))
|
||||
.flatten()
|
||||
.ignore_err()
|
||||
.map(<[u8]>::len)
|
||||
.ready_fold_default(|acc: usize, len| acc.saturating_add(len))
|
||||
.await;
|
||||
let query_time = timer.elapsed();
|
||||
self.write_str(&format!("```\n{result:#?}\n```\n\nQuery completed in {query_time:?}"))
|
||||
.await
|
||||
}
|
||||
|
||||
let query_time = timer.elapsed();
|
||||
self.write_str(&format!("```\n{result:#?}\n\n```\n\nQuery completed in {query_time:?}"))
|
||||
.await
|
||||
}
|
||||
pub(super) async fn raw_vals_total(
|
||||
&self,
|
||||
map: Option<String>,
|
||||
prefix: Option<String>,
|
||||
) -> Result {
|
||||
let prefix = prefix.as_deref().unwrap_or(EMPTY);
|
||||
|
||||
#[admin_command]
|
||||
pub(super) async fn raw_vals_sizes(&self, map: Option<String>, prefix: Option<String>) -> Result {
|
||||
let prefix = prefix.as_deref().unwrap_or(EMPTY);
|
||||
let timer = Instant::now();
|
||||
let result = with_maps_or(map.as_deref(), self.services)
|
||||
.map(|map| map.raw_stream_prefix(&prefix))
|
||||
.flatten()
|
||||
.ignore_err()
|
||||
.map(at!(1))
|
||||
.map(<[u8]>::len)
|
||||
.ready_fold_default(|acc: usize, len| acc.saturating_add(len))
|
||||
.await;
|
||||
|
||||
let timer = Instant::now();
|
||||
let result = with_maps_or(map.as_deref(), self.services)
|
||||
.map(|map| map.raw_stream_prefix(&prefix))
|
||||
.flatten()
|
||||
.ignore_err()
|
||||
.map(at!(1))
|
||||
.map(<[u8]>::len)
|
||||
.ready_fold_default(|mut map: BTreeMap<_, usize>, len| {
|
||||
let entry = map.entry(len).or_default();
|
||||
*entry = entry.saturating_add(1);
|
||||
map
|
||||
})
|
||||
.await;
|
||||
let query_time = timer.elapsed();
|
||||
self.write_str(&format!("```\n{result:#?}\n\n```\n\nQuery completed in {query_time:?}"))
|
||||
.await
|
||||
}
|
||||
|
||||
let query_time = timer.elapsed();
|
||||
self.write_str(&format!("```\n{result:#?}\n```\n\nQuery completed in {query_time:?}"))
|
||||
.await
|
||||
}
|
||||
pub(super) async fn raw_iter(&self, map: String, prefix: Option<String>) -> Result {
|
||||
writeln!(self, "```").await?;
|
||||
|
||||
#[admin_command]
|
||||
pub(super) async fn raw_vals_total(&self, map: Option<String>, prefix: Option<String>) -> Result {
|
||||
let prefix = prefix.as_deref().unwrap_or(EMPTY);
|
||||
let map = self.services.db.get(&map)?;
|
||||
let timer = Instant::now();
|
||||
prefix
|
||||
.as_deref()
|
||||
.map_or_else(
|
||||
|| map.raw_stream().boxed(),
|
||||
|prefix| map.raw_stream_prefix(prefix).boxed(),
|
||||
)
|
||||
.map_ok(apply!(2, String::from_utf8_lossy))
|
||||
.map_ok(apply!(2, Cow::into_owned))
|
||||
.try_for_each(|keyval| writeln!(self, "{keyval:?}"))
|
||||
.boxed()
|
||||
.await?;
|
||||
|
||||
let timer = Instant::now();
|
||||
let result = with_maps_or(map.as_deref(), self.services)
|
||||
.map(|map| map.raw_stream_prefix(&prefix))
|
||||
.flatten()
|
||||
.ignore_err()
|
||||
.map(at!(1))
|
||||
.map(<[u8]>::len)
|
||||
.ready_fold_default(|acc: usize, len| acc.saturating_add(len))
|
||||
.await;
|
||||
let query_time = timer.elapsed();
|
||||
self.write_str(&format!("\n```\n\nQuery completed in {query_time:?}"))
|
||||
.await
|
||||
}
|
||||
|
||||
let query_time = timer.elapsed();
|
||||
self.write_str(&format!("```\n{result:#?}\n\n```\n\nQuery completed in {query_time:?}"))
|
||||
.await
|
||||
}
|
||||
pub(super) async fn raw_keys_from(
|
||||
&self,
|
||||
map: String,
|
||||
start: String,
|
||||
limit: Option<usize>,
|
||||
) -> Result {
|
||||
writeln!(self, "```").await?;
|
||||
|
||||
#[admin_command]
|
||||
pub(super) async fn raw_iter(&self, map: String, prefix: Option<String>) -> Result {
|
||||
writeln!(self, "```").await?;
|
||||
let map = self.services.db.get(&map)?;
|
||||
let timer = Instant::now();
|
||||
map.raw_keys_from(&start)
|
||||
.map_ok(String::from_utf8_lossy)
|
||||
.take(limit.unwrap_or(usize::MAX))
|
||||
.try_for_each(|str| writeln!(self, "{str:?}"))
|
||||
.boxed()
|
||||
.await?;
|
||||
|
||||
let map = self.services.db.get(&map)?;
|
||||
let timer = Instant::now();
|
||||
prefix
|
||||
.as_deref()
|
||||
.map_or_else(|| map.raw_stream().boxed(), |prefix| map.raw_stream_prefix(prefix).boxed())
|
||||
.map_ok(apply!(2, String::from_utf8_lossy))
|
||||
.map_ok(apply!(2, Cow::into_owned))
|
||||
.try_for_each(|keyval| writeln!(self, "{keyval:?}"))
|
||||
.boxed()
|
||||
.await?;
|
||||
let query_time = timer.elapsed();
|
||||
self.write_str(&format!("\n```\n\nQuery completed in {query_time:?}"))
|
||||
.await
|
||||
}
|
||||
|
||||
let query_time = timer.elapsed();
|
||||
self.write_str(&format!("\n```\n\nQuery completed in {query_time:?}"))
|
||||
.await
|
||||
}
|
||||
pub(super) async fn raw_iter_from(
|
||||
&self,
|
||||
map: String,
|
||||
start: String,
|
||||
limit: Option<usize>,
|
||||
) -> Result {
|
||||
let map = self.services.db.get(&map)?;
|
||||
let timer = Instant::now();
|
||||
let result = map
|
||||
.raw_stream_from(&start)
|
||||
.map_ok(apply!(2, String::from_utf8_lossy))
|
||||
.map_ok(apply!(2, Cow::into_owned))
|
||||
.take(limit.unwrap_or(usize::MAX))
|
||||
.try_collect::<Vec<(String, String)>>()
|
||||
.await?;
|
||||
|
||||
#[admin_command]
|
||||
pub(super) async fn raw_keys_from(
|
||||
&self,
|
||||
map: String,
|
||||
start: String,
|
||||
limit: Option<usize>,
|
||||
) -> Result {
|
||||
writeln!(self, "```").await?;
|
||||
let query_time = timer.elapsed();
|
||||
self.write_str(&format!("Query completed in {query_time:?}:\n\n```rs\n{result:#?}\n```"))
|
||||
.await
|
||||
}
|
||||
|
||||
let map = self.services.db.get(&map)?;
|
||||
let timer = Instant::now();
|
||||
map.raw_keys_from(&start)
|
||||
.map_ok(String::from_utf8_lossy)
|
||||
.take(limit.unwrap_or(usize::MAX))
|
||||
.try_for_each(|str| writeln!(self, "{str:?}"))
|
||||
.boxed()
|
||||
.await?;
|
||||
pub(super) async fn raw_del(&self, map: String, key: String) -> Result {
|
||||
let map = self.services.db.get(&map)?;
|
||||
let timer = Instant::now();
|
||||
map.remove(&key);
|
||||
|
||||
let query_time = timer.elapsed();
|
||||
self.write_str(&format!("\n```\n\nQuery completed in {query_time:?}"))
|
||||
.await
|
||||
}
|
||||
let query_time = timer.elapsed();
|
||||
self.write_str(&format!("Operation completed in {query_time:?}"))
|
||||
.await
|
||||
}
|
||||
|
||||
#[admin_command]
|
||||
pub(super) async fn raw_iter_from(
|
||||
&self,
|
||||
map: String,
|
||||
start: String,
|
||||
limit: Option<usize>,
|
||||
) -> Result {
|
||||
let map = self.services.db.get(&map)?;
|
||||
let timer = Instant::now();
|
||||
let result = map
|
||||
.raw_stream_from(&start)
|
||||
.map_ok(apply!(2, String::from_utf8_lossy))
|
||||
.map_ok(apply!(2, Cow::into_owned))
|
||||
.take(limit.unwrap_or(usize::MAX))
|
||||
.try_collect::<Vec<(String, String)>>()
|
||||
.await?;
|
||||
pub(super) async fn raw_get(&self, map: String, key: String) -> Result {
|
||||
let map = self.services.db.get(&map)?;
|
||||
let timer = Instant::now();
|
||||
let handle = map.get(&key).await?;
|
||||
|
||||
let query_time = timer.elapsed();
|
||||
self.write_str(&format!("Query completed in {query_time:?}:\n\n```rs\n{result:#?}\n```"))
|
||||
.await
|
||||
}
|
||||
let query_time = timer.elapsed();
|
||||
let result = String::from_utf8_lossy(&handle);
|
||||
self.write_str(&format!("Query completed in {query_time:?}:\n\n```rs\n{result:?}\n```"))
|
||||
.await
|
||||
}
|
||||
|
||||
#[admin_command]
|
||||
pub(super) async fn raw_del(&self, map: String, key: String) -> Result {
|
||||
let map = self.services.db.get(&map)?;
|
||||
let timer = Instant::now();
|
||||
map.remove(&key);
|
||||
pub(super) async fn raw_maps(&self) -> Result {
|
||||
let list: Vec<_> = self.services.db.iter().map(at!(0)).copied().collect();
|
||||
|
||||
let query_time = timer.elapsed();
|
||||
self.write_str(&format!("Operation completed in {query_time:?}"))
|
||||
.await
|
||||
}
|
||||
|
||||
#[admin_command]
|
||||
pub(super) async fn raw_get(&self, map: String, key: String) -> Result {
|
||||
let map = self.services.db.get(&map)?;
|
||||
let timer = Instant::now();
|
||||
let handle = map.get(&key).await?;
|
||||
|
||||
let query_time = timer.elapsed();
|
||||
let result = String::from_utf8_lossy(&handle);
|
||||
self.write_str(&format!("Query completed in {query_time:?}:\n\n```rs\n{result:?}\n```"))
|
||||
.await
|
||||
}
|
||||
|
||||
#[admin_command]
|
||||
pub(super) async fn raw_maps(&self) -> Result {
|
||||
let list: Vec<_> = self.services.db.iter().map(at!(0)).copied().collect();
|
||||
|
||||
self.write_str(&format!("{list:#?}")).await
|
||||
self.write_str(&format!("{list:#?}")).await
|
||||
}
|
||||
}
|
||||
|
||||
fn with_maps_or<'a>(
|
||||
|
||||
+48
-47
@@ -3,7 +3,7 @@ use conduwuit::{Err, Result, utils::time};
|
||||
use futures::StreamExt;
|
||||
use ruma::OwnedServerName;
|
||||
|
||||
use crate::{admin_command, admin_command_dispatch};
|
||||
use crate::admin_command_dispatch;
|
||||
|
||||
#[admin_command_dispatch]
|
||||
#[derive(Debug, Subcommand)]
|
||||
@@ -39,67 +39,68 @@ pub enum ResolverCommand {
|
||||
},
|
||||
}
|
||||
|
||||
#[admin_command]
|
||||
async fn destinations_cache(&self, server_name: Option<OwnedServerName>) -> Result {
|
||||
use service::resolver::cache::CachedDest;
|
||||
impl crate::Context<'_> {
|
||||
async fn destinations_cache(&self, server_name: Option<OwnedServerName>) -> Result {
|
||||
use service::resolver::cache::CachedDest;
|
||||
|
||||
writeln!(self, "| Server Name | Destination | Hostname | Expires |").await?;
|
||||
writeln!(self, "| ----------- | ----------- | -------- | ------- |").await?;
|
||||
writeln!(self, "| Server Name | Destination | Hostname | Expires |").await?;
|
||||
writeln!(self, "| ----------- | ----------- | -------- | ------- |").await?;
|
||||
|
||||
let mut destinations = self.services.resolver.cache.destinations().boxed();
|
||||
let mut destinations = self.services.resolver.cache.destinations().boxed();
|
||||
|
||||
while let Some((name, CachedDest { dest, host, expire })) = destinations.next().await {
|
||||
if let Some(server_name) = server_name.as_ref() {
|
||||
if name != *server_name {
|
||||
continue;
|
||||
while let Some((name, CachedDest { dest, host, expire })) = destinations.next().await {
|
||||
if let Some(server_name) = server_name.as_ref() {
|
||||
if name != *server_name {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
let expire = time::format(expire, "%+");
|
||||
self.write_str(&format!("| {name} | {dest} | {host} | {expire} |\n"))
|
||||
.await?;
|
||||
}
|
||||
|
||||
let expire = time::format(expire, "%+");
|
||||
self.write_str(&format!("| {name} | {dest} | {host} | {expire} |\n"))
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
async fn overrides_cache(&self, server_name: Option<String>) -> Result {
|
||||
use service::resolver::cache::CachedOverride;
|
||||
|
||||
#[admin_command]
|
||||
async fn overrides_cache(&self, server_name: Option<String>) -> Result {
|
||||
use service::resolver::cache::CachedOverride;
|
||||
writeln!(self, "| Server Name | IP | Port | Expires | Overriding |").await?;
|
||||
writeln!(self, "| ----------- | --- | ----:| ------- | ---------- |").await?;
|
||||
|
||||
writeln!(self, "| Server Name | IP | Port | Expires | Overriding |").await?;
|
||||
writeln!(self, "| ----------- | --- | ----:| ------- | ---------- |").await?;
|
||||
let mut overrides = self.services.resolver.cache.overrides().boxed();
|
||||
|
||||
let mut overrides = self.services.resolver.cache.overrides().boxed();
|
||||
|
||||
while let Some((name, CachedOverride { ips, port, expire, overriding })) =
|
||||
overrides.next().await
|
||||
{
|
||||
if let Some(server_name) = server_name.as_ref() {
|
||||
if name != *server_name {
|
||||
continue;
|
||||
while let Some((name, CachedOverride { ips, port, expire, overriding })) =
|
||||
overrides.next().await
|
||||
{
|
||||
if let Some(server_name) = server_name.as_ref() {
|
||||
if name != *server_name {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
let expire = time::format(expire, "%+");
|
||||
self.write_str(&format!(
|
||||
"| {name} | {ips:?} | {port} | {expire} | {overriding:?} |\n"
|
||||
))
|
||||
.await?;
|
||||
}
|
||||
|
||||
let expire = time::format(expire, "%+");
|
||||
self.write_str(&format!("| {name} | {ips:?} | {port} | {expire} | {overriding:?} |\n"))
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[admin_command]
|
||||
async fn flush_cache(&self, name: Option<OwnedServerName>, all: bool) -> Result {
|
||||
if all {
|
||||
self.services.resolver.cache.clear().await;
|
||||
writeln!(self, "Resolver caches cleared!").await
|
||||
} else if let Some(name) = name {
|
||||
self.services.resolver.cache.del_destination(&name);
|
||||
self.services.resolver.cache.del_override(&name);
|
||||
self.write_str(&format!("Cleared {name} from resolver caches!"))
|
||||
.await
|
||||
} else {
|
||||
Err!("Missing name. Supply a name or use --all to flush the whole cache.")
|
||||
async fn flush_cache(&self, name: Option<OwnedServerName>, all: bool) -> Result {
|
||||
if all {
|
||||
self.services.resolver.cache.clear().await;
|
||||
writeln!(self, "Resolver caches cleared!").await
|
||||
} else if let Some(name) = name {
|
||||
self.services.resolver.cache.del_destination(&name);
|
||||
self.services.resolver.cache.del_override(&name);
|
||||
self.write_str(&format!("Cleared {name} from resolver caches!"))
|
||||
.await
|
||||
} else {
|
||||
Err!("Missing name. Supply a name or use --all to flush the whole cache.")
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -3,7 +3,7 @@ use conduwuit::{PduCount, Result, utils::stream::TryTools};
|
||||
use futures::TryStreamExt;
|
||||
use ruma::OwnedRoomOrAliasId;
|
||||
|
||||
use crate::{admin_command, admin_command_dispatch};
|
||||
use crate::admin_command_dispatch;
|
||||
|
||||
#[admin_command_dispatch]
|
||||
#[derive(Debug, Subcommand)]
|
||||
@@ -23,39 +23,39 @@ pub enum RoomTimelineCommand {
|
||||
},
|
||||
}
|
||||
|
||||
#[admin_command]
|
||||
pub(super) async fn last(&self, room_id: OwnedRoomOrAliasId) -> Result {
|
||||
let room_id = self.services.rooms.alias.resolve(&room_id).await?;
|
||||
impl crate::Context<'_> {
|
||||
pub(super) async fn last(&self, room_id: OwnedRoomOrAliasId) -> Result {
|
||||
let room_id = self.services.rooms.alias.resolve(&room_id).await?;
|
||||
|
||||
let result = self
|
||||
.services
|
||||
.rooms
|
||||
.timeline
|
||||
.last_timeline_count(&room_id)
|
||||
.await?;
|
||||
let result = self
|
||||
.services
|
||||
.rooms
|
||||
.timeline
|
||||
.last_timeline_count(&room_id)
|
||||
.await?;
|
||||
|
||||
self.write_str(&format!("{result:#?}")).await
|
||||
}
|
||||
self.write_str(&format!("{result:#?}")).await
|
||||
}
|
||||
|
||||
#[admin_command]
|
||||
pub(super) async fn pdus(
|
||||
&self,
|
||||
room_id: OwnedRoomOrAliasId,
|
||||
from: Option<String>,
|
||||
limit: Option<usize>,
|
||||
) -> Result {
|
||||
let room_id = self.services.rooms.alias.resolve(&room_id).await?;
|
||||
pub(super) async fn pdus(
|
||||
&self,
|
||||
room_id: OwnedRoomOrAliasId,
|
||||
from: Option<String>,
|
||||
limit: Option<usize>,
|
||||
) -> Result {
|
||||
let room_id = self.services.rooms.alias.resolve(&room_id).await?;
|
||||
|
||||
let from: Option<PduCount> = from.as_deref().map(str::parse).transpose()?;
|
||||
let from: Option<PduCount> = from.as_deref().map(str::parse).transpose()?;
|
||||
|
||||
let result: Vec<_> = self
|
||||
.services
|
||||
.rooms
|
||||
.timeline
|
||||
.pdus_rev(&room_id, from)
|
||||
.try_take(limit.unwrap_or(3))
|
||||
.try_collect()
|
||||
.await?;
|
||||
let result: Vec<_> = self
|
||||
.services
|
||||
.rooms
|
||||
.timeline
|
||||
.pdus_rev(&room_id, from)
|
||||
.try_take(limit.unwrap_or(3))
|
||||
.try_collect()
|
||||
.await?;
|
||||
|
||||
self.write_str(&format!("```\n{result:#?}\n```")).await
|
||||
self.write_str(&format!("```\n{result:#?}\n```")).await
|
||||
}
|
||||
}
|
||||
+16
-16
@@ -2,7 +2,7 @@ use clap::Subcommand;
|
||||
use conduwuit::Result;
|
||||
use ruma::{OwnedEventId, OwnedRoomOrAliasId};
|
||||
|
||||
use crate::{admin_command, admin_command_dispatch};
|
||||
use crate::admin_command_dispatch;
|
||||
|
||||
#[admin_command_dispatch]
|
||||
#[derive(Debug, Subcommand)]
|
||||
@@ -17,23 +17,23 @@ pub enum ShortCommand {
|
||||
},
|
||||
}
|
||||
|
||||
#[admin_command]
|
||||
pub(super) async fn short_event_id(&self, event_id: OwnedEventId) -> Result {
|
||||
let shortid = self
|
||||
.services
|
||||
.rooms
|
||||
.short
|
||||
.get_shorteventid(&event_id)
|
||||
.await?;
|
||||
impl crate::Context<'_> {
|
||||
pub(super) async fn short_event_id(&self, event_id: OwnedEventId) -> Result {
|
||||
let shortid = self
|
||||
.services
|
||||
.rooms
|
||||
.short
|
||||
.get_shorteventid(&event_id)
|
||||
.await?;
|
||||
|
||||
self.write_str(&format!("{shortid:#?}")).await
|
||||
}
|
||||
self.write_str(&format!("{shortid:#?}")).await
|
||||
}
|
||||
|
||||
#[admin_command]
|
||||
pub(super) async fn short_room_id(&self, room_id: OwnedRoomOrAliasId) -> Result {
|
||||
let room_id = self.services.rooms.alias.resolve(&room_id).await?;
|
||||
pub(super) async fn short_room_id(&self, room_id: OwnedRoomOrAliasId) -> Result {
|
||||
let room_id = self.services.rooms.alias.resolve(&room_id).await?;
|
||||
|
||||
let shortid = self.services.rooms.short.get_shortroomid(&room_id).await?;
|
||||
let shortid = self.services.rooms.short.get_shortroomid(&room_id).await?;
|
||||
|
||||
self.write_str(&format!("{shortid:#?}")).await
|
||||
self.write_str(&format!("{shortid:#?}")).await
|
||||
}
|
||||
}
|
||||
+255
-259
@@ -3,7 +3,7 @@ use conduwuit::Result;
|
||||
use futures::stream::StreamExt;
|
||||
use ruma::{OwnedDeviceId, OwnedRoomId, OwnedUserId};
|
||||
|
||||
use crate::{admin_command, admin_command_dispatch};
|
||||
use crate::admin_command_dispatch;
|
||||
|
||||
#[admin_command_dispatch]
|
||||
#[derive(Debug, Subcommand)]
|
||||
@@ -92,263 +92,259 @@ pub enum UsersCommand {
|
||||
},
|
||||
}
|
||||
|
||||
#[admin_command]
|
||||
async fn get_shared_rooms(&self, user_a: OwnedUserId, user_b: OwnedUserId) -> Result {
|
||||
let timer = tokio::time::Instant::now();
|
||||
let result: Vec<_> = self
|
||||
.services
|
||||
.rooms
|
||||
.state_cache
|
||||
.get_shared_rooms(&user_a, &user_b)
|
||||
.collect()
|
||||
.await;
|
||||
let query_time = timer.elapsed();
|
||||
impl crate::Context<'_> {
|
||||
async fn get_shared_rooms(&self, user_a: OwnedUserId, user_b: OwnedUserId) -> Result {
|
||||
let timer = tokio::time::Instant::now();
|
||||
let result: Vec<_> = self
|
||||
.services
|
||||
.rooms
|
||||
.state_cache
|
||||
.get_shared_rooms(&user_a, &user_b)
|
||||
.collect()
|
||||
.await;
|
||||
let query_time = timer.elapsed();
|
||||
|
||||
self.write_str(&format!("Query completed in {query_time:?}:\n\n```rs\n{result:#?}\n```"))
|
||||
.await
|
||||
}
|
||||
|
||||
#[admin_command]
|
||||
async fn get_backup_session(
|
||||
&self,
|
||||
user_id: OwnedUserId,
|
||||
version: String,
|
||||
room_id: OwnedRoomId,
|
||||
session_id: String,
|
||||
) -> Result {
|
||||
let timer = tokio::time::Instant::now();
|
||||
let result = self
|
||||
.services
|
||||
.key_backups
|
||||
.get_session(&user_id, &version, &room_id, &session_id)
|
||||
.await;
|
||||
let query_time = timer.elapsed();
|
||||
|
||||
self.write_str(&format!("Query completed in {query_time:?}:\n\n```rs\n{result:#?}\n```"))
|
||||
.await
|
||||
}
|
||||
|
||||
#[admin_command]
|
||||
async fn get_room_backups(
|
||||
&self,
|
||||
user_id: OwnedUserId,
|
||||
version: String,
|
||||
room_id: OwnedRoomId,
|
||||
) -> Result {
|
||||
let timer = tokio::time::Instant::now();
|
||||
let result = self
|
||||
.services
|
||||
.key_backups
|
||||
.get_room(&user_id, &version, &room_id)
|
||||
.await;
|
||||
let query_time = timer.elapsed();
|
||||
|
||||
self.write_str(&format!("Query completed in {query_time:?}:\n\n```rs\n{result:#?}\n```"))
|
||||
.await
|
||||
}
|
||||
|
||||
#[admin_command]
|
||||
async fn get_all_backups(&self, user_id: OwnedUserId, version: String) -> Result {
|
||||
let timer = tokio::time::Instant::now();
|
||||
let result = self.services.key_backups.get_all(&user_id, &version).await;
|
||||
let query_time = timer.elapsed();
|
||||
|
||||
self.write_str(&format!("Query completed in {query_time:?}:\n\n```rs\n{result:#?}\n```"))
|
||||
.await
|
||||
}
|
||||
|
||||
#[admin_command]
|
||||
async fn get_backup_algorithm(&self, user_id: OwnedUserId, version: String) -> Result {
|
||||
let timer = tokio::time::Instant::now();
|
||||
let result = self
|
||||
.services
|
||||
.key_backups
|
||||
.get_backup(&user_id, &version)
|
||||
.await;
|
||||
let query_time = timer.elapsed();
|
||||
|
||||
self.write_str(&format!("Query completed in {query_time:?}:\n\n```rs\n{result:#?}\n```"))
|
||||
.await
|
||||
}
|
||||
|
||||
#[admin_command]
|
||||
async fn get_latest_backup_version(&self, user_id: OwnedUserId) -> Result {
|
||||
let timer = tokio::time::Instant::now();
|
||||
let result = self
|
||||
.services
|
||||
.key_backups
|
||||
.get_latest_backup_version(&user_id)
|
||||
.await;
|
||||
let query_time = timer.elapsed();
|
||||
|
||||
self.write_str(&format!("Query completed in {query_time:?}:\n\n```rs\n{result:#?}\n```"))
|
||||
.await
|
||||
}
|
||||
|
||||
#[admin_command]
|
||||
async fn get_latest_backup(&self, user_id: OwnedUserId) -> Result {
|
||||
let timer = tokio::time::Instant::now();
|
||||
let result = self.services.key_backups.get_latest_backup(&user_id).await;
|
||||
let query_time = timer.elapsed();
|
||||
|
||||
self.write_str(&format!("Query completed in {query_time:?}:\n\n```rs\n{result:#?}\n```"))
|
||||
.await
|
||||
}
|
||||
|
||||
#[admin_command]
|
||||
async fn iter_users(&self) -> Result {
|
||||
let timer = tokio::time::Instant::now();
|
||||
let result: Vec<OwnedUserId> = self.services.users.stream().map(Into::into).collect().await;
|
||||
|
||||
let query_time = timer.elapsed();
|
||||
|
||||
self.write_str(&format!("Query completed in {query_time:?}:\n\n```rs\n{result:#?}\n```"))
|
||||
.await
|
||||
}
|
||||
|
||||
#[admin_command]
|
||||
async fn iter_users2(&self) -> Result {
|
||||
let timer = tokio::time::Instant::now();
|
||||
let result: Vec<_> = self.services.users.stream().collect().await;
|
||||
let result: Vec<_> = result
|
||||
.into_iter()
|
||||
.map(|user_id| String::from_utf8_lossy(user_id.as_bytes()).into_owned())
|
||||
.collect();
|
||||
|
||||
let query_time = timer.elapsed();
|
||||
|
||||
self.write_str(&format!("Query completed in {query_time:?}:\n\n```rs\n{result:?}\n```"))
|
||||
.await
|
||||
}
|
||||
|
||||
#[admin_command]
|
||||
async fn count_users(&self) -> Result {
|
||||
let timer = tokio::time::Instant::now();
|
||||
let result = self.services.users.count().await;
|
||||
let query_time = timer.elapsed();
|
||||
|
||||
self.write_str(&format!("Query completed in {query_time:?}:\n\n```rs\n{result:#?}\n```"))
|
||||
.await
|
||||
}
|
||||
|
||||
#[admin_command]
|
||||
async fn list_devices(&self, user_id: OwnedUserId) -> Result {
|
||||
let timer = tokio::time::Instant::now();
|
||||
let devices = self
|
||||
.services
|
||||
.users
|
||||
.all_device_ids(&user_id)
|
||||
.collect::<Vec<_>>()
|
||||
.await;
|
||||
|
||||
let query_time = timer.elapsed();
|
||||
|
||||
self.write_str(&format!("Query completed in {query_time:?}:\n\n```rs\n{devices:#?}\n```"))
|
||||
.await
|
||||
}
|
||||
|
||||
#[admin_command]
|
||||
async fn list_devices_metadata(&self, user_id: OwnedUserId) -> Result {
|
||||
let timer = tokio::time::Instant::now();
|
||||
let devices = self
|
||||
.services
|
||||
.users
|
||||
.all_devices_metadata(&user_id)
|
||||
.collect::<Vec<_>>()
|
||||
.await;
|
||||
let query_time = timer.elapsed();
|
||||
|
||||
self.write_str(&format!("Query completed in {query_time:?}:\n\n```rs\n{devices:#?}\n```"))
|
||||
.await
|
||||
}
|
||||
|
||||
#[admin_command]
|
||||
async fn get_device_metadata(&self, user_id: OwnedUserId, device_id: OwnedDeviceId) -> Result {
|
||||
let timer = tokio::time::Instant::now();
|
||||
let device = self
|
||||
.services
|
||||
.users
|
||||
.get_device_metadata(&user_id, &device_id)
|
||||
.await;
|
||||
let query_time = timer.elapsed();
|
||||
|
||||
self.write_str(&format!("Query completed in {query_time:?}:\n\n```rs\n{device:#?}\n```"))
|
||||
.await
|
||||
}
|
||||
|
||||
#[admin_command]
|
||||
async fn get_devices_version(&self, user_id: OwnedUserId) -> Result {
|
||||
let timer = tokio::time::Instant::now();
|
||||
let device = self.services.users.get_devicelist_version(&user_id).await;
|
||||
let query_time = timer.elapsed();
|
||||
|
||||
self.write_str(&format!("Query completed in {query_time:?}:\n\n```rs\n{device:#?}\n```"))
|
||||
.await
|
||||
}
|
||||
|
||||
#[admin_command]
|
||||
async fn count_one_time_keys(&self, user_id: OwnedUserId, device_id: OwnedDeviceId) -> Result {
|
||||
let timer = tokio::time::Instant::now();
|
||||
let result = self
|
||||
.services
|
||||
.users
|
||||
.count_one_time_keys(&user_id, &device_id)
|
||||
.await;
|
||||
let query_time = timer.elapsed();
|
||||
|
||||
self.write_str(&format!("Query completed in {query_time:?}:\n\n```rs\n{result:#?}\n```"))
|
||||
.await
|
||||
}
|
||||
|
||||
#[admin_command]
|
||||
async fn get_device_keys(&self, user_id: OwnedUserId, device_id: OwnedDeviceId) -> Result {
|
||||
let timer = tokio::time::Instant::now();
|
||||
let result = self
|
||||
.services
|
||||
.users
|
||||
.get_device_keys(&user_id, &device_id)
|
||||
.await;
|
||||
let query_time = timer.elapsed();
|
||||
|
||||
self.write_str(&format!("Query completed in {query_time:?}:\n\n```rs\n{result:#?}\n```"))
|
||||
.await
|
||||
}
|
||||
|
||||
#[admin_command]
|
||||
async fn get_user_signing_key(&self, user_id: OwnedUserId) -> Result {
|
||||
let timer = tokio::time::Instant::now();
|
||||
let result = self.services.users.get_user_signing_key(&user_id).await;
|
||||
let query_time = timer.elapsed();
|
||||
|
||||
self.write_str(&format!("Query completed in {query_time:?}:\n\n```rs\n{result:#?}\n```"))
|
||||
.await
|
||||
}
|
||||
|
||||
#[admin_command]
|
||||
async fn get_master_key(&self, user_id: OwnedUserId) -> Result {
|
||||
let timer = tokio::time::Instant::now();
|
||||
let result = self
|
||||
.services
|
||||
.users
|
||||
.get_master_key(None, &user_id, &|_| true)
|
||||
.await;
|
||||
let query_time = timer.elapsed();
|
||||
|
||||
self.write_str(&format!("Query completed in {query_time:?}:\n\n```rs\n{result:#?}\n```"))
|
||||
.await
|
||||
}
|
||||
|
||||
#[admin_command]
|
||||
async fn get_to_device_events(&self, user_id: OwnedUserId, device_id: OwnedDeviceId) -> Result {
|
||||
let timer = tokio::time::Instant::now();
|
||||
let result = self
|
||||
.services
|
||||
.users
|
||||
.get_to_device_events(&user_id, &device_id, None, None)
|
||||
.collect::<Vec<_>>()
|
||||
.await;
|
||||
let query_time = timer.elapsed();
|
||||
|
||||
self.write_str(&format!("Query completed in {query_time:?}:\n\n```rs\n{result:#?}\n```"))
|
||||
.await
|
||||
self.write_str(&format!("Query completed in {query_time:?}:\n\n```rs\n{result:#?}\n```"))
|
||||
.await
|
||||
}
|
||||
|
||||
async fn get_backup_session(
|
||||
&self,
|
||||
user_id: OwnedUserId,
|
||||
version: String,
|
||||
room_id: OwnedRoomId,
|
||||
session_id: String,
|
||||
) -> Result {
|
||||
let timer = tokio::time::Instant::now();
|
||||
let result = self
|
||||
.services
|
||||
.key_backups
|
||||
.get_session(&user_id, &version, &room_id, &session_id)
|
||||
.await;
|
||||
let query_time = timer.elapsed();
|
||||
|
||||
self.write_str(&format!("Query completed in {query_time:?}:\n\n```rs\n{result:#?}\n```"))
|
||||
.await
|
||||
}
|
||||
|
||||
async fn get_room_backups(
|
||||
&self,
|
||||
user_id: OwnedUserId,
|
||||
version: String,
|
||||
room_id: OwnedRoomId,
|
||||
) -> Result {
|
||||
let timer = tokio::time::Instant::now();
|
||||
let result = self
|
||||
.services
|
||||
.key_backups
|
||||
.get_room(&user_id, &version, &room_id)
|
||||
.await;
|
||||
let query_time = timer.elapsed();
|
||||
|
||||
self.write_str(&format!("Query completed in {query_time:?}:\n\n```rs\n{result:#?}\n```"))
|
||||
.await
|
||||
}
|
||||
|
||||
async fn get_all_backups(&self, user_id: OwnedUserId, version: String) -> Result {
|
||||
let timer = tokio::time::Instant::now();
|
||||
let result = self.services.key_backups.get_all(&user_id, &version).await;
|
||||
let query_time = timer.elapsed();
|
||||
|
||||
self.write_str(&format!("Query completed in {query_time:?}:\n\n```rs\n{result:#?}\n```"))
|
||||
.await
|
||||
}
|
||||
|
||||
async fn get_backup_algorithm(&self, user_id: OwnedUserId, version: String) -> Result {
|
||||
let timer = tokio::time::Instant::now();
|
||||
let result = self
|
||||
.services
|
||||
.key_backups
|
||||
.get_backup(&user_id, &version)
|
||||
.await;
|
||||
let query_time = timer.elapsed();
|
||||
|
||||
self.write_str(&format!("Query completed in {query_time:?}:\n\n```rs\n{result:#?}\n```"))
|
||||
.await
|
||||
}
|
||||
|
||||
async fn get_latest_backup_version(&self, user_id: OwnedUserId) -> Result {
|
||||
let timer = tokio::time::Instant::now();
|
||||
let result = self
|
||||
.services
|
||||
.key_backups
|
||||
.get_latest_backup_version(&user_id)
|
||||
.await;
|
||||
let query_time = timer.elapsed();
|
||||
|
||||
self.write_str(&format!("Query completed in {query_time:?}:\n\n```rs\n{result:#?}\n```"))
|
||||
.await
|
||||
}
|
||||
|
||||
async fn get_latest_backup(&self, user_id: OwnedUserId) -> Result {
|
||||
let timer = tokio::time::Instant::now();
|
||||
let result = self.services.key_backups.get_latest_backup(&user_id).await;
|
||||
let query_time = timer.elapsed();
|
||||
|
||||
self.write_str(&format!("Query completed in {query_time:?}:\n\n```rs\n{result:#?}\n```"))
|
||||
.await
|
||||
}
|
||||
|
||||
async fn iter_users(&self) -> Result {
|
||||
let timer = tokio::time::Instant::now();
|
||||
let result: Vec<OwnedUserId> =
|
||||
self.services.users.stream().map(Into::into).collect().await;
|
||||
|
||||
let query_time = timer.elapsed();
|
||||
|
||||
self.write_str(&format!("Query completed in {query_time:?}:\n\n```rs\n{result:#?}\n```"))
|
||||
.await
|
||||
}
|
||||
|
||||
async fn iter_users2(&self) -> Result {
|
||||
let timer = tokio::time::Instant::now();
|
||||
let result: Vec<_> = self.services.users.stream().collect().await;
|
||||
let result: Vec<_> = result
|
||||
.into_iter()
|
||||
.map(|user_id| String::from_utf8_lossy(user_id.as_bytes()).into_owned())
|
||||
.collect();
|
||||
|
||||
let query_time = timer.elapsed();
|
||||
|
||||
self.write_str(&format!("Query completed in {query_time:?}:\n\n```rs\n{result:?}\n```"))
|
||||
.await
|
||||
}
|
||||
|
||||
async fn count_users(&self) -> Result {
|
||||
let timer = tokio::time::Instant::now();
|
||||
let result = self.services.users.count().await;
|
||||
let query_time = timer.elapsed();
|
||||
|
||||
self.write_str(&format!("Query completed in {query_time:?}:\n\n```rs\n{result:#?}\n```"))
|
||||
.await
|
||||
}
|
||||
|
||||
async fn list_devices(&self, user_id: OwnedUserId) -> Result {
|
||||
let timer = tokio::time::Instant::now();
|
||||
let devices = self
|
||||
.services
|
||||
.users
|
||||
.all_device_ids(&user_id)
|
||||
.collect::<Vec<_>>()
|
||||
.await;
|
||||
|
||||
let query_time = timer.elapsed();
|
||||
|
||||
self.write_str(&format!("Query completed in {query_time:?}:\n\n```rs\n{devices:#?}\n```"))
|
||||
.await
|
||||
}
|
||||
|
||||
async fn list_devices_metadata(&self, user_id: OwnedUserId) -> Result {
|
||||
let timer = tokio::time::Instant::now();
|
||||
let devices = self
|
||||
.services
|
||||
.users
|
||||
.all_devices_metadata(&user_id)
|
||||
.collect::<Vec<_>>()
|
||||
.await;
|
||||
let query_time = timer.elapsed();
|
||||
|
||||
self.write_str(&format!("Query completed in {query_time:?}:\n\n```rs\n{devices:#?}\n```"))
|
||||
.await
|
||||
}
|
||||
|
||||
async fn get_device_metadata(
|
||||
&self,
|
||||
user_id: OwnedUserId,
|
||||
device_id: OwnedDeviceId,
|
||||
) -> Result {
|
||||
let timer = tokio::time::Instant::now();
|
||||
let device = self
|
||||
.services
|
||||
.users
|
||||
.get_device_metadata(&user_id, &device_id)
|
||||
.await;
|
||||
let query_time = timer.elapsed();
|
||||
|
||||
self.write_str(&format!("Query completed in {query_time:?}:\n\n```rs\n{device:#?}\n```"))
|
||||
.await
|
||||
}
|
||||
|
||||
async fn get_devices_version(&self, user_id: OwnedUserId) -> Result {
|
||||
let timer = tokio::time::Instant::now();
|
||||
let device = self.services.users.get_devicelist_version(&user_id).await;
|
||||
let query_time = timer.elapsed();
|
||||
|
||||
self.write_str(&format!("Query completed in {query_time:?}:\n\n```rs\n{device:#?}\n```"))
|
||||
.await
|
||||
}
|
||||
|
||||
async fn count_one_time_keys(
|
||||
&self,
|
||||
user_id: OwnedUserId,
|
||||
device_id: OwnedDeviceId,
|
||||
) -> Result {
|
||||
let timer = tokio::time::Instant::now();
|
||||
let result = self
|
||||
.services
|
||||
.users
|
||||
.count_one_time_keys(&user_id, &device_id)
|
||||
.await;
|
||||
let query_time = timer.elapsed();
|
||||
|
||||
self.write_str(&format!("Query completed in {query_time:?}:\n\n```rs\n{result:#?}\n```"))
|
||||
.await
|
||||
}
|
||||
|
||||
async fn get_device_keys(&self, user_id: OwnedUserId, device_id: OwnedDeviceId) -> Result {
|
||||
let timer = tokio::time::Instant::now();
|
||||
let result = self
|
||||
.services
|
||||
.users
|
||||
.get_device_keys(&user_id, &device_id)
|
||||
.await;
|
||||
let query_time = timer.elapsed();
|
||||
|
||||
self.write_str(&format!("Query completed in {query_time:?}:\n\n```rs\n{result:#?}\n```"))
|
||||
.await
|
||||
}
|
||||
|
||||
async fn get_user_signing_key(&self, user_id: OwnedUserId) -> Result {
|
||||
let timer = tokio::time::Instant::now();
|
||||
let result = self.services.users.get_user_signing_key(&user_id).await;
|
||||
let query_time = timer.elapsed();
|
||||
|
||||
self.write_str(&format!("Query completed in {query_time:?}:\n\n```rs\n{result:#?}\n```"))
|
||||
.await
|
||||
}
|
||||
|
||||
async fn get_master_key(&self, user_id: OwnedUserId) -> Result {
|
||||
let timer = tokio::time::Instant::now();
|
||||
let result = self
|
||||
.services
|
||||
.users
|
||||
.get_master_key(None, &user_id, &|_| true)
|
||||
.await;
|
||||
let query_time = timer.elapsed();
|
||||
|
||||
self.write_str(&format!("Query completed in {query_time:?}:\n\n```rs\n{result:#?}\n```"))
|
||||
.await
|
||||
}
|
||||
|
||||
async fn get_to_device_events(
|
||||
&self,
|
||||
user_id: OwnedUserId,
|
||||
device_id: OwnedDeviceId,
|
||||
) -> Result {
|
||||
let timer = tokio::time::Instant::now();
|
||||
let result = self
|
||||
.services
|
||||
.users
|
||||
.get_to_device_events(&user_id, &device_id, None, None)
|
||||
.collect::<Vec<_>>()
|
||||
.await;
|
||||
let query_time = timer.elapsed();
|
||||
|
||||
self.write_str(&format!("Query completed in {query_time:?}:\n\n```rs\n{result:#?}\n```"))
|
||||
.await
|
||||
}
|
||||
}
|
||||
+72
-72
@@ -2,83 +2,83 @@ use conduwuit::{Err, Result};
|
||||
use futures::StreamExt;
|
||||
use ruma::OwnedRoomId;
|
||||
|
||||
use crate::{PAGE_SIZE, admin_command, get_room_info};
|
||||
use crate::{PAGE_SIZE, get_room_info};
|
||||
|
||||
#[allow(clippy::fn_params_excessive_bools)]
|
||||
#[admin_command]
|
||||
pub(super) async fn list_rooms(
|
||||
&self,
|
||||
page: Option<usize>,
|
||||
exclude_disabled: bool,
|
||||
exclude_banned: bool,
|
||||
include_empty: bool,
|
||||
no_details: bool,
|
||||
) -> Result {
|
||||
// TODO: i know there's a way to do this with clap, but i can't seem to find it
|
||||
let page = page.unwrap_or(1);
|
||||
let mut rooms = self
|
||||
.services
|
||||
.rooms
|
||||
.metadata
|
||||
.iter_ids()
|
||||
.filter_map(|room_id| async move {
|
||||
(!exclude_disabled || !self.services.rooms.metadata.is_disabled(&room_id).await)
|
||||
.then_some(room_id)
|
||||
})
|
||||
.filter_map(|room_id| async move {
|
||||
(!exclude_banned || !self.services.rooms.metadata.is_banned(&room_id).await)
|
||||
.then_some(room_id)
|
||||
})
|
||||
.then(async |room_id| get_room_info(self.services, &room_id).await)
|
||||
.then(|(room_id, total_members, name)| async move {
|
||||
let local_members: Vec<_> = self
|
||||
.services
|
||||
.rooms
|
||||
.state_cache
|
||||
.active_local_users_in_room(&room_id)
|
||||
.collect()
|
||||
.await;
|
||||
let local_members = local_members.len();
|
||||
(room_id, total_members, local_members, name)
|
||||
})
|
||||
.filter_map(|(room_id, total_members, local_members, name)| async move {
|
||||
(include_empty || local_members > 0).then_some((room_id, total_members, name))
|
||||
})
|
||||
.collect::<Vec<_>>()
|
||||
.await;
|
||||
impl crate::Context<'_> {
|
||||
#[allow(clippy::fn_params_excessive_bools)]
|
||||
pub(super) async fn list_rooms(
|
||||
&self,
|
||||
page: Option<usize>,
|
||||
exclude_disabled: bool,
|
||||
exclude_banned: bool,
|
||||
include_empty: bool,
|
||||
no_details: bool,
|
||||
) -> Result {
|
||||
// TODO: i know there's a way to do this with clap, but i can't seem to find it
|
||||
let page = page.unwrap_or(1);
|
||||
let mut rooms = self
|
||||
.services
|
||||
.rooms
|
||||
.metadata
|
||||
.iter_ids()
|
||||
.filter_map(|room_id| async move {
|
||||
(!exclude_disabled || !self.services.rooms.metadata.is_disabled(&room_id).await)
|
||||
.then_some(room_id)
|
||||
})
|
||||
.filter_map(|room_id| async move {
|
||||
(!exclude_banned || !self.services.rooms.metadata.is_banned(&room_id).await)
|
||||
.then_some(room_id)
|
||||
})
|
||||
.then(async |room_id| get_room_info(self.services, &room_id).await)
|
||||
.then(|(room_id, total_members, name)| async move {
|
||||
let local_members: Vec<_> = self
|
||||
.services
|
||||
.rooms
|
||||
.state_cache
|
||||
.active_local_users_in_room(&room_id)
|
||||
.collect()
|
||||
.await;
|
||||
let local_members = local_members.len();
|
||||
(room_id, total_members, local_members, name)
|
||||
})
|
||||
.filter_map(|(room_id, total_members, local_members, name)| async move {
|
||||
(include_empty || local_members > 0).then_some((room_id, total_members, name))
|
||||
})
|
||||
.collect::<Vec<_>>()
|
||||
.await;
|
||||
|
||||
rooms.sort_by_key(|r| r.1);
|
||||
rooms.reverse();
|
||||
rooms.sort_by_key(|r| r.1);
|
||||
rooms.reverse();
|
||||
|
||||
let rooms = rooms
|
||||
.into_iter()
|
||||
.skip(page.saturating_sub(1).saturating_mul(PAGE_SIZE))
|
||||
.take(PAGE_SIZE)
|
||||
.collect::<Vec<_>>();
|
||||
let rooms = rooms
|
||||
.into_iter()
|
||||
.skip(page.saturating_sub(1).saturating_mul(PAGE_SIZE))
|
||||
.take(PAGE_SIZE)
|
||||
.collect::<Vec<_>>();
|
||||
|
||||
if rooms.is_empty() {
|
||||
return Err!("No more rooms.");
|
||||
if rooms.is_empty() {
|
||||
return Err!("No more rooms.");
|
||||
}
|
||||
|
||||
let body = rooms
|
||||
.iter()
|
||||
.map(|(id, members, name)| {
|
||||
if no_details {
|
||||
format!("{id}")
|
||||
} else {
|
||||
format!("{id}\tMembers: {members}\tName: {name}")
|
||||
}
|
||||
})
|
||||
.collect::<Vec<_>>()
|
||||
.join("\n");
|
||||
|
||||
self.write_str(&format!("Rooms ({}):\n```\n{body}\n```", rooms.len()))
|
||||
.await
|
||||
}
|
||||
|
||||
let body = rooms
|
||||
.iter()
|
||||
.map(|(id, members, name)| {
|
||||
if no_details {
|
||||
format!("{id}")
|
||||
} else {
|
||||
format!("{id}\tMembers: {members}\tName: {name}")
|
||||
}
|
||||
})
|
||||
.collect::<Vec<_>>()
|
||||
.join("\n");
|
||||
pub(super) async fn exists(&self, room_id: OwnedRoomId) -> Result {
|
||||
let result = self.services.rooms.metadata.exists(&room_id).await;
|
||||
|
||||
self.write_str(&format!("Rooms ({}):\n```\n{body}\n```", rooms.len()))
|
||||
.await
|
||||
}
|
||||
|
||||
#[admin_command]
|
||||
pub(super) async fn exists(&self, room_id: OwnedRoomId) -> Result {
|
||||
let result = self.services.rooms.metadata.exists(&room_id).await;
|
||||
|
||||
self.write_str(&format!("{result}")).await
|
||||
self.write_str(&format!("{result}")).await
|
||||
}
|
||||
}
|
||||
+54
-54
@@ -3,7 +3,7 @@ use conduwuit::{Err, Result, utils::ReadyExt};
|
||||
use futures::StreamExt;
|
||||
use ruma::OwnedRoomId;
|
||||
|
||||
use crate::{admin_command, admin_command_dispatch};
|
||||
use crate::admin_command_dispatch;
|
||||
|
||||
#[admin_command_dispatch]
|
||||
#[derive(Debug, Subcommand)]
|
||||
@@ -26,62 +26,62 @@ pub enum RoomInfoCommand {
|
||||
},
|
||||
}
|
||||
|
||||
#[admin_command]
|
||||
async fn list_joined_members(&self, room_id: OwnedRoomId, local_only: bool) -> Result {
|
||||
let room_name = self
|
||||
.services
|
||||
.rooms
|
||||
.state_accessor
|
||||
.get_name(&room_id)
|
||||
.await
|
||||
.unwrap_or_else(|_| room_id.to_string());
|
||||
impl crate::Context<'_> {
|
||||
async fn list_joined_members(&self, room_id: OwnedRoomId, local_only: bool) -> Result {
|
||||
let room_name = self
|
||||
.services
|
||||
.rooms
|
||||
.state_accessor
|
||||
.get_name(&room_id)
|
||||
.await
|
||||
.unwrap_or_else(|_| room_id.to_string());
|
||||
|
||||
let member_info: Vec<_> = self
|
||||
.services
|
||||
.rooms
|
||||
.state_cache
|
||||
.room_members(&room_id)
|
||||
.ready_filter(|user_id| {
|
||||
local_only
|
||||
.then(|| self.services.globals.user_is_local(user_id))
|
||||
.unwrap_or(true)
|
||||
})
|
||||
.filter_map(|user_id| async move {
|
||||
Some((
|
||||
self.services
|
||||
.users
|
||||
.displayname(&user_id)
|
||||
.await
|
||||
.unwrap_or_else(|_| user_id.to_string()),
|
||||
user_id,
|
||||
))
|
||||
})
|
||||
.collect()
|
||||
.await;
|
||||
let member_info: Vec<_> = self
|
||||
.services
|
||||
.rooms
|
||||
.state_cache
|
||||
.room_members(&room_id)
|
||||
.ready_filter(|user_id| {
|
||||
local_only
|
||||
.then(|| self.services.globals.user_is_local(user_id))
|
||||
.unwrap_or(true)
|
||||
})
|
||||
.filter_map(|user_id| async move {
|
||||
Some((
|
||||
self.services
|
||||
.users
|
||||
.displayname(&user_id)
|
||||
.await
|
||||
.unwrap_or_else(|_| user_id.to_string()),
|
||||
user_id,
|
||||
))
|
||||
})
|
||||
.collect()
|
||||
.await;
|
||||
|
||||
let num = member_info.len();
|
||||
let body = member_info
|
||||
.into_iter()
|
||||
.map(|(displayname, mxid)| format!("{mxid} | {displayname}"))
|
||||
.collect::<Vec<_>>()
|
||||
.join("\n");
|
||||
let num = member_info.len();
|
||||
let body = member_info
|
||||
.into_iter()
|
||||
.map(|(displayname, mxid)| format!("{mxid} | {displayname}"))
|
||||
.collect::<Vec<_>>()
|
||||
.join("\n");
|
||||
|
||||
self.write_str(&format!("{num} Members in Room \"{room_name}\":\n```\n{body}\n```"))
|
||||
.await
|
||||
}
|
||||
self.write_str(&format!("{num} Members in Room \"{room_name}\":\n```\n{body}\n```"))
|
||||
.await
|
||||
}
|
||||
|
||||
#[admin_command]
|
||||
async fn view_room_topic(&self, room_id: OwnedRoomId) -> Result {
|
||||
let Ok(room_topic) = self
|
||||
.services
|
||||
.rooms
|
||||
.state_accessor
|
||||
.get_room_topic(&room_id)
|
||||
.await
|
||||
else {
|
||||
return Err!("Room does not have a room topic set.");
|
||||
};
|
||||
async fn view_room_topic(&self, room_id: OwnedRoomId) -> Result {
|
||||
let Ok(room_topic) = self
|
||||
.services
|
||||
.rooms
|
||||
.state_accessor
|
||||
.get_room_topic(&room_id)
|
||||
.await
|
||||
else {
|
||||
return Err!("Room does not have a room topic set.");
|
||||
};
|
||||
|
||||
self.write_str(&format!("Room topic:\n```\n{room_topic}\n```"))
|
||||
.await
|
||||
self.write_str(&format!("Room topic:\n```\n{room_topic}\n```"))
|
||||
.await
|
||||
}
|
||||
}
|
||||
+352
-351
@@ -8,7 +8,7 @@ use conduwuit::{
|
||||
use futures::{FutureExt, StreamExt};
|
||||
use ruma::{OwnedRoomId, OwnedRoomOrAliasId, RoomAliasId, RoomId, RoomOrAliasId};
|
||||
|
||||
use crate::{admin_command, admin_command_dispatch, get_room_info};
|
||||
use crate::{admin_command_dispatch, get_room_info};
|
||||
|
||||
#[admin_command_dispatch]
|
||||
#[derive(Debug, Subcommand)]
|
||||
@@ -45,238 +45,72 @@ pub enum RoomModerationCommand {
|
||||
},
|
||||
}
|
||||
|
||||
#[admin_command]
|
||||
async fn ban_room(&self, room: OwnedRoomOrAliasId) -> Result {
|
||||
debug!("Got room alias or ID: {}", room);
|
||||
impl crate::Context<'_> {
|
||||
async fn ban_room(&self, room: OwnedRoomOrAliasId) -> Result {
|
||||
debug!("Got room alias or ID: {}", room);
|
||||
|
||||
let admin_room_alias = &self.services.globals.admin_alias;
|
||||
let admin_room_alias = &self.services.globals.admin_alias;
|
||||
|
||||
if let Ok(admin_room_id) = self.services.admin.get_admin_room().await {
|
||||
if room.to_string().eq(&admin_room_id) || room.to_string().eq(admin_room_alias) {
|
||||
return Err!("Not allowed to ban the admin room.");
|
||||
if let Ok(admin_room_id) = self.services.admin.get_admin_room().await {
|
||||
if room.to_string().eq(&admin_room_id) || room.to_string().eq(admin_room_alias) {
|
||||
return Err!("Not allowed to ban the admin room.");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let room_id = if room.is_room_id() {
|
||||
let room_id = match RoomId::parse(&room) {
|
||||
| Ok(room_id) => room_id,
|
||||
| Err(e) => {
|
||||
return Err!(
|
||||
"Failed to parse room ID {room}. Please note that this requires a full room \
|
||||
ID (`!awIh6gGInaS5wLQJwa:example.com`) or a room alias \
|
||||
(`#roomalias:example.com`): {e}"
|
||||
);
|
||||
},
|
||||
let room_id = if room.is_room_id() {
|
||||
let room_id = match RoomId::parse(&room) {
|
||||
| Ok(room_id) => room_id,
|
||||
| Err(e) => {
|
||||
return Err!(
|
||||
"Failed to parse room ID {room}. Please note that this requires a full \
|
||||
room ID (`!awIh6gGInaS5wLQJwa:example.com`) or a room alias \
|
||||
(`#roomalias:example.com`): {e}"
|
||||
);
|
||||
},
|
||||
};
|
||||
|
||||
debug!("Room specified is a room ID, banning room ID");
|
||||
|
||||
room_id.clone()
|
||||
} else if room.is_room_alias_id() {
|
||||
let room_alias = match RoomAliasId::parse(&room) {
|
||||
| Ok(room_alias) => room_alias,
|
||||
| Err(e) => {
|
||||
return Err!(
|
||||
"Failed to parse room ID {room}. Please note that this requires a full \
|
||||
room ID (`!awIh6gGInaS5wLQJwa:example.com`) or a room alias \
|
||||
(`#roomalias:example.com`): {e}"
|
||||
);
|
||||
},
|
||||
};
|
||||
|
||||
debug!(
|
||||
"Room specified is not a room ID, attempting to resolve room alias to a room ID \
|
||||
locally, if not using get_alias_helper to fetch room ID remotely"
|
||||
);
|
||||
|
||||
match self.services.rooms.alias.resolve_alias(&room_alias).await {
|
||||
| Ok((room_id, servers)) => {
|
||||
debug!(
|
||||
%room_id,
|
||||
?servers,
|
||||
"Got federation response fetching room ID for room {room}"
|
||||
);
|
||||
room_id
|
||||
},
|
||||
| Err(e) => {
|
||||
return Err!("Failed to resolve room alias {room} to a room ID: {e}");
|
||||
},
|
||||
}
|
||||
} else {
|
||||
return Err!(
|
||||
"Room specified is not a room ID or room alias. Please note that this requires \
|
||||
a full room ID (`!awIh6gGInaS5wLQJwa:example.com`) or a room alias \
|
||||
(`#roomalias:example.com`)",
|
||||
);
|
||||
};
|
||||
|
||||
debug!("Room specified is a room ID, banning room ID");
|
||||
|
||||
room_id.clone()
|
||||
} else if room.is_room_alias_id() {
|
||||
let room_alias = match RoomAliasId::parse(&room) {
|
||||
| Ok(room_alias) => room_alias,
|
||||
| Err(e) => {
|
||||
return Err!(
|
||||
"Failed to parse room ID {room}. Please note that this requires a full room \
|
||||
ID (`!awIh6gGInaS5wLQJwa:example.com`) or a room alias \
|
||||
(`#roomalias:example.com`): {e}"
|
||||
);
|
||||
},
|
||||
};
|
||||
|
||||
debug!(
|
||||
"Room specified is not a room ID, attempting to resolve room alias to a room ID \
|
||||
locally, if not using get_alias_helper to fetch room ID remotely"
|
||||
);
|
||||
|
||||
match self.services.rooms.alias.resolve_alias(&room_alias).await {
|
||||
| Ok((room_id, servers)) => {
|
||||
debug!(
|
||||
%room_id,
|
||||
?servers,
|
||||
"Got federation response fetching room ID for room {room}"
|
||||
);
|
||||
room_id
|
||||
},
|
||||
| Err(e) => {
|
||||
return Err!("Failed to resolve room alias {room} to a room ID: {e}");
|
||||
},
|
||||
}
|
||||
} else {
|
||||
return Err!(
|
||||
"Room specified is not a room ID or room alias. Please note that this requires a \
|
||||
full room ID (`!awIh6gGInaS5wLQJwa:example.com`) or a room alias \
|
||||
(`#roomalias:example.com`)",
|
||||
);
|
||||
};
|
||||
|
||||
info!("Making all users leave the room {room_id} and forgetting it");
|
||||
let mut users = self
|
||||
.services
|
||||
.rooms
|
||||
.state_cache
|
||||
.room_members(&room_id)
|
||||
.ready_filter(|user| self.services.globals.user_is_local(user))
|
||||
.boxed();
|
||||
|
||||
while let Some(ref user_id) = users.next().await {
|
||||
info!(
|
||||
"Attempting leave for user {user_id} in room {room_id} (ignoring all errors, \
|
||||
evicting admins too)",
|
||||
);
|
||||
|
||||
if let Err(e) = leave_room(self.services, user_id, &room_id, None)
|
||||
.boxed()
|
||||
.await
|
||||
{
|
||||
warn!("Failed to leave room: {e}");
|
||||
}
|
||||
|
||||
self.services.rooms.state_cache.forget(&room_id, user_id);
|
||||
}
|
||||
|
||||
self.services
|
||||
.rooms
|
||||
.alias
|
||||
.local_aliases_for_room(&room_id)
|
||||
.for_each(|local_alias| async move {
|
||||
self.services
|
||||
.rooms
|
||||
.alias
|
||||
.remove_alias(&local_alias, &self.services.globals.server_user)
|
||||
.await
|
||||
.ok();
|
||||
})
|
||||
.await;
|
||||
|
||||
self.services.rooms.directory.set_not_public(&room_id); // remove from the room directory
|
||||
self.services.rooms.metadata.ban_room(&room_id, true); // prevent further joins
|
||||
self.services.rooms.metadata.disable_room(&room_id, true); // disable federation
|
||||
|
||||
self.write_str(
|
||||
"Room banned, removed all our local users, and disabled incoming federation with room.",
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
#[admin_command]
|
||||
async fn ban_list_of_rooms(&self) -> Result {
|
||||
if self.body.len() < 2
|
||||
|| !self.body[0].trim().starts_with("```")
|
||||
|| self.body.last().unwrap_or(&"").trim() != "```"
|
||||
{
|
||||
return Err!("Expected code block in command body. Add --help for details.",);
|
||||
}
|
||||
|
||||
let rooms_s = self
|
||||
.body
|
||||
.to_vec()
|
||||
.drain(1..self.body.len().saturating_sub(1))
|
||||
.collect::<Vec<_>>();
|
||||
|
||||
let admin_room_alias = &self.services.globals.admin_alias;
|
||||
|
||||
let mut room_ban_count: usize = 0;
|
||||
let mut room_ids: Vec<OwnedRoomId> = Vec::new();
|
||||
|
||||
for &room in &rooms_s {
|
||||
match <&RoomOrAliasId>::try_from(room) {
|
||||
| Ok(room_alias_or_id) => {
|
||||
if let Ok(admin_room_id) = self.services.admin.get_admin_room().await {
|
||||
if room.to_owned().eq(&admin_room_id) || room.to_owned().eq(admin_room_alias)
|
||||
{
|
||||
warn!("User specified admin room in bulk ban list, ignoring");
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
if room_alias_or_id.is_room_id() {
|
||||
let room_id = match RoomId::parse(room_alias_or_id) {
|
||||
| Ok(room_id) => room_id,
|
||||
| Err(e) => {
|
||||
// ignore rooms we failed to parse
|
||||
warn!(
|
||||
"Error parsing room \"{room}\" during bulk room banning, \
|
||||
ignoring error and logging here: {e}"
|
||||
);
|
||||
continue;
|
||||
},
|
||||
};
|
||||
|
||||
room_ids.push(room_id.clone());
|
||||
}
|
||||
|
||||
if room_alias_or_id.is_room_alias_id() {
|
||||
match RoomAliasId::parse(room_alias_or_id) {
|
||||
| Ok(room_alias) => {
|
||||
let room_id = match self
|
||||
.services
|
||||
.rooms
|
||||
.alias
|
||||
.resolve_local_alias(&room_alias)
|
||||
.await
|
||||
{
|
||||
| Ok(room_id) => room_id,
|
||||
| _ => {
|
||||
debug!(
|
||||
"We don't have this room alias to a room ID locally, \
|
||||
attempting to fetch room ID over federation"
|
||||
);
|
||||
|
||||
match self
|
||||
.services
|
||||
.rooms
|
||||
.alias
|
||||
.resolve_alias(&room_alias)
|
||||
.await
|
||||
{
|
||||
| Ok((room_id, servers)) => {
|
||||
debug!(
|
||||
%room_id,
|
||||
?servers,
|
||||
"Got federation response fetching room ID for \
|
||||
{room}",
|
||||
);
|
||||
room_id
|
||||
},
|
||||
| Err(e) => {
|
||||
warn!(
|
||||
"Failed to resolve room alias {room} to a room \
|
||||
ID: {e}"
|
||||
);
|
||||
continue;
|
||||
},
|
||||
}
|
||||
},
|
||||
};
|
||||
|
||||
room_ids.push(room_id);
|
||||
},
|
||||
| Err(e) => {
|
||||
warn!(
|
||||
"Error parsing room \"{room}\" during bulk room banning, \
|
||||
ignoring error and logging here: {e}"
|
||||
);
|
||||
continue;
|
||||
},
|
||||
}
|
||||
}
|
||||
},
|
||||
| Err(e) => {
|
||||
warn!(
|
||||
"Error parsing room \"{room}\" during bulk room banning, ignoring error and \
|
||||
logging here: {e}"
|
||||
);
|
||||
continue;
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
for room_id in room_ids {
|
||||
debug!("Banned {room_id} successfully");
|
||||
room_ban_count = room_ban_count.saturating_add(1);
|
||||
|
||||
debug!("Making all users leave the room {room_id} and forgetting it");
|
||||
info!("Making all users leave the room {room_id} and forgetting it");
|
||||
let mut users = self
|
||||
.services
|
||||
.rooms
|
||||
@@ -286,7 +120,7 @@ async fn ban_list_of_rooms(&self) -> Result {
|
||||
.boxed();
|
||||
|
||||
while let Some(ref user_id) = users.next().await {
|
||||
debug!(
|
||||
info!(
|
||||
"Attempting leave for user {user_id} in room {room_id} (ignoring all errors, \
|
||||
evicting admins too)",
|
||||
);
|
||||
@@ -301,7 +135,6 @@ async fn ban_list_of_rooms(&self) -> Result {
|
||||
self.services.rooms.state_cache.forget(&room_id, user_id);
|
||||
}
|
||||
|
||||
// remove any local aliases, ignore errors
|
||||
self.services
|
||||
.rooms
|
||||
.alias
|
||||
@@ -316,139 +149,307 @@ async fn ban_list_of_rooms(&self) -> Result {
|
||||
})
|
||||
.await;
|
||||
|
||||
self.services.rooms.metadata.ban_room(&room_id, true);
|
||||
// unpublish from room directory, ignore errors
|
||||
self.services.rooms.directory.set_not_public(&room_id);
|
||||
self.services.rooms.metadata.disable_room(&room_id, true);
|
||||
self.services.rooms.directory.set_not_public(&room_id); // remove from the room directory
|
||||
self.services.rooms.metadata.ban_room(&room_id, true); // prevent further joins
|
||||
self.services.rooms.metadata.disable_room(&room_id, true); // disable federation
|
||||
|
||||
self.write_str(
|
||||
"Room banned, removed all our local users, and disabled incoming federation with \
|
||||
room.",
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
self.write_str(&format!(
|
||||
"Finished bulk room ban, banned {room_ban_count} total rooms, evicted all users, and \
|
||||
disabled incoming federation with the room."
|
||||
))
|
||||
.await
|
||||
}
|
||||
async fn ban_list_of_rooms(&self) -> Result {
|
||||
if self.body.len() < 2
|
||||
|| !self.body[0].trim().starts_with("```")
|
||||
|| self.body.last().unwrap_or(&"").trim() != "```"
|
||||
{
|
||||
return Err!("Expected code block in command body. Add --help for details.",);
|
||||
}
|
||||
|
||||
#[admin_command]
|
||||
async fn unban_room(&self, room: OwnedRoomOrAliasId) -> Result {
|
||||
let room_id = if room.is_room_id() {
|
||||
let room_id = match RoomId::parse(&room) {
|
||||
| Ok(room_id) => room_id,
|
||||
| Err(e) => {
|
||||
return Err!(
|
||||
"Failed to parse room ID {room}. Please note that this requires a full room \
|
||||
ID (`!awIh6gGInaS5wLQJwa:example.com`) or a room alias \
|
||||
(`#roomalias:example.com`): {e}"
|
||||
let rooms_s = self
|
||||
.body
|
||||
.to_vec()
|
||||
.drain(1..self.body.len().saturating_sub(1))
|
||||
.collect::<Vec<_>>();
|
||||
|
||||
let admin_room_alias = &self.services.globals.admin_alias;
|
||||
|
||||
let mut room_ban_count: usize = 0;
|
||||
let mut room_ids: Vec<OwnedRoomId> = Vec::new();
|
||||
|
||||
for &room in &rooms_s {
|
||||
match <&RoomOrAliasId>::try_from(room) {
|
||||
| Ok(room_alias_or_id) => {
|
||||
if let Ok(admin_room_id) = self.services.admin.get_admin_room().await {
|
||||
if room.to_owned().eq(&admin_room_id)
|
||||
|| room.to_owned().eq(admin_room_alias)
|
||||
{
|
||||
warn!("User specified admin room in bulk ban list, ignoring");
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
if room_alias_or_id.is_room_id() {
|
||||
let room_id = match RoomId::parse(room_alias_or_id) {
|
||||
| Ok(room_id) => room_id,
|
||||
| Err(e) => {
|
||||
// ignore rooms we failed to parse
|
||||
warn!(
|
||||
"Error parsing room \"{room}\" during bulk room banning, \
|
||||
ignoring error and logging here: {e}"
|
||||
);
|
||||
continue;
|
||||
},
|
||||
};
|
||||
|
||||
room_ids.push(room_id.clone());
|
||||
}
|
||||
|
||||
if room_alias_or_id.is_room_alias_id() {
|
||||
match RoomAliasId::parse(room_alias_or_id) {
|
||||
| Ok(room_alias) => {
|
||||
let room_id = match self
|
||||
.services
|
||||
.rooms
|
||||
.alias
|
||||
.resolve_local_alias(&room_alias)
|
||||
.await
|
||||
{
|
||||
| Ok(room_id) => room_id,
|
||||
| _ => {
|
||||
debug!(
|
||||
"We don't have this room alias to a room ID \
|
||||
locally, attempting to fetch room ID over \
|
||||
federation"
|
||||
);
|
||||
|
||||
match self
|
||||
.services
|
||||
.rooms
|
||||
.alias
|
||||
.resolve_alias(&room_alias)
|
||||
.await
|
||||
{
|
||||
| Ok((room_id, servers)) => {
|
||||
debug!(
|
||||
%room_id,
|
||||
?servers,
|
||||
"Got federation response fetching room ID for \
|
||||
{room}",
|
||||
);
|
||||
room_id
|
||||
},
|
||||
| Err(e) => {
|
||||
warn!(
|
||||
"Failed to resolve room alias {room} to a \
|
||||
room ID: {e}"
|
||||
);
|
||||
continue;
|
||||
},
|
||||
}
|
||||
},
|
||||
};
|
||||
|
||||
room_ids.push(room_id);
|
||||
},
|
||||
| Err(e) => {
|
||||
warn!(
|
||||
"Error parsing room \"{room}\" during bulk room banning, \
|
||||
ignoring error and logging here: {e}"
|
||||
);
|
||||
continue;
|
||||
},
|
||||
}
|
||||
}
|
||||
},
|
||||
| Err(e) => {
|
||||
warn!(
|
||||
"Error parsing room \"{room}\" during bulk room banning, ignoring error \
|
||||
and logging here: {e}"
|
||||
);
|
||||
continue;
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
for room_id in room_ids {
|
||||
debug!("Banned {room_id} successfully");
|
||||
room_ban_count = room_ban_count.saturating_add(1);
|
||||
|
||||
debug!("Making all users leave the room {room_id} and forgetting it");
|
||||
let mut users = self
|
||||
.services
|
||||
.rooms
|
||||
.state_cache
|
||||
.room_members(&room_id)
|
||||
.ready_filter(|user| self.services.globals.user_is_local(user))
|
||||
.boxed();
|
||||
|
||||
while let Some(ref user_id) = users.next().await {
|
||||
debug!(
|
||||
"Attempting leave for user {user_id} in room {room_id} (ignoring all \
|
||||
errors, evicting admins too)",
|
||||
);
|
||||
},
|
||||
|
||||
if let Err(e) = leave_room(self.services, user_id, &room_id, None)
|
||||
.boxed()
|
||||
.await
|
||||
{
|
||||
warn!("Failed to leave room: {e}");
|
||||
}
|
||||
|
||||
self.services.rooms.state_cache.forget(&room_id, user_id);
|
||||
}
|
||||
|
||||
// remove any local aliases, ignore errors
|
||||
self.services
|
||||
.rooms
|
||||
.alias
|
||||
.local_aliases_for_room(&room_id)
|
||||
.for_each(|local_alias| async move {
|
||||
self.services
|
||||
.rooms
|
||||
.alias
|
||||
.remove_alias(&local_alias, &self.services.globals.server_user)
|
||||
.await
|
||||
.ok();
|
||||
})
|
||||
.await;
|
||||
|
||||
self.services.rooms.metadata.ban_room(&room_id, true);
|
||||
// unpublish from room directory, ignore errors
|
||||
self.services.rooms.directory.set_not_public(&room_id);
|
||||
self.services.rooms.metadata.disable_room(&room_id, true);
|
||||
}
|
||||
|
||||
self.write_str(&format!(
|
||||
"Finished bulk room ban, banned {room_ban_count} total rooms, evicted all users, \
|
||||
and disabled incoming federation with the room."
|
||||
))
|
||||
.await
|
||||
}
|
||||
|
||||
async fn unban_room(&self, room: OwnedRoomOrAliasId) -> Result {
|
||||
let room_id = if room.is_room_id() {
|
||||
let room_id = match RoomId::parse(&room) {
|
||||
| Ok(room_id) => room_id,
|
||||
| Err(e) => {
|
||||
return Err!(
|
||||
"Failed to parse room ID {room}. Please note that this requires a full \
|
||||
room ID (`!awIh6gGInaS5wLQJwa:example.com`) or a room alias \
|
||||
(`#roomalias:example.com`): {e}"
|
||||
);
|
||||
},
|
||||
};
|
||||
|
||||
debug!("Room specified is a room ID, unbanning room ID");
|
||||
self.services.rooms.metadata.ban_room(&room_id, false);
|
||||
|
||||
room_id.clone()
|
||||
} else if room.is_room_alias_id() {
|
||||
let room_alias = match RoomAliasId::parse(&room) {
|
||||
| Ok(room_alias) => room_alias,
|
||||
| Err(e) => {
|
||||
return Err!(
|
||||
"Failed to parse room ID {room}. Please note that this requires a full \
|
||||
room ID (`!awIh6gGInaS5wLQJwa:example.com`) or a room alias \
|
||||
(`#roomalias:example.com`): {e}"
|
||||
);
|
||||
},
|
||||
};
|
||||
|
||||
debug!(
|
||||
"Room specified is not a room ID, attempting to resolve room alias to a room ID \
|
||||
locally, if not using get_alias_helper to fetch room ID remotely"
|
||||
);
|
||||
|
||||
let room_id = match self
|
||||
.services
|
||||
.rooms
|
||||
.alias
|
||||
.resolve_local_alias(&room_alias)
|
||||
.await
|
||||
{
|
||||
| Ok(room_id) => room_id,
|
||||
| _ => {
|
||||
debug!(
|
||||
"We don't have this room alias to a room ID locally, attempting to \
|
||||
fetch room ID over federation"
|
||||
);
|
||||
|
||||
match self.services.rooms.alias.resolve_alias(&room_alias).await {
|
||||
| Ok((room_id, servers)) => {
|
||||
debug!(
|
||||
%room_id,
|
||||
?servers,
|
||||
"Got federation response fetching room ID for room {room}"
|
||||
);
|
||||
room_id
|
||||
},
|
||||
| Err(e) => {
|
||||
return Err!("Failed to resolve room alias {room} to a room ID: {e}");
|
||||
},
|
||||
}
|
||||
},
|
||||
};
|
||||
|
||||
self.services.rooms.metadata.ban_room(&room_id, false);
|
||||
|
||||
room_id
|
||||
} else {
|
||||
return Err!(
|
||||
"Room specified is not a room ID or room alias. Please note that this requires \
|
||||
a full room ID (`!awIh6gGInaS5wLQJwa:example.com`) or a room alias \
|
||||
(`#roomalias:example.com`)",
|
||||
);
|
||||
};
|
||||
|
||||
debug!("Room specified is a room ID, unbanning room ID");
|
||||
self.services.rooms.metadata.ban_room(&room_id, false);
|
||||
self.services.rooms.metadata.disable_room(&room_id, false);
|
||||
self.write_str("Room unbanned and federation re-enabled.")
|
||||
.await
|
||||
}
|
||||
|
||||
room_id.clone()
|
||||
} else if room.is_room_alias_id() {
|
||||
let room_alias = match RoomAliasId::parse(&room) {
|
||||
| Ok(room_alias) => room_alias,
|
||||
| Err(e) => {
|
||||
return Err!(
|
||||
"Failed to parse room ID {room}. Please note that this requires a full room \
|
||||
ID (`!awIh6gGInaS5wLQJwa:example.com`) or a room alias \
|
||||
(`#roomalias:example.com`): {e}"
|
||||
);
|
||||
},
|
||||
};
|
||||
|
||||
debug!(
|
||||
"Room specified is not a room ID, attempting to resolve room alias to a room ID \
|
||||
locally, if not using get_alias_helper to fetch room ID remotely"
|
||||
);
|
||||
|
||||
let room_id = match self
|
||||
async fn list_banned_rooms(&self, no_details: bool) -> Result {
|
||||
let room_ids: Vec<OwnedRoomId> = self
|
||||
.services
|
||||
.rooms
|
||||
.alias
|
||||
.resolve_local_alias(&room_alias)
|
||||
.await
|
||||
{
|
||||
| Ok(room_id) => room_id,
|
||||
| _ => {
|
||||
debug!(
|
||||
"We don't have this room alias to a room ID locally, attempting to fetch \
|
||||
room ID over federation"
|
||||
);
|
||||
.metadata
|
||||
.list_banned_rooms()
|
||||
.map(Into::into)
|
||||
.collect()
|
||||
.await;
|
||||
|
||||
match self.services.rooms.alias.resolve_alias(&room_alias).await {
|
||||
| Ok((room_id, servers)) => {
|
||||
debug!(
|
||||
%room_id,
|
||||
?servers,
|
||||
"Got federation response fetching room ID for room {room}"
|
||||
);
|
||||
room_id
|
||||
},
|
||||
| Err(e) => {
|
||||
return Err!("Failed to resolve room alias {room} to a room ID: {e}");
|
||||
},
|
||||
if room_ids.is_empty() {
|
||||
return Err!("No rooms are banned.");
|
||||
}
|
||||
|
||||
let mut rooms = room_ids
|
||||
.iter()
|
||||
.stream()
|
||||
.then(|room_id| get_room_info(self.services, room_id))
|
||||
.collect::<Vec<_>>()
|
||||
.await;
|
||||
|
||||
rooms.sort_by_key(|r| r.1);
|
||||
rooms.reverse();
|
||||
|
||||
let num = rooms.len();
|
||||
|
||||
let body = rooms
|
||||
.iter()
|
||||
.map(|(id, members, name)| {
|
||||
if no_details {
|
||||
format!("{id}")
|
||||
} else {
|
||||
format!("{id}\tMembers: {members}\tName: {name}")
|
||||
}
|
||||
},
|
||||
};
|
||||
})
|
||||
.collect::<Vec<_>>()
|
||||
.join("\n");
|
||||
|
||||
self.services.rooms.metadata.ban_room(&room_id, false);
|
||||
|
||||
room_id
|
||||
} else {
|
||||
return Err!(
|
||||
"Room specified is not a room ID or room alias. Please note that this requires a \
|
||||
full room ID (`!awIh6gGInaS5wLQJwa:example.com`) or a room alias \
|
||||
(`#roomalias:example.com`)",
|
||||
);
|
||||
};
|
||||
|
||||
self.services.rooms.metadata.disable_room(&room_id, false);
|
||||
self.write_str("Room unbanned and federation re-enabled.")
|
||||
.await
|
||||
}
|
||||
|
||||
#[admin_command]
|
||||
async fn list_banned_rooms(&self, no_details: bool) -> Result {
|
||||
let room_ids: Vec<OwnedRoomId> = self
|
||||
.services
|
||||
.rooms
|
||||
.metadata
|
||||
.list_banned_rooms()
|
||||
.map(Into::into)
|
||||
.collect()
|
||||
.await;
|
||||
|
||||
if room_ids.is_empty() {
|
||||
return Err!("No rooms are banned.");
|
||||
self.write_str(&format!("Rooms Banned ({num}):\n```\n{body}\n```"))
|
||||
.await
|
||||
}
|
||||
|
||||
let mut rooms = room_ids
|
||||
.iter()
|
||||
.stream()
|
||||
.then(|room_id| get_room_info(self.services, room_id))
|
||||
.collect::<Vec<_>>()
|
||||
.await;
|
||||
|
||||
rooms.sort_by_key(|r| r.1);
|
||||
rooms.reverse();
|
||||
|
||||
let num = rooms.len();
|
||||
|
||||
let body = rooms
|
||||
.iter()
|
||||
.map(|(id, members, name)| {
|
||||
if no_details {
|
||||
format!("{id}")
|
||||
} else {
|
||||
format!("{id}\tMembers: {members}\tName: {name}")
|
||||
}
|
||||
})
|
||||
.collect::<Vec<_>>()
|
||||
.join("\n");
|
||||
|
||||
self.write_str(&format!("Rooms Banned ({num}):\n```\n{body}\n```"))
|
||||
.await
|
||||
}
|
||||
+197
-210
@@ -7,243 +7,230 @@ use conduwuit::{
|
||||
};
|
||||
use futures::TryStreamExt;
|
||||
|
||||
use crate::admin_command;
|
||||
impl crate::Context<'_> {
|
||||
pub(super) async fn uptime(&self) -> Result {
|
||||
let elapsed = self
|
||||
.services
|
||||
.server
|
||||
.started
|
||||
.elapsed()
|
||||
.expect("standard duration");
|
||||
|
||||
#[admin_command]
|
||||
pub(super) async fn uptime(&self) -> Result {
|
||||
let elapsed = self
|
||||
.services
|
||||
.server
|
||||
.started
|
||||
.elapsed()
|
||||
.expect("standard duration");
|
||||
let result = time::pretty(elapsed);
|
||||
self.write_str(&format!("{result}.")).await
|
||||
}
|
||||
|
||||
let result = time::pretty(elapsed);
|
||||
self.write_str(&format!("{result}.")).await
|
||||
}
|
||||
pub(super) async fn show_config(&self) -> Result {
|
||||
self.bail_restricted()?;
|
||||
|
||||
#[admin_command]
|
||||
pub(super) async fn show_config(&self) -> Result {
|
||||
self.bail_restricted()?;
|
||||
self.write_str(&format!("{}", *self.services.server.config))
|
||||
.await
|
||||
}
|
||||
|
||||
self.write_str(&format!("{}", *self.services.server.config))
|
||||
.await
|
||||
}
|
||||
pub(super) async fn reload_config(&self, path: Option<PathBuf>) -> Result {
|
||||
// The path argument is only what's optionally passed via the admin command,
|
||||
// so we need to merge it with the existing paths if any were given at startup.
|
||||
let mut paths = Vec::new();
|
||||
|
||||
#[admin_command]
|
||||
pub(super) async fn reload_config(&self, path: Option<PathBuf>) -> Result {
|
||||
// The path argument is only what's optionally passed via the admin command,
|
||||
// so we need to merge it with the existing paths if any were given at startup.
|
||||
let mut paths = Vec::new();
|
||||
// Add previously saved paths to the argument list
|
||||
self.services
|
||||
.config
|
||||
.config_paths
|
||||
.clone()
|
||||
.unwrap_or_default()
|
||||
.iter()
|
||||
.for_each(|p| paths.push(p.to_owned()));
|
||||
|
||||
// Add previously saved paths to the argument list
|
||||
self.services
|
||||
.config
|
||||
.config_paths
|
||||
.clone()
|
||||
.unwrap_or_default()
|
||||
.iter()
|
||||
.for_each(|p| paths.push(p.to_owned()));
|
||||
|
||||
// If a path is given, and it's not already in the list,
|
||||
// add it last, so that it overrides earlier files
|
||||
if let Some(p) = path {
|
||||
if !paths.contains(&p) {
|
||||
paths.push(p);
|
||||
// If a path is given, and it's not already in the list,
|
||||
// add it last, so that it overrides earlier files
|
||||
if let Some(p) = path {
|
||||
if !paths.contains(&p) {
|
||||
paths.push(p);
|
||||
}
|
||||
}
|
||||
|
||||
self.services.config.reload(&paths)?;
|
||||
|
||||
self.write_str(&format!("Successfully reconfigured from paths: {paths:?}"))
|
||||
.await
|
||||
}
|
||||
|
||||
self.services.config.reload(&paths)?;
|
||||
pub(super) async fn memory_usage(&self) -> Result {
|
||||
let services_usage = self.services.memory_usage().await?;
|
||||
let database_usage = self.services.db.db.memory_usage()?;
|
||||
let allocator_usage = conduwuit::alloc::memory_usage()
|
||||
.map_or(String::new(), |s| format!("\nAllocator:\n{s}"));
|
||||
|
||||
self.write_str(&format!("Successfully reconfigured from paths: {paths:?}"))
|
||||
self.write_str(&format!(
|
||||
"Services:\n{services_usage}\nDatabase:\n{database_usage}{allocator_usage}",
|
||||
))
|
||||
.await
|
||||
}
|
||||
|
||||
#[admin_command]
|
||||
pub(super) async fn memory_usage(&self) -> Result {
|
||||
let services_usage = self.services.memory_usage().await?;
|
||||
let database_usage = self.services.db.db.memory_usage()?;
|
||||
let allocator_usage =
|
||||
conduwuit::alloc::memory_usage().map_or(String::new(), |s| format!("\nAllocator:\n{s}"));
|
||||
|
||||
self.write_str(&format!(
|
||||
"Services:\n{services_usage}\nDatabase:\n{database_usage}{allocator_usage}",
|
||||
))
|
||||
.await
|
||||
}
|
||||
|
||||
#[admin_command]
|
||||
pub(super) async fn clear_caches(&self) -> Result {
|
||||
self.services.clear_cache().await;
|
||||
|
||||
self.write_str("Done.").await
|
||||
}
|
||||
|
||||
#[admin_command]
|
||||
pub(super) async fn list_backups(&self) -> Result {
|
||||
self.services
|
||||
.db
|
||||
.db
|
||||
.backup_list()?
|
||||
.try_stream()
|
||||
.try_for_each(|result| writeln!(self, "{result}"))
|
||||
.await
|
||||
}
|
||||
|
||||
#[admin_command]
|
||||
pub(super) async fn backup_database(&self) -> Result {
|
||||
self.bail_restricted()?;
|
||||
|
||||
let db = Arc::clone(&self.services.db);
|
||||
let result = self
|
||||
.services
|
||||
.server
|
||||
.runtime()
|
||||
.spawn_blocking(move || match db.db.backup() {
|
||||
| Ok(()) => "Done".to_owned(),
|
||||
| Err(e) => format!("Failed: {e}"),
|
||||
})
|
||||
.await?;
|
||||
|
||||
let count = self.services.db.db.backup_count()?;
|
||||
self.write_str(&format!("{result}. Currently have {count} backups."))
|
||||
.await
|
||||
}
|
||||
|
||||
#[admin_command]
|
||||
pub(super) async fn admin_notice(&self, message: Vec<String>) -> Result {
|
||||
let message = message.join(" ");
|
||||
self.services.admin.send_text(&message).await;
|
||||
|
||||
self.write_str("Notice was sent to #admins").await
|
||||
}
|
||||
|
||||
#[admin_command]
|
||||
pub(super) async fn reload_mods(&self) -> Result {
|
||||
self.bail_restricted()?;
|
||||
|
||||
self.services.server.reload()?;
|
||||
|
||||
self.write_str("Reloading server...").await
|
||||
}
|
||||
|
||||
#[admin_command]
|
||||
#[cfg(unix)]
|
||||
pub(super) async fn restart(&self, force: bool) -> Result {
|
||||
use conduwuit::utils::sys::current_exe_deleted;
|
||||
|
||||
if !force && current_exe_deleted() {
|
||||
return Err!(
|
||||
"The server cannot be restarted because the executable changed. If this is expected \
|
||||
use --force to override."
|
||||
);
|
||||
}
|
||||
|
||||
self.services.server.restart()?;
|
||||
pub(super) async fn clear_caches(&self) -> Result {
|
||||
self.services.clear_cache().await;
|
||||
|
||||
self.write_str("Restarting server...").await
|
||||
}
|
||||
|
||||
#[admin_command]
|
||||
pub(super) async fn shutdown(&self) -> Result {
|
||||
self.bail_restricted()?;
|
||||
|
||||
warn!("shutdown command");
|
||||
self.services.server.shutdown()?;
|
||||
|
||||
self.write_str("Shutting down server...").await
|
||||
}
|
||||
|
||||
#[admin_command]
|
||||
pub(super) async fn list_features(&self) -> Result {
|
||||
let mut enabled_features = conduwuit::info::introspection::ENABLED_FEATURES
|
||||
.lock()
|
||||
.expect("locked")
|
||||
.values()
|
||||
.flat_map(|f| f.iter())
|
||||
.collect::<Vec<_>>();
|
||||
|
||||
enabled_features.sort_unstable();
|
||||
enabled_features.dedup();
|
||||
|
||||
let mut available_features = conduwuit::build_metadata::WORKSPACE_FEATURES
|
||||
.iter()
|
||||
.flat_map(|(_, f)| f.iter())
|
||||
.collect::<Vec<_>>();
|
||||
|
||||
available_features.sort_unstable();
|
||||
available_features.dedup();
|
||||
|
||||
let mut features = String::new();
|
||||
|
||||
for feature in available_features {
|
||||
let active = enabled_features.contains(&feature);
|
||||
let emoji = if active { "✅" } else { "❌" };
|
||||
let remark = if active { "[enabled]" } else { "" };
|
||||
writeln!(features, "{emoji} {feature} {remark}")?;
|
||||
self.write_str("Done.").await
|
||||
}
|
||||
|
||||
self.write_str(&features).await
|
||||
}
|
||||
|
||||
#[admin_command]
|
||||
pub(super) async fn build_info(&self) -> Result {
|
||||
use conduwuit::build_metadata::built;
|
||||
pub(super) async fn list_backups(&self) -> Result {
|
||||
self.services
|
||||
.db
|
||||
.db
|
||||
.backup_list()?
|
||||
.try_stream()
|
||||
.try_for_each(|result| writeln!(self, "{result}"))
|
||||
.await
|
||||
}
|
||||
|
||||
let mut info = String::new();
|
||||
pub(super) async fn backup_database(&self) -> Result {
|
||||
self.bail_restricted()?;
|
||||
|
||||
let db = Arc::clone(&self.services.db);
|
||||
let result = self
|
||||
.services
|
||||
.server
|
||||
.runtime()
|
||||
.spawn_blocking(move || match db.db.backup() {
|
||||
| Ok(()) => "Done".to_owned(),
|
||||
| Err(e) => format!("Failed: {e}"),
|
||||
})
|
||||
.await?;
|
||||
|
||||
let count = self.services.db.db.backup_count()?;
|
||||
self.write_str(&format!("{result}. Currently have {count} backups."))
|
||||
.await
|
||||
}
|
||||
|
||||
// Version information
|
||||
writeln!(info, "# Build Information\n")?;
|
||||
writeln!(info, "**Version:** {}", built::PKG_VERSION)?;
|
||||
writeln!(info, "**Package:** {}", built::PKG_NAME)?;
|
||||
writeln!(info, "**Description:** {}", built::PKG_DESCRIPTION)?;
|
||||
pub(super) async fn admin_notice(&self, message: Vec<String>) -> Result {
|
||||
let message = message.join(" ");
|
||||
self.services.admin.send_text(&message).await;
|
||||
|
||||
// Git information
|
||||
writeln!(info, "\n## Git Information\n")?;
|
||||
if let Some(hash) = conduwuit::build_metadata::GIT_COMMIT_HASH {
|
||||
writeln!(info, "**Commit Hash:** {hash}")?;
|
||||
}
|
||||
if let Some(hash) = conduwuit::build_metadata::GIT_COMMIT_HASH_SHORT {
|
||||
writeln!(info, "**Commit Hash (short):** {hash}")?;
|
||||
}
|
||||
if let Some(url) = conduwuit::build_metadata::GIT_REMOTE_WEB_URL {
|
||||
writeln!(info, "**Repository:** {url}")?;
|
||||
self.write_str("Notice was sent to #admins").await
|
||||
}
|
||||
if let Some(url) = conduwuit::build_metadata::GIT_REMOTE_COMMIT_URL {
|
||||
writeln!(info, "**Commit URL:** {url}")?;
|
||||
|
||||
pub(super) async fn reload_mods(&self) -> Result {
|
||||
self.bail_restricted()?;
|
||||
|
||||
self.services.server.reload()?;
|
||||
|
||||
self.write_str("Reloading server...").await
|
||||
}
|
||||
|
||||
// Build environment
|
||||
writeln!(info, "\n## Build Environment\n")?;
|
||||
writeln!(info, "**Profile:** {}", built::PROFILE)?;
|
||||
writeln!(info, "**Optimization Level:** {}", built::OPT_LEVEL)?;
|
||||
writeln!(info, "**Debug:** {}", built::DEBUG)?;
|
||||
writeln!(info, "**Target:** {}", built::TARGET)?;
|
||||
writeln!(info, "**Host:** {}", built::HOST)?;
|
||||
#[cfg(unix)]
|
||||
pub(super) async fn restart(&self, force: bool) -> Result {
|
||||
use conduwuit::utils::sys::current_exe_deleted;
|
||||
|
||||
// Rust compiler information
|
||||
writeln!(info, "\n## Compiler Information\n")?;
|
||||
writeln!(info, "**Rustc Version:** {}", built::RUSTC_VERSION)?;
|
||||
if !built::RUSTDOC_VERSION.is_empty() {
|
||||
writeln!(info, "**Rustdoc Version:** {}", built::RUSTDOC_VERSION)?;
|
||||
if !force && current_exe_deleted() {
|
||||
return Err!(
|
||||
"The server cannot be restarted because the executable changed. If this is \
|
||||
expected use --force to override."
|
||||
);
|
||||
}
|
||||
|
||||
self.services.server.restart()?;
|
||||
|
||||
self.write_str("Restarting server...").await
|
||||
}
|
||||
|
||||
// Target configuration
|
||||
writeln!(info, "\n## Target Configuration\n")?;
|
||||
writeln!(info, "**Architecture:** {}", built::CFG_TARGET_ARCH)?;
|
||||
writeln!(info, "**OS:** {}", built::CFG_OS)?;
|
||||
writeln!(info, "**Family:** {}", built::CFG_FAMILY)?;
|
||||
writeln!(info, "**Endianness:** {}", built::CFG_ENDIAN)?;
|
||||
writeln!(info, "**Pointer Width:** {} bits", built::CFG_POINTER_WIDTH)?;
|
||||
if !built::CFG_ENV.is_empty() {
|
||||
writeln!(info, "**Environment:** {}", built::CFG_ENV)?;
|
||||
pub(super) async fn shutdown(&self) -> Result {
|
||||
self.bail_restricted()?;
|
||||
|
||||
warn!("shutdown command");
|
||||
self.services.server.shutdown()?;
|
||||
|
||||
self.write_str("Shutting down server...").await
|
||||
}
|
||||
|
||||
// CI information
|
||||
if let Some(ci) = built::CI_PLATFORM {
|
||||
writeln!(info, "\n## CI Platform\n")?;
|
||||
writeln!(info, "**Platform:** {ci}")?;
|
||||
pub(super) async fn list_features(&self) -> Result {
|
||||
let mut enabled_features = conduwuit::info::introspection::ENABLED_FEATURES
|
||||
.lock()
|
||||
.expect("locked")
|
||||
.values()
|
||||
.flat_map(|f| f.iter())
|
||||
.collect::<Vec<_>>();
|
||||
|
||||
enabled_features.sort_unstable();
|
||||
enabled_features.dedup();
|
||||
|
||||
let mut available_features = conduwuit::build_metadata::WORKSPACE_FEATURES
|
||||
.iter()
|
||||
.flat_map(|(_, f)| f.iter())
|
||||
.collect::<Vec<_>>();
|
||||
|
||||
available_features.sort_unstable();
|
||||
available_features.dedup();
|
||||
|
||||
let mut features = String::new();
|
||||
|
||||
for feature in available_features {
|
||||
let active = enabled_features.contains(&feature);
|
||||
let emoji = if active { "✅" } else { "❌" };
|
||||
let remark = if active { "[enabled]" } else { "" };
|
||||
writeln!(features, "{emoji} {feature} {remark}")?;
|
||||
}
|
||||
|
||||
self.write_str(&features).await
|
||||
}
|
||||
|
||||
self.write_str(&info).await
|
||||
pub(super) async fn build_info(&self) -> Result {
|
||||
use conduwuit::build_metadata::built;
|
||||
|
||||
let mut info = String::new();
|
||||
|
||||
// Version information
|
||||
writeln!(info, "# Build Information\n")?;
|
||||
writeln!(info, "**Version:** {}", built::PKG_VERSION)?;
|
||||
writeln!(info, "**Package:** {}", built::PKG_NAME)?;
|
||||
writeln!(info, "**Description:** {}", built::PKG_DESCRIPTION)?;
|
||||
|
||||
// Git information
|
||||
writeln!(info, "\n## Git Information\n")?;
|
||||
if let Some(hash) = conduwuit::build_metadata::GIT_COMMIT_HASH {
|
||||
writeln!(info, "**Commit Hash:** {hash}")?;
|
||||
}
|
||||
if let Some(hash) = conduwuit::build_metadata::GIT_COMMIT_HASH_SHORT {
|
||||
writeln!(info, "**Commit Hash (short):** {hash}")?;
|
||||
}
|
||||
if let Some(url) = conduwuit::build_metadata::GIT_REMOTE_WEB_URL {
|
||||
writeln!(info, "**Repository:** {url}")?;
|
||||
}
|
||||
if let Some(url) = conduwuit::build_metadata::GIT_REMOTE_COMMIT_URL {
|
||||
writeln!(info, "**Commit URL:** {url}")?;
|
||||
}
|
||||
|
||||
// Build environment
|
||||
writeln!(info, "\n## Build Environment\n")?;
|
||||
writeln!(info, "**Profile:** {}", built::PROFILE)?;
|
||||
writeln!(info, "**Optimization Level:** {}", built::OPT_LEVEL)?;
|
||||
writeln!(info, "**Debug:** {}", built::DEBUG)?;
|
||||
writeln!(info, "**Target:** {}", built::TARGET)?;
|
||||
writeln!(info, "**Host:** {}", built::HOST)?;
|
||||
|
||||
// Rust compiler information
|
||||
writeln!(info, "\n## Compiler Information\n")?;
|
||||
writeln!(info, "**Rustc Version:** {}", built::RUSTC_VERSION)?;
|
||||
if !built::RUSTDOC_VERSION.is_empty() {
|
||||
writeln!(info, "**Rustdoc Version:** {}", built::RUSTDOC_VERSION)?;
|
||||
}
|
||||
|
||||
// Target configuration
|
||||
writeln!(info, "\n## Target Configuration\n")?;
|
||||
writeln!(info, "**Architecture:** {}", built::CFG_TARGET_ARCH)?;
|
||||
writeln!(info, "**OS:** {}", built::CFG_OS)?;
|
||||
writeln!(info, "**Family:** {}", built::CFG_FAMILY)?;
|
||||
writeln!(info, "**Endianness:** {}", built::CFG_ENDIAN)?;
|
||||
writeln!(info, "**Pointer Width:** {} bits", built::CFG_POINTER_WIDTH)?;
|
||||
if !built::CFG_ENV.is_empty() {
|
||||
writeln!(info, "**Environment:** {}", built::CFG_ENV)?;
|
||||
}
|
||||
|
||||
// CI information
|
||||
if let Some(ci) = built::CI_PLATFORM {
|
||||
writeln!(info, "\n## CI Platform\n")?;
|
||||
writeln!(info, "**Platform:** {ci}")?;
|
||||
}
|
||||
|
||||
self.write_str(&info).await
|
||||
}
|
||||
}
|
||||
+66
-66
@@ -1,76 +1,76 @@
|
||||
use conduwuit::{Err, Result, utils};
|
||||
use conduwuit_macros::admin_command;
|
||||
use futures::StreamExt;
|
||||
use service::registration_tokens::TokenExpires;
|
||||
|
||||
#[admin_command]
|
||||
pub(super) async fn issue_token(&self, expires: super::TokenExpires) -> Result {
|
||||
let expires = {
|
||||
if expires.immortal {
|
||||
None
|
||||
} else if let Some(max_uses) = expires.max_uses {
|
||||
Some(TokenExpires::AfterUses(max_uses))
|
||||
} else if expires.once {
|
||||
Some(TokenExpires::AfterUses(1))
|
||||
} else if let Some(max_age) = expires
|
||||
.max_age
|
||||
.as_deref()
|
||||
.map(|max_age| utils::time::timepoint_from_now(utils::time::parse_duration(max_age)?))
|
||||
.transpose()?
|
||||
{
|
||||
Some(TokenExpires::AfterTime(max_age))
|
||||
} else {
|
||||
unreachable!();
|
||||
}
|
||||
};
|
||||
impl crate::Context<'_> {
|
||||
pub(super) async fn issue_token(&self, expires: super::TokenExpires) -> Result {
|
||||
let expires = {
|
||||
if expires.immortal {
|
||||
None
|
||||
} else if let Some(max_uses) = expires.max_uses {
|
||||
Some(TokenExpires::AfterUses(max_uses))
|
||||
} else if expires.once {
|
||||
Some(TokenExpires::AfterUses(1))
|
||||
} else if let Some(max_age) = expires
|
||||
.max_age
|
||||
.as_deref()
|
||||
.map(|max_age| {
|
||||
utils::time::timepoint_from_now(utils::time::parse_duration(max_age)?)
|
||||
})
|
||||
.transpose()?
|
||||
{
|
||||
Some(TokenExpires::AfterTime(max_age))
|
||||
} else {
|
||||
unreachable!();
|
||||
}
|
||||
};
|
||||
|
||||
let (token, info) = self
|
||||
.services
|
||||
.registration_tokens
|
||||
.issue_token(self.sender_or_service_user().into(), expires);
|
||||
let (token, info) = self
|
||||
.services
|
||||
.registration_tokens
|
||||
.issue_token(self.sender_or_service_user().into(), expires);
|
||||
|
||||
self.write_str(&format!(
|
||||
"New registration token issued: `{token}`. {}.",
|
||||
if let Some(expires) = info.expires {
|
||||
format!("{expires}")
|
||||
} else {
|
||||
"Never expires".to_owned()
|
||||
}
|
||||
))
|
||||
.await
|
||||
}
|
||||
|
||||
#[admin_command]
|
||||
pub(super) async fn revoke_token(&self, token: String) -> Result {
|
||||
let Some(token) = self
|
||||
.services
|
||||
.registration_tokens
|
||||
.validate_token(token)
|
||||
self.write_str(&format!(
|
||||
"New registration token issued: `{token}`. {}.",
|
||||
if let Some(expires) = info.expires {
|
||||
format!("{expires}")
|
||||
} else {
|
||||
"Never expires".to_owned()
|
||||
}
|
||||
))
|
||||
.await
|
||||
else {
|
||||
return Err!("This token does not exist or has already expired.");
|
||||
};
|
||||
|
||||
self.services.registration_tokens.revoke_token(token)?;
|
||||
|
||||
self.write_str("Token revoked successfully.").await
|
||||
}
|
||||
|
||||
#[admin_command]
|
||||
pub(super) async fn list_tokens(&self) -> Result {
|
||||
let tokens: Vec<_> = self
|
||||
.services
|
||||
.registration_tokens
|
||||
.iterate_tokens()
|
||||
.collect()
|
||||
.await;
|
||||
|
||||
self.write_str(&format!("Found {} registration tokens:\n", tokens.len()))
|
||||
.await?;
|
||||
|
||||
for token in tokens {
|
||||
self.write_str(&format!("- {token}\n")).await?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
pub(super) async fn revoke_token(&self, token: String) -> Result {
|
||||
let Some(token) = self
|
||||
.services
|
||||
.registration_tokens
|
||||
.validate_token(token)
|
||||
.await
|
||||
else {
|
||||
return Err!("This token does not exist or has already expired.");
|
||||
};
|
||||
|
||||
self.services.registration_tokens.revoke_token(token)?;
|
||||
|
||||
self.write_str("Token revoked successfully.").await
|
||||
}
|
||||
|
||||
pub(super) async fn list_tokens(&self) -> Result {
|
||||
let tokens: Vec<_> = self
|
||||
.services
|
||||
.registration_tokens
|
||||
.iterate_tokens()
|
||||
.collect()
|
||||
.await;
|
||||
|
||||
self.write_str(&format!("Found {} registration tokens:\n", tokens.len()))
|
||||
.await?;
|
||||
|
||||
for token in tokens {
|
||||
self.write_str(&format!("- {token}\n")).await?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
+1154
-1158
File diff suppressed because it is too large.
Load diff
@@ -134,7 +134,7 @@ pub(crate) async fn register_route(
|
||||
};
|
||||
|
||||
// Create user
|
||||
services.users.create(&user_id, password).await?;
|
||||
services.users.create(&user_id, password)?;
|
||||
|
||||
// Set an initial display name
|
||||
let mut displayname = user_id.localpart().to_owned();
|
||||
|
||||
@@ -32,7 +32,7 @@ pub(crate) async fn get_backfill_route(
|
||||
room_id: &body.room_id,
|
||||
event_id: None,
|
||||
}
|
||||
.check()
|
||||
.assert()
|
||||
.await?;
|
||||
if !services
|
||||
.rooms
|
||||
|
||||
@@ -44,7 +44,7 @@ pub(crate) async fn get_event_route(
|
||||
room_id,
|
||||
event_id: Some(&body.event_id),
|
||||
}
|
||||
.check()
|
||||
.assert()
|
||||
.await?;
|
||||
|
||||
if !services
|
||||
|
||||
@@ -23,7 +23,7 @@ pub(crate) async fn get_event_authorization_route(
|
||||
room_id: &body.room_id,
|
||||
event_id: None,
|
||||
}
|
||||
.check()
|
||||
.assert()
|
||||
.await?;
|
||||
|
||||
if services
|
||||
|
||||
@@ -4,15 +4,11 @@ use axum::extract::State;
|
||||
use conduwuit::{Err, Event, Result, debug, info, trace, utils::to_canonical_object, warn};
|
||||
use ruma::{OwnedEventId, api::federation::event::get_missing_events};
|
||||
use serde_json::{json, value::RawValue};
|
||||
use service::rooms::event_handler::GET_MISSING_EVENTS_MAX_BATCH_SIZE;
|
||||
|
||||
use super::AccessCheck;
|
||||
use crate::Ruma;
|
||||
|
||||
/// arbitrary number but synapse's is 20 and we can handle lots of these anyways
|
||||
const LIMIT_MAX: usize = 50;
|
||||
/// spec says default is 10
|
||||
const LIMIT_DEFAULT: usize = 10;
|
||||
|
||||
/// # `POST /_matrix/federation/v1/get_missing_events/{roomId}`
|
||||
///
|
||||
/// Retrieves events that the sender is missing.
|
||||
@@ -26,7 +22,7 @@ pub(crate) async fn get_missing_events_route(
|
||||
room_id: &body.room_id,
|
||||
event_id: None,
|
||||
}
|
||||
.check()
|
||||
.assert()
|
||||
.await?;
|
||||
|
||||
if !services
|
||||
@@ -45,8 +41,8 @@ pub(crate) async fn get_missing_events_route(
|
||||
let limit = body
|
||||
.limit
|
||||
.try_into()
|
||||
.unwrap_or(LIMIT_DEFAULT)
|
||||
.min(LIMIT_MAX);
|
||||
.unwrap_or(10)
|
||||
.min(GET_MISSING_EVENTS_MAX_BATCH_SIZE);
|
||||
|
||||
let room_version = services.rooms.state.get_room_version(&body.room_id).await?;
|
||||
|
||||
|
||||
+11
-3
@@ -7,7 +7,7 @@ use std::{
|
||||
use axum::extract::State;
|
||||
use axum_client_ip::ClientIp;
|
||||
use conduwuit::{
|
||||
Err, Error, Result, debug, debug_warn, err, error,
|
||||
Err, Error, Result, debug, debug_error, debug_warn, err, error,
|
||||
result::LogErr,
|
||||
state_res::lexicographical_topological_sort,
|
||||
trace,
|
||||
@@ -133,6 +133,7 @@ async fn wait_for_result(
|
||||
}
|
||||
|
||||
#[instrument(
|
||||
name="transaction"
|
||||
skip_all,
|
||||
fields(
|
||||
id = ?body.transaction_id.as_str(),
|
||||
@@ -174,8 +175,14 @@ async fn process_inbound_transaction(
|
||||
|
||||
for (id, result) in &results {
|
||||
if let Err(e) = result {
|
||||
if matches!(e, Error::BadRequest(ErrorKind::NotFound, _)) {
|
||||
debug_warn!("Incoming PDU failed {id}: {e:?}");
|
||||
match e {
|
||||
| Error::BadRequest(
|
||||
ErrorKind::Forbidden | ErrorKind::InvalidParam | ErrorKind::BadJson,
|
||||
..,
|
||||
) => {
|
||||
debug_warn!("Incoming PDU {id} failed: {e:?}");
|
||||
},
|
||||
| _ => debug_error!("Incoming PDU {id} failed: {e:?}"),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -381,6 +388,7 @@ async fn handle_room(
|
||||
.rooms
|
||||
.event_handler
|
||||
.handle_incoming_pdu(origin, room_id, &event_id, value, true)
|
||||
.boxed()
|
||||
.await
|
||||
.map(|_| ());
|
||||
results.push((event_id, result));
|
||||
|
||||
@@ -21,7 +21,7 @@ pub(crate) async fn get_room_state_route(
|
||||
room_id: &body.room_id,
|
||||
event_id: None,
|
||||
}
|
||||
.check()
|
||||
.assert()
|
||||
.await?;
|
||||
|
||||
if services
|
||||
|
||||
@@ -22,7 +22,7 @@ pub(crate) async fn get_room_state_ids_route(
|
||||
room_id: &body.room_id,
|
||||
event_id: None,
|
||||
}
|
||||
.check()
|
||||
.assert()
|
||||
.await?;
|
||||
|
||||
if services
|
||||
|
||||
+47
-43
@@ -1,4 +1,4 @@
|
||||
use conduwuit::{Err, Result, implement, is_false};
|
||||
use conduwuit::{Err, Result, is_false};
|
||||
use conduwuit_service::Services;
|
||||
use futures::{FutureExt, future::OptionFuture, join};
|
||||
use ruma::{EventId, RoomId, ServerName};
|
||||
@@ -10,52 +10,56 @@ pub(super) struct AccessCheck<'a> {
|
||||
pub(super) event_id: Option<&'a EventId>,
|
||||
}
|
||||
|
||||
#[implement(AccessCheck, params = "<'_>")]
|
||||
pub(super) async fn check(&self) -> Result {
|
||||
let acl_check = self
|
||||
.services
|
||||
.rooms
|
||||
.event_handler
|
||||
.acl_check(self.origin, self.room_id)
|
||||
.map(|result| result.is_ok());
|
||||
impl AccessCheck<'_> {
|
||||
/// Asserts that the server has access to the room and event (if any).
|
||||
/// If the server is permitted, `Ok(())` is returned. Otherwise, a Forbidden
|
||||
/// error is returned.
|
||||
pub(super) async fn assert(&self) -> Result {
|
||||
let acl_check = self
|
||||
.services
|
||||
.rooms
|
||||
.event_handler
|
||||
.acl_check(self.origin, self.room_id)
|
||||
.map(|result| result.is_ok());
|
||||
|
||||
let world_readable = self
|
||||
.services
|
||||
.rooms
|
||||
.state_accessor
|
||||
.is_world_readable(self.room_id);
|
||||
let world_readable = self
|
||||
.services
|
||||
.rooms
|
||||
.state_accessor
|
||||
.is_world_readable(self.room_id);
|
||||
|
||||
let server_in_room = self
|
||||
.services
|
||||
.rooms
|
||||
.state_cache
|
||||
.server_in_room(self.origin, self.room_id);
|
||||
let server_in_room = self
|
||||
.services
|
||||
.rooms
|
||||
.state_cache
|
||||
.server_in_room(self.origin, self.room_id);
|
||||
|
||||
let server_can_see: OptionFuture<_> = self
|
||||
.event_id
|
||||
.map(|event_id| {
|
||||
self.services.rooms.state_accessor.server_can_see_event(
|
||||
self.origin,
|
||||
self.room_id,
|
||||
event_id,
|
||||
)
|
||||
})
|
||||
.into();
|
||||
let server_can_see: OptionFuture<_> = self
|
||||
.event_id
|
||||
.map(|event_id| {
|
||||
self.services.rooms.state_accessor.server_can_see_event(
|
||||
self.origin,
|
||||
self.room_id,
|
||||
event_id,
|
||||
)
|
||||
})
|
||||
.into();
|
||||
|
||||
let (world_readable, server_in_room, server_can_see, acl_check) =
|
||||
join!(world_readable, server_in_room, server_can_see, acl_check);
|
||||
let (world_readable, server_in_room, server_can_see, acl_check) =
|
||||
join!(world_readable, server_in_room, server_can_see, acl_check);
|
||||
|
||||
if !acl_check {
|
||||
return Err!(Request(Forbidden("Server access denied.")));
|
||||
if !acl_check {
|
||||
return Err!(Request(Forbidden("Server access denied.")));
|
||||
}
|
||||
|
||||
if !world_readable && !server_in_room {
|
||||
return Err!(Request(Forbidden("Server is not in room.")));
|
||||
}
|
||||
|
||||
if server_can_see.is_some_and(is_false!()) {
|
||||
return Err!(Request(Forbidden("Server is not allowed to see event.")));
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
if !world_readable && !server_in_room {
|
||||
return Err!(Request(Forbidden("Server is not in room.")));
|
||||
}
|
||||
|
||||
if server_can_see.is_some_and(is_false!()) {
|
||||
return Err!(Request(Forbidden("Server is not allowed to see event.")));
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
+43
-45
@@ -10,7 +10,7 @@ use std::{
|
||||
};
|
||||
|
||||
use super::Config;
|
||||
use crate::{Result, implement};
|
||||
use crate::Result;
|
||||
|
||||
/// The configuration manager is an indirection to reload the configuration for
|
||||
/// the server while it is running. In order to not burden or clutter the many
|
||||
@@ -39,6 +39,48 @@ impl Manager {
|
||||
active: AtomicPtr::new(Arc::into_raw(config).cast_mut()),
|
||||
}
|
||||
}
|
||||
|
||||
/// Update the active configuration, returning prior configuration.
|
||||
#[tracing::instrument(skip_all, level = "info")]
|
||||
pub fn update(&self, config: Config) -> Result<Arc<Config>> {
|
||||
let config = Arc::new(config);
|
||||
let new = Arc::into_raw(config);
|
||||
let old = self.active.swap(new.cast_mut(), Ordering::AcqRel);
|
||||
|
||||
// SAFETY: The old active pointer was set using an Arc::into_raw(). We're
|
||||
// obliged to reconstitute that into Arc otherwise it will leak.
|
||||
Ok(unsafe { Arc::from_raw(old) })
|
||||
}
|
||||
|
||||
fn load(&self, handle: &mut [Option<Arc<Config>>]) -> &'static Arc<Config> {
|
||||
let config = self.active.load(Ordering::Acquire);
|
||||
|
||||
// Branch taken after config reload or first access by this thread.
|
||||
if handle[INDEX.get()]
|
||||
.as_ref()
|
||||
.is_none_or(|handle| !ptr::eq(config, Arc::as_ptr(handle)))
|
||||
{
|
||||
INDEX.set(INDEX.get().wrapping_add(1).wrapping_rem(HISTORY));
|
||||
return load_miss(handle, INDEX.get(), config);
|
||||
}
|
||||
|
||||
let config: &Arc<Config> = handle[INDEX.get()]
|
||||
.as_ref()
|
||||
.expect("handle was already cached for this thread");
|
||||
|
||||
// SAFETY: The caller should not hold multiple references at a time directly
|
||||
// into Config, as a subsequent reference might invalidate the thread's cache
|
||||
// causing another reference to dangle.
|
||||
//
|
||||
// This is a highly unusual pattern as most config values are copied by value or
|
||||
// used immediately without running overlap with another value. Even if it does
|
||||
// actually occur somewhere, the window of danger is limited to the config being
|
||||
// reloaded while the reference is held and another access is made by the same
|
||||
// thread into a different config value. This is mitigated by creating a buffer
|
||||
// of old configs rather than discarding at the earliest opportunity; the odds
|
||||
// of this scenario are thus astronomical.
|
||||
unsafe { std::mem::transmute(config) }
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for Manager {
|
||||
@@ -57,50 +99,6 @@ impl Deref for Manager {
|
||||
fn deref(&self) -> &Self::Target { HANDLE.with_borrow_mut(|handle| self.load(handle)) }
|
||||
}
|
||||
|
||||
/// Update the active configuration, returning prior configuration.
|
||||
#[implement(Manager)]
|
||||
#[tracing::instrument(skip_all, level = "info")]
|
||||
pub fn update(&self, config: Config) -> Result<Arc<Config>> {
|
||||
let config = Arc::new(config);
|
||||
let new = Arc::into_raw(config);
|
||||
let old = self.active.swap(new.cast_mut(), Ordering::AcqRel);
|
||||
|
||||
// SAFETY: The old active pointer was set using an Arc::into_raw(). We're
|
||||
// obliged to reconstitute that into Arc otherwise it will leak.
|
||||
Ok(unsafe { Arc::from_raw(old) })
|
||||
}
|
||||
|
||||
#[implement(Manager)]
|
||||
fn load(&self, handle: &mut [Option<Arc<Config>>]) -> &'static Arc<Config> {
|
||||
let config = self.active.load(Ordering::Acquire);
|
||||
|
||||
// Branch taken after config reload or first access by this thread.
|
||||
if handle[INDEX.get()]
|
||||
.as_ref()
|
||||
.is_none_or(|handle| !ptr::eq(config, Arc::as_ptr(handle)))
|
||||
{
|
||||
INDEX.set(INDEX.get().wrapping_add(1).wrapping_rem(HISTORY));
|
||||
return load_miss(handle, INDEX.get(), config);
|
||||
}
|
||||
|
||||
let config: &Arc<Config> = handle[INDEX.get()]
|
||||
.as_ref()
|
||||
.expect("handle was already cached for this thread");
|
||||
|
||||
// SAFETY: The caller should not hold multiple references at a time directly
|
||||
// into Config, as a subsequent reference might invalidate the thread's cache
|
||||
// causing another reference to dangle.
|
||||
//
|
||||
// This is a highly unusual pattern as most config values are copied by value or
|
||||
// used immediately without running overlap with another value. Even if it does
|
||||
// actually occur somewhere, the window of danger is limited to the config being
|
||||
// reloaded while the reference is held and another access is made by the same
|
||||
// thread into a different config value. This is mitigated by creating a buffer
|
||||
// of old configs rather than discarding at the earliest opportunity; the odds
|
||||
// of this scenario are thus astronomical.
|
||||
unsafe { std::mem::transmute(config) }
|
||||
}
|
||||
|
||||
#[tracing::instrument(
|
||||
name = "miss",
|
||||
level = "trace",
|
||||
|
||||
+14
-2
@@ -375,7 +375,7 @@ pub struct Config {
|
||||
#[serde(default = "default_max_request_size")]
|
||||
pub max_request_size: usize,
|
||||
|
||||
/// default: 192
|
||||
/// default: 1024
|
||||
#[serde(default = "default_max_fetch_prev_events")]
|
||||
pub max_fetch_prev_events: u16,
|
||||
|
||||
@@ -781,6 +781,16 @@ pub struct Config {
|
||||
/// a substitute for moderation bots.
|
||||
pub default_room_acl_deny: Option<Vec<String>>,
|
||||
|
||||
/// The number of forward extremities to tolerate in a room before
|
||||
/// attempting to manually squash them with a "dummy event". Setting this
|
||||
/// above 20 will hinder its efficacy, and setting it below 5 will cause
|
||||
/// more dummy events to be sent than necessary (which increases federation
|
||||
/// traffic).
|
||||
///
|
||||
/// default: 10
|
||||
#[serde(default = "default_extremity_threshold")]
|
||||
pub dummy_event_threshold: u8,
|
||||
|
||||
/// display: nested
|
||||
#[serde(default)]
|
||||
pub well_known: WellKnownConfig,
|
||||
@@ -2549,7 +2559,7 @@ fn default_pusher_timeout() -> u64 { 60 }
|
||||
|
||||
fn default_pusher_idle_timeout() -> u64 { 15 }
|
||||
|
||||
fn default_max_fetch_prev_events() -> u16 { 192_u16 }
|
||||
fn default_max_fetch_prev_events() -> u16 { 1024 }
|
||||
|
||||
fn default_max_concurrent_inbound_transactions() -> usize { 150 }
|
||||
|
||||
@@ -2652,6 +2662,8 @@ fn default_rocksdb_stats_level() -> u8 { 1 }
|
||||
#[inline]
|
||||
pub fn default_default_room_version() -> RoomVersionId { RoomVersionId::V12 }
|
||||
|
||||
fn default_extremity_threshold() -> u8 { 10 }
|
||||
|
||||
fn default_ip_range_denylist() -> Vec<String> {
|
||||
vec![
|
||||
"127.0.0.0/8".to_owned(),
|
||||
|
||||
@@ -62,7 +62,7 @@ impl Default for PartialPdu {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
event_type: "m.room.message".into(),
|
||||
content: Box::<RawJsonValue>::default(),
|
||||
content: to_raw_value("{}").unwrap(),
|
||||
unsigned: None,
|
||||
state_key: None,
|
||||
redacts: None,
|
||||
|
||||
@@ -1,32 +1,34 @@
|
||||
use ruma::{RoomVersionId, canonical_json::redact_content_in_place};
|
||||
use serde_json::{Value as JsonValue, json, value::to_raw_value};
|
||||
|
||||
use crate::{Err, Result, err, implement};
|
||||
use crate::{Err, Result, err};
|
||||
|
||||
#[implement(super::Pdu)]
|
||||
pub fn redact(&mut self, room_version_id: &RoomVersionId, reason: JsonValue) -> Result {
|
||||
let Some(rules) = room_version_id.rules() else {
|
||||
return Err!("Cannot redact event for unknown room version {room_version_id}");
|
||||
};
|
||||
impl super::Pdu {
|
||||
pub fn redact(&mut self, room_version_id: &RoomVersionId, reason: JsonValue) -> Result {
|
||||
let Some(rules) = room_version_id.rules() else {
|
||||
return Err!("Cannot redact event for unknown room version {room_version_id}");
|
||||
};
|
||||
|
||||
self.unsigned = None;
|
||||
self.unsigned = None;
|
||||
|
||||
let mut content = serde_json::from_str(self.content.get())
|
||||
.map_err(|e| err!(Request(BadJson("Failed to deserialize content into type: {e}"))))?;
|
||||
let mut content = serde_json::from_str(self.content.get()).map_err(|e| {
|
||||
err!(Request(BadJson("Failed to deserialize content into type: {e}")))
|
||||
})?;
|
||||
|
||||
redact_content_in_place(&mut content, &rules.redaction, self.kind.to_string());
|
||||
redact_content_in_place(&mut content, &rules.redaction, self.kind.to_string());
|
||||
|
||||
let reason = serde_json::to_value(reason).expect("Failed to preserialize reason");
|
||||
let reason = serde_json::to_value(reason).expect("Failed to preserialize reason");
|
||||
|
||||
let redacted_because = json!({
|
||||
"redacted_because": reason,
|
||||
});
|
||||
let redacted_because = json!({
|
||||
"redacted_because": reason,
|
||||
});
|
||||
|
||||
self.unsigned = to_raw_value(&redacted_because)
|
||||
.expect("Failed to serialize unsigned")
|
||||
.into();
|
||||
self.unsigned = to_raw_value(&redacted_because)
|
||||
.expect("Failed to serialize unsigned")
|
||||
.into();
|
||||
|
||||
self.content = to_raw_value(&content).expect("Failed to serialize content");
|
||||
self.content = to_raw_value(&content).expect("Failed to serialize content");
|
||||
|
||||
Ok(())
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
@@ -4,85 +4,83 @@ use ruma::MilliSecondsSinceUnixEpoch;
|
||||
use serde_json::value::{RawValue as RawJsonValue, Value as JsonValue, to_raw_value};
|
||||
|
||||
use super::Pdu;
|
||||
use crate::{Result, err, implement, result::LogErr};
|
||||
use crate::{Result, err, result::LogErr};
|
||||
|
||||
/// Set the `unsigned` field of the PDU using only information in the PDU.
|
||||
/// Some unsigned data is already set within the database (eg. prev events,
|
||||
/// threads). Once this is done, other data must be calculated from the database
|
||||
/// (eg. relations) This is for server-to-client events.
|
||||
/// Backfill handles this itself.
|
||||
#[implement(Pdu)]
|
||||
pub fn set_unsigned(&mut self, user_id: Option<&ruma::UserId>) {
|
||||
if Some(self.sender.borrow()) != user_id {
|
||||
self.remove_transaction_id().log_err().ok();
|
||||
impl Pdu {
|
||||
/// Set the `unsigned` field of the PDU using only information in the PDU.
|
||||
/// Some unsigned data is already set within the database (eg. prev events,
|
||||
/// threads). Once this is done, other data must be calculated from the
|
||||
/// database (eg. relations) This is for server-to-client events.
|
||||
/// Backfill handles this itself.
|
||||
pub fn set_unsigned(&mut self, user_id: Option<&ruma::UserId>) {
|
||||
if Some(self.sender.borrow()) != user_id {
|
||||
self.remove_transaction_id().log_err().ok();
|
||||
}
|
||||
self.add_age().log_err().ok();
|
||||
}
|
||||
|
||||
pub fn remove_transaction_id(&mut self) -> Result {
|
||||
use BTreeMap as Map;
|
||||
|
||||
let Some(unsigned) = &self.unsigned else {
|
||||
return Ok(());
|
||||
};
|
||||
|
||||
let mut unsigned: Map<&str, Box<RawJsonValue>> = serde_json::from_str(unsigned.get())
|
||||
.map_err(|e| err!(Database("Invalid unsigned in pdu event: {e}")))?;
|
||||
|
||||
unsigned.remove("transaction_id");
|
||||
self.unsigned = to_raw_value(&unsigned)
|
||||
.map(Some)
|
||||
.expect("unsigned is valid");
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn add_age(&mut self) -> Result {
|
||||
use BTreeMap as Map;
|
||||
|
||||
let mut unsigned: Map<&str, Box<RawJsonValue>> = self
|
||||
.unsigned
|
||||
.as_deref()
|
||||
.map(RawJsonValue::get)
|
||||
.map_or_else(|| Ok(Map::new()), serde_json::from_str)
|
||||
.map_err(|e| err!(Database("Invalid unsigned in pdu event: {e}")))?;
|
||||
|
||||
// deliberately allowing for the possibility of negative age
|
||||
let now: i128 = MilliSecondsSinceUnixEpoch::now().get().into();
|
||||
let then: i128 = self.origin_server_ts.into();
|
||||
let this_age = now.saturating_sub(then);
|
||||
|
||||
unsigned.insert("age", to_raw_value(&this_age)?);
|
||||
self.unsigned = Some(to_raw_value(&unsigned)?);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn add_relation(&mut self, name: &str, pdu: Option<&Self>) -> Result {
|
||||
use serde_json::Map;
|
||||
|
||||
let mut unsigned: Map<String, JsonValue> = self
|
||||
.unsigned
|
||||
.as_deref()
|
||||
.map(RawJsonValue::get)
|
||||
.map_or_else(|| Ok(Map::new()), serde_json::from_str)
|
||||
.map_err(|e| err!(Database("Invalid unsigned in pdu event: {e}")))?;
|
||||
|
||||
let pdu = pdu
|
||||
.map(serde_json::to_value)
|
||||
.transpose()?
|
||||
.unwrap_or_else(|| JsonValue::Object(Map::new()));
|
||||
|
||||
unsigned
|
||||
.entry("m.relations")
|
||||
.or_insert(JsonValue::Object(Map::new()))
|
||||
.as_object_mut()
|
||||
.map(|object| object.insert(name.to_owned(), pdu));
|
||||
|
||||
self.unsigned = Some(to_raw_value(&unsigned)?);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
self.add_age().log_err().ok();
|
||||
}
|
||||
|
||||
#[implement(Pdu)]
|
||||
pub fn remove_transaction_id(&mut self) -> Result {
|
||||
use BTreeMap as Map;
|
||||
|
||||
let Some(unsigned) = &self.unsigned else {
|
||||
return Ok(());
|
||||
};
|
||||
|
||||
let mut unsigned: Map<&str, Box<RawJsonValue>> = serde_json::from_str(unsigned.get())
|
||||
.map_err(|e| err!(Database("Invalid unsigned in pdu event: {e}")))?;
|
||||
|
||||
unsigned.remove("transaction_id");
|
||||
self.unsigned = to_raw_value(&unsigned)
|
||||
.map(Some)
|
||||
.expect("unsigned is valid");
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[implement(Pdu)]
|
||||
pub fn add_age(&mut self) -> Result {
|
||||
use BTreeMap as Map;
|
||||
|
||||
let mut unsigned: Map<&str, Box<RawJsonValue>> = self
|
||||
.unsigned
|
||||
.as_deref()
|
||||
.map(RawJsonValue::get)
|
||||
.map_or_else(|| Ok(Map::new()), serde_json::from_str)
|
||||
.map_err(|e| err!(Database("Invalid unsigned in pdu event: {e}")))?;
|
||||
|
||||
// deliberately allowing for the possibility of negative age
|
||||
let now: i128 = MilliSecondsSinceUnixEpoch::now().get().into();
|
||||
let then: i128 = self.origin_server_ts.into();
|
||||
let this_age = now.saturating_sub(then);
|
||||
|
||||
unsigned.insert("age", to_raw_value(&this_age)?);
|
||||
self.unsigned = Some(to_raw_value(&unsigned)?);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[implement(Pdu)]
|
||||
pub fn add_relation(&mut self, name: &str, pdu: Option<&Pdu>) -> Result {
|
||||
use serde_json::Map;
|
||||
|
||||
let mut unsigned: Map<String, JsonValue> = self
|
||||
.unsigned
|
||||
.as_deref()
|
||||
.map(RawJsonValue::get)
|
||||
.map_or_else(|| Ok(Map::new()), serde_json::from_str)
|
||||
.map_err(|e| err!(Database("Invalid unsigned in pdu event: {e}")))?;
|
||||
|
||||
let pdu = pdu
|
||||
.map(serde_json::to_value)
|
||||
.transpose()?
|
||||
.unwrap_or_else(|| JsonValue::Object(Map::new()));
|
||||
|
||||
unsigned
|
||||
.entry("m.relations")
|
||||
.or_insert(JsonValue::Object(Map::new()))
|
||||
.as_object_mut()
|
||||
.map(|object| object.insert(name.to_owned(), pdu));
|
||||
|
||||
self.unsigned = Some(to_raw_value(&unsigned)?);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -44,5 +44,6 @@ pub fn unstable_features() -> BTreeMap<String, bool> {
|
||||
("uk.timedout.msc4323".to_owned(), true), /* agnostic suspend (https://github.com/matrix-org/matrix-spec-proposals/pull/4323) */
|
||||
("org.matrix.msc4155".to_owned(), true), /* invite filtering (https://github.com/matrix-org/matrix-spec-proposals/pull/4155) */
|
||||
("computer.gingershaped.msc4466".to_owned(), true), /* profile change propagation (https://github.com/matrix-org/matrix-spec-proposals/pull/4466) */
|
||||
("org.matrix.msc4380.stable".to_owned(), true),
|
||||
])
|
||||
}
|
||||
+1
-1
@@ -41,7 +41,7 @@ pub use info::{
|
||||
pub use matrix::{Event, EventTypeExt, Pdu, PduCount, PduEvent, PduId, pdu, state_res};
|
||||
pub use parking_lot::{Mutex as SyncMutex, RwLock as SyncRwLock};
|
||||
pub use server::Server;
|
||||
pub use utils::{implement, result, result::Result};
|
||||
pub use utils::{result, result::Result};
|
||||
|
||||
pub use crate as conduwuit_core;
|
||||
|
||||
|
||||
@@ -22,8 +22,6 @@ mod tests;
|
||||
pub mod time;
|
||||
pub mod with_lock;
|
||||
|
||||
pub use ::conduwuit_macros::implement;
|
||||
|
||||
pub use self::{
|
||||
arrayvec::ArrayVecExt,
|
||||
bool::BoolExt,
|
||||
|
||||
@@ -27,3 +27,12 @@ pub fn continue_exponential_backoff(
|
||||
let min = cmp::min(min, max);
|
||||
elapsed < min
|
||||
}
|
||||
|
||||
/// Determines the minimum number of backoff seconds
|
||||
#[must_use]
|
||||
pub fn min_exp_backoff_duration(min: u64, max: u64, retries: u32) -> Duration {
|
||||
let min = Duration::from_secs(min)
|
||||
.saturating_mul(retries)
|
||||
.saturating_mul(retries);
|
||||
Duration::from_secs(max).min(min)
|
||||
}
|
||||
@@ -1,92 +1,88 @@
|
||||
use std::{ffi::OsString, path::PathBuf};
|
||||
|
||||
use conduwuit::{Err, Result, error, implement, info, utils::time::rfc2822_from_seconds, warn};
|
||||
use conduwuit::{Err, Result, error, info, utils::time::rfc2822_from_seconds, warn};
|
||||
use rocksdb::backup::{BackupEngine, BackupEngineOptions};
|
||||
|
||||
use super::Engine;
|
||||
use crate::util::map_err;
|
||||
|
||||
#[implement(Engine)]
|
||||
#[tracing::instrument(skip(self), level = "info")]
|
||||
pub fn backup(&self) -> Result {
|
||||
let mut engine = self.backup_engine()?;
|
||||
let config = &self.ctx.server.config;
|
||||
if config.database_backups_to_keep > 0 {
|
||||
engine
|
||||
.create_new_backup_flush(&self.db, true)
|
||||
.map_err(map_err)?;
|
||||
impl super::Engine {
|
||||
#[tracing::instrument(skip(self), level = "info")]
|
||||
pub fn backup(&self) -> Result {
|
||||
let mut engine = self.backup_engine()?;
|
||||
let config = &self.ctx.server.config;
|
||||
if config.database_backups_to_keep > 0 {
|
||||
engine
|
||||
.create_new_backup_flush(&self.db, true)
|
||||
.map_err(map_err)?;
|
||||
|
||||
let engine_info = engine.get_backup_info();
|
||||
let info = &engine_info.last().expect("backup engine info is not empty");
|
||||
info!(
|
||||
"Created database backup #{} using {} bytes in {} files",
|
||||
info.backup_id, info.size, info.num_files,
|
||||
);
|
||||
}
|
||||
|
||||
if config.database_backups_to_keep >= 0 {
|
||||
let keep = u32::try_from(config.database_backups_to_keep)?;
|
||||
if let Err(e) = engine.purge_old_backups(keep.try_into()?) {
|
||||
error!("Failed to purge old backup: {e:?}");
|
||||
let engine_info = engine.get_backup_info();
|
||||
let info = &engine_info.last().expect("backup engine info is not empty");
|
||||
info!(
|
||||
"Created database backup #{} using {} bytes in {} files",
|
||||
info.backup_id, info.size, info.num_files,
|
||||
);
|
||||
}
|
||||
|
||||
if config.database_backups_to_keep >= 0 {
|
||||
let keep = u32::try_from(config.database_backups_to_keep)?;
|
||||
if let Err(e) = engine.purge_old_backups(keep.try_into()?) {
|
||||
error!("Failed to purge old backup: {e:?}");
|
||||
}
|
||||
}
|
||||
|
||||
if config.database_backups_to_keep == 0 {
|
||||
warn!("Configuration item `database_backups_to_keep` is set to 0.");
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
if config.database_backups_to_keep == 0 {
|
||||
warn!("Configuration item `database_backups_to_keep` is set to 0.");
|
||||
pub fn backup_list(&self) -> Result<impl Iterator<Item = String> + Send> {
|
||||
let info = self.backup_engine()?.get_backup_info();
|
||||
|
||||
if info.is_empty() {
|
||||
return Err!("No backups found.");
|
||||
}
|
||||
|
||||
let list = info.into_iter().map(|info| {
|
||||
format!(
|
||||
"#{} {}: {} bytes, {} files",
|
||||
info.backup_id,
|
||||
rfc2822_from_seconds(info.timestamp),
|
||||
info.size,
|
||||
info.num_files,
|
||||
)
|
||||
});
|
||||
|
||||
Ok(list)
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
pub fn backup_count(&self) -> Result<usize> {
|
||||
let info = self.backup_engine()?.get_backup_info();
|
||||
|
||||
#[implement(Engine)]
|
||||
pub fn backup_list(&self) -> Result<impl Iterator<Item = String> + Send> {
|
||||
let info = self.backup_engine()?.get_backup_info();
|
||||
|
||||
if info.is_empty() {
|
||||
return Err!("No backups found.");
|
||||
Ok(info.len())
|
||||
}
|
||||
|
||||
let list = info.into_iter().map(|info| {
|
||||
format!(
|
||||
"#{} {}: {} bytes, {} files",
|
||||
info.backup_id,
|
||||
rfc2822_from_seconds(info.timestamp),
|
||||
info.size,
|
||||
info.num_files,
|
||||
)
|
||||
});
|
||||
|
||||
Ok(list)
|
||||
}
|
||||
|
||||
#[implement(Engine)]
|
||||
pub fn backup_count(&self) -> Result<usize> {
|
||||
let info = self.backup_engine()?.get_backup_info();
|
||||
|
||||
Ok(info.len())
|
||||
}
|
||||
|
||||
#[implement(Engine)]
|
||||
fn backup_engine(&self) -> Result<BackupEngine> {
|
||||
let path = self.backup_path()?;
|
||||
let options = BackupEngineOptions::new(path).map_err(map_err)?;
|
||||
BackupEngine::open(&options, &self.ctx.env.lock()).map_err(map_err)
|
||||
}
|
||||
|
||||
#[implement(Engine)]
|
||||
fn backup_path(&self) -> Result<OsString> {
|
||||
let path = self
|
||||
.ctx
|
||||
.server
|
||||
.config
|
||||
.database_backup_path
|
||||
.clone()
|
||||
.map(PathBuf::into_os_string)
|
||||
.unwrap_or_default();
|
||||
|
||||
if path.is_empty() {
|
||||
return Err!(Config("database_backup_path", "Configure path to enable backups"));
|
||||
fn backup_engine(&self) -> Result<BackupEngine> {
|
||||
let path = self.backup_path()?;
|
||||
let options = BackupEngineOptions::new(path).map_err(map_err)?;
|
||||
BackupEngine::open(&options, &self.ctx.env.lock()).map_err(map_err)
|
||||
}
|
||||
|
||||
Ok(path)
|
||||
fn backup_path(&self) -> Result<OsString> {
|
||||
let path = self
|
||||
.ctx
|
||||
.server
|
||||
.config
|
||||
.database_backup_path
|
||||
.clone()
|
||||
.map(PathBuf::into_os_string)
|
||||
.unwrap_or_default();
|
||||
|
||||
if path.is_empty() {
|
||||
return Err!(Config("database_backup_path", "Configure path to enable backups"));
|
||||
}
|
||||
|
||||
Ok(path)
|
||||
}
|
||||
}
|
||||
@@ -1,15 +1,15 @@
|
||||
use conduwuit::{Result, implement};
|
||||
use conduwuit::Result;
|
||||
use rocksdb::LiveFile as SstFile;
|
||||
|
||||
use super::Engine;
|
||||
use crate::util::map_err;
|
||||
|
||||
#[implement(Engine)]
|
||||
pub fn file_list(&self) -> impl Iterator<Item = Result<SstFile>> + Send + use<> {
|
||||
self.db
|
||||
.live_files()
|
||||
.map_err(map_err)
|
||||
.into_iter()
|
||||
.flat_map(Vec::into_iter)
|
||||
.map(Ok)
|
||||
impl super::Engine {
|
||||
pub fn file_list(&self) -> impl Iterator<Item = Result<SstFile>> + Send + use<> {
|
||||
self.db
|
||||
.live_files()
|
||||
.map_err(map_err)
|
||||
.into_iter()
|
||||
.flat_map(Vec::into_iter)
|
||||
.map(Ok)
|
||||
}
|
||||
}
|
||||
@@ -1,30 +1,31 @@
|
||||
use std::fmt::Write;
|
||||
|
||||
use conduwuit::{Result, implement};
|
||||
use conduwuit::Result;
|
||||
use rocksdb::perf::get_memory_usage_stats;
|
||||
|
||||
use super::Engine;
|
||||
use crate::or_else;
|
||||
|
||||
#[implement(Engine)]
|
||||
pub fn memory_usage(&self) -> Result<String> {
|
||||
let mut res = String::new();
|
||||
let stats = get_memory_usage_stats(Some(&[&self.db]), Some(&[&*self.ctx.row_cache.lock()]))
|
||||
.or_else(or_else)?;
|
||||
let mibs = |input| f64::from(u32::try_from(input / 1024).unwrap_or(0)) / 1024.0;
|
||||
writeln!(
|
||||
res,
|
||||
"Memory buffers: {:.2} MiB\nPending write: {:.2} MiB\nTable readers: {:.2} MiB\nRow \
|
||||
cache: {:.2} MiB",
|
||||
mibs(stats.mem_table_total),
|
||||
mibs(stats.mem_table_unflushed),
|
||||
mibs(stats.mem_table_readers_total),
|
||||
mibs(u64::try_from(self.ctx.row_cache.lock().get_usage())?),
|
||||
)?;
|
||||
impl super::Engine {
|
||||
pub fn memory_usage(&self) -> Result<String> {
|
||||
let mut res = String::new();
|
||||
let stats =
|
||||
get_memory_usage_stats(Some(&[&self.db]), Some(&[&*self.ctx.row_cache.lock()]))
|
||||
.or_else(or_else)?;
|
||||
let mibs = |input| f64::from(u32::try_from(input / 1024).unwrap_or(0)) / 1024.0;
|
||||
writeln!(
|
||||
res,
|
||||
"Memory buffers: {:.2} MiB\nPending write: {:.2} MiB\nTable readers: {:.2} MiB\nRow \
|
||||
cache: {:.2} MiB",
|
||||
mibs(stats.mem_table_total),
|
||||
mibs(stats.mem_table_unflushed),
|
||||
mibs(stats.mem_table_readers_total),
|
||||
mibs(u64::try_from(self.ctx.row_cache.lock().get_usage())?),
|
||||
)?;
|
||||
|
||||
for (name, cache) in &*self.ctx.col_cache.lock() {
|
||||
writeln!(res, "{name} cache: {:.2} MiB", mibs(u64::try_from(cache.get_usage())?))?;
|
||||
for (name, cache) in &*self.ctx.col_cache.lock() {
|
||||
writeln!(res, "{name} cache: {:.2} MiB", mibs(u64::try_from(cache.get_usage())?))?;
|
||||
}
|
||||
|
||||
Ok(res)
|
||||
}
|
||||
|
||||
Ok(res)
|
||||
}
|
||||
+94
-95
@@ -4,11 +4,11 @@ use std::{
|
||||
sync::{Arc, atomic::AtomicU32},
|
||||
};
|
||||
|
||||
use conduwuit::{Result, debug, implement, info, warn};
|
||||
use conduwuit::{Result, debug, info, warn};
|
||||
use rocksdb::{ColumnFamilyDescriptor, Options};
|
||||
|
||||
use super::{
|
||||
Db, Engine,
|
||||
Db,
|
||||
cf_opts::cf_options,
|
||||
db_opts::db_options,
|
||||
descriptor::{self, Descriptor},
|
||||
@@ -16,104 +16,103 @@ use super::{
|
||||
};
|
||||
use crate::{Context, or_else};
|
||||
|
||||
#[implement(Engine)]
|
||||
#[tracing::instrument(skip_all, level = "info")]
|
||||
pub(crate) async fn open(ctx: Arc<Context>, desc: &[Descriptor]) -> Result<Arc<Self>> {
|
||||
let server = &ctx.server;
|
||||
let config = &server.config;
|
||||
let path = &config.database_path;
|
||||
impl super::Engine {
|
||||
#[tracing::instrument(skip_all, level = "info")]
|
||||
pub(crate) async fn open(ctx: Arc<Context>, desc: &[Descriptor]) -> Result<Arc<Self>> {
|
||||
let server = &ctx.server;
|
||||
let config = &server.config;
|
||||
let path = &config.database_path;
|
||||
|
||||
let db_opts = db_options(config, &ctx.env.lock(), &ctx.row_cache.lock())?;
|
||||
let db_opts = db_options(config, &ctx.env.lock(), &ctx.row_cache.lock())?;
|
||||
|
||||
let cfds = Self::configure_cfds(&ctx, &db_opts, desc)?;
|
||||
let num_cfds = cfds.len();
|
||||
debug!("Configured {num_cfds} column descriptors...");
|
||||
let cfds = Self::configure_cfds(&ctx, &db_opts, desc)?;
|
||||
let num_cfds = cfds.len();
|
||||
debug!("Configured {num_cfds} column descriptors...");
|
||||
|
||||
let load_time = std::time::Instant::now();
|
||||
if config.rocksdb_repair {
|
||||
repair(&db_opts, &config.database_path)?;
|
||||
let load_time = std::time::Instant::now();
|
||||
if config.rocksdb_repair {
|
||||
repair(&db_opts, &config.database_path)?;
|
||||
}
|
||||
|
||||
debug!("Opening database...");
|
||||
let db = Db::open_cf_descriptors(&db_opts, path, cfds).or_else(or_else)?;
|
||||
|
||||
info!(
|
||||
columns = num_cfds,
|
||||
sequence = %db.latest_sequence_number(),
|
||||
time = ?load_time.elapsed(),
|
||||
"Opened database."
|
||||
);
|
||||
|
||||
Ok(Arc::new(Self {
|
||||
db,
|
||||
pool: ctx.pool.clone(),
|
||||
ctx: ctx.clone(),
|
||||
checksums: config.rocksdb_checksums,
|
||||
corks: AtomicU32::new(0),
|
||||
}))
|
||||
}
|
||||
|
||||
debug!("Opening database...");
|
||||
let db = Db::open_cf_descriptors(&db_opts, path, cfds).or_else(or_else)?;
|
||||
#[tracing::instrument(name = "configure", skip_all, level = "debug")]
|
||||
fn configure_cfds(
|
||||
ctx: &Arc<Context>,
|
||||
db_opts: &Options,
|
||||
desc: &[Descriptor],
|
||||
) -> Result<Vec<ColumnFamilyDescriptor>> {
|
||||
let server = &ctx.server;
|
||||
let config = &server.config;
|
||||
let path = &config.database_path;
|
||||
let existing = Self::discover_cfs(path, db_opts);
|
||||
|
||||
info!(
|
||||
columns = num_cfds,
|
||||
sequence = %db.latest_sequence_number(),
|
||||
time = ?load_time.elapsed(),
|
||||
"Opened database."
|
||||
);
|
||||
let creating = desc.iter().filter(|desc| !existing.contains(desc.name));
|
||||
|
||||
Ok(Arc::new(Self {
|
||||
db,
|
||||
pool: ctx.pool.clone(),
|
||||
ctx: ctx.clone(),
|
||||
checksums: config.rocksdb_checksums,
|
||||
corks: AtomicU32::new(0),
|
||||
}))
|
||||
}
|
||||
|
||||
#[implement(Engine)]
|
||||
#[tracing::instrument(name = "configure", skip_all, level = "debug")]
|
||||
fn configure_cfds(
|
||||
ctx: &Arc<Context>,
|
||||
db_opts: &Options,
|
||||
desc: &[Descriptor],
|
||||
) -> Result<Vec<ColumnFamilyDescriptor>> {
|
||||
let server = &ctx.server;
|
||||
let config = &server.config;
|
||||
let path = &config.database_path;
|
||||
let existing = Self::discover_cfs(path, db_opts);
|
||||
|
||||
let creating = desc.iter().filter(|desc| !existing.contains(desc.name));
|
||||
|
||||
let missing = existing
|
||||
.iter()
|
||||
.filter(|&name| name != "default")
|
||||
.filter(|&name| !desc.iter().any(|desc| desc.name == name));
|
||||
|
||||
debug!(
|
||||
existing = existing.len(),
|
||||
described = desc.len(),
|
||||
missing = missing.clone().count(),
|
||||
creating = creating.clone().count(),
|
||||
"Discovered database columns"
|
||||
);
|
||||
|
||||
missing.clone().for_each(|name| {
|
||||
debug!("Found unrecognized column {name:?} in existing database.");
|
||||
});
|
||||
|
||||
creating.map(|desc| desc.name).for_each(|name| {
|
||||
debug!("Creating new column {name:?} not previously found in existing database.");
|
||||
});
|
||||
|
||||
let missing_descriptors = missing.clone().map(|_| descriptor::DROPPED);
|
||||
|
||||
let cfopts: Vec<_> = desc
|
||||
.iter()
|
||||
.copied()
|
||||
.chain(missing_descriptors)
|
||||
.map(|ref desc| cf_options(ctx, db_opts.clone(), desc))
|
||||
.collect::<Result<_>>()?;
|
||||
|
||||
let cfds: Vec<_> = desc
|
||||
.iter()
|
||||
.map(|desc| desc.name)
|
||||
.map(ToOwned::to_owned)
|
||||
.chain(missing.cloned())
|
||||
.zip(cfopts.into_iter())
|
||||
.map(|(name, opts)| ColumnFamilyDescriptor::new(name, opts))
|
||||
.collect();
|
||||
|
||||
Ok(cfds)
|
||||
}
|
||||
|
||||
#[implement(Engine)]
|
||||
#[tracing::instrument(name = "discover", skip_all, level = "debug")]
|
||||
fn discover_cfs(path: &Path, opts: &Options) -> BTreeSet<String> {
|
||||
Db::list_cf(opts, path)
|
||||
.unwrap_or_default()
|
||||
.into_iter()
|
||||
.collect::<BTreeSet<_>>()
|
||||
let missing = existing
|
||||
.iter()
|
||||
.filter(|&name| name != "default")
|
||||
.filter(|&name| !desc.iter().any(|desc| desc.name == name));
|
||||
|
||||
debug!(
|
||||
existing = existing.len(),
|
||||
described = desc.len(),
|
||||
missing = missing.clone().count(),
|
||||
creating = creating.clone().count(),
|
||||
"Discovered database columns"
|
||||
);
|
||||
|
||||
missing.clone().for_each(|name| {
|
||||
debug!("Found unrecognized column {name:?} in existing database.");
|
||||
});
|
||||
|
||||
creating.map(|desc| desc.name).for_each(|name| {
|
||||
debug!("Creating new column {name:?} not previously found in existing database.");
|
||||
});
|
||||
|
||||
let missing_descriptors = missing.clone().map(|_| descriptor::DROPPED);
|
||||
|
||||
let cfopts: Vec<_> = desc
|
||||
.iter()
|
||||
.copied()
|
||||
.chain(missing_descriptors)
|
||||
.map(|ref desc| cf_options(ctx, db_opts.clone(), desc))
|
||||
.collect::<Result<_>>()?;
|
||||
|
||||
let cfds: Vec<_> = desc
|
||||
.iter()
|
||||
.map(|desc| desc.name)
|
||||
.map(ToOwned::to_owned)
|
||||
.chain(missing.cloned())
|
||||
.zip(cfopts.into_iter())
|
||||
.map(|(name, opts)| ColumnFamilyDescriptor::new(name, opts))
|
||||
.collect();
|
||||
|
||||
Ok(cfds)
|
||||
}
|
||||
|
||||
#[tracing::instrument(name = "discover", skip_all, level = "debug")]
|
||||
fn discover_cfs(path: &Path, opts: &Options) -> BTreeSet<String> {
|
||||
Db::list_cf(opts, path)
|
||||
.unwrap_or_default()
|
||||
.into_iter()
|
||||
.collect::<BTreeSet<_>>()
|
||||
}
|
||||
}
|
||||
+20
-19
@@ -1,30 +1,31 @@
|
||||
use std::sync::Arc;
|
||||
|
||||
use conduwuit::{
|
||||
Result, implement,
|
||||
Result,
|
||||
utils::stream::{ReadyExt, TryIgnore},
|
||||
};
|
||||
use futures::{Stream, TryStreamExt};
|
||||
|
||||
use crate::keyval::Key;
|
||||
|
||||
/// Delete all data stored in this map. !!! USE WITH CAUTION !!!
|
||||
///
|
||||
/// See for_clear() with additional details.
|
||||
#[implement(super::Map)]
|
||||
#[tracing::instrument(level = "trace")]
|
||||
pub async fn clear(self: &Arc<Self>) {
|
||||
self.for_clear().ignore_err().ready_for_each(|_| ()).await;
|
||||
}
|
||||
impl super::Map {
|
||||
/// Delete all data stored in this map. !!! USE WITH CAUTION !!!
|
||||
///
|
||||
/// See for_clear() with additional details.
|
||||
#[tracing::instrument(level = "trace")]
|
||||
pub async fn clear(self: &Arc<Self>) {
|
||||
// drops your entire database cutely :3
|
||||
self.for_clear().ignore_err().ready_for_each(|_| ()).await;
|
||||
}
|
||||
|
||||
/// Delete all data stored in this map. !!! USE WITH CAUTION !!!
|
||||
///
|
||||
/// Provides stream of keys undergoing deletion along with any errors.
|
||||
///
|
||||
/// Note this operation applies to a snapshot of the data when invoked.
|
||||
/// Additional data written during or after this call may be missed.
|
||||
#[implement(super::Map)]
|
||||
#[tracing::instrument(level = "trace")]
|
||||
pub fn for_clear(self: &Arc<Self>) -> impl Stream<Item = Result<Key<'_>>> + Send {
|
||||
self.raw_keys().inspect_ok(|key| self.remove(key))
|
||||
/// Delete all data stored in this map. !!! USE WITH CAUTION !!!
|
||||
///
|
||||
/// Provides stream of keys undergoing deletion along with any errors.
|
||||
///
|
||||
/// Note this operation applies to a snapshot of the data when invoked.
|
||||
/// Additional data written during or after this call may be missed.
|
||||
#[tracing::instrument(level = "trace")]
|
||||
pub fn for_clear(self: &Arc<Self>) -> impl Stream<Item = Result<Key<'_>>> + Send {
|
||||
self.raw_keys().inspect_ok(|key| self.remove(key))
|
||||
}
|
||||
}
|
||||
+36
-35
@@ -1,4 +1,4 @@
|
||||
use conduwuit::{Err, Result, implement};
|
||||
use conduwuit::{Err, Result};
|
||||
use rocksdb::{BottommostLevelCompaction, CompactOptions};
|
||||
|
||||
use crate::keyval::KeyBuf;
|
||||
@@ -23,40 +23,41 @@ pub struct Options {
|
||||
pub exclusive: bool,
|
||||
}
|
||||
|
||||
#[implement(super::Map)]
|
||||
#[tracing::instrument(
|
||||
name = "compact",
|
||||
level = "info",
|
||||
skip(self),
|
||||
fields(%self),
|
||||
)]
|
||||
pub fn compact_blocking(&self, opts: Options) -> Result {
|
||||
let mut co = CompactOptions::default();
|
||||
co.set_exclusive_manual_compaction(opts.exclusive);
|
||||
co.set_bottommost_level_compaction(match opts.exhaustive {
|
||||
| true => BottommostLevelCompaction::Force,
|
||||
| false => BottommostLevelCompaction::ForceOptimized,
|
||||
});
|
||||
impl super::Map {
|
||||
#[tracing::instrument(
|
||||
name = "compact",
|
||||
skip(self),
|
||||
fields(%self),
|
||||
)]
|
||||
pub fn compact_blocking(&self, opts: Options) -> Result {
|
||||
let mut co = CompactOptions::default();
|
||||
co.set_exclusive_manual_compaction(opts.exclusive);
|
||||
co.set_bottommost_level_compaction(match opts.exhaustive {
|
||||
| true => BottommostLevelCompaction::Force,
|
||||
| false => BottommostLevelCompaction::ForceOptimized,
|
||||
});
|
||||
|
||||
match opts.level {
|
||||
| (None, None) => {
|
||||
co.set_change_level(true);
|
||||
co.set_target_level(-1);
|
||||
},
|
||||
| (None, Some(level)) => {
|
||||
co.set_change_level(true);
|
||||
co.set_target_level(level.try_into()?);
|
||||
},
|
||||
| (Some(level), None) => {
|
||||
co.set_change_level(false);
|
||||
co.set_target_level(level.try_into()?);
|
||||
},
|
||||
| (Some(_), Some(_)) => return Err!("compacting between specific levels not supported"),
|
||||
match opts.level {
|
||||
| (None, None) => {
|
||||
co.set_change_level(true);
|
||||
co.set_target_level(-1);
|
||||
},
|
||||
| (None, Some(level)) => {
|
||||
co.set_change_level(true);
|
||||
co.set_target_level(level.try_into()?);
|
||||
},
|
||||
| (Some(level), None) => {
|
||||
co.set_change_level(false);
|
||||
co.set_target_level(level.try_into()?);
|
||||
},
|
||||
| (Some(_), Some(_)) =>
|
||||
return Err!("compacting between specific levels not supported"),
|
||||
}
|
||||
|
||||
self.db
|
||||
.db
|
||||
.compact_range_cf_opt(&self.cf(), opts.range.0, opts.range.1, &co);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
self.db
|
||||
.db
|
||||
.compact_range_cf_opt(&self.cf(), opts.range.0, opts.range.1, &co);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -3,7 +3,7 @@ use std::{convert::AsRef, fmt::Debug, future::Future, io::Write, sync::Arc};
|
||||
use conduwuit::{
|
||||
Result,
|
||||
arrayvec::ArrayVec,
|
||||
err, implement,
|
||||
err,
|
||||
utils::{future::TryExtExt, result::FlatOk},
|
||||
};
|
||||
use futures::FutureExt;
|
||||
@@ -11,93 +11,90 @@ use serde::Serialize;
|
||||
|
||||
use crate::{keyval::KeyBuf, ser};
|
||||
|
||||
/// Returns true if the map contains the key.
|
||||
/// - key is serialized into allocated buffer
|
||||
/// - harder errors may not be reported
|
||||
#[inline]
|
||||
#[implement(super::Map)]
|
||||
pub fn contains<K>(
|
||||
self: &Arc<Self>,
|
||||
key: &K,
|
||||
) -> impl Future<Output = bool> + Send + '_ + use<'_, K>
|
||||
where
|
||||
K: Serialize + ?Sized + Debug,
|
||||
{
|
||||
let mut buf = KeyBuf::new();
|
||||
self.bcontains(key, &mut buf)
|
||||
}
|
||||
impl super::Map {
|
||||
/// Returns true if the map contains the key.
|
||||
/// - key is serialized into allocated buffer
|
||||
/// - harder errors may not be reported
|
||||
#[inline]
|
||||
pub fn contains<K>(
|
||||
self: &Arc<Self>,
|
||||
key: &K,
|
||||
) -> impl Future<Output = bool> + Send + '_ + use<'_, K>
|
||||
where
|
||||
K: Serialize + ?Sized + Debug,
|
||||
{
|
||||
let mut buf = KeyBuf::new();
|
||||
self.bcontains(key, &mut buf)
|
||||
}
|
||||
|
||||
/// Returns true if the map contains the key.
|
||||
/// - key is serialized into stack-buffer
|
||||
/// - harder errors will panic
|
||||
#[inline]
|
||||
#[implement(super::Map)]
|
||||
pub fn acontains<const MAX: usize, K>(
|
||||
self: &Arc<Self>,
|
||||
key: &K,
|
||||
) -> impl Future<Output = bool> + Send + '_ + use<'_, MAX, K>
|
||||
where
|
||||
K: Serialize + ?Sized + Debug,
|
||||
{
|
||||
let mut buf = ArrayVec::<u8, MAX>::new();
|
||||
self.bcontains(key, &mut buf)
|
||||
}
|
||||
/// Returns true if the map contains the key.
|
||||
/// - key is serialized into stack-buffer
|
||||
/// - harder errors will panic
|
||||
#[inline]
|
||||
pub fn acontains<const MAX: usize, K>(
|
||||
self: &Arc<Self>,
|
||||
key: &K,
|
||||
) -> impl Future<Output = bool> + Send + '_ + use<'_, MAX, K>
|
||||
where
|
||||
K: Serialize + ?Sized + Debug,
|
||||
{
|
||||
let mut buf = ArrayVec::<u8, MAX>::new();
|
||||
self.bcontains(key, &mut buf)
|
||||
}
|
||||
|
||||
/// Returns true if the map contains the key.
|
||||
/// - key is serialized into provided buffer
|
||||
/// - harder errors will panic
|
||||
#[implement(super::Map)]
|
||||
#[tracing::instrument(skip(self, buf), fields(%self), level = "trace")]
|
||||
pub fn bcontains<K, B>(
|
||||
self: &Arc<Self>,
|
||||
key: &K,
|
||||
buf: &mut B,
|
||||
) -> impl Future<Output = bool> + Send + '_ + use<'_, K, B>
|
||||
where
|
||||
K: Serialize + ?Sized + Debug,
|
||||
B: Write + AsRef<[u8]>,
|
||||
{
|
||||
let key = ser::serialize(buf, key).expect("failed to serialize query key");
|
||||
self.exists(key).is_ok()
|
||||
}
|
||||
/// Returns true if the map contains the key.
|
||||
/// - key is serialized into provided buffer
|
||||
/// - harder errors will panic
|
||||
#[tracing::instrument(skip(self, buf), fields(%self), level = "trace")]
|
||||
pub fn bcontains<K, B>(
|
||||
self: &Arc<Self>,
|
||||
key: &K,
|
||||
buf: &mut B,
|
||||
) -> impl Future<Output = bool> + Send + '_ + use<'_, K, B>
|
||||
where
|
||||
K: Serialize + ?Sized + Debug,
|
||||
B: Write + AsRef<[u8]>,
|
||||
{
|
||||
let key = ser::serialize(buf, key).expect("failed to serialize query key");
|
||||
self.exists(key).is_ok()
|
||||
}
|
||||
|
||||
/// Returns Ok if the map contains the key.
|
||||
/// - key is raw
|
||||
#[inline]
|
||||
#[implement(super::Map)]
|
||||
pub fn exists<'a, K>(
|
||||
self: &'a Arc<Self>,
|
||||
key: &K,
|
||||
) -> impl Future<Output = Result> + Send + 'a + use<'a, K>
|
||||
where
|
||||
K: AsRef<[u8]> + ?Sized + Debug + 'a,
|
||||
{
|
||||
self.get(key).map(|res| res.map(|_| ()))
|
||||
}
|
||||
/// Returns Ok if the map contains the key.
|
||||
/// - key is raw
|
||||
#[inline]
|
||||
pub fn exists<'a, K>(
|
||||
self: &'a Arc<Self>,
|
||||
key: &K,
|
||||
) -> impl Future<Output = Result> + Send + 'a + use<'a, K>
|
||||
where
|
||||
K: AsRef<[u8]> + ?Sized + Debug + 'a,
|
||||
{
|
||||
self.get(key).map(|res| res.map(|_| ()))
|
||||
}
|
||||
|
||||
/// Returns Ok if the map contains the key; NotFound otherwise. Harder errors
|
||||
/// may not always be reported properly.
|
||||
#[implement(super::Map)]
|
||||
#[tracing::instrument(skip(self, key), fields(%self), level = "trace")]
|
||||
pub fn exists_blocking<K>(&self, key: &K) -> Result
|
||||
where
|
||||
K: AsRef<[u8]> + ?Sized + Debug,
|
||||
{
|
||||
self.maybe_exists(key)
|
||||
.then(|| self.get_blocking(key))
|
||||
.flat_ok()
|
||||
.map(|_| ())
|
||||
.ok_or_else(|| err!(Request(NotFound("Not found in database"))))
|
||||
}
|
||||
/// Returns Ok if the map contains the key; NotFound otherwise. Harder
|
||||
/// errors may not always be reported properly.
|
||||
#[tracing::instrument(skip(self, key), fields(%self), level = "trace")]
|
||||
pub fn exists_blocking<K>(&self, key: &K) -> Result
|
||||
where
|
||||
K: AsRef<[u8]> + ?Sized + Debug,
|
||||
{
|
||||
self.maybe_exists(key)
|
||||
.then(|| self.get_blocking(key))
|
||||
.flat_ok()
|
||||
.map(|_| ())
|
||||
.ok_or_else(|| err!(Request(NotFound("Not found in database"))))
|
||||
}
|
||||
|
||||
/// Rocksdb limits this to kBlockCacheTier internally so this is not actually a
|
||||
/// blocking call; in case that changes we set this as well in our read_options.
|
||||
#[implement(super::Map)]
|
||||
pub(crate) fn maybe_exists<K>(&self, key: &K) -> bool
|
||||
where
|
||||
K: AsRef<[u8]> + ?Sized,
|
||||
{
|
||||
self.db
|
||||
.db
|
||||
.key_may_exist_cf_opt(&self.cf(), key, &self.cache_read_options)
|
||||
/// Rocksdb limits this to kBlockCacheTier internally so this is not
|
||||
/// actually a blocking call; in case that changes we set this as well in
|
||||
/// our read_options.
|
||||
pub(crate) fn maybe_exists<K>(&self, key: &K) -> bool
|
||||
where
|
||||
K: AsRef<[u8]> + ?Sized,
|
||||
{
|
||||
self.db
|
||||
.db
|
||||
.key_may_exist_cf_opt(&self.cf(), key, &self.cache_read_options)
|
||||
}
|
||||
}
|
||||
+58
-62
@@ -1,72 +1,68 @@
|
||||
use std::{fmt::Debug, future::Future, sync::Arc};
|
||||
|
||||
use conduwuit::implement;
|
||||
use futures::stream::StreamExt;
|
||||
use serde::Serialize;
|
||||
|
||||
/// Count the total number of entries in the map.
|
||||
#[implement(super::Map)]
|
||||
#[inline]
|
||||
pub fn count(self: &Arc<Self>) -> impl Future<Output = usize> + Send + '_ {
|
||||
self.raw_keys().count()
|
||||
}
|
||||
impl super::Map {
|
||||
/// Count the total number of entries in the map.
|
||||
#[inline]
|
||||
pub fn count(self: &Arc<Self>) -> impl Future<Output = usize> + Send + '_ {
|
||||
self.raw_keys().count()
|
||||
}
|
||||
|
||||
/// Count the number of entries in the map starting from a lower-bound.
|
||||
///
|
||||
/// - From is a structured key
|
||||
#[implement(super::Map)]
|
||||
#[inline]
|
||||
pub fn count_from<'a, P>(
|
||||
self: &'a Arc<Self>,
|
||||
from: &P,
|
||||
) -> impl Future<Output = usize> + Send + 'a + use<'a, P>
|
||||
where
|
||||
P: Serialize + ?Sized + Debug + 'a,
|
||||
{
|
||||
self.keys_from_raw(from).count()
|
||||
}
|
||||
/// Count the number of entries in the map starting from a lower-bound.
|
||||
///
|
||||
/// - From is a structured key
|
||||
#[inline]
|
||||
pub fn count_from<'a, P>(
|
||||
self: &'a Arc<Self>,
|
||||
from: &P,
|
||||
) -> impl Future<Output = usize> + Send + 'a + use<'a, P>
|
||||
where
|
||||
P: Serialize + ?Sized + Debug + 'a,
|
||||
{
|
||||
self.keys_from_raw(from).count()
|
||||
}
|
||||
|
||||
/// Count the number of entries in the map starting from a lower-bound.
|
||||
///
|
||||
/// - From is a raw
|
||||
#[implement(super::Map)]
|
||||
#[inline]
|
||||
pub fn raw_count_from<'a, P>(
|
||||
self: &'a Arc<Self>,
|
||||
from: &'a P,
|
||||
) -> impl Future<Output = usize> + Send + 'a
|
||||
where
|
||||
P: AsRef<[u8]> + ?Sized + Debug + Sync + 'a,
|
||||
{
|
||||
self.raw_keys_from(from).count()
|
||||
}
|
||||
/// Count the number of entries in the map starting from a lower-bound.
|
||||
///
|
||||
/// - From is a raw
|
||||
#[inline]
|
||||
pub fn raw_count_from<'a, P>(
|
||||
self: &'a Arc<Self>,
|
||||
from: &'a P,
|
||||
) -> impl Future<Output = usize> + Send + 'a
|
||||
where
|
||||
P: AsRef<[u8]> + ?Sized + Debug + Sync + 'a,
|
||||
{
|
||||
self.raw_keys_from(from).count()
|
||||
}
|
||||
|
||||
/// Count the number of entries in the map matching a prefix.
|
||||
///
|
||||
/// - Prefix is structured key
|
||||
#[implement(super::Map)]
|
||||
#[inline]
|
||||
pub fn count_prefix<'a, P>(
|
||||
self: &'a Arc<Self>,
|
||||
prefix: &P,
|
||||
) -> impl Future<Output = usize> + Send + 'a + use<'a, P>
|
||||
where
|
||||
P: Serialize + ?Sized + Debug + 'a,
|
||||
{
|
||||
self.keys_prefix_raw(prefix).count()
|
||||
}
|
||||
/// Count the number of entries in the map matching a prefix.
|
||||
///
|
||||
/// - Prefix is structured key
|
||||
#[inline]
|
||||
pub fn count_prefix<'a, P>(
|
||||
self: &'a Arc<Self>,
|
||||
prefix: &P,
|
||||
) -> impl Future<Output = usize> + Send + 'a + use<'a, P>
|
||||
where
|
||||
P: Serialize + ?Sized + Debug + 'a,
|
||||
{
|
||||
self.keys_prefix_raw(prefix).count()
|
||||
}
|
||||
|
||||
/// Count the number of entries in the map matching a prefix.
|
||||
///
|
||||
/// - Prefix is raw
|
||||
#[implement(super::Map)]
|
||||
#[inline]
|
||||
pub fn raw_count_prefix<'a, P>(
|
||||
self: &'a Arc<Self>,
|
||||
prefix: &'a P,
|
||||
) -> impl Future<Output = usize> + Send + 'a
|
||||
where
|
||||
P: AsRef<[u8]> + ?Sized + Debug + Sync + 'a,
|
||||
{
|
||||
self.raw_keys_prefix(prefix).count()
|
||||
/// Count the number of entries in the map matching a prefix.
|
||||
///
|
||||
/// - Prefix is raw
|
||||
#[inline]
|
||||
pub fn raw_count_prefix<'a, P>(
|
||||
self: &'a Arc<Self>,
|
||||
prefix: &'a P,
|
||||
) -> impl Future<Output = usize> + Send + 'a
|
||||
where
|
||||
P: AsRef<[u8]> + ?Sized + Debug + Sync + 'a,
|
||||
{
|
||||
self.raw_keys_prefix(prefix).count()
|
||||
}
|
||||
}
|
||||
+62
-64
@@ -1,6 +1,6 @@
|
||||
use std::{convert::AsRef, fmt::Debug, sync::Arc};
|
||||
|
||||
use conduwuit::{Err, Result, err, implement, utils::result::MapExpect};
|
||||
use conduwuit::{Err, Result, err, utils::result::MapExpect};
|
||||
use futures::{Future, FutureExt, TryFutureExt, future::ready};
|
||||
use rocksdb::{DBPinnableSlice, ReadOptions};
|
||||
use tokio::task;
|
||||
@@ -10,74 +10,72 @@ use crate::{
|
||||
util::{is_incomplete, map_err, or_else},
|
||||
};
|
||||
|
||||
/// Fetch a value from the database into cache, returning a reference-handle
|
||||
/// asynchronously. The key is referenced directly to perform the query.
|
||||
#[implement(super::Map)]
|
||||
#[tracing::instrument(skip(self, key), fields(%self), level = "trace")]
|
||||
pub fn get<K>(
|
||||
self: &Arc<Self>,
|
||||
key: &K,
|
||||
) -> impl Future<Output = Result<Handle<'_>>> + Send + use<'_, K>
|
||||
where
|
||||
K: AsRef<[u8]> + Debug + ?Sized,
|
||||
{
|
||||
use crate::pool::Get;
|
||||
impl super::Map {
|
||||
/// Fetch a value from the database into cache, returning a reference-handle
|
||||
/// asynchronously. The key is referenced directly to perform the query.
|
||||
#[tracing::instrument(skip(self, key), fields(%self), level = "trace")]
|
||||
pub fn get<K>(
|
||||
self: &Arc<Self>,
|
||||
key: &K,
|
||||
) -> impl Future<Output = Result<Handle<'_>>> + Send + use<'_, K>
|
||||
where
|
||||
K: AsRef<[u8]> + Debug + ?Sized,
|
||||
{
|
||||
use crate::pool::Get;
|
||||
|
||||
let cached = self.get_cached(key);
|
||||
if matches!(cached, Err(_) | Ok(Some(_))) {
|
||||
return task::consume_budget()
|
||||
.map(move |()| cached.map_expect("data found in cache"))
|
||||
.boxed();
|
||||
let cached = self.get_cached(key);
|
||||
if matches!(cached, Err(_) | Ok(Some(_))) {
|
||||
return task::consume_budget()
|
||||
.map(move |()| cached.map_expect("data found in cache"))
|
||||
.boxed();
|
||||
}
|
||||
|
||||
debug_assert!(matches!(cached, Ok(None)), "expected status Incomplete");
|
||||
let cmd = Get {
|
||||
map: self.clone(),
|
||||
key: [key.as_ref().into()].into(),
|
||||
res: None,
|
||||
};
|
||||
|
||||
self.db
|
||||
.pool
|
||||
.execute_get(cmd)
|
||||
.and_then(|mut res| ready(res.remove(0)))
|
||||
.boxed()
|
||||
}
|
||||
|
||||
debug_assert!(matches!(cached, Ok(None)), "expected status Incomplete");
|
||||
let cmd = Get {
|
||||
map: self.clone(),
|
||||
key: [key.as_ref().into()].into(),
|
||||
res: None,
|
||||
};
|
||||
/// Fetch a value from the cache without I/O.
|
||||
#[tracing::instrument(skip(self, key), name = "cache", level = "trace")]
|
||||
pub(crate) fn get_cached<K>(&self, key: &K) -> Result<Option<Handle<'_>>>
|
||||
where
|
||||
K: AsRef<[u8]> + Debug + ?Sized,
|
||||
{
|
||||
let res = self.get_blocking_opts(key, &self.cache_read_options);
|
||||
cached_handle_from(res)
|
||||
}
|
||||
|
||||
self.db
|
||||
.pool
|
||||
.execute_get(cmd)
|
||||
.and_then(|mut res| ready(res.remove(0)))
|
||||
.boxed()
|
||||
}
|
||||
/// Fetch a value from the database into cache, returning a
|
||||
/// reference-handle. The key is referenced directly to perform the query.
|
||||
/// This is a thread- blocking call.
|
||||
#[tracing::instrument(skip(self, key), name = "blocking", level = "trace")]
|
||||
pub fn get_blocking<K>(&self, key: &K) -> Result<Handle<'_>>
|
||||
where
|
||||
K: AsRef<[u8]> + ?Sized,
|
||||
{
|
||||
let res = self.get_blocking_opts(key, &self.read_options);
|
||||
handle_from(res)
|
||||
}
|
||||
|
||||
/// Fetch a value from the cache without I/O.
|
||||
#[implement(super::Map)]
|
||||
#[tracing::instrument(skip(self, key), name = "cache", level = "trace")]
|
||||
pub(crate) fn get_cached<K>(&self, key: &K) -> Result<Option<Handle<'_>>>
|
||||
where
|
||||
K: AsRef<[u8]> + Debug + ?Sized,
|
||||
{
|
||||
let res = self.get_blocking_opts(key, &self.cache_read_options);
|
||||
cached_handle_from(res)
|
||||
}
|
||||
|
||||
/// Fetch a value from the database into cache, returning a reference-handle.
|
||||
/// The key is referenced directly to perform the query. This is a thread-
|
||||
/// blocking call.
|
||||
#[implement(super::Map)]
|
||||
#[tracing::instrument(skip(self, key), name = "blocking", level = "trace")]
|
||||
pub fn get_blocking<K>(&self, key: &K) -> Result<Handle<'_>>
|
||||
where
|
||||
K: AsRef<[u8]> + ?Sized,
|
||||
{
|
||||
let res = self.get_blocking_opts(key, &self.read_options);
|
||||
handle_from(res)
|
||||
}
|
||||
|
||||
#[implement(super::Map)]
|
||||
fn get_blocking_opts<K>(
|
||||
&self,
|
||||
key: &K,
|
||||
read_options: &ReadOptions,
|
||||
) -> Result<Option<DBPinnableSlice<'_>>, rocksdb::Error>
|
||||
where
|
||||
K: AsRef<[u8]> + ?Sized,
|
||||
{
|
||||
self.db.db.get_pinned_cf_opt(&self.cf(), key, read_options)
|
||||
fn get_blocking_opts<K>(
|
||||
&self,
|
||||
key: &K,
|
||||
read_options: &ReadOptions,
|
||||
) -> Result<Option<DBPinnableSlice<'_>>, rocksdb::Error>
|
||||
where
|
||||
K: AsRef<[u8]> + ?Sized,
|
||||
{
|
||||
self.db.db.get_pinned_cf_opt(&self.cf(), key, read_options)
|
||||
}
|
||||
}
|
||||
|
||||
#[inline]
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
use std::{convert::AsRef, sync::Arc};
|
||||
|
||||
use conduwuit::{
|
||||
Result, implement,
|
||||
Result,
|
||||
utils::{
|
||||
IterStream,
|
||||
stream::{WidebandExt, automatic_amplification, automatic_width},
|
||||
@@ -34,60 +34,59 @@ where
|
||||
}
|
||||
}
|
||||
|
||||
#[implement(super::Map)]
|
||||
#[tracing::instrument(skip(self, keys), level = "trace")]
|
||||
pub(crate) fn get_batch<'a, S, K>(
|
||||
self: &'a Arc<Self>,
|
||||
keys: S,
|
||||
) -> impl Stream<Item = Result<Handle<'a>>> + Send + 'a
|
||||
where
|
||||
S: Stream<Item = K> + Send + 'a,
|
||||
K: AsRef<[u8]> + Send + Sync + 'a,
|
||||
{
|
||||
use crate::pool::Get;
|
||||
impl super::Map {
|
||||
#[tracing::instrument(skip(self, keys), level = "trace")]
|
||||
pub(crate) fn get_batch<'a, S, K>(
|
||||
self: &'a Arc<Self>,
|
||||
keys: S,
|
||||
) -> impl Stream<Item = Result<Handle<'a>>> + Send + 'a
|
||||
where
|
||||
S: Stream<Item = K> + Send + 'a,
|
||||
K: AsRef<[u8]> + Send + Sync + 'a,
|
||||
{
|
||||
use crate::pool::Get;
|
||||
|
||||
keys.ready_chunks(automatic_amplification())
|
||||
.widen_then(automatic_width(), |chunk| {
|
||||
self.db.pool.execute_get(Get {
|
||||
map: self.clone(),
|
||||
key: chunk.iter().map(AsRef::as_ref).map(Into::into).collect(),
|
||||
res: None,
|
||||
keys.ready_chunks(automatic_amplification())
|
||||
.widen_then(automatic_width(), |chunk| {
|
||||
self.db.pool.execute_get(Get {
|
||||
map: self.clone(),
|
||||
key: chunk.iter().map(AsRef::as_ref).map(Into::into).collect(),
|
||||
res: None,
|
||||
})
|
||||
})
|
||||
})
|
||||
.map_ok(|results| results.into_iter().stream())
|
||||
.try_flatten()
|
||||
}
|
||||
.map_ok(|results| results.into_iter().stream())
|
||||
.try_flatten()
|
||||
}
|
||||
|
||||
#[implement(super::Map)]
|
||||
#[tracing::instrument(name = "batch_blocking", level = "trace", skip_all)]
|
||||
pub(crate) fn get_batch_blocking<'a, I, K>(
|
||||
&self,
|
||||
keys: I,
|
||||
) -> impl Iterator<Item = Result<Handle<'_>>> + Send + use<'_, I, K>
|
||||
where
|
||||
I: Iterator<Item = &'a K> + ExactSizeIterator + Send,
|
||||
K: AsRef<[u8]> + Send + ?Sized + Sync + 'a,
|
||||
{
|
||||
self.get_batch_blocking_opts(keys, &self.read_options)
|
||||
.map(handle_from)
|
||||
}
|
||||
#[tracing::instrument(name = "batch_blocking", level = "trace", skip_all)]
|
||||
pub(crate) fn get_batch_blocking<'a, I, K>(
|
||||
&self,
|
||||
keys: I,
|
||||
) -> impl Iterator<Item = Result<Handle<'_>>> + Send + use<'_, I, K>
|
||||
where
|
||||
I: Iterator<Item = &'a K> + ExactSizeIterator + Send,
|
||||
K: AsRef<[u8]> + Send + ?Sized + Sync + 'a,
|
||||
{
|
||||
self.get_batch_blocking_opts(keys, &self.read_options)
|
||||
.map(handle_from)
|
||||
}
|
||||
|
||||
#[implement(super::Map)]
|
||||
fn get_batch_blocking_opts<'a, I, K>(
|
||||
&self,
|
||||
keys: I,
|
||||
read_options: &ReadOptions,
|
||||
) -> impl Iterator<Item = Result<Option<DBPinnableSlice<'_>>, rocksdb::Error>> + Send + use<'_, I, K>
|
||||
where
|
||||
I: Iterator<Item = &'a K> + ExactSizeIterator + Send,
|
||||
K: AsRef<[u8]> + Send + ?Sized + Sync + 'a,
|
||||
{
|
||||
// Optimization can be `true` if key vector is pre-sorted **by the column
|
||||
// comparator**.
|
||||
const SORTED: bool = false;
|
||||
fn get_batch_blocking_opts<'a, I, K>(
|
||||
&self,
|
||||
keys: I,
|
||||
read_options: &ReadOptions,
|
||||
) -> impl Iterator<Item = Result<Option<DBPinnableSlice<'_>>, rocksdb::Error>> + Send + use<'_, I, K>
|
||||
where
|
||||
I: Iterator<Item = &'a K> + ExactSizeIterator + Send,
|
||||
K: AsRef<[u8]> + Send + ?Sized + Sync + 'a,
|
||||
{
|
||||
// Optimization can be `true` if key vector is pre-sorted **by the column
|
||||
// comparator**.
|
||||
const SORTED: bool = false;
|
||||
|
||||
self.db
|
||||
.db
|
||||
.batched_multi_get_cf_opt(&self.cf(), keys, SORTED, read_options)
|
||||
.into_iter()
|
||||
self.db
|
||||
.db
|
||||
.batched_multi_get_cf_opt(&self.cf(), keys, SORTED, read_options)
|
||||
.into_iter()
|
||||
}
|
||||
}
|
||||
+203
-214
@@ -5,7 +5,7 @@
|
||||
|
||||
use std::{convert::AsRef, fmt::Debug, io::Write};
|
||||
|
||||
use conduwuit::{arrayvec::ArrayVec, implement};
|
||||
use conduwuit::arrayvec::ArrayVec;
|
||||
use rocksdb::WriteBatchWithTransaction;
|
||||
use serde::Serialize;
|
||||
|
||||
@@ -15,223 +15,212 @@ use crate::{
|
||||
util::or_else,
|
||||
};
|
||||
|
||||
/// Insert Key/Value
|
||||
///
|
||||
/// - Key is serialized
|
||||
/// - Val is serialized
|
||||
#[implement(super::Map)]
|
||||
#[inline]
|
||||
pub fn put<K, V>(&self, key: K, val: V)
|
||||
where
|
||||
K: Serialize + Debug,
|
||||
V: Serialize,
|
||||
{
|
||||
let mut key_buf = KeyBuf::new();
|
||||
let mut val_buf = ValBuf::new();
|
||||
self.bput(key, val, (&mut key_buf, &mut val_buf));
|
||||
}
|
||||
|
||||
/// Insert Key/Value
|
||||
///
|
||||
/// - Key is serialized
|
||||
/// - Val is raw
|
||||
#[implement(super::Map)]
|
||||
#[inline]
|
||||
pub fn put_raw<K, V>(&self, key: K, val: V)
|
||||
where
|
||||
K: Serialize + Debug,
|
||||
V: AsRef<[u8]>,
|
||||
{
|
||||
let mut key_buf = KeyBuf::new();
|
||||
self.bput_raw(key, val, &mut key_buf);
|
||||
}
|
||||
|
||||
/// Insert Key/Value
|
||||
///
|
||||
/// - Key is raw
|
||||
/// - Val is serialized
|
||||
#[implement(super::Map)]
|
||||
#[inline]
|
||||
pub fn raw_put<K, V>(&self, key: K, val: V)
|
||||
where
|
||||
K: AsRef<[u8]>,
|
||||
V: Serialize,
|
||||
{
|
||||
let mut val_buf = ValBuf::new();
|
||||
self.raw_bput(key, val, &mut val_buf);
|
||||
}
|
||||
|
||||
/// Insert Key/Value
|
||||
///
|
||||
/// - Key is serialized
|
||||
/// - Val is serialized to stack-buffer
|
||||
#[implement(super::Map)]
|
||||
#[inline]
|
||||
pub fn put_aput<const VMAX: usize, K, V>(&self, key: K, val: V)
|
||||
where
|
||||
K: Serialize + Debug,
|
||||
V: Serialize,
|
||||
{
|
||||
let mut key_buf = KeyBuf::new();
|
||||
let mut val_buf = ArrayVec::<u8, VMAX>::new();
|
||||
self.bput(key, val, (&mut key_buf, &mut val_buf));
|
||||
}
|
||||
|
||||
/// Insert Key/Value
|
||||
///
|
||||
/// - Key is serialized to stack-buffer
|
||||
/// - Val is serialized
|
||||
#[implement(super::Map)]
|
||||
#[inline]
|
||||
pub fn aput_put<const KMAX: usize, K, V>(&self, key: K, val: V)
|
||||
where
|
||||
K: Serialize + Debug,
|
||||
V: Serialize,
|
||||
{
|
||||
let mut key_buf = ArrayVec::<u8, KMAX>::new();
|
||||
let mut val_buf = ValBuf::new();
|
||||
self.bput(key, val, (&mut key_buf, &mut val_buf));
|
||||
}
|
||||
|
||||
/// Insert Key/Value
|
||||
///
|
||||
/// - Key is serialized to stack-buffer
|
||||
/// - Val is serialized to stack-buffer
|
||||
#[implement(super::Map)]
|
||||
#[inline]
|
||||
pub fn aput<const KMAX: usize, const VMAX: usize, K, V>(&self, key: K, val: V)
|
||||
where
|
||||
K: Serialize + Debug,
|
||||
V: Serialize,
|
||||
{
|
||||
let mut key_buf = ArrayVec::<u8, KMAX>::new();
|
||||
let mut val_buf = ArrayVec::<u8, VMAX>::new();
|
||||
self.bput(key, val, (&mut key_buf, &mut val_buf));
|
||||
}
|
||||
|
||||
/// Insert Key/Value
|
||||
///
|
||||
/// - Key is serialized to stack-buffer
|
||||
/// - Val is raw
|
||||
#[implement(super::Map)]
|
||||
#[inline]
|
||||
pub fn aput_raw<const KMAX: usize, K, V>(&self, key: K, val: V)
|
||||
where
|
||||
K: Serialize + Debug,
|
||||
V: AsRef<[u8]>,
|
||||
{
|
||||
let mut key_buf = ArrayVec::<u8, KMAX>::new();
|
||||
self.bput_raw(key, val, &mut key_buf);
|
||||
}
|
||||
|
||||
/// Insert Key/Value
|
||||
///
|
||||
/// - Key is raw
|
||||
/// - Val is serialized to stack-buffer
|
||||
#[implement(super::Map)]
|
||||
#[inline]
|
||||
pub fn raw_aput<const VMAX: usize, K, V>(&self, key: K, val: V)
|
||||
where
|
||||
K: AsRef<[u8]>,
|
||||
V: Serialize,
|
||||
{
|
||||
let mut val_buf = ArrayVec::<u8, VMAX>::new();
|
||||
self.raw_bput(key, val, &mut val_buf);
|
||||
}
|
||||
|
||||
/// Insert Key/Value
|
||||
///
|
||||
/// - Key is serialized to supplied buffer
|
||||
/// - Val is serialized to supplied buffer
|
||||
#[implement(super::Map)]
|
||||
pub fn bput<K, V, Bk, Bv>(&self, key: K, val: V, mut buf: (Bk, Bv))
|
||||
where
|
||||
K: Serialize + Debug,
|
||||
V: Serialize,
|
||||
Bk: Write + AsRef<[u8]>,
|
||||
Bv: Write + AsRef<[u8]>,
|
||||
{
|
||||
let val = ser::serialize(&mut buf.1, val).expect("failed to serialize insertion val");
|
||||
self.bput_raw(key, val, &mut buf.0);
|
||||
}
|
||||
|
||||
/// Insert Key/Value
|
||||
///
|
||||
/// - Key is serialized to supplied buffer
|
||||
/// - Val is raw
|
||||
#[implement(super::Map)]
|
||||
#[tracing::instrument(skip(self, val, buf), level = "trace")]
|
||||
pub fn bput_raw<K, V, Bk>(&self, key: K, val: V, mut buf: Bk)
|
||||
where
|
||||
K: Serialize + Debug,
|
||||
V: AsRef<[u8]>,
|
||||
Bk: Write + AsRef<[u8]>,
|
||||
{
|
||||
let key = ser::serialize(&mut buf, key).expect("failed to serialize insertion key");
|
||||
self.insert(&key, val);
|
||||
}
|
||||
|
||||
/// Insert Key/Value
|
||||
///
|
||||
/// - Key is raw
|
||||
/// - Val is serialized to supplied buffer
|
||||
#[implement(super::Map)]
|
||||
pub fn raw_bput<K, V, Bv>(&self, key: K, val: V, mut buf: Bv)
|
||||
where
|
||||
K: AsRef<[u8]>,
|
||||
V: Serialize,
|
||||
Bv: Write + AsRef<[u8]>,
|
||||
{
|
||||
let val = ser::serialize(&mut buf, val).expect("failed to serialize insertion val");
|
||||
self.insert(&key, val);
|
||||
}
|
||||
|
||||
/// Insert Key/Value
|
||||
///
|
||||
/// - Key is raw
|
||||
/// - Val is raw
|
||||
#[implement(super::Map)]
|
||||
#[tracing::instrument(skip_all, fields(%self), level = "trace")]
|
||||
pub fn insert<K, V>(&self, key: &K, val: V)
|
||||
where
|
||||
K: AsRef<[u8]> + ?Sized,
|
||||
V: AsRef<[u8]>,
|
||||
{
|
||||
let write_options = &self.write_options;
|
||||
self.db
|
||||
.db
|
||||
.put_cf_opt(&self.cf(), key, val, write_options)
|
||||
.or_else(or_else)
|
||||
.expect("database insert error");
|
||||
|
||||
if !self.db.corked() {
|
||||
self.db.flush().expect("database flush error");
|
||||
impl super::Map {
|
||||
/// Insert Key/Value
|
||||
///
|
||||
/// - Key is serialized
|
||||
/// - Val is serialized
|
||||
#[inline]
|
||||
pub fn put<K, V>(&self, key: K, val: V)
|
||||
where
|
||||
K: Serialize + Debug,
|
||||
V: Serialize,
|
||||
{
|
||||
let mut key_buf = KeyBuf::new();
|
||||
let mut val_buf = ValBuf::new();
|
||||
self.bput(key, val, (&mut key_buf, &mut val_buf));
|
||||
}
|
||||
|
||||
self.watchers.wake(key.as_ref());
|
||||
}
|
||||
|
||||
#[implement(super::Map)]
|
||||
#[tracing::instrument(skip(self, iter), fields(%self), level = "trace")]
|
||||
pub fn insert_batch<'a, I, K, V>(&'a self, iter: I)
|
||||
where
|
||||
I: Iterator<Item = (K, V)> + Send + Debug,
|
||||
K: AsRef<[u8]> + Sized + Debug + 'a,
|
||||
V: AsRef<[u8]> + Sized + 'a,
|
||||
{
|
||||
let mut batch = WriteBatchWithTransaction::<false>::default();
|
||||
for (key, val) in iter {
|
||||
batch.put_cf(&self.cf(), key.as_ref(), val.as_ref());
|
||||
/// Insert Key/Value
|
||||
///
|
||||
/// - Key is serialized
|
||||
/// - Val is raw
|
||||
#[inline]
|
||||
pub fn put_raw<K, V>(&self, key: K, val: V)
|
||||
where
|
||||
K: Serialize + Debug,
|
||||
V: AsRef<[u8]>,
|
||||
{
|
||||
let mut key_buf = KeyBuf::new();
|
||||
self.bput_raw(key, val, &mut key_buf);
|
||||
}
|
||||
|
||||
let write_options = &self.write_options;
|
||||
self.db
|
||||
.db
|
||||
.write_opt(&batch, write_options)
|
||||
.or_else(or_else)
|
||||
.expect("database insert batch error");
|
||||
/// Insert Key/Value
|
||||
///
|
||||
/// - Key is raw
|
||||
/// - Val is serialized
|
||||
#[inline]
|
||||
pub fn raw_put<K, V>(&self, key: K, val: V)
|
||||
where
|
||||
K: AsRef<[u8]>,
|
||||
V: Serialize,
|
||||
{
|
||||
let mut val_buf = ValBuf::new();
|
||||
self.raw_bput(key, val, &mut val_buf);
|
||||
}
|
||||
|
||||
if !self.db.corked() {
|
||||
self.db.flush().expect("database flush error");
|
||||
/// Insert Key/Value
|
||||
///
|
||||
/// - Key is serialized
|
||||
/// - Val is serialized to stack-buffer
|
||||
#[inline]
|
||||
pub fn put_aput<const VMAX: usize, K, V>(&self, key: K, val: V)
|
||||
where
|
||||
K: Serialize + Debug,
|
||||
V: Serialize,
|
||||
{
|
||||
let mut key_buf = KeyBuf::new();
|
||||
let mut val_buf = ArrayVec::<u8, VMAX>::new();
|
||||
self.bput(key, val, (&mut key_buf, &mut val_buf));
|
||||
}
|
||||
|
||||
/// Insert Key/Value
|
||||
///
|
||||
/// - Key is serialized to stack-buffer
|
||||
/// - Val is serialized
|
||||
#[inline]
|
||||
pub fn aput_put<const KMAX: usize, K, V>(&self, key: K, val: V)
|
||||
where
|
||||
K: Serialize + Debug,
|
||||
V: Serialize,
|
||||
{
|
||||
let mut key_buf = ArrayVec::<u8, KMAX>::new();
|
||||
let mut val_buf = ValBuf::new();
|
||||
self.bput(key, val, (&mut key_buf, &mut val_buf));
|
||||
}
|
||||
|
||||
/// Insert Key/Value
|
||||
///
|
||||
/// - Key is serialized to stack-buffer
|
||||
/// - Val is serialized to stack-buffer
|
||||
#[inline]
|
||||
pub fn aput<const KMAX: usize, const VMAX: usize, K, V>(&self, key: K, val: V)
|
||||
where
|
||||
K: Serialize + Debug,
|
||||
V: Serialize,
|
||||
{
|
||||
let mut key_buf = ArrayVec::<u8, KMAX>::new();
|
||||
let mut val_buf = ArrayVec::<u8, VMAX>::new();
|
||||
self.bput(key, val, (&mut key_buf, &mut val_buf));
|
||||
}
|
||||
|
||||
/// Insert Key/Value
|
||||
///
|
||||
/// - Key is serialized to stack-buffer
|
||||
/// - Val is raw
|
||||
#[inline]
|
||||
pub fn aput_raw<const KMAX: usize, K, V>(&self, key: K, val: V)
|
||||
where
|
||||
K: Serialize + Debug,
|
||||
V: AsRef<[u8]>,
|
||||
{
|
||||
let mut key_buf = ArrayVec::<u8, KMAX>::new();
|
||||
self.bput_raw(key, val, &mut key_buf);
|
||||
}
|
||||
|
||||
/// Insert Key/Value
|
||||
///
|
||||
/// - Key is raw
|
||||
/// - Val is serialized to stack-buffer
|
||||
#[inline]
|
||||
pub fn raw_aput<const VMAX: usize, K, V>(&self, key: K, val: V)
|
||||
where
|
||||
K: AsRef<[u8]>,
|
||||
V: Serialize,
|
||||
{
|
||||
let mut val_buf = ArrayVec::<u8, VMAX>::new();
|
||||
self.raw_bput(key, val, &mut val_buf);
|
||||
}
|
||||
|
||||
/// Insert Key/Value
|
||||
///
|
||||
/// - Key is serialized to supplied buffer
|
||||
/// - Val is serialized to supplied buffer
|
||||
pub fn bput<K, V, Bk, Bv>(&self, key: K, val: V, mut buf: (Bk, Bv))
|
||||
where
|
||||
K: Serialize + Debug,
|
||||
V: Serialize,
|
||||
Bk: Write + AsRef<[u8]>,
|
||||
Bv: Write + AsRef<[u8]>,
|
||||
{
|
||||
let val = ser::serialize(&mut buf.1, val).expect("failed to serialize insertion val");
|
||||
self.bput_raw(key, val, &mut buf.0);
|
||||
}
|
||||
|
||||
/// Insert Key/Value
|
||||
///
|
||||
/// - Key is serialized to supplied buffer
|
||||
/// - Val is raw
|
||||
#[tracing::instrument(skip(self, val, buf), level = "trace")]
|
||||
pub fn bput_raw<K, V, Bk>(&self, key: K, val: V, mut buf: Bk)
|
||||
where
|
||||
K: Serialize + Debug,
|
||||
V: AsRef<[u8]>,
|
||||
Bk: Write + AsRef<[u8]>,
|
||||
{
|
||||
let key = ser::serialize(&mut buf, key).expect("failed to serialize insertion key");
|
||||
self.insert(&key, val);
|
||||
}
|
||||
|
||||
/// Insert Key/Value
|
||||
///
|
||||
/// - Key is raw
|
||||
/// - Val is serialized to supplied buffer
|
||||
pub fn raw_bput<K, V, Bv>(&self, key: K, val: V, mut buf: Bv)
|
||||
where
|
||||
K: AsRef<[u8]>,
|
||||
V: Serialize,
|
||||
Bv: Write + AsRef<[u8]>,
|
||||
{
|
||||
let val = ser::serialize(&mut buf, val).expect("failed to serialize insertion val");
|
||||
self.insert(&key, val);
|
||||
}
|
||||
|
||||
/// Insert Key/Value
|
||||
///
|
||||
/// - Key is raw
|
||||
/// - Val is raw
|
||||
#[tracing::instrument(skip_all, fields(%self), level = "trace")]
|
||||
pub fn insert<K, V>(&self, key: &K, val: V)
|
||||
where
|
||||
K: AsRef<[u8]> + ?Sized,
|
||||
V: AsRef<[u8]>,
|
||||
{
|
||||
let write_options = &self.write_options;
|
||||
self.db
|
||||
.db
|
||||
.put_cf_opt(&self.cf(), key, val, write_options)
|
||||
.or_else(or_else)
|
||||
.expect("database insert error");
|
||||
|
||||
if !self.db.corked() {
|
||||
self.db.flush().expect("database flush error");
|
||||
}
|
||||
|
||||
self.watchers.wake(key.as_ref());
|
||||
}
|
||||
|
||||
#[tracing::instrument(skip(self, iter), fields(%self), level = "trace")]
|
||||
pub fn insert_batch<'a, I, K, V>(&'a self, iter: I)
|
||||
where
|
||||
I: Iterator<Item = (K, V)> + Send + Debug,
|
||||
K: AsRef<[u8]> + Sized + Debug + 'a,
|
||||
V: AsRef<[u8]> + Sized + 'a,
|
||||
{
|
||||
let mut batch = WriteBatchWithTransaction::<false>::default();
|
||||
for (key, val) in iter {
|
||||
batch.put_cf(&self.cf(), key.as_ref(), val.as_ref());
|
||||
}
|
||||
|
||||
let write_options = &self.write_options;
|
||||
self.db
|
||||
.db
|
||||
.write_opt(&batch, write_options)
|
||||
.or_else(or_else)
|
||||
.expect("database insert batch error");
|
||||
|
||||
if !self.db.corked() {
|
||||
self.db.flush().expect("database flush error");
|
||||
}
|
||||
}
|
||||
}
|
||||
+36
-37
@@ -1,6 +1,6 @@
|
||||
use std::sync::Arc;
|
||||
|
||||
use conduwuit::{Result, implement};
|
||||
use conduwuit::Result;
|
||||
use futures::{FutureExt, Stream, StreamExt, TryFutureExt, TryStreamExt};
|
||||
use rocksdb::Direction;
|
||||
use serde::Deserialize;
|
||||
@@ -9,43 +9,42 @@ use tokio::task;
|
||||
use super::stream::is_cached;
|
||||
use crate::{keyval, keyval::Key, stream};
|
||||
|
||||
#[implement(super::Map)]
|
||||
pub fn keys<'a, K>(self: &'a Arc<Self>) -> impl Stream<Item = Result<Key<'a, K>>> + Send
|
||||
where
|
||||
K: Deserialize<'a> + Send,
|
||||
{
|
||||
self.raw_keys().map(keyval::result_deserialize_key::<K>)
|
||||
}
|
||||
|
||||
#[implement(super::Map)]
|
||||
#[tracing::instrument(skip(self), fields(%self), level = "trace")]
|
||||
pub fn raw_keys(self: &Arc<Self>) -> impl Stream<Item = Result<Key<'_>>> + Send {
|
||||
use crate::pool::Seek;
|
||||
|
||||
let opts = super::iter_options_default(&self.db);
|
||||
let state = stream::State::new(self, opts);
|
||||
if is_cached(self) {
|
||||
let state = state.init_fwd(None);
|
||||
return task::consume_budget()
|
||||
.map(move |()| stream::Keys::<'_>::from(state))
|
||||
.into_stream()
|
||||
.flatten()
|
||||
.boxed();
|
||||
impl super::Map {
|
||||
pub fn keys<'a, K>(self: &'a Arc<Self>) -> impl Stream<Item = Result<Key<'a, K>>> + Send
|
||||
where
|
||||
K: Deserialize<'a> + Send,
|
||||
{
|
||||
self.raw_keys().map(keyval::result_deserialize_key::<K>)
|
||||
}
|
||||
|
||||
let seek = Seek {
|
||||
map: self.clone(),
|
||||
dir: Direction::Forward,
|
||||
state: crate::pool::into_send_seek(state),
|
||||
key: None,
|
||||
res: None,
|
||||
};
|
||||
pub fn raw_keys(self: &Arc<Self>) -> impl Stream<Item = Result<Key<'_>>> + Send {
|
||||
use crate::pool::Seek;
|
||||
|
||||
self.db
|
||||
.pool
|
||||
.execute_iter(seek)
|
||||
.ok_into::<stream::Keys<'_>>()
|
||||
.into_stream()
|
||||
.try_flatten()
|
||||
.boxed()
|
||||
let opts = super::iter_options_default(&self.db);
|
||||
let state = stream::State::new(self, opts);
|
||||
if is_cached(self) {
|
||||
let state = state.init_fwd(None);
|
||||
return task::consume_budget()
|
||||
.map(move |()| stream::Keys::<'_>::from(state))
|
||||
.into_stream()
|
||||
.flatten()
|
||||
.boxed();
|
||||
}
|
||||
|
||||
let seek = Seek {
|
||||
map: self.clone(),
|
||||
dir: Direction::Forward,
|
||||
state: crate::pool::into_send_seek(state),
|
||||
key: None,
|
||||
res: None,
|
||||
};
|
||||
|
||||
self.db
|
||||
.pool
|
||||
.execute_iter(seek)
|
||||
.ok_into::<stream::Keys<'_>>()
|
||||
.into_stream()
|
||||
.try_flatten()
|
||||
.boxed()
|
||||
}
|
||||
}
|
||||
@@ -1,6 +1,6 @@
|
||||
use std::{convert::AsRef, fmt::Debug, sync::Arc};
|
||||
|
||||
use conduwuit::{Result, implement};
|
||||
use conduwuit::Result;
|
||||
use futures::{FutureExt, Stream, StreamExt, TryFutureExt, TryStreamExt};
|
||||
use rocksdb::Direction;
|
||||
use serde::{Deserialize, Serialize};
|
||||
@@ -11,73 +11,71 @@ use crate::{
|
||||
stream,
|
||||
};
|
||||
|
||||
#[implement(super::Map)]
|
||||
pub fn keys_from<'a, K, P>(
|
||||
self: &'a Arc<Self>,
|
||||
from: &P,
|
||||
) -> impl Stream<Item = Result<Key<'a, K>>> + Send + use<'a, K, P>
|
||||
where
|
||||
P: Serialize + ?Sized + Debug,
|
||||
K: Deserialize<'a> + Send,
|
||||
{
|
||||
self.keys_from_raw(from).map(result_deserialize_key::<K>)
|
||||
}
|
||||
|
||||
#[implement(super::Map)]
|
||||
#[tracing::instrument(skip(self), level = "trace")]
|
||||
pub fn keys_from_raw<P>(
|
||||
self: &Arc<Self>,
|
||||
from: &P,
|
||||
) -> impl Stream<Item = Result<Key<'_>>> + Send + use<'_, P>
|
||||
where
|
||||
P: Serialize + ?Sized + Debug,
|
||||
{
|
||||
let key = serialize_key(from).expect("failed to serialize query key");
|
||||
self.raw_keys_from(&key)
|
||||
}
|
||||
|
||||
#[implement(super::Map)]
|
||||
pub fn keys_raw_from<'a, K, P>(
|
||||
self: &'a Arc<Self>,
|
||||
from: &P,
|
||||
) -> impl Stream<Item = Result<Key<'a, K>>> + Send + use<'a, K, P>
|
||||
where
|
||||
P: AsRef<[u8]> + ?Sized + Debug + Sync,
|
||||
K: Deserialize<'a> + Send,
|
||||
{
|
||||
self.raw_keys_from(from).map(result_deserialize_key::<K>)
|
||||
}
|
||||
|
||||
#[implement(super::Map)]
|
||||
#[tracing::instrument(skip(self, from), fields(%self), level = "trace")]
|
||||
pub fn raw_keys_from<P>(
|
||||
self: &Arc<Self>,
|
||||
from: &P,
|
||||
) -> impl Stream<Item = Result<Key<'_>>> + Send + use<'_, P>
|
||||
where
|
||||
P: AsRef<[u8]> + ?Sized + Debug,
|
||||
{
|
||||
use crate::pool::Seek;
|
||||
|
||||
let opts = super::iter_options_default(&self.db);
|
||||
let state = stream::State::new(self, opts);
|
||||
if is_cached(self, from) {
|
||||
return stream::Keys::<'_>::from(state.init_fwd(from.as_ref().into())).boxed();
|
||||
impl super::Map {
|
||||
pub fn keys_from<'a, K, P>(
|
||||
self: &'a Arc<Self>,
|
||||
from: &P,
|
||||
) -> impl Stream<Item = Result<Key<'a, K>>> + Send + use<'a, K, P>
|
||||
where
|
||||
P: Serialize + ?Sized + Debug,
|
||||
K: Deserialize<'a> + Send,
|
||||
{
|
||||
self.keys_from_raw(from).map(result_deserialize_key::<K>)
|
||||
}
|
||||
|
||||
let seek = Seek {
|
||||
map: self.clone(),
|
||||
dir: Direction::Forward,
|
||||
key: Some(from.as_ref().into()),
|
||||
state: crate::pool::into_send_seek(state),
|
||||
res: None,
|
||||
};
|
||||
#[tracing::instrument(skip(self), level = "trace")]
|
||||
pub fn keys_from_raw<P>(
|
||||
self: &Arc<Self>,
|
||||
from: &P,
|
||||
) -> impl Stream<Item = Result<Key<'_>>> + Send + use<'_, P>
|
||||
where
|
||||
P: Serialize + ?Sized + Debug,
|
||||
{
|
||||
let key = serialize_key(from).expect("failed to serialize query key");
|
||||
self.raw_keys_from(&key)
|
||||
}
|
||||
|
||||
self.db
|
||||
.pool
|
||||
.execute_iter(seek)
|
||||
.ok_into::<stream::Keys<'_>>()
|
||||
.into_stream()
|
||||
.try_flatten()
|
||||
.boxed()
|
||||
pub fn keys_raw_from<'a, K, P>(
|
||||
self: &'a Arc<Self>,
|
||||
from: &P,
|
||||
) -> impl Stream<Item = Result<Key<'a, K>>> + Send + use<'a, K, P>
|
||||
where
|
||||
P: AsRef<[u8]> + ?Sized + Debug + Sync,
|
||||
K: Deserialize<'a> + Send,
|
||||
{
|
||||
self.raw_keys_from(from).map(result_deserialize_key::<K>)
|
||||
}
|
||||
|
||||
#[tracing::instrument(skip(self, from), fields(%self), level = "trace")]
|
||||
pub fn raw_keys_from<P>(
|
||||
self: &Arc<Self>,
|
||||
from: &P,
|
||||
) -> impl Stream<Item = Result<Key<'_>>> + Send + use<'_, P>
|
||||
where
|
||||
P: AsRef<[u8]> + ?Sized + Debug,
|
||||
{
|
||||
use crate::pool::Seek;
|
||||
|
||||
let opts = super::iter_options_default(&self.db);
|
||||
let state = stream::State::new(self, opts);
|
||||
if is_cached(self, from) {
|
||||
return stream::Keys::<'_>::from(state.init_fwd(from.as_ref().into())).boxed();
|
||||
}
|
||||
|
||||
let seek = Seek {
|
||||
map: self.clone(),
|
||||
dir: Direction::Forward,
|
||||
key: Some(from.as_ref().into()),
|
||||
state: crate::pool::into_send_seek(state),
|
||||
res: None,
|
||||
};
|
||||
|
||||
self.db
|
||||
.pool
|
||||
.execute_iter(seek)
|
||||
.ok_into::<stream::Keys<'_>>()
|
||||
.into_stream()
|
||||
.try_flatten()
|
||||
.boxed()
|
||||
}
|
||||
}
|
||||
@@ -1,59 +1,57 @@
|
||||
use std::{convert::AsRef, fmt::Debug, sync::Arc};
|
||||
|
||||
use conduwuit::{Result, implement};
|
||||
use conduwuit::Result;
|
||||
use futures::{Stream, StreamExt, TryStreamExt, future};
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use crate::keyval::{Key, result_deserialize_key, serialize_key};
|
||||
|
||||
#[implement(super::Map)]
|
||||
pub fn keys_prefix<'a, K, P>(
|
||||
self: &'a Arc<Self>,
|
||||
prefix: &P,
|
||||
) -> impl Stream<Item = Result<Key<'a, K>>> + Send + use<'a, K, P>
|
||||
where
|
||||
P: Serialize + ?Sized + Debug,
|
||||
K: Deserialize<'a> + Send,
|
||||
{
|
||||
self.keys_prefix_raw(prefix)
|
||||
.map(result_deserialize_key::<K>)
|
||||
}
|
||||
impl super::Map {
|
||||
pub fn keys_prefix<'a, K, P>(
|
||||
self: &'a Arc<Self>,
|
||||
prefix: &P,
|
||||
) -> impl Stream<Item = Result<Key<'a, K>>> + Send + use<'a, K, P>
|
||||
where
|
||||
P: Serialize + ?Sized + Debug,
|
||||
K: Deserialize<'a> + Send,
|
||||
{
|
||||
self.keys_prefix_raw(prefix)
|
||||
.map(result_deserialize_key::<K>)
|
||||
}
|
||||
|
||||
#[implement(super::Map)]
|
||||
#[tracing::instrument(skip(self), level = "trace")]
|
||||
pub fn keys_prefix_raw<P>(
|
||||
self: &Arc<Self>,
|
||||
prefix: &P,
|
||||
) -> impl Stream<Item = Result<Key<'_>>> + Send + use<'_, P>
|
||||
where
|
||||
P: Serialize + ?Sized + Debug,
|
||||
{
|
||||
let key = serialize_key(prefix).expect("failed to serialize query key");
|
||||
self.raw_keys_from(&key)
|
||||
.try_take_while(move |k: &Key<'_>| future::ok(k.starts_with(&key)))
|
||||
}
|
||||
#[tracing::instrument(skip(self), level = "trace")]
|
||||
pub fn keys_prefix_raw<P>(
|
||||
self: &Arc<Self>,
|
||||
prefix: &P,
|
||||
) -> impl Stream<Item = Result<Key<'_>>> + Send + use<'_, P>
|
||||
where
|
||||
P: Serialize + ?Sized + Debug,
|
||||
{
|
||||
let key = serialize_key(prefix).expect("failed to serialize query key");
|
||||
self.raw_keys_from(&key)
|
||||
.try_take_while(move |k: &Key<'_>| future::ok(k.starts_with(&key)))
|
||||
}
|
||||
|
||||
#[implement(super::Map)]
|
||||
pub fn keys_raw_prefix<'a, K, P>(
|
||||
self: &'a Arc<Self>,
|
||||
prefix: &'a P,
|
||||
) -> impl Stream<Item = Result<Key<'a, K>>> + Send + 'a
|
||||
where
|
||||
P: AsRef<[u8]> + ?Sized + Debug + Sync + 'a,
|
||||
K: Deserialize<'a> + Send + 'a,
|
||||
{
|
||||
self.raw_keys_prefix(prefix)
|
||||
.map(result_deserialize_key::<K>)
|
||||
}
|
||||
pub fn keys_raw_prefix<'a, K, P>(
|
||||
self: &'a Arc<Self>,
|
||||
prefix: &'a P,
|
||||
) -> impl Stream<Item = Result<Key<'a, K>>> + Send + 'a
|
||||
where
|
||||
P: AsRef<[u8]> + ?Sized + Debug + Sync + 'a,
|
||||
K: Deserialize<'a> + Send + 'a,
|
||||
{
|
||||
self.raw_keys_prefix(prefix)
|
||||
.map(result_deserialize_key::<K>)
|
||||
}
|
||||
|
||||
#[implement(super::Map)]
|
||||
pub fn raw_keys_prefix<'a, P>(
|
||||
self: &'a Arc<Self>,
|
||||
prefix: &'a P,
|
||||
) -> impl Stream<Item = Result<Key<'a>>> + Send + 'a
|
||||
where
|
||||
P: AsRef<[u8]> + ?Sized + Debug + Sync + 'a,
|
||||
{
|
||||
self.raw_keys_from(prefix)
|
||||
.try_take_while(|k: &Key<'_>| future::ok(k.starts_with(prefix.as_ref())))
|
||||
pub fn raw_keys_prefix<'a, P>(
|
||||
self: &'a Arc<Self>,
|
||||
prefix: &'a P,
|
||||
) -> impl Stream<Item = Result<Key<'a>>> + Send + 'a
|
||||
where
|
||||
P: AsRef<[u8]> + ?Sized + Debug + Sync + 'a,
|
||||
{
|
||||
self.raw_keys_from(prefix)
|
||||
.try_take_while(|k: &Key<'_>| future::ok(k.starts_with(prefix.as_ref())))
|
||||
}
|
||||
}
|
||||
+46
-46
@@ -1,56 +1,56 @@
|
||||
use std::{convert::AsRef, fmt::Debug, io::Write, sync::Arc};
|
||||
|
||||
use conduwuit::{Result, arrayvec::ArrayVec, implement};
|
||||
use conduwuit::{Result, arrayvec::ArrayVec};
|
||||
use futures::Future;
|
||||
use serde::Serialize;
|
||||
|
||||
use crate::{Handle, keyval::KeyBuf, ser};
|
||||
|
||||
/// Fetch a value from the database into cache, returning a reference-handle
|
||||
/// asynchronously. The key is serialized into an allocated buffer to perform
|
||||
/// the query.
|
||||
#[implement(super::Map)]
|
||||
#[inline]
|
||||
pub fn qry<K>(
|
||||
self: &Arc<Self>,
|
||||
key: &K,
|
||||
) -> impl Future<Output = Result<Handle<'_>>> + Send + use<'_, K>
|
||||
where
|
||||
K: Serialize + ?Sized + Debug,
|
||||
{
|
||||
let mut buf = KeyBuf::new();
|
||||
self.bqry(key, &mut buf)
|
||||
}
|
||||
impl super::Map {
|
||||
/// Fetch a value from the database into cache, returning a reference-handle
|
||||
/// asynchronously. The key is serialized into an allocated buffer to
|
||||
/// perform the query.
|
||||
#[inline]
|
||||
pub fn qry<K>(
|
||||
self: &Arc<Self>,
|
||||
key: &K,
|
||||
) -> impl Future<Output = Result<Handle<'_>>> + Send + use<'_, K>
|
||||
where
|
||||
K: Serialize + ?Sized + Debug,
|
||||
{
|
||||
let mut buf = KeyBuf::new();
|
||||
self.bqry(key, &mut buf)
|
||||
}
|
||||
|
||||
/// Fetch a value from the database into cache, returning a reference-handle
|
||||
/// asynchronously. The key is serialized into a fixed-sized buffer to perform
|
||||
/// the query. The maximum size is supplied as const generic parameter.
|
||||
#[implement(super::Map)]
|
||||
#[inline]
|
||||
pub fn aqry<const MAX: usize, K>(
|
||||
self: &Arc<Self>,
|
||||
key: &K,
|
||||
) -> impl Future<Output = Result<Handle<'_>>> + Send + use<'_, MAX, K>
|
||||
where
|
||||
K: Serialize + ?Sized + Debug,
|
||||
{
|
||||
let mut buf = ArrayVec::<u8, MAX>::new();
|
||||
self.bqry(key, &mut buf)
|
||||
}
|
||||
/// Fetch a value from the database into cache, returning a reference-handle
|
||||
/// asynchronously. The key is serialized into a fixed-sized buffer to
|
||||
/// perform the query. The maximum size is supplied as const generic
|
||||
/// parameter.
|
||||
#[inline]
|
||||
pub fn aqry<const MAX: usize, K>(
|
||||
self: &Arc<Self>,
|
||||
key: &K,
|
||||
) -> impl Future<Output = Result<Handle<'_>>> + Send + use<'_, MAX, K>
|
||||
where
|
||||
K: Serialize + ?Sized + Debug,
|
||||
{
|
||||
let mut buf = ArrayVec::<u8, MAX>::new();
|
||||
self.bqry(key, &mut buf)
|
||||
}
|
||||
|
||||
/// Fetch a value from the database into cache, returning a reference-handle
|
||||
/// asynchronously. The key is serialized into a user-supplied Writer.
|
||||
#[implement(super::Map)]
|
||||
#[tracing::instrument(skip(self, buf), level = "trace")]
|
||||
pub fn bqry<K, B>(
|
||||
self: &Arc<Self>,
|
||||
key: &K,
|
||||
buf: &mut B,
|
||||
) -> impl Future<Output = Result<Handle<'_>>> + Send + use<'_, K, B>
|
||||
where
|
||||
K: Serialize + ?Sized + Debug,
|
||||
B: Write + AsRef<[u8]>,
|
||||
{
|
||||
let key = ser::serialize(buf, key).expect("failed to serialize query key");
|
||||
self.get(key)
|
||||
/// Fetch a value from the database into cache, returning a reference-handle
|
||||
/// asynchronously. The key is serialized into a user-supplied Writer.
|
||||
#[tracing::instrument(skip(self, buf), level = "trace")]
|
||||
pub fn bqry<K, B>(
|
||||
self: &Arc<Self>,
|
||||
key: &K,
|
||||
buf: &mut B,
|
||||
) -> impl Future<Output = Result<Handle<'_>>> + Send + use<'_, K, B>
|
||||
where
|
||||
K: Serialize + ?Sized + Debug,
|
||||
B: Write + AsRef<[u8]>,
|
||||
{
|
||||
let key = ser::serialize(buf, key).expect("failed to serialize query key");
|
||||
self.get(key)
|
||||
}
|
||||
}
|
||||
@@ -1,7 +1,7 @@
|
||||
use std::{fmt::Debug, sync::Arc};
|
||||
|
||||
use conduwuit::{
|
||||
Result, implement,
|
||||
Result,
|
||||
utils::{
|
||||
IterStream,
|
||||
stream::{WidebandExt, automatic_amplification, automatic_width},
|
||||
@@ -32,30 +32,31 @@ where
|
||||
}
|
||||
}
|
||||
|
||||
#[implement(super::Map)]
|
||||
#[tracing::instrument(skip(self, keys), level = "trace")]
|
||||
pub(crate) fn qry_batch<'a, S, K>(
|
||||
self: &'a Arc<Self>,
|
||||
keys: S,
|
||||
) -> impl Stream<Item = Result<Handle<'a>>> + Send + 'a
|
||||
where
|
||||
S: Stream<Item = K> + Send + 'a,
|
||||
K: Serialize + Debug + 'a,
|
||||
{
|
||||
use crate::pool::Get;
|
||||
impl super::Map {
|
||||
#[tracing::instrument(skip(self, keys), level = "trace")]
|
||||
pub(crate) fn qry_batch<'a, S, K>(
|
||||
self: &'a Arc<Self>,
|
||||
keys: S,
|
||||
) -> impl Stream<Item = Result<Handle<'a>>> + Send + 'a
|
||||
where
|
||||
S: Stream<Item = K> + Send + 'a,
|
||||
K: Serialize + Debug + 'a,
|
||||
{
|
||||
use crate::pool::Get;
|
||||
|
||||
keys.ready_chunks(automatic_amplification())
|
||||
.widen_then(automatic_width(), |chunk| {
|
||||
let keys = chunk
|
||||
.iter()
|
||||
.map(ser::serialize_to::<KeyBuf, _>)
|
||||
.map(|result| result.expect("failed to serialize query key"))
|
||||
.collect();
|
||||
keys.ready_chunks(automatic_amplification())
|
||||
.widen_then(automatic_width(), |chunk| {
|
||||
let keys = chunk
|
||||
.iter()
|
||||
.map(ser::serialize_to::<KeyBuf, _>)
|
||||
.map(|result| result.expect("failed to serialize query key"))
|
||||
.collect();
|
||||
|
||||
self.db
|
||||
.pool
|
||||
.execute_get(Get { map: self.clone(), key: keys, res: None })
|
||||
})
|
||||
.map_ok(|results| results.into_iter().stream())
|
||||
.try_flatten()
|
||||
self.db
|
||||
.pool
|
||||
.execute_get(Get { map: self.clone(), key: keys, res: None })
|
||||
})
|
||||
.map_ok(|results| results.into_iter().stream())
|
||||
.try_flatten()
|
||||
}
|
||||
}
|
||||
+41
-43
@@ -1,55 +1,53 @@
|
||||
use std::{convert::AsRef, fmt::Debug, io::Write};
|
||||
|
||||
use conduwuit::{arrayvec::ArrayVec, implement};
|
||||
use conduwuit::arrayvec::ArrayVec;
|
||||
use serde::Serialize;
|
||||
|
||||
use crate::{keyval::KeyBuf, ser, util::or_else};
|
||||
|
||||
#[implement(super::Map)]
|
||||
#[inline]
|
||||
pub fn del<K>(&self, key: K)
|
||||
where
|
||||
K: Serialize + Debug,
|
||||
{
|
||||
let mut buf = KeyBuf::new();
|
||||
self.bdel(key, &mut buf);
|
||||
}
|
||||
impl super::Map {
|
||||
#[inline]
|
||||
pub fn del<K>(&self, key: K)
|
||||
where
|
||||
K: Serialize + Debug,
|
||||
{
|
||||
let mut buf = KeyBuf::new();
|
||||
self.bdel(key, &mut buf);
|
||||
}
|
||||
|
||||
#[implement(super::Map)]
|
||||
#[inline]
|
||||
pub fn adel<const MAX: usize, K>(&self, key: K)
|
||||
where
|
||||
K: Serialize + Debug,
|
||||
{
|
||||
let mut buf = ArrayVec::<u8, MAX>::new();
|
||||
self.bdel(key, &mut buf);
|
||||
}
|
||||
#[inline]
|
||||
pub fn adel<const MAX: usize, K>(&self, key: K)
|
||||
where
|
||||
K: Serialize + Debug,
|
||||
{
|
||||
let mut buf = ArrayVec::<u8, MAX>::new();
|
||||
self.bdel(key, &mut buf);
|
||||
}
|
||||
|
||||
#[implement(super::Map)]
|
||||
#[tracing::instrument(skip(self, buf), level = "trace")]
|
||||
pub fn bdel<K, B>(&self, key: K, buf: &mut B)
|
||||
where
|
||||
K: Serialize + Debug,
|
||||
B: Write + AsRef<[u8]>,
|
||||
{
|
||||
let key = ser::serialize(buf, key).expect("failed to serialize deletion key");
|
||||
self.remove(key);
|
||||
}
|
||||
#[tracing::instrument(skip(self, buf), level = "trace")]
|
||||
pub fn bdel<K, B>(&self, key: K, buf: &mut B)
|
||||
where
|
||||
K: Serialize + Debug,
|
||||
B: Write + AsRef<[u8]>,
|
||||
{
|
||||
let key = ser::serialize(buf, key).expect("failed to serialize deletion key");
|
||||
self.remove(key);
|
||||
}
|
||||
|
||||
#[implement(super::Map)]
|
||||
#[tracing::instrument(skip(self, key), fields(%self), level = "trace")]
|
||||
pub fn remove<K>(&self, key: &K)
|
||||
where
|
||||
K: AsRef<[u8]> + ?Sized + Debug,
|
||||
{
|
||||
let write_options = &self.write_options;
|
||||
self.db
|
||||
.db
|
||||
.delete_cf_opt(&self.cf(), key, write_options)
|
||||
.or_else(or_else)
|
||||
.expect("database remove error");
|
||||
#[tracing::instrument(skip(self, key), fields(%self), level = "trace")]
|
||||
pub fn remove<K>(&self, key: &K)
|
||||
where
|
||||
K: AsRef<[u8]> + ?Sized + Debug,
|
||||
{
|
||||
let write_options = &self.write_options;
|
||||
self.db
|
||||
.db
|
||||
.delete_cf_opt(&self.cf(), key, write_options)
|
||||
.or_else(or_else)
|
||||
.expect("database remove error");
|
||||
|
||||
if !self.db.corked() {
|
||||
self.db.flush().expect("database flush error");
|
||||
if !self.db.corked() {
|
||||
self.db.flush().expect("database flush error");
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,6 +1,6 @@
|
||||
use std::sync::Arc;
|
||||
|
||||
use conduwuit::{Result, implement};
|
||||
use conduwuit::Result;
|
||||
use futures::{FutureExt, Stream, StreamExt, TryFutureExt, TryStreamExt};
|
||||
use rocksdb::Direction;
|
||||
use serde::Deserialize;
|
||||
@@ -9,43 +9,43 @@ use tokio::task;
|
||||
use super::rev_stream::is_cached;
|
||||
use crate::{keyval, keyval::Key, stream};
|
||||
|
||||
#[implement(super::Map)]
|
||||
pub fn rev_keys<'a, K>(self: &'a Arc<Self>) -> impl Stream<Item = Result<Key<'a, K>>> + Send
|
||||
where
|
||||
K: Deserialize<'a> + Send,
|
||||
{
|
||||
self.rev_raw_keys().map(keyval::result_deserialize_key::<K>)
|
||||
}
|
||||
|
||||
#[implement(super::Map)]
|
||||
#[tracing::instrument(skip(self), fields(%self), level = "trace")]
|
||||
pub fn rev_raw_keys(self: &Arc<Self>) -> impl Stream<Item = Result<Key<'_>>> + Send {
|
||||
use crate::pool::Seek;
|
||||
|
||||
let opts = super::iter_options_default(&self.db);
|
||||
let state = stream::State::new(self, opts);
|
||||
if is_cached(self) {
|
||||
let state = state.init_rev(None);
|
||||
return task::consume_budget()
|
||||
.map(move |()| stream::KeysRev::<'_>::from(state))
|
||||
.into_stream()
|
||||
.flatten()
|
||||
.boxed();
|
||||
impl super::Map {
|
||||
pub fn rev_keys<'a, K>(self: &'a Arc<Self>) -> impl Stream<Item = Result<Key<'a, K>>> + Send
|
||||
where
|
||||
K: Deserialize<'a> + Send,
|
||||
{
|
||||
self.rev_raw_keys().map(keyval::result_deserialize_key::<K>)
|
||||
}
|
||||
|
||||
let seek = Seek {
|
||||
map: self.clone(),
|
||||
dir: Direction::Reverse,
|
||||
state: crate::pool::into_send_seek(state),
|
||||
key: None,
|
||||
res: None,
|
||||
};
|
||||
#[tracing::instrument(skip(self), fields(%self), level = "trace")]
|
||||
pub fn rev_raw_keys(self: &Arc<Self>) -> impl Stream<Item = Result<Key<'_>>> + Send {
|
||||
use crate::pool::Seek;
|
||||
|
||||
self.db
|
||||
.pool
|
||||
.execute_iter(seek)
|
||||
.ok_into::<stream::KeysRev<'_>>()
|
||||
.into_stream()
|
||||
.try_flatten()
|
||||
.boxed()
|
||||
let opts = super::iter_options_default(&self.db);
|
||||
let state = stream::State::new(self, opts);
|
||||
if is_cached(self) {
|
||||
let state = state.init_rev(None);
|
||||
return task::consume_budget()
|
||||
.map(move |()| stream::KeysRev::<'_>::from(state))
|
||||
.into_stream()
|
||||
.flatten()
|
||||
.boxed();
|
||||
}
|
||||
|
||||
let seek = Seek {
|
||||
map: self.clone(),
|
||||
dir: Direction::Reverse,
|
||||
state: crate::pool::into_send_seek(state),
|
||||
key: None,
|
||||
res: None,
|
||||
};
|
||||
|
||||
self.db
|
||||
.pool
|
||||
.execute_iter(seek)
|
||||
.ok_into::<stream::KeysRev<'_>>()
|
||||
.into_stream()
|
||||
.try_flatten()
|
||||
.boxed()
|
||||
}
|
||||
}
|
||||
@@ -1,6 +1,6 @@
|
||||
use std::{convert::AsRef, fmt::Debug, sync::Arc};
|
||||
|
||||
use conduwuit::{Result, implement};
|
||||
use conduwuit::Result;
|
||||
use futures::{FutureExt, Stream, StreamExt, TryFutureExt, TryStreamExt};
|
||||
use rocksdb::Direction;
|
||||
use serde::{Deserialize, Serialize};
|
||||
@@ -11,75 +11,73 @@ use crate::{
|
||||
stream,
|
||||
};
|
||||
|
||||
#[implement(super::Map)]
|
||||
pub fn rev_keys_from<'a, K, P>(
|
||||
self: &'a Arc<Self>,
|
||||
from: &P,
|
||||
) -> impl Stream<Item = Result<Key<'a, K>>> + Send + use<'a, K, P>
|
||||
where
|
||||
P: Serialize + ?Sized + Debug,
|
||||
K: Deserialize<'a> + Send,
|
||||
{
|
||||
self.rev_keys_from_raw(from)
|
||||
.map(result_deserialize_key::<K>)
|
||||
}
|
||||
|
||||
#[implement(super::Map)]
|
||||
#[tracing::instrument(skip(self), level = "trace")]
|
||||
pub fn rev_keys_from_raw<P>(
|
||||
self: &Arc<Self>,
|
||||
from: &P,
|
||||
) -> impl Stream<Item = Result<Key<'_>>> + Send + use<'_, P>
|
||||
where
|
||||
P: Serialize + ?Sized + Debug,
|
||||
{
|
||||
let key = serialize_key(from).expect("failed to serialize query key");
|
||||
self.rev_raw_keys_from(&key)
|
||||
}
|
||||
|
||||
#[implement(super::Map)]
|
||||
pub fn rev_keys_raw_from<'a, K, P>(
|
||||
self: &'a Arc<Self>,
|
||||
from: &P,
|
||||
) -> impl Stream<Item = Result<Key<'a, K>>> + Send + use<'a, K, P>
|
||||
where
|
||||
P: AsRef<[u8]> + ?Sized + Debug + Sync,
|
||||
K: Deserialize<'a> + Send,
|
||||
{
|
||||
self.rev_raw_keys_from(from)
|
||||
.map(result_deserialize_key::<K>)
|
||||
}
|
||||
|
||||
#[implement(super::Map)]
|
||||
#[tracing::instrument(skip(self, from), fields(%self), level = "trace")]
|
||||
pub fn rev_raw_keys_from<P>(
|
||||
self: &Arc<Self>,
|
||||
from: &P,
|
||||
) -> impl Stream<Item = Result<Key<'_>>> + Send + use<'_, P>
|
||||
where
|
||||
P: AsRef<[u8]> + ?Sized + Debug,
|
||||
{
|
||||
use crate::pool::Seek;
|
||||
|
||||
let opts = super::iter_options_default(&self.db);
|
||||
let state = stream::State::new(self, opts);
|
||||
if is_cached(self, from) {
|
||||
return stream::KeysRev::<'_>::from(state.init_rev(from.as_ref().into())).boxed();
|
||||
impl super::Map {
|
||||
pub fn rev_keys_from<'a, K, P>(
|
||||
self: &'a Arc<Self>,
|
||||
from: &P,
|
||||
) -> impl Stream<Item = Result<Key<'a, K>>> + Send + use<'a, K, P>
|
||||
where
|
||||
P: Serialize + ?Sized + Debug,
|
||||
K: Deserialize<'a> + Send,
|
||||
{
|
||||
self.rev_keys_from_raw(from)
|
||||
.map(result_deserialize_key::<K>)
|
||||
}
|
||||
|
||||
let seek = Seek {
|
||||
map: self.clone(),
|
||||
dir: Direction::Reverse,
|
||||
key: Some(from.as_ref().into()),
|
||||
state: crate::pool::into_send_seek(state),
|
||||
res: None,
|
||||
};
|
||||
#[tracing::instrument(skip(self), level = "trace")]
|
||||
pub fn rev_keys_from_raw<P>(
|
||||
self: &Arc<Self>,
|
||||
from: &P,
|
||||
) -> impl Stream<Item = Result<Key<'_>>> + Send + use<'_, P>
|
||||
where
|
||||
P: Serialize + ?Sized + Debug,
|
||||
{
|
||||
let key = serialize_key(from).expect("failed to serialize query key");
|
||||
self.rev_raw_keys_from(&key)
|
||||
}
|
||||
|
||||
self.db
|
||||
.pool
|
||||
.execute_iter(seek)
|
||||
.ok_into::<stream::KeysRev<'_>>()
|
||||
.into_stream()
|
||||
.try_flatten()
|
||||
.boxed()
|
||||
pub fn rev_keys_raw_from<'a, K, P>(
|
||||
self: &'a Arc<Self>,
|
||||
from: &P,
|
||||
) -> impl Stream<Item = Result<Key<'a, K>>> + Send + use<'a, K, P>
|
||||
where
|
||||
P: AsRef<[u8]> + ?Sized + Debug + Sync,
|
||||
K: Deserialize<'a> + Send,
|
||||
{
|
||||
self.rev_raw_keys_from(from)
|
||||
.map(result_deserialize_key::<K>)
|
||||
}
|
||||
|
||||
#[tracing::instrument(skip(self, from), fields(%self), level = "trace")]
|
||||
pub fn rev_raw_keys_from<P>(
|
||||
self: &Arc<Self>,
|
||||
from: &P,
|
||||
) -> impl Stream<Item = Result<Key<'_>>> + Send + use<'_, P>
|
||||
where
|
||||
P: AsRef<[u8]> + ?Sized + Debug,
|
||||
{
|
||||
use crate::pool::Seek;
|
||||
|
||||
let opts = super::iter_options_default(&self.db);
|
||||
let state = stream::State::new(self, opts);
|
||||
if is_cached(self, from) {
|
||||
return stream::KeysRev::<'_>::from(state.init_rev(from.as_ref().into())).boxed();
|
||||
}
|
||||
|
||||
let seek = Seek {
|
||||
map: self.clone(),
|
||||
dir: Direction::Reverse,
|
||||
key: Some(from.as_ref().into()),
|
||||
state: crate::pool::into_send_seek(state),
|
||||
res: None,
|
||||
};
|
||||
|
||||
self.db
|
||||
.pool
|
||||
.execute_iter(seek)
|
||||
.ok_into::<stream::KeysRev<'_>>()
|
||||
.into_stream()
|
||||
.try_flatten()
|
||||
.boxed()
|
||||
}
|
||||
}
|
||||
@@ -1,59 +1,57 @@
|
||||
use std::{convert::AsRef, fmt::Debug, sync::Arc};
|
||||
|
||||
use conduwuit::{Result, implement};
|
||||
use conduwuit::Result;
|
||||
use futures::{Stream, StreamExt, TryStreamExt, future};
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use crate::keyval::{Key, result_deserialize_key, serialize_key};
|
||||
|
||||
#[implement(super::Map)]
|
||||
pub fn rev_keys_prefix<'a, K, P>(
|
||||
self: &'a Arc<Self>,
|
||||
prefix: &P,
|
||||
) -> impl Stream<Item = Result<Key<'a, K>>> + Send + use<'a, K, P>
|
||||
where
|
||||
P: Serialize + ?Sized + Debug,
|
||||
K: Deserialize<'a> + Send,
|
||||
{
|
||||
self.rev_keys_prefix_raw(prefix)
|
||||
.map(result_deserialize_key::<K>)
|
||||
}
|
||||
impl super::Map {
|
||||
pub fn rev_keys_prefix<'a, K, P>(
|
||||
self: &'a Arc<Self>,
|
||||
prefix: &P,
|
||||
) -> impl Stream<Item = Result<Key<'a, K>>> + Send + use<'a, K, P>
|
||||
where
|
||||
P: Serialize + ?Sized + Debug,
|
||||
K: Deserialize<'a> + Send,
|
||||
{
|
||||
self.rev_keys_prefix_raw(prefix)
|
||||
.map(result_deserialize_key::<K>)
|
||||
}
|
||||
|
||||
#[implement(super::Map)]
|
||||
#[tracing::instrument(skip(self), level = "trace")]
|
||||
pub fn rev_keys_prefix_raw<P>(
|
||||
self: &Arc<Self>,
|
||||
prefix: &P,
|
||||
) -> impl Stream<Item = Result<Key<'_>>> + Send + use<'_, P>
|
||||
where
|
||||
P: Serialize + ?Sized + Debug,
|
||||
{
|
||||
let key = serialize_key(prefix).expect("failed to serialize query key");
|
||||
self.rev_raw_keys_from(&key)
|
||||
.try_take_while(move |k: &Key<'_>| future::ok(k.starts_with(&key)))
|
||||
}
|
||||
#[tracing::instrument(skip(self), level = "trace")]
|
||||
pub fn rev_keys_prefix_raw<P>(
|
||||
self: &Arc<Self>,
|
||||
prefix: &P,
|
||||
) -> impl Stream<Item = Result<Key<'_>>> + Send + use<'_, P>
|
||||
where
|
||||
P: Serialize + ?Sized + Debug,
|
||||
{
|
||||
let key = serialize_key(prefix).expect("failed to serialize query key");
|
||||
self.rev_raw_keys_from(&key)
|
||||
.try_take_while(move |k: &Key<'_>| future::ok(k.starts_with(&key)))
|
||||
}
|
||||
|
||||
#[implement(super::Map)]
|
||||
pub fn rev_keys_raw_prefix<'a, K, P>(
|
||||
self: &'a Arc<Self>,
|
||||
prefix: &'a P,
|
||||
) -> impl Stream<Item = Result<Key<'a, K>>> + Send + 'a
|
||||
where
|
||||
P: AsRef<[u8]> + ?Sized + Debug + Sync + 'a,
|
||||
K: Deserialize<'a> + Send + 'a,
|
||||
{
|
||||
self.rev_raw_keys_prefix(prefix)
|
||||
.map(result_deserialize_key::<K>)
|
||||
}
|
||||
pub fn rev_keys_raw_prefix<'a, K, P>(
|
||||
self: &'a Arc<Self>,
|
||||
prefix: &'a P,
|
||||
) -> impl Stream<Item = Result<Key<'a, K>>> + Send + 'a
|
||||
where
|
||||
P: AsRef<[u8]> + ?Sized + Debug + Sync + 'a,
|
||||
K: Deserialize<'a> + Send + 'a,
|
||||
{
|
||||
self.rev_raw_keys_prefix(prefix)
|
||||
.map(result_deserialize_key::<K>)
|
||||
}
|
||||
|
||||
#[implement(super::Map)]
|
||||
pub fn rev_raw_keys_prefix<'a, P>(
|
||||
self: &'a Arc<Self>,
|
||||
prefix: &'a P,
|
||||
) -> impl Stream<Item = Result<Key<'a>>> + Send + 'a
|
||||
where
|
||||
P: AsRef<[u8]> + ?Sized + Debug + Sync + 'a,
|
||||
{
|
||||
self.rev_raw_keys_from(prefix)
|
||||
.try_take_while(|k: &Key<'_>| future::ok(k.starts_with(prefix.as_ref())))
|
||||
pub fn rev_raw_keys_prefix<'a, P>(
|
||||
self: &'a Arc<Self>,
|
||||
prefix: &'a P,
|
||||
) -> impl Stream<Item = Result<Key<'a>>> + Send + 'a
|
||||
where
|
||||
P: AsRef<[u8]> + ?Sized + Debug + Sync + 'a,
|
||||
{
|
||||
self.rev_raw_keys_from(prefix)
|
||||
.try_take_while(|k: &Key<'_>| future::ok(k.starts_with(prefix.as_ref())))
|
||||
}
|
||||
}
|
||||
@@ -1,6 +1,6 @@
|
||||
use std::sync::Arc;
|
||||
|
||||
use conduwuit::{Result, implement};
|
||||
use conduwuit::Result;
|
||||
use futures::{FutureExt, Stream, StreamExt, TryFutureExt, TryStreamExt};
|
||||
use rocksdb::Direction;
|
||||
use serde::Deserialize;
|
||||
@@ -8,55 +8,55 @@ use tokio::task;
|
||||
|
||||
use crate::{keyval, keyval::KeyVal, stream};
|
||||
|
||||
/// Iterate key-value entries in the map from the end.
|
||||
///
|
||||
/// - Result is deserialized
|
||||
#[implement(super::Map)]
|
||||
pub fn rev_stream<'a, K, V>(
|
||||
self: &'a Arc<Self>,
|
||||
) -> impl Stream<Item = Result<KeyVal<'a, K, V>>> + Send
|
||||
where
|
||||
K: Deserialize<'a> + Send,
|
||||
V: Deserialize<'a> + Send,
|
||||
{
|
||||
self.rev_raw_stream()
|
||||
.map(keyval::result_deserialize::<K, V>)
|
||||
}
|
||||
|
||||
/// Iterate key-value entries in the map from the end.
|
||||
///
|
||||
/// - Result is raw
|
||||
#[implement(super::Map)]
|
||||
#[tracing::instrument(skip(self), fields(%self), level = "trace")]
|
||||
pub fn rev_raw_stream(self: &Arc<Self>) -> impl Stream<Item = Result<KeyVal<'_>>> + Send {
|
||||
use crate::pool::Seek;
|
||||
|
||||
let opts = super::iter_options_default(&self.db);
|
||||
let state = stream::State::new(self, opts);
|
||||
if is_cached(self) {
|
||||
let state = state.init_rev(None);
|
||||
return task::consume_budget()
|
||||
.map(move |()| stream::ItemsRev::<'_>::from(state))
|
||||
.into_stream()
|
||||
.flatten()
|
||||
.boxed();
|
||||
impl super::Map {
|
||||
/// Iterate key-value entries in the map from the end.
|
||||
///
|
||||
/// - Result is deserialized
|
||||
pub fn rev_stream<'a, K, V>(
|
||||
self: &'a Arc<Self>,
|
||||
) -> impl Stream<Item = Result<KeyVal<'a, K, V>>> + Send
|
||||
where
|
||||
K: Deserialize<'a> + Send,
|
||||
V: Deserialize<'a> + Send,
|
||||
{
|
||||
self.rev_raw_stream()
|
||||
.map(keyval::result_deserialize::<K, V>)
|
||||
}
|
||||
|
||||
let seek = Seek {
|
||||
map: self.clone(),
|
||||
dir: Direction::Reverse,
|
||||
state: crate::pool::into_send_seek(state),
|
||||
key: None,
|
||||
res: None,
|
||||
};
|
||||
/// Iterate key-value entries in the map from the end.
|
||||
///
|
||||
/// - Result is raw
|
||||
#[tracing::instrument(skip(self), fields(%self), level = "trace")]
|
||||
pub fn rev_raw_stream(self: &Arc<Self>) -> impl Stream<Item = Result<KeyVal<'_>>> + Send {
|
||||
use crate::pool::Seek;
|
||||
|
||||
self.db
|
||||
.pool
|
||||
.execute_iter(seek)
|
||||
.ok_into::<stream::ItemsRev<'_>>()
|
||||
.into_stream()
|
||||
.try_flatten()
|
||||
.boxed()
|
||||
let opts = super::iter_options_default(&self.db);
|
||||
let state = stream::State::new(self, opts);
|
||||
if is_cached(self) {
|
||||
let state = state.init_rev(None);
|
||||
return task::consume_budget()
|
||||
.map(move |()| stream::ItemsRev::<'_>::from(state))
|
||||
.into_stream()
|
||||
.flatten()
|
||||
.boxed();
|
||||
}
|
||||
|
||||
let seek = Seek {
|
||||
map: self.clone(),
|
||||
dir: Direction::Reverse,
|
||||
state: crate::pool::into_send_seek(state),
|
||||
key: None,
|
||||
res: None,
|
||||
};
|
||||
|
||||
self.db
|
||||
.pool
|
||||
.execute_iter(seek)
|
||||
.ok_into::<stream::ItemsRev<'_>>()
|
||||
.into_stream()
|
||||
.try_flatten()
|
||||
.boxed()
|
||||
}
|
||||
}
|
||||
|
||||
#[tracing::instrument(
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
use std::{convert::AsRef, fmt::Debug, sync::Arc};
|
||||
|
||||
use conduwuit::{Result, implement};
|
||||
use conduwuit::Result;
|
||||
use futures::{FutureExt, Stream, StreamExt, TryFutureExt, TryStreamExt};
|
||||
use rocksdb::Direction;
|
||||
use serde::{Deserialize, Serialize};
|
||||
@@ -12,100 +12,98 @@ use crate::{
|
||||
util::is_incomplete,
|
||||
};
|
||||
|
||||
/// Iterate key-value entries in the map starting from upper-bound.
|
||||
///
|
||||
/// - Query is serialized
|
||||
/// - Result is deserialized
|
||||
#[implement(super::Map)]
|
||||
pub fn rev_stream_from<'a, K, V, P>(
|
||||
self: &'a Arc<Self>,
|
||||
from: &P,
|
||||
) -> impl Stream<Item = Result<KeyVal<'a, K, V>>> + Send + use<'a, K, V, P>
|
||||
where
|
||||
P: Serialize + ?Sized + Debug,
|
||||
K: Deserialize<'a> + Send,
|
||||
V: Deserialize<'a> + Send,
|
||||
{
|
||||
self.rev_stream_from_raw(from)
|
||||
.map(result_deserialize::<K, V>)
|
||||
}
|
||||
|
||||
/// Iterate key-value entries in the map starting from upper-bound.
|
||||
///
|
||||
/// - Query is serialized
|
||||
/// - Result is raw
|
||||
#[implement(super::Map)]
|
||||
#[tracing::instrument(skip(self), level = "trace")]
|
||||
pub fn rev_stream_from_raw<P>(
|
||||
self: &Arc<Self>,
|
||||
from: &P,
|
||||
) -> impl Stream<Item = Result<KeyVal<'_>>> + Send + use<'_, P>
|
||||
where
|
||||
P: Serialize + ?Sized + Debug,
|
||||
{
|
||||
let key = serialize_key(from).expect("failed to serialize query key");
|
||||
self.rev_raw_stream_from(&key)
|
||||
}
|
||||
|
||||
/// Iterate key-value entries in the map starting from upper-bound.
|
||||
///
|
||||
/// - Query is raw
|
||||
/// - Result is deserialized
|
||||
#[implement(super::Map)]
|
||||
pub fn rev_stream_raw_from<'a, K, V, P>(
|
||||
self: &'a Arc<Self>,
|
||||
from: &P,
|
||||
) -> impl Stream<Item = Result<KeyVal<'a, K, V>>> + Send + use<'a, K, V, P>
|
||||
where
|
||||
P: AsRef<[u8]> + ?Sized + Debug + Sync,
|
||||
K: Deserialize<'a> + Send,
|
||||
V: Deserialize<'a> + Send,
|
||||
{
|
||||
self.rev_raw_stream_from(from)
|
||||
.map(result_deserialize::<K, V>)
|
||||
}
|
||||
|
||||
/// Iterate key-value entries in the map starting from upper-bound.
|
||||
///
|
||||
/// - Query is raw
|
||||
/// - Result is raw
|
||||
#[implement(super::Map)]
|
||||
#[tracing::instrument(skip(self, from), fields(%self), level = "trace")]
|
||||
pub fn rev_raw_stream_from<P>(
|
||||
self: &Arc<Self>,
|
||||
from: &P,
|
||||
) -> impl Stream<Item = Result<KeyVal<'_>>> + Send + use<'_, P>
|
||||
where
|
||||
P: AsRef<[u8]> + ?Sized + Debug,
|
||||
{
|
||||
use crate::pool::Seek;
|
||||
|
||||
let opts = super::iter_options_default(&self.db);
|
||||
let state = stream::State::new(self, opts);
|
||||
if is_cached(self, from) {
|
||||
let state = state.init_rev(from.as_ref().into());
|
||||
return task::consume_budget()
|
||||
.map(move |()| stream::ItemsRev::<'_>::from(state))
|
||||
.into_stream()
|
||||
.flatten()
|
||||
.boxed();
|
||||
impl super::Map {
|
||||
/// Iterate key-value entries in the map starting from upper-bound.
|
||||
///
|
||||
/// - Query is serialized
|
||||
/// - Result is deserialized
|
||||
pub fn rev_stream_from<'a, K, V, P>(
|
||||
self: &'a Arc<Self>,
|
||||
from: &P,
|
||||
) -> impl Stream<Item = Result<KeyVal<'a, K, V>>> + Send + use<'a, K, V, P>
|
||||
where
|
||||
P: Serialize + ?Sized + Debug,
|
||||
K: Deserialize<'a> + Send,
|
||||
V: Deserialize<'a> + Send,
|
||||
{
|
||||
self.rev_stream_from_raw(from)
|
||||
.map(result_deserialize::<K, V>)
|
||||
}
|
||||
|
||||
let seek = Seek {
|
||||
map: self.clone(),
|
||||
dir: Direction::Reverse,
|
||||
key: Some(from.as_ref().into()),
|
||||
state: crate::pool::into_send_seek(state),
|
||||
res: None,
|
||||
};
|
||||
/// Iterate key-value entries in the map starting from upper-bound.
|
||||
///
|
||||
/// - Query is serialized
|
||||
/// - Result is raw
|
||||
#[tracing::instrument(skip(self), level = "trace")]
|
||||
pub fn rev_stream_from_raw<P>(
|
||||
self: &Arc<Self>,
|
||||
from: &P,
|
||||
) -> impl Stream<Item = Result<KeyVal<'_>>> + Send + use<'_, P>
|
||||
where
|
||||
P: Serialize + ?Sized + Debug,
|
||||
{
|
||||
let key = serialize_key(from).expect("failed to serialize query key");
|
||||
self.rev_raw_stream_from(&key)
|
||||
}
|
||||
|
||||
self.db
|
||||
.pool
|
||||
.execute_iter(seek)
|
||||
.ok_into::<stream::ItemsRev<'_>>()
|
||||
.into_stream()
|
||||
.try_flatten()
|
||||
.boxed()
|
||||
/// Iterate key-value entries in the map starting from upper-bound.
|
||||
///
|
||||
/// - Query is raw
|
||||
/// - Result is deserialized
|
||||
pub fn rev_stream_raw_from<'a, K, V, P>(
|
||||
self: &'a Arc<Self>,
|
||||
from: &P,
|
||||
) -> impl Stream<Item = Result<KeyVal<'a, K, V>>> + Send + use<'a, K, V, P>
|
||||
where
|
||||
P: AsRef<[u8]> + ?Sized + Debug + Sync,
|
||||
K: Deserialize<'a> + Send,
|
||||
V: Deserialize<'a> + Send,
|
||||
{
|
||||
self.rev_raw_stream_from(from)
|
||||
.map(result_deserialize::<K, V>)
|
||||
}
|
||||
|
||||
/// Iterate key-value entries in the map starting from upper-bound.
|
||||
///
|
||||
/// - Query is raw
|
||||
/// - Result is raw
|
||||
#[tracing::instrument(skip(self, from), fields(%self), level = "trace")]
|
||||
pub fn rev_raw_stream_from<P>(
|
||||
self: &Arc<Self>,
|
||||
from: &P,
|
||||
) -> impl Stream<Item = Result<KeyVal<'_>>> + Send + use<'_, P>
|
||||
where
|
||||
P: AsRef<[u8]> + ?Sized + Debug,
|
||||
{
|
||||
use crate::pool::Seek;
|
||||
|
||||
let opts = super::iter_options_default(&self.db);
|
||||
let state = stream::State::new(self, opts);
|
||||
if is_cached(self, from) {
|
||||
let state = state.init_rev(from.as_ref().into());
|
||||
return task::consume_budget()
|
||||
.map(move |()| stream::ItemsRev::<'_>::from(state))
|
||||
.into_stream()
|
||||
.flatten()
|
||||
.boxed();
|
||||
}
|
||||
|
||||
let seek = Seek {
|
||||
map: self.clone(),
|
||||
dir: Direction::Reverse,
|
||||
key: Some(from.as_ref().into()),
|
||||
state: crate::pool::into_send_seek(state),
|
||||
res: None,
|
||||
};
|
||||
|
||||
self.db
|
||||
.pool
|
||||
.execute_iter(seek)
|
||||
.ok_into::<stream::ItemsRev<'_>>()
|
||||
.into_stream()
|
||||
.try_flatten()
|
||||
.boxed()
|
||||
}
|
||||
}
|
||||
|
||||
#[tracing::instrument(
|
||||
|
||||
@@ -1,77 +1,75 @@
|
||||
use std::{convert::AsRef, fmt::Debug, sync::Arc};
|
||||
|
||||
use conduwuit::{Result, implement};
|
||||
use conduwuit::Result;
|
||||
use futures::{Stream, StreamExt, TryStreamExt, future};
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use crate::keyval::{KeyVal, result_deserialize, serialize_key};
|
||||
|
||||
/// Iterate key-value entries in the map where the key matches a prefix.
|
||||
///
|
||||
/// - Query is serialized
|
||||
/// - Result is deserialized
|
||||
#[implement(super::Map)]
|
||||
pub fn rev_stream_prefix<'a, K, V, P>(
|
||||
self: &'a Arc<Self>,
|
||||
prefix: &P,
|
||||
) -> impl Stream<Item = Result<KeyVal<'a, K, V>>> + Send + use<'a, K, V, P>
|
||||
where
|
||||
P: Serialize + ?Sized + Debug,
|
||||
K: Deserialize<'a> + Send,
|
||||
V: Deserialize<'a> + Send,
|
||||
{
|
||||
self.rev_stream_prefix_raw(prefix)
|
||||
.map(result_deserialize::<K, V>)
|
||||
}
|
||||
impl super::Map {
|
||||
/// Iterate key-value entries in the map where the key matches a prefix.
|
||||
///
|
||||
/// - Query is serialized
|
||||
/// - Result is deserialized
|
||||
pub fn rev_stream_prefix<'a, K, V, P>(
|
||||
self: &'a Arc<Self>,
|
||||
prefix: &P,
|
||||
) -> impl Stream<Item = Result<KeyVal<'a, K, V>>> + Send + use<'a, K, V, P>
|
||||
where
|
||||
P: Serialize + ?Sized + Debug,
|
||||
K: Deserialize<'a> + Send,
|
||||
V: Deserialize<'a> + Send,
|
||||
{
|
||||
self.rev_stream_prefix_raw(prefix)
|
||||
.map(result_deserialize::<K, V>)
|
||||
}
|
||||
|
||||
/// Iterate key-value entries in the map where the key matches a prefix.
|
||||
///
|
||||
/// - Query is serialized
|
||||
/// - Result is raw
|
||||
#[implement(super::Map)]
|
||||
#[tracing::instrument(skip(self), level = "trace")]
|
||||
pub fn rev_stream_prefix_raw<P>(
|
||||
self: &Arc<Self>,
|
||||
prefix: &P,
|
||||
) -> impl Stream<Item = Result<KeyVal<'_>>> + Send + use<'_, P>
|
||||
where
|
||||
P: Serialize + ?Sized + Debug,
|
||||
{
|
||||
let key = serialize_key(prefix).expect("failed to serialize query key");
|
||||
self.rev_raw_stream_from(&key)
|
||||
.try_take_while(move |(k, _): &KeyVal<'_>| future::ok(k.starts_with(&key)))
|
||||
}
|
||||
/// Iterate key-value entries in the map where the key matches a prefix.
|
||||
///
|
||||
/// - Query is serialized
|
||||
/// - Result is raw
|
||||
#[tracing::instrument(skip(self), level = "trace")]
|
||||
pub fn rev_stream_prefix_raw<P>(
|
||||
self: &Arc<Self>,
|
||||
prefix: &P,
|
||||
) -> impl Stream<Item = Result<KeyVal<'_>>> + Send + use<'_, P>
|
||||
where
|
||||
P: Serialize + ?Sized + Debug,
|
||||
{
|
||||
let key = serialize_key(prefix).expect("failed to serialize query key");
|
||||
self.rev_raw_stream_from(&key)
|
||||
.try_take_while(move |(k, _): &KeyVal<'_>| future::ok(k.starts_with(&key)))
|
||||
}
|
||||
|
||||
/// Iterate key-value entries in the map where the key matches a prefix.
|
||||
///
|
||||
/// - Query is raw
|
||||
/// - Result is deserialized
|
||||
#[implement(super::Map)]
|
||||
pub fn rev_stream_raw_prefix<'a, K, V, P>(
|
||||
self: &'a Arc<Self>,
|
||||
prefix: &'a P,
|
||||
) -> impl Stream<Item = Result<KeyVal<'a, K, V>>> + Send + 'a
|
||||
where
|
||||
P: AsRef<[u8]> + ?Sized + Debug + Sync + 'a,
|
||||
K: Deserialize<'a> + Send + 'a,
|
||||
V: Deserialize<'a> + Send + 'a,
|
||||
{
|
||||
self.rev_raw_stream_prefix(prefix)
|
||||
.map(result_deserialize::<K, V>)
|
||||
}
|
||||
/// Iterate key-value entries in the map where the key matches a prefix.
|
||||
///
|
||||
/// - Query is raw
|
||||
/// - Result is deserialized
|
||||
pub fn rev_stream_raw_prefix<'a, K, V, P>(
|
||||
self: &'a Arc<Self>,
|
||||
prefix: &'a P,
|
||||
) -> impl Stream<Item = Result<KeyVal<'a, K, V>>> + Send + 'a
|
||||
where
|
||||
P: AsRef<[u8]> + ?Sized + Debug + Sync + 'a,
|
||||
K: Deserialize<'a> + Send + 'a,
|
||||
V: Deserialize<'a> + Send + 'a,
|
||||
{
|
||||
self.rev_raw_stream_prefix(prefix)
|
||||
.map(result_deserialize::<K, V>)
|
||||
}
|
||||
|
||||
/// Iterate key-value entries in the map where the key matches a prefix.
|
||||
///
|
||||
/// - Query is raw
|
||||
/// - Result is raw
|
||||
#[implement(super::Map)]
|
||||
pub fn rev_raw_stream_prefix<'a, P>(
|
||||
self: &'a Arc<Self>,
|
||||
prefix: &'a P,
|
||||
) -> impl Stream<Item = Result<KeyVal<'a>>> + Send + 'a
|
||||
where
|
||||
P: AsRef<[u8]> + ?Sized + Debug + Sync + 'a,
|
||||
{
|
||||
self.rev_raw_stream_from(prefix)
|
||||
.try_take_while(|(k, _): &KeyVal<'_>| future::ok(k.starts_with(prefix.as_ref())))
|
||||
/// Iterate key-value entries in the map where the key matches a prefix.
|
||||
///
|
||||
/// - Query is raw
|
||||
/// - Result is raw
|
||||
pub fn rev_raw_stream_prefix<'a, P>(
|
||||
self: &'a Arc<Self>,
|
||||
prefix: &'a P,
|
||||
) -> impl Stream<Item = Result<KeyVal<'a>>> + Send + 'a
|
||||
where
|
||||
P: AsRef<[u8]> + ?Sized + Debug + Sync + 'a,
|
||||
{
|
||||
self.rev_raw_stream_from(prefix)
|
||||
.try_take_while(|(k, _): &KeyVal<'_>| future::ok(k.starts_with(prefix.as_ref())))
|
||||
}
|
||||
}
|
||||
+46
-46
@@ -1,6 +1,6 @@
|
||||
use std::sync::Arc;
|
||||
|
||||
use conduwuit::{Result, implement};
|
||||
use conduwuit::Result;
|
||||
use futures::{FutureExt, Stream, StreamExt, TryFutureExt, TryStreamExt};
|
||||
use rocksdb::Direction;
|
||||
use serde::Deserialize;
|
||||
@@ -8,54 +8,54 @@ use tokio::task;
|
||||
|
||||
use crate::{keyval, keyval::KeyVal, stream};
|
||||
|
||||
/// Iterate key-value entries in the map from the beginning.
|
||||
///
|
||||
/// - Result is deserialized
|
||||
#[implement(super::Map)]
|
||||
pub fn stream<'a, K, V>(
|
||||
self: &'a Arc<Self>,
|
||||
) -> impl Stream<Item = Result<KeyVal<'a, K, V>>> + Send
|
||||
where
|
||||
K: Deserialize<'a> + Send,
|
||||
V: Deserialize<'a> + Send,
|
||||
{
|
||||
self.raw_stream().map(keyval::result_deserialize::<K, V>)
|
||||
}
|
||||
|
||||
/// Iterate key-value entries in the map from the beginning.
|
||||
///
|
||||
/// - Result is raw
|
||||
#[implement(super::Map)]
|
||||
#[tracing::instrument(skip(self), fields(%self), level = "trace")]
|
||||
pub fn raw_stream(self: &Arc<Self>) -> impl Stream<Item = Result<KeyVal<'_>>> + Send {
|
||||
use crate::pool::Seek;
|
||||
|
||||
let opts = super::iter_options_default(&self.db);
|
||||
let state = stream::State::new(self, opts);
|
||||
if is_cached(self) {
|
||||
let state = state.init_fwd(None);
|
||||
return task::consume_budget()
|
||||
.map(move |()| stream::Items::<'_>::from(state))
|
||||
.into_stream()
|
||||
.flatten()
|
||||
.boxed();
|
||||
impl super::Map {
|
||||
/// Iterate key-value entries in the map from the beginning.
|
||||
///
|
||||
/// - Result is deserialized
|
||||
pub fn stream<'a, K, V>(
|
||||
self: &'a Arc<Self>,
|
||||
) -> impl Stream<Item = Result<KeyVal<'a, K, V>>> + Send
|
||||
where
|
||||
K: Deserialize<'a> + Send,
|
||||
V: Deserialize<'a> + Send,
|
||||
{
|
||||
self.raw_stream().map(keyval::result_deserialize::<K, V>)
|
||||
}
|
||||
|
||||
let seek = Seek {
|
||||
map: self.clone(),
|
||||
dir: Direction::Forward,
|
||||
state: crate::pool::into_send_seek(state),
|
||||
key: None,
|
||||
res: None,
|
||||
};
|
||||
/// Iterate key-value entries in the map from the beginning.
|
||||
///
|
||||
/// - Result is raw
|
||||
#[tracing::instrument(skip(self), fields(%self), level = "trace")]
|
||||
pub fn raw_stream(self: &Arc<Self>) -> impl Stream<Item = Result<KeyVal<'_>>> + Send {
|
||||
use crate::pool::Seek;
|
||||
|
||||
self.db
|
||||
.pool
|
||||
.execute_iter(seek)
|
||||
.ok_into::<stream::Items<'_>>()
|
||||
.into_stream()
|
||||
.try_flatten()
|
||||
.boxed()
|
||||
let opts = super::iter_options_default(&self.db);
|
||||
let state = stream::State::new(self, opts);
|
||||
if is_cached(self) {
|
||||
let state = state.init_fwd(None);
|
||||
return task::consume_budget()
|
||||
.map(move |()| stream::Items::<'_>::from(state))
|
||||
.into_stream()
|
||||
.flatten()
|
||||
.boxed();
|
||||
}
|
||||
|
||||
let seek = Seek {
|
||||
map: self.clone(),
|
||||
dir: Direction::Forward,
|
||||
state: crate::pool::into_send_seek(state),
|
||||
key: None,
|
||||
res: None,
|
||||
};
|
||||
|
||||
self.db
|
||||
.pool
|
||||
.execute_iter(seek)
|
||||
.ok_into::<stream::Items<'_>>()
|
||||
.into_stream()
|
||||
.try_flatten()
|
||||
.boxed()
|
||||
}
|
||||
}
|
||||
|
||||
#[tracing::instrument(
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
use std::{convert::AsRef, fmt::Debug, sync::Arc};
|
||||
|
||||
use conduwuit::{Result, implement};
|
||||
use conduwuit::Result;
|
||||
use futures::{FutureExt, Stream, StreamExt, TryFutureExt, TryStreamExt};
|
||||
use rocksdb::Direction;
|
||||
use serde::{Deserialize, Serialize};
|
||||
@@ -11,105 +11,103 @@ use crate::{
|
||||
stream,
|
||||
};
|
||||
|
||||
/// Iterate key-value entries in the map starting from lower-bound.
|
||||
///
|
||||
/// - Query is serialized
|
||||
/// - Result is deserialized
|
||||
#[implement(super::Map)]
|
||||
pub fn stream_from<'a, K, V, P>(
|
||||
self: &'a Arc<Self>,
|
||||
from: &P,
|
||||
) -> impl Stream<Item = Result<KeyVal<'a, K, V>>> + Send + use<'a, K, V, P>
|
||||
where
|
||||
P: Serialize + ?Sized + Debug,
|
||||
K: Deserialize<'a> + Send,
|
||||
V: Deserialize<'a> + Send,
|
||||
{
|
||||
self.stream_from_raw(from).map(result_deserialize::<K, V>)
|
||||
}
|
||||
|
||||
/// Iterate key-value entries in the map starting from lower-bound.
|
||||
///
|
||||
/// - Query is serialized
|
||||
/// - Result is raw
|
||||
#[implement(super::Map)]
|
||||
#[tracing::instrument(skip(self), level = "trace")]
|
||||
pub fn stream_from_raw<P>(
|
||||
self: &Arc<Self>,
|
||||
from: &P,
|
||||
) -> impl Stream<Item = Result<KeyVal<'_>>> + Send + use<'_, P>
|
||||
where
|
||||
P: Serialize + ?Sized + Debug,
|
||||
{
|
||||
let key = serialize_key(from).expect("failed to serialize query key");
|
||||
self.raw_stream_from(&key)
|
||||
}
|
||||
|
||||
/// Iterate key-value entries in the map starting from lower-bound.
|
||||
///
|
||||
/// - Query is raw
|
||||
/// - Result is deserialized
|
||||
#[implement(super::Map)]
|
||||
pub fn stream_raw_from<'a, K, V, P>(
|
||||
self: &'a Arc<Self>,
|
||||
from: &P,
|
||||
) -> impl Stream<Item = Result<KeyVal<'a, K, V>>> + Send + use<'a, K, V, P>
|
||||
where
|
||||
P: AsRef<[u8]> + ?Sized + Debug + Sync,
|
||||
K: Deserialize<'a> + Send,
|
||||
V: Deserialize<'a> + Send,
|
||||
{
|
||||
self.raw_stream_from(from).map(result_deserialize::<K, V>)
|
||||
}
|
||||
|
||||
/// Iterate key-value entries in the map starting from lower-bound.
|
||||
///
|
||||
/// - Query is raw
|
||||
/// - Result is raw
|
||||
#[implement(super::Map)]
|
||||
#[tracing::instrument(skip(self, from), fields(%self), level = "trace")]
|
||||
pub fn raw_stream_from<P>(
|
||||
self: &Arc<Self>,
|
||||
from: &P,
|
||||
) -> impl Stream<Item = Result<KeyVal<'_>>> + Send + use<'_, P>
|
||||
where
|
||||
P: AsRef<[u8]> + ?Sized + Debug,
|
||||
{
|
||||
use crate::pool::Seek;
|
||||
|
||||
let opts = super::iter_options_default(&self.db);
|
||||
let state = stream::State::new(self, opts);
|
||||
if is_cached(self, from) {
|
||||
let state = state.init_fwd(from.as_ref().into());
|
||||
return task::consume_budget()
|
||||
.map(move |()| stream::Items::<'_>::from(state))
|
||||
.into_stream()
|
||||
.flatten()
|
||||
.boxed();
|
||||
impl super::Map {
|
||||
/// Iterate key-value entries in the map starting from lower-bound.
|
||||
///
|
||||
/// - Query is serialized
|
||||
/// - Result is deserialized
|
||||
pub fn stream_from<'a, K, V, P>(
|
||||
self: &'a Arc<Self>,
|
||||
from: &P,
|
||||
) -> impl Stream<Item = Result<KeyVal<'a, K, V>>> + Send + use<'a, K, V, P>
|
||||
where
|
||||
P: Serialize + ?Sized + Debug,
|
||||
K: Deserialize<'a> + Send,
|
||||
V: Deserialize<'a> + Send,
|
||||
{
|
||||
self.stream_from_raw(from).map(result_deserialize::<K, V>)
|
||||
}
|
||||
|
||||
let seek = Seek {
|
||||
map: self.clone(),
|
||||
dir: Direction::Forward,
|
||||
key: Some(from.as_ref().into()),
|
||||
state: crate::pool::into_send_seek(state),
|
||||
res: None,
|
||||
};
|
||||
/// Iterate key-value entries in the map starting from lower-bound.
|
||||
///
|
||||
/// - Query is serialized
|
||||
/// - Result is raw
|
||||
#[tracing::instrument(skip(self), level = "trace")]
|
||||
pub fn stream_from_raw<P>(
|
||||
self: &Arc<Self>,
|
||||
from: &P,
|
||||
) -> impl Stream<Item = Result<KeyVal<'_>>> + Send + use<'_, P>
|
||||
where
|
||||
P: Serialize + ?Sized + Debug,
|
||||
{
|
||||
let key = serialize_key(from).expect("failed to serialize query key");
|
||||
self.raw_stream_from(&key)
|
||||
}
|
||||
|
||||
self.db
|
||||
.pool
|
||||
.execute_iter(seek)
|
||||
.ok_into::<stream::Items<'_>>()
|
||||
.into_stream()
|
||||
.try_flatten()
|
||||
.boxed()
|
||||
/// Iterate key-value entries in the map starting from lower-bound.
|
||||
///
|
||||
/// - Query is raw
|
||||
/// - Result is deserialized
|
||||
pub fn stream_raw_from<'a, K, V, P>(
|
||||
self: &'a Arc<Self>,
|
||||
from: &P,
|
||||
) -> impl Stream<Item = Result<KeyVal<'a, K, V>>> + Send + use<'a, K, V, P>
|
||||
where
|
||||
P: AsRef<[u8]> + ?Sized + Debug + Sync,
|
||||
K: Deserialize<'a> + Send,
|
||||
V: Deserialize<'a> + Send,
|
||||
{
|
||||
self.raw_stream_from(from).map(result_deserialize::<K, V>)
|
||||
}
|
||||
|
||||
/// Iterate key-value entries in the map starting from lower-bound.
|
||||
///
|
||||
/// - Query is raw
|
||||
/// - Result is raw
|
||||
#[tracing::instrument(skip(self, from), fields(%self), level = "trace")]
|
||||
pub fn raw_stream_from<P>(
|
||||
self: &Arc<Self>,
|
||||
from: &P,
|
||||
) -> impl Stream<Item = Result<KeyVal<'_>>> + Send + use<'_, P>
|
||||
where
|
||||
P: AsRef<[u8]> + ?Sized + Debug,
|
||||
{
|
||||
use crate::pool::Seek;
|
||||
|
||||
let opts = super::iter_options_default(&self.db);
|
||||
let state = stream::State::new(self, opts);
|
||||
if is_cached(self, from) {
|
||||
let state = state.init_fwd(from.as_ref().into());
|
||||
return task::consume_budget()
|
||||
.map(move |()| stream::Items::<'_>::from(state))
|
||||
.into_stream()
|
||||
.flatten()
|
||||
.boxed();
|
||||
}
|
||||
|
||||
let seek = Seek {
|
||||
map: self.clone(),
|
||||
dir: Direction::Forward,
|
||||
key: Some(from.as_ref().into()),
|
||||
state: crate::pool::into_send_seek(state),
|
||||
res: None,
|
||||
};
|
||||
|
||||
self.db
|
||||
.pool
|
||||
.execute_iter(seek)
|
||||
.ok_into::<stream::Items<'_>>()
|
||||
.into_stream()
|
||||
.try_flatten()
|
||||
.boxed()
|
||||
}
|
||||
}
|
||||
|
||||
#[tracing::instrument(
|
||||
name = "cached",
|
||||
level = "trace",
|
||||
skip(map, from),
|
||||
fields(%map),
|
||||
name = "cached",
|
||||
level = "trace",
|
||||
skip(map, from),
|
||||
fields(%map),
|
||||
)]
|
||||
pub(super) fn is_cached<P>(map: &Arc<super::Map>, from: &P) -> bool
|
||||
where
|
||||
|
||||
@@ -1,77 +1,75 @@
|
||||
use std::{convert::AsRef, fmt::Debug, sync::Arc};
|
||||
|
||||
use conduwuit::{Result, implement};
|
||||
use conduwuit::Result;
|
||||
use futures::{Stream, StreamExt, TryStreamExt, future};
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use crate::keyval::{KeyVal, result_deserialize, serialize_key};
|
||||
|
||||
/// Iterate key-value entries in the map where the key matches a prefix.
|
||||
///
|
||||
/// - Query is serialized
|
||||
/// - Result is deserialized
|
||||
#[implement(super::Map)]
|
||||
pub fn stream_prefix<'a, K, V, P>(
|
||||
self: &'a Arc<Self>,
|
||||
prefix: &P,
|
||||
) -> impl Stream<Item = Result<KeyVal<'a, K, V>>> + Send + use<'a, K, V, P>
|
||||
where
|
||||
P: Serialize + ?Sized + Debug,
|
||||
K: Deserialize<'a> + Send,
|
||||
V: Deserialize<'a> + Send,
|
||||
{
|
||||
self.stream_prefix_raw(prefix)
|
||||
.map(result_deserialize::<K, V>)
|
||||
}
|
||||
impl super::Map {
|
||||
/// Iterate key-value entries in the map where the key matches a prefix.
|
||||
///
|
||||
/// - Query is serialized
|
||||
/// - Result is deserialized
|
||||
pub fn stream_prefix<'a, K, V, P>(
|
||||
self: &'a Arc<Self>,
|
||||
prefix: &P,
|
||||
) -> impl Stream<Item = Result<KeyVal<'a, K, V>>> + Send + use<'a, K, V, P>
|
||||
where
|
||||
P: Serialize + ?Sized + Debug,
|
||||
K: Deserialize<'a> + Send,
|
||||
V: Deserialize<'a> + Send,
|
||||
{
|
||||
self.stream_prefix_raw(prefix)
|
||||
.map(result_deserialize::<K, V>)
|
||||
}
|
||||
|
||||
/// Iterate key-value entries in the map where the key matches a prefix.
|
||||
///
|
||||
/// - Query is serialized
|
||||
/// - Result is raw
|
||||
#[implement(super::Map)]
|
||||
#[tracing::instrument(skip(self), level = "trace")]
|
||||
pub fn stream_prefix_raw<P>(
|
||||
self: &Arc<Self>,
|
||||
prefix: &P,
|
||||
) -> impl Stream<Item = Result<KeyVal<'_>>> + Send + use<'_, P>
|
||||
where
|
||||
P: Serialize + ?Sized + Debug,
|
||||
{
|
||||
let key = serialize_key(prefix).expect("failed to serialize query key");
|
||||
self.raw_stream_from(&key)
|
||||
.try_take_while(move |(k, _): &KeyVal<'_>| future::ok(k.starts_with(&key)))
|
||||
}
|
||||
/// Iterate key-value entries in the map where the key matches a prefix.
|
||||
///
|
||||
/// - Query is serialized
|
||||
/// - Result is raw
|
||||
#[tracing::instrument(skip(self), level = "trace")]
|
||||
pub fn stream_prefix_raw<P>(
|
||||
self: &Arc<Self>,
|
||||
prefix: &P,
|
||||
) -> impl Stream<Item = Result<KeyVal<'_>>> + Send + use<'_, P>
|
||||
where
|
||||
P: Serialize + ?Sized + Debug,
|
||||
{
|
||||
let key = serialize_key(prefix).expect("failed to serialize query key");
|
||||
self.raw_stream_from(&key)
|
||||
.try_take_while(move |(k, _): &KeyVal<'_>| future::ok(k.starts_with(&key)))
|
||||
}
|
||||
|
||||
/// Iterate key-value entries in the map where the key matches a prefix.
|
||||
///
|
||||
/// - Query is raw
|
||||
/// - Result is deserialized
|
||||
#[implement(super::Map)]
|
||||
pub fn stream_raw_prefix<'a, K, V, P>(
|
||||
self: &'a Arc<Self>,
|
||||
prefix: &'a P,
|
||||
) -> impl Stream<Item = Result<KeyVal<'a, K, V>>> + Send + 'a
|
||||
where
|
||||
P: AsRef<[u8]> + ?Sized + Debug + Sync + 'a,
|
||||
K: Deserialize<'a> + Send + 'a,
|
||||
V: Deserialize<'a> + Send + 'a,
|
||||
{
|
||||
self.raw_stream_prefix(prefix)
|
||||
.map(result_deserialize::<K, V>)
|
||||
}
|
||||
/// Iterate key-value entries in the map where the key matches a prefix.
|
||||
///
|
||||
/// - Query is raw
|
||||
/// - Result is deserialized
|
||||
pub fn stream_raw_prefix<'a, K, V, P>(
|
||||
self: &'a Arc<Self>,
|
||||
prefix: &'a P,
|
||||
) -> impl Stream<Item = Result<KeyVal<'a, K, V>>> + Send + 'a
|
||||
where
|
||||
P: AsRef<[u8]> + ?Sized + Debug + Sync + 'a,
|
||||
K: Deserialize<'a> + Send + 'a,
|
||||
V: Deserialize<'a> + Send + 'a,
|
||||
{
|
||||
self.raw_stream_prefix(prefix)
|
||||
.map(result_deserialize::<K, V>)
|
||||
}
|
||||
|
||||
/// Iterate key-value entries in the map where the key matches a prefix.
|
||||
///
|
||||
/// - Query is raw
|
||||
/// - Result is raw
|
||||
#[implement(super::Map)]
|
||||
pub fn raw_stream_prefix<'a, P>(
|
||||
self: &'a Arc<Self>,
|
||||
prefix: &'a P,
|
||||
) -> impl Stream<Item = Result<KeyVal<'a>>> + Send + 'a
|
||||
where
|
||||
P: AsRef<[u8]> + ?Sized + Debug + Sync + 'a,
|
||||
{
|
||||
self.raw_stream_from(prefix)
|
||||
.try_take_while(|(k, _): &KeyVal<'_>| future::ok(k.starts_with(prefix.as_ref())))
|
||||
/// Iterate key-value entries in the map where the key matches a prefix.
|
||||
///
|
||||
/// - Query is raw
|
||||
/// - Result is raw
|
||||
pub fn raw_stream_prefix<'a, P>(
|
||||
self: &'a Arc<Self>,
|
||||
prefix: &'a P,
|
||||
) -> impl Stream<Item = Result<KeyVal<'a>>> + Send + 'a
|
||||
where
|
||||
P: AsRef<[u8]> + ?Sized + Debug + Sync + 'a,
|
||||
{
|
||||
self.raw_stream_from(prefix)
|
||||
.try_take_while(|(k, _): &KeyVal<'_>| future::ok(k.starts_with(prefix.as_ref())))
|
||||
}
|
||||
}
|
||||
@@ -187,6 +187,10 @@ pub(super) static MAPS: &[Descriptor] = &[
|
||||
val_size_hint: Some(8),
|
||||
..descriptor::RANDOM_SMALL
|
||||
},
|
||||
Descriptor {
|
||||
name: "roomid_mindepth",
|
||||
..descriptor::RANDOM_SMALL
|
||||
},
|
||||
Descriptor {
|
||||
name: "roomserverids",
|
||||
..descriptor::RANDOM_SMALL
|
||||
|
||||
+322
-332
@@ -12,7 +12,7 @@ use std::{
|
||||
|
||||
use async_channel::{QueueStrategy, Receiver, RecvError, Sender};
|
||||
use conduwuit::{
|
||||
Error, Result, Server, SyncMutex, debug, err, error, implement,
|
||||
Error, Result, Server, SyncMutex, debug, err, error,
|
||||
result::DebugInspect,
|
||||
smallvec::SmallVec,
|
||||
trace,
|
||||
@@ -72,29 +72,332 @@ const BATCH_INLINE: usize = 1;
|
||||
const WORKER_STACK_SIZE: usize = 1_048_576;
|
||||
const WORKER_NAME: &str = "conduwuit:db";
|
||||
|
||||
#[implement(Pool)]
|
||||
pub(crate) fn new(server: &Arc<Server>) -> Result<Arc<Self>> {
|
||||
const CHAN_SCHED: (QueueStrategy, QueueStrategy) = (QueueStrategy::Fifo, QueueStrategy::Lifo);
|
||||
impl Pool {
|
||||
pub(crate) fn new(server: &Arc<Server>) -> Result<Arc<Self>> {
|
||||
const CHAN_SCHED: (QueueStrategy, QueueStrategy) =
|
||||
(QueueStrategy::Fifo, QueueStrategy::Lifo);
|
||||
|
||||
let (total_workers, queue_sizes, topology) = configure(server);
|
||||
let (total_workers, queue_sizes, topology) = configure(server);
|
||||
|
||||
let (senders, receivers): (Vec<_>, Vec<_>) = queue_sizes
|
||||
.into_iter()
|
||||
.map(|cap| async_channel::bounded_with_queue_strategy(cap, CHAN_SCHED))
|
||||
.unzip();
|
||||
let (senders, receivers): (Vec<_>, Vec<_>) = queue_sizes
|
||||
.into_iter()
|
||||
.map(|cap| async_channel::bounded_with_queue_strategy(cap, CHAN_SCHED))
|
||||
.unzip();
|
||||
|
||||
let pool = Arc::new(Self {
|
||||
server: server.clone(),
|
||||
queues: senders,
|
||||
workers: Vec::new().into(),
|
||||
topology,
|
||||
busy: AtomicUsize::default(),
|
||||
queued_max: AtomicUsize::default(),
|
||||
});
|
||||
let pool = Arc::new(Self {
|
||||
server: server.clone(),
|
||||
queues: senders,
|
||||
workers: Vec::new().into(),
|
||||
topology,
|
||||
busy: AtomicUsize::default(),
|
||||
queued_max: AtomicUsize::default(),
|
||||
});
|
||||
|
||||
pool.spawn_until(&receivers, total_workers)?;
|
||||
pool.spawn_until(&receivers, total_workers)?;
|
||||
|
||||
Ok(pool)
|
||||
Ok(pool)
|
||||
}
|
||||
|
||||
#[tracing::instrument(skip_all, level = "debug")]
|
||||
pub(crate) fn close(&self) {
|
||||
let workers = take(&mut *self.workers.lock());
|
||||
|
||||
let senders = self.queues.iter().map(Sender::sender_count).sum::<usize>();
|
||||
|
||||
let receivers = self
|
||||
.queues
|
||||
.iter()
|
||||
.map(Sender::receiver_count)
|
||||
.sum::<usize>();
|
||||
|
||||
for queue in &self.queues {
|
||||
queue.close();
|
||||
}
|
||||
|
||||
if workers.is_empty() {
|
||||
return;
|
||||
}
|
||||
|
||||
debug!(
|
||||
senders,
|
||||
receivers,
|
||||
queues = self.queues.len(),
|
||||
workers = workers.len(),
|
||||
"Closing pool. Waiting for workers to join..."
|
||||
);
|
||||
|
||||
workers
|
||||
.into_iter()
|
||||
.map(JoinHandle::join)
|
||||
.map(|result| result.map_err(Error::from_panic))
|
||||
.enumerate()
|
||||
.for_each(|(id, result)| match result {
|
||||
| Ok(()) => trace!(%id, "worker joined"),
|
||||
| Err(error) => error!(%id, "worker joined with error: {error}"),
|
||||
});
|
||||
}
|
||||
|
||||
fn spawn_until(self: &Arc<Self>, recv: &[Receiver<Cmd>], count: usize) -> Result {
|
||||
let mut workers = self.workers.lock();
|
||||
while workers.len() < count {
|
||||
self.clone().spawn_one(&mut workers, recv)?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tracing::instrument(
|
||||
name = "spawn",
|
||||
level = "trace",
|
||||
skip_all,
|
||||
fields(id = %workers.len())
|
||||
)]
|
||||
fn spawn_one(
|
||||
self: Arc<Self>,
|
||||
workers: &mut Vec<JoinHandle<()>>,
|
||||
recv: &[Receiver<Cmd>],
|
||||
) -> Result {
|
||||
debug_assert!(!self.queues.is_empty(), "Must have at least one queue");
|
||||
debug_assert!(!recv.is_empty(), "Must have at least one receiver");
|
||||
|
||||
let id = workers.len();
|
||||
let group = id.overflowing_rem(self.queues.len()).0;
|
||||
let recv = recv[group].clone();
|
||||
|
||||
let handle = thread::Builder::new()
|
||||
.name(WORKER_NAME.into())
|
||||
.stack_size(WORKER_STACK_SIZE)
|
||||
.spawn(move || self.worker(id, &recv))?;
|
||||
|
||||
workers.push(handle);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tracing::instrument(level = "trace", name = "get", skip(self, cmd))]
|
||||
pub(crate) async fn execute_get(self: &Arc<Self>, mut cmd: Get) -> Result<BatchResult<'_>> {
|
||||
let (send, recv) = oneshot::channel();
|
||||
_ = cmd.res.insert(send);
|
||||
|
||||
let queue = self.select_queue();
|
||||
self.execute(queue, Cmd::Get(cmd))
|
||||
.and_then(move |()| {
|
||||
recv.map_ok(into_recv_get)
|
||||
.map_err(|e| err!(error!("recv failed {e:?}")))
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
#[tracing::instrument(level = "trace", name = "iter", skip(self, cmd))]
|
||||
pub(crate) async fn execute_iter(
|
||||
self: &Arc<Self>,
|
||||
mut cmd: Seek,
|
||||
) -> Result<stream::State<'_>> {
|
||||
let (send, recv) = oneshot::channel();
|
||||
_ = cmd.res.insert(send);
|
||||
|
||||
let queue = self.select_queue();
|
||||
self.execute(queue, Cmd::Iter(cmd))
|
||||
.and_then(|()| {
|
||||
recv.map_ok(into_recv_seek)
|
||||
.map_err(|e| err!(error!("recv failed {e:?}")))
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
fn select_queue(&self) -> &Sender<Cmd> {
|
||||
let core_id = get_affinity().next().unwrap_or(0);
|
||||
let chan_id = self.topology[core_id];
|
||||
self.queues.get(chan_id).unwrap_or_else(|| &self.queues[0])
|
||||
}
|
||||
|
||||
#[tracing::instrument(
|
||||
level = "trace",
|
||||
name = "execute",
|
||||
skip(self, cmd),
|
||||
fields(
|
||||
task = ?tokio::task::try_id(),
|
||||
receivers = queue.receiver_count(),
|
||||
queued = queue.len(),
|
||||
queued_max = self.queued_max.load(Ordering::Relaxed),
|
||||
),
|
||||
)]
|
||||
async fn execute(&self, queue: &Sender<Cmd>, cmd: Cmd) -> Result {
|
||||
if cfg!(debug_assertions) {
|
||||
self.queued_max.fetch_max(queue.len(), Ordering::Relaxed);
|
||||
}
|
||||
|
||||
queue
|
||||
.send(cmd)
|
||||
.await
|
||||
.map_err(|e| err!(error!("send failed {e:?}")))
|
||||
}
|
||||
|
||||
#[tracing::instrument(
|
||||
parent = None,
|
||||
level = "debug",
|
||||
skip(self, recv),
|
||||
fields(
|
||||
tid = ?thread::current().id(),
|
||||
),
|
||||
)]
|
||||
fn worker(self: Arc<Self>, id: usize, recv: &Receiver<Cmd>) {
|
||||
self.worker_init(id);
|
||||
self.worker_loop(recv);
|
||||
}
|
||||
|
||||
fn worker_init(&self, id: usize) {
|
||||
let group = id.overflowing_rem(self.queues.len()).0;
|
||||
let affinity = self
|
||||
.topology
|
||||
.iter()
|
||||
.enumerate()
|
||||
.filter(|_| self.queues.len() > 1)
|
||||
.filter(|_| self.server.config.db_pool_affinity)
|
||||
.filter_map(|(core_id, &queue_id)| (group == queue_id).then_some(core_id))
|
||||
.filter_map(nth_core_available);
|
||||
|
||||
// affinity is empty (no-op) if there's only one queue
|
||||
set_affinity(affinity.clone());
|
||||
|
||||
#[cfg(all(not(target_env = "msvc"), feature = "jemalloc"))]
|
||||
if affinity.clone().count() == 1 && conduwuit::alloc::je::is_affine_arena() {
|
||||
use conduwuit::{
|
||||
alloc::je::this_thread::{arena_id, set_arena},
|
||||
result::LogDebugErr,
|
||||
};
|
||||
|
||||
let id = affinity.clone().next().expect("at least one id");
|
||||
|
||||
if let Ok(arena) = arena_id() {
|
||||
if arena != id {
|
||||
set_arena(id).log_debug_err().ok();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
debug!(
|
||||
%group,
|
||||
affinity = ?affinity.collect::<Vec<_>>(),
|
||||
"worker ready"
|
||||
);
|
||||
}
|
||||
|
||||
fn worker_loop(self: &Arc<Self>, recv: &Receiver<Cmd>) {
|
||||
// initial +1 needed prior to entering wait
|
||||
self.busy.fetch_add(1, Ordering::Relaxed);
|
||||
|
||||
while let Ok(cmd) = self.worker_wait(recv) {
|
||||
Self::worker_handle(cmd);
|
||||
}
|
||||
}
|
||||
|
||||
#[tracing::instrument(
|
||||
name = "wait",
|
||||
level = "trace",
|
||||
skip_all,
|
||||
fields(
|
||||
receivers = recv.receiver_count(),
|
||||
queued = recv.len(),
|
||||
busy = self.busy.fetch_sub(1, Ordering::Relaxed) - 1,
|
||||
),
|
||||
)]
|
||||
fn worker_wait(self: &Arc<Self>, recv: &Receiver<Cmd>) -> Result<Cmd, RecvError> {
|
||||
recv.recv_blocking().debug_inspect(|_| {
|
||||
self.busy.fetch_add(1, Ordering::Relaxed);
|
||||
})
|
||||
}
|
||||
|
||||
fn worker_handle(cmd: Cmd) {
|
||||
match cmd {
|
||||
| Cmd::Get(cmd) if cmd.key.len() == 1 => Self::handle_get(cmd),
|
||||
| Cmd::Get(cmd) => Self::handle_batch(cmd),
|
||||
| Cmd::Iter(cmd) => Self::handle_iter(cmd),
|
||||
}
|
||||
}
|
||||
|
||||
#[tracing::instrument(
|
||||
name = "iter",
|
||||
level = "trace",
|
||||
skip_all,
|
||||
fields(%cmd.map),
|
||||
)]
|
||||
fn handle_iter(mut cmd: Seek) {
|
||||
let chan = cmd.res.take().expect("missing result channel");
|
||||
|
||||
if chan.is_canceled() {
|
||||
return;
|
||||
}
|
||||
|
||||
let from = cmd.key.as_deref();
|
||||
|
||||
let result = match cmd.dir {
|
||||
| Direction::Forward => cmd.state.init_fwd(from),
|
||||
| Direction::Reverse => cmd.state.init_rev(from),
|
||||
};
|
||||
|
||||
let chan_result = chan.send(into_send_seek(result));
|
||||
|
||||
let _chan_sent = chan_result.is_ok();
|
||||
}
|
||||
|
||||
#[tracing::instrument(
|
||||
name = "batch",
|
||||
level = "trace",
|
||||
skip_all,
|
||||
fields(
|
||||
%cmd.map,
|
||||
keys = %cmd.key.len(),
|
||||
),
|
||||
)]
|
||||
fn handle_batch(mut cmd: Get) {
|
||||
debug_assert!(cmd.key.len() > 1, "should have more than one key");
|
||||
debug_assert!(!cmd.key.iter().any(SmallVec::is_empty), "querying for empty key");
|
||||
|
||||
let chan = cmd.res.take().expect("missing result channel");
|
||||
|
||||
if chan.is_canceled() {
|
||||
return;
|
||||
}
|
||||
|
||||
let keys = cmd.key.iter();
|
||||
|
||||
let result: SmallVec<_> = cmd.map.get_batch_blocking(keys).collect();
|
||||
|
||||
let chan_result = chan.send(into_send_get(result));
|
||||
|
||||
let _chan_sent = chan_result.is_ok();
|
||||
}
|
||||
|
||||
#[tracing::instrument(
|
||||
name = "get",
|
||||
level = "trace",
|
||||
skip_all,
|
||||
fields(%cmd.map),
|
||||
)]
|
||||
fn handle_get(mut cmd: Get) {
|
||||
debug_assert!(!cmd.key[0].is_empty(), "querying for empty key");
|
||||
|
||||
// Obtain the result channel.
|
||||
let chan = cmd.res.take().expect("missing result channel");
|
||||
|
||||
// It is worth checking if the future was dropped while the command was queued
|
||||
// so we can bail without paying for any query.
|
||||
if chan.is_canceled() {
|
||||
return;
|
||||
}
|
||||
|
||||
// Perform the actual database query. We reuse our database::Map interface but
|
||||
// limited to the blocking calls, rather than creating another surface directly
|
||||
// with rocksdb here.
|
||||
let result = cmd.map.get_blocking(&cmd.key[0]);
|
||||
|
||||
// Send the result back to the submitter.
|
||||
let chan_result = chan.send(into_send_get([result].into()));
|
||||
|
||||
// If the future was dropped during the query this will fail acceptably.
|
||||
let _chan_sent = chan_result.is_ok();
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for Pool {
|
||||
@@ -112,319 +415,6 @@ impl Drop for Pool {
|
||||
}
|
||||
}
|
||||
|
||||
#[implement(Pool)]
|
||||
#[tracing::instrument(skip_all, level = "debug")]
|
||||
pub(crate) fn close(&self) {
|
||||
let workers = take(&mut *self.workers.lock());
|
||||
|
||||
let senders = self.queues.iter().map(Sender::sender_count).sum::<usize>();
|
||||
|
||||
let receivers = self
|
||||
.queues
|
||||
.iter()
|
||||
.map(Sender::receiver_count)
|
||||
.sum::<usize>();
|
||||
|
||||
for queue in &self.queues {
|
||||
queue.close();
|
||||
}
|
||||
|
||||
if workers.is_empty() {
|
||||
return;
|
||||
}
|
||||
|
||||
debug!(
|
||||
senders,
|
||||
receivers,
|
||||
queues = self.queues.len(),
|
||||
workers = workers.len(),
|
||||
"Closing pool. Waiting for workers to join..."
|
||||
);
|
||||
|
||||
workers
|
||||
.into_iter()
|
||||
.map(JoinHandle::join)
|
||||
.map(|result| result.map_err(Error::from_panic))
|
||||
.enumerate()
|
||||
.for_each(|(id, result)| match result {
|
||||
| Ok(()) => trace!(%id, "worker joined"),
|
||||
| Err(error) => error!(%id, "worker joined with error: {error}"),
|
||||
});
|
||||
}
|
||||
|
||||
#[implement(Pool)]
|
||||
fn spawn_until(self: &Arc<Self>, recv: &[Receiver<Cmd>], count: usize) -> Result {
|
||||
let mut workers = self.workers.lock();
|
||||
while workers.len() < count {
|
||||
self.clone().spawn_one(&mut workers, recv)?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[implement(Pool)]
|
||||
#[tracing::instrument(
|
||||
name = "spawn",
|
||||
level = "trace",
|
||||
skip_all,
|
||||
fields(id = %workers.len())
|
||||
)]
|
||||
fn spawn_one(
|
||||
self: Arc<Self>,
|
||||
workers: &mut Vec<JoinHandle<()>>,
|
||||
recv: &[Receiver<Cmd>],
|
||||
) -> Result {
|
||||
debug_assert!(!self.queues.is_empty(), "Must have at least one queue");
|
||||
debug_assert!(!recv.is_empty(), "Must have at least one receiver");
|
||||
|
||||
let id = workers.len();
|
||||
let group = id.overflowing_rem(self.queues.len()).0;
|
||||
let recv = recv[group].clone();
|
||||
|
||||
let handle = thread::Builder::new()
|
||||
.name(WORKER_NAME.into())
|
||||
.stack_size(WORKER_STACK_SIZE)
|
||||
.spawn(move || self.worker(id, &recv))?;
|
||||
|
||||
workers.push(handle);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[implement(Pool)]
|
||||
#[tracing::instrument(level = "trace", name = "get", skip(self, cmd))]
|
||||
pub(crate) async fn execute_get(self: &Arc<Self>, mut cmd: Get) -> Result<BatchResult<'_>> {
|
||||
let (send, recv) = oneshot::channel();
|
||||
_ = cmd.res.insert(send);
|
||||
|
||||
let queue = self.select_queue();
|
||||
self.execute(queue, Cmd::Get(cmd))
|
||||
.and_then(move |()| {
|
||||
recv.map_ok(into_recv_get)
|
||||
.map_err(|e| err!(error!("recv failed {e:?}")))
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
#[implement(Pool)]
|
||||
#[tracing::instrument(level = "trace", name = "iter", skip(self, cmd))]
|
||||
pub(crate) async fn execute_iter(self: &Arc<Self>, mut cmd: Seek) -> Result<stream::State<'_>> {
|
||||
let (send, recv) = oneshot::channel();
|
||||
_ = cmd.res.insert(send);
|
||||
|
||||
let queue = self.select_queue();
|
||||
self.execute(queue, Cmd::Iter(cmd))
|
||||
.and_then(|()| {
|
||||
recv.map_ok(into_recv_seek)
|
||||
.map_err(|e| err!(error!("recv failed {e:?}")))
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
#[implement(Pool)]
|
||||
fn select_queue(&self) -> &Sender<Cmd> {
|
||||
let core_id = get_affinity().next().unwrap_or(0);
|
||||
let chan_id = self.topology[core_id];
|
||||
self.queues.get(chan_id).unwrap_or_else(|| &self.queues[0])
|
||||
}
|
||||
|
||||
#[implement(Pool)]
|
||||
#[tracing::instrument(
|
||||
level = "trace",
|
||||
name = "execute",
|
||||
skip(self, cmd),
|
||||
fields(
|
||||
task = ?tokio::task::try_id(),
|
||||
receivers = queue.receiver_count(),
|
||||
queued = queue.len(),
|
||||
queued_max = self.queued_max.load(Ordering::Relaxed),
|
||||
),
|
||||
)]
|
||||
async fn execute(&self, queue: &Sender<Cmd>, cmd: Cmd) -> Result {
|
||||
if cfg!(debug_assertions) {
|
||||
self.queued_max.fetch_max(queue.len(), Ordering::Relaxed);
|
||||
}
|
||||
|
||||
queue
|
||||
.send(cmd)
|
||||
.await
|
||||
.map_err(|e| err!(error!("send failed {e:?}")))
|
||||
}
|
||||
|
||||
#[implement(Pool)]
|
||||
#[tracing::instrument(
|
||||
parent = None,
|
||||
level = "debug",
|
||||
skip(self, recv),
|
||||
fields(
|
||||
tid = ?thread::current().id(),
|
||||
),
|
||||
)]
|
||||
fn worker(self: Arc<Self>, id: usize, recv: &Receiver<Cmd>) {
|
||||
self.worker_init(id);
|
||||
self.worker_loop(recv);
|
||||
}
|
||||
|
||||
#[implement(Pool)]
|
||||
fn worker_init(&self, id: usize) {
|
||||
let group = id.overflowing_rem(self.queues.len()).0;
|
||||
let affinity = self
|
||||
.topology
|
||||
.iter()
|
||||
.enumerate()
|
||||
.filter(|_| self.queues.len() > 1)
|
||||
.filter(|_| self.server.config.db_pool_affinity)
|
||||
.filter_map(|(core_id, &queue_id)| (group == queue_id).then_some(core_id))
|
||||
.filter_map(nth_core_available);
|
||||
|
||||
// affinity is empty (no-op) if there's only one queue
|
||||
set_affinity(affinity.clone());
|
||||
|
||||
#[cfg(all(not(target_env = "msvc"), feature = "jemalloc"))]
|
||||
if affinity.clone().count() == 1 && conduwuit::alloc::je::is_affine_arena() {
|
||||
use conduwuit::{
|
||||
alloc::je::this_thread::{arena_id, set_arena},
|
||||
result::LogDebugErr,
|
||||
};
|
||||
|
||||
let id = affinity.clone().next().expect("at least one id");
|
||||
|
||||
if let Ok(arena) = arena_id() {
|
||||
if arena != id {
|
||||
set_arena(id).log_debug_err().ok();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
debug!(
|
||||
%group,
|
||||
affinity = ?affinity.collect::<Vec<_>>(),
|
||||
"worker ready"
|
||||
);
|
||||
}
|
||||
|
||||
#[implement(Pool)]
|
||||
fn worker_loop(self: &Arc<Self>, recv: &Receiver<Cmd>) {
|
||||
// initial +1 needed prior to entering wait
|
||||
self.busy.fetch_add(1, Ordering::Relaxed);
|
||||
|
||||
while let Ok(cmd) = self.worker_wait(recv) {
|
||||
Pool::worker_handle(cmd);
|
||||
}
|
||||
}
|
||||
|
||||
#[implement(Pool)]
|
||||
#[tracing::instrument(
|
||||
name = "wait",
|
||||
level = "trace",
|
||||
skip_all,
|
||||
fields(
|
||||
receivers = recv.receiver_count(),
|
||||
queued = recv.len(),
|
||||
busy = self.busy.fetch_sub(1, Ordering::Relaxed) - 1,
|
||||
),
|
||||
)]
|
||||
fn worker_wait(self: &Arc<Self>, recv: &Receiver<Cmd>) -> Result<Cmd, RecvError> {
|
||||
recv.recv_blocking().debug_inspect(|_| {
|
||||
self.busy.fetch_add(1, Ordering::Relaxed);
|
||||
})
|
||||
}
|
||||
|
||||
#[implement(Pool)]
|
||||
fn worker_handle(cmd: Cmd) {
|
||||
match cmd {
|
||||
| Cmd::Get(cmd) if cmd.key.len() == 1 => Pool::handle_get(cmd),
|
||||
| Cmd::Get(cmd) => Pool::handle_batch(cmd),
|
||||
| Cmd::Iter(cmd) => Pool::handle_iter(cmd),
|
||||
}
|
||||
}
|
||||
|
||||
#[implement(Pool)]
|
||||
#[tracing::instrument(
|
||||
name = "iter",
|
||||
level = "trace",
|
||||
skip_all,
|
||||
fields(%cmd.map),
|
||||
)]
|
||||
fn handle_iter(mut cmd: Seek) {
|
||||
let chan = cmd.res.take().expect("missing result channel");
|
||||
|
||||
if chan.is_canceled() {
|
||||
return;
|
||||
}
|
||||
|
||||
let from = cmd.key.as_deref();
|
||||
|
||||
let result = match cmd.dir {
|
||||
| Direction::Forward => cmd.state.init_fwd(from),
|
||||
| Direction::Reverse => cmd.state.init_rev(from),
|
||||
};
|
||||
|
||||
let chan_result = chan.send(into_send_seek(result));
|
||||
|
||||
let _chan_sent = chan_result.is_ok();
|
||||
}
|
||||
|
||||
#[implement(Pool)]
|
||||
#[tracing::instrument(
|
||||
name = "batch",
|
||||
level = "trace",
|
||||
skip_all,
|
||||
fields(
|
||||
%cmd.map,
|
||||
keys = %cmd.key.len(),
|
||||
),
|
||||
)]
|
||||
fn handle_batch(mut cmd: Get) {
|
||||
debug_assert!(cmd.key.len() > 1, "should have more than one key");
|
||||
debug_assert!(!cmd.key.iter().any(SmallVec::is_empty), "querying for empty key");
|
||||
|
||||
let chan = cmd.res.take().expect("missing result channel");
|
||||
|
||||
if chan.is_canceled() {
|
||||
return;
|
||||
}
|
||||
|
||||
let keys = cmd.key.iter();
|
||||
|
||||
let result: SmallVec<_> = cmd.map.get_batch_blocking(keys).collect();
|
||||
|
||||
let chan_result = chan.send(into_send_get(result));
|
||||
|
||||
let _chan_sent = chan_result.is_ok();
|
||||
}
|
||||
|
||||
#[implement(Pool)]
|
||||
#[tracing::instrument(
|
||||
name = "get",
|
||||
level = "trace",
|
||||
skip_all,
|
||||
fields(%cmd.map),
|
||||
)]
|
||||
fn handle_get(mut cmd: Get) {
|
||||
debug_assert!(!cmd.key[0].is_empty(), "querying for empty key");
|
||||
|
||||
// Obtain the result channel.
|
||||
let chan = cmd.res.take().expect("missing result channel");
|
||||
|
||||
// It is worth checking if the future was dropped while the command was queued
|
||||
// so we can bail without paying for any query.
|
||||
if chan.is_canceled() {
|
||||
return;
|
||||
}
|
||||
|
||||
// Perform the actual database query. We reuse our database::Map interface but
|
||||
// limited to the blocking calls, rather than creating another surface directly
|
||||
// with rocksdb here.
|
||||
let result = cmd.map.get_blocking(&cmd.key[0]);
|
||||
|
||||
// Send the result back to the submitter.
|
||||
let chan_result = chan.send(into_send_get([result].into()));
|
||||
|
||||
// If the future was dropped during the query this will fail acceptably.
|
||||
let _chan_sent = chan_result.is_ok();
|
||||
}
|
||||
|
||||
fn into_send_get(result: BatchResult<'_>) -> BatchResult<'static> {
|
||||
// SAFETY: Necessary to send the Handle (rust_rocksdb::PinnableSlice) through
|
||||
// the channel. The lifetime on the handle is a device by rust-rocksdb to
|
||||
|
||||
+1
-10
@@ -2,19 +2,10 @@ use itertools::Itertools;
|
||||
use proc_macro::{Span, TokenStream};
|
||||
use proc_macro2::TokenStream as TokenStream2;
|
||||
use quote::{ToTokens, quote};
|
||||
use syn::{Attribute, Error, Fields, Ident, ItemEnum, ItemFn, Meta, Variant, parse_quote};
|
||||
use syn::{Error, Fields, Ident, ItemEnum, Meta, Variant};
|
||||
|
||||
use crate::{Result, utils::camel_to_snake_string};
|
||||
|
||||
pub(super) fn command(mut item: ItemFn, _args: &[Meta]) -> Result<TokenStream> {
|
||||
let attr: Attribute = parse_quote! {
|
||||
#[conduwuit_macros::implement(crate::Context, params = "<'_>")]
|
||||
};
|
||||
|
||||
item.attrs.push(attr);
|
||||
Ok(item.into_token_stream().into())
|
||||
}
|
||||
|
||||
pub(super) fn command_dispatch(item: ItemEnum, _args: &[Meta]) -> Result<TokenStream> {
|
||||
let name = &item.ident;
|
||||
let arm: Vec<TokenStream2> = item.variants.iter().map(dispatch_arm).try_collect()?;
|
||||
|
||||
@@ -1,35 +0,0 @@
|
||||
use proc_macro::{Span, TokenStream};
|
||||
use quote::quote;
|
||||
use syn::{Error, ItemFn, Meta, Path};
|
||||
use utils::get_named_generics;
|
||||
|
||||
use crate::{Result, utils};
|
||||
|
||||
pub(super) fn implement(item: ItemFn, args: &[Meta]) -> Result<TokenStream> {
|
||||
let generics = get_named_generics(args, "generics")?;
|
||||
let receiver = get_receiver(args)?;
|
||||
let params = get_named_generics(args, "params")?;
|
||||
let input = item;
|
||||
let out = quote! {
|
||||
impl #generics #receiver #params {
|
||||
#input
|
||||
}
|
||||
};
|
||||
|
||||
Ok(out.into())
|
||||
}
|
||||
|
||||
fn get_receiver(args: &[Meta]) -> Result<Path> {
|
||||
let receiver = &args.first().ok_or_else(|| {
|
||||
Error::new(Span::call_site().into(), "Missing required argument to receiver")
|
||||
})?;
|
||||
|
||||
let Meta::Path(receiver) = receiver else {
|
||||
return Err(Error::new(
|
||||
Span::call_site().into(),
|
||||
"First argument is not path to receiver",
|
||||
));
|
||||
};
|
||||
|
||||
Ok(receiver.clone())
|
||||
}
|
||||
@@ -2,7 +2,6 @@ mod admin;
|
||||
mod build_info;
|
||||
mod config;
|
||||
mod debug;
|
||||
mod implement;
|
||||
mod refutable;
|
||||
mod utils;
|
||||
|
||||
@@ -15,11 +14,6 @@ use syn::{
|
||||
|
||||
pub(crate) type Result<T> = std::result::Result<T, Error>;
|
||||
|
||||
#[proc_macro_attribute]
|
||||
pub fn admin_command(args: TokenStream, input: TokenStream) -> TokenStream {
|
||||
attribute_macro::<ItemFn, _>(args, input, admin::command)
|
||||
}
|
||||
|
||||
#[proc_macro_attribute]
|
||||
pub fn admin_command_dispatch(args: TokenStream, input: TokenStream) -> TokenStream {
|
||||
attribute_macro::<ItemEnum, _>(args, input, admin::command_dispatch)
|
||||
@@ -35,11 +29,6 @@ pub fn refutable(args: TokenStream, input: TokenStream) -> TokenStream {
|
||||
attribute_macro::<ItemFn, _>(args, input, refutable::refutable)
|
||||
}
|
||||
|
||||
#[proc_macro_attribute]
|
||||
pub fn implement(args: TokenStream, input: TokenStream) -> TokenStream {
|
||||
attribute_macro::<ItemFn, _>(args, input, implement::implement)
|
||||
}
|
||||
|
||||
#[proc_macro_attribute]
|
||||
pub fn config_example_generator(args: TokenStream, input: TokenStream) -> TokenStream {
|
||||
attribute_macro::<ItemStruct, _>(args, input, config::example_generator)
|
||||
|
||||
+1
-22
@@ -1,8 +1,6 @@
|
||||
use std::collections::HashMap;
|
||||
|
||||
use syn::{Expr, ExprLit, Generics, Lit, Meta, MetaNameValue, parse_str};
|
||||
|
||||
use crate::Result;
|
||||
use syn::{Expr, ExprLit, Lit, Meta, MetaNameValue};
|
||||
|
||||
pub(crate) fn get_simple_settings(args: &[Meta]) -> HashMap<String, String> {
|
||||
args.iter().fold(HashMap::new(), |mut map, arg| {
|
||||
@@ -44,25 +42,6 @@ pub(crate) fn legacy_is_cargo_build() -> bool {
|
||||
|
||||
pub(crate) fn is_cargo_test() -> bool { std::env::args().any(|flag| flag == "--test") }
|
||||
|
||||
pub(crate) fn get_named_generics(args: &[Meta], name: &str) -> Result<Generics> {
|
||||
const DEFAULT: &str = "<>";
|
||||
|
||||
parse_str::<Generics>(&get_named_string(args, name).unwrap_or_else(|| DEFAULT.to_owned()))
|
||||
}
|
||||
|
||||
pub(crate) fn get_named_string(args: &[Meta], name: &str) -> Option<String> {
|
||||
args.iter().find_map(|arg| {
|
||||
let value = arg.require_name_value().ok()?;
|
||||
let Expr::Lit(ref lit) = value.value else {
|
||||
return None;
|
||||
};
|
||||
let Lit::Str(ref str) = lit.lit else {
|
||||
return None;
|
||||
};
|
||||
value.path.is_ident(name).then_some(str.value())
|
||||
})
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub(crate) fn camel_to_snake_string(s: &str) -> String {
|
||||
let mut output = String::with_capacity(
|
||||
|
||||
@@ -1,8 +1,13 @@
|
||||
//! Types for invite filtering ([MSC4155]).
|
||||
//! Types for invite filtering ([MSC4155]) and
|
||||
//! invite blocking ([invite-blocking]).
|
||||
//!
|
||||
//! MSC4155: https://github.com/matrix-org/matrix-spec-proposals/pull/4155
|
||||
//! invite-blocking: https://spec.matrix.org/v1.18/client-server-api/#invite-permission
|
||||
|
||||
use ruma::{ServerName, UserId, exports::ruma_macros::EventContent};
|
||||
use ruma::{
|
||||
ServerName, UserId, events::invite_permission_config::InvitePermissionAction,
|
||||
exports::ruma_macros::EventContent,
|
||||
};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use wildmatch::WildMatch;
|
||||
|
||||
@@ -25,6 +30,21 @@ pub struct InvitePermissionConfigEventContent {
|
||||
#[serde(default = "ruma::serde::default_true")]
|
||||
pub enabled: bool,
|
||||
|
||||
/// The default action chosen by the user that the homeserver should perform
|
||||
/// automatically when receiving an invitation for this account.
|
||||
///
|
||||
/// A missing, invalid or unsupported value means that the user wants to
|
||||
/// receive invites as normal. Other parts of the specification might still
|
||||
/// have effects on invites, like [ignoring users].
|
||||
///
|
||||
/// [ignoring users]: https://spec.matrix.org/v1.18/client-server-api/#ignoring-users
|
||||
#[serde(
|
||||
default,
|
||||
deserialize_with = "ruma::serde::default_on_error",
|
||||
skip_serializing_if = "Option::is_none"
|
||||
)]
|
||||
pub default_action: Option<InvitePermissionAction>,
|
||||
|
||||
/// A list of globs matching users which are allowed to send an invite.
|
||||
/// Entries in this list supersede entries in the ignored and blocked lists.
|
||||
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
||||
@@ -77,6 +97,7 @@ impl InvitePermissionConfigEventContent {
|
||||
) -> Self {
|
||||
Self {
|
||||
enabled,
|
||||
default_action: None,
|
||||
allowed_users,
|
||||
ignored_users,
|
||||
blocked_users,
|
||||
@@ -86,12 +107,21 @@ impl InvitePermissionConfigEventContent {
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns true if the default action is configured and set to block.
|
||||
fn are_all_blocked(&self) -> bool {
|
||||
self.default_action
|
||||
.clone()
|
||||
.is_some_and(|action| action == InvitePermissionAction::Block)
|
||||
}
|
||||
|
||||
/// Test the filters against a user id. This function will check both the
|
||||
/// user rules _and_ the server rules.
|
||||
#[must_use]
|
||||
#[allow(clippy::if_same_then_else)]
|
||||
pub fn user_filter_level(&self, user: &UserId) -> FilterLevel {
|
||||
if !self.enabled {
|
||||
if self.are_all_blocked() {
|
||||
FilterLevel::Block
|
||||
} else if !self.enabled {
|
||||
FilterLevel::Allow
|
||||
} else if Self::matches(&self.allowed_users, user.as_str()) {
|
||||
FilterLevel::Allow
|
||||
@@ -107,7 +137,9 @@ impl InvitePermissionConfigEventContent {
|
||||
/// Test the filters against a server name. Port numbers are ignored.
|
||||
#[must_use]
|
||||
pub fn server_filter_level(&self, server: &ServerName) -> FilterLevel {
|
||||
if !self.enabled {
|
||||
if self.are_all_blocked() {
|
||||
FilterLevel::Block
|
||||
} else if !self.enabled {
|
||||
FilterLevel::Allow
|
||||
} else {
|
||||
let server = server.host();
|
||||
@@ -132,7 +164,12 @@ impl InvitePermissionConfigEventContent {
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use ruma::{ServerName, UserId, events::GlobalAccountDataEvent};
|
||||
use assign::assign;
|
||||
use ruma::{
|
||||
ServerName, UserId,
|
||||
events::{GlobalAccountDataEvent, invite_permission_config::InvitePermissionAction},
|
||||
user_id,
|
||||
};
|
||||
use serde_json::{from_value as from_json_value, json};
|
||||
|
||||
use crate::invite_permission_config::{FilterLevel, InvitePermissionConfigEventContent};
|
||||
@@ -268,4 +305,58 @@ mod tests {
|
||||
FilterLevel::Ignore
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn v118_blocking_enabled_blocks_all() {
|
||||
let event = assign!(
|
||||
InvitePermissionConfigEventContent::default(),
|
||||
{ default_action: Some(InvitePermissionAction::Block) }
|
||||
);
|
||||
let fixtures = [
|
||||
user_id!("@alice:goodguys.org"),
|
||||
user_id!("@alice:goodguys.org:8080"),
|
||||
user_id!("@bob:bar.com"),
|
||||
user_id!("@bob:Bar.com"),
|
||||
user_id!("@kevin:reallybadguys.org"),
|
||||
];
|
||||
assert!(
|
||||
fixtures
|
||||
.iter()
|
||||
.all(|f| event.user_filter_level(f) == FilterLevel::Block)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn v118_blocking_enabled_is_uninfluenced_by_msc4155() {
|
||||
let toggled = InvitePermissionConfigEventContent {
|
||||
enabled: false,
|
||||
default_action: Some(InvitePermissionAction::Block),
|
||||
..Default::default()
|
||||
};
|
||||
let only_goodguys = InvitePermissionConfigEventContent {
|
||||
enabled: true,
|
||||
default_action: Some(InvitePermissionAction::Block),
|
||||
allowed_servers: vec!["goodguys.org".to_owned()],
|
||||
ignored_servers: vec!["reallybadguys.org".to_owned()],
|
||||
blocked_servers: vec!["*".to_owned()],
|
||||
..Default::default()
|
||||
};
|
||||
let fixtures = [
|
||||
user_id!("@alice:goodguys.org"),
|
||||
user_id!("@alice:goodguys.org:8080"),
|
||||
user_id!("@bob:bar.com"),
|
||||
user_id!("@bob:Bar.com"),
|
||||
user_id!("@kevin:reallybadguys.org"),
|
||||
];
|
||||
assert!(
|
||||
fixtures
|
||||
.iter()
|
||||
.all(|f| toggled.user_filter_level(f) == FilterLevel::Block)
|
||||
);
|
||||
assert!(
|
||||
fixtures
|
||||
.iter()
|
||||
.all(|f| only_goodguys.user_filter_level(f) == FilterLevel::Block)
|
||||
);
|
||||
}
|
||||
}
|
||||
+104
-103
@@ -1,7 +1,7 @@
|
||||
use std::sync::Arc;
|
||||
|
||||
use conduwuit::{
|
||||
Err, Result, err, implement,
|
||||
Err, Result, err,
|
||||
utils::{ReadyExt, result::LogErr, stream::TryIgnore},
|
||||
};
|
||||
use database::{Deserialized, Handle, Ignore, Json, Map};
|
||||
@@ -54,114 +54,115 @@ impl crate::Service for Service {
|
||||
fn name(&self) -> &str { crate::service::make_name(std::module_path!()) }
|
||||
}
|
||||
|
||||
/// Places one event in the account data of the user and removes the
|
||||
/// previous entry.
|
||||
#[allow(clippy::needless_pass_by_value)]
|
||||
#[implement(Service)]
|
||||
pub async fn update(
|
||||
&self,
|
||||
room_id: Option<&RoomId>,
|
||||
user_id: &UserId,
|
||||
event_type: RoomAccountDataEventType,
|
||||
data: &serde_json::Value,
|
||||
) -> Result<()> {
|
||||
if data.get("type").is_none() || data.get("content").is_none() {
|
||||
return Err!(Request(InvalidParam("Account data doesn't have all required fields.")));
|
||||
impl Service {
|
||||
/// Places one event in the account data of the user and removes the
|
||||
/// previous entry.
|
||||
#[allow(clippy::needless_pass_by_value)]
|
||||
pub async fn update(
|
||||
&self,
|
||||
room_id: Option<&RoomId>,
|
||||
user_id: &UserId,
|
||||
event_type: RoomAccountDataEventType,
|
||||
data: &serde_json::Value,
|
||||
) -> Result<()> {
|
||||
if data.get("type").is_none() || data.get("content").is_none() {
|
||||
return Err!(Request(InvalidParam("Account data doesn't have all required fields.")));
|
||||
}
|
||||
|
||||
let count = self.services.globals.next_count().unwrap();
|
||||
let roomuserdataid = (room_id, user_id, count, &event_type);
|
||||
self.db
|
||||
.roomuserdataid_accountdata
|
||||
.put(roomuserdataid, Json(data));
|
||||
|
||||
let key = (room_id, user_id, &event_type);
|
||||
let prev = self.db.roomusertype_roomuserdataid.qry(&key).await;
|
||||
self.db.roomusertype_roomuserdataid.put(key, roomuserdataid);
|
||||
|
||||
// Remove old entry
|
||||
if let Ok(prev) = prev {
|
||||
self.db.roomuserdataid_accountdata.remove(&prev);
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
let count = self.services.globals.next_count().unwrap();
|
||||
let roomuserdataid = (room_id, user_id, count, &event_type);
|
||||
self.db
|
||||
.roomuserdataid_accountdata
|
||||
.put(roomuserdataid, Json(data));
|
||||
|
||||
let key = (room_id, user_id, &event_type);
|
||||
let prev = self.db.roomusertype_roomuserdataid.qry(&key).await;
|
||||
self.db.roomusertype_roomuserdataid.put(key, roomuserdataid);
|
||||
|
||||
// Remove old entry
|
||||
if let Ok(prev) = prev {
|
||||
self.db.roomuserdataid_accountdata.remove(&prev);
|
||||
/// Searches the room account data for a specific kind.
|
||||
pub async fn get_global<T>(
|
||||
&self,
|
||||
user_id: &UserId,
|
||||
kind: GlobalAccountDataEventType,
|
||||
) -> Result<T>
|
||||
where
|
||||
T: for<'de> Deserialize<'de>,
|
||||
{
|
||||
self.get_raw(None, user_id, &kind.to_string())
|
||||
.await
|
||||
.deserialized()
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
/// Searches the global account data for a specific kind.
|
||||
pub async fn get_room<T>(
|
||||
&self,
|
||||
room_id: &RoomId,
|
||||
user_id: &UserId,
|
||||
kind: RoomAccountDataEventType,
|
||||
) -> Result<T>
|
||||
where
|
||||
T: for<'de> Deserialize<'de>,
|
||||
{
|
||||
self.get_raw(Some(room_id), user_id, &kind.to_string())
|
||||
.await
|
||||
.deserialized()
|
||||
}
|
||||
|
||||
/// Searches the room account data for a specific kind.
|
||||
#[implement(Service)]
|
||||
pub async fn get_global<T>(&self, user_id: &UserId, kind: GlobalAccountDataEventType) -> Result<T>
|
||||
where
|
||||
T: for<'de> Deserialize<'de>,
|
||||
{
|
||||
self.get_raw(None, user_id, &kind.to_string())
|
||||
.await
|
||||
.deserialized()
|
||||
}
|
||||
pub async fn get_raw(
|
||||
&self,
|
||||
room_id: Option<&RoomId>,
|
||||
user_id: &UserId,
|
||||
kind: &str,
|
||||
) -> Result<Handle<'_>> {
|
||||
let key = (room_id, user_id, kind.to_owned());
|
||||
self.db
|
||||
.roomusertype_roomuserdataid
|
||||
.qry(&key)
|
||||
.and_then(|roomuserdataid| self.db.roomuserdataid_accountdata.get(&roomuserdataid))
|
||||
.await
|
||||
}
|
||||
|
||||
/// Searches the global account data for a specific kind.
|
||||
#[implement(Service)]
|
||||
pub async fn get_room<T>(
|
||||
&self,
|
||||
room_id: &RoomId,
|
||||
user_id: &UserId,
|
||||
kind: RoomAccountDataEventType,
|
||||
) -> Result<T>
|
||||
where
|
||||
T: for<'de> Deserialize<'de>,
|
||||
{
|
||||
self.get_raw(Some(room_id), user_id, &kind.to_string())
|
||||
.await
|
||||
.deserialized()
|
||||
}
|
||||
/// Returns all changes to the account data that happened after `since`.
|
||||
pub fn changes_since<'a>(
|
||||
&'a self,
|
||||
room_id: Option<&'a RoomId>,
|
||||
user_id: &'a UserId,
|
||||
since: Option<u64>,
|
||||
to: Option<u64>,
|
||||
) -> impl Stream<Item = AnyRawAccountDataEvent> + Send + 'a {
|
||||
type Key = (Option<OwnedRoomId>, OwnedUserId, u64, Ignore);
|
||||
|
||||
#[implement(Service)]
|
||||
pub async fn get_raw(
|
||||
&self,
|
||||
room_id: Option<&RoomId>,
|
||||
user_id: &UserId,
|
||||
kind: &str,
|
||||
) -> Result<Handle<'_>> {
|
||||
let key = (room_id, user_id, kind.to_owned());
|
||||
self.db
|
||||
.roomusertype_roomuserdataid
|
||||
.qry(&key)
|
||||
.and_then(|roomuserdataid| self.db.roomuserdataid_accountdata.get(&roomuserdataid))
|
||||
.await
|
||||
}
|
||||
// Skip the data that's exactly at since, because we sent that last time
|
||||
// ...unless this is an initial sync, in which case send everything
|
||||
let first_possible = (room_id, user_id, since.map_or(0, |since| since.saturating_add(1)));
|
||||
|
||||
/// Returns all changes to the account data that happened after `since`.
|
||||
#[implement(Service)]
|
||||
pub fn changes_since<'a>(
|
||||
&'a self,
|
||||
room_id: Option<&'a RoomId>,
|
||||
user_id: &'a UserId,
|
||||
since: Option<u64>,
|
||||
to: Option<u64>,
|
||||
) -> impl Stream<Item = AnyRawAccountDataEvent> + Send + 'a {
|
||||
type Key = (Option<OwnedRoomId>, OwnedUserId, u64, Ignore);
|
||||
|
||||
// Skip the data that's exactly at since, because we sent that last time
|
||||
// ...unless this is an initial sync, in which case send everything
|
||||
let first_possible = (room_id, user_id, since.map_or(0, |since| since.saturating_add(1)));
|
||||
|
||||
self.db
|
||||
.roomuserdataid_accountdata
|
||||
.stream_from(&first_possible)
|
||||
.ignore_err()
|
||||
.ready_take_while(move |((room_id_, user_id_, count, _), _): &(Key, _)| {
|
||||
room_id == room_id_.as_deref()
|
||||
&& user_id == user_id_
|
||||
&& to.is_none_or(|to| *count <= to)
|
||||
})
|
||||
.map(move |(_, v)| {
|
||||
match room_id {
|
||||
| Some(_) => serde_json::from_slice::<Raw<AnyRoomAccountDataEvent>>(v)
|
||||
.map(AnyRawAccountDataEvent::Room),
|
||||
| None => serde_json::from_slice::<Raw<AnyGlobalAccountDataEvent>>(v)
|
||||
.map(AnyRawAccountDataEvent::Global),
|
||||
}
|
||||
.map_err(|e| err!(Database("Database contains invalid account data: {e}")))
|
||||
.log_err()
|
||||
})
|
||||
.ignore_err()
|
||||
self.db
|
||||
.roomuserdataid_accountdata
|
||||
.stream_from(&first_possible)
|
||||
.ignore_err()
|
||||
.ready_take_while(move |((room_id_, user_id_, count, _), _): &(Key, _)| {
|
||||
room_id == room_id_.as_deref()
|
||||
&& user_id == user_id_
|
||||
&& to.is_none_or(|to| *count <= to)
|
||||
})
|
||||
.map(move |(_, v)| {
|
||||
match room_id {
|
||||
| Some(_) => serde_json::from_slice::<Raw<AnyRoomAccountDataEvent>>(v)
|
||||
.map(AnyRawAccountDataEvent::Room),
|
||||
| None => serde_json::from_slice::<Raw<AnyGlobalAccountDataEvent>>(v)
|
||||
.map(AnyRawAccountDataEvent::Global),
|
||||
}
|
||||
.map_err(|e| err!(Database("Database contains invalid account data: {e}")))
|
||||
.log_err()
|
||||
})
|
||||
.ignore_err()
|
||||
}
|
||||
}
|
||||
+76
-37
@@ -1,20 +1,22 @@
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
use conduwuit::{Result, info, pdu::PartialPdu};
|
||||
use futures::FutureExt;
|
||||
use ruma::{
|
||||
RoomId, RoomVersionId,
|
||||
events::room::{
|
||||
canonical_alias::RoomCanonicalAliasEventContent,
|
||||
create::RoomCreateEventContent,
|
||||
guest_access::{GuestAccess, RoomGuestAccessEventContent},
|
||||
history_visibility::{HistoryVisibility, RoomHistoryVisibilityEventContent},
|
||||
join_rules::{JoinRule, RoomJoinRulesEventContent},
|
||||
member::{MembershipState, RoomMemberEventContent},
|
||||
name::RoomNameEventContent,
|
||||
power_levels::RoomPowerLevelsEventContent,
|
||||
topic::RoomTopicEventContent,
|
||||
Int, RoomId,
|
||||
events::{
|
||||
TimelineEventType,
|
||||
room::{
|
||||
canonical_alias::RoomCanonicalAliasEventContent,
|
||||
create::RoomCreateEventContent,
|
||||
guest_access::{GuestAccess, RoomGuestAccessEventContent},
|
||||
history_visibility::{HistoryVisibility, RoomHistoryVisibilityEventContent},
|
||||
join_rules::{JoinRule, RoomJoinRulesEventContent},
|
||||
member::{MembershipState, RoomMemberEventContent},
|
||||
name::RoomNameEventContent,
|
||||
power_levels::RoomPowerLevelsEventContent,
|
||||
topic::RoomTopicEventContent,
|
||||
},
|
||||
},
|
||||
room_version_rules::RoomIdFormatVersion,
|
||||
};
|
||||
|
||||
use crate::Services;
|
||||
@@ -24,47 +26,68 @@ use crate::Services;
|
||||
/// Users in this room are considered admins by conduwuit, and the room can be
|
||||
/// used to issue admin commands by talking to the server user inside it.
|
||||
pub async fn create_admin_room(services: &Services) -> Result {
|
||||
let room_id = RoomId::new_v1(services.globals.server_name());
|
||||
let room_version = &RoomVersionId::V11;
|
||||
let room_version = services.config.default_room_version.clone();
|
||||
let room_version_rules = room_version
|
||||
.rules()
|
||||
.expect("default_room_version must be supported");
|
||||
let room_id = match room_version_rules.room_id_format {
|
||||
| RoomIdFormatVersion::V1 => {
|
||||
let room_id = RoomId::new_v1(services.globals.server_name());
|
||||
services
|
||||
.rooms
|
||||
.short
|
||||
.get_or_create_shortroomid(&room_id)
|
||||
.await;
|
||||
Some(room_id)
|
||||
},
|
||||
| RoomIdFormatVersion::V2 => None,
|
||||
| _ => panic!("Unknown room version format"),
|
||||
};
|
||||
|
||||
let _short_id = services
|
||||
.rooms
|
||||
.short
|
||||
.get_or_create_shortroomid(&room_id)
|
||||
.await;
|
||||
|
||||
let state_lock = services.rooms.state.mutex.lock(room_id.as_str()).await;
|
||||
let state_lock = services.rooms.state.mutex.lock("!new-room").await;
|
||||
|
||||
// Create a user for the server
|
||||
let server_user = services.globals.server_user.as_ref();
|
||||
services.users.create(server_user, None).await?;
|
||||
services.users.create(server_user, None)?;
|
||||
|
||||
let mut create_content = {
|
||||
use RoomVersionId::*;
|
||||
match room_version {
|
||||
| V1 | V2 | V3 | V4 | V5 | V6 | V7 | V8 | V9 | V10 =>
|
||||
RoomCreateEventContent::new_v1(server_user.into()),
|
||||
| _ => RoomCreateEventContent::new_v11(),
|
||||
}
|
||||
let mut create_content = if room_version_rules.authorization.use_room_create_sender {
|
||||
RoomCreateEventContent::new_v1(server_user.into())
|
||||
} else {
|
||||
RoomCreateEventContent::new_v11()
|
||||
};
|
||||
|
||||
create_content.federate = true;
|
||||
create_content.room_version = room_version.clone();
|
||||
|
||||
info!("Creating admin room {} with version {}", room_id, room_version);
|
||||
info!(
|
||||
"Creating admin room {} with version {}",
|
||||
room_id
|
||||
.clone()
|
||||
.map_or_else(|| "<not known ahead of time>".to_owned(), |id| id.as_str().to_owned()),
|
||||
room_version
|
||||
);
|
||||
|
||||
// 1. The room create event
|
||||
services
|
||||
let create_event_id = services
|
||||
.rooms
|
||||
.timeline
|
||||
.build_and_append_pdu(
|
||||
PartialPdu::state(String::new(), &create_content),
|
||||
server_user,
|
||||
Some(&room_id),
|
||||
room_id.as_deref(),
|
||||
&state_lock,
|
||||
)
|
||||
.boxed()
|
||||
.await?;
|
||||
let room_id = room_id.unwrap_or_else(|| {
|
||||
RoomId::new_v2(
|
||||
create_event_id
|
||||
.as_str()
|
||||
.strip_prefix("$")
|
||||
.expect("event ID must start with a $ sigil"),
|
||||
)
|
||||
.expect("event ID without sigil must be a valid room ID")
|
||||
});
|
||||
|
||||
// 2. Make server user/bot join
|
||||
services
|
||||
@@ -83,11 +106,27 @@ pub async fn create_admin_room(services: &Services) -> Result {
|
||||
.await?;
|
||||
|
||||
// 3. Power levels
|
||||
let users = BTreeMap::from_iter([(server_user.into(), 69420.into())]);
|
||||
|
||||
let mut power_levels_content =
|
||||
RoomPowerLevelsEventContent::new(&room_version.rules().unwrap().authorization);
|
||||
power_levels_content.users = users;
|
||||
RoomPowerLevelsEventContent::new(&room_version_rules.authorization);
|
||||
if !room_version_rules
|
||||
.authorization
|
||||
.explicitly_privilege_room_creators
|
||||
{
|
||||
power_levels_content
|
||||
.users
|
||||
.insert(server_user.into(), Int::MAX);
|
||||
}
|
||||
// Prevent common foot-shotguns
|
||||
power_levels_content
|
||||
.events
|
||||
.insert(TimelineEventType::RoomTombstone, Int::MAX);
|
||||
power_levels_content
|
||||
.events
|
||||
.insert(TimelineEventType::RoomEncryption, Int::MAX);
|
||||
power_levels_content
|
||||
.events
|
||||
.insert(TimelineEventType::RoomCanonicalAlias, Int::MAX);
|
||||
power_levels_content.invite = power_levels_content.state_default;
|
||||
|
||||
services
|
||||
.rooms
|
||||
|
||||
+100
-107
@@ -1,4 +1,4 @@
|
||||
use conduwuit::{Err, Result, debug, debug_info, error, implement, info};
|
||||
use conduwuit::{Err, Result, debug, debug_info, error, info};
|
||||
use ruma::events::room::message::RoomMessageEventContent;
|
||||
use tokio::time::{Duration, sleep};
|
||||
|
||||
@@ -6,127 +6,120 @@ use crate::admin::InvocationSource;
|
||||
|
||||
pub(super) const SIGNAL: &str = "SIGUSR2";
|
||||
|
||||
/// Possibly spawn the terminal console at startup if configured.
|
||||
#[implement(super::Service)]
|
||||
pub(super) async fn console_auto_start(&self) {
|
||||
#[cfg(feature = "console")]
|
||||
if self.services.server.config.admin_console_automatic {
|
||||
// Allow more of the startup sequence to execute before spawning
|
||||
tokio::task::yield_now().await;
|
||||
self.console.start().await;
|
||||
impl super::Service {
|
||||
/// Possibly spawn the terminal console at startup if configured.
|
||||
pub(super) async fn console_auto_start(&self) {
|
||||
#[cfg(feature = "console")]
|
||||
if self.services.server.config.admin_console_automatic {
|
||||
// Allow more of the startup sequence to execute before spawning
|
||||
tokio::task::yield_now().await;
|
||||
self.console.start().await;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Shutdown the console when the admin worker terminates.
|
||||
#[implement(super::Service)]
|
||||
pub(super) async fn console_auto_stop(&self) {
|
||||
#[cfg(feature = "console")]
|
||||
self.console.close().await;
|
||||
}
|
||||
/// Shutdown the console when the admin worker terminates.
|
||||
pub(super) async fn console_auto_stop(&self) {
|
||||
#[cfg(feature = "console")]
|
||||
self.console.close().await;
|
||||
}
|
||||
|
||||
/// Execute admin commands after startup
|
||||
#[implement(super::Service)]
|
||||
pub async fn startup_execute(&self) -> Result {
|
||||
// List of commands to execute
|
||||
let commands = &self.services.server.config.admin_execute;
|
||||
/// Execute admin commands after startup
|
||||
pub async fn startup_execute(&self) -> Result {
|
||||
// List of commands to execute
|
||||
let commands = &self.services.server.config.admin_execute;
|
||||
|
||||
// Determine if we're running in smoketest-mode which will change some behaviors
|
||||
let smoketest = self.services.server.config.test.contains("smoke");
|
||||
// Determine if we're running in smoketest-mode which will change some behaviors
|
||||
let smoketest = self.services.server.config.test.contains("smoke");
|
||||
|
||||
// When true, errors are ignored and startup continues.
|
||||
let errors = !smoketest && self.services.server.config.admin_execute_errors_ignore;
|
||||
// When true, errors are ignored and startup continues.
|
||||
let errors = !smoketest && self.services.server.config.admin_execute_errors_ignore;
|
||||
|
||||
//TODO: remove this after run-states are broadcast
|
||||
sleep(Duration::from_millis(500)).await;
|
||||
//TODO: remove this after run-states are broadcast
|
||||
sleep(Duration::from_millis(500)).await;
|
||||
|
||||
for (i, command) in commands.iter().enumerate() {
|
||||
if let Err(e) = self.execute_command(i, command.clone()).await {
|
||||
if !errors {
|
||||
return Err(e);
|
||||
for (i, command) in commands.iter().enumerate() {
|
||||
if let Err(e) = self.execute_command(i, command.clone()).await {
|
||||
if !errors {
|
||||
return Err(e);
|
||||
}
|
||||
}
|
||||
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
|
||||
// The smoketest functionality is placed here for now and simply initiates
|
||||
// shutdown after all commands have executed.
|
||||
if smoketest {
|
||||
debug_info!("Smoketest mode. All commands complete. Shutting down now...");
|
||||
self.services
|
||||
.server
|
||||
.shutdown()
|
||||
.inspect_err(error::inspect_log)
|
||||
.expect("Error shutting down from smoketest");
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Execute admin commands after signal
|
||||
#[implement(super::Service)]
|
||||
pub(super) async fn signal_execute(&self) -> Result {
|
||||
// List of commands to execute
|
||||
let commands = self.services.server.config.admin_signal_execute.clone();
|
||||
|
||||
// When true, errors are ignored and execution continues.
|
||||
let ignore_errors = self.services.server.config.admin_execute_errors_ignore;
|
||||
|
||||
for (i, command) in commands.iter().enumerate() {
|
||||
if let Err(e) = self.execute_command(i, command.clone()).await {
|
||||
if !ignore_errors {
|
||||
return Err(e);
|
||||
}
|
||||
// The smoketest functionality is placed here for now and simply initiates
|
||||
// shutdown after all commands have executed.
|
||||
if smoketest {
|
||||
debug_info!("Smoketest mode. All commands complete. Shutting down now...");
|
||||
self.services
|
||||
.server
|
||||
.shutdown()
|
||||
.inspect_err(error::inspect_log)
|
||||
.expect("Error shutting down from smoketest");
|
||||
}
|
||||
|
||||
tokio::task::yield_now().await;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
/// Execute admin commands after signal
|
||||
pub(super) async fn signal_execute(&self) -> Result {
|
||||
// List of commands to execute
|
||||
let commands = self.services.server.config.admin_signal_execute.clone();
|
||||
|
||||
/// Execute one admin command after startup or signal
|
||||
#[implement(super::Service)]
|
||||
async fn execute_command(&self, i: usize, command: String) -> Result {
|
||||
debug!("Execute command #{i}: executing {command:?}");
|
||||
// When true, errors are ignored and execution continues.
|
||||
let ignore_errors = self.services.server.config.admin_execute_errors_ignore;
|
||||
|
||||
match self
|
||||
.command_in_place(command, None, InvocationSource::Console)
|
||||
.await
|
||||
{
|
||||
| Ok(Some(output)) => Self::execute_command_output(i, &output),
|
||||
| Err(output) => Self::execute_command_error(i, &output),
|
||||
| Ok(None) => {
|
||||
info!("Execute command #{i} completed (no output).");
|
||||
Ok(())
|
||||
},
|
||||
for (i, command) in commands.iter().enumerate() {
|
||||
if let Err(e) = self.execute_command(i, command.clone()).await {
|
||||
if !ignore_errors {
|
||||
return Err(e);
|
||||
}
|
||||
}
|
||||
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Execute one admin command after startup or signal
|
||||
async fn execute_command(&self, i: usize, command: String) -> Result {
|
||||
debug!("Execute command #{i}: executing {command:?}");
|
||||
|
||||
match self
|
||||
.command_in_place(command, None, InvocationSource::Console)
|
||||
.await
|
||||
{
|
||||
| Ok(Some(output)) => Self::execute_command_output(i, &output),
|
||||
| Err(output) => Self::execute_command_error(i, &output),
|
||||
| Ok(None) => {
|
||||
info!("Execute command #{i} completed (no output).");
|
||||
Ok(())
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "console")]
|
||||
fn execute_command_output(i: usize, content: &RoomMessageEventContent) -> Result {
|
||||
debug_info!("Execute command #{i} completed:");
|
||||
super::console::print(content.body());
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(feature = "console")]
|
||||
fn execute_command_error(i: usize, content: &RoomMessageEventContent) -> Result {
|
||||
super::console::print_err(content.body());
|
||||
Err!(debug_error!("Execute command #{i} failed."))
|
||||
}
|
||||
|
||||
#[cfg(not(feature = "console"))]
|
||||
fn execute_command_output(i: usize, content: &RoomMessageEventContent) -> Result {
|
||||
info!("Execute command #{i} completed:\n{:#}", content.body());
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(not(feature = "console"))]
|
||||
fn execute_command_error(i: usize, content: &RoomMessageEventContent) -> Result {
|
||||
Err!(error!("Execute command #{i} failed:\n{:#}", content.body()))
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "console")]
|
||||
#[implement(super::Service)]
|
||||
fn execute_command_output(i: usize, content: &RoomMessageEventContent) -> Result {
|
||||
debug_info!("Execute command #{i} completed:");
|
||||
super::console::print(content.body());
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(feature = "console")]
|
||||
#[implement(super::Service)]
|
||||
fn execute_command_error(i: usize, content: &RoomMessageEventContent) -> Result {
|
||||
super::console::print_err(content.body());
|
||||
Err!(debug_error!("Execute command #{i} failed."))
|
||||
}
|
||||
|
||||
#[cfg(not(feature = "console"))]
|
||||
#[implement(super::Service)]
|
||||
fn execute_command_output(i: usize, content: &RoomMessageEventContent) -> Result {
|
||||
info!("Execute command #{i} completed:\n{:#}", content.body());
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(not(feature = "console"))]
|
||||
#[implement(super::Service)]
|
||||
fn execute_command_error(i: usize, content: &RoomMessageEventContent) -> Result {
|
||||
Err!(error!("Execute command #{i} failed:\n{:#}", content.body()))
|
||||
}
|
||||
+233
-199
@@ -1,10 +1,8 @@
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
use conduwuit::{
|
||||
Err, Result, debug_info, debug_warn, error, implement, matrix::pdu::PartialPdu, warn,
|
||||
};
|
||||
use conduwuit::{Err, Result, debug_info, debug_warn, error, matrix::pdu::PartialPdu, warn};
|
||||
use ruma::{
|
||||
RoomId, UserId,
|
||||
Int, RoomId, UserId,
|
||||
events::{
|
||||
RoomAccountDataEventType, StateEventType,
|
||||
room::{
|
||||
@@ -13,205 +11,241 @@ use ruma::{
|
||||
},
|
||||
tag::{TagEvent, TagEventContent, TagInfo},
|
||||
},
|
||||
int,
|
||||
};
|
||||
|
||||
/// Invite the user to the conduwuit admin room.
|
||||
///
|
||||
/// This is equivalent to granting server admin privileges.
|
||||
#[implement(super::Service)]
|
||||
pub async fn make_user_admin(&self, user_id: &UserId) -> Result {
|
||||
let Ok(room_id) = self.get_admin_room().await else {
|
||||
debug_warn!(
|
||||
"make_user_admin was called without an admin room being available or created"
|
||||
);
|
||||
return Ok(());
|
||||
};
|
||||
|
||||
let state_lock = self.services.state.mutex.lock(room_id.as_str()).await;
|
||||
|
||||
if self.services.state_cache.is_joined(user_id, &room_id).await {
|
||||
return Err!(debug_warn!("User is already joined in the admin room"));
|
||||
}
|
||||
if self
|
||||
.services
|
||||
.state_cache
|
||||
.is_invited(user_id, &room_id)
|
||||
.await
|
||||
{
|
||||
return Err!(debug_warn!("User is already pending an invitation to the admin room"));
|
||||
}
|
||||
|
||||
// Use the server user to grant the new admin's power level
|
||||
let server_user = self.services.globals.server_user.as_ref();
|
||||
|
||||
// if this is our local user, just forcefully join them in the room. otherwise,
|
||||
// invite the remote user.
|
||||
if self.services.globals.user_is_local(user_id) {
|
||||
debug_info!("Inviting local user {user_id} to admin room {room_id}");
|
||||
self.services
|
||||
.timeline
|
||||
.build_and_append_pdu(
|
||||
PartialPdu::state(
|
||||
String::from(user_id),
|
||||
&RoomMemberEventContent::new(MembershipState::Invite),
|
||||
),
|
||||
server_user,
|
||||
Some(&room_id),
|
||||
&state_lock,
|
||||
)
|
||||
.await?;
|
||||
|
||||
debug_info!("Force joining local user {user_id} to admin room {room_id}");
|
||||
self.services
|
||||
.timeline
|
||||
.build_and_append_pdu(
|
||||
PartialPdu::state(
|
||||
String::from(user_id),
|
||||
&RoomMemberEventContent::new(MembershipState::Join),
|
||||
),
|
||||
user_id,
|
||||
Some(&room_id),
|
||||
&state_lock,
|
||||
)
|
||||
.await?;
|
||||
} else {
|
||||
debug_info!("Inviting remote user {user_id} to admin room {room_id}");
|
||||
self.services
|
||||
.timeline
|
||||
.build_and_append_pdu(
|
||||
PartialPdu::state(
|
||||
user_id.to_string(),
|
||||
&RoomMemberEventContent::new(MembershipState::Invite),
|
||||
),
|
||||
server_user,
|
||||
Some(&room_id),
|
||||
&state_lock,
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
|
||||
// Set power levels
|
||||
let mut room_power_levels = self
|
||||
.services
|
||||
.state_accessor
|
||||
.room_state_get_content::<RoomPowerLevelsEventContent>(
|
||||
&room_id,
|
||||
&StateEventType::RoomPowerLevels,
|
||||
"",
|
||||
)
|
||||
.await
|
||||
.expect("admin room should have power levels");
|
||||
|
||||
room_power_levels
|
||||
.users
|
||||
.insert(server_user.into(), 69420.into());
|
||||
room_power_levels.users.insert(user_id.into(), 100.into());
|
||||
|
||||
self.services
|
||||
.timeline
|
||||
.build_and_append_pdu(
|
||||
PartialPdu::state(String::new(), &room_power_levels),
|
||||
server_user,
|
||||
Some(&room_id),
|
||||
&state_lock,
|
||||
)
|
||||
.await?;
|
||||
|
||||
// Set room tag
|
||||
let room_tag = self.services.server.config.admin_room_tag.as_str();
|
||||
if !room_tag.is_empty() {
|
||||
if let Err(e) = self.set_room_tag(&room_id, user_id, room_tag).await {
|
||||
error!(%room_id, %user_id, %room_tag, "Failed to set tag for admin grant: {e}");
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[implement(super::Service)]
|
||||
async fn set_room_tag(&self, room_id: &RoomId, user_id: &UserId, tag: &str) -> Result {
|
||||
let mut event = self
|
||||
.services
|
||||
.account_data
|
||||
.get_room(room_id, user_id, RoomAccountDataEventType::Tag)
|
||||
.await
|
||||
.unwrap_or_else(|_| TagEvent::new(TagEventContent::new(BTreeMap::new())));
|
||||
|
||||
event
|
||||
.content
|
||||
.tags
|
||||
.insert(tag.to_owned().into(), TagInfo::new());
|
||||
|
||||
self.services
|
||||
.account_data
|
||||
.update(
|
||||
Some(room_id),
|
||||
user_id,
|
||||
RoomAccountDataEventType::Tag,
|
||||
&serde_json::to_value(event)?,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
/// Demote an admin, removing its rights.
|
||||
#[implement(super::Service)]
|
||||
pub async fn revoke_admin(&self, user_id: &UserId) -> Result {
|
||||
use MembershipState::{Invite, Join, Knock, Leave};
|
||||
|
||||
if self
|
||||
.services
|
||||
.server
|
||||
.config
|
||||
.admins_list
|
||||
.contains(&user_id.to_owned())
|
||||
{
|
||||
warn!(
|
||||
"Revoking the admin status of {user_id} will not work correctly as they are within \
|
||||
the admins_list config."
|
||||
);
|
||||
}
|
||||
|
||||
let Ok(room_id) = self.get_admin_room().await else {
|
||||
return Err!(error!("No admin room available or created."));
|
||||
};
|
||||
|
||||
let state_lock = self.services.state.mutex.lock(room_id.as_str()).await;
|
||||
|
||||
let mut member_content = match self
|
||||
.services
|
||||
.state_accessor
|
||||
.get_member(&room_id, user_id)
|
||||
.await
|
||||
{
|
||||
| Err(e) if e.is_not_found() => return Err!("{user_id} was never an admin."),
|
||||
|
||||
| Err(e) => return Err!(error!(?e, "Failure occurred while attempting revoke.")),
|
||||
|
||||
| Ok(event) if !matches!(event.membership, Invite | Knock | Join) => {
|
||||
return Err!("Cannot revoke {user_id} in membership state {:?}.", event.membership);
|
||||
},
|
||||
|
||||
| Ok(event) => {
|
||||
assert!(
|
||||
matches!(event.membership, Invite | Knock | Join),
|
||||
"Incorrect membership state to remove user."
|
||||
impl super::Service {
|
||||
/// Invite the user to the conduwuit admin room.
|
||||
///
|
||||
/// This is equivalent to granting server admin privileges.
|
||||
pub async fn make_user_admin(&self, user_id: &UserId) -> Result {
|
||||
let Ok(room_id) = self.get_admin_room().await else {
|
||||
debug_warn!(
|
||||
"make_user_admin was called without an admin room being available or created"
|
||||
);
|
||||
return Ok(());
|
||||
};
|
||||
|
||||
event
|
||||
},
|
||||
};
|
||||
let state_lock = self.services.state.mutex.lock(room_id.as_str()).await;
|
||||
|
||||
member_content.membership = Leave;
|
||||
member_content.reason = Some("Admin Revoked".to_owned());
|
||||
if self.services.state_cache.is_joined(user_id, &room_id).await {
|
||||
return Err!(debug_warn!("User is already joined in the admin room"));
|
||||
}
|
||||
if self
|
||||
.services
|
||||
.state_cache
|
||||
.is_invited(user_id, &room_id)
|
||||
.await
|
||||
{
|
||||
return Err!(debug_warn!("User is already pending an invitation to the admin room"));
|
||||
}
|
||||
|
||||
self.services
|
||||
.timeline
|
||||
.build_and_append_pdu(
|
||||
PartialPdu::state(user_id.to_string(), &member_content),
|
||||
self.services.globals.server_user.as_ref(),
|
||||
Some(&room_id),
|
||||
&state_lock,
|
||||
)
|
||||
.await
|
||||
.map(|_| ())
|
||||
// Use the server user to grant the new admin's power level
|
||||
let server_user = self.services.globals.server_user.as_ref();
|
||||
|
||||
// if this is our local user, just forcefully join them in the room. otherwise,
|
||||
// invite the remote user.
|
||||
if self.services.globals.user_is_local(user_id) {
|
||||
debug_info!("Inviting local user {user_id} to admin room {room_id}");
|
||||
self.services
|
||||
.timeline
|
||||
.build_and_append_pdu(
|
||||
PartialPdu::state(
|
||||
String::from(user_id),
|
||||
&RoomMemberEventContent::new(MembershipState::Invite),
|
||||
),
|
||||
server_user,
|
||||
Some(&room_id),
|
||||
&state_lock,
|
||||
)
|
||||
.await?;
|
||||
|
||||
debug_info!("Force joining local user {user_id} to admin room {room_id}");
|
||||
self.services
|
||||
.timeline
|
||||
.build_and_append_pdu(
|
||||
PartialPdu::state(
|
||||
String::from(user_id),
|
||||
&RoomMemberEventContent::new(MembershipState::Join),
|
||||
),
|
||||
user_id,
|
||||
Some(&room_id),
|
||||
&state_lock,
|
||||
)
|
||||
.await?;
|
||||
} else {
|
||||
debug_info!("Inviting remote user {user_id} to admin room {room_id}");
|
||||
self.services
|
||||
.timeline
|
||||
.build_and_append_pdu(
|
||||
PartialPdu::state(
|
||||
user_id.to_string(),
|
||||
&RoomMemberEventContent::new(MembershipState::Invite),
|
||||
),
|
||||
server_user,
|
||||
Some(&room_id),
|
||||
&state_lock,
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
|
||||
// Set power levels
|
||||
let mut room_power_levels = self
|
||||
.services
|
||||
.state_accessor
|
||||
.room_state_get_content::<RoomPowerLevelsEventContent>(
|
||||
&room_id,
|
||||
&StateEventType::RoomPowerLevels,
|
||||
"",
|
||||
)
|
||||
.await
|
||||
.expect("admin room should have power levels");
|
||||
|
||||
let server_user_power_level = room_power_levels
|
||||
.users
|
||||
.get(server_user)
|
||||
.copied()
|
||||
.unwrap_or(Int::MAX);
|
||||
room_power_levels
|
||||
.users
|
||||
.insert(user_id.into(), server_user_power_level.saturating_sub(int!(1)));
|
||||
|
||||
self.services
|
||||
.timeline
|
||||
.build_and_append_pdu(
|
||||
PartialPdu::state(String::new(), &room_power_levels),
|
||||
server_user,
|
||||
Some(&room_id),
|
||||
&state_lock,
|
||||
)
|
||||
.await?;
|
||||
|
||||
// Set room tag
|
||||
let room_tag = self.services.server.config.admin_room_tag.as_str();
|
||||
if !room_tag.is_empty() {
|
||||
if let Err(e) = self.set_room_tag(&room_id, user_id, room_tag).await {
|
||||
error!(%room_id, %user_id, %room_tag, "Failed to set tag for admin grant: {e}");
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn set_room_tag(&self, room_id: &RoomId, user_id: &UserId, tag: &str) -> Result {
|
||||
let mut event = self
|
||||
.services
|
||||
.account_data
|
||||
.get_room(room_id, user_id, RoomAccountDataEventType::Tag)
|
||||
.await
|
||||
.unwrap_or_else(|_| TagEvent::new(TagEventContent::new(BTreeMap::new())));
|
||||
|
||||
event
|
||||
.content
|
||||
.tags
|
||||
.insert(tag.to_owned().into(), TagInfo::new());
|
||||
|
||||
self.services
|
||||
.account_data
|
||||
.update(
|
||||
Some(room_id),
|
||||
user_id,
|
||||
RoomAccountDataEventType::Tag,
|
||||
&serde_json::to_value(event)?,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
/// Demote an admin, removing its rights.
|
||||
pub async fn revoke_admin(&self, user_id: &UserId) -> Result {
|
||||
use MembershipState::{Invite, Join, Knock, Leave};
|
||||
|
||||
if self
|
||||
.services
|
||||
.server
|
||||
.config
|
||||
.admins_list
|
||||
.contains(&user_id.to_owned())
|
||||
{
|
||||
warn!(
|
||||
"Revoking the admin status of {user_id} will not work correctly as they are \
|
||||
within the admins_list config."
|
||||
);
|
||||
}
|
||||
|
||||
let Ok(room_id) = self.get_admin_room().await else {
|
||||
return Err!(error!("No admin room available or created."));
|
||||
};
|
||||
|
||||
let state_lock = self.services.state.mutex.lock(room_id.as_str()).await;
|
||||
|
||||
let mut room_power_levels = self
|
||||
.services
|
||||
.state_accessor
|
||||
.room_state_get_content::<RoomPowerLevelsEventContent>(
|
||||
&room_id,
|
||||
&StateEventType::RoomPowerLevels,
|
||||
"",
|
||||
)
|
||||
.await
|
||||
.expect("admin room should have power levels");
|
||||
if room_power_levels.users.remove(user_id).is_some() {
|
||||
// drop the target's power level
|
||||
self.services
|
||||
.timeline
|
||||
.build_and_append_pdu(
|
||||
PartialPdu::state(String::new(), &room_power_levels),
|
||||
self.services.globals.server_user.as_ref(),
|
||||
Some(&room_id),
|
||||
&state_lock,
|
||||
)
|
||||
.await
|
||||
.inspect_err(|e| {
|
||||
error!(?e, "Failed to update power levels while revoking admin.");
|
||||
})
|
||||
.ok();
|
||||
// Don't treat this as fatal. removing them from the room is more
|
||||
// important.
|
||||
}
|
||||
|
||||
let mut member_content = match self
|
||||
.services
|
||||
.state_accessor
|
||||
.get_member(&room_id, user_id)
|
||||
.await
|
||||
{
|
||||
| Err(e) if e.is_not_found() => return Err!("{user_id} was never an admin."),
|
||||
|
||||
| Err(e) => return Err!(error!(?e, "Failure occurred while attempting revoke.")),
|
||||
|
||||
| Ok(event) if !matches!(event.membership, Invite | Knock | Join) => {
|
||||
return Err!(
|
||||
"Cannot revoke {user_id} in membership state {:?}.",
|
||||
event.membership
|
||||
);
|
||||
},
|
||||
|
||||
| Ok(event) => {
|
||||
assert!(
|
||||
matches!(event.membership, Invite | Knock | Join),
|
||||
"Incorrect membership state to remove user."
|
||||
);
|
||||
|
||||
event
|
||||
},
|
||||
};
|
||||
|
||||
member_content.membership = Leave;
|
||||
member_content.reason = Some("Admin Revoked".to_owned());
|
||||
|
||||
self.services
|
||||
.timeline
|
||||
.build_and_append_pdu(
|
||||
PartialPdu::state(user_id.to_string(), &member_content),
|
||||
self.services.globals.server_user.as_ref(),
|
||||
Some(&room_id),
|
||||
&state_lock,
|
||||
)
|
||||
.await
|
||||
.map(|_| ())
|
||||
}
|
||||
}
|
||||
@@ -109,10 +109,7 @@ impl Service {
|
||||
)?;
|
||||
|
||||
if !self.services.users.exists(&appservice_user_id).await {
|
||||
self.services
|
||||
.users
|
||||
.create(&appservice_user_id, None)
|
||||
.await?;
|
||||
self.services.users.create(&appservice_user_id, None)?;
|
||||
} else if self
|
||||
.services
|
||||
.users
|
||||
|
||||
+13
-12
@@ -1,6 +1,6 @@
|
||||
use std::{sync::Arc, time::Duration};
|
||||
|
||||
use conduwuit::{Config, Result, err, implement, trace};
|
||||
use conduwuit::{Config, Result, err, trace};
|
||||
use either::Either;
|
||||
use ipaddress::IPAddress;
|
||||
use reqwest::redirect;
|
||||
@@ -13,7 +13,7 @@ pub struct Service {
|
||||
pub extern_media: reqwest::Client,
|
||||
pub well_known: reqwest::Client,
|
||||
pub federation: reqwest::Client,
|
||||
pub synapse: reqwest::Client,
|
||||
pub federation_slow: reqwest::Client,
|
||||
pub sender: reqwest::Client,
|
||||
pub appservice: reqwest::Client,
|
||||
pub pusher: reqwest::Client,
|
||||
@@ -85,7 +85,7 @@ impl crate::Service for Service {
|
||||
.redirect(redirect::Policy::limited(3))
|
||||
.build()?,
|
||||
|
||||
synapse: base(config)?
|
||||
federation_slow: base(config)?
|
||||
.dns_resolver(resolver.resolver.hooked.clone())
|
||||
.connect_timeout(Duration::from_secs(config.federation_conn_timeout))
|
||||
.read_timeout(Duration::from_secs(config.federation_timeout.saturating_mul(6)))
|
||||
@@ -141,6 +141,16 @@ impl crate::Service for Service {
|
||||
fn name(&self) -> &str { service::make_name(std::module_path!()) }
|
||||
}
|
||||
|
||||
impl Service {
|
||||
#[inline]
|
||||
#[must_use]
|
||||
pub fn valid_cidr_range(&self, ip: &IPAddress) -> bool {
|
||||
self.cidr_range_denylist
|
||||
.iter()
|
||||
.all(|cidr| !cidr.includes(ip))
|
||||
}
|
||||
}
|
||||
|
||||
fn base(config: &Config) -> Result<reqwest::ClientBuilder> {
|
||||
let mut builder = reqwest::Client::builder()
|
||||
.hickory_dns(true)
|
||||
@@ -227,12 +237,3 @@ fn builder_interface(
|
||||
Ok(builder)
|
||||
}
|
||||
}
|
||||
|
||||
#[inline]
|
||||
#[must_use]
|
||||
#[implement(Service)]
|
||||
pub fn valid_cidr_range(&self, ip: &IPAddress) -> bool {
|
||||
self.cidr_range_denylist
|
||||
.iter()
|
||||
.all(|cidr| !cidr.includes(ip))
|
||||
}
|
||||
+46
-47
@@ -4,7 +4,7 @@ use async_trait::async_trait;
|
||||
use conduwuit::{
|
||||
Result, Server,
|
||||
config::{Config, check},
|
||||
error, implement,
|
||||
error,
|
||||
};
|
||||
use url::Url;
|
||||
|
||||
@@ -16,28 +16,6 @@ pub struct Service {
|
||||
|
||||
const SIGNAL: &str = "SIGUSR1";
|
||||
|
||||
impl Service {
|
||||
/// Get the registration token set in the config file, if it exists.
|
||||
#[must_use]
|
||||
pub fn get_config_file_token(&self) -> Option<ValidToken> {
|
||||
self.registration_token
|
||||
.clone()
|
||||
.map(|token| ValidToken { token, source: ValidTokenSource::Config })
|
||||
}
|
||||
|
||||
/// Get the base domain to use for user-facing URLs.
|
||||
#[must_use]
|
||||
pub fn get_client_domain(&self) -> Url {
|
||||
self.well_known.client.clone().unwrap_or_else(|| {
|
||||
let host = self.server_name.host();
|
||||
format!("https://{host}")
|
||||
.as_str()
|
||||
.try_into()
|
||||
.expect("server name should be a valid host")
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl crate::Service for Service {
|
||||
fn build(args: crate::Args<'_>) -> Result<Arc<Self>> {
|
||||
@@ -66,32 +44,53 @@ impl Deref for Service {
|
||||
fn deref(&self) -> &Self::Target { &self.server.config }
|
||||
}
|
||||
|
||||
#[implement(Service)]
|
||||
fn handle_reload(&self) -> Result {
|
||||
if self.server.config.config_reload_signal {
|
||||
#[cfg(all(feature = "systemd", target_os = "linux"))]
|
||||
sd_notify::notify(&[
|
||||
sd_notify::NotifyState::Reloading,
|
||||
sd_notify::NotifyState::monotonic_usec_now().expect("Failed to read monotonic time"),
|
||||
])
|
||||
.expect("failed to notify systemd of reloading state");
|
||||
|
||||
let config_paths = self.server.config.config_paths.clone().unwrap_or_default();
|
||||
self.reload(&config_paths)?;
|
||||
|
||||
#[cfg(all(feature = "systemd", target_os = "linux"))]
|
||||
sd_notify::notify(&[sd_notify::NotifyState::Ready])
|
||||
.expect("failed to notify systemd of ready state");
|
||||
impl Service {
|
||||
/// Get the registration token set in the config file, if it exists.
|
||||
#[must_use]
|
||||
pub fn get_config_file_token(&self) -> Option<ValidToken> {
|
||||
self.registration_token
|
||||
.clone()
|
||||
.map(|token| ValidToken { token, source: ValidTokenSource::Config })
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
/// Get the base domain to use for user-facing URLs.
|
||||
#[must_use]
|
||||
pub fn get_client_domain(&self) -> Url {
|
||||
self.well_known.client.clone().unwrap_or_else(|| {
|
||||
let host = self.server_name.host();
|
||||
format!("https://{host}")
|
||||
.as_str()
|
||||
.try_into()
|
||||
.expect("server name should be a valid host")
|
||||
})
|
||||
}
|
||||
|
||||
#[implement(Service)]
|
||||
pub fn reload(&self, paths: &[PathBuf]) -> Result<Arc<Config>> {
|
||||
let old = self.server.config.clone();
|
||||
let new = Config::load(paths).and_then(|raw| Config::new(&raw))?;
|
||||
fn handle_reload(&self) -> Result {
|
||||
if self.server.config.config_reload_signal {
|
||||
#[cfg(all(feature = "systemd", target_os = "linux"))]
|
||||
sd_notify::notify(&[
|
||||
sd_notify::NotifyState::Reloading,
|
||||
sd_notify::NotifyState::monotonic_usec_now()
|
||||
.expect("Failed to read monotonic time"),
|
||||
])
|
||||
.expect("failed to notify systemd of reloading state");
|
||||
|
||||
check::reload(&old, &new)?;
|
||||
self.server.config.update(new)
|
||||
let config_paths = self.server.config.config_paths.clone().unwrap_or_default();
|
||||
self.reload(&config_paths)?;
|
||||
|
||||
#[cfg(all(feature = "systemd", target_os = "linux"))]
|
||||
sd_notify::notify(&[sd_notify::NotifyState::Ready])
|
||||
.expect("failed to notify systemd of ready state");
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn reload(&self, paths: &[PathBuf]) -> Result<Arc<Config>> {
|
||||
let old = self.server.config.clone();
|
||||
let new = Config::load(paths).and_then(|raw| Config::new(&raw))?;
|
||||
|
||||
check::reload(&old, &new)?;
|
||||
self.server.config.update(new)
|
||||
}
|
||||
}
|
||||
+196
-164
@@ -1,8 +1,8 @@
|
||||
use std::{borrow::Cow, fmt::Debug, mem};
|
||||
use std::{borrow::Cow, fmt::Debug, mem, time::Instant};
|
||||
|
||||
use bytes::Bytes;
|
||||
use conduwuit::{
|
||||
Err, Error, Result, debug, debug_error, debug_warn, err, implement, trace,
|
||||
Err, Error, Result, debug, debug_error, debug_info, debug_warn, err, trace,
|
||||
utils::response::LimitReadExt,
|
||||
};
|
||||
use ipaddress::IPAddress;
|
||||
@@ -20,181 +20,213 @@ use ruma::{
|
||||
|
||||
use crate::{SUPPORTED_VERSIONS, resolver::actual::ActualDest};
|
||||
|
||||
/// Sends a request to a federation server
|
||||
#[implement(super::Service)]
|
||||
#[tracing::instrument(skip_all, name = "request", level = "debug")]
|
||||
pub async fn execute<'i, T>(&self, dest: &ServerName, request: T) -> Result<T::IncomingResponse>
|
||||
where
|
||||
T: OutgoingRequest<
|
||||
Authentication = ServerSignatures,
|
||||
PathBuilder: PathBuilder<Input<'i>: FederationPathBuilderInput>,
|
||||
> + Debug
|
||||
+ Send,
|
||||
{
|
||||
let client = &self.services.client.federation;
|
||||
self.execute_signed(client, dest, request).await
|
||||
}
|
||||
|
||||
/// Like execute() but with a very large timeout
|
||||
#[implement(super::Service)]
|
||||
#[tracing::instrument(skip_all, name = "synapse", level = "debug")]
|
||||
pub async fn execute_synapse<'i, T>(
|
||||
&self,
|
||||
dest: &ServerName,
|
||||
request: T,
|
||||
) -> Result<T::IncomingResponse>
|
||||
where
|
||||
T: OutgoingRequest<
|
||||
Authentication = ServerSignatures,
|
||||
PathBuilder: PathBuilder<Input<'i>: FederationPathBuilderInput>,
|
||||
> + Debug
|
||||
+ Send,
|
||||
{
|
||||
let client = &self.services.client.synapse;
|
||||
self.execute_signed(client, dest, request).await
|
||||
}
|
||||
|
||||
#[implement(super::Service)]
|
||||
pub async fn execute_unauthenticated<'i, T>(
|
||||
&self,
|
||||
dest: &ServerName,
|
||||
request: T,
|
||||
) -> Result<T::IncomingResponse>
|
||||
where
|
||||
T: OutgoingRequest<
|
||||
Authentication = NoAuthentication,
|
||||
PathBuilder: PathBuilder<Input<'i>: FederationPathBuilderInput>,
|
||||
> + Debug
|
||||
+ Send,
|
||||
{
|
||||
let client = &self.services.client.federation;
|
||||
|
||||
self.execute_on(client, dest, request, ()).await
|
||||
}
|
||||
|
||||
#[implement(super::Service)]
|
||||
pub async fn execute_signed<'i, T>(
|
||||
&self,
|
||||
client: &Client,
|
||||
dest: &ServerName,
|
||||
request: T,
|
||||
) -> Result<T::IncomingResponse>
|
||||
where
|
||||
T: OutgoingRequest<
|
||||
Authentication = ServerSignatures,
|
||||
PathBuilder: PathBuilder<Input<'i>: FederationPathBuilderInput>,
|
||||
> + Send,
|
||||
{
|
||||
let authentication = ServerSignaturesInput::new(
|
||||
self.services.server.name.clone(),
|
||||
dest.to_owned(),
|
||||
self.services.server_keys.keypair(),
|
||||
);
|
||||
|
||||
self.execute_on(client, dest, request, authentication).await
|
||||
}
|
||||
|
||||
#[implement(super::Service)]
|
||||
#[tracing::instrument(name = "fed", level = "info", skip(self, client, request, authentication))]
|
||||
pub async fn execute_on<'i, T, PathBuilderInput>(
|
||||
&self,
|
||||
client: &Client,
|
||||
dest: &ServerName,
|
||||
request: T,
|
||||
authentication: <T::Authentication as AuthScheme>::Input<'_>,
|
||||
) -> Result<T::IncomingResponse>
|
||||
where
|
||||
T: OutgoingRequest<PathBuilder: PathBuilder<Input<'i> = PathBuilderInput>> + Send,
|
||||
PathBuilderInput: FederationPathBuilderInput,
|
||||
{
|
||||
if !self.services.server.config.allow_federation {
|
||||
return Err!(Config("allow_federation", "Federation is disabled."));
|
||||
impl super::Service {
|
||||
/// Sends a signed request to a remote server over federation.
|
||||
#[tracing::instrument(skip_all, name = "request", level = "debug")]
|
||||
pub async fn execute<'i, T>(
|
||||
&self,
|
||||
dest: &ServerName,
|
||||
request: T,
|
||||
) -> Result<T::IncomingResponse>
|
||||
where
|
||||
T: OutgoingRequest<
|
||||
Authentication = ServerSignatures,
|
||||
PathBuilder: PathBuilder<Input<'i>: FederationPathBuilderInput>,
|
||||
> + Debug
|
||||
+ Send,
|
||||
{
|
||||
let client = &self.services.client.federation;
|
||||
self.execute_signed(client, dest, request).await
|
||||
}
|
||||
|
||||
if self.services.moderation.is_remote_server_forbidden(dest) {
|
||||
return Err!(Request(Forbidden(debug_warn!("Federation with {dest} is not allowed."))));
|
||||
/// Sends a signed request to a remote server over federation, with a
|
||||
/// significantly higher timeout to account for slow endpoints.
|
||||
#[tracing::instrument(skip_all, name = "request_slow", level = "debug")]
|
||||
pub async fn execute_slow<'i, T>(
|
||||
&self,
|
||||
dest: &ServerName,
|
||||
request: T,
|
||||
) -> Result<T::IncomingResponse>
|
||||
where
|
||||
T: OutgoingRequest<
|
||||
Authentication = ServerSignatures,
|
||||
PathBuilder: PathBuilder<Input<'i>: FederationPathBuilderInput>,
|
||||
> + Debug
|
||||
+ Send,
|
||||
{
|
||||
let client = &self.services.client.federation_slow;
|
||||
self.execute_signed(client, dest, request).await
|
||||
}
|
||||
|
||||
let actual = self.services.resolver.get_actual_dest(dest).await?;
|
||||
/// Sends an unsigned (unauthenticated) request to a remote server over
|
||||
/// federation.
|
||||
pub async fn execute_unauthenticated<'i, T>(
|
||||
&self,
|
||||
dest: &ServerName,
|
||||
request: T,
|
||||
) -> Result<T::IncomingResponse>
|
||||
where
|
||||
T: OutgoingRequest<
|
||||
Authentication = NoAuthentication,
|
||||
PathBuilder: PathBuilder<Input<'i>: FederationPathBuilderInput>,
|
||||
> + Debug
|
||||
+ Send,
|
||||
{
|
||||
let client = &self.services.client.federation;
|
||||
|
||||
let request = Request::try_from(request.try_into_http_request::<Vec<u8>>(
|
||||
actual.string().as_str(),
|
||||
authentication,
|
||||
PathBuilderInput::create(),
|
||||
)?)?;
|
||||
self.validate_url(request.url())?;
|
||||
self.services.server.check_running()?;
|
||||
|
||||
self.perform::<T>(dest, &actual, request, client).await
|
||||
}
|
||||
|
||||
#[implement(super::Service)]
|
||||
async fn perform<T>(
|
||||
&self,
|
||||
dest: &ServerName,
|
||||
actual: &ActualDest,
|
||||
request: Request,
|
||||
client: &Client,
|
||||
) -> Result<T::IncomingResponse>
|
||||
where
|
||||
T: OutgoingRequest + Send,
|
||||
{
|
||||
let url = request.url().clone();
|
||||
let method = request.method().clone();
|
||||
|
||||
debug!(%method, %url, "Sending request");
|
||||
match client.execute(request).await {
|
||||
| Ok(response) =>
|
||||
self.handle_response::<T>(dest, actual, &method, &url, response)
|
||||
.await,
|
||||
| Err(error) =>
|
||||
Err(handle_error(actual, &method, &url, error).expect_err("always returns error")),
|
||||
self.execute_on(client, dest, request, ()).await
|
||||
}
|
||||
}
|
||||
|
||||
#[implement(super::Service)]
|
||||
fn validate_url(&self, url: &Url) -> Result<()> {
|
||||
if let Some(url_host) = url.host_str() {
|
||||
if let Ok(ip) = IPAddress::parse(url_host) {
|
||||
trace!("Checking request URL IP {ip:?}");
|
||||
self.services.resolver.validate_ip(&ip)?;
|
||||
/// Sends a signed request to a remote server over federation, given the
|
||||
/// specific client.
|
||||
pub async fn execute_signed<'i, T>(
|
||||
&self,
|
||||
client: &Client,
|
||||
dest: &ServerName,
|
||||
request: T,
|
||||
) -> Result<T::IncomingResponse>
|
||||
where
|
||||
T: OutgoingRequest<
|
||||
Authentication = ServerSignatures,
|
||||
PathBuilder: PathBuilder<Input<'i>: FederationPathBuilderInput>,
|
||||
> + Send,
|
||||
{
|
||||
let authentication = ServerSignaturesInput::new(
|
||||
self.services.server.name.clone(),
|
||||
dest.to_owned(),
|
||||
self.services.server_keys.keypair(),
|
||||
);
|
||||
|
||||
self.execute_on(client, dest, request, authentication).await
|
||||
}
|
||||
|
||||
/// Prepares and sends a federation request with the given client.
|
||||
/// If federation is disabled, an error is returned here. Likewise, if the
|
||||
/// destination is forbidden.
|
||||
/// Preparation includes resolving the server before sending the request.
|
||||
#[tracing::instrument(
|
||||
name = "fed",
|
||||
level = "info",
|
||||
skip(self, client, request, authentication)
|
||||
)]
|
||||
pub async fn execute_on<'i, T, PathBuilderInput>(
|
||||
&self,
|
||||
client: &Client,
|
||||
dest: &ServerName,
|
||||
request: T,
|
||||
authentication: <T::Authentication as AuthScheme>::Input<'_>,
|
||||
) -> Result<T::IncomingResponse>
|
||||
where
|
||||
T: OutgoingRequest<PathBuilder: PathBuilder<Input<'i> = PathBuilderInput>> + Send,
|
||||
PathBuilderInput: FederationPathBuilderInput,
|
||||
{
|
||||
if !self.services.server.config.allow_federation {
|
||||
return Err!(Config("allow_federation", "Federation is disabled."));
|
||||
}
|
||||
|
||||
if self.services.moderation.is_remote_server_forbidden(dest) {
|
||||
return Err!(Request(Forbidden(debug_warn!(
|
||||
"Federation with {dest} is not allowed."
|
||||
))));
|
||||
}
|
||||
|
||||
let actual = self.services.resolver.get_actual_dest(dest).await?;
|
||||
|
||||
let request = Request::try_from(request.try_into_http_request::<Vec<u8>>(
|
||||
actual.string().as_str(),
|
||||
authentication,
|
||||
PathBuilderInput::create(),
|
||||
)?)?;
|
||||
self.validate_url(request.url())?;
|
||||
self.services.server.check_running()?;
|
||||
|
||||
self.perform::<T>(dest, &actual, request, client).await
|
||||
}
|
||||
|
||||
/// Actually sends the federation request, handling the response before
|
||||
/// returning.
|
||||
///
|
||||
/// Timing info and request logs are available when debug logs are enabled.
|
||||
async fn perform<T>(
|
||||
&self,
|
||||
dest: &ServerName,
|
||||
actual: &ActualDest,
|
||||
request: Request,
|
||||
client: &Client,
|
||||
) -> Result<T::IncomingResponse>
|
||||
where
|
||||
T: OutgoingRequest + Send,
|
||||
{
|
||||
let url = request.url().clone();
|
||||
let method = request.method().clone();
|
||||
|
||||
debug_info!(
|
||||
"Sending request: {} matrix-federation://{dest}{}",
|
||||
method.as_str(),
|
||||
url.path(),
|
||||
);
|
||||
let start = Instant::now();
|
||||
let response = client.execute(request).await;
|
||||
debug_info!(
|
||||
"Request finished in {:?}: {} matrix-federation://{dest}{}",
|
||||
start.elapsed(),
|
||||
method.as_str(),
|
||||
url.path(),
|
||||
);
|
||||
match response {
|
||||
| Ok(response) =>
|
||||
self.handle_response::<T>(dest, actual, &method, &url, response)
|
||||
.await,
|
||||
| Err(error) =>
|
||||
Err(handle_error(actual, &method, &url, error).expect_err("always returns error")),
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
fn validate_url(&self, url: &Url) -> Result<()> {
|
||||
if let Some(url_host) = url.host_str() {
|
||||
if let Ok(ip) = IPAddress::parse(url_host) {
|
||||
trace!("Checking request URL IP {ip:?}");
|
||||
self.services.resolver.validate_ip(&ip)?;
|
||||
}
|
||||
}
|
||||
|
||||
#[implement(super::Service)]
|
||||
async fn handle_response<T>(
|
||||
&self,
|
||||
dest: &ServerName,
|
||||
actual: &ActualDest,
|
||||
method: &Method,
|
||||
url: &Url,
|
||||
response: Response,
|
||||
) -> Result<T::IncomingResponse>
|
||||
where
|
||||
T: OutgoingRequest + Send,
|
||||
{
|
||||
const HUGE_ENDPOINTS: [&str; 2] =
|
||||
["/_matrix/federation/v2/send_join/", "/_matrix/federation/v2/state/"];
|
||||
let size_limit: u64 = if HUGE_ENDPOINTS.iter().any(|e| url.path().starts_with(e)) {
|
||||
// Some federation endpoints can return huge response bodies, so we'll bump the
|
||||
// limit for those endpoints specifically.
|
||||
self.services
|
||||
.server
|
||||
.config
|
||||
.max_request_size
|
||||
.saturating_mul(10)
|
||||
} else {
|
||||
self.services.server.config.max_request_size
|
||||
Ok(())
|
||||
}
|
||||
.try_into()
|
||||
.expect("size_limit (usize) should fit within a u64");
|
||||
let response = into_http_response(dest, actual, method, url, response, size_limit).await?;
|
||||
|
||||
T::IncomingResponse::try_from_http_response(response)
|
||||
.map_err(|e| err!(BadServerResponse("Server returned bad 200 response: {e:?}")))
|
||||
/// Handles a successful response from a remote server, reading and
|
||||
/// deserializing the response body (if any).
|
||||
async fn handle_response<T>(
|
||||
&self,
|
||||
dest: &ServerName,
|
||||
actual: &ActualDest,
|
||||
method: &Method,
|
||||
url: &Url,
|
||||
response: Response,
|
||||
) -> Result<T::IncomingResponse>
|
||||
where
|
||||
T: OutgoingRequest + Send,
|
||||
{
|
||||
const HUGE_ENDPOINTS: [&str; 2] =
|
||||
["/_matrix/federation/v2/send_join/", "/_matrix/federation/v2/state/"];
|
||||
let size_limit: u64 = if HUGE_ENDPOINTS.iter().any(|e| url.path().starts_with(e)) {
|
||||
// Some federation endpoints can return huge response bodies, so we'll bump the
|
||||
// limit for those endpoints specifically.
|
||||
self.services
|
||||
.server
|
||||
.config
|
||||
.max_request_size
|
||||
.saturating_mul(10)
|
||||
} else {
|
||||
self.services.server.config.max_request_size
|
||||
}
|
||||
.try_into()
|
||||
.expect("size_limit (usize) should fit within a u64");
|
||||
let response =
|
||||
into_http_response(dest, actual, method, url, response, size_limit).await?;
|
||||
|
||||
T::IncomingResponse::try_from_http_response(response)
|
||||
.map_err(|e| err!(BadServerResponse("Server returned bad 200 response: {e:?}")))
|
||||
}
|
||||
}
|
||||
|
||||
async fn into_http_response(
|
||||
|
||||
+231
-240
@@ -1,7 +1,7 @@
|
||||
use std::{collections::BTreeMap, sync::Arc};
|
||||
|
||||
use conduwuit::{
|
||||
Err, Result, err, implement,
|
||||
Err, Result, err,
|
||||
utils::stream::{ReadyExt, TryIgnore},
|
||||
};
|
||||
use database::{Deserialized, Ignore, Interfix, Json, Map};
|
||||
@@ -46,253 +46,244 @@ impl crate::Service for Service {
|
||||
fn name(&self) -> &str { crate::service::make_name(std::module_path!()) }
|
||||
}
|
||||
|
||||
#[implement(Service)]
|
||||
pub fn create_backup(
|
||||
&self,
|
||||
user_id: &UserId,
|
||||
backup_metadata: &Raw<BackupAlgorithm>,
|
||||
) -> Result<String> {
|
||||
let version = self.services.globals.next_count()?.to_string();
|
||||
let count = self.services.globals.next_count()?;
|
||||
impl Service {
|
||||
pub fn create_backup(
|
||||
&self,
|
||||
user_id: &UserId,
|
||||
backup_metadata: &Raw<BackupAlgorithm>,
|
||||
) -> Result<String> {
|
||||
let version = self.services.globals.next_count()?.to_string();
|
||||
let count = self.services.globals.next_count()?;
|
||||
|
||||
let key = (user_id, &version);
|
||||
self.db.backupid_algorithm.put(key, Json(backup_metadata));
|
||||
let key = (user_id, &version);
|
||||
self.db.backupid_algorithm.put(key, Json(backup_metadata));
|
||||
|
||||
self.db.backupid_etag.put(key, count);
|
||||
self.db.backupid_etag.put(key, count);
|
||||
|
||||
Ok(version)
|
||||
}
|
||||
|
||||
#[implement(Service)]
|
||||
pub async fn delete_backup(&self, user_id: &UserId, version: &str) {
|
||||
let key = (user_id, version);
|
||||
self.db.backupid_algorithm.del(key);
|
||||
self.db.backupid_etag.del(key);
|
||||
|
||||
let key = (user_id, version, Interfix);
|
||||
self.db
|
||||
.backupkeyid_backup
|
||||
.keys_prefix_raw(&key)
|
||||
.ignore_err()
|
||||
.ready_for_each(|outdated_key| {
|
||||
self.db.backupkeyid_backup.remove(outdated_key);
|
||||
})
|
||||
.await;
|
||||
}
|
||||
|
||||
#[implement(Service)]
|
||||
pub async fn update_backup<'a>(
|
||||
&self,
|
||||
user_id: &UserId,
|
||||
version: &'a str,
|
||||
backup_metadata: &Raw<BackupAlgorithm>,
|
||||
) -> Result<&'a str> {
|
||||
let key = (user_id, version);
|
||||
if self.db.backupid_algorithm.qry(&key).await.is_err() {
|
||||
return Err!(Request(NotFound("Tried to update nonexistent backup.")));
|
||||
Ok(version)
|
||||
}
|
||||
|
||||
let count = self.services.globals.next_count().unwrap();
|
||||
self.db.backupid_etag.put(key, count);
|
||||
self.db
|
||||
.backupid_algorithm
|
||||
.put_raw(key, backup_metadata.json().get());
|
||||
pub async fn delete_backup(&self, user_id: &UserId, version: &str) {
|
||||
let key = (user_id, version);
|
||||
self.db.backupid_algorithm.del(key);
|
||||
self.db.backupid_etag.del(key);
|
||||
|
||||
Ok(version)
|
||||
}
|
||||
|
||||
#[implement(Service)]
|
||||
pub async fn get_latest_backup_version(&self, user_id: &UserId) -> Result<String> {
|
||||
type Key<'a> = (OwnedUserId, &'a str);
|
||||
|
||||
let last_possible_key = (user_id, u64::MAX);
|
||||
self.db
|
||||
.backupid_algorithm
|
||||
.rev_keys_from(&last_possible_key)
|
||||
.ignore_err()
|
||||
.ready_take_while(|(user_id_, _): &Key<'_>| *user_id_ == user_id)
|
||||
.map(|(_, version): Key<'_>| version.to_owned())
|
||||
.next()
|
||||
.await
|
||||
.ok_or_else(|| err!(Request(NotFound("No backup versions found"))))
|
||||
}
|
||||
|
||||
#[implement(Service)]
|
||||
pub async fn get_latest_backup(
|
||||
&self,
|
||||
user_id: &UserId,
|
||||
) -> Result<(String, Raw<BackupAlgorithm>)> {
|
||||
type Key<'a> = (OwnedUserId, &'a str);
|
||||
type KeyVal<'a> = (Key<'a>, Raw<BackupAlgorithm>);
|
||||
|
||||
let last_possible_key = (user_id, u64::MAX);
|
||||
self.db
|
||||
.backupid_algorithm
|
||||
.rev_stream_from(&last_possible_key)
|
||||
.ignore_err()
|
||||
.ready_take_while(|((user_id_, _), _): &KeyVal<'_>| *user_id_ == user_id)
|
||||
.map(|((_, version), algorithm): KeyVal<'_>| (version.to_owned(), algorithm))
|
||||
.next()
|
||||
.await
|
||||
.ok_or_else(|| err!(Request(NotFound("No backup found"))))
|
||||
}
|
||||
|
||||
#[implement(Service)]
|
||||
pub async fn get_backup(&self, user_id: &UserId, version: &str) -> Result<Raw<BackupAlgorithm>> {
|
||||
let key = (user_id, version);
|
||||
self.db.backupid_algorithm.qry(&key).await.deserialized()
|
||||
}
|
||||
|
||||
#[implement(Service)]
|
||||
pub async fn add_key(
|
||||
&self,
|
||||
user_id: &UserId,
|
||||
version: &str,
|
||||
room_id: &RoomId,
|
||||
session_id: &str,
|
||||
key_data: &Raw<KeyBackupData>,
|
||||
) -> Result<()> {
|
||||
let key = (user_id, version);
|
||||
if self.db.backupid_algorithm.qry(&key).await.is_err() {
|
||||
return Err!(Request(NotFound("Tried to update nonexistent backup.")));
|
||||
let key = (user_id, version, Interfix);
|
||||
self.db
|
||||
.backupkeyid_backup
|
||||
.keys_prefix_raw(&key)
|
||||
.ignore_err()
|
||||
.ready_for_each(|outdated_key| {
|
||||
self.db.backupkeyid_backup.remove(outdated_key);
|
||||
})
|
||||
.await;
|
||||
}
|
||||
|
||||
let count = self.services.globals.next_count().unwrap();
|
||||
self.db.backupid_etag.put(key, count);
|
||||
pub async fn update_backup<'a>(
|
||||
&self,
|
||||
user_id: &UserId,
|
||||
version: &'a str,
|
||||
backup_metadata: &Raw<BackupAlgorithm>,
|
||||
) -> Result<&'a str> {
|
||||
let key = (user_id, version);
|
||||
if self.db.backupid_algorithm.qry(&key).await.is_err() {
|
||||
return Err!(Request(NotFound("Tried to update nonexistent backup.")));
|
||||
}
|
||||
|
||||
let key = (user_id, version, room_id, session_id);
|
||||
self.db
|
||||
.backupkeyid_backup
|
||||
.put_raw(key, key_data.json().get());
|
||||
let count = self.services.globals.next_count().unwrap();
|
||||
self.db.backupid_etag.put(key, count);
|
||||
self.db
|
||||
.backupid_algorithm
|
||||
.put_raw(key, backup_metadata.json().get());
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[implement(Service)]
|
||||
pub async fn count_keys(&self, user_id: &UserId, version: &str) -> usize {
|
||||
let prefix = (user_id, version);
|
||||
self.db
|
||||
.backupkeyid_backup
|
||||
.keys_prefix_raw(&prefix)
|
||||
.count()
|
||||
.await
|
||||
}
|
||||
|
||||
#[implement(Service)]
|
||||
pub async fn get_etag(&self, user_id: &UserId, version: &str) -> String {
|
||||
let key = (user_id, version);
|
||||
self.db
|
||||
.backupid_etag
|
||||
.qry(&key)
|
||||
.await
|
||||
.deserialized::<u64>()
|
||||
.as_ref()
|
||||
.map(ToString::to_string)
|
||||
.expect("Backup has no etag.")
|
||||
}
|
||||
|
||||
#[implement(Service)]
|
||||
pub async fn get_all(
|
||||
&self,
|
||||
user_id: &UserId,
|
||||
version: &str,
|
||||
) -> BTreeMap<OwnedRoomId, RoomKeyBackup> {
|
||||
type Key<'a> = (Ignore, Ignore, OwnedRoomId, &'a str);
|
||||
type KeyVal<'a> = (Key<'a>, Raw<KeyBackupData>);
|
||||
|
||||
let mut rooms = BTreeMap::<OwnedRoomId, RoomKeyBackup>::new();
|
||||
let default = || RoomKeyBackup::new(BTreeMap::new());
|
||||
|
||||
let prefix = (user_id, version, Interfix);
|
||||
self.db
|
||||
.backupkeyid_backup
|
||||
.stream_prefix(&prefix)
|
||||
.ignore_err()
|
||||
.ready_for_each(|((_, _, room_id, session_id), key_backup_data): KeyVal<'_>| {
|
||||
rooms
|
||||
.entry(room_id)
|
||||
.or_insert_with(default)
|
||||
.sessions
|
||||
.insert(session_id.into(), key_backup_data);
|
||||
})
|
||||
.await;
|
||||
|
||||
rooms
|
||||
}
|
||||
|
||||
#[implement(Service)]
|
||||
pub async fn get_room(
|
||||
&self,
|
||||
user_id: &UserId,
|
||||
version: &str,
|
||||
room_id: &RoomId,
|
||||
) -> BTreeMap<String, Raw<KeyBackupData>> {
|
||||
type KeyVal<'a> = ((Ignore, Ignore, Ignore, &'a str), Raw<KeyBackupData>);
|
||||
|
||||
let prefix = (user_id, version, room_id, Interfix);
|
||||
self.db
|
||||
.backupkeyid_backup
|
||||
.stream_prefix(&prefix)
|
||||
.ignore_err()
|
||||
.map(|((.., session_id), key_backup_data): KeyVal<'_>| {
|
||||
(session_id.to_owned(), key_backup_data)
|
||||
})
|
||||
.collect()
|
||||
.await
|
||||
}
|
||||
|
||||
#[implement(Service)]
|
||||
pub async fn get_session(
|
||||
&self,
|
||||
user_id: &UserId,
|
||||
version: &str,
|
||||
room_id: &RoomId,
|
||||
session_id: &str,
|
||||
) -> Result<Raw<KeyBackupData>> {
|
||||
let key = (user_id, version, room_id, session_id);
|
||||
|
||||
self.db.backupkeyid_backup.qry(&key).await.deserialized()
|
||||
}
|
||||
|
||||
#[implement(Service)]
|
||||
pub async fn delete_all_keys(&self, user_id: &UserId, version: &str) {
|
||||
let key = (user_id, version, Interfix);
|
||||
self.db
|
||||
.backupkeyid_backup
|
||||
.keys_prefix_raw(&key)
|
||||
.ignore_err()
|
||||
.ready_for_each(|outdated_key| self.db.backupkeyid_backup.remove(outdated_key))
|
||||
.await;
|
||||
}
|
||||
|
||||
#[implement(Service)]
|
||||
pub async fn delete_room_keys(&self, user_id: &UserId, version: &str, room_id: &RoomId) {
|
||||
let key = (user_id, version, room_id, Interfix);
|
||||
self.db
|
||||
.backupkeyid_backup
|
||||
.keys_prefix_raw(&key)
|
||||
.ignore_err()
|
||||
.ready_for_each(|outdated_key| {
|
||||
self.db.backupkeyid_backup.remove(outdated_key);
|
||||
})
|
||||
.await;
|
||||
}
|
||||
|
||||
#[implement(Service)]
|
||||
pub async fn delete_room_key(
|
||||
&self,
|
||||
user_id: &UserId,
|
||||
version: &str,
|
||||
room_id: &RoomId,
|
||||
session_id: &str,
|
||||
) {
|
||||
let key = (user_id, version, room_id, session_id);
|
||||
self.db
|
||||
.backupkeyid_backup
|
||||
.keys_prefix_raw(&key)
|
||||
.ignore_err()
|
||||
.ready_for_each(|outdated_key| {
|
||||
self.db.backupkeyid_backup.remove(outdated_key);
|
||||
})
|
||||
.await;
|
||||
Ok(version)
|
||||
}
|
||||
|
||||
pub async fn get_latest_backup_version(&self, user_id: &UserId) -> Result<String> {
|
||||
type Key<'a> = (OwnedUserId, &'a str);
|
||||
|
||||
let last_possible_key = (user_id, u64::MAX);
|
||||
self.db
|
||||
.backupid_algorithm
|
||||
.rev_keys_from(&last_possible_key)
|
||||
.ignore_err()
|
||||
.ready_take_while(|(user_id_, _): &Key<'_>| *user_id_ == user_id)
|
||||
.map(|(_, version): Key<'_>| version.to_owned())
|
||||
.next()
|
||||
.await
|
||||
.ok_or_else(|| err!(Request(NotFound("No backup versions found"))))
|
||||
}
|
||||
|
||||
pub async fn get_latest_backup(
|
||||
&self,
|
||||
user_id: &UserId,
|
||||
) -> Result<(String, Raw<BackupAlgorithm>)> {
|
||||
type Key<'a> = (OwnedUserId, &'a str);
|
||||
type KeyVal<'a> = (Key<'a>, Raw<BackupAlgorithm>);
|
||||
|
||||
let last_possible_key = (user_id, u64::MAX);
|
||||
self.db
|
||||
.backupid_algorithm
|
||||
.rev_stream_from(&last_possible_key)
|
||||
.ignore_err()
|
||||
.ready_take_while(|((user_id_, _), _): &KeyVal<'_>| *user_id_ == user_id)
|
||||
.map(|((_, version), algorithm): KeyVal<'_>| (version.to_owned(), algorithm))
|
||||
.next()
|
||||
.await
|
||||
.ok_or_else(|| err!(Request(NotFound("No backup found"))))
|
||||
}
|
||||
|
||||
pub async fn get_backup(
|
||||
&self,
|
||||
user_id: &UserId,
|
||||
version: &str,
|
||||
) -> Result<Raw<BackupAlgorithm>> {
|
||||
let key = (user_id, version);
|
||||
self.db.backupid_algorithm.qry(&key).await.deserialized()
|
||||
}
|
||||
|
||||
pub async fn add_key(
|
||||
&self,
|
||||
user_id: &UserId,
|
||||
version: &str,
|
||||
room_id: &RoomId,
|
||||
session_id: &str,
|
||||
key_data: &Raw<KeyBackupData>,
|
||||
) -> Result<()> {
|
||||
let key = (user_id, version);
|
||||
if self.db.backupid_algorithm.qry(&key).await.is_err() {
|
||||
return Err!(Request(NotFound("Tried to update nonexistent backup.")));
|
||||
}
|
||||
|
||||
let count = self.services.globals.next_count().unwrap();
|
||||
self.db.backupid_etag.put(key, count);
|
||||
|
||||
let key = (user_id, version, room_id, session_id);
|
||||
self.db
|
||||
.backupkeyid_backup
|
||||
.put_raw(key, key_data.json().get());
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn count_keys(&self, user_id: &UserId, version: &str) -> usize {
|
||||
let prefix = (user_id, version);
|
||||
self.db
|
||||
.backupkeyid_backup
|
||||
.keys_prefix_raw(&prefix)
|
||||
.count()
|
||||
.await
|
||||
}
|
||||
|
||||
pub async fn get_etag(&self, user_id: &UserId, version: &str) -> String {
|
||||
let key = (user_id, version);
|
||||
self.db
|
||||
.backupid_etag
|
||||
.qry(&key)
|
||||
.await
|
||||
.deserialized::<u64>()
|
||||
.as_ref()
|
||||
.map(ToString::to_string)
|
||||
.expect("Backup has no etag.")
|
||||
}
|
||||
|
||||
pub async fn get_all(
|
||||
&self,
|
||||
user_id: &UserId,
|
||||
version: &str,
|
||||
) -> BTreeMap<OwnedRoomId, RoomKeyBackup> {
|
||||
type Key<'a> = (Ignore, Ignore, OwnedRoomId, &'a str);
|
||||
type KeyVal<'a> = (Key<'a>, Raw<KeyBackupData>);
|
||||
|
||||
let mut rooms = BTreeMap::<OwnedRoomId, RoomKeyBackup>::new();
|
||||
let default = || RoomKeyBackup::new(BTreeMap::new());
|
||||
|
||||
let prefix = (user_id, version, Interfix);
|
||||
self.db
|
||||
.backupkeyid_backup
|
||||
.stream_prefix(&prefix)
|
||||
.ignore_err()
|
||||
.ready_for_each(|((_, _, room_id, session_id), key_backup_data): KeyVal<'_>| {
|
||||
rooms
|
||||
.entry(room_id)
|
||||
.or_insert_with(default)
|
||||
.sessions
|
||||
.insert(session_id.into(), key_backup_data);
|
||||
})
|
||||
.await;
|
||||
|
||||
rooms
|
||||
}
|
||||
|
||||
pub async fn get_room(
|
||||
&self,
|
||||
user_id: &UserId,
|
||||
version: &str,
|
||||
room_id: &RoomId,
|
||||
) -> BTreeMap<String, Raw<KeyBackupData>> {
|
||||
type KeyVal<'a> = ((Ignore, Ignore, Ignore, &'a str), Raw<KeyBackupData>);
|
||||
|
||||
let prefix = (user_id, version, room_id, Interfix);
|
||||
self.db
|
||||
.backupkeyid_backup
|
||||
.stream_prefix(&prefix)
|
||||
.ignore_err()
|
||||
.map(|((.., session_id), key_backup_data): KeyVal<'_>| {
|
||||
(session_id.to_owned(), key_backup_data)
|
||||
})
|
||||
.collect()
|
||||
.await
|
||||
}
|
||||
|
||||
pub async fn get_session(
|
||||
&self,
|
||||
user_id: &UserId,
|
||||
version: &str,
|
||||
room_id: &RoomId,
|
||||
session_id: &str,
|
||||
) -> Result<Raw<KeyBackupData>> {
|
||||
let key = (user_id, version, room_id, session_id);
|
||||
|
||||
self.db.backupkeyid_backup.qry(&key).await.deserialized()
|
||||
}
|
||||
|
||||
pub async fn delete_all_keys(&self, user_id: &UserId, version: &str) {
|
||||
let key = (user_id, version, Interfix);
|
||||
self.db
|
||||
.backupkeyid_backup
|
||||
.keys_prefix_raw(&key)
|
||||
.ignore_err()
|
||||
.ready_for_each(|outdated_key| self.db.backupkeyid_backup.remove(outdated_key))
|
||||
.await;
|
||||
}
|
||||
|
||||
pub async fn delete_room_keys(&self, user_id: &UserId, version: &str, room_id: &RoomId) {
|
||||
let key = (user_id, version, room_id, Interfix);
|
||||
self.db
|
||||
.backupkeyid_backup
|
||||
.keys_prefix_raw(&key)
|
||||
.ignore_err()
|
||||
.ready_for_each(|outdated_key| {
|
||||
self.db.backupkeyid_backup.remove(outdated_key);
|
||||
})
|
||||
.await;
|
||||
}
|
||||
|
||||
pub async fn delete_room_key(
|
||||
&self,
|
||||
user_id: &UserId,
|
||||
version: &str,
|
||||
room_id: &RoomId,
|
||||
session_id: &str,
|
||||
) {
|
||||
let key = (user_id, version, room_id, session_id);
|
||||
self.db
|
||||
.backupkeyid_backup
|
||||
.keys_prefix_raw(&key)
|
||||
.ignore_err()
|
||||
.ready_for_each(|outdated_key| {
|
||||
self.db.backupkeyid_backup.remove(outdated_key);
|
||||
})
|
||||
.await;
|
||||
}
|
||||
}
|
||||
+378
-384
@@ -10,7 +10,6 @@ use std::time::SystemTime;
|
||||
#[cfg(feature = "url_preview")]
|
||||
use conduwuit::utils::response::LimitReadExt;
|
||||
use conduwuit::{Err, Result, debug, err};
|
||||
use conduwuit_core::implement;
|
||||
use ipaddress::IPAddress;
|
||||
#[cfg(feature = "url_preview")]
|
||||
use ruma::OwnedMxcUri;
|
||||
@@ -49,407 +48,402 @@ pub struct UrlPreviewData {
|
||||
pub audio_size: Option<usize>,
|
||||
}
|
||||
|
||||
#[implement(Service)]
|
||||
pub async fn remove_url_preview(&self, url: &str) -> Result<()> {
|
||||
// TODO: also remove the downloaded image
|
||||
self.db.remove_url_preview(url)
|
||||
}
|
||||
|
||||
#[implement(Service)]
|
||||
pub async fn clear_url_previews(&self) { self.db.clear_url_previews().await; }
|
||||
|
||||
#[implement(Service)]
|
||||
pub async fn set_url_preview(&self, url: &str, data: &UrlPreviewData) -> Result<()> {
|
||||
let now = SystemTime::now()
|
||||
.duration_since(SystemTime::UNIX_EPOCH)
|
||||
.expect("valid system time");
|
||||
self.db.set_url_preview(url, data, now)
|
||||
}
|
||||
|
||||
#[implement(Service)]
|
||||
pub async fn get_url_preview(&self, url: &Url) -> Result<UrlPreviewData> {
|
||||
if let Ok(preview) = self.db.get_url_preview(url.as_str()).await {
|
||||
return Ok(preview);
|
||||
impl Service {
|
||||
pub async fn remove_url_preview(&self, url: &str) -> Result<()> {
|
||||
// TODO: also remove the downloaded image
|
||||
self.db.remove_url_preview(url)
|
||||
}
|
||||
|
||||
// ensure that only one request is made per URL
|
||||
let _request_lock = self.url_preview_mutex.lock(url.as_str()).await;
|
||||
pub async fn clear_url_previews(&self) { self.db.clear_url_previews().await; }
|
||||
|
||||
match self.db.get_url_preview(url.as_str()).await {
|
||||
| Ok(preview) => Ok(preview),
|
||||
| Err(_) => self.request_url_preview(url).await,
|
||||
pub async fn set_url_preview(&self, url: &str, data: &UrlPreviewData) -> Result<()> {
|
||||
let now = SystemTime::now()
|
||||
.duration_since(SystemTime::UNIX_EPOCH)
|
||||
.expect("valid system time");
|
||||
self.db.set_url_preview(url, data, now)
|
||||
}
|
||||
}
|
||||
|
||||
#[implement(Service)]
|
||||
async fn request_url_preview(&self, url: &Url) -> Result<UrlPreviewData> {
|
||||
if let Ok(ip) = IPAddress::parse(url.host_str().expect("URL previously validated")) {
|
||||
if !self.services.client.valid_cidr_range(&ip) {
|
||||
return Err!(Request(Forbidden("Requesting from this address is forbidden")));
|
||||
pub async fn get_url_preview(&self, url: &Url) -> Result<UrlPreviewData> {
|
||||
if let Ok(preview) = self.db.get_url_preview(url.as_str()).await {
|
||||
return Ok(preview);
|
||||
}
|
||||
|
||||
// ensure that only one request is made per URL
|
||||
let _request_lock = self.url_preview_mutex.lock(url.as_str()).await;
|
||||
|
||||
match self.db.get_url_preview(url.as_str()).await {
|
||||
| Ok(preview) => Ok(preview),
|
||||
| Err(_) => self.request_url_preview(url).await,
|
||||
}
|
||||
}
|
||||
|
||||
let client = &self.services.client.url_preview;
|
||||
let response = client.head(url.as_str()).send().await?;
|
||||
|
||||
debug!(%url, "URL preview response headers: {:?}", response.headers());
|
||||
|
||||
if let Some(remote_addr) = response.remote_addr() {
|
||||
debug!(%url, "URL preview response remote address: {:?}", remote_addr);
|
||||
|
||||
if let Ok(ip) = IPAddress::parse(remote_addr.ip().to_string()) {
|
||||
async fn request_url_preview(&self, url: &Url) -> Result<UrlPreviewData> {
|
||||
if let Ok(ip) = IPAddress::parse(url.host_str().expect("URL previously validated")) {
|
||||
if !self.services.client.valid_cidr_range(&ip) {
|
||||
return Err!(Request(Forbidden("Requesting from this address is forbidden")));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let Some(content_type) = response.headers().get(reqwest::header::CONTENT_TYPE) else {
|
||||
return Err!(Request(Unknown("Unknown or invalid Content-Type header")));
|
||||
};
|
||||
let client = &self.services.client.url_preview;
|
||||
let response = client.head(url.as_str()).send().await?;
|
||||
|
||||
let content_type = content_type
|
||||
.to_str()
|
||||
.map_err(|e| err!(Request(Unknown("Unknown or invalid Content-Type header: {e}"))))?;
|
||||
debug!(%url, "URL preview response headers: {:?}", response.headers());
|
||||
|
||||
let data = match content_type {
|
||||
| html if html.starts_with("text/html") => self.download_html(url.as_str()).await?,
|
||||
| img if img.starts_with("image/") => self.download_image(url.as_str(), None).await?,
|
||||
| video if video.starts_with("video/") => self.download_video(url.as_str(), None).await?,
|
||||
| audio if audio.starts_with("audio/") => self.download_audio(url.as_str(), None).await?,
|
||||
| _ => return Err!(Request(Unknown("Unsupported Content-Type"))),
|
||||
};
|
||||
if let Some(remote_addr) = response.remote_addr() {
|
||||
debug!(%url, "URL preview response remote address: {:?}", remote_addr);
|
||||
|
||||
self.set_url_preview(url.as_str(), &data).await?;
|
||||
|
||||
Ok(data)
|
||||
}
|
||||
|
||||
#[cfg(feature = "url_preview")]
|
||||
#[implement(Service)]
|
||||
pub async fn download_image(
|
||||
&self,
|
||||
url: &str,
|
||||
preview_data: Option<UrlPreviewData>,
|
||||
) -> Result<UrlPreviewData> {
|
||||
use conduwuit::utils::random_string;
|
||||
use image::ImageReader;
|
||||
|
||||
use crate::media::mxc::Mxc;
|
||||
|
||||
let mut preview_data = preview_data.unwrap_or_default();
|
||||
|
||||
let image = self
|
||||
.services
|
||||
.client
|
||||
.url_preview
|
||||
.get(url)
|
||||
.send()
|
||||
.await?
|
||||
.limit_read(
|
||||
self.services
|
||||
.server
|
||||
.config
|
||||
.max_request_size
|
||||
.try_into()
|
||||
.expect("u64 should fit in usize"),
|
||||
)
|
||||
.await?;
|
||||
|
||||
let mxc = Mxc {
|
||||
server_name: self.services.globals.server_name(),
|
||||
media_id: &random_string(super::MXC_LENGTH),
|
||||
};
|
||||
|
||||
self.create(&mxc, None, None, None, &image).await?;
|
||||
|
||||
preview_data.image = Some(mxc.to_string());
|
||||
if preview_data.image_height.is_none() || preview_data.image_width.is_none() {
|
||||
let cursor = std::io::Cursor::new(&image);
|
||||
let (width, height) = match ImageReader::new(cursor).with_guessed_format() {
|
||||
| Err(_) => (None, None),
|
||||
| Ok(reader) => match reader.into_dimensions() {
|
||||
| Err(_) => (None, None),
|
||||
| Ok((width, height)) => (Some(width), Some(height)),
|
||||
},
|
||||
};
|
||||
|
||||
preview_data.image_width = width;
|
||||
preview_data.image_height = height;
|
||||
}
|
||||
|
||||
Ok(preview_data)
|
||||
}
|
||||
|
||||
#[cfg(feature = "url_preview")]
|
||||
#[implement(Service)]
|
||||
pub async fn download_video(
|
||||
&self,
|
||||
url: &str,
|
||||
preview_data: Option<UrlPreviewData>,
|
||||
) -> Result<UrlPreviewData> {
|
||||
let mut preview_data = preview_data.unwrap_or_default();
|
||||
|
||||
if self.services.globals.url_preview_allow_audio_video() {
|
||||
let (url, size) = self.download_media(url).await?;
|
||||
preview_data.video = Some(url.to_string());
|
||||
preview_data.video_size = Some(size);
|
||||
}
|
||||
|
||||
Ok(preview_data)
|
||||
}
|
||||
|
||||
#[cfg(feature = "url_preview")]
|
||||
#[implement(Service)]
|
||||
pub async fn download_audio(
|
||||
&self,
|
||||
url: &str,
|
||||
preview_data: Option<UrlPreviewData>,
|
||||
) -> Result<UrlPreviewData> {
|
||||
let mut preview_data = preview_data.unwrap_or_default();
|
||||
|
||||
if self.services.globals.url_preview_allow_audio_video() {
|
||||
let (url, size) = self.download_media(url).await?;
|
||||
preview_data.audio = Some(url.to_string());
|
||||
preview_data.audio_size = Some(size);
|
||||
}
|
||||
|
||||
Ok(preview_data)
|
||||
}
|
||||
|
||||
#[cfg(feature = "url_preview")]
|
||||
#[implement(Service)]
|
||||
pub async fn download_media(&self, url: &str) -> Result<(OwnedMxcUri, usize)> {
|
||||
use conduwuit::utils::random_string;
|
||||
use http::header::CONTENT_TYPE;
|
||||
|
||||
let response = self.services.client.url_preview.get(url).send().await?;
|
||||
let content_type = response.headers().get(CONTENT_TYPE).cloned();
|
||||
let media = response
|
||||
.limit_read(
|
||||
self.services
|
||||
.server
|
||||
.config
|
||||
.max_request_size
|
||||
.try_into()
|
||||
.expect("u64 should fit in usize"),
|
||||
)
|
||||
.await?;
|
||||
|
||||
let mxc = Mxc {
|
||||
server_name: self.services.globals.server_name(),
|
||||
media_id: &random_string(super::MXC_LENGTH),
|
||||
};
|
||||
|
||||
let content_type = content_type.and_then(|v| v.to_str().map(ToOwned::to_owned).ok());
|
||||
self.create(&mxc, None, None, content_type.as_deref(), &media)
|
||||
.await?;
|
||||
|
||||
Ok((OwnedMxcUri::from(mxc.to_string()), media.len()))
|
||||
}
|
||||
|
||||
#[cfg(not(feature = "url_preview"))]
|
||||
#[implement(Service)]
|
||||
pub async fn download_image(
|
||||
&self,
|
||||
_url: &str,
|
||||
_preview_data: Option<UrlPreviewData>,
|
||||
) -> Result<UrlPreviewData> {
|
||||
Err!(FeatureDisabled("url_preview"))
|
||||
}
|
||||
|
||||
#[cfg(not(feature = "url_preview"))]
|
||||
#[implement(Service)]
|
||||
pub async fn download_video(
|
||||
&self,
|
||||
_url: &str,
|
||||
_preview_data: Option<UrlPreviewData>,
|
||||
) -> Result<UrlPreviewData> {
|
||||
Err!(FeatureDisabled("url_preview"))
|
||||
}
|
||||
|
||||
#[cfg(not(feature = "url_preview"))]
|
||||
#[implement(Service)]
|
||||
pub async fn download_audio(
|
||||
&self,
|
||||
_url: &str,
|
||||
_preview_data: Option<UrlPreviewData>,
|
||||
) -> Result<UrlPreviewData> {
|
||||
Err!(FeatureDisabled("url_preview"))
|
||||
}
|
||||
|
||||
#[cfg(not(feature = "url_preview"))]
|
||||
#[implement(Service)]
|
||||
pub async fn download_media(&self, _url: &str) -> Result<UrlPreviewData> {
|
||||
Err!(FeatureDisabled("url_preview"))
|
||||
}
|
||||
|
||||
#[cfg(feature = "url_preview")]
|
||||
#[implement(Service)]
|
||||
async fn download_html(&self, url: &str) -> Result<UrlPreviewData> {
|
||||
use webpage::HTML;
|
||||
|
||||
let client = &self.services.client.url_preview;
|
||||
let body = client
|
||||
.get(url)
|
||||
.send()
|
||||
.await?
|
||||
.limit_read_text(
|
||||
self.services
|
||||
.server
|
||||
.config
|
||||
.max_request_size
|
||||
.try_into()
|
||||
.expect("u64 should fit in usize"),
|
||||
)
|
||||
.await?;
|
||||
let Ok(html) = HTML::from_string(body.clone(), Some(url.to_owned())) else {
|
||||
return Err!(Request(Unknown("Failed to parse HTML")));
|
||||
};
|
||||
|
||||
let mut preview_data = UrlPreviewData::default();
|
||||
|
||||
if let Some(obj) = html.opengraph.images.first() {
|
||||
preview_data = self.download_image(&obj.url, Some(preview_data)).await?;
|
||||
}
|
||||
|
||||
if let Some(obj) = html.opengraph.videos.first() {
|
||||
preview_data = self.download_video(&obj.url, Some(preview_data)).await?;
|
||||
preview_data.video_width = obj.properties.get("width").and_then(|v| v.parse().ok());
|
||||
preview_data.video_height = obj.properties.get("height").and_then(|v| v.parse().ok());
|
||||
}
|
||||
|
||||
if let Some(obj) = html.opengraph.audios.first() {
|
||||
preview_data = self.download_audio(&obj.url, Some(preview_data)).await?;
|
||||
}
|
||||
|
||||
let props = html.opengraph.properties;
|
||||
|
||||
/* use OpenGraph title/description, but fall back to HTML if not available */
|
||||
preview_data.title = props.get("title").cloned().or(html.title);
|
||||
preview_data.description = props.get("description").cloned().or(html.description);
|
||||
|
||||
Ok(preview_data)
|
||||
}
|
||||
|
||||
#[cfg(not(feature = "url_preview"))]
|
||||
#[implement(Service)]
|
||||
async fn download_html(&self, _url: &str) -> Result<UrlPreviewData> {
|
||||
Err!(FeatureDisabled("url_preview"))
|
||||
}
|
||||
|
||||
#[implement(Service)]
|
||||
pub fn url_preview_allowed(&self, url: &Url) -> bool {
|
||||
if ["http", "https"]
|
||||
.iter()
|
||||
.all(|&scheme| scheme != url.scheme().to_lowercase())
|
||||
{
|
||||
debug!("Ignoring non-HTTP/HTTPS URL to preview: {}", url);
|
||||
return false;
|
||||
}
|
||||
|
||||
let host = match url.host_str() {
|
||||
| None => {
|
||||
debug!("Ignoring URL preview for a URL that does not have a host (?): {}", url);
|
||||
return false;
|
||||
},
|
||||
| Some(h) => h.to_owned(),
|
||||
};
|
||||
|
||||
let allowlist_domain_contains = self
|
||||
.services
|
||||
.globals
|
||||
.url_preview_domain_contains_allowlist();
|
||||
let allowlist_domain_explicit = self
|
||||
.services
|
||||
.globals
|
||||
.url_preview_domain_explicit_allowlist();
|
||||
let denylist_domain_explicit = self.services.globals.url_preview_domain_explicit_denylist();
|
||||
let allowlist_url_contains = self.services.globals.url_preview_url_contains_allowlist();
|
||||
|
||||
if allowlist_domain_contains.contains(&"*".to_owned())
|
||||
|| allowlist_domain_explicit.contains(&"*".to_owned())
|
||||
|| allowlist_url_contains.contains(&"*".to_owned())
|
||||
{
|
||||
debug!("Config key contains * which is allowing all URL previews. Allowing URL {}", url);
|
||||
return true;
|
||||
}
|
||||
|
||||
if !host.is_empty() {
|
||||
if denylist_domain_explicit.contains(&host) {
|
||||
debug!(
|
||||
"Host {} is not allowed by url_preview_domain_explicit_denylist (check 1/4)",
|
||||
&host
|
||||
);
|
||||
return false;
|
||||
}
|
||||
|
||||
if allowlist_domain_explicit.contains(&host) {
|
||||
debug!(
|
||||
"Host {} is allowed by url_preview_domain_explicit_allowlist (check 2/4)",
|
||||
&host
|
||||
);
|
||||
return true;
|
||||
}
|
||||
|
||||
if allowlist_domain_contains
|
||||
.iter()
|
||||
.any(|domain_s| domain_s.contains(&host.clone()))
|
||||
{
|
||||
debug!(
|
||||
"Host {} is allowed by url_preview_domain_contains_allowlist (check 3/4)",
|
||||
&host
|
||||
);
|
||||
return true;
|
||||
}
|
||||
|
||||
if allowlist_url_contains
|
||||
.iter()
|
||||
.any(|url_s| url.to_string().contains(url_s))
|
||||
{
|
||||
debug!("URL {} is allowed by url_preview_url_contains_allowlist (check 4/4)", &host);
|
||||
return true;
|
||||
}
|
||||
|
||||
// check root domain if available and if user has root domain checks
|
||||
if self.services.globals.url_preview_check_root_domain() {
|
||||
debug!("Checking root domain");
|
||||
match host.split_once('.') {
|
||||
| None => return false,
|
||||
| Some((_, root_domain)) => {
|
||||
if denylist_domain_explicit.contains(&root_domain.to_owned()) {
|
||||
debug!(
|
||||
"Root domain {} is not allowed by \
|
||||
url_preview_domain_explicit_denylist (check 1/3)",
|
||||
&root_domain
|
||||
);
|
||||
return false;
|
||||
}
|
||||
|
||||
if allowlist_domain_explicit.contains(&root_domain.to_owned()) {
|
||||
debug!(
|
||||
"Root domain {} is allowed by url_preview_domain_explicit_allowlist \
|
||||
(check 2/3)",
|
||||
&root_domain
|
||||
);
|
||||
return true;
|
||||
}
|
||||
|
||||
if allowlist_domain_contains
|
||||
.iter()
|
||||
.any(|domain_s| domain_s.contains(&root_domain.to_owned()))
|
||||
{
|
||||
debug!(
|
||||
"Root domain {} is allowed by url_preview_domain_contains_allowlist \
|
||||
(check 3/3)",
|
||||
&root_domain
|
||||
);
|
||||
return true;
|
||||
}
|
||||
},
|
||||
if let Ok(ip) = IPAddress::parse(remote_addr.ip().to_string()) {
|
||||
if !self.services.client.valid_cidr_range(&ip) {
|
||||
return Err!(Request(Forbidden("Requesting from this address is forbidden")));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let Some(content_type) = response.headers().get(reqwest::header::CONTENT_TYPE) else {
|
||||
return Err!(Request(Unknown("Unknown or invalid Content-Type header")));
|
||||
};
|
||||
|
||||
let content_type = content_type
|
||||
.to_str()
|
||||
.map_err(|e| err!(Request(Unknown("Unknown or invalid Content-Type header: {e}"))))?;
|
||||
|
||||
let data = match content_type {
|
||||
| html if html.starts_with("text/html") => self.download_html(url.as_str()).await?,
|
||||
| img if img.starts_with("image/") => self.download_image(url.as_str(), None).await?,
|
||||
| video if video.starts_with("video/") =>
|
||||
self.download_video(url.as_str(), None).await?,
|
||||
| audio if audio.starts_with("audio/") =>
|
||||
self.download_audio(url.as_str(), None).await?,
|
||||
| _ => return Err!(Request(Unknown("Unsupported Content-Type"))),
|
||||
};
|
||||
|
||||
self.set_url_preview(url.as_str(), &data).await?;
|
||||
|
||||
Ok(data)
|
||||
}
|
||||
|
||||
false
|
||||
#[cfg(feature = "url_preview")]
|
||||
pub async fn download_image(
|
||||
&self,
|
||||
url: &str,
|
||||
preview_data: Option<UrlPreviewData>,
|
||||
) -> Result<UrlPreviewData> {
|
||||
use conduwuit::utils::random_string;
|
||||
use image::ImageReader;
|
||||
|
||||
use crate::media::mxc::Mxc;
|
||||
|
||||
let mut preview_data = preview_data.unwrap_or_default();
|
||||
|
||||
let image = self
|
||||
.services
|
||||
.client
|
||||
.url_preview
|
||||
.get(url)
|
||||
.send()
|
||||
.await?
|
||||
.limit_read(
|
||||
self.services
|
||||
.server
|
||||
.config
|
||||
.max_request_size
|
||||
.try_into()
|
||||
.expect("u64 should fit in usize"),
|
||||
)
|
||||
.await?;
|
||||
|
||||
let mxc = Mxc {
|
||||
server_name: self.services.globals.server_name(),
|
||||
media_id: &random_string(super::MXC_LENGTH),
|
||||
};
|
||||
|
||||
self.create(&mxc, None, None, None, &image).await?;
|
||||
|
||||
preview_data.image = Some(mxc.to_string());
|
||||
if preview_data.image_height.is_none() || preview_data.image_width.is_none() {
|
||||
let cursor = std::io::Cursor::new(&image);
|
||||
let (width, height) = match ImageReader::new(cursor).with_guessed_format() {
|
||||
| Err(_) => (None, None),
|
||||
| Ok(reader) => match reader.into_dimensions() {
|
||||
| Err(_) => (None, None),
|
||||
| Ok((width, height)) => (Some(width), Some(height)),
|
||||
},
|
||||
};
|
||||
|
||||
preview_data.image_width = width;
|
||||
preview_data.image_height = height;
|
||||
}
|
||||
|
||||
Ok(preview_data)
|
||||
}
|
||||
|
||||
#[cfg(feature = "url_preview")]
|
||||
pub async fn download_video(
|
||||
&self,
|
||||
url: &str,
|
||||
preview_data: Option<UrlPreviewData>,
|
||||
) -> Result<UrlPreviewData> {
|
||||
let mut preview_data = preview_data.unwrap_or_default();
|
||||
|
||||
if self.services.globals.url_preview_allow_audio_video() {
|
||||
let (url, size) = self.download_media(url).await?;
|
||||
preview_data.video = Some(url.to_string());
|
||||
preview_data.video_size = Some(size);
|
||||
}
|
||||
|
||||
Ok(preview_data)
|
||||
}
|
||||
|
||||
#[cfg(feature = "url_preview")]
|
||||
pub async fn download_audio(
|
||||
&self,
|
||||
url: &str,
|
||||
preview_data: Option<UrlPreviewData>,
|
||||
) -> Result<UrlPreviewData> {
|
||||
let mut preview_data = preview_data.unwrap_or_default();
|
||||
|
||||
if self.services.globals.url_preview_allow_audio_video() {
|
||||
let (url, size) = self.download_media(url).await?;
|
||||
preview_data.audio = Some(url.to_string());
|
||||
preview_data.audio_size = Some(size);
|
||||
}
|
||||
|
||||
Ok(preview_data)
|
||||
}
|
||||
|
||||
#[cfg(feature = "url_preview")]
|
||||
pub async fn download_media(&self, url: &str) -> Result<(OwnedMxcUri, usize)> {
|
||||
use conduwuit::utils::random_string;
|
||||
use http::header::CONTENT_TYPE;
|
||||
|
||||
let response = self.services.client.url_preview.get(url).send().await?;
|
||||
let content_type = response.headers().get(CONTENT_TYPE).cloned();
|
||||
let media = response
|
||||
.limit_read(
|
||||
self.services
|
||||
.server
|
||||
.config
|
||||
.max_request_size
|
||||
.try_into()
|
||||
.expect("u64 should fit in usize"),
|
||||
)
|
||||
.await?;
|
||||
|
||||
let mxc = Mxc {
|
||||
server_name: self.services.globals.server_name(),
|
||||
media_id: &random_string(super::MXC_LENGTH),
|
||||
};
|
||||
|
||||
let content_type = content_type.and_then(|v| v.to_str().map(ToOwned::to_owned).ok());
|
||||
self.create(&mxc, None, None, content_type.as_deref(), &media)
|
||||
.await?;
|
||||
|
||||
Ok((OwnedMxcUri::from(mxc.to_string()), media.len()))
|
||||
}
|
||||
|
||||
#[cfg(not(feature = "url_preview"))]
|
||||
pub async fn download_image(
|
||||
&self,
|
||||
_url: &str,
|
||||
_preview_data: Option<UrlPreviewData>,
|
||||
) -> Result<UrlPreviewData> {
|
||||
Err!(FeatureDisabled("url_preview"))
|
||||
}
|
||||
|
||||
#[cfg(not(feature = "url_preview"))]
|
||||
pub async fn download_video(
|
||||
&self,
|
||||
_url: &str,
|
||||
_preview_data: Option<UrlPreviewData>,
|
||||
) -> Result<UrlPreviewData> {
|
||||
Err!(FeatureDisabled("url_preview"))
|
||||
}
|
||||
|
||||
#[cfg(not(feature = "url_preview"))]
|
||||
pub async fn download_audio(
|
||||
&self,
|
||||
_url: &str,
|
||||
_preview_data: Option<UrlPreviewData>,
|
||||
) -> Result<UrlPreviewData> {
|
||||
Err!(FeatureDisabled("url_preview"))
|
||||
}
|
||||
|
||||
#[cfg(not(feature = "url_preview"))]
|
||||
pub async fn download_media(&self, _url: &str) -> Result<UrlPreviewData> {
|
||||
Err!(FeatureDisabled("url_preview"))
|
||||
}
|
||||
|
||||
#[cfg(feature = "url_preview")]
|
||||
async fn download_html(&self, url: &str) -> Result<UrlPreviewData> {
|
||||
use webpage::HTML;
|
||||
|
||||
let client = &self.services.client.url_preview;
|
||||
let body = client
|
||||
.get(url)
|
||||
.send()
|
||||
.await?
|
||||
.limit_read_text(
|
||||
self.services
|
||||
.server
|
||||
.config
|
||||
.max_request_size
|
||||
.try_into()
|
||||
.expect("u64 should fit in usize"),
|
||||
)
|
||||
.await?;
|
||||
let Ok(html) = HTML::from_string(body.clone(), Some(url.to_owned())) else {
|
||||
return Err!(Request(Unknown("Failed to parse HTML")));
|
||||
};
|
||||
|
||||
let mut preview_data = UrlPreviewData::default();
|
||||
|
||||
if let Some(obj) = html.opengraph.images.first() {
|
||||
preview_data = self.download_image(&obj.url, Some(preview_data)).await?;
|
||||
}
|
||||
|
||||
if let Some(obj) = html.opengraph.videos.first() {
|
||||
preview_data = self.download_video(&obj.url, Some(preview_data)).await?;
|
||||
preview_data.video_width = obj.properties.get("width").and_then(|v| v.parse().ok());
|
||||
preview_data.video_height = obj.properties.get("height").and_then(|v| v.parse().ok());
|
||||
}
|
||||
|
||||
if let Some(obj) = html.opengraph.audios.first() {
|
||||
preview_data = self.download_audio(&obj.url, Some(preview_data)).await?;
|
||||
}
|
||||
|
||||
let props = html.opengraph.properties;
|
||||
|
||||
/* use OpenGraph title/description, but fall back to HTML if not available */
|
||||
preview_data.title = props.get("title").cloned().or(html.title);
|
||||
preview_data.description = props.get("description").cloned().or(html.description);
|
||||
|
||||
Ok(preview_data)
|
||||
}
|
||||
|
||||
#[cfg(not(feature = "url_preview"))]
|
||||
async fn download_html(&self, _url: &str) -> Result<UrlPreviewData> {
|
||||
Err!(FeatureDisabled("url_preview"))
|
||||
}
|
||||
|
||||
pub fn url_preview_allowed(&self, url: &Url) -> bool {
|
||||
if ["http", "https"]
|
||||
.iter()
|
||||
.all(|&scheme| scheme != url.scheme().to_lowercase())
|
||||
{
|
||||
debug!("Ignoring non-HTTP/HTTPS URL to preview: {}", url);
|
||||
return false;
|
||||
}
|
||||
|
||||
let host = match url.host_str() {
|
||||
| None => {
|
||||
debug!("Ignoring URL preview for a URL that does not have a host (?): {}", url);
|
||||
return false;
|
||||
},
|
||||
| Some(h) => h.to_owned(),
|
||||
};
|
||||
|
||||
let allowlist_domain_contains = self
|
||||
.services
|
||||
.globals
|
||||
.url_preview_domain_contains_allowlist();
|
||||
let allowlist_domain_explicit = self
|
||||
.services
|
||||
.globals
|
||||
.url_preview_domain_explicit_allowlist();
|
||||
let denylist_domain_explicit =
|
||||
self.services.globals.url_preview_domain_explicit_denylist();
|
||||
let allowlist_url_contains = self.services.globals.url_preview_url_contains_allowlist();
|
||||
|
||||
if allowlist_domain_contains.contains(&"*".to_owned())
|
||||
|| allowlist_domain_explicit.contains(&"*".to_owned())
|
||||
|| allowlist_url_contains.contains(&"*".to_owned())
|
||||
{
|
||||
debug!(
|
||||
"Config key contains * which is allowing all URL previews. Allowing URL {}",
|
||||
url
|
||||
);
|
||||
return true;
|
||||
}
|
||||
|
||||
if !host.is_empty() {
|
||||
if denylist_domain_explicit.contains(&host) {
|
||||
debug!(
|
||||
"Host {} is not allowed by url_preview_domain_explicit_denylist (check 1/4)",
|
||||
&host
|
||||
);
|
||||
return false;
|
||||
}
|
||||
|
||||
if allowlist_domain_explicit.contains(&host) {
|
||||
debug!(
|
||||
"Host {} is allowed by url_preview_domain_explicit_allowlist (check 2/4)",
|
||||
&host
|
||||
);
|
||||
return true;
|
||||
}
|
||||
|
||||
if allowlist_domain_contains
|
||||
.iter()
|
||||
.any(|domain_s| domain_s.contains(&host.clone()))
|
||||
{
|
||||
debug!(
|
||||
"Host {} is allowed by url_preview_domain_contains_allowlist (check 3/4)",
|
||||
&host
|
||||
);
|
||||
return true;
|
||||
}
|
||||
|
||||
if allowlist_url_contains
|
||||
.iter()
|
||||
.any(|url_s| url.to_string().contains(url_s))
|
||||
{
|
||||
debug!(
|
||||
"URL {} is allowed by url_preview_url_contains_allowlist (check 4/4)",
|
||||
&host
|
||||
);
|
||||
return true;
|
||||
}
|
||||
|
||||
// check root domain if available and if user has root domain checks
|
||||
if self.services.globals.url_preview_check_root_domain() {
|
||||
debug!("Checking root domain");
|
||||
match host.split_once('.') {
|
||||
| None => return false,
|
||||
| Some((_, root_domain)) => {
|
||||
if denylist_domain_explicit.contains(&root_domain.to_owned()) {
|
||||
debug!(
|
||||
"Root domain {} is not allowed by \
|
||||
url_preview_domain_explicit_denylist (check 1/3)",
|
||||
&root_domain
|
||||
);
|
||||
return false;
|
||||
}
|
||||
|
||||
if allowlist_domain_explicit.contains(&root_domain.to_owned()) {
|
||||
debug!(
|
||||
"Root domain {} is allowed by \
|
||||
url_preview_domain_explicit_allowlist (check 2/3)",
|
||||
&root_domain
|
||||
);
|
||||
return true;
|
||||
}
|
||||
|
||||
if allowlist_domain_contains
|
||||
.iter()
|
||||
.any(|domain_s| domain_s.contains(&root_domain.to_owned()))
|
||||
{
|
||||
debug!(
|
||||
"Root domain {} is allowed by \
|
||||
url_preview_domain_contains_allowlist (check 3/3)",
|
||||
&root_domain
|
||||
);
|
||||
return true;
|
||||
}
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
false
|
||||
}
|
||||
}
|
||||
+385
-397
@@ -1,7 +1,7 @@
|
||||
use std::{fmt::Debug, time::Duration};
|
||||
|
||||
use conduwuit::{
|
||||
Err, Error, Result, debug_warn, err, implement,
|
||||
Err, Error, Result, debug_warn, err,
|
||||
utils::{content_disposition::make_content_disposition, response::LimitReadExt},
|
||||
};
|
||||
use http::header::{CONTENT_DISPOSITION, CONTENT_TYPE, HeaderValue};
|
||||
@@ -24,436 +24,424 @@ use ruma::{
|
||||
use super::{Dim, FileMeta};
|
||||
use crate::{federation::FederationPathBuilderInput, media::mxc::Mxc};
|
||||
|
||||
#[implement(super::Service)]
|
||||
pub async fn fetch_remote_thumbnail(
|
||||
&self,
|
||||
mxc: &Mxc<'_>,
|
||||
user: Option<&UserId>,
|
||||
server: Option<&ServerName>,
|
||||
timeout_ms: Duration,
|
||||
dim: &Dim,
|
||||
) -> Result<FileMeta> {
|
||||
self.check_fetch_authorized(mxc)?;
|
||||
impl super::Service {
|
||||
pub async fn fetch_remote_thumbnail(
|
||||
&self,
|
||||
mxc: &Mxc<'_>,
|
||||
user: Option<&UserId>,
|
||||
server: Option<&ServerName>,
|
||||
timeout_ms: Duration,
|
||||
dim: &Dim,
|
||||
) -> Result<FileMeta> {
|
||||
self.check_fetch_authorized(mxc)?;
|
||||
|
||||
let result = self
|
||||
.fetch_thumbnail_authenticated(mxc, user, server, timeout_ms, dim)
|
||||
.await;
|
||||
|
||||
if let Err(Error::Request(NotFound, ..)) = &result {
|
||||
return self
|
||||
.fetch_thumbnail_unauthenticated(mxc, user, server, timeout_ms, dim)
|
||||
let result = self
|
||||
.fetch_thumbnail_authenticated(mxc, user, server, timeout_ms, dim)
|
||||
.await;
|
||||
|
||||
if let Err(Error::Request(NotFound, ..)) = &result {
|
||||
return self
|
||||
.fetch_thumbnail_unauthenticated(mxc, user, server, timeout_ms, dim)
|
||||
.await;
|
||||
}
|
||||
|
||||
result
|
||||
}
|
||||
|
||||
result
|
||||
}
|
||||
pub async fn fetch_remote_content(
|
||||
&self,
|
||||
mxc: &Mxc<'_>,
|
||||
user: Option<&UserId>,
|
||||
server: Option<&ServerName>,
|
||||
timeout_ms: Duration,
|
||||
) -> Result<FileMeta> {
|
||||
self.check_fetch_authorized(mxc)?;
|
||||
|
||||
#[implement(super::Service)]
|
||||
pub async fn fetch_remote_content(
|
||||
&self,
|
||||
mxc: &Mxc<'_>,
|
||||
user: Option<&UserId>,
|
||||
server: Option<&ServerName>,
|
||||
timeout_ms: Duration,
|
||||
) -> Result<FileMeta> {
|
||||
self.check_fetch_authorized(mxc)?;
|
||||
let result = self
|
||||
.fetch_content_authenticated(mxc, user, server, timeout_ms)
|
||||
.await
|
||||
.inspect_err(|error| {
|
||||
debug_warn!(
|
||||
%mxc,
|
||||
?user,
|
||||
?server,
|
||||
?error,
|
||||
"Authenticated fetch of remote content failed"
|
||||
);
|
||||
});
|
||||
|
||||
let result = self
|
||||
.fetch_content_authenticated(mxc, user, server, timeout_ms)
|
||||
.await
|
||||
.inspect_err(|error| {
|
||||
debug_warn!(
|
||||
%mxc,
|
||||
?user,
|
||||
?server,
|
||||
?error,
|
||||
"Authenticated fetch of remote content failed"
|
||||
);
|
||||
});
|
||||
if let Err(Error::Request(Unrecognized, ..)) = &result {
|
||||
return self
|
||||
.fetch_content_unauthenticated(mxc, user, server, timeout_ms)
|
||||
.await;
|
||||
}
|
||||
|
||||
if let Err(Error::Request(Unrecognized, ..)) = &result {
|
||||
return self
|
||||
.fetch_content_unauthenticated(mxc, user, server, timeout_ms)
|
||||
.await;
|
||||
result
|
||||
}
|
||||
|
||||
result
|
||||
}
|
||||
async fn fetch_thumbnail_authenticated(
|
||||
&self,
|
||||
mxc: &Mxc<'_>,
|
||||
user: Option<&UserId>,
|
||||
server: Option<&ServerName>,
|
||||
timeout_ms: Duration,
|
||||
dim: &Dim,
|
||||
) -> Result<FileMeta> {
|
||||
use federation::authenticated_media::get_content_thumbnail::v1::{Request, Response};
|
||||
|
||||
#[implement(super::Service)]
|
||||
async fn fetch_thumbnail_authenticated(
|
||||
&self,
|
||||
mxc: &Mxc<'_>,
|
||||
user: Option<&UserId>,
|
||||
server: Option<&ServerName>,
|
||||
timeout_ms: Duration,
|
||||
dim: &Dim,
|
||||
) -> Result<FileMeta> {
|
||||
use federation::authenticated_media::get_content_thumbnail::v1::{Request, Response};
|
||||
let mut request = Request::new(mxc.media_id.into(), dim.width.into(), dim.height.into());
|
||||
request.method = Some(dim.method.clone());
|
||||
request.animated = Some(true);
|
||||
request.timeout_ms = timeout_ms;
|
||||
|
||||
let mut request = Request::new(mxc.media_id.into(), dim.width.into(), dim.height.into());
|
||||
request.method = Some(dim.method.clone());
|
||||
request.animated = Some(true);
|
||||
request.timeout_ms = timeout_ms;
|
||||
let Response { content, .. } = self.federation_request(mxc, server, request).await?;
|
||||
|
||||
let Response { content, .. } = self.federation_request(mxc, server, request).await?;
|
||||
|
||||
match content {
|
||||
| FileOrLocation::File(content) =>
|
||||
self.handle_thumbnail_file(mxc, user, dim, content).await,
|
||||
| FileOrLocation::Location(location) => self.handle_location(mxc, user, &location).await,
|
||||
| _ => Err!("Unknown content in response"),
|
||||
match content {
|
||||
| FileOrLocation::File(content) =>
|
||||
self.handle_thumbnail_file(mxc, user, dim, content).await,
|
||||
| FileOrLocation::Location(location) =>
|
||||
self.handle_location(mxc, user, &location).await,
|
||||
| _ => Err!("Unknown content in response"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[implement(super::Service)]
|
||||
async fn fetch_content_authenticated(
|
||||
&self,
|
||||
mxc: &Mxc<'_>,
|
||||
user: Option<&UserId>,
|
||||
server: Option<&ServerName>,
|
||||
timeout_ms: Duration,
|
||||
) -> Result<FileMeta> {
|
||||
use federation::authenticated_media::get_content::v1::{Request, Response};
|
||||
async fn fetch_content_authenticated(
|
||||
&self,
|
||||
mxc: &Mxc<'_>,
|
||||
user: Option<&UserId>,
|
||||
server: Option<&ServerName>,
|
||||
timeout_ms: Duration,
|
||||
) -> Result<FileMeta> {
|
||||
use federation::authenticated_media::get_content::v1::{Request, Response};
|
||||
|
||||
let mut request = Request::new(mxc.media_id.into());
|
||||
request.timeout_ms = timeout_ms;
|
||||
let mut request = Request::new(mxc.media_id.into());
|
||||
request.timeout_ms = timeout_ms;
|
||||
|
||||
let Response { content, .. } = self.federation_request(mxc, server, request).await?;
|
||||
let Response { content, .. } = self.federation_request(mxc, server, request).await?;
|
||||
|
||||
match content {
|
||||
| FileOrLocation::File(content) => self.handle_content_file(mxc, user, content).await,
|
||||
| FileOrLocation::Location(location) => self.handle_location(mxc, user, &location).await,
|
||||
| _ => Err!("Unknown content in response"),
|
||||
match content {
|
||||
| FileOrLocation::File(content) => self.handle_content_file(mxc, user, content).await,
|
||||
| FileOrLocation::Location(location) =>
|
||||
self.handle_location(mxc, user, &location).await,
|
||||
| _ => Err!("Unknown content in response"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(deprecated)]
|
||||
#[implement(super::Service)]
|
||||
async fn fetch_thumbnail_unauthenticated(
|
||||
&self,
|
||||
mxc: &Mxc<'_>,
|
||||
user: Option<&UserId>,
|
||||
server: Option<&ServerName>,
|
||||
timeout_ms: Duration,
|
||||
dim: &Dim,
|
||||
) -> Result<FileMeta> {
|
||||
use media::get_content_thumbnail::v3::{Request, Response};
|
||||
#[allow(deprecated)]
|
||||
async fn fetch_thumbnail_unauthenticated(
|
||||
&self,
|
||||
mxc: &Mxc<'_>,
|
||||
user: Option<&UserId>,
|
||||
server: Option<&ServerName>,
|
||||
timeout_ms: Duration,
|
||||
dim: &Dim,
|
||||
) -> Result<FileMeta> {
|
||||
use media::get_content_thumbnail::v3::{Request, Response};
|
||||
|
||||
let mut request = Request::new(
|
||||
mxc.media_id.into(),
|
||||
mxc.server_name.into(),
|
||||
dim.width.into(),
|
||||
dim.height.into(),
|
||||
);
|
||||
request.allow_redirect = true;
|
||||
request.allow_remote = true;
|
||||
request.animated = Some(true);
|
||||
request.method = Some(dim.method.clone());
|
||||
request.timeout_ms = timeout_ms;
|
||||
let mut request = Request::new(
|
||||
mxc.media_id.into(),
|
||||
mxc.server_name.into(),
|
||||
dim.width.into(),
|
||||
dim.height.into(),
|
||||
);
|
||||
request.allow_redirect = true;
|
||||
request.allow_remote = true;
|
||||
request.animated = Some(true);
|
||||
request.method = Some(dim.method.clone());
|
||||
request.timeout_ms = timeout_ms;
|
||||
|
||||
let Response {
|
||||
file, content_type, content_disposition, ..
|
||||
} = self
|
||||
.federation_request_legacy_media(mxc, server, request)
|
||||
.await?;
|
||||
let Response {
|
||||
file, content_type, content_disposition, ..
|
||||
} = self
|
||||
.federation_request_legacy_media(mxc, server, request)
|
||||
.await?;
|
||||
|
||||
let content = Content::new(file, content_type.unwrap(), content_disposition.unwrap());
|
||||
let content = Content::new(file, content_type.unwrap(), content_disposition.unwrap());
|
||||
|
||||
self.handle_thumbnail_file(mxc, user, dim, content).await
|
||||
}
|
||||
self.handle_thumbnail_file(mxc, user, dim, content).await
|
||||
}
|
||||
|
||||
#[allow(deprecated)]
|
||||
#[implement(super::Service)]
|
||||
async fn fetch_content_unauthenticated(
|
||||
&self,
|
||||
mxc: &Mxc<'_>,
|
||||
user: Option<&UserId>,
|
||||
server: Option<&ServerName>,
|
||||
timeout_ms: Duration,
|
||||
) -> Result<FileMeta> {
|
||||
use media::get_content::v3::{Request, Response};
|
||||
#[allow(deprecated)]
|
||||
async fn fetch_content_unauthenticated(
|
||||
&self,
|
||||
mxc: &Mxc<'_>,
|
||||
user: Option<&UserId>,
|
||||
server: Option<&ServerName>,
|
||||
timeout_ms: Duration,
|
||||
) -> Result<FileMeta> {
|
||||
use media::get_content::v3::{Request, Response};
|
||||
|
||||
let mut request = Request::new(mxc.media_id.into(), mxc.server_name.into());
|
||||
request.allow_remote = true;
|
||||
request.allow_redirect = true;
|
||||
request.timeout_ms = timeout_ms;
|
||||
let mut request = Request::new(mxc.media_id.into(), mxc.server_name.into());
|
||||
request.allow_remote = true;
|
||||
request.allow_redirect = true;
|
||||
request.timeout_ms = timeout_ms;
|
||||
|
||||
let Response {
|
||||
file, content_type, content_disposition, ..
|
||||
} = self
|
||||
.federation_request_legacy_media(mxc, server, request)
|
||||
.await?;
|
||||
let Response {
|
||||
file, content_type, content_disposition, ..
|
||||
} = self
|
||||
.federation_request_legacy_media(mxc, server, request)
|
||||
.await?;
|
||||
|
||||
let content = Content::new(file, content_type.unwrap(), content_disposition.unwrap());
|
||||
let content = Content::new(file, content_type.unwrap(), content_disposition.unwrap());
|
||||
|
||||
self.handle_content_file(mxc, user, content).await
|
||||
}
|
||||
self.handle_content_file(mxc, user, content).await
|
||||
}
|
||||
|
||||
#[implement(super::Service)]
|
||||
async fn handle_thumbnail_file(
|
||||
&self,
|
||||
mxc: &Mxc<'_>,
|
||||
user: Option<&UserId>,
|
||||
dim: &Dim,
|
||||
content: Content,
|
||||
) -> Result<FileMeta> {
|
||||
let content_disposition = make_content_disposition(
|
||||
content.content_disposition.as_ref(),
|
||||
content.content_type.as_deref(),
|
||||
None,
|
||||
);
|
||||
async fn handle_thumbnail_file(
|
||||
&self,
|
||||
mxc: &Mxc<'_>,
|
||||
user: Option<&UserId>,
|
||||
dim: &Dim,
|
||||
content: Content,
|
||||
) -> Result<FileMeta> {
|
||||
let content_disposition = make_content_disposition(
|
||||
content.content_disposition.as_ref(),
|
||||
content.content_type.as_deref(),
|
||||
None,
|
||||
);
|
||||
|
||||
self.upload_thumbnail(
|
||||
mxc,
|
||||
user,
|
||||
Some(&content_disposition),
|
||||
content.content_type.as_deref(),
|
||||
dim,
|
||||
&content.file,
|
||||
)
|
||||
.await
|
||||
.map(|()| FileMeta {
|
||||
content: Some(content.file),
|
||||
content_type: content.content_type,
|
||||
content_disposition: Some(content_disposition),
|
||||
})
|
||||
}
|
||||
|
||||
#[implement(super::Service)]
|
||||
async fn handle_content_file(
|
||||
&self,
|
||||
mxc: &Mxc<'_>,
|
||||
user: Option<&UserId>,
|
||||
content: Content,
|
||||
) -> Result<FileMeta> {
|
||||
let content_disposition = make_content_disposition(
|
||||
content.content_disposition.as_ref(),
|
||||
content.content_type.as_deref(),
|
||||
None,
|
||||
);
|
||||
|
||||
self.create(
|
||||
mxc,
|
||||
user,
|
||||
Some(&content_disposition),
|
||||
content.content_type.as_deref(),
|
||||
&content.file,
|
||||
)
|
||||
.await
|
||||
.map(|()| FileMeta {
|
||||
content: Some(content.file),
|
||||
content_type: content.content_type,
|
||||
content_disposition: Some(content_disposition),
|
||||
})
|
||||
}
|
||||
|
||||
#[implement(super::Service)]
|
||||
async fn handle_location(
|
||||
&self,
|
||||
mxc: &Mxc<'_>,
|
||||
user: Option<&UserId>,
|
||||
location: &str,
|
||||
) -> Result<FileMeta> {
|
||||
self.location_request(location).await.map_err(|error| {
|
||||
err!(Request(NotFound(
|
||||
debug_warn!(%mxc, user = user.map(tracing::field::display), ?location, ?error, "Fetching media from location failed")
|
||||
)))
|
||||
})
|
||||
}
|
||||
|
||||
#[implement(super::Service)]
|
||||
async fn location_request(&self, location: &str) -> Result<FileMeta> {
|
||||
let response = self
|
||||
.services
|
||||
.client
|
||||
.extern_media
|
||||
.get(location)
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
let content_type = response
|
||||
.headers()
|
||||
.get(CONTENT_TYPE)
|
||||
.map(HeaderValue::to_str)
|
||||
.and_then(Result::ok)
|
||||
.map(str::to_owned);
|
||||
|
||||
let content_disposition = response
|
||||
.headers()
|
||||
.get(CONTENT_DISPOSITION)
|
||||
.map(HeaderValue::as_bytes)
|
||||
.map(TryFrom::try_from)
|
||||
.and_then(Result::ok);
|
||||
|
||||
response
|
||||
.limit_read(
|
||||
self.services
|
||||
.server
|
||||
.config
|
||||
.max_request_size
|
||||
.try_into()
|
||||
.expect("u64 should fit in usize"),
|
||||
self.upload_thumbnail(
|
||||
mxc,
|
||||
user,
|
||||
Some(&content_disposition),
|
||||
content.content_type.as_deref(),
|
||||
dim,
|
||||
&content.file,
|
||||
)
|
||||
.await
|
||||
.map(|content| FileMeta {
|
||||
content: Some(content),
|
||||
content_type: content_type.clone(),
|
||||
content_disposition: Some(make_content_disposition(
|
||||
content_disposition.as_ref(),
|
||||
content_type.as_deref(),
|
||||
None,
|
||||
)),
|
||||
.map(|()| FileMeta {
|
||||
content: Some(content.file),
|
||||
content_type: content.content_type,
|
||||
content_disposition: Some(content_disposition),
|
||||
})
|
||||
}
|
||||
|
||||
#[implement(super::Service)]
|
||||
async fn federation_request<'i, Request>(
|
||||
&self,
|
||||
mxc: &Mxc<'_>,
|
||||
server: Option<&ServerName>,
|
||||
request: Request,
|
||||
) -> Result<Request::IncomingResponse>
|
||||
where
|
||||
Request: OutgoingRequest<
|
||||
Authentication = ServerSignatures,
|
||||
PathBuilder: PathBuilder<Input<'i>: FederationPathBuilderInput>,
|
||||
> + Debug
|
||||
+ Send,
|
||||
{
|
||||
self.services
|
||||
.sending
|
||||
.send_federation_request(server.unwrap_or(mxc.server_name), request)
|
||||
.await
|
||||
}
|
||||
|
||||
#[implement(super::Service)]
|
||||
async fn federation_request_legacy_media<'i, Request>(
|
||||
&self,
|
||||
mxc: &Mxc<'_>,
|
||||
server: Option<&ServerName>,
|
||||
request: Request,
|
||||
) -> Result<Request::IncomingResponse>
|
||||
where
|
||||
Request: OutgoingRequest<
|
||||
Authentication = NoAccessToken,
|
||||
PathBuilder: PathBuilder<Input<'i>: FederationPathBuilderInput>,
|
||||
> + Debug
|
||||
+ Send,
|
||||
{
|
||||
self.services
|
||||
.sending
|
||||
.send_legacy_media_request(server.unwrap_or(mxc.server_name), request)
|
||||
.await
|
||||
}
|
||||
|
||||
#[implement(super::Service)]
|
||||
#[allow(deprecated)]
|
||||
pub async fn fetch_remote_thumbnail_legacy(
|
||||
&self,
|
||||
body: &media::get_content_thumbnail::v3::Request,
|
||||
) -> Result<media::get_content_thumbnail::v3::Response> {
|
||||
let mxc = Mxc {
|
||||
server_name: &body.server_name,
|
||||
media_id: &body.media_id,
|
||||
};
|
||||
|
||||
let mut request = media::get_content_thumbnail::v3::Request::new(
|
||||
body.media_id.clone(),
|
||||
body.server_name.clone(),
|
||||
body.width,
|
||||
body.height,
|
||||
);
|
||||
request.method.clone_from(&body.method);
|
||||
request.allow_remote = body.allow_remote;
|
||||
request.allow_redirect = body.allow_redirect;
|
||||
request.animated = body.animated;
|
||||
request.timeout_ms = body.timeout_ms;
|
||||
|
||||
self.check_legacy_freeze()?;
|
||||
self.check_fetch_authorized(&mxc)?;
|
||||
let response = self
|
||||
.services
|
||||
.sending
|
||||
.send_legacy_media_request(mxc.server_name, request)
|
||||
.await?;
|
||||
|
||||
let dim = Dim::from_ruma(body.width, body.height, body.method.clone())?;
|
||||
self.upload_thumbnail(
|
||||
&mxc,
|
||||
None,
|
||||
None,
|
||||
response.content_type.as_deref(),
|
||||
&dim,
|
||||
&response.file,
|
||||
)
|
||||
.await?;
|
||||
|
||||
Ok(response)
|
||||
}
|
||||
|
||||
#[implement(super::Service)]
|
||||
#[allow(deprecated)]
|
||||
pub async fn fetch_remote_content_legacy(
|
||||
&self,
|
||||
mxc: &Mxc<'_>,
|
||||
allow_redirect: bool,
|
||||
timeout_ms: Duration,
|
||||
) -> Result<media::get_content::v3::Response, Error> {
|
||||
let mut request =
|
||||
media::get_content::v3::Request::new(mxc.media_id.into(), mxc.server_name.into());
|
||||
request.allow_remote = true;
|
||||
request.allow_redirect = allow_redirect;
|
||||
request.timeout_ms = timeout_ms;
|
||||
|
||||
self.check_legacy_freeze()?;
|
||||
self.check_fetch_authorized(mxc)?;
|
||||
let response = self
|
||||
.services
|
||||
.sending
|
||||
.send_legacy_media_request(mxc.server_name, request)
|
||||
.await?;
|
||||
|
||||
let content_disposition = make_content_disposition(
|
||||
response.content_disposition.as_ref(),
|
||||
response.content_type.as_deref(),
|
||||
None,
|
||||
);
|
||||
|
||||
self.create(
|
||||
mxc,
|
||||
None,
|
||||
Some(&content_disposition),
|
||||
response.content_type.as_deref(),
|
||||
&response.file,
|
||||
)
|
||||
.await?;
|
||||
|
||||
Ok(response)
|
||||
}
|
||||
|
||||
#[implement(super::Service)]
|
||||
fn check_fetch_authorized(&self, mxc: &Mxc<'_>) -> Result<()> {
|
||||
if self
|
||||
.services
|
||||
.moderation
|
||||
.is_remote_server_media_downloads_forbidden(mxc.server_name)
|
||||
{
|
||||
// we'll lie to the client and say the blocked server's media was not found and
|
||||
// log. the client has no way of telling anyways so this is a security bonus.
|
||||
debug_warn!(%mxc, "Received request for media on blocklisted server");
|
||||
return Err!(Request(NotFound("Media not found.")));
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
async fn handle_content_file(
|
||||
&self,
|
||||
mxc: &Mxc<'_>,
|
||||
user: Option<&UserId>,
|
||||
content: Content,
|
||||
) -> Result<FileMeta> {
|
||||
let content_disposition = make_content_disposition(
|
||||
content.content_disposition.as_ref(),
|
||||
content.content_type.as_deref(),
|
||||
None,
|
||||
);
|
||||
|
||||
#[implement(super::Service)]
|
||||
fn check_legacy_freeze(&self) -> Result<()> {
|
||||
self.services
|
||||
.server
|
||||
.config
|
||||
.freeze_legacy_media
|
||||
.then_some(())
|
||||
.ok_or(err!(Request(NotFound("Remote media is frozen."))))
|
||||
self.create(
|
||||
mxc,
|
||||
user,
|
||||
Some(&content_disposition),
|
||||
content.content_type.as_deref(),
|
||||
&content.file,
|
||||
)
|
||||
.await
|
||||
.map(|()| FileMeta {
|
||||
content: Some(content.file),
|
||||
content_type: content.content_type,
|
||||
content_disposition: Some(content_disposition),
|
||||
})
|
||||
}
|
||||
|
||||
async fn handle_location(
|
||||
&self,
|
||||
mxc: &Mxc<'_>,
|
||||
user: Option<&UserId>,
|
||||
location: &str,
|
||||
) -> Result<FileMeta> {
|
||||
self.location_request(location).await.map_err(|error| {
|
||||
err!(Request(NotFound(
|
||||
debug_warn!(%mxc, user = user.map(tracing::field::display), ?location, ?error, "Fetching media from location failed")
|
||||
)))
|
||||
})
|
||||
}
|
||||
|
||||
async fn location_request(&self, location: &str) -> Result<FileMeta> {
|
||||
let response = self
|
||||
.services
|
||||
.client
|
||||
.extern_media
|
||||
.get(location)
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
let content_type = response
|
||||
.headers()
|
||||
.get(CONTENT_TYPE)
|
||||
.map(HeaderValue::to_str)
|
||||
.and_then(Result::ok)
|
||||
.map(str::to_owned);
|
||||
|
||||
let content_disposition = response
|
||||
.headers()
|
||||
.get(CONTENT_DISPOSITION)
|
||||
.map(HeaderValue::as_bytes)
|
||||
.map(TryFrom::try_from)
|
||||
.and_then(Result::ok);
|
||||
|
||||
response
|
||||
.limit_read(
|
||||
self.services
|
||||
.server
|
||||
.config
|
||||
.max_request_size
|
||||
.try_into()
|
||||
.expect("u64 should fit in usize"),
|
||||
)
|
||||
.await
|
||||
.map(|content| FileMeta {
|
||||
content: Some(content),
|
||||
content_type: content_type.clone(),
|
||||
content_disposition: Some(make_content_disposition(
|
||||
content_disposition.as_ref(),
|
||||
content_type.as_deref(),
|
||||
None,
|
||||
)),
|
||||
})
|
||||
}
|
||||
|
||||
async fn federation_request<'i, Request>(
|
||||
&self,
|
||||
mxc: &Mxc<'_>,
|
||||
server: Option<&ServerName>,
|
||||
request: Request,
|
||||
) -> Result<Request::IncomingResponse>
|
||||
where
|
||||
Request: OutgoingRequest<
|
||||
Authentication = ServerSignatures,
|
||||
PathBuilder: PathBuilder<Input<'i>: FederationPathBuilderInput>,
|
||||
> + Debug
|
||||
+ Send,
|
||||
{
|
||||
self.services
|
||||
.sending
|
||||
.send_federation_request(server.unwrap_or(mxc.server_name), request)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn federation_request_legacy_media<'i, Request>(
|
||||
&self,
|
||||
mxc: &Mxc<'_>,
|
||||
server: Option<&ServerName>,
|
||||
request: Request,
|
||||
) -> Result<Request::IncomingResponse>
|
||||
where
|
||||
Request: OutgoingRequest<
|
||||
Authentication = NoAccessToken,
|
||||
PathBuilder: PathBuilder<Input<'i>: FederationPathBuilderInput>,
|
||||
> + Debug
|
||||
+ Send,
|
||||
{
|
||||
self.services
|
||||
.sending
|
||||
.send_legacy_media_request(server.unwrap_or(mxc.server_name), request)
|
||||
.await
|
||||
}
|
||||
|
||||
#[allow(deprecated)]
|
||||
pub async fn fetch_remote_thumbnail_legacy(
|
||||
&self,
|
||||
body: &media::get_content_thumbnail::v3::Request,
|
||||
) -> Result<media::get_content_thumbnail::v3::Response> {
|
||||
let mxc = Mxc {
|
||||
server_name: &body.server_name,
|
||||
media_id: &body.media_id,
|
||||
};
|
||||
|
||||
let mut request = media::get_content_thumbnail::v3::Request::new(
|
||||
body.media_id.clone(),
|
||||
body.server_name.clone(),
|
||||
body.width,
|
||||
body.height,
|
||||
);
|
||||
request.method.clone_from(&body.method);
|
||||
request.allow_remote = body.allow_remote;
|
||||
request.allow_redirect = body.allow_redirect;
|
||||
request.animated = body.animated;
|
||||
request.timeout_ms = body.timeout_ms;
|
||||
|
||||
self.check_legacy_freeze()?;
|
||||
self.check_fetch_authorized(&mxc)?;
|
||||
let response = self
|
||||
.services
|
||||
.sending
|
||||
.send_legacy_media_request(mxc.server_name, request)
|
||||
.await?;
|
||||
|
||||
let dim = Dim::from_ruma(body.width, body.height, body.method.clone())?;
|
||||
self.upload_thumbnail(
|
||||
&mxc,
|
||||
None,
|
||||
None,
|
||||
response.content_type.as_deref(),
|
||||
&dim,
|
||||
&response.file,
|
||||
)
|
||||
.await?;
|
||||
|
||||
Ok(response)
|
||||
}
|
||||
|
||||
#[allow(deprecated)]
|
||||
pub async fn fetch_remote_content_legacy(
|
||||
&self,
|
||||
mxc: &Mxc<'_>,
|
||||
allow_redirect: bool,
|
||||
timeout_ms: Duration,
|
||||
) -> Result<media::get_content::v3::Response, Error> {
|
||||
let mut request =
|
||||
media::get_content::v3::Request::new(mxc.media_id.into(), mxc.server_name.into());
|
||||
request.allow_remote = true;
|
||||
request.allow_redirect = allow_redirect;
|
||||
request.timeout_ms = timeout_ms;
|
||||
|
||||
self.check_legacy_freeze()?;
|
||||
self.check_fetch_authorized(mxc)?;
|
||||
let response = self
|
||||
.services
|
||||
.sending
|
||||
.send_legacy_media_request(mxc.server_name, request)
|
||||
.await?;
|
||||
|
||||
let content_disposition = make_content_disposition(
|
||||
response.content_disposition.as_ref(),
|
||||
response.content_type.as_deref(),
|
||||
None,
|
||||
);
|
||||
|
||||
self.create(
|
||||
mxc,
|
||||
None,
|
||||
Some(&content_disposition),
|
||||
response.content_type.as_deref(),
|
||||
&response.file,
|
||||
)
|
||||
.await?;
|
||||
|
||||
Ok(response)
|
||||
}
|
||||
|
||||
fn check_fetch_authorized(&self, mxc: &Mxc<'_>) -> Result<()> {
|
||||
if self
|
||||
.services
|
||||
.moderation
|
||||
.is_remote_server_media_downloads_forbidden(mxc.server_name)
|
||||
{
|
||||
// we'll lie to the client and say the blocked server's media was not found and
|
||||
// log. the client has no way of telling anyways so this is a security bonus.
|
||||
debug_warn!(%mxc, "Received request for media on blocklisted server");
|
||||
return Err!(Request(NotFound("Media not found.")));
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn check_legacy_freeze(&self) -> Result<()> {
|
||||
self.services
|
||||
.server
|
||||
.config
|
||||
.freeze_legacy_media
|
||||
.then_some(())
|
||||
.ok_or(err!(Request(NotFound("Remote media is frozen."))))
|
||||
}
|
||||
}
|
||||
@@ -7,7 +7,7 @@
|
||||
|
||||
use std::{cmp, num::Saturating as Sat};
|
||||
|
||||
use conduwuit::{Result, checked, err, implement};
|
||||
use conduwuit::{Result, checked, err};
|
||||
use ruma::{UInt, UserId, http_headers::ContentDisposition, media::Method};
|
||||
use tokio::{
|
||||
fs,
|
||||
@@ -74,80 +74,77 @@ impl super::Service {
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Using saved thumbnail
|
||||
#[implement(super::Service)]
|
||||
#[tracing::instrument(name = "saved", level = "debug", skip(self, data))]
|
||||
async fn get_thumbnail_saved(&self, data: Metadata) -> Result<Option<FileMeta>> {
|
||||
let mut content = Vec::new();
|
||||
let path = self.get_media_file(&data.key);
|
||||
fs::File::open(path)
|
||||
.await?
|
||||
.read_to_end(&mut content)
|
||||
.await?;
|
||||
/// Using saved thumbnail
|
||||
#[tracing::instrument(name = "saved", level = "debug", skip(self, data))]
|
||||
async fn get_thumbnail_saved(&self, data: Metadata) -> Result<Option<FileMeta>> {
|
||||
let mut content = Vec::new();
|
||||
let path = self.get_media_file(&data.key);
|
||||
fs::File::open(path)
|
||||
.await?
|
||||
.read_to_end(&mut content)
|
||||
.await?;
|
||||
|
||||
Ok(Some(into_filemeta(data, content)))
|
||||
}
|
||||
|
||||
/// Generate a thumbnail
|
||||
#[cfg(feature = "media_thumbnail")]
|
||||
#[implement(super::Service)]
|
||||
#[tracing::instrument(name = "generate", level = "debug", skip(self, data))]
|
||||
async fn get_thumbnail_generate(
|
||||
&self,
|
||||
mxc: &Mxc<'_>,
|
||||
dim: &Dim,
|
||||
data: Metadata,
|
||||
) -> Result<Option<FileMeta>> {
|
||||
let mut content = Vec::new();
|
||||
let path = self.get_media_file(&data.key);
|
||||
fs::File::open(path)
|
||||
.await?
|
||||
.read_to_end(&mut content)
|
||||
.await?;
|
||||
|
||||
let Ok(image) = image::load_from_memory(&content) else {
|
||||
// Couldn't parse file to generate thumbnail, send original
|
||||
return Ok(Some(into_filemeta(data, content)));
|
||||
};
|
||||
|
||||
if dim.width > image.width() || dim.height > image.height() {
|
||||
return Ok(Some(into_filemeta(data, content)));
|
||||
Ok(Some(into_filemeta(data, content)))
|
||||
}
|
||||
|
||||
let mut thumbnail_bytes = Vec::new();
|
||||
let thumbnail = thumbnail_generate(&image, dim)?;
|
||||
let mut cursor = std::io::Cursor::new(&mut thumbnail_bytes);
|
||||
thumbnail
|
||||
.write_to(&mut cursor, image::ImageFormat::Png)
|
||||
.map_err(|error| err!(error!(%error, "Error writing PNG thumbnail.")))?;
|
||||
/// Generate a thumbnail
|
||||
#[cfg(feature = "media_thumbnail")]
|
||||
#[tracing::instrument(name = "generate", level = "debug", skip(self, data))]
|
||||
async fn get_thumbnail_generate(
|
||||
&self,
|
||||
mxc: &Mxc<'_>,
|
||||
dim: &Dim,
|
||||
data: Metadata,
|
||||
) -> Result<Option<FileMeta>> {
|
||||
let mut content = Vec::new();
|
||||
let path = self.get_media_file(&data.key);
|
||||
fs::File::open(path)
|
||||
.await?
|
||||
.read_to_end(&mut content)
|
||||
.await?;
|
||||
|
||||
// Save thumbnail in database so we don't have to generate it again next time
|
||||
let thumbnail_key = self.db.create_file_metadata(
|
||||
mxc,
|
||||
None,
|
||||
dim,
|
||||
data.content_disposition.as_ref(),
|
||||
data.content_type.as_deref(),
|
||||
)?;
|
||||
let Ok(image) = image::load_from_memory(&content) else {
|
||||
// Couldn't parse file to generate thumbnail, send original
|
||||
return Ok(Some(into_filemeta(data, content)));
|
||||
};
|
||||
|
||||
let mut f = self.create_media_file(&thumbnail_key).await?;
|
||||
f.write_all(&thumbnail_bytes).await?;
|
||||
if dim.width > image.width() || dim.height > image.height() {
|
||||
return Ok(Some(into_filemeta(data, content)));
|
||||
}
|
||||
|
||||
Ok(Some(into_filemeta(data, thumbnail_bytes)))
|
||||
}
|
||||
let mut thumbnail_bytes = Vec::new();
|
||||
let thumbnail = thumbnail_generate(&image, dim)?;
|
||||
let mut cursor = std::io::Cursor::new(&mut thumbnail_bytes);
|
||||
thumbnail
|
||||
.write_to(&mut cursor, image::ImageFormat::Png)
|
||||
.map_err(|error| err!(error!(%error, "Error writing PNG thumbnail.")))?;
|
||||
|
||||
#[cfg(not(feature = "media_thumbnail"))]
|
||||
#[implement(super::Service)]
|
||||
#[tracing::instrument(name = "fallback", level = "debug", skip_all)]
|
||||
async fn get_thumbnail_generate(
|
||||
&self,
|
||||
_mxc: &Mxc<'_>,
|
||||
_dim: &Dim,
|
||||
data: Metadata,
|
||||
) -> Result<Option<FileMeta>> {
|
||||
self.get_thumbnail_saved(data).await
|
||||
// Save thumbnail in database so we don't have to generate it again next time
|
||||
let thumbnail_key = self.db.create_file_metadata(
|
||||
mxc,
|
||||
None,
|
||||
dim,
|
||||
data.content_disposition.as_ref(),
|
||||
data.content_type.as_deref(),
|
||||
)?;
|
||||
|
||||
let mut f = self.create_media_file(&thumbnail_key).await?;
|
||||
f.write_all(&thumbnail_bytes).await?;
|
||||
|
||||
Ok(Some(into_filemeta(data, thumbnail_bytes)))
|
||||
}
|
||||
|
||||
#[cfg(not(feature = "media_thumbnail"))]
|
||||
#[tracing::instrument(name = "fallback", level = "debug", skip_all)]
|
||||
async fn get_thumbnail_generate(
|
||||
&self,
|
||||
_mxc: &Mxc<'_>,
|
||||
_dim: &Dim,
|
||||
data: Metadata,
|
||||
) -> Result<Option<FileMeta>> {
|
||||
self.get_thumbnail_saved(data).await
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "media_thumbnail")]
|
||||
|
||||
+66
-55
@@ -1,6 +1,6 @@
|
||||
use std::sync::Arc;
|
||||
|
||||
use conduwuit::{Result, implement};
|
||||
use conduwuit::Result;
|
||||
use ruma::ServerName;
|
||||
|
||||
use crate::{Dep, config};
|
||||
@@ -27,67 +27,78 @@ impl crate::Service for Service {
|
||||
fn name(&self) -> &str { crate::service::make_name(std::module_path!()) }
|
||||
}
|
||||
|
||||
#[implement(Service)]
|
||||
#[must_use]
|
||||
pub fn is_remote_server_ignored(&self, server_name: &ServerName) -> bool {
|
||||
// We must never block federating with ourselves
|
||||
if server_name == self.services.config.server_name {
|
||||
return false;
|
||||
impl Service {
|
||||
/// Returns true if the remote server is *ignored*. Ignored server names are
|
||||
/// still permitted in federation, but should not have their events sent to
|
||||
/// local users.
|
||||
#[must_use]
|
||||
pub fn is_remote_server_ignored(&self, server_name: &ServerName) -> bool {
|
||||
// We must never block federating with ourselves
|
||||
if server_name == self.services.config.server_name {
|
||||
return false;
|
||||
}
|
||||
|
||||
self.services
|
||||
.config
|
||||
.ignore_messages_from_server_names
|
||||
.is_match(server_name.host())
|
||||
}
|
||||
|
||||
self.services
|
||||
.config
|
||||
.ignore_messages_from_server_names
|
||||
.is_match(server_name.host())
|
||||
}
|
||||
/// Returns true if the remote server is *forbidden*. Forbidden server names
|
||||
/// must be blocked in federation in both directions - no inbound nor
|
||||
/// outbound requests from the remote are permitted.
|
||||
#[must_use]
|
||||
pub fn is_remote_server_forbidden(&self, server_name: &ServerName) -> bool {
|
||||
// We must never block federating with ourselves
|
||||
if server_name == self.services.config.server_name {
|
||||
return false;
|
||||
}
|
||||
|
||||
#[implement(Service)]
|
||||
#[must_use]
|
||||
pub fn is_remote_server_forbidden(&self, server_name: &ServerName) -> bool {
|
||||
// We must never block federating with ourselves
|
||||
if server_name == self.services.config.server_name {
|
||||
return false;
|
||||
}
|
||||
|
||||
// Check if server is explicitly allowed
|
||||
if self
|
||||
.services
|
||||
.config
|
||||
.allowed_remote_server_names
|
||||
.is_match(server_name.host())
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
// Check if server is explicitly forbidden
|
||||
self.services
|
||||
.config
|
||||
.forbidden_remote_server_names
|
||||
.is_match(server_name.host())
|
||||
}
|
||||
|
||||
#[implement(Service)]
|
||||
#[must_use]
|
||||
pub fn is_remote_server_room_directory_forbidden(&self, server_name: &ServerName) -> bool {
|
||||
// Forbidden if NOT (allowed is empty OR allowed contains server OR is self)
|
||||
// OR forbidden contains server
|
||||
self.is_remote_server_forbidden(server_name)
|
||||
|| self
|
||||
// Check if server is explicitly allowed
|
||||
if self
|
||||
.services
|
||||
.config
|
||||
.forbidden_remote_room_directory_server_names
|
||||
.allowed_remote_server_names
|
||||
.is_match(server_name.host())
|
||||
}
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
#[implement(Service)]
|
||||
#[must_use]
|
||||
pub fn is_remote_server_media_downloads_forbidden(&self, server_name: &ServerName) -> bool {
|
||||
// Forbidden if NOT (allowed is empty OR allowed contains server OR is self)
|
||||
// OR forbidden contains server
|
||||
self.is_remote_server_forbidden(server_name)
|
||||
|| self
|
||||
.services
|
||||
// Check if server is explicitly forbidden
|
||||
self.services
|
||||
.config
|
||||
.prevent_media_downloads_from
|
||||
.forbidden_remote_server_names
|
||||
.is_match(server_name.host())
|
||||
}
|
||||
|
||||
/// Returns true if the remote server is *forbidden*, or if it is only
|
||||
/// forbidden from having its room directory checked. Does not prevent the
|
||||
/// remote from reading our room directory.
|
||||
#[must_use]
|
||||
pub fn is_remote_server_room_directory_forbidden(&self, server_name: &ServerName) -> bool {
|
||||
// Forbidden if NOT (allowed is empty OR allowed contains server OR is self)
|
||||
// OR forbidden contains server
|
||||
self.is_remote_server_forbidden(server_name)
|
||||
|| self
|
||||
.services
|
||||
.config
|
||||
.forbidden_remote_room_directory_server_names
|
||||
.is_match(server_name.host())
|
||||
}
|
||||
|
||||
/// Returns true if the remote server is *forbidden*, or if it is only
|
||||
/// forbidden from serving media to us (i.e. we aren't allowed to download
|
||||
/// media from the remote). Does not prevent the remote downloading media
|
||||
/// from us.
|
||||
#[must_use]
|
||||
pub fn is_remote_server_media_downloads_forbidden(&self, server_name: &ServerName) -> bool {
|
||||
// Forbidden if NOT (allowed is empty OR allowed contains server OR is self)
|
||||
// OR forbidden contains server
|
||||
self.is_remote_server_forbidden(server_name)
|
||||
|| self
|
||||
.services
|
||||
.config
|
||||
.prevent_media_downloads_from
|
||||
.is_match(server_name.host())
|
||||
}
|
||||
}
|
||||
@@ -3,7 +3,7 @@ use std::{net::IpAddr, sync::Arc, time::SystemTime};
|
||||
use conduwuit::{
|
||||
Result,
|
||||
arrayvec::ArrayVec,
|
||||
at, err, implement,
|
||||
at, err,
|
||||
utils::{math::Expected, rand, stream::TryIgnore},
|
||||
};
|
||||
use database::{Cbor, Deserialized, Map};
|
||||
@@ -43,83 +43,70 @@ impl Cache {
|
||||
overrides: args.db["servername_override"].clone(),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[implement(Cache)]
|
||||
pub async fn clear(&self) { join(self.clear_destinations(), self.clear_overrides()).await; }
|
||||
pub async fn clear(&self) { join(self.clear_destinations(), self.clear_overrides()).await; }
|
||||
|
||||
#[implement(Cache)]
|
||||
pub async fn clear_destinations(&self) { self.destinations.clear().await; }
|
||||
pub async fn clear_destinations(&self) { self.destinations.clear().await; }
|
||||
|
||||
#[implement(Cache)]
|
||||
pub async fn clear_overrides(&self) { self.overrides.clear().await; }
|
||||
pub async fn clear_overrides(&self) { self.overrides.clear().await; }
|
||||
|
||||
#[implement(Cache)]
|
||||
pub fn del_destination(&self, name: &ServerName) { self.destinations.remove(name); }
|
||||
pub fn del_destination(&self, name: &ServerName) { self.destinations.remove(name); }
|
||||
|
||||
#[implement(Cache)]
|
||||
pub fn del_override(&self, name: &ServerName) { self.overrides.remove(name); }
|
||||
pub fn del_override(&self, name: &ServerName) { self.overrides.remove(name); }
|
||||
|
||||
#[implement(Cache)]
|
||||
pub fn set_destination(&self, name: &ServerName, dest: &CachedDest) {
|
||||
self.destinations.raw_put(name, Cbor(dest));
|
||||
}
|
||||
pub fn set_destination(&self, name: &ServerName, dest: &CachedDest) {
|
||||
self.destinations.raw_put(name, Cbor(dest));
|
||||
}
|
||||
|
||||
#[implement(Cache)]
|
||||
pub fn set_override(&self, name: &str, over: &CachedOverride) {
|
||||
self.overrides.raw_put(name, Cbor(over));
|
||||
}
|
||||
pub fn set_override(&self, name: &str, over: &CachedOverride) {
|
||||
self.overrides.raw_put(name, Cbor(over));
|
||||
}
|
||||
|
||||
#[implement(Cache)]
|
||||
#[must_use]
|
||||
pub async fn has_destination(&self, destination: &ServerName) -> bool {
|
||||
self.get_destination(destination).await.is_ok()
|
||||
}
|
||||
#[must_use]
|
||||
pub async fn has_destination(&self, destination: &ServerName) -> bool {
|
||||
self.get_destination(destination).await.is_ok()
|
||||
}
|
||||
|
||||
#[implement(Cache)]
|
||||
#[must_use]
|
||||
pub async fn has_override(&self, destination: &str) -> bool {
|
||||
self.get_override(destination)
|
||||
.await
|
||||
.iter()
|
||||
.any(CachedOverride::valid)
|
||||
}
|
||||
#[must_use]
|
||||
pub async fn has_override(&self, destination: &str) -> bool {
|
||||
self.get_override(destination)
|
||||
.await
|
||||
.iter()
|
||||
.any(CachedOverride::valid)
|
||||
}
|
||||
|
||||
#[implement(Cache)]
|
||||
pub async fn get_destination(&self, name: &ServerName) -> Result<CachedDest> {
|
||||
self.destinations
|
||||
.get(name)
|
||||
.await
|
||||
.deserialized::<Cbor<_>>()
|
||||
.map(at!(0))
|
||||
.into_iter()
|
||||
.find(CachedDest::valid)
|
||||
.ok_or(err!(Request(NotFound("Expired from cache"))))
|
||||
}
|
||||
pub async fn get_destination(&self, name: &ServerName) -> Result<CachedDest> {
|
||||
self.destinations
|
||||
.get(name)
|
||||
.await
|
||||
.deserialized::<Cbor<_>>()
|
||||
.map(at!(0))
|
||||
.into_iter()
|
||||
.find(CachedDest::valid)
|
||||
.ok_or(err!(Request(NotFound("Expired from cache"))))
|
||||
}
|
||||
|
||||
#[implement(Cache)]
|
||||
pub async fn get_override(&self, name: &str) -> Result<CachedOverride> {
|
||||
self.overrides
|
||||
.get(name)
|
||||
.await
|
||||
.deserialized::<Cbor<_>>()
|
||||
.map(at!(0))
|
||||
}
|
||||
pub async fn get_override(&self, name: &str) -> Result<CachedOverride> {
|
||||
self.overrides
|
||||
.get(name)
|
||||
.await
|
||||
.deserialized::<Cbor<_>>()
|
||||
.map(at!(0))
|
||||
}
|
||||
|
||||
#[implement(Cache)]
|
||||
pub fn destinations(&self) -> impl Stream<Item = (OwnedServerName, CachedDest)> + Send + '_ {
|
||||
self.destinations
|
||||
.stream()
|
||||
.ignore_err()
|
||||
.map(|item: (OwnedServerName, Cbor<_>)| (item.0, item.1.0))
|
||||
}
|
||||
pub fn destinations(&self) -> impl Stream<Item = (OwnedServerName, CachedDest)> + Send + '_ {
|
||||
self.destinations
|
||||
.stream()
|
||||
.ignore_err()
|
||||
.map(|item: (OwnedServerName, Cbor<_>)| (item.0, item.1.0))
|
||||
}
|
||||
|
||||
#[implement(Cache)]
|
||||
pub fn overrides(&self) -> impl Stream<Item = (OwnedServerName, CachedOverride)> + Send + '_ {
|
||||
self.overrides
|
||||
.stream()
|
||||
.ignore_err()
|
||||
.map(|item: (OwnedServerName, Cbor<_>)| (item.0, item.1.0))
|
||||
pub fn overrides(&self) -> impl Stream<Item = (OwnedServerName, CachedOverride)> + Send + '_ {
|
||||
self.overrides
|
||||
.stream()
|
||||
.ignore_err()
|
||||
.map(|item: (OwnedServerName, Cbor<_>)| (item.0, item.1.0))
|
||||
}
|
||||
}
|
||||
|
||||
impl CachedDest {
|
||||
|
||||
@@ -1,51 +1,50 @@
|
||||
use conduwuit::{
|
||||
Result, debug, debug_error, debug_info, implement, trace, utils::response::LimitReadExt,
|
||||
};
|
||||
use conduwuit::{Result, debug, debug_error, debug_info, trace, utils::response::LimitReadExt};
|
||||
use ruma::ServerName;
|
||||
|
||||
#[implement(super::Service)]
|
||||
#[tracing::instrument(name = "well-known", level = "debug", skip(self, dest))]
|
||||
pub(super) async fn request_well_known(&self, dest: &str) -> Result<Option<String>> {
|
||||
trace!("Requesting well known for {dest}");
|
||||
let response = self
|
||||
.services
|
||||
.client
|
||||
.well_known
|
||||
.get(format!("https://{dest}/.well-known/matrix/server"))
|
||||
.send()
|
||||
.await;
|
||||
impl super::Service {
|
||||
#[tracing::instrument(name = "well-known", level = "debug", skip(self, dest))]
|
||||
pub(super) async fn request_well_known(&self, dest: &str) -> Result<Option<String>> {
|
||||
trace!("Requesting well known for {dest}");
|
||||
let response = self
|
||||
.services
|
||||
.client
|
||||
.well_known
|
||||
.get(format!("https://{dest}/.well-known/matrix/server"))
|
||||
.send()
|
||||
.await;
|
||||
|
||||
trace!("response: {response:?}");
|
||||
if let Err(e) = &response {
|
||||
debug!("error: {e:?}");
|
||||
return Ok(None);
|
||||
trace!("response: {response:?}");
|
||||
if let Err(e) = &response {
|
||||
debug!("error: {e:?}");
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
let response = response?;
|
||||
if !response.status().is_success() {
|
||||
debug!("response not 2XX");
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
let Ok(text) = response.limit_read_text(8192).await else {
|
||||
debug!("failed to read well-known response (too large or non-text content)");
|
||||
return Ok(None);
|
||||
};
|
||||
trace!("response text: {text:?}");
|
||||
|
||||
let body: serde_json::Value = serde_json::from_str(&text).unwrap_or_default();
|
||||
|
||||
let m_server = body
|
||||
.get("m.server")
|
||||
.unwrap_or(&serde_json::Value::Null)
|
||||
.as_str()
|
||||
.unwrap_or_default();
|
||||
|
||||
if ServerName::parse(m_server).is_err() {
|
||||
debug_error!("response content missing or invalid");
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
debug_info!("{dest:?} found at {m_server:?}");
|
||||
Ok(Some(m_server.to_owned()))
|
||||
}
|
||||
|
||||
let response = response?;
|
||||
if !response.status().is_success() {
|
||||
debug!("response not 2XX");
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
let Ok(text) = response.limit_read_text(8192).await else {
|
||||
debug!("failed to read well-known response (too large or non-text content)");
|
||||
return Ok(None);
|
||||
};
|
||||
trace!("response text: {text:?}");
|
||||
|
||||
let body: serde_json::Value = serde_json::from_str(&text).unwrap_or_default();
|
||||
|
||||
let m_server = body
|
||||
.get("m.server")
|
||||
.unwrap_or(&serde_json::Value::Null)
|
||||
.as_str()
|
||||
.unwrap_or_default();
|
||||
|
||||
if ServerName::parse(m_server).is_err() {
|
||||
debug_error!("response content missing or invalid");
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
debug_info!("{dest:?} found at {m_server:?}");
|
||||
Ok(Some(m_server.to_owned()))
|
||||
}
|
||||
@@ -1,40 +1,42 @@
|
||||
use conduwuit::{Result, debug, error, implement};
|
||||
use conduwuit::{Result, debug, error};
|
||||
use federation::query::get_room_information::v1::Response;
|
||||
use ruma::{OwnedRoomId, OwnedServerName, RoomAliasId, ServerName, api::federation};
|
||||
|
||||
#[implement(super::Service)]
|
||||
pub(super) async fn remote_resolve(
|
||||
&self,
|
||||
room_alias: &RoomAliasId,
|
||||
) -> Result<(OwnedRoomId, Vec<OwnedServerName>)> {
|
||||
debug!("Asking {} to resolve {room_alias:?}", room_alias.server_name());
|
||||
match self
|
||||
.remote_request(room_alias, room_alias.server_name())
|
||||
.await
|
||||
{
|
||||
| Err(e) => {
|
||||
error!("Unable to resolve remote room alias {}: {e}", room_alias);
|
||||
Err(e)
|
||||
},
|
||||
| Ok(Response { room_id, servers, .. }) => {
|
||||
debug!("Remote resolved {room_alias:?} to {room_id:?} with servers {servers:?}");
|
||||
Ok((room_id, servers))
|
||||
},
|
||||
impl super::Service {
|
||||
/// Resolves the given room alias to a room ID and a list of servers that
|
||||
/// are in the room by asking the remote server which owns it.
|
||||
pub(super) async fn remote_resolve(
|
||||
&self,
|
||||
room_alias: &RoomAliasId,
|
||||
) -> Result<(OwnedRoomId, Vec<OwnedServerName>)> {
|
||||
debug!("Asking {} to resolve {room_alias:?}", room_alias.server_name());
|
||||
match self
|
||||
.remote_request(room_alias, room_alias.server_name())
|
||||
.await
|
||||
{
|
||||
| Err(e) => {
|
||||
error!("Unable to resolve remote room alias {}: {e}", room_alias);
|
||||
Err(e)
|
||||
},
|
||||
| Ok(Response { room_id, servers, .. }) => {
|
||||
debug!("Remote resolved {room_alias:?} to {room_id:?} with servers {servers:?}");
|
||||
Ok((room_id, servers))
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
async fn remote_request(
|
||||
&self,
|
||||
room_alias: &RoomAliasId,
|
||||
server: &ServerName,
|
||||
) -> Result<Response> {
|
||||
use federation::query::get_room_information::v1::Request;
|
||||
|
||||
let request = Request::new(room_alias.to_owned());
|
||||
|
||||
self.services
|
||||
.sending
|
||||
.send_federation_request(server, request)
|
||||
.await
|
||||
}
|
||||
}
|
||||
|
||||
#[implement(super::Service)]
|
||||
async fn remote_request(
|
||||
&self,
|
||||
room_alias: &RoomAliasId,
|
||||
server: &ServerName,
|
||||
) -> Result<Response> {
|
||||
use federation::query::get_room_information::v1::Request;
|
||||
|
||||
let request = Request::new(room_alias.to_owned());
|
||||
|
||||
self.services
|
||||
.sending
|
||||
.send_federation_request(server, request)
|
||||
.await
|
||||
}
|
||||
+195
-204
@@ -8,12 +8,12 @@ use std::{
|
||||
};
|
||||
|
||||
use conduwuit::{
|
||||
Err, Result, at, debug, debug_error, implement, trace,
|
||||
Err, Result, at, debug, debug_error, trace,
|
||||
utils::{
|
||||
IterStream,
|
||||
stream::{ReadyExt, TryBroadbandExt},
|
||||
},
|
||||
validated, warn,
|
||||
validated,
|
||||
};
|
||||
use futures::{FutureExt, Stream, StreamExt, TryFutureExt, TryStreamExt};
|
||||
use ruma::{EventId, OwnedEventId, RoomId};
|
||||
@@ -47,213 +47,204 @@ impl crate::Service for Service {
|
||||
fn name(&self) -> &str { crate::service::make_name(std::module_path!()) }
|
||||
}
|
||||
|
||||
#[implement(Service)]
|
||||
pub fn event_ids_iter<'a, I>(
|
||||
&'a self,
|
||||
room_id: &'a RoomId,
|
||||
starting_events: I,
|
||||
) -> impl Stream<Item = Result<OwnedEventId>> + Send + 'a
|
||||
where
|
||||
I: Iterator<Item = &'a EventId> + Clone + Debug + ExactSizeIterator + Send + 'a,
|
||||
{
|
||||
self.get_auth_chain(room_id, starting_events)
|
||||
.map_ok(|chain| {
|
||||
self.services
|
||||
.short
|
||||
.multi_get_eventid_from_short(chain.into_iter().stream())
|
||||
.ready_filter(Result::is_ok)
|
||||
})
|
||||
.try_flatten_stream()
|
||||
}
|
||||
|
||||
#[implement(Service)]
|
||||
#[tracing::instrument(name = "auth_chain", level = "debug", skip_all)]
|
||||
pub async fn get_auth_chain<'a, I>(
|
||||
&'a self,
|
||||
room_id: &RoomId,
|
||||
starting_events: I,
|
||||
) -> Result<Vec<ShortEventId>>
|
||||
where
|
||||
I: Iterator<Item = &'a EventId> + Clone + Debug + ExactSizeIterator + Send + 'a,
|
||||
{
|
||||
const NUM_BUCKETS: usize = 50; //TODO: change possible w/o disrupting db?
|
||||
const BUCKET: Bucket<'_> = BTreeSet::new();
|
||||
|
||||
let started = Instant::now();
|
||||
let mut starting_ids = self
|
||||
.services
|
||||
.short
|
||||
.multi_get_or_create_shorteventid(starting_events.clone())
|
||||
.zip(starting_events.clone().stream())
|
||||
.boxed();
|
||||
|
||||
let mut buckets = [BUCKET; NUM_BUCKETS];
|
||||
while let Some((short, starting_event)) = starting_ids.next().await {
|
||||
let bucket: usize = short.try_into()?;
|
||||
let bucket: usize = validated!(bucket % NUM_BUCKETS);
|
||||
buckets[bucket].insert((short, starting_event));
|
||||
impl Service {
|
||||
pub fn event_ids_iter<'a, I>(
|
||||
&'a self,
|
||||
room_id: &'a RoomId,
|
||||
starting_events: I,
|
||||
) -> impl Stream<Item = Result<OwnedEventId>> + Send + 'a
|
||||
where
|
||||
I: Iterator<Item = &'a EventId> + Clone + Debug + ExactSizeIterator + Send + 'a,
|
||||
{
|
||||
self.get_auth_chain(room_id, starting_events)
|
||||
.map_ok(|chain| {
|
||||
self.services
|
||||
.short
|
||||
.multi_get_eventid_from_short(chain.into_iter().stream())
|
||||
.ready_filter(Result::is_ok)
|
||||
})
|
||||
.try_flatten_stream()
|
||||
}
|
||||
|
||||
debug!(
|
||||
starting_events = ?starting_events.count(),
|
||||
elapsed = ?started.elapsed(),
|
||||
"start",
|
||||
);
|
||||
#[tracing::instrument(name = "auth_chain", skip_all)]
|
||||
pub async fn get_auth_chain<'a, I>(
|
||||
&'a self,
|
||||
room_id: &RoomId,
|
||||
starting_events: I,
|
||||
) -> Result<Vec<ShortEventId>>
|
||||
where
|
||||
I: Iterator<Item = &'a EventId> + Clone + Debug + ExactSizeIterator + Send + 'a,
|
||||
{
|
||||
const NUM_BUCKETS: usize = 50; //TODO: change possible w/o disrupting db?
|
||||
const BUCKET: Bucket<'_> = BTreeSet::new();
|
||||
|
||||
let full_auth_chain: Vec<ShortEventId> = buckets
|
||||
.into_iter()
|
||||
.try_stream()
|
||||
.broad_and_then(|chunk| self.get_auth_chain_outer(room_id, started, chunk))
|
||||
.try_collect()
|
||||
.map_ok(|auth_chain: Vec<_>| auth_chain.into_iter().flatten().collect())
|
||||
.map_ok(|mut full_auth_chain: Vec<_>| {
|
||||
full_auth_chain.sort_unstable();
|
||||
full_auth_chain.dedup();
|
||||
full_auth_chain
|
||||
})
|
||||
.boxed()
|
||||
.await?;
|
||||
let started = Instant::now();
|
||||
let mut starting_ids = self
|
||||
.services
|
||||
.short
|
||||
.multi_get_or_create_shorteventid(starting_events.clone())
|
||||
.zip(starting_events.clone().stream())
|
||||
.boxed();
|
||||
|
||||
debug!(
|
||||
chain_length = ?full_auth_chain.len(),
|
||||
elapsed = ?started.elapsed(),
|
||||
"done",
|
||||
);
|
||||
|
||||
Ok(full_auth_chain)
|
||||
}
|
||||
|
||||
#[implement(Service)]
|
||||
async fn get_auth_chain_outer(
|
||||
&self,
|
||||
room_id: &RoomId,
|
||||
started: Instant,
|
||||
chunk: Bucket<'_>,
|
||||
) -> Result<Vec<ShortEventId>> {
|
||||
let chunk_key: Vec<ShortEventId> = chunk.iter().map(at!(0)).collect();
|
||||
|
||||
if chunk_key.is_empty() {
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
|
||||
if let Ok(cached) = self.get_cached_eventid_authchain(&chunk_key).await {
|
||||
return Ok(cached.to_vec());
|
||||
}
|
||||
|
||||
let chunk_cache: Vec<_> = chunk
|
||||
.into_iter()
|
||||
.try_stream::<conduwuit::Error>()
|
||||
.broad_and_then(|(shortid, event_id)| async move {
|
||||
if let Ok(cached) = self.get_cached_eventid_authchain(&[shortid]).await {
|
||||
return Ok(cached.to_vec());
|
||||
}
|
||||
|
||||
let auth_chain = self.get_auth_chain_inner(room_id, event_id).await?;
|
||||
self.cache_auth_chain_vec(vec![shortid], auth_chain.as_slice());
|
||||
debug!(
|
||||
%event_id,
|
||||
elapsed = ?started.elapsed(),
|
||||
"Cache missed event"
|
||||
);
|
||||
|
||||
Ok(auth_chain)
|
||||
})
|
||||
.try_collect()
|
||||
.map_ok(|chunk_cache: Vec<_>| chunk_cache.into_iter().flatten().collect())
|
||||
.map_ok(|mut chunk_cache: Vec<_>| {
|
||||
chunk_cache.sort_unstable();
|
||||
chunk_cache.dedup();
|
||||
chunk_cache
|
||||
})
|
||||
.await?;
|
||||
|
||||
self.cache_auth_chain_vec(chunk_key, chunk_cache.as_slice());
|
||||
debug!(
|
||||
chunk_cache_length = ?chunk_cache.len(),
|
||||
elapsed = ?started.elapsed(),
|
||||
"Cache missed chunk",
|
||||
);
|
||||
|
||||
Ok(chunk_cache)
|
||||
}
|
||||
|
||||
#[implement(Service)]
|
||||
#[tracing::instrument(name = "inner", level = "trace", skip(self, room_id))]
|
||||
async fn get_auth_chain_inner(
|
||||
&self,
|
||||
room_id: &RoomId,
|
||||
event_id: &EventId,
|
||||
) -> Result<Vec<ShortEventId>> {
|
||||
let mut todo: VecDeque<_> = [event_id.to_owned()].into();
|
||||
let mut found = HashSet::new();
|
||||
|
||||
while let Some(event_id) = todo.pop_front() {
|
||||
trace!(%event_id, "processing auth event");
|
||||
|
||||
match self.services.timeline.get_pdu(&event_id).await {
|
||||
| Err(e) => {
|
||||
debug_error!(%event_id, ?e, "Could not find pdu mentioned in auth events");
|
||||
},
|
||||
| Ok(pdu) => {
|
||||
if let Some(claimed_room_id) = pdu.room_id.clone() {
|
||||
if claimed_room_id != *room_id {
|
||||
return Err!(Request(Forbidden(error!(
|
||||
%event_id,
|
||||
%room_id,
|
||||
wrong_room_id = ?pdu.room_id.unwrap(),
|
||||
"auth event for incorrect room"
|
||||
))));
|
||||
}
|
||||
}
|
||||
|
||||
for auth_event in &pdu.auth_events {
|
||||
let sauthevent = self
|
||||
.services
|
||||
.short
|
||||
.get_or_create_shorteventid(auth_event)
|
||||
.await;
|
||||
|
||||
if found.insert(sauthevent) {
|
||||
trace!(%event_id, ?auth_event, "adding auth event to processing queue");
|
||||
|
||||
todo.push_back(auth_event.clone());
|
||||
}
|
||||
}
|
||||
},
|
||||
let mut buckets = [BUCKET; NUM_BUCKETS];
|
||||
while let Some((short, starting_event)) = starting_ids.next().await {
|
||||
let bucket: usize = short.try_into()?;
|
||||
let bucket: usize = validated!(bucket % NUM_BUCKETS);
|
||||
buckets[bucket].insert((short, starting_event));
|
||||
}
|
||||
|
||||
trace!(
|
||||
starting_events = ?starting_events.count(),
|
||||
elapsed = ?started.elapsed(),
|
||||
"start",
|
||||
);
|
||||
|
||||
let full_auth_chain: Vec<ShortEventId> = buckets
|
||||
.into_iter()
|
||||
.try_stream()
|
||||
.broad_and_then(|chunk| self.get_auth_chain_outer(room_id, started, chunk))
|
||||
.try_collect()
|
||||
.map_ok(|auth_chain: Vec<_>| auth_chain.into_iter().flatten().collect())
|
||||
.map_ok(|mut full_auth_chain: Vec<_>| {
|
||||
full_auth_chain.sort_unstable();
|
||||
full_auth_chain.dedup();
|
||||
full_auth_chain
|
||||
})
|
||||
.boxed()
|
||||
.await?;
|
||||
|
||||
trace!(
|
||||
chain_length = ?full_auth_chain.len(),
|
||||
elapsed = ?started.elapsed(),
|
||||
"done",
|
||||
);
|
||||
|
||||
Ok(full_auth_chain)
|
||||
}
|
||||
|
||||
Ok(found.into_iter().collect())
|
||||
async fn get_auth_chain_outer(
|
||||
&self,
|
||||
room_id: &RoomId,
|
||||
started: Instant,
|
||||
chunk: Bucket<'_>,
|
||||
) -> Result<Vec<ShortEventId>> {
|
||||
let chunk_key: Vec<ShortEventId> = chunk.iter().map(at!(0)).collect();
|
||||
|
||||
if chunk_key.is_empty() {
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
|
||||
if let Ok(cached) = self.get_cached_eventid_authchain(&chunk_key).await {
|
||||
return Ok(cached.to_vec());
|
||||
}
|
||||
|
||||
let chunk_cache: Vec<_> = chunk
|
||||
.into_iter()
|
||||
.try_stream::<conduwuit::Error>()
|
||||
.broad_and_then(|(shortid, event_id)| async move {
|
||||
if let Ok(cached) = self.get_cached_eventid_authchain(&[shortid]).await {
|
||||
return Ok(cached.to_vec());
|
||||
}
|
||||
|
||||
let auth_chain = self.get_auth_chain_inner(room_id, event_id).await?;
|
||||
self.cache_auth_chain_vec(vec![shortid], auth_chain.as_slice());
|
||||
debug!(
|
||||
%event_id,
|
||||
elapsed = ?started.elapsed(),
|
||||
"Cache missed event"
|
||||
);
|
||||
|
||||
Ok(auth_chain)
|
||||
})
|
||||
.try_collect()
|
||||
.map_ok(|chunk_cache: Vec<_>| chunk_cache.into_iter().flatten().collect())
|
||||
.map_ok(|mut chunk_cache: Vec<_>| {
|
||||
chunk_cache.sort_unstable();
|
||||
chunk_cache.dedup();
|
||||
chunk_cache
|
||||
})
|
||||
.await?;
|
||||
|
||||
self.cache_auth_chain_vec(chunk_key, chunk_cache.as_slice());
|
||||
debug!(
|
||||
chunk_cache_length = ?chunk_cache.len(),
|
||||
elapsed = ?started.elapsed(),
|
||||
"Cache missed chunk",
|
||||
);
|
||||
|
||||
Ok(chunk_cache)
|
||||
}
|
||||
|
||||
#[tracing::instrument(name = "inner", level = "trace", skip(self, room_id))]
|
||||
async fn get_auth_chain_inner(
|
||||
&self,
|
||||
room_id: &RoomId,
|
||||
event_id: &EventId,
|
||||
) -> Result<Vec<ShortEventId>> {
|
||||
let mut todo: VecDeque<_> = [event_id.to_owned()].into();
|
||||
let mut found = HashSet::new();
|
||||
|
||||
while let Some(event_id) = todo.pop_front() {
|
||||
trace!(%event_id, "processing auth event");
|
||||
|
||||
match self.services.timeline.get_pdu(&event_id).await {
|
||||
| Err(e) => {
|
||||
debug_error!(%event_id, ?e, "Could not find pdu mentioned in auth events");
|
||||
},
|
||||
| Ok(pdu) => {
|
||||
if let Some(claimed_room_id) = pdu.room_id.clone() {
|
||||
if claimed_room_id != *room_id {
|
||||
return Err!(Request(Forbidden(error!(
|
||||
%event_id,
|
||||
%room_id,
|
||||
wrong_room_id = ?pdu.room_id.unwrap(),
|
||||
"auth event for incorrect room"
|
||||
))));
|
||||
}
|
||||
}
|
||||
|
||||
for auth_event in &pdu.auth_events {
|
||||
let sauthevent = self
|
||||
.services
|
||||
.short
|
||||
.get_or_create_shorteventid(auth_event)
|
||||
.await;
|
||||
|
||||
if found.insert(sauthevent) {
|
||||
trace!(%event_id, ?auth_event, "adding auth event to processing queue");
|
||||
|
||||
todo.push_back(auth_event.clone());
|
||||
}
|
||||
}
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
Ok(found.into_iter().collect())
|
||||
}
|
||||
|
||||
#[inline]
|
||||
pub async fn get_cached_eventid_authchain(&self, key: &[u64]) -> Result<Arc<[ShortEventId]>> {
|
||||
self.db.get_cached_eventid_authchain(key).await
|
||||
}
|
||||
|
||||
pub fn cache_auth_chain(&self, key: Vec<u64>, auth_chain: &HashSet<ShortEventId>) {
|
||||
let val: Arc<[ShortEventId]> = auth_chain.iter().copied().collect();
|
||||
|
||||
self.db.cache_auth_chain(key, val);
|
||||
}
|
||||
|
||||
pub fn cache_auth_chain_vec(&self, key: Vec<u64>, auth_chain: &[ShortEventId]) {
|
||||
let val: Arc<[ShortEventId]> = auth_chain.iter().copied().collect();
|
||||
|
||||
self.db.cache_auth_chain(key, val);
|
||||
}
|
||||
|
||||
pub fn get_cache_usage(&self) -> (usize, usize) {
|
||||
let cache = self.db.auth_chain_cache.lock();
|
||||
|
||||
(cache.len(), cache.capacity())
|
||||
}
|
||||
|
||||
pub fn clear_cache(&self) { self.db.auth_chain_cache.lock().clear(); }
|
||||
}
|
||||
|
||||
#[implement(Service)]
|
||||
#[inline]
|
||||
pub async fn get_cached_eventid_authchain(&self, key: &[u64]) -> Result<Arc<[ShortEventId]>> {
|
||||
self.db.get_cached_eventid_authchain(key).await
|
||||
}
|
||||
|
||||
#[implement(Service)]
|
||||
#[tracing::instrument(skip_all, level = "debug")]
|
||||
pub fn cache_auth_chain(&self, key: Vec<u64>, auth_chain: &HashSet<ShortEventId>) {
|
||||
let val: Arc<[ShortEventId]> = auth_chain.iter().copied().collect();
|
||||
|
||||
self.db.cache_auth_chain(key, val);
|
||||
}
|
||||
|
||||
#[implement(Service)]
|
||||
#[tracing::instrument(skip_all, level = "debug")]
|
||||
pub fn cache_auth_chain_vec(&self, key: Vec<u64>, auth_chain: &[ShortEventId]) {
|
||||
let val: Arc<[ShortEventId]> = auth_chain.iter().copied().collect();
|
||||
|
||||
self.db.cache_auth_chain(key, val);
|
||||
}
|
||||
|
||||
#[implement(Service)]
|
||||
pub fn get_cache_usage(&self) -> (usize, usize) {
|
||||
let cache = self.db.auth_chain_cache.lock();
|
||||
|
||||
(cache.len(), cache.capacity())
|
||||
}
|
||||
|
||||
#[implement(Service)]
|
||||
pub fn clear_cache(&self) { self.db.auth_chain_cache.lock().clear(); }
|
||||
@@ -1,6 +1,6 @@
|
||||
use std::sync::Arc;
|
||||
|
||||
use conduwuit::{Result, implement, utils::stream::TryIgnore};
|
||||
use conduwuit::{Result, utils::stream::TryIgnore};
|
||||
use database::Map;
|
||||
use futures::Stream;
|
||||
use ruma::{OwnedRoomId, RoomId, api::client::room::Visibility};
|
||||
@@ -25,27 +25,30 @@ impl crate::Service for Service {
|
||||
fn name(&self) -> &str { crate::service::make_name(std::module_path!()) }
|
||||
}
|
||||
|
||||
#[implement(Service)]
|
||||
pub fn set_public(&self, room_id: &RoomId) { self.db.publicroomids.insert(room_id, []); }
|
||||
impl Service {
|
||||
/// Marks a room as public, adding it to the room directory. Has no relation
|
||||
/// to the join rule.
|
||||
pub fn set_public(&self, room_id: &RoomId) { self.db.publicroomids.insert(room_id, []); }
|
||||
|
||||
#[implement(Service)]
|
||||
pub fn set_not_public(&self, room_id: &RoomId) { self.db.publicroomids.remove(room_id); }
|
||||
/// Removes a room from the public room directory.
|
||||
pub fn set_not_public(&self, room_id: &RoomId) { self.db.publicroomids.remove(room_id); }
|
||||
|
||||
#[implement(Service)]
|
||||
pub fn public_rooms(&self) -> impl Stream<Item = OwnedRoomId> + Send {
|
||||
self.db.publicroomids.keys().ignore_err()
|
||||
}
|
||||
/// Lists all public rooms in the directory.
|
||||
pub fn public_rooms(&self) -> impl Stream<Item = OwnedRoomId> + Send {
|
||||
self.db.publicroomids.keys().ignore_err()
|
||||
}
|
||||
|
||||
#[implement(Service)]
|
||||
pub async fn is_public_room(&self, room_id: &RoomId) -> bool {
|
||||
self.visibility(room_id).await == Visibility::Public
|
||||
}
|
||||
/// Checks if a room is public (per visibility, not join rule).
|
||||
pub async fn is_public_room(&self, room_id: &RoomId) -> bool {
|
||||
self.visibility(room_id).await == Visibility::Public
|
||||
}
|
||||
|
||||
#[implement(Service)]
|
||||
pub async fn visibility(&self, room_id: &RoomId) -> Visibility {
|
||||
if self.db.publicroomids.get(room_id).await.is_ok() {
|
||||
Visibility::Public
|
||||
} else {
|
||||
Visibility::Private
|
||||
/// Fetches the visibility of a specific room ID.
|
||||
pub async fn visibility(&self, room_id: &RoomId) -> Visibility {
|
||||
if self.db.publicroomids.get(room_id).await.is_ok() {
|
||||
Visibility::Public
|
||||
} else {
|
||||
Visibility::Private
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,42 +1,40 @@
|
||||
use conduwuit::{Err, Result, debug, implement, trace, warn};
|
||||
use conduwuit::{Err, Result, debug, debug_warn, trace};
|
||||
use ruma::{
|
||||
RoomId, ServerName,
|
||||
events::{StateEventType, room::server_acl::RoomServerAclEventContent},
|
||||
};
|
||||
|
||||
/// Returns Ok if the acl allows the server
|
||||
#[implement(super::Service)]
|
||||
#[tracing::instrument(skip_all, level = "debug")]
|
||||
pub async fn acl_check(&self, server_name: &ServerName, room_id: &RoomId) -> Result {
|
||||
let Ok(acl_event_content) = self
|
||||
.services
|
||||
.state_accessor
|
||||
.room_state_get_content(room_id, &StateEventType::RoomServerAcl, "")
|
||||
.await
|
||||
.map(|c: RoomServerAclEventContent| c)
|
||||
.inspect(|acl| trace!(%room_id, "ACL content found: {acl:?}"))
|
||||
.inspect_err(|e| trace!(%room_id, "No ACL content found: {e:?}"))
|
||||
else {
|
||||
return Ok(());
|
||||
};
|
||||
impl super::Service {
|
||||
/// Returns Ok if the acl allows the server, otherwise Forbidden.
|
||||
pub async fn acl_check(&self, server_name: &ServerName, room_id: &RoomId) -> Result {
|
||||
let Ok(acl_event_content) = self
|
||||
.services
|
||||
.state_accessor
|
||||
.room_state_get_content(room_id, &StateEventType::RoomServerAcl, "")
|
||||
.await
|
||||
.map(|c: RoomServerAclEventContent| c)
|
||||
else {
|
||||
return Ok(());
|
||||
};
|
||||
|
||||
if acl_event_content.allow.is_empty() {
|
||||
warn!(%room_id, "Ignoring broken ACL event (allow key is empty)");
|
||||
return Ok(());
|
||||
}
|
||||
if acl_event_content.allow.is_empty() {
|
||||
debug_warn!(%room_id, "Ignoring broken ACL event (allow key is empty)");
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
if acl_event_content.deny.contains(&String::from("*"))
|
||||
&& acl_event_content.allow.contains(&String::from("*"))
|
||||
{
|
||||
warn!(%room_id, "Ignoring broken ACL event (allow key and deny key both contain wildcard \"*\"");
|
||||
return Ok(());
|
||||
}
|
||||
if acl_event_content.deny.contains(&String::from("*"))
|
||||
&& acl_event_content.allow.contains(&String::from("*"))
|
||||
{
|
||||
debug_warn!(%room_id, "Ignoring broken ACL event (allow key and deny key both contain wildcard \"*\"");
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
if acl_event_content.is_allowed(server_name) {
|
||||
trace!("server {server_name} is allowed by ACL");
|
||||
Ok(())
|
||||
} else {
|
||||
debug!("Server {server_name} was denied by room ACL in {room_id}");
|
||||
Err!(Request(Forbidden("Server was denied by room ACL")))
|
||||
if acl_event_content.is_allowed(server_name) {
|
||||
trace!("server {server_name} is allowed by ACL");
|
||||
Ok(())
|
||||
} else {
|
||||
debug!("Server {server_name} was denied by room ACL in {room_id}");
|
||||
Err!(Request(Forbidden("Server was denied by room ACL")))
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,233 +1,667 @@
|
||||
use std::{
|
||||
collections::{BTreeMap, HashSet, VecDeque, hash_map},
|
||||
collections::{HashMap, HashSet, VecDeque},
|
||||
time::Instant,
|
||||
};
|
||||
|
||||
use assign::assign;
|
||||
#[cfg(debug_assertions)]
|
||||
use conduwuit::error;
|
||||
use conduwuit::{
|
||||
Event, PduEvent, debug, debug_warn, implement, matrix::event::gen_event_id_canonical_json,
|
||||
trace, utils::continue_exponential_backoff_secs, warn,
|
||||
Err, Event, PduEvent, debug, debug_error, debug_info, debug_warn, err,
|
||||
state_res::lexicographical_topological_sort,
|
||||
trace,
|
||||
utils::{IterStream, math::Expected, stream::BroadbandExt},
|
||||
warn,
|
||||
};
|
||||
use futures::{StreamExt, future::select_ok};
|
||||
use ruma::{
|
||||
CanonicalJsonValue, EventId, OwnedEventId, RoomId, ServerName,
|
||||
api::federation::event::get_event,
|
||||
CanonicalJsonObject, CanonicalJsonValue, EventId, MilliSecondsSinceUnixEpoch, OwnedEventId,
|
||||
OwnedServerName, RoomId, ServerName, UInt,
|
||||
api::federation::event::{get_event, get_missing_events},
|
||||
int,
|
||||
room_version_rules::RoomVersionRules,
|
||||
};
|
||||
|
||||
use super::get_room_version_rules;
|
||||
use crate::rooms::event_handler::parse_incoming_pdu::expect_event_id_array;
|
||||
|
||||
/// Find the event and auth it. Once the event is validated (steps 1 - 8)
|
||||
/// it is appended to the outliers Tree.
|
||||
pub const GET_MISSING_EVENTS_MAX_BATCH_SIZE: usize = 50;
|
||||
|
||||
/// Attempts to build a localised directed acyclic graph out of the given PDUs,
|
||||
/// returning them in a topologically sorted order.
|
||||
///
|
||||
/// Returns pdu and if we fetched it over federation the raw json.
|
||||
///
|
||||
/// a. Look in the main timeline (pduid_pdu tree)
|
||||
/// b. Look at outlier pdu tree
|
||||
/// c. Ask origin server over federation
|
||||
/// d. TODO: Ask other servers over federation?
|
||||
#[implement(super::Service)]
|
||||
pub(super) async fn fetch_and_handle_outliers<'a, Pdu, Events>(
|
||||
&self,
|
||||
origin: &'a ServerName,
|
||||
events: Events,
|
||||
create_event: &'a Pdu,
|
||||
room_id: &'a RoomId,
|
||||
) -> Vec<(PduEvent, Option<BTreeMap<String, CanonicalJsonValue>>)>
|
||||
where
|
||||
Pdu: Event + Send + Sync,
|
||||
Events: Iterator<Item = &'a EventId> + Clone + Send,
|
||||
{
|
||||
let back_off = |id| match self
|
||||
.services
|
||||
.globals
|
||||
.bad_event_ratelimiter
|
||||
.write()
|
||||
.entry(id)
|
||||
{
|
||||
| hash_map::Entry::Vacant(e) => {
|
||||
e.insert((Instant::now(), 1));
|
||||
},
|
||||
| hash_map::Entry::Occupied(mut e) => {
|
||||
*e.get_mut() = (Instant::now(), e.get().1.saturating_add(1));
|
||||
},
|
||||
};
|
||||
/// This is used to attempt to process PDUs in an order that respects their
|
||||
/// dependencies, however it is ultimately the sender's responsibility to send
|
||||
/// them in a processable order, so this is just a best effort attempt. It does
|
||||
/// not account for power levels or other tie breaks.
|
||||
pub async fn build_local_dag<S: std::hash::BuildHasher + Send + Sync>(
|
||||
pdu_map: &HashMap<OwnedEventId, &CanonicalJsonObject, S>,
|
||||
) -> conduwuit::Result<Vec<OwnedEventId>> {
|
||||
debug_assert!(pdu_map.len() >= 2, "needless call to build_local_dag with less than 2 PDUs");
|
||||
let mut dag: HashMap<OwnedEventId, HashSet<OwnedEventId>> =
|
||||
HashMap::with_capacity(pdu_map.len());
|
||||
let mut id_origin_ts: HashMap<OwnedEventId, _> = HashMap::with_capacity(pdu_map.len());
|
||||
|
||||
let mut events_with_auth_events = Vec::with_capacity(events.clone().count());
|
||||
trace!("Fetching {} outlier pdus", events.clone().count());
|
||||
for (event_id, value) in pdu_map {
|
||||
// We already checked that these properties are correct in parse_incoming_pdu,
|
||||
// so it's safe to unwrap here.
|
||||
// We also filter to remove any prev_events that are not in this pdu_map, as we
|
||||
// need to have at least one event with zero out degrees for the lexico-topo
|
||||
// sort below. If there are multiple events with omitted prevs, they will be
|
||||
// ordered by timestamp, then event ID. At that point though, it's unlikely to
|
||||
// matter.
|
||||
let prev_events = value
|
||||
.get("prev_events")
|
||||
.unwrap()
|
||||
.as_array()
|
||||
.unwrap()
|
||||
.iter()
|
||||
.map(|v| EventId::parse(v.as_str().unwrap()).unwrap())
|
||||
.filter(|id| pdu_map.contains_key(id))
|
||||
.collect();
|
||||
|
||||
for id in events {
|
||||
// a. Look in the main timeline (pduid_pdu tree)
|
||||
// b. Look at outlier pdu tree
|
||||
// (get_pdu_json checks both)
|
||||
if let Ok(local_pdu) = self.services.timeline.get_pdu(id).await {
|
||||
trace!("Found {id} in main timeline or outlier tree");
|
||||
events_with_auth_events.push((id.to_owned(), Some(local_pdu), vec![]));
|
||||
continue;
|
||||
}
|
||||
dag.insert(event_id.clone(), prev_events);
|
||||
let origin_server_ts = value
|
||||
.get("origin_server_ts")
|
||||
.and_then(CanonicalJsonValue::as_integer)
|
||||
.unwrap_or_default();
|
||||
id_origin_ts.insert(event_id.clone(), origin_server_ts);
|
||||
}
|
||||
|
||||
// c. Ask origin server over federation
|
||||
// We also handle its auth chain here so we don't get a stack overflow in
|
||||
// handle_outlier_pdu.
|
||||
let mut todo_auth_events: VecDeque<_> = [id.to_owned()].into();
|
||||
let mut events_in_reverse_order = Vec::with_capacity(todo_auth_events.len());
|
||||
debug!(count = dag.len(), "Sorting incoming events with partial graph");
|
||||
lexicographical_topological_sort(&dag, &async |node_id| {
|
||||
// Note: we don't bother fetching power levels because that would massively slow
|
||||
// this function down. This is a best-effort attempt to order events correctly
|
||||
// for processing, however ultimately that should be the sender's job.
|
||||
let ts = id_origin_ts
|
||||
.get(&node_id)
|
||||
.copied()
|
||||
.unwrap_or_else(|| int!(0))
|
||||
.to_string()
|
||||
.parse::<u64>()
|
||||
.ok()
|
||||
.and_then(UInt::new)
|
||||
.unwrap_or_default();
|
||||
Ok((int!(0), MilliSecondsSinceUnixEpoch(ts)))
|
||||
})
|
||||
.await
|
||||
.inspect(|sorted| {
|
||||
debug_assert_eq!(
|
||||
sorted.len(),
|
||||
pdu_map.len(),
|
||||
"Sorted graph was not the same size as the input graph"
|
||||
);
|
||||
})
|
||||
.map_err(|e| err!("failed to resolve local graph: {e}"))
|
||||
}
|
||||
|
||||
let mut events_all = HashSet::with_capacity(todo_auth_events.len());
|
||||
while let Some(next_id) = todo_auth_events.pop_front() {
|
||||
if let Some((time, tries)) = self
|
||||
.services
|
||||
.globals
|
||||
.bad_event_ratelimiter
|
||||
.read()
|
||||
.get(&*next_id)
|
||||
{
|
||||
// Exponential backoff
|
||||
const MIN_DURATION: u64 = 60 * 2;
|
||||
const MAX_DURATION: u64 = 60 * 60 * 8;
|
||||
if continue_exponential_backoff_secs(
|
||||
MIN_DURATION,
|
||||
MAX_DURATION,
|
||||
time.elapsed(),
|
||||
*tries,
|
||||
) {
|
||||
debug_warn!(
|
||||
tried = ?*tries,
|
||||
elapsed = ?time.elapsed(),
|
||||
"Backing off from {next_id}",
|
||||
);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
impl super::Service {
|
||||
/// Uses `POST /_matrix/federation/v1/get_missing_events/{room_id}` to fill
|
||||
/// gaps in the DAG.
|
||||
///
|
||||
/// This function walks backwards from `head`, fetching incrementally (by a
|
||||
/// factor of 10) more events until the remote we're fetching from either
|
||||
/// stops returning new events, or the min_depth is reached.
|
||||
///
|
||||
/// This function does not persist the events, but does validate them. The
|
||||
/// caller is responsible for passing them through handle_incoming_pdu or
|
||||
/// related functions.
|
||||
///
|
||||
/// Only the one `via` is asked for missing events, as multiplexing remotes
|
||||
/// may result in the event tree being walked in a gappy or disordered
|
||||
/// manner.
|
||||
///
|
||||
/// ## Parameters
|
||||
///
|
||||
/// - `room_id`: The room's ID.
|
||||
/// - `head`: The event we are potentially missing prev_events for.
|
||||
/// - `tail`: The most recently known events in the graph (typically forward
|
||||
/// extremities).
|
||||
/// - `via`: The server to ask for missing events.
|
||||
/// - `min_depth`: Don't process events with a `depth` lower than this
|
||||
/// value. Not massively useful, but can help short-circuit infinite loops
|
||||
/// and weird edge paths.
|
||||
#[tracing::instrument(name = "get_missing_events_bulk", skip_all)]
|
||||
pub async fn get_missing_events(
|
||||
&self,
|
||||
room_id: &RoomId,
|
||||
head: &PduEvent,
|
||||
tail: Vec<OwnedEventId>,
|
||||
via: &ServerName,
|
||||
min_depth: UInt,
|
||||
) -> conduwuit::Result<HashMap<OwnedEventId, PduEvent>> {
|
||||
let start = Instant::now();
|
||||
#[cfg(debug_assertions)]
|
||||
{
|
||||
let missing_count = head
|
||||
.prev_events()
|
||||
.stream()
|
||||
.fold(0_u8, |i, event_id| async move {
|
||||
if self.services.timeline.pdu_exists(event_id).await {
|
||||
i.expected_add(1)
|
||||
} else {
|
||||
i
|
||||
}
|
||||
})
|
||||
.await;
|
||||
debug_assert_ne!(
|
||||
missing_count, 0,
|
||||
"event passed to get_missing_events is not missing any events (wasteful call)"
|
||||
);
|
||||
};
|
||||
assert!(!tail.is_empty(), "empty tail");
|
||||
assert_ne!(via, self.services.globals.server_name(), "cannot ask ourselves for events");
|
||||
|
||||
if events_all.contains(&next_id) {
|
||||
continue;
|
||||
}
|
||||
// The iteration limit is in place to ensure that if the remote server leaves us
|
||||
// in a state of infinite recursion (as old versions of continuwuity and
|
||||
// predecessors would), we give up. However, get_missing_events doesn't return
|
||||
// that many events per-request. Synapse returns 20, and conduwuit+ return 50.
|
||||
// This means with a hard iteration limit, we might give up too early, before
|
||||
// we get a chance to even come close to max_fetch_prev_events. As such, we'll
|
||||
// calculate the limit based on that config option and the aforementioned
|
||||
// averages.
|
||||
let max_fetch = self.services.server.config.max_fetch_prev_events;
|
||||
let iteration_limit = max_fetch.saturating_div(20).max(10);
|
||||
|
||||
if self.services.timeline.pdu_exists(&next_id).await {
|
||||
trace!("Found {next_id} in db");
|
||||
continue;
|
||||
}
|
||||
|
||||
debug!("Fetching {next_id} over federation from {origin}.");
|
||||
match self
|
||||
let mut discovered = HashMap::with_capacity(head.prev_events.len());
|
||||
let mut latest_events: Vec<OwnedEventId> = vec![head.event_id().to_owned()];
|
||||
debug!(elapsed=?start.elapsed(),
|
||||
%room_id,
|
||||
event_id=%head.event_id(),
|
||||
%iteration_limit,
|
||||
"Fetching any missing events for head event",
|
||||
);
|
||||
for iteration in 0..iteration_limit {
|
||||
let limit = iteration
|
||||
.expected_add(1)
|
||||
.saturating_mul(10)
|
||||
.min(GET_MISSING_EVENTS_MAX_BATCH_SIZE.try_into().expect(
|
||||
"GET_MISSING_EVENTS_MAX_BATCH_SIZE (usize) should fit in u16 (<=65536)",
|
||||
))
|
||||
.max(
|
||||
// This max call ensures we fetch *at least* all the prev events the
|
||||
// head has.
|
||||
u16::try_from(head.prev_events.len())
|
||||
.expect("cannot have more than 20 prev events, which fits in u16"),
|
||||
);
|
||||
debug_info!(elapsed=?start.elapsed(),
|
||||
%limit,
|
||||
%via,
|
||||
%iteration,
|
||||
%iteration_limit,
|
||||
discovered=discovered.len(),
|
||||
%min_depth,
|
||||
"Attempting to gap fill missing events"
|
||||
);
|
||||
let response: get_missing_events::v1::Response = self
|
||||
.services
|
||||
.sending
|
||||
.send_federation_request(
|
||||
origin,
|
||||
get_event::v1::Request::new((*next_id).to_owned()),
|
||||
via,
|
||||
assign!(
|
||||
get_missing_events::v1::Request::new(
|
||||
room_id.to_owned(),
|
||||
tail.clone(),
|
||||
latest_events.clone()
|
||||
),
|
||||
{limit: limit.into(), min_depth}
|
||||
),
|
||||
)
|
||||
.await
|
||||
{
|
||||
| Ok(res) => {
|
||||
debug!("Got {next_id} over federation from {origin}");
|
||||
let Ok(room_version_rules) = get_room_version_rules(create_event) else {
|
||||
back_off((*next_id).to_owned());
|
||||
continue;
|
||||
};
|
||||
.await?;
|
||||
|
||||
let Ok((calculated_event_id, value)) =
|
||||
gen_event_id_canonical_json(&res.pdu, &room_version_rules)
|
||||
else {
|
||||
back_off((*next_id).to_owned());
|
||||
continue;
|
||||
};
|
||||
|
||||
if calculated_event_id != *next_id {
|
||||
warn!(
|
||||
"Server didn't return event id we requested: requested: {next_id}, \
|
||||
we got {calculated_event_id}. Event: {:?}",
|
||||
&res.pdu
|
||||
);
|
||||
}
|
||||
|
||||
if let Some(auth_events) = value
|
||||
.get("auth_events")
|
||||
.and_then(CanonicalJsonValue::as_array)
|
||||
{
|
||||
for auth_event in auth_events {
|
||||
match serde_json::from_value::<OwnedEventId>(
|
||||
auth_event.clone().into(),
|
||||
) {
|
||||
| Ok(auth_event) => {
|
||||
trace!(
|
||||
"Found auth event id {auth_event} for event {next_id}"
|
||||
);
|
||||
todo_auth_events.push_back(auth_event);
|
||||
},
|
||||
| _ => {
|
||||
warn!("Auth event id is not valid");
|
||||
},
|
||||
}
|
||||
}
|
||||
} else {
|
||||
warn!("Auth event list invalid");
|
||||
}
|
||||
|
||||
events_in_reverse_order.push((next_id.clone(), value));
|
||||
events_all.insert(next_id);
|
||||
},
|
||||
| Err(e) => {
|
||||
warn!("Failed to fetch auth event {next_id} from {origin}: {e}");
|
||||
back_off((*next_id).to_owned());
|
||||
},
|
||||
if response.events.is_empty() {
|
||||
debug_info!(
|
||||
elapsed=?start.elapsed(),
|
||||
%via,
|
||||
"Finished gap filling missing events (remote returned no more events)."
|
||||
);
|
||||
break;
|
||||
}
|
||||
}
|
||||
debug_info!(
|
||||
elapsed=?start.elapsed(),
|
||||
"Got {} events back from remote",
|
||||
response.events.len()
|
||||
);
|
||||
|
||||
events_with_auth_events.push((id.to_owned(), None, events_in_reverse_order));
|
||||
}
|
||||
|
||||
let mut pdus = Vec::with_capacity(events_with_auth_events.len());
|
||||
for (id, local_pdu, events_in_reverse_order) in events_with_auth_events {
|
||||
// a. Look in the main timeline (pduid_pdu tree)
|
||||
// b. Look at outlier pdu tree
|
||||
// (get_pdu_json checks both)
|
||||
if let Some(local_pdu) = local_pdu {
|
||||
trace!("Found {id} in main timeline or outlier tree");
|
||||
pdus.push((local_pdu.clone(), None));
|
||||
}
|
||||
|
||||
for (next_id, value) in events_in_reverse_order.into_iter().rev() {
|
||||
if let Some((time, tries)) = self
|
||||
.services
|
||||
.globals
|
||||
.bad_event_ratelimiter
|
||||
.read()
|
||||
.get(&*next_id)
|
||||
{
|
||||
// Exponential backoff
|
||||
const MIN_DURATION: u64 = 5 * 60;
|
||||
const MAX_DURATION: u64 = 60 * 60 * 24;
|
||||
if continue_exponential_backoff_secs(
|
||||
MIN_DURATION,
|
||||
MAX_DURATION,
|
||||
time.elapsed(),
|
||||
*tries,
|
||||
) {
|
||||
debug!("Backing off from {next_id}");
|
||||
latest_events.clear();
|
||||
for raw_event in response.events {
|
||||
let (_, event_id, pdu_json) = self.parse_incoming_pdu(&raw_event).await?;
|
||||
let pdu = PduEvent::from_id_val(&event_id, pdu_json).map_err(|e| {
|
||||
err!(Request(BadJson("Failed to parse gapfilled event {event_id}: {e}")))
|
||||
})?;
|
||||
if discovered.contains_key(&event_id) {
|
||||
// We already received this event.
|
||||
trace!("Already received {event_id}");
|
||||
continue;
|
||||
}
|
||||
if self
|
||||
.services
|
||||
.timeline
|
||||
.non_outlier_pdu_exists(&event_id)
|
||||
.await
|
||||
{
|
||||
// NOTE: we explicitly check for *non*-outlier events here, as if we end
|
||||
// up discovering outlier events, we will be able to upgrade them
|
||||
// immediately.
|
||||
trace!("Already have {event_id} as a timeline PDU");
|
||||
continue;
|
||||
}
|
||||
|
||||
if pdu.depth < min_depth {
|
||||
debug_warn!(
|
||||
elapsed=?start.elapsed(),
|
||||
"Received PDU with depth {} below min_depth {}",
|
||||
pdu.depth,
|
||||
min_depth
|
||||
);
|
||||
discovered.insert(event_id.clone(), pdu);
|
||||
continue;
|
||||
}
|
||||
|
||||
for prev_event_id in pdu.prev_events() {
|
||||
if discovered.contains_key(prev_event_id) {
|
||||
// We already received this event.
|
||||
trace!("Already received prev event {prev_event_id}");
|
||||
continue;
|
||||
}
|
||||
if self
|
||||
.services
|
||||
.timeline
|
||||
.non_outlier_pdu_exists(prev_event_id)
|
||||
.await
|
||||
{
|
||||
// NOTE: we explicitly check for *non*-outlier events here, as if we end
|
||||
// up discovering outlier events, we will be able to upgrade them
|
||||
// immediately.
|
||||
trace!("Already have prev event {prev_event_id} as a timeline PDU");
|
||||
continue;
|
||||
}
|
||||
if let Ok(outlier) = self.services.timeline.get_pdu(prev_event_id).await {
|
||||
// We already have this PDU as an outlier, don't ask for
|
||||
// it. However, if we are missing any prev events for it, add it to the
|
||||
// latest events anyway.
|
||||
let outlier_missing_prevs = outlier
|
||||
.prev_events()
|
||||
.stream()
|
||||
.fold(0_u8, |i, event_id| async move {
|
||||
if self.services.timeline.pdu_exists(event_id).await {
|
||||
i.expected_add(1)
|
||||
} else {
|
||||
i
|
||||
}
|
||||
})
|
||||
.await;
|
||||
if outlier_missing_prevs > 0 {
|
||||
trace!("Missing {outlier_missing_prevs} PDU(s) for prev event");
|
||||
latest_events.push(prev_event_id.to_owned());
|
||||
}
|
||||
trace!("Had {prev_event_id} as an outlier already, skipping discovery");
|
||||
discovered.insert(prev_event_id.to_owned(), outlier);
|
||||
continue;
|
||||
}
|
||||
trace!("Missing prev {prev_event_id} of {event_id}");
|
||||
latest_events.push(prev_event_id.to_owned());
|
||||
}
|
||||
trace!("Discovered {event_id}");
|
||||
discovered.insert(event_id.clone(), pdu);
|
||||
}
|
||||
|
||||
trace!("Handling outlier {next_id}");
|
||||
if latest_events.is_empty() {
|
||||
debug!(elapsed=?start.elapsed(),
|
||||
%limit,
|
||||
%via,
|
||||
%iteration,
|
||||
discovered=discovered.len(),
|
||||
"No more events to fetch."
|
||||
);
|
||||
break;
|
||||
}
|
||||
if discovered.len() >= self.services.server.config.max_fetch_prev_events.into() {
|
||||
// Stupid hack, debug_error!() drops the log to a DEBUG when not in debug mode,
|
||||
// which is bad because this should at least produce a warning. It's an error in
|
||||
// debug mode because this can be important, but typically not much can be done
|
||||
// about it as a user.
|
||||
#[cfg(debug_assertions)]
|
||||
error!(elapsed=?start.elapsed(),
|
||||
discovered=discovered.len(),
|
||||
max_fetch_prev_events=self.services.server.config.max_fetch_prev_events,
|
||||
%iteration,
|
||||
%iteration_limit,
|
||||
%via,
|
||||
event_id=%head.event_id(),
|
||||
%room_id,
|
||||
"Encountered a gap too large to fill, giving up"
|
||||
);
|
||||
#[cfg(not(debug_assertions))]
|
||||
warn!(elapsed=?start.elapsed(),
|
||||
discovered=discovered.len(),
|
||||
max_fetch_prev_events=self.services.server.config.max_fetch_prev_events,
|
||||
%iteration,
|
||||
%iteration_limit,
|
||||
%via,
|
||||
event_id=%head.event_id(),
|
||||
%room_id,
|
||||
"Encountered a gap too large to fill"
|
||||
);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
trace!(elapsed=?start.elapsed(), "Finished get_missing_events");
|
||||
Ok(discovered)
|
||||
}
|
||||
|
||||
/// Sends a `GET /_matrix/federation/v1/event/{event_id}` request to the
|
||||
/// target `remote`, parses the resulting PDU, and ensures the remote
|
||||
/// returned the correct event.
|
||||
/// Allows `fetch_and_handle_missing_events` to atomically fetch events from
|
||||
/// multiple remotes in parallel.
|
||||
async fn fetch_event_via(
|
||||
&self,
|
||||
remote: OwnedServerName,
|
||||
event_id: OwnedEventId,
|
||||
room_version_rules: &RoomVersionRules,
|
||||
) -> conduwuit::Result<(OwnedEventId, CanonicalJsonObject)> {
|
||||
let res = self
|
||||
.services
|
||||
.sending
|
||||
.send_federation_request(&remote, get_event::v1::Request::new(event_id.clone()))
|
||||
.await?;
|
||||
|
||||
let (calculated_event_id, value) = self
|
||||
.parse_incoming_pdu_with_known_room(&res.pdu, room_version_rules)
|
||||
.await?;
|
||||
|
||||
if calculated_event_id != event_id {
|
||||
Err!(Request(BadJson(warn!(
|
||||
expected=%event_id,
|
||||
received=%calculated_event_id,
|
||||
"Server didn't return event id we requested",
|
||||
))))
|
||||
} else {
|
||||
Ok((event_id, value))
|
||||
}
|
||||
}
|
||||
|
||||
async fn fetch_event_vias(
|
||||
&self,
|
||||
candidates: impl Iterator<Item = &OwnedServerName>,
|
||||
event_id: &EventId,
|
||||
room_version_rules: &RoomVersionRules,
|
||||
) -> conduwuit::Result<(OwnedEventId, CanonicalJsonObject)> {
|
||||
if let Ok(pdu_json) = self.services.timeline.get_pdu_json(event_id).await {
|
||||
return Ok((event_id.to_owned(), pdu_json));
|
||||
}
|
||||
let futures = candidates
|
||||
.map(|remote| {
|
||||
Box::pin(self.fetch_event_via(
|
||||
remote.to_owned(),
|
||||
event_id.to_owned(),
|
||||
room_version_rules,
|
||||
))
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
select_ok(futures).await.map(|(res, _)| res)
|
||||
}
|
||||
|
||||
/// Asks remote servers for any individual events that are missing, also
|
||||
/// known as "atomic fetch". Should only be used for fetching missing auth
|
||||
/// events or resolving missing events from state_ids. For all other uses,
|
||||
/// use get_missing_events.
|
||||
///
|
||||
/// This function manually walks auth_events trees in a breadth-first
|
||||
/// search, and persists all fetched events as outliers when all the
|
||||
/// backwards extremities have been resolved.
|
||||
#[tracing::instrument(name = "get_missing_auth_events_atomic", skip_all)]
|
||||
pub(super) async fn fetch_and_handle_auth_events<Pdu>(
|
||||
&self,
|
||||
origin: &ServerName,
|
||||
events: Vec<OwnedEventId>,
|
||||
create_event: &Pdu,
|
||||
room_id: &RoomId,
|
||||
) -> HashMap<OwnedEventId, PduEvent>
|
||||
where
|
||||
Pdu: Event + Send + Sync,
|
||||
{
|
||||
let start = Instant::now();
|
||||
let room_version_rules =
|
||||
&get_room_version_rules(create_event).unwrap_or(RoomVersionRules::V1);
|
||||
let mut candidates = self
|
||||
.services
|
||||
.timeline
|
||||
.candidate_backfill_servers(room_id)
|
||||
.await;
|
||||
candidates.insert(origin.to_owned());
|
||||
assert!(!candidates.is_empty(), "no candidates to fetch missing events from");
|
||||
let mut discovered_events =
|
||||
HashMap::with_capacity(events.len().saturating_add(events.len().saturating_mul(3)));
|
||||
trace!(
|
||||
elapsed=?start.elapsed(),
|
||||
"Fetching {} unknown PDUs on demand from {} candidates",
|
||||
events.len(),
|
||||
candidates.len()
|
||||
);
|
||||
|
||||
let mut seen: HashMap<OwnedEventId, u8> = HashMap::new();
|
||||
for apex_event_id in &events {
|
||||
let mut todo: VecDeque<OwnedEventId> = [apex_event_id.to_owned()].into();
|
||||
|
||||
while let Some(target_id) = todo.pop_front() {
|
||||
if discovered_events.contains_key(&target_id) {
|
||||
continue;
|
||||
}
|
||||
if let Ok(local_pdu) = self.services.timeline.get_pdu(&target_id).await {
|
||||
trace!(elapsed=?start.elapsed(), "Found {target_id} in db");
|
||||
let mut obj = local_pdu.into_canonical_object();
|
||||
obj.remove("event_id");
|
||||
discovered_events.insert(target_id.clone(), obj);
|
||||
continue;
|
||||
}
|
||||
let attempts = seen.get(&*target_id).copied().unwrap_or_default();
|
||||
if attempts >= 5 {
|
||||
debug_error!(
|
||||
elapsed=?start.elapsed(),
|
||||
%attempts,
|
||||
%target_id,
|
||||
"Could not fetch missing event after 5 attempts, giving up"
|
||||
);
|
||||
continue;
|
||||
}
|
||||
|
||||
debug!(elapsed=?start.elapsed(),"Fetching {target_id} over federation");
|
||||
let value = match self
|
||||
.fetch_event_vias(candidates.iter(), &target_id, room_version_rules)
|
||||
.await
|
||||
{
|
||||
| Ok((_, x)) => x,
|
||||
| Err(e) => {
|
||||
warn!(elapsed=?start.elapsed(),"failed to fetch missing event {target_id} from any candidate: {e}");
|
||||
continue;
|
||||
},
|
||||
};
|
||||
let auth_events =
|
||||
match expect_event_id_array(&value, "auth_events").map_err(|e| {
|
||||
err!(Request(BadJson(warn!(
|
||||
elapsed=?start.elapsed(),
|
||||
event_id=%target_id,
|
||||
"Failed to parse event fetched from remote: {e}"
|
||||
))))
|
||||
}) {
|
||||
| Ok(auth_events) => auth_events,
|
||||
| Err(e) => {
|
||||
warn!(
|
||||
elapsed=?start.elapsed(),
|
||||
?e,
|
||||
"event {target_id} is malformed (bad auth_events), skipping"
|
||||
);
|
||||
continue;
|
||||
},
|
||||
};
|
||||
let mut have_all_auth = true;
|
||||
for auth_event_id in auth_events {
|
||||
if let Ok(local_pdu) = self.services.timeline.get_pdu(&auth_event_id).await {
|
||||
trace!(elapsed=?start.elapsed(),"Found auth event {auth_event_id} in db");
|
||||
let mut obj = local_pdu.into_canonical_object();
|
||||
obj.remove("event_id");
|
||||
discovered_events.insert(auth_event_id.clone(), obj);
|
||||
continue;
|
||||
}
|
||||
if discovered_events.contains_key(&auth_event_id) {
|
||||
trace!(elapsed=?start.elapsed(),%auth_event_id, "Already found auth event");
|
||||
continue;
|
||||
}
|
||||
debug!(elapsed=?start.elapsed(),"Missing auth event {auth_event_id} for event {target_id}");
|
||||
seen.insert(
|
||||
auth_event_id.clone(),
|
||||
seen.get(&auth_event_id)
|
||||
.copied()
|
||||
.unwrap_or_default()
|
||||
.saturating_add(1),
|
||||
);
|
||||
todo.push_back(auth_event_id);
|
||||
have_all_auth = false;
|
||||
}
|
||||
// Insert this PDU back at the end of the queue so that it will be resolved once
|
||||
// all of its auth events have been fetched.
|
||||
if have_all_auth {
|
||||
debug!(elapsed=?start.elapsed(),%target_id, "Have all auth events");
|
||||
discovered_events.insert(target_id, value);
|
||||
} else {
|
||||
debug_warn!(elapsed=?start.elapsed(),
|
||||
"Fetched {target_id} but missing some auth events, will have to re-fetch."
|
||||
);
|
||||
seen.insert(target_id.clone(), attempts.saturating_add(1));
|
||||
todo.push_back(target_id);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let refmap: HashMap<OwnedEventId, &CanonicalJsonObject> = discovered_events
|
||||
.iter()
|
||||
.map(|(id, data)| (id.clone(), data))
|
||||
.collect();
|
||||
let seeded_ordered = build_local_dag(&refmap)
|
||||
.await
|
||||
.expect("failed to build local DAG");
|
||||
let mut pdus = HashMap::with_capacity(seeded_ordered.len());
|
||||
for discovered_event_id in seeded_ordered {
|
||||
let pdu_json = discovered_events.remove(&discovered_event_id).unwrap();
|
||||
debug_info!(
|
||||
elapsed=?start.elapsed(),
|
||||
"Handling missing event {discovered_event_id} as outlier"
|
||||
);
|
||||
assert_eq!(pdu_json.get("event_id"), None, "pdu_json had event_id");
|
||||
match Box::pin(self.handle_outlier_pdu(
|
||||
origin,
|
||||
create_event,
|
||||
&next_id,
|
||||
&discovered_event_id,
|
||||
room_id,
|
||||
value.clone(),
|
||||
true,
|
||||
pdu_json,
|
||||
))
|
||||
.await
|
||||
{
|
||||
| Ok((pdu, json)) =>
|
||||
if next_id == *id {
|
||||
trace!("Handled outlier {next_id} (original request)");
|
||||
pdus.push((pdu, Some(json)));
|
||||
},
|
||||
| Err(e) => {
|
||||
warn!("Authentication of event {next_id} failed: {e:?}");
|
||||
back_off(next_id);
|
||||
| Ok((pdu, _)) => {
|
||||
trace!(elapsed=?start.elapsed(), "Persisted {discovered_event_id}");
|
||||
let _ = pdus.insert(discovered_event_id, pdu);
|
||||
},
|
||||
| Err(e) => warn!(
|
||||
elapsed=?start.elapsed(),
|
||||
"Authentication of event {discovered_event_id} failed: {e:?}"
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
trace!(
|
||||
elapsed=?start.elapsed(),
|
||||
"Finished fetch_and_handle_missing_events: fetched and handled {} missing PDUs",
|
||||
pdus.len()
|
||||
);
|
||||
pdus.retain(|id, _| events.contains(id)); // Only return state events
|
||||
trace!(elapsed=?start.elapsed(), "Filtered return value down to {} PDUs", pdus.len());
|
||||
pdus
|
||||
}
|
||||
|
||||
/// Similar to `fetch_and_handle_missing_events`, but simply walks the
|
||||
/// prev events tree instead of the auth events tree. Additionally, it does
|
||||
/// not *handle* fetched PDUs in any capacity.
|
||||
#[tracing::instrument(name = "get_missing_prev_events_atomic", skip_all)]
|
||||
pub(super) async fn fetch_prev_events<Pdu>(
|
||||
&self,
|
||||
origin: &ServerName,
|
||||
events: Vec<OwnedEventId>,
|
||||
create_event: &Pdu,
|
||||
room_id: &RoomId,
|
||||
) -> HashMap<OwnedEventId, PduEvent>
|
||||
where
|
||||
Pdu: Event + Send + Sync,
|
||||
{
|
||||
let room_version_rules =
|
||||
&get_room_version_rules(create_event).unwrap_or(RoomVersionRules::V1);
|
||||
let mut candidates = self
|
||||
.services
|
||||
.timeline
|
||||
.candidate_backfill_servers(room_id)
|
||||
.await;
|
||||
candidates.insert(origin.to_owned());
|
||||
|
||||
let mut todo: VecDeque<OwnedEventId> = VecDeque::from(events);
|
||||
let mut discovered_events = HashMap::new();
|
||||
while let Some(next_id) = todo.pop_front() {
|
||||
if discovered_events.len() >= self.services.server.config.max_fetch_prev_events.into()
|
||||
{
|
||||
debug_warn!(
|
||||
"Encountered a gap too large to fill, giving up (fetched {} events)",
|
||||
discovered_events.len()
|
||||
);
|
||||
break;
|
||||
}
|
||||
if discovered_events.contains_key(&next_id) {
|
||||
continue;
|
||||
}
|
||||
let pdu = match self
|
||||
.fetch_event_vias(candidates.iter(), &next_id, room_version_rules)
|
||||
.await
|
||||
{
|
||||
| Ok((_, data)) => data,
|
||||
| Err(e) => {
|
||||
warn!("Failed to fetch prev event {next_id} from any candidate: {e}");
|
||||
continue;
|
||||
},
|
||||
};
|
||||
|
||||
let prev_events = match expect_event_id_array(&pdu, "prev_events").map_err(|e| {
|
||||
err!(Request(BadJson(warn!(
|
||||
event_id=%next_id,
|
||||
"Failed to parse event fetched from remote: {e}"
|
||||
))))
|
||||
}) {
|
||||
| Ok(auth_events) => auth_events,
|
||||
| Err(e) => {
|
||||
warn!(?e, "event {next_id} is malformed (bad prev_events), skipping");
|
||||
continue;
|
||||
},
|
||||
};
|
||||
let missing_prev = prev_events
|
||||
.iter()
|
||||
.stream()
|
||||
.broad_filter_map(|event_id| async {
|
||||
if discovered_events.contains_key(event_id)
|
||||
|| self.services.timeline.pdu_exists(event_id).await
|
||||
{
|
||||
None
|
||||
} else {
|
||||
Some(event_id.to_owned())
|
||||
}
|
||||
})
|
||||
.collect::<Vec<_>>()
|
||||
.await;
|
||||
todo.extend(missing_prev);
|
||||
discovered_events.insert(
|
||||
next_id.clone(),
|
||||
PduEvent::from_id_val(&next_id, pdu).expect("fetched PDU was already validated"),
|
||||
);
|
||||
}
|
||||
|
||||
discovered_events
|
||||
}
|
||||
trace!("Fetched and handled {} outlier pdus", pdus.len());
|
||||
pdus
|
||||
}
|
||||
@@ -1,128 +1,155 @@
|
||||
use std::{
|
||||
collections::{BTreeMap, HashMap, HashSet, VecDeque},
|
||||
iter::once,
|
||||
};
|
||||
use std::{collections::HashMap, time::Instant};
|
||||
|
||||
use conduwuit::{
|
||||
Event, PduEvent, Result, debug_warn, err, implement,
|
||||
state_res::{self},
|
||||
};
|
||||
use futures::{FutureExt, future};
|
||||
use ruma::{
|
||||
CanonicalJsonValue, EventId, MilliSecondsSinceUnixEpoch, OwnedEventId, RoomId, ServerName,
|
||||
int, uint,
|
||||
Event, PduEvent, debug, debug_info, debug_warn, trace,
|
||||
utils::{BoolExt, IterStream, stream::BroadbandExt},
|
||||
};
|
||||
use futures::StreamExt;
|
||||
use ruma::{CanonicalJsonObject, MilliSecondsSinceUnixEpoch, OwnedEventId, RoomId, ServerName};
|
||||
|
||||
use super::check_room_id;
|
||||
use crate::rooms::event_handler::build_local_dag;
|
||||
|
||||
#[implement(super::Service)]
|
||||
#[tracing::instrument(
|
||||
level = "debug",
|
||||
skip_all,
|
||||
fields(%origin),
|
||||
)]
|
||||
#[allow(clippy::type_complexity)]
|
||||
pub(super) async fn fetch_prev<'a, Pdu, Events>(
|
||||
&self,
|
||||
origin: &ServerName,
|
||||
create_event: &Pdu,
|
||||
room_id: &RoomId,
|
||||
first_ts_in_room: MilliSecondsSinceUnixEpoch,
|
||||
initial_set: Events,
|
||||
) -> Result<(
|
||||
Vec<OwnedEventId>,
|
||||
HashMap<OwnedEventId, (PduEvent, BTreeMap<String, CanonicalJsonValue>)>,
|
||||
)>
|
||||
where
|
||||
Pdu: Event + Send + Sync,
|
||||
Events: Iterator<Item = &'a EventId> + Clone + Send,
|
||||
{
|
||||
let num_ids = initial_set.clone().count();
|
||||
let mut eventid_info = HashMap::new();
|
||||
let mut graph: HashMap<OwnedEventId, _> = HashMap::with_capacity(num_ids);
|
||||
let mut todo_outlier_stack: VecDeque<OwnedEventId> =
|
||||
initial_set.map(ToOwned::to_owned).collect();
|
||||
|
||||
let mut amount = 0;
|
||||
|
||||
while let Some(prev_event_id) = todo_outlier_stack.pop_front() {
|
||||
self.services.server.check_running()?;
|
||||
|
||||
match self
|
||||
.fetch_and_handle_outliers(
|
||||
origin,
|
||||
once(prev_event_id.as_ref()),
|
||||
create_event,
|
||||
room_id,
|
||||
)
|
||||
.boxed()
|
||||
.await
|
||||
.pop()
|
||||
{
|
||||
| Some((pdu, mut json_opt)) => {
|
||||
check_room_id(room_id, &pdu)?;
|
||||
|
||||
let limit = self.services.server.config.max_fetch_prev_events;
|
||||
if amount > limit {
|
||||
debug_warn!("Max prev event limit reached! Limit: {limit}");
|
||||
graph.insert(prev_event_id.clone(), HashSet::new());
|
||||
continue;
|
||||
}
|
||||
|
||||
if json_opt.is_none() {
|
||||
json_opt = self
|
||||
.services
|
||||
.outlier
|
||||
.get_outlier_pdu_json(&prev_event_id)
|
||||
.await
|
||||
.ok();
|
||||
}
|
||||
|
||||
if let Some(json) = json_opt {
|
||||
if pdu.origin_server_ts() > first_ts_in_room {
|
||||
amount = amount.saturating_add(1);
|
||||
for prev_prev in pdu.prev_events() {
|
||||
if !graph.contains_key(prev_prev) {
|
||||
todo_outlier_stack.push_back(prev_prev.to_owned());
|
||||
}
|
||||
}
|
||||
|
||||
graph.insert(
|
||||
prev_event_id.clone(),
|
||||
pdu.prev_events().map(ToOwned::to_owned).collect(),
|
||||
);
|
||||
} else {
|
||||
// Time based check failed
|
||||
graph.insert(prev_event_id.clone(), HashSet::new());
|
||||
}
|
||||
|
||||
eventid_info.insert(prev_event_id.clone(), (pdu, json));
|
||||
} else {
|
||||
// Get json failed, so this was not fetched over federation
|
||||
graph.insert(prev_event_id.clone(), HashSet::new());
|
||||
}
|
||||
},
|
||||
| _ => {
|
||||
// Fetch and handle failed
|
||||
graph.insert(prev_event_id.clone(), HashSet::new());
|
||||
},
|
||||
impl super::Service {
|
||||
/// Fetches any missing prev_events for this event and persists them before
|
||||
/// returning.
|
||||
pub(super) async fn fetch_prevs(
|
||||
&self,
|
||||
room_id: &RoomId,
|
||||
create_event: &PduEvent,
|
||||
incoming_pdu: &PduEvent,
|
||||
origin: &ServerName,
|
||||
first_ts_in_room: MilliSecondsSinceUnixEpoch,
|
||||
) -> conduwuit::Result<()> {
|
||||
let start = Instant::now();
|
||||
let mut missing = incoming_pdu
|
||||
.prev_events()
|
||||
.stream()
|
||||
.broad_filter_map(|event_id| async move {
|
||||
self.services
|
||||
.timeline
|
||||
.get_non_outlier_pdu_json(event_id)
|
||||
.await
|
||||
.is_ok()
|
||||
.or(|| event_id.to_owned())
|
||||
})
|
||||
.collect::<Vec<_>>()
|
||||
.await;
|
||||
if missing.is_empty() {
|
||||
debug!(elapsed=?start.elapsed(), event_id=%incoming_pdu.event_id(), "No missing prev events.");
|
||||
return Ok(());
|
||||
}
|
||||
debug!(elapsed=?start.elapsed(), %room_id, event_id=%incoming_pdu.event_id(), ?missing, "Fetching previous events");
|
||||
let tail = self
|
||||
.services
|
||||
.state
|
||||
.get_forward_extremities(room_id)
|
||||
.collect::<Vec<_>>()
|
||||
.await;
|
||||
|
||||
let mut gapfilled = self
|
||||
.get_missing_events(
|
||||
room_id,
|
||||
incoming_pdu,
|
||||
tail,
|
||||
origin,
|
||||
self.services
|
||||
.metadata
|
||||
.get_mindepth(room_id)
|
||||
.await
|
||||
.saturating_sub(
|
||||
u8::try_from(incoming_pdu.prev_events.len())
|
||||
.unwrap()
|
||||
.saturating_mul(2)
|
||||
.into(),
|
||||
),
|
||||
)
|
||||
.await?;
|
||||
debug_info!(elapsed=?start.elapsed(), "Fetched {} missing events", gapfilled.len());
|
||||
missing.retain(|eid| !gapfilled.contains_key(eid));
|
||||
if !missing.is_empty() {
|
||||
debug_warn!(elapsed=?start.elapsed(), "Still missing {} events, falling back to atomic fetch.", missing.len());
|
||||
gapfilled.extend(
|
||||
self.fetch_prev_events(origin, missing, create_event, room_id)
|
||||
.await,
|
||||
);
|
||||
}
|
||||
|
||||
// Persist all fetched events
|
||||
let mapped = gapfilled
|
||||
.iter()
|
||||
.map(|(eid, evt)| {
|
||||
let mut obj = evt.to_canonical_object();
|
||||
obj.remove("event_id"); // event_id is inserted by backfill_missing_events
|
||||
(eid.clone(), obj)
|
||||
})
|
||||
.collect::<HashMap<_, _>>();
|
||||
|
||||
let to_persist = if mapped.len() <= 1 {
|
||||
mapped.keys().map(ToOwned::to_owned).collect()
|
||||
} else {
|
||||
let refmap: HashMap<OwnedEventId, &CanonicalJsonObject> =
|
||||
mapped.iter().map(|(id, data)| (id.clone(), data)).collect();
|
||||
build_local_dag(&refmap).await?
|
||||
};
|
||||
|
||||
let job_start = Instant::now();
|
||||
trace!("Starting to persist {} prev events", to_persist.len());
|
||||
for (i, event_id) in to_persist.iter().enumerate() {
|
||||
debug!(
|
||||
elapsed=?start.elapsed(),
|
||||
"Persisting fetched prev event: {event_id} ({}/{})",
|
||||
i.saturating_add(1),
|
||||
to_persist.len(),
|
||||
);
|
||||
let obj = mapped.get(event_id).cloned().unwrap();
|
||||
let persist_start = Instant::now();
|
||||
match self
|
||||
.handle_outlier_pdu(origin, create_event, event_id, room_id, obj)
|
||||
.await
|
||||
{
|
||||
| Ok((pdu, val)) if pdu.origin_server_ts() >= first_ts_in_room => {
|
||||
Box::pin(self.upgrade_outlier_to_timeline_pdu(
|
||||
pdu,
|
||||
val,
|
||||
create_event,
|
||||
origin,
|
||||
room_id,
|
||||
))
|
||||
.await
|
||||
.inspect_err(|e| {
|
||||
debug_warn!(
|
||||
total_elapsed=?start.elapsed(),
|
||||
job_elapsed=?job_start.elapsed(),
|
||||
task_elapsed=?persist_start.elapsed(),
|
||||
"Failed to upgrade prev event {event_id}: {e}",
|
||||
);
|
||||
})
|
||||
.inspect(|_| {
|
||||
debug_info!(
|
||||
total_elapsed=?start.elapsed(),
|
||||
job_elapsed=?job_start.elapsed(),
|
||||
task_elapsed=?persist_start.elapsed(),
|
||||
"Upgraded prev event {event_id}",
|
||||
);
|
||||
})
|
||||
.ok();
|
||||
},
|
||||
| Err(e) => debug_warn!(
|
||||
total_elapsed=?start.elapsed(),
|
||||
job_elapsed=?job_start.elapsed(),
|
||||
task_elapsed=?persist_start.elapsed(),
|
||||
"Failed to persist prev event {event_id}: {e}",
|
||||
),
|
||||
| _ => {},
|
||||
}
|
||||
}
|
||||
|
||||
// NOTE because i keep forgetting: the caller persists incoming_pdu.
|
||||
// we only care about its prev events
|
||||
trace!(
|
||||
total_elapsed=?start.elapsed(),
|
||||
persist_elapsed=?job_start.elapsed(),
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
let event_fetch = |event_id| {
|
||||
let origin_server_ts = eventid_info
|
||||
.get(&event_id)
|
||||
.map_or_else(|| uint!(0), |info| info.0.origin_server_ts().get());
|
||||
|
||||
// This return value is the key used for sorting events,
|
||||
// events are then sorted by power level, time,
|
||||
// and lexically by event_id.
|
||||
future::ok((int!(0), MilliSecondsSinceUnixEpoch(origin_server_ts)))
|
||||
};
|
||||
|
||||
let sorted = state_res::lexicographical_topological_sort(&graph, &event_fetch)
|
||||
.await
|
||||
.map_err(|e| err!(Database(error!("Error sorting prev events: {e}"))))?;
|
||||
|
||||
Ok((sorted, eventid_info))
|
||||
}
|
||||
@@ -1,86 +1,392 @@
|
||||
use std::collections::{HashMap, hash_map};
|
||||
|
||||
use conduwuit::{Err, Event, Result, debug, debug_warn, err, implement};
|
||||
use futures::FutureExt;
|
||||
use ruma::{
|
||||
EventId, OwnedEventId, RoomId, ServerName, api::federation::event::get_room_state_ids,
|
||||
events::StateEventType,
|
||||
use std::{
|
||||
cmp::max,
|
||||
collections::{HashMap, HashSet, hash_map},
|
||||
hash::{BuildHasherDefault, DefaultHasher},
|
||||
time::{Duration, Instant},
|
||||
};
|
||||
|
||||
use crate::rooms::short::ShortStateKey;
|
||||
use conduwuit::{
|
||||
Err, Event, PduEvent, Result, debug, debug_warn, err, info, trace,
|
||||
utils::{BoolExt, IterStream},
|
||||
warn,
|
||||
};
|
||||
use futures::{StreamExt, TryFutureExt, future::select_ok};
|
||||
use ruma::{
|
||||
EventId, OwnedEventId, OwnedRoomId, RoomId, ServerName,
|
||||
api::federation::event::{get_room_state, get_room_state_ids},
|
||||
};
|
||||
|
||||
/// Call /state_ids to find out what the state at this pdu is. We trust the
|
||||
/// server's response to some extend (sic), but we still do a lot of checks
|
||||
/// on the events
|
||||
#[implement(super::Service)]
|
||||
#[tracing::instrument(
|
||||
level = "debug",
|
||||
skip_all,
|
||||
fields(%origin),
|
||||
)]
|
||||
pub(super) async fn fetch_state<Pdu>(
|
||||
&self,
|
||||
origin: &ServerName,
|
||||
create_event: &Pdu,
|
||||
room_id: &RoomId,
|
||||
event_id: &EventId,
|
||||
) -> Result<Option<HashMap<u64, OwnedEventId>>>
|
||||
where
|
||||
Pdu: Event + Send + Sync,
|
||||
{
|
||||
let res = self
|
||||
.services
|
||||
.sending
|
||||
.send_federation_request(
|
||||
origin,
|
||||
get_room_state_ids::v1::Request::new(event_id.to_owned(), room_id.to_owned()),
|
||||
)
|
||||
.await
|
||||
.inspect_err(|e| debug_warn!("Fetching state for event failed: {e}"))?;
|
||||
use crate::{conduwuit::utils::stream::BroadbandExt, rooms::short::ShortStateKey};
|
||||
|
||||
debug!("Fetching state events");
|
||||
let state_ids = res.pdu_ids.iter().map(AsRef::as_ref);
|
||||
let state_vec = self
|
||||
.fetch_and_handle_outliers(origin, state_ids, create_event, room_id)
|
||||
.boxed()
|
||||
.await;
|
||||
|
||||
let mut state: HashMap<ShortStateKey, OwnedEventId> = HashMap::with_capacity(state_vec.len());
|
||||
for (pdu, _) in state_vec {
|
||||
let state_key = pdu
|
||||
.state_key()
|
||||
.ok_or_else(|| err!(Database("Found non-state pdu in state events.")))?;
|
||||
|
||||
let shortstatekey = self
|
||||
impl super::Service {
|
||||
/// Asks a remote server what the state at this event is.
|
||||
/// It first attempts to call `GET /_matrix/federation/v1/state_ids` (fast).
|
||||
/// If any events are missing, they are fetched from the remote, and
|
||||
/// persisted as outliers, before being returned back to this function. If
|
||||
/// we are missing a lot of events locally (>=50), this function falls back
|
||||
/// to requesting the full state in PDU format from the remote (`GET
|
||||
/// /_matrix/federation/v1/state, very slow in large rooms), and persists
|
||||
/// them directly.
|
||||
#[tracing::instrument(skip_all)]
|
||||
pub(super) async fn fetch_state(
|
||||
&self,
|
||||
origin: &ServerName,
|
||||
create_event: &PduEvent,
|
||||
room_id: &RoomId,
|
||||
event_id: &EventId,
|
||||
) -> Result<HashMap<u64, OwnedEventId>> {
|
||||
let start = Instant::now();
|
||||
trace!(%origin, "Asking remote for state_ids");
|
||||
let res: get_room_state_ids::v1::Response = match self
|
||||
.services
|
||||
.short
|
||||
.get_or_create_shortstatekey(&pdu.kind().to_string().into(), state_key)
|
||||
.sending
|
||||
.send_federation_request(
|
||||
origin,
|
||||
get_room_state_ids::v1::Request::new(event_id.to_owned(), room_id.to_owned()),
|
||||
)
|
||||
.await
|
||||
.inspect_err(
|
||||
|e| debug_warn!(elapsed=?start.elapsed(), "Fetching state for event failed: {e}"),
|
||||
) {
|
||||
| Ok(resp) => Ok(resp),
|
||||
| Err(e) =>
|
||||
if e.is_not_found() {
|
||||
self.fetch_state_ids_from_backfill_servers(
|
||||
event_id.to_owned(),
|
||||
room_id.to_owned(),
|
||||
)
|
||||
.await
|
||||
} else {
|
||||
Err(e)
|
||||
},
|
||||
}?;
|
||||
|
||||
debug!(elapsed=?start.elapsed(), events = res.pdu_ids.len(), "Fetching state events");
|
||||
let mut state_events: HashMap<OwnedEventId, PduEvent> =
|
||||
HashMap::with_capacity(res.pdu_ids.len());
|
||||
let to_fetch: Vec<OwnedEventId> = res
|
||||
.pdu_ids
|
||||
.clone()
|
||||
.into_iter()
|
||||
.stream()
|
||||
.broad_filter_map(|event_id| async move {
|
||||
self.services
|
||||
.timeline
|
||||
.pdu_exists(&event_id)
|
||||
.await
|
||||
.or_some(event_id)
|
||||
})
|
||||
.collect()
|
||||
.await;
|
||||
if to_fetch.is_empty() {
|
||||
debug!(elapsed=?start.elapsed(), "All required state events are already known.");
|
||||
state_events = res
|
||||
.pdu_ids
|
||||
.iter()
|
||||
.stream()
|
||||
.broad_filter_map(|event_id| async move {
|
||||
Some((
|
||||
event_id.clone(),
|
||||
self.services
|
||||
.timeline
|
||||
.get_pdu(event_id)
|
||||
.await
|
||||
.expect("Event disappeared between filtering and fetching"),
|
||||
))
|
||||
})
|
||||
.collect()
|
||||
.await;
|
||||
assert_eq!(
|
||||
state_events.len(),
|
||||
res.pdu_ids.len(),
|
||||
"Failed to load all required state events despite allegedly knowing all of them \
|
||||
already",
|
||||
);
|
||||
} else {
|
||||
let total_count = res.pdu_ids.len();
|
||||
let missing_count = to_fetch.len();
|
||||
let missing_threshold = max(50, total_count >> 2);
|
||||
if missing_count >= missing_threshold {
|
||||
// If there's more than 50 events to fetch, or we're missing 25% or more of the
|
||||
// state, we would need to make a lot of atomic requests, so we'll just try
|
||||
// to fetch the full state from the remote instead.
|
||||
// Since this endpoint might fail in huge rooms, we fall back to atomic fetch
|
||||
// anyway.
|
||||
warn!(
|
||||
elapsed=?start.elapsed(),
|
||||
%missing_count,
|
||||
%total_count,
|
||||
%missing_threshold,
|
||||
"Fetching full state from remote server for event"
|
||||
);
|
||||
let state_response = tokio::time::timeout(
|
||||
Duration::from_secs(30),
|
||||
self.fetch_full_state(origin, create_event, room_id, event_id),
|
||||
)
|
||||
.await;
|
||||
info!(
|
||||
elapsed=?start.elapsed(),
|
||||
%missing_count,
|
||||
%total_count,
|
||||
%missing_threshold,
|
||||
"Fetched full state from remote server for event"
|
||||
);
|
||||
let fetched_state = match state_response {
|
||||
| Ok(Ok(state)) => {
|
||||
// Filter to ensure we only use the PDUs we were expecting, preventing
|
||||
// arbitrary state injection.
|
||||
// Atomic fetch does not have this problem as each PDU is evaluated
|
||||
// individually.
|
||||
let expected: &HashSet<OwnedEventId, BuildHasherDefault<DefaultHasher>> =
|
||||
&HashSet::from_iter(res.pdu_ids.clone());
|
||||
state
|
||||
.into_iter()
|
||||
.stream()
|
||||
.broad_filter_map(|(event_id, pdu)| async move {
|
||||
expected.contains(&event_id).then_some((event_id, pdu))
|
||||
})
|
||||
.collect()
|
||||
.await
|
||||
},
|
||||
| Ok(Err(e)) => {
|
||||
warn!(
|
||||
elapsed=?start.elapsed(),
|
||||
error=?e,
|
||||
%origin,
|
||||
"Failed to fetch full state from remote, falling back to atomic fetch"
|
||||
);
|
||||
self.fetch_and_handle_auth_events(
|
||||
origin,
|
||||
res.pdu_ids.clone(),
|
||||
create_event,
|
||||
room_id,
|
||||
)
|
||||
.await
|
||||
},
|
||||
| Err(e) => {
|
||||
warn!(
|
||||
elapsed=?start.elapsed(),
|
||||
error=?e,
|
||||
%origin,
|
||||
"Remote did not return room state in an acceptable timeframe, falling back to atomic fetch"
|
||||
);
|
||||
self.fetch_and_handle_auth_events(
|
||||
origin,
|
||||
res.pdu_ids.clone(),
|
||||
create_event,
|
||||
room_id,
|
||||
)
|
||||
.await
|
||||
},
|
||||
};
|
||||
|
||||
match state.entry(shortstatekey) {
|
||||
| hash_map::Entry::Vacant(v) => {
|
||||
v.insert(pdu.event_id().to_owned());
|
||||
},
|
||||
| hash_map::Entry::Occupied(_) => {
|
||||
return Err!(Database(
|
||||
"State event's type and state_key combination exists multiple times: {}, {}",
|
||||
pdu.kind(),
|
||||
state_key
|
||||
));
|
||||
},
|
||||
assert!(
|
||||
!fetched_state.is_empty(),
|
||||
"fetch_full_state or fetch_and_handle_missing_events returned empty state \
|
||||
map"
|
||||
);
|
||||
state_events.extend(fetched_state);
|
||||
} else {
|
||||
state_events = res
|
||||
.pdu_ids
|
||||
.iter()
|
||||
.stream()
|
||||
.broad_filter_map(|event_id| async move {
|
||||
self.services
|
||||
.timeline
|
||||
.get_pdu(event_id)
|
||||
.await
|
||||
.map(|p| (event_id.to_owned(), p))
|
||||
.ok()
|
||||
})
|
||||
.collect()
|
||||
.await;
|
||||
assert!(
|
||||
!state_events.is_empty(),
|
||||
"Only missing {} events but read-ahead state vec was empty",
|
||||
to_fetch.len()
|
||||
);
|
||||
debug!(
|
||||
elapsed=?start.elapsed(),
|
||||
to_fetch = to_fetch.len(),
|
||||
"Fetching missing events for state from remote"
|
||||
);
|
||||
let fetched_state = self
|
||||
.fetch_and_handle_auth_events(origin, to_fetch, create_event, room_id)
|
||||
.await;
|
||||
state_events.extend(fetched_state);
|
||||
}
|
||||
}
|
||||
if state_events.is_empty() {
|
||||
return Ok(HashMap::new());
|
||||
}
|
||||
|
||||
let mut state: HashMap<ShortStateKey, OwnedEventId> =
|
||||
HashMap::with_capacity(state_events.len());
|
||||
debug!(elapsed=?start.elapsed(), events = state_events.len(), "Processing state events");
|
||||
for (event_id, pdu) in state_events {
|
||||
let state_key = pdu.state_key().ok_or_else(|| {
|
||||
err!(Request(BadJson("Found non-state pdu in state events: {event_id}")))
|
||||
})?;
|
||||
|
||||
let shortstatekey = self
|
||||
.services
|
||||
.short
|
||||
.get_or_create_shortstatekey(&pdu.kind().to_string().into(), state_key)
|
||||
.await;
|
||||
|
||||
match state.entry(shortstatekey) {
|
||||
| hash_map::Entry::Vacant(v) => {
|
||||
v.insert(pdu.event_id().to_owned());
|
||||
},
|
||||
| hash_map::Entry::Occupied(existing) => {
|
||||
return Err!(Request(Forbidden(
|
||||
"State event's type and state_key combination exists multiple times \
|
||||
({event_id} + {}): ({}, \"{}\")",
|
||||
existing.get(),
|
||||
pdu.kind(),
|
||||
state_key,
|
||||
)));
|
||||
},
|
||||
}
|
||||
}
|
||||
trace!(elapsed=?start.elapsed(), "fetch_state finished");
|
||||
Ok(state)
|
||||
}
|
||||
|
||||
// The original create event must still be in the state
|
||||
let create_shortstatekey = self
|
||||
.services
|
||||
.short
|
||||
.get_shortstatekey(&StateEventType::RoomCreate, "")
|
||||
.await?;
|
||||
|
||||
if state.get(&create_shortstatekey).map(AsRef::as_ref) != Some(create_event.event_id()) {
|
||||
return Err!(Database("Incoming event refers to wrong create event."));
|
||||
async fn fetch_state_ids_from_backfill_servers(
|
||||
&self,
|
||||
event_id: OwnedEventId,
|
||||
room_id: OwnedRoomId,
|
||||
) -> Result<get_room_state_ids::v1::Response> {
|
||||
let candidates = self
|
||||
.services
|
||||
.timeline
|
||||
.candidate_backfill_servers(&room_id)
|
||||
.await;
|
||||
if candidates.is_empty() {
|
||||
return Err!(Request(NotFound(
|
||||
"Cannot ask any other servers for the state at this event"
|
||||
)));
|
||||
}
|
||||
debug!(%room_id, ?candidates, "Asking backfill servers for state_ids");
|
||||
let futures = candidates.iter().map(|server_name| {
|
||||
Box::pin(
|
||||
self.services
|
||||
.sending
|
||||
.send_federation_request(
|
||||
server_name,
|
||||
get_room_state_ids::v1::Request::new(event_id.clone(), room_id.clone()),
|
||||
)
|
||||
.inspect_err(|e| {
|
||||
debug_warn!("Fallback fetching state for event failed: {e}");
|
||||
}),
|
||||
)
|
||||
});
|
||||
Ok(select_ok(futures).await?.0)
|
||||
}
|
||||
|
||||
Ok(Some(state))
|
||||
/// Fetches the full state via `GET /_matrix/federation/v1/state` from a
|
||||
/// remote server, and persists all the incoming auth chain events and
|
||||
/// state events as outliers, for use later.
|
||||
///
|
||||
/// Any events that cannot be persisted are dropped with a warning.
|
||||
pub(super) async fn fetch_full_state(
|
||||
&self,
|
||||
origin: &ServerName,
|
||||
create_event: &PduEvent,
|
||||
room_id: &RoomId,
|
||||
event_id: &EventId,
|
||||
) -> Result<HashMap<OwnedEventId, PduEvent>> {
|
||||
let start = Instant::now();
|
||||
trace!("Fetching full state from remote server");
|
||||
let res: get_room_state::v1::Response = self
|
||||
.services
|
||||
.sending
|
||||
.send_federation_request(
|
||||
origin,
|
||||
get_room_state::v1::Request::new(event_id.to_owned(), room_id.to_owned()),
|
||||
)
|
||||
.await
|
||||
.inspect_err(|e| debug_warn!("Fetching state for event failed: {e}"))?;
|
||||
debug!(elapsed=?start.elapsed(), count = res.auth_chain.len(), "Handling incoming auth chain...");
|
||||
res.auth_chain
|
||||
.iter()
|
||||
.stream()
|
||||
.broad_filter_map(|raw_event_json| async {
|
||||
if let Some(parsed) = self.parse_incoming_pdu(raw_event_json).await.ok()
|
||||
&& parsed.0 == room_id
|
||||
{
|
||||
Some(parsed)
|
||||
} else {
|
||||
None
|
||||
}
|
||||
})
|
||||
.for_each_concurrent(
|
||||
None,
|
||||
|(incoming_room_id, incoming_event_id, incoming_event_json)| async move {
|
||||
self.handle_outlier_pdu(
|
||||
origin,
|
||||
create_event,
|
||||
&incoming_event_id,
|
||||
&incoming_room_id,
|
||||
incoming_event_json,
|
||||
)
|
||||
.await
|
||||
.inspect_err(|e| {
|
||||
warn!(
|
||||
%incoming_room_id,
|
||||
%incoming_event_id,
|
||||
?e,
|
||||
"Failed to handle auth chain event from state fetch"
|
||||
);
|
||||
})
|
||||
.ok();
|
||||
},
|
||||
)
|
||||
.await;
|
||||
debug!(elapsed=?start.elapsed(), count = res.pdus.len(), "Handling incoming state PDUs...");
|
||||
let r = res
|
||||
.pdus
|
||||
.iter()
|
||||
.stream()
|
||||
.broad_filter_map(|raw_event_json| async {
|
||||
if let Some(parsed) = self.parse_incoming_pdu(raw_event_json).await.ok()
|
||||
&& parsed.0 == room_id
|
||||
{
|
||||
Some(parsed)
|
||||
} else {
|
||||
None
|
||||
}
|
||||
})
|
||||
.broad_filter_map(
|
||||
|(incoming_room_id, incoming_event_id, incoming_event_json)| async move {
|
||||
self.handle_outlier_pdu(
|
||||
origin,
|
||||
create_event,
|
||||
&incoming_event_id,
|
||||
&incoming_room_id,
|
||||
incoming_event_json,
|
||||
)
|
||||
.await
|
||||
.inspect_err(|e| {
|
||||
warn!(
|
||||
elapsed=?start.elapsed(),
|
||||
%incoming_room_id,
|
||||
%incoming_event_id,
|
||||
?e,
|
||||
"Failed to handle state event from state fetch"
|
||||
);
|
||||
})
|
||||
.ok()
|
||||
},
|
||||
)
|
||||
.fold(HashMap::new(), |mut acc, (event, _)| async move {
|
||||
acc.insert(event.event_id().to_owned(), event);
|
||||
acc
|
||||
})
|
||||
.await;
|
||||
trace!(elapsed=?start.elapsed(), "fetch_full_state finished");
|
||||
Ok(r)
|
||||
}
|
||||
}
|
||||
Loaded 100 of 151 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user