From 8e71ed7b630358441c5239e987ff97f120abf4c9 Mon Sep 17 00:00:00 2001 From: Henry-Hiles Date: Sat, 20 Jun 2026 13:19:35 -0400 Subject: [PATCH 01/70] fix: dynamically link rocksdb on dynamic nix builds Fixes liburing error. --- .forgejo/workflows/update-flake-hashes.yml | 3 ++ nix/default.nix | 1 + nix/packages/continuwuity.nix | 22 +++++++++++-- nix/packages/default.nix | 3 ++ nix/packages/rocksdb.nix | 36 ++++++++++++++++++++++ nix/rocksdb-updater.nix | 14 +++++++++ 6 files changed, 77 insertions(+), 2 deletions(-) create mode 100644 nix/packages/rocksdb.nix create mode 100644 nix/rocksdb-updater.nix diff --git a/.forgejo/workflows/update-flake-hashes.yml b/.forgejo/workflows/update-flake-hashes.yml index 57b0bd601..baf9b4b0a 100644 --- a/.forgejo/workflows/update-flake-hashes.yml +++ b/.forgejo/workflows/update-flake-hashes.yml @@ -45,6 +45,9 @@ jobs: rm new_toolchain_hash.txt + - name: Update rocksdb + run: nix run .#update-rocksdb + - name: Show diff run: git diff flake.nix nix diff --git a/nix/default.nix b/nix/default.nix index 742e34d5a..a49df0826 100644 --- a/nix/default.nix +++ b/nix/default.nix @@ -4,5 +4,6 @@ ./packages ./devshell.nix ./fmt.nix + ./rocksdb-updater.nix ]; } diff --git a/nix/packages/continuwuity.nix b/nix/packages/continuwuity.nix index c10312947..51a34b849 100644 --- a/nix/packages/continuwuity.nix +++ b/nix/packages/continuwuity.nix @@ -2,9 +2,10 @@ lib, self, stdenv, - liburing, + rocksdb, craneLib, pkg-config, + liburing, rustPlatform, cargoExtraArgs ? "", rustflags ? "", @@ -36,12 +37,17 @@ let pkg-config rustPlatform.bindgenHook ]; + buildInputs = lib.optionals stdenv.hostPlatform.isLinux [ liburing ]; - doCheck = false; + env = { CARGO_PROFILE = profile; RUSTFLAGS = rustflags; } + // (lib.optionalAttrs (rocksdb != null) { + ROCKSDB_INCLUDE_DIR = "${rocksdb}/include"; + ROCKSDB_LIB_DIR = "${rocksdb}/lib"; + }) // (lib.optionalAttrs (target_cpu != null) { TARGET_CPU = target_cpu; }); @@ -52,6 +58,18 @@ craneLib.buildPackage ( inherit cargoExtraArgs; cargoArtifacts = craneLib.buildDepsOnly attrs; + # Needed to make continuwuity link to rocksdb + postFixup = lib.optionalString (stdenv.hostPlatform.isLinux && rocksdb != null) '' + old_rpath="$(patchelf --print-rpath $out/bin/conduwuit)" + extra_rpath="${ + lib.makeLibraryPath [ + rocksdb + ] + }" + + patchelf --set-rpath "$old_rpath:$extra_rpath" $out/bin/conduwuit + ''; + meta = { description = "A community-driven Matrix homeserver in Rust"; mainProgram = "conduwuit"; diff --git a/nix/packages/default.nix b/nix/packages/default.nix index d73b4b022..37915b079 100644 --- a/nix/packages/default.nix +++ b/nix/packages/default.nix @@ -36,7 +36,10 @@ default = pkgs.callPackage ./continuwuity.nix { inherit self craneLib; + liburing = (if isStatic then pkgs.pkgsStatic else pkgs).liburing; + rocksdb = if isStatic then null else pkgs.callPackage ./rocksdb.nix { }; + # extra features via `cargoExtraArgs` cargoExtraArgs = "-F http3"; # extra RUSTFLAGS via `rustflags` diff --git a/nix/packages/rocksdb.nix b/nix/packages/rocksdb.nix new file mode 100644 index 000000000..b4427dbc4 --- /dev/null +++ b/nix/packages/rocksdb.nix @@ -0,0 +1,36 @@ +{ + # stdenv, + # enableJemalloc ? stdenv.hostPlatform.isLinux, + enableJemalloc ? false, + rocksdb, + fetchFromGitea, + rust-jemalloc-sys-unprefixed, + ... +}: +(rocksdb.override { + # rocksdb fails to build with prefixed jemalloc, which is required on + # darwin due to [1]. In this case, fall back to building rocksdb with + # libc malloc. This should not cause conflicts, because all of the + # jemalloc symbols are prefixed. + # + # [1]: https://github.com/tikv/jemallocator/blob/ab0676d77e81268cd09b059260c75b38dbef2d51/jemalloc-sys/src/env.rs#L17 + jemalloc = rust-jemalloc-sys-unprefixed; + inherit enableJemalloc; +}).overrideAttrs + ({ + version = "continuwuity-v0.5.0-unstable-2026-05-19"; + src = fetchFromGitea { + domain = "forgejo.ellis.link"; + owner = "continuwuation"; + repo = "rocksdb"; + rev = "3756b2b905e13216d8b56bcc783d814e7b073aff"; + hash = "sha256-rSv4fr2bf9JJwdodgeuPCuceeh7k97KVxrAOC0wyPQY="; + }; + + # We have this already at https://forgejo.ellis.link/continuwuation/rocksdb/commit/a935c0273e1ba44eacf88ce3685a9b9831486155 + # Unsetting `patches` so we don't have to revert it and make this nix exclusive + patches = [ ]; + + # Unset postPatch, as our version override breaks version-specific sed calls in the original package + postPatch = ""; + }) diff --git a/nix/rocksdb-updater.nix b/nix/rocksdb-updater.nix new file mode 100644 index 000000000..38f900651 --- /dev/null +++ b/nix/rocksdb-updater.nix @@ -0,0 +1,14 @@ +{ + perSystem = + { pkgs, ... }: + { + apps.update-rocksdb = { + type = "app"; + program = pkgs.writeShellApplication { + name = "update-rocksdb"; + runtimeInputs = [ pkgs.nix-update ]; + text = "nix-update rocksdb -F --version branch"; + }; + }; + }; +} From 182c5a120efaa00c9b578ea2c07dc4c333a7c691 Mon Sep 17 00:00:00 2001 From: Henry-Hiles Date: Sat, 20 Jun 2026 13:45:25 -0400 Subject: [PATCH 02/70] fix: add rocksdb as a flake package This fixes the flake-hashes workflow. --- nix/packages/default.nix | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/nix/packages/default.nix b/nix/packages/default.nix index 37915b079..13e124dbd 100644 --- a/nix/packages/default.nix +++ b/nix/packages/default.nix @@ -38,7 +38,7 @@ inherit self craneLib; liburing = (if isStatic then pkgs.pkgsStatic else pkgs).liburing; - rocksdb = if isStatic then null else pkgs.callPackage ./rocksdb.nix { }; + rocksdb = if isStatic then null else self'.packages.rocksdb; # extra features via `cargoExtraArgs` cargoExtraArgs = "-F http3"; @@ -65,7 +65,10 @@ }; in - (mkPackages pkgs) + { + rocksdb = pkgs.callPackage ./rocksdb.nix { }; + } + // (mkPackages pkgs) // (lib.mapAttrs' (name: value: lib.nameValuePair "${name}-static-x86_64" value) ( mkPackages ( import inputs.nixpkgs { From 40935cf96a7edd1ee15f81241cbcd7a202a2667d Mon Sep 17 00:00:00 2001 From: Henry-Hiles Date: Sat, 20 Jun 2026 15:32:33 -0400 Subject: [PATCH 03/70] fix: use toolchain declared in rust-toolchain.toml for cross rust-std --- nix/packages/default.nix | 7 ++++--- nix/rust.nix | 22 +++++++++++++--------- 2 files changed, 17 insertions(+), 12 deletions(-) diff --git a/nix/packages/default.nix b/nix/packages/default.nix index 13e124dbd..afc032557 100644 --- a/nix/packages/default.nix +++ b/nix/packages/default.nix @@ -9,8 +9,10 @@ self', lib, pkgs, + inputs', system, craneLib, + mkToolchain, ... }: { @@ -19,7 +21,7 @@ mkPackages = pkgs: let - fnx = inputs.fenix.packages.${system}; + fnx = inputs'.fenix.packages; isStatic = pkgs.stdenv.hostPlatform.isMusl; @@ -28,7 +30,7 @@ if isStatic then fnx.combine [ self'.packages.stable-toolchain - (fnx.targets.${pkgs.stdenv.hostPlatform.config}.stable).rust-std + (mkToolchain fnx.targets.${pkgs.stdenv.hostPlatform.config}).rust-std ] else self'.packages.stable-toolchain @@ -62,7 +64,6 @@ in { inherit default max-perf max-perf-haswell; - }; in { diff --git a/nix/rust.nix b/nix/rust.nix index f56823839..5dd5a614a 100644 --- a/nix/rust.nix +++ b/nix/rust.nix @@ -2,22 +2,26 @@ { perSystem = { - system, lib, + inputs', pkgs, ... }: + let + mkToolchain = + target: + target.fromToolchainName { + name = (lib.importTOML "${inputs.self}/rust-toolchain.toml").toolchain.channel; + sha256 = "sha256-mvUGEOHYJpn3ikC5hckneuGixaC+yGrkMM/liDIDgoU="; + }; + in { + _module.args = { inherit mkToolchain; }; + packages = let - fnx = inputs.fenix.packages.${system}; - - stable-toolchain = fnx.fromToolchainFile { - dir = inputs.self; - - # See also `rust-toolchain.toml` - sha256 = "sha256-mvUGEOHYJpn3ikC5hckneuGixaC+yGrkMM/liDIDgoU="; - }; + fnx = inputs'.fenix.packages; + stable-toolchain = (mkToolchain fnx).toolchain; in { inherit stable-toolchain; From 4078062331665a7842fb231eaaeb596ab6193a42 Mon Sep 17 00:00:00 2001 From: Henry-Hiles Date: Sat, 20 Jun 2026 15:35:09 -0400 Subject: [PATCH 04/70] fix: change the update-flake-hashes repo to work with the new rust.nix structure --- .forgejo/workflows/update-flake-hashes.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.forgejo/workflows/update-flake-hashes.yml b/.forgejo/workflows/update-flake-hashes.yml index baf9b4b0a..b318e1029 100644 --- a/.forgejo/workflows/update-flake-hashes.yml +++ b/.forgejo/workflows/update-flake-hashes.yml @@ -27,7 +27,7 @@ jobs: - name: Get new toolchain hash run: | # Set the current sha256 to an empty hash to make `nix build` calculate a new one - awk '/fromToolchainFile *\{/{found=1; print; next} found && /sha256 =/{sub(/sha256 = .*/, "sha256 = lib.fakeSha256;"); found=0} 1' nix/rust.nix > temp.nix + awk '/fromToolchainName *\{/{found=1; print; next} found && /sha256 =/{sub(/sha256 = .*/, "sha256 = lib.fakeSha256;"); found=0} 1' nix/rust.nix > temp.nix mv temp.nix nix/rust.nix # Build continuwuity and filter for the new hash @@ -39,7 +39,7 @@ jobs: sed -i "s|lib.fakeSha256|\"$new_hash\"|" nix/rust.nix echo "New hash:" - awk -F'"' '/fromToolchainFile/{found=1; next} found && /sha256 =/{print $2; found=0}' nix/rust.nix + awk -F'"' '/fromToolchainName/{found=1; next} found && /sha256 =/{print $2; found=0}' nix/rust.nix echo "Expected new hash:" cat new_toolchain_hash.txt From 86fe98c90d081a1df6aaa5877db6af30477a9eaf Mon Sep 17 00:00:00 2001 From: Jade Date: Sat, 20 Jun 2026 23:32:56 +0000 Subject: [PATCH 05/70] chore: Admin announcement --- docs/public/.well-known/continuwuity/announcements | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/public/.well-known/continuwuity/announcements b/docs/public/.well-known/continuwuity/announcements index bae764c78..bf783f6c9 100644 --- a/docs/public/.well-known/continuwuity/announcements +++ b/docs/public/.well-known/continuwuity/announcements @@ -6,10 +6,10 @@ "message": "Welcome to Continuwuity! Important announcements about the project will appear here." }, { - "id": 13, + "id": 14, "mention_room": true, "date": "2026-05-08", - "message": "[v0.5.9](https://forgejo.ellis.link/continuwuation/continuwuity/releases/tag/v0.5.9) has been released, fixing a few low-severity federation-related vulnerabilities. It is recommended you read the changelog and update as soon as possible. There are no new features or other changes in this release, only related bugfixes. Deployments tracking the main branch should also update to the latest commit." + "message": "[v0.5.10](https://forgejo.ellis.link/continuwuation/continuwuity/releases/tag/v0.5.10) has been released. It is a security release, so we suggest you update as soon as possible. Don't forget to also join [our announcements room](https://matrix.to/#/!jIdNjSM5X-V5JVx2h2kAhUZIIQ08GyzPL55NFZAH1vM/%24K1ISNKIqfNiZzsNVCaTt2E7ZtNeP6Dsy6sbz9l3rO0A?via=ellis.link&via=gingershaped.computer&via=matrix.org)." } ] } From ed382123915e762d3e64f43580dc7570d47831cb Mon Sep 17 00:00:00 2001 From: Jade Ellis Date: Sun, 21 Jun 2026 01:03:14 +0100 Subject: [PATCH 06/70] fix(ci): Correct version regex in debian build --- .forgejo/workflows/build-debian.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.forgejo/workflows/build-debian.yml b/.forgejo/workflows/build-debian.yml index d41099bce..5942da89d 100644 --- a/.forgejo/workflows/build-debian.yml +++ b/.forgejo/workflows/build-debian.yml @@ -82,10 +82,10 @@ jobs: # VERSION is the package version, COMPONENT is used in # apt's repository config like a git repo branch VERSION=$BASE_VERSION - if [[ ${{ forge.ref_name }} =~ ^v+[0-9]\.+[0-9]\.+[0-9]$ ]]; then + if [[ ${{ forge.ref_name }} =~ ^v+[0-9]+\.+[0-9]+\.+[0-9]+$ ]]; then # Use the "stable" component for tagged semver releases COMPONENT="stable" - elif [[ ${{ forge.ref_name }} =~ ^v+[0-9]\.+[0-9]\.+[0-9] ]]; then + elif [[ ${{ forge.ref_name }} =~ ^v+[0-9]+\.+[0-9]+\.+[0-9]+ ]]; then # Use the "unstable" component for tagged semver pre-releases COMPONENT="unstable" else From 52c1544e6f512f1d0d9e2eb842789bdc1ec513d5 Mon Sep 17 00:00:00 2001 From: stratself Date: Sun, 21 Jun 2026 02:36:32 +0000 Subject: [PATCH 07/70] fix: Correct date in announcements --- docs/public/.well-known/continuwuity/announcements | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/public/.well-known/continuwuity/announcements b/docs/public/.well-known/continuwuity/announcements index bf783f6c9..5d809d8f3 100644 --- a/docs/public/.well-known/continuwuity/announcements +++ b/docs/public/.well-known/continuwuity/announcements @@ -8,7 +8,7 @@ { "id": 14, "mention_room": true, - "date": "2026-05-08", + "date": "2026-06-20", "message": "[v0.5.10](https://forgejo.ellis.link/continuwuation/continuwuity/releases/tag/v0.5.10) has been released. It is a security release, so we suggest you update as soon as possible. Don't forget to also join [our announcements room](https://matrix.to/#/!jIdNjSM5X-V5JVx2h2kAhUZIIQ08GyzPL55NFZAH1vM/%24K1ISNKIqfNiZzsNVCaTt2E7ZtNeP6Dsy6sbz9l3rO0A?via=ellis.link&via=gingershaped.computer&via=matrix.org)." } ] From a01035e63a37b86221ba76bd536c103a8407c1d6 Mon Sep 17 00:00:00 2001 From: Renovate Bot Date: Mon, 22 Jun 2026 05:03:19 +0000 Subject: [PATCH 08/70] chore(deps): update github-actions-non-major to v43.234.0 --- .forgejo/workflows/renovate.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.forgejo/workflows/renovate.yml b/.forgejo/workflows/renovate.yml index 45ae30bb7..bef741da8 100644 --- a/.forgejo/workflows/renovate.yml +++ b/.forgejo/workflows/renovate.yml @@ -43,7 +43,7 @@ jobs: name: Renovate runs-on: ubuntu-latest container: - image: ghcr.io/renovatebot/renovate:43.222.1@sha256:b9af3f59f3f4d92b2c41e9f4ca3ffe92400503f20158d0bd67d07a3fdbe781d2 + image: ghcr.io/renovatebot/renovate:43.234.0@sha256:bff111bfe347c559c615b658b28721eba5b7bb32a7b7901ea321336767209fe1 options: --tmpfs /tmp:exec steps: - name: Checkout From eeef60d5405ff131fe93280ec30ecaa2ab9857c1 Mon Sep 17 00:00:00 2001 From: Renovate Bot Date: Tue, 23 Jun 2026 05:01:57 +0000 Subject: [PATCH 09/70] chore(deps): update github-actions-digest --- .forgejo/actions/timelord/action.yml | 2 +- .forgejo/workflows/build-nix.yml | 2 +- .forgejo/workflows/mirror-images.yml | 2 +- .forgejo/workflows/release-image.yml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.forgejo/actions/timelord/action.yml b/.forgejo/actions/timelord/action.yml index 2d62b94b6..cc9a0790a 100644 --- a/.forgejo/actions/timelord/action.yml +++ b/.forgejo/actions/timelord/action.yml @@ -71,7 +71,7 @@ runs: - name: Install timelord-cli and git-warp-time if: steps.check-binaries.outputs.need-install == 'true' - uses: https://github.com/taiki-e/install-action@15449e3094499af05d8d964a1c884208e4b8b595 # v2 + uses: https://github.com/taiki-e/install-action@9e1e5806d4a4822de933115878265be9aaa786d9 # v2 with: tool: git-warp-time,timelord-cli@3.0.1 diff --git a/.forgejo/workflows/build-nix.yml b/.forgejo/workflows/build-nix.yml index dcd99c1ba..36b28e288 100644 --- a/.forgejo/workflows/build-nix.yml +++ b/.forgejo/workflows/build-nix.yml @@ -65,7 +65,7 @@ jobs: path: binaries merge-multiple: true - name: Create Release and Upload - uses: https://github.com/softprops/action-gh-release@b4309332981a82ec1c5618f44dd2e27cc8bfbfda # v3 + uses: https://github.com/softprops/action-gh-release@718ea10b132b3b2eba29c1007bb80653f286566b # v3 with: draft: true files: binaries/* diff --git a/.forgejo/workflows/mirror-images.yml b/.forgejo/workflows/mirror-images.yml index b75e23c7e..95cedc815 100644 --- a/.forgejo/workflows/mirror-images.yml +++ b/.forgejo/workflows/mirror-images.yml @@ -55,7 +55,7 @@ jobs: # repositories: continuwuity - name: Install regsync - uses: https://github.com/regclient/actions/regsync-installer@14f9d37db17b5dc41fefd1ffdd1af4b9e2490560 # main + uses: https://github.com/regclient/actions/regsync-installer@4b4db1dcc7dad75ad67a788a380f75a20cc8a040 # main - name: Check what images need mirroring run: | diff --git a/.forgejo/workflows/release-image.yml b/.forgejo/workflows/release-image.yml index 8f6cc7ad9..d9e9b6e84 100644 --- a/.forgejo/workflows/release-image.yml +++ b/.forgejo/workflows/release-image.yml @@ -216,7 +216,7 @@ jobs: path: binaries merge-multiple: true - name: Create Release and Upload - uses: https://github.com/softprops/action-gh-release@b4309332981a82ec1c5618f44dd2e27cc8bfbfda # v3 + uses: https://github.com/softprops/action-gh-release@718ea10b132b3b2eba29c1007bb80653f286566b # v3 with: draft: true files: binaries/* From 6b259d15ee6afda59ee9c8ad03ca313ef84f582a Mon Sep 17 00:00:00 2001 From: Renovate Bot Date: Mon, 22 Jun 2026 05:05:26 +0000 Subject: [PATCH 10/70] chore(deps): update rust-non-major --- Cargo.lock | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 6f5a59782..0e93a919c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -560,9 +560,9 @@ checksum = "8f1fe948ff07f4bd06c30984e69f5b4899c516a3ef74f34df92a2df2ab535495" [[package]] name = "bytes" -version = "1.11.1" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33" +checksum = "8ae3f5d315924270530207e2a68396c3cc547f6dca3fbdca317cfb1a51edb593" [[package]] name = "bytesize" @@ -3945,9 +3945,9 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.45" +version = "1.0.46" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924" +checksum = "dfbc457d0c7a0759a614551b11a6409e5951f6c7537be1f1b7682b9ae9230368" dependencies = [ "proc-macro2", ] @@ -5144,9 +5144,9 @@ checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" [[package]] name = "syn" -version = "2.0.117" +version = "2.0.118" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99" +checksum = "1b9ae57f904213ebb649ce6895b8a66c66f0203b9319718f69a5612a065b1422" dependencies = [ "proc-macro2", "quote", From d81f4df61ccc228e61061acf0dac0bae42ebeffb Mon Sep 17 00:00:00 2001 From: Renovate Bot Date: Wed, 17 Jun 2026 05:04:23 +0000 Subject: [PATCH 11/70] chore(deps): update rust crate itertools to 0.15.0 --- Cargo.lock | 17 +++++++++++++---- Cargo.toml | 2 +- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 0e93a919c..081dc19e7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -889,7 +889,7 @@ dependencies = [ "http-body-util", "hyper", "ipaddress", - "itertools 0.14.0", + "itertools 0.15.0", "lettre", "log", "rand 0.10.1", @@ -942,7 +942,7 @@ dependencies = [ "http-body-util", "hyper-util", "ipaddress", - "itertools 0.14.0", + "itertools 0.15.0", "lettre", "libc", "libloading 0.9.0", @@ -1004,7 +1004,7 @@ name = "conduwuit_macros" version = "26.6.0-alpha.1" dependencies = [ "cargo_toml", - "itertools 0.14.0", + "itertools 0.15.0", "proc-macro2", "quote", "syn", @@ -1070,7 +1070,7 @@ dependencies = [ "http", "image", "ipaddress", - "itertools 0.14.0", + "itertools 0.15.0", "lettre", "log", "loole", @@ -2641,6 +2641,15 @@ dependencies = [ "either", ] +[[package]] +name = "itertools" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b4baf93f58d4425749ca49a51c50ebab072c5df6994d08fed93541c331481dc" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "1.0.18" diff --git a/Cargo.toml b/Cargo.toml index e0e163bcf..519de9d2d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -316,7 +316,7 @@ default-features = false # Used to make working with iterators easier, was already a transitive depdendency [workspace.dependencies.itertools] -version = "0.14.0" +version = "0.15.0" # to parse user-friendly time durations in admin commands #TODO: overlaps chrono? From 2adb9a7941b577fcfd45c993ef9f9bf9f2406532 Mon Sep 17 00:00:00 2001 From: Renovate Bot Date: Mon, 22 Jun 2026 05:10:58 +0000 Subject: [PATCH 12/70] chore(deps): update actions/checkout action to v7 --- .forgejo/workflows/build-debian.yml | 2 +- .forgejo/workflows/build-fedora.yml | 2 +- .forgejo/workflows/check-changelog.yml | 2 +- .forgejo/workflows/documentation.yml | 2 +- .forgejo/workflows/mirror-images.yml | 2 +- .forgejo/workflows/prek-checks.yml | 6 +++--- .forgejo/workflows/release-image.yml | 8 ++++---- .forgejo/workflows/renovate.yml | 2 +- .forgejo/workflows/update-flake-hashes.yml | 2 +- 9 files changed, 14 insertions(+), 14 deletions(-) diff --git a/.forgejo/workflows/build-debian.yml b/.forgejo/workflows/build-debian.yml index 5942da89d..cd1128124 100644 --- a/.forgejo/workflows/build-debian.yml +++ b/.forgejo/workflows/build-debian.yml @@ -43,7 +43,7 @@ jobs: # fi - name: Checkout repository with full history - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 with: fetch-depth: 0 ref: ${{ github.ref_name }} diff --git a/.forgejo/workflows/build-fedora.yml b/.forgejo/workflows/build-fedora.yml index bd2bbc035..79b953c2d 100644 --- a/.forgejo/workflows/build-fedora.yml +++ b/.forgejo/workflows/build-fedora.yml @@ -30,7 +30,7 @@ jobs: echo "Fedora version: $VERSION" - name: Checkout repository with full history - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 with: fetch-depth: 0 ref: ${{ github.ref_name }} diff --git a/.forgejo/workflows/check-changelog.yml b/.forgejo/workflows/check-changelog.yml index 0d31f980a..fb21354aa 100644 --- a/.forgejo/workflows/check-changelog.yml +++ b/.forgejo/workflows/check-changelog.yml @@ -15,7 +15,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout repository - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 with: ref: ${{ github.event.pull_request.head.sha }} fetch-depth: 0 diff --git a/.forgejo/workflows/documentation.yml b/.forgejo/workflows/documentation.yml index 72a1e7c4d..abc525e64 100644 --- a/.forgejo/workflows/documentation.yml +++ b/.forgejo/workflows/documentation.yml @@ -21,7 +21,7 @@ jobs: steps: - name: Sync repository - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 with: persist-credentials: false fetch-depth: 0 diff --git a/.forgejo/workflows/mirror-images.yml b/.forgejo/workflows/mirror-images.yml index 95cedc815..6e0845c51 100644 --- a/.forgejo/workflows/mirror-images.yml +++ b/.forgejo/workflows/mirror-images.yml @@ -41,7 +41,7 @@ jobs: DOCKER_MIRROR_TOKEN: ${{ secrets.DOCKER_MIRROR_TOKEN }} steps: - name: Checkout repository - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 with: persist-credentials: false diff --git a/.forgejo/workflows/prek-checks.yml b/.forgejo/workflows/prek-checks.yml index a59f352a5..5fe740ed9 100644 --- a/.forgejo/workflows/prek-checks.yml +++ b/.forgejo/workflows/prek-checks.yml @@ -17,7 +17,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout repository - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 with: persist-credentials: false @@ -48,7 +48,7 @@ jobs: rust: ${{ steps.filter.outputs.rust }} steps: - name: Checkout repository - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 with: persist-credentials: false @@ -70,7 +70,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 with: persist-credentials: false diff --git a/.forgejo/workflows/release-image.yml b/.forgejo/workflows/release-image.yml index d9e9b6e84..ee387bd92 100644 --- a/.forgejo/workflows/release-image.yml +++ b/.forgejo/workflows/release-image.yml @@ -46,7 +46,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 with: persist-credentials: false - name: Prepare Docker build environment @@ -100,7 +100,7 @@ jobs: needs: build-release steps: - name: Checkout repository - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 with: persist-credentials: false - name: Create multi-platform manifest @@ -133,7 +133,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 with: persist-credentials: false - name: Prepare max-perf Docker build environment @@ -187,7 +187,7 @@ jobs: needs: build-maxperf steps: - name: Checkout repository - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 with: persist-credentials: false - name: Create max-perf manifest diff --git a/.forgejo/workflows/renovate.yml b/.forgejo/workflows/renovate.yml index bef741da8..a00624759 100644 --- a/.forgejo/workflows/renovate.yml +++ b/.forgejo/workflows/renovate.yml @@ -47,7 +47,7 @@ jobs: options: --tmpfs /tmp:exec steps: - name: Checkout - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 with: show-progress: false diff --git a/.forgejo/workflows/update-flake-hashes.yml b/.forgejo/workflows/update-flake-hashes.yml index b318e1029..87af65ce3 100644 --- a/.forgejo/workflows/update-flake-hashes.yml +++ b/.forgejo/workflows/update-flake-hashes.yml @@ -14,7 +14,7 @@ jobs: update-flake-hashes: runs-on: ubuntu-latest steps: - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 with: persist-credentials: true token: ${{ secrets.FORGEJO_TOKEN }} From de24fcbb8c1bfd6411cacf4c138f5a0c249be789 Mon Sep 17 00:00:00 2001 From: Renovate Bot Date: Sun, 21 Jun 2026 05:05:02 +0000 Subject: [PATCH 13/70] chore(deps): update dependency cargo-bins/cargo-binstall to v1.20.1 --- docker/Dockerfile | 2 +- docker/musl.Dockerfile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 74c318762..5ac22b161 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -50,7 +50,7 @@ EOF # Developer tool versions # renovate: datasource=github-releases depName=cargo-bins/cargo-binstall -ENV BINSTALL_VERSION=1.20.0 +ENV BINSTALL_VERSION=1.20.1 # renovate: datasource=github-releases depName=psastras/sbom-rs ENV CARGO_SBOM_VERSION=0.9.1 # renovate: datasource=crate depName=lddtree diff --git a/docker/musl.Dockerfile b/docker/musl.Dockerfile index ed6e72643..8bae5fdb3 100644 --- a/docker/musl.Dockerfile +++ b/docker/musl.Dockerfile @@ -18,7 +18,7 @@ RUN --mount=type=cache,target=/etc/apk/cache apk add \ # Developer tool versions # renovate: datasource=github-releases depName=cargo-bins/cargo-binstall -ENV BINSTALL_VERSION=1.20.0 +ENV BINSTALL_VERSION=1.20.1 # renovate: datasource=github-releases depName=psastras/sbom-rs ENV CARGO_SBOM_VERSION=0.9.1 # renovate: datasource=crate depName=lddtree From 3b455218f5937773dac6c262400cb6a7f3b6d04d Mon Sep 17 00:00:00 2001 From: Renovate Bot Date: Wed, 17 Jun 2026 05:04:23 +0000 Subject: [PATCH 14/70] chore(deps): update rust crate itertools to 0.15.0 From 4d40e20fddb33f2d0d0d16ae6d9a214cd1b221c1 Mon Sep 17 00:00:00 2001 From: ginger Date: Tue, 23 Jun 2026 14:56:14 +0000 Subject: [PATCH 15/70] docs: Fix typo --- docs/troubleshooting.mdx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/troubleshooting.mdx b/docs/troubleshooting.mdx index adc89ad31..b98edc583 100644 --- a/docs/troubleshooting.mdx +++ b/docs/troubleshooting.mdx @@ -10,7 +10,7 @@ Please check that your issues are not due to problems with your Docker setup. ### Slow joins to rooms -Some slowness is to be expected if you're the first person on your homserver to join a room (which will +Some slowness is to be expected if you're the first person on your homeserver to join a room (which will always be the case for single-user homeservers). In this situation, your homeserver has to verify the signatures of all of the state events sent by other servers before your join. To make this process as fast as possible, make sure you have multiple fast, trusted servers listed in `trusted_servers` in your configuration, and ensure From 6dac46e1ad4d47a95c406ba51b854b81d25abf70 Mon Sep 17 00:00:00 2001 From: ginger Date: Wed, 24 Jun 2026 13:00:27 +0000 Subject: [PATCH 16/70] Update SECURITY.md --- SECURITY.md | 1 + 1 file changed, 1 insertion(+) diff --git a/SECURITY.md b/SECURITY.md index ee912970d..10c3c014d 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -23,6 +23,7 @@ We appreciate the efforts of security researchers and the community in identifyi 1. **Contact members of the team directly** over E2EE private message. - [@jade:ellis.link](https://matrix.to/#/@jade:ellis.link) - [@nex:nexy7574.co.uk](https://matrix.to/#/@nex:nexy7574.co.uk) + - [@ginger:gingershaped.computer](https://matrix.to/#/@ginger:gingershaped.computer) 2. **Email the security team** at [security@continuwuity.org](mailto:security@continuwuity.org). This is not E2EE, so don't include sensitive details. 3. **Do not disclose the vulnerability publicly** until it has been addressed 4. **Provide detailed information** about the vulnerability, including: From b6baf663990c0c8524f25261eb1f0ee330a47a34 Mon Sep 17 00:00:00 2001 From: timedout Date: Wed, 24 Jun 2026 18:59:49 +0100 Subject: [PATCH 17/70] feat: Support MSC4380 invite blocking fix: Re-add silently dropped support for MSC4155 --- src/ruminuwuity/invite_permission_config.rs | 97 ++++++++++++++++++++- src/service/users/mod.rs | 28 ++++-- 2 files changed, 114 insertions(+), 11 deletions(-) diff --git a/src/ruminuwuity/invite_permission_config.rs b/src/ruminuwuity/invite_permission_config.rs index ed37b753c..2208ea2e3 100644 --- a/src/ruminuwuity/invite_permission_config.rs +++ b/src/ruminuwuity/invite_permission_config.rs @@ -1,8 +1,13 @@ -//! Types for invite filtering ([MSC4155]). +//! Types for invite filtering ([MSC4155]) and +//! invite blocking ([invite-blocking]). //! //! MSC4155: https://github.com/matrix-org/matrix-spec-proposals/pull/4155 +//! invite-blocking: https://spec.matrix.org/v1.18/client-server-api/#invite-permission -use ruma::{ServerName, UserId, exports::ruma_macros::EventContent}; +use ruma::{ + ServerName, UserId, events::invite_permission_config::InvitePermissionAction, + exports::ruma_macros::EventContent, +}; use serde::{Deserialize, Serialize}; use wildmatch::WildMatch; @@ -25,6 +30,21 @@ pub struct InvitePermissionConfigEventContent { #[serde(default = "ruma::serde::default_true")] pub enabled: bool, + /// The default action chosen by the user that the homeserver should perform + /// automatically when receiving an invitation for this account. + /// + /// A missing, invalid or unsupported value means that the user wants to + /// receive invites as normal. Other parts of the specification might still + /// have effects on invites, like [ignoring users]. + /// + /// [ignoring users]: https://spec.matrix.org/v1.18/client-server-api/#ignoring-users + #[serde( + default, + deserialize_with = "ruma::serde::default_on_error", + skip_serializing_if = "Option::is_none" + )] + pub default_action: Option, + /// A list of globs matching users which are allowed to send an invite. /// Entries in this list supersede entries in the ignored and blocked lists. #[serde(default, skip_serializing_if = "Vec::is_empty")] @@ -77,6 +97,7 @@ impl InvitePermissionConfigEventContent { ) -> Self { Self { enabled, + default_action: None, allowed_users, ignored_users, blocked_users, @@ -86,12 +107,21 @@ impl InvitePermissionConfigEventContent { } } + /// Returns true if the default action is configured and set to block. + fn are_all_blocked(&self) -> bool { + self.default_action + .clone() + .is_some_and(|action| action == InvitePermissionAction::Block) + } + /// Test the filters against a user id. This function will check both the /// user rules _and_ the server rules. #[must_use] #[allow(clippy::if_same_then_else)] pub fn user_filter_level(&self, user: &UserId) -> FilterLevel { - if !self.enabled { + if self.are_all_blocked() { + FilterLevel::Block + } else if !self.enabled { FilterLevel::Allow } else if Self::matches(&self.allowed_users, user.as_str()) { FilterLevel::Allow @@ -132,7 +162,12 @@ impl InvitePermissionConfigEventContent { #[cfg(test)] mod tests { - use ruma::{ServerName, UserId, events::GlobalAccountDataEvent}; + use assign::assign; + use ruma::{ + ServerName, UserId, + events::{GlobalAccountDataEvent, invite_permission_config::InvitePermissionAction}, + user_id, + }; use serde_json::{from_value as from_json_value, json}; use crate::invite_permission_config::{FilterLevel, InvitePermissionConfigEventContent}; @@ -268,4 +303,58 @@ mod tests { FilterLevel::Ignore ); } + + #[test] + fn v118_blocking_enabled_blocks_all() { + let event = assign!( + InvitePermissionConfigEventContent::default(), + { default_action: Some(InvitePermissionAction::Block) } + ); + let fixtures = [ + user_id!("@alice:goodguys.org"), + user_id!("@alice:goodguys.org:8080"), + user_id!("@bob:bar.com"), + user_id!("@bob:Bar.com"), + user_id!("@kevin:reallybadguys.org"), + ]; + assert!( + fixtures + .iter() + .all(|f| event.user_filter_level(f) == FilterLevel::Block) + ); + } + + #[test] + fn v118_blocking_enabled_is_uninfluenced_by_msc4155() { + let toggled = InvitePermissionConfigEventContent { + enabled: false, + default_action: Some(InvitePermissionAction::Block), + ..Default::default() + }; + let only_goodguys = InvitePermissionConfigEventContent { + enabled: true, + default_action: Some(InvitePermissionAction::Block), + allowed_servers: vec!["goodguys.org".to_owned()], + ignored_servers: vec!["reallybadguys.org".to_owned()], + blocked_servers: vec!["*".to_owned()], + ..Default::default() + }; + let fixtures = [ + user_id!("@alice:goodguys.org"), + user_id!("@alice:goodguys.org:8080"), + user_id!("@bob:bar.com"), + user_id!("@bob:Bar.com"), + user_id!("@kevin:reallybadguys.org"), + ]; + assert!( + fixtures + .iter() + .all(|f| toggled.user_filter_level(f) == FilterLevel::Block) + ); + assert!( + fixtures + .iter() + .all(|f| only_goodguys.user_filter_level(f) == FilterLevel::Block) + ); + } } diff --git a/src/service/users/mod.rs b/src/service/users/mod.rs index 4dafb39f9..ec245450f 100644 --- a/src/service/users/mod.rs +++ b/src/service/users/mod.rs @@ -164,13 +164,27 @@ impl Service { if self.user_is_ignored(sender_user, recipient_user).await { FilterLevel::Ignore } else { - self.services - .account_data - .get_global(recipient_user, GlobalAccountDataEventType::InvitePermissionConfig) - .await - .map_or(FilterLevel::Allow, |config: InvitePermissionConfigEvent| { - config.content.user_filter_level(sender_user) - }) + let (stable, unstable) = tokio::join!( + self.services + .account_data + .get_global::( + recipient_user, + GlobalAccountDataEventType::InvitePermissionConfig + ), + self.services + .account_data + .get_global::( + recipient_user, + "org.matrix.msc4155.invite_permission_config".into() + ) // TODO: MSC4155 probably needs upstreaming to ruma at some point + ); + if stable.is_err() && unstable.is_err() { + return FilterLevel::Allow; + } + stable + .unwrap_or_else(|_| unstable.unwrap()) + .content + .user_filter_level(sender_user) } } From b009ffd31c050cc5b72184ea1b522c1762901152 Mon Sep 17 00:00:00 2001 From: timedout Date: Wed, 24 Jun 2026 19:03:38 +0100 Subject: [PATCH 18/70] chore(1875): Add changelogs --- changelog.d/1875.bugfix | 1 + changelog.d/1875.feature | 1 + 2 files changed, 2 insertions(+) create mode 100644 changelog.d/1875.bugfix create mode 100644 changelog.d/1875.feature diff --git a/changelog.d/1875.bugfix b/changelog.d/1875.bugfix new file mode 100644 index 000000000..6b2a4a4b4 --- /dev/null +++ b/changelog.d/1875.bugfix @@ -0,0 +1 @@ +Re-added silently dropped support for MSC4155 invite filtering. Contributed by @nex. diff --git a/changelog.d/1875.feature b/changelog.d/1875.feature new file mode 100644 index 000000000..8407f1d19 --- /dev/null +++ b/changelog.d/1875.feature @@ -0,0 +1 @@ +Added support for MSC4380 invite blocking, which has become part of the Matrix specification in v1.18. Contributed by @nex. From f19f5c71a5b725265e0d4befef508db7dcb2f312 Mon Sep 17 00:00:00 2001 From: timedout Date: Wed, 24 Jun 2026 19:13:35 +0100 Subject: [PATCH 19/70] chore(1875): Remove extra changelog entry --- changelog.d/1875.bugfix | 1 - 1 file changed, 1 deletion(-) delete mode 100644 changelog.d/1875.bugfix diff --git a/changelog.d/1875.bugfix b/changelog.d/1875.bugfix deleted file mode 100644 index 6b2a4a4b4..000000000 --- a/changelog.d/1875.bugfix +++ /dev/null @@ -1 +0,0 @@ -Re-added silently dropped support for MSC4155 invite filtering. Contributed by @nex. From 63a1a3d9abe59d3dbe8bf0c7dd2ed4d6142161d4 Mon Sep 17 00:00:00 2001 From: timedout Date: Wed, 24 Jun 2026 20:05:34 +0100 Subject: [PATCH 20/70] feat: Advertise stable prefix in unstable features --- src/core/matrix/versions.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/core/matrix/versions.rs b/src/core/matrix/versions.rs index bf3d49f0f..ffc4fe082 100644 --- a/src/core/matrix/versions.rs +++ b/src/core/matrix/versions.rs @@ -44,5 +44,6 @@ pub fn unstable_features() -> BTreeMap { ("uk.timedout.msc4323".to_owned(), true), /* agnostic suspend (https://github.com/matrix-org/matrix-spec-proposals/pull/4323) */ ("org.matrix.msc4155".to_owned(), true), /* invite filtering (https://github.com/matrix-org/matrix-spec-proposals/pull/4155) */ ("computer.gingershaped.msc4466".to_owned(), true), /* profile change propagation (https://github.com/matrix-org/matrix-spec-proposals/pull/4466) */ + ("org.matrix.msc4380.stable".to_owned(), true), ]) } From 2a4b9f932341b2331cee9bac161fce5bf87125ad Mon Sep 17 00:00:00 2001 From: timedout Date: Wed, 24 Jun 2026 20:06:02 +0100 Subject: [PATCH 21/70] fix: Check `are_all_blocked` in `server_filter_level` --- src/ruminuwuity/invite_permission_config.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/ruminuwuity/invite_permission_config.rs b/src/ruminuwuity/invite_permission_config.rs index 2208ea2e3..cc7458d80 100644 --- a/src/ruminuwuity/invite_permission_config.rs +++ b/src/ruminuwuity/invite_permission_config.rs @@ -137,7 +137,9 @@ impl InvitePermissionConfigEventContent { /// Test the filters against a server name. Port numbers are ignored. #[must_use] pub fn server_filter_level(&self, server: &ServerName) -> FilterLevel { - if !self.enabled { + if self.are_all_blocked() { + FilterLevel::Block + } else if !self.enabled { FilterLevel::Allow } else { let server = server.host(); From 0483d3e155dcc39e622c933af17e268e93b2597f Mon Sep 17 00:00:00 2001 From: Jade Ellis Date: Sat, 27 Jun 2026 00:54:09 +0100 Subject: [PATCH 22/70] ci: Don't checkout repo in pull_request_target --- .forgejo/workflows/check-changelog.yml | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/.forgejo/workflows/check-changelog.yml b/.forgejo/workflows/check-changelog.yml index fb21354aa..f4e0f8112 100644 --- a/.forgejo/workflows/check-changelog.yml +++ b/.forgejo/workflows/check-changelog.yml @@ -14,23 +14,19 @@ jobs: name: Check changelog is added runs-on: ubuntu-latest steps: - - name: Checkout repository - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 - with: - ref: ${{ github.event.pull_request.head.sha }} - fetch-depth: 0 - persist-credentials: false - sparse-checkout: . - - name: Check for changelog entry id: check_files run: | - git fetch origin ${GITHUB_BASE_REF} + AUTH=$(echo -n "x-access-token:${{ secrets.GITHUB_TOKEN }}" | base64 -w 0) + git config --global http.${{ github.server_url }}/.extraheader "Authorization: basic $AUTH" + git clone "${{ github.event.repository.clone_url }}" repo.git --bare + + git -C repo.git fetch origin ${{ github.event.pull_request.head.ref }} # Check for Added (A) or Modified (M) files in changelog.d - CHANGELOG_CHANGES=$(git diff --name-status origin/${GITHUB_BASE_REF}...HEAD -- changelog.d/) + CHANGELOG_CHANGES=$(git -C repo.git diff --name-status ${{ github.event.pull_request.base.sha }}...${{ github.event.pull_request.head.sha }} -- changelog.d/) - SRC_CHANGES=$(git diff --name-status origin/${GITHUB_BASE_REF}...HEAD -- src/) + SRC_CHANGES=$(git -C repo.git diff --name-status ${{ github.event.pull_request.base.sha }}...${{ github.event.pull_request.head.sha }} -- src/) echo "Changes in changelog.d/:" echo "$CHANGELOG_CHANGES" From cee51d571755447c022a62b25fc771e73d7127e4 Mon Sep 17 00:00:00 2001 From: Jade Ellis Date: Sat, 27 Jun 2026 00:57:45 +0100 Subject: [PATCH 23/70] fix(ci): Correct remote ref fetch --- .forgejo/workflows/check-changelog.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.forgejo/workflows/check-changelog.yml b/.forgejo/workflows/check-changelog.yml index f4e0f8112..a6060a0f3 100644 --- a/.forgejo/workflows/check-changelog.yml +++ b/.forgejo/workflows/check-changelog.yml @@ -21,7 +21,7 @@ jobs: git config --global http.${{ github.server_url }}/.extraheader "Authorization: basic $AUTH" git clone "${{ github.event.repository.clone_url }}" repo.git --bare - git -C repo.git fetch origin ${{ github.event.pull_request.head.ref }} + git -C repo.git fetch origin refs/pull/${{ github.event.pull_request.number }}/merge # Check for Added (A) or Modified (M) files in changelog.d CHANGELOG_CHANGES=$(git -C repo.git diff --name-status ${{ github.event.pull_request.base.sha }}...${{ github.event.pull_request.head.sha }} -- changelog.d/) From 8a495a7d7f57103995704a5eb1bb42f99b515859 Mon Sep 17 00:00:00 2001 From: Jade Ellis Date: Sat, 27 Jun 2026 01:00:33 +0100 Subject: [PATCH 24/70] fix(ci): Correct remote ref fetch --- .forgejo/workflows/check-changelog.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.forgejo/workflows/check-changelog.yml b/.forgejo/workflows/check-changelog.yml index a6060a0f3..eb4ecb66a 100644 --- a/.forgejo/workflows/check-changelog.yml +++ b/.forgejo/workflows/check-changelog.yml @@ -21,7 +21,7 @@ jobs: git config --global http.${{ github.server_url }}/.extraheader "Authorization: basic $AUTH" git clone "${{ github.event.repository.clone_url }}" repo.git --bare - git -C repo.git fetch origin refs/pull/${{ github.event.pull_request.number }}/merge + git -C repo.git fetch origin pull/${{ github.event.pull_request.number }}/head # Check for Added (A) or Modified (M) files in changelog.d CHANGELOG_CHANGES=$(git -C repo.git diff --name-status ${{ github.event.pull_request.base.sha }}...${{ github.event.pull_request.head.sha }} -- changelog.d/) From 4b2f032ae6bcab7647535d606df06ede8feea564 Mon Sep 17 00:00:00 2001 From: 31a05b9c Date: Wed, 24 Jun 2026 10:15:04 +0000 Subject: [PATCH 25/70] feat: `get-state-at` admin command --- src/admin/debug/commands.rs | 40 ++++++++++++++++++++++++++++++++++++- src/admin/debug/mod.rs | 8 ++++++++ 2 files changed, 47 insertions(+), 1 deletion(-) diff --git a/src/admin/debug/commands.rs b/src/admin/debug/commands.rs index f9a678f19..1aa9cb062 100644 --- a/src/admin/debug/commands.rs +++ b/src/admin/debug/commands.rs @@ -6,7 +6,7 @@ use std::{ }; use conduwuit::{ - Err, Result, debug_error, err, info, + Err, Result, at, debug_error, err, info, matrix::{ Event, pdu::{PduEvent, PduId, RawPduId}, @@ -504,6 +504,44 @@ pub(super) async fn get_remote_pdu( .await } +#[admin_command] +pub(super) async fn get_state_at(&self, event_id: OwnedEventId) -> Result { + self.bail_restricted()?; + + let shortstatehash = self + .services + .rooms + .state_accessor + .pdu_shortstatehash(&event_id) + .await?; + + let state_ids: Vec = self + .services + .rooms + .state_accessor + .state_full_ids(shortstatehash) + .map(at!(1)) + .collect() + .await; + + let pdus: Vec = state_ids + .iter() + .try_stream() + .and_then(|id| self.services.rooms.timeline.get_pdu_json(id)) + .try_collect() + .await?; + + let json = serde_json::to_string_pretty(&pdus).map_err(|e| { + err!(Database( + "Failed to convert room state events to pretty JSON, possible invalid room state \ + events in our database {e}", + )) + })?; + + let out = format!("```json\n{json}\n```"); + self.write_str(&out).await +} + #[admin_command] pub(super) async fn get_room_state(&self, room: OwnedRoomOrAliasId) -> Result { self.bail_restricted()?; diff --git a/src/admin/debug/mod.rs b/src/admin/debug/mod.rs index a9478ce82..2e1b62e0b 100644 --- a/src/admin/debug/mod.rs +++ b/src/admin/debug/mod.rs @@ -95,6 +95,14 @@ pub enum DebugCommand { room_id: OwnedRoomOrAliasId, }, + /// Gets all the room state events at the specified event. + /// + /// State at event might not be available for some PDUs, such as rejected + /// ones. + GetStateAt { + event_id: OwnedEventId, + }, + /// Get and display signing keys from local cache or remote server. GetSigningKeys { server_name: Option, From 21e93f8feb00410e3a55f1766578346eb371aa5f Mon Sep 17 00:00:00 2001 From: 31a05b9c Date: Thu, 25 Jun 2026 01:35:31 +0000 Subject: [PATCH 26/70] chore: changelog --- changelog.d/1877.feature | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/1877.feature diff --git a/changelog.d/1877.feature b/changelog.d/1877.feature new file mode 100644 index 000000000..2d3033cbf --- /dev/null +++ b/changelog.d/1877.feature @@ -0,0 +1 @@ +Added `!admin debug get-state-at` command From 574d3e91640fae1b7106587e4ad89cc31e9ecd21 Mon Sep 17 00:00:00 2001 From: timedout Date: Thu, 25 Jun 2026 23:49:24 +0100 Subject: [PATCH 27/70] style: Use `impl` instead of `implement` in moderation service --- src/service/moderation.rs | 108 +++++++++++++++++++------------------- 1 file changed, 53 insertions(+), 55 deletions(-) diff --git a/src/service/moderation.rs b/src/service/moderation.rs index c3e55a1d4..edbaa97b6 100644 --- a/src/service/moderation.rs +++ b/src/service/moderation.rs @@ -1,6 +1,6 @@ use std::sync::Arc; -use conduwuit::{Result, implement}; +use conduwuit::Result; use ruma::ServerName; use crate::{Dep, config}; @@ -27,67 +27,65 @@ impl crate::Service for Service { fn name(&self) -> &str { crate::service::make_name(std::module_path!()) } } -#[implement(Service)] -#[must_use] -pub fn is_remote_server_ignored(&self, server_name: &ServerName) -> bool { - // We must never block federating with ourselves - if server_name == self.services.config.server_name { - return false; +impl Service { + #[must_use] + pub fn is_remote_server_ignored(&self, server_name: &ServerName) -> bool { + // We must never block federating with ourselves + if server_name == self.services.config.server_name { + return false; + } + + self.services + .config + .ignore_messages_from_server_names + .is_match(server_name.host()) } - self.services - .config - .ignore_messages_from_server_names - .is_match(server_name.host()) -} + #[must_use] + pub fn is_remote_server_forbidden(&self, server_name: &ServerName) -> bool { + // We must never block federating with ourselves + if server_name == self.services.config.server_name { + return false; + } -#[implement(Service)] -#[must_use] -pub fn is_remote_server_forbidden(&self, server_name: &ServerName) -> bool { - // We must never block federating with ourselves - if server_name == self.services.config.server_name { - return false; - } - - // Check if server is explicitly allowed - if self - .services - .config - .allowed_remote_server_names - .is_match(server_name.host()) - { - return false; - } - - // Check if server is explicitly forbidden - self.services - .config - .forbidden_remote_server_names - .is_match(server_name.host()) -} - -#[implement(Service)] -#[must_use] -pub fn is_remote_server_room_directory_forbidden(&self, server_name: &ServerName) -> bool { - // Forbidden if NOT (allowed is empty OR allowed contains server OR is self) - // OR forbidden contains server - self.is_remote_server_forbidden(server_name) - || self + // Check if server is explicitly allowed + if self .services .config - .forbidden_remote_room_directory_server_names + .allowed_remote_server_names .is_match(server_name.host()) -} + { + return false; + } -#[implement(Service)] -#[must_use] -pub fn is_remote_server_media_downloads_forbidden(&self, server_name: &ServerName) -> bool { - // Forbidden if NOT (allowed is empty OR allowed contains server OR is self) - // OR forbidden contains server - self.is_remote_server_forbidden(server_name) - || self - .services + // Check if server is explicitly forbidden + self.services .config - .prevent_media_downloads_from + .forbidden_remote_server_names .is_match(server_name.host()) + } + + #[must_use] + pub fn is_remote_server_room_directory_forbidden(&self, server_name: &ServerName) -> bool { + // Forbidden if NOT (allowed is empty OR allowed contains server OR is self) + // OR forbidden contains server + self.is_remote_server_forbidden(server_name) + || self + .services + .config + .forbidden_remote_room_directory_server_names + .is_match(server_name.host()) + } + + #[must_use] + pub fn is_remote_server_media_downloads_forbidden(&self, server_name: &ServerName) -> bool { + // Forbidden if NOT (allowed is empty OR allowed contains server OR is self) + // OR forbidden contains server + self.is_remote_server_forbidden(server_name) + || self + .services + .config + .prevent_media_downloads_from + .is_match(server_name.host()) + } } From aff6315ae1671912f56cc715463f7860246ab7ac Mon Sep 17 00:00:00 2001 From: timedout Date: Thu, 25 Jun 2026 23:52:09 +0100 Subject: [PATCH 28/70] style: Document moderation service --- src/service/moderation.rs | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/service/moderation.rs b/src/service/moderation.rs index edbaa97b6..2d1d4fbe9 100644 --- a/src/service/moderation.rs +++ b/src/service/moderation.rs @@ -28,6 +28,9 @@ impl crate::Service for Service { } impl Service { + /// Returns true if the remote server is *ignored*. Ignored server names are + /// still permitted in federation, but should not have their events sent to + /// local users. #[must_use] pub fn is_remote_server_ignored(&self, server_name: &ServerName) -> bool { // We must never block federating with ourselves @@ -41,6 +44,9 @@ impl Service { .is_match(server_name.host()) } + /// Returns true if the remote server is *forbidden*. Forbidden server names + /// must be blocked in federation in both directions - no inbound nor + /// outbound requests from the remote are permitted. #[must_use] pub fn is_remote_server_forbidden(&self, server_name: &ServerName) -> bool { // We must never block federating with ourselves @@ -65,6 +71,9 @@ impl Service { .is_match(server_name.host()) } + /// Returns true if the remote server is *forbidden*, or if it is only + /// forbidden from having its room directory checked. Does not prevent the + /// remote from reading our room directory. #[must_use] pub fn is_remote_server_room_directory_forbidden(&self, server_name: &ServerName) -> bool { // Forbidden if NOT (allowed is empty OR allowed contains server OR is self) @@ -77,6 +86,10 @@ impl Service { .is_match(server_name.host()) } + /// Returns true if the remote server is *forbidden*, or if it is only + /// forbidden from serving media to us (i.e. we aren't allowed to download + /// media from the remote). Does not prevent the remote downloading media + /// from us. #[must_use] pub fn is_remote_server_media_downloads_forbidden(&self, server_name: &ServerName) -> bool { // Forbidden if NOT (allowed is empty OR allowed contains server OR is self) From 17cff1fb36395444c612a9a925eb670296080a09 Mon Sep 17 00:00:00 2001 From: timedout Date: Fri, 26 Jun 2026 00:31:14 +0100 Subject: [PATCH 29/70] style: Document & refactor users service --- src/service/users/dehydrated_device.rs | 217 +++++++++++-------------- src/service/users/mod.rs | 138 ++++++++++------ 2 files changed, 188 insertions(+), 167 deletions(-) diff --git a/src/service/users/dehydrated_device.rs b/src/service/users/dehydrated_device.rs index cacce5f6b..302cf46f9 100644 --- a/src/service/users/dehydrated_device.rs +++ b/src/service/users/dehydrated_device.rs @@ -1,4 +1,4 @@ -use conduwuit::{Err, Result, implement, trace}; +use conduwuit::{Err, Result, trace}; use conduwuit_database::{Deserialized, Json}; use ruma::{ DeviceId, OwnedDeviceId, UserId, @@ -18,132 +18,107 @@ pub struct DehydratedDevice { pub device_data: Raw, } -/// Creates or recreates the user's dehydrated device. -#[implement(super::Service)] -#[tracing::instrument( - level = "info", - skip_all, - fields( - %user_id, - device_id = %request.device_id, - display_name = ?request.initial_device_display_name, - ) -)] -pub async fn set_dehydrated_device(&self, user_id: &UserId, request: Request) -> Result { - assert!( - self.exists(user_id).await, - "Tried to create dehydrated device for non-existent user" - ); +impl super::Service { + /// Creates or recreates the user's dehydrated device. + pub async fn set_dehydrated_device(&self, user_id: &UserId, request: Request) -> Result { + assert!( + self.exists(user_id).await, + "Tried to create dehydrated device for non-existent user" + ); - let existing_id = self.get_dehydrated_device_id(user_id).await; + let existing_id = self.get_dehydrated_device_id(user_id).await; - if existing_id.is_err() - && self - .get_device_metadata(user_id, &request.device_id) - .await - .is_ok() - { - return Err!("A hydrated device already exists with that ID."); - } - - if let Ok(existing_id) = existing_id { - self.remove_device(user_id, &existing_id).await; - } - - self.create_device( - user_id, - &request.device_id, - "", - request.initial_device_display_name.clone(), - None, - ) - .await?; - - trace!(device_data = ?request.device_data); - self.db.userid_dehydrateddevice.raw_put( - user_id, - Json(&DehydratedDevice { - device_id: request.device_id.clone(), - device_data: request.device_data, - }), - ); - - trace!(device_keys = ?request.device_keys); - self.add_device_keys(user_id, &request.device_id, &request.device_keys) - .await; - - trace!(one_time_keys = ?request.one_time_keys); - for (one_time_key_key, one_time_key_value) in &request.one_time_keys { - self.add_one_time_key(user_id, &request.device_id, one_time_key_key, one_time_key_value) - .await?; - } - - Ok(()) -} - -/// Removes a user's dehydrated device. -/// -/// Calling this directly will remove the dehydrated data but leak the frontage -/// device. Thus this is called by the regular device interface such that the -/// dehydrated data will not leak instead. -/// -/// If device_id is given, the user's dehydrated device must match or this is a -/// no-op, but an Err is still returned to indicate that. Otherwise returns the -/// removed dehydrated device_id. -#[implement(super::Service)] -#[tracing::instrument( - level = "debug", - skip_all, - fields( - %user_id, - device_id = ?maybe_device_id, - ) -)] -pub(super) async fn remove_dehydrated_device( - &self, - user_id: &UserId, - maybe_device_id: Option<&DeviceId>, -) -> Result { - let Ok(device_id) = self.get_dehydrated_device_id(user_id).await else { - return Err!(Request(NotFound("No dehydrated device for this user."))); - }; - - if let Some(maybe_device_id) = maybe_device_id { - if maybe_device_id != device_id { - return Err!(Request(NotFound("Not the user's dehydrated device."))); + if existing_id.is_err() + && self + .get_device_metadata(user_id, &request.device_id) + .await + .is_ok() + { + return Err!("A hydrated device already exists with that ID."); } + + if let Ok(existing_id) = existing_id { + self.remove_device(user_id, &existing_id).await; + } + + self.create_device( + user_id, + &request.device_id, + "", + request.initial_device_display_name.clone(), + None, + ) + .await?; + + trace!(device_data = ?request.device_data); + self.db.userid_dehydrateddevice.raw_put( + user_id, + Json(&DehydratedDevice { + device_id: request.device_id.clone(), + device_data: request.device_data, + }), + ); + + trace!(device_keys = ?request.device_keys); + self.add_device_keys(user_id, &request.device_id, &request.device_keys) + .await; + + trace!(one_time_keys = ?request.one_time_keys); + for (one_time_key_key, one_time_key_value) in &request.one_time_keys { + self.add_one_time_key( + user_id, + &request.device_id, + one_time_key_key, + one_time_key_value, + ) + .await?; + } + + Ok(()) } - self.db.userid_dehydrateddevice.remove(user_id); + /// Removes a user's dehydrated device. + /// + /// Calling this directly will remove the dehydrated data but leak the + /// frontage device. Thus this is called by the regular device interface + /// such that the dehydrated data will not leak instead. + /// + /// If device_id is given, the user's dehydrated device must match or this + /// is a no-op, but an Err is still returned to indicate that. Otherwise + /// returns the removed dehydrated device_id. + pub(super) async fn remove_dehydrated_device( + &self, + user_id: &UserId, + maybe_device_id: Option<&DeviceId>, + ) -> Result { + let Ok(device_id) = self.get_dehydrated_device_id(user_id).await else { + return Err!(Request(NotFound("No dehydrated device for this user."))); + }; - Ok(device_id) -} + if let Some(maybe_device_id) = maybe_device_id { + if maybe_device_id != device_id { + return Err!(Request(NotFound("Not the user's dehydrated device."))); + } + } -/// Get the device_id of the user's dehydrated device. -#[implement(super::Service)] -#[tracing::instrument( - level = "debug", - skip_all, - fields(%user_id) -)] -pub async fn get_dehydrated_device_id(&self, user_id: &UserId) -> Result { - self.get_dehydrated_device(user_id) - .await - .map(|device| device.device_id) -} + self.db.userid_dehydrateddevice.remove(user_id); -/// Get the dehydrated device private data -#[implement(super::Service)] -#[tracing::instrument( - level = "debug", - skip_all, - fields(%user_id), - ret, -)] -pub async fn get_dehydrated_device(&self, user_id: &UserId) -> Result { - self.db - .userid_dehydrateddevice - .get(user_id) - .await - .deserialized() + Ok(device_id) + } + + /// Get the device_id of the user's dehydrated device. + pub async fn get_dehydrated_device_id(&self, user_id: &UserId) -> Result { + self.get_dehydrated_device(user_id) + .await + .map(|device| device.device_id) + } + + /// Get the dehydrated device private data + pub async fn get_dehydrated_device(&self, user_id: &UserId) -> Result { + self.db + .userid_dehydrateddevice + .get(user_id) + .await + .deserialized() + } } diff --git a/src/service/users/mod.rs b/src/service/users/mod.rs index ec245450f..429facfd1 100644 --- a/src/service/users/mod.rs +++ b/src/service/users/mod.rs @@ -140,7 +140,7 @@ impl crate::Service for Service { impl Service { /// Returns true/false based on whether the recipient/receiving user has - /// blocked the sender + /// ignored the sender. pub async fn user_is_ignored(&self, sender_user: &UserId, recipient_user: &UserId) -> bool { self.services .account_data @@ -213,7 +213,8 @@ impl Service { // password: // ) - /// Deactivate account + /// Deactivates an account, removing all of their device IDs and unsetting + /// their password. pub async fn deactivate_account(&self, user_id: &UserId) -> Result<()> { // Remove all associated devices self.all_device_ids(user_id) @@ -247,6 +248,7 @@ impl Service { self.db.userid_suspension.remove(user_id); } + /// Locks an account, preventing it being used until it is unlocked. pub async fn lock_account(&self, user_id: &UserId, locking_user: &UserId) { // NOTE: Locking is basically just suspension with a more severe effect, // so we'll just re-use the suspension data structure to store the lock state. @@ -265,6 +267,7 @@ impl Service { self.db.userid_lock.raw_put(user_id, Json(suspension)); } + /// Unlocks an account, allowing the user to log in and use it again. pub async fn unlock_account(&self, user_id: &UserId) { self.db.userid_lock.remove(user_id); } /// Check if the provided user ID belongs to an existing (possibly @@ -275,7 +278,8 @@ impl Service { && self.db.userid_password.get(user_id).await.is_ok() } - /// Check if account is deactivated + /// Check if account is deactivated (has an empty password). Returns a + /// NotFound error if the user does not exist. pub async fn is_deactivated(&self, user_id: &UserId) -> Result { self.db .userid_password @@ -285,7 +289,7 @@ impl Service { .await } - /// Check if account is suspended + /// Check if account is suspended. Returns false if the user does not exist. pub async fn is_suspended(&self, user_id: &UserId) -> Result { match self .db @@ -304,6 +308,8 @@ impl Service { } } + /// Returns true if the user is locked. Returns false if the user does not + /// exist or is not locked. pub async fn is_locked(&self, user_id: &UserId) -> Result { match self .db @@ -322,12 +328,16 @@ impl Service { } } + /// Disables login for a user, preventing them from creating new devices, + /// but allows them to continue using their existing sessions unimpeded. pub fn disable_login(&self, user_id: &UserId) { self.db.userid_logindisabled.insert(user_id, ""); } + /// Re-enables login for a user, allowing them to create new devices again. pub fn enable_login(&self, user_id: &UserId) { self.db.userid_logindisabled.remove(user_id); } + /// Returns true if the target user's login is disabled. pub async fn is_login_disabled(&self, user_id: &UserId) -> bool { self.db .userid_logindisabled @@ -336,21 +346,23 @@ impl Service { .is_ok() } - /// Check if account is active, infallible + /// Check if account is active (not deactivated) pub async fn is_active(&self, user_id: &UserId) -> bool { !self.is_deactivated(user_id).await.unwrap_or(true) } - /// Check if account is active, infallible + /// Check if account is a local user, and is active (not deactivated) pub async fn is_active_local(&self, user_id: &UserId) -> bool { self.services.globals.user_is_local(user_id) && self.is_active(user_id).await } - /// Returns the number of users registered on this server. + /// Returns the number of users registered on this server, including + /// deactivated users. #[inline] pub async fn count(&self) -> usize { self.db.userid_password.count().await } - /// Find out which user an access token belongs to. + /// Find out which user an access token belongs to. Will panic if the access + /// token is empty. pub async fn find_from_token(&self, token: &str) -> Result<(OwnedUserId, OwnedDeviceId)> { assert!(!token.is_empty(), "Empty access token"); self.db.token_userdeviceid.get(token).await.deserialized() @@ -361,10 +373,10 @@ impl Service { self.db.userid_password.keys().ignore_err() } - /// Returns a list of local users as list of usernames. + /// Returns a list of active local users. /// - /// A user account is considered `local` if the length of it's password is - /// greater then zero. + /// A user account is considered `local` if the associated password is not + /// empty. pub fn list_local_users(&self) -> impl Stream + Send + '_ { self.db .userid_password @@ -384,21 +396,26 @@ impl Service { /// Check a user's password. pub async fn check_password(&self, user_id: &UserId, password: &str) -> Result { - let (hash, user_id): (String, OwnedUserId) = if let Ok(hash) = - self.db.userid_password.get(user_id).await.deserialized() - { - (hash, user_id.to_owned()) - } else { - // We also check the lowercased version of the user ID to handle legacy user IDs - // better - let lowercase_user_id = UserId::parse(user_id.as_str().to_lowercase()).unwrap(); - + let (hash, user_id): (String, OwnedUserId) = if let Ok(hash) = self.db.userid_password.get(user_id).await.deserialized() { - (hash, lowercase_user_id) + (hash, user_id.to_owned()) } else { - return Err!(Request(InvalidParam("This user cannot log in with a password."))); - } - }; + // We also check the lowercased version of the user ID to handle legacy user IDs + // better + let lowercase_user_id = UserId::parse(user_id.as_str().to_lowercase()).unwrap(); + + if let Ok(hash) = self + .db + .userid_password + .get(lowercase_user_id.as_str()) + .await + .deserialized() + { + (hash, lowercase_user_id) + } else { + return Err!(Request(Forbidden("This user cannot log in with a password."))); + } + }; if hash.is_empty() { return Err!(Request(UserDeactivated("This user is deactivated"))); @@ -443,7 +460,8 @@ impl Service { } } - /// Adds a new device to a user. + /// Adds a new device to a user. The user must exist, otherwise InvalidParam + /// is returned. pub async fn create_device( &self, user_id: &UserId, @@ -514,6 +532,7 @@ impl Service { .map(|(_, device_id): (Ignore, OwnedDeviceId)| device_id) } + /// Gets the access token associated with a device. pub async fn get_token(&self, user_id: &UserId, device_id: &DeviceId) -> Result { let key = (user_id, device_id); self.db.userdeviceid_token.qry(&key).await.deserialized() @@ -524,19 +543,12 @@ impl Service { loop { let token = utils::random_string(32); - // Check for collision with appservice tokens - if self - .services - .appservice - .find_from_token(&token) - .await - .is_ok() - { - continue; - } - - // Check for collision with user tokens - if self.db.token_userdeviceid.get(&token).await.is_ok() { + // Check for collision with existing appservice and user tokens + let (appservice, usr) = tokio::join!( + self.services.appservice.find_from_token(&token), + self.db.token_userdeviceid.get(&token) + ); + if appservice.is_ok() || usr.is_ok() { continue; } @@ -586,6 +598,7 @@ impl Service { Ok(()) } + /// Adds a single one-time key to a device. pub async fn add_one_time_key( &self, user_id: &UserId, @@ -628,7 +641,7 @@ impl Service { } /// Save a fallback key for the given user, device, and algorithm - /// This key will replace an existing fallback key + /// This key will replace an existing fallback key. pub async fn add_fallback_key( &self, user_id: &UserId, @@ -660,6 +673,8 @@ impl Service { Ok(()) } + /// Returns the timestamp at when the last OTK update for the user was, or 0 + /// if the keys have never been updated. pub async fn last_one_time_keys_update(&self, user_id: &UserId) -> u64 { self.db .userid_lastonetimekeyupdate @@ -669,6 +684,10 @@ impl Service { .unwrap_or(0) } + /// Consumes a one-time key belonging to the device of a given algorithm. If + /// no one-time keys are available, a fallback key is returned instead (if + /// available). If neither an OTK nor fallback key are available, NotFound + /// is returned. pub async fn take_one_time_key( &self, user_id: &UserId, @@ -751,6 +770,8 @@ impl Service { Err(err!(Request(NotFound("No one-time key or fallback key found")))) } + /// Returns the number of one-time keys the given device has. Does not count + /// fallback keys. pub async fn count_one_time_keys( &self, user_id: &UserId, @@ -781,6 +802,7 @@ impl Service { algorithm_counts } + /// Returns a list of *unused* fallback key types. pub async fn list_unused_fallback_key_types( &self, user_id: &UserId, @@ -809,6 +831,8 @@ impl Service { unused_algorithms } + /// Adds device identity keys to a device, overwriting existing ones if they + /// exist. Dispatches a device key update. pub async fn add_device_keys( &self, user_id: &UserId, @@ -821,6 +845,8 @@ impl Service { self.mark_device_key_update(user_id).await; } + /// Adds cross-signing keys for the given user. If notify is true, a device + /// key update is dispatched. pub async fn add_cross_signing_keys( &self, user_id: &UserId, @@ -898,6 +924,8 @@ impl Service { Ok(()) } + /// Appends a new signature to the given key ID. Dispatches a new device key + /// update. pub async fn sign_key( &self, target_id: &UserId, @@ -943,6 +971,7 @@ impl Service { Ok(()) } + /// Returns a stream of changed keys between the two counts. #[inline] pub fn keys_changed<'a>( &'a self, @@ -954,6 +983,8 @@ impl Service { .map(|(user_id, ..)| user_id) } + /// Returns a stream of changed keys between the two counts in the given + /// room specifically. #[inline] pub fn room_keys_changed<'a>( &'a self, @@ -985,6 +1016,8 @@ impl Service { .map(|((_, count), user_id): KeyVal<'_>| (user_id, count)) } + /// Marks that a user's device keys have been updated, so that other users + /// can be notified of the change. pub async fn mark_device_key_update(&self, user_id: &UserId) { let count = self.services.globals.next_count().unwrap(); @@ -1009,6 +1042,7 @@ impl Service { self.db.keychangeid_userid.put_raw(key, user_id); } + /// Returns the device identity keys for a given device. pub async fn get_device_keys<'a>( &'a self, user_id: &'a UserId, @@ -1018,6 +1052,7 @@ impl Service { self.db.keyid_key.qry(&key_id).await.deserialized() } + /// Gets a specific cross-signing key. pub async fn get_key( &self, key_id: &[u8], @@ -1035,6 +1070,7 @@ impl Service { Ok(Raw::from_json(raw_value)) } + /// Gets a user's master cross-signing key. pub async fn get_master_key( &self, sender_user: Option<&UserId>, @@ -1050,6 +1086,7 @@ impl Service { .await } + /// Gets a user's self-signing cross-signing key. pub async fn get_self_signing_key( &self, sender_user: Option<&UserId>, @@ -1065,6 +1102,7 @@ impl Service { .await } + /// Gets a user's user-signing cross-signing key. pub async fn get_user_signing_key(&self, user_id: &UserId) -> Result> { self.db .userid_usersigningkeyid @@ -1074,6 +1112,7 @@ impl Service { .deserialized() } + /// Pushes a new to-device event into a device's inbox. pub async fn add_to_device_event( &self, sender: &UserId, @@ -1095,6 +1134,7 @@ impl Service { ); } + /// Gets all to-device events between the two counts. pub fn get_to_device_events<'a>( &'a self, user_id: &'a UserId, @@ -1118,6 +1158,8 @@ impl Service { .map(|((_, _, count), event)| (count, event)) } + /// Removes to-device events from the target device's inbox, until the given + /// count. pub async fn remove_to_device_events( &self, user_id: &UserId, @@ -1152,14 +1194,13 @@ impl Service { ) -> Result<()> { increment(&self.db.userid_devicelistversion, user_id.as_bytes()); self.update_device_metadata_no_increment(user_id, device_id, device) - .await } - // Updates device metadata without incrementing the device list version. - // This is namely used for updating the last_seen_ip and last_seen_ts values, - // as those do not need a device list version bump due to them not being - // relevant to other consumers. - pub async fn update_device_metadata_no_increment( + /// Updates device metadata without incrementing the device list version. + /// This is namely used for updating the last_seen_ip and last_seen_ts + /// values, as those do not need a device list version bump due to them not + /// being relevant to other consumers. + fn update_device_metadata_no_increment( &self, user_id: &UserId, device_id: &DeviceId, @@ -1171,6 +1212,9 @@ impl Service { Ok(()) } + /// Updates the last seen timestamp for a device. Silently does nothing if + /// the last update was less than 10 seconds ago, or the device does not + /// exist. pub async fn update_device_last_seen( &self, user_id: &UserId, @@ -1190,7 +1234,6 @@ impl Service { device.last_seen_ts = Some(now); self.update_device_metadata_no_increment(user_id, device_id, &device) - .await .ok(); } } @@ -1209,6 +1252,7 @@ impl Service { .deserialized() } + /// Gets the most recent device list version for a user. pub async fn get_devicelist_version(&self, user_id: &UserId) -> Result { self.db .userid_devicelistversion @@ -1217,6 +1261,7 @@ impl Service { .deserialized() } + /// Gets metadata for all devices belonging to the target user. pub fn all_devices_metadata<'a>( &'a self, user_id: &'a UserId, @@ -1239,6 +1284,7 @@ impl Service { filter_id } + /// Fetches a filter from a filter ID belonging to a user. pub async fn get_filter( &self, user_id: &UserId, From 6312ff80885c3278863220587104affe8936d05f Mon Sep 17 00:00:00 2001 From: timedout Date: Fri, 26 Jun 2026 00:34:19 +0100 Subject: [PATCH 30/70] style: Document & refactor sync service --- src/service/sync/watch.rs | 179 +++++++++++++++++++------------------- 1 file changed, 91 insertions(+), 88 deletions(-) diff --git a/src/service/sync/watch.rs b/src/service/sync/watch.rs index ce8f20f5f..3795b4338 100644 --- a/src/service/sync/watch.rs +++ b/src/service/sync/watch.rs @@ -1,112 +1,115 @@ -use conduwuit::{Result, implement, trace}; +use conduwuit::{Result, trace}; use futures::{FutureExt, StreamExt, pin_mut, stream::FuturesUnordered}; use ruma::{DeviceId, UserId}; -#[implement(super::Service)] -#[tracing::instrument(skip(self), level = "debug")] -pub async fn watch(&self, user_id: &UserId, device_id: &DeviceId) -> Result { - let userid_bytes = user_id.as_bytes().to_vec(); - let mut userid_prefix = userid_bytes.clone(); - userid_prefix.push(0xFF); +impl super::Service { + /// Watches for changes that might wake the sync loop for the given user + + /// device. + #[tracing::instrument(skip(self), level = "debug")] + pub async fn watch(&self, user_id: &UserId, device_id: &DeviceId) -> Result { + let userid_bytes = user_id.as_bytes().to_vec(); + let mut userid_prefix = userid_bytes.clone(); + userid_prefix.push(0xFF); - let mut userdeviceid_prefix = userid_prefix.clone(); - userdeviceid_prefix.extend_from_slice(device_id.as_bytes()); - userdeviceid_prefix.push(0xFF); + let mut userdeviceid_prefix = userid_prefix.clone(); + userdeviceid_prefix.extend_from_slice(device_id.as_bytes()); + userdeviceid_prefix.push(0xFF); - let mut futures = FuturesUnordered::new(); + let mut futures = FuturesUnordered::new(); - // Return when *any* user changed their key - // TODO: only send for user they share a room with - futures.push(self.db.todeviceid_events.watch_prefix(&userdeviceid_prefix)); + // Return when *any* user changed their key + // TODO: only send for user they share a room with + futures.push(self.db.todeviceid_events.watch_prefix(&userdeviceid_prefix)); - futures.push(self.db.userroomid_joined.watch_prefix(&userid_prefix)); - futures.push(self.db.userroomid_invitestate.watch_prefix(&userid_prefix)); - futures.push(self.db.userroomid_leftstate.watch_prefix(&userid_prefix)); - futures.push( - self.db - .userroomid_notificationcount - .watch_prefix(&userid_prefix), - ); - futures.push( - self.db - .userroomid_highlightcount - .watch_prefix(&userid_prefix), - ); + futures.push(self.db.userroomid_joined.watch_prefix(&userid_prefix)); + futures.push(self.db.userroomid_invitestate.watch_prefix(&userid_prefix)); + futures.push(self.db.userroomid_leftstate.watch_prefix(&userid_prefix)); + futures.push( + self.db + .userroomid_notificationcount + .watch_prefix(&userid_prefix), + ); + futures.push( + self.db + .userroomid_highlightcount + .watch_prefix(&userid_prefix), + ); - // Events for rooms we are in - let rooms_joined = self.services.state_cache.rooms_joined(user_id); + // Events for rooms we are in + let rooms_joined = self.services.state_cache.rooms_joined(user_id); - pin_mut!(rooms_joined); - while let Some(room_id) = rooms_joined.next().await { - let Ok(short_roomid) = self.services.short.get_shortroomid(&room_id).await else { - continue; - }; + pin_mut!(rooms_joined); + while let Some(room_id) = rooms_joined.next().await { + let Ok(short_roomid) = self.services.short.get_shortroomid(&room_id).await else { + continue; + }; - let roomid_bytes = room_id.as_bytes().to_vec(); - let mut roomid_prefix = roomid_bytes.clone(); - roomid_prefix.push(0xFF); + let roomid_bytes = room_id.as_bytes().to_vec(); + let mut roomid_prefix = roomid_bytes.clone(); + roomid_prefix.push(0xFF); - // Key changes - futures.push(self.db.keychangeid_userid.watch_prefix(&roomid_prefix)); + // Key changes + futures.push(self.db.keychangeid_userid.watch_prefix(&roomid_prefix)); - // Room account data - let mut roomuser_prefix = roomid_prefix.clone(); - roomuser_prefix.extend_from_slice(&userid_prefix); + // Room account data + let mut roomuser_prefix = roomid_prefix.clone(); + roomuser_prefix.extend_from_slice(&userid_prefix); + + futures.push( + self.db + .roomusertype_roomuserdataid + .watch_prefix(&roomuser_prefix), + ); + + // PDUs + let short_roomid = short_roomid.to_be_bytes().to_vec(); + futures.push(self.db.pduid_pdu.watch_prefix(&short_roomid)); + + // EDUs + let typing_room_id = room_id.clone(); + let typing_wait_for_update = async move { + self.services.typing.wait_for_update(&typing_room_id).await; + }; + + futures.push(typing_wait_for_update.boxed()); + futures.push( + self.db + .readreceiptid_readreceipt + .watch_prefix(&roomid_prefix), + ); + } + + let mut globaluserdata_prefix = vec![0xFF]; + globaluserdata_prefix.extend_from_slice(&userid_prefix); futures.push( self.db .roomusertype_roomuserdataid - .watch_prefix(&roomuser_prefix), + .watch_prefix(&globaluserdata_prefix), ); - // PDUs - let short_roomid = short_roomid.to_be_bytes().to_vec(); - futures.push(self.db.pduid_pdu.watch_prefix(&short_roomid)); + // More key changes (used when user is not joined to any rooms) + futures.push(self.db.keychangeid_userid.watch_prefix(&userid_prefix)); - // EDUs - let typing_room_id = room_id.clone(); - let typing_wait_for_update = async move { - self.services.typing.wait_for_update(&typing_room_id).await; - }; - - futures.push(typing_wait_for_update.boxed()); + // One time keys futures.push( self.db - .readreceiptid_readreceipt - .watch_prefix(&roomid_prefix), + .userid_lastonetimekeyupdate + .watch_prefix(&userid_bytes), ); + + // Server shutdown + futures.push(self.services.server.until_shutdown().boxed()); + + if !self.services.server.running() { + return Ok(()); + } + + // Wait until one of them finds something + trace!(futures = futures.len(), "watch started"); + futures.next().await; + trace!(futures = futures.len(), "watch finished"); + + Ok(()) } - - let mut globaluserdata_prefix = vec![0xFF]; - globaluserdata_prefix.extend_from_slice(&userid_prefix); - - futures.push( - self.db - .roomusertype_roomuserdataid - .watch_prefix(&globaluserdata_prefix), - ); - - // More key changes (used when user is not joined to any rooms) - futures.push(self.db.keychangeid_userid.watch_prefix(&userid_prefix)); - - // One time keys - futures.push( - self.db - .userid_lastonetimekeyupdate - .watch_prefix(&userid_bytes), - ); - - // Server shutdown - futures.push(self.services.server.until_shutdown().boxed()); - - if !self.services.server.running() { - return Ok(()); - } - - // Wait until one of them finds something - trace!(futures = futures.len(), "watch started"); - futures.next().await; - trace!(futures = futures.len(), "watch finished"); - - Ok(()) } From a3994fe1a3d0061c33e5448663582702bcedc806 Mon Sep 17 00:00:00 2001 From: timedout Date: Fri, 26 Jun 2026 01:10:36 +0100 Subject: [PATCH 31/70] style: Document & refactor server_keys service --- src/service/server_keys/acquire.rs | 418 +++++++++++++++-------------- src/service/server_keys/get.rs | 233 ++++++++-------- src/service/server_keys/mod.rs | 199 +++++++------- src/service/server_keys/request.rs | 178 ++++++------ src/service/server_keys/sign.rs | 42 +-- src/service/server_keys/verify.rs | 122 ++++----- 6 files changed, 606 insertions(+), 586 deletions(-) diff --git a/src/service/server_keys/acquire.rs b/src/service/server_keys/acquire.rs index a14056257..4451f738d 100644 --- a/src/service/server_keys/acquire.rs +++ b/src/service/server_keys/acquire.rs @@ -4,9 +4,7 @@ use std::{ time::Duration, }; -use conduwuit::{ - debug, debug_error, debug_warn, error, implement, info, result::FlatOk, trace, warn, -}; +use conduwuit::{debug, debug_error, debug_warn, error, info, result::FlatOk, trace, warn}; use futures::{StreamExt, stream::FuturesUnordered}; use ruma::{ CanonicalJsonObject, OwnedServerName, OwnedServerSigningKeyId, ServerName, @@ -19,218 +17,228 @@ use super::key_exists; type Batch = BTreeMap>; -#[implement(super::Service)] -pub async fn acquire_events_pubkeys<'a, I>(&self, events: I) -where - I: Iterator> + Send, -{ - type Batch = BTreeMap>; - type Signatures = BTreeMap>; +impl super::Service { + /// Returns all verify keys required to verify the incoming events. + pub async fn acquire_events_pubkeys<'a, I>(&self, events: I) + where + I: Iterator> + Send, + { + type Batch = BTreeMap>; + type Signatures = BTreeMap>; - let mut batch = Batch::new(); - events - .cloned() - .map(Raw::::from_json) - .map(|event| event.get_field::("signatures")) - .filter_map(FlatOk::flat_ok) - .flat_map(IntoIterator::into_iter) - .for_each(|(server, sigs)| { - batch.entry(server).or_default().extend(sigs.into_keys()); - }); + let mut batch = Batch::new(); + events + .cloned() + .map(Raw::::from_json) + .map(|event| event.get_field::("signatures")) + .filter_map(FlatOk::flat_ok) + .flat_map(IntoIterator::into_iter) + .for_each(|(server, sigs)| { + batch.entry(server).or_default().extend(sigs.into_keys()); + }); - let batch = batch - .iter() - .map(|(server, keys)| (server.borrow(), keys.iter().map(Borrow::borrow))); - - self.acquire_pubkeys(batch).await; -} - -#[implement(super::Service)] -pub async fn acquire_pubkeys<'a, S, K>(&self, batch: S) -where - S: Iterator + Send + Clone, - K: Iterator + Send + Clone, -{ - let notary_only = self.services.server.config.only_query_trusted_key_servers; - let notary_first_always = self.services.server.config.query_trusted_key_servers_first; - let notary_first_on_join = self - .services - .server - .config - .query_trusted_key_servers_first_on_join; - - let requested_servers = batch.clone().count(); - let requested_keys = batch.clone().flat_map(|(_, key_ids)| key_ids).count(); - - debug!("acquire {requested_keys} keys from {requested_servers}"); - - let mut missing = self.acquire_locals(batch).await; - let mut missing_keys = keys_count(&missing); - let mut missing_servers = missing.len(); - if missing_servers == 0 { - return; - } - - info!("{missing_keys} keys for {missing_servers} servers will be acquired"); - - if notary_first_always || notary_first_on_join { - missing = self.acquire_notary(missing.into_iter()).await; - missing_keys = keys_count(&missing); - missing_servers = missing.len(); - if missing_keys == 0 { - return; - } - - warn!( - "missing {missing_keys} keys for {missing_servers} servers from all notaries first" - ); - } - - if !notary_only { - missing = self.acquire_origins(missing.into_iter()).await; - missing_keys = keys_count(&missing); - missing_servers = missing.len(); - if missing_keys == 0 { - return; - } - - debug_warn!("missing {missing_keys} keys for {missing_servers} servers unreachable"); - } - - if !notary_first_always && !notary_first_on_join { - missing = self.acquire_notary(missing.into_iter()).await; - missing_keys = keys_count(&missing); - missing_servers = missing.len(); - if missing_keys == 0 { - return; - } - - debug_warn!( - "still missing {missing_keys} keys for {missing_servers} servers from all notaries." - ); - } - - if missing_keys > 0 { - warn!( - "did not obtain {missing_keys} keys for {missing_servers} servers out of \ - {requested_keys} total keys for {requested_servers} total servers." - ); - } - - for (server, key_ids) in missing { - debug_warn!(%server, ?key_ids, "missing"); - } -} - -#[implement(super::Service)] -async fn acquire_locals<'a, S, K>(&self, batch: S) -> Batch -where - S: Iterator + Send, - K: Iterator + Send, -{ - let mut missing = Batch::new(); - for (server, key_ids) in batch { - for key_id in key_ids { - if !self.verify_key_exists(server, key_id).await { - missing - .entry(server.into()) - .or_default() - .push(key_id.into()); - } - } - } - - missing -} - -#[implement(super::Service)] -async fn acquire_origins(&self, batch: I) -> Batch -where - I: Iterator)> + Send, -{ - let timeout = Instant::now() - .checked_add(Duration::from_secs(45)) - .expect("timeout overflows"); - - let mut requests: FuturesUnordered<_> = batch - .map(|(origin, key_ids)| self.acquire_origin(origin, key_ids, timeout)) - .collect(); - - let mut missing = Batch::new(); - while let Some((origin, key_ids)) = requests.next().await { - if !key_ids.is_empty() { - missing.insert(origin, key_ids); - } - } - - missing -} - -#[implement(super::Service)] -async fn acquire_origin( - &self, - origin: OwnedServerName, - mut key_ids: Vec, - timeout: Instant, -) -> (OwnedServerName, Vec) { - match timeout_at(timeout, self.server_request(&origin)).await { - | Err(e) => debug_warn!(%origin, "timed out: {e}"), - | Ok(Err(e)) => debug_error!(%origin, "{e}"), - | Ok(Ok(server_keys)) => { - trace!( - %origin, - ?key_ids, - ?server_keys, - "received server_keys" - ); - - self.add_signing_keys(server_keys.clone()).await; - key_ids.retain(|key_id| !key_exists(&server_keys, key_id)); - }, - } - - (origin, key_ids) -} - -#[implement(super::Service)] -async fn acquire_notary(&self, batch: I) -> Batch -where - I: Iterator)> + Send, -{ - let mut missing: Batch = batch.collect(); - for notary in self.services.globals.trusted_servers() { - let missing_keys = keys_count(&missing); - let missing_servers = missing.len(); - debug!( - "Asking notary {notary} for {missing_keys} missing keys from {missing_servers} \ - servers" - ); - - let batch = missing + let batch = batch .iter() .map(|(server, keys)| (server.borrow(), keys.iter().map(Borrow::borrow))); - match self.batch_notary_request(notary, batch).await { - | Err(e) => error!("Failed to contact notary {notary:?}: {e}"), - | Ok(results) => - for server_keys in results { - self.acquire_notary_result(&mut missing, server_keys).await; - }, + self.acquire_pubkeys(batch).await; + } + + /// Acquires all the verify keys required in the provided batch, using + /// notaries and direct-fetch where applicable. + pub async fn acquire_pubkeys<'a, S, K>(&self, batch: S) + where + S: Iterator + Send + Clone, + K: Iterator + Send + Clone, + { + let notary_only = self.services.server.config.only_query_trusted_key_servers; + let notary_first_always = self.services.server.config.query_trusted_key_servers_first; + let notary_first_on_join = self + .services + .server + .config + .query_trusted_key_servers_first_on_join; + + let requested_servers = batch.clone().count(); + let requested_keys = batch.clone().flat_map(|(_, key_ids)| key_ids).count(); + + debug!("acquire {requested_keys} keys from {requested_servers}"); + + let mut missing = self.acquire_locals(batch).await; + let mut missing_keys = keys_count(&missing); + let mut missing_servers = missing.len(); + if missing_servers == 0 { + return; + } + + info!("{missing_keys} keys for {missing_servers} servers will be acquired"); + + if notary_first_always || notary_first_on_join { + missing = self.acquire_notary(missing.into_iter()).await; + missing_keys = keys_count(&missing); + missing_servers = missing.len(); + if missing_keys == 0 { + return; + } + + warn!( + "missing {missing_keys} keys for {missing_servers} servers from all notaries \ + first" + ); + } + + if !notary_only { + missing = self.acquire_origins(missing.into_iter()).await; + missing_keys = keys_count(&missing); + missing_servers = missing.len(); + if missing_keys == 0 { + return; + } + + debug_warn!("missing {missing_keys} keys for {missing_servers} servers unreachable"); + } + + if !notary_first_always && !notary_first_on_join { + missing = self.acquire_notary(missing.into_iter()).await; + missing_keys = keys_count(&missing); + missing_servers = missing.len(); + if missing_keys == 0 { + return; + } + + debug_warn!( + "still missing {missing_keys} keys for {missing_servers} servers from all \ + notaries." + ); + } + + if missing_keys > 0 { + warn!( + "did not obtain {missing_keys} keys for {missing_servers} servers out of \ + {requested_keys} total keys for {requested_servers} total servers." + ); + } + + for (server, key_ids) in missing { + debug_warn!(%server, ?key_ids, "missing"); } } - missing -} + /// Filters the batch to return only keys which we do not already have + /// locally. + async fn acquire_locals<'a, S, K>(&self, batch: S) -> Batch + where + S: Iterator + Send, + K: Iterator + Send, + { + let mut missing = Batch::new(); + for (server, key_ids) in batch { + for key_id in key_ids { + if !self.verify_key_exists(server, key_id).await { + missing + .entry(server.into()) + .or_default() + .push(key_id.into()); + } + } + } -#[implement(super::Service)] -async fn acquire_notary_result(&self, missing: &mut Batch, server_keys: ServerSigningKeys) { - let server = &server_keys.server_name; - self.add_signing_keys(server_keys.clone()).await; + missing + } - if let Some(key_ids) = missing.get_mut(server) { - key_ids.retain(|key_id| !key_exists(&server_keys, key_id)); - if key_ids.is_empty() { - missing.remove(server); + /// Fetches verification keys from the origin servers directly, returning + /// any keys that could not be fetched. + async fn acquire_origins(&self, batch: I) -> Batch + where + I: Iterator)> + Send, + { + let timeout = Instant::now() + .checked_add(Duration::from_secs(45)) + .expect("timeout overflows"); + + let mut requests: FuturesUnordered<_> = batch + .map(|(origin, key_ids)| self.acquire_origin(origin, key_ids, timeout)) + .collect(); + + let mut missing = Batch::new(); + while let Some((origin, key_ids)) = requests.next().await { + if !key_ids.is_empty() { + missing.insert(origin, key_ids); + } + } + + missing + } + + /// Fetches verification keys from an origin directly, returning any keys + /// that were fetched. + async fn acquire_origin( + &self, + origin: OwnedServerName, + mut key_ids: Vec, + timeout: Instant, + ) -> (OwnedServerName, Vec) { + match timeout_at(timeout, self.server_request(&origin)).await { + | Err(e) => debug_warn!(%origin, "timed out: {e}"), + | Ok(Err(e)) => debug_error!(%origin, "{e}"), + | Ok(Ok(server_keys)) => { + trace!( + %origin, + ?key_ids, + ?server_keys, + "received server_keys" + ); + + self.add_signing_keys(server_keys.clone()).await; + key_ids.retain(|key_id| !key_exists(&server_keys, key_id)); + }, + } + + (origin, key_ids) + } + + /// Fetches signing keys for the given batch from the configured notary + /// servers, returning any keys that could not be fetched. + async fn acquire_notary(&self, batch: I) -> Batch + where + I: Iterator)> + Send, + { + let mut missing: Batch = batch.collect(); + for notary in self.services.globals.trusted_servers() { + let missing_keys = keys_count(&missing); + let missing_servers = missing.len(); + debug!( + "Asking notary {notary} for {missing_keys} missing keys from {missing_servers} \ + servers" + ); + + let batch = missing + .iter() + .map(|(server, keys)| (server.borrow(), keys.iter().map(Borrow::borrow))); + + match self.batch_notary_request(notary, batch).await { + | Err(e) => error!("Failed to contact notary {notary:?}: {e}"), + | Ok(results) => + for server_keys in results { + self.acquire_notary_result(&mut missing, server_keys).await; + }, + } + } + + missing + } + + /// Contacts a notary and fetches signing keys for the given batch, removing + /// keys from the missing batch when they are found. + async fn acquire_notary_result(&self, missing: &mut Batch, server_keys: ServerSigningKeys) { + let server = &server_keys.server_name; + self.add_signing_keys(server_keys.clone()).await; + + if let Some(key_ids) = missing.get_mut(server) { + key_ids.retain(|key_id| !key_exists(&server_keys, key_id)); + if key_ids.is_empty() { + missing.remove(server); + } } } } diff --git a/src/service/server_keys/get.rs b/src/service/server_keys/get.rs index 7d334eb98..a05340704 100644 --- a/src/service/server_keys/get.rs +++ b/src/service/server_keys/get.rs @@ -1,6 +1,6 @@ use std::borrow::Borrow; -use conduwuit::{Err, Result, debug_error, implement, trace}; +use conduwuit::{Err, Result, trace}; use ruma::{ CanonicalJsonObject, ServerName, ServerSigningKeyId, api::federation::discovery::VerifyKey, room_version_rules::RoomVersionRules, @@ -9,133 +9,138 @@ use ruma::{ use super::{PubKeyMap, PubKeys, extract_key}; use crate::server_keys::util::required_keys; -#[implement(super::Service)] -pub async fn get_event_keys( - &self, - object: &CanonicalJsonObject, - room_version_rules: &RoomVersionRules, -) -> Result { - let required = match required_keys(object, &room_version_rules.signatures) { - | Ok(required) => required, - | Err(e) => { - debug_error!("Failed to determine keys required to verify: {e}"); - return Err!(BadServerResponse("Failed to determine keys required to verify: {e}")); - }, - }; - trace!(?required, "Keys required to verify event"); +impl super::Service { + /// Fetches the verify keys required to verify the incoming event. + pub async fn get_event_keys( + &self, + object: &CanonicalJsonObject, + room_version_rules: &RoomVersionRules, + ) -> Result { + let required = match required_keys(object, &room_version_rules.signatures) { + | Ok(required) => required, + | Err(e) => { + return Err!(BadServerResponse( + "Failed to determine keys required to verify event: {e}" + )); + }, + }; - let batch = required - .iter() - .map(|(s, ids)| (s.borrow(), ids.iter().map(Borrow::borrow))); + let batch = required + .iter() + .map(|(s, ids)| (s.borrow(), ids.iter().map(Borrow::borrow))); - Ok(self.get_pubkeys(batch).await) -} - -#[implement(super::Service)] -pub async fn get_pubkeys<'a, S, K>(&self, batch: S) -> PubKeyMap -where - S: Iterator + Send, - K: Iterator + Send, -{ - let mut keys = PubKeyMap::new(); - for (server, key_ids) in batch { - let pubkeys = self.get_pubkeys_for(server, key_ids).await; - keys.insert(server.into(), pubkeys); + Ok(self.get_pubkeys(batch).await) } - keys -} - -#[implement(super::Service)] -pub async fn get_pubkeys_for<'a, I>(&self, origin: &ServerName, key_ids: I) -> PubKeys -where - I: Iterator + Send, -{ - let mut keys = PubKeys::new(); - for key_id in key_ids { - if let Ok(verify_key) = self.get_verify_key(origin, key_id).await { - keys.insert(key_id.into(), verify_key.key); + /// Fetches all verify keys for the given batch. Any keys which could not be + /// fetched are elided from the resulting map. + pub async fn get_pubkeys<'a, S, K>(&self, batch: S) -> PubKeyMap + where + S: Iterator + Send, + K: Iterator + Send, + { + let mut keys = PubKeyMap::new(); + for (server, key_ids) in batch { + let pubkeys = self.get_pubkeys_for(server, key_ids).await; + keys.insert(server.into(), pubkeys); } + + keys } - keys -} - -#[implement(super::Service)] -#[tracing::instrument(skip(self), level = "debug")] -pub async fn get_verify_key( - &self, - origin: &ServerName, - key_id: &ServerSigningKeyId, -) -> Result { - let notary_first = self.services.server.config.query_trusted_key_servers_first; - let notary_only = self.services.server.config.only_query_trusted_key_servers; - - if let Some(result) = self.verify_keys_for(origin).await.remove(key_id) { - trace!("Found key in cache"); - return Ok(result); - } - - if notary_first { - if let Ok(result) = self.get_verify_key_from_notaries(origin, key_id).await { - return Ok(result); - } - } - - if !notary_only { - if let Ok(result) = self.get_verify_key_from_origin(origin, key_id).await { - return Ok(result); - } - } - - if !notary_first { - if let Ok(result) = self.get_verify_key_from_notaries(origin, key_id).await { - return Ok(result); - } - } - - Err!(BadServerResponse(debug_error!( - %key_id, - %origin, - "Failed to fetch federation signing-key" - ))) -} - -#[implement(super::Service)] -async fn get_verify_key_from_notaries( - &self, - origin: &ServerName, - key_id: &ServerSigningKeyId, -) -> Result { - for notary in self.services.globals.trusted_servers() { - if let Ok(server_keys) = self.notary_request(notary, origin).await { - for server_key in server_keys.clone() { - self.add_signing_keys(server_key).await; + /// Gets verify keys for the given server name and a set of key IDs. Any + /// keys that cannot be found are elided from the resulting set. + pub async fn get_pubkeys_for<'a, I>(&self, origin: &ServerName, key_ids: I) -> PubKeys + where + I: Iterator + Send, + { + let mut keys = PubKeys::new(); + for key_id in key_ids { + if let Ok(verify_key) = self.get_verify_key(origin, key_id).await { + keys.insert(key_id.into(), verify_key.key); } + } - for server_key in server_keys { - if let Some(result) = extract_key(server_key, key_id) { - return Ok(result); + keys + } + + /// Fetches a single verify key via origin and notaries. + pub async fn get_verify_key( + &self, + origin: &ServerName, + key_id: &ServerSigningKeyId, + ) -> Result { + let notary_first = self.services.server.config.query_trusted_key_servers_first; + let notary_only = self.services.server.config.only_query_trusted_key_servers; + + if let Some(result) = self.verify_keys_for(origin).await.remove(key_id) { + trace!("Found key in cache"); + return Ok(result); + } + + if notary_first { + if let Ok(result) = self.get_verify_key_from_notaries(origin, key_id).await { + return Ok(result); + } + } + + if !notary_only { + if let Ok(result) = self.get_verify_key_from_origin(origin, key_id).await { + return Ok(result); + } + } + + if !notary_first { + if let Ok(result) = self.get_verify_key_from_notaries(origin, key_id).await { + return Ok(result); + } + } + + Err!(BadServerResponse(debug_error!( + %key_id, + %origin, + "Failed to fetch federation signing-key" + ))) + } + + /// Fetches verify keys from notary servers, adding them to the cache, and + /// returning the requested key if found. + async fn get_verify_key_from_notaries( + &self, + origin: &ServerName, + key_id: &ServerSigningKeyId, + ) -> Result { + for notary in self.services.globals.trusted_servers() { + if let Ok(server_keys) = self.notary_request(notary, origin).await { + for server_key in server_keys.clone() { + self.add_signing_keys(server_key).await; + } + + for server_key in server_keys { + if let Some(result) = extract_key(server_key, key_id) { + return Ok(result); + } } } } + + Err!(Request(NotFound("Failed to fetch signing-key from notaries"))) } - Err!(Request(NotFound("Failed to fetch signing-key from notaries"))) -} - -#[implement(super::Service)] -async fn get_verify_key_from_origin( - &self, - origin: &ServerName, - key_id: &ServerSigningKeyId, -) -> Result { - if let Ok(server_key) = self.server_request(origin).await { - self.add_signing_keys(server_key.clone()).await; - if let Some(result) = extract_key(server_key, key_id) { - return Ok(result); + /// Fetches verify keys from the origin server, adding them to the cache, + /// and returning the requested key if found. + async fn get_verify_key_from_origin( + &self, + origin: &ServerName, + key_id: &ServerSigningKeyId, + ) -> Result { + if let Ok(server_key) = self.server_request(origin).await { + self.add_signing_keys(server_key.clone()).await; + if let Some(result) = extract_key(server_key, key_id) { + return Ok(result); + } } - } - Err!(Request(NotFound("Failed to fetch signing-key from origin"))) + Err!(Request(NotFound("Failed to fetch signing-key from origin"))) + } } diff --git a/src/service/server_keys/mod.rs b/src/service/server_keys/mod.rs index 5df3e5193..b79b2b10d 100644 --- a/src/service/server_keys/mod.rs +++ b/src/service/server_keys/mod.rs @@ -9,7 +9,7 @@ mod verify; use std::{collections::BTreeMap, sync::Arc, time::Duration}; use conduwuit::{ - Result, Server, debug_error, debug_warn, implement, trace, + Result, Server, utils::{IterStream, timepoint_from_now}, }; use database::{Deserialized, Json, Map}; @@ -73,123 +73,124 @@ impl crate::Service for Service { fn name(&self) -> &str { crate::service::make_name(std::module_path!()) } } -#[implement(Service)] -#[inline] -pub fn keypair(&self) -> &Ed25519KeyPair { &self.keypair } +impl Service { + /// Fetches the server's active keypair. + pub fn keypair(&self) -> &Ed25519KeyPair { &self.keypair } -#[implement(Service)] -#[inline] -pub fn active_key_id(&self) -> &ServerSigningKeyId { self.active_verify_key().0 } + /// Fetches the server's active signing key ID. + pub fn active_key_id(&self) -> &ServerSigningKeyId { self.active_verify_key().0 } -#[implement(Service)] -#[inline] -pub fn active_verify_key(&self) -> (&ServerSigningKeyId, &VerifyKey) { - debug_assert!(self.verify_keys.len() <= 1, "more than one active verify_key"); - self.verify_keys - .iter() - .next() - .map(|(id, key)| (id.as_ref(), key)) - .expect("missing active verify_key") -} + /// Fetches the server's active signing key. Panics if there's more than 1 + /// active key. + pub fn active_verify_key(&self) -> (&ServerSigningKeyId, &VerifyKey) { + debug_assert!(self.verify_keys.len() <= 1, "more than one active verify_key"); + self.verify_keys + .iter() + .next() + .map(|(id, key)| (id.as_ref(), key)) + .expect("missing active verify_key") + } -#[implement(Service)] -async fn add_signing_keys(&self, new_keys: ServerSigningKeys) { - let origin = &new_keys.server_name; + /// Combines signing keys from the incoming response to the existing + /// database set. + async fn add_signing_keys(&self, new_keys: ServerSigningKeys) { + let origin = &new_keys.server_name; - // (timo) Not atomic, but this is not critical - let mut keys: ServerSigningKeys = self - .db - .server_signingkeys - .get(origin) - .await - .deserialized() - .unwrap_or_else(|_| { - // Just insert "now", it doesn't matter - ServerSigningKeys::new(origin.to_owned(), MilliSecondsSinceUnixEpoch::now()) - }); + // (timo) Not atomic, but this is not critical + let mut keys: ServerSigningKeys = self + .db + .server_signingkeys + .get(origin) + .await + .deserialized() + .unwrap_or_else(|_| { + // Just insert "now", it doesn't matter + ServerSigningKeys::new(origin.to_owned(), MilliSecondsSinceUnixEpoch::now()) + }); - keys.verify_keys.extend(new_keys.verify_keys); - keys.old_verify_keys.extend(new_keys.old_verify_keys); - self.db.server_signingkeys.raw_put(origin, Json(&keys)); -} + keys.verify_keys.extend(new_keys.verify_keys); + keys.old_verify_keys.extend(new_keys.old_verify_keys); + self.db.server_signingkeys.raw_put(origin, Json(&keys)); + } -#[implement(Service)] -#[tracing::instrument(skip(self, object), level = "debug")] -pub async fn required_keys_exist( - &self, - object: &CanonicalJsonObject, - room_version_rules: &RoomVersionRules, -) -> bool { - trace!(?object, "Checking required keys exist"); - let Ok(required_keys) = required_keys(object, &room_version_rules.signatures) else { - debug_error!("Failed to determine required keys"); - return false; - }; - trace!(?required_keys, "Required keys to verify event"); - required_keys - .iter() - .flat_map(|(server, key_ids)| key_ids.iter().map(move |key_id| (server, key_id))) - .stream() - .all(|(server, key_id)| self.verify_key_exists(server, key_id)) - .await -} + /// Checks if the keys required to verify an incoming PDU are already known + /// locally. + pub async fn required_keys_exist( + &self, + object: &CanonicalJsonObject, + room_version_rules: &RoomVersionRules, + ) -> bool { + let Ok(required_keys) = required_keys(object, &room_version_rules.signatures) else { + return false; + }; + required_keys + .iter() + .flat_map(|(server, key_ids)| key_ids.iter().map(move |key_id| (server, key_id))) + .stream() + .all(|(server, key_id)| self.verify_key_exists(server, key_id)) + .await + } -#[implement(Service)] -#[tracing::instrument(skip(self), level = "debug")] -pub async fn verify_key_exists(&self, origin: &ServerName, key_id: &ServerSigningKeyId) -> bool { - type KeysMap = BTreeMap>; + /// Checks if a single verify key belonging to the target server is already + /// known locally. + pub async fn verify_key_exists( + &self, + origin: &ServerName, + key_id: &ServerSigningKeyId, + ) -> bool { + type KeysMap = BTreeMap>; - let Ok(keys) = self - .db - .server_signingkeys - .get(origin) - .await - .deserialized::>() - else { - debug_warn!("No known signing keys found for {origin}"); - return false; - }; + let Ok(keys) = self + .db + .server_signingkeys + .get(origin) + .await + .deserialized::>() + else { + return false; + }; - if let Ok(Some(verify_keys)) = keys.get_field::("verify_keys") { - if verify_keys.contains_key(key_id) { - return true; + if let Ok(Some(verify_keys)) = keys.get_field::("verify_keys") { + if verify_keys.contains_key(key_id) { + return true; + } } - } - if let Ok(Some(old_verify_keys)) = keys.get_field::("old_verify_keys") { - if old_verify_keys.contains_key(key_id) { - return true; + if let Ok(Some(old_verify_keys)) = keys.get_field::("old_verify_keys") { + if old_verify_keys.contains_key(key_id) { + return true; + } } + + false } - debug_warn!("Key {key_id} not found for {origin}"); - false -} + /// Returns all verify keys for the origin. + pub async fn verify_keys_for(&self, origin: &ServerName) -> VerifyKeys { + let mut keys = self + .signing_keys_for(origin) + .await + .map_or(BTreeMap::new(), |keys| merge_old_keys(keys).verify_keys); -#[implement(Service)] -pub async fn verify_keys_for(&self, origin: &ServerName) -> VerifyKeys { - let mut keys = self - .signing_keys_for(origin) - .await - .map_or(BTreeMap::new(), |keys| merge_old_keys(keys).verify_keys); + if self.services.globals.server_is_ours(origin) { + keys.extend(self.verify_keys.clone()); + } - if self.services.globals.server_is_ours(origin) { - keys.extend(self.verify_keys.clone()); + keys } - keys -} + /// Returns the stored server signing keys response for the origin. + pub async fn signing_keys_for(&self, origin: &ServerName) -> Result { + self.db.server_signingkeys.get(origin).await.deserialized() + } -#[implement(Service)] -pub async fn signing_keys_for(&self, origin: &ServerName) -> Result { - self.db.server_signingkeys.get(origin).await.deserialized() -} - -#[implement(Service)] -fn minimum_valid_ts(&self) -> MilliSecondsSinceUnixEpoch { - let timepoint = - timepoint_from_now(self.minimum_valid).expect("SystemTime should not overflow"); - MilliSecondsSinceUnixEpoch::from_system_time(timepoint).expect("UInt should not overflow") + /// Returns the minimum_valid_ts for a key response from the current point + /// in time, plus the minimum valid time. + fn minimum_valid_ts(&self) -> MilliSecondsSinceUnixEpoch { + let timepoint = + timepoint_from_now(self.minimum_valid).expect("SystemTime should not overflow"); + MilliSecondsSinceUnixEpoch::from_system_time(timepoint).expect("UInt should not overflow") + } } fn merge_old_keys(mut keys: ServerSigningKeys) -> ServerSigningKeys { diff --git a/src/service/server_keys/request.rs b/src/service/server_keys/request.rs index c8823fdc7..043e894a9 100644 --- a/src/service/server_keys/request.rs +++ b/src/service/server_keys/request.rs @@ -1,6 +1,7 @@ use std::{collections::BTreeMap, fmt::Debug}; -use conduwuit::{Err, Result, debug, implement}; +use conduwuit::{Err, Result, debug}; +use get_remote_server_keys_batch::v2::Request; use ruma::{ OwnedServerName, OwnedServerSigningKeyId, ServerName, ServerSigningKeyId, api::federation::discovery::{ @@ -9,51 +10,79 @@ use ruma::{ get_server_keys, }, }; +type RumaBatch = BTreeMap>; -#[implement(super::Service)] -pub(super) async fn batch_notary_request<'a, S, K>( - &self, - notary: &ServerName, - batch: S, -) -> Result> -where - S: Iterator + Send, - K: Iterator + Send, -{ - use get_remote_server_keys_batch::v2::Request; - type RumaBatch = BTreeMap>; - - let mut criteria = QueryCriteria::new(); - criteria.minimum_valid_until_ts = Some(self.minimum_valid_ts()); - - let mut server_keys = batch.fold(RumaBatch::new(), |mut batch, (server, key_ids)| { - batch - .entry(server.into()) - .or_default() - .extend(key_ids.map(|key_id| (key_id.into(), criteria.clone()))); - - batch - }); - - debug_assert!(!server_keys.is_empty(), "empty batch request to notary"); - - let mut results = Vec::new(); - while let Some(batch) = server_keys - .keys() - .rev() - .take(self.services.server.config.trusted_server_batch_size) - .next_back() - .cloned() +impl super::Service { + /// Handles a batched notary request. + /// If the batch is empty, an empty vec is returned. + pub(super) async fn batch_notary_request<'a, S, K>( + &self, + notary: &ServerName, + batch: S, + ) -> Result> + where + S: Iterator + Send, + K: Iterator + Send, { - let request = Request::new(server_keys.split_off(&batch)); + let mut criteria = QueryCriteria::new(); + criteria.minimum_valid_until_ts = Some(self.minimum_valid_ts()); - debug!( - ?notary, - ?batch, - remaining = %server_keys.len(), - requesting = ?request.server_keys.keys(), - "notary request" - ); + let mut server_keys = batch.fold(RumaBatch::new(), |mut batch, (server, key_ids)| { + batch + .entry(server.into()) + .or_default() + .extend(key_ids.map(|key_id| (key_id.into(), criteria.clone()))); + + batch + }); + + if server_keys.is_empty() { + return Ok(vec![]); + } + + let mut results = Vec::new(); + while let Some(batch) = server_keys + .keys() + .rev() + .take(self.services.server.config.trusted_server_batch_size) + .next_back() + .cloned() + { + let request = Request::new(server_keys.split_off(&batch)); + + debug!( + ?notary, + ?batch, + remaining = %server_keys.len(), + requesting = ?request.server_keys.keys(), + "notary request" + ); + + let response = self + .services + .sending + .send_unauthenticated_request(notary, request) + .await? + .server_keys + .into_iter() + .map(|key| key.deserialize()) + .filter_map(Result::ok); + + results.extend(response); + } + + Ok(results) + } + + /// Makes a request to a notary server to fetch signing keys. + pub async fn notary_request( + &self, + notary: &ServerName, + target: &ServerName, + ) -> Result + Clone + Debug + Send + use<>> { + use get_remote_server_keys::v2::Request; + + let request = Request::new(target.into(), self.minimum_valid_ts()); let response = self .services @@ -65,54 +94,29 @@ where .map(|key| key.deserialize()) .filter_map(Result::ok); - results.extend(response); + Ok(response) } - Ok(results) -} + /// Makes a request to an origin server to fetch signing keys. + pub async fn server_request(&self, target: &ServerName) -> Result { + use get_server_keys::v2::Request; -#[implement(super::Service)] -pub async fn notary_request( - &self, - notary: &ServerName, - target: &ServerName, -) -> Result + Clone + Debug + Send + use<>> { - use get_remote_server_keys::v2::Request; + let server_signing_key = self + .services + .sending + .send_unauthenticated_request(target, Request::new()) + .await + .map(|response| response.server_key) + .and_then(|key| key.deserialize().map_err(Into::into))?; - let request = Request::new(target.into(), self.minimum_valid_ts()); + if server_signing_key.server_name != target { + return Err!(BadServerResponse(debug_warn!( + requested = ?target, + response = ?server_signing_key.server_name, + "Server responded with bogus server_name" + ))); + } - let response = self - .services - .sending - .send_unauthenticated_request(notary, request) - .await? - .server_keys - .into_iter() - .map(|key| key.deserialize()) - .filter_map(Result::ok); - - Ok(response) -} - -#[implement(super::Service)] -pub async fn server_request(&self, target: &ServerName) -> Result { - use get_server_keys::v2::Request; - - let server_signing_key = self - .services - .sending - .send_unauthenticated_request(target, Request::new()) - .await - .map(|response| response.server_key) - .and_then(|key| key.deserialize().map_err(Into::into))?; - - if server_signing_key.server_name != target { - return Err!(BadServerResponse(debug_warn!( - requested = ?target, - response = ?server_signing_key.server_name, - "Server responded with bogus server_name" - ))); + Ok(server_signing_key) } - - Ok(server_signing_key) } diff --git a/src/service/server_keys/sign.rs b/src/service/server_keys/sign.rs index c28fcd26b..405c955e2 100644 --- a/src/service/server_keys/sign.rs +++ b/src/service/server_keys/sign.rs @@ -1,23 +1,29 @@ -use conduwuit::{Result, implement}; -use ruma::{CanonicalJsonObject, room_version_rules::RoomVersionRules}; +use conduwuit::Result; +use ruma::{ + CanonicalJsonObject, room_version_rules::RoomVersionRules, signatures::hash_and_sign_event, +}; -#[implement(super::Service)] -pub fn sign_json(&self, object: &mut CanonicalJsonObject) -> Result { - use ruma::signatures::sign_json; +impl super::Service { + /// Signs an arbitrary JSON object. The object is mutated. + pub fn sign_json(&self, object: &mut CanonicalJsonObject) -> Result { + use ruma::signatures::sign_json; - let server_name = self.services.globals.server_name().as_str(); - sign_json(server_name, self.keypair(), object).map_err(Into::into) -} + let server_name = self.services.globals.server_name().as_str(); + sign_json(server_name, self.keypair(), object).map_err(Into::into) + } -#[implement(super::Service)] -pub fn hash_and_sign_event( - &self, - object: &mut CanonicalJsonObject, - room_version_rules: &RoomVersionRules, -) -> Result { - use ruma::signatures::hash_and_sign_event; - - let server_name = self.services.globals.server_name().as_str(); - hash_and_sign_event(server_name, self.keypair(), object, &room_version_rules.redaction) + /// Hashes and signs an event JSON object. The object is mutated. + pub fn hash_and_sign_event( + &self, + object: &mut CanonicalJsonObject, + room_version_rules: &RoomVersionRules, + ) -> Result { + hash_and_sign_event( + self.services.globals.server_name().as_str(), + self.keypair(), + object, + &room_version_rules.redaction, + ) .map_err(Into::into) + } } diff --git a/src/service/server_keys/verify.rs b/src/service/server_keys/verify.rs index a109b0ff5..8d28e10df 100644 --- a/src/service/server_keys/verify.rs +++ b/src/service/server_keys/verify.rs @@ -1,77 +1,73 @@ -use conduwuit::{ - Err, Result, debug_warn, implement, matrix::event::gen_event_id_canonical_json, trace, -}; +use conduwuit::{Err, Result, matrix::event::gen_event_id_canonical_json}; use ruma::{ CanonicalJsonObject, CanonicalJsonValue, OwnedEventId, room_version_rules::RoomVersionRules, signatures::Verified, }; use serde_json::value::RawValue as RawJsonValue; -#[implement(super::Service)] -pub async fn validate_and_add_event_id( - &self, - pdu: &RawJsonValue, - room_version_rules: &RoomVersionRules, -) -> Result<(OwnedEventId, CanonicalJsonObject)> { - let (event_id, mut value) = gen_event_id_canonical_json(pdu, room_version_rules)?; - if let Err(e) = self.verify_event(&value, room_version_rules).await { - return Err!(BadServerResponse(debug_error!( - "Event {event_id} failed verification: {e:?}" - ))); +impl super::Service { + /// Validates the incoming event, and then inserts the calculated event ID + /// into the event_id field. + pub async fn validate_and_add_event_id( + &self, + pdu: &RawJsonValue, + room_version_rules: &RoomVersionRules, + ) -> Result<(OwnedEventId, CanonicalJsonObject)> { + let (event_id, mut value) = gen_event_id_canonical_json(pdu, room_version_rules)?; + if let Err(e) = self.verify_event(&value, room_version_rules).await { + return Err!(BadServerResponse(debug_error!( + "Event {event_id} failed verification: {e:?}" + ))); + } + + value.insert("event_id".into(), CanonicalJsonValue::String(event_id.as_str().into())); + + Ok((event_id, value)) } - value.insert("event_id".into(), CanonicalJsonValue::String(event_id.as_str().into())); + /// Validates the incoming event only using locally known verification keys. + /// Inserts the calculated event ID into the event_id field upon success. + pub async fn validate_and_add_event_id_no_fetch( + &self, + pdu: &RawJsonValue, + room_version_rules: &RoomVersionRules, + ) -> Result<(OwnedEventId, CanonicalJsonObject)> { + let (event_id, mut value) = gen_event_id_canonical_json(pdu, room_version_rules)?; + if !self.required_keys_exist(&value, room_version_rules).await { + return Err!(BadServerResponse(debug_warn!( + "Event {event_id} cannot be verified: missing keys." + ))); + } + if let Err(e) = self.verify_event(&value, room_version_rules).await { + return Err!(BadServerResponse(debug_error!( + "Event {event_id} failed verification: {e:?}" + ))); + } - Ok((event_id, value)) -} + value.insert("event_id".into(), CanonicalJsonValue::String(event_id.as_str().into())); -#[implement(super::Service)] -pub async fn validate_and_add_event_id_no_fetch( - &self, - pdu: &RawJsonValue, - room_version_rules: &RoomVersionRules, -) -> Result<(OwnedEventId, CanonicalJsonObject)> { - trace!(?pdu, "Validating PDU without fetching keys"); - let (event_id, mut value) = gen_event_id_canonical_json(pdu, room_version_rules)?; - trace!(event_id = event_id.as_str(), "Generated event ID, checking required keys"); - if !self.required_keys_exist(&value, room_version_rules).await { - debug_warn!( - "Event {event_id} is missing required keys, cannot verify without fetching keys" - ); - return Err!(BadServerResponse(debug_warn!( - "Event {event_id} cannot be verified: missing keys." - ))); + Ok((event_id, value)) } - trace!("All required keys exist, verifying event"); - if let Err(e) = self.verify_event(&value, room_version_rules).await { - debug_warn!("Event verification failed"); - return Err!(BadServerResponse(debug_error!( - "Event {event_id} failed verification: {e:?}" - ))); + + /// Verifies the incoming event after fetching the keys required to do so. + pub async fn verify_event( + &self, + event: &CanonicalJsonObject, + room_version_rules: &RoomVersionRules, + ) -> Result { + let keys = self.get_event_keys(event, room_version_rules).await?; + ruma::signatures::verify_event(&keys, event, room_version_rules).map_err(Into::into) } - trace!("Event verified successfully"); - value.insert("event_id".into(), CanonicalJsonValue::String(event_id.as_str().into())); - - Ok((event_id, value)) -} - -#[implement(super::Service)] -pub async fn verify_event( - &self, - event: &CanonicalJsonObject, - room_version_rules: &RoomVersionRules, -) -> Result { - let keys = self.get_event_keys(event, room_version_rules).await?; - ruma::signatures::verify_event(&keys, event, room_version_rules).map_err(Into::into) -} - -#[implement(super::Service)] -pub async fn verify_json( - &self, - event: &CanonicalJsonObject, - room_version_rules: &RoomVersionRules, -) -> Result { - let keys = self.get_event_keys(event, room_version_rules).await?; - ruma::signatures::verify_json(&keys, event).map_err(Into::into) + /// Verifies an arbitrary JSON object after fetching the keys required to do + /// so. + pub async fn verify_json( + &self, + event: &CanonicalJsonObject, + room_version_rules: &RoomVersionRules, + ) -> Result { + // TODO: is this actually used anywhere? + let keys = self.get_event_keys(event, room_version_rules).await?; + ruma::signatures::verify_json(&keys, event).map_err(Into::into) + } } From aa2f506996840bd9b667709f2c44807d9bc62ea2 Mon Sep 17 00:00:00 2001 From: timedout Date: Fri, 26 Jun 2026 01:20:08 +0100 Subject: [PATCH 32/70] style: Document & refactor sending service --- src/service/sending/appservice.rs | 192 +++++++++++++++--------------- src/service/sending/dest.rs | 75 ++++++------ 2 files changed, 136 insertions(+), 131 deletions(-) diff --git a/src/service/sending/appservice.rs b/src/service/sending/appservice.rs index a9bd50d43..e7242af42 100644 --- a/src/service/sending/appservice.rs +++ b/src/service/sending/appservice.rs @@ -2,7 +2,7 @@ use std::{fmt::Debug, mem}; use bytes::BytesMut; use conduwuit::{ - Err, Result, debug_error, err, implement, trace, utils, utils::response::LimitReadExt, warn, + Err, Result, debug_error, err, trace, utils, utils::response::LimitReadExt, warn, }; use ruma::api::{ IncomingResponse, OutgoingRequest, @@ -11,99 +11,103 @@ use ruma::api::{ path_builder::SinglePath, }; -/// Sends a request to an appservice -/// -/// Only returns Ok(None) if there is no url specified in the appservice -/// registration file -#[implement(super::Service)] -pub async fn send_appservice_request( - &self, - registration: Registration, - request: T, -) -> Result> -where - T: OutgoingRequest + Debug + Send, -{ - let Some(dest) = registration.url else { - return Ok(None); - }; +impl super::Service { + /// Sends a request to an appservice + /// + /// Only returns Ok(None) if there is no url specified in the appservice + /// registration file + pub async fn send_appservice_request( + &self, + registration: Registration, + request: T, + ) -> Result> + where + T: OutgoingRequest + Debug + Send, + { + let Some(dest) = registration.url else { + return Ok(None); + }; - if dest == *"null" || dest.is_empty() { - return Ok(None); + if dest == *"null" || dest.is_empty() { + return Ok(None); + } + + trace!("Appservice URL \"{dest}\", Appservice ID: {}", registration.id); + + let hs_token = registration.hs_token.as_str(); + let mut http_request = request + .try_into_http_request::(&dest, SendAccessToken::Appservice(hs_token), ()) + .map_err(|e| { + err!(BadServerResponse( + warn!(appservice = %registration.id, "Failed to find destination {dest}: {e:?}") + )) + })? + .map(BytesMut::freeze); + + let mut parts = http_request.uri().clone().into_parts(); + let old_path_and_query = parts.path_and_query.unwrap().as_str().to_owned(); + let symbol = if old_path_and_query.contains('?') { "&" } else { "?" }; + + parts.path_and_query = Some( + (old_path_and_query + symbol + "access_token=" + hs_token) + .parse() + .unwrap(), + ); + *http_request.uri_mut() = parts.try_into().expect("our manipulation is always valid"); + + let reqwest_request = reqwest::Request::try_from(http_request)?; + + let client = &self.services.client.appservice; + + let mut response = client.execute(reqwest_request).await.map_err(|e| { + warn!( + "Could not send request to appservice \"{}\" at {dest}: {e:?}", + registration.id + ); + e + })?; + + // reqwest::Response -> http::Response conversion + let status = response.status(); + let mut http_response_builder = http::Response::builder() + .status(status) + .version(response.version()); + mem::swap( + response.headers_mut(), + http_response_builder + .headers_mut() + .expect("http::response::Builder is usable"), + ); + + let body = response + .limit_read( + self.server + .config + .max_request_size + .try_into() + .expect("usize fits into u64"), + ) + .await?; + + if !status.is_success() { + debug_error!("Appservice response bytes: {:?}", utils::string_from_bytes(&body)); + return Err!(BadServerResponse(warn!( + "Appservice \"{}\" returned unsuccessful HTTP response {status} at {dest}", + registration.id + ))); + } + + let response = T::IncomingResponse::try_from_http_response( + http_response_builder + .body(body) + .expect("reqwest body is valid http body"), + ); + + response.map(Some).map_err(|e| { + err!(BadServerResponse(warn!( + "Appservice \"{}\" returned invalid/malformed response bytes {dest}: {e}", + registration.id + ))) + }) } - - trace!("Appservice URL \"{dest}\", Appservice ID: {}", registration.id); - - let hs_token = registration.hs_token.as_str(); - let mut http_request = request - .try_into_http_request::(&dest, SendAccessToken::Appservice(hs_token), ()) - .map_err(|e| { - err!(BadServerResponse( - warn!(appservice = %registration.id, "Failed to find destination {dest}: {e:?}") - )) - })? - .map(BytesMut::freeze); - - let mut parts = http_request.uri().clone().into_parts(); - let old_path_and_query = parts.path_and_query.unwrap().as_str().to_owned(); - let symbol = if old_path_and_query.contains('?') { "&" } else { "?" }; - - parts.path_and_query = Some( - (old_path_and_query + symbol + "access_token=" + hs_token) - .parse() - .unwrap(), - ); - *http_request.uri_mut() = parts.try_into().expect("our manipulation is always valid"); - - let reqwest_request = reqwest::Request::try_from(http_request)?; - - let client = &self.services.client.appservice; - - let mut response = client.execute(reqwest_request).await.map_err(|e| { - warn!("Could not send request to appservice \"{}\" at {dest}: {e:?}", registration.id); - e - })?; - - // reqwest::Response -> http::Response conversion - let status = response.status(); - let mut http_response_builder = http::Response::builder() - .status(status) - .version(response.version()); - mem::swap( - response.headers_mut(), - http_response_builder - .headers_mut() - .expect("http::response::Builder is usable"), - ); - - let body = response - .limit_read( - self.server - .config - .max_request_size - .try_into() - .expect("usize fits into u64"), - ) - .await?; - - if !status.is_success() { - debug_error!("Appservice response bytes: {:?}", utils::string_from_bytes(&body)); - return Err!(BadServerResponse(warn!( - "Appservice \"{}\" returned unsuccessful HTTP response {status} at {dest}", - registration.id - ))); - } - - let response = T::IncomingResponse::try_from_http_response( - http_response_builder - .body(body) - .expect("reqwest body is valid http body"), - ); - - response.map(Some).map_err(|e| { - err!(BadServerResponse(warn!( - "Appservice \"{}\" returned invalid/malformed response bytes {dest}: {e}", - registration.id - ))) - }) } diff --git a/src/service/sending/dest.rs b/src/service/sending/dest.rs index 4099d3722..878b0fc41 100644 --- a/src/service/sending/dest.rs +++ b/src/service/sending/dest.rs @@ -1,6 +1,5 @@ use std::fmt::Debug; -use conduwuit::implement; use ruma::{OwnedServerName, OwnedUserId}; #[derive(Clone, Debug, PartialEq, Eq, Hash)] @@ -10,44 +9,46 @@ pub enum Destination { Federation(OwnedServerName), } -#[implement(Destination)] -#[must_use] -pub(super) fn get_prefix(&self) -> Vec { - match self { - | Self::Federation(server) => { - let len = server.as_bytes().len().saturating_add(1); +impl Destination { + /// Gets the prefix for this destination. + #[must_use] + pub(super) fn get_prefix(&self) -> Vec { + match self { + | Self::Federation(server) => { + let len = server.as_bytes().len().saturating_add(1); - let mut p = Vec::with_capacity(len); - p.extend_from_slice(server.as_bytes()); - p.push(0xFF); - p - }, - | Self::Appservice(server) => { - let sigil = b"+"; - let len = sigil.len().saturating_add(server.len()).saturating_add(1); + let mut p = Vec::with_capacity(len); + p.extend_from_slice(server.as_bytes()); + p.push(0xFF); + p + }, + | Self::Appservice(server) => { + let sigil = b"+"; + let len = sigil.len().saturating_add(server.len()).saturating_add(1); - let mut p = Vec::with_capacity(len); - p.extend_from_slice(sigil); - p.extend_from_slice(server.as_bytes()); - p.push(0xFF); - p - }, - | Self::Push(user, pushkey) => { - let sigil = b"$"; - let len = sigil - .len() - .saturating_add(user.as_bytes().len()) - .saturating_add(1) - .saturating_add(pushkey.len()) - .saturating_add(1); + let mut p = Vec::with_capacity(len); + p.extend_from_slice(sigil); + p.extend_from_slice(server.as_bytes()); + p.push(0xFF); + p + }, + | Self::Push(user, pushkey) => { + let sigil = b"$"; + let len = sigil + .len() + .saturating_add(user.as_bytes().len()) + .saturating_add(1) + .saturating_add(pushkey.len()) + .saturating_add(1); - let mut p = Vec::with_capacity(len); - p.extend_from_slice(sigil); - p.extend_from_slice(user.as_bytes()); - p.push(0xFF); - p.extend_from_slice(pushkey.as_bytes()); - p.push(0xFF); - p - }, + let mut p = Vec::with_capacity(len); + p.extend_from_slice(sigil); + p.extend_from_slice(user.as_bytes()); + p.push(0xFF); + p.extend_from_slice(pushkey.as_bytes()); + p.push(0xFF); + p + }, + } } } From 199f6eb67b29c3af5ce2f0d61cf0f4420422c1ca Mon Sep 17 00:00:00 2001 From: timedout Date: Fri, 26 Jun 2026 03:06:03 +0100 Subject: [PATCH 33/70] style: Document & refactor rooms/user and rooms/timeline services --- src/service/rooms/membership/mod.rs | 3 +- src/service/rooms/timeline/append.rs | 701 ++++++++++++++----------- src/service/rooms/timeline/backfill.rs | 508 +++++++++--------- src/service/rooms/timeline/build.rs | 466 ++++++++-------- src/service/rooms/timeline/create.rs | 528 ++++++++++--------- src/service/rooms/timeline/redact.rs | 83 +-- src/service/rooms/user/mod.rs | 85 +-- 7 files changed, 1241 insertions(+), 1133 deletions(-) diff --git a/src/service/rooms/membership/mod.rs b/src/service/rooms/membership/mod.rs index ff69e095e..0cbad3509 100644 --- a/src/service/rooms/membership/mod.rs +++ b/src/service/rooms/membership/mod.rs @@ -282,8 +282,7 @@ impl Service { remote_servers = %servers.len(), "Could not join room locally, attempting remote join", ); - self.join_remote_room(sender_user, room_id, reason, servers, state_lock) - .await + Box::pin(self.join_remote_room(sender_user, room_id, reason, servers, state_lock)).await } #[tracing::instrument(skip_all, fields(%sender_user, %room_id), name = "join_remote_room", level = "info")] diff --git a/src/service/rooms/timeline/append.rs b/src/service/rooms/timeline/append.rs index 8f999a737..76e19f10f 100644 --- a/src/service/rooms/timeline/append.rs +++ b/src/service/rooms/timeline/append.rs @@ -3,16 +3,22 @@ use std::{ sync::Arc, }; -use conduwuit::trace; +use conduwuit::{ + debug_warn, + pdu::{Count, ShortRoomId}, + trace, + utils::{IterStream, TryFutureExtExt, stream::BroadbandExt}, + warn, +}; use conduwuit_core::{ - Result, err, error, implement, + Result, err, error, matrix::{ event::Event, pdu::{PduCount, PduEvent, PduId, RawPduId}, }, utils::{self, ReadyExt}, }; -use futures::StreamExt; +use futures::{StreamExt, TryFutureExt}; use ruma::{ CanonicalJsonObject, CanonicalJsonValue, EventId, RoomVersionId, UserId, events::{ @@ -24,206 +30,245 @@ use ruma::{ }; use super::{ExtractBody, ExtractRelatesTo, ExtractRelatesToEventId, RoomMutexGuard}; -use crate::{appservice::NamespaceRegex, rooms::state_compressor::CompressedState}; +use crate::{appservice::RegistrationInfo, rooms::state_compressor::CompressedState}; -/// Append the incoming event setting the state snapshot to the state from -/// the server that sent the event. -#[implement(super::Service)] -#[tracing::instrument(level = "debug", skip_all)] -#[allow(clippy::too_many_arguments)] -pub async fn append_incoming_pdu<'a, Leaves>( - &'a self, - pdu: &'a PduEvent, - pdu_json: CanonicalJsonObject, - new_room_leaves: Leaves, - state_ids_compressed: Arc, - soft_fail: bool, - state_lock: &'a RoomMutexGuard, - room_id: &'a ruma::RoomId, -) -> Result> -where - Leaves: Iterator + Send + 'a, -{ - // We append to state before appending the pdu, so we don't have a moment in - // time with the pdu without it's state. This is okay because append_pdu can't - // fail. - self.services - .state - .set_event_state(&pdu.event_id, room_id, state_ids_compressed) - .await?; +impl super::Service { + /// Append the incoming event setting the state snapshot to the state from + /// the server that sent the event. + #[allow(clippy::too_many_arguments)] + pub async fn append_incoming_pdu<'a, Leaves>( + &'a self, + pdu: &'a PduEvent, + pdu_json: CanonicalJsonObject, + new_room_leaves: Leaves, + state_ids_compressed: Arc, + soft_fail: bool, + state_lock: &'a RoomMutexGuard, + room_id: &'a ruma::RoomId, + ) -> Result> + where + Leaves: Iterator + Send + 'a, + { + // We append to state before appending the pdu, so we don't have a moment in + // time with the pdu without it's state. This is okay because append_pdu can't + // fail. + self.services + .state + .set_event_state(&pdu.event_id, room_id, state_ids_compressed) + .await?; - if soft_fail { - // Nothing else to do with a soft-failed event. - return Ok(None); - } - - let pdu_id = self - .append_pdu(pdu, pdu_json, new_room_leaves, state_lock, room_id) - .await?; - - // Process admin commands for federation events - if *pdu.kind() == TimelineEventType::RoomMessage { - let content: ExtractBody = pdu.get_content()?; - if let Some(body) = content.body { - if let Some(source) = self - .services - .admin - .is_admin_command(pdu, &body, false) - .await - { - self.services.admin.command_with_sender( - body, - Some(pdu.event_id().into()), - source, - pdu.sender.clone(), - )?; - } + if soft_fail { + // Nothing else to do with a soft-failed event. + return Ok(None); } - } - Ok(Some(pdu_id)) -} + let pdu_id = self + .append_pdu(pdu, pdu_json, new_room_leaves, state_lock, room_id) + .await?; -/// Creates a new persisted data unit and adds it to a room. -/// -/// By this point the incoming event should be fully authenticated, no auth -/// happens in `append_pdu`. -/// -/// Returns pdu id -#[implement(super::Service)] -#[tracing::instrument(level = "debug", skip_all)] -pub async fn append_pdu<'a, Leaves>( - &'a self, - pdu: &'a PduEvent, - mut pdu_json: CanonicalJsonObject, - leaves: Leaves, - state_lock: &'a RoomMutexGuard, - room_id: &'a ruma::RoomId, -) -> Result -where - Leaves: Iterator + Send + 'a, -{ - // Coalesce database writes for the remainder of this scope. - let _cork = self.db.db.cork_and_flush(); - - let shortroomid = self - .services - .short - .get_shortroomid(room_id) - .await - .map_err(|_| err!(Database("Room does not exist")))?; - - // Make unsigned fields correct. This is not properly documented in the spec, - // but state events need to have previous content in the unsigned field, so - // clients can easily interpret things like membership changes - if let Some(state_key) = pdu.state_key() { - if let CanonicalJsonValue::Object(unsigned) = pdu_json - .entry("unsigned".to_owned()) - .or_insert_with(|| CanonicalJsonValue::Object(BTreeMap::default())) - { - if let Ok(shortstatehash) = self - .services - .state_accessor - .pdu_shortstatehash(pdu.event_id()) - .await - { - if let Ok(prev_state) = self + // Process admin commands for federation events + if *pdu.kind() == TimelineEventType::RoomMessage { + let content: ExtractBody = pdu.get_content()?; + if let Some(body) = content.body { + if let Some(source) = self .services - .state_accessor - .state_get(shortstatehash, &pdu.kind().to_string().into(), state_key) + .admin + .is_admin_command(pdu, &body, false) .await { - unsigned.insert( - "prev_content".to_owned(), - CanonicalJsonValue::Object( - utils::to_canonical_object(prev_state.get_content_as_value()) - .map_err(|e| { - err!(Database(error!( - "Failed to convert prev_state to canonical JSON: {e}", - ))) - })?, - ), - ); - unsigned.insert( - String::from("prev_sender"), - CanonicalJsonValue::String(prev_state.sender().to_string()), - ); - unsigned.insert( - String::from("replaces_state"), - CanonicalJsonValue::String(prev_state.event_id().to_string()), - ); + self.services.admin.command_with_sender( + body, + Some(pdu.event_id().into()), + source, + pdu.sender.clone(), + )?; } } - } else { - error!("Invalid unsigned type in pdu."); } + + Ok(Some(pdu_id)) } - // We must keep track of all events that have been referenced. - self.services - .pdu_metadata - .mark_as_referenced(room_id, pdu.prev_events().map(AsRef::as_ref)); + /// Populates the unsigned data of a PDU + async fn populate_unsigned(&self, pdu: &PduEvent, pdu_json: &mut CanonicalJsonObject) { + let Some(state_key) = pdu.state_key() else { + return; // Non-state events can't have replaced state + }; - trace!("setting forward extremities"); - self.services - .state - .set_forward_extremities(room_id, leaves, state_lock) - .await; + let CanonicalJsonValue::Object(unsigned) = pdu_json + .entry("unsigned".into()) + .or_insert_with(|| CanonicalJsonValue::Object(BTreeMap::new())) + else { + return; // This shouldn't be reachable, really. + }; - let insert_lock = self.mutex_insert.lock(room_id).await; + let Ok(shortstatehash) = self + .services + .state_accessor + .pdu_shortstatehash(pdu.event_id()) + .await + else { + return; + }; + let Ok(prev_state) = self + .services + .state_accessor + .state_get(shortstatehash, &pdu.kind().to_string().into(), state_key) + .await + else { + return; + }; + unsigned.insert( + "prev_content".to_owned(), + CanonicalJsonValue::Object( + utils::to_canonical_object(prev_state.get_content_as_value()) + .expect("Failed to convert prev_content into canonical JSON object"), + ), + ); + unsigned.insert( + String::from("prev_sender"), + CanonicalJsonValue::String(prev_state.sender().to_string()), + ); + unsigned.insert( + String::from("replaces_state"), + CanonicalJsonValue::String(prev_state.event_id().to_string()), + ); + } - let count1 = self.services.globals.next_count().unwrap(); + /// Creates a new persisted data unit and adds it to a room. + /// + /// By this point the incoming event should be fully authenticated, no auth + /// happens in `append_pdu`. + /// + /// Returns pdu id + pub async fn append_pdu<'a, Leaves>( + &'a self, + pdu: &'a PduEvent, + mut pdu_json: CanonicalJsonObject, + leaves: Leaves, + state_lock: &'a RoomMutexGuard, + room_id: &'a ruma::RoomId, + ) -> Result + where + Leaves: Iterator + Send + 'a, + { + // Coalesce database writes for the remainder of this scope. + let _cork = self.db.db.cork_and_flush(); - // Mark as read first so the sending client doesn't get a notification even if - // appending fails - self.services - .read_receipt - .private_read_set(room_id, pdu.sender(), count1); + let shortroomid = self + .services + .short + .get_shortroomid(room_id) + .await + .map_err(|_| err!(Database("Room does not exist")))?; - self.services - .user - .reset_notification_counts(pdu.sender(), room_id); + // Make unsigned fields correct. This is not properly documented in the spec, + // but state events need to have previous content in the unsigned field, so + // clients can easily interpret things like membership changes - let count2 = PduCount::Normal(self.services.globals.next_count().unwrap()); - let pdu_id: RawPduId = PduId { shortroomid, shorteventid: count2 }.into(); + // TODO: This needs to be refactored to add this information on the fly. + // Because the prev content becomes part of the unsigned object in the PDU, we + // unintentionally leak redacted or hidden content to local users. + // See: https://forgejo.ellis.link/continuwuation/continuwuity/issues/1103 + self.populate_unsigned(pdu, &mut pdu_json).await; - // Insert pdu - self.db.append_pdu(&pdu_id, pdu, &pdu_json, count2).await; + // We must keep track of all events that have been referenced. + self.services + .pdu_metadata + .mark_as_referenced(room_id, pdu.prev_events().map(AsRef::as_ref)); - drop(insert_lock); + trace!("setting forward extremities"); + self.services + .state + .set_forward_extremities(room_id, leaves, state_lock) + .await; - // See if the event matches any known pushers via power level - if *pdu.kind() != TimelineEventType::RoomCreate { + let insert_lock = self.mutex_insert.lock(room_id).await; + + let count1 = self.services.globals.next_count().unwrap(); + + // Mark as read first so the sending client doesn't get a notification even if + // appending fails + // TODO: Is this necessary? appending doesn't seem that fallible, and if it is, + // there's bigger issues than ghost notifications. + self.services + .read_receipt + .private_read_set(room_id, pdu.sender(), count1); + + self.services + .user + .reset_notification_counts(pdu.sender(), room_id); + + let count2 = PduCount::Normal(self.services.globals.next_count().unwrap()); + let pdu_id: RawPduId = PduId { shortroomid, shorteventid: count2 }.into(); + + // Insert pdu + self.db.append_pdu(&pdu_id, pdu, &pdu_json, count2).await; + + drop(insert_lock); + + // See if the event matches any known pushers via power level + if *pdu.kind() != TimelineEventType::RoomCreate { + tokio::join!( + self.notify_local_users(pdu, &pdu_id), + self.handle_pdu_effects(pdu, &pdu_id, shortroomid) + .inspect_err(|e| { + error!( + "failed to handle PDU effects of incoming PDU {}: {e:?}", + pdu.event_id() + ); + }) + .ok(), + self.aggregate_relations(pdu, count2), + ); + } + + self.send_to_interested_appservices(pdu, &pdu_id).await; + + Ok(pdu_id) + } + + /// Notifies local users of the incoming event with a power levels context. + async fn notify_local_users(&self, pdu: &PduEvent, pdu_id: &RawPduId) { let power_levels = self .services .state_accessor - .get_room_power_levels(room_id) + .get_room_power_levels(pdu.room_id().unwrap()) + .await; + let mut push_targets: HashSet<_> = self + .services + .state_cache + .active_local_users_in_room(pdu.room_id().unwrap()) + // Don't notify the sender of their own events, and don't send from ignored users + .ready_filter(|user| *user != pdu.sender()) + .filter_map(|recipient_user| async move { + (!self.services.users.user_is_ignored(pdu.sender(), &recipient_user).await).then_some(recipient_user) + }) + .collect() .await; - let mut push_target: HashSet<_> = self - .services - .state_cache - .active_local_users_in_room(room_id) - // Don't notify the sender of their own events, and dont send from ignored users - .ready_filter(|user| *user != pdu.sender()) - .filter_map(|recipient_user| async move { (!self.services.users.user_is_ignored(pdu.sender(), &recipient_user).await).then_some(recipient_user) }) - .collect() - .await; - let mut notifies = Vec::with_capacity(push_target.len().saturating_add(1)); - let mut highlights = Vec::with_capacity(push_target.len().saturating_add(1)); + let mut notifies = Vec::with_capacity(push_targets.len().saturating_add(1)); + let mut highlights = Vec::with_capacity(push_targets.len().saturating_add(1)); if *pdu.kind() == TimelineEventType::RoomMember { if let Some(state_key) = pdu.state_key() { - let target_user_id = UserId::parse(state_key)?; - - if self.services.users.is_active_local(&target_user_id).await { - push_target.insert(target_user_id.clone()); + match UserId::parse(state_key) { + | Ok(target_user_id) => { + if self.services.users.is_active_local(&target_user_id).await { + push_targets.insert(target_user_id.clone()); + } + }, + | Err(e) => debug_warn!(user_id=?state_key, ?e, "failed to parse user ID"), } } } + if push_targets.is_empty() { + return; + } + let serialized = pdu.to_format(); - for user in &push_target { + for user in &push_targets { let rules_for_user = self .services .account_data @@ -240,7 +285,13 @@ where for action in self .services .pusher - .get_actions(user, &rules_for_user, power_levels.clone(), &serialized, room_id) + .get_actions( + user, + &rules_for_user, + power_levels.clone(), + &serialized, + pdu.room_id().unwrap(), + ) .await { match action { @@ -273,163 +324,197 @@ where .ready_for_each(|push_key| { self.services .sending - .send_pdu_push(&pdu_id, user, push_key.to_owned()) + .send_pdu_push(pdu_id, user, push_key.to_owned()) .expect("TODO: replace with future"); }) .await; } self.db - .increment_notification_counts(room_id, notifies, highlights); + .increment_notification_counts(pdu.room_id().unwrap(), notifies, highlights); } - match *pdu.kind() { - | TimelineEventType::RoomRedaction => { - use RoomVersionId::*; + /// Handles PDU effects based on the type of incoming event. + /// For redaction events, handles redacting. Memberships update the + /// membership cache. Et cetera. + async fn handle_pdu_effects( + &self, + pdu: &PduEvent, + pdu_id: &RawPduId, + short_room_id: ShortRoomId, + ) -> Result { + let room_id = pdu.room_id().unwrap(); + match *pdu.kind() { + | TimelineEventType::RoomRedaction => { + use RoomVersionId::*; - let room_version_id = self.services.state.get_room_version(room_id).await?; - match room_version_id { - | V1 | V2 | V3 | V4 | V5 | V6 | V7 | V8 | V9 | V10 => { - if let Some(redact_id) = pdu.redacts() { - if self - .services - .state_accessor - .user_can_redact(redact_id, pdu.sender(), room_id, false) - .await? - { - self.redact_pdu(redact_id, pdu, shortroomid).await?; + // TODO: support delayed redaction (MSC2815) + let room_version_id = self.services.state.get_room_version(room_id).await?; + match room_version_id { + | V1 | V2 | V3 | V4 | V5 | V6 | V7 | V8 | V9 | V10 => { + if let Some(redact_id) = pdu.redacts() { + if self + .services + .state_accessor + .user_can_redact(redact_id, pdu.sender(), room_id, false) + .await? + { + self.redact_pdu(redact_id, pdu, short_room_id).await?; + } } - } - }, - | _ => { - let content: RoomRedactionEventContent = pdu.get_content()?; - if let Some(redact_id) = &content.redacts { - if self - .services - .state_accessor - .user_can_redact(redact_id, pdu.sender(), room_id, false) - .await? - { - self.redact_pdu(redact_id, pdu, shortroomid).await?; + }, + | _ => { + let content: RoomRedactionEventContent = pdu.get_content()?; + if let Some(redact_id) = &content.redacts { + if self + .services + .state_accessor + .user_can_redact(redact_id, pdu.sender(), room_id, false) + .await? + { + self.redact_pdu(redact_id, pdu, short_room_id).await?; + } } - } - }, - } - }, - | TimelineEventType::RoomMember => { - if let Some(state_key) = pdu.state_key() { - // if the state_key fails - let target_user_id = - UserId::parse(state_key).expect("This state_key was previously validated"); - - // Update our membership info, we do this here incase a user is invited or - // knocked and immediately leaves we need the DB to record the invite or - // knock event for auth - self.services - .state_cache - .update_membership(room_id, &target_user_id, pdu, true) - .await?; - } - }, - | TimelineEventType::RoomMessage => { - let content: ExtractBody = pdu.get_content()?; - if let Some(body) = content.body { - self.services.search.index_pdu(shortroomid, &pdu_id, &body); - } - }, - | _ => {}, - } - - // CONCERN: If we receive events with a relation out-of-order, we never write - // their relation / thread. We need some kind of way to trigger when we receive - // this event, and potentially a way to rebuild the table entirely. - - if let Ok(content) = pdu.get_content::() { - if let Ok(related_pducount) = self.get_pdu_count(&content.relates_to.event_id).await { - self.services - .pdu_metadata - .add_relation(count2, related_pducount); - } - } - - if let Ok(content) = pdu.get_content::() { - match content.relates_to { - | Relation::Reply(in_reply_to) => { - // We need to do it again here, because replies don't have - // event_id as a top level field - if let Ok(related_pducount) = - self.get_pdu_count(&in_reply_to.in_reply_to.event_id).await - { - self.services - .pdu_metadata - .add_relation(count2, related_pducount); + }, } }, - | Relation::Thread(thread) => { - self.services - .threads - .add_to_thread(&thread.event_id, pdu) - .await?; + | TimelineEventType::RoomMember => { + if let Some(state_key) = pdu.state_key() { + // if the state_key fails + let target_user_id = UserId::parse(state_key) + .expect("This state_key was previously validated"); + + // Update our membership info, we do this here incase a user is invited or + // knocked and immediately leaves we need the DB to record the invite or + // knock event for auth + self.services + .state_cache + .update_membership(room_id, &target_user_id, pdu, true) + .await?; + } }, - | _ => {}, // TODO: Aggregate other types + | TimelineEventType::RoomMessage => { + let content: ExtractBody = pdu.get_content()?; + if let Some(body) = content.body { + self.services.search.index_pdu(short_room_id, pdu_id, &body); + } + }, + | _ => {}, + } + + Ok(()) + } + + /// Adds relation data to the incoming event and events it relates to. + async fn aggregate_relations(&self, pdu: &PduEvent, count2: Count) { + // CONCERN: If we receive events with a relation out-of-order, we never write + // their relation / thread. We need some kind of way to trigger when we receive + // this event, and potentially a way to rebuild the table entirely. + + if let Ok(content) = pdu.get_content::() { + if let Ok(related_pducount) = self.get_pdu_count(&content.relates_to.event_id).await { + self.services + .pdu_metadata + .add_relation(count2, related_pducount); + } + } + + if let Ok(content) = pdu.get_content::() { + match content.relates_to { + | Relation::Reply(in_reply_to) => { + // We need to do it again here, because replies don't have + // event_id as a top level field + if let Ok(related_pducount) = + self.get_pdu_count(&in_reply_to.in_reply_to.event_id).await + { + self.services + .pdu_metadata + .add_relation(count2, related_pducount); + } + }, + | Relation::Thread(thread) => { + self.services + .threads + .add_to_thread(&thread.event_id, pdu) + .await + .inspect_err(|e| { + warn!( + "Failed to add incoming event {} to thread {}: {e:?}", + pdu.event_id(), + thread.event_id + ); + }) + .ok(); + }, + | _ => {}, // TODO: Aggregate other types + } } } - for appservice in self.services.appservice.read().await.values() { - if self - .services - .state_cache - .appservice_in_room(room_id, appservice) + /// Determines if an appservice is interested in a particular event. + async fn is_appservice_interested( + &self, + appservice: &RegistrationInfo, + pdu: &PduEvent, + ) -> bool { + let room_id = pdu.room_id().unwrap(); + let target = if *pdu.kind() == TimelineEventType::RoomMember { + pdu.state_key().and_then(|sk| UserId::parse(sk).ok()) + } else { + None + }; + + let (target_matches_sender, target_matches_namespace) = match target { + | Some(target) => ( + appservice.registration.sender_localpart.as_str() == target.localpart(), + appservice.users.is_match(target.as_str()), + ), + | _ => (false, false), + }; + let sender_matches_namespace = appservice.users.is_match(pdu.sender().as_str()); + + if target_matches_sender || target_matches_namespace || sender_matches_namespace { + return true; + } + + let aliases = appservice.aliases.clone(); + self.services + .alias + .local_aliases_for_room(room_id) + .ready_any(move |room_alias| aliases.is_match(room_alias.as_str())) .await - { - self.services - .sending - .send_pdu_appservice(appservice.registration.id.clone(), pdu_id)?; - continue; - } - - // If the RoomMember event has a non-empty state_key, it is targeted at someone. - // If it is our appservice user, we send this PDU to it. - if *pdu.kind() == TimelineEventType::RoomMember { - if let Some(state_key_uid) = &pdu - .state_key - .as_ref() - .and_then(|state_key| UserId::parse(state_key.as_str()).ok()) - { - let appservice_uid = appservice.registration.sender_localpart.as_str(); - if state_key_uid == appservice_uid { - self.services - .sending - .send_pdu_appservice(appservice.registration.id.clone(), pdu_id)?; - continue; - } - } - } - - let matching_users = |users: &NamespaceRegex| { - appservice.users.is_match(pdu.sender().as_str()) - || *pdu.kind() == TimelineEventType::RoomMember - && pdu - .state_key - .as_ref() - .is_some_and(|state_key| users.is_match(state_key)) - }; - let matching_aliases = |aliases: NamespaceRegex| { - self.services - .alias - .local_aliases_for_room(room_id) - .ready_any(move |room_alias| aliases.is_match(room_alias.as_str())) - }; - - if matching_aliases(appservice.aliases.clone()).await - || appservice.rooms.is_match(room_id.as_str()) - || matching_users(&appservice.users) - { - self.services - .sending - .send_pdu_appservice(appservice.registration.id.clone(), pdu_id)?; - } } - Ok(pdu_id) + /// Notifies interested appservices of a new PDU. + async fn send_to_interested_appservices(&self, pdu: &PduEvent, pdu_id: &RawPduId) { + let interested_appservices = self + .services + .appservice + .read() + .await + .values() + .map(ToOwned::to_owned) // TODO: is this to_owned expensive? + .collect::>(); + interested_appservices + .stream() + .broad_filter_map(|appservice| async move { + self.is_appservice_interested(&appservice, pdu) + .await + .then_some(appservice) + }) + .for_each_concurrent(None, |appservice| async move { + self.services + .sending + .send_pdu_appservice(appservice.registration.id.clone(), *pdu_id) + .inspect_err(|e| { + warn!( + "failed to send PDU {} to appservice {}: {e:?}", + pdu.event_id(), + appservice.registration.id + ); + }) + .ok(); + }) + .await; + } } diff --git a/src/service/rooms/timeline/backfill.rs b/src/service/rooms/timeline/backfill.rs index 9930cf466..26c36bf69 100644 --- a/src/service/rooms/timeline/backfill.rs +++ b/src/service/rooms/timeline/backfill.rs @@ -2,7 +2,7 @@ use std::collections::HashSet; use conduwuit::{Err, PduEvent}; use conduwuit_core::{ - Result, debug, debug_warn, err, implement, info, + Result, debug, debug_warn, err, info, matrix::{ event::Event, pdu::{PduCount, PduId, RawPduId}, @@ -18,261 +18,267 @@ use serde_json::value::RawValue as RawJsonValue; use super::ExtractBody; -#[implement(super::Service)] -#[tracing::instrument(name = "backfill", level = "trace", skip(self))] -pub async fn backfill_if_required(&self, room_id: &RoomId, from: PduCount) -> Result<()> { - if self - .services - .state_cache - .room_joined_count(room_id) - .await - .is_ok_and(|count| count <= 1) - && !self - .services - .state_accessor - .is_world_readable(room_id) - .await - { - // Room is empty (1 user or none), there is no one that can backfill - debug_warn!("Room {room_id} is empty, skipping backfill"); - return Ok(()); - } - - let first_pdu = self - .first_item_in_room(room_id) - .await - .expect("Room is not empty"); - - if first_pdu.0 < from { - // No backfill required, there are still events between them - debug!("No backfill required in room {room_id}, {:?} < {from}", first_pdu.0); - return Ok(()); - } - - let servers = self.candidate_backfill_servers(room_id).await; - - let mut federated_room = false; - - for backfill_server in servers { - if !self.services.globals.server_is_ours(&backfill_server) { - federated_room = true; - } - info!("Asking {backfill_server} for backfill in {room_id}"); - let response = self - .services - .sending - .send_federation_request( - &backfill_server, - federation::backfill::get_backfill::v1::Request::new( - room_id.to_owned(), - vec![first_pdu.1.event_id().to_owned()], - uint!(100), - ), - ) - .await; - match response { - | Ok(response) => { - for pdu in response.pdus { - if let Err(e) = self.backfill_pdu(&backfill_server, pdu).boxed().await { - debug_warn!("Failed to add backfilled pdu in room {room_id}: {e}"); - } - } - return Ok(()); - }, - | Err(e) => { - warn!("{backfill_server} failed to provide backfill for room {room_id}: {e}"); - }, - } - } - - if federated_room { - warn!("No servers could backfill, but backfill was needed in room {room_id}"); - } - Ok(()) -} - -#[implement(super::Service)] -#[tracing::instrument(name = "get_remote_pdu", level = "debug", skip(self))] -pub async fn get_remote_pdu(&self, room_id: &RoomId, event_id: &EventId) -> Result { - let local = self.get_pdu(event_id).await; - if local.is_ok() { - // We already have this PDU, no need to backfill - debug!("We already have {event_id} in {room_id}, no need to backfill."); - return local; - } - debug!("Preparing to fetch event {event_id} in room {room_id} from remote servers."); - // Similar to backfill_if_required, but only for a single PDU - // Fetch a list of servers to try - if self - .services - .state_cache - .room_joined_count(room_id) - .await - .is_ok_and(|count| count <= 1) - && !self - .services - .state_accessor - .is_world_readable(room_id) - .await - { - // Room is empty (1 user or none), there is no one that can backfill - return Err!(Request(NotFound("No one can backfill this PDU, room is empty."))); - } - - let servers = self.candidate_backfill_servers(room_id).await; - - for backfill_server in servers { - info!("Asking {backfill_server} for event {}", event_id); - let value = self - .services - .sending - .send_federation_request( - &backfill_server, - federation::event::get_event::v1::Request::new(event_id.to_owned()), - ) - .await - .and_then(|response| { - serde_json::from_str::(response.pdu.get()).map_err(|e| { - err!(BadServerResponse(debug_warn!( - "Error parsing incoming event {e:?} from {backfill_server}" - ))) - }) - }); - let pdu = match value { - | Ok(value) => { - self.services - .event_handler - .handle_incoming_pdu(&backfill_server, room_id, event_id, value, false) - .boxed() - .await?; - debug!("Successfully backfilled {event_id} from {backfill_server}"); - Some(self.get_pdu(event_id).await) - }, - | Err(e) => { - warn!("{backfill_server} failed to provide backfill for room {room_id}: {e}"); - None - }, - }; - if let Some(pdu) = pdu { - debug!("Fetched {event_id} from {backfill_server}"); - return pdu; - } - } - - Err!("No servers could be used to fetch {} in {}.", room_id, event_id) -} - -#[implement(super::Service)] -#[tracing::instrument(skip(self, pdu), level = "debug")] -pub async fn backfill_pdu(&self, origin: &ServerName, pdu: Box) -> Result<()> { - let (room_id, event_id, value) = self.services.event_handler.parse_incoming_pdu(&pdu).await?; - - // Lock so we cannot backfill the same pdu twice at the same time - let mutex_lock = self - .services - .event_handler - .mutex_federation - .lock(room_id.as_str()) - .await; - - // Skip the PDU if we already have it as a timeline event - if let Ok(pdu_id) = self.get_pdu_id(&event_id).await { - debug!("We already know {event_id} at {pdu_id:?}"); - return Ok(()); - } - - self.services - .event_handler - .handle_incoming_pdu(origin, &room_id, &event_id, value, false) - .boxed() - .await?; - - let value = self.get_pdu_json(&event_id).await?; - - let pdu = self.get_pdu(&event_id).await?; - - let shortroomid = self.services.short.get_shortroomid(&room_id).await?; - - let insert_lock = self.mutex_insert.lock(room_id.as_str()).await; - - let count: i64 = self.services.globals.next_count().unwrap().try_into()?; - - let pdu_id: RawPduId = PduId { - shortroomid, - shorteventid: PduCount::Backfilled(validated!(0 - count)), - } - .into(); - - // Insert pdu - self.db.prepend_backfill_pdu(&pdu_id, &event_id, &value); - - drop(insert_lock); - - if pdu.kind == TimelineEventType::RoomMessage { - let content: ExtractBody = pdu.get_content()?; - if let Some(body) = content.body { - self.services.search.index_pdu(shortroomid, &pdu_id, &body); - } - } - drop(mutex_lock); - - debug!("Prepended backfill pdu"); - Ok(()) -} - -#[implement(super::Service)] -async fn candidate_backfill_servers(&self, room_id: &RoomId) -> HashSet { - let mut candidate_backfill_servers = HashSet::new(); - - let power_levels = self - .services - .state_accessor - .get_room_power_levels(room_id) - .await; - - // Insert servers of room creators - if let Some(creators) = &power_levels.rules.privileged_creators { - for creator in creators { - candidate_backfill_servers.insert(creator.server_name().to_owned()); - } - } - - // Insert servers of remote users with higher-than-default PL - for (user_id, level) in &power_levels.users { - if !self.services.globals.user_is_local(user_id) && *level > power_levels.users_default { - candidate_backfill_servers.insert(user_id.server_name().to_owned()); - } - } - - // Insert the canonical room alias server - if let Ok(canonical_alias) = self - .services - .state_accessor - .get_canonical_alias(room_id) - .await - { - candidate_backfill_servers.insert(canonical_alias.server_name().to_owned()); - } - - // Insert all trusted servers in the config - candidate_backfill_servers - .extend(self.services.server.config.trusted_servers.iter().cloned()); - - // Remove our own name, we can't request backfill from ourselves - candidate_backfill_servers.remove(self.services.globals.server_name()); - - // Remove all servers that aren't in the room - for server in candidate_backfill_servers.clone() { - if !self +impl super::Service { + /// Performs backfill, if it is required. + #[tracing::instrument(name = "backfill", level = "trace", skip(self))] + pub async fn backfill_if_required(&self, room_id: &RoomId, from: PduCount) -> Result<()> { + if self .services .state_cache - .server_in_room(&server, room_id) + .room_joined_count(room_id) .await + .is_ok_and(|count| count <= 1) + && !self + .services + .state_accessor + .is_world_readable(room_id) + .await { - candidate_backfill_servers.remove(&server); + // Room is empty (1 user or none), there is no one that can backfill + debug_warn!("Room {room_id} is empty, skipping backfill"); + return Ok(()); } + + let first_pdu = self + .first_item_in_room(room_id) + .await + .expect("Room is not empty"); + + if first_pdu.0 < from { + // No backfill required, there are still events between them + debug!("No backfill required in room {room_id}, {:?} < {from}", first_pdu.0); + return Ok(()); + } + + let servers = self.candidate_backfill_servers(room_id).await; + + let mut federated_room = false; + + for backfill_server in servers { + if !self.services.globals.server_is_ours(&backfill_server) { + federated_room = true; + } + info!("Asking {backfill_server} for backfill in {room_id}"); + let response = self + .services + .sending + .send_federation_request( + &backfill_server, + federation::backfill::get_backfill::v1::Request::new( + room_id.to_owned(), + vec![first_pdu.1.event_id().to_owned()], + uint!(100), + ), + ) + .await; + match response { + | Ok(response) => { + for pdu in response.pdus { + if let Err(e) = self.backfill_pdu(&backfill_server, pdu).boxed().await { + debug_warn!("Failed to add backfilled pdu in room {room_id}: {e}"); + } + } + return Ok(()); + }, + | Err(e) => { + warn!("{backfill_server} failed to provide backfill for room {room_id}: {e}"); + }, + } + } + + if federated_room { + warn!("No servers could backfill, but backfill was needed in room {room_id}"); + } + Ok(()) } - debug!(?candidate_backfill_servers, "Found candidate servers for backfill"); - candidate_backfill_servers + /// Fetches a single PDU from a remote server, without persisting it. + #[tracing::instrument(name = "get_remote_pdu", level = "debug", skip(self))] + pub async fn get_remote_pdu(&self, room_id: &RoomId, event_id: &EventId) -> Result { + let local = self.get_pdu(event_id).await; + if local.is_ok() { + // We already have this PDU, no need to backfill + debug!("We already have {event_id} in {room_id}, no need to backfill."); + return local; + } + debug!("Preparing to fetch event {event_id} in room {room_id} from remote servers."); + // Similar to backfill_if_required, but only for a single PDU + // Fetch a list of servers to try + if self + .services + .state_cache + .room_joined_count(room_id) + .await + .is_ok_and(|count| count <= 1) + && !self + .services + .state_accessor + .is_world_readable(room_id) + .await + { + // Room is empty (1 user or none), there is no one that can backfill + return Err!(Request(NotFound("No one can backfill this PDU, room is empty."))); + } + + let servers = self.candidate_backfill_servers(room_id).await; + + for backfill_server in servers { + info!("Asking {backfill_server} for event {}", event_id); + let value = self + .services + .sending + .send_federation_request( + &backfill_server, + federation::event::get_event::v1::Request::new(event_id.to_owned()), + ) + .await + .and_then(|response| { + serde_json::from_str::(response.pdu.get()).map_err(|e| { + err!(BadServerResponse(debug_warn!( + "Error parsing incoming event {e:?} from {backfill_server}" + ))) + }) + }); + let pdu = match value { + | Ok(value) => { + self.services + .event_handler + .handle_incoming_pdu(&backfill_server, room_id, event_id, value, false) + .boxed() + .await?; + debug!("Successfully backfilled {event_id} from {backfill_server}"); + Some(self.get_pdu(event_id).await) + }, + | Err(e) => { + warn!("{backfill_server} failed to provide backfill for room {room_id}: {e}"); + None + }, + }; + if let Some(pdu) = pdu { + debug!("Fetched {event_id} from {backfill_server}"); + return pdu; + } + } + + Err!("No servers could be used to fetch {} in {}.", room_id, event_id) + } + + /// Backfills a single PDU. + #[tracing::instrument(skip(self, pdu), level = "debug")] + pub async fn backfill_pdu(&self, origin: &ServerName, pdu: Box) -> Result<()> { + let (room_id, event_id, value) = + self.services.event_handler.parse_incoming_pdu(&pdu).await?; + + // Lock so we cannot backfill the same pdu twice at the same time + let mutex_lock = self + .services + .event_handler + .mutex_federation + .lock(room_id.as_str()) + .await; + + // Skip the PDU if we already have it as a timeline event + if let Ok(pdu_id) = self.get_pdu_id(&event_id).await { + debug!("We already know {event_id} at {pdu_id:?}"); + return Ok(()); + } + + self.services + .event_handler + .handle_incoming_pdu(origin, &room_id, &event_id, value, false) + .boxed() + .await?; + + let value = self.get_pdu_json(&event_id).await?; + + let pdu = self.get_pdu(&event_id).await?; + + let shortroomid = self.services.short.get_shortroomid(&room_id).await?; + + let insert_lock = self.mutex_insert.lock(room_id.as_str()).await; + + let count: i64 = self.services.globals.next_count().unwrap().try_into()?; + + let pdu_id: RawPduId = PduId { + shortroomid, + shorteventid: PduCount::Backfilled(validated!(0 - count)), + } + .into(); + + // Insert pdu + self.db.prepend_backfill_pdu(&pdu_id, &event_id, &value); + + drop(insert_lock); + + if pdu.kind == TimelineEventType::RoomMessage { + let content: ExtractBody = pdu.get_content()?; + if let Some(body) = content.body { + self.services.search.index_pdu(shortroomid, &pdu_id, &body); + } + } + drop(mutex_lock); + + debug!("Prepended backfill pdu"); + Ok(()) + } + + /// Determines which servers are trusted enough to provide backfill in a + /// room. + async fn candidate_backfill_servers(&self, room_id: &RoomId) -> HashSet { + let mut candidate_backfill_servers = HashSet::new(); + + let power_levels = self + .services + .state_accessor + .get_room_power_levels(room_id) + .await; + + // Insert servers of room creators + if let Some(creators) = &power_levels.rules.privileged_creators { + for creator in creators { + candidate_backfill_servers.insert(creator.server_name().to_owned()); + } + } + + // Insert servers of remote users with higher-than-default PL + for (user_id, level) in &power_levels.users { + if !self.services.globals.user_is_local(user_id) + && *level > power_levels.users_default + { + candidate_backfill_servers.insert(user_id.server_name().to_owned()); + } + } + + // Insert the canonical room alias server + if let Ok(canonical_alias) = self + .services + .state_accessor + .get_canonical_alias(room_id) + .await + { + candidate_backfill_servers.insert(canonical_alias.server_name().to_owned()); + } + + // Insert all trusted servers in the config + candidate_backfill_servers + .extend(self.services.server.config.trusted_servers.iter().cloned()); + + // Remove our own name, we can't request backfill from ourselves + candidate_backfill_servers.remove(self.services.globals.server_name()); + + // Remove all servers that aren't in the room + for server in candidate_backfill_servers.clone() { + if !self + .services + .state_cache + .server_in_room(&server, room_id) + .await + { + candidate_backfill_servers.remove(&server); + } + } + + debug!(?candidate_backfill_servers, "Found candidate servers for backfill"); + candidate_backfill_servers + } } diff --git a/src/service/rooms/timeline/build.rs b/src/service/rooms/timeline/build.rs index 85f1f88d9..8ff04b024 100644 --- a/src/service/rooms/timeline/build.rs +++ b/src/service/rooms/timeline/build.rs @@ -2,13 +2,13 @@ use std::{collections::HashSet, iter::once}; use conduwuit::trace; use conduwuit_core::{ - Err, Result, implement, + Err, Result, matrix::{event::Event, pdu::PartialPdu}, utils::{IterStream, ReadyExt}, }; use futures::{FutureExt, StreamExt}; use ruma::{ - OwnedEventId, OwnedServerName, RoomId, RoomVersionId, UserId, + OwnedEventId, OwnedServerName, RoomId, UserId, events::{ TimelineEventType, room::{ @@ -20,246 +20,248 @@ use ruma::{ use super::{ExtractBody, RoomMutexGuard}; -/// Creates a new persisted data unit and adds it to a room. This function -/// takes a roomid_mutex_state, meaning that only this function is able to -/// mutate the room state. -#[implement(super::Service)] -#[tracing::instrument(skip(self, state_lock, partial_pdu), level = "trace")] -pub async fn build_and_append_pdu( - &self, - partial_pdu: PartialPdu, - sender: &UserId, - room_id: Option<&RoomId>, - state_lock: &RoomMutexGuard, -) -> Result { - let (pdu, pdu_json) = self - .create_hash_and_sign_event(partial_pdu, sender, room_id, state_lock) - .await?; +impl super::Service { + /// Creates a new persisted data unit and adds it to a room. This function + /// takes a roomid_mutex_state, meaning that only this function is able to + /// mutate the room state. + #[tracing::instrument(skip(self, state_lock, partial_pdu), level = "trace")] + pub async fn build_and_append_pdu( + &self, + partial_pdu: PartialPdu, + sender: &UserId, + room_id: Option<&RoomId>, + state_lock: &RoomMutexGuard, + ) -> Result { + let (pdu, pdu_json) = self + .create_hash_and_sign_event(partial_pdu, sender, room_id, state_lock) + .await?; - let room_id = pdu.room_id_or_hash(); - if self.services.admin.is_admin_room(&room_id).await { - self.check_pdu_for_admin_room(&pdu, sender).boxed().await?; - } + let room_id = pdu.room_id_or_hash(); + if self.services.admin.is_admin_room(&room_id).await { + self.check_pdu_for_admin_room(&pdu, sender).boxed().await?; + } - // If redaction event is not authorized, do not append it to the timeline - if *pdu.kind() == TimelineEventType::RoomRedaction { - use RoomVersionId::*; - trace!("Running redaction checks for room {room_id}"); - match self.services.state.get_room_version(&room_id).await? { - | V1 | V2 | V3 | V4 | V5 | V6 | V7 | V8 | V9 | V10 => { - if let Some(redact_id) = pdu.redacts() { - if !self - .services - .state_accessor - .user_can_redact(redact_id, pdu.sender(), &room_id, false) - .await? - { - return Err!(Request(Forbidden("User cannot redact this event."))); + // If redaction event is not authorized, do not append it to the timeline + if *pdu.kind() == TimelineEventType::RoomRedaction { + use ruma::RoomVersionId::*; + trace!("Running redaction checks for room {room_id}"); + match self.services.state.get_room_version(&room_id).await? { + | V1 | V2 | V3 | V4 | V5 | V6 | V7 | V8 | V9 | V10 => { + if let Some(redact_id) = pdu.redacts() { + if !self + .services + .state_accessor + .user_can_redact(redact_id, pdu.sender(), &room_id, false) + .await? + { + return Err!(Request(Forbidden("User cannot redact this event."))); + } } - } - }, - | _ => { - let content: RoomRedactionEventContent = pdu.get_content()?; - if let Some(redact_id) = &content.redacts { - if !self - .services - .state_accessor - .user_can_redact(redact_id, pdu.sender(), &room_id, false) - .await? - { - return Err!(Request(Forbidden("User cannot redact this event."))); + }, + | _ => { + let content: RoomRedactionEventContent = pdu.get_content()?; + if let Some(redact_id) = &content.redacts { + if !self + .services + .state_accessor + .user_can_redact(redact_id, pdu.sender(), &room_id, false) + .await? + { + return Err!(Request(Forbidden("User cannot redact this event."))); + } } - } - }, - } - } - - if *pdu.kind() == TimelineEventType::RoomMember { - trace!("Running room member checks for room {room_id}"); - let content: RoomMemberEventContent = pdu.get_content()?; - - if content.join_authorized_via_users_server.is_some() - && content.membership != MembershipState::Join - { - return Err!(Request(BadJson( - "join_authorised_via_users_server is only for member joins" - ))); - } - - if content - .join_authorized_via_users_server - .as_ref() - .is_some_and(|authorising_user| { - !self.services.globals.user_is_local(authorising_user) - }) { - return Err!(Request(InvalidParam( - "Authorising user does not belong to this homeserver" - ))); - } - } - if *pdu.kind() == TimelineEventType::RoomCreate { - trace!("Creating shortroomid for {room_id}"); - self.services - .short - .get_or_create_shortroomid(&room_id) - .await; - } - - // We append to state before appending the pdu, so we don't have a moment in - // time with the pdu without it's state. This is okay because append_pdu can't - // fail. - trace!("Appending {} state for room {room_id}", pdu.event_id()); - let statehashid = self.services.state.append_to_state(&pdu, &room_id).await?; - trace!("State hash ID for {room_id}: {statehashid:?}"); - - trace!("Generating raw ID for PDU {}", pdu.event_id()); - let pdu_id = self - .append_pdu( - &pdu, - pdu_json, - // Since this PDU references all pdu_leaves we can update the leaves - // of the room - once(pdu.event_id()), - state_lock, - &room_id, - ) - .boxed() - .await?; - - // Process admin commands for locally sent events - if *pdu.kind() == TimelineEventType::RoomMessage { - let content: ExtractBody = pdu.get_content()?; - if let Some(body) = content.body { - if let Some(source) = self - .services - .admin - .is_admin_command(&pdu, &body, true) - .await - { - self.services.admin.command_with_sender( - body, - Some(pdu.event_id().into()), - source, - pdu.sender.clone(), - )?; + }, } } - } - - // We set the room state after inserting the pdu, so that we never have a moment - // in time where events in the current room state do not exist - trace!("Setting room state for room {room_id}"); - self.services - .state - .set_room_state(&room_id, statehashid, state_lock); - - let mut servers: HashSet = self - .services - .state_cache - .room_servers(&room_id) - .collect() - .await; - - // In case we are kicking or banning a user, we need to inform their server of - // the change - if *pdu.kind() == TimelineEventType::RoomMember { - if let Some(state_key_uid) = &pdu - .state_key - .as_ref() - .and_then(|state_key| UserId::parse(state_key.as_str()).ok()) - { - servers.insert(state_key_uid.server_name().to_owned()); - } - } - - // Remove our server from the server list since it will be added to it by - // room_servers() and/or the if statement above - servers.remove(self.services.globals.server_name()); - - trace!("Sending PDU {} to {} servers", pdu.event_id(), servers.len()); - self.services - .sending - .send_pdu_servers(servers.stream(), &pdu_id) - .await?; - - trace!("Event {} in room {:?} has been appended", pdu.event_id(), room_id); - Ok(pdu.event_id().to_owned()) -} - -/// Assert invariants about the admin room, to prevent (for example) all admins -/// from leaving or being banned from the room -#[implement(super::Service)] -#[tracing::instrument(skip_all, level = "debug")] -async fn check_pdu_for_admin_room(&self, pdu: &Pdu, sender: &UserId) -> Result -where - Pdu: Event + Send + Sync, -{ - match pdu.kind() { - | TimelineEventType::RoomEncryption => { - return Err!(Request(Forbidden(error!("Encryption not supported in admins room.")))); - }, - | TimelineEventType::RoomMember => { - let target = pdu - .state_key() - .filter(|v| v.starts_with('@')) - .unwrap_or(sender.as_str()); - - let server_user = &self.services.globals.server_user.to_string(); + if *pdu.kind() == TimelineEventType::RoomMember { + trace!("Running room member checks for room {room_id}"); let content: RoomMemberEventContent = pdu.get_content()?; - match content.membership { - | MembershipState::Leave => { - if target == server_user { - return Err!(Request(Forbidden(error!( - "Server user cannot leave the admins room." - )))); - } - let count = self - .services - .state_cache - .room_members(&pdu.room_id_or_hash()) - .ready_filter(|user| self.services.globals.user_is_local(user)) - .ready_filter(|user| *user != target) - .boxed() - .count() - .await; - - if count < 2 { - return Err!(Request(Forbidden(error!( - "Last admin cannot leave the admins room." - )))); - } - }, - - | MembershipState::Ban if pdu.state_key().is_some() => { - if target == server_user { - return Err!(Request(Forbidden(error!( - "Server cannot be banned from admins room." - )))); - } - - let count = self - .services - .state_cache - .room_members(&pdu.room_id_or_hash()) - .ready_filter(|user| self.services.globals.user_is_local(user)) - .ready_filter(|user| *user != target) - .boxed() - .count() - .await; - - if count < 2 { - return Err!(Request(Forbidden(error!( - "Last admin cannot be banned from admins room." - )))); - } - }, - | _ => {}, + if content.join_authorized_via_users_server.is_some() + && content.membership != MembershipState::Join + { + return Err!(Request(BadJson( + "join_authorised_via_users_server is only for member joins" + ))); } - }, - | _ => {}, + + if content + .join_authorized_via_users_server + .as_ref() + .is_some_and(|authorising_user| { + !self.services.globals.user_is_local(authorising_user) + }) { + return Err!(Request(InvalidParam( + "Authorising user does not belong to this homeserver" + ))); + } + } + if *pdu.kind() == TimelineEventType::RoomCreate { + trace!("Creating shortroomid for {room_id}"); + self.services + .short + .get_or_create_shortroomid(&room_id) + .await; + } + + // We append to state before appending the pdu, so we don't have a moment in + // time with the pdu without it's state. This is okay because append_pdu can't + // fail. + trace!("Appending {} state for room {room_id}", pdu.event_id()); + let statehashid = self.services.state.append_to_state(&pdu, &room_id).await?; + trace!("State hash ID for {room_id}: {statehashid:?}"); + + trace!("Generating raw ID for PDU {}", pdu.event_id()); + let pdu_id = self + .append_pdu( + &pdu, + pdu_json, + // Since this PDU references all pdu_leaves we can update the leaves + // of the room + once(pdu.event_id()), + state_lock, + &room_id, + ) + .boxed() + .await?; + + // Process admin commands for locally sent events + if *pdu.kind() == TimelineEventType::RoomMessage { + let content: ExtractBody = pdu.get_content()?; + if let Some(body) = content.body { + if let Some(source) = self + .services + .admin + .is_admin_command(&pdu, &body, true) + .await + { + self.services.admin.command_with_sender( + body, + Some(pdu.event_id().into()), + source, + pdu.sender.clone(), + )?; + } + } + } + + // We set the room state after inserting the pdu, so that we never have a moment + // in time where events in the current room state do not exist + trace!("Setting room state for room {room_id}"); + self.services + .state + .set_room_state(&room_id, statehashid, state_lock); + + let mut servers: HashSet = self + .services + .state_cache + .room_servers(&room_id) + .collect() + .await; + + // In case we are kicking or banning a user, we need to inform their server of + // the change + if *pdu.kind() == TimelineEventType::RoomMember { + if let Some(state_key_uid) = &pdu + .state_key + .as_ref() + .and_then(|state_key| UserId::parse(state_key.as_str()).ok()) + { + servers.insert(state_key_uid.server_name().to_owned()); + } + } + + // Remove our server from the server list since it will be added to it by + // room_servers() and/or the if statement above + servers.remove(self.services.globals.server_name()); + + trace!("Sending PDU {} to {} servers", pdu.event_id(), servers.len()); + self.services + .sending + .send_pdu_servers(servers.stream(), &pdu_id) + .await?; + + trace!("Event {} in room {:?} has been appended", pdu.event_id(), room_id); + Ok(pdu.event_id().to_owned()) } - Ok(()) + /// Assert invariants about the admin room, to prevent (for example) all + /// admins from leaving or being banned from the room + #[tracing::instrument(skip_all, level = "debug")] + async fn check_pdu_for_admin_room(&self, pdu: &Pdu, sender: &UserId) -> Result + where + Pdu: Event + Send + Sync, + { + match pdu.kind() { + | TimelineEventType::RoomEncryption => { + return Err!(Request(Forbidden(error!( + "Encryption not supported in admins room." + )))); + }, + | TimelineEventType::RoomMember => { + let target = pdu + .state_key() + .filter(|v| v.starts_with('@')) + .unwrap_or(sender.as_str()); + + let server_user = &self.services.globals.server_user.to_string(); + + let content: RoomMemberEventContent = pdu.get_content()?; + match content.membership { + | MembershipState::Leave => { + if target == server_user { + return Err!(Request(Forbidden(error!( + "Server user cannot leave the admins room." + )))); + } + + let count = self + .services + .state_cache + .room_members(&pdu.room_id_or_hash()) + .ready_filter(|user| self.services.globals.user_is_local(user)) + .ready_filter(|user| *user != target) + .boxed() + .count() + .await; + + if count < 2 { + return Err!(Request(Forbidden(error!( + "Last admin cannot leave the admins room." + )))); + } + }, + + | MembershipState::Ban if pdu.state_key().is_some() => { + if target == server_user { + return Err!(Request(Forbidden(error!( + "Server cannot be banned from admins room." + )))); + } + + let count = self + .services + .state_cache + .room_members(&pdu.room_id_or_hash()) + .ready_filter(|user| self.services.globals.user_is_local(user)) + .ready_filter(|user| *user != target) + .boxed() + .count() + .await; + + if count < 2 { + return Err!(Request(Forbidden(error!( + "Last admin cannot be banned from admins room." + )))); + } + }, + | _ => {}, + } + }, + | _ => {}, + } + + Ok(()) + } } diff --git a/src/service/rooms/timeline/create.rs b/src/service/rooms/timeline/create.rs index d86b9c442..9cdd3ea34 100644 --- a/src/service/rooms/timeline/create.rs +++ b/src/service/rooms/timeline/create.rs @@ -2,7 +2,7 @@ use std::{cmp, collections::HashMap}; use conduwuit::{smallstr::SmallString, trace}; use conduwuit_core::{ - Err, Error, Result, err, implement, + Err, Error, Result, err, matrix::{ event::{Event, gen_event_id}, pdu::{EventHash, PartialPdu, PduEvent}, @@ -73,277 +73,289 @@ fn room_version_from_event( } } -// Creates an event, but does not hash or sign it. -#[implement(super::Service)] -pub async fn create_event( - &self, - partial_pdu: PartialPdu, - sender: &UserId, - room_id: Option<&RoomId>, - _mutex_lock: &RoomMutexGuard, -) -> Result<(PduEvent, RoomVersionRules)> { - let PartialPdu { - event_type, - content, - unsigned, - state_key, - redacts, - timestamp, - } = partial_pdu; +impl super::Service { + /// Creates a new PDU and runs an auth check before returning it. Does not + /// hash or sign the PDU, meaning it can be mutated after. + pub async fn create_event( + &self, + partial_pdu: PartialPdu, + sender: &UserId, + room_id: Option<&RoomId>, + _mutex_lock: &RoomMutexGuard, + ) -> Result<(PduEvent, RoomVersionRules)> { + let PartialPdu { + event_type, + content, + unsigned, + state_key, + redacts, + timestamp, + } = partial_pdu; - trace!( - "Creating event of type {} in room {}", - event_type, - room_id.as_ref().map_or("None", |id| id.as_str()) - ); - let room_version = match room_id { - | Some(room_id) => { - trace!(%room_id, "Looking up existing room ID"); - self.services - .state - .get_room_version(room_id) + trace!( + "Creating event of type {} in room {}", + event_type, + room_id.as_ref().map_or("None", |id| id.as_str()) + ); + let room_version = match room_id { + | Some(room_id) => { + trace!(%room_id, "Looking up existing room ID"); + self.services + .state + .get_room_version(room_id) + .await + .or_else(|_| { + room_version_from_event( + room_id.to_owned(), + &event_type.clone(), + &content.clone(), + ) + })? + }, + | None => { + trace!("No room ID, assuming room creation"); + room_version_from_event( + RoomId::new_v1(self.services.globals.server_name()), + &event_type.clone(), + &content.clone(), + )? + }, + }; + + let Some(room_version_rules) = room_version.rules() else { + return Err!(Request(UnsupportedRoomVersion("Unsupported room version"))); + }; + + let prev_events: Vec = match room_id { + | Some(room_id) => + self.services + .state + .get_forward_extremities(room_id) + .take(20) + .map(Into::into) + .collect() + .await, + | None => Vec::new(), + }; + + let auth_events: HashMap<(StateEventType, SmallString<[u8; 48]>), PduEvent> = + match room_id { + | Some(room_id) => + self.services + .state + .get_auth_events( + room_id, + &event_type, + sender, + state_key.as_deref(), + &content, + &room_version_rules, + ) + .await?, + | None => HashMap::new(), + }; + // Our depth is the maximum depth of prev_events + 1 + let depth = match room_id { + | Some(_) => prev_events + .iter() + .stream() + .map(Ok) + .and_then(|event_id| self.get_pdu(event_id)) + .and_then(|pdu| future::ok(pdu.depth)) + .ignore_err() + .ready_fold(uint!(0), cmp::max) .await - .or_else(|_| { - room_version_from_event( - room_id.to_owned(), - &event_type.clone(), - &content.clone(), - ) - })? - }, - | None => { - trace!("No room ID, assuming room creation"); - room_version_from_event( - RoomId::new_v1(self.services.globals.server_name()), - &event_type.clone(), - &content.clone(), - )? - }, - }; + .saturating_add(uint!(1)), + | None => uint!(1), + }; - let Some(room_version_rules) = room_version.rules() else { - return Err!(Request(UnsupportedRoomVersion("Unsupported room version"))); - }; + let mut unsigned = unsigned.unwrap_or_default(); - let prev_events: Vec = match room_id { - | Some(room_id) => - self.services - .state - .get_forward_extremities(room_id) - .take(20) - .map(Into::into) - .collect() - .await, - | None => Vec::new(), - }; - - let auth_events: HashMap<(StateEventType, SmallString<[u8; 48]>), PduEvent> = match room_id { - | Some(room_id) => - self.services - .state - .get_auth_events( - room_id, - &event_type, - sender, - state_key.as_deref(), - &content, - &room_version_rules, - ) - .await?, - | None => HashMap::new(), - }; - // Our depth is the maximum depth of prev_events + 1 - let depth = match room_id { - | Some(_) => prev_events - .iter() - .stream() - .map(Ok) - .and_then(|event_id| self.get_pdu(event_id)) - .and_then(|pdu| future::ok(pdu.depth)) - .ignore_err() - .ready_fold(uint!(0), cmp::max) - .await - .saturating_add(uint!(1)), - | None => uint!(1), - }; - - let mut unsigned = unsigned.unwrap_or_default(); - - if let Some(room_id) = room_id { - if let Some(state_key) = &state_key { - if let Ok(prev_pdu) = self - .services - .state_accessor - .room_state_get(room_id, &event_type.clone().to_string().into(), state_key) - .await - { - unsigned.insert("prev_content".to_owned(), prev_pdu.get_content_as_value()); - unsigned - .insert("prev_sender".to_owned(), serde_json::to_value(prev_pdu.sender())?); - unsigned.insert( - "replaces_state".to_owned(), - serde_json::to_value(prev_pdu.event_id())?, - ); + if let Some(room_id) = room_id { + if let Some(state_key) = &state_key { + if let Ok(prev_pdu) = self + .services + .state_accessor + .room_state_get(room_id, &event_type.clone().to_string().into(), state_key) + .await + { + unsigned.insert("prev_content".to_owned(), prev_pdu.get_content_as_value()); + unsigned.insert( + "prev_sender".to_owned(), + serde_json::to_value(prev_pdu.sender())?, + ); + unsigned.insert( + "replaces_state".to_owned(), + serde_json::to_value(prev_pdu.event_id())?, + ); + } } } - } - let pdu = PduEvent { - event_id: ruma::event_id!("$thiswillbefilledinlater").into(), - room_id: room_id.map(ToOwned::to_owned), - sender: sender.to_owned(), - origin: None, - origin_server_ts: timestamp.map_or_else( - || { - utils::millis_since_unix_epoch() - .try_into() - .expect("u64 fits into UInt") + let pdu = PduEvent { + event_id: ruma::event_id!("$thiswillbefilledinlater").into(), + room_id: room_id.map(ToOwned::to_owned), + sender: sender.to_owned(), + origin: None, + origin_server_ts: timestamp.map_or_else( + || { + utils::millis_since_unix_epoch() + .try_into() + .expect("u64 fits into UInt") + }, + |ts| ts.get(), + ), + kind: event_type, + content, + state_key, + prev_events, + depth, + auth_events: auth_events + .values() + .map(|pdu| pdu.event_id.clone()) + .collect(), + redacts, + unsigned: if unsigned.is_empty() { + None + } else { + Some(to_raw_value(&unsigned)?) }, - |ts| ts.get(), - ), - kind: event_type, - content, - state_key, - prev_events, - depth, - auth_events: auth_events - .values() - .map(|pdu| pdu.event_id.clone()) - .collect(), - redacts, - unsigned: if unsigned.is_empty() { - None - } else { - Some(to_raw_value(&unsigned)?) - }, - hashes: EventHash { sha256: String::new() }, - signatures: None, - }; - - let auth_fetch = |k: &StateEventType, s: &str| { - let key = (k.clone(), s.into()); - ready(auth_events.get(&key).map(ToOwned::to_owned)) - }; - - let room_id_or_hash = pdu.room_id_or_hash(); - let create_pdu = match &pdu.kind { - | TimelineEventType::RoomCreate => None, - | _ => Some( - self.services - .state_accessor - .room_state_get(&room_id_or_hash, &StateEventType::RoomCreate, "") - .await - .map_err(|e| { - err!(Request(Forbidden(warn!("Failed to fetch room create event: {e}")))) - })?, - ), - }; - let create_event = match &pdu.kind { - | TimelineEventType::RoomCreate => &pdu, - | _ => create_pdu.as_ref().unwrap().as_pdu(), - }; - - let auth_check = state_res::auth_check( - &room_version_rules, - &pdu, - None, // TODO: third_party_invite - auth_fetch, - create_event, - ) - .await - .map_err(|e| err!(Request(Forbidden(warn!("Auth check failed: {e:?}")))))?; - - if !auth_check { - return Err!(Request(Forbidden("Event is not authorized."))); - } - trace!( - "Event {} in room {} is authorized", - pdu.event_id, - pdu.room_id.as_ref().map_or("None", |id| id.as_str()) - ); - Ok((pdu, room_version_rules)) -} - -#[implement(super::Service)] -pub async fn create_hash_and_sign_event( - &self, - partial_pdu: PartialPdu, - sender: &UserId, - room_id: Option<&RoomId>, - mutex_lock: &RoomMutexGuard, /* Take mutex guard to make sure users get the room - * state mutex */ -) -> Result<(PduEvent, CanonicalJsonObject)> { - if !self.services.globals.user_is_local(sender) { - return Err!(Request(Forbidden("Sender must be a local user"))); - } - let (mut pdu, room_version_rules) = self - .create_event(partial_pdu, sender, room_id, mutex_lock) - .await?; - // Hash and sign - let mut pdu_json = utils::to_canonical_object(&pdu).map_err(|e| { - err!(Request(BadJson(warn!("Failed to convert PDU to canonical JSON: {e}")))) - })?; - pdu_json.remove("event_id"); - - trace!("hashing and signing event {}", pdu.event_id); - if let Err(e) = self - .services - .server_keys - .hash_and_sign_event(&mut pdu_json, &room_version_rules) - { - return match e { - | Error::SignatureJson(ruma::signatures::JsonError::PduTooLarge) => { - Err!(Request(TooLarge("Message/PDU is too long (exceeds 65535 bytes)"))) - }, - | _ => Err!(Request(Unknown(warn!("Signing event failed: {e}")))), + hashes: EventHash { sha256: String::new() }, + signatures: None, }; - } - // Generate event id - pdu.event_id = gen_event_id(&pdu_json, &room_version_rules)?; - pdu_json.insert("event_id".into(), CanonicalJsonValue::String(pdu.event_id.clone().into())); - // Verify that the *full* PDU isn't over 64KiB. - if !pdu_fits(&mut pdu_json.clone()) { - // feckin huge PDU mate - return Err!(Request(TooLarge("Message/PDU is too long (exceeds 65535 bytes)"))); - } - // Check with the policy server - if room_id.is_some() { + let auth_fetch = |k: &StateEventType, s: &str| { + let key = (k.clone(), s.into()); + ready(auth_events.get(&key).map(ToOwned::to_owned)) + }; + + let room_id_or_hash = pdu.room_id_or_hash(); + let create_pdu = match &pdu.kind { + | TimelineEventType::RoomCreate => None, + | _ => Some( + self.services + .state_accessor + .room_state_get(&room_id_or_hash, &StateEventType::RoomCreate, "") + .await + .map_err(|e| { + err!(Request(Forbidden(warn!("Failed to fetch room create event: {e}")))) + })?, + ), + }; + let create_event = match &pdu.kind { + | TimelineEventType::RoomCreate => &pdu, + | _ => create_pdu.as_ref().unwrap().as_pdu(), + }; + + let auth_check = state_res::auth_check( + &room_version_rules, + &pdu, + None, // TODO: third_party_invite + auth_fetch, + create_event, + ) + .await + .map_err(|e| err!(Request(Forbidden(warn!("Auth check failed: {e:?}")))))?; + + if !auth_check { + return Err!(Request(Forbidden("Event is not authorized."))); + } trace!( - "Checking event in room {} with policy server", + "Event {} in room {} is authorized", + pdu.event_id, pdu.room_id.as_ref().map_or("None", |id| id.as_str()) ); - // We need to remove the event ID before getting a PS signature on the event. - // Note that we seemingly pointlessly add it above just to remove it here, but - // it's important to make sure the event ID isn't the field that makes the - // difference between an illegally-large event and one that is okay. - pdu_json.remove("event_id"); - self.services - .event_handler - .policy_server_allows_event( - &pdu, - &mut pdu_json, - pdu.room_id().expect("has room ID"), - &room_version_rules, - false, - ) - .await?; - pdu_json - .insert("event_id".into(), CanonicalJsonValue::String(pdu.event_id.clone().into())); + Ok((pdu, room_version_rules)) } - // Generate short event id - trace!( - "Generating short event ID for {} in room {}", - pdu.event_id, - pdu.room_id.as_ref().map_or("None", |id| id.as_str()) - ); - let _shorteventid = self - .services - .short - .get_or_create_shorteventid(&pdu.event_id) - .await; + /// Creates, checks, hashes, and signs a new PDU. The resulting PDU is + /// immutable. Since the PDU is immutable, the `event_id` field is + /// populated. + /// + /// TODO: The `event_id` field should be a separate return option, not + /// embedded. + pub async fn create_hash_and_sign_event( + &self, + partial_pdu: PartialPdu, + sender: &UserId, + room_id: Option<&RoomId>, + mutex_lock: &RoomMutexGuard, + ) -> Result<(PduEvent, CanonicalJsonObject)> { + if !self.services.globals.user_is_local(sender) { + return Err!(Request(Forbidden("Sender must be a local user"))); + } + let (mut pdu, room_version_rules) = self + .create_event(partial_pdu, sender, room_id, mutex_lock) + .await?; + // Hash and sign + let mut pdu_json = utils::to_canonical_object(&pdu).map_err(|e| { + err!(Request(BadJson(warn!("Failed to convert PDU to canonical JSON: {e}")))) + })?; + pdu_json.remove("event_id"); - trace!("New PDU created: {pdu:?}"); - Ok((pdu, pdu_json)) + trace!("hashing and signing event {}", pdu.event_id); + if let Err(e) = self + .services + .server_keys + .hash_and_sign_event(&mut pdu_json, &room_version_rules) + { + return match e { + | Error::SignatureJson(ruma::signatures::JsonError::PduTooLarge) => { + Err!(Request(TooLarge("Message/PDU is too long (exceeds 65535 bytes)"))) + }, + | _ => Err!(Request(Unknown(warn!("Signing event failed: {e}")))), + }; + } + // Generate event id + pdu.event_id = gen_event_id(&pdu_json, &room_version_rules)?; + pdu_json + .insert("event_id".into(), CanonicalJsonValue::String(pdu.event_id.clone().into())); + // Verify that the *full* PDU isn't over 64KiB. + if !pdu_fits(&mut pdu_json.clone()) { + // feckin huge PDU mate + return Err!(Request(TooLarge("Message/PDU is too long (exceeds 65535 bytes)"))); + } + + // Check with the policy server + if room_id.is_some() { + trace!( + "Checking event in room {} with policy server", + pdu.room_id.as_ref().map_or("None", |id| id.as_str()) + ); + // We need to remove the event ID before getting a PS signature on the event. + // Note that we seemingly pointlessly add it above just to remove it here, but + // it's important to make sure the event ID isn't the field that makes the + // difference between an illegally-large event and one that is okay. + pdu_json.remove("event_id"); + self.services + .event_handler + .policy_server_allows_event( + &pdu, + &mut pdu_json, + pdu.room_id().expect("has room ID"), + &room_version_rules, + false, + ) + .await?; + pdu_json.insert( + "event_id".into(), + CanonicalJsonValue::String(pdu.event_id.clone().into()), + ); + } + + // Generate short event id + trace!( + "Generating short event ID for {} in room {}", + pdu.event_id, + pdu.room_id.as_ref().map_or("None", |id| id.as_str()) + ); + let _shorteventid = self + .services + .short + .get_or_create_shorteventid(&pdu.event_id) + .await; + + trace!("New PDU created: {pdu:?}"); + Ok((pdu, pdu_json)) + } } diff --git a/src/service/rooms/timeline/redact.rs b/src/service/rooms/timeline/redact.rs index b3aff9eca..5509feeb5 100644 --- a/src/service/rooms/timeline/redact.rs +++ b/src/service/rooms/timeline/redact.rs @@ -1,5 +1,5 @@ use conduwuit_core::{ - Result, err, implement, + Result, err, matrix::event::Event, utils::{self}, }; @@ -8,48 +8,49 @@ use ruma::EventId; use super::ExtractBody; use crate::rooms::short::ShortRoomId; -/// Replace a PDU with the redacted form. -#[implement(super::Service)] -#[tracing::instrument(name = "redact", level = "debug", skip(self))] -pub async fn redact_pdu( - &self, - event_id: &EventId, - reason: &Pdu, - shortroomid: ShortRoomId, -) -> Result { - // TODO: Don't reserialize, keep original json - let Ok(pdu_id) = self.get_pdu_id(event_id).await else { - // If event does not exist, just noop - return Ok(()); - }; +impl super::Service { + /// Replace a PDU with the redacted form. + #[tracing::instrument(name = "redact", level = "debug", skip(self))] + pub async fn redact_pdu( + &self, + event_id: &EventId, + reason: &Pdu, + shortroomid: ShortRoomId, + ) -> Result { + // TODO: Don't reserialize, keep original json + let Ok(pdu_id) = self.get_pdu_id(event_id).await else { + // If event does not exist, just noop + return Ok(()); + }; - let mut pdu = self - .get_pdu_from_id(&pdu_id) - .await - .map(Event::into_pdu) - .map_err(|e| { - err!(Database(error!(?pdu_id, %event_id, ?e, "PDU ID points to invalid PDU."))) + let mut pdu = self + .get_pdu_from_id(&pdu_id) + .await + .map(Event::into_pdu) + .map_err(|e| { + err!(Database(error!(?pdu_id, %event_id, ?e, "PDU ID points to invalid PDU."))) + })?; + + if let Ok(content) = pdu.get_content::() { + if let Some(body) = content.body { + self.services + .search + .deindex_pdu(shortroomid, &pdu_id, &body); + } + } + + let room_version_id = self + .services + .state + .get_room_version(&pdu.room_id_or_hash()) + .await?; + + pdu.redact(&room_version_id, reason.to_value())?; + + let obj = utils::to_canonical_object(&pdu).map_err(|e| { + err!(Database(error!(%event_id, ?e, "Failed to convert PDU to canonical JSON"))) })?; - if let Ok(content) = pdu.get_content::() { - if let Some(body) = content.body { - self.services - .search - .deindex_pdu(shortroomid, &pdu_id, &body); - } + self.replace_pdu(&pdu_id, &obj).await } - - let room_version_id = self - .services - .state - .get_room_version(&pdu.room_id_or_hash()) - .await?; - - pdu.redact(&room_version_id, reason.to_value())?; - - let obj = utils::to_canonical_object(&pdu).map_err(|e| { - err!(Database(error!(%event_id, ?e, "Failed to convert PDU to canonical JSON"))) - })?; - - self.replace_pdu(&pdu_id, &obj).await } diff --git a/src/service/rooms/user/mod.rs b/src/service/rooms/user/mod.rs index c6a9f9cca..9705e2e3f 100644 --- a/src/service/rooms/user/mod.rs +++ b/src/service/rooms/user/mod.rs @@ -1,6 +1,6 @@ use std::sync::Arc; -use conduwuit::{Result, implement}; +use conduwuit::Result; use database::{Deserialized, Map}; use ruma::{RoomId, UserId}; @@ -38,48 +38,51 @@ impl crate::Service for Service { fn name(&self) -> &str { crate::service::make_name(std::module_path!()) } } -#[implement(Service)] -pub fn reset_notification_counts(&self, user_id: &UserId, room_id: &RoomId) { - let userroom_id = (user_id, room_id); - self.db.userroomid_highlightcount.put(userroom_id, 0_u64); - self.db.userroomid_notificationcount.put(userroom_id, 0_u64); +impl Service { + /// Resets the notification counts for a room the user is in. + pub fn reset_notification_counts(&self, user_id: &UserId, room_id: &RoomId) { + let userroom_id = (user_id, room_id); + self.db.userroomid_highlightcount.put(userroom_id, 0_u64); + self.db.userroomid_notificationcount.put(userroom_id, 0_u64); - let roomuser_id = (room_id, user_id); - let count = self.services.globals.next_count().unwrap(); - self.db - .roomuserid_lastnotificationread - .put(roomuser_id, count); -} + let roomuser_id = (room_id, user_id); + let count = self.services.globals.next_count().unwrap(); + self.db + .roomuserid_lastnotificationread + .put(roomuser_id, count); + } -#[implement(Service)] -pub async fn notification_count(&self, user_id: &UserId, room_id: &RoomId) -> u64 { - let key = (user_id, room_id); - self.db - .userroomid_notificationcount - .qry(&key) - .await - .deserialized() - .unwrap_or(0) -} + /// Gets the notification count for a room the user is in. + pub async fn notification_count(&self, user_id: &UserId, room_id: &RoomId) -> u64 { + let key = (user_id, room_id); + self.db + .userroomid_notificationcount + .qry(&key) + .await + .deserialized() + .unwrap_or(0) + } -#[implement(Service)] -pub async fn highlight_count(&self, user_id: &UserId, room_id: &RoomId) -> u64 { - let key = (user_id, room_id); - self.db - .userroomid_highlightcount - .qry(&key) - .await - .deserialized() - .unwrap_or(0) -} + /// Gets the number of events that highlighted the user in a given room. + /// These aren't necessarily notifications. + pub async fn highlight_count(&self, user_id: &UserId, room_id: &RoomId) -> u64 { + let key = (user_id, room_id); + self.db + .userroomid_highlightcount + .qry(&key) + .await + .deserialized() + .unwrap_or(0) + } -#[implement(Service)] -pub async fn last_notification_read(&self, user_id: &UserId, room_id: &RoomId) -> u64 { - let key = (room_id, user_id); - self.db - .roomuserid_lastnotificationread - .qry(&key) - .await - .deserialized() - .unwrap_or(0) + /// Returns the last notification the user read in the room, or 0 if none. + pub async fn last_notification_read(&self, user_id: &UserId, room_id: &RoomId) -> u64 { + let key = (room_id, user_id); + self.db + .roomuserid_lastnotificationread + .qry(&key) + .await + .deserialized() + .unwrap_or(0) + } } From 6603798d60d9710cdfd1d1ce91dc3d2709836d38 Mon Sep 17 00:00:00 2001 From: timedout Date: Fri, 26 Jun 2026 03:14:11 +0100 Subject: [PATCH 34/70] style: Document & refactor rooms/state_compressor service --- src/service/rooms/state_compressor/mod.rs | 717 +++++++++++----------- 1 file changed, 354 insertions(+), 363 deletions(-) diff --git a/src/service/rooms/state_compressor/mod.rs b/src/service/rooms/state_compressor/mod.rs index 039429a36..d0919aefb 100644 --- a/src/service/rooms/state_compressor/mod.rs +++ b/src/service/rooms/state_compressor/mod.rs @@ -9,7 +9,7 @@ use async_trait::async_trait; use conduwuit::{ Result, SyncMutex, arrayvec::ArrayVec, - at, checked, err, expected, implement, utils, + at, checked, err, expected, utils, utils::{bytes, math::usize_from_f64, stream::IterStream}, }; use database::Map; @@ -115,400 +115,390 @@ impl crate::Service for Service { fn name(&self) -> &str { crate::service::make_name(std::module_path!()) } } -/// Returns a stack with info on shortstatehash, full state, added diff and -/// removed diff for the selected shortstatehash and each parent layer. -#[implement(Service)] -#[tracing::instrument(name = "load", level = "debug", skip(self))] -pub async fn load_shortstatehash_info( - &self, - shortstatehash: ShortStateHash, -) -> Result { - if let Some(r) = self.stateinfo_cache.lock().get_mut(&shortstatehash) { - return Ok(r.clone()); +impl Service { + /// Returns a stack with info on shortstatehash, full state, added diff and + /// removed diff for the selected shortstatehash and each parent layer. + pub async fn load_shortstatehash_info( + &self, + shortstatehash: ShortStateHash, + ) -> Result { + if let Some(r) = self.stateinfo_cache.lock().get_mut(&shortstatehash) { + return Ok(r.clone()); + } + + let stack = self.new_shortstatehash_info(shortstatehash).await?; + + self.cache_shortstatehash_info(shortstatehash, stack.clone()) + .await?; + + Ok(stack) } - let stack = self.new_shortstatehash_info(shortstatehash).await?; + /// Returns a stack with info on shortstatehash, full state, added diff and + /// removed diff for the selected shortstatehash and each parent layer. + async fn cache_shortstatehash_info( + &self, + shortstatehash: ShortStateHash, + stack: ShortStateInfoVec, + ) -> Result { + self.stateinfo_cache.lock().insert(shortstatehash, stack); - self.cache_shortstatehash_info(shortstatehash, stack.clone()) - .await?; + Ok(()) + } - Ok(stack) -} + /// Inserts a new shortstatehash info entry. + async fn new_shortstatehash_info( + &self, + shortstatehash: ShortStateHash, + ) -> Result { + let StateDiff { parent, added, removed } = self.get_statediff(shortstatehash).await?; -/// Returns a stack with info on shortstatehash, full state, added diff and -/// removed diff for the selected shortstatehash and each parent layer. -#[implement(Service)] -#[tracing::instrument( - name = "cache", - level = "debug", - skip_all, - fields( - ?shortstatehash, - stack = stack.len(), - ), - )] -async fn cache_shortstatehash_info( - &self, - shortstatehash: ShortStateHash, - stack: ShortStateInfoVec, -) -> Result { - self.stateinfo_cache.lock().insert(shortstatehash, stack); + let Some(parent) = parent else { + return Ok(vec![ShortStateInfo { + shortstatehash, + full_state: added.clone(), + added, + removed, + }]); + }; - Ok(()) -} + let mut stack = Box::pin(self.load_shortstatehash_info(parent)).await?; + let top = stack.last().expect("at least one frame"); -#[implement(Service)] -async fn new_shortstatehash_info( - &self, - shortstatehash: ShortStateHash, -) -> Result { - let StateDiff { parent, added, removed } = self.get_statediff(shortstatehash).await?; + let mut full_state = (*top.full_state).clone(); + full_state.extend(added.iter().copied()); - let Some(parent) = parent else { - return Ok(vec![ShortStateInfo { + let removed = (*removed).clone(); + for r in &removed { + full_state.remove(r); + } + + stack.push(ShortStateInfo { shortstatehash, - full_state: added.clone(), added, - removed, - }]); - }; + removed: Arc::new(removed), + full_state: Arc::new(full_state), + }); - let mut stack = Box::pin(self.load_shortstatehash_info(parent)).await?; - let top = stack.last().expect("at least one frame"); - - let mut full_state = (*top.full_state).clone(); - full_state.extend(added.iter().copied()); - - let removed = (*removed).clone(); - for r in &removed { - full_state.remove(r); + Ok(stack) } - stack.push(ShortStateInfo { - shortstatehash, - added, - removed: Arc::new(removed), - full_state: Arc::new(full_state), - }); + /// Compresses the input state event stream + pub fn compress_state_events<'a, I>( + &'a self, + state: I, + ) -> impl Stream + Send + 'a + where + I: Iterator + Clone + Debug + Send + 'a, + { + let event_ids = state.clone().map(at!(1)); - Ok(stack) -} + let short_event_ids = self + .services + .short + .multi_get_or_create_shorteventid(event_ids); -#[implement(Service)] -pub fn compress_state_events<'a, I>( - &'a self, - state: I, -) -> impl Stream + Send + 'a -where - I: Iterator + Clone + Debug + Send + 'a, -{ - let event_ids = state.clone().map(at!(1)); + state + .stream() + .map(at!(0)) + .zip(short_event_ids) + .map(|(shortstatekey, shorteventid)| { + compress_state_event(*shortstatekey, shorteventid) + }) + } - let short_event_ids = self - .services - .short - .multi_get_or_create_shorteventid(event_ids); + /// Compresses a single state event by creating a short event ID for it. + pub async fn compress_state_event( + &self, + shortstatekey: ShortStateKey, + event_id: &EventId, + ) -> CompressedStateEvent { + let shorteventid = self + .services + .short + .get_or_create_shorteventid(event_id) + .await; - state - .stream() - .map(at!(0)) - .zip(short_event_ids) - .map(|(shortstatekey, shorteventid)| compress_state_event(*shortstatekey, shorteventid)) -} + compress_state_event(shortstatekey, shorteventid) + } -#[implement(Service)] -pub async fn compress_state_event( - &self, - shortstatekey: ShortStateKey, - event_id: &EventId, -) -> CompressedStateEvent { - let shorteventid = self - .services - .short - .get_or_create_shorteventid(event_id) - .await; + /// Creates a new shortstatehash that often is just a diff to an already + /// existing shortstatehash and therefore very efficient. + /// + /// There are multiple layers of diffs. The bottom layer 0 always contains + /// the full state. Layer 1 contains diffs to states of layer 0, layer 2 + /// diffs to layer 1 and so on. If layer n > 0 grows too big, it will be + /// combined with layer n-1 to create a new diff on layer n-1 that's + /// based on layer n-2. If that layer is also too big, it will recursively + /// fix above layers too. + /// + /// * `shortstatehash` - Shortstatehash of this state + /// * `statediffnew` - Added to base. Each vec is shortstatekey+shorteventid + /// * `statediffremoved` - Removed from base. Each vec is + /// shortstatekey+shorteventid + /// * `diff_to_sibling` - Approximately how much the diff grows each time + /// for this layer + /// * `parent_states` - A stack with info on shortstatehash, full state, + /// added diff and removed diff for each parent layer + pub fn save_state_from_diff( + &self, + shortstatehash: ShortStateHash, + statediffnew: Arc, + statediffremoved: Arc, + diff_to_sibling: usize, + mut parent_states: ParentStatesVec, + ) -> Result { + let statediffnew_len = statediffnew.len(); + let statediffremoved_len = statediffremoved.len(); + let diffsum = checked!(statediffnew_len + statediffremoved_len)?; - compress_state_event(shortstatekey, shorteventid) -} + if parent_states.len() > 3 { + // Number of layers + // To many layers, we have to go deeper + let parent = parent_states.pop().expect("parent must have a state"); -/// Creates a new shortstatehash that often is just a diff to an already -/// existing shortstatehash and therefore very efficient. -/// -/// There are multiple layers of diffs. The bottom layer 0 always contains -/// the full state. Layer 1 contains diffs to states of layer 0, layer 2 -/// diffs to layer 1 and so on. If layer n > 0 grows too big, it will be -/// combined with layer n-1 to create a new diff on layer n-1 that's -/// based on layer n-2. If that layer is also too big, it will recursively -/// fix above layers too. -/// -/// * `shortstatehash` - Shortstatehash of this state -/// * `statediffnew` - Added to base. Each vec is shortstatekey+shorteventid -/// * `statediffremoved` - Removed from base. Each vec is -/// shortstatekey+shorteventid -/// * `diff_to_sibling` - Approximately how much the diff grows each time for -/// this layer -/// * `parent_states` - A stack with info on shortstatehash, full state, added -/// diff and removed diff for each parent layer -#[implement(Service)] -pub fn save_state_from_diff( - &self, - shortstatehash: ShortStateHash, - statediffnew: Arc, - statediffremoved: Arc, - diff_to_sibling: usize, - mut parent_states: ParentStatesVec, -) -> Result { - let statediffnew_len = statediffnew.len(); - let statediffremoved_len = statediffremoved.len(); - let diffsum = checked!(statediffnew_len + statediffremoved_len)?; + let mut parent_new = (*parent.added).clone(); + let mut parent_removed = (*parent.removed).clone(); + + for removed in statediffremoved.iter() { + if !parent_new.remove(removed) { + // It was not added in the parent and we removed it + parent_removed.insert(*removed); + } + // Else it was added in the parent and we removed it again. We + // can forget this change + } + + for new in statediffnew.iter() { + if !parent_removed.remove(new) { + // It was not touched in the parent and we added it + parent_new.insert(*new); + } + // Else it was removed in the parent and we added it again. We + // can forget this change + } + + self.save_state_from_diff( + shortstatehash, + Arc::new(parent_new), + Arc::new(parent_removed), + diffsum, + parent_states, + )?; + + return Ok(()); + } + + if parent_states.is_empty() { + // There is no parent layer, create a new state + self.save_statediff(shortstatehash, &StateDiff { + parent: None, + added: statediffnew, + removed: statediffremoved, + }); + + return Ok(()); + } + + // Else we have two options. + // 1. We add the current diff on top of the parent layer. + // 2. We replace a layer above - if parent_states.len() > 3 { - // Number of layers - // To many layers, we have to go deeper let parent = parent_states.pop().expect("parent must have a state"); + let parent_added_len = parent.added.len(); + let parent_removed_len = parent.removed.len(); + let parent_diff = checked!(parent_added_len + parent_removed_len)?; - let mut parent_new = (*parent.added).clone(); - let mut parent_removed = (*parent.removed).clone(); + if checked!(diffsum * diffsum)? >= checked!(2 * diff_to_sibling * parent_diff)? { + // Diff too big, we replace above layer(s) + let mut parent_new = (*parent.added).clone(); + let mut parent_removed = (*parent.removed).clone(); - for removed in statediffremoved.iter() { - if !parent_new.remove(removed) { - // It was not added in the parent and we removed it - parent_removed.insert(*removed); + for removed in statediffremoved.iter() { + if !parent_new.remove(removed) { + // It was not added in the parent and we removed it + parent_removed.insert(*removed); + } + // Else it was added in the parent and we removed it again. We + // can forget this change } - // Else it was added in the parent and we removed it again. We - // can forget this change - } - for new in statediffnew.iter() { - if !parent_removed.remove(new) { - // It was not touched in the parent and we added it - parent_new.insert(*new); + for new in statediffnew.iter() { + if !parent_removed.remove(new) { + // It was not touched in the parent and we added it + parent_new.insert(*new); + } + // Else it was removed in the parent and we added it again. We + // can forget this change } - // Else it was removed in the parent and we added it again. We - // can forget this change - } - self.save_state_from_diff( - shortstatehash, - Arc::new(parent_new), - Arc::new(parent_removed), - diffsum, - parent_states, - )?; - - return Ok(()); - } - - if parent_states.is_empty() { - // There is no parent layer, create a new state - self.save_statediff(shortstatehash, &StateDiff { - parent: None, - added: statediffnew, - removed: statediffremoved, - }); - - return Ok(()); - } - - // Else we have two options. - // 1. We add the current diff on top of the parent layer. - // 2. We replace a layer above - - let parent = parent_states.pop().expect("parent must have a state"); - let parent_added_len = parent.added.len(); - let parent_removed_len = parent.removed.len(); - let parent_diff = checked!(parent_added_len + parent_removed_len)?; - - if checked!(diffsum * diffsum)? >= checked!(2 * diff_to_sibling * parent_diff)? { - // Diff too big, we replace above layer(s) - let mut parent_new = (*parent.added).clone(); - let mut parent_removed = (*parent.removed).clone(); - - for removed in statediffremoved.iter() { - if !parent_new.remove(removed) { - // It was not added in the parent and we removed it - parent_removed.insert(*removed); - } - // Else it was added in the parent and we removed it again. We - // can forget this change - } - - for new in statediffnew.iter() { - if !parent_removed.remove(new) { - // It was not touched in the parent and we added it - parent_new.insert(*new); - } - // Else it was removed in the parent and we added it again. We - // can forget this change - } - - self.save_state_from_diff( - shortstatehash, - Arc::new(parent_new), - Arc::new(parent_removed), - diffsum, - parent_states, - )?; - } else { - // Diff small enough, we add diff as layer on top of parent - self.save_statediff(shortstatehash, &StateDiff { - parent: Some(parent.shortstatehash), - added: statediffnew, - removed: statediffremoved, - }); - } - - Ok(()) -} - -/// Returns the new shortstatehash, and the state diff from the previous -/// room state -#[implement(Service)] -#[tracing::instrument(skip(self, new_state_ids_compressed), level = "debug")] -pub async fn save_state( - &self, - room_id: &RoomId, - new_state_ids_compressed: Arc, -) -> Result { - let previous_shortstatehash = self - .services - .state - .get_room_shortstatehash(room_id) - .await - .ok(); - - let state_hash = - utils::calculate_hash(new_state_ids_compressed.iter().map(|bytes| &bytes[..])); - - let (new_shortstatehash, already_existed) = self - .services - .short - .get_or_create_shortstatehash(&state_hash) - .await; - - if Some(new_shortstatehash) == previous_shortstatehash { - return Ok(HashSetCompressStateEvent { - shortstatehash: new_shortstatehash, - ..Default::default() - }); - } - - let states_parents = if let Some(p) = previous_shortstatehash { - self.load_shortstatehash_info(p).await.unwrap_or_default() - } else { - ShortStateInfoVec::new() - }; - - let (statediffnew, statediffremoved) = if let Some(parent_stateinfo) = states_parents.last() { - let statediffnew: CompressedState = new_state_ids_compressed - .difference(&parent_stateinfo.full_state) - .copied() - .collect(); - - let statediffremoved: CompressedState = parent_stateinfo - .full_state - .difference(&new_state_ids_compressed) - .copied() - .collect(); - - (Arc::new(statediffnew), Arc::new(statediffremoved)) - } else { - (new_state_ids_compressed, Arc::new(CompressedState::new())) - }; - - if !already_existed { - self.save_state_from_diff( - new_shortstatehash, - statediffnew.clone(), - statediffremoved.clone(), - 2, // every state change is 2 event changes on average - states_parents, - )?; - } - - Ok(HashSetCompressStateEvent { - shortstatehash: new_shortstatehash, - added: statediffnew, - removed: statediffremoved, - }) -} - -#[implement(Service)] -#[tracing::instrument(skip(self), level = "debug", name = "get")] -async fn get_statediff(&self, shortstatehash: ShortStateHash) -> Result { - const BUFSIZE: usize = size_of::(); - const STRIDE: usize = size_of::(); - - let value = self - .db - .shortstatehash_statediff - .aqry::(&shortstatehash) - .await - .map_err(|e| { - err!(Database("Failed to find StateDiff from short {shortstatehash:?}: {e}")) - })?; - - let parent = utils::u64_from_bytes(&value[0..size_of::()]) - .ok() - .take_if(|parent| *parent != 0); - - debug_assert!(value.len().is_multiple_of(STRIDE), "value not aligned to stride"); - let _num_values = value.len() / STRIDE; - - let mut add_mode = true; - let mut added = CompressedState::new(); - let mut removed = CompressedState::new(); - - let mut i = STRIDE; - while let Some(v) = value.get(i..expected!(i + 2 * STRIDE)) { - if add_mode && v.starts_with(&0_u64.to_be_bytes()) { - add_mode = false; - i = expected!(i + STRIDE); - continue; - } - if add_mode { - added.insert(v.try_into()?); + self.save_state_from_diff( + shortstatehash, + Arc::new(parent_new), + Arc::new(parent_removed), + diffsum, + parent_states, + )?; } else { - removed.insert(v.try_into()?); + // Diff small enough, we add diff as layer on top of parent + self.save_statediff(shortstatehash, &StateDiff { + parent: Some(parent.shortstatehash), + added: statediffnew, + removed: statediffremoved, + }); } - i = expected!(i + 2 * STRIDE); + + Ok(()) } - Ok(StateDiff { - parent, - added: Arc::new(added), - removed: Arc::new(removed), - }) + /// Returns the new shortstatehash, and the state diff from the previous + /// room state + pub async fn save_state( + &self, + room_id: &RoomId, + new_state_ids_compressed: Arc, + ) -> Result { + let previous_shortstatehash = self + .services + .state + .get_room_shortstatehash(room_id) + .await + .ok(); + + let state_hash = + utils::calculate_hash(new_state_ids_compressed.iter().map(|bytes| &bytes[..])); + + let (new_shortstatehash, already_existed) = self + .services + .short + .get_or_create_shortstatehash(&state_hash) + .await; + + if Some(new_shortstatehash) == previous_shortstatehash { + return Ok(HashSetCompressStateEvent { + shortstatehash: new_shortstatehash, + ..Default::default() + }); + } + + let states_parents = if let Some(p) = previous_shortstatehash { + self.load_shortstatehash_info(p).await.unwrap_or_default() + } else { + ShortStateInfoVec::new() + }; + + let (statediffnew, statediffremoved) = + if let Some(parent_stateinfo) = states_parents.last() { + let statediffnew: CompressedState = new_state_ids_compressed + .difference(&parent_stateinfo.full_state) + .copied() + .collect(); + + let statediffremoved: CompressedState = parent_stateinfo + .full_state + .difference(&new_state_ids_compressed) + .copied() + .collect(); + + (Arc::new(statediffnew), Arc::new(statediffremoved)) + } else { + (new_state_ids_compressed, Arc::new(CompressedState::new())) + }; + + if !already_existed { + self.save_state_from_diff( + new_shortstatehash, + statediffnew.clone(), + statediffremoved.clone(), + 2, // every state change is 2 event changes on average + states_parents, + )?; + } + + Ok(HashSetCompressStateEvent { + shortstatehash: new_shortstatehash, + added: statediffnew, + removed: statediffremoved, + }) + } + + /// Gets the diff uin by the shortstatehash. The diff is a set of added and + /// removed + async fn get_statediff(&self, shortstatehash: ShortStateHash) -> Result { + const BUFSIZE: usize = size_of::(); + const STRIDE: usize = size_of::(); + + let value = self + .db + .shortstatehash_statediff + .aqry::(&shortstatehash) + .await + .map_err(|e| { + err!(Database("Failed to find StateDiff from short {shortstatehash:?}: {e}")) + })?; + + let parent = utils::u64_from_bytes(&value[0..size_of::()]) + .ok() + .take_if(|parent| *parent != 0); + + debug_assert!(value.len().is_multiple_of(STRIDE), "value not aligned to stride"); + let _num_values = value.len() / STRIDE; + + let mut add_mode = true; + let mut added = CompressedState::new(); + let mut removed = CompressedState::new(); + + let mut i = STRIDE; + while let Some(v) = value.get(i..expected!(i + 2 * STRIDE)) { + if add_mode && v.starts_with(&0_u64.to_be_bytes()) { + add_mode = false; + i = expected!(i + STRIDE); + continue; + } + if add_mode { + added.insert(v.try_into()?); + } else { + removed.insert(v.try_into()?); + } + i = expected!(i + 2 * STRIDE); + } + + Ok(StateDiff { + parent, + added: Arc::new(added), + removed: Arc::new(removed), + }) + } + + /// Stores a new state diff with a shortstatehash. + fn save_statediff(&self, shortstatehash: ShortStateHash, diff: &StateDiff) { + let mut value = Vec::::with_capacity( + 2_usize + .saturating_add(diff.added.len()) + .saturating_add(diff.removed.len()), + ); + + let parent = diff.parent.unwrap_or(0_u64); + value.extend_from_slice(&parent.to_be_bytes()); + + for new in diff.added.iter() { + value.extend_from_slice(&new[..]); + } + + if !diff.removed.is_empty() { + value.extend_from_slice(&0_u64.to_be_bytes()); + for removed in diff.removed.iter() { + value.extend_from_slice(&removed[..]); + } + } + + self.db + .shortstatehash_statediff + .insert(&shortstatehash.to_be_bytes(), &value); + } } -#[implement(Service)] -fn save_statediff(&self, shortstatehash: ShortStateHash, diff: &StateDiff) { - let mut value = Vec::::with_capacity( - 2_usize - .saturating_add(diff.added.len()) - .saturating_add(diff.removed.len()), - ); - - let parent = diff.parent.unwrap_or(0_u64); - value.extend_from_slice(&parent.to_be_bytes()); - - for new in diff.added.iter() { - value.extend_from_slice(&new[..]); - } - - if !diff.removed.is_empty() { - value.extend_from_slice(&0_u64.to_be_bytes()); - for removed in diff.removed.iter() { - value.extend_from_slice(&removed[..]); - } - } - - self.db - .shortstatehash_statediff - .insert(&shortstatehash.to_be_bytes(), &value); -} - -#[inline] +/// Compresses a single state event. #[must_use] pub(crate) fn compress_state_event( shortstatekey: ShortStateKey, @@ -524,7 +514,8 @@ pub(crate) fn compress_state_event( .expect("failed to create CompressedStateEvent") } -#[inline] +/// Parses a compressed state event identifier into a (ssh, short eventid) +/// tuple. #[must_use] pub fn parse_compressed_state_event( compressed_event: CompressedStateEvent, @@ -537,7 +528,7 @@ pub fn parse_compressed_state_event( (shortstatekey, shorteventid) } -#[inline] +/// Gets the size of a compressed state group. fn compressed_state_size(compressed_state: &CompressedState) -> usize { compressed_state .len() From 4844c6ae3e63d06f45761cf37d905c50ccb0ec4c Mon Sep 17 00:00:00 2001 From: timedout Date: Fri, 26 Jun 2026 03:54:10 +0100 Subject: [PATCH 35/70] style: Document & refactor rooms/state_cache service --- src/service/rooms/state_cache/mod.rs | 775 +++++++++++------------- src/service/rooms/state_cache/update.rs | 623 ++++++++++--------- src/service/rooms/state_cache/via.rs | 139 +++-- 3 files changed, 745 insertions(+), 792 deletions(-) diff --git a/src/service/rooms/state_cache/mod.rs b/src/service/rooms/state_cache/mod.rs index c54970638..46ef9a769 100644 --- a/src/service/rooms/state_cache/mod.rs +++ b/src/service/rooms/state_cache/mod.rs @@ -4,10 +4,9 @@ mod via; use std::{collections::HashMap, sync::Arc}; use conduwuit::{ - Pdu, Result, SyncRwLock, implement, + Pdu, Result, SyncRwLock, result::LogErr, utils::{ReadyExt, stream::TryIgnore}, - warn, }; use database::{Deserialized, Ignore, Interfix, Map}; use futures::{Stream, StreamExt, future::join5, pin_mut}; @@ -93,448 +92,414 @@ impl crate::Service for Service { fn name(&self) -> &str { crate::service::make_name(std::module_path!()) } } -#[implement(Service)] -#[tracing::instrument(level = "trace", skip_all)] -pub async fn appservice_in_room(&self, room_id: &RoomId, appservice: &RegistrationInfo) -> bool { - if let Some(cached) = self - .appservice_in_room_cache - .read() - .get(room_id) - .and_then(|map| map.get(&appservice.registration.id)) - .copied() - { - return cached; +impl Service { + /// Checks if an appservice (or any user in its claimed namespace) is in the + /// target room. + pub async fn appservice_in_room( + &self, + room_id: &RoomId, + appservice: &RegistrationInfo, + ) -> bool { + if let Some(cached) = self + .appservice_in_room_cache + .read() + .get(room_id) + .and_then(|map| map.get(&appservice.registration.id)) + .copied() + { + return cached; + } + + let appservice_user_id = UserId::parse_with_server_name( + appservice.registration.sender_localpart.as_str(), + self.services.globals.server_name(), + ); + + let Ok(bridge_user_id) = appservice_user_id.log_err() else { + return false; + }; + + let (as_user_joined, any_as_namespace_user_joined) = tokio::join!( + self.is_joined(&bridge_user_id, room_id), + self.room_members(room_id) + .ready_any(|user_id| appservice.users.is_match(user_id.as_str())) + ); + let in_room = as_user_joined || any_as_namespace_user_joined; + + self.appservice_in_room_cache + .write() + .entry(room_id.into()) + .or_default() + .insert(appservice.registration.id.clone(), in_room); + + in_room } - let bridge_user_id = UserId::parse_with_server_name( - appservice.registration.sender_localpart.as_str(), - self.services.globals.server_name(), - ); + /// Gets the size of the AS joined room cache and its capacity. + pub fn get_appservice_in_room_cache_usage(&self) -> (usize, usize) { + let cache = self.appservice_in_room_cache.read(); - let Ok(bridge_user_id) = bridge_user_id.log_err() else { - return false; - }; + (cache.len(), cache.capacity()) + } - let in_room = self.is_joined(&bridge_user_id, room_id).await - || self - .room_members(room_id) - .ready_any(|user_id| appservice.users.is_match(user_id.as_str())) - .await; + /// Clears the internal cache of appservice joined rooms. + pub fn clear_appservice_in_room_cache(&self) { + self.appservice_in_room_cache.write().clear(); + } - self.appservice_in_room_cache - .write() - .entry(room_id.into()) - .or_default() - .insert(appservice.registration.id.clone(), in_room); + /// Returns an iterator of all servers participating in this room. + pub fn room_servers<'a>( + &'a self, + room_id: &'a RoomId, + ) -> impl Stream + Send + 'a { + let prefix = (room_id, Interfix); + self.db + .roomserverids + .keys_prefix(&prefix) + .ignore_err() + .map(|(_, server): (Ignore, OwnedServerName)| server) + } - in_room -} + /// Determines if a server name has at least one joined member in a room. + pub async fn server_in_room<'a>( + &'a self, + server: &'a ServerName, + room_id: &'a RoomId, + ) -> bool { + let key = (server, room_id); + self.db.serverroomids.qry(&key).await.is_ok() + } -#[implement(Service)] -pub fn get_appservice_in_room_cache_usage(&self) -> (usize, usize) { - let cache = self.appservice_in_room_cache.read(); + /// Returns an iterator of all rooms a server participates in (as far as we + /// know). + pub fn server_rooms<'a>( + &'a self, + server: &'a ServerName, + ) -> impl Stream + Send + 'a { + let prefix = (server, Interfix); + self.db + .serverroomids + .keys_prefix(&prefix) + .ignore_err() + .map(|(_, room_id): (Ignore, OwnedRoomId)| room_id) + } - (cache.len(), cache.capacity()) -} + /// Returns true if server can see user by sharing at least one room. + pub async fn server_sees_user(&self, server: &ServerName, user_id: &UserId) -> bool { + self.server_rooms(server) + .any(async |room_id| self.is_joined(user_id, &room_id).await) + .await + } -#[implement(Service)] -#[tracing::instrument(level = "debug", skip_all)] -pub fn clear_appservice_in_room_cache(&self) { self.appservice_in_room_cache.write().clear(); } + /// Returns true if user_a and user_b share at least one room. + pub async fn user_sees_user(&self, user_a: &UserId, user_b: &UserId) -> bool { + let get_shared_rooms = self.get_shared_rooms(user_a, user_b); -/// Returns an iterator of all servers participating in this room. -#[implement(Service)] -#[tracing::instrument(skip(self), level = "debug")] -pub fn room_servers<'a>( - &'a self, - room_id: &'a RoomId, -) -> impl Stream + Send + 'a { - let prefix = (room_id, Interfix); - self.db - .roomserverids - .keys_prefix(&prefix) - .ignore_err() - .map(|(_, server): (Ignore, OwnedServerName)| server) -} + pin_mut!(get_shared_rooms); + get_shared_rooms.next().await.is_some() + } -#[implement(Service)] -#[tracing::instrument(skip(self), level = "trace")] -pub async fn server_in_room<'a>(&'a self, server: &'a ServerName, room_id: &'a RoomId) -> bool { - let key = (server, room_id); - self.db.serverroomids.qry(&key).await.is_ok() -} + /// Lists the rooms common between two users. + pub fn get_shared_rooms<'a>( + &'a self, + user_a: &'a UserId, + user_b: &'a UserId, + ) -> impl Stream + Send + 'a { + use conduwuit::utils::set; -/// Returns an iterator of all rooms a server participates in (as far as we -/// know). -#[implement(Service)] -#[tracing::instrument(skip(self), level = "debug")] -pub fn server_rooms<'a>( - &'a self, - server: &'a ServerName, -) -> impl Stream + Send + 'a { - let prefix = (server, Interfix); - self.db - .serverroomids - .keys_prefix(&prefix) - .ignore_err() - .map(|(_, room_id): (Ignore, OwnedRoomId)| room_id) -} + let a = self.rooms_joined(user_a); + let b = self.rooms_joined(user_b); + set::intersection_sorted_stream2(a, b) + } -/// Returns true if server can see user by sharing at least one room. -#[implement(Service)] -#[tracing::instrument(skip(self), level = "trace")] -pub async fn server_sees_user(&self, server: &ServerName, user_id: &UserId) -> bool { - self.server_rooms(server) - .any(async |room_id| self.is_joined(user_id, &room_id).await) - .await -} + /// Returns an iterator of all joined members of a room. + pub fn room_members<'a>( + &'a self, + room_id: &'a RoomId, + ) -> impl Stream + Send + 'a { + let prefix = (room_id, Interfix); + self.db + .roomuserid_joined + .keys_prefix(&prefix) + .ignore_err() + .map(|(_, user_id): (Ignore, OwnedUserId)| user_id) + } -/// Returns true if user_a and user_b share at least one room. -#[implement(Service)] -#[tracing::instrument(skip(self), level = "trace")] -pub async fn user_sees_user(&self, user_a: &UserId, user_b: &UserId) -> bool { - let get_shared_rooms = self.get_shared_rooms(user_a, user_b); + /// Returns the number of users which are currently in a room. + pub async fn room_joined_count(&self, room_id: &RoomId) -> Result { + self.db.roomid_joinedcount.get(room_id).await.deserialized() + } - pin_mut!(get_shared_rooms); - get_shared_rooms.next().await.is_some() -} + /// Returns an iterator of all our local users in the room, even if they're + /// deactivated. + pub fn local_users_in_room<'a>( + &'a self, + room_id: &'a RoomId, + ) -> impl Stream + Send + 'a { + self.room_members(room_id) + .ready_filter(|user| self.services.globals.user_is_local(user)) + } -/// List the rooms common between two users -#[implement(Service)] -#[tracing::instrument(skip(self), level = "debug")] -pub fn get_shared_rooms<'a>( - &'a self, - user_a: &'a UserId, - user_b: &'a UserId, -) -> impl Stream + Send + 'a { - use conduwuit::utils::set; + /// Returns an iterator of all our local joined users in a room who are + /// active (not deactivated) + pub fn active_local_users_in_room<'a>( + &'a self, + room_id: &'a RoomId, + ) -> impl Stream + Send + 'a { + self.local_users_in_room(room_id) + .filter_map(async |user_id| { + if self.services.users.is_active(&user_id).await { + Some(user_id) + } else { + None + } + }) + } - let a = self.rooms_joined(user_a); - let b = self.rooms_joined(user_b); - set::intersection_sorted_stream2(a, b) -} + /// Returns the number of users which are currently invited to a room + pub async fn room_invited_count(&self, room_id: &RoomId) -> Result { + self.db + .roomid_invitedcount + .get(room_id) + .await + .deserialized() + } -/// Returns an iterator of all joined members of a room. -#[implement(Service)] -#[tracing::instrument(skip(self), level = "debug")] -pub fn room_members<'a>( - &'a self, - room_id: &'a RoomId, -) -> impl Stream + Send + 'a { - let prefix = (room_id, Interfix); - self.db - .roomuserid_joined - .keys_prefix(&prefix) - .ignore_err() - .map(|(_, user_id): (Ignore, OwnedUserId)| user_id) -} + /// Returns an iterator over all User IDs who ever joined a room, even if + /// they are no longer joined. + pub fn room_useroncejoined<'a>( + &'a self, + room_id: &'a RoomId, + ) -> impl Stream + Send + 'a { + let prefix = (room_id, Interfix); + self.db + .roomuseroncejoinedids + .keys_prefix(&prefix) + .ignore_err() + .map(|(_, user_id): (Ignore, OwnedUserId)| user_id) + } -/// Returns the number of users which are currently in a room -#[implement(Service)] -#[tracing::instrument(skip(self), level = "trace")] -pub async fn room_joined_count(&self, room_id: &RoomId) -> Result { - self.db.roomid_joinedcount.get(room_id).await.deserialized() -} + /// Returns an iterator over all invited members of a room. + pub fn room_members_invited<'a>( + &'a self, + room_id: &'a RoomId, + ) -> impl Stream + Send + 'a { + let prefix = (room_id, Interfix); + self.db + .roomuserid_invitecount + .keys_prefix(&prefix) + .ignore_err() + .map(|(_, user_id): (Ignore, OwnedUserId)| user_id) + } -#[implement(Service)] -#[tracing::instrument(skip(self), level = "debug")] -/// Returns an iterator of all our local users in the room, even if they're -/// deactivated -pub fn local_users_in_room<'a>( - &'a self, - room_id: &'a RoomId, -) -> impl Stream + Send + 'a { - self.room_members(room_id) - .ready_filter(|user| self.services.globals.user_is_local(user)) -} + /// Returns an iterator over all knocking members of a room. + pub fn room_members_knocked<'a>( + &'a self, + room_id: &'a RoomId, + ) -> impl Stream + Send + 'a { + let prefix = (room_id, Interfix); + self.db + .roomuserid_knockedcount + .keys_prefix(&prefix) + .ignore_err() + .map(|(_, user_id): (Ignore, OwnedUserId)| user_id) + } -/// Returns an iterator of all our local joined users in a room who are -/// active (not deactivated) -#[implement(Service)] -#[tracing::instrument(skip(self), level = "trace")] -pub fn active_local_users_in_room<'a>( - &'a self, - room_id: &'a RoomId, -) -> impl Stream + Send + 'a { - self.local_users_in_room(room_id) - .filter_map(async |user_id| { - if self.services.users.is_active(&user_id).await { - Some(user_id) - } else { - None - } - }) -} + /// Gets the invite count value. + pub async fn get_invite_count(&self, room_id: &RoomId, user_id: &UserId) -> Result { + let key = (room_id, user_id); + self.db + .roomuserid_invitecount + .qry(&key) + .await + .deserialized() + } -/// Returns the number of users which are currently invited to a room -#[implement(Service)] -#[tracing::instrument(skip(self), level = "trace")] -pub async fn room_invited_count(&self, room_id: &RoomId) -> Result { - self.db - .roomid_invitedcount - .get(room_id) - .await - .deserialized() -} + /// Gets the knock count value. + pub async fn get_knock_count(&self, room_id: &RoomId, user_id: &UserId) -> Result { + let key = (room_id, user_id); + self.db + .roomuserid_knockedcount + .qry(&key) + .await + .deserialized() + } -/// Returns an iterator over all User IDs who ever joined a room. -#[implement(Service)] -#[tracing::instrument(skip(self), level = "debug")] -pub fn room_useroncejoined<'a>( - &'a self, - room_id: &'a RoomId, -) -> impl Stream + Send + 'a { - let prefix = (room_id, Interfix); - self.db - .roomuseroncejoinedids - .keys_prefix(&prefix) - .ignore_err() - .map(|(_, user_id): (Ignore, OwnedUserId)| user_id) -} + /// Gets the left count value. + pub async fn get_left_count(&self, room_id: &RoomId, user_id: &UserId) -> Result { + let key = (room_id, user_id); + self.db.roomuserid_leftcount.qry(&key).await.deserialized() + } -/// Returns an iterator over all invited members of a room. -#[implement(Service)] -#[tracing::instrument(skip(self), level = "debug")] -pub fn room_members_invited<'a>( - &'a self, - room_id: &'a RoomId, -) -> impl Stream + Send + 'a { - let prefix = (room_id, Interfix); - self.db - .roomuserid_invitecount - .keys_prefix(&prefix) - .ignore_err() - .map(|(_, user_id): (Ignore, OwnedUserId)| user_id) -} + /// Returns an iterator over all rooms this user joined. + pub fn rooms_joined<'a>( + &'a self, + user_id: &'a UserId, + ) -> impl Stream + Send + 'a { + self.db + .userroomid_joined + .keys_raw_prefix(user_id) + .ignore_err() + .map(|(_, room_id): (Ignore, OwnedRoomId)| room_id) + } -/// Returns an iterator over all knocked members of a room. -#[implement(Service)] -#[tracing::instrument(skip(self), level = "debug")] -pub fn room_members_knocked<'a>( - &'a self, - room_id: &'a RoomId, -) -> impl Stream + Send + 'a { - let prefix = (room_id, Interfix); - self.db - .roomuserid_knockedcount - .keys_prefix(&prefix) - .ignore_err() - .map(|(_, user_id): (Ignore, OwnedUserId)| user_id) -} + /// Returns an iterator over all rooms a user was invited to. + pub fn rooms_invited<'a>( + &'a self, + user_id: &'a UserId, + ) -> impl Stream + Send + 'a { + type KeyVal = (Key, Raw>); + type Key = (OwnedUserId, OwnedRoomId); -#[implement(Service)] -#[tracing::instrument(skip(self), level = "trace")] -pub async fn get_invite_count(&self, room_id: &RoomId, user_id: &UserId) -> Result { - let key = (room_id, user_id); - self.db - .roomuserid_invitecount - .qry(&key) - .await - .deserialized() -} + let prefix = (user_id, Interfix); + self.db + .userroomid_invitestate + .stream_prefix(&prefix) + .ignore_err() + .map(|((_, room_id), state): KeyVal| (room_id, state)) + .map(|(room_id, state)| Ok((room_id, state.deserialize_as_unchecked()?))) + .ignore_err() + } -#[implement(Service)] -#[tracing::instrument(skip(self), level = "trace")] -pub async fn get_knock_count(&self, room_id: &RoomId, user_id: &UserId) -> Result { - let key = (room_id, user_id); - self.db - .roomuserid_knockedcount - .qry(&key) - .await - .deserialized() -} + /// Returns an iterator over all rooms a user is currently knocking on. + pub fn rooms_knocked<'a>( + &'a self, + user_id: &'a UserId, + ) -> impl Stream + Send + 'a { + type KeyVal = (Key, Raw>); + type Key = (OwnedUserId, OwnedRoomId); -#[implement(Service)] -#[tracing::instrument(skip(self), level = "trace")] -pub async fn get_left_count(&self, room_id: &RoomId, user_id: &UserId) -> Result { - let key = (room_id, user_id); - self.db.roomuserid_leftcount.qry(&key).await.deserialized() -} + let prefix = (user_id, Interfix); + self.db + .userroomid_knockedstate + .stream_prefix(&prefix) + .ignore_err() + .map(|((_, room_id), state): KeyVal| (room_id, state)) + .map(|(room_id, state)| Ok((room_id, state.deserialize_as_unchecked()?))) + .ignore_err() + } -/// Returns an iterator over all rooms this user joined. -#[implement(Service)] -#[tracing::instrument(skip(self), level = "debug")] -pub fn rooms_joined<'a>( - &'a self, - user_id: &'a UserId, -) -> impl Stream + Send + 'a { - self.db - .userroomid_joined - .keys_raw_prefix(user_id) - .ignore_err() - .map(|(_, room_id): (Ignore, OwnedRoomId)| room_id) -} + /// Returns the invite state for a user in a room, if they are invited. + pub async fn invite_state( + &self, + user_id: &UserId, + room_id: &RoomId, + ) -> Result>> { + let key = (user_id, room_id); + self.db + .userroomid_invitestate + .qry(&key) + .await + .deserialized() + .and_then(|val: Raw>| { + val.deserialize_as_unchecked().map_err(Into::into) + }) + } -/// Returns an iterator over all rooms a user was invited to. -#[implement(Service)] -#[tracing::instrument(skip(self), level = "debug")] -pub fn rooms_invited<'a>( - &'a self, - user_id: &'a UserId, -) -> impl Stream + Send + 'a { - type KeyVal = (Key, Raw>); - type Key = (OwnedUserId, OwnedRoomId); + /// Returns the knock state for a user in a room, if they are knocking. + pub async fn knock_state( + &self, + user_id: &UserId, + room_id: &RoomId, + ) -> Result>> { + let key = (user_id, room_id); + self.db + .userroomid_knockedstate + .qry(&key) + .await + .deserialized() + .and_then(|val: Raw>| { + val.deserialize_as_unchecked().map_err(Into::into) + }) + } - let prefix = (user_id, Interfix); - self.db - .userroomid_invitestate - .stream_prefix(&prefix) - .ignore_err() - .map(|((_, room_id), state): KeyVal| (room_id, state)) - .map(|(room_id, state)| Ok((room_id, state.deserialize_as_unchecked()?))) - .ignore_err() -} + /// Returns the left state for a user in a room, if they have left. + pub async fn left_state(&self, user_id: &UserId, room_id: &RoomId) -> Result> { + let key = (user_id, room_id); + self.db.userroomid_leftstate.qry(&key).await.deserialized() + } -/// Returns an iterator over all rooms a user is currently knocking. -#[implement(Service)] -#[tracing::instrument(skip(self), level = "trace")] -pub fn rooms_knocked<'a>( - &'a self, - user_id: &'a UserId, -) -> impl Stream + Send + 'a { - type KeyVal = (Key, Raw>); - type Key = (OwnedUserId, OwnedRoomId); + /// Returns an iterator over all rooms a user left. + pub fn rooms_left<'a>( + &'a self, + user_id: &'a UserId, + ) -> impl Stream)> + Send + 'a { + type KeyVal = (Key, Raw>); + type Key = (OwnedUserId, OwnedRoomId); - let prefix = (user_id, Interfix); - self.db - .userroomid_knockedstate - .stream_prefix(&prefix) - .ignore_err() - .map(|((_, room_id), state): KeyVal| (room_id, state)) - .map(|(room_id, state)| Ok((room_id, state.deserialize_as_unchecked()?))) - .ignore_err() -} + let prefix = (user_id, Interfix); + self.db + .userroomid_leftstate + .stream_prefix(&prefix) + .ignore_err() + .map(|((_, room_id), state): KeyVal| (room_id, state)) + .map(|(room_id, state)| Ok((room_id, state.deserialize()?))) + .ignore_err() + } -#[implement(Service)] -#[tracing::instrument(skip(self), level = "trace")] -pub async fn invite_state( - &self, - user_id: &UserId, - room_id: &RoomId, -) -> Result>> { - let key = (user_id, room_id); - self.db - .userroomid_invitestate - .qry(&key) - .await - .deserialized() - .and_then(|val: Raw>| { - val.deserialize_as_unchecked().map_err(Into::into) - }) -} + /// Determines a user's membership in a target room. If the user has never + /// been a member of the target room at any point in time, None is returned. + pub async fn user_membership( + &self, + user_id: &UserId, + room_id: &RoomId, + ) -> Option { + let states = join5( + self.is_joined(user_id, room_id), + self.is_left(user_id, room_id), + self.is_knocked(user_id, room_id), + self.is_invited(user_id, room_id), + self.once_joined(user_id, room_id), + ) + .await; -#[implement(Service)] -#[tracing::instrument(skip(self), level = "trace")] -pub async fn knock_state( - &self, - user_id: &UserId, - room_id: &RoomId, -) -> Result>> { - let key = (user_id, room_id); - self.db - .userroomid_knockedstate - .qry(&key) - .await - .deserialized() - .and_then(|val: Raw>| { - val.deserialize_as_unchecked().map_err(Into::into) - }) -} + match states { + | (true, ..) => Some(MembershipState::Join), + | (_, true, ..) => Some(MembershipState::Leave), + | (_, _, true, ..) => Some(MembershipState::Knock), + | (_, _, _, true, ..) => Some(MembershipState::Invite), + | (false, false, false, false, true) => Some(MembershipState::Ban), + | _ => None, + } + } -#[implement(Service)] -#[tracing::instrument(skip(self), level = "trace")] -pub async fn left_state(&self, user_id: &UserId, room_id: &RoomId) -> Result> { - let key = (user_id, room_id); - self.db.userroomid_leftstate.qry(&key).await.deserialized() -} + /// Checks if a user has joined at least once before. + pub async fn once_joined(&self, user_id: &UserId, room_id: &RoomId) -> bool { + let key = (user_id, room_id); + self.db.roomuseroncejoinedids.qry(&key).await.is_ok() + } -/// Returns an iterator over all rooms a user left. -#[implement(Service)] -#[tracing::instrument(skip(self), level = "debug")] -pub fn rooms_left<'a>( - &'a self, - user_id: &'a UserId, -) -> impl Stream)> + Send + 'a { - type KeyVal = (Key, Raw>); - type Key = (OwnedUserId, OwnedRoomId); + /// Checks if a user is currently joined to a room. + pub async fn is_joined<'a>(&'a self, user_id: &'a UserId, room_id: &'a RoomId) -> bool { + let key = (user_id, room_id); + self.db.userroomid_joined.qry(&key).await.is_ok() + } - let prefix = (user_id, Interfix); - self.db - .userroomid_leftstate - .stream_prefix(&prefix) - .ignore_err() - .map(|((_, room_id), state): KeyVal| (room_id, state)) - .map(|(room_id, state)| Ok((room_id, state.deserialize()?))) - .ignore_err() -} + /// Checks if a user is currently knocking on a room. + pub async fn is_knocked<'a>(&'a self, user_id: &'a UserId, room_id: &'a RoomId) -> bool { + let key = (user_id, room_id); + self.db.userroomid_knockedstate.qry(&key).await.is_ok() + } -#[implement(Service)] -#[tracing::instrument(skip(self), level = "trace")] -pub async fn user_membership( - &self, - user_id: &UserId, - room_id: &RoomId, -) -> Option { - let states = join5( - self.is_joined(user_id, room_id), - self.is_left(user_id, room_id), - self.is_knocked(user_id, room_id), - self.is_invited(user_id, room_id), - self.once_joined(user_id, room_id), - ) - .await; + /// Checks if a user is currently invited to a room. + pub async fn is_invited(&self, user_id: &UserId, room_id: &RoomId) -> bool { + let key = (user_id, room_id); + self.db.userroomid_invitestate.qry(&key).await.is_ok() + } - match states { - | (true, ..) => Some(MembershipState::Join), - | (_, true, ..) => Some(MembershipState::Leave), - | (_, _, true, ..) => Some(MembershipState::Knock), - | (_, _, _, true, ..) => Some(MembershipState::Invite), - | (false, false, false, false, true) => Some(MembershipState::Ban), - | _ => None, + /// Checks if a user has left the room. + pub async fn is_left(&self, user_id: &UserId, room_id: &RoomId) -> bool { + let key = (user_id, room_id); + self.db.userroomid_leftstate.qry(&key).await.is_ok() + } + + /// Fetches the sender who invited a target user. + pub async fn invite_sender(&self, user_id: &UserId, room_id: &RoomId) -> Result { + let key = (user_id, room_id); + self.db + .userroomid_invitesender + .qry(&key) + .await + .deserialized() } } - -#[implement(Service)] -#[tracing::instrument(skip(self), level = "debug")] -pub async fn once_joined(&self, user_id: &UserId, room_id: &RoomId) -> bool { - let key = (user_id, room_id); - self.db.roomuseroncejoinedids.qry(&key).await.is_ok() -} - -#[implement(Service)] -#[tracing::instrument(skip(self), level = "trace")] -pub async fn is_joined<'a>(&'a self, user_id: &'a UserId, room_id: &'a RoomId) -> bool { - let key = (user_id, room_id); - self.db.userroomid_joined.qry(&key).await.is_ok() -} - -#[implement(Service)] -#[tracing::instrument(skip(self), level = "trace")] -pub async fn is_knocked<'a>(&'a self, user_id: &'a UserId, room_id: &'a RoomId) -> bool { - let key = (user_id, room_id); - self.db.userroomid_knockedstate.qry(&key).await.is_ok() -} - -#[implement(Service)] -#[tracing::instrument(skip(self), level = "trace")] -pub async fn is_invited(&self, user_id: &UserId, room_id: &RoomId) -> bool { - let key = (user_id, room_id); - self.db.userroomid_invitestate.qry(&key).await.is_ok() -} - -#[implement(Service)] -#[tracing::instrument(skip(self), level = "trace")] -pub async fn is_left(&self, user_id: &UserId, room_id: &RoomId) -> bool { - let key = (user_id, room_id); - self.db.userroomid_leftstate.qry(&key).await.is_ok() -} - -#[implement(Service)] -#[tracing::instrument(skip(self), level = "trace")] -pub async fn invite_sender(&self, user_id: &UserId, room_id: &RoomId) -> Result { - let key = (user_id, room_id); - self.db - .userroomid_invitesender - .qry(&key) - .await - .deserialized() -} diff --git a/src/service/rooms/state_cache/update.rs b/src/service/rooms/state_cache/update.rs index 0103713f4..20c6a700f 100644 --- a/src/service/rooms/state_cache/update.rs +++ b/src/service/rooms/state_cache/update.rs @@ -1,6 +1,6 @@ use std::collections::HashSet; -use conduwuit::{Err, Event, Pdu, Result, implement, is_not_empty, utils::ReadyExt, warn}; +use conduwuit::{Err, Event, Pdu, Result, is_not_empty, utils::ReadyExt}; use database::{Json, serialize_key}; use futures::StreamExt; use ruma::{ @@ -19,369 +19,360 @@ use ruma::{ }; use ruminuwuity::invite_permission_config::FilterLevel; -/// Update current membership data. -#[implement(super::Service)] -#[tracing::instrument( - level = "debug", - skip_all, - fields( - %room_id, - %user_id, - ?pdu, - ), - )] -#[allow(clippy::too_many_arguments)] -pub async fn update_membership( - &self, - room_id: &RoomId, - user_id: &UserId, - pdu: &Pdu, - update_joined_count: bool, -) -> Result { - let membership = pdu.get_content::()?; +impl super::Service { + /// Update current membership data. + pub async fn update_membership( + &self, + room_id: &RoomId, + user_id: &UserId, + pdu: &Pdu, + update_joined_count: bool, + ) -> Result { + let membership = pdu.get_content::()?; - // Keep track what remote users exist by adding them as "deactivated" users - // - // TODO: use futures to update remote profiles without blocking the membership - // update - #[allow(clippy::collapsible_if)] - if !self.services.globals.user_is_local(user_id) { - if !self.services.users.exists(user_id).await { + // Keep track what remote users exist by adding them as "deactivated" users + // + // TODO: use futures to update remote profiles without blocking the membership + // update + #[allow(clippy::collapsible_if)] + if !self.services.globals.user_is_local(user_id) + && !self.services.users.exists(user_id).await + { self.services.users.create(user_id, None).await?; } - } - match &membership.membership { - | MembershipState::Join => { - // Check if the user never joined this room - if !self.once_joined(user_id, room_id).await { - // Add the user ID to the join list then - self.mark_as_once_joined(user_id, room_id); + match &membership.membership { + | MembershipState::Join => { + // Check if the user never joined this room + if !self.once_joined(user_id, room_id).await { + // Add the user ID to the join list then + self.mark_as_once_joined(user_id, room_id); - // Check if the room has a predecessor - if let Ok(Some(predecessor)) = self - .services - .state_accessor - .room_state_get_content(room_id, &StateEventType::RoomCreate, "") - .await - .map(|content: RoomCreateEventContent| content.predecessor) - { - // Copy old tags to new room - if let Ok(tag_event) = self + // Check if the room has a predecessor + if let Ok(Some(predecessor)) = self .services - .account_data - .get_room(&predecessor.room_id, user_id, RoomAccountDataEventType::Tag) + .state_accessor + .room_state_get_content(room_id, &StateEventType::RoomCreate, "") .await + .map(|content: RoomCreateEventContent| content.predecessor) { - self.services + // Copy old tags to new room + if let Ok(tag_event) = self + .services .account_data - .update( - Some(room_id), + .get_room( + &predecessor.room_id, user_id, RoomAccountDataEventType::Tag, - &tag_event, ) .await - .ok(); - } - - // Copy direct chat flag - if let Ok(mut direct_event) = self - .services - .account_data - .get_global::(user_id, GlobalAccountDataEventType::Direct) - .await - { - let mut room_ids_updated = false; - for room_ids in direct_event.content.0.values_mut() { - if room_ids.iter().any(|r| r == &predecessor.room_id) { - room_ids.push(room_id.to_owned()); - room_ids_updated = true; - } - } - - if room_ids_updated { + { self.services .account_data .update( - None, + Some(room_id), user_id, - GlobalAccountDataEventType::Direct.to_string().into(), - &serde_json::to_value(&direct_event) - .expect("to json always works"), + RoomAccountDataEventType::Tag, + &tag_event, ) - .await?; + .await + .ok(); + } + + // Copy direct chat flag + if let Ok(mut direct_event) = self + .services + .account_data + .get_global::( + user_id, + GlobalAccountDataEventType::Direct, + ) + .await + { + let mut room_ids_updated = false; + for room_ids in direct_event.content.0.values_mut() { + if room_ids.iter().any(|r| r == &predecessor.room_id) { + room_ids.push(room_id.to_owned()); + room_ids_updated = true; + } + } + + if room_ids_updated { + self.services + .account_data + .update( + None, + user_id, + GlobalAccountDataEventType::Direct.to_string().into(), + &serde_json::to_value(&direct_event) + .expect("to json always works"), + ) + .await?; + } } } } - } - self.mark_as_joined(user_id, room_id); - }, - | MembershipState::Invite => { - let last_state = self - .services - .state - .summary_stripped(pdu, room_id, user_id) - .await; + self.mark_as_joined(user_id, room_id); + }, + | MembershipState::Invite => { + let last_state = self + .services + .state + .summary_stripped(pdu, room_id, user_id) + .await; - self.mark_as_invited(user_id, room_id, pdu.sender(), last_state, None) - .await?; - }, - | MembershipState::Leave | MembershipState::Ban => { - self.mark_as_left(user_id, room_id, Some(pdu.clone())).await; - }, - | _ => {}, + self.mark_as_invited(user_id, room_id, pdu.sender(), last_state, None) + .await?; + }, + | MembershipState::Leave | MembershipState::Ban => { + self.mark_as_left(user_id, room_id, Some(pdu.clone())).await; + }, + | _ => {}, + } + + if update_joined_count { + self.update_joined_count(room_id).await; + } + + Ok(()) } - if update_joined_count { - self.update_joined_count(room_id).await; + /// Updates membership counts for the target room. + pub async fn update_joined_count(&self, room_id: &RoomId) { + let mut joinedcount = 0_u64; + let mut invitedcount = 0_u64; + let mut knockedcount = 0_u64; + let mut joined_servers = HashSet::new(); + + self.room_members(room_id) + .ready_for_each(|joined| { + joined_servers.insert(joined.server_name().to_owned()); + joinedcount = joinedcount.saturating_add(1); + }) + .await; + + invitedcount = invitedcount.saturating_add( + self.room_members_invited(room_id) + .count() + .await + .try_into() + .unwrap_or(0), + ); + + knockedcount = knockedcount.saturating_add( + self.room_members_knocked(room_id) + .count() + .await + .try_into() + .unwrap_or(0), + ); + + self.db.roomid_joinedcount.raw_put(room_id, joinedcount); + self.db.roomid_invitedcount.raw_put(room_id, invitedcount); + self.db + .roomuserid_knockedcount + .raw_put(room_id, knockedcount); + + self.room_servers(room_id) + .ready_for_each(|old_joined_server| { + if joined_servers.remove(&old_joined_server) { + return; + } + + // Server not in room anymore + let roomserver_id = (room_id, old_joined_server.clone()); + let serverroom_id = (old_joined_server, room_id); + + self.db.roomserverids.del(roomserver_id); + self.db.serverroomids.del(serverroom_id); + }) + .await; + + // Now only new servers are in joined_servers anymore + for server in &joined_servers { + let roomserver_id = (room_id, server); + let serverroom_id = (server, room_id); + + self.db.roomserverids.put_raw(roomserver_id, []); + self.db.serverroomids.put_raw(serverroom_id, []); + } + + self.appservice_in_room_cache.write().remove(room_id); } - Ok(()) -} + /// Direct DB function to directly mark a user as joined. It is not + /// recommended to use this directly. You most likely should use + /// `update_membership` instead + pub fn mark_as_joined(&self, user_id: &UserId, room_id: &RoomId) { + let userroom_id = (user_id, room_id); + let userroom_id = serialize_key(userroom_id).expect("failed to serialize userroom_id"); -#[implement(super::Service)] -#[tracing::instrument(level = "debug", skip(self))] -pub async fn update_joined_count(&self, room_id: &RoomId) { - let mut joinedcount = 0_u64; - let mut invitedcount = 0_u64; - let mut knockedcount = 0_u64; - let mut joined_servers = HashSet::new(); + let roomuser_id = (room_id, user_id); + let roomuser_id = serialize_key(roomuser_id).expect("failed to serialize roomuser_id"); - self.room_members(room_id) - .ready_for_each(|joined| { - joined_servers.insert(joined.server_name().to_owned()); - joinedcount = joinedcount.saturating_add(1); - }) - .await; + self.db.userroomid_joined.insert(&userroom_id, []); + self.db.roomuserid_joined.insert(&roomuser_id, []); - invitedcount = invitedcount.saturating_add( - self.room_members_invited(room_id) - .count() - .await - .try_into() - .unwrap_or(0), - ); + self.db.userroomid_invitestate.remove(&userroom_id); + self.db.roomuserid_invitecount.remove(&roomuser_id); + self.db.userroomid_invitesender.remove(&userroom_id); - knockedcount = knockedcount.saturating_add( - self.room_members_knocked(room_id) - .count() - .await - .try_into() - .unwrap_or(0), - ); + self.db.userroomid_leftstate.remove(&userroom_id); + self.db.roomuserid_leftcount.remove(&roomuser_id); - self.db.roomid_joinedcount.raw_put(room_id, joinedcount); - self.db.roomid_invitedcount.raw_put(room_id, invitedcount); - self.db - .roomuserid_knockedcount - .raw_put(room_id, knockedcount); + self.db.userroomid_knockedstate.remove(&userroom_id); + self.db.roomuserid_knockedcount.remove(&roomuser_id); - self.room_servers(room_id) - .ready_for_each(|old_joined_server| { - if joined_servers.remove(&old_joined_server) { - return; - } - - // Server not in room anymore - let roomserver_id = (room_id, old_joined_server.clone()); - let serverroom_id = (old_joined_server, room_id); - - self.db.roomserverids.del(roomserver_id); - self.db.serverroomids.del(serverroom_id); - }) - .await; - - // Now only new servers are in joined_servers anymore - for server in &joined_servers { - let roomserver_id = (room_id, server); - let serverroom_id = (server, room_id); - - self.db.roomserverids.put_raw(roomserver_id, []); - self.db.serverroomids.put_raw(serverroom_id, []); + self.db.roomid_inviteviaservers.remove(room_id); } - self.appservice_in_room_cache.write().remove(room_id); -} + /// Mark a user as having left a room. + /// + /// `leave_pdu` represents the m.room.member event which the user sent to + /// leave the room. If this is None, no event was actually sent, but we + /// must still behave as if the user is no longer in the room. This may + /// occur, for example, if the room being left has been server-banned by an + /// administrator. + pub async fn mark_as_left(&self, user_id: &UserId, room_id: &RoomId, leave_pdu: Option) { + let userroom_id = (user_id, room_id); + let userroom_id = serialize_key(userroom_id).expect("failed to serialize userroom_id"); -/// Direct DB function to directly mark a user as joined. It is not -/// recommended to use this directly. You most likely should use -/// `update_membership` instead -#[implement(super::Service)] -#[tracing::instrument(skip(self), level = "debug")] -pub fn mark_as_joined(&self, user_id: &UserId, room_id: &RoomId) { - let userroom_id = (user_id, room_id); - let userroom_id = serialize_key(userroom_id).expect("failed to serialize userroom_id"); + let roomuser_id = (room_id, user_id); + let roomuser_id = serialize_key(roomuser_id).expect("failed to serialize roomuser_id"); - let roomuser_id = (room_id, user_id); - let roomuser_id = serialize_key(roomuser_id).expect("failed to serialize roomuser_id"); + self.db + .userroomid_leftstate + .raw_put(&userroom_id, Json(leave_pdu)); + self.db + .roomuserid_leftcount + .raw_aput::<8, _, _>(&roomuser_id, self.services.globals.next_count().unwrap()); - self.db.userroomid_joined.insert(&userroom_id, []); - self.db.roomuserid_joined.insert(&roomuser_id, []); + self.db.userroomid_joined.remove(&userroom_id); + self.db.roomuserid_joined.remove(&roomuser_id); - self.db.userroomid_invitestate.remove(&userroom_id); - self.db.roomuserid_invitecount.remove(&roomuser_id); - self.db.userroomid_invitesender.remove(&userroom_id); + self.db.userroomid_invitestate.remove(&userroom_id); + self.db.roomuserid_invitecount.remove(&roomuser_id); + self.db.userroomid_invitesender.remove(&userroom_id); - self.db.userroomid_leftstate.remove(&userroom_id); - self.db.roomuserid_leftcount.remove(&roomuser_id); + self.db.userroomid_knockedstate.remove(&userroom_id); + self.db.roomuserid_knockedcount.remove(&roomuser_id); - self.db.userroomid_knockedstate.remove(&userroom_id); - self.db.roomuserid_knockedcount.remove(&roomuser_id); + self.db.roomid_inviteviaservers.remove(room_id); - self.db.roomid_inviteviaservers.remove(room_id); -} - -/// Mark a user as having left a room. -/// -/// `leave_pdu` represents the m.room.member event which the user sent to leave -/// the room. If this is None, no event was actually sent, but we must still -/// behave as if the user is no longer in the room. This may occur, for example, -/// if the room being left has been server-banned by an administrator. -#[implement(super::Service)] -#[tracing::instrument(skip(self), level = "debug")] -pub async fn mark_as_left(&self, user_id: &UserId, room_id: &RoomId, leave_pdu: Option) { - let userroom_id = (user_id, room_id); - let userroom_id = serialize_key(userroom_id).expect("failed to serialize userroom_id"); - - let roomuser_id = (room_id, user_id); - let roomuser_id = serialize_key(roomuser_id).expect("failed to serialize roomuser_id"); - - self.db - .userroomid_leftstate - .raw_put(&userroom_id, Json(leave_pdu)); - self.db - .roomuserid_leftcount - .raw_aput::<8, _, _>(&roomuser_id, self.services.globals.next_count().unwrap()); - - self.db.userroomid_joined.remove(&userroom_id); - self.db.roomuserid_joined.remove(&roomuser_id); - - self.db.userroomid_invitestate.remove(&userroom_id); - self.db.roomuserid_invitecount.remove(&roomuser_id); - self.db.userroomid_invitesender.remove(&userroom_id); - - self.db.userroomid_knockedstate.remove(&userroom_id); - self.db.roomuserid_knockedcount.remove(&roomuser_id); - - self.db.roomid_inviteviaservers.remove(room_id); - - if self.services.globals.user_is_local(user_id) - && (self.services.config.forget_forced_upon_leave - || self.services.metadata.is_banned(room_id).await - || self.services.metadata.is_disabled(room_id).await) - { - self.forget(room_id, user_id); + if self.services.globals.user_is_local(user_id) + && (self.services.config.forget_forced_upon_leave + || self.services.metadata.is_banned(room_id).await + || self.services.metadata.is_disabled(room_id).await) + { + self.forget(room_id, user_id); + } } -} -/// Direct DB function to directly mark a user as knocked. It is not -/// recommended to use this directly. You most likely should use -/// `update_membership` instead -#[implement(super::Service)] -#[tracing::instrument(skip(self), level = "debug")] -pub fn mark_as_knocked( - &self, - user_id: &UserId, - room_id: &RoomId, - knocked_state: Option>>, -) { - let userroom_id = (user_id, room_id); - let userroom_id = serialize_key(userroom_id).expect("failed to serialize userroom_id"); - - let roomuser_id = (room_id, user_id); - let roomuser_id = serialize_key(roomuser_id).expect("failed to serialize roomuser_id"); - - self.db - .userroomid_knockedstate - .raw_put(&userroom_id, Json(knocked_state.unwrap_or_default())); - self.db - .roomuserid_knockedcount - .raw_aput::<8, _, _>(&roomuser_id, self.services.globals.next_count().unwrap()); - - self.db.userroomid_joined.remove(&userroom_id); - self.db.roomuserid_joined.remove(&roomuser_id); - - self.db.userroomid_invitestate.remove(&userroom_id); - self.db.roomuserid_invitecount.remove(&roomuser_id); - self.db.userroomid_invitesender.remove(&userroom_id); - - self.db.userroomid_leftstate.remove(&userroom_id); - self.db.roomuserid_leftcount.remove(&roomuser_id); - - self.db.roomid_inviteviaservers.remove(room_id); -} - -/// Makes a user forget a room. -#[implement(super::Service)] -#[tracing::instrument(skip(self), level = "debug")] -pub fn forget(&self, room_id: &RoomId, user_id: &UserId) { - let userroom_id = (user_id, room_id); - let roomuser_id = (room_id, user_id); - - self.db.userroomid_leftstate.del(userroom_id); - self.db.roomuserid_leftcount.del(roomuser_id); -} - -#[implement(super::Service)] -#[tracing::instrument(level = "debug", skip(self))] -fn mark_as_once_joined(&self, user_id: &UserId, room_id: &RoomId) { - let key = (user_id, room_id); - self.db.roomuseroncejoinedids.put_raw(key, []); -} - -#[implement(super::Service)] -#[tracing::instrument(level = "debug", skip(self, last_state, invite_via))] -pub async fn mark_as_invited( - &self, - user_id: &UserId, - room_id: &RoomId, - sender_user: &UserId, - last_state: Vec, - invite_via: Option>, -) -> Result<()> { - // return an error for blocked invites. ignored invites aren't handled here - // since the recipient's membership should still be changed to `invite`. - // they're filtered out in the individual /sync handlers - if matches!( - self.services - .users - .invite_filter_level(sender_user, user_id) - .await, - FilterLevel::Block + /// Direct DB function to directly mark a user as knocked. It is not + /// recommended to use this directly. You most likely should use + /// `update_membership` instead + pub fn mark_as_knocked( + &self, + user_id: &UserId, + room_id: &RoomId, + knocked_state: Option>>, ) { - return Err!(Request(InviteBlocked("{user_id} has blocked invites from {sender_user}."))); + let userroom_id = (user_id, room_id); + let userroom_id = serialize_key(userroom_id).expect("failed to serialize userroom_id"); + + let roomuser_id = (room_id, user_id); + let roomuser_id = serialize_key(roomuser_id).expect("failed to serialize roomuser_id"); + + self.db + .userroomid_knockedstate + .raw_put(&userroom_id, Json(knocked_state.unwrap_or_default())); + self.db + .roomuserid_knockedcount + .raw_aput::<8, _, _>(&roomuser_id, self.services.globals.next_count().unwrap()); + + self.db.userroomid_joined.remove(&userroom_id); + self.db.roomuserid_joined.remove(&roomuser_id); + + self.db.userroomid_invitestate.remove(&userroom_id); + self.db.roomuserid_invitecount.remove(&roomuser_id); + self.db.userroomid_invitesender.remove(&userroom_id); + + self.db.userroomid_leftstate.remove(&userroom_id); + self.db.roomuserid_leftcount.remove(&roomuser_id); + + self.db.roomid_inviteviaservers.remove(room_id); } - let roomuser_id = (room_id, user_id); - let roomuser_id = serialize_key(roomuser_id).expect("failed to serialize roomuser_id"); + /// Makes a user forget a room. + pub fn forget(&self, room_id: &RoomId, user_id: &UserId) { + let userroom_id = (user_id, room_id); + let roomuser_id = (room_id, user_id); - let userroom_id = (user_id, room_id); - let userroom_id = serialize_key(userroom_id).expect("failed to serialize userroom_id"); - - self.db - .userroomid_invitestate - .raw_put(&userroom_id, Json(last_state)); - self.db - .roomuserid_invitecount - .raw_aput::<8, _, _>(&roomuser_id, self.services.globals.next_count().unwrap()); - self.db - .userroomid_invitesender - .insert(&userroom_id, sender_user); - - self.db.userroomid_joined.remove(&userroom_id); - self.db.roomuserid_joined.remove(&roomuser_id); - - self.db.userroomid_leftstate.remove(&userroom_id); - self.db.roomuserid_leftcount.remove(&roomuser_id); - - self.db.userroomid_knockedstate.remove(&userroom_id); - self.db.roomuserid_knockedcount.remove(&roomuser_id); - - if let Some(servers) = invite_via.filter(is_not_empty!()) { - self.add_servers_invite_via(room_id, servers).await; + self.db.userroomid_leftstate.del(userroom_id); + self.db.roomuserid_leftcount.del(roomuser_id); } - Ok(()) + /// Immutably a user as having joined a room. + fn mark_as_once_joined(&self, user_id: &UserId, room_id: &RoomId) { + let key = (user_id, room_id); + self.db.roomuseroncejoinedids.put_raw(key, []); + } + + /// Marks a user as having been invited to a room, saving the relevant + /// invite state. + pub async fn mark_as_invited( + &self, + user_id: &UserId, + room_id: &RoomId, + sender_user: &UserId, + last_state: Vec, + invite_via: Option>, + ) -> Result<()> { + // return an error for blocked invites. ignored invites aren't handled here + // since the recipient's membership should still be changed to `invite`. + // they're filtered out in the individual /sync handlers + if matches!( + self.services + .users + .invite_filter_level(sender_user, user_id) + .await, + FilterLevel::Block + ) { + return Err!(Request(InviteBlocked( + "{user_id} has blocked invites from {sender_user}." + ))); + } + + let roomuser_id = (room_id, user_id); + let roomuser_id = serialize_key(roomuser_id).expect("failed to serialize roomuser_id"); + + let userroom_id = (user_id, room_id); + let userroom_id = serialize_key(userroom_id).expect("failed to serialize userroom_id"); + + self.db + .userroomid_invitestate + .raw_put(&userroom_id, Json(last_state)); + self.db + .roomuserid_invitecount + .raw_aput::<8, _, _>(&roomuser_id, self.services.globals.next_count().unwrap()); + self.db + .userroomid_invitesender + .insert(&userroom_id, sender_user); + + self.db.userroomid_joined.remove(&userroom_id); + self.db.roomuserid_joined.remove(&roomuser_id); + + self.db.userroomid_leftstate.remove(&userroom_id); + self.db.roomuserid_leftcount.remove(&roomuser_id); + + self.db.userroomid_knockedstate.remove(&userroom_id); + self.db.roomuserid_knockedcount.remove(&roomuser_id); + + if let Some(servers) = invite_via.filter(is_not_empty!()) { + self.add_servers_invite_via(room_id, servers).await; + } + + Ok(()) + } } diff --git a/src/service/rooms/state_cache/via.rs b/src/service/rooms/state_cache/via.rs index 1ffc4770a..7494a9123 100644 --- a/src/service/rooms/state_cache/via.rs +++ b/src/service/rooms/state_cache/via.rs @@ -1,7 +1,6 @@ use conduwuit::{ - Result, implement, + Result, utils::{StreamTools, stream::TryIgnore}, - warn, }; use database::Ignore; use futures::{Stream, StreamExt, stream::iter}; @@ -12,80 +11,78 @@ use ruma::{ int, }; -#[implement(super::Service)] -#[tracing::instrument(level = "debug", skip(self, servers))] -pub async fn add_servers_invite_via(&self, room_id: &RoomId, servers: Vec) { - let mut servers: Vec<_> = self - .servers_invite_via(room_id) - .chain(iter(servers.into_iter())) - .collect() - .await; +impl super::Service { + /// Adds N servers that are likely to be in the room in the future. + pub async fn add_servers_invite_via(&self, room_id: &RoomId, servers: Vec) { + let mut servers: Vec<_> = self + .servers_invite_via(room_id) + .chain(iter(servers)) + .collect() + .await; - servers.sort_unstable(); - servers.dedup(); + servers.sort_unstable(); + servers.dedup(); - let servers = servers - .iter() - .map(|server| server.as_bytes()) - .collect_vec() - .join(&[0xFF][..]); + let servers = servers + .iter() + .map(|server| server.as_bytes()) + .collect_vec() + .join(&[0xFF][..]); - self.db - .roomid_inviteviaservers - .insert(room_id.as_bytes(), &servers); -} - -/// Gets up to five servers that are likely to be in the room in the -/// distant future. -/// -/// See -#[implement(super::Service)] -#[tracing::instrument(skip(self), level = "trace")] -pub async fn servers_route_via(&self, room_id: &RoomId) -> Result> { - let most_powerful_user_server = self - .services - .state_accessor - .room_state_get_content(room_id, &StateEventType::RoomPowerLevels, "") - .await - .map(|content: RoomPowerLevelsEventContent| { - content - .users - .iter() - .max_by_key(|(_, power)| *power) - .and_then(|x| (x.1 >= &int!(50)).then_some(x)) - .map(|(user, _power)| user.server_name().to_owned()) - }); - - let mut servers: Vec = self - .room_members(room_id) - .counts_by(|user| user.server_name().to_owned()) - .await - .into_iter() - .sorted_by_key(|(_, users)| *users) - .map(|(server, _)| server) - .rev() - .take(5) - .collect(); - - if let Ok(Some(server)) = most_powerful_user_server { - servers.insert(0, server); - servers.truncate(5); + self.db + .roomid_inviteviaservers + .insert(room_id.as_bytes(), &servers); } - Ok(servers) -} + /// Gets up to five servers that are likely to be in the room in the + /// distant future. + /// + /// See + pub async fn servers_route_via(&self, room_id: &RoomId) -> Result> { + let most_powerful_user_server = self + .services + .state_accessor + .room_state_get_content(room_id, &StateEventType::RoomPowerLevels, "") + .await + .map(|content: RoomPowerLevelsEventContent| { + content + .users + .iter() + .max_by_key(|(_, power)| *power) + .and_then(|x| (x.1 >= &int!(50)).then_some(x)) + .map(|(user, _power)| user.server_name().to_owned()) + }); -#[implement(super::Service)] -#[tracing::instrument(skip(self), level = "debug")] -pub fn servers_invite_via<'a>( - &'a self, - room_id: &'a RoomId, -) -> impl Stream + Send + 'a { - type KeyVal = (Ignore, Vec); + let mut servers: Vec = self + .room_members(room_id) + .counts_by(|user| user.server_name().to_owned()) + .await + .into_iter() + .sorted_by_key(|(_, users)| *users) + .map(|(server, _)| server) + .rev() + .take(5) + .collect(); - self.db - .roomid_inviteviaservers - .stream_raw_prefix(room_id) - .ignore_err() - .map(|(_, mut servers): KeyVal| servers.pop().expect("at least one server")) + if let Ok(Some(server)) = most_powerful_user_server { + servers.insert(0, server); + servers.truncate(5); + } + + Ok(servers) + } + + /// Fetches a list of servers that may be in the room as of an invite. + pub fn servers_invite_via<'a>( + &'a self, + room_id: &'a RoomId, + ) -> impl Stream + Send + 'a { + type KeyVal = (Ignore, Vec); + + self.db + .roomid_inviteviaservers + .stream_raw_prefix(room_id) + .ignore_err() + .map(|(_, mut servers): KeyVal| servers.pop().expect("at least one server")) + } } From 4d30ee5d3329b1f785c1bfbd54cd30011985b954 Mon Sep 17 00:00:00 2001 From: timedout Date: Fri, 26 Jun 2026 04:17:47 +0100 Subject: [PATCH 36/70] style: Document & refactor rooms/state_accessor service --- src/service/rooms/state_accessor/mod.rs | 17 +- .../rooms/state_accessor/room_state.rs | 181 ++-- .../rooms/state_accessor/server_can.rs | 90 +- src/service/rooms/state_accessor/state.rs | 802 +++++++++--------- src/service/rooms/state_accessor/user_can.rs | 249 +++--- 5 files changed, 673 insertions(+), 666 deletions(-) diff --git a/src/service/rooms/state_accessor/mod.rs b/src/service/rooms/state_accessor/mod.rs index ae331fa28..8d2918302 100644 --- a/src/service/rooms/state_accessor/mod.rs +++ b/src/service/rooms/state_accessor/mod.rs @@ -70,12 +70,14 @@ impl crate::Service for Service { } impl Service { + /// Gets the name of a room, if any. pub async fn get_name(&self, room_id: &RoomId) -> Result { self.room_state_get_content(room_id, &StateEventType::RoomName, "") .await .map(|c: RoomNameEventContent| c.name) } + /// Gets the avatar event of a room, if any. pub async fn get_avatar(&self, room_id: &RoomId) -> JsOption { let content = self .room_state_get_content(room_id, &StateEventType::RoomAvatar, "") @@ -85,6 +87,7 @@ impl Service { JsOption::from_option(content) } + /// Gets a member's event content, if one exists. pub async fn get_member( &self, room_id: &RoomId, @@ -124,7 +127,13 @@ impl Service { pub async fn get_room_topic(&self, room_id: &RoomId) -> Result { self.room_state_get_content(room_id, &StateEventType::RoomTopic, "") .await - .map(|c: RoomTopicEventContent| c.topic) + .map(|c: RoomTopicEventContent| { + c.topic_block + .text + .find_plain() + .map(ToOwned::to_owned) + .unwrap_or(c.topic) + }) } /// Returns the join rules for a given room (`JoinRule` type). Will default @@ -135,6 +144,7 @@ impl Service { .map_or(JoinRule::Invite, |c: RoomJoinRulesEventContent| c.join_rule) } + /// Gets the room type of a room, typically a space. pub async fn get_room_type(&self, room_id: &RoomId) -> Result { self.room_state_get_content(room_id, &StateEventType::RoomCreate, "") .await @@ -156,6 +166,7 @@ impl Service { .map(|content: RoomEncryptionEventContent| content.algorithm) } + /// Checks if the given room is one with encryption enabled. pub async fn is_encrypted_room(&self, room_id: &RoomId) -> bool { self.room_state_get(room_id, &StateEventType::RoomEncryption, "") .await @@ -166,7 +177,7 @@ impl Service { pub async fn get_room_create_event(&self, room_id: &RoomId) -> Pdu { self.room_state_get(room_id, &StateEventType::RoomCreate, "") .await - .expect("room should have a create event") + .expect("room must have a create event") } /// Get a set of the room's creators. This will always contain a single user @@ -202,7 +213,7 @@ impl Service { creators } - /// Get the room's power levels. This will never fail -- if the room has no + /// Get the room's power levels. This will never fail - if the room has no /// power level state event, the default power levels for the room's /// version will be returned. pub async fn get_room_power_levels(&self, room_id: &RoomId) -> RoomPowerLevels { diff --git a/src/service/rooms/state_accessor/room_state.rs b/src/service/rooms/state_accessor/room_state.rs index fbc7ed5d4..d1dfbbacc 100644 --- a/src/service/rooms/state_accessor/room_state.rs +++ b/src/service/rooms/state_accessor/room_state.rs @@ -1,112 +1,103 @@ use std::borrow::Borrow; use conduwuit::{ - Pdu, Result, err, implement, + Pdu, Result, err, matrix::{Event, StateKey}, }; use futures::{Stream, StreamExt, TryFutureExt}; use ruma::{EventId, RoomId, events::StateEventType}; use serde::Deserialize; -/// Returns a single PDU from `room_id` with key (`event_type`,`state_key`). -#[implement(super::Service)] -pub async fn room_state_get_content( - &self, - room_id: &RoomId, - event_type: &StateEventType, - state_key: &str, -) -> Result -where - T: for<'de> Deserialize<'de>, -{ - self.room_state_get(room_id, event_type, state_key) - .await - .and_then(|event| event.get_content()) -} +impl super::Service { + /// Returns a single PDU from `room_id` with key (`event_type`,`state_key`). + pub async fn room_state_get_content( + &self, + room_id: &RoomId, + event_type: &StateEventType, + state_key: &str, + ) -> Result + where + T: for<'de> Deserialize<'de>, + { + self.room_state_get(room_id, event_type, state_key) + .await + .and_then(|event| event.get_content()) + } -/// Returns the full room state. -#[implement(super::Service)] -#[tracing::instrument(skip(self), level = "debug")] -pub fn room_state_full<'a>( - &'a self, - room_id: &'a RoomId, -) -> impl Stream> + Send + 'a { - self.services - .state - .get_room_shortstatehash(room_id) - .map_ok(|shortstatehash| self.state_full(shortstatehash).map(Ok).boxed()) - .map_err(move |e| err!(Database("Missing state for {room_id:?}: {e:?}"))) - .try_flatten_stream() -} + /// Returns the full current room state. + pub fn room_state_full<'a>( + &'a self, + room_id: &'a RoomId, + ) -> impl Stream> + Send + 'a { + self.services + .state + .get_room_shortstatehash(room_id) + .map_ok(|shortstatehash| self.state_full(shortstatehash).map(Ok).boxed()) + .map_err(move |e| err!(Database("Missing state for {room_id:?}: {e:?}"))) + .try_flatten_stream() + } -/// Returns the full room state pdus -#[implement(super::Service)] -#[tracing::instrument(skip(self), level = "debug")] -pub fn room_state_full_pdus<'a>( - &'a self, - room_id: &'a RoomId, -) -> impl Stream> + Send + 'a { - self.services - .state - .get_room_shortstatehash(room_id) - .map_ok(|shortstatehash| self.state_full_pdus(shortstatehash).map(Ok).boxed()) - .map_err(move |e| err!(Database("Missing state for {room_id:?}: {e:?}"))) - .try_flatten_stream() -} + /// Returns the full room state pdus + pub fn room_state_full_pdus<'a>( + &'a self, + room_id: &'a RoomId, + ) -> impl Stream> + Send + 'a { + self.services + .state + .get_room_shortstatehash(room_id) + .map_ok(|shortstatehash| self.state_full_pdus(shortstatehash).map(Ok).boxed()) + .map_err(move |e| err!(Database("Missing state for {room_id:?}: {e:?}"))) + .try_flatten_stream() + } -/// Returns a single EventId from `room_id` with key (`event_type`, -/// `state_key`). -#[implement(super::Service)] -#[tracing::instrument(skip(self), level = "debug")] -pub async fn room_state_get_id( - &self, - room_id: &RoomId, - event_type: &StateEventType, - state_key: &str, -) -> Result -where - Id: for<'de> Deserialize<'de> + Sized + ToOwned, - ::Owned: Borrow, -{ - self.services - .state - .get_room_shortstatehash(room_id) - .and_then(|shortstatehash| self.state_get_id(shortstatehash, event_type, state_key)) - .await -} + /// Returns a single EventId from `room_id` with key (`event_type`, + /// `state_key`). + pub async fn room_state_get_id( + &self, + room_id: &RoomId, + event_type: &StateEventType, + state_key: &str, + ) -> Result + where + Id: for<'de> Deserialize<'de> + Sized + ToOwned, + ::Owned: Borrow, + { + self.services + .state + .get_room_shortstatehash(room_id) + .and_then(|shortstatehash| self.state_get_id(shortstatehash, event_type, state_key)) + .await + } -/// Returns a single PDU from `room_id` with key (`event_type`, -/// `state_key`). -#[implement(super::Service)] -#[tracing::instrument(skip(self), level = "debug")] -pub async fn room_state_get( - &self, - room_id: &RoomId, - event_type: &StateEventType, - state_key: &str, -) -> Result { - self.services - .state - .get_room_shortstatehash(room_id) - .and_then(|shortstatehash| self.state_get(shortstatehash, event_type, state_key)) - .await -} + /// Returns a single PDU from `room_id` with key (`event_type`, + /// `state_key`). + pub async fn room_state_get( + &self, + room_id: &RoomId, + event_type: &StateEventType, + state_key: &str, + ) -> Result { + self.services + .state + .get_room_shortstatehash(room_id) + .and_then(|shortstatehash| self.state_get(shortstatehash, event_type, state_key)) + .await + } -/// Returns all state keys for the given `room_id` and `event_type`. -#[implement(super::Service)] -#[tracing::instrument(skip(self), level = "debug")] -pub async fn room_state_keys( - &self, - room_id: &RoomId, - event_type: &StateEventType, -) -> Result> { - let shortstatehash = self.services.state.get_room_shortstatehash(room_id).await?; + /// Returns all state keys for the given `room_id` and `event_type`. + pub async fn room_state_keys( + &self, + room_id: &RoomId, + event_type: &StateEventType, + ) -> Result> { + let shortstatehash = self.services.state.get_room_shortstatehash(room_id).await?; - let state_keys: Vec = self - .state_keys(shortstatehash, event_type) - .map(|state_key| state_key.to_string()) - .collect() - .await; + let state_keys: Vec = self + .state_keys(shortstatehash, event_type) + .map(|state_key| state_key.to_string()) + .collect() + .await; - Ok(state_keys) + Ok(state_keys) + } } diff --git a/src/service/rooms/state_accessor/server_can.rs b/src/service/rooms/state_accessor/server_can.rs index b992007aa..bbc47529b 100644 --- a/src/service/rooms/state_accessor/server_can.rs +++ b/src/service/rooms/state_accessor/server_can.rs @@ -1,4 +1,4 @@ -use conduwuit::{implement, utils::stream::ReadyExt, warn}; +use conduwuit::{debug_warn, utils::stream::ReadyExt}; use futures::StreamExt; use ruma::{ EventId, RoomId, ServerName, @@ -8,51 +8,53 @@ use ruma::{ }, }; -/// Whether a server is allowed to see an event through federation, based on -/// the room's history_visibility at that event's state. -#[implement(super::Service)] -#[tracing::instrument(skip_all, level = "trace")] -pub async fn server_can_see_event( - &self, - origin: &ServerName, - room_id: &RoomId, - event_id: &EventId, -) -> bool { - let Ok(shortstatehash) = self.pdu_shortstatehash(event_id).await else { - warn!( - "Unable to visibility check event {} in room {} for server {}: shortstatehash not \ - found", - event_id, room_id, origin - ); - return false; - }; +impl super::Service { + /// Whether a server is allowed to see an event through federation, based on + /// the room's history_visibility at that event's state. + pub async fn server_can_see_event( + &self, + origin: &ServerName, + room_id: &RoomId, + event_id: &EventId, + ) -> bool { + let Ok(shortstatehash) = self.pdu_shortstatehash(event_id).await else { + debug_warn!( + "Unable to visibility check event {} in room {} for server {}: shortstatehash \ + not found", + event_id, + room_id, + origin + ); + return false; + }; - let history_visibility = self - .state_get_content(shortstatehash, &StateEventType::RoomHistoryVisibility, "") - .await - .map_or(HistoryVisibility::Shared, |c: RoomHistoryVisibilityEventContent| { - c.history_visibility - }); + let history_visibility = self + .state_get_content(shortstatehash, &StateEventType::RoomHistoryVisibility, "") + .await + .map_or(HistoryVisibility::Shared, |c: RoomHistoryVisibilityEventContent| { + c.history_visibility + }); - let current_server_members = self - .services - .state_cache - .room_members(room_id) - .ready_filter(|member| member.server_name() == origin); + let current_server_members = self + .services + .state_cache + .room_members(room_id) + .ready_filter(|member| member.server_name() == origin); - match history_visibility { - | HistoryVisibility::Invited => { - // Allow if any member on requesting server was AT LEAST invited, else deny - current_server_members - .any(async |member| self.user_was_invited(shortstatehash, &member).await) - .await - }, - | HistoryVisibility::Joined => { - // Allow if any member on requested server was joined, else deny - current_server_members - .any(async |member| self.user_was_joined(shortstatehash, &member).await) - .await - }, - | HistoryVisibility::WorldReadable | HistoryVisibility::Shared | _ => true, + match history_visibility { + | HistoryVisibility::Invited => { + // Allow if any member on requesting server was AT LEAST invited, else deny + current_server_members + .any(async |member| self.user_was_invited(shortstatehash, &member).await) + .await + }, + | HistoryVisibility::Joined => { + // Allow if any member on requested server was joined, else deny + current_server_members + .any(async |member| self.user_was_joined(shortstatehash, &member).await) + .await + }, + | HistoryVisibility::WorldReadable | HistoryVisibility::Shared | _ => true, + } } } diff --git a/src/service/rooms/state_accessor/state.rs b/src/service/rooms/state_accessor/state.rs index 8aabbfcf2..2ca26b3ce 100644 --- a/src/service/rooms/state_accessor/state.rs +++ b/src/service/rooms/state_accessor/state.rs @@ -1,7 +1,7 @@ use std::{borrow::Borrow, ops::Deref, sync::Arc}; use conduwuit::{ - Pdu, Result, at, err, implement, + Pdu, Result, at, err, matrix::{Event, StateKey}, pair_of, utils::{ @@ -25,402 +25,406 @@ use crate::rooms::{ state_compressor::{CompressedState, compress_state_event, parse_compressed_state_event}, }; -/// The user was a joined member at this state (potentially in the past) -#[implement(super::Service)] -#[inline] -pub async fn user_was_joined(&self, shortstatehash: ShortStateHash, user_id: &UserId) -> bool { - self.user_membership(shortstatehash, user_id).await == MembershipState::Join -} - -/// The user was an invited or joined room member at this state (potentially -/// in the past) -#[implement(super::Service)] -#[inline] -pub async fn user_was_invited(&self, shortstatehash: ShortStateHash, user_id: &UserId) -> bool { - let s = self.user_membership(shortstatehash, user_id).await; - s == MembershipState::Join || s == MembershipState::Invite -} - -/// Get membership for given user in state -#[implement(super::Service)] -pub async fn user_membership( - &self, - shortstatehash: ShortStateHash, - user_id: &UserId, -) -> MembershipState { - self.state_get_content(shortstatehash, &StateEventType::RoomMember, user_id.as_str()) - .await - .map_or(MembershipState::Leave, |c: RoomMemberEventContent| c.membership) -} - -/// Returns a single PDU from `room_id` with key (`event_type`,`state_key`). -#[implement(super::Service)] -pub async fn state_get_content( - &self, - shortstatehash: ShortStateHash, - event_type: &StateEventType, - state_key: &str, -) -> Result -where - T: for<'de> Deserialize<'de>, -{ - self.state_get(shortstatehash, event_type, state_key) - .await - .and_then(|event| event.get_content()) -} - -#[implement(super::Service)] -pub async fn state_contains( - &self, - shortstatehash: ShortStateHash, - event_type: &StateEventType, - state_key: &str, -) -> bool { - let Ok(shortstatekey) = self - .services - .short - .get_shortstatekey(event_type, state_key) - .await - else { - return false; - }; - - self.state_contains_shortstatekey(shortstatehash, shortstatekey) - .await -} - -#[implement(super::Service)] -pub async fn state_contains_type( - &self, - shortstatehash: ShortStateHash, - event_type: &StateEventType, -) -> bool { - let state_keys = self.state_keys(shortstatehash, event_type); - - pin_mut!(state_keys); - state_keys.next().await.is_some() -} - -#[implement(super::Service)] -pub async fn state_contains_shortstatekey( - &self, - shortstatehash: ShortStateHash, - shortstatekey: ShortStateKey, -) -> bool { - let start = compress_state_event(shortstatekey, 0); - let end = compress_state_event(shortstatekey, u64::MAX); - - self.load_full_state(shortstatehash) - .map_ok(|full_state| full_state.range(start..=end).next().copied()) - .await - .flat_ok() - .is_some() -} - -/// Returns a single PDU from `room_id` with key (`event_type`, -/// `state_key`). -#[implement(super::Service)] -pub async fn state_get( - &self, - shortstatehash: ShortStateHash, - event_type: &StateEventType, - state_key: &str, -) -> Result { - self.state_get_id(shortstatehash, event_type, state_key) - .and_then(async |event_id: OwnedEventId| self.services.timeline.get_pdu(&event_id).await) - .await -} - -/// Returns a single EventId from `room_id` with key (`event_type`, -/// `state_key`). -#[implement(super::Service)] -pub async fn state_get_id( - &self, - shortstatehash: ShortStateHash, - event_type: &StateEventType, - state_key: &str, -) -> Result -where - Id: for<'de> Deserialize<'de> + Sized + ToOwned, - ::Owned: Borrow, -{ - let shorteventid = self - .state_get_shortid(shortstatehash, event_type, state_key) - .await?; - - self.services - .short - .get_eventid_from_short(shorteventid) - .await -} - -/// Returns a single EventId from `room_id` with key (`event_type`, -/// `state_key`). -#[implement(super::Service)] -pub async fn state_get_shortid( - &self, - shortstatehash: ShortStateHash, - event_type: &StateEventType, - state_key: &str, -) -> Result { - let shortstatekey = self - .services - .short - .get_shortstatekey(event_type, state_key) - .await?; - - let start = compress_state_event(shortstatekey, 0); - let end = compress_state_event(shortstatekey, u64::MAX); - self.load_full_state(shortstatehash) - .map_ok(|full_state| { - full_state - .range(start..=end) - .next() - .copied() - .map(parse_compressed_state_event) - .map(at!(1)) - .ok_or(err!(Request(NotFound("Not found in room state")))) - }) - .await? -} - -/// Iterates the state_keys for an event_type in the state; current state -/// event_id included. -#[implement(super::Service)] -pub fn state_keys_with_ids<'a, Id>( - &'a self, - shortstatehash: ShortStateHash, - event_type: &'a StateEventType, -) -> impl Stream + Send + 'a -where - Id: for<'de> Deserialize<'de> + Send + Sized + ToOwned + 'a, - ::Owned: Borrow, -{ - let state_keys_with_short_ids = self - .state_keys_with_shortids(shortstatehash, event_type) - .unzip() - .map(|(ssks, sids): (Vec, Vec)| (ssks, sids)) - .shared(); - - let state_keys = state_keys_with_short_ids - .clone() - .map(at!(0)) - .map(Vec::into_iter) - .map(IterStream::stream) - .flatten_stream(); - - let shorteventids = state_keys_with_short_ids - .map(at!(1)) - .map(Vec::into_iter) - .map(IterStream::stream) - .flatten_stream(); - - self.services - .short - .multi_get_eventid_from_short(shorteventids) - .zip(state_keys) - .ready_filter_map(|(eid, sk)| eid.map(move |eid| (sk, eid)).ok()) -} - -/// Iterates the state_keys for an event_type in the state; current state -/// event_id included. -#[implement(super::Service)] -pub fn state_keys_with_shortids<'a>( - &'a self, - shortstatehash: ShortStateHash, - event_type: &'a StateEventType, -) -> impl Stream + Send + 'a { - let short_ids = self - .state_full_shortids(shortstatehash) - .ignore_err() - .unzip() - .map(|(ssks, sids): (Vec, Vec)| (ssks, sids)) - .boxed() - .shared(); - - let shortstatekeys = short_ids - .clone() - .map(at!(0)) - .map(Vec::into_iter) - .map(IterStream::stream) - .flatten_stream(); - - let shorteventids = short_ids - .map(at!(1)) - .map(Vec::into_iter) - .map(IterStream::stream) - .flatten_stream(); - - self.services - .short - .multi_get_statekey_from_short(shortstatekeys) - .zip(shorteventids) - .ready_filter_map(|(res, id)| res.map(|res| (res, id)).ok()) - .ready_filter_map(move |((event_type_, state_key), event_id)| { - event_type_.eq(event_type).then_some((state_key, event_id)) - }) -} - -/// Iterates the state_keys for an event_type in the state -#[implement(super::Service)] -pub fn state_keys<'a>( - &'a self, - shortstatehash: ShortStateHash, - event_type: &'a StateEventType, -) -> impl Stream + Send + 'a { - let short_ids = self - .state_full_shortids(shortstatehash) - .ignore_err() - .map(at!(0)); - - self.services - .short - .multi_get_statekey_from_short(short_ids) - .ready_filter_map(Result::ok) - .ready_filter_map(move |(event_type_, state_key)| { - event_type_.eq(event_type).then_some(state_key) - }) -} - -/// Returns the state events removed between the interval (present in .0 but -/// not in .1) -#[implement(super::Service)] -#[inline] -pub async fn state_removed( - &self, - shortstatehash: pair_of!(ShortStateHash), -) -> Result> { - self.state_added((shortstatehash.1, shortstatehash.0)).await -} - -/// Returns the state events added between the interval (present in .1 but -/// not in .0) -#[implement(super::Service)] -pub async fn state_added( - &self, - shortstatehash: pair_of!(ShortStateHash), -) -> Result> { - let full_state_a = self.load_full_state(shortstatehash.0).await?; - let full_state_b = self.load_full_state(shortstatehash.1).await?; - - Ok(full_state_b - .difference(&full_state_a) - .copied() - .map(parse_compressed_state_event) - .collect()) -} - -#[implement(super::Service)] -pub fn state_full( - &self, - shortstatehash: ShortStateHash, -) -> impl Stream + Send + '_ { - self.state_full_pdus(shortstatehash) - .ready_filter_map(|pdu| { - Some(((pdu.kind().to_string().into(), pdu.state_key()?.into()), pdu)) - }) -} - -#[implement(super::Service)] -pub fn state_full_pdus( - &self, - shortstatehash: ShortStateHash, -) -> impl Stream + Send + '_ { - let short_ids = self - .state_full_shortids(shortstatehash) - .ignore_err() - .map(at!(1)); - - self.services - .short - .multi_get_eventid_from_short(short_ids) - .ready_filter_map(Result::ok) - .broad_filter_map(move |event_id: OwnedEventId| async move { - self.services.timeline.get_pdu(&event_id).await.ok() - }) -} - -/// Builds a StateMap by iterating over all keys that start -/// with state_hash, this gives the full state for the given state_hash. -#[implement(super::Service)] -pub fn state_full_ids<'a, Id>( - &'a self, - shortstatehash: ShortStateHash, -) -> impl Stream + Send + 'a -where - Id: for<'de> Deserialize<'de> + Send + Sized + ToOwned + 'a, - ::Owned: Borrow, -{ - let shortids = self - .state_full_shortids(shortstatehash) - .ignore_err() - .unzip() - .shared(); - - let shortstatekeys = shortids - .clone() - .map(at!(0)) - .map(Vec::into_iter) - .map(IterStream::stream) - .flatten_stream(); - - let shorteventids = shortids - .map(at!(1)) - .map(Vec::into_iter) - .map(IterStream::stream) - .flatten_stream(); - - self.services - .short - .multi_get_eventid_from_short(shorteventids) - .zip(shortstatekeys) - .ready_filter_map(|(event_id, shortstatekey)| Some((shortstatekey, event_id.ok()?))) -} - -#[implement(super::Service)] -pub fn state_full_shortids( - &self, - shortstatehash: ShortStateHash, -) -> impl Stream> + Send + '_ { - self.load_full_state(shortstatehash) - .map_ok(|full_state| { - full_state - .deref() - .iter() - .copied() - .map(parse_compressed_state_event) - .collect() - }) - .map_ok(Vec::into_iter) - .map_ok(IterStream::try_stream) - .try_flatten_stream() - .boxed() -} - -#[implement(super::Service)] -#[tracing::instrument(name = "load", level = "debug", skip(self))] -async fn load_full_state(&self, shortstatehash: ShortStateHash) -> Result> { - self.services - .state_compressor - .load_shortstatehash_info(shortstatehash) - .map_err(|e| err!(Database("Missing state IDs: {e}"))) - .map_ok(|vec| vec.last().expect("at least one layer").full_state.clone()) - .await -} - -/// Returns the state hash for this pdu. -#[implement(super::Service)] -pub async fn pdu_shortstatehash(&self, event_id: &EventId) -> Result { - const BUFSIZE: usize = size_of::(); - - self.services - .short - .get_shorteventid(event_id) - .and_then(|shorteventid| { - self.db - .shorteventid_shortstatehash - .aqry::(&shorteventid) - }) - .await - .deserialized() +impl super::Service { + /// The user was a joined member at this state (potentially in the past) + pub async fn user_was_joined( + &self, + shortstatehash: ShortStateHash, + user_id: &UserId, + ) -> bool { + self.user_membership(shortstatehash, user_id).await == MembershipState::Join + } + + /// The user was an invited or joined room member at this state (potentially + /// in the past) + pub async fn user_was_invited( + &self, + shortstatehash: ShortStateHash, + user_id: &UserId, + ) -> bool { + matches!( + self.user_membership(shortstatehash, user_id).await, + MembershipState::Invite | MembershipState::Join + ) + } + + /// Get membership for given user in state. If the user has no membership, + /// leave is returned. + pub async fn user_membership( + &self, + shortstatehash: ShortStateHash, + user_id: &UserId, + ) -> MembershipState { + self.state_get_content(shortstatehash, &StateEventType::RoomMember, user_id.as_str()) + .await + .map_or(MembershipState::Leave, |c: RoomMemberEventContent| c.membership) + } + + /// Returns a single PDU from `room_id` with key (`event_type`,`state_key`). + pub async fn state_get_content( + &self, + shortstatehash: ShortStateHash, + event_type: &StateEventType, + state_key: &str, + ) -> Result + where + T: for<'de> Deserialize<'de>, + { + self.state_get(shortstatehash, event_type, state_key) + .await + .and_then(|event| event.get_content()) + } + + /// Returns true if the state at the given snapshot contains a specific + /// state key. Returns false if the shortstatehash is not found. + pub async fn state_contains( + &self, + shortstatehash: ShortStateHash, + event_type: &StateEventType, + state_key: &str, + ) -> bool { + let Ok(shortstatekey) = self + .services + .short + .get_shortstatekey(event_type, state_key) + .await + else { + return false; + }; + + self.state_contains_shortstatekey(shortstatehash, shortstatekey) + .await + } + + /// Returns true if the state at the given snapshot contains a specific + /// event type, regardless of state key. Returns false if the shortstatehash + /// is not found. + pub async fn state_contains_type( + &self, + shortstatehash: ShortStateHash, + event_type: &StateEventType, + ) -> bool { + let state_keys = self.state_keys(shortstatehash, event_type); + + pin_mut!(state_keys); + state_keys.next().await.is_some() + } + + /// Returns true if the given snapshot contains a compressed state ID. + /// Returns false if the snapshot cannot be found. + pub async fn state_contains_shortstatekey( + &self, + shortstatehash: ShortStateHash, + shortstatekey: ShortStateKey, + ) -> bool { + let start = compress_state_event(shortstatekey, 0); + let end = compress_state_event(shortstatekey, u64::MAX); + + self.load_full_state(shortstatehash) + .map_ok(|full_state| full_state.range(start..=end).next().copied()) + .await + .flat_ok() + .is_some() + } + + /// Returns a single PDU from `room_id` with key (`event_type`, + /// `state_key`). + pub async fn state_get( + &self, + shortstatehash: ShortStateHash, + event_type: &StateEventType, + state_key: &str, + ) -> Result { + self.state_get_id(shortstatehash, event_type, state_key) + .and_then(async |event_id: OwnedEventId| { + self.services.timeline.get_pdu(&event_id).await + }) + .await + } + + /// Returns a single EventId from `room_id` with key (`event_type`, + /// `state_key`). + pub async fn state_get_id( + &self, + shortstatehash: ShortStateHash, + event_type: &StateEventType, + state_key: &str, + ) -> Result + where + Id: for<'de> Deserialize<'de> + Sized + ToOwned, + ::Owned: Borrow, + { + let shorteventid = self + .state_get_shortid(shortstatehash, event_type, state_key) + .await?; + + self.services + .short + .get_eventid_from_short(shorteventid) + .await + } + + /// Returns a single EventId from `room_id` with key (`event_type`, + /// `state_key`). + pub async fn state_get_shortid( + &self, + shortstatehash: ShortStateHash, + event_type: &StateEventType, + state_key: &str, + ) -> Result { + let shortstatekey = self + .services + .short + .get_shortstatekey(event_type, state_key) + .await?; + + let start = compress_state_event(shortstatekey, 0); + let end = compress_state_event(shortstatekey, u64::MAX); + self.load_full_state(shortstatehash) + .map_ok(|full_state| { + full_state + .range(start..=end) + .next() + .copied() + .map(parse_compressed_state_event) + .map(at!(1)) + .ok_or(err!(Request(NotFound("Not found in room state")))) + }) + .await? + } + + /// Iterates the state_keys for an event_type in the state; current state + /// event_id included. + pub fn state_keys_with_ids<'a, Id>( + &'a self, + shortstatehash: ShortStateHash, + event_type: &'a StateEventType, + ) -> impl Stream + Send + 'a + where + Id: for<'de> Deserialize<'de> + Send + Sized + ToOwned + 'a, + ::Owned: Borrow, + { + let state_keys_with_short_ids = self + .state_keys_with_shortids(shortstatehash, event_type) + .unzip() + .map(|(ssks, sids): (Vec, Vec)| (ssks, sids)) + .shared(); + + let state_keys = state_keys_with_short_ids + .clone() + .map(at!(0)) + .map(Vec::into_iter) + .map(IterStream::stream) + .flatten_stream(); + + let shorteventids = state_keys_with_short_ids + .map(at!(1)) + .map(Vec::into_iter) + .map(IterStream::stream) + .flatten_stream(); + + self.services + .short + .multi_get_eventid_from_short(shorteventids) + .zip(state_keys) + .ready_filter_map(|(eid, sk)| eid.map(move |eid| (sk, eid)).ok()) + } + + /// Iterates the state_keys for an event_type in the state; current state + /// event_id included. + pub fn state_keys_with_shortids<'a>( + &'a self, + shortstatehash: ShortStateHash, + event_type: &'a StateEventType, + ) -> impl Stream + Send + 'a { + let short_ids = self + .state_full_shortids(shortstatehash) + .ignore_err() + .unzip() + .map(|(ssks, sids): (Vec, Vec)| (ssks, sids)) + .boxed() + .shared(); + + let shortstatekeys = short_ids + .clone() + .map(at!(0)) + .map(Vec::into_iter) + .map(IterStream::stream) + .flatten_stream(); + + let shorteventids = short_ids + .map(at!(1)) + .map(Vec::into_iter) + .map(IterStream::stream) + .flatten_stream(); + + self.services + .short + .multi_get_statekey_from_short(shortstatekeys) + .zip(shorteventids) + .ready_filter_map(|(res, id)| res.map(|res| (res, id)).ok()) + .ready_filter_map(move |((event_type_, state_key), event_id)| { + event_type_.eq(event_type).then_some((state_key, event_id)) + }) + } + + /// Iterates the state_keys for an event_type in the state + pub fn state_keys<'a>( + &'a self, + shortstatehash: ShortStateHash, + event_type: &'a StateEventType, + ) -> impl Stream + Send + 'a { + let short_ids = self + .state_full_shortids(shortstatehash) + .ignore_err() + .map(at!(0)); + + self.services + .short + .multi_get_statekey_from_short(short_ids) + .ready_filter_map(Result::ok) + .ready_filter_map(move |(event_type_, state_key)| { + event_type_.eq(event_type).then_some(state_key) + }) + } + + /// Returns the state events removed between the interval (present in .0 but + /// not in .1) + pub async fn state_removed( + &self, + shortstatehash: pair_of!(ShortStateHash), + ) -> Result> { + self.state_added((shortstatehash.1, shortstatehash.0)).await + } + + /// Returns the state events added between the interval (present in .1 but + /// not in .0) + pub async fn state_added( + &self, + shortstatehash: pair_of!(ShortStateHash), + ) -> Result> { + let full_state_a = self.load_full_state(shortstatehash.0).await?; + let full_state_b = self.load_full_state(shortstatehash.1).await?; + + Ok(full_state_b + .difference(&full_state_a) + .copied() + .map(parse_compressed_state_event) + .collect()) + } + + /// Returns the full set of state keys in the state snapshot. + pub fn state_full( + &self, + shortstatehash: ShortStateHash, + ) -> impl Stream + Send + '_ { + self.state_full_pdus(shortstatehash) + .ready_filter_map(|pdu| { + Some(((pdu.kind().to_string().into(), pdu.state_key()?.into()), pdu)) + }) + } + + /// Fetches all PDUs belonging to the given state snapshot. + pub fn state_full_pdus( + &self, + shortstatehash: ShortStateHash, + ) -> impl Stream + Send + '_ { + let short_ids = self + .state_full_shortids(shortstatehash) + .ignore_err() + .map(at!(1)); + + self.services + .short + .multi_get_eventid_from_short(short_ids) + .ready_filter_map(Result::ok) + .broad_filter_map(move |event_id: OwnedEventId| async move { + self.services.timeline.get_pdu(&event_id).await.ok() + }) + } + + /// Builds a StateMap by iterating over all keys that start + /// with state_hash, this gives the full state for the given state_hash. + pub fn state_full_ids<'a, Id>( + &'a self, + shortstatehash: ShortStateHash, + ) -> impl Stream + Send + 'a + where + Id: for<'de> Deserialize<'de> + Send + Sized + ToOwned + 'a, + ::Owned: Borrow, + { + let shortids = self + .state_full_shortids(shortstatehash) + .ignore_err() + .unzip() + .shared(); + + let shortstatekeys = shortids + .clone() + .map(at!(0)) + .map(Vec::into_iter) + .map(IterStream::stream) + .flatten_stream(); + + let shorteventids = shortids + .map(at!(1)) + .map(Vec::into_iter) + .map(IterStream::stream) + .flatten_stream(); + + self.services + .short + .multi_get_eventid_from_short(shorteventids) + .zip(shortstatekeys) + .ready_filter_map(|(event_id, shortstatekey)| Some((shortstatekey, event_id.ok()?))) + } + + /// Fetches all compressed state IDs that make up the given state snapshot. + pub fn state_full_shortids( + &self, + shortstatehash: ShortStateHash, + ) -> impl Stream> + Send + '_ { + self.load_full_state(shortstatehash) + .map_ok(|full_state| { + full_state + .deref() + .iter() + .copied() + .map(parse_compressed_state_event) + .collect() + }) + .map_ok(Vec::into_iter) + .map_ok(IterStream::try_stream) + .try_flatten_stream() + .boxed() + } + + /// Loads the full state for a state snapshot + async fn load_full_state( + &self, + shortstatehash: ShortStateHash, + ) -> Result> { + self.services + .state_compressor + .load_shortstatehash_info(shortstatehash) + .map_err(|e| err!(Database("Missing state IDs: {e}"))) + .map_ok(|vec| vec.last().expect("at least one layer").full_state.clone()) + .await + } + + /// Returns the state hash for this pdu. + pub async fn pdu_shortstatehash(&self, event_id: &EventId) -> Result { + const BUFSIZE: usize = size_of::(); + + self.services + .short + .get_shorteventid(event_id) + .and_then(|shorteventid| { + self.db + .shorteventid_shortstatehash + .aqry::(&shorteventid) + }) + .await + .deserialized() + } } diff --git a/src/service/rooms/state_accessor/user_can.rs b/src/service/rooms/state_accessor/user_can.rs index fb74b27ad..b27a57304 100644 --- a/src/service/rooms/state_accessor/user_can.rs +++ b/src/service/rooms/state_accessor/user_can.rs @@ -1,4 +1,4 @@ -use conduwuit::{Err, Result, implement, matrix::Event, pdu::PartialPdu}; +use conduwuit::{Err, Result, matrix::Event, pdu::PartialPdu}; use ruma::{ EventId, RoomId, UserId, events::{ @@ -12,140 +12,139 @@ use ruma::{ use crate::rooms::state::RoomMutexGuard; -/// Checks if a given user can redact a given event -/// -/// If federation is true, it allows redaction events from any user of the -/// same server as the original event sender -#[implement(super::Service)] -pub async fn user_can_redact( - &self, - redacts: &EventId, - sender: &UserId, - room_id: &RoomId, - federation: bool, -) -> Result { - let redacting_event = self.services.timeline.get_pdu(redacts).await; +impl super::Service { + /// Checks if a given user can redact a given event + /// + /// If federation is true, it allows redaction events from any user of the + /// same server as the original event sender + pub async fn user_can_redact( + &self, + redacts: &EventId, + sender: &UserId, + room_id: &RoomId, + federation: bool, + ) -> Result { + let redacting_event = self.services.timeline.get_pdu(redacts).await; - if redacting_event - .as_ref() - .is_ok_and(|pdu| *pdu.kind() == TimelineEventType::RoomCreate) - { - return Err!(Request(Forbidden("Redacting m.room.create is not safe, forbidding."))); + if redacting_event + .as_ref() + .is_ok_and(|pdu| *pdu.kind() == TimelineEventType::RoomCreate) + { + return Err!(Request(Forbidden("Redacting m.room.create is not safe, forbidding."))); + } + + if redacting_event + .as_ref() + .is_ok_and(|pdu| *pdu.kind() == TimelineEventType::RoomServerAcl) + { + return Err!(Request(Forbidden( + "Redacting m.room.server_acl will result in the room being inaccessible for \ + everyone (empty allow key), forbidding." + ))); + } + + let power_levels = self.get_room_power_levels(room_id).await; + + if power_levels.user_can_redact_event_of_other(sender) { + return Ok(true); + } + + if power_levels.user_can_redact_own_event(sender) { + let is_own_event = match redacting_event { + | Ok(redacting_event) => + if federation { + redacting_event.sender().server_name() == sender.server_name() + } else { + redacting_event.sender() == sender + }, + | _ => false, + }; + + return Ok(is_own_event); + } + + Ok(false) } - if redacting_event - .as_ref() - .is_ok_and(|pdu| *pdu.kind() == TimelineEventType::RoomServerAcl) - { - return Err!(Request(Forbidden( - "Redacting m.room.server_acl will result in the room being inaccessible for \ - everyone (empty allow key), forbidding." - ))); - } - - let power_levels = self.get_room_power_levels(room_id).await; - - if power_levels.user_can_redact_event_of_other(sender) { - return Ok(true); - } - - if power_levels.user_can_redact_own_event(sender) { - let is_own_event = match redacting_event { - | Ok(redacting_event) => - if federation { - redacting_event.sender().server_name() == sender.server_name() - } else { - redacting_event.sender() == sender - }, - | _ => false, + /// Whether a user is allowed to see an event, based on + /// the room's history_visibility at that event's state. + pub async fn user_can_see_event( + &self, + user_id: &UserId, + room_id: &RoomId, + event_id: &EventId, + ) -> bool { + let Ok(shortstatehash) = self.pdu_shortstatehash(event_id).await else { + return true; }; - return Ok(is_own_event); + let currently_member = self.services.state_cache.is_joined(user_id, room_id).await; + + let history_visibility = self + .state_get_content(shortstatehash, &StateEventType::RoomHistoryVisibility, "") + .await + .map_or(HistoryVisibility::Shared, |c: RoomHistoryVisibilityEventContent| { + c.history_visibility + }); + + match history_visibility { + | HistoryVisibility::Invited => { + // Allow if any member on requesting server was AT LEAST invited, else deny + self.user_was_invited(shortstatehash, user_id).await + }, + | HistoryVisibility::Joined => { + // Allow if any member on requested server was joined, else deny + self.user_was_joined(shortstatehash, user_id).await + }, + | HistoryVisibility::WorldReadable => true, + | HistoryVisibility::Shared | _ => currently_member, + } } - Ok(false) -} + /// Whether a user is allowed to see an event, based on + /// the room's history_visibility at that event's state. + pub async fn user_can_see_state_events(&self, user_id: &UserId, room_id: &RoomId) -> bool { + if self.services.state_cache.is_joined(user_id, room_id).await { + return true; + } -/// Whether a user is allowed to see an event, based on -/// the room's history_visibility at that event's state. -#[implement(super::Service)] -#[tracing::instrument(skip_all, level = "trace")] -pub async fn user_can_see_event( - &self, - user_id: &UserId, - room_id: &RoomId, - event_id: &EventId, -) -> bool { - let Ok(shortstatehash) = self.pdu_shortstatehash(event_id).await else { - return true; - }; + let history_visibility = self + .room_state_get_content(room_id, &StateEventType::RoomHistoryVisibility, "") + .await + .map_or(HistoryVisibility::Shared, |c: RoomHistoryVisibilityEventContent| { + c.history_visibility + }); - let currently_member = self.services.state_cache.is_joined(user_id, room_id).await; - - let history_visibility = self - .state_get_content(shortstatehash, &StateEventType::RoomHistoryVisibility, "") - .await - .map_or(HistoryVisibility::Shared, |c: RoomHistoryVisibilityEventContent| { - c.history_visibility - }); - - match history_visibility { - | HistoryVisibility::Invited => { - // Allow if any member on requesting server was AT LEAST invited, else deny - self.user_was_invited(shortstatehash, user_id).await - }, - | HistoryVisibility::Joined => { - // Allow if any member on requested server was joined, else deny - self.user_was_joined(shortstatehash, user_id).await - }, - | HistoryVisibility::WorldReadable => true, - | HistoryVisibility::Shared | _ => currently_member, - } -} - -/// Whether a user is allowed to see an event, based on -/// the room's history_visibility at that event's state. -#[implement(super::Service)] -#[tracing::instrument(skip_all, level = "trace")] -pub async fn user_can_see_state_events(&self, user_id: &UserId, room_id: &RoomId) -> bool { - if self.services.state_cache.is_joined(user_id, room_id).await { - return true; + match history_visibility { + | HistoryVisibility::Invited => + self.services.state_cache.is_invited(user_id, room_id).await, + | HistoryVisibility::WorldReadable => true, + | _ => false, + } } - let history_visibility = self - .room_state_get_content(room_id, &StateEventType::RoomHistoryVisibility, "") - .await - .map_or(HistoryVisibility::Shared, |c: RoomHistoryVisibilityEventContent| { - c.history_visibility - }); - - match history_visibility { - | HistoryVisibility::Invited => - self.services.state_cache.is_invited(user_id, room_id).await, - | HistoryVisibility::WorldReadable => true, - | _ => false, + /// Checks if the target user can send a specific invite by creating and + /// discarding a new PDU. If creating the PDU fails, the user is not + /// permitted to issue the invite. + pub async fn user_can_invite( + &self, + room_id: &RoomId, + sender: &UserId, + target_user: &UserId, + state_lock: &RoomMutexGuard, + ) -> bool { + self.services + .timeline + .create_event( + PartialPdu::state( + target_user.as_str(), + &RoomMemberEventContent::new(MembershipState::Invite), + ), + sender, + Some(room_id), + state_lock, + ) + .await + .is_ok() } } - -#[implement(super::Service)] -pub async fn user_can_invite( - &self, - room_id: &RoomId, - sender: &UserId, - target_user: &UserId, - state_lock: &RoomMutexGuard, -) -> bool { - self.services - .timeline - .create_hash_and_sign_event( - PartialPdu::state( - target_user.as_str(), - &RoomMemberEventContent::new(MembershipState::Invite), - ), - sender, - Some(room_id), - state_lock, - ) - .await - .is_ok() -} From ecbe035b361a292d8b02b421b3ddf0a94b201a9a Mon Sep 17 00:00:00 2001 From: timedout Date: Fri, 26 Jun 2026 04:25:10 +0100 Subject: [PATCH 37/70] style: Document & refactor rooms/short service --- src/service/rooms/short/mod.rs | 440 +++++++++++++++++---------------- 1 file changed, 224 insertions(+), 216 deletions(-) diff --git a/src/service/rooms/short/mod.rs b/src/service/rooms/short/mod.rs index 20685442b..53d70fe85 100644 --- a/src/service/rooms/short/mod.rs +++ b/src/service/rooms/short/mod.rs @@ -2,7 +2,7 @@ use std::{borrow::Borrow, fmt::Debug, mem::size_of_val, sync::Arc}; pub use conduwuit::matrix::pdu::{ShortEventId, ShortId, ShortRoomId, ShortStateKey}; use conduwuit::{ - Result, err, implement, + Result, err, matrix::StateKey, pair_of, utils::{self, IterStream, ReadyExt}, @@ -57,232 +57,240 @@ impl crate::Service for Service { fn name(&self) -> &str { crate::service::make_name(std::module_path!()) } } -#[implement(Service)] -pub async fn get_or_create_shorteventid(&self, event_id: &EventId) -> ShortEventId { - if let Ok(shorteventid) = self.get_shorteventid(event_id).await { - return shorteventid; +impl Service { + /// Gets or creates a short event ID + pub async fn get_or_create_shorteventid(&self, event_id: &EventId) -> ShortEventId { + if let Ok(shorteventid) = self.get_shorteventid(event_id).await { + return shorteventid; + } + + self.create_shorteventid(event_id) } - self.create_shorteventid(event_id) -} - -#[implement(Service)] -pub fn multi_get_or_create_shorteventid<'a, I>( - &'a self, - event_ids: I, -) -> impl Stream + Send + 'a -where - I: Iterator + Clone + Debug + Send + 'a, -{ - event_ids - .clone() - .stream() - .get(&self.db.eventid_shorteventid) - .zip(event_ids.into_iter().stream()) - .map(|(result, event_id)| match result { - | Ok(ref short) => utils::u64_from_u8(short), - | Err(_) => self.create_shorteventid(event_id), - }) -} - -#[implement(Service)] -fn create_shorteventid(&self, event_id: &EventId) -> ShortEventId { - const BUFSIZE: usize = size_of::(); - - let short = self.services.globals.next_count().unwrap(); - debug_assert!(size_of_val(&short) == BUFSIZE, "buffer requirement changed"); - - self.db - .eventid_shorteventid - .raw_aput::(event_id, short); - - self.db - .shorteventid_eventid - .aput_raw::(short, event_id); - - short -} - -#[implement(Service)] -pub async fn get_shorteventid(&self, event_id: &EventId) -> Result { - self.db - .eventid_shorteventid - .get(event_id) - .await - .deserialized() -} - -#[implement(Service)] -pub async fn get_or_create_shortstatekey( - &self, - event_type: &StateEventType, - state_key: &str, -) -> ShortStateKey { - const BUFSIZE: usize = size_of::(); - - if let Ok(shortstatekey) = self.get_shortstatekey(event_type, state_key).await { - return shortstatekey; - } - - let key = (event_type, state_key); - let shortstatekey = self.services.globals.next_count().unwrap(); - debug_assert!(size_of_val(&shortstatekey) == BUFSIZE, "buffer requirement changed"); - - self.db - .statekey_shortstatekey - .put_aput::(key, shortstatekey); - - self.db - .shortstatekey_statekey - .aput_put::(shortstatekey, key); - - shortstatekey -} - -#[implement(Service)] -pub async fn get_shortstatekey( - &self, - event_type: &StateEventType, - state_key: &str, -) -> Result { - let key = (event_type, state_key); - self.db - .statekey_shortstatekey - .qry(&key) - .await - .deserialized() -} - -#[implement(Service)] -pub async fn get_eventid_from_short(&self, shorteventid: ShortEventId) -> Result -where - Id: for<'de> Deserialize<'de> + Sized + ToOwned, - ::Owned: Borrow, -{ - const BUFSIZE: usize = size_of::(); - - self.db - .shorteventid_eventid - .aqry::(&shorteventid) - .await - .deserialized() - .map_err(|e| err!(Database("Failed to find EventId from short {shorteventid:?}: {e:?}"))) -} - -#[implement(Service)] -pub fn multi_get_eventid_from_short<'a, Id, S>( - &'a self, - shorteventid: S, -) -> impl Stream> + Send + 'a -where - S: Stream + Send + 'a, - Id: for<'de> Deserialize<'de> + Sized + ToOwned + 'a, - ::Owned: Borrow, -{ - shorteventid - .qry(&self.db.shorteventid_eventid) - .map(Deserialized::deserialized) -} - -#[implement(Service)] -pub async fn get_statekey_from_short( - &self, - shortstatekey: ShortStateKey, -) -> Result<(StateEventType, StateKey)> { - const BUFSIZE: usize = size_of::(); - - self.db - .shortstatekey_statekey - .aqry::(&shortstatekey) - .await - .deserialized() - .map_err(|e| { - err!(Database( - "Failed to find (StateEventType, state_key) from short {shortstatekey:?}: {e:?}" - )) - }) -} - -#[implement(Service)] -pub fn multi_get_statekey_from_short<'a, S>( - &'a self, - shortstatekey: S, -) -> impl Stream> + Send + 'a -where - S: Stream + Send + 'a, -{ - shortstatekey - .qry(&self.db.shortstatekey_statekey) - .map(Deserialized::deserialized) -} - -/// Returns (shortstatehash, already_existed) -#[implement(Service)] -pub async fn get_or_create_shortstatehash(&self, state_hash: &[u8]) -> (ShortStateHash, bool) { - const BUFSIZE: usize = size_of::(); - - if let Ok(shortstatehash) = self - .db - .statehash_shortstatehash - .get(state_hash) - .await - .deserialized() + /// Gets or creates multiple short event IDs. + pub fn multi_get_or_create_shorteventid<'a, I>( + &'a self, + event_ids: I, + ) -> impl Stream + Send + 'a + where + I: Iterator + Clone + Debug + Send + 'a, { - return (shortstatehash, true); + event_ids + .clone() + .stream() + .get(&self.db.eventid_shorteventid) + .zip(event_ids.into_iter().stream()) + .map(|(result, event_id)| match result { + | Ok(ref short) => utils::u64_from_u8(short), + | Err(_) => self.create_shorteventid(event_id), + }) } - let shortstatehash = self.services.globals.next_count().unwrap(); - debug_assert!(size_of_val(&shortstatehash) == BUFSIZE, "buffer requirement changed"); + /// Creates a short event ID + fn create_shorteventid(&self, event_id: &EventId) -> ShortEventId { + const BUFSIZE: usize = size_of::(); - self.db - .statehash_shortstatehash - .raw_aput::(state_hash, shortstatehash); + let short = self.services.globals.next_count().unwrap(); + debug_assert!(size_of_val(&short) == BUFSIZE, "buffer requirement changed"); - (shortstatehash, false) -} + self.db + .eventid_shorteventid + .raw_aput::(event_id, short); -#[implement(Service)] -pub async fn get_shortroomid(&self, room_id: &RoomId) -> Result { - self.db.roomid_shortroomid.get(room_id).await.deserialized() -} + self.db + .shorteventid_eventid + .aput_raw::(short, event_id); -#[implement(Service)] -pub async fn get_or_create_shortroomid(&self, room_id: &RoomId) -> ShortRoomId { - self.db - .roomid_shortroomid - .get(room_id) - .await - .deserialized() - .unwrap_or_else(|_| { - const BUFSIZE: usize = size_of::(); + short + } - let short = self.services.globals.next_count().unwrap(); - debug_assert!(size_of_val(&short) == BUFSIZE, "buffer requirement changed"); + /// Gets a short event ID. + pub async fn get_shorteventid(&self, event_id: &EventId) -> Result { + self.db + .eventid_shorteventid + .get(event_id) + .await + .deserialized() + } - self.db - .roomid_shortroomid - .raw_aput::(room_id, short); + /// Gets or creates a short ID for a state key pair. + pub async fn get_or_create_shortstatekey( + &self, + event_type: &StateEventType, + state_key: &str, + ) -> ShortStateKey { + const BUFSIZE: usize = size_of::(); - short + if let Ok(shortstatekey) = self.get_shortstatekey(event_type, state_key).await { + return shortstatekey; + } + + let key = (event_type, state_key); + let shortstatekey = self.services.globals.next_count().unwrap(); + debug_assert!(size_of_val(&shortstatekey) == BUFSIZE, "buffer requirement changed"); + + self.db + .statekey_shortstatekey + .put_aput::(key, shortstatekey); + + self.db + .shortstatekey_statekey + .aput_put::(shortstatekey, key); + + shortstatekey + } + + /// Gets a short ID for a state key pair. + pub async fn get_shortstatekey( + &self, + event_type: &StateEventType, + state_key: &str, + ) -> Result { + let key = (event_type, state_key); + self.db + .statekey_shortstatekey + .qry(&key) + .await + .deserialized() + } + + /// Gets a full event ID from a short event ID. + pub async fn get_eventid_from_short(&self, shorteventid: ShortEventId) -> Result + where + Id: for<'de> Deserialize<'de> + Sized + ToOwned, + ::Owned: Borrow, + { + const BUFSIZE: usize = size_of::(); + + self.db + .shorteventid_eventid + .aqry::(&shorteventid) + .await + .deserialized() + .map_err(|e| { + err!(Database("Failed to find EventId from short {shorteventid:?}: {e:?}")) + }) + } + + /// Gets multiple full event IDs from a short event ID. + pub fn multi_get_eventid_from_short<'a, Id, S>( + &'a self, + shorteventid: S, + ) -> impl Stream> + Send + 'a + where + S: Stream + Send + 'a, + Id: for<'de> Deserialize<'de> + Sized + ToOwned + 'a, + ::Owned: Borrow, + { + shorteventid + .qry(&self.db.shorteventid_eventid) + .map(Deserialized::deserialized) + } + + /// Gets a state key pair from a short state key ID. + pub async fn get_statekey_from_short( + &self, + shortstatekey: ShortStateKey, + ) -> Result<(StateEventType, StateKey)> { + const BUFSIZE: usize = size_of::(); + + self.db + .shortstatekey_statekey + .aqry::(&shortstatekey) + .await + .deserialized() + .map_err(|e| { + err!(Database( + "Failed to find (StateEventType, state_key) from short {shortstatekey:?}: \ + {e:?}" + )) + }) + } + + /// Gets multiple state key pairs from their short IDs. + pub fn multi_get_statekey_from_short<'a, S>( + &'a self, + shortstatekey: S, + ) -> impl Stream> + Send + 'a + where + S: Stream + Send + 'a, + { + shortstatekey + .qry(&self.db.shortstatekey_statekey) + .map(Deserialized::deserialized) + } + + /// Gets or creates a short state hash ID. The boolean indicates whether a + /// new short ID was created. + pub async fn get_or_create_shortstatehash( + &self, + state_hash: &[u8], + ) -> (ShortStateHash, bool) { + const BUFSIZE: usize = size_of::(); + + if let Ok(shortstatehash) = self + .db + .statehash_shortstatehash + .get(state_hash) + .await + .deserialized() + { + return (shortstatehash, true); + } + + let shortstatehash = self.services.globals.next_count().unwrap(); + debug_assert!(size_of_val(&shortstatehash) == BUFSIZE, "buffer requirement changed"); + + self.db + .statehash_shortstatehash + .raw_aput::(state_hash, shortstatehash); + + (shortstatehash, false) + } + + /// Gets a short room ID. + pub async fn get_shortroomid(&self, room_id: &RoomId) -> Result { + self.db.roomid_shortroomid.get(room_id).await.deserialized() + } + + /// Gets or creates a short room ID. + pub async fn get_or_create_shortroomid(&self, room_id: &RoomId) -> ShortRoomId { + self.db + .roomid_shortroomid + .get(room_id) + .await + .deserialized() + .unwrap_or_else(|_| { + const BUFSIZE: usize = size_of::(); + + let short = self.services.globals.next_count().unwrap(); + debug_assert!(size_of_val(&short) == BUFSIZE, "buffer requirement changed"); + + self.db + .roomid_shortroomid + .raw_aput::(room_id, short); + + short + }) + } + + /// Gets the state map associated with a short state hash. + pub async fn multi_get_state_from_short<'a, S>( + &'a self, + short_state: S, + ) -> impl Stream> + Send + 'a + where + S: Stream + Send + 'a, + { + let (short_state_keys, short_event_ids): pair_of!(Vec<_>) = short_state.unzip().await; + + StreamExt::zip( + self.multi_get_statekey_from_short(stream::iter(short_state_keys)), + self.multi_get_eventid_from_short(stream::iter(short_event_ids)), + ) + .ready_filter_map(|state_event| match state_event { + | (Ok(state_key), Ok(event_id)) => Some(Ok((state_key, event_id))), + | (Err(e), _) | (_, Err(e)) => Some(Err(e)), }) -} - -#[implement(Service)] -pub async fn multi_get_state_from_short<'a, S>( - &'a self, - short_state: S, -) -> impl Stream> + Send + 'a -where - S: Stream + Send + 'a, -{ - let (short_state_keys, short_event_ids): pair_of!(Vec<_>) = short_state.unzip().await; - - StreamExt::zip( - self.multi_get_statekey_from_short(stream::iter(short_state_keys)), - self.multi_get_eventid_from_short(stream::iter(short_event_ids)), - ) - .ready_filter_map(|state_event| match state_event { - | (Ok(state_key), Ok(event_id)) => Some(Ok((state_key, event_id))), - | (Err(e), _) | (_, Err(e)) => Some(Err(e)), - }) + } } From 4bf0816bccd40137c2056be059abcfe72bd05932 Mon Sep 17 00:00:00 2001 From: timedout Date: Fri, 26 Jun 2026 04:31:47 +0100 Subject: [PATCH 38/70] style: Document & refactor rooms/search service --- src/service/rooms/search/mod.rs | 291 ++++++++++++++++---------------- 1 file changed, 147 insertions(+), 144 deletions(-) diff --git a/src/service/rooms/search/mod.rs b/src/service/rooms/search/mod.rs index 241d952ff..71ed82610 100644 --- a/src/service/rooms/search/mod.rs +++ b/src/service/rooms/search/mod.rs @@ -3,7 +3,7 @@ use std::sync::Arc; use conduwuit::{ PduCount, PduEvent, Result, arrayvec::ArrayVec, - debug_warn, implement, + debug_warn, matrix::event::{Event, Matches}, utils::{ ArrayVecExt, IterStream, ReadyExt, set, @@ -70,158 +70,161 @@ impl crate::Service for Service { fn name(&self) -> &str { crate::service::make_name(std::module_path!()) } } -#[implement(Service)] -pub fn index_pdu(&self, shortroomid: ShortRoomId, pdu_id: &RawPduId, message_body: &str) { - let batch = tokenize(message_body) - .map(|word| { +impl Service { + /// Adds a single PDU to the search index. + pub fn index_pdu(&self, shortroomid: ShortRoomId, pdu_id: &RawPduId, message_body: &str) { + let batch = tokenize(message_body) + .map(|word| { + let mut key = shortroomid.to_be_bytes().to_vec(); + key.extend_from_slice(word.as_bytes()); + key.push(0xFF); + key.extend_from_slice(pdu_id.as_ref()); // TODO: currently we save the room id a second time here + key + }) + .collect::>(); + + self.db + .tokenids + .insert_batch(batch.iter().map(|k| (k.as_slice(), &[]))); + } + + /// Removes a single PDU from the search index. + pub fn deindex_pdu(&self, shortroomid: ShortRoomId, pdu_id: &RawPduId, message_body: &str) { + let batch = tokenize(message_body).map(|word| { let mut key = shortroomid.to_be_bytes().to_vec(); key.extend_from_slice(word.as_bytes()); key.push(0xFF); key.extend_from_slice(pdu_id.as_ref()); // TODO: currently we save the room id a second time here key - }) - .collect::>(); - - self.db - .tokenids - .insert_batch(batch.iter().map(|k| (k.as_slice(), &[]))); -} - -#[implement(Service)] -pub fn deindex_pdu(&self, shortroomid: ShortRoomId, pdu_id: &RawPduId, message_body: &str) { - let batch = tokenize(message_body).map(|word| { - let mut key = shortroomid.to_be_bytes().to_vec(); - key.extend_from_slice(word.as_bytes()); - key.push(0xFF); - key.extend_from_slice(pdu_id.as_ref()); // TODO: currently we save the room id a second time here - key - }); - - for token in batch { - self.db.tokenids.remove(&token); - } -} - -#[implement(Service)] -pub async fn search_pdus<'a>( - &'a self, - query: &'a RoomQuery<'a>, - sender_user: &'a UserId, -) -> Result<(usize, impl Stream + Send + 'a)> { - let pdu_ids: Vec<_> = self.search_pdu_ids(query).await?.collect().await; - - let filter = &query.criteria.filter; - let count = pdu_ids.len(); - let pdus = pdu_ids - .into_iter() - .stream() - .wide_filter_map(move |result_pdu_id: RawPduId| async move { - self.services - .timeline - .get_pdu_from_id(&result_pdu_id) - .await - .ok() - }) - .ready_filter(|pdu| !pdu.is_redacted()) - .ready_filter(move |pdu| filter.matches(pdu)) - .wide_filter_map(move |pdu| async move { - self.services - .state_accessor - .user_can_see_event(query.user_id?, pdu.room_id().unwrap(), pdu.event_id()) - .await - .then_some(pdu) - }) - .skip(query.skip) - .take(query.limit) - .map(move |mut pdu| { - pdu.set_unsigned(query.user_id); - - pdu - }) - .then(async move |mut pdu| { - if let Err(e) = self - .services - .pdu_metadata - .add_bundled_aggregations_to_pdu(sender_user, &mut pdu) - .await - { - debug_warn!("Failed to add bundled aggregations: {e}"); - } - pdu }); - Ok((count, pdus)) -} - -// result is modeled as a stream such that callers don't have to be refactored -// though an additional async/wrap still exists for now -#[implement(Service)] -pub async fn search_pdu_ids<'a>( - &'a self, - query: &'a RoomQuery<'_>, -) -> Result + Send + 'a + use<'a>> { - let shortroomid = self.services.short.get_shortroomid(query.room_id).await?; - - let pdu_ids = self.search_pdu_ids_query_room(query, shortroomid).await; - - let iters = pdu_ids.into_iter().map(IntoIterator::into_iter); - - Ok(set::intersection(iters).stream()) -} - -#[implement(Service)] -async fn search_pdu_ids_query_room( - &self, - query: &RoomQuery<'_>, - shortroomid: ShortRoomId, -) -> Vec> { - tokenize(&query.criteria.search_term) - .stream() - .wide_then(|word| async move { - self.search_pdu_ids_query_words(shortroomid, &word) - .collect::>() - .await - }) - .collect::>() - .await -} - -/// Iterate over PduId's containing a word -#[implement(Service)] -fn search_pdu_ids_query_words<'a>( - &'a self, - shortroomid: ShortRoomId, - word: &'a str, -) -> impl Stream + Send + 'a { - self.search_pdu_ids_query_word(shortroomid, word) - .map(move |key| -> RawPduId { - let key = &key[prefix_len(word)..]; - key.into() - }) -} - -/// Iterate over raw database results for a word -#[implement(Service)] -fn search_pdu_ids_query_word<'a>( - &'a self, - shortroomid: ShortRoomId, - word: &'a str, -) -> impl Stream> + Send + 'a + use<'a> { - // rustc says const'ing this not yet stable - let end_id: RawPduId = PduId { - shortroomid, - shorteventid: PduCount::max(), + for token in batch { + self.db.tokenids.remove(&token); + } } - .into(); - // Newest pdus first - let end = make_tokenid(shortroomid, word, &end_id); - let prefix = make_prefix(shortroomid, word); - self.db - .tokenids - .rev_raw_keys_from(&end) - .ignore_err() - .ready_take_while(move |key| key.starts_with(&prefix)) + /// Searches through the search index to find PDUs matching the query. + /// Filters history visibility based on what the sender can see, and bundles + /// aggregations before returning. + pub async fn search_pdus<'a>( + &'a self, + query: &'a RoomQuery<'a>, + sender_user: &'a UserId, + ) -> Result<(usize, impl Stream + Send + 'a)> { + let pdu_ids: Vec<_> = self.search_pdu_ids(query).await?.collect().await; + + let filter = &query.criteria.filter; + let count = pdu_ids.len(); + let pdus = pdu_ids + .into_iter() + .stream() + .wide_filter_map(move |result_pdu_id: RawPduId| async move { + self.services + .timeline + .get_pdu_from_id(&result_pdu_id) + .await + .ok() + }) + .ready_filter(|pdu| !pdu.is_redacted()) + .ready_filter(move |pdu| filter.matches(pdu)) + .wide_filter_map(move |pdu| async move { + self.services + .state_accessor + .user_can_see_event(query.user_id?, pdu.room_id().unwrap(), pdu.event_id()) + .await + .then_some(pdu) + }) + .skip(query.skip) + .take(query.limit) + .map(move |mut pdu| { + pdu.set_unsigned(query.user_id); + + pdu + }) + .then(async move |mut pdu| { + if let Err(e) = self + .services + .pdu_metadata + .add_bundled_aggregations_to_pdu(sender_user, &mut pdu) + .await + { + debug_warn!("Failed to add bundled aggregations: {e}"); + } + pdu + }); + + Ok((count, pdus)) + } + + // TODO: The next 4 functions can be, theoretically, squashed into one. + /// Same as search_pdus (searches the search index for matching PDUs), but + /// only returns their short IDs with no filtering or bundled aggregations. + pub async fn search_pdu_ids<'a>( + &'a self, + query: &'a RoomQuery<'_>, + ) -> Result + Send + 'a + use<'a>> { + // result is modeled as a stream such that callers don't have to be refactored + // though an additional async/wrap still exists for now + let shortroomid = self.services.short.get_shortroomid(query.room_id).await?; + + let pdu_ids = self.search_pdu_ids_query_room(query, shortroomid).await; + + let iters = pdu_ids.into_iter().map(IntoIterator::into_iter); + + Ok(set::intersection(iters).stream()) + } + + async fn search_pdu_ids_query_room( + &self, + query: &RoomQuery<'_>, + shortroomid: ShortRoomId, + ) -> Vec> { + tokenize(&query.criteria.search_term) + .stream() + .wide_then(|word| async move { + self.search_pdu_ids_query_words(shortroomid, &word) + .collect::>() + .await + }) + .collect::>() + .await + } + + /// Iterate over PduId's containing a word + fn search_pdu_ids_query_words<'a>( + &'a self, + shortroomid: ShortRoomId, + word: &'a str, + ) -> impl Stream + Send + 'a { + self.search_pdu_ids_query_word(shortroomid, word) + .map(move |key| -> RawPduId { + let key = &key[prefix_len(word)..]; + key.into() + }) + } + + /// Iterate over raw database results for a word + fn search_pdu_ids_query_word<'a>( + &'a self, + shortroomid: ShortRoomId, + word: &'a str, + ) -> impl Stream> + Send + 'a + use<'a> { + // rustc says const'ing this not yet stable + let end_id: RawPduId = PduId { + shortroomid, + shorteventid: PduCount::max(), + } + .into(); + + // Newest pdus first + let end = make_tokenid(shortroomid, word, &end_id); + let prefix = make_prefix(shortroomid, word); + self.db + .tokenids + .rev_raw_keys_from(&end) + .ignore_err() + .ready_take_while(move |key| key.starts_with(&prefix)) + } } /// Splits a string into tokens used as keys in the search inverted index From 5e0e82e7e82e913f5e1c0dfa3e5811781db69e75 Mon Sep 17 00:00:00 2001 From: timedout Date: Fri, 26 Jun 2026 04:33:34 +0100 Subject: [PATCH 39/70] style: Document & refactor rooms/outlier service --- src/service/rooms/outlier/mod.rs | 44 +++++++++++++++----------------- 1 file changed, 21 insertions(+), 23 deletions(-) diff --git a/src/service/rooms/outlier/mod.rs b/src/service/rooms/outlier/mod.rs index 6ab2c026a..55a3ce7b5 100644 --- a/src/service/rooms/outlier/mod.rs +++ b/src/service/rooms/outlier/mod.rs @@ -24,29 +24,27 @@ impl crate::Service for Service { fn name(&self) -> &str { crate::service::make_name(std::module_path!()) } } -/// Returns the pdu from the outlier tree. -#[implement(Service)] -pub async fn get_outlier_pdu_json(&self, event_id: &EventId) -> Result { - self.db - .eventid_outlierpdu - .get(event_id) - .await - .deserialized() -} +impl Service { + /// Returns the pdu from the outlier tree. + pub async fn get_outlier_pdu_json(&self, event_id: &EventId) -> Result { + self.db + .eventid_outlierpdu + .get(event_id) + .await + .deserialized() + } -/// Returns the pdu from the outlier tree. -#[implement(Service)] -pub async fn get_pdu_outlier(&self, event_id: &EventId) -> Result { - self.db - .eventid_outlierpdu - .get(event_id) - .await - .deserialized() -} + /// Returns the pdu from the outlier tree. + pub async fn get_pdu_outlier(&self, event_id: &EventId) -> Result { + self.db + .eventid_outlierpdu + .get(event_id) + .await + .deserialized() + } -/// Append the PDU as an outlier. -#[implement(Service)] -#[tracing::instrument(skip(self, pdu), level = "debug")] -pub fn add_pdu_outlier(&self, event_id: &EventId, pdu: &CanonicalJsonObject) { - self.db.eventid_outlierpdu.raw_put(event_id, Json(pdu)); + /// Append the PDU as an outlier. + pub fn add_pdu_outlier(&self, event_id: &EventId, pdu: &CanonicalJsonObject) { + self.db.eventid_outlierpdu.raw_put(event_id, Json(pdu)); + } } From 0d1b130127e1db97911a8bb56a10b1636e7b7bd7 Mon Sep 17 00:00:00 2001 From: timedout Date: Fri, 26 Jun 2026 04:36:01 +0100 Subject: [PATCH 40/70] style: Document & refactor rooms/metadata service --- src/service/rooms/metadata/mod.rs | 105 +++++++++++++++--------------- 1 file changed, 53 insertions(+), 52 deletions(-) diff --git a/src/service/rooms/metadata/mod.rs b/src/service/rooms/metadata/mod.rs index 310ab2c76..f0de153f7 100644 --- a/src/service/rooms/metadata/mod.rs +++ b/src/service/rooms/metadata/mod.rs @@ -41,60 +41,61 @@ impl crate::Service for Service { fn name(&self) -> &str { crate::service::make_name(std::module_path!()) } } -#[implement(Service)] -pub async fn exists(&self, room_id: &RoomId) -> bool { - let Ok(prefix) = self.services.short.get_shortroomid(room_id).await else { - return false; - }; +impl Service { + /// Checks if a room exists by checking if there are any PDUs in that room. + /// If the short room ID is not found or there are no PDUs in the room, it + /// does not exist. + pub async fn exists(&self, room_id: &RoomId) -> bool { + let Ok(prefix) = self.services.short.get_shortroomid(room_id).await else { + return false; + }; - // Look for PDUs in that room. - self.db - .pduid_pdu - .keys_prefix_raw(&prefix) - .ignore_err() - .next() - .await - .is_some() -} + // Look for PDUs in that room. + self.db + .pduid_pdu + .keys_prefix_raw(&prefix) + .ignore_err() + .next() + .await + .is_some() + } -#[implement(Service)] -pub fn iter_ids(&self) -> impl Stream + Send + '_ { - self.db.roomid_shortroomid.keys().ignore_err() -} + /// Returns a stream of every room ID in the database. + pub fn iter_ids(&self) -> impl Stream + Send + '_ { + self.db.roomid_shortroomid.keys().ignore_err() + } -#[implement(Service)] -#[inline] -pub fn disable_room(&self, room_id: &RoomId, disabled: bool) { - if disabled { - self.db.disabledroomids.insert(room_id, []); - } else { - self.db.disabledroomids.remove(room_id); + /// Disables a room, preventing it from engaging in federation, but still + /// allowing it to be used locally. + pub fn disable_room(&self, room_id: &RoomId, disabled: bool) { + if disabled { + self.db.disabledroomids.insert(room_id, []); + } else { + self.db.disabledroomids.remove(room_id); + } + } + + pub async fn is_disabled(&self, room_id: &RoomId) -> bool { + self.db.disabledroomids.get(room_id).await.is_ok() + } + + /// Bans a room, ensuring it cannot be used neither locally nor over + /// federation. + pub fn ban_room(&self, room_id: &RoomId, banned: bool) { + if banned { + self.db.bannedroomids.insert(room_id, []); + } else { + self.db.bannedroomids.remove(room_id); + } + } + + /// Checks if a room is currently banned. + pub async fn is_banned(&self, room_id: &RoomId) -> bool { + self.db.bannedroomids.get(room_id).await.is_ok() + } + + /// Lists all rooms that are currently banned. + pub fn list_banned_rooms(&self) -> impl Stream + Send + '_ { + self.db.bannedroomids.keys().ignore_err() } } - -#[implement(Service)] -#[inline] -pub fn ban_room(&self, room_id: &RoomId, banned: bool) { - if banned { - self.db.bannedroomids.insert(room_id, []); - } else { - self.db.bannedroomids.remove(room_id); - } -} - -#[implement(Service)] -pub fn list_banned_rooms(&self) -> impl Stream + Send + '_ { - self.db.bannedroomids.keys().ignore_err() -} - -#[implement(Service)] -#[inline] -pub async fn is_disabled(&self, room_id: &RoomId) -> bool { - self.db.disabledroomids.get(room_id).await.is_ok() -} - -#[implement(Service)] -#[inline] -pub async fn is_banned(&self, room_id: &RoomId) -> bool { - self.db.bannedroomids.get(room_id).await.is_ok() -} From 0a162b7bc46bbe509782e1349afd6798b0b8a1bb Mon Sep 17 00:00:00 2001 From: timedout Date: Fri, 26 Jun 2026 04:43:40 +0100 Subject: [PATCH 41/70] style: Document & refactor rooms/lazy_loading service --- src/service/rooms/lazy_loading/mod.rs | 144 +++++++++++++------------- src/service/rooms/metadata/mod.rs | 2 +- src/service/rooms/outlier/mod.rs | 2 +- 3 files changed, 74 insertions(+), 74 deletions(-) diff --git a/src/service/rooms/lazy_loading/mod.rs b/src/service/rooms/lazy_loading/mod.rs index 71fe82960..60abdbdfe 100644 --- a/src/service/rooms/lazy_loading/mod.rs +++ b/src/service/rooms/lazy_loading/mod.rs @@ -3,7 +3,7 @@ use std::{collections::HashSet, sync::Arc}; use conduwuit::{ - Result, implement, + Result, utils::{IterStream, ReadyExt, stream::TryIgnore}, }; use database::{Database, Deserialized, Handle, Interfix, Map, Qry}; @@ -55,87 +55,87 @@ impl crate::Service for Service { fn name(&self) -> &str { crate::service::make_name(std::module_path!()) } } -#[implement(Service)] -#[tracing::instrument(skip(self), level = "debug")] -pub async fn reset(&self, ctx: &Context<'_>) { - let prefix = (ctx.user_id, ctx.device_id, ctx.room_id, Interfix); - self.db - .lazyloadedids - .keys_prefix_raw(&prefix) - .ignore_err() - .ready_for_each(|key| self.db.lazyloadedids.remove(key)) - .await; -} +impl Service { + /// Resets the lazy loading context. + pub async fn reset(&self, ctx: &Context<'_>) { + let prefix = (ctx.user_id, ctx.device_id, ctx.room_id, Interfix); + self.db + .lazyloadedids + .keys_prefix_raw(&prefix) + .ignore_err() + .ready_for_each(|key| self.db.lazyloadedids.remove(key)) + .await; + } -/// Returns only the subset of `senders` which should be sent to the client -/// according to the provided lazy loading context. -#[implement(Service)] -#[tracing::instrument(name = "retain", level = "debug", skip_all)] -pub async fn retain_lazy_members(&self, senders: MemberSet, ctx: &Context<'_>) -> MemberSet { - debug_assert!( - ctx.options.is_none_or(Options::is_enabled), - "lazy loading should be enabled by your options" - ); + /// Returns only the subset of `senders` which should be sent to the client + /// according to the provided lazy loading context. + pub async fn retain_lazy_members(&self, senders: MemberSet, ctx: &Context<'_>) -> MemberSet { + debug_assert!( + ctx.options.is_none_or(Options::is_enabled), + "lazy loading should be enabled by your options" + ); - let include_redundant = cfg!(feature = "element_hacks") - || ctx.options.is_some_and(Options::include_redundant_members); + let include_redundant = cfg!(feature = "element_hacks") + || ctx.options.is_some_and(Options::include_redundant_members); - let witness = self - .witness(ctx, senders.iter().map(AsRef::as_ref)) - .zip(senders.iter().stream()); + let witness = self + .witness(ctx, senders.iter().map(AsRef::as_ref)) + .zip(senders.iter().stream()); - pin_mut!(witness); - let _cork = self.db.db.cork(); - let mut senders = MemberSet::with_capacity(senders.len()); - while let Some((status, sender)) = witness.next().await { - if include_redundant || status == Status::Unseen { - senders.insert(sender.clone()); - continue; - } - - if let Status::Seen(seen) = status { - if seen == 0 || ctx.token == Some(seen) { + pin_mut!(witness); + let _cork = self.db.db.cork(); + let mut senders = MemberSet::with_capacity(senders.len()); + while let Some((status, sender)) = witness.next().await { + if include_redundant || status == Status::Unseen { senders.insert(sender.clone()); continue; } + + if let Status::Seen(seen) = status { + if seen == 0 || ctx.token == Some(seen) { + senders.insert(sender.clone()); + continue; + } + } } + + senders } - senders -} + /// Witnesses a set of users via the lazy loading context, returning a + /// stream of their statuses. + fn witness<'a, I>( + &'a self, + ctx: &'a Context<'a>, + senders: I, + ) -> impl Stream + Send + 'a + where + I: Iterator + Send + Clone + 'a, + { + let make_key = + |sender: &'a UserId| -> Key<'a> { (ctx.user_id, ctx.device_id, ctx.room_id, sender) }; -#[implement(Service)] -fn witness<'a, I>( - &'a self, - ctx: &'a Context<'a>, - senders: I, -) -> impl Stream + Send + 'a -where - I: Iterator + Send + Clone + 'a, -{ - let make_key = - |sender: &'a UserId| -> Key<'a> { (ctx.user_id, ctx.device_id, ctx.room_id, sender) }; + senders + .clone() + .stream() + .map(make_key) + .qry(&self.db.lazyloadedids) + .map(into_status) + .zip(senders.stream()) + .map(move |(status, sender)| { + if matches!(status, Status::Unseen) { + self.db + .lazyloadedids + .put_aput::<8, _, _>(make_key(sender), 0_u64); + } else if matches!(status, Status::Seen(0)) { + self.db + .lazyloadedids + .put_aput::<8, _, _>(make_key(sender), ctx.token.unwrap_or(0_u64)); + } - senders - .clone() - .stream() - .map(make_key) - .qry(&self.db.lazyloadedids) - .map(into_status) - .zip(senders.stream()) - .map(move |(status, sender)| { - if matches!(status, Status::Unseen) { - self.db - .lazyloadedids - .put_aput::<8, _, _>(make_key(sender), 0_u64); - } else if matches!(status, Status::Seen(0)) { - self.db - .lazyloadedids - .put_aput::<8, _, _>(make_key(sender), ctx.token.unwrap_or(0_u64)); - } - - status - }) + status + }) + } } fn into_status(result: Result>) -> Status { @@ -146,6 +146,8 @@ fn into_status(result: Result>) -> Status { } impl Options for LazyLoadOptions { + fn is_enabled(&self) -> bool { !self.is_disabled() } + fn include_redundant_members(&self) -> bool { if let Self::Enabled { include_redundant_members } = self { *include_redundant_members @@ -153,6 +155,4 @@ impl Options for LazyLoadOptions { false } } - - fn is_enabled(&self) -> bool { !self.is_disabled() } } diff --git a/src/service/rooms/metadata/mod.rs b/src/service/rooms/metadata/mod.rs index f0de153f7..b224ac232 100644 --- a/src/service/rooms/metadata/mod.rs +++ b/src/service/rooms/metadata/mod.rs @@ -1,6 +1,6 @@ use std::sync::Arc; -use conduwuit::{Result, implement, utils::stream::TryIgnore}; +use conduwuit::{Result, utils::stream::TryIgnore}; use database::Map; use futures::{Stream, StreamExt}; use ruma::{OwnedRoomId, RoomId}; diff --git a/src/service/rooms/outlier/mod.rs b/src/service/rooms/outlier/mod.rs index 55a3ce7b5..afd671999 100644 --- a/src/service/rooms/outlier/mod.rs +++ b/src/service/rooms/outlier/mod.rs @@ -1,6 +1,6 @@ use std::sync::Arc; -use conduwuit::{Result, implement, matrix::PduEvent}; +use conduwuit::{Result, matrix::PduEvent}; use database::{Deserialized, Json, Map}; use ruma::{CanonicalJsonObject, EventId}; From 19f4c24c005a31f51268a562eea504d462cdaa2e Mon Sep 17 00:00:00 2001 From: timedout Date: Fri, 26 Jun 2026 04:55:57 +0100 Subject: [PATCH 42/70] style: Document & refactor rooms/event_handler service --- src/service/rooms/event_handler/acl_check.rs | 62 +- .../fetch_and_handle_outliers.rs | 402 +++++----- src/service/rooms/event_handler/fetch_prev.rs | 198 +++-- .../rooms/event_handler/fetch_state.rs | 143 ++-- .../event_handler/handle_incoming_pdu.rs | 401 +++++----- .../rooms/event_handler/handle_outlier_pdu.rs | 419 +++++------ .../rooms/event_handler/handle_prev_pdu.rs | 140 ++-- src/service/rooms/event_handler/mod.rs | 3 + .../rooms/event_handler/parse_incoming_pdu.rs | 100 +-- .../rooms/event_handler/policy_server.rs | 645 ++++++++-------- .../rooms/event_handler/resolve_state.rs | 194 +++-- .../rooms/event_handler/state_at_incoming.rs | 323 ++++---- .../event_handler/upgrade_outlier_pdu.rs | 700 +++++++++--------- 13 files changed, 1884 insertions(+), 1846 deletions(-) diff --git a/src/service/rooms/event_handler/acl_check.rs b/src/service/rooms/event_handler/acl_check.rs index f847015bb..c3fcfea1c 100644 --- a/src/service/rooms/event_handler/acl_check.rs +++ b/src/service/rooms/event_handler/acl_check.rs @@ -1,42 +1,40 @@ -use conduwuit::{Err, Result, debug, implement, trace, warn}; +use conduwuit::{Err, Result, debug, debug_warn, trace}; use ruma::{ RoomId, ServerName, events::{StateEventType, room::server_acl::RoomServerAclEventContent}, }; -/// Returns Ok if the acl allows the server -#[implement(super::Service)] -#[tracing::instrument(skip_all, level = "debug")] -pub async fn acl_check(&self, server_name: &ServerName, room_id: &RoomId) -> Result { - let Ok(acl_event_content) = self - .services - .state_accessor - .room_state_get_content(room_id, &StateEventType::RoomServerAcl, "") - .await - .map(|c: RoomServerAclEventContent| c) - .inspect(|acl| trace!(%room_id, "ACL content found: {acl:?}")) - .inspect_err(|e| trace!(%room_id, "No ACL content found: {e:?}")) - else { - return Ok(()); - }; +impl super::Service { + /// Returns Ok if the acl allows the server, otherwise Forbidden. + pub async fn acl_check(&self, server_name: &ServerName, room_id: &RoomId) -> Result { + let Ok(acl_event_content) = self + .services + .state_accessor + .room_state_get_content(room_id, &StateEventType::RoomServerAcl, "") + .await + .map(|c: RoomServerAclEventContent| c) + else { + return Ok(()); + }; - if acl_event_content.allow.is_empty() { - warn!(%room_id, "Ignoring broken ACL event (allow key is empty)"); - return Ok(()); - } + if acl_event_content.allow.is_empty() { + debug_warn!(%room_id, "Ignoring broken ACL event (allow key is empty)"); + return Ok(()); + } - if acl_event_content.deny.contains(&String::from("*")) - && acl_event_content.allow.contains(&String::from("*")) - { - warn!(%room_id, "Ignoring broken ACL event (allow key and deny key both contain wildcard \"*\""); - return Ok(()); - } + if acl_event_content.deny.contains(&String::from("*")) + && acl_event_content.allow.contains(&String::from("*")) + { + debug_warn!(%room_id, "Ignoring broken ACL event (allow key and deny key both contain wildcard \"*\""); + return Ok(()); + } - if acl_event_content.is_allowed(server_name) { - trace!("server {server_name} is allowed by ACL"); - Ok(()) - } else { - debug!("Server {server_name} was denied by room ACL in {room_id}"); - Err!(Request(Forbidden("Server was denied by room ACL"))) + if acl_event_content.is_allowed(server_name) { + trace!("server {server_name} is allowed by ACL"); + Ok(()) + } else { + debug!("Server {server_name} was denied by room ACL in {room_id}"); + Err!(Request(Forbidden("Server was denied by room ACL"))) + } } } diff --git a/src/service/rooms/event_handler/fetch_and_handle_outliers.rs b/src/service/rooms/event_handler/fetch_and_handle_outliers.rs index e465131fb..09d819b77 100644 --- a/src/service/rooms/event_handler/fetch_and_handle_outliers.rs +++ b/src/service/rooms/event_handler/fetch_and_handle_outliers.rs @@ -4,8 +4,8 @@ use std::{ }; use conduwuit::{ - Event, PduEvent, debug, debug_warn, implement, matrix::event::gen_event_id_canonical_json, - trace, utils::continue_exponential_backoff_secs, warn, + Event, PduEvent, debug, debug_warn, matrix::event::gen_event_id_canonical_json, trace, + utils::continue_exponential_backoff_secs, warn, }; use ruma::{ CanonicalJsonValue, EventId, OwnedEventId, RoomId, ServerName, @@ -14,220 +14,222 @@ use ruma::{ use super::get_room_version_rules; -/// Find the event and auth it. Once the event is validated (steps 1 - 8) -/// it is appended to the outliers Tree. -/// -/// Returns pdu and if we fetched it over federation the raw json. -/// -/// a. Look in the main timeline (pduid_pdu tree) -/// b. Look at outlier pdu tree -/// c. Ask origin server over federation -/// d. TODO: Ask other servers over federation? -#[implement(super::Service)] -pub(super) async fn fetch_and_handle_outliers<'a, Pdu, Events>( - &self, - origin: &'a ServerName, - events: Events, - create_event: &'a Pdu, - room_id: &'a RoomId, -) -> Vec<(PduEvent, Option>)> -where - Pdu: Event + Send + Sync, - Events: Iterator + Clone + Send, -{ - let back_off = |id| match self - .services - .globals - .bad_event_ratelimiter - .write() - .entry(id) +impl super::Service { + /// Find the event and auth it. Once the event is validated (steps 1 - 8) + /// it is appended to the outliers Tree. + /// + /// Returns pdu and if we fetched it over federation the raw json. + /// + /// a. Look in the main timeline (pduid_pdu tree) + /// b. Look at outlier pdu tree + /// c. Ask origin server over federation + /// d. TODO: Ask other servers over federation? + pub(super) async fn fetch_and_handle_outliers<'a, Pdu, Events>( + &self, + origin: &'a ServerName, + events: Events, + create_event: &'a Pdu, + room_id: &'a RoomId, + ) -> Vec<(PduEvent, Option>)> + where + Pdu: Event + Send + Sync, + Events: Iterator + Clone + Send, { - | hash_map::Entry::Vacant(e) => { - e.insert((Instant::now(), 1)); - }, - | hash_map::Entry::Occupied(mut e) => { - *e.get_mut() = (Instant::now(), e.get().1.saturating_add(1)); - }, - }; + let back_off = |id| match self + .services + .globals + .bad_event_ratelimiter + .write() + .entry(id) + { + | hash_map::Entry::Vacant(e) => { + e.insert((Instant::now(), 1)); + }, + | hash_map::Entry::Occupied(mut e) => { + *e.get_mut() = (Instant::now(), e.get().1.saturating_add(1)); + }, + }; - let mut events_with_auth_events = Vec::with_capacity(events.clone().count()); - trace!("Fetching {} outlier pdus", events.clone().count()); + let mut events_with_auth_events = Vec::with_capacity(events.clone().count()); + trace!("Fetching {} outlier pdus", events.clone().count()); - for id in events { - // a. Look in the main timeline (pduid_pdu tree) - // b. Look at outlier pdu tree - // (get_pdu_json checks both) - if let Ok(local_pdu) = self.services.timeline.get_pdu(id).await { - trace!("Found {id} in main timeline or outlier tree"); - events_with_auth_events.push((id.to_owned(), Some(local_pdu), vec![])); - continue; - } - - // c. Ask origin server over federation - // We also handle its auth chain here so we don't get a stack overflow in - // handle_outlier_pdu. - let mut todo_auth_events: VecDeque<_> = [id.to_owned()].into(); - let mut events_in_reverse_order = Vec::with_capacity(todo_auth_events.len()); - - let mut events_all = HashSet::with_capacity(todo_auth_events.len()); - while let Some(next_id) = todo_auth_events.pop_front() { - if let Some((time, tries)) = self - .services - .globals - .bad_event_ratelimiter - .read() - .get(&*next_id) - { - // Exponential backoff - const MIN_DURATION: u64 = 60 * 2; - const MAX_DURATION: u64 = 60 * 60 * 8; - if continue_exponential_backoff_secs( - MIN_DURATION, - MAX_DURATION, - time.elapsed(), - *tries, - ) { - debug_warn!( - tried = ?*tries, - elapsed = ?time.elapsed(), - "Backing off from {next_id}", - ); - continue; - } - } - - if events_all.contains(&next_id) { + for id in events { + // a. Look in the main timeline (pduid_pdu tree) + // b. Look at outlier pdu tree + // (get_pdu_json checks both) + if let Ok(local_pdu) = self.services.timeline.get_pdu(id).await { + trace!("Found {id} in main timeline or outlier tree"); + events_with_auth_events.push((id.to_owned(), Some(local_pdu), vec![])); continue; } - if self.services.timeline.pdu_exists(&next_id).await { - trace!("Found {next_id} in db"); - continue; - } + // c. Ask origin server over federation + // We also handle its auth chain here so we don't get a stack overflow in + // handle_outlier_pdu. + let mut todo_auth_events: VecDeque<_> = [id.to_owned()].into(); + let mut events_in_reverse_order = Vec::with_capacity(todo_auth_events.len()); - debug!("Fetching {next_id} over federation from {origin}."); - match self - .services - .sending - .send_federation_request( - origin, - get_event::v1::Request::new((*next_id).to_owned()), - ) - .await - { - | Ok(res) => { - debug!("Got {next_id} over federation from {origin}"); - let Ok(room_version_rules) = get_room_version_rules(create_event) else { - back_off((*next_id).to_owned()); - continue; - }; - - let Ok((calculated_event_id, value)) = - gen_event_id_canonical_json(&res.pdu, &room_version_rules) - else { - back_off((*next_id).to_owned()); - continue; - }; - - if calculated_event_id != *next_id { - warn!( - "Server didn't return event id we requested: requested: {next_id}, \ - we got {calculated_event_id}. Event: {:?}", - &res.pdu + let mut events_all = HashSet::with_capacity(todo_auth_events.len()); + while let Some(next_id) = todo_auth_events.pop_front() { + if let Some((time, tries)) = self + .services + .globals + .bad_event_ratelimiter + .read() + .get(&*next_id) + { + // Exponential backoff + const MIN_DURATION: u64 = 60 * 2; + const MAX_DURATION: u64 = 60 * 60 * 8; + if continue_exponential_backoff_secs( + MIN_DURATION, + MAX_DURATION, + time.elapsed(), + *tries, + ) { + debug_warn!( + tried = ?*tries, + elapsed = ?time.elapsed(), + "Backing off from {next_id}", ); + continue; } + } - if let Some(auth_events) = value - .get("auth_events") - .and_then(CanonicalJsonValue::as_array) - { - for auth_event in auth_events { - match serde_json::from_value::( - auth_event.clone().into(), - ) { - | Ok(auth_event) => { - trace!( - "Found auth event id {auth_event} for event {next_id}" - ); - todo_auth_events.push_back(auth_event); - }, - | _ => { - warn!("Auth event id is not valid"); - }, - } - } - } else { - warn!("Auth event list invalid"); - } - - events_in_reverse_order.push((next_id.clone(), value)); - events_all.insert(next_id); - }, - | Err(e) => { - warn!("Failed to fetch auth event {next_id} from {origin}: {e}"); - back_off((*next_id).to_owned()); - }, - } - } - - events_with_auth_events.push((id.to_owned(), None, events_in_reverse_order)); - } - - let mut pdus = Vec::with_capacity(events_with_auth_events.len()); - for (id, local_pdu, events_in_reverse_order) in events_with_auth_events { - // a. Look in the main timeline (pduid_pdu tree) - // b. Look at outlier pdu tree - // (get_pdu_json checks both) - if let Some(local_pdu) = local_pdu { - trace!("Found {id} in main timeline or outlier tree"); - pdus.push((local_pdu.clone(), None)); - } - - for (next_id, value) in events_in_reverse_order.into_iter().rev() { - if let Some((time, tries)) = self - .services - .globals - .bad_event_ratelimiter - .read() - .get(&*next_id) - { - // Exponential backoff - const MIN_DURATION: u64 = 5 * 60; - const MAX_DURATION: u64 = 60 * 60 * 24; - if continue_exponential_backoff_secs( - MIN_DURATION, - MAX_DURATION, - time.elapsed(), - *tries, - ) { - debug!("Backing off from {next_id}"); + if events_all.contains(&next_id) { continue; } + + if self.services.timeline.pdu_exists(&next_id).await { + trace!("Found {next_id} in db"); + continue; + } + + debug!("Fetching {next_id} over federation from {origin}."); + match self + .services + .sending + .send_federation_request( + origin, + get_event::v1::Request::new((*next_id).to_owned()), + ) + .await + { + | Ok(res) => { + debug!("Got {next_id} over federation from {origin}"); + let Ok(room_version_rules) = get_room_version_rules(create_event) else { + back_off((*next_id).to_owned()); + continue; + }; + + let Ok((calculated_event_id, value)) = + gen_event_id_canonical_json(&res.pdu, &room_version_rules) + else { + back_off((*next_id).to_owned()); + continue; + }; + + if calculated_event_id != *next_id { + warn!( + "Server didn't return event id we requested: requested: \ + {next_id}, we got {calculated_event_id}. Event: {:?}", + &res.pdu + ); + } + + if let Some(auth_events) = value + .get("auth_events") + .and_then(CanonicalJsonValue::as_array) + { + for auth_event in auth_events { + match serde_json::from_value::( + auth_event.clone().into(), + ) { + | Ok(auth_event) => { + trace!( + "Found auth event id {auth_event} for event \ + {next_id}" + ); + todo_auth_events.push_back(auth_event); + }, + | _ => { + warn!("Auth event id is not valid"); + }, + } + } + } else { + warn!("Auth event list invalid"); + } + + events_in_reverse_order.push((next_id.clone(), value)); + events_all.insert(next_id); + }, + | Err(e) => { + warn!("Failed to fetch auth event {next_id} from {origin}: {e}"); + back_off((*next_id).to_owned()); + }, + } } - trace!("Handling outlier {next_id}"); - match Box::pin(self.handle_outlier_pdu( - origin, - create_event, - &next_id, - room_id, - value.clone(), - true, - )) - .await - { - | Ok((pdu, json)) => - if next_id == *id { - trace!("Handled outlier {next_id} (original request)"); - pdus.push((pdu, Some(json))); + events_with_auth_events.push((id.to_owned(), None, events_in_reverse_order)); + } + + let mut pdus = Vec::with_capacity(events_with_auth_events.len()); + for (id, local_pdu, events_in_reverse_order) in events_with_auth_events { + // a. Look in the main timeline (pduid_pdu tree) + // b. Look at outlier pdu tree + // (get_pdu_json checks both) + if let Some(local_pdu) = local_pdu { + trace!("Found {id} in main timeline or outlier tree"); + pdus.push((local_pdu.clone(), None)); + } + + for (next_id, value) in events_in_reverse_order.into_iter().rev() { + if let Some((time, tries)) = self + .services + .globals + .bad_event_ratelimiter + .read() + .get(&*next_id) + { + // Exponential backoff + const MIN_DURATION: u64 = 5 * 60; + const MAX_DURATION: u64 = 60 * 60 * 24; + if continue_exponential_backoff_secs( + MIN_DURATION, + MAX_DURATION, + time.elapsed(), + *tries, + ) { + debug!("Backing off from {next_id}"); + continue; + } + } + + trace!("Handling outlier {next_id}"); + match Box::pin(self.handle_outlier_pdu( + origin, + create_event, + &next_id, + room_id, + value.clone(), + true, + )) + .await + { + | Ok((pdu, json)) => + if next_id == *id { + trace!("Handled outlier {next_id} (original request)"); + pdus.push((pdu, Some(json))); + }, + | Err(e) => { + warn!("Authentication of event {next_id} failed: {e:?}"); + back_off(next_id); }, - | Err(e) => { - warn!("Authentication of event {next_id} failed: {e:?}"); - back_off(next_id); - }, + } } } + trace!("Fetched and handled {} outlier pdus", pdus.len()); + pdus } - trace!("Fetched and handled {} outlier pdus", pdus.len()); - pdus } diff --git a/src/service/rooms/event_handler/fetch_prev.rs b/src/service/rooms/event_handler/fetch_prev.rs index efc7a434d..19c7a1089 100644 --- a/src/service/rooms/event_handler/fetch_prev.rs +++ b/src/service/rooms/event_handler/fetch_prev.rs @@ -4,7 +4,7 @@ use std::{ }; use conduwuit::{ - Event, PduEvent, Result, debug_warn, err, implement, + Event, PduEvent, Result, debug_warn, err, state_res::{self}, }; use futures::{FutureExt, future}; @@ -15,114 +15,110 @@ use ruma::{ use super::check_room_id; -#[implement(super::Service)] -#[tracing::instrument( - level = "debug", - skip_all, - fields(%origin), -)] -#[allow(clippy::type_complexity)] -pub(super) async fn fetch_prev<'a, Pdu, Events>( - &self, - origin: &ServerName, - create_event: &Pdu, - room_id: &RoomId, - first_ts_in_room: MilliSecondsSinceUnixEpoch, - initial_set: Events, -) -> Result<( - Vec, - HashMap)>, -)> -where - Pdu: Event + Send + Sync, - Events: Iterator + Clone + Send, -{ - let num_ids = initial_set.clone().count(); - let mut eventid_info = HashMap::new(); - let mut graph: HashMap = HashMap::with_capacity(num_ids); - let mut todo_outlier_stack: VecDeque = - initial_set.map(ToOwned::to_owned).collect(); +impl super::Service { + #[allow(clippy::type_complexity)] + pub(super) async fn fetch_prev<'a, Pdu, Events>( + &self, + origin: &ServerName, + create_event: &Pdu, + room_id: &RoomId, + first_ts_in_room: MilliSecondsSinceUnixEpoch, + initial_set: Events, + ) -> Result<( + Vec, + HashMap)>, + )> + where + Pdu: Event + Send + Sync, + Events: Iterator + Clone + Send, + { + let num_ids = initial_set.clone().count(); + let mut eventid_info = HashMap::new(); + let mut graph: HashMap = HashMap::with_capacity(num_ids); + let mut todo_outlier_stack: VecDeque = + initial_set.map(ToOwned::to_owned).collect(); - let mut amount = 0; + let mut amount = 0; - while let Some(prev_event_id) = todo_outlier_stack.pop_front() { - self.services.server.check_running()?; + while let Some(prev_event_id) = todo_outlier_stack.pop_front() { + self.services.server.check_running()?; - match self - .fetch_and_handle_outliers( - origin, - once(prev_event_id.as_ref()), - create_event, - room_id, - ) - .boxed() - .await - .pop() - { - | Some((pdu, mut json_opt)) => { - check_room_id(room_id, &pdu)?; + match self + .fetch_and_handle_outliers( + origin, + once(prev_event_id.as_ref()), + create_event, + room_id, + ) + .boxed() + .await + .pop() + { + | Some((pdu, mut json_opt)) => { + check_room_id(room_id, &pdu)?; - let limit = self.services.server.config.max_fetch_prev_events; - if amount > limit { - debug_warn!("Max prev event limit reached! Limit: {limit}"); - graph.insert(prev_event_id.clone(), HashSet::new()); - continue; - } - - if json_opt.is_none() { - json_opt = self - .services - .outlier - .get_outlier_pdu_json(&prev_event_id) - .await - .ok(); - } - - if let Some(json) = json_opt { - if pdu.origin_server_ts() > first_ts_in_room { - amount = amount.saturating_add(1); - for prev_prev in pdu.prev_events() { - if !graph.contains_key(prev_prev) { - todo_outlier_stack.push_back(prev_prev.to_owned()); - } - } - - graph.insert( - prev_event_id.clone(), - pdu.prev_events().map(ToOwned::to_owned).collect(), - ); - } else { - // Time based check failed + let limit = self.services.server.config.max_fetch_prev_events; + if amount > limit { + debug_warn!("Max prev event limit reached! Limit: {limit}"); graph.insert(prev_event_id.clone(), HashSet::new()); + continue; } - eventid_info.insert(prev_event_id.clone(), (pdu, json)); - } else { - // Get json failed, so this was not fetched over federation + if json_opt.is_none() { + json_opt = self + .services + .outlier + .get_outlier_pdu_json(&prev_event_id) + .await + .ok(); + } + + if let Some(json) = json_opt { + if pdu.origin_server_ts() > first_ts_in_room { + amount = amount.saturating_add(1); + for prev_prev in pdu.prev_events() { + if !graph.contains_key(prev_prev) { + todo_outlier_stack.push_back(prev_prev.to_owned()); + } + } + + graph.insert( + prev_event_id.clone(), + pdu.prev_events().map(ToOwned::to_owned).collect(), + ); + } else { + // Time based check failed + graph.insert(prev_event_id.clone(), HashSet::new()); + } + + eventid_info.insert(prev_event_id.clone(), (pdu, json)); + } else { + // Get json failed, so this was not fetched over federation + graph.insert(prev_event_id.clone(), HashSet::new()); + } + }, + | _ => { + // Fetch and handle failed graph.insert(prev_event_id.clone(), HashSet::new()); - } - }, - | _ => { - // Fetch and handle failed - graph.insert(prev_event_id.clone(), HashSet::new()); - }, + }, + } } + + let event_fetch = |event_id| { + let origin_server_ts = eventid_info + .get(&event_id) + .map_or_else(|| uint!(0), |info| info.0.origin_server_ts().get()); + + // This return value is the key used for sorting events, + // events are then sorted by power level, time, + // and lexically by event_id. + future::ok((int!(0), MilliSecondsSinceUnixEpoch(origin_server_ts))) + }; + + let sorted = state_res::lexicographical_topological_sort(&graph, &event_fetch) + .await + .map_err(|e| err!(Database(error!("Error sorting prev events: {e}"))))?; + + Ok((sorted, eventid_info)) } - - let event_fetch = |event_id| { - let origin_server_ts = eventid_info - .get(&event_id) - .map_or_else(|| uint!(0), |info| info.0.origin_server_ts().get()); - - // This return value is the key used for sorting events, - // events are then sorted by power level, time, - // and lexically by event_id. - future::ok((int!(0), MilliSecondsSinceUnixEpoch(origin_server_ts))) - }; - - let sorted = state_res::lexicographical_topological_sort(&graph, &event_fetch) - .await - .map_err(|e| err!(Database(error!("Error sorting prev events: {e}"))))?; - - Ok((sorted, eventid_info)) } diff --git a/src/service/rooms/event_handler/fetch_state.rs b/src/service/rooms/event_handler/fetch_state.rs index 790b4ec9e..d245b35e7 100644 --- a/src/service/rooms/event_handler/fetch_state.rs +++ b/src/service/rooms/event_handler/fetch_state.rs @@ -1,6 +1,6 @@ use std::collections::{HashMap, hash_map}; -use conduwuit::{Err, Event, Result, debug, debug_warn, err, implement}; +use conduwuit::{Err, Event, Result, debug, debug_warn, err}; use futures::FutureExt; use ruma::{ EventId, OwnedEventId, RoomId, ServerName, api::federation::event::get_room_state_ids, @@ -9,78 +9,81 @@ use ruma::{ use crate::rooms::short::ShortStateKey; -/// Call /state_ids to find out what the state at this pdu is. We trust the -/// server's response to some extend (sic), but we still do a lot of checks -/// on the events -#[implement(super::Service)] -#[tracing::instrument( - level = "debug", - skip_all, - fields(%origin), -)] -pub(super) async fn fetch_state( - &self, - origin: &ServerName, - create_event: &Pdu, - room_id: &RoomId, - event_id: &EventId, -) -> Result>> -where - Pdu: Event + Send + Sync, -{ - let res = self - .services - .sending - .send_federation_request( - origin, - get_room_state_ids::v1::Request::new(event_id.to_owned(), room_id.to_owned()), - ) - .await - .inspect_err(|e| debug_warn!("Fetching state for event failed: {e}"))?; - - debug!("Fetching state events"); - let state_ids = res.pdu_ids.iter().map(AsRef::as_ref); - let state_vec = self - .fetch_and_handle_outliers(origin, state_ids, create_event, room_id) - .boxed() - .await; - - let mut state: HashMap = HashMap::with_capacity(state_vec.len()); - for (pdu, _) in state_vec { - let state_key = pdu - .state_key() - .ok_or_else(|| err!(Database("Found non-state pdu in state events.")))?; - - let shortstatekey = self +impl super::Service { + /// Call /state_ids to find out what the state at this pdu is. We trust the + /// server's response to some extent (sic), but we still do a lot of checks + /// on the events + #[tracing::instrument( + level = "debug", + skip_all, + fields(%origin), + )] + pub(super) async fn fetch_state( + &self, + origin: &ServerName, + create_event: &Pdu, + room_id: &RoomId, + event_id: &EventId, + ) -> Result>> + where + Pdu: Event + Send + Sync, + { + let res = self .services - .short - .get_or_create_shortstatekey(&pdu.kind().to_string().into(), state_key) + .sending + .send_federation_request( + origin, + get_room_state_ids::v1::Request::new(event_id.to_owned(), room_id.to_owned()), + ) + .await + .inspect_err(|e| debug_warn!("Fetching state for event failed: {e}"))?; + + debug!("Fetching state events"); + let state_ids = res.pdu_ids.iter().map(AsRef::as_ref); + let state_vec = self + .fetch_and_handle_outliers(origin, state_ids, create_event, room_id) + .boxed() .await; - match state.entry(shortstatekey) { - | hash_map::Entry::Vacant(v) => { - v.insert(pdu.event_id().to_owned()); - }, - | hash_map::Entry::Occupied(_) => { - return Err!(Database( - "State event's type and state_key combination exists multiple times: {}, {}", - pdu.kind(), - state_key - )); - }, + let mut state: HashMap = + HashMap::with_capacity(state_vec.len()); + for (pdu, _) in state_vec { + let state_key = pdu + .state_key() + .ok_or_else(|| err!(Database("Found non-state pdu in state events.")))?; + + let shortstatekey = self + .services + .short + .get_or_create_shortstatekey(&pdu.kind().to_string().into(), state_key) + .await; + + match state.entry(shortstatekey) { + | hash_map::Entry::Vacant(v) => { + v.insert(pdu.event_id().to_owned()); + }, + | hash_map::Entry::Occupied(_) => { + return Err!(Database( + "State event's type and state_key combination exists multiple times: \ + {}, {}", + pdu.kind(), + state_key + )); + }, + } } + + // The original create event must still be in the state + let create_shortstatekey = self + .services + .short + .get_shortstatekey(&StateEventType::RoomCreate, "") + .await?; + + if state.get(&create_shortstatekey).map(AsRef::as_ref) != Some(create_event.event_id()) { + return Err!(Database("Incoming event refers to wrong create event.")); + } + + Ok(Some(state)) } - - // The original create event must still be in the state - let create_shortstatekey = self - .services - .short - .get_shortstatekey(&StateEventType::RoomCreate, "") - .await?; - - if state.get(&create_shortstatekey).map(AsRef::as_ref) != Some(create_event.event_id()) { - return Err!(Database("Incoming event refers to wrong create event.")); - } - - Ok(Some(state)) } diff --git a/src/service/rooms/event_handler/handle_incoming_pdu.rs b/src/service/rooms/event_handler/handle_incoming_pdu.rs index 204dfa715..d59ecf69e 100644 --- a/src/service/rooms/event_handler/handle_incoming_pdu.rs +++ b/src/service/rooms/event_handler/handle_incoming_pdu.rs @@ -5,7 +5,7 @@ use std::{ use conduwuit::{ Err, Event, PduEvent, Result, debug::INFO_SPAN_LEVEL, debug_error, debug_info, defer, err, - implement, info, trace, utils::stream::IterStream, warn, + info, trace, utils::stream::IterStream, warn, }; use futures::{ FutureExt, TryFutureExt, TryStreamExt, @@ -81,218 +81,233 @@ async fn should_rescind_invite( Ok(None) } -/// When receiving an event one needs to: -/// 0. Check the server is in the room -/// 1. Skip the PDU if we already know about it -/// 1.1. Remove unsigned field -/// 2. Check signatures, otherwise drop -/// 3. Check content hash, redact if doesn't match -/// 4. Fetch any missing auth events doing all checks listed here starting at 1. -/// These are not timeline events -/// 5. Reject "due to auth events" if can't get all the auth events or some of -/// the auth events are also rejected "due to auth events" -/// 6. Reject "due to auth events" if the event doesn't pass auth based on the -/// auth events -/// 7. Persist this event as an outlier -/// 8. If not timeline event: stop -/// 9. Fetch any missing prev events doing all checks listed here starting at 1. -/// These are timeline events -/// 10. Fetch missing state and auth chain events by calling `/state_ids` at -/// backwards extremities doing all the checks in this list starting at -/// 1. These are not timeline events -/// 11. Check the auth of the event passes based on the state of the event -/// 12. Ensure that the state is derived from the previous current state (i.e. -/// we calculated by doing state res where one of the inputs was a -/// previously trusted set of state, don't just trust a set of state we got -/// from a remote) -/// 13. Use state resolution to find new room state -/// 14. Check if the event passes auth based on the "current state" of the room, -/// if not soft fail it -#[implement(super::Service)] -#[tracing::instrument( +impl super::Service { + /// When receiving an event one needs to: + /// 0. Check the server is in the room + /// 1. Skip the PDU if we already know about it + /// 1.1. Remove unsigned field + /// 2. Check signatures, otherwise drop + /// 3. Check content hash, redact if doesn't match + /// 4. Fetch any missing auth events doing all checks listed here starting + /// at 1. These are not timeline events + /// 5. Reject "due to auth events" if can't get all the auth events or some + /// of the auth events are also rejected "due to auth events" + /// 6. Reject "due to auth events" if the event doesn't pass auth based on + /// the auth events + /// 7. Persist this event as an outlier + /// 8. If not timeline event: stop + /// 9. Fetch any missing prev events doing all checks listed here starting + /// at 1. These are timeline events + /// 10. Fetch missing state and auth chain events by calling `/state_ids` at + /// backwards extremities doing all the checks in this list starting at + /// 1. These are not timeline events + /// 11. Check the auth of the event passes based on the state of the event + /// 12. Ensure that the state is derived from the previous current state + /// (i.e. we calculated by doing state res where one of the inputs was a + /// previously trusted set of state, don't just trust a set of state we + /// got from a remote) + /// 13. Use state resolution to find new room state + /// 14. Check if the event passes auth based on the "current state" of the + /// room, if not soft fail it + #[tracing::instrument( name = "pdu", level = INFO_SPAN_LEVEL, skip_all, fields(%room_id, %event_id), -)] -pub async fn handle_incoming_pdu<'a>( - &self, - origin: &'a ServerName, - room_id: &'a RoomId, - event_id: &'a EventId, - value: BTreeMap, - is_timeline_event: bool, -) -> Result> { - // 1. Skip the PDU if we already have it as a timeline event - if let Ok(pdu_id) = self.services.timeline.get_pdu_id(event_id).await { - return Ok(Some(pdu_id)); - } - if !pdu_fits(&mut value.clone()) { - warn!( - "dropping incoming PDU {event_id} in room {room_id} from {origin} because it \ - exceeds 65535 bytes or is otherwise too large." - ); - return Err!(Request(TooLarge("PDU is too large"))); - } - trace!("processing incoming PDU from {origin} for room {room_id} with event id {event_id}"); - - // 1.1 Check we even know about the room - let meta_exists = self.services.metadata.exists(room_id).map(Ok); - - // 1.2 Check if the room is disabled - let is_disabled = self.services.metadata.is_disabled(room_id).map(Ok); - - // 1.3.1 Check room ACL on origin field/server - let origin_acl_check = self.acl_check(origin, room_id); - - // 1.3.2 Check room ACL on sender's server name - let sender: OwnedUserId = value - .get("sender") - .and_then(|v| v.as_str()) - .ok_or_else(|| err!("No sender in object")) - .and_then(|v| Ok(UserId::parse(v)?)) - .map_err(|e| err!(Request(InvalidParam("PDU does not have a valid sender key: {e}"))))?; - - let sender_acl_check: OptionFuture<_> = sender - .server_name() - .ne(origin) - .then(|| self.acl_check(sender.server_name(), room_id)) - .into(); - - let (meta_exists, is_disabled, (), ()) = try_join4( - meta_exists, - is_disabled, - origin_acl_check, - sender_acl_check.map(|o| o.unwrap_or(Ok(()))), - ) - .await - .inspect_err(|e| debug_error!(%origin, "failed to handle incoming PDU {event_id}: {e}"))?; - - if is_disabled { - return Err!(Request(Forbidden("Federation of this room is disabled by this server."))); - } - - if !self - .services - .state_cache - .server_in_room(self.services.globals.server_name(), room_id) - .await - { - // Is this a federated invite rescind? - // copied from https://github.com/element-hq/synapse/blob/7e4588a/synapse/handlers/federation_event.py#L255-L300 - if value.get("type").and_then(|t| t.as_str()) == Some("m.room.member") { - if let Some(pdu) = - should_rescind_invite(&self.services, &mut value.clone(), &sender, room_id) - .await? - { - debug_info!( - "Invite to {room_id} appears to have been rescinded by {sender}, marking as \ - left" - ); - self.services - .state_cache - .mark_as_left(&sender, room_id, Some(pdu)) - .await; - return Ok(None); - } + )] + pub async fn handle_incoming_pdu<'a>( + &self, + origin: &'a ServerName, + room_id: &'a RoomId, + event_id: &'a EventId, + value: BTreeMap, + is_timeline_event: bool, + ) -> Result> { + // 1. Skip the PDU if we already have it as a timeline event + if let Ok(pdu_id) = self.services.timeline.get_pdu_id(event_id).await { + return Ok(Some(pdu_id)); } - info!( - %origin, - %room_id, - "Dropping inbound PDU for room we aren't participating in" + if !pdu_fits(&mut value.clone()) { + warn!( + "dropping incoming PDU {event_id} in room {room_id} from {origin} because it \ + exceeds 65535 bytes or is otherwise too large." + ); + return Err!(Request(TooLarge("PDU is too large"))); + } + trace!( + "processing incoming PDU from {origin} for room {room_id} with event id {event_id}" ); - return Err!(Request(NotFound("This server is not participating in that room."))); - } - if !meta_exists { - return Err!(Request(NotFound("Room is unknown to this server"))); - } + // 1.1 Check we even know about the room + let meta_exists = self.services.metadata.exists(room_id).map(Ok); - // Fetch create event - let create_event = &self - .services - .state_accessor - .get_room_create_event(room_id) - .await; + // 1.2 Check if the room is disabled + let is_disabled = self.services.metadata.is_disabled(room_id).map(Ok); - let start_time = Instant::now(); - self.federation_handletime - .write() - .insert(room_id.into(), (event_id.to_owned(), start_time)); + // 1.3.1 Check room ACL on origin field/server + let origin_acl_check = self.acl_check(origin, room_id); - defer! {{ + // 1.3.2 Check room ACL on sender's server name + let sender: OwnedUserId = value + .get("sender") + .and_then(|v| v.as_str()) + .ok_or_else(|| err!("No sender in object")) + .and_then(|v| Ok(UserId::parse(v)?)) + .map_err(|e| { + err!(Request(InvalidParam("PDU does not have a valid sender key: {e}"))) + })?; + + let sender_acl_check: OptionFuture<_> = sender + .server_name() + .ne(origin) + .then(|| self.acl_check(sender.server_name(), room_id)) + .into(); + + let (meta_exists, is_disabled, (), ()) = try_join4( + meta_exists, + is_disabled, + origin_acl_check, + sender_acl_check.map(|o| o.unwrap_or(Ok(()))), + ) + .await + .inspect_err( + |e| debug_error!(%origin, "failed to handle incoming PDU {event_id}: {e}"), + )?; + + if is_disabled { + return Err!(Request(Forbidden( + "Federation of this room is disabled by this server." + ))); + } + + if !self + .services + .state_cache + .server_in_room(self.services.globals.server_name(), room_id) + .await + { + // Is this a federated invite rescind? + // copied from https://github.com/element-hq/synapse/blob/7e4588a/synapse/handlers/federation_event.py#L255-L300 + if value.get("type").and_then(|t| t.as_str()) == Some("m.room.member") { + if let Some(pdu) = + should_rescind_invite(&self.services, &mut value.clone(), &sender, room_id) + .await? + { + debug_info!( + "Invite to {room_id} appears to have been rescinded by {sender}, \ + marking as left" + ); + self.services + .state_cache + .mark_as_left(&sender, room_id, Some(pdu)) + .await; + return Ok(None); + } + } + info!( + %origin, + %room_id, + "Dropping inbound PDU for room we aren't participating in" + ); + return Err!(Request(NotFound("This server is not participating in that room."))); + } + + if !meta_exists { + return Err!(Request(NotFound("Room is unknown to this server"))); + } + + // Fetch create event + let create_event = &self + .services + .state_accessor + .get_room_create_event(room_id) + .await; + + let start_time = Instant::now(); self.federation_handletime .write() - .remove(room_id); - }}; + .insert(room_id.into(), (event_id.to_owned(), start_time)); - let (incoming_pdu, val) = self - .handle_outlier_pdu(origin, create_event, event_id, room_id, value, false) - .await?; + defer! {{ + self.federation_handletime + .write() + .remove(room_id); + }} - // 8. if not timeline event: stop - if !is_timeline_event { - return Ok(None); - } + let (incoming_pdu, val) = self + .handle_outlier_pdu(origin, create_event, event_id, room_id, value, false) + .await?; - // Skip old events - let first_ts_in_room = self - .services - .timeline - .first_pdu_in_room(room_id) - .await? - .origin_server_ts(); + // 8. if not timeline event: stop + if !is_timeline_event { + return Ok(None); + } - // 9. Fetch any missing prev events doing all checks listed here starting at 1. - // These are timeline events - let (sorted_prev_events, mut eventid_info) = self - .fetch_prev(origin, create_event, room_id, first_ts_in_room, incoming_pdu.prev_events()) - .await?; + // Skip old events + let first_ts_in_room = self + .services + .timeline + .first_pdu_in_room(room_id) + .await? + .origin_server_ts(); - debug!( - events = ?sorted_prev_events, - "Handling previous events" - ); - - sorted_prev_events - .iter() - .try_stream() - .map_ok(AsRef::as_ref) - .try_for_each(|prev_id| { - self.handle_prev_pdu( + // 9. Fetch any missing prev events doing all checks listed here starting at 1. + // These are timeline events + let (sorted_prev_events, mut eventid_info) = self + .fetch_prev( origin, - event_id, - room_id, - eventid_info.remove(prev_id), create_event, + room_id, first_ts_in_room, - prev_id, + incoming_pdu.prev_events(), ) - .inspect_err(move |e| { - warn!("Prev {prev_id} failed: {e}"); - match self - .services - .globals - .bad_event_ratelimiter - .write() - .entry(prev_id.into()) - { - | hash_map::Entry::Vacant(e) => { - e.insert((Instant::now(), 1)); - }, - | hash_map::Entry::Occupied(mut e) => { - let tries = e.get().1.saturating_add(1); - *e.get_mut() = (Instant::now(), tries); - }, - } - }) - .map(|_| self.services.server.check_running()) - }) - .boxed() - .await?; + .await?; - // Done with prev events, now handling the incoming event - self.upgrade_outlier_to_timeline_pdu(incoming_pdu, val, create_event, origin, room_id) - .boxed() - .await + debug!( + events = ?sorted_prev_events, + "Handling previous events" + ); + + sorted_prev_events + .iter() + .try_stream() + .map_ok(AsRef::as_ref) + .try_for_each(|prev_id| { + self.handle_prev_pdu( + origin, + event_id, + room_id, + eventid_info.remove(prev_id), + create_event, + first_ts_in_room, + prev_id, + ) + .inspect_err(move |e| { + warn!("Prev {prev_id} failed: {e}"); + match self + .services + .globals + .bad_event_ratelimiter + .write() + .entry(prev_id.into()) + { + | hash_map::Entry::Vacant(e) => { + e.insert((Instant::now(), 1)); + }, + | hash_map::Entry::Occupied(mut e) => { + let tries = e.get().1.saturating_add(1); + *e.get_mut() = (Instant::now(), tries); + }, + } + }) + .map(|_| self.services.server.check_running()) + }) + .boxed() + .await?; + + // Done with prev events, now handling the incoming event + self.upgrade_outlier_to_timeline_pdu(incoming_pdu, val, create_event, origin, room_id) + .boxed() + .await + } } diff --git a/src/service/rooms/event_handler/handle_outlier_pdu.rs b/src/service/rooms/event_handler/handle_outlier_pdu.rs index dc08ded5e..ec344433c 100644 --- a/src/service/rooms/event_handler/handle_outlier_pdu.rs +++ b/src/service/rooms/event_handler/handle_outlier_pdu.rs @@ -1,8 +1,7 @@ use std::collections::{BTreeMap, HashMap, hash_map}; use conduwuit::{ - Err, Event, PduEvent, Result, debug, debug_info, debug_warn, err, implement, state_res, - trace, warn, + Err, Event, PduEvent, Result, debug, debug_info, debug_warn, err, state_res, trace, warn, }; use futures::future::ready; use ruma::{ @@ -13,224 +12,232 @@ use ruma::{ use super::{check_room_id, get_room_version_rules}; use crate::rooms::timeline::pdu_fits; -#[implement(super::Service)] -#[allow(clippy::too_many_arguments)] -pub(super) async fn handle_outlier_pdu<'a, Pdu>( - &self, - origin: &'a ServerName, - create_event: &'a Pdu, - event_id: &'a EventId, - room_id: &'a RoomId, - mut value: CanonicalJsonObject, - auth_events_known: bool, -) -> Result<(PduEvent, BTreeMap)> -where - Pdu: Event + Send + Sync, -{ - if !pdu_fits(&mut value.clone()) { - warn!( - "dropping incoming PDU {event_id} in room {room_id} from {origin} because it \ - exceeds 65535 bytes or is otherwise too large." - ); - return Err!(Request(TooLarge("PDU is too large"))); - } - // 1. Remove unsigned field - value.remove("unsigned"); - - // 2. Check signatures, otherwise drop - // 3. check content hash, redact if doesn't match - let room_version_rules = get_room_version_rules(create_event)?; - let mut incoming_pdu = match self - .services - .server_keys - .verify_event(&value, &room_version_rules) - .await +impl super::Service { + /// Handles a PDU as an outlier, performing basic checks like signatures and + /// hashes, proclaimed event auth, and then adding it to the outlier tree. + #[allow(clippy::too_many_arguments)] + pub(super) async fn handle_outlier_pdu<'a, Pdu>( + &self, + origin: &'a ServerName, + create_event: &'a Pdu, + event_id: &'a EventId, + room_id: &'a RoomId, + mut value: CanonicalJsonObject, + auth_events_known: bool, + ) -> Result<(PduEvent, BTreeMap)> + where + Pdu: Event + Send + Sync, { - | Ok(ruma::signatures::Verified::All) => value, - | Ok(ruma::signatures::Verified::Signatures) => { - // Redact - debug_info!("Calculated hash does not match (redaction): {event_id}"); - let Ok(obj) = - ruma::canonical_json::redact(value, &room_version_rules.redaction, None) - else { - return Err!(Request(InvalidParam("Redaction failed"))); - }; - - // Skip the PDU if it is redacted and we already have it as an outlier event - if self.services.timeline.pdu_exists(event_id).await { - return Err!(Request(InvalidParam( - "Event was redacted and we already knew about it" - ))); - } - - obj - }, - | Err(e) => { - return Err!(Request(InvalidParam(debug_error!( - "Signature verification failed for {event_id}: {e}" - )))); - }, - }; - - // Now that we have checked the signature and hashes we can add the eventID and - // convert to our PduEvent type - incoming_pdu - .insert("event_id".to_owned(), CanonicalJsonValue::String(event_id.as_str().to_owned())); - - let pdu_event = serde_json::from_value::( - serde_json::to_value(&incoming_pdu).expect("CanonicalJsonObj is a valid JsonValue"), - ) - .map_err(|e| err!(Request(BadJson(debug_warn!("Event is not a valid PDU: {e}")))))?; - - check_room_id(room_id, &pdu_event)?; - - // Fetch all auth events - let mut auth_events: HashMap = HashMap::new(); - - for aid in pdu_event.auth_events() { - if self.services.pdu_metadata.is_event_rejected(aid).await { - debug_warn!( - "Rejecting incoming event {} which depends on rejected auth event {aid}", - event_id, + if !pdu_fits(&mut value.clone()) { + warn!( + "dropping incoming PDU {event_id} in room {room_id} from {origin} because it \ + exceeds 65535 bytes or is otherwise too large." ); - self.services.pdu_metadata.mark_event_rejected(event_id); - return Err!(Request(InvalidParam("Event has rejected auth event: {aid}"))); + return Err!(Request(TooLarge("PDU is too large"))); } + // 1. Remove unsigned field + value.remove("unsigned"); - if let Ok(auth_event) = self.services.timeline.get_pdu(aid).await { - check_room_id(room_id, &auth_event)?; - trace!("Found auth event {aid} for outlier event {event_id} locally"); - auth_events.insert(aid.to_owned(), auth_event); - } else { - debug_warn!("Could not find auth event {aid} for outlier event {event_id} locally"); - } - } - - // Fetch any missing ones & reject invalid ones - let missing_auth_events = if auth_events_known { - pdu_event - .auth_events() - .filter(|id| !auth_events.contains_key(*id)) - .collect::>() - } else { - pdu_event.auth_events().collect::>() - }; - if !missing_auth_events.is_empty() || !auth_events_known { - debug_info!( - "Fetching {} missing auth events for outlier event {event_id}", - missing_auth_events.len() - ); - for (pdu, _) in self - .fetch_and_handle_outliers( - origin, - missing_auth_events.iter().copied(), - create_event, - room_id, - ) + // 2. Check signatures, otherwise drop + // 3. check content hash, redact if doesn't match + let room_version_rules = get_room_version_rules(create_event)?; + let mut incoming_pdu = match self + .services + .server_keys + .verify_event(&value, &room_version_rules) .await { - auth_events.insert(pdu.event_id().to_owned(), pdu); - } - } else { - debug!("No missing auth events for outlier event {event_id}"); - } - // reject if we are still missing some - let still_missing = pdu_event - .auth_events() - .filter(|id| !auth_events.contains_key(*id)) - .collect::>(); - if !still_missing.is_empty() { - // Don't reject: this could be a temporary condition - // TODO: use get_missing_events? - return Err!(Request(InvalidParam( - "Could not fetch all auth events for outlier event {event_id}, still missing: \ - {still_missing:?}" - ))); - } + | Ok(ruma::signatures::Verified::All) => value, + | Ok(ruma::signatures::Verified::Signatures) => { + // Redact + debug_info!("Calculated hash does not match (redaction): {event_id}"); + let Ok(obj) = + ruma::canonical_json::redact(value, &room_version_rules.redaction, None) + else { + return Err!(Request(InvalidParam("Redaction failed"))); + }; - // 6. Reject "due to auth events" if the event doesn't pass auth based on the - // auth events - debug!("Checking based on auth events"); - let mut auth_events_by_key: HashMap<_, _> = HashMap::with_capacity(auth_events.len()); - // Build map of auth events - for id in pdu_event.auth_events() { - let auth_event = auth_events - .get(id) - .expect("we just checked that we have all auth events") - .to_owned(); + // Skip the PDU if it is redacted and we already have it as an outlier event + if self.services.timeline.pdu_exists(event_id).await { + return Err!(Request(InvalidParam( + "Event was redacted and we already knew about it" + ))); + } - check_room_id(room_id, &auth_event)?; - - match auth_events_by_key.entry(( - auth_event.kind.to_string().into(), - auth_event - .state_key - .clone() - .expect("all auth events have state keys"), - )) { - | hash_map::Entry::Vacant(v) => { - v.insert(auth_event); + obj }, - | hash_map::Entry::Occupied(_) => { - self.services - .outlier - .add_pdu_outlier(pdu_event.event_id(), &incoming_pdu); + | Err(e) => { + return Err!(Request(InvalidParam(debug_error!( + "Signature verification failed for {event_id}: {e}" + )))); + }, + }; + + // Now that we have checked the signature and hashes we can add the eventID and + // convert to our PduEvent type + incoming_pdu.insert( + "event_id".to_owned(), + CanonicalJsonValue::String(event_id.as_str().to_owned()), + ); + + let pdu_event = serde_json::from_value::( + serde_json::to_value(&incoming_pdu).expect("CanonicalJsonObj is a valid JsonValue"), + ) + .map_err(|e| err!(Request(BadJson(debug_warn!("Event is not a valid PDU: {e}")))))?; + + check_room_id(room_id, &pdu_event)?; + + // Fetch all auth events + let mut auth_events: HashMap = HashMap::new(); + + for aid in pdu_event.auth_events() { + if self.services.pdu_metadata.is_event_rejected(aid).await { + debug_warn!( + "Rejecting incoming event {} which depends on rejected auth event {aid}", + event_id, + ); self.services.pdu_metadata.mark_event_rejected(event_id); - return Err!(Request(InvalidParam( - "Auth event's type and state_key combination exists multiple times: {}, {}", - auth_event.kind, - auth_event.state_key().unwrap_or("") - ))); - }, + return Err!(Request(InvalidParam("Event has rejected auth event: {aid}"))); + } + + if let Ok(auth_event) = self.services.timeline.get_pdu(aid).await { + check_room_id(room_id, &auth_event)?; + trace!("Found auth event {aid} for outlier event {event_id} locally"); + auth_events.insert(aid.to_owned(), auth_event); + } else { + debug_warn!( + "Could not find auth event {aid} for outlier event {event_id} locally" + ); + } } - } - // The original create event must be in the auth events - if !matches!( - auth_events_by_key.get(&(StateEventType::RoomCreate, String::new().into())), - Some(_) | None - ) { - self.services.pdu_metadata.mark_event_rejected(event_id); + // Fetch any missing ones & reject invalid ones + let missing_auth_events = if auth_events_known { + pdu_event + .auth_events() + .filter(|id| !auth_events.contains_key(*id)) + .collect::>() + } else { + pdu_event.auth_events().collect::>() + }; + if !missing_auth_events.is_empty() || !auth_events_known { + debug_info!( + "Fetching {} missing auth events for outlier event {event_id}", + missing_auth_events.len() + ); + for (pdu, _) in self + .fetch_and_handle_outliers( + origin, + missing_auth_events.iter().copied(), + create_event, + room_id, + ) + .await + { + auth_events.insert(pdu.event_id().to_owned(), pdu); + } + } else { + debug!("No missing auth events for outlier event {event_id}"); + } + // reject if we are still missing some + let still_missing = pdu_event + .auth_events() + .filter(|id| !auth_events.contains_key(*id)) + .collect::>(); + if !still_missing.is_empty() { + // Don't reject: this could be a temporary condition + // TODO: use get_missing_events? + return Err!(Request(InvalidParam( + "Could not fetch all auth events for outlier event {event_id}, still missing: \ + {still_missing:?}" + ))); + } + + // 6. Reject "due to auth events" if the event doesn't pass auth based on the + // auth events + debug!("Checking based on auth events"); + let mut auth_events_by_key: HashMap<_, _> = HashMap::with_capacity(auth_events.len()); + // Build map of auth events + for id in pdu_event.auth_events() { + let auth_event = auth_events + .get(id) + .expect("we just checked that we have all auth events") + .to_owned(); + + check_room_id(room_id, &auth_event)?; + + match auth_events_by_key.entry(( + auth_event.kind.to_string().into(), + auth_event + .state_key + .clone() + .expect("all auth events have state keys"), + )) { + | hash_map::Entry::Vacant(v) => { + v.insert(auth_event); + }, + | hash_map::Entry::Occupied(_) => { + self.services + .outlier + .add_pdu_outlier(pdu_event.event_id(), &incoming_pdu); + self.services.pdu_metadata.mark_event_rejected(event_id); + return Err!(Request(InvalidParam( + "Auth event's type and state_key combination exists multiple times: {}, \ + {}", + auth_event.kind, + auth_event.state_key().unwrap_or("") + ))); + }, + } + } + + // The original create event must be in the auth events + if !matches!( + auth_events_by_key.get(&(StateEventType::RoomCreate, String::new().into())), + Some(_) | None + ) { + self.services.pdu_metadata.mark_event_rejected(event_id); + self.services + .outlier + .add_pdu_outlier(pdu_event.event_id(), &incoming_pdu); + return Err!(Request(InvalidParam("Incoming event refers to wrong create event."))); + } + + let state_fetch = |ty: &StateEventType, sk: &str| { + let key = (ty.to_owned(), sk.into()); + ready(auth_events_by_key.get(&key).map(ToOwned::to_owned)) + }; + + // PDU check: 3 + let auth_check = state_res::event_auth::auth_check( + &room_version_rules, + &pdu_event, + None, // TODO: third party invite + state_fetch, + create_event.as_pdu(), + ) + .await + .map_err(|e| err!(Request(Forbidden("Auth check failed: {e:?}"))))?; + + if !auth_check { + self.services.pdu_metadata.mark_event_rejected(event_id); + self.services + .outlier + .add_pdu_outlier(pdu_event.event_id(), &incoming_pdu); + return Err!(Request(Forbidden( + "Event authorisation fails based on event's claimed auth events" + ))); + } + + trace!("Validation successful."); + + // 7. Persist the event as an outlier. self.services .outlier .add_pdu_outlier(pdu_event.event_id(), &incoming_pdu); - return Err!(Request(InvalidParam("Incoming event refers to wrong create event."))); + + trace!("Added pdu as outlier."); + + Ok((pdu_event, incoming_pdu)) } - - let state_fetch = |ty: &StateEventType, sk: &str| { - let key = (ty.to_owned(), sk.into()); - ready(auth_events_by_key.get(&key).map(ToOwned::to_owned)) - }; - - // PDU check: 3 - let auth_check = state_res::event_auth::auth_check( - &room_version_rules, - &pdu_event, - None, // TODO: third party invite - state_fetch, - create_event.as_pdu(), - ) - .await - .map_err(|e| err!(Request(Forbidden("Auth check failed: {e:?}"))))?; - - if !auth_check { - self.services.pdu_metadata.mark_event_rejected(event_id); - self.services - .outlier - .add_pdu_outlier(pdu_event.event_id(), &incoming_pdu); - return Err!(Request(Forbidden( - "Event authorisation fails based on event's claimed auth events" - ))); - } - - trace!("Validation successful."); - - // 7. Persist the event as an outlier. - self.services - .outlier - .add_pdu_outlier(pdu_event.event_id(), &incoming_pdu); - - trace!("Added pdu as outlier."); - - Ok((pdu_event, incoming_pdu)) } diff --git a/src/service/rooms/event_handler/handle_prev_pdu.rs b/src/service/rooms/event_handler/handle_prev_pdu.rs index cb4978d94..6eb0d1383 100644 --- a/src/service/rooms/event_handler/handle_prev_pdu.rs +++ b/src/service/rooms/event_handler/handle_prev_pdu.rs @@ -1,89 +1,95 @@ use std::{collections::BTreeMap, time::Instant}; use conduwuit::{ - Err, Event, PduEvent, Result, debug::INFO_SPAN_LEVEL, defer, implement, + Err, Event, PduEvent, Result, debug::INFO_SPAN_LEVEL, defer, utils::continue_exponential_backoff_secs, }; use ruma::{CanonicalJsonValue, EventId, MilliSecondsSinceUnixEpoch, RoomId, ServerName}; use tracing::debug; -#[implement(super::Service)] -#[allow(clippy::type_complexity)] -#[allow(clippy::too_many_arguments)] -#[tracing::instrument( +impl super::Service { + #[allow(clippy::type_complexity)] + #[allow(clippy::too_many_arguments)] + #[tracing::instrument( name = "prev", level = INFO_SPAN_LEVEL, skip_all, fields(%prev_id), -)] -pub(super) async fn handle_prev_pdu<'a, Pdu>( - &self, - origin: &'a ServerName, - event_id: &'a EventId, - room_id: &'a RoomId, - eventid_info: Option<(PduEvent, BTreeMap)>, - create_event: &'a Pdu, - first_ts_in_room: MilliSecondsSinceUnixEpoch, - prev_id: &'a EventId, -) -> Result -where - Pdu: Event + Send + Sync, -{ - // Check for disabled again because it might have changed - if self.services.metadata.is_disabled(room_id).await { - return Err!(Request(Forbidden(debug_warn!( - "Federaton of room {room_id} is currently disabled on this server. Request by \ - origin {origin} and event ID {event_id}" - )))); - } - - if let Some((time, tries)) = self - .services - .globals - .bad_event_ratelimiter - .read() - .get(prev_id) + )] + pub(super) async fn handle_prev_pdu<'a, Pdu>( + &self, + origin: &'a ServerName, + event_id: &'a EventId, + room_id: &'a RoomId, + eventid_info: Option<(PduEvent, BTreeMap)>, + create_event: &'a Pdu, + first_ts_in_room: MilliSecondsSinceUnixEpoch, + prev_id: &'a EventId, + ) -> Result + where + Pdu: Event + Send + Sync, { - // Exponential backoff - const MIN_DURATION: u64 = 5 * 60; - const MAX_DURATION: u64 = 60 * 60 * 24; - if continue_exponential_backoff_secs(MIN_DURATION, MAX_DURATION, time.elapsed(), *tries) { - debug!( - ?tries, - duration = ?time.elapsed(), - "Backing off from prev_event" - ); + // Check for disabled again because it might have changed + if self.services.metadata.is_disabled(room_id).await { + return Err!(Request(Forbidden(debug_warn!( + "Federaton of room {room_id} is currently disabled on this server. Request by \ + origin {origin} and event ID {event_id}" + )))); + } + + if let Some((time, tries)) = self + .services + .globals + .bad_event_ratelimiter + .read() + .get(prev_id) + { + // Exponential backoff + const MIN_DURATION: u64 = 5 * 60; + const MAX_DURATION: u64 = 60 * 60 * 24; + if continue_exponential_backoff_secs( + MIN_DURATION, + MAX_DURATION, + time.elapsed(), + *tries, + ) { + debug!( + ?tries, + duration = ?time.elapsed(), + "Backing off from prev_event" + ); + return Ok(()); + } + } + + let Some((pdu, json)) = eventid_info else { + return Ok(()); + }; + + // Skip old events + if pdu.origin_server_ts() < first_ts_in_room { return Ok(()); } - } - let Some((pdu, json)) = eventid_info else { - return Ok(()); - }; - - // Skip old events - if pdu.origin_server_ts() < first_ts_in_room { - return Ok(()); - } - - let start_time = Instant::now(); - self.federation_handletime - .write() - .insert(room_id.into(), ((*prev_id).to_owned(), start_time)); - - defer! {{ + let start_time = Instant::now(); self.federation_handletime .write() - .remove(room_id); - }}; + .insert(room_id.into(), ((*prev_id).to_owned(), start_time)); - self.upgrade_outlier_to_timeline_pdu(pdu, json, create_event, origin, room_id) - .await?; + defer! {{ + self.federation_handletime + .write() + .remove(room_id); + }}; - debug!( - elapsed = ?start_time.elapsed(), - "Handled prev_event", - ); + self.upgrade_outlier_to_timeline_pdu(pdu, json, create_event, origin, room_id) + .await?; - Ok(()) + debug!( + elapsed = ?start_time.elapsed(), + "Handled prev_event", + ); + + Ok(()) + } } diff --git a/src/service/rooms/event_handler/mod.rs b/src/service/rooms/event_handler/mod.rs index 6197bc87e..8a1922652 100644 --- a/src/service/rooms/event_handler/mod.rs +++ b/src/service/rooms/event_handler/mod.rs @@ -94,10 +94,13 @@ impl crate::Service for Service { } impl Service { + /// Checks if a single event exists. Alias for + /// `self.services.timeline.pdu_exists`. async fn event_exists(&self, event_id: OwnedEventId) -> bool { self.services.timeline.pdu_exists(&event_id).await } + /// Fetches a single PDU, returning None if there is an error. async fn event_fetch(&self, event_id: OwnedEventId) -> Option { self.services.timeline.get_pdu(&event_id).await.ok() } diff --git a/src/service/rooms/event_handler/parse_incoming_pdu.rs b/src/service/rooms/event_handler/parse_incoming_pdu.rs index b4af6b4a8..6726f4bd3 100644 --- a/src/service/rooms/event_handler/parse_incoming_pdu.rs +++ b/src/service/rooms/event_handler/parse_incoming_pdu.rs @@ -1,7 +1,7 @@ use std::str::FromStr; use conduwuit::{ - Err, Result, err, implement, + Err, Result, err, matrix::event::{gen_event_id, gen_event_id_canonical_json}, }; use itertools::Itertools; @@ -76,55 +76,57 @@ fn expect_event_id_array(value: &CanonicalJsonObject, field: &str) -> Result Result { - // Since v3: - // `event_id` should not be present on the PDU. - // NOTE: The above is ignored since technically it's still allowed to be - // included, but should be ignored instead. - // `auth_events` and `prev_events` must be an array of event IDs - let auth_events = expect_event_id_array(pdu, "auth_events")?; - if auth_events.len() > 10 { - return Err!(Request(BadJson("PDU has too many auth events"))); +impl super::Service { + /// Performs some basic validation on the PDU to make sure it's not + /// obviously malformed. This is not a full validation, but guards against + /// extreme errors. + /// + /// Currently, this just validates that prev/auth events are within + /// acceptable ranges. Other servers do some additional things like + /// checking depth range, but serde will do that later when converting the + /// object to a PduEvent. + pub fn validate_pdu(&self, pdu: &CanonicalJsonObject) -> Result { + // Since v3: + // `event_id` should not be present on the PDU. + // NOTE: The above is ignored since technically it's still allowed to be + // included, but should be ignored instead. + // `auth_events` and `prev_events` must be an array of event IDs + let auth_events = expect_event_id_array(pdu, "auth_events")?; + if auth_events.len() > 10 { + return Err!(Request(BadJson("PDU has too many auth events"))); + } + let prev_events = expect_event_id_array(pdu, "prev_events")?; + if prev_events.len() > 20 { + return Err!(Request(BadJson("PDU has too many prev events"))); + } + Ok(()) } - let prev_events = expect_event_id_array(pdu, "prev_events")?; - if prev_events.len() > 20 { - return Err!(Request(BadJson("PDU has too many prev events"))); - } - Ok(()) -} -#[implement(super::Service)] -pub async fn parse_incoming_pdu(&self, pdu: &RawJsonValue) -> Result { - let value = serde_json::from_str::(pdu.get()).map_err(|e| { - err!(BadServerResponse(debug_warn!("Error parsing incoming event {e:?}"))) - })?; - let event_type = value - .get("type") - .and_then(CanonicalJsonValue::as_str) - .ok_or_else(|| err!(Request(InvalidParam("Missing or invalid type in pdu"))))?; - - let room_id = extract_room_id(event_type, &value)?; - - let room_version_rules = self - .services - .state - .get_room_version(&room_id) - .await - .unwrap_or(RoomVersionId::V1) - .rules() - .unwrap(); - - let (event_id, value) = - gen_event_id_canonical_json(pdu, &room_version_rules).map_err(|e| { - err!(Request(InvalidParam("Could not convert event to canonical json: {e}"))) + pub async fn parse_incoming_pdu(&self, pdu: &RawJsonValue) -> Result { + let value = serde_json::from_str::(pdu.get()).map_err(|e| { + err!(BadServerResponse(debug_warn!("Error parsing incoming event {e:?}"))) })?; - self.validate_pdu(&value)?; - Ok((room_id, event_id, value)) + let event_type = value + .get("type") + .and_then(CanonicalJsonValue::as_str) + .ok_or_else(|| err!(Request(InvalidParam("Missing or invalid type in pdu"))))?; + + let room_id = extract_room_id(event_type, &value)?; + + let room_version_rules = self + .services + .state + .get_room_version(&room_id) + .await + .unwrap_or(RoomVersionId::V1) + .rules() + .unwrap(); + + let (event_id, value) = + gen_event_id_canonical_json(pdu, &room_version_rules).map_err(|e| { + err!(Request(InvalidParam("Could not convert event to canonical json: {e}"))) + })?; + self.validate_pdu(&value)?; + Ok((room_id, event_id, value)) + } } diff --git a/src/service/rooms/event_handler/policy_server.rs b/src/service/rooms/event_handler/policy_server.rs index cdd142277..330d48ef6 100644 --- a/src/service/rooms/event_handler/policy_server.rs +++ b/src/service/rooms/event_handler/policy_server.rs @@ -6,8 +6,8 @@ use std::{collections::BTreeMap, time::Duration}; use conduwuit::{ - Err, Error, Event, PduEvent, Result, debug, debug_error, debug_info, debug_warn, error, - implement, info, state_res::EventTypeExt, trace, utils::to_canonical_object, warn, + Err, Error, Event, PduEvent, Result, debug, debug_error, debug_info, debug_warn, error, info, + state_res::EventTypeExt, trace, utils::to_canonical_object, warn, }; use http::StatusCode; use ruma::{ @@ -93,348 +93,351 @@ pub(super) fn extract_signature( .and_then(|signature| Base64::::parse(signature).ok()) } -/// Verifies the existing policy server signature, and/or fetches a new one -/// immediately. -/// -/// If `incoming` is `true`, the event is checked for an existing signature. If -/// it has a valid one, `Ok` is returned. If it does not have a valid signature, -/// the function falls through to fetching a new one (which may be a soft-fail -/// in a future version). -/// -/// If the event is the `m.room.policy` configuration state event, -/// this check is skipped. Similarly, if there is no policy server configured in -/// the PDU's room, the configuration event is malformed, or the configured -/// server is not present in the room, the check is also skipped. -/// -/// If the policy server marks the event as spam, the relevant error is -/// returned. Otherwise, the incoming PDU JSON is mutated to include the new -/// policy server signature. Transient errors such as rate-limits are handled, -/// so any error returned by this function should be treated as final. -#[implement(super::Service)] -#[tracing::instrument(skip(self, pdu, pdu_json, room_version_rules), level = "info")] -pub async fn policy_server_allows_event( - &self, - pdu: &PduEvent, - pdu_json: &mut CanonicalJsonObject, - room_id: &RoomId, - room_version_rules: &RoomVersionRules, - incoming: bool, -) -> Result<()> { - assert!( - !pdu_json.contains_key("event_id"), - "event_id should be removed from the JSON before calling policy_server_allows_event" - ); - if pdu.event_type().with_state_key("") == (StateEventType::RoomPolicy, "".into()) { - return Ok(()); - } - let ps = match self - .services - .state_accessor - .room_state_get_content::( - room_id, - &StateEventType::RoomPolicy, - "", - ) - .await - { - | Ok(s) => s, - | Err(e) => - return if e.is_not_found() || e.kind() == ErrorKind::BadJson { - debug!(%e, "no policy server configured"); - Ok(()) - } else { - Err!("failed to load m.room.policy state event: {e}") - }, - }; - - let Some(ps_key) = ps.public_keys.get(&SigningKeyAlgorithm::Ed25519) else { - debug!( - "room has a policy server configured, but no valid public keys; skipping spam check" +impl super::Service { + /// Verifies the existing policy server signature, and/or fetches a new one + /// immediately. + /// + /// If `incoming` is `true`, the event is checked for an existing signature. + /// If it has a valid one, `Ok` is returned. If it does not have a valid + /// signature, the function falls through to fetching a new one (which may + /// be a soft-fail in a future version). + /// + /// If the event is the `m.room.policy` configuration state event, + /// this check is skipped. Similarly, if there is no policy server + /// configured in the PDU's room, the configuration event is malformed, or + /// the configured server is not present in the room, the check is also + /// skipped. + /// + /// If the policy server marks the event as spam, the relevant error is + /// returned. Otherwise, the incoming PDU JSON is mutated to include the new + /// policy server signature. Transient errors such as rate-limits are + /// handled, so any error returned by this function should be treated as + /// final. + #[tracing::instrument(skip(self, pdu, pdu_json, room_version_rules), level = "info")] + pub async fn policy_server_allows_event( + &self, + pdu: &PduEvent, + pdu_json: &mut CanonicalJsonObject, + room_id: &RoomId, + room_version_rules: &RoomVersionRules, + incoming: bool, + ) -> Result<()> { + assert!( + !pdu_json.contains_key("event_id"), + "event_id should be removed from the JSON before calling policy_server_allows_event" ); - return Ok(()); - }; + if pdu.event_type().with_state_key("") == (StateEventType::RoomPolicy, "".into()) { + return Ok(()); + } + let ps = match self + .services + .state_accessor + .room_state_get_content::( + room_id, + &StateEventType::RoomPolicy, + "", + ) + .await + { + | Ok(s) => s, + | Err(e) => + return if e.is_not_found() || e.kind() == ErrorKind::BadJson { + debug!(%e, "no policy server configured"); + Ok(()) + } else { + Err!("failed to load m.room.policy state event: {e}") + }, + }; - if !self - .services - .state_cache - .server_in_room(&ps.via, room_id) - .await - { - debug!( - via = %ps.via, - "Policy server is not in the room, skipping spam check" - ); - return Ok(()); - } + let Some(ps_key) = ps.public_keys.get(&SigningKeyAlgorithm::Ed25519) else { + debug!( + "room has a policy server configured, but no valid public keys; skipping spam \ + check" + ); + return Ok(()); + }; - if incoming { - if verify_policy_signature(&ps.via, ps_key, pdu_json, &room_version_rules.redaction) { + if !self + .services + .state_cache + .server_in_room(&ps.via, room_id) + .await + { debug!( via = %ps.via, - "Event is incoming and has a valid policy server signature" + "Policy server is not in the room, skipping spam check" ); return Ok(()); } - // N.B. In a future room version, this will be a soft failure specifically. - debug_info!( - via = %ps.via, - "Event is incoming but does not have a valid policy server signature; asking policy \ - server to sign it now" - ); - } - if ps.via == self.services.globals.server_name() - && !self.services.server.config.federation_loopback - { - error!( - %ps.via, - %room_id, - "Cannot ask ourselves for a policy signature if `federation_loopback=false`", - ); - return Ok(()); - } - - let outgoing = self - .services - .sending - .convert_to_outgoing_federation_event(pdu_json.clone()) - .await; - - debug_info!( - %ps.via, - "Asking policy server to sign event" - ); - if let Err(e) = self - .fetch_policy_server_signature(pdu, pdu_json, &ps.via, outgoing, room_id, ps_key, 0) - .await - { - if e.is_not_found() { - return Ok(()); - } - return Err(e); - } - trace!( - "Got successful response for fetching PS signature, ensuring it is signed with the \ - expected key." - ); - if verify_policy_signature(&ps.via, ps_key, pdu_json, &room_version_rules.redaction) { - Ok(()) - } else if incoming { - Err!(Request(Forbidden("Policy server signature is invalid"))) - } else { - Err(Error::Request( - ErrorKind::Unknown, - "Policy server signature is invalid".into(), - StatusCode::BAD_GATEWAY, - )) - } -} - -/// Handles an error returned by the policy server. If the error is one that -/// should be returned to the user, it is propagated, otherwise the request may -/// be retried (for example, when rate-limited). -#[allow(clippy::too_many_arguments)] -#[implement(super::Service)] -async fn handle_policy_server_error( - &self, - error: Error, - pdu: &PduEvent, - pdu_json: &mut CanonicalJsonObject, - via: &ServerName, - outgoing: Box, - room_id: &RoomId, - policy_server_key: &Base64>, - retries: u8, - timeout: Duration, -) -> Result<()> { - match error.status_code() { - | StatusCode::OK => unreachable!("ok response passed to handle_policy_server_error"), - | StatusCode::BAD_REQUEST => { - if matches!(error.kind(), ErrorKind::Forbidden) { - warn!( - via = %via, - event_id = %pdu.event_id(), - %room_id, - error = ?error, - "Policy server marked the event as spam" + if incoming { + if verify_policy_signature(&ps.via, ps_key, pdu_json, &room_version_rules.redaction) { + debug!( + via = %ps.via, + "Event is incoming and has a valid policy server signature" ); - return Err(error); + return Ok(()); } - error!( - via = %via, - event_id = %pdu.event_id(), - %room_id, - error = ?error.to_string(), - "Policy server could not understand our request", - ); - Err!(BadServerResponse("Error communicating with policy server")) - }, - | StatusCode::FORBIDDEN => { - Err!(Request(Forbidden( - "Policy server refused to sign the event due to the room ACL" - ))) - }, - | StatusCode::NOT_FOUND => { + // N.B. In a future room version, this will be a soft failure specifically. debug_info!( - via = %via, - event_id = %pdu.event_id(), - %room_id, - "Policy server is not actually a policy server or is not protecting this room: {}", - error.message() + via = %ps.via, + "Event is incoming but does not have a valid policy server signature; asking policy \ + server to sign it now" ); - Err(error) - }, - | StatusCode::TOO_MANY_REQUESTS => { - if let Some(retry_after) = error.retry_after() { - if retries >= 5 { + } + + if ps.via == self.services.globals.server_name() + && !self.services.server.config.federation_loopback + { + error!( + %ps.via, + %room_id, + "Cannot ask ourselves for a policy signature if `federation_loopback=false`", + ); + return Ok(()); + } + + let outgoing = self + .services + .sending + .convert_to_outgoing_federation_event(pdu_json.clone()) + .await; + + debug_info!( + %ps.via, + "Asking policy server to sign event" + ); + if let Err(e) = self + .fetch_policy_server_signature(pdu, pdu_json, &ps.via, outgoing, room_id, ps_key, 0) + .await + { + if e.is_not_found() { + return Ok(()); + } + return Err(e); + } + trace!( + "Got successful response for fetching PS signature, ensuring it is signed with the \ + expected key." + ); + if verify_policy_signature(&ps.via, ps_key, pdu_json, &room_version_rules.redaction) { + Ok(()) + } else if incoming { + Err!(Request(Forbidden("Policy server signature is invalid"))) + } else { + Err(Error::Request( + ErrorKind::Unknown, + "Policy server signature is invalid".into(), + StatusCode::BAD_GATEWAY, + )) + } + } + + /// Handles an error returned by the policy server. If the error is one that + /// should be returned to the user, it is propagated, otherwise the request + /// may be retried (for example, when rate-limited). + #[allow(clippy::too_many_arguments)] + async fn handle_policy_server_error( + &self, + error: Error, + pdu: &PduEvent, + pdu_json: &mut CanonicalJsonObject, + via: &ServerName, + outgoing: Box, + room_id: &RoomId, + policy_server_key: &Base64>, + retries: u8, + timeout: Duration, + ) -> Result<()> { + match error.status_code() { + | StatusCode::OK => unreachable!("ok response passed to handle_policy_server_error"), + | StatusCode::BAD_REQUEST => { + if matches!(error.kind(), ErrorKind::Forbidden) { warn!( via = %via, event_id = %pdu.event_id(), - room_id = %room_id, - retries, - "Policy server rate-limited us too many times; giving up" + %room_id, + error = ?error, + "Policy server marked the event as spam" ); - return Err(error); // Error should be passed to c2s + return Err(error); } - let saturated = retry_after.min(timeout); - // ^ don't wait more than 60 seconds - info!( + error!( + via = %via, + event_id = %pdu.event_id(), + %room_id, + error = ?error.to_string(), + "Policy server could not understand our request", + ); + Err!(BadServerResponse("Error communicating with policy server")) + }, + | StatusCode::FORBIDDEN => { + Err!(Request(Forbidden( + "Policy server refused to sign the event due to the room ACL" + ))) + }, + | StatusCode::NOT_FOUND => { + debug_info!( + via = %via, + event_id = %pdu.event_id(), + %room_id, + "Policy server is not actually a policy server or is not protecting this room: {}", + error.message() + ); + Err(error) + }, + | StatusCode::TOO_MANY_REQUESTS => { + if let Some(retry_after) = error.retry_after() { + if retries >= 5 { + warn!( + via = %via, + event_id = %pdu.event_id(), + room_id = %room_id, + retries, + "Policy server rate-limited us too many times; giving up" + ); + return Err(error); // Error should be passed to c2s + } + let saturated = retry_after.min(timeout); + // ^ don't wait more than 60 seconds + info!( + via = %via, + event_id = %pdu.event_id(), + room_id = %room_id, + retry_after = %saturated.as_secs(), + retries, + "Policy server rate-limited us; retrying after {retry_after:?}" + ); + tokio::select! { + () = self.server_shutdown.notified() => (), + () = sleep(saturated) => (), + } + if !self.services.server.running() { + return Err(error); + } + return Box::pin(self.fetch_policy_server_signature( + pdu, + pdu_json, + via, + outgoing, + room_id, + policy_server_key, + retries.saturating_add(1), + )) + .await; + } + warn!( via = %via, event_id = %pdu.event_id(), room_id = %room_id, - retry_after = %saturated.as_secs(), retries, - "Policy server rate-limited us; retrying after {retry_after:?}" + "Policy server rate-limited us without giving a retry window; giving up" ); - tokio::select! { - () = self.server_shutdown.notified() => (), - () = sleep(saturated) => (), - } - if !self.services.server.running() { - return Err(error); - } - return Box::pin(self.fetch_policy_server_signature( - pdu, - pdu_json, - via, - outgoing, - room_id, - policy_server_key, - retries.saturating_add(1), - )) - .await; - } - warn!( - via = %via, - event_id = %pdu.event_id(), - room_id = %room_id, - retries, - "Policy server rate-limited us without giving a retry window; giving up" + Err(error) + }, + | _ => Err!(BadServerResponse( + "Unexpected response from policy server: {}/{:?}", + error.status_code(), + error.kind() + )), + } + } + + /// Asks a remote policy server for a signature on this event. + /// If the policy server signs this event, the original data is mutated. + /// Otherwise, the error is handled and potentially returned. + #[allow(clippy::too_many_arguments)] + #[tracing::instrument(skip_all, fields(event_id=%pdu.event_id(), via=%via), level = "info")] + pub async fn fetch_policy_server_signature( + &self, + pdu: &PduEvent, + pdu_json: &mut CanonicalJsonObject, + via: &ServerName, + outgoing: Box, + room_id: &RoomId, + policy_server_key: &Base64>, + retries: u8, + ) -> Result<()> { + let timeout = + Duration::from_secs(self.services.server.config.policy_server_request_timeout); + debug!("Requesting policy server signature"); + let response = tokio::time::timeout( + timeout, + self.services + .sending + .send_federation_request(via, sign_event::v1::Request::new(outgoing.clone())), + ) + .await; + + let response = match response { + | Ok(Ok(response)) => response, + | Ok(Err(e)) => { + debug_error!("Error from policy server: {:?}", e); + return self + .handle_policy_server_error( + e, + pdu, + pdu_json, + via, + outgoing, + room_id, + policy_server_key, + retries, + timeout, + ) + .await; + }, + | Err(elapsed) => { + warn!( + %via, + event_id = %pdu.event_id(), + %room_id, + %elapsed, + "Policy server signature request timed out" + ); + return Err!(Request(Forbidden("Policy server did not respond in time"))); + }, + }; + + let Some(signatures) = response.signatures.get(via) else { + // NOTE: Legacy policy servers return a `200 {}` to indicate that the event was + // flagged as spam. We'll make a distinction in the error message in case + // it's unexpected. + return Err!(Request(Forbidden("Policy server did not sign the event"))); + }; + if response.signatures.len() > 1 { + warn!(?response.signatures, "Misbehaving policy server: returned signatures for extraneous servers"); + // TODO: This should return an error but doesn't because some + // servers do this. It's safe for us to not explode for now + // because we only care about the signature for `via`, but + // ideally we'd want to enforce this more strictly in the future. + } + + let Some(signature) = signatures.get( + &KeyId::parse(POLICY_SERVER_KEY_ID_ED25519).expect("policy server key ID is valid"), + ) else { + return Err!(BadServerResponse( + "Policy server did not return a signature with the expected key ID", + )); + }; + if signatures.len() > 1 { + info!(?signatures, "Misbehaving policy server: returned extraneous signatures"); + } + + pdu_json + .entry("signatures".to_owned()) + .or_insert_with(|| CanonicalJsonValue::Object(BTreeMap::default())) + .as_object_mut() + .expect("`signatures` field must be an object") + .entry(via.to_string()) + .or_insert_with(|| CanonicalJsonValue::Object(BTreeMap::default())) + .as_object_mut() + .expect("`signatures[via]` field must be an object") + .insert( + POLICY_SERVER_KEY_ID_ED25519.to_owned(), + CanonicalJsonValue::String(signature.clone()), ); - Err(error) - }, - | _ => Err!(BadServerResponse( - "Unexpected response from policy server: {}/{:?}", - error.status_code(), - error.kind() - )), + debug_info!("Policy server allowed event"); + Ok(()) } } - -/// Asks a remote policy server for a signature on this event. -/// If the policy server signs this event, the original data is mutated. -/// Otherwise, the error is handled and potentially returned. -#[allow(clippy::too_many_arguments)] -#[implement(super::Service)] -#[tracing::instrument(skip_all, fields(event_id=%pdu.event_id(), via=%via), level = "info")] -pub async fn fetch_policy_server_signature( - &self, - pdu: &PduEvent, - pdu_json: &mut CanonicalJsonObject, - via: &ServerName, - outgoing: Box, - room_id: &RoomId, - policy_server_key: &Base64>, - retries: u8, -) -> Result<()> { - let timeout = Duration::from_secs(self.services.server.config.policy_server_request_timeout); - debug!("Requesting policy server signature"); - let response = tokio::time::timeout( - timeout, - self.services - .sending - .send_federation_request(via, sign_event::v1::Request::new(outgoing.clone())), - ) - .await; - - let response = match response { - | Ok(Ok(response)) => response, - | Ok(Err(e)) => { - debug_error!("Error from policy server: {:?}", e); - return self - .handle_policy_server_error( - e, - pdu, - pdu_json, - via, - outgoing, - room_id, - policy_server_key, - retries, - timeout, - ) - .await; - }, - | Err(elapsed) => { - warn!( - %via, - event_id = %pdu.event_id(), - %room_id, - %elapsed, - "Policy server signature request timed out" - ); - return Err!(Request(Forbidden("Policy server did not respond in time"))); - }, - }; - - let Some(signatures) = response.signatures.get(via) else { - // NOTE: Legacy policy servers return a `200 {}` to indicate that the event was - // flagged as spam. We'll make a distinction in the error message in case - // it's unexpected. - return Err!(Request(Forbidden("Policy server did not sign the event"))); - }; - if response.signatures.len() > 1 { - warn!(?response.signatures, "Misbehaving policy server: returned signatures for extraneous servers"); - // TODO: This should return an error but doesn't because some servers do - // this. It's safe for us to not explode for now because we only care - // about the signature for `via`, but ideally we'd want to enforce - // this more strictly in the future. - } - - let Some(signature) = signatures - .get(&KeyId::parse(POLICY_SERVER_KEY_ID_ED25519).expect("policy server key ID is valid")) - else { - return Err!(BadServerResponse( - "Policy server did not return a signature with the expected key ID", - )); - }; - if signatures.len() > 1 { - info!(?signatures, "Misbehaving policy server: returned extraneous signatures"); - } - - pdu_json - .entry("signatures".to_owned()) - .or_insert_with(|| CanonicalJsonValue::Object(BTreeMap::default())) - .as_object_mut() - .expect("`signatures` field must be an object") - .entry(via.to_string()) - .or_insert_with(|| CanonicalJsonValue::Object(BTreeMap::default())) - .as_object_mut() - .expect("`signatures[via]` field must be an object") - .insert( - POLICY_SERVER_KEY_ID_ED25519.to_owned(), - CanonicalJsonValue::String(signature.clone()), - ); - debug_info!("Policy server allowed event"); - Ok(()) -} diff --git a/src/service/rooms/event_handler/resolve_state.rs b/src/service/rooms/event_handler/resolve_state.rs index 471876658..4f35ceecb 100644 --- a/src/service/rooms/event_handler/resolve_state.rs +++ b/src/service/rooms/event_handler/resolve_state.rs @@ -5,7 +5,7 @@ use std::{ }; use conduwuit::{ - Error, Result, err, implement, + Error, Result, err, state_res::{self, StateMap}, trace, utils::stream::{IterStream, ReadyExt, TryWidebandExt, WidebandExt}, @@ -15,110 +15,108 @@ use ruma::{OwnedEventId, RoomId, room_version_rules::RoomVersionRules}; use crate::rooms::state_compressor::CompressedState; -#[implement(super::Service)] -#[tracing::instrument(name = "resolve", level = "debug", skip_all)] -pub async fn resolve_state( - &self, - room_id: &RoomId, - room_version_rules: &RoomVersionRules, - incoming_state: HashMap, -) -> Result> { - trace!("Loading current room state ids"); - let current_sstatehash = self - .services - .state - .get_room_shortstatehash(room_id) - .map_err(|e| err!(Database(error!("No state for {room_id:?}: {e:?}")))) - .await?; +impl super::Service { + pub async fn resolve_state( + &self, + room_id: &RoomId, + room_version_rules: &RoomVersionRules, + incoming_state: HashMap, + ) -> Result> { + trace!("Loading current room state ids"); + let current_sstatehash = self + .services + .state + .get_room_shortstatehash(room_id) + .map_err(|e| err!(Database(error!("No state for {room_id:?}: {e:?}")))) + .await?; - let current_state_ids: HashMap<_, _> = self - .services - .state_accessor - .state_full_ids(current_sstatehash) - .collect() - .await; + let current_state_ids: HashMap<_, _> = self + .services + .state_accessor + .state_full_ids(current_sstatehash) + .collect() + .await; - trace!("Loading fork states"); - let fork_states = [current_state_ids, incoming_state]; - let auth_chain_sets = fork_states - .iter() - .try_stream() - .wide_and_then(|state| { - self.services - .auth_chain - .event_ids_iter(room_id, state.values().map(Borrow::borrow)) - .try_collect() - }) - .try_collect::>>(); + trace!("Loading fork states"); + let fork_states = [current_state_ids, incoming_state]; + let auth_chain_sets = fork_states + .iter() + .try_stream() + .wide_and_then(|state| { + self.services + .auth_chain + .event_ids_iter(room_id, state.values().map(Borrow::borrow)) + .try_collect() + }) + .try_collect::>>(); - let fork_states = fork_states - .iter() - .stream() - .wide_then(|fork_state| { - let shortstatekeys = fork_state.keys().copied().stream(); - let event_ids = fork_state.values().cloned().stream(); - self.services - .short - .multi_get_statekey_from_short(shortstatekeys) - .zip(event_ids) - .ready_filter_map(|(ty_sk, id)| Some((ty_sk.ok()?, id))) - .collect() - }) - .map(Ok::<_, Error>) - .try_collect::>>(); + let fork_states = fork_states + .iter() + .stream() + .wide_then(|fork_state| { + let shortstatekeys = fork_state.keys().copied().stream(); + let event_ids = fork_state.values().cloned().stream(); + self.services + .short + .multi_get_statekey_from_short(shortstatekeys) + .zip(event_ids) + .ready_filter_map(|(ty_sk, id)| Some((ty_sk.ok()?, id))) + .collect() + }) + .map(Ok::<_, Error>) + .try_collect::>>(); - let (fork_states, auth_chain_sets) = try_join(fork_states, auth_chain_sets).await?; + let (fork_states, auth_chain_sets) = try_join(fork_states, auth_chain_sets).await?; - trace!("Resolving state"); - let state = self - .state_resolution(room_version_rules, fork_states.iter(), &auth_chain_sets) - .boxed() - .await?; + trace!("Resolving state"); + let state = self + .state_resolution(room_version_rules, fork_states.iter(), &auth_chain_sets) + .boxed() + .await?; - trace!("State resolution done."); - let state_events: Vec<_> = state - .iter() - .stream() - .wide_then(|((event_type, state_key), event_id)| { - self.services - .short - .get_or_create_shortstatekey(event_type, state_key) - .map(move |shortstatekey| (shortstatekey, event_id)) - }) - .collect() - .await; + trace!("State resolution done."); + let state_events: Vec<_> = state + .iter() + .stream() + .wide_then(|((event_type, state_key), event_id)| { + self.services + .short + .get_or_create_shortstatekey(event_type, state_key) + .map(move |shortstatekey| (shortstatekey, event_id)) + }) + .collect() + .await; - trace!("Compressing state..."); - let new_room_state: CompressedState = self - .services - .state_compressor - .compress_state_events(state_events.iter().map(|(ssk, eid)| (ssk, (*eid).borrow()))) - .collect() - .await; + trace!("Compressing state..."); + let new_room_state: CompressedState = self + .services + .state_compressor + .compress_state_events(state_events.iter().map(|(ssk, eid)| (ssk, (*eid).borrow()))) + .collect() + .await; - Ok(Arc::new(new_room_state)) -} + Ok(Arc::new(new_room_state)) + } -#[implement(super::Service)] -#[tracing::instrument(name = "ruma", level = "debug", skip_all)] -pub async fn state_resolution<'a, StateSets>( - &'a self, - room_version_rules: &'a RoomVersionRules, - state_sets: StateSets, - auth_chain_sets: &'a [HashSet], -) -> Result> -where - StateSets: Iterator> + Clone + Send, -{ - let event_fetch = |event_id| self.event_fetch(event_id); - let event_exists = |event_id| self.event_exists(event_id); - state_res::resolve( - room_version_rules, - state_sets, - auth_chain_sets, - &event_fetch, - &event_exists, - ) - .map_err(|e| err!(error!("State resolution failed: {e:?}"))) - .await + pub async fn state_resolution<'a, StateSets>( + &'a self, + room_version_rules: &'a RoomVersionRules, + state_sets: StateSets, + auth_chain_sets: &'a [HashSet], + ) -> Result> + where + StateSets: Iterator> + Clone + Send, + { + let event_fetch = |event_id| self.event_fetch(event_id); + let event_exists = |event_id| self.event_exists(event_id); + state_res::resolve( + room_version_rules, + state_sets, + auth_chain_sets, + &event_fetch, + &event_exists, + ) + .map_err(|e| err!(error!("State resolution failed: {e:?}"))) + .await + } } diff --git a/src/service/rooms/event_handler/state_at_incoming.rs b/src/service/rooms/event_handler/state_at_incoming.rs index 9f11ac4b8..3dc7fc9f3 100644 --- a/src/service/rooms/event_handler/state_at_incoming.rs +++ b/src/service/rooms/event_handler/state_at_incoming.rs @@ -5,7 +5,7 @@ use std::{ }; use conduwuit::{ - Result, debug, err, error, implement, + Result, debug, err, error, matrix::{Event, StateMap}, trace, utils::stream::{BroadbandExt, IterStream, ReadyExt, TryBroadbandExt, TryWidebandExt}, @@ -15,180 +15,181 @@ use ruma::{OwnedEventId, RoomId, room_version_rules::RoomVersionRules}; use crate::rooms::short::ShortStateHash; -// TODO: if we know the prev_events of the incoming event we can avoid the -#[implement(super::Service)] -// request and build the state from a known point and resolve if > 1 prev_event -#[tracing::instrument(name = "state", level = "debug", skip_all)] -pub(super) async fn state_at_incoming_degree_one( - &self, - incoming_pdu: &Pdu, -) -> Result>> -where - Pdu: Event + Send + Sync, -{ - let prev_event = incoming_pdu - .prev_events() - .next() - .expect("at least one prev_event"); +impl super::Service { + // TODO: if we know the prev_events of the incoming event we can avoid the + // request and build the state from a known point and resolve if > 1 prev_event + #[tracing::instrument(name = "state", level = "debug", skip_all)] + pub(super) async fn state_at_incoming_degree_one( + &self, + incoming_pdu: &Pdu, + ) -> Result>> + where + Pdu: Event + Send + Sync, + { + let prev_event = incoming_pdu + .prev_events() + .next() + .expect("at least one prev_event"); - let Ok(prev_event_sstatehash) = self - .services - .state_accessor - .pdu_shortstatehash(prev_event) - .await - else { - return Ok(None); - }; - - let mut state: HashMap<_, _> = self - .services - .state_accessor - .state_full_ids(prev_event_sstatehash) - .collect() - .await; - - debug!("Using cached state"); - let prev_pdu = self - .services - .timeline - .get_pdu(prev_event) - .await - .map_err(|e| err!(Database("Could not find prev event, but we know the state: {e:?}")))?; - - if let Some(state_key) = &prev_pdu.state_key { - let shortstatekey = self + let Ok(prev_event_sstatehash) = self .services - .short - .get_or_create_shortstatekey(&prev_pdu.kind.to_string().into(), state_key) + .state_accessor + .pdu_shortstatehash(prev_event) + .await + else { + return Ok(None); + }; + + let mut state: HashMap<_, _> = self + .services + .state_accessor + .state_full_ids(prev_event_sstatehash) + .collect() .await; - state.insert(shortstatekey, prev_event.to_owned()); - // Now it's the state after the pdu + debug!("Using cached state"); + let prev_pdu = self + .services + .timeline + .get_pdu(prev_event) + .await + .map_err(|e| { + err!(Database("Could not find prev event, but we know the state: {e:?}")) + })?; + + if let Some(state_key) = &prev_pdu.state_key { + let shortstatekey = self + .services + .short + .get_or_create_shortstatekey(&prev_pdu.kind.to_string().into(), state_key) + .await; + + state.insert(shortstatekey, prev_event.to_owned()); + // Now it's the state after the pdu + } + + debug_assert!(!state.is_empty(), "should be returning None for empty HashMap result"); + + Ok(Some(state)) } - debug_assert!(!state.is_empty(), "should be returning None for empty HashMap result"); - - Ok(Some(state)) -} - -#[implement(super::Service)] -#[tracing::instrument(name = "state", level = "debug", skip_all)] -pub(super) async fn state_at_incoming_resolved( - &self, - incoming_pdu: &Pdu, - room_id: &RoomId, - room_version_rules: &RoomVersionRules, -) -> Result>> -where - Pdu: Event + Send + Sync, -{ - trace!("Calculating extremity statehashes..."); - let Ok(extremity_sstatehashes) = incoming_pdu - .prev_events() - .try_stream() - .broad_and_then(|prev_eventid| { - self.services - .timeline - .get_pdu(prev_eventid) - .map_ok(move |prev_event| (prev_eventid, prev_event)) - }) - .broad_and_then(|(prev_eventid, prev_event)| { - self.services - .state_accessor - .pdu_shortstatehash(prev_eventid) - .map_ok(move |sstatehash| (sstatehash, prev_event)) - }) - .try_collect::>() - .await - else { - return Ok(None); - }; - - trace!("Calculating fork states..."); - let (fork_states, auth_chain_sets): (Vec>, Vec>) = - extremity_sstatehashes - .into_iter() + #[tracing::instrument(name = "state", level = "debug", skip_all)] + pub(super) async fn state_at_incoming_resolved( + &self, + incoming_pdu: &Pdu, + room_id: &RoomId, + room_version_rules: &RoomVersionRules, + ) -> Result>> + where + Pdu: Event + Send + Sync, + { + trace!("Calculating extremity statehashes..."); + let Ok(extremity_sstatehashes) = incoming_pdu + .prev_events() .try_stream() - .wide_and_then(|(sstatehash, prev_event)| { - self.state_at_incoming_fork(room_id, sstatehash, prev_event) + .broad_and_then(|prev_eventid| { + self.services + .timeline + .get_pdu(prev_eventid) + .map_ok(move |prev_event| (prev_eventid, prev_event)) }) - .try_collect() - .map_ok(Vec::into_iter) - .map_ok(Iterator::unzip) - .await?; + .broad_and_then(|(prev_eventid, prev_event)| { + self.services + .state_accessor + .pdu_shortstatehash(prev_eventid) + .map_ok(move |sstatehash| (sstatehash, prev_event)) + }) + .try_collect::>() + .await + else { + return Ok(None); + }; - let Ok(new_state) = self - .state_resolution(room_version_rules, fork_states.iter(), &auth_chain_sets) - .boxed() - .await - .inspect_err(|e| error!("State resolution failed: {e:?}")) - else { - return Ok(None); - }; + trace!("Calculating fork states..."); + let (fork_states, auth_chain_sets): (Vec>, Vec>) = + extremity_sstatehashes + .into_iter() + .try_stream() + .wide_and_then(|(sstatehash, prev_event)| { + self.state_at_incoming_fork(room_id, sstatehash, prev_event) + }) + .try_collect() + .map_ok(Vec::into_iter) + .map_ok(Iterator::unzip) + .await?; - new_state - .into_iter() - .stream() - .broad_then(|((event_type, state_key), event_id)| async move { - self.services - .short - .get_or_create_shortstatekey(&event_type, &state_key) - .map(move |shortstatekey| (shortstatekey, event_id)) - .await - }) - .collect() - .map(Some) - .map(Ok) - .await -} + let Ok(new_state) = self + .state_resolution(room_version_rules, fork_states.iter(), &auth_chain_sets) + .boxed() + .await + .inspect_err(|e| error!("State resolution failed: {e:?}")) + else { + return Ok(None); + }; -#[implement(super::Service)] -async fn state_at_incoming_fork( - &self, - room_id: &RoomId, - sstatehash: ShortStateHash, - prev_event: Pdu, -) -> Result<(StateMap, HashSet)> -where - Pdu: Event, -{ - let mut leaf_state: HashMap<_, _> = self - .services - .state_accessor - .state_full_ids(sstatehash) - .collect() - .await; - - if let Some(state_key) = prev_event.state_key() { - let shortstatekey = self - .services - .short - .get_or_create_shortstatekey(&prev_event.kind().to_string().into(), state_key) - .await; - - let event_id = prev_event.event_id(); - leaf_state.insert(shortstatekey, event_id.to_owned()); - // Now it's the state after the pdu + new_state + .into_iter() + .stream() + .broad_then(|((event_type, state_key), event_id)| async move { + self.services + .short + .get_or_create_shortstatekey(&event_type, &state_key) + .map(move |shortstatekey| (shortstatekey, event_id)) + .await + }) + .collect() + .map(Some) + .map(Ok) + .await } - let auth_chain = self - .services - .auth_chain - .event_ids_iter(room_id, leaf_state.values().map(Borrow::borrow)) - .try_collect(); + async fn state_at_incoming_fork( + &self, + room_id: &RoomId, + sstatehash: ShortStateHash, + prev_event: Pdu, + ) -> Result<(StateMap, HashSet)> + where + Pdu: Event, + { + let mut leaf_state: HashMap<_, _> = self + .services + .state_accessor + .state_full_ids(sstatehash) + .collect() + .await; - let fork_state = leaf_state - .iter() - .stream() - .broad_then(|(k, id)| { - self.services + if let Some(state_key) = prev_event.state_key() { + let shortstatekey = self + .services .short - .get_statekey_from_short(*k) - .map_ok(|(ty, sk)| ((ty, sk), id.clone())) - }) - .ready_filter_map(Result::ok) - .collect() - .map(Ok); + .get_or_create_shortstatekey(&prev_event.kind().to_string().into(), state_key) + .await; - try_join(fork_state, auth_chain).await + let event_id = prev_event.event_id(); + leaf_state.insert(shortstatekey, event_id.to_owned()); + // Now it's the state after the pdu + } + + let auth_chain = self + .services + .auth_chain + .event_ids_iter(room_id, leaf_state.values().map(Borrow::borrow)) + .try_collect(); + + let fork_state = leaf_state + .iter() + .stream() + .broad_then(|(k, id)| { + self.services + .short + .get_statekey_from_short(*k) + .map_ok(|(ty, sk)| ((ty, sk), id.clone())) + }) + .ready_filter_map(Result::ok) + .collect() + .map(Ok); + + try_join(fork_state, auth_chain).await + } } diff --git a/src/service/rooms/event_handler/upgrade_outlier_pdu.rs b/src/service/rooms/event_handler/upgrade_outlier_pdu.rs index 0a711b468..523dde46b 100644 --- a/src/service/rooms/event_handler/upgrade_outlier_pdu.rs +++ b/src/service/rooms/event_handler/upgrade_outlier_pdu.rs @@ -1,7 +1,7 @@ use std::{borrow::Borrow, sync::Arc, time::Instant}; use conduwuit::{ - Err, Result, debug, debug_info, err, implement, info, is_equal_to, + Err, Result, debug, debug_info, err, info, is_equal_to, matrix::{Event, EventTypeExt, PduEvent, StateKey, state_res}, trace, utils::{ @@ -22,366 +22,370 @@ use crate::rooms::{ timeline::RawPduId, }; -#[implement(super::Service)] -pub(super) async fn upgrade_outlier_to_timeline_pdu( - &self, - incoming_pdu: PduEvent, - mut val: CanonicalJsonObject, - create_event: &Pdu, - origin: &ServerName, - room_id: &RoomId, -) -> Result> -where - Pdu: Event + Send + Sync, -{ - // Skip the PDU if we already have it as a timeline event - if let Ok(pduid) = self - .services - .timeline - .get_pdu_id(incoming_pdu.event_id()) - .await +impl super::Service { + pub(super) async fn upgrade_outlier_to_timeline_pdu( + &self, + incoming_pdu: PduEvent, + mut val: CanonicalJsonObject, + create_event: &Pdu, + origin: &ServerName, + room_id: &RoomId, + ) -> Result> + where + Pdu: Event + Send + Sync, { - return Ok(Some(pduid)); - } - - let (rejected, soft_failed) = join!( - self.services - .pdu_metadata - .is_event_rejected(incoming_pdu.event_id()), - self.services - .pdu_metadata - .is_event_soft_failed(incoming_pdu.event_id()) - ); - if rejected { - return Err!(Request(InvalidParam("Event has been rejected"))); - } else if soft_failed { - return Err!(Request(InvalidParam("Event has been soft-failed"))); - } - - debug!( - event_id = %incoming_pdu.event_id, - "Upgrading PDU from outlier to timeline" - ); - let timer = Instant::now(); - let room_version_rules = get_room_version_rules(create_event)?; - - // 10. Fetch missing state and auth chain events by calling /state_ids at - // backwards extremities doing all the checks in this list starting at 1. - // These are not timeline events. - - debug!( - event_id = %incoming_pdu.event_id, - "Resolving state at event" - ); - let mut state_at_incoming_event = if incoming_pdu.prev_events().count() == 1 { - self.state_at_incoming_degree_one(&incoming_pdu).await? - } else { - self.state_at_incoming_resolved(&incoming_pdu, room_id, &room_version_rules) - .await? - }; - - if state_at_incoming_event.is_none() { - state_at_incoming_event = self - .fetch_state(origin, create_event, room_id, incoming_pdu.event_id()) - .await?; - } - - let state_at_incoming_event = - state_at_incoming_event.expect("we always set this to some above"); - - debug!( - event_id = %incoming_pdu.event_id, - "Performing auth check to upgrade" - ); - // 11. Check the auth of the event passes based on the state of the event - let state_fetch_state = &state_at_incoming_event; - let state_fetch = |k: StateEventType, s: StateKey| async move { - let shortstatekey = self.services.short.get_shortstatekey(&k, &s).await.ok()?; - - let event_id = state_fetch_state.get(&shortstatekey)?; - self.services.timeline.get_pdu(event_id).await.ok() - }; - - debug!( - event_id = %incoming_pdu.event_id, - "Running initial auth check" - ); - // PDU check: 5 - let auth_check = state_res::event_auth::auth_check( - &room_version_rules, - &incoming_pdu, - None, // TODO: third party invite - |ty, sk| state_fetch(ty.clone(), sk.into()), - create_event.as_pdu(), - ) - .await - .map_err(|e| err!(Request(Forbidden("Auth check failed: {e:?}"))))?; - - if !auth_check { - self.services - .pdu_metadata - .mark_event_rejected(incoming_pdu.event_id()); - return Err!(Request(Forbidden( - "Event authorisation fails based on the state before the event" - ))); - } - - debug!( - event_id = %incoming_pdu.event_id, - "Gathering auth events" - ); - let auth_events = self - .services - .state - .get_auth_events( - room_id, - incoming_pdu.kind(), - incoming_pdu.sender(), - incoming_pdu.state_key(), - incoming_pdu.content(), - &room_version_rules, - ) - .await?; - - let state_fetch = |k: &StateEventType, s: &str| { - let key = k.with_state_key(s); - ready(auth_events.get(&key).map(ToOwned::to_owned)) - }; - - debug!( - event_id = %incoming_pdu.event_id, - "Running auth check with claimed state auth" - ); - // PDU check: 6 - let auth_check = state_res::event_auth::auth_check( - &room_version_rules, - &incoming_pdu, - None, // third-party invite - state_fetch, - create_event.as_pdu(), - ) - .await - .map_err(|e| err!(Request(Forbidden("Auth check failed: {e:?}"))))?; - if !auth_check { - warn!( - event_id = %incoming_pdu.event_id, - "Event authorization fails based on the current state of the room" - ); - } - - // Soft fail check before doing state res - debug!( - event_id = %incoming_pdu.event_id, - "Performing soft-fail check" - ); - let mut soft_fail = match (auth_check, incoming_pdu.redacts_id(&room_version_rules)) { - | (false, _) => true, - | (true, None) => false, - | (true, Some(redact_id)) => { - if !self - .services - .state_accessor - .user_can_redact(&redact_id, incoming_pdu.sender(), room_id, true) - .await? - { - warn!(redacts = %redact_id, "User is not allowed to redact event"); - true - } else { - false - } - }, - }; - - // 13. Use state resolution to find new room state - // We start looking at current room state now, so lets lock the room - trace!( - room_id = %room_id, - "Locking the room" - ); - let state_lock = self.services.state.mutex.lock(room_id).await; - - let state_ids_compressed: Arc = self - .services - .state_compressor - .compress_state_events( - state_at_incoming_event - .iter() - .map(|(ssk, eid)| (ssk, eid.borrow())), - ) - .collect() - .map(Arc::new) - .await; - - if incoming_pdu.state_key().is_some() { - debug!("Event is a state-event. Deriving new room state"); - - // We also add state after incoming event to the fork states - let mut state_after = state_at_incoming_event.clone(); - if let Some(state_key) = incoming_pdu.state_key() { - let shortstatekey = self - .services - .short - .get_or_create_shortstatekey(&incoming_pdu.kind().to_string().into(), state_key) - .await; - - let event_id = incoming_pdu.event_id(); - state_after.insert(shortstatekey, event_id.to_owned()); - } - - let new_room_state = self - .resolve_state(room_id, &room_version_rules, state_after) - .await?; - - // Set the new room state to the resolved state - debug!("Forcing new room state"); - let HashSetCompressStateEvent { shortstatehash, added, removed } = self + // Skip the PDU if we already have it as a timeline event + if let Ok(pduid) = self .services - .state_compressor - .save_state(room_id, new_room_state) - .await?; - - self.services - .state - .force_state(room_id, shortstatehash, added, removed, &state_lock) - .await?; - } - - if !soft_fail { - // Don't call the below checks on events that have already soft-failed, there's - // no reason to re-calculate that. - // 14-pre. ask the policy server to sign the event, if possible - debug!(event_id = %incoming_pdu.event_id, "Checking policy server for event"); - let tmp_evt_id = val.remove("event_id"); - if let Err(e) = self - .policy_server_allows_event( - &incoming_pdu, - &mut val, - room_id, - &room_version_rules, - true, - ) + .timeline + .get_pdu_id(incoming_pdu.event_id()) .await { - if matches!(e.kind(), ErrorKind::Forbidden) { - info!( - event_id = %incoming_pdu.event_id, - error = %e, - "Event has been marked as spam by policy server: {}", - e.message(), - ); - soft_fail = true; - } else { - return Err(e); - } - } else { - debug!( - event_id = %incoming_pdu.event_id, - "Event has passed policy server check." - ); - } - if let Some(id) = tmp_evt_id { - val.insert("event_id".to_owned(), id); + return Ok(Some(pduid)); } - // Additionally, if this is a redaction for a soft-failed event, we soft-fail it - // also. - - // TODO: this is supposed to hide redactions from policy servers, however, for - // full efficacy it also needs to hide redactions for unknown events. This - // needs to be investigated at a later time. - if let Some(redact_id) = incoming_pdu.redacts_id(&room_version_rules) { - debug!( - redact_id = %redact_id, - "Checking if redaction is for a soft-failed event" - ); - if self - .services + let (rejected, soft_failed) = join!( + self.services .pdu_metadata - .is_event_soft_failed(&redact_id) + .is_event_rejected(incoming_pdu.event_id()), + self.services + .pdu_metadata + .is_event_soft_failed(incoming_pdu.event_id()) + ); + if rejected { + return Err!(Request(InvalidParam("Event has been rejected"))); + } else if soft_failed { + return Err!(Request(InvalidParam("Event has been soft-failed"))); + } + + debug!( + event_id = %incoming_pdu.event_id, + "Upgrading PDU from outlier to timeline" + ); + let timer = Instant::now(); + let room_version_rules = get_room_version_rules(create_event)?; + + // 10. Fetch missing state and auth chain events by calling /state_ids at + // backwards extremities doing all the checks in this list starting at 1. + // These are not timeline events. + + debug!( + event_id = %incoming_pdu.event_id, + "Resolving state at event" + ); + let mut state_at_incoming_event = if incoming_pdu.prev_events().count() == 1 { + self.state_at_incoming_degree_one(&incoming_pdu).await? + } else { + self.state_at_incoming_resolved(&incoming_pdu, room_id, &room_version_rules) + .await? + }; + + if state_at_incoming_event.is_none() { + state_at_incoming_event = self + .fetch_state(origin, create_event, room_id, incoming_pdu.event_id()) + .await?; + } + + let state_at_incoming_event = + state_at_incoming_event.expect("we always set this to some above"); + + debug!( + event_id = %incoming_pdu.event_id, + "Performing auth check to upgrade" + ); + // 11. Check the auth of the event passes based on the state of the event + let state_fetch_state = &state_at_incoming_event; + let state_fetch = |k: StateEventType, s: StateKey| async move { + let shortstatekey = self.services.short.get_shortstatekey(&k, &s).await.ok()?; + + let event_id = state_fetch_state.get(&shortstatekey)?; + self.services.timeline.get_pdu(event_id).await.ok() + }; + + debug!( + event_id = %incoming_pdu.event_id, + "Running initial auth check" + ); + // PDU check: 5 + let auth_check = state_res::event_auth::auth_check( + &room_version_rules, + &incoming_pdu, + None, // TODO: third party invite + |ty, sk| state_fetch(ty.clone(), sk.into()), + create_event.as_pdu(), + ) + .await + .map_err(|e| err!(Request(Forbidden("Auth check failed: {e:?}"))))?; + + if !auth_check { + self.services + .pdu_metadata + .mark_event_rejected(incoming_pdu.event_id()); + return Err!(Request(Forbidden( + "Event authorisation fails based on the state before the event" + ))); + } + + debug!( + event_id = %incoming_pdu.event_id, + "Gathering auth events" + ); + let auth_events = self + .services + .state + .get_auth_events( + room_id, + incoming_pdu.kind(), + incoming_pdu.sender(), + incoming_pdu.state_key(), + incoming_pdu.content(), + &room_version_rules, + ) + .await?; + + let state_fetch = |k: &StateEventType, s: &str| { + let key = k.with_state_key(s); + ready(auth_events.get(&key).map(ToOwned::to_owned)) + }; + + debug!( + event_id = %incoming_pdu.event_id, + "Running auth check with claimed state auth" + ); + // PDU check: 6 + let auth_check = state_res::event_auth::auth_check( + &room_version_rules, + &incoming_pdu, + None, // third-party invite + state_fetch, + create_event.as_pdu(), + ) + .await + .map_err(|e| err!(Request(Forbidden("Auth check failed: {e:?}"))))?; + if !auth_check { + warn!( + event_id = %incoming_pdu.event_id, + "Event authorization fails based on the current state of the room" + ); + } + + // Soft fail check before doing state res + debug!( + event_id = %incoming_pdu.event_id, + "Performing soft-fail check" + ); + let mut soft_fail = match (auth_check, incoming_pdu.redacts_id(&room_version_rules)) { + | (false, _) => true, + | (true, None) => false, + | (true, Some(redact_id)) => { + if !self + .services + .state_accessor + .user_can_redact(&redact_id, incoming_pdu.sender(), room_id, true) + .await? + { + warn!(redacts = %redact_id, "User is not allowed to redact event"); + true + } else { + false + } + }, + }; + + // 13. Use state resolution to find new room state + // We start looking at current room state now, so lets lock the room + trace!( + room_id = %room_id, + "Locking the room" + ); + let state_lock = self.services.state.mutex.lock(room_id).await; + + let state_ids_compressed: Arc = self + .services + .state_compressor + .compress_state_events( + state_at_incoming_event + .iter() + .map(|(ssk, eid)| (ssk, eid.borrow())), + ) + .collect() + .map(Arc::new) + .await; + + if incoming_pdu.state_key().is_some() { + debug!("Event is a state-event. Deriving new room state"); + + // We also add state after incoming event to the fork states + let mut state_after = state_at_incoming_event.clone(); + if let Some(state_key) = incoming_pdu.state_key() { + let shortstatekey = self + .services + .short + .get_or_create_shortstatekey( + &incoming_pdu.kind().to_string().into(), + state_key, + ) + .await; + + let event_id = incoming_pdu.event_id(); + state_after.insert(shortstatekey, event_id.to_owned()); + } + + let new_room_state = self + .resolve_state(room_id, &room_version_rules, state_after) + .await?; + + // Set the new room state to the resolved state + debug!("Forcing new room state"); + let HashSetCompressStateEvent { shortstatehash, added, removed } = self + .services + .state_compressor + .save_state(room_id, new_room_state) + .await?; + + self.services + .state + .force_state(room_id, shortstatehash, added, removed, &state_lock) + .await?; + } + + if !soft_fail { + // Don't call the below checks on events that have already soft-failed, there's + // no reason to re-calculate that. + // 14-pre. ask the policy server to sign the event, if possible + debug!(event_id = %incoming_pdu.event_id, "Checking policy server for event"); + let tmp_evt_id = val.remove("event_id"); + if let Err(e) = self + .policy_server_allows_event( + &incoming_pdu, + &mut val, + room_id, + &room_version_rules, + true, + ) .await { - info!( - redact_id = %redact_id, - "Redaction is for a soft-failed event" + if matches!(e.kind(), ErrorKind::Forbidden) { + info!( + event_id = %incoming_pdu.event_id, + error = %e, + "Event has been marked as spam by policy server: {}", + e.message(), + ); + soft_fail = true; + } else { + return Err(e); + } + } else { + debug!( + event_id = %incoming_pdu.event_id, + "Event has passed policy server check." ); - soft_fail = true; + } + if let Some(id) = tmp_evt_id { + val.insert("event_id".to_owned(), id); + } + + // Additionally, if this is a redaction for a soft-failed event, we soft-fail it + // also. + + // TODO: this is supposed to hide redactions from policy servers, however, for + // full efficacy it also needs to hide redactions for unknown events. This + // needs to be investigated at a later time. + if let Some(redact_id) = incoming_pdu.redacts_id(&room_version_rules) { + debug!( + redact_id = %redact_id, + "Checking if redaction is for a soft-failed event" + ); + if self + .services + .pdu_metadata + .is_event_soft_failed(&redact_id) + .await + { + info!( + redact_id = %redact_id, + "Redaction is for a soft-failed event" + ); + soft_fail = true; + } } } - } - trace!("Appending pdu to timeline"); - let mut extremities: Vec<_> = self - .services - .state - .get_forward_extremities(room_id) - .collect() - .await; - if !soft_fail { - // Per https://spec.matrix.org/unstable/server-server-api/#soft-failure, soft-failed events - // are not added as forward extremities. - - // Now we calculate the set of extremities this room has after the incoming - // event has been applied. We start with the previous extremities (aka leaves) - trace!("Calculating extremities"); - extremities = extremities - .into_iter() - .stream() - .ready_filter(|event_id| { - // Remove any that are referenced by this incoming event's prev_events - !incoming_pdu.prev_events().any(is_equal_to!(event_id)) - }) - .broad_filter_map(|event_id| async move { - // Only keep those extremities were not referenced yet - self.services - .pdu_metadata - .is_event_referenced(room_id, &event_id) - .await - .eq(&false) - .then_some(event_id) - }) - .collect::>() + trace!("Appending pdu to timeline"); + let mut extremities: Vec<_> = self + .services + .state + .get_forward_extremities(room_id) + .collect() .await; - extremities.push(incoming_pdu.event_id().to_owned()); - debug!( - "Retained {} extremities checked against {} prev_events", - extremities.len(), - incoming_pdu.prev_events().count() - ); - assert!(!extremities.is_empty(), "extremities must not empty"); + if !soft_fail { + // Per https://spec.matrix.org/unstable/server-server-api/#soft-failure, soft-failed events + // are not added as forward extremities. + + // Now we calculate the set of extremities this room has after the incoming + // event has been applied. We start with the previous extremities (aka leaves) + trace!("Calculating extremities"); + extremities = extremities + .into_iter() + .stream() + .ready_filter(|event_id| { + // Remove any that are referenced by this incoming event's prev_events + !incoming_pdu.prev_events().any(is_equal_to!(event_id)) + }) + .broad_filter_map(|event_id| async move { + // Only keep those extremities were not referenced yet + self.services + .pdu_metadata + .is_event_referenced(room_id, &event_id) + .await + .eq(&false) + .then_some(event_id) + }) + .collect::>() + .await; + extremities.push(incoming_pdu.event_id().to_owned()); + debug!( + "Retained {} extremities checked against {} prev_events", + extremities.len(), + incoming_pdu.prev_events().count() + ); + assert!(!extremities.is_empty(), "extremities must not empty"); + } + + let pdu_id = self + .services + .timeline + .append_incoming_pdu( + &incoming_pdu, + val, + extremities.iter().map(Borrow::borrow), + state_ids_compressed, + soft_fail, + &state_lock, + room_id, + ) + .await?; + if soft_fail { + self.services + .pdu_metadata + .mark_event_soft_failed(incoming_pdu.event_id()); + + info!( + elapsed = ?timer.elapsed(), + event_id = %incoming_pdu.event_id, + "Event was soft failed" + ); + } else { + debug_info!( + elapsed = ?timer.elapsed(), + "Accepted", + ); + } + + // Event has passed all auth/stateres checks + drop(state_lock); + + Ok(pdu_id) } - - let pdu_id = self - .services - .timeline - .append_incoming_pdu( - &incoming_pdu, - val, - extremities.iter().map(Borrow::borrow), - state_ids_compressed, - soft_fail, - &state_lock, - room_id, - ) - .await?; - if soft_fail { - self.services - .pdu_metadata - .mark_event_soft_failed(incoming_pdu.event_id()); - - info!( - elapsed = ?timer.elapsed(), - event_id = %incoming_pdu.event_id, - "Event was soft failed" - ); - } else { - debug_info!( - elapsed = ?timer.elapsed(), - "Accepted", - ); - } - - // Event has passed all auth/stateres checks - drop(state_lock); - - Ok(pdu_id) } From f9eed42cb77a750c3aff4cdf3a1dc7f749b56675 Mon Sep 17 00:00:00 2001 From: timedout Date: Fri, 26 Jun 2026 04:58:52 +0100 Subject: [PATCH 43/70] style: Document & refactor rooms/directory service --- src/service/rooms/directory/mod.rs | 41 ++++++++++++++++-------------- 1 file changed, 22 insertions(+), 19 deletions(-) diff --git a/src/service/rooms/directory/mod.rs b/src/service/rooms/directory/mod.rs index 775764fa1..c1e7650c3 100644 --- a/src/service/rooms/directory/mod.rs +++ b/src/service/rooms/directory/mod.rs @@ -1,6 +1,6 @@ use std::sync::Arc; -use conduwuit::{Result, implement, utils::stream::TryIgnore}; +use conduwuit::{Result, utils::stream::TryIgnore}; use database::Map; use futures::Stream; use ruma::{OwnedRoomId, RoomId, api::client::room::Visibility}; @@ -25,27 +25,30 @@ impl crate::Service for Service { fn name(&self) -> &str { crate::service::make_name(std::module_path!()) } } -#[implement(Service)] -pub fn set_public(&self, room_id: &RoomId) { self.db.publicroomids.insert(room_id, []); } +impl Service { + /// Marks a room as public, adding it to the room directory. Has no relation + /// to the join rule. + pub fn set_public(&self, room_id: &RoomId) { self.db.publicroomids.insert(room_id, []); } -#[implement(Service)] -pub fn set_not_public(&self, room_id: &RoomId) { self.db.publicroomids.remove(room_id); } + /// Removes a room from the public room directory. + pub fn set_not_public(&self, room_id: &RoomId) { self.db.publicroomids.remove(room_id); } -#[implement(Service)] -pub fn public_rooms(&self) -> impl Stream + Send { - self.db.publicroomids.keys().ignore_err() -} + /// Lists all public rooms in the directory. + pub fn public_rooms(&self) -> impl Stream + Send { + self.db.publicroomids.keys().ignore_err() + } -#[implement(Service)] -pub async fn is_public_room(&self, room_id: &RoomId) -> bool { - self.visibility(room_id).await == Visibility::Public -} + /// Checks if a room is public (per visibility, not join rule). + pub async fn is_public_room(&self, room_id: &RoomId) -> bool { + self.visibility(room_id).await == Visibility::Public + } -#[implement(Service)] -pub async fn visibility(&self, room_id: &RoomId) -> Visibility { - if self.db.publicroomids.get(room_id).await.is_ok() { - Visibility::Public - } else { - Visibility::Private + /// Fetches the visibility of a specific room ID. + pub async fn visibility(&self, room_id: &RoomId) -> Visibility { + if self.db.publicroomids.get(room_id).await.is_ok() { + Visibility::Public + } else { + Visibility::Private + } } } From 10b5ca8e3a78f273c1e0beae024e5ef73d8b6651 Mon Sep 17 00:00:00 2001 From: timedout Date: Fri, 26 Jun 2026 05:05:42 +0100 Subject: [PATCH 44/70] style: Document & refactor rooms/directory service --- src/service/rooms/auth_chain/mod.rs | 399 ++++++++++++++-------------- 1 file changed, 195 insertions(+), 204 deletions(-) diff --git a/src/service/rooms/auth_chain/mod.rs b/src/service/rooms/auth_chain/mod.rs index 936d1193c..fc68d2c9d 100644 --- a/src/service/rooms/auth_chain/mod.rs +++ b/src/service/rooms/auth_chain/mod.rs @@ -8,12 +8,12 @@ use std::{ }; use conduwuit::{ - Err, Result, at, debug, debug_error, implement, trace, + Err, Result, at, debug, debug_error, trace, utils::{ IterStream, stream::{ReadyExt, TryBroadbandExt}, }, - validated, warn, + validated, }; use futures::{FutureExt, Stream, StreamExt, TryFutureExt, TryStreamExt}; use ruma::{EventId, OwnedEventId, RoomId}; @@ -47,213 +47,204 @@ impl crate::Service for Service { fn name(&self) -> &str { crate::service::make_name(std::module_path!()) } } -#[implement(Service)] -pub fn event_ids_iter<'a, I>( - &'a self, - room_id: &'a RoomId, - starting_events: I, -) -> impl Stream> + Send + 'a -where - I: Iterator + Clone + Debug + ExactSizeIterator + Send + 'a, -{ - self.get_auth_chain(room_id, starting_events) - .map_ok(|chain| { - self.services - .short - .multi_get_eventid_from_short(chain.into_iter().stream()) - .ready_filter(Result::is_ok) - }) - .try_flatten_stream() -} - -#[implement(Service)] -#[tracing::instrument(name = "auth_chain", level = "debug", skip_all)] -pub async fn get_auth_chain<'a, I>( - &'a self, - room_id: &RoomId, - starting_events: I, -) -> Result> -where - I: Iterator + Clone + Debug + ExactSizeIterator + Send + 'a, -{ - const NUM_BUCKETS: usize = 50; //TODO: change possible w/o disrupting db? - const BUCKET: Bucket<'_> = BTreeSet::new(); - - let started = Instant::now(); - let mut starting_ids = self - .services - .short - .multi_get_or_create_shorteventid(starting_events.clone()) - .zip(starting_events.clone().stream()) - .boxed(); - - let mut buckets = [BUCKET; NUM_BUCKETS]; - while let Some((short, starting_event)) = starting_ids.next().await { - let bucket: usize = short.try_into()?; - let bucket: usize = validated!(bucket % NUM_BUCKETS); - buckets[bucket].insert((short, starting_event)); +impl Service { + pub fn event_ids_iter<'a, I>( + &'a self, + room_id: &'a RoomId, + starting_events: I, + ) -> impl Stream> + Send + 'a + where + I: Iterator + Clone + Debug + ExactSizeIterator + Send + 'a, + { + self.get_auth_chain(room_id, starting_events) + .map_ok(|chain| { + self.services + .short + .multi_get_eventid_from_short(chain.into_iter().stream()) + .ready_filter(Result::is_ok) + }) + .try_flatten_stream() } - debug!( - starting_events = ?starting_events.count(), - elapsed = ?started.elapsed(), - "start", - ); + #[tracing::instrument(name = "auth_chain", skip_all)] + pub async fn get_auth_chain<'a, I>( + &'a self, + room_id: &RoomId, + starting_events: I, + ) -> Result> + where + I: Iterator + Clone + Debug + ExactSizeIterator + Send + 'a, + { + const NUM_BUCKETS: usize = 50; //TODO: change possible w/o disrupting db? + const BUCKET: Bucket<'_> = BTreeSet::new(); - let full_auth_chain: Vec = buckets - .into_iter() - .try_stream() - .broad_and_then(|chunk| self.get_auth_chain_outer(room_id, started, chunk)) - .try_collect() - .map_ok(|auth_chain: Vec<_>| auth_chain.into_iter().flatten().collect()) - .map_ok(|mut full_auth_chain: Vec<_>| { - full_auth_chain.sort_unstable(); - full_auth_chain.dedup(); - full_auth_chain - }) - .boxed() - .await?; + let started = Instant::now(); + let mut starting_ids = self + .services + .short + .multi_get_or_create_shorteventid(starting_events.clone()) + .zip(starting_events.clone().stream()) + .boxed(); - debug!( - chain_length = ?full_auth_chain.len(), - elapsed = ?started.elapsed(), - "done", - ); - - Ok(full_auth_chain) -} - -#[implement(Service)] -async fn get_auth_chain_outer( - &self, - room_id: &RoomId, - started: Instant, - chunk: Bucket<'_>, -) -> Result> { - let chunk_key: Vec = chunk.iter().map(at!(0)).collect(); - - if chunk_key.is_empty() { - return Ok(Vec::new()); - } - - if let Ok(cached) = self.get_cached_eventid_authchain(&chunk_key).await { - return Ok(cached.to_vec()); - } - - let chunk_cache: Vec<_> = chunk - .into_iter() - .try_stream::() - .broad_and_then(|(shortid, event_id)| async move { - if let Ok(cached) = self.get_cached_eventid_authchain(&[shortid]).await { - return Ok(cached.to_vec()); - } - - let auth_chain = self.get_auth_chain_inner(room_id, event_id).await?; - self.cache_auth_chain_vec(vec![shortid], auth_chain.as_slice()); - debug!( - %event_id, - elapsed = ?started.elapsed(), - "Cache missed event" - ); - - Ok(auth_chain) - }) - .try_collect() - .map_ok(|chunk_cache: Vec<_>| chunk_cache.into_iter().flatten().collect()) - .map_ok(|mut chunk_cache: Vec<_>| { - chunk_cache.sort_unstable(); - chunk_cache.dedup(); - chunk_cache - }) - .await?; - - self.cache_auth_chain_vec(chunk_key, chunk_cache.as_slice()); - debug!( - chunk_cache_length = ?chunk_cache.len(), - elapsed = ?started.elapsed(), - "Cache missed chunk", - ); - - Ok(chunk_cache) -} - -#[implement(Service)] -#[tracing::instrument(name = "inner", level = "trace", skip(self, room_id))] -async fn get_auth_chain_inner( - &self, - room_id: &RoomId, - event_id: &EventId, -) -> Result> { - let mut todo: VecDeque<_> = [event_id.to_owned()].into(); - let mut found = HashSet::new(); - - while let Some(event_id) = todo.pop_front() { - trace!(%event_id, "processing auth event"); - - match self.services.timeline.get_pdu(&event_id).await { - | Err(e) => { - debug_error!(%event_id, ?e, "Could not find pdu mentioned in auth events"); - }, - | Ok(pdu) => { - if let Some(claimed_room_id) = pdu.room_id.clone() { - if claimed_room_id != *room_id { - return Err!(Request(Forbidden(error!( - %event_id, - %room_id, - wrong_room_id = ?pdu.room_id.unwrap(), - "auth event for incorrect room" - )))); - } - } - - for auth_event in &pdu.auth_events { - let sauthevent = self - .services - .short - .get_or_create_shorteventid(auth_event) - .await; - - if found.insert(sauthevent) { - trace!(%event_id, ?auth_event, "adding auth event to processing queue"); - - todo.push_back(auth_event.clone()); - } - } - }, + let mut buckets = [BUCKET; NUM_BUCKETS]; + while let Some((short, starting_event)) = starting_ids.next().await { + let bucket: usize = short.try_into()?; + let bucket: usize = validated!(bucket % NUM_BUCKETS); + buckets[bucket].insert((short, starting_event)); } + + trace!( + starting_events = ?starting_events.count(), + elapsed = ?started.elapsed(), + "start", + ); + + let full_auth_chain: Vec = buckets + .into_iter() + .try_stream() + .broad_and_then(|chunk| self.get_auth_chain_outer(room_id, started, chunk)) + .try_collect() + .map_ok(|auth_chain: Vec<_>| auth_chain.into_iter().flatten().collect()) + .map_ok(|mut full_auth_chain: Vec<_>| { + full_auth_chain.sort_unstable(); + full_auth_chain.dedup(); + full_auth_chain + }) + .boxed() + .await?; + + trace!( + chain_length = ?full_auth_chain.len(), + elapsed = ?started.elapsed(), + "done", + ); + + Ok(full_auth_chain) } - Ok(found.into_iter().collect()) + async fn get_auth_chain_outer( + &self, + room_id: &RoomId, + started: Instant, + chunk: Bucket<'_>, + ) -> Result> { + let chunk_key: Vec = chunk.iter().map(at!(0)).collect(); + + if chunk_key.is_empty() { + return Ok(Vec::new()); + } + + if let Ok(cached) = self.get_cached_eventid_authchain(&chunk_key).await { + return Ok(cached.to_vec()); + } + + let chunk_cache: Vec<_> = chunk + .into_iter() + .try_stream::() + .broad_and_then(|(shortid, event_id)| async move { + if let Ok(cached) = self.get_cached_eventid_authchain(&[shortid]).await { + return Ok(cached.to_vec()); + } + + let auth_chain = self.get_auth_chain_inner(room_id, event_id).await?; + self.cache_auth_chain_vec(vec![shortid], auth_chain.as_slice()); + debug!( + %event_id, + elapsed = ?started.elapsed(), + "Cache missed event" + ); + + Ok(auth_chain) + }) + .try_collect() + .map_ok(|chunk_cache: Vec<_>| chunk_cache.into_iter().flatten().collect()) + .map_ok(|mut chunk_cache: Vec<_>| { + chunk_cache.sort_unstable(); + chunk_cache.dedup(); + chunk_cache + }) + .await?; + + self.cache_auth_chain_vec(chunk_key, chunk_cache.as_slice()); + debug!( + chunk_cache_length = ?chunk_cache.len(), + elapsed = ?started.elapsed(), + "Cache missed chunk", + ); + + Ok(chunk_cache) + } + + #[tracing::instrument(name = "inner", level = "trace", skip(self, room_id))] + async fn get_auth_chain_inner( + &self, + room_id: &RoomId, + event_id: &EventId, + ) -> Result> { + let mut todo: VecDeque<_> = [event_id.to_owned()].into(); + let mut found = HashSet::new(); + + while let Some(event_id) = todo.pop_front() { + trace!(%event_id, "processing auth event"); + + match self.services.timeline.get_pdu(&event_id).await { + | Err(e) => { + debug_error!(%event_id, ?e, "Could not find pdu mentioned in auth events"); + }, + | Ok(pdu) => { + if let Some(claimed_room_id) = pdu.room_id.clone() { + if claimed_room_id != *room_id { + return Err!(Request(Forbidden(error!( + %event_id, + %room_id, + wrong_room_id = ?pdu.room_id.unwrap(), + "auth event for incorrect room" + )))); + } + } + + for auth_event in &pdu.auth_events { + let sauthevent = self + .services + .short + .get_or_create_shorteventid(auth_event) + .await; + + if found.insert(sauthevent) { + trace!(%event_id, ?auth_event, "adding auth event to processing queue"); + + todo.push_back(auth_event.clone()); + } + } + }, + } + } + + Ok(found.into_iter().collect()) + } + + #[inline] + pub async fn get_cached_eventid_authchain(&self, key: &[u64]) -> Result> { + self.db.get_cached_eventid_authchain(key).await + } + + pub fn cache_auth_chain(&self, key: Vec, auth_chain: &HashSet) { + let val: Arc<[ShortEventId]> = auth_chain.iter().copied().collect(); + + self.db.cache_auth_chain(key, val); + } + + pub fn cache_auth_chain_vec(&self, key: Vec, auth_chain: &[ShortEventId]) { + let val: Arc<[ShortEventId]> = auth_chain.iter().copied().collect(); + + self.db.cache_auth_chain(key, val); + } + + pub fn get_cache_usage(&self) -> (usize, usize) { + let cache = self.db.auth_chain_cache.lock(); + + (cache.len(), cache.capacity()) + } + + pub fn clear_cache(&self) { self.db.auth_chain_cache.lock().clear(); } } - -#[implement(Service)] -#[inline] -pub async fn get_cached_eventid_authchain(&self, key: &[u64]) -> Result> { - self.db.get_cached_eventid_authchain(key).await -} - -#[implement(Service)] -#[tracing::instrument(skip_all, level = "debug")] -pub fn cache_auth_chain(&self, key: Vec, auth_chain: &HashSet) { - let val: Arc<[ShortEventId]> = auth_chain.iter().copied().collect(); - - self.db.cache_auth_chain(key, val); -} - -#[implement(Service)] -#[tracing::instrument(skip_all, level = "debug")] -pub fn cache_auth_chain_vec(&self, key: Vec, auth_chain: &[ShortEventId]) { - let val: Arc<[ShortEventId]> = auth_chain.iter().copied().collect(); - - self.db.cache_auth_chain(key, val); -} - -#[implement(Service)] -pub fn get_cache_usage(&self) -> (usize, usize) { - let cache = self.db.auth_chain_cache.lock(); - - (cache.len(), cache.capacity()) -} - -#[implement(Service)] -pub fn clear_cache(&self) { self.db.auth_chain_cache.lock().clear(); } From 77dab8fd20d07ed6f3666e9a254b0702d9d68013 Mon Sep 17 00:00:00 2001 From: timedout Date: Fri, 26 Jun 2026 05:10:50 +0100 Subject: [PATCH 45/70] style: Document & refactor rooms/alias service --- src/service/rooms/alias/remote.rs | 72 ++++++++++++++++--------------- 1 file changed, 37 insertions(+), 35 deletions(-) diff --git a/src/service/rooms/alias/remote.rs b/src/service/rooms/alias/remote.rs index cc73990af..5ec464f7f 100644 --- a/src/service/rooms/alias/remote.rs +++ b/src/service/rooms/alias/remote.rs @@ -1,40 +1,42 @@ -use conduwuit::{Result, debug, error, implement}; +use conduwuit::{Result, debug, error}; use federation::query::get_room_information::v1::Response; use ruma::{OwnedRoomId, OwnedServerName, RoomAliasId, ServerName, api::federation}; -#[implement(super::Service)] -pub(super) async fn remote_resolve( - &self, - room_alias: &RoomAliasId, -) -> Result<(OwnedRoomId, Vec)> { - debug!("Asking {} to resolve {room_alias:?}", room_alias.server_name()); - match self - .remote_request(room_alias, room_alias.server_name()) - .await - { - | Err(e) => { - error!("Unable to resolve remote room alias {}: {e}", room_alias); - Err(e) - }, - | Ok(Response { room_id, servers, .. }) => { - debug!("Remote resolved {room_alias:?} to {room_id:?} with servers {servers:?}"); - Ok((room_id, servers)) - }, +impl super::Service { + /// Resolves the given room alias to a room ID and a list of servers that + /// are in the room by asking the remote server which owns it. + pub(super) async fn remote_resolve( + &self, + room_alias: &RoomAliasId, + ) -> Result<(OwnedRoomId, Vec)> { + debug!("Asking {} to resolve {room_alias:?}", room_alias.server_name()); + match self + .remote_request(room_alias, room_alias.server_name()) + .await + { + | Err(e) => { + error!("Unable to resolve remote room alias {}: {e}", room_alias); + Err(e) + }, + | Ok(Response { room_id, servers, .. }) => { + debug!("Remote resolved {room_alias:?} to {room_id:?} with servers {servers:?}"); + Ok((room_id, servers)) + }, + } + } + + async fn remote_request( + &self, + room_alias: &RoomAliasId, + server: &ServerName, + ) -> Result { + use federation::query::get_room_information::v1::Request; + + let request = Request::new(room_alias.to_owned()); + + self.services + .sending + .send_federation_request(server, request) + .await } } - -#[implement(super::Service)] -async fn remote_request( - &self, - room_alias: &RoomAliasId, - server: &ServerName, -) -> Result { - use federation::query::get_room_information::v1::Request; - - let request = Request::new(room_alias.to_owned()); - - self.services - .sending - .send_federation_request(server, request) - .await -} From 0c9230f35ece775a6a51046c2c7de7874989bd0f Mon Sep 17 00:00:00 2001 From: timedout Date: Fri, 26 Jun 2026 05:15:26 +0100 Subject: [PATCH 46/70] style: Document & refactor resolver service good riddance to this thing --- src/service/resolver/cache.rs | 117 +++++++++++++---------------- src/service/resolver/well_known.rs | 91 +++++++++++----------- 2 files changed, 97 insertions(+), 111 deletions(-) diff --git a/src/service/resolver/cache.rs b/src/service/resolver/cache.rs index 6bad6402d..b0268a3cd 100644 --- a/src/service/resolver/cache.rs +++ b/src/service/resolver/cache.rs @@ -3,7 +3,7 @@ use std::{net::IpAddr, sync::Arc, time::SystemTime}; use conduwuit::{ Result, arrayvec::ArrayVec, - at, err, implement, + at, err, utils::{math::Expected, rand, stream::TryIgnore}, }; use database::{Cbor, Deserialized, Map}; @@ -43,83 +43,70 @@ impl Cache { overrides: args.db["servername_override"].clone(), }) } -} -#[implement(Cache)] -pub async fn clear(&self) { join(self.clear_destinations(), self.clear_overrides()).await; } + pub async fn clear(&self) { join(self.clear_destinations(), self.clear_overrides()).await; } -#[implement(Cache)] -pub async fn clear_destinations(&self) { self.destinations.clear().await; } + pub async fn clear_destinations(&self) { self.destinations.clear().await; } -#[implement(Cache)] -pub async fn clear_overrides(&self) { self.overrides.clear().await; } + pub async fn clear_overrides(&self) { self.overrides.clear().await; } -#[implement(Cache)] -pub fn del_destination(&self, name: &ServerName) { self.destinations.remove(name); } + pub fn del_destination(&self, name: &ServerName) { self.destinations.remove(name); } -#[implement(Cache)] -pub fn del_override(&self, name: &ServerName) { self.overrides.remove(name); } + pub fn del_override(&self, name: &ServerName) { self.overrides.remove(name); } -#[implement(Cache)] -pub fn set_destination(&self, name: &ServerName, dest: &CachedDest) { - self.destinations.raw_put(name, Cbor(dest)); -} + pub fn set_destination(&self, name: &ServerName, dest: &CachedDest) { + self.destinations.raw_put(name, Cbor(dest)); + } -#[implement(Cache)] -pub fn set_override(&self, name: &str, over: &CachedOverride) { - self.overrides.raw_put(name, Cbor(over)); -} + pub fn set_override(&self, name: &str, over: &CachedOverride) { + self.overrides.raw_put(name, Cbor(over)); + } -#[implement(Cache)] -#[must_use] -pub async fn has_destination(&self, destination: &ServerName) -> bool { - self.get_destination(destination).await.is_ok() -} + #[must_use] + pub async fn has_destination(&self, destination: &ServerName) -> bool { + self.get_destination(destination).await.is_ok() + } -#[implement(Cache)] -#[must_use] -pub async fn has_override(&self, destination: &str) -> bool { - self.get_override(destination) - .await - .iter() - .any(CachedOverride::valid) -} + #[must_use] + pub async fn has_override(&self, destination: &str) -> bool { + self.get_override(destination) + .await + .iter() + .any(CachedOverride::valid) + } -#[implement(Cache)] -pub async fn get_destination(&self, name: &ServerName) -> Result { - self.destinations - .get(name) - .await - .deserialized::>() - .map(at!(0)) - .into_iter() - .find(CachedDest::valid) - .ok_or(err!(Request(NotFound("Expired from cache")))) -} + pub async fn get_destination(&self, name: &ServerName) -> Result { + self.destinations + .get(name) + .await + .deserialized::>() + .map(at!(0)) + .into_iter() + .find(CachedDest::valid) + .ok_or(err!(Request(NotFound("Expired from cache")))) + } -#[implement(Cache)] -pub async fn get_override(&self, name: &str) -> Result { - self.overrides - .get(name) - .await - .deserialized::>() - .map(at!(0)) -} + pub async fn get_override(&self, name: &str) -> Result { + self.overrides + .get(name) + .await + .deserialized::>() + .map(at!(0)) + } -#[implement(Cache)] -pub fn destinations(&self) -> impl Stream + Send + '_ { - self.destinations - .stream() - .ignore_err() - .map(|item: (OwnedServerName, Cbor<_>)| (item.0, item.1.0)) -} + pub fn destinations(&self) -> impl Stream + Send + '_ { + self.destinations + .stream() + .ignore_err() + .map(|item: (OwnedServerName, Cbor<_>)| (item.0, item.1.0)) + } -#[implement(Cache)] -pub fn overrides(&self) -> impl Stream + Send + '_ { - self.overrides - .stream() - .ignore_err() - .map(|item: (OwnedServerName, Cbor<_>)| (item.0, item.1.0)) + pub fn overrides(&self) -> impl Stream + Send + '_ { + self.overrides + .stream() + .ignore_err() + .map(|item: (OwnedServerName, Cbor<_>)| (item.0, item.1.0)) + } } impl CachedDest { diff --git a/src/service/resolver/well_known.rs b/src/service/resolver/well_known.rs index 237c5cd09..2a4ecf222 100644 --- a/src/service/resolver/well_known.rs +++ b/src/service/resolver/well_known.rs @@ -1,51 +1,50 @@ -use conduwuit::{ - Result, debug, debug_error, debug_info, implement, trace, utils::response::LimitReadExt, -}; +use conduwuit::{Result, debug, debug_error, debug_info, trace, utils::response::LimitReadExt}; use ruma::ServerName; -#[implement(super::Service)] -#[tracing::instrument(name = "well-known", level = "debug", skip(self, dest))] -pub(super) async fn request_well_known(&self, dest: &str) -> Result> { - trace!("Requesting well known for {dest}"); - let response = self - .services - .client - .well_known - .get(format!("https://{dest}/.well-known/matrix/server")) - .send() - .await; +impl super::Service { + #[tracing::instrument(name = "well-known", level = "debug", skip(self, dest))] + pub(super) async fn request_well_known(&self, dest: &str) -> Result> { + trace!("Requesting well known for {dest}"); + let response = self + .services + .client + .well_known + .get(format!("https://{dest}/.well-known/matrix/server")) + .send() + .await; - trace!("response: {response:?}"); - if let Err(e) = &response { - debug!("error: {e:?}"); - return Ok(None); + trace!("response: {response:?}"); + if let Err(e) = &response { + debug!("error: {e:?}"); + return Ok(None); + } + + let response = response?; + if !response.status().is_success() { + debug!("response not 2XX"); + return Ok(None); + } + + let Ok(text) = response.limit_read_text(8192).await else { + debug!("failed to read well-known response (too large or non-text content)"); + return Ok(None); + }; + trace!("response text: {text:?}"); + + let body: serde_json::Value = serde_json::from_str(&text).unwrap_or_default(); + + let m_server = body + .get("m.server") + .unwrap_or(&serde_json::Value::Null) + .as_str() + .unwrap_or_default(); + + if ServerName::parse(m_server).is_err() { + debug_error!("response content missing or invalid"); + return Ok(None); + } + + debug_info!("{dest:?} found at {m_server:?}"); + Ok(Some(m_server.to_owned())) } - - let response = response?; - if !response.status().is_success() { - debug!("response not 2XX"); - return Ok(None); - } - - let Ok(text) = response.limit_read_text(8192).await else { - debug!("failed to read well-known response (too large or non-text content)"); - return Ok(None); - }; - trace!("response text: {text:?}"); - - let body: serde_json::Value = serde_json::from_str(&text).unwrap_or_default(); - - let m_server = body - .get("m.server") - .unwrap_or(&serde_json::Value::Null) - .as_str() - .unwrap_or_default(); - - if ServerName::parse(m_server).is_err() { - debug_error!("response content missing or invalid"); - return Ok(None); - } - - debug_info!("{dest:?} found at {m_server:?}"); - Ok(Some(m_server.to_owned())) } From b1c5b3d0006c99b4b2147fc069324bfbbdae1098 Mon Sep 17 00:00:00 2001 From: timedout Date: Fri, 26 Jun 2026 05:20:30 +0100 Subject: [PATCH 47/70] style: Refactor media service --- src/service/media/preview.rs | 767 ++++++++++++++++---------------- src/service/media/remote.rs | 782 ++++++++++++++++----------------- src/service/media/thumbnail.rs | 131 +++--- 3 files changed, 832 insertions(+), 848 deletions(-) diff --git a/src/service/media/preview.rs b/src/service/media/preview.rs index f0270d3b3..21e096019 100644 --- a/src/service/media/preview.rs +++ b/src/service/media/preview.rs @@ -10,7 +10,6 @@ use std::time::SystemTime; #[cfg(feature = "url_preview")] use conduwuit::utils::response::LimitReadExt; use conduwuit::{Err, Result, debug, err}; -use conduwuit_core::implement; use ipaddress::IPAddress; #[cfg(feature = "url_preview")] use ruma::OwnedMxcUri; @@ -49,407 +48,407 @@ pub struct UrlPreviewData { pub audio_size: Option, } -#[implement(Service)] -pub async fn remove_url_preview(&self, url: &str) -> Result<()> { - // TODO: also remove the downloaded image - self.db.remove_url_preview(url) -} - -#[implement(Service)] -pub async fn clear_url_previews(&self) { self.db.clear_url_previews().await; } - -#[implement(Service)] -pub async fn set_url_preview(&self, url: &str, data: &UrlPreviewData) -> Result<()> { - let now = SystemTime::now() - .duration_since(SystemTime::UNIX_EPOCH) - .expect("valid system time"); - self.db.set_url_preview(url, data, now) -} - -#[implement(Service)] -pub async fn get_url_preview(&self, url: &Url) -> Result { - if let Ok(preview) = self.db.get_url_preview(url.as_str()).await { - return Ok(preview); +impl Service { + pub async fn remove_url_preview(&self, url: &str) -> Result<()> { + // TODO: also remove the downloaded image + self.db.remove_url_preview(url) } - // ensure that only one request is made per URL - let _request_lock = self.url_preview_mutex.lock(url.as_str()).await; + pub async fn clear_url_previews(&self) { self.db.clear_url_previews().await; } - match self.db.get_url_preview(url.as_str()).await { - | Ok(preview) => Ok(preview), - | Err(_) => self.request_url_preview(url).await, + pub async fn set_url_preview(&self, url: &str, data: &UrlPreviewData) -> Result<()> { + let now = SystemTime::now() + .duration_since(SystemTime::UNIX_EPOCH) + .expect("valid system time"); + self.db.set_url_preview(url, data, now) } -} -#[implement(Service)] -async fn request_url_preview(&self, url: &Url) -> Result { - if let Ok(ip) = IPAddress::parse(url.host_str().expect("URL previously validated")) { - if !self.services.client.valid_cidr_range(&ip) { - return Err!(Request(Forbidden("Requesting from this address is forbidden"))); + pub async fn get_url_preview(&self, url: &Url) -> Result { + if let Ok(preview) = self.db.get_url_preview(url.as_str()).await { + return Ok(preview); + } + + // ensure that only one request is made per URL + let _request_lock = self.url_preview_mutex.lock(url.as_str()).await; + + match self.db.get_url_preview(url.as_str()).await { + | Ok(preview) => Ok(preview), + | Err(_) => self.request_url_preview(url).await, } } - let client = &self.services.client.url_preview; - let response = client.head(url.as_str()).send().await?; - - debug!(%url, "URL preview response headers: {:?}", response.headers()); - - if let Some(remote_addr) = response.remote_addr() { - debug!(%url, "URL preview response remote address: {:?}", remote_addr); - - if let Ok(ip) = IPAddress::parse(remote_addr.ip().to_string()) { + async fn request_url_preview(&self, url: &Url) -> Result { + if let Ok(ip) = IPAddress::parse(url.host_str().expect("URL previously validated")) { if !self.services.client.valid_cidr_range(&ip) { return Err!(Request(Forbidden("Requesting from this address is forbidden"))); } } - } - let Some(content_type) = response.headers().get(reqwest::header::CONTENT_TYPE) else { - return Err!(Request(Unknown("Unknown or invalid Content-Type header"))); - }; + let client = &self.services.client.url_preview; + let response = client.head(url.as_str()).send().await?; - let content_type = content_type - .to_str() - .map_err(|e| err!(Request(Unknown("Unknown or invalid Content-Type header: {e}"))))?; + debug!(%url, "URL preview response headers: {:?}", response.headers()); - let data = match content_type { - | html if html.starts_with("text/html") => self.download_html(url.as_str()).await?, - | img if img.starts_with("image/") => self.download_image(url.as_str(), None).await?, - | video if video.starts_with("video/") => self.download_video(url.as_str(), None).await?, - | audio if audio.starts_with("audio/") => self.download_audio(url.as_str(), None).await?, - | _ => return Err!(Request(Unknown("Unsupported Content-Type"))), - }; + if let Some(remote_addr) = response.remote_addr() { + debug!(%url, "URL preview response remote address: {:?}", remote_addr); - self.set_url_preview(url.as_str(), &data).await?; - - Ok(data) -} - -#[cfg(feature = "url_preview")] -#[implement(Service)] -pub async fn download_image( - &self, - url: &str, - preview_data: Option, -) -> Result { - use conduwuit::utils::random_string; - use image::ImageReader; - - use crate::media::mxc::Mxc; - - let mut preview_data = preview_data.unwrap_or_default(); - - let image = self - .services - .client - .url_preview - .get(url) - .send() - .await? - .limit_read( - self.services - .server - .config - .max_request_size - .try_into() - .expect("u64 should fit in usize"), - ) - .await?; - - let mxc = Mxc { - server_name: self.services.globals.server_name(), - media_id: &random_string(super::MXC_LENGTH), - }; - - self.create(&mxc, None, None, None, &image).await?; - - preview_data.image = Some(mxc.to_string()); - if preview_data.image_height.is_none() || preview_data.image_width.is_none() { - let cursor = std::io::Cursor::new(&image); - let (width, height) = match ImageReader::new(cursor).with_guessed_format() { - | Err(_) => (None, None), - | Ok(reader) => match reader.into_dimensions() { - | Err(_) => (None, None), - | Ok((width, height)) => (Some(width), Some(height)), - }, - }; - - preview_data.image_width = width; - preview_data.image_height = height; - } - - Ok(preview_data) -} - -#[cfg(feature = "url_preview")] -#[implement(Service)] -pub async fn download_video( - &self, - url: &str, - preview_data: Option, -) -> Result { - let mut preview_data = preview_data.unwrap_or_default(); - - if self.services.globals.url_preview_allow_audio_video() { - let (url, size) = self.download_media(url).await?; - preview_data.video = Some(url.to_string()); - preview_data.video_size = Some(size); - } - - Ok(preview_data) -} - -#[cfg(feature = "url_preview")] -#[implement(Service)] -pub async fn download_audio( - &self, - url: &str, - preview_data: Option, -) -> Result { - let mut preview_data = preview_data.unwrap_or_default(); - - if self.services.globals.url_preview_allow_audio_video() { - let (url, size) = self.download_media(url).await?; - preview_data.audio = Some(url.to_string()); - preview_data.audio_size = Some(size); - } - - Ok(preview_data) -} - -#[cfg(feature = "url_preview")] -#[implement(Service)] -pub async fn download_media(&self, url: &str) -> Result<(OwnedMxcUri, usize)> { - use conduwuit::utils::random_string; - use http::header::CONTENT_TYPE; - - let response = self.services.client.url_preview.get(url).send().await?; - let content_type = response.headers().get(CONTENT_TYPE).cloned(); - let media = response - .limit_read( - self.services - .server - .config - .max_request_size - .try_into() - .expect("u64 should fit in usize"), - ) - .await?; - - let mxc = Mxc { - server_name: self.services.globals.server_name(), - media_id: &random_string(super::MXC_LENGTH), - }; - - let content_type = content_type.and_then(|v| v.to_str().map(ToOwned::to_owned).ok()); - self.create(&mxc, None, None, content_type.as_deref(), &media) - .await?; - - Ok((OwnedMxcUri::from(mxc.to_string()), media.len())) -} - -#[cfg(not(feature = "url_preview"))] -#[implement(Service)] -pub async fn download_image( - &self, - _url: &str, - _preview_data: Option, -) -> Result { - Err!(FeatureDisabled("url_preview")) -} - -#[cfg(not(feature = "url_preview"))] -#[implement(Service)] -pub async fn download_video( - &self, - _url: &str, - _preview_data: Option, -) -> Result { - Err!(FeatureDisabled("url_preview")) -} - -#[cfg(not(feature = "url_preview"))] -#[implement(Service)] -pub async fn download_audio( - &self, - _url: &str, - _preview_data: Option, -) -> Result { - Err!(FeatureDisabled("url_preview")) -} - -#[cfg(not(feature = "url_preview"))] -#[implement(Service)] -pub async fn download_media(&self, _url: &str) -> Result { - Err!(FeatureDisabled("url_preview")) -} - -#[cfg(feature = "url_preview")] -#[implement(Service)] -async fn download_html(&self, url: &str) -> Result { - use webpage::HTML; - - let client = &self.services.client.url_preview; - let body = client - .get(url) - .send() - .await? - .limit_read_text( - self.services - .server - .config - .max_request_size - .try_into() - .expect("u64 should fit in usize"), - ) - .await?; - let Ok(html) = HTML::from_string(body.clone(), Some(url.to_owned())) else { - return Err!(Request(Unknown("Failed to parse HTML"))); - }; - - let mut preview_data = UrlPreviewData::default(); - - if let Some(obj) = html.opengraph.images.first() { - preview_data = self.download_image(&obj.url, Some(preview_data)).await?; - } - - if let Some(obj) = html.opengraph.videos.first() { - preview_data = self.download_video(&obj.url, Some(preview_data)).await?; - preview_data.video_width = obj.properties.get("width").and_then(|v| v.parse().ok()); - preview_data.video_height = obj.properties.get("height").and_then(|v| v.parse().ok()); - } - - if let Some(obj) = html.opengraph.audios.first() { - preview_data = self.download_audio(&obj.url, Some(preview_data)).await?; - } - - let props = html.opengraph.properties; - - /* use OpenGraph title/description, but fall back to HTML if not available */ - preview_data.title = props.get("title").cloned().or(html.title); - preview_data.description = props.get("description").cloned().or(html.description); - - Ok(preview_data) -} - -#[cfg(not(feature = "url_preview"))] -#[implement(Service)] -async fn download_html(&self, _url: &str) -> Result { - Err!(FeatureDisabled("url_preview")) -} - -#[implement(Service)] -pub fn url_preview_allowed(&self, url: &Url) -> bool { - if ["http", "https"] - .iter() - .all(|&scheme| scheme != url.scheme().to_lowercase()) - { - debug!("Ignoring non-HTTP/HTTPS URL to preview: {}", url); - return false; - } - - let host = match url.host_str() { - | None => { - debug!("Ignoring URL preview for a URL that does not have a host (?): {}", url); - return false; - }, - | Some(h) => h.to_owned(), - }; - - let allowlist_domain_contains = self - .services - .globals - .url_preview_domain_contains_allowlist(); - let allowlist_domain_explicit = self - .services - .globals - .url_preview_domain_explicit_allowlist(); - let denylist_domain_explicit = self.services.globals.url_preview_domain_explicit_denylist(); - let allowlist_url_contains = self.services.globals.url_preview_url_contains_allowlist(); - - if allowlist_domain_contains.contains(&"*".to_owned()) - || allowlist_domain_explicit.contains(&"*".to_owned()) - || allowlist_url_contains.contains(&"*".to_owned()) - { - debug!("Config key contains * which is allowing all URL previews. Allowing URL {}", url); - return true; - } - - if !host.is_empty() { - if denylist_domain_explicit.contains(&host) { - debug!( - "Host {} is not allowed by url_preview_domain_explicit_denylist (check 1/4)", - &host - ); - return false; - } - - if allowlist_domain_explicit.contains(&host) { - debug!( - "Host {} is allowed by url_preview_domain_explicit_allowlist (check 2/4)", - &host - ); - return true; - } - - if allowlist_domain_contains - .iter() - .any(|domain_s| domain_s.contains(&host.clone())) - { - debug!( - "Host {} is allowed by url_preview_domain_contains_allowlist (check 3/4)", - &host - ); - return true; - } - - if allowlist_url_contains - .iter() - .any(|url_s| url.to_string().contains(url_s)) - { - debug!("URL {} is allowed by url_preview_url_contains_allowlist (check 4/4)", &host); - return true; - } - - // check root domain if available and if user has root domain checks - if self.services.globals.url_preview_check_root_domain() { - debug!("Checking root domain"); - match host.split_once('.') { - | None => return false, - | Some((_, root_domain)) => { - if denylist_domain_explicit.contains(&root_domain.to_owned()) { - debug!( - "Root domain {} is not allowed by \ - url_preview_domain_explicit_denylist (check 1/3)", - &root_domain - ); - return false; - } - - if allowlist_domain_explicit.contains(&root_domain.to_owned()) { - debug!( - "Root domain {} is allowed by url_preview_domain_explicit_allowlist \ - (check 2/3)", - &root_domain - ); - return true; - } - - if allowlist_domain_contains - .iter() - .any(|domain_s| domain_s.contains(&root_domain.to_owned())) - { - debug!( - "Root domain {} is allowed by url_preview_domain_contains_allowlist \ - (check 3/3)", - &root_domain - ); - return true; - } - }, + if let Ok(ip) = IPAddress::parse(remote_addr.ip().to_string()) { + if !self.services.client.valid_cidr_range(&ip) { + return Err!(Request(Forbidden("Requesting from this address is forbidden"))); + } } } + + let Some(content_type) = response.headers().get(reqwest::header::CONTENT_TYPE) else { + return Err!(Request(Unknown("Unknown or invalid Content-Type header"))); + }; + + let content_type = content_type + .to_str() + .map_err(|e| err!(Request(Unknown("Unknown or invalid Content-Type header: {e}"))))?; + + let data = match content_type { + | html if html.starts_with("text/html") => self.download_html(url.as_str()).await?, + | img if img.starts_with("image/") => self.download_image(url.as_str(), None).await?, + | video if video.starts_with("video/") => + self.download_video(url.as_str(), None).await?, + | audio if audio.starts_with("audio/") => + self.download_audio(url.as_str(), None).await?, + | _ => return Err!(Request(Unknown("Unsupported Content-Type"))), + }; + + self.set_url_preview(url.as_str(), &data).await?; + + Ok(data) } - false + #[cfg(feature = "url_preview")] + + pub async fn download_image( + &self, + url: &str, + preview_data: Option, + ) -> Result { + use conduwuit::utils::random_string; + use image::ImageReader; + + use crate::media::mxc::Mxc; + + let mut preview_data = preview_data.unwrap_or_default(); + + let image = self + .services + .client + .url_preview + .get(url) + .send() + .await? + .limit_read( + self.services + .server + .config + .max_request_size + .try_into() + .expect("u64 should fit in usize"), + ) + .await?; + + let mxc = Mxc { + server_name: self.services.globals.server_name(), + media_id: &random_string(super::MXC_LENGTH), + }; + + self.create(&mxc, None, None, None, &image).await?; + + preview_data.image = Some(mxc.to_string()); + if preview_data.image_height.is_none() || preview_data.image_width.is_none() { + let cursor = std::io::Cursor::new(&image); + let (width, height) = match ImageReader::new(cursor).with_guessed_format() { + | Err(_) => (None, None), + | Ok(reader) => match reader.into_dimensions() { + | Err(_) => (None, None), + | Ok((width, height)) => (Some(width), Some(height)), + }, + }; + + preview_data.image_width = width; + preview_data.image_height = height; + } + + Ok(preview_data) + } + + #[cfg(feature = "url_preview")] + + pub async fn download_video( + &self, + url: &str, + preview_data: Option, + ) -> Result { + let mut preview_data = preview_data.unwrap_or_default(); + + if self.services.globals.url_preview_allow_audio_video() { + let (url, size) = self.download_media(url).await?; + preview_data.video = Some(url.to_string()); + preview_data.video_size = Some(size); + } + + Ok(preview_data) + } + + #[cfg(feature = "url_preview")] + + pub async fn download_audio( + &self, + url: &str, + preview_data: Option, + ) -> Result { + let mut preview_data = preview_data.unwrap_or_default(); + + if self.services.globals.url_preview_allow_audio_video() { + let (url, size) = self.download_media(url).await?; + preview_data.audio = Some(url.to_string()); + preview_data.audio_size = Some(size); + } + + Ok(preview_data) + } + + #[cfg(feature = "url_preview")] + + pub async fn download_media(&self, url: &str) -> Result<(OwnedMxcUri, usize)> { + use conduwuit::utils::random_string; + use http::header::CONTENT_TYPE; + + let response = self.services.client.url_preview.get(url).send().await?; + let content_type = response.headers().get(CONTENT_TYPE).cloned(); + let media = response + .limit_read( + self.services + .server + .config + .max_request_size + .try_into() + .expect("u64 should fit in usize"), + ) + .await?; + + let mxc = Mxc { + server_name: self.services.globals.server_name(), + media_id: &random_string(super::MXC_LENGTH), + }; + + let content_type = content_type.and_then(|v| v.to_str().map(ToOwned::to_owned).ok()); + self.create(&mxc, None, None, content_type.as_deref(), &media) + .await?; + + Ok((OwnedMxcUri::from(mxc.to_string()), media.len())) + } + + #[cfg(not(feature = "url_preview"))] + pub async fn download_image( + &self, + _url: &str, + _preview_data: Option, + ) -> Result { + Err!(FeatureDisabled("url_preview")) + } + + #[cfg(not(feature = "url_preview"))] + pub async fn download_video( + &self, + _url: &str, + _preview_data: Option, + ) -> Result { + Err!(FeatureDisabled("url_preview")) + } + + #[cfg(not(feature = "url_preview"))] + pub async fn download_audio( + &self, + _url: &str, + _preview_data: Option, + ) -> Result { + Err!(FeatureDisabled("url_preview")) + } + + #[cfg(not(feature = "url_preview"))] + pub async fn download_media(&self, _url: &str) -> Result { + Err!(FeatureDisabled("url_preview")) + } + + #[cfg(feature = "url_preview")] + + async fn download_html(&self, url: &str) -> Result { + use webpage::HTML; + + let client = &self.services.client.url_preview; + let body = client + .get(url) + .send() + .await? + .limit_read_text( + self.services + .server + .config + .max_request_size + .try_into() + .expect("u64 should fit in usize"), + ) + .await?; + let Ok(html) = HTML::from_string(body.clone(), Some(url.to_owned())) else { + return Err!(Request(Unknown("Failed to parse HTML"))); + }; + + let mut preview_data = UrlPreviewData::default(); + + if let Some(obj) = html.opengraph.images.first() { + preview_data = self.download_image(&obj.url, Some(preview_data)).await?; + } + + if let Some(obj) = html.opengraph.videos.first() { + preview_data = self.download_video(&obj.url, Some(preview_data)).await?; + preview_data.video_width = obj.properties.get("width").and_then(|v| v.parse().ok()); + preview_data.video_height = obj.properties.get("height").and_then(|v| v.parse().ok()); + } + + if let Some(obj) = html.opengraph.audios.first() { + preview_data = self.download_audio(&obj.url, Some(preview_data)).await?; + } + + let props = html.opengraph.properties; + + /* use OpenGraph title/description, but fall back to HTML if not available */ + preview_data.title = props.get("title").cloned().or(html.title); + preview_data.description = props.get("description").cloned().or(html.description); + + Ok(preview_data) + } + + #[cfg(not(feature = "url_preview"))] + async fn download_html(&self, _url: &str) -> Result { + Err!(FeatureDisabled("url_preview")) + } + + pub fn url_preview_allowed(&self, url: &Url) -> bool { + if ["http", "https"] + .iter() + .all(|&scheme| scheme != url.scheme().to_lowercase()) + { + debug!("Ignoring non-HTTP/HTTPS URL to preview: {}", url); + return false; + } + + let host = match url.host_str() { + | None => { + debug!("Ignoring URL preview for a URL that does not have a host (?): {}", url); + return false; + }, + | Some(h) => h.to_owned(), + }; + + let allowlist_domain_contains = self + .services + .globals + .url_preview_domain_contains_allowlist(); + let allowlist_domain_explicit = self + .services + .globals + .url_preview_domain_explicit_allowlist(); + let denylist_domain_explicit = + self.services.globals.url_preview_domain_explicit_denylist(); + let allowlist_url_contains = self.services.globals.url_preview_url_contains_allowlist(); + + if allowlist_domain_contains.contains(&"*".to_owned()) + || allowlist_domain_explicit.contains(&"*".to_owned()) + || allowlist_url_contains.contains(&"*".to_owned()) + { + debug!( + "Config key contains * which is allowing all URL previews. Allowing URL {}", + url + ); + return true; + } + + if !host.is_empty() { + if denylist_domain_explicit.contains(&host) { + debug!( + "Host {} is not allowed by url_preview_domain_explicit_denylist (check 1/4)", + &host + ); + return false; + } + + if allowlist_domain_explicit.contains(&host) { + debug!( + "Host {} is allowed by url_preview_domain_explicit_allowlist (check 2/4)", + &host + ); + return true; + } + + if allowlist_domain_contains + .iter() + .any(|domain_s| domain_s.contains(&host.clone())) + { + debug!( + "Host {} is allowed by url_preview_domain_contains_allowlist (check 3/4)", + &host + ); + return true; + } + + if allowlist_url_contains + .iter() + .any(|url_s| url.to_string().contains(url_s)) + { + debug!( + "URL {} is allowed by url_preview_url_contains_allowlist (check 4/4)", + &host + ); + return true; + } + + // check root domain if available and if user has root domain checks + if self.services.globals.url_preview_check_root_domain() { + debug!("Checking root domain"); + match host.split_once('.') { + | None => return false, + | Some((_, root_domain)) => { + if denylist_domain_explicit.contains(&root_domain.to_owned()) { + debug!( + "Root domain {} is not allowed by \ + url_preview_domain_explicit_denylist (check 1/3)", + &root_domain + ); + return false; + } + + if allowlist_domain_explicit.contains(&root_domain.to_owned()) { + debug!( + "Root domain {} is allowed by \ + url_preview_domain_explicit_allowlist (check 2/3)", + &root_domain + ); + return true; + } + + if allowlist_domain_contains + .iter() + .any(|domain_s| domain_s.contains(&root_domain.to_owned())) + { + debug!( + "Root domain {} is allowed by \ + url_preview_domain_contains_allowlist (check 3/3)", + &root_domain + ); + return true; + } + }, + } + } + } + + false + } } diff --git a/src/service/media/remote.rs b/src/service/media/remote.rs index c5862a576..4a1ca828b 100644 --- a/src/service/media/remote.rs +++ b/src/service/media/remote.rs @@ -1,7 +1,7 @@ use std::{fmt::Debug, time::Duration}; use conduwuit::{ - Err, Error, Result, debug_warn, err, implement, + Err, Error, Result, debug_warn, err, utils::{content_disposition::make_content_disposition, response::LimitReadExt}, }; use http::header::{CONTENT_DISPOSITION, CONTENT_TYPE, HeaderValue}; @@ -24,436 +24,424 @@ use ruma::{ use super::{Dim, FileMeta}; use crate::{federation::FederationPathBuilderInput, media::mxc::Mxc}; -#[implement(super::Service)] -pub async fn fetch_remote_thumbnail( - &self, - mxc: &Mxc<'_>, - user: Option<&UserId>, - server: Option<&ServerName>, - timeout_ms: Duration, - dim: &Dim, -) -> Result { - self.check_fetch_authorized(mxc)?; +impl super::Service { + pub async fn fetch_remote_thumbnail( + &self, + mxc: &Mxc<'_>, + user: Option<&UserId>, + server: Option<&ServerName>, + timeout_ms: Duration, + dim: &Dim, + ) -> Result { + self.check_fetch_authorized(mxc)?; - let result = self - .fetch_thumbnail_authenticated(mxc, user, server, timeout_ms, dim) - .await; - - if let Err(Error::Request(NotFound, ..)) = &result { - return self - .fetch_thumbnail_unauthenticated(mxc, user, server, timeout_ms, dim) + let result = self + .fetch_thumbnail_authenticated(mxc, user, server, timeout_ms, dim) .await; + + if let Err(Error::Request(NotFound, ..)) = &result { + return self + .fetch_thumbnail_unauthenticated(mxc, user, server, timeout_ms, dim) + .await; + } + + result } - result -} + pub async fn fetch_remote_content( + &self, + mxc: &Mxc<'_>, + user: Option<&UserId>, + server: Option<&ServerName>, + timeout_ms: Duration, + ) -> Result { + self.check_fetch_authorized(mxc)?; -#[implement(super::Service)] -pub async fn fetch_remote_content( - &self, - mxc: &Mxc<'_>, - user: Option<&UserId>, - server: Option<&ServerName>, - timeout_ms: Duration, -) -> Result { - self.check_fetch_authorized(mxc)?; + let result = self + .fetch_content_authenticated(mxc, user, server, timeout_ms) + .await + .inspect_err(|error| { + debug_warn!( + %mxc, + ?user, + ?server, + ?error, + "Authenticated fetch of remote content failed" + ); + }); - let result = self - .fetch_content_authenticated(mxc, user, server, timeout_ms) - .await - .inspect_err(|error| { - debug_warn!( - %mxc, - ?user, - ?server, - ?error, - "Authenticated fetch of remote content failed" - ); - }); + if let Err(Error::Request(Unrecognized, ..)) = &result { + return self + .fetch_content_unauthenticated(mxc, user, server, timeout_ms) + .await; + } - if let Err(Error::Request(Unrecognized, ..)) = &result { - return self - .fetch_content_unauthenticated(mxc, user, server, timeout_ms) - .await; + result } - result -} + async fn fetch_thumbnail_authenticated( + &self, + mxc: &Mxc<'_>, + user: Option<&UserId>, + server: Option<&ServerName>, + timeout_ms: Duration, + dim: &Dim, + ) -> Result { + use federation::authenticated_media::get_content_thumbnail::v1::{Request, Response}; -#[implement(super::Service)] -async fn fetch_thumbnail_authenticated( - &self, - mxc: &Mxc<'_>, - user: Option<&UserId>, - server: Option<&ServerName>, - timeout_ms: Duration, - dim: &Dim, -) -> Result { - use federation::authenticated_media::get_content_thumbnail::v1::{Request, Response}; + let mut request = Request::new(mxc.media_id.into(), dim.width.into(), dim.height.into()); + request.method = Some(dim.method.clone()); + request.animated = Some(true); + request.timeout_ms = timeout_ms; - let mut request = Request::new(mxc.media_id.into(), dim.width.into(), dim.height.into()); - request.method = Some(dim.method.clone()); - request.animated = Some(true); - request.timeout_ms = timeout_ms; + let Response { content, .. } = self.federation_request(mxc, server, request).await?; - let Response { content, .. } = self.federation_request(mxc, server, request).await?; - - match content { - | FileOrLocation::File(content) => - self.handle_thumbnail_file(mxc, user, dim, content).await, - | FileOrLocation::Location(location) => self.handle_location(mxc, user, &location).await, - | _ => Err!("Unknown content in response"), + match content { + | FileOrLocation::File(content) => + self.handle_thumbnail_file(mxc, user, dim, content).await, + | FileOrLocation::Location(location) => + self.handle_location(mxc, user, &location).await, + | _ => Err!("Unknown content in response"), + } } -} -#[implement(super::Service)] -async fn fetch_content_authenticated( - &self, - mxc: &Mxc<'_>, - user: Option<&UserId>, - server: Option<&ServerName>, - timeout_ms: Duration, -) -> Result { - use federation::authenticated_media::get_content::v1::{Request, Response}; + async fn fetch_content_authenticated( + &self, + mxc: &Mxc<'_>, + user: Option<&UserId>, + server: Option<&ServerName>, + timeout_ms: Duration, + ) -> Result { + use federation::authenticated_media::get_content::v1::{Request, Response}; - let mut request = Request::new(mxc.media_id.into()); - request.timeout_ms = timeout_ms; + let mut request = Request::new(mxc.media_id.into()); + request.timeout_ms = timeout_ms; - let Response { content, .. } = self.federation_request(mxc, server, request).await?; + let Response { content, .. } = self.federation_request(mxc, server, request).await?; - match content { - | FileOrLocation::File(content) => self.handle_content_file(mxc, user, content).await, - | FileOrLocation::Location(location) => self.handle_location(mxc, user, &location).await, - | _ => Err!("Unknown content in response"), + match content { + | FileOrLocation::File(content) => self.handle_content_file(mxc, user, content).await, + | FileOrLocation::Location(location) => + self.handle_location(mxc, user, &location).await, + | _ => Err!("Unknown content in response"), + } } -} -#[allow(deprecated)] -#[implement(super::Service)] -async fn fetch_thumbnail_unauthenticated( - &self, - mxc: &Mxc<'_>, - user: Option<&UserId>, - server: Option<&ServerName>, - timeout_ms: Duration, - dim: &Dim, -) -> Result { - use media::get_content_thumbnail::v3::{Request, Response}; + #[allow(deprecated)] + async fn fetch_thumbnail_unauthenticated( + &self, + mxc: &Mxc<'_>, + user: Option<&UserId>, + server: Option<&ServerName>, + timeout_ms: Duration, + dim: &Dim, + ) -> Result { + use media::get_content_thumbnail::v3::{Request, Response}; - let mut request = Request::new( - mxc.media_id.into(), - mxc.server_name.into(), - dim.width.into(), - dim.height.into(), - ); - request.allow_redirect = true; - request.allow_remote = true; - request.animated = Some(true); - request.method = Some(dim.method.clone()); - request.timeout_ms = timeout_ms; + let mut request = Request::new( + mxc.media_id.into(), + mxc.server_name.into(), + dim.width.into(), + dim.height.into(), + ); + request.allow_redirect = true; + request.allow_remote = true; + request.animated = Some(true); + request.method = Some(dim.method.clone()); + request.timeout_ms = timeout_ms; - let Response { - file, content_type, content_disposition, .. - } = self - .federation_request_legacy_media(mxc, server, request) - .await?; + let Response { + file, content_type, content_disposition, .. + } = self + .federation_request_legacy_media(mxc, server, request) + .await?; - let content = Content::new(file, content_type.unwrap(), content_disposition.unwrap()); + let content = Content::new(file, content_type.unwrap(), content_disposition.unwrap()); - self.handle_thumbnail_file(mxc, user, dim, content).await -} + self.handle_thumbnail_file(mxc, user, dim, content).await + } -#[allow(deprecated)] -#[implement(super::Service)] -async fn fetch_content_unauthenticated( - &self, - mxc: &Mxc<'_>, - user: Option<&UserId>, - server: Option<&ServerName>, - timeout_ms: Duration, -) -> Result { - use media::get_content::v3::{Request, Response}; + #[allow(deprecated)] + async fn fetch_content_unauthenticated( + &self, + mxc: &Mxc<'_>, + user: Option<&UserId>, + server: Option<&ServerName>, + timeout_ms: Duration, + ) -> Result { + use media::get_content::v3::{Request, Response}; - let mut request = Request::new(mxc.media_id.into(), mxc.server_name.into()); - request.allow_remote = true; - request.allow_redirect = true; - request.timeout_ms = timeout_ms; + let mut request = Request::new(mxc.media_id.into(), mxc.server_name.into()); + request.allow_remote = true; + request.allow_redirect = true; + request.timeout_ms = timeout_ms; - let Response { - file, content_type, content_disposition, .. - } = self - .federation_request_legacy_media(mxc, server, request) - .await?; + let Response { + file, content_type, content_disposition, .. + } = self + .federation_request_legacy_media(mxc, server, request) + .await?; - let content = Content::new(file, content_type.unwrap(), content_disposition.unwrap()); + let content = Content::new(file, content_type.unwrap(), content_disposition.unwrap()); - self.handle_content_file(mxc, user, content).await -} + self.handle_content_file(mxc, user, content).await + } -#[implement(super::Service)] -async fn handle_thumbnail_file( - &self, - mxc: &Mxc<'_>, - user: Option<&UserId>, - dim: &Dim, - content: Content, -) -> Result { - let content_disposition = make_content_disposition( - content.content_disposition.as_ref(), - content.content_type.as_deref(), - None, - ); + async fn handle_thumbnail_file( + &self, + mxc: &Mxc<'_>, + user: Option<&UserId>, + dim: &Dim, + content: Content, + ) -> Result { + let content_disposition = make_content_disposition( + content.content_disposition.as_ref(), + content.content_type.as_deref(), + None, + ); - self.upload_thumbnail( - mxc, - user, - Some(&content_disposition), - content.content_type.as_deref(), - dim, - &content.file, - ) - .await - .map(|()| FileMeta { - content: Some(content.file), - content_type: content.content_type, - content_disposition: Some(content_disposition), - }) -} - -#[implement(super::Service)] -async fn handle_content_file( - &self, - mxc: &Mxc<'_>, - user: Option<&UserId>, - content: Content, -) -> Result { - let content_disposition = make_content_disposition( - content.content_disposition.as_ref(), - content.content_type.as_deref(), - None, - ); - - self.create( - mxc, - user, - Some(&content_disposition), - content.content_type.as_deref(), - &content.file, - ) - .await - .map(|()| FileMeta { - content: Some(content.file), - content_type: content.content_type, - content_disposition: Some(content_disposition), - }) -} - -#[implement(super::Service)] -async fn handle_location( - &self, - mxc: &Mxc<'_>, - user: Option<&UserId>, - location: &str, -) -> Result { - self.location_request(location).await.map_err(|error| { - err!(Request(NotFound( - debug_warn!(%mxc, user = user.map(tracing::field::display), ?location, ?error, "Fetching media from location failed") - ))) - }) -} - -#[implement(super::Service)] -async fn location_request(&self, location: &str) -> Result { - let response = self - .services - .client - .extern_media - .get(location) - .send() - .await?; - - let content_type = response - .headers() - .get(CONTENT_TYPE) - .map(HeaderValue::to_str) - .and_then(Result::ok) - .map(str::to_owned); - - let content_disposition = response - .headers() - .get(CONTENT_DISPOSITION) - .map(HeaderValue::as_bytes) - .map(TryFrom::try_from) - .and_then(Result::ok); - - response - .limit_read( - self.services - .server - .config - .max_request_size - .try_into() - .expect("u64 should fit in usize"), + self.upload_thumbnail( + mxc, + user, + Some(&content_disposition), + content.content_type.as_deref(), + dim, + &content.file, ) .await - .map(|content| FileMeta { - content: Some(content), - content_type: content_type.clone(), - content_disposition: Some(make_content_disposition( - content_disposition.as_ref(), - content_type.as_deref(), - None, - )), + .map(|()| FileMeta { + content: Some(content.file), + content_type: content.content_type, + content_disposition: Some(content_disposition), }) -} - -#[implement(super::Service)] -async fn federation_request<'i, Request>( - &self, - mxc: &Mxc<'_>, - server: Option<&ServerName>, - request: Request, -) -> Result -where - Request: OutgoingRequest< - Authentication = ServerSignatures, - PathBuilder: PathBuilder: FederationPathBuilderInput>, - > + Debug - + Send, -{ - self.services - .sending - .send_federation_request(server.unwrap_or(mxc.server_name), request) - .await -} - -#[implement(super::Service)] -async fn federation_request_legacy_media<'i, Request>( - &self, - mxc: &Mxc<'_>, - server: Option<&ServerName>, - request: Request, -) -> Result -where - Request: OutgoingRequest< - Authentication = NoAccessToken, - PathBuilder: PathBuilder: FederationPathBuilderInput>, - > + Debug - + Send, -{ - self.services - .sending - .send_legacy_media_request(server.unwrap_or(mxc.server_name), request) - .await -} - -#[implement(super::Service)] -#[allow(deprecated)] -pub async fn fetch_remote_thumbnail_legacy( - &self, - body: &media::get_content_thumbnail::v3::Request, -) -> Result { - let mxc = Mxc { - server_name: &body.server_name, - media_id: &body.media_id, - }; - - let mut request = media::get_content_thumbnail::v3::Request::new( - body.media_id.clone(), - body.server_name.clone(), - body.width, - body.height, - ); - request.method.clone_from(&body.method); - request.allow_remote = body.allow_remote; - request.allow_redirect = body.allow_redirect; - request.animated = body.animated; - request.timeout_ms = body.timeout_ms; - - self.check_legacy_freeze()?; - self.check_fetch_authorized(&mxc)?; - let response = self - .services - .sending - .send_legacy_media_request(mxc.server_name, request) - .await?; - - let dim = Dim::from_ruma(body.width, body.height, body.method.clone())?; - self.upload_thumbnail( - &mxc, - None, - None, - response.content_type.as_deref(), - &dim, - &response.file, - ) - .await?; - - Ok(response) -} - -#[implement(super::Service)] -#[allow(deprecated)] -pub async fn fetch_remote_content_legacy( - &self, - mxc: &Mxc<'_>, - allow_redirect: bool, - timeout_ms: Duration, -) -> Result { - let mut request = - media::get_content::v3::Request::new(mxc.media_id.into(), mxc.server_name.into()); - request.allow_remote = true; - request.allow_redirect = allow_redirect; - request.timeout_ms = timeout_ms; - - self.check_legacy_freeze()?; - self.check_fetch_authorized(mxc)?; - let response = self - .services - .sending - .send_legacy_media_request(mxc.server_name, request) - .await?; - - let content_disposition = make_content_disposition( - response.content_disposition.as_ref(), - response.content_type.as_deref(), - None, - ); - - self.create( - mxc, - None, - Some(&content_disposition), - response.content_type.as_deref(), - &response.file, - ) - .await?; - - Ok(response) -} - -#[implement(super::Service)] -fn check_fetch_authorized(&self, mxc: &Mxc<'_>) -> Result<()> { - if self - .services - .moderation - .is_remote_server_media_downloads_forbidden(mxc.server_name) - { - // we'll lie to the client and say the blocked server's media was not found and - // log. the client has no way of telling anyways so this is a security bonus. - debug_warn!(%mxc, "Received request for media on blocklisted server"); - return Err!(Request(NotFound("Media not found."))); } - Ok(()) -} + async fn handle_content_file( + &self, + mxc: &Mxc<'_>, + user: Option<&UserId>, + content: Content, + ) -> Result { + let content_disposition = make_content_disposition( + content.content_disposition.as_ref(), + content.content_type.as_deref(), + None, + ); -#[implement(super::Service)] -fn check_legacy_freeze(&self) -> Result<()> { - self.services - .server - .config - .freeze_legacy_media - .then_some(()) - .ok_or(err!(Request(NotFound("Remote media is frozen.")))) + self.create( + mxc, + user, + Some(&content_disposition), + content.content_type.as_deref(), + &content.file, + ) + .await + .map(|()| FileMeta { + content: Some(content.file), + content_type: content.content_type, + content_disposition: Some(content_disposition), + }) + } + + async fn handle_location( + &self, + mxc: &Mxc<'_>, + user: Option<&UserId>, + location: &str, + ) -> Result { + self.location_request(location).await.map_err(|error| { + err!(Request(NotFound( + debug_warn!(%mxc, user = user.map(tracing::field::display), ?location, ?error, "Fetching media from location failed") + ))) + }) + } + + async fn location_request(&self, location: &str) -> Result { + let response = self + .services + .client + .extern_media + .get(location) + .send() + .await?; + + let content_type = response + .headers() + .get(CONTENT_TYPE) + .map(HeaderValue::to_str) + .and_then(Result::ok) + .map(str::to_owned); + + let content_disposition = response + .headers() + .get(CONTENT_DISPOSITION) + .map(HeaderValue::as_bytes) + .map(TryFrom::try_from) + .and_then(Result::ok); + + response + .limit_read( + self.services + .server + .config + .max_request_size + .try_into() + .expect("u64 should fit in usize"), + ) + .await + .map(|content| FileMeta { + content: Some(content), + content_type: content_type.clone(), + content_disposition: Some(make_content_disposition( + content_disposition.as_ref(), + content_type.as_deref(), + None, + )), + }) + } + + async fn federation_request<'i, Request>( + &self, + mxc: &Mxc<'_>, + server: Option<&ServerName>, + request: Request, + ) -> Result + where + Request: OutgoingRequest< + Authentication = ServerSignatures, + PathBuilder: PathBuilder: FederationPathBuilderInput>, + > + Debug + + Send, + { + self.services + .sending + .send_federation_request(server.unwrap_or(mxc.server_name), request) + .await + } + + async fn federation_request_legacy_media<'i, Request>( + &self, + mxc: &Mxc<'_>, + server: Option<&ServerName>, + request: Request, + ) -> Result + where + Request: OutgoingRequest< + Authentication = NoAccessToken, + PathBuilder: PathBuilder: FederationPathBuilderInput>, + > + Debug + + Send, + { + self.services + .sending + .send_legacy_media_request(server.unwrap_or(mxc.server_name), request) + .await + } + + #[allow(deprecated)] + pub async fn fetch_remote_thumbnail_legacy( + &self, + body: &media::get_content_thumbnail::v3::Request, + ) -> Result { + let mxc = Mxc { + server_name: &body.server_name, + media_id: &body.media_id, + }; + + let mut request = media::get_content_thumbnail::v3::Request::new( + body.media_id.clone(), + body.server_name.clone(), + body.width, + body.height, + ); + request.method.clone_from(&body.method); + request.allow_remote = body.allow_remote; + request.allow_redirect = body.allow_redirect; + request.animated = body.animated; + request.timeout_ms = body.timeout_ms; + + self.check_legacy_freeze()?; + self.check_fetch_authorized(&mxc)?; + let response = self + .services + .sending + .send_legacy_media_request(mxc.server_name, request) + .await?; + + let dim = Dim::from_ruma(body.width, body.height, body.method.clone())?; + self.upload_thumbnail( + &mxc, + None, + None, + response.content_type.as_deref(), + &dim, + &response.file, + ) + .await?; + + Ok(response) + } + + #[allow(deprecated)] + pub async fn fetch_remote_content_legacy( + &self, + mxc: &Mxc<'_>, + allow_redirect: bool, + timeout_ms: Duration, + ) -> Result { + let mut request = + media::get_content::v3::Request::new(mxc.media_id.into(), mxc.server_name.into()); + request.allow_remote = true; + request.allow_redirect = allow_redirect; + request.timeout_ms = timeout_ms; + + self.check_legacy_freeze()?; + self.check_fetch_authorized(mxc)?; + let response = self + .services + .sending + .send_legacy_media_request(mxc.server_name, request) + .await?; + + let content_disposition = make_content_disposition( + response.content_disposition.as_ref(), + response.content_type.as_deref(), + None, + ); + + self.create( + mxc, + None, + Some(&content_disposition), + response.content_type.as_deref(), + &response.file, + ) + .await?; + + Ok(response) + } + + fn check_fetch_authorized(&self, mxc: &Mxc<'_>) -> Result<()> { + if self + .services + .moderation + .is_remote_server_media_downloads_forbidden(mxc.server_name) + { + // we'll lie to the client and say the blocked server's media was not found and + // log. the client has no way of telling anyways so this is a security bonus. + debug_warn!(%mxc, "Received request for media on blocklisted server"); + return Err!(Request(NotFound("Media not found."))); + } + + Ok(()) + } + + fn check_legacy_freeze(&self) -> Result<()> { + self.services + .server + .config + .freeze_legacy_media + .then_some(()) + .ok_or(err!(Request(NotFound("Remote media is frozen.")))) + } } diff --git a/src/service/media/thumbnail.rs b/src/service/media/thumbnail.rs index 162ac0ba0..8a23bb086 100644 --- a/src/service/media/thumbnail.rs +++ b/src/service/media/thumbnail.rs @@ -7,7 +7,7 @@ use std::{cmp, num::Saturating as Sat}; -use conduwuit::{Result, checked, err, implement}; +use conduwuit::{Result, checked, err}; use ruma::{UInt, UserId, http_headers::ContentDisposition, media::Method}; use tokio::{ fs, @@ -74,80 +74,77 @@ impl super::Service { }, } } -} -/// Using saved thumbnail -#[implement(super::Service)] -#[tracing::instrument(name = "saved", level = "debug", skip(self, data))] -async fn get_thumbnail_saved(&self, data: Metadata) -> Result> { - let mut content = Vec::new(); - let path = self.get_media_file(&data.key); - fs::File::open(path) - .await? - .read_to_end(&mut content) - .await?; + /// Using saved thumbnail + #[tracing::instrument(name = "saved", level = "debug", skip(self, data))] + async fn get_thumbnail_saved(&self, data: Metadata) -> Result> { + let mut content = Vec::new(); + let path = self.get_media_file(&data.key); + fs::File::open(path) + .await? + .read_to_end(&mut content) + .await?; - Ok(Some(into_filemeta(data, content))) -} - -/// Generate a thumbnail -#[cfg(feature = "media_thumbnail")] -#[implement(super::Service)] -#[tracing::instrument(name = "generate", level = "debug", skip(self, data))] -async fn get_thumbnail_generate( - &self, - mxc: &Mxc<'_>, - dim: &Dim, - data: Metadata, -) -> Result> { - let mut content = Vec::new(); - let path = self.get_media_file(&data.key); - fs::File::open(path) - .await? - .read_to_end(&mut content) - .await?; - - let Ok(image) = image::load_from_memory(&content) else { - // Couldn't parse file to generate thumbnail, send original - return Ok(Some(into_filemeta(data, content))); - }; - - if dim.width > image.width() || dim.height > image.height() { - return Ok(Some(into_filemeta(data, content))); + Ok(Some(into_filemeta(data, content))) } - let mut thumbnail_bytes = Vec::new(); - let thumbnail = thumbnail_generate(&image, dim)?; - let mut cursor = std::io::Cursor::new(&mut thumbnail_bytes); - thumbnail - .write_to(&mut cursor, image::ImageFormat::Png) - .map_err(|error| err!(error!(%error, "Error writing PNG thumbnail.")))?; + /// Generate a thumbnail + #[cfg(feature = "media_thumbnail")] + #[tracing::instrument(name = "generate", level = "debug", skip(self, data))] + async fn get_thumbnail_generate( + &self, + mxc: &Mxc<'_>, + dim: &Dim, + data: Metadata, + ) -> Result> { + let mut content = Vec::new(); + let path = self.get_media_file(&data.key); + fs::File::open(path) + .await? + .read_to_end(&mut content) + .await?; - // Save thumbnail in database so we don't have to generate it again next time - let thumbnail_key = self.db.create_file_metadata( - mxc, - None, - dim, - data.content_disposition.as_ref(), - data.content_type.as_deref(), - )?; + let Ok(image) = image::load_from_memory(&content) else { + // Couldn't parse file to generate thumbnail, send original + return Ok(Some(into_filemeta(data, content))); + }; - let mut f = self.create_media_file(&thumbnail_key).await?; - f.write_all(&thumbnail_bytes).await?; + if dim.width > image.width() || dim.height > image.height() { + return Ok(Some(into_filemeta(data, content))); + } - Ok(Some(into_filemeta(data, thumbnail_bytes))) -} + let mut thumbnail_bytes = Vec::new(); + let thumbnail = thumbnail_generate(&image, dim)?; + let mut cursor = std::io::Cursor::new(&mut thumbnail_bytes); + thumbnail + .write_to(&mut cursor, image::ImageFormat::Png) + .map_err(|error| err!(error!(%error, "Error writing PNG thumbnail.")))?; -#[cfg(not(feature = "media_thumbnail"))] -#[implement(super::Service)] -#[tracing::instrument(name = "fallback", level = "debug", skip_all)] -async fn get_thumbnail_generate( - &self, - _mxc: &Mxc<'_>, - _dim: &Dim, - data: Metadata, -) -> Result> { - self.get_thumbnail_saved(data).await + // Save thumbnail in database so we don't have to generate it again next time + let thumbnail_key = self.db.create_file_metadata( + mxc, + None, + dim, + data.content_disposition.as_ref(), + data.content_type.as_deref(), + )?; + + let mut f = self.create_media_file(&thumbnail_key).await?; + f.write_all(&thumbnail_bytes).await?; + + Ok(Some(into_filemeta(data, thumbnail_bytes))) + } + + #[cfg(not(feature = "media_thumbnail"))] + #[tracing::instrument(name = "fallback", level = "debug", skip_all)] + async fn get_thumbnail_generate( + &self, + _mxc: &Mxc<'_>, + _dim: &Dim, + data: Metadata, + ) -> Result> { + self.get_thumbnail_saved(data).await + } } #[cfg(feature = "media_thumbnail")] From ae572ec1085410a0b069b854a16d8f2eb8bcbf4f Mon Sep 17 00:00:00 2001 From: timedout Date: Fri, 26 Jun 2026 05:22:05 +0100 Subject: [PATCH 48/70] style: Refactor key_backups service --- src/service/key_backups/mod.rs | 471 ++++++++++++++++----------------- 1 file changed, 231 insertions(+), 240 deletions(-) diff --git a/src/service/key_backups/mod.rs b/src/service/key_backups/mod.rs index 3adbc46b1..f2f3ce813 100644 --- a/src/service/key_backups/mod.rs +++ b/src/service/key_backups/mod.rs @@ -1,7 +1,7 @@ use std::{collections::BTreeMap, sync::Arc}; use conduwuit::{ - Err, Result, err, implement, + Err, Result, err, utils::stream::{ReadyExt, TryIgnore}, }; use database::{Deserialized, Ignore, Interfix, Json, Map}; @@ -46,253 +46,244 @@ impl crate::Service for Service { fn name(&self) -> &str { crate::service::make_name(std::module_path!()) } } -#[implement(Service)] -pub fn create_backup( - &self, - user_id: &UserId, - backup_metadata: &Raw, -) -> Result { - let version = self.services.globals.next_count()?.to_string(); - let count = self.services.globals.next_count()?; +impl Service { + pub fn create_backup( + &self, + user_id: &UserId, + backup_metadata: &Raw, + ) -> Result { + let version = self.services.globals.next_count()?.to_string(); + let count = self.services.globals.next_count()?; - let key = (user_id, &version); - self.db.backupid_algorithm.put(key, Json(backup_metadata)); + let key = (user_id, &version); + self.db.backupid_algorithm.put(key, Json(backup_metadata)); - self.db.backupid_etag.put(key, count); + self.db.backupid_etag.put(key, count); - Ok(version) -} - -#[implement(Service)] -pub async fn delete_backup(&self, user_id: &UserId, version: &str) { - let key = (user_id, version); - self.db.backupid_algorithm.del(key); - self.db.backupid_etag.del(key); - - let key = (user_id, version, Interfix); - self.db - .backupkeyid_backup - .keys_prefix_raw(&key) - .ignore_err() - .ready_for_each(|outdated_key| { - self.db.backupkeyid_backup.remove(outdated_key); - }) - .await; -} - -#[implement(Service)] -pub async fn update_backup<'a>( - &self, - user_id: &UserId, - version: &'a str, - backup_metadata: &Raw, -) -> Result<&'a str> { - let key = (user_id, version); - if self.db.backupid_algorithm.qry(&key).await.is_err() { - return Err!(Request(NotFound("Tried to update nonexistent backup."))); + Ok(version) } - let count = self.services.globals.next_count().unwrap(); - self.db.backupid_etag.put(key, count); - self.db - .backupid_algorithm - .put_raw(key, backup_metadata.json().get()); + pub async fn delete_backup(&self, user_id: &UserId, version: &str) { + let key = (user_id, version); + self.db.backupid_algorithm.del(key); + self.db.backupid_etag.del(key); - Ok(version) -} - -#[implement(Service)] -pub async fn get_latest_backup_version(&self, user_id: &UserId) -> Result { - type Key<'a> = (OwnedUserId, &'a str); - - let last_possible_key = (user_id, u64::MAX); - self.db - .backupid_algorithm - .rev_keys_from(&last_possible_key) - .ignore_err() - .ready_take_while(|(user_id_, _): &Key<'_>| *user_id_ == user_id) - .map(|(_, version): Key<'_>| version.to_owned()) - .next() - .await - .ok_or_else(|| err!(Request(NotFound("No backup versions found")))) -} - -#[implement(Service)] -pub async fn get_latest_backup( - &self, - user_id: &UserId, -) -> Result<(String, Raw)> { - type Key<'a> = (OwnedUserId, &'a str); - type KeyVal<'a> = (Key<'a>, Raw); - - let last_possible_key = (user_id, u64::MAX); - self.db - .backupid_algorithm - .rev_stream_from(&last_possible_key) - .ignore_err() - .ready_take_while(|((user_id_, _), _): &KeyVal<'_>| *user_id_ == user_id) - .map(|((_, version), algorithm): KeyVal<'_>| (version.to_owned(), algorithm)) - .next() - .await - .ok_or_else(|| err!(Request(NotFound("No backup found")))) -} - -#[implement(Service)] -pub async fn get_backup(&self, user_id: &UserId, version: &str) -> Result> { - let key = (user_id, version); - self.db.backupid_algorithm.qry(&key).await.deserialized() -} - -#[implement(Service)] -pub async fn add_key( - &self, - user_id: &UserId, - version: &str, - room_id: &RoomId, - session_id: &str, - key_data: &Raw, -) -> Result<()> { - let key = (user_id, version); - if self.db.backupid_algorithm.qry(&key).await.is_err() { - return Err!(Request(NotFound("Tried to update nonexistent backup."))); + let key = (user_id, version, Interfix); + self.db + .backupkeyid_backup + .keys_prefix_raw(&key) + .ignore_err() + .ready_for_each(|outdated_key| { + self.db.backupkeyid_backup.remove(outdated_key); + }) + .await; } - let count = self.services.globals.next_count().unwrap(); - self.db.backupid_etag.put(key, count); + pub async fn update_backup<'a>( + &self, + user_id: &UserId, + version: &'a str, + backup_metadata: &Raw, + ) -> Result<&'a str> { + let key = (user_id, version); + if self.db.backupid_algorithm.qry(&key).await.is_err() { + return Err!(Request(NotFound("Tried to update nonexistent backup."))); + } - let key = (user_id, version, room_id, session_id); - self.db - .backupkeyid_backup - .put_raw(key, key_data.json().get()); + let count = self.services.globals.next_count().unwrap(); + self.db.backupid_etag.put(key, count); + self.db + .backupid_algorithm + .put_raw(key, backup_metadata.json().get()); - Ok(()) -} - -#[implement(Service)] -pub async fn count_keys(&self, user_id: &UserId, version: &str) -> usize { - let prefix = (user_id, version); - self.db - .backupkeyid_backup - .keys_prefix_raw(&prefix) - .count() - .await -} - -#[implement(Service)] -pub async fn get_etag(&self, user_id: &UserId, version: &str) -> String { - let key = (user_id, version); - self.db - .backupid_etag - .qry(&key) - .await - .deserialized::() - .as_ref() - .map(ToString::to_string) - .expect("Backup has no etag.") -} - -#[implement(Service)] -pub async fn get_all( - &self, - user_id: &UserId, - version: &str, -) -> BTreeMap { - type Key<'a> = (Ignore, Ignore, OwnedRoomId, &'a str); - type KeyVal<'a> = (Key<'a>, Raw); - - let mut rooms = BTreeMap::::new(); - let default = || RoomKeyBackup::new(BTreeMap::new()); - - let prefix = (user_id, version, Interfix); - self.db - .backupkeyid_backup - .stream_prefix(&prefix) - .ignore_err() - .ready_for_each(|((_, _, room_id, session_id), key_backup_data): KeyVal<'_>| { - rooms - .entry(room_id) - .or_insert_with(default) - .sessions - .insert(session_id.into(), key_backup_data); - }) - .await; - - rooms -} - -#[implement(Service)] -pub async fn get_room( - &self, - user_id: &UserId, - version: &str, - room_id: &RoomId, -) -> BTreeMap> { - type KeyVal<'a> = ((Ignore, Ignore, Ignore, &'a str), Raw); - - let prefix = (user_id, version, room_id, Interfix); - self.db - .backupkeyid_backup - .stream_prefix(&prefix) - .ignore_err() - .map(|((.., session_id), key_backup_data): KeyVal<'_>| { - (session_id.to_owned(), key_backup_data) - }) - .collect() - .await -} - -#[implement(Service)] -pub async fn get_session( - &self, - user_id: &UserId, - version: &str, - room_id: &RoomId, - session_id: &str, -) -> Result> { - let key = (user_id, version, room_id, session_id); - - self.db.backupkeyid_backup.qry(&key).await.deserialized() -} - -#[implement(Service)] -pub async fn delete_all_keys(&self, user_id: &UserId, version: &str) { - let key = (user_id, version, Interfix); - self.db - .backupkeyid_backup - .keys_prefix_raw(&key) - .ignore_err() - .ready_for_each(|outdated_key| self.db.backupkeyid_backup.remove(outdated_key)) - .await; -} - -#[implement(Service)] -pub async fn delete_room_keys(&self, user_id: &UserId, version: &str, room_id: &RoomId) { - let key = (user_id, version, room_id, Interfix); - self.db - .backupkeyid_backup - .keys_prefix_raw(&key) - .ignore_err() - .ready_for_each(|outdated_key| { - self.db.backupkeyid_backup.remove(outdated_key); - }) - .await; -} - -#[implement(Service)] -pub async fn delete_room_key( - &self, - user_id: &UserId, - version: &str, - room_id: &RoomId, - session_id: &str, -) { - let key = (user_id, version, room_id, session_id); - self.db - .backupkeyid_backup - .keys_prefix_raw(&key) - .ignore_err() - .ready_for_each(|outdated_key| { - self.db.backupkeyid_backup.remove(outdated_key); - }) - .await; + Ok(version) + } + + pub async fn get_latest_backup_version(&self, user_id: &UserId) -> Result { + type Key<'a> = (OwnedUserId, &'a str); + + let last_possible_key = (user_id, u64::MAX); + self.db + .backupid_algorithm + .rev_keys_from(&last_possible_key) + .ignore_err() + .ready_take_while(|(user_id_, _): &Key<'_>| *user_id_ == user_id) + .map(|(_, version): Key<'_>| version.to_owned()) + .next() + .await + .ok_or_else(|| err!(Request(NotFound("No backup versions found")))) + } + + pub async fn get_latest_backup( + &self, + user_id: &UserId, + ) -> Result<(String, Raw)> { + type Key<'a> = (OwnedUserId, &'a str); + type KeyVal<'a> = (Key<'a>, Raw); + + let last_possible_key = (user_id, u64::MAX); + self.db + .backupid_algorithm + .rev_stream_from(&last_possible_key) + .ignore_err() + .ready_take_while(|((user_id_, _), _): &KeyVal<'_>| *user_id_ == user_id) + .map(|((_, version), algorithm): KeyVal<'_>| (version.to_owned(), algorithm)) + .next() + .await + .ok_or_else(|| err!(Request(NotFound("No backup found")))) + } + + pub async fn get_backup( + &self, + user_id: &UserId, + version: &str, + ) -> Result> { + let key = (user_id, version); + self.db.backupid_algorithm.qry(&key).await.deserialized() + } + + pub async fn add_key( + &self, + user_id: &UserId, + version: &str, + room_id: &RoomId, + session_id: &str, + key_data: &Raw, + ) -> Result<()> { + let key = (user_id, version); + if self.db.backupid_algorithm.qry(&key).await.is_err() { + return Err!(Request(NotFound("Tried to update nonexistent backup."))); + } + + let count = self.services.globals.next_count().unwrap(); + self.db.backupid_etag.put(key, count); + + let key = (user_id, version, room_id, session_id); + self.db + .backupkeyid_backup + .put_raw(key, key_data.json().get()); + + Ok(()) + } + + pub async fn count_keys(&self, user_id: &UserId, version: &str) -> usize { + let prefix = (user_id, version); + self.db + .backupkeyid_backup + .keys_prefix_raw(&prefix) + .count() + .await + } + + pub async fn get_etag(&self, user_id: &UserId, version: &str) -> String { + let key = (user_id, version); + self.db + .backupid_etag + .qry(&key) + .await + .deserialized::() + .as_ref() + .map(ToString::to_string) + .expect("Backup has no etag.") + } + + pub async fn get_all( + &self, + user_id: &UserId, + version: &str, + ) -> BTreeMap { + type Key<'a> = (Ignore, Ignore, OwnedRoomId, &'a str); + type KeyVal<'a> = (Key<'a>, Raw); + + let mut rooms = BTreeMap::::new(); + let default = || RoomKeyBackup::new(BTreeMap::new()); + + let prefix = (user_id, version, Interfix); + self.db + .backupkeyid_backup + .stream_prefix(&prefix) + .ignore_err() + .ready_for_each(|((_, _, room_id, session_id), key_backup_data): KeyVal<'_>| { + rooms + .entry(room_id) + .or_insert_with(default) + .sessions + .insert(session_id.into(), key_backup_data); + }) + .await; + + rooms + } + + pub async fn get_room( + &self, + user_id: &UserId, + version: &str, + room_id: &RoomId, + ) -> BTreeMap> { + type KeyVal<'a> = ((Ignore, Ignore, Ignore, &'a str), Raw); + + let prefix = (user_id, version, room_id, Interfix); + self.db + .backupkeyid_backup + .stream_prefix(&prefix) + .ignore_err() + .map(|((.., session_id), key_backup_data): KeyVal<'_>| { + (session_id.to_owned(), key_backup_data) + }) + .collect() + .await + } + + pub async fn get_session( + &self, + user_id: &UserId, + version: &str, + room_id: &RoomId, + session_id: &str, + ) -> Result> { + let key = (user_id, version, room_id, session_id); + + self.db.backupkeyid_backup.qry(&key).await.deserialized() + } + + pub async fn delete_all_keys(&self, user_id: &UserId, version: &str) { + let key = (user_id, version, Interfix); + self.db + .backupkeyid_backup + .keys_prefix_raw(&key) + .ignore_err() + .ready_for_each(|outdated_key| self.db.backupkeyid_backup.remove(outdated_key)) + .await; + } + + pub async fn delete_room_keys(&self, user_id: &UserId, version: &str, room_id: &RoomId) { + let key = (user_id, version, room_id, Interfix); + self.db + .backupkeyid_backup + .keys_prefix_raw(&key) + .ignore_err() + .ready_for_each(|outdated_key| { + self.db.backupkeyid_backup.remove(outdated_key); + }) + .await; + } + + pub async fn delete_room_key( + &self, + user_id: &UserId, + version: &str, + room_id: &RoomId, + session_id: &str, + ) { + let key = (user_id, version, room_id, session_id); + self.db + .backupkeyid_backup + .keys_prefix_raw(&key) + .ignore_err() + .ready_for_each(|outdated_key| { + self.db.backupkeyid_backup.remove(outdated_key); + }) + .await; + } } From 580d9b031837ad091d9c0c51f03ce5f9f5a75b82 Mon Sep 17 00:00:00 2001 From: timedout Date: Fri, 26 Jun 2026 05:24:42 +0100 Subject: [PATCH 49/70] style: Refactor federation service --- src/service/federation/execute.rs | 332 +++++++++++++++--------------- 1 file changed, 168 insertions(+), 164 deletions(-) diff --git a/src/service/federation/execute.rs b/src/service/federation/execute.rs index 0c147c1ef..ee588df7c 100644 --- a/src/service/federation/execute.rs +++ b/src/service/federation/execute.rs @@ -2,8 +2,7 @@ use std::{borrow::Cow, fmt::Debug, mem}; use bytes::Bytes; use conduwuit::{ - Err, Error, Result, debug, debug_error, debug_warn, err, implement, trace, - utils::response::LimitReadExt, + Err, Error, Result, debug, debug_error, debug_warn, err, trace, utils::response::LimitReadExt, }; use ipaddress::IPAddress; use reqwest::{Client, Method, Request, Response, Url}; @@ -20,181 +19,186 @@ use ruma::{ use crate::{SUPPORTED_VERSIONS, resolver::actual::ActualDest}; -/// Sends a request to a federation server -#[implement(super::Service)] -#[tracing::instrument(skip_all, name = "request", level = "debug")] -pub async fn execute<'i, T>(&self, dest: &ServerName, request: T) -> Result -where - T: OutgoingRequest< - Authentication = ServerSignatures, - PathBuilder: PathBuilder: FederationPathBuilderInput>, - > + Debug - + Send, -{ - let client = &self.services.client.federation; - self.execute_signed(client, dest, request).await -} - -/// Like execute() but with a very large timeout -#[implement(super::Service)] -#[tracing::instrument(skip_all, name = "synapse", level = "debug")] -pub async fn execute_synapse<'i, T>( - &self, - dest: &ServerName, - request: T, -) -> Result -where - T: OutgoingRequest< - Authentication = ServerSignatures, - PathBuilder: PathBuilder: FederationPathBuilderInput>, - > + Debug - + Send, -{ - let client = &self.services.client.synapse; - self.execute_signed(client, dest, request).await -} - -#[implement(super::Service)] -pub async fn execute_unauthenticated<'i, T>( - &self, - dest: &ServerName, - request: T, -) -> Result -where - T: OutgoingRequest< - Authentication = NoAuthentication, - PathBuilder: PathBuilder: FederationPathBuilderInput>, - > + Debug - + Send, -{ - let client = &self.services.client.federation; - - self.execute_on(client, dest, request, ()).await -} - -#[implement(super::Service)] -pub async fn execute_signed<'i, T>( - &self, - client: &Client, - dest: &ServerName, - request: T, -) -> Result -where - T: OutgoingRequest< - Authentication = ServerSignatures, - PathBuilder: PathBuilder: FederationPathBuilderInput>, - > + Send, -{ - let authentication = ServerSignaturesInput::new( - self.services.server.name.clone(), - dest.to_owned(), - self.services.server_keys.keypair(), - ); - - self.execute_on(client, dest, request, authentication).await -} - -#[implement(super::Service)] -#[tracing::instrument(name = "fed", level = "info", skip(self, client, request, authentication))] -pub async fn execute_on<'i, T, PathBuilderInput>( - &self, - client: &Client, - dest: &ServerName, - request: T, - authentication: ::Input<'_>, -) -> Result -where - T: OutgoingRequest = PathBuilderInput>> + Send, - PathBuilderInput: FederationPathBuilderInput, -{ - if !self.services.server.config.allow_federation { - return Err!(Config("allow_federation", "Federation is disabled.")); +impl super::Service { + /// Sends a request to a federation server + #[tracing::instrument(skip_all, name = "request", level = "debug")] + pub async fn execute<'i, T>( + &self, + dest: &ServerName, + request: T, + ) -> Result + where + T: OutgoingRequest< + Authentication = ServerSignatures, + PathBuilder: PathBuilder: FederationPathBuilderInput>, + > + Debug + + Send, + { + let client = &self.services.client.federation; + self.execute_signed(client, dest, request).await } - if self.services.moderation.is_remote_server_forbidden(dest) { - return Err!(Request(Forbidden(debug_warn!("Federation with {dest} is not allowed.")))); + /// Like execute() but with a very large timeout + #[tracing::instrument(skip_all, name = "synapse", level = "debug")] + pub async fn execute_synapse<'i, T>( + &self, + dest: &ServerName, + request: T, + ) -> Result + where + T: OutgoingRequest< + Authentication = ServerSignatures, + PathBuilder: PathBuilder: FederationPathBuilderInput>, + > + Debug + + Send, + { + let client = &self.services.client.synapse; + self.execute_signed(client, dest, request).await } - let actual = self.services.resolver.get_actual_dest(dest).await?; + pub async fn execute_unauthenticated<'i, T>( + &self, + dest: &ServerName, + request: T, + ) -> Result + where + T: OutgoingRequest< + Authentication = NoAuthentication, + PathBuilder: PathBuilder: FederationPathBuilderInput>, + > + Debug + + Send, + { + let client = &self.services.client.federation; - let request = Request::try_from(request.try_into_http_request::>( - actual.string().as_str(), - authentication, - PathBuilderInput::create(), - )?)?; - self.validate_url(request.url())?; - self.services.server.check_running()?; - - self.perform::(dest, &actual, request, client).await -} - -#[implement(super::Service)] -async fn perform( - &self, - dest: &ServerName, - actual: &ActualDest, - request: Request, - client: &Client, -) -> Result -where - T: OutgoingRequest + Send, -{ - let url = request.url().clone(); - let method = request.method().clone(); - - debug!(%method, %url, "Sending request"); - match client.execute(request).await { - | Ok(response) => - self.handle_response::(dest, actual, &method, &url, response) - .await, - | Err(error) => - Err(handle_error(actual, &method, &url, error).expect_err("always returns error")), + self.execute_on(client, dest, request, ()).await } -} -#[implement(super::Service)] -fn validate_url(&self, url: &Url) -> Result<()> { - if let Some(url_host) = url.host_str() { - if let Ok(ip) = IPAddress::parse(url_host) { - trace!("Checking request URL IP {ip:?}"); - self.services.resolver.validate_ip(&ip)?; + pub async fn execute_signed<'i, T>( + &self, + client: &Client, + dest: &ServerName, + request: T, + ) -> Result + where + T: OutgoingRequest< + Authentication = ServerSignatures, + PathBuilder: PathBuilder: FederationPathBuilderInput>, + > + Send, + { + let authentication = ServerSignaturesInput::new( + self.services.server.name.clone(), + dest.to_owned(), + self.services.server_keys.keypair(), + ); + + self.execute_on(client, dest, request, authentication).await + } + + #[tracing::instrument( + name = "fed", + level = "info", + skip(self, client, request, authentication) + )] + pub async fn execute_on<'i, T, PathBuilderInput>( + &self, + client: &Client, + dest: &ServerName, + request: T, + authentication: ::Input<'_>, + ) -> Result + where + T: OutgoingRequest = PathBuilderInput>> + Send, + PathBuilderInput: FederationPathBuilderInput, + { + if !self.services.server.config.allow_federation { + return Err!(Config("allow_federation", "Federation is disabled.")); + } + + if self.services.moderation.is_remote_server_forbidden(dest) { + return Err!(Request(Forbidden(debug_warn!( + "Federation with {dest} is not allowed." + )))); + } + + let actual = self.services.resolver.get_actual_dest(dest).await?; + + let request = Request::try_from(request.try_into_http_request::>( + actual.string().as_str(), + authentication, + PathBuilderInput::create(), + )?)?; + self.validate_url(request.url())?; + self.services.server.check_running()?; + + self.perform::(dest, &actual, request, client).await + } + + async fn perform( + &self, + dest: &ServerName, + actual: &ActualDest, + request: Request, + client: &Client, + ) -> Result + where + T: OutgoingRequest + Send, + { + let url = request.url().clone(); + let method = request.method().clone(); + + debug!(%method, %url, "Sending request"); + match client.execute(request).await { + | Ok(response) => + self.handle_response::(dest, actual, &method, &url, response) + .await, + | Err(error) => + Err(handle_error(actual, &method, &url, error).expect_err("always returns error")), } } - Ok(()) -} + fn validate_url(&self, url: &Url) -> Result<()> { + if let Some(url_host) = url.host_str() { + if let Ok(ip) = IPAddress::parse(url_host) { + trace!("Checking request URL IP {ip:?}"); + self.services.resolver.validate_ip(&ip)?; + } + } -#[implement(super::Service)] -async fn handle_response( - &self, - dest: &ServerName, - actual: &ActualDest, - method: &Method, - url: &Url, - response: Response, -) -> Result -where - T: OutgoingRequest + Send, -{ - const HUGE_ENDPOINTS: [&str; 2] = - ["/_matrix/federation/v2/send_join/", "/_matrix/federation/v2/state/"]; - let size_limit: u64 = if HUGE_ENDPOINTS.iter().any(|e| url.path().starts_with(e)) { - // Some federation endpoints can return huge response bodies, so we'll bump the - // limit for those endpoints specifically. - self.services - .server - .config - .max_request_size - .saturating_mul(10) - } else { - self.services.server.config.max_request_size + Ok(()) } - .try_into() - .expect("size_limit (usize) should fit within a u64"); - let response = into_http_response(dest, actual, method, url, response, size_limit).await?; - T::IncomingResponse::try_from_http_response(response) - .map_err(|e| err!(BadServerResponse("Server returned bad 200 response: {e:?}"))) + async fn handle_response( + &self, + dest: &ServerName, + actual: &ActualDest, + method: &Method, + url: &Url, + response: Response, + ) -> Result + where + T: OutgoingRequest + Send, + { + const HUGE_ENDPOINTS: [&str; 2] = + ["/_matrix/federation/v2/send_join/", "/_matrix/federation/v2/state/"]; + let size_limit: u64 = if HUGE_ENDPOINTS.iter().any(|e| url.path().starts_with(e)) { + // Some federation endpoints can return huge response bodies, so we'll bump the + // limit for those endpoints specifically. + self.services + .server + .config + .max_request_size + .saturating_mul(10) + } else { + self.services.server.config.max_request_size + } + .try_into() + .expect("size_limit (usize) should fit within a u64"); + let response = + into_http_response(dest, actual, method, url, response, size_limit).await?; + + T::IncomingResponse::try_from_http_response(response) + .map_err(|e| err!(BadServerResponse("Server returned bad 200 response: {e:?}"))) + } } async fn into_http_response( From 6f20b7bc3f1d49cd2a36c0d065f3e760c1eda500 Mon Sep 17 00:00:00 2001 From: timedout Date: Fri, 26 Jun 2026 05:26:17 +0100 Subject: [PATCH 50/70] style: Refactor config service --- src/service/config/mod.rs | 93 +++++++++++++++++++-------------------- 1 file changed, 46 insertions(+), 47 deletions(-) diff --git a/src/service/config/mod.rs b/src/service/config/mod.rs index d115d645f..a7effeb26 100644 --- a/src/service/config/mod.rs +++ b/src/service/config/mod.rs @@ -4,7 +4,7 @@ use async_trait::async_trait; use conduwuit::{ Result, Server, config::{Config, check}, - error, implement, + error, }; use url::Url; @@ -16,28 +16,6 @@ pub struct Service { const SIGNAL: &str = "SIGUSR1"; -impl Service { - /// Get the registration token set in the config file, if it exists. - #[must_use] - pub fn get_config_file_token(&self) -> Option { - self.registration_token - .clone() - .map(|token| ValidToken { token, source: ValidTokenSource::Config }) - } - - /// Get the base domain to use for user-facing URLs. - #[must_use] - pub fn get_client_domain(&self) -> Url { - self.well_known.client.clone().unwrap_or_else(|| { - let host = self.server_name.host(); - format!("https://{host}") - .as_str() - .try_into() - .expect("server name should be a valid host") - }) - } -} - #[async_trait] impl crate::Service for Service { fn build(args: crate::Args<'_>) -> Result> { @@ -66,32 +44,53 @@ impl Deref for Service { fn deref(&self) -> &Self::Target { &self.server.config } } -#[implement(Service)] -fn handle_reload(&self) -> Result { - if self.server.config.config_reload_signal { - #[cfg(all(feature = "systemd", target_os = "linux"))] - sd_notify::notify(&[ - sd_notify::NotifyState::Reloading, - sd_notify::NotifyState::monotonic_usec_now().expect("Failed to read monotonic time"), - ]) - .expect("failed to notify systemd of reloading state"); - - let config_paths = self.server.config.config_paths.clone().unwrap_or_default(); - self.reload(&config_paths)?; - - #[cfg(all(feature = "systemd", target_os = "linux"))] - sd_notify::notify(&[sd_notify::NotifyState::Ready]) - .expect("failed to notify systemd of ready state"); +impl Service { + /// Get the registration token set in the config file, if it exists. + #[must_use] + pub fn get_config_file_token(&self) -> Option { + self.registration_token + .clone() + .map(|token| ValidToken { token, source: ValidTokenSource::Config }) } - Ok(()) -} + /// Get the base domain to use for user-facing URLs. + #[must_use] + pub fn get_client_domain(&self) -> Url { + self.well_known.client.clone().unwrap_or_else(|| { + let host = self.server_name.host(); + format!("https://{host}") + .as_str() + .try_into() + .expect("server name should be a valid host") + }) + } -#[implement(Service)] -pub fn reload(&self, paths: &[PathBuf]) -> Result> { - let old = self.server.config.clone(); - let new = Config::load(paths).and_then(|raw| Config::new(&raw))?; + fn handle_reload(&self) -> Result { + if self.server.config.config_reload_signal { + #[cfg(all(feature = "systemd", target_os = "linux"))] + sd_notify::notify(&[ + sd_notify::NotifyState::Reloading, + sd_notify::NotifyState::monotonic_usec_now() + .expect("Failed to read monotonic time"), + ]) + .expect("failed to notify systemd of reloading state"); - check::reload(&old, &new)?; - self.server.config.update(new) + let config_paths = self.server.config.config_paths.clone().unwrap_or_default(); + self.reload(&config_paths)?; + + #[cfg(all(feature = "systemd", target_os = "linux"))] + sd_notify::notify(&[sd_notify::NotifyState::Ready]) + .expect("failed to notify systemd of ready state"); + } + + Ok(()) + } + + pub fn reload(&self, paths: &[PathBuf]) -> Result> { + let old = self.server.config.clone(); + let new = Config::load(paths).and_then(|raw| Config::new(&raw))?; + + check::reload(&old, &new)?; + self.server.config.update(new) + } } From 9b76c9a08545087aef1e6a4203b2699410b72bc7 Mon Sep 17 00:00:00 2001 From: timedout Date: Fri, 26 Jun 2026 05:57:59 +0100 Subject: [PATCH 51/70] feat: Support arbitrary room versions while creating the admin room Also changes the default power levels so people stop shooting their feet off --- src/service/admin/create.rs | 111 ++++++++++++++++++++++++------------ 1 file changed, 75 insertions(+), 36 deletions(-) diff --git a/src/service/admin/create.rs b/src/service/admin/create.rs index 15c25c6b5..50bb03ac8 100644 --- a/src/service/admin/create.rs +++ b/src/service/admin/create.rs @@ -1,20 +1,22 @@ -use std::collections::BTreeMap; - use conduwuit::{Result, info, pdu::PartialPdu}; use futures::FutureExt; use ruma::{ - RoomId, RoomVersionId, - events::room::{ - canonical_alias::RoomCanonicalAliasEventContent, - create::RoomCreateEventContent, - guest_access::{GuestAccess, RoomGuestAccessEventContent}, - history_visibility::{HistoryVisibility, RoomHistoryVisibilityEventContent}, - join_rules::{JoinRule, RoomJoinRulesEventContent}, - member::{MembershipState, RoomMemberEventContent}, - name::RoomNameEventContent, - power_levels::RoomPowerLevelsEventContent, - topic::RoomTopicEventContent, + Int, RoomId, + events::{ + TimelineEventType, + room::{ + canonical_alias::RoomCanonicalAliasEventContent, + create::RoomCreateEventContent, + guest_access::{GuestAccess, RoomGuestAccessEventContent}, + history_visibility::{HistoryVisibility, RoomHistoryVisibilityEventContent}, + join_rules::{JoinRule, RoomJoinRulesEventContent}, + member::{MembershipState, RoomMemberEventContent}, + name::RoomNameEventContent, + power_levels::RoomPowerLevelsEventContent, + topic::RoomTopicEventContent, + }, }, + room_version_rules::RoomIdFormatVersion, }; use crate::Services; @@ -24,47 +26,68 @@ use crate::Services; /// Users in this room are considered admins by conduwuit, and the room can be /// used to issue admin commands by talking to the server user inside it. pub async fn create_admin_room(services: &Services) -> Result { - let room_id = RoomId::new_v1(services.globals.server_name()); - let room_version = &RoomVersionId::V11; + let room_version = services.config.default_room_version.clone(); + let room_version_rules = room_version + .rules() + .expect("default_room_version must be supported"); + let room_id = match room_version_rules.room_id_format { + | RoomIdFormatVersion::V1 => { + let room_id = RoomId::new_v1(services.globals.server_name()); + services + .rooms + .short + .get_or_create_shortroomid(&room_id) + .await; + Some(room_id) + }, + | RoomIdFormatVersion::V2 => None, + | _ => panic!("Unknown room version format"), + }; - let _short_id = services - .rooms - .short - .get_or_create_shortroomid(&room_id) - .await; - - let state_lock = services.rooms.state.mutex.lock(room_id.as_str()).await; + let state_lock = services.rooms.state.mutex.lock("!new-room").await; // Create a user for the server let server_user = services.globals.server_user.as_ref(); services.users.create(server_user, None).await?; - let mut create_content = { - use RoomVersionId::*; - match room_version { - | V1 | V2 | V3 | V4 | V5 | V6 | V7 | V8 | V9 | V10 => - RoomCreateEventContent::new_v1(server_user.into()), - | _ => RoomCreateEventContent::new_v11(), - } + let mut create_content = if room_version_rules.authorization.use_room_create_sender { + RoomCreateEventContent::new_v1(server_user.into()) + } else { + RoomCreateEventContent::new_v11() }; create_content.federate = true; create_content.room_version = room_version.clone(); - info!("Creating admin room {} with version {}", room_id, room_version); + info!( + "Creating admin room {} with version {}", + room_id + .clone() + .map_or_else(|| "".to_owned(), |id| id.as_str().to_owned()), + room_version + ); // 1. The room create event - services + let create_event_id = services .rooms .timeline .build_and_append_pdu( PartialPdu::state(String::new(), &create_content), server_user, - Some(&room_id), + room_id.as_deref(), &state_lock, ) .boxed() .await?; + let room_id = room_id.unwrap_or_else(|| { + RoomId::new_v2( + create_event_id + .as_str() + .strip_prefix("$") + .expect("event ID must start with a $ sigil"), + ) + .expect("event ID without sigil must be a valid room ID") + }); // 2. Make server user/bot join services @@ -83,11 +106,27 @@ pub async fn create_admin_room(services: &Services) -> Result { .await?; // 3. Power levels - let users = BTreeMap::from_iter([(server_user.into(), 69420.into())]); - let mut power_levels_content = - RoomPowerLevelsEventContent::new(&room_version.rules().unwrap().authorization); - power_levels_content.users = users; + RoomPowerLevelsEventContent::new(&room_version_rules.authorization); + if !room_version_rules + .authorization + .explicitly_privilege_room_creators + { + power_levels_content + .users + .insert(server_user.into(), Int::MAX); + } + // Prevent common foot-shotguns + power_levels_content + .events + .insert(TimelineEventType::RoomTombstone, Int::MAX); + power_levels_content + .events + .insert(TimelineEventType::RoomEncryption, Int::MAX); + power_levels_content + .events + .insert(TimelineEventType::RoomCanonicalAlias, Int::MAX); + power_levels_content.invite = power_levels_content.state_default; services .rooms From 0bb97a53c545c7e0c9b366bfe77fb3467d6f5c48 Mon Sep 17 00:00:00 2001 From: timedout Date: Fri, 26 Jun 2026 05:59:48 +0100 Subject: [PATCH 52/70] style: Refactor account_data service --- src/service/account_data/mod.rs | 207 ++++++++++++++++---------------- 1 file changed, 104 insertions(+), 103 deletions(-) diff --git a/src/service/account_data/mod.rs b/src/service/account_data/mod.rs index 9fb17adaa..5622826a5 100644 --- a/src/service/account_data/mod.rs +++ b/src/service/account_data/mod.rs @@ -1,7 +1,7 @@ use std::sync::Arc; use conduwuit::{ - Err, Result, err, implement, + Err, Result, err, utils::{ReadyExt, result::LogErr, stream::TryIgnore}, }; use database::{Deserialized, Handle, Ignore, Json, Map}; @@ -54,114 +54,115 @@ impl crate::Service for Service { fn name(&self) -> &str { crate::service::make_name(std::module_path!()) } } -/// Places one event in the account data of the user and removes the -/// previous entry. -#[allow(clippy::needless_pass_by_value)] -#[implement(Service)] -pub async fn update( - &self, - room_id: Option<&RoomId>, - user_id: &UserId, - event_type: RoomAccountDataEventType, - data: &serde_json::Value, -) -> Result<()> { - if data.get("type").is_none() || data.get("content").is_none() { - return Err!(Request(InvalidParam("Account data doesn't have all required fields."))); +impl Service { + /// Places one event in the account data of the user and removes the + /// previous entry. + #[allow(clippy::needless_pass_by_value)] + pub async fn update( + &self, + room_id: Option<&RoomId>, + user_id: &UserId, + event_type: RoomAccountDataEventType, + data: &serde_json::Value, + ) -> Result<()> { + if data.get("type").is_none() || data.get("content").is_none() { + return Err!(Request(InvalidParam("Account data doesn't have all required fields."))); + } + + let count = self.services.globals.next_count().unwrap(); + let roomuserdataid = (room_id, user_id, count, &event_type); + self.db + .roomuserdataid_accountdata + .put(roomuserdataid, Json(data)); + + let key = (room_id, user_id, &event_type); + let prev = self.db.roomusertype_roomuserdataid.qry(&key).await; + self.db.roomusertype_roomuserdataid.put(key, roomuserdataid); + + // Remove old entry + if let Ok(prev) = prev { + self.db.roomuserdataid_accountdata.remove(&prev); + } + + Ok(()) } - let count = self.services.globals.next_count().unwrap(); - let roomuserdataid = (room_id, user_id, count, &event_type); - self.db - .roomuserdataid_accountdata - .put(roomuserdataid, Json(data)); - - let key = (room_id, user_id, &event_type); - let prev = self.db.roomusertype_roomuserdataid.qry(&key).await; - self.db.roomusertype_roomuserdataid.put(key, roomuserdataid); - - // Remove old entry - if let Ok(prev) = prev { - self.db.roomuserdataid_accountdata.remove(&prev); + /// Searches the room account data for a specific kind. + pub async fn get_global( + &self, + user_id: &UserId, + kind: GlobalAccountDataEventType, + ) -> Result + where + T: for<'de> Deserialize<'de>, + { + self.get_raw(None, user_id, &kind.to_string()) + .await + .deserialized() } - Ok(()) -} + /// Searches the global account data for a specific kind. + pub async fn get_room( + &self, + room_id: &RoomId, + user_id: &UserId, + kind: RoomAccountDataEventType, + ) -> Result + where + T: for<'de> Deserialize<'de>, + { + self.get_raw(Some(room_id), user_id, &kind.to_string()) + .await + .deserialized() + } -/// Searches the room account data for a specific kind. -#[implement(Service)] -pub async fn get_global(&self, user_id: &UserId, kind: GlobalAccountDataEventType) -> Result -where - T: for<'de> Deserialize<'de>, -{ - self.get_raw(None, user_id, &kind.to_string()) - .await - .deserialized() -} + pub async fn get_raw( + &self, + room_id: Option<&RoomId>, + user_id: &UserId, + kind: &str, + ) -> Result> { + let key = (room_id, user_id, kind.to_owned()); + self.db + .roomusertype_roomuserdataid + .qry(&key) + .and_then(|roomuserdataid| self.db.roomuserdataid_accountdata.get(&roomuserdataid)) + .await + } -/// Searches the global account data for a specific kind. -#[implement(Service)] -pub async fn get_room( - &self, - room_id: &RoomId, - user_id: &UserId, - kind: RoomAccountDataEventType, -) -> Result -where - T: for<'de> Deserialize<'de>, -{ - self.get_raw(Some(room_id), user_id, &kind.to_string()) - .await - .deserialized() -} + /// Returns all changes to the account data that happened after `since`. + pub fn changes_since<'a>( + &'a self, + room_id: Option<&'a RoomId>, + user_id: &'a UserId, + since: Option, + to: Option, + ) -> impl Stream + Send + 'a { + type Key = (Option, OwnedUserId, u64, Ignore); -#[implement(Service)] -pub async fn get_raw( - &self, - room_id: Option<&RoomId>, - user_id: &UserId, - kind: &str, -) -> Result> { - let key = (room_id, user_id, kind.to_owned()); - self.db - .roomusertype_roomuserdataid - .qry(&key) - .and_then(|roomuserdataid| self.db.roomuserdataid_accountdata.get(&roomuserdataid)) - .await -} + // Skip the data that's exactly at since, because we sent that last time + // ...unless this is an initial sync, in which case send everything + let first_possible = (room_id, user_id, since.map_or(0, |since| since.saturating_add(1))); -/// Returns all changes to the account data that happened after `since`. -#[implement(Service)] -pub fn changes_since<'a>( - &'a self, - room_id: Option<&'a RoomId>, - user_id: &'a UserId, - since: Option, - to: Option, -) -> impl Stream + Send + 'a { - type Key = (Option, OwnedUserId, u64, Ignore); - - // Skip the data that's exactly at since, because we sent that last time - // ...unless this is an initial sync, in which case send everything - let first_possible = (room_id, user_id, since.map_or(0, |since| since.saturating_add(1))); - - self.db - .roomuserdataid_accountdata - .stream_from(&first_possible) - .ignore_err() - .ready_take_while(move |((room_id_, user_id_, count, _), _): &(Key, _)| { - room_id == room_id_.as_deref() - && user_id == user_id_ - && to.is_none_or(|to| *count <= to) - }) - .map(move |(_, v)| { - match room_id { - | Some(_) => serde_json::from_slice::>(v) - .map(AnyRawAccountDataEvent::Room), - | None => serde_json::from_slice::>(v) - .map(AnyRawAccountDataEvent::Global), - } - .map_err(|e| err!(Database("Database contains invalid account data: {e}"))) - .log_err() - }) - .ignore_err() + self.db + .roomuserdataid_accountdata + .stream_from(&first_possible) + .ignore_err() + .ready_take_while(move |((room_id_, user_id_, count, _), _): &(Key, _)| { + room_id == room_id_.as_deref() + && user_id == user_id_ + && to.is_none_or(|to| *count <= to) + }) + .map(move |(_, v)| { + match room_id { + | Some(_) => serde_json::from_slice::>(v) + .map(AnyRawAccountDataEvent::Room), + | None => serde_json::from_slice::>(v) + .map(AnyRawAccountDataEvent::Global), + } + .map_err(|e| err!(Database("Database contains invalid account data: {e}"))) + .log_err() + }) + .ignore_err() + } } From fed9a599d97d0f9c2bf9723b24126e65fd82eebe Mon Sep 17 00:00:00 2001 From: timedout Date: Fri, 26 Jun 2026 06:09:43 +0100 Subject: [PATCH 53/70] style: Refactor remaining parts of admin service --- src/service/admin/execute.rs | 207 ++++++++--------- src/service/admin/grant.rs | 432 +++++++++++++++++++---------------- src/service/client/mod.rs | 21 +- 3 files changed, 344 insertions(+), 316 deletions(-) diff --git a/src/service/admin/execute.rs b/src/service/admin/execute.rs index 0df7b9882..9aeb80ce3 100644 --- a/src/service/admin/execute.rs +++ b/src/service/admin/execute.rs @@ -1,4 +1,4 @@ -use conduwuit::{Err, Result, debug, debug_info, error, implement, info}; +use conduwuit::{Err, Result, debug, debug_info, error, info}; use ruma::events::room::message::RoomMessageEventContent; use tokio::time::{Duration, sleep}; @@ -6,127 +6,120 @@ use crate::admin::InvocationSource; pub(super) const SIGNAL: &str = "SIGUSR2"; -/// Possibly spawn the terminal console at startup if configured. -#[implement(super::Service)] -pub(super) async fn console_auto_start(&self) { - #[cfg(feature = "console")] - if self.services.server.config.admin_console_automatic { - // Allow more of the startup sequence to execute before spawning - tokio::task::yield_now().await; - self.console.start().await; +impl super::Service { + /// Possibly spawn the terminal console at startup if configured. + pub(super) async fn console_auto_start(&self) { + #[cfg(feature = "console")] + if self.services.server.config.admin_console_automatic { + // Allow more of the startup sequence to execute before spawning + tokio::task::yield_now().await; + self.console.start().await; + } } -} -/// Shutdown the console when the admin worker terminates. -#[implement(super::Service)] -pub(super) async fn console_auto_stop(&self) { - #[cfg(feature = "console")] - self.console.close().await; -} + /// Shutdown the console when the admin worker terminates. + pub(super) async fn console_auto_stop(&self) { + #[cfg(feature = "console")] + self.console.close().await; + } -/// Execute admin commands after startup -#[implement(super::Service)] -pub async fn startup_execute(&self) -> Result { - // List of commands to execute - let commands = &self.services.server.config.admin_execute; + /// Execute admin commands after startup + pub async fn startup_execute(&self) -> Result { + // List of commands to execute + let commands = &self.services.server.config.admin_execute; - // Determine if we're running in smoketest-mode which will change some behaviors - let smoketest = self.services.server.config.test.contains("smoke"); + // Determine if we're running in smoketest-mode which will change some behaviors + let smoketest = self.services.server.config.test.contains("smoke"); - // When true, errors are ignored and startup continues. - let errors = !smoketest && self.services.server.config.admin_execute_errors_ignore; + // When true, errors are ignored and startup continues. + let errors = !smoketest && self.services.server.config.admin_execute_errors_ignore; - //TODO: remove this after run-states are broadcast - sleep(Duration::from_millis(500)).await; + //TODO: remove this after run-states are broadcast + sleep(Duration::from_millis(500)).await; - for (i, command) in commands.iter().enumerate() { - if let Err(e) = self.execute_command(i, command.clone()).await { - if !errors { - return Err(e); + for (i, command) in commands.iter().enumerate() { + if let Err(e) = self.execute_command(i, command.clone()).await { + if !errors { + return Err(e); + } } + + tokio::task::yield_now().await; } - tokio::task::yield_now().await; - } - - // The smoketest functionality is placed here for now and simply initiates - // shutdown after all commands have executed. - if smoketest { - debug_info!("Smoketest mode. All commands complete. Shutting down now..."); - self.services - .server - .shutdown() - .inspect_err(error::inspect_log) - .expect("Error shutting down from smoketest"); - } - - Ok(()) -} - -/// Execute admin commands after signal -#[implement(super::Service)] -pub(super) async fn signal_execute(&self) -> Result { - // List of commands to execute - let commands = self.services.server.config.admin_signal_execute.clone(); - - // When true, errors are ignored and execution continues. - let ignore_errors = self.services.server.config.admin_execute_errors_ignore; - - for (i, command) in commands.iter().enumerate() { - if let Err(e) = self.execute_command(i, command.clone()).await { - if !ignore_errors { - return Err(e); - } + // The smoketest functionality is placed here for now and simply initiates + // shutdown after all commands have executed. + if smoketest { + debug_info!("Smoketest mode. All commands complete. Shutting down now..."); + self.services + .server + .shutdown() + .inspect_err(error::inspect_log) + .expect("Error shutting down from smoketest"); } - tokio::task::yield_now().await; + Ok(()) } - Ok(()) -} + /// Execute admin commands after signal + pub(super) async fn signal_execute(&self) -> Result { + // List of commands to execute + let commands = self.services.server.config.admin_signal_execute.clone(); -/// Execute one admin command after startup or signal -#[implement(super::Service)] -async fn execute_command(&self, i: usize, command: String) -> Result { - debug!("Execute command #{i}: executing {command:?}"); + // When true, errors are ignored and execution continues. + let ignore_errors = self.services.server.config.admin_execute_errors_ignore; - match self - .command_in_place(command, None, InvocationSource::Console) - .await - { - | Ok(Some(output)) => Self::execute_command_output(i, &output), - | Err(output) => Self::execute_command_error(i, &output), - | Ok(None) => { - info!("Execute command #{i} completed (no output)."); - Ok(()) - }, + for (i, command) in commands.iter().enumerate() { + if let Err(e) = self.execute_command(i, command.clone()).await { + if !ignore_errors { + return Err(e); + } + } + + tokio::task::yield_now().await; + } + + Ok(()) + } + + /// Execute one admin command after startup or signal + async fn execute_command(&self, i: usize, command: String) -> Result { + debug!("Execute command #{i}: executing {command:?}"); + + match self + .command_in_place(command, None, InvocationSource::Console) + .await + { + | Ok(Some(output)) => Self::execute_command_output(i, &output), + | Err(output) => Self::execute_command_error(i, &output), + | Ok(None) => { + info!("Execute command #{i} completed (no output)."); + Ok(()) + }, + } + } + + #[cfg(feature = "console")] + fn execute_command_output(i: usize, content: &RoomMessageEventContent) -> Result { + debug_info!("Execute command #{i} completed:"); + super::console::print(content.body()); + Ok(()) + } + + #[cfg(feature = "console")] + fn execute_command_error(i: usize, content: &RoomMessageEventContent) -> Result { + super::console::print_err(content.body()); + Err!(debug_error!("Execute command #{i} failed.")) + } + + #[cfg(not(feature = "console"))] + fn execute_command_output(i: usize, content: &RoomMessageEventContent) -> Result { + info!("Execute command #{i} completed:\n{:#}", content.body()); + Ok(()) + } + + #[cfg(not(feature = "console"))] + fn execute_command_error(i: usize, content: &RoomMessageEventContent) -> Result { + Err!(error!("Execute command #{i} failed:\n{:#}", content.body())) } } - -#[cfg(feature = "console")] -#[implement(super::Service)] -fn execute_command_output(i: usize, content: &RoomMessageEventContent) -> Result { - debug_info!("Execute command #{i} completed:"); - super::console::print(content.body()); - Ok(()) -} - -#[cfg(feature = "console")] -#[implement(super::Service)] -fn execute_command_error(i: usize, content: &RoomMessageEventContent) -> Result { - super::console::print_err(content.body()); - Err!(debug_error!("Execute command #{i} failed.")) -} - -#[cfg(not(feature = "console"))] -#[implement(super::Service)] -fn execute_command_output(i: usize, content: &RoomMessageEventContent) -> Result { - info!("Execute command #{i} completed:\n{:#}", content.body()); - Ok(()) -} - -#[cfg(not(feature = "console"))] -#[implement(super::Service)] -fn execute_command_error(i: usize, content: &RoomMessageEventContent) -> Result { - Err!(error!("Execute command #{i} failed:\n{:#}", content.body())) -} diff --git a/src/service/admin/grant.rs b/src/service/admin/grant.rs index 624c6eade..938a15019 100644 --- a/src/service/admin/grant.rs +++ b/src/service/admin/grant.rs @@ -1,10 +1,8 @@ use std::collections::BTreeMap; -use conduwuit::{ - Err, Result, debug_info, debug_warn, error, implement, matrix::pdu::PartialPdu, warn, -}; +use conduwuit::{Err, Result, debug_info, debug_warn, error, matrix::pdu::PartialPdu, warn}; use ruma::{ - RoomId, UserId, + Int, RoomId, UserId, events::{ RoomAccountDataEventType, StateEventType, room::{ @@ -13,205 +11,241 @@ use ruma::{ }, tag::{TagEvent, TagEventContent, TagInfo}, }, + int, }; -/// Invite the user to the conduwuit admin room. -/// -/// This is equivalent to granting server admin privileges. -#[implement(super::Service)] -pub async fn make_user_admin(&self, user_id: &UserId) -> Result { - let Ok(room_id) = self.get_admin_room().await else { - debug_warn!( - "make_user_admin was called without an admin room being available or created" - ); - return Ok(()); - }; - - let state_lock = self.services.state.mutex.lock(room_id.as_str()).await; - - if self.services.state_cache.is_joined(user_id, &room_id).await { - return Err!(debug_warn!("User is already joined in the admin room")); - } - if self - .services - .state_cache - .is_invited(user_id, &room_id) - .await - { - return Err!(debug_warn!("User is already pending an invitation to the admin room")); - } - - // Use the server user to grant the new admin's power level - let server_user = self.services.globals.server_user.as_ref(); - - // if this is our local user, just forcefully join them in the room. otherwise, - // invite the remote user. - if self.services.globals.user_is_local(user_id) { - debug_info!("Inviting local user {user_id} to admin room {room_id}"); - self.services - .timeline - .build_and_append_pdu( - PartialPdu::state( - String::from(user_id), - &RoomMemberEventContent::new(MembershipState::Invite), - ), - server_user, - Some(&room_id), - &state_lock, - ) - .await?; - - debug_info!("Force joining local user {user_id} to admin room {room_id}"); - self.services - .timeline - .build_and_append_pdu( - PartialPdu::state( - String::from(user_id), - &RoomMemberEventContent::new(MembershipState::Join), - ), - user_id, - Some(&room_id), - &state_lock, - ) - .await?; - } else { - debug_info!("Inviting remote user {user_id} to admin room {room_id}"); - self.services - .timeline - .build_and_append_pdu( - PartialPdu::state( - user_id.to_string(), - &RoomMemberEventContent::new(MembershipState::Invite), - ), - server_user, - Some(&room_id), - &state_lock, - ) - .await?; - } - - // Set power levels - let mut room_power_levels = self - .services - .state_accessor - .room_state_get_content::( - &room_id, - &StateEventType::RoomPowerLevels, - "", - ) - .await - .expect("admin room should have power levels"); - - room_power_levels - .users - .insert(server_user.into(), 69420.into()); - room_power_levels.users.insert(user_id.into(), 100.into()); - - self.services - .timeline - .build_and_append_pdu( - PartialPdu::state(String::new(), &room_power_levels), - server_user, - Some(&room_id), - &state_lock, - ) - .await?; - - // Set room tag - let room_tag = self.services.server.config.admin_room_tag.as_str(); - if !room_tag.is_empty() { - if let Err(e) = self.set_room_tag(&room_id, user_id, room_tag).await { - error!(%room_id, %user_id, %room_tag, "Failed to set tag for admin grant: {e}"); - } - } - - Ok(()) -} - -#[implement(super::Service)] -async fn set_room_tag(&self, room_id: &RoomId, user_id: &UserId, tag: &str) -> Result { - let mut event = self - .services - .account_data - .get_room(room_id, user_id, RoomAccountDataEventType::Tag) - .await - .unwrap_or_else(|_| TagEvent::new(TagEventContent::new(BTreeMap::new()))); - - event - .content - .tags - .insert(tag.to_owned().into(), TagInfo::new()); - - self.services - .account_data - .update( - Some(room_id), - user_id, - RoomAccountDataEventType::Tag, - &serde_json::to_value(event)?, - ) - .await -} - -/// Demote an admin, removing its rights. -#[implement(super::Service)] -pub async fn revoke_admin(&self, user_id: &UserId) -> Result { - use MembershipState::{Invite, Join, Knock, Leave}; - - if self - .services - .server - .config - .admins_list - .contains(&user_id.to_owned()) - { - warn!( - "Revoking the admin status of {user_id} will not work correctly as they are within \ - the admins_list config." - ); - } - - let Ok(room_id) = self.get_admin_room().await else { - return Err!(error!("No admin room available or created.")); - }; - - let state_lock = self.services.state.mutex.lock(room_id.as_str()).await; - - let mut member_content = match self - .services - .state_accessor - .get_member(&room_id, user_id) - .await - { - | Err(e) if e.is_not_found() => return Err!("{user_id} was never an admin."), - - | Err(e) => return Err!(error!(?e, "Failure occurred while attempting revoke.")), - - | Ok(event) if !matches!(event.membership, Invite | Knock | Join) => { - return Err!("Cannot revoke {user_id} in membership state {:?}.", event.membership); - }, - - | Ok(event) => { - assert!( - matches!(event.membership, Invite | Knock | Join), - "Incorrect membership state to remove user." +impl super::Service { + /// Invite the user to the conduwuit admin room. + /// + /// This is equivalent to granting server admin privileges. + pub async fn make_user_admin(&self, user_id: &UserId) -> Result { + let Ok(room_id) = self.get_admin_room().await else { + debug_warn!( + "make_user_admin was called without an admin room being available or created" ); + return Ok(()); + }; - event - }, - }; + let state_lock = self.services.state.mutex.lock(room_id.as_str()).await; - member_content.membership = Leave; - member_content.reason = Some("Admin Revoked".to_owned()); + if self.services.state_cache.is_joined(user_id, &room_id).await { + return Err!(debug_warn!("User is already joined in the admin room")); + } + if self + .services + .state_cache + .is_invited(user_id, &room_id) + .await + { + return Err!(debug_warn!("User is already pending an invitation to the admin room")); + } - self.services - .timeline - .build_and_append_pdu( - PartialPdu::state(user_id.to_string(), &member_content), - self.services.globals.server_user.as_ref(), - Some(&room_id), - &state_lock, - ) - .await - .map(|_| ()) + // Use the server user to grant the new admin's power level + let server_user = self.services.globals.server_user.as_ref(); + + // if this is our local user, just forcefully join them in the room. otherwise, + // invite the remote user. + if self.services.globals.user_is_local(user_id) { + debug_info!("Inviting local user {user_id} to admin room {room_id}"); + self.services + .timeline + .build_and_append_pdu( + PartialPdu::state( + String::from(user_id), + &RoomMemberEventContent::new(MembershipState::Invite), + ), + server_user, + Some(&room_id), + &state_lock, + ) + .await?; + + debug_info!("Force joining local user {user_id} to admin room {room_id}"); + self.services + .timeline + .build_and_append_pdu( + PartialPdu::state( + String::from(user_id), + &RoomMemberEventContent::new(MembershipState::Join), + ), + user_id, + Some(&room_id), + &state_lock, + ) + .await?; + } else { + debug_info!("Inviting remote user {user_id} to admin room {room_id}"); + self.services + .timeline + .build_and_append_pdu( + PartialPdu::state( + user_id.to_string(), + &RoomMemberEventContent::new(MembershipState::Invite), + ), + server_user, + Some(&room_id), + &state_lock, + ) + .await?; + } + + // Set power levels + let mut room_power_levels = self + .services + .state_accessor + .room_state_get_content::( + &room_id, + &StateEventType::RoomPowerLevels, + "", + ) + .await + .expect("admin room should have power levels"); + + let server_user_power_level = room_power_levels + .users + .get(server_user) + .copied() + .unwrap_or(Int::MAX); + room_power_levels + .users + .insert(user_id.into(), server_user_power_level.saturating_sub(int!(1))); + + self.services + .timeline + .build_and_append_pdu( + PartialPdu::state(String::new(), &room_power_levels), + server_user, + Some(&room_id), + &state_lock, + ) + .await?; + + // Set room tag + let room_tag = self.services.server.config.admin_room_tag.as_str(); + if !room_tag.is_empty() { + if let Err(e) = self.set_room_tag(&room_id, user_id, room_tag).await { + error!(%room_id, %user_id, %room_tag, "Failed to set tag for admin grant: {e}"); + } + } + + Ok(()) + } + + async fn set_room_tag(&self, room_id: &RoomId, user_id: &UserId, tag: &str) -> Result { + let mut event = self + .services + .account_data + .get_room(room_id, user_id, RoomAccountDataEventType::Tag) + .await + .unwrap_or_else(|_| TagEvent::new(TagEventContent::new(BTreeMap::new()))); + + event + .content + .tags + .insert(tag.to_owned().into(), TagInfo::new()); + + self.services + .account_data + .update( + Some(room_id), + user_id, + RoomAccountDataEventType::Tag, + &serde_json::to_value(event)?, + ) + .await + } + + /// Demote an admin, removing its rights. + pub async fn revoke_admin(&self, user_id: &UserId) -> Result { + use MembershipState::{Invite, Join, Knock, Leave}; + + if self + .services + .server + .config + .admins_list + .contains(&user_id.to_owned()) + { + warn!( + "Revoking the admin status of {user_id} will not work correctly as they are \ + within the admins_list config." + ); + } + + let Ok(room_id) = self.get_admin_room().await else { + return Err!(error!("No admin room available or created.")); + }; + + let state_lock = self.services.state.mutex.lock(room_id.as_str()).await; + + let mut room_power_levels = self + .services + .state_accessor + .room_state_get_content::( + &room_id, + &StateEventType::RoomPowerLevels, + "", + ) + .await + .expect("admin room should have power levels"); + if room_power_levels.users.remove(user_id).is_some() { + // drop the target's power level + self.services + .timeline + .build_and_append_pdu( + PartialPdu::state(String::new(), &room_power_levels), + self.services.globals.server_user.as_ref(), + Some(&room_id), + &state_lock, + ) + .await + .inspect_err(|e| { + error!(?e, "Failed to update power levels while revoking admin."); + }) + .ok(); + // Don't treat this as fatal. removing them from the room is more + // important. + } + + let mut member_content = match self + .services + .state_accessor + .get_member(&room_id, user_id) + .await + { + | Err(e) if e.is_not_found() => return Err!("{user_id} was never an admin."), + + | Err(e) => return Err!(error!(?e, "Failure occurred while attempting revoke.")), + + | Ok(event) if !matches!(event.membership, Invite | Knock | Join) => { + return Err!( + "Cannot revoke {user_id} in membership state {:?}.", + event.membership + ); + }, + + | Ok(event) => { + assert!( + matches!(event.membership, Invite | Knock | Join), + "Incorrect membership state to remove user." + ); + + event + }, + }; + + member_content.membership = Leave; + member_content.reason = Some("Admin Revoked".to_owned()); + + self.services + .timeline + .build_and_append_pdu( + PartialPdu::state(user_id.to_string(), &member_content), + self.services.globals.server_user.as_ref(), + Some(&room_id), + &state_lock, + ) + .await + .map(|_| ()) + } } diff --git a/src/service/client/mod.rs b/src/service/client/mod.rs index 4792bd7ad..8492714da 100644 --- a/src/service/client/mod.rs +++ b/src/service/client/mod.rs @@ -1,6 +1,6 @@ use std::{sync::Arc, time::Duration}; -use conduwuit::{Config, Result, err, implement, trace}; +use conduwuit::{Config, Result, err, trace}; use either::Either; use ipaddress::IPAddress; use reqwest::redirect; @@ -141,6 +141,16 @@ impl crate::Service for Service { fn name(&self) -> &str { service::make_name(std::module_path!()) } } +impl Service { + #[inline] + #[must_use] + pub fn valid_cidr_range(&self, ip: &IPAddress) -> bool { + self.cidr_range_denylist + .iter() + .all(|cidr| !cidr.includes(ip)) + } +} + fn base(config: &Config) -> Result { let mut builder = reqwest::Client::builder() .hickory_dns(true) @@ -227,12 +237,3 @@ fn builder_interface( Ok(builder) } } - -#[inline] -#[must_use] -#[implement(Service)] -pub fn valid_cidr_range(&self, ip: &IPAddress) -> bool { - self.cidr_range_denylist - .iter() - .all(|cidr| !cidr.includes(ip)) -} From c54e56a73eeb626537441ad962f6cb5e458bc8b4 Mon Sep 17 00:00:00 2001 From: timedout Date: Fri, 26 Jun 2026 06:11:22 +0100 Subject: [PATCH 54/70] style: Remove `implement` for AccessCheck, rename `check` to `assert` --- src/api/server/backfill.rs | 2 +- src/api/server/event.rs | 2 +- src/api/server/event_auth.rs | 2 +- src/api/server/get_missing_events.rs | 2 +- src/api/server/state.rs | 2 +- src/api/server/state_ids.rs | 2 +- src/api/server/utils.rs | 88 +++++++++++++++------------- 7 files changed, 52 insertions(+), 48 deletions(-) diff --git a/src/api/server/backfill.rs b/src/api/server/backfill.rs index 7e0b18f00..73cb0f7ae 100644 --- a/src/api/server/backfill.rs +++ b/src/api/server/backfill.rs @@ -32,7 +32,7 @@ pub(crate) async fn get_backfill_route( room_id: &body.room_id, event_id: None, } - .check() + .assert() .await?; if !services .rooms diff --git a/src/api/server/event.rs b/src/api/server/event.rs index ff82f3d00..9e0c8d751 100644 --- a/src/api/server/event.rs +++ b/src/api/server/event.rs @@ -44,7 +44,7 @@ pub(crate) async fn get_event_route( room_id, event_id: Some(&body.event_id), } - .check() + .assert() .await?; if !services diff --git a/src/api/server/event_auth.rs b/src/api/server/event_auth.rs index 66fae5e01..fd24cefd1 100644 --- a/src/api/server/event_auth.rs +++ b/src/api/server/event_auth.rs @@ -23,7 +23,7 @@ pub(crate) async fn get_event_authorization_route( room_id: &body.room_id, event_id: None, } - .check() + .assert() .await?; if services diff --git a/src/api/server/get_missing_events.rs b/src/api/server/get_missing_events.rs index a4915bb8b..a5b3c41d3 100644 --- a/src/api/server/get_missing_events.rs +++ b/src/api/server/get_missing_events.rs @@ -26,7 +26,7 @@ pub(crate) async fn get_missing_events_route( room_id: &body.room_id, event_id: None, } - .check() + .assert() .await?; if !services diff --git a/src/api/server/state.rs b/src/api/server/state.rs index 709c62fb6..90aa4253e 100644 --- a/src/api/server/state.rs +++ b/src/api/server/state.rs @@ -21,7 +21,7 @@ pub(crate) async fn get_room_state_route( room_id: &body.room_id, event_id: None, } - .check() + .assert() .await?; if services diff --git a/src/api/server/state_ids.rs b/src/api/server/state_ids.rs index bd7414f7a..81fadb167 100644 --- a/src/api/server/state_ids.rs +++ b/src/api/server/state_ids.rs @@ -22,7 +22,7 @@ pub(crate) async fn get_room_state_ids_route( room_id: &body.room_id, event_id: None, } - .check() + .assert() .await?; if services diff --git a/src/api/server/utils.rs b/src/api/server/utils.rs index e698f235e..7b9103876 100644 --- a/src/api/server/utils.rs +++ b/src/api/server/utils.rs @@ -10,52 +10,56 @@ pub(super) struct AccessCheck<'a> { pub(super) event_id: Option<&'a EventId>, } -#[implement(AccessCheck, params = "<'_>")] -pub(super) async fn check(&self) -> Result { - let acl_check = self - .services - .rooms - .event_handler - .acl_check(self.origin, self.room_id) - .map(|result| result.is_ok()); +impl<'a> AccessCheck { + /// Asserts that the server has access to the room and event (if any). + /// If the server is permitted, `Ok(())` is returned. Otherwise, a Forbidden + /// error is returned. + pub(super) async fn assert(&self) -> Result { + let acl_check = self + .services + .rooms + .event_handler + .acl_check(self.origin, self.room_id) + .map(|result| result.is_ok()); - let world_readable = self - .services - .rooms - .state_accessor - .is_world_readable(self.room_id); + let world_readable = self + .services + .rooms + .state_accessor + .is_world_readable(self.room_id); - let server_in_room = self - .services - .rooms - .state_cache - .server_in_room(self.origin, self.room_id); + let server_in_room = self + .services + .rooms + .state_cache + .server_in_room(self.origin, self.room_id); - let server_can_see: OptionFuture<_> = self - .event_id - .map(|event_id| { - self.services.rooms.state_accessor.server_can_see_event( - self.origin, - self.room_id, - event_id, - ) - }) - .into(); + let server_can_see: OptionFuture<_> = self + .event_id + .map(|event_id| { + self.services.rooms.state_accessor.server_can_see_event( + self.origin, + self.room_id, + event_id, + ) + }) + .into(); - let (world_readable, server_in_room, server_can_see, acl_check) = - join!(world_readable, server_in_room, server_can_see, acl_check); + let (world_readable, server_in_room, server_can_see, acl_check) = + join!(world_readable, server_in_room, server_can_see, acl_check); - if !acl_check { - return Err!(Request(Forbidden("Server access denied."))); + if !acl_check { + return Err!(Request(Forbidden("Server access denied."))); + } + + if !world_readable && !server_in_room { + return Err!(Request(Forbidden("Server is not in room."))); + } + + if server_can_see.is_some_and(is_false!()) { + return Err!(Request(Forbidden("Server is not allowed to see event."))); + } + + Ok(()) } - - if !world_readable && !server_in_room { - return Err!(Request(Forbidden("Server is not in room."))); - } - - if server_can_see.is_some_and(is_false!()) { - return Err!(Request(Forbidden("Server is not allowed to see event."))); - } - - Ok(()) } From fdcb7e3957b5cf3ade276dbe6ac232d7e675bd59 Mon Sep 17 00:00:00 2001 From: timedout Date: Fri, 26 Jun 2026 06:12:30 +0100 Subject: [PATCH 55/70] style: Remove `implement` for config `Manager` --- src/core/config/manager.rs | 88 +++++++++++++++++++------------------- 1 file changed, 43 insertions(+), 45 deletions(-) diff --git a/src/core/config/manager.rs b/src/core/config/manager.rs index d9b262259..6060b80b5 100644 --- a/src/core/config/manager.rs +++ b/src/core/config/manager.rs @@ -10,7 +10,7 @@ use std::{ }; use super::Config; -use crate::{Result, implement}; +use crate::Result; /// The configuration manager is an indirection to reload the configuration for /// the server while it is running. In order to not burden or clutter the many @@ -39,6 +39,48 @@ impl Manager { active: AtomicPtr::new(Arc::into_raw(config).cast_mut()), } } + + /// Update the active configuration, returning prior configuration. + #[tracing::instrument(skip_all, level = "info")] + pub fn update(&self, config: Config) -> Result> { + let config = Arc::new(config); + let new = Arc::into_raw(config); + let old = self.active.swap(new.cast_mut(), Ordering::AcqRel); + + // SAFETY: The old active pointer was set using an Arc::into_raw(). We're + // obliged to reconstitute that into Arc otherwise it will leak. + Ok(unsafe { Arc::from_raw(old) }) + } + + fn load(&self, handle: &mut [Option>]) -> &'static Arc { + let config = self.active.load(Ordering::Acquire); + + // Branch taken after config reload or first access by this thread. + if handle[INDEX.get()] + .as_ref() + .is_none_or(|handle| !ptr::eq(config, Arc::as_ptr(handle))) + { + INDEX.set(INDEX.get().wrapping_add(1).wrapping_rem(HISTORY)); + return load_miss(handle, INDEX.get(), config); + } + + let config: &Arc = handle[INDEX.get()] + .as_ref() + .expect("handle was already cached for this thread"); + + // SAFETY: The caller should not hold multiple references at a time directly + // into Config, as a subsequent reference might invalidate the thread's cache + // causing another reference to dangle. + // + // This is a highly unusual pattern as most config values are copied by value or + // used immediately without running overlap with another value. Even if it does + // actually occur somewhere, the window of danger is limited to the config being + // reloaded while the reference is held and another access is made by the same + // thread into a different config value. This is mitigated by creating a buffer + // of old configs rather than discarding at the earliest opportunity; the odds + // of this scenario are thus astronomical. + unsafe { std::mem::transmute(config) } + } } impl Drop for Manager { @@ -57,50 +99,6 @@ impl Deref for Manager { fn deref(&self) -> &Self::Target { HANDLE.with_borrow_mut(|handle| self.load(handle)) } } -/// Update the active configuration, returning prior configuration. -#[implement(Manager)] -#[tracing::instrument(skip_all, level = "info")] -pub fn update(&self, config: Config) -> Result> { - let config = Arc::new(config); - let new = Arc::into_raw(config); - let old = self.active.swap(new.cast_mut(), Ordering::AcqRel); - - // SAFETY: The old active pointer was set using an Arc::into_raw(). We're - // obliged to reconstitute that into Arc otherwise it will leak. - Ok(unsafe { Arc::from_raw(old) }) -} - -#[implement(Manager)] -fn load(&self, handle: &mut [Option>]) -> &'static Arc { - let config = self.active.load(Ordering::Acquire); - - // Branch taken after config reload or first access by this thread. - if handle[INDEX.get()] - .as_ref() - .is_none_or(|handle| !ptr::eq(config, Arc::as_ptr(handle))) - { - INDEX.set(INDEX.get().wrapping_add(1).wrapping_rem(HISTORY)); - return load_miss(handle, INDEX.get(), config); - } - - let config: &Arc = handle[INDEX.get()] - .as_ref() - .expect("handle was already cached for this thread"); - - // SAFETY: The caller should not hold multiple references at a time directly - // into Config, as a subsequent reference might invalidate the thread's cache - // causing another reference to dangle. - // - // This is a highly unusual pattern as most config values are copied by value or - // used immediately without running overlap with another value. Even if it does - // actually occur somewhere, the window of danger is limited to the config being - // reloaded while the reference is held and another access is made by the same - // thread into a different config value. This is mitigated by creating a buffer - // of old configs rather than discarding at the earliest opportunity; the odds - // of this scenario are thus astronomical. - unsafe { std::mem::transmute(config) } -} - #[tracing::instrument( name = "miss", level = "trace", From 86e1101b40681661c9c7e12395ca16deb8b1ffc4 Mon Sep 17 00:00:00 2001 From: timedout Date: Fri, 26 Jun 2026 06:15:20 +0100 Subject: [PATCH 56/70] style: Remove `implement` for database pool --- src/database/pool.rs | 654 +++++++++++++++++++++---------------------- 1 file changed, 322 insertions(+), 332 deletions(-) diff --git a/src/database/pool.rs b/src/database/pool.rs index b00fd0eff..6eb8e6013 100644 --- a/src/database/pool.rs +++ b/src/database/pool.rs @@ -12,7 +12,7 @@ use std::{ use async_channel::{QueueStrategy, Receiver, RecvError, Sender}; use conduwuit::{ - Error, Result, Server, SyncMutex, debug, err, error, implement, + Error, Result, Server, SyncMutex, debug, err, error, result::DebugInspect, smallvec::SmallVec, trace, @@ -72,29 +72,332 @@ const BATCH_INLINE: usize = 1; const WORKER_STACK_SIZE: usize = 1_048_576; const WORKER_NAME: &str = "conduwuit:db"; -#[implement(Pool)] -pub(crate) fn new(server: &Arc) -> Result> { - const CHAN_SCHED: (QueueStrategy, QueueStrategy) = (QueueStrategy::Fifo, QueueStrategy::Lifo); +impl Pool { + pub(crate) fn new(server: &Arc) -> Result> { + const CHAN_SCHED: (QueueStrategy, QueueStrategy) = + (QueueStrategy::Fifo, QueueStrategy::Lifo); - let (total_workers, queue_sizes, topology) = configure(server); + let (total_workers, queue_sizes, topology) = configure(server); - let (senders, receivers): (Vec<_>, Vec<_>) = queue_sizes - .into_iter() - .map(|cap| async_channel::bounded_with_queue_strategy(cap, CHAN_SCHED)) - .unzip(); + let (senders, receivers): (Vec<_>, Vec<_>) = queue_sizes + .into_iter() + .map(|cap| async_channel::bounded_with_queue_strategy(cap, CHAN_SCHED)) + .unzip(); - let pool = Arc::new(Self { - server: server.clone(), - queues: senders, - workers: Vec::new().into(), - topology, - busy: AtomicUsize::default(), - queued_max: AtomicUsize::default(), - }); + let pool = Arc::new(Self { + server: server.clone(), + queues: senders, + workers: Vec::new().into(), + topology, + busy: AtomicUsize::default(), + queued_max: AtomicUsize::default(), + }); - pool.spawn_until(&receivers, total_workers)?; + pool.spawn_until(&receivers, total_workers)?; - Ok(pool) + Ok(pool) + } + + #[tracing::instrument(skip_all, level = "debug")] + pub(crate) fn close(&self) { + let workers = take(&mut *self.workers.lock()); + + let senders = self.queues.iter().map(Sender::sender_count).sum::(); + + let receivers = self + .queues + .iter() + .map(Sender::receiver_count) + .sum::(); + + for queue in &self.queues { + queue.close(); + } + + if workers.is_empty() { + return; + } + + debug!( + senders, + receivers, + queues = self.queues.len(), + workers = workers.len(), + "Closing pool. Waiting for workers to join..." + ); + + workers + .into_iter() + .map(JoinHandle::join) + .map(|result| result.map_err(Error::from_panic)) + .enumerate() + .for_each(|(id, result)| match result { + | Ok(()) => trace!(%id, "worker joined"), + | Err(error) => error!(%id, "worker joined with error: {error}"), + }); + } + + fn spawn_until(self: &Arc, recv: &[Receiver], count: usize) -> Result { + let mut workers = self.workers.lock(); + while workers.len() < count { + self.clone().spawn_one(&mut workers, recv)?; + } + + Ok(()) + } + + #[tracing::instrument( + name = "spawn", + level = "trace", + skip_all, + fields(id = %workers.len()) + )] + fn spawn_one( + self: Arc, + workers: &mut Vec>, + recv: &[Receiver], + ) -> Result { + debug_assert!(!self.queues.is_empty(), "Must have at least one queue"); + debug_assert!(!recv.is_empty(), "Must have at least one receiver"); + + let id = workers.len(); + let group = id.overflowing_rem(self.queues.len()).0; + let recv = recv[group].clone(); + + let handle = thread::Builder::new() + .name(WORKER_NAME.into()) + .stack_size(WORKER_STACK_SIZE) + .spawn(move || self.worker(id, &recv))?; + + workers.push(handle); + + Ok(()) + } + + #[tracing::instrument(level = "trace", name = "get", skip(self, cmd))] + pub(crate) async fn execute_get(self: &Arc, mut cmd: Get) -> Result> { + let (send, recv) = oneshot::channel(); + _ = cmd.res.insert(send); + + let queue = self.select_queue(); + self.execute(queue, Cmd::Get(cmd)) + .and_then(move |()| { + recv.map_ok(into_recv_get) + .map_err(|e| err!(error!("recv failed {e:?}"))) + }) + .await + } + + #[tracing::instrument(level = "trace", name = "iter", skip(self, cmd))] + pub(crate) async fn execute_iter( + self: &Arc, + mut cmd: Seek, + ) -> Result> { + let (send, recv) = oneshot::channel(); + _ = cmd.res.insert(send); + + let queue = self.select_queue(); + self.execute(queue, Cmd::Iter(cmd)) + .and_then(|()| { + recv.map_ok(into_recv_seek) + .map_err(|e| err!(error!("recv failed {e:?}"))) + }) + .await + } + + fn select_queue(&self) -> &Sender { + let core_id = get_affinity().next().unwrap_or(0); + let chan_id = self.topology[core_id]; + self.queues.get(chan_id).unwrap_or_else(|| &self.queues[0]) + } + + #[tracing::instrument( + level = "trace", + name = "execute", + skip(self, cmd), + fields( + task = ?tokio::task::try_id(), + receivers = queue.receiver_count(), + queued = queue.len(), + queued_max = self.queued_max.load(Ordering::Relaxed), + ), + )] + async fn execute(&self, queue: &Sender, cmd: Cmd) -> Result { + if cfg!(debug_assertions) { + self.queued_max.fetch_max(queue.len(), Ordering::Relaxed); + } + + queue + .send(cmd) + .await + .map_err(|e| err!(error!("send failed {e:?}"))) + } + + #[tracing::instrument( + parent = None, + level = "debug", + skip(self, recv), + fields( + tid = ?thread::current().id(), + ), + )] + fn worker(self: Arc, id: usize, recv: &Receiver) { + self.worker_init(id); + self.worker_loop(recv); + } + + fn worker_init(&self, id: usize) { + let group = id.overflowing_rem(self.queues.len()).0; + let affinity = self + .topology + .iter() + .enumerate() + .filter(|_| self.queues.len() > 1) + .filter(|_| self.server.config.db_pool_affinity) + .filter_map(|(core_id, &queue_id)| (group == queue_id).then_some(core_id)) + .filter_map(nth_core_available); + + // affinity is empty (no-op) if there's only one queue + set_affinity(affinity.clone()); + + #[cfg(all(not(target_env = "msvc"), feature = "jemalloc"))] + if affinity.clone().count() == 1 && conduwuit::alloc::je::is_affine_arena() { + use conduwuit::{ + alloc::je::this_thread::{arena_id, set_arena}, + result::LogDebugErr, + }; + + let id = affinity.clone().next().expect("at least one id"); + + if let Ok(arena) = arena_id() { + if arena != id { + set_arena(id).log_debug_err().ok(); + } + } + } + + debug!( + %group, + affinity = ?affinity.collect::>(), + "worker ready" + ); + } + + fn worker_loop(self: &Arc, recv: &Receiver) { + // initial +1 needed prior to entering wait + self.busy.fetch_add(1, Ordering::Relaxed); + + while let Ok(cmd) = self.worker_wait(recv) { + Self::worker_handle(cmd); + } + } + + #[tracing::instrument( + name = "wait", + level = "trace", + skip_all, + fields( + receivers = recv.receiver_count(), + queued = recv.len(), + busy = self.busy.fetch_sub(1, Ordering::Relaxed) - 1, + ), + )] + fn worker_wait(self: &Arc, recv: &Receiver) -> Result { + recv.recv_blocking().debug_inspect(|_| { + self.busy.fetch_add(1, Ordering::Relaxed); + }) + } + + fn worker_handle(cmd: Cmd) { + match cmd { + | Cmd::Get(cmd) if cmd.key.len() == 1 => Self::handle_get(cmd), + | Cmd::Get(cmd) => Self::handle_batch(cmd), + | Cmd::Iter(cmd) => Self::handle_iter(cmd), + } + } + + #[tracing::instrument( + name = "iter", + level = "trace", + skip_all, + fields(%cmd.map), + )] + fn handle_iter(mut cmd: Seek) { + let chan = cmd.res.take().expect("missing result channel"); + + if chan.is_canceled() { + return; + } + + let from = cmd.key.as_deref(); + + let result = match cmd.dir { + | Direction::Forward => cmd.state.init_fwd(from), + | Direction::Reverse => cmd.state.init_rev(from), + }; + + let chan_result = chan.send(into_send_seek(result)); + + let _chan_sent = chan_result.is_ok(); + } + + #[tracing::instrument( + name = "batch", + level = "trace", + skip_all, + fields( + %cmd.map, + keys = %cmd.key.len(), + ), + )] + fn handle_batch(mut cmd: Get) { + debug_assert!(cmd.key.len() > 1, "should have more than one key"); + debug_assert!(!cmd.key.iter().any(SmallVec::is_empty), "querying for empty key"); + + let chan = cmd.res.take().expect("missing result channel"); + + if chan.is_canceled() { + return; + } + + let keys = cmd.key.iter(); + + let result: SmallVec<_> = cmd.map.get_batch_blocking(keys).collect(); + + let chan_result = chan.send(into_send_get(result)); + + let _chan_sent = chan_result.is_ok(); + } + + #[tracing::instrument( + name = "get", + level = "trace", + skip_all, + fields(%cmd.map), + )] + fn handle_get(mut cmd: Get) { + debug_assert!(!cmd.key[0].is_empty(), "querying for empty key"); + + // Obtain the result channel. + let chan = cmd.res.take().expect("missing result channel"); + + // It is worth checking if the future was dropped while the command was queued + // so we can bail without paying for any query. + if chan.is_canceled() { + return; + } + + // Perform the actual database query. We reuse our database::Map interface but + // limited to the blocking calls, rather than creating another surface directly + // with rocksdb here. + let result = cmd.map.get_blocking(&cmd.key[0]); + + // Send the result back to the submitter. + let chan_result = chan.send(into_send_get([result].into())); + + // If the future was dropped during the query this will fail acceptably. + let _chan_sent = chan_result.is_ok(); + } } impl Drop for Pool { @@ -112,319 +415,6 @@ impl Drop for Pool { } } -#[implement(Pool)] -#[tracing::instrument(skip_all, level = "debug")] -pub(crate) fn close(&self) { - let workers = take(&mut *self.workers.lock()); - - let senders = self.queues.iter().map(Sender::sender_count).sum::(); - - let receivers = self - .queues - .iter() - .map(Sender::receiver_count) - .sum::(); - - for queue in &self.queues { - queue.close(); - } - - if workers.is_empty() { - return; - } - - debug!( - senders, - receivers, - queues = self.queues.len(), - workers = workers.len(), - "Closing pool. Waiting for workers to join..." - ); - - workers - .into_iter() - .map(JoinHandle::join) - .map(|result| result.map_err(Error::from_panic)) - .enumerate() - .for_each(|(id, result)| match result { - | Ok(()) => trace!(%id, "worker joined"), - | Err(error) => error!(%id, "worker joined with error: {error}"), - }); -} - -#[implement(Pool)] -fn spawn_until(self: &Arc, recv: &[Receiver], count: usize) -> Result { - let mut workers = self.workers.lock(); - while workers.len() < count { - self.clone().spawn_one(&mut workers, recv)?; - } - - Ok(()) -} - -#[implement(Pool)] -#[tracing::instrument( - name = "spawn", - level = "trace", - skip_all, - fields(id = %workers.len()) -)] -fn spawn_one( - self: Arc, - workers: &mut Vec>, - recv: &[Receiver], -) -> Result { - debug_assert!(!self.queues.is_empty(), "Must have at least one queue"); - debug_assert!(!recv.is_empty(), "Must have at least one receiver"); - - let id = workers.len(); - let group = id.overflowing_rem(self.queues.len()).0; - let recv = recv[group].clone(); - - let handle = thread::Builder::new() - .name(WORKER_NAME.into()) - .stack_size(WORKER_STACK_SIZE) - .spawn(move || self.worker(id, &recv))?; - - workers.push(handle); - - Ok(()) -} - -#[implement(Pool)] -#[tracing::instrument(level = "trace", name = "get", skip(self, cmd))] -pub(crate) async fn execute_get(self: &Arc, mut cmd: Get) -> Result> { - let (send, recv) = oneshot::channel(); - _ = cmd.res.insert(send); - - let queue = self.select_queue(); - self.execute(queue, Cmd::Get(cmd)) - .and_then(move |()| { - recv.map_ok(into_recv_get) - .map_err(|e| err!(error!("recv failed {e:?}"))) - }) - .await -} - -#[implement(Pool)] -#[tracing::instrument(level = "trace", name = "iter", skip(self, cmd))] -pub(crate) async fn execute_iter(self: &Arc, mut cmd: Seek) -> Result> { - let (send, recv) = oneshot::channel(); - _ = cmd.res.insert(send); - - let queue = self.select_queue(); - self.execute(queue, Cmd::Iter(cmd)) - .and_then(|()| { - recv.map_ok(into_recv_seek) - .map_err(|e| err!(error!("recv failed {e:?}"))) - }) - .await -} - -#[implement(Pool)] -fn select_queue(&self) -> &Sender { - let core_id = get_affinity().next().unwrap_or(0); - let chan_id = self.topology[core_id]; - self.queues.get(chan_id).unwrap_or_else(|| &self.queues[0]) -} - -#[implement(Pool)] -#[tracing::instrument( - level = "trace", - name = "execute", - skip(self, cmd), - fields( - task = ?tokio::task::try_id(), - receivers = queue.receiver_count(), - queued = queue.len(), - queued_max = self.queued_max.load(Ordering::Relaxed), - ), -)] -async fn execute(&self, queue: &Sender, cmd: Cmd) -> Result { - if cfg!(debug_assertions) { - self.queued_max.fetch_max(queue.len(), Ordering::Relaxed); - } - - queue - .send(cmd) - .await - .map_err(|e| err!(error!("send failed {e:?}"))) -} - -#[implement(Pool)] -#[tracing::instrument( - parent = None, - level = "debug", - skip(self, recv), - fields( - tid = ?thread::current().id(), - ), -)] -fn worker(self: Arc, id: usize, recv: &Receiver) { - self.worker_init(id); - self.worker_loop(recv); -} - -#[implement(Pool)] -fn worker_init(&self, id: usize) { - let group = id.overflowing_rem(self.queues.len()).0; - let affinity = self - .topology - .iter() - .enumerate() - .filter(|_| self.queues.len() > 1) - .filter(|_| self.server.config.db_pool_affinity) - .filter_map(|(core_id, &queue_id)| (group == queue_id).then_some(core_id)) - .filter_map(nth_core_available); - - // affinity is empty (no-op) if there's only one queue - set_affinity(affinity.clone()); - - #[cfg(all(not(target_env = "msvc"), feature = "jemalloc"))] - if affinity.clone().count() == 1 && conduwuit::alloc::je::is_affine_arena() { - use conduwuit::{ - alloc::je::this_thread::{arena_id, set_arena}, - result::LogDebugErr, - }; - - let id = affinity.clone().next().expect("at least one id"); - - if let Ok(arena) = arena_id() { - if arena != id { - set_arena(id).log_debug_err().ok(); - } - } - } - - debug!( - %group, - affinity = ?affinity.collect::>(), - "worker ready" - ); -} - -#[implement(Pool)] -fn worker_loop(self: &Arc, recv: &Receiver) { - // initial +1 needed prior to entering wait - self.busy.fetch_add(1, Ordering::Relaxed); - - while let Ok(cmd) = self.worker_wait(recv) { - Pool::worker_handle(cmd); - } -} - -#[implement(Pool)] -#[tracing::instrument( - name = "wait", - level = "trace", - skip_all, - fields( - receivers = recv.receiver_count(), - queued = recv.len(), - busy = self.busy.fetch_sub(1, Ordering::Relaxed) - 1, - ), -)] -fn worker_wait(self: &Arc, recv: &Receiver) -> Result { - recv.recv_blocking().debug_inspect(|_| { - self.busy.fetch_add(1, Ordering::Relaxed); - }) -} - -#[implement(Pool)] -fn worker_handle(cmd: Cmd) { - match cmd { - | Cmd::Get(cmd) if cmd.key.len() == 1 => Pool::handle_get(cmd), - | Cmd::Get(cmd) => Pool::handle_batch(cmd), - | Cmd::Iter(cmd) => Pool::handle_iter(cmd), - } -} - -#[implement(Pool)] -#[tracing::instrument( - name = "iter", - level = "trace", - skip_all, - fields(%cmd.map), -)] -fn handle_iter(mut cmd: Seek) { - let chan = cmd.res.take().expect("missing result channel"); - - if chan.is_canceled() { - return; - } - - let from = cmd.key.as_deref(); - - let result = match cmd.dir { - | Direction::Forward => cmd.state.init_fwd(from), - | Direction::Reverse => cmd.state.init_rev(from), - }; - - let chan_result = chan.send(into_send_seek(result)); - - let _chan_sent = chan_result.is_ok(); -} - -#[implement(Pool)] -#[tracing::instrument( - name = "batch", - level = "trace", - skip_all, - fields( - %cmd.map, - keys = %cmd.key.len(), - ), -)] -fn handle_batch(mut cmd: Get) { - debug_assert!(cmd.key.len() > 1, "should have more than one key"); - debug_assert!(!cmd.key.iter().any(SmallVec::is_empty), "querying for empty key"); - - let chan = cmd.res.take().expect("missing result channel"); - - if chan.is_canceled() { - return; - } - - let keys = cmd.key.iter(); - - let result: SmallVec<_> = cmd.map.get_batch_blocking(keys).collect(); - - let chan_result = chan.send(into_send_get(result)); - - let _chan_sent = chan_result.is_ok(); -} - -#[implement(Pool)] -#[tracing::instrument( - name = "get", - level = "trace", - skip_all, - fields(%cmd.map), -)] -fn handle_get(mut cmd: Get) { - debug_assert!(!cmd.key[0].is_empty(), "querying for empty key"); - - // Obtain the result channel. - let chan = cmd.res.take().expect("missing result channel"); - - // It is worth checking if the future was dropped while the command was queued - // so we can bail without paying for any query. - if chan.is_canceled() { - return; - } - - // Perform the actual database query. We reuse our database::Map interface but - // limited to the blocking calls, rather than creating another surface directly - // with rocksdb here. - let result = cmd.map.get_blocking(&cmd.key[0]); - - // Send the result back to the submitter. - let chan_result = chan.send(into_send_get([result].into())); - - // If the future was dropped during the query this will fail acceptably. - let _chan_sent = chan_result.is_ok(); -} - fn into_send_get(result: BatchResult<'_>) -> BatchResult<'static> { // SAFETY: Necessary to send the Handle (rust_rocksdb::PinnableSlice) through // the channel. The lifetime on the handle is a device by rust-rocksdb to From ba0ffee7cfadb19b1dd74e294497823cc8b6b4d6 Mon Sep 17 00:00:00 2001 From: timedout Date: Fri, 26 Jun 2026 06:17:41 +0100 Subject: [PATCH 57/70] fix: Style errors and compile problems --- src/api/server/utils.rs | 4 ++-- src/service/media/preview.rs | 5 ----- 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/src/api/server/utils.rs b/src/api/server/utils.rs index 7b9103876..52912dfdc 100644 --- a/src/api/server/utils.rs +++ b/src/api/server/utils.rs @@ -1,4 +1,4 @@ -use conduwuit::{Err, Result, implement, is_false}; +use conduwuit::{Err, Result, is_false}; use conduwuit_service::Services; use futures::{FutureExt, future::OptionFuture, join}; use ruma::{EventId, RoomId, ServerName}; @@ -10,7 +10,7 @@ pub(super) struct AccessCheck<'a> { pub(super) event_id: Option<&'a EventId>, } -impl<'a> AccessCheck { +impl AccessCheck<'_> { /// Asserts that the server has access to the room and event (if any). /// If the server is permitted, `Ok(())` is returned. Otherwise, a Forbidden /// error is returned. diff --git a/src/service/media/preview.rs b/src/service/media/preview.rs index 21e096019..54b321fba 100644 --- a/src/service/media/preview.rs +++ b/src/service/media/preview.rs @@ -123,7 +123,6 @@ impl Service { } #[cfg(feature = "url_preview")] - pub async fn download_image( &self, url: &str, @@ -179,7 +178,6 @@ impl Service { } #[cfg(feature = "url_preview")] - pub async fn download_video( &self, url: &str, @@ -197,7 +195,6 @@ impl Service { } #[cfg(feature = "url_preview")] - pub async fn download_audio( &self, url: &str, @@ -215,7 +212,6 @@ impl Service { } #[cfg(feature = "url_preview")] - pub async fn download_media(&self, url: &str) -> Result<(OwnedMxcUri, usize)> { use conduwuit::utils::random_string; use http::header::CONTENT_TYPE; @@ -278,7 +274,6 @@ impl Service { } #[cfg(feature = "url_preview")] - async fn download_html(&self, url: &str) -> Result { use webpage::HTML; From 85122c62cde249bd489217084b3b4c7c1d17a053 Mon Sep 17 00:00:00 2001 From: timedout Date: Fri, 26 Jun 2026 06:18:24 +0100 Subject: [PATCH 58/70] style: Eliminate `implement` in database map keys --- src/database/map/keys.rs | 73 ++++++++++++++++++++-------------------- 1 file changed, 36 insertions(+), 37 deletions(-) diff --git a/src/database/map/keys.rs b/src/database/map/keys.rs index ac044e91f..67cd99511 100644 --- a/src/database/map/keys.rs +++ b/src/database/map/keys.rs @@ -1,6 +1,6 @@ use std::sync::Arc; -use conduwuit::{Result, implement}; +use conduwuit::Result; use futures::{FutureExt, Stream, StreamExt, TryFutureExt, TryStreamExt}; use rocksdb::Direction; use serde::Deserialize; @@ -9,43 +9,42 @@ use tokio::task; use super::stream::is_cached; use crate::{keyval, keyval::Key, stream}; -#[implement(super::Map)] -pub fn keys<'a, K>(self: &'a Arc) -> impl Stream>> + Send -where - K: Deserialize<'a> + Send, -{ - self.raw_keys().map(keyval::result_deserialize_key::) -} - -#[implement(super::Map)] -#[tracing::instrument(skip(self), fields(%self), level = "trace")] -pub fn raw_keys(self: &Arc) -> impl Stream>> + Send { - use crate::pool::Seek; - - let opts = super::iter_options_default(&self.db); - let state = stream::State::new(self, opts); - if is_cached(self) { - let state = state.init_fwd(None); - return task::consume_budget() - .map(move |()| stream::Keys::<'_>::from(state)) - .into_stream() - .flatten() - .boxed(); +impl super::Map { + pub fn keys<'a, K>(self: &'a Arc) -> impl Stream>> + Send + where + K: Deserialize<'a> + Send, + { + self.raw_keys().map(keyval::result_deserialize_key::) } - let seek = Seek { - map: self.clone(), - dir: Direction::Forward, - state: crate::pool::into_send_seek(state), - key: None, - res: None, - }; + pub fn raw_keys(self: &Arc) -> impl Stream>> + Send { + use crate::pool::Seek; - self.db - .pool - .execute_iter(seek) - .ok_into::>() - .into_stream() - .try_flatten() - .boxed() + let opts = super::iter_options_default(&self.db); + let state = stream::State::new(self, opts); + if is_cached(self) { + let state = state.init_fwd(None); + return task::consume_budget() + .map(move |()| stream::Keys::<'_>::from(state)) + .into_stream() + .flatten() + .boxed(); + } + + let seek = Seek { + map: self.clone(), + dir: Direction::Forward, + state: crate::pool::into_send_seek(state), + key: None, + res: None, + }; + + self.db + .pool + .execute_iter(seek) + .ok_into::>() + .into_stream() + .try_flatten() + .boxed() + } } From e7b302955f4c4e3329f6d91c103b69ef58256619 Mon Sep 17 00:00:00 2001 From: timedout Date: Fri, 26 Jun 2026 06:29:53 +0100 Subject: [PATCH 59/70] style: Eliminate `implement` database --- src/database/engine/backup.rs | 144 +++++---- src/database/engine/files.rs | 20 +- src/database/engine/memory_usage.rs | 43 +-- src/database/engine/open.rs | 187 ++++++------ src/database/map/clear.rs | 39 +-- src/database/map/compact.rs | 69 ++--- src/database/map/contains.rs | 167 +++++------ src/database/map/count.rs | 120 ++++---- src/database/map/get.rs | 126 ++++---- src/database/map/get_batch.rs | 103 ++++--- src/database/map/insert.rs | 417 +++++++++++++------------- src/database/map/keys_from.rs | 130 ++++---- src/database/map/keys_prefix.rs | 94 +++--- src/database/map/qry.rs | 92 +++--- src/database/map/qry_batch.rs | 49 +-- src/database/map/remove.rs | 84 +++--- src/database/map/rev_keys.rs | 74 ++--- src/database/map/rev_keys_from.rs | 136 +++++---- src/database/map/rev_keys_prefix.rs | 92 +++--- src/database/map/rev_stream.rs | 94 +++--- src/database/map/rev_stream_from.rs | 182 ++++++----- src/database/map/rev_stream_prefix.rs | 130 ++++---- src/database/map/stream.rs | 92 +++--- src/database/map/stream_from.rs | 186 ++++++------ src/database/map/stream_prefix.rs | 130 ++++---- 25 files changed, 1480 insertions(+), 1520 deletions(-) diff --git a/src/database/engine/backup.rs b/src/database/engine/backup.rs index 69d510f33..9c6d899c7 100644 --- a/src/database/engine/backup.rs +++ b/src/database/engine/backup.rs @@ -1,92 +1,88 @@ use std::{ffi::OsString, path::PathBuf}; -use conduwuit::{Err, Result, error, implement, info, utils::time::rfc2822_from_seconds, warn}; +use conduwuit::{Err, Result, error, info, utils::time::rfc2822_from_seconds, warn}; use rocksdb::backup::{BackupEngine, BackupEngineOptions}; -use super::Engine; use crate::util::map_err; -#[implement(Engine)] -#[tracing::instrument(skip(self), level = "info")] -pub fn backup(&self) -> Result { - let mut engine = self.backup_engine()?; - let config = &self.ctx.server.config; - if config.database_backups_to_keep > 0 { - engine - .create_new_backup_flush(&self.db, true) - .map_err(map_err)?; +impl super::Engine { + #[tracing::instrument(skip(self), level = "info")] + pub fn backup(&self) -> Result { + let mut engine = self.backup_engine()?; + let config = &self.ctx.server.config; + if config.database_backups_to_keep > 0 { + engine + .create_new_backup_flush(&self.db, true) + .map_err(map_err)?; - let engine_info = engine.get_backup_info(); - let info = &engine_info.last().expect("backup engine info is not empty"); - info!( - "Created database backup #{} using {} bytes in {} files", - info.backup_id, info.size, info.num_files, - ); - } - - if config.database_backups_to_keep >= 0 { - let keep = u32::try_from(config.database_backups_to_keep)?; - if let Err(e) = engine.purge_old_backups(keep.try_into()?) { - error!("Failed to purge old backup: {e:?}"); + let engine_info = engine.get_backup_info(); + let info = &engine_info.last().expect("backup engine info is not empty"); + info!( + "Created database backup #{} using {} bytes in {} files", + info.backup_id, info.size, info.num_files, + ); } + + if config.database_backups_to_keep >= 0 { + let keep = u32::try_from(config.database_backups_to_keep)?; + if let Err(e) = engine.purge_old_backups(keep.try_into()?) { + error!("Failed to purge old backup: {e:?}"); + } + } + + if config.database_backups_to_keep == 0 { + warn!("Configuration item `database_backups_to_keep` is set to 0."); + } + + Ok(()) } - if config.database_backups_to_keep == 0 { - warn!("Configuration item `database_backups_to_keep` is set to 0."); + pub fn backup_list(&self) -> Result + Send> { + let info = self.backup_engine()?.get_backup_info(); + + if info.is_empty() { + return Err!("No backups found."); + } + + let list = info.into_iter().map(|info| { + format!( + "#{} {}: {} bytes, {} files", + info.backup_id, + rfc2822_from_seconds(info.timestamp), + info.size, + info.num_files, + ) + }); + + Ok(list) } - Ok(()) -} + pub fn backup_count(&self) -> Result { + let info = self.backup_engine()?.get_backup_info(); -#[implement(Engine)] -pub fn backup_list(&self) -> Result + Send> { - let info = self.backup_engine()?.get_backup_info(); - - if info.is_empty() { - return Err!("No backups found."); + Ok(info.len()) } - let list = info.into_iter().map(|info| { - format!( - "#{} {}: {} bytes, {} files", - info.backup_id, - rfc2822_from_seconds(info.timestamp), - info.size, - info.num_files, - ) - }); - - Ok(list) -} - -#[implement(Engine)] -pub fn backup_count(&self) -> Result { - let info = self.backup_engine()?.get_backup_info(); - - Ok(info.len()) -} - -#[implement(Engine)] -fn backup_engine(&self) -> Result { - let path = self.backup_path()?; - let options = BackupEngineOptions::new(path).map_err(map_err)?; - BackupEngine::open(&options, &self.ctx.env.lock()).map_err(map_err) -} - -#[implement(Engine)] -fn backup_path(&self) -> Result { - let path = self - .ctx - .server - .config - .database_backup_path - .clone() - .map(PathBuf::into_os_string) - .unwrap_or_default(); - - if path.is_empty() { - return Err!(Config("database_backup_path", "Configure path to enable backups")); + fn backup_engine(&self) -> Result { + let path = self.backup_path()?; + let options = BackupEngineOptions::new(path).map_err(map_err)?; + BackupEngine::open(&options, &self.ctx.env.lock()).map_err(map_err) } - Ok(path) + fn backup_path(&self) -> Result { + let path = self + .ctx + .server + .config + .database_backup_path + .clone() + .map(PathBuf::into_os_string) + .unwrap_or_default(); + + if path.is_empty() { + return Err!(Config("database_backup_path", "Configure path to enable backups")); + } + + Ok(path) + } } diff --git a/src/database/engine/files.rs b/src/database/engine/files.rs index 1f38a63c7..acbda4189 100644 --- a/src/database/engine/files.rs +++ b/src/database/engine/files.rs @@ -1,15 +1,15 @@ -use conduwuit::{Result, implement}; +use conduwuit::Result; use rocksdb::LiveFile as SstFile; -use super::Engine; use crate::util::map_err; -#[implement(Engine)] -pub fn file_list(&self) -> impl Iterator> + Send + use<> { - self.db - .live_files() - .map_err(map_err) - .into_iter() - .flat_map(Vec::into_iter) - .map(Ok) +impl super::Engine { + pub fn file_list(&self) -> impl Iterator> + Send + use<> { + self.db + .live_files() + .map_err(map_err) + .into_iter() + .flat_map(Vec::into_iter) + .map(Ok) + } } diff --git a/src/database/engine/memory_usage.rs b/src/database/engine/memory_usage.rs index 21af35c8a..d5e5945d8 100644 --- a/src/database/engine/memory_usage.rs +++ b/src/database/engine/memory_usage.rs @@ -1,30 +1,31 @@ use std::fmt::Write; -use conduwuit::{Result, implement}; +use conduwuit::Result; use rocksdb::perf::get_memory_usage_stats; -use super::Engine; use crate::or_else; -#[implement(Engine)] -pub fn memory_usage(&self) -> Result { - let mut res = String::new(); - let stats = get_memory_usage_stats(Some(&[&self.db]), Some(&[&*self.ctx.row_cache.lock()])) - .or_else(or_else)?; - let mibs = |input| f64::from(u32::try_from(input / 1024).unwrap_or(0)) / 1024.0; - writeln!( - res, - "Memory buffers: {:.2} MiB\nPending write: {:.2} MiB\nTable readers: {:.2} MiB\nRow \ - cache: {:.2} MiB", - mibs(stats.mem_table_total), - mibs(stats.mem_table_unflushed), - mibs(stats.mem_table_readers_total), - mibs(u64::try_from(self.ctx.row_cache.lock().get_usage())?), - )?; +impl super::Engine { + pub fn memory_usage(&self) -> Result { + let mut res = String::new(); + let stats = + get_memory_usage_stats(Some(&[&self.db]), Some(&[&*self.ctx.row_cache.lock()])) + .or_else(or_else)?; + let mibs = |input| f64::from(u32::try_from(input / 1024).unwrap_or(0)) / 1024.0; + writeln!( + res, + "Memory buffers: {:.2} MiB\nPending write: {:.2} MiB\nTable readers: {:.2} MiB\nRow \ + cache: {:.2} MiB", + mibs(stats.mem_table_total), + mibs(stats.mem_table_unflushed), + mibs(stats.mem_table_readers_total), + mibs(u64::try_from(self.ctx.row_cache.lock().get_usage())?), + )?; - for (name, cache) in &*self.ctx.col_cache.lock() { - writeln!(res, "{name} cache: {:.2} MiB", mibs(u64::try_from(cache.get_usage())?))?; + for (name, cache) in &*self.ctx.col_cache.lock() { + writeln!(res, "{name} cache: {:.2} MiB", mibs(u64::try_from(cache.get_usage())?))?; + } + + Ok(res) } - - Ok(res) } diff --git a/src/database/engine/open.rs b/src/database/engine/open.rs index b52d72f6c..094ff8f1b 100644 --- a/src/database/engine/open.rs +++ b/src/database/engine/open.rs @@ -8,7 +8,7 @@ use conduwuit::{Result, debug, implement, info, warn}; use rocksdb::{ColumnFamilyDescriptor, Options}; use super::{ - Db, Engine, + Db, cf_opts::cf_options, db_opts::db_options, descriptor::{self, Descriptor}, @@ -16,104 +16,103 @@ use super::{ }; use crate::{Context, or_else}; -#[implement(Engine)] -#[tracing::instrument(skip_all, level = "info")] -pub(crate) async fn open(ctx: Arc, desc: &[Descriptor]) -> Result> { - let server = &ctx.server; - let config = &server.config; - let path = &config.database_path; +impl super::Engine { + #[tracing::instrument(skip_all, level = "info")] + pub(crate) async fn open(ctx: Arc, desc: &[Descriptor]) -> Result> { + let server = &ctx.server; + let config = &server.config; + let path = &config.database_path; - let db_opts = db_options(config, &ctx.env.lock(), &ctx.row_cache.lock())?; + let db_opts = db_options(config, &ctx.env.lock(), &ctx.row_cache.lock())?; - let cfds = Self::configure_cfds(&ctx, &db_opts, desc)?; - let num_cfds = cfds.len(); - debug!("Configured {num_cfds} column descriptors..."); + let cfds = Self::configure_cfds(&ctx, &db_opts, desc)?; + let num_cfds = cfds.len(); + debug!("Configured {num_cfds} column descriptors..."); - let load_time = std::time::Instant::now(); - if config.rocksdb_repair { - repair(&db_opts, &config.database_path)?; + let load_time = std::time::Instant::now(); + if config.rocksdb_repair { + repair(&db_opts, &config.database_path)?; + } + + debug!("Opening database..."); + let db = Db::open_cf_descriptors(&db_opts, path, cfds).or_else(or_else)?; + + info!( + columns = num_cfds, + sequence = %db.latest_sequence_number(), + time = ?load_time.elapsed(), + "Opened database." + ); + + Ok(Arc::new(Self { + db, + pool: ctx.pool.clone(), + ctx: ctx.clone(), + checksums: config.rocksdb_checksums, + corks: AtomicU32::new(0), + })) } - debug!("Opening database..."); - let db = Db::open_cf_descriptors(&db_opts, path, cfds).or_else(or_else)?; + #[tracing::instrument(name = "configure", skip_all, level = "debug")] + fn configure_cfds( + ctx: &Arc, + db_opts: &Options, + desc: &[Descriptor], + ) -> Result> { + let server = &ctx.server; + let config = &server.config; + let path = &config.database_path; + let existing = Self::discover_cfs(path, db_opts); - info!( - columns = num_cfds, - sequence = %db.latest_sequence_number(), - time = ?load_time.elapsed(), - "Opened database." - ); + let creating = desc.iter().filter(|desc| !existing.contains(desc.name)); - Ok(Arc::new(Self { - db, - pool: ctx.pool.clone(), - ctx: ctx.clone(), - checksums: config.rocksdb_checksums, - corks: AtomicU32::new(0), - })) -} - -#[implement(Engine)] -#[tracing::instrument(name = "configure", skip_all, level = "debug")] -fn configure_cfds( - ctx: &Arc, - db_opts: &Options, - desc: &[Descriptor], -) -> Result> { - let server = &ctx.server; - let config = &server.config; - let path = &config.database_path; - let existing = Self::discover_cfs(path, db_opts); - - let creating = desc.iter().filter(|desc| !existing.contains(desc.name)); - - let missing = existing - .iter() - .filter(|&name| name != "default") - .filter(|&name| !desc.iter().any(|desc| desc.name == name)); - - debug!( - existing = existing.len(), - described = desc.len(), - missing = missing.clone().count(), - creating = creating.clone().count(), - "Discovered database columns" - ); - - missing.clone().for_each(|name| { - debug!("Found unrecognized column {name:?} in existing database."); - }); - - creating.map(|desc| desc.name).for_each(|name| { - debug!("Creating new column {name:?} not previously found in existing database."); - }); - - let missing_descriptors = missing.clone().map(|_| descriptor::DROPPED); - - let cfopts: Vec<_> = desc - .iter() - .copied() - .chain(missing_descriptors) - .map(|ref desc| cf_options(ctx, db_opts.clone(), desc)) - .collect::>()?; - - let cfds: Vec<_> = desc - .iter() - .map(|desc| desc.name) - .map(ToOwned::to_owned) - .chain(missing.cloned()) - .zip(cfopts.into_iter()) - .map(|(name, opts)| ColumnFamilyDescriptor::new(name, opts)) - .collect(); - - Ok(cfds) -} - -#[implement(Engine)] -#[tracing::instrument(name = "discover", skip_all, level = "debug")] -fn discover_cfs(path: &Path, opts: &Options) -> BTreeSet { - Db::list_cf(opts, path) - .unwrap_or_default() - .into_iter() - .collect::>() + let missing = existing + .iter() + .filter(|&name| name != "default") + .filter(|&name| !desc.iter().any(|desc| desc.name == name)); + + debug!( + existing = existing.len(), + described = desc.len(), + missing = missing.clone().count(), + creating = creating.clone().count(), + "Discovered database columns" + ); + + missing.clone().for_each(|name| { + debug!("Found unrecognized column {name:?} in existing database."); + }); + + creating.map(|desc| desc.name).for_each(|name| { + debug!("Creating new column {name:?} not previously found in existing database."); + }); + + let missing_descriptors = missing.clone().map(|_| descriptor::DROPPED); + + let cfopts: Vec<_> = desc + .iter() + .copied() + .chain(missing_descriptors) + .map(|ref desc| cf_options(ctx, db_opts.clone(), desc)) + .collect::>()?; + + let cfds: Vec<_> = desc + .iter() + .map(|desc| desc.name) + .map(ToOwned::to_owned) + .chain(missing.cloned()) + .zip(cfopts.into_iter()) + .map(|(name, opts)| ColumnFamilyDescriptor::new(name, opts)) + .collect(); + + Ok(cfds) + } + + #[tracing::instrument(name = "discover", skip_all, level = "debug")] + fn discover_cfs(path: &Path, opts: &Options) -> BTreeSet { + Db::list_cf(opts, path) + .unwrap_or_default() + .into_iter() + .collect::>() + } } diff --git a/src/database/map/clear.rs b/src/database/map/clear.rs index 321ec79cc..f95a608ed 100644 --- a/src/database/map/clear.rs +++ b/src/database/map/clear.rs @@ -1,30 +1,31 @@ use std::sync::Arc; use conduwuit::{ - Result, implement, + Result, utils::stream::{ReadyExt, TryIgnore}, }; use futures::{Stream, TryStreamExt}; use crate::keyval::Key; -/// Delete all data stored in this map. !!! USE WITH CAUTION !!! -/// -/// See for_clear() with additional details. -#[implement(super::Map)] -#[tracing::instrument(level = "trace")] -pub async fn clear(self: &Arc) { - self.for_clear().ignore_err().ready_for_each(|_| ()).await; -} +impl super::Map { + /// Delete all data stored in this map. !!! USE WITH CAUTION !!! + /// + /// See for_clear() with additional details. + #[tracing::instrument(level = "trace")] + pub async fn clear(self: &Arc) { + // drops your entire database cutely :3 + self.for_clear().ignore_err().ready_for_each(|_| ()).await; + } -/// Delete all data stored in this map. !!! USE WITH CAUTION !!! -/// -/// Provides stream of keys undergoing deletion along with any errors. -/// -/// Note this operation applies to a snapshot of the data when invoked. -/// Additional data written during or after this call may be missed. -#[implement(super::Map)] -#[tracing::instrument(level = "trace")] -pub fn for_clear(self: &Arc) -> impl Stream>> + Send { - self.raw_keys().inspect_ok(|key| self.remove(key)) + /// Delete all data stored in this map. !!! USE WITH CAUTION !!! + /// + /// Provides stream of keys undergoing deletion along with any errors. + /// + /// Note this operation applies to a snapshot of the data when invoked. + /// Additional data written during or after this call may be missed. + #[tracing::instrument(level = "trace")] + pub fn for_clear(self: &Arc) -> impl Stream>> + Send { + self.raw_keys().inspect_ok(|key| self.remove(key)) + } } diff --git a/src/database/map/compact.rs b/src/database/map/compact.rs index a4c0add0c..ae6890ccd 100644 --- a/src/database/map/compact.rs +++ b/src/database/map/compact.rs @@ -23,40 +23,41 @@ pub struct Options { pub exclusive: bool, } -#[implement(super::Map)] -#[tracing::instrument( - name = "compact", - level = "info", - skip(self), - fields(%self), -)] -pub fn compact_blocking(&self, opts: Options) -> Result { - let mut co = CompactOptions::default(); - co.set_exclusive_manual_compaction(opts.exclusive); - co.set_bottommost_level_compaction(match opts.exhaustive { - | true => BottommostLevelCompaction::Force, - | false => BottommostLevelCompaction::ForceOptimized, - }); +impl super::Map { + #[tracing::instrument( + name = "compact", + skip(self), + fields(%self), + )] + pub fn compact_blocking(&self, opts: Options) -> Result { + let mut co = CompactOptions::default(); + co.set_exclusive_manual_compaction(opts.exclusive); + co.set_bottommost_level_compaction(match opts.exhaustive { + | true => BottommostLevelCompaction::Force, + | false => BottommostLevelCompaction::ForceOptimized, + }); - match opts.level { - | (None, None) => { - co.set_change_level(true); - co.set_target_level(-1); - }, - | (None, Some(level)) => { - co.set_change_level(true); - co.set_target_level(level.try_into()?); - }, - | (Some(level), None) => { - co.set_change_level(false); - co.set_target_level(level.try_into()?); - }, - | (Some(_), Some(_)) => return Err!("compacting between specific levels not supported"), + match opts.level { + | (None, None) => { + co.set_change_level(true); + co.set_target_level(-1); + }, + | (None, Some(level)) => { + co.set_change_level(true); + co.set_target_level(level.try_into()?); + }, + | (Some(level), None) => { + co.set_change_level(false); + co.set_target_level(level.try_into()?); + }, + | (Some(_), Some(_)) => + return Err!("compacting between specific levels not supported"), + } + + self.db + .db + .compact_range_cf_opt(&self.cf(), opts.range.0, opts.range.1, &co); + + Ok(()) } - - self.db - .db - .compact_range_cf_opt(&self.cf(), opts.range.0, opts.range.1, &co); - - Ok(()) } diff --git a/src/database/map/contains.rs b/src/database/map/contains.rs index 474818e81..0785a87e3 100644 --- a/src/database/map/contains.rs +++ b/src/database/map/contains.rs @@ -3,7 +3,7 @@ use std::{convert::AsRef, fmt::Debug, future::Future, io::Write, sync::Arc}; use conduwuit::{ Result, arrayvec::ArrayVec, - err, implement, + err, utils::{future::TryExtExt, result::FlatOk}, }; use futures::FutureExt; @@ -11,93 +11,90 @@ use serde::Serialize; use crate::{keyval::KeyBuf, ser}; -/// Returns true if the map contains the key. -/// - key is serialized into allocated buffer -/// - harder errors may not be reported -#[inline] -#[implement(super::Map)] -pub fn contains( - self: &Arc, - key: &K, -) -> impl Future + Send + '_ + use<'_, K> -where - K: Serialize + ?Sized + Debug, -{ - let mut buf = KeyBuf::new(); - self.bcontains(key, &mut buf) -} +impl super::Map { + /// Returns true if the map contains the key. + /// - key is serialized into allocated buffer + /// - harder errors may not be reported + #[inline] + pub fn contains( + self: &Arc, + key: &K, + ) -> impl Future + Send + '_ + use<'_, K> + where + K: Serialize + ?Sized + Debug, + { + let mut buf = KeyBuf::new(); + self.bcontains(key, &mut buf) + } -/// Returns true if the map contains the key. -/// - key is serialized into stack-buffer -/// - harder errors will panic -#[inline] -#[implement(super::Map)] -pub fn acontains( - self: &Arc, - key: &K, -) -> impl Future + Send + '_ + use<'_, MAX, K> -where - K: Serialize + ?Sized + Debug, -{ - let mut buf = ArrayVec::::new(); - self.bcontains(key, &mut buf) -} + /// Returns true if the map contains the key. + /// - key is serialized into stack-buffer + /// - harder errors will panic + #[inline] + pub fn acontains( + self: &Arc, + key: &K, + ) -> impl Future + Send + '_ + use<'_, MAX, K> + where + K: Serialize + ?Sized + Debug, + { + let mut buf = ArrayVec::::new(); + self.bcontains(key, &mut buf) + } -/// Returns true if the map contains the key. -/// - key is serialized into provided buffer -/// - harder errors will panic -#[implement(super::Map)] -#[tracing::instrument(skip(self, buf), fields(%self), level = "trace")] -pub fn bcontains( - self: &Arc, - key: &K, - buf: &mut B, -) -> impl Future + Send + '_ + use<'_, K, B> -where - K: Serialize + ?Sized + Debug, - B: Write + AsRef<[u8]>, -{ - let key = ser::serialize(buf, key).expect("failed to serialize query key"); - self.exists(key).is_ok() -} + /// Returns true if the map contains the key. + /// - key is serialized into provided buffer + /// - harder errors will panic + #[tracing::instrument(skip(self, buf), fields(%self), level = "trace")] + pub fn bcontains( + self: &Arc, + key: &K, + buf: &mut B, + ) -> impl Future + Send + '_ + use<'_, K, B> + where + K: Serialize + ?Sized + Debug, + B: Write + AsRef<[u8]>, + { + let key = ser::serialize(buf, key).expect("failed to serialize query key"); + self.exists(key).is_ok() + } -/// Returns Ok if the map contains the key. -/// - key is raw -#[inline] -#[implement(super::Map)] -pub fn exists<'a, K>( - self: &'a Arc, - key: &K, -) -> impl Future + Send + 'a + use<'a, K> -where - K: AsRef<[u8]> + ?Sized + Debug + 'a, -{ - self.get(key).map(|res| res.map(|_| ())) -} + /// Returns Ok if the map contains the key. + /// - key is raw + #[inline] + pub fn exists<'a, K>( + self: &'a Arc, + key: &K, + ) -> impl Future + Send + 'a + use<'a, K> + where + K: AsRef<[u8]> + ?Sized + Debug + 'a, + { + self.get(key).map(|res| res.map(|_| ())) + } -/// Returns Ok if the map contains the key; NotFound otherwise. Harder errors -/// may not always be reported properly. -#[implement(super::Map)] -#[tracing::instrument(skip(self, key), fields(%self), level = "trace")] -pub fn exists_blocking(&self, key: &K) -> Result -where - K: AsRef<[u8]> + ?Sized + Debug, -{ - self.maybe_exists(key) - .then(|| self.get_blocking(key)) - .flat_ok() - .map(|_| ()) - .ok_or_else(|| err!(Request(NotFound("Not found in database")))) -} + /// Returns Ok if the map contains the key; NotFound otherwise. Harder + /// errors may not always be reported properly. + #[tracing::instrument(skip(self, key), fields(%self), level = "trace")] + pub fn exists_blocking(&self, key: &K) -> Result + where + K: AsRef<[u8]> + ?Sized + Debug, + { + self.maybe_exists(key) + .then(|| self.get_blocking(key)) + .flat_ok() + .map(|_| ()) + .ok_or_else(|| err!(Request(NotFound("Not found in database")))) + } -/// Rocksdb limits this to kBlockCacheTier internally so this is not actually a -/// blocking call; in case that changes we set this as well in our read_options. -#[implement(super::Map)] -pub(crate) fn maybe_exists(&self, key: &K) -> bool -where - K: AsRef<[u8]> + ?Sized, -{ - self.db - .db - .key_may_exist_cf_opt(&self.cf(), key, &self.cache_read_options) + /// Rocksdb limits this to kBlockCacheTier internally so this is not + /// actually a blocking call; in case that changes we set this as well in + /// our read_options. + pub(crate) fn maybe_exists(&self, key: &K) -> bool + where + K: AsRef<[u8]> + ?Sized, + { + self.db + .db + .key_may_exist_cf_opt(&self.cf(), key, &self.cache_read_options) + } } diff --git a/src/database/map/count.rs b/src/database/map/count.rs index 78f9e2e39..09af94de8 100644 --- a/src/database/map/count.rs +++ b/src/database/map/count.rs @@ -1,72 +1,68 @@ use std::{fmt::Debug, future::Future, sync::Arc}; -use conduwuit::implement; use futures::stream::StreamExt; use serde::Serialize; -/// Count the total number of entries in the map. -#[implement(super::Map)] -#[inline] -pub fn count(self: &Arc) -> impl Future + Send + '_ { - self.raw_keys().count() -} +impl super::Map { + /// Count the total number of entries in the map. + #[inline] + pub fn count(self: &Arc) -> impl Future + Send + '_ { + self.raw_keys().count() + } -/// Count the number of entries in the map starting from a lower-bound. -/// -/// - From is a structured key -#[implement(super::Map)] -#[inline] -pub fn count_from<'a, P>( - self: &'a Arc, - from: &P, -) -> impl Future + Send + 'a + use<'a, P> -where - P: Serialize + ?Sized + Debug + 'a, -{ - self.keys_from_raw(from).count() -} + /// Count the number of entries in the map starting from a lower-bound. + /// + /// - From is a structured key + #[inline] + pub fn count_from<'a, P>( + self: &'a Arc, + from: &P, + ) -> impl Future + Send + 'a + use<'a, P> + where + P: Serialize + ?Sized + Debug + 'a, + { + self.keys_from_raw(from).count() + } -/// Count the number of entries in the map starting from a lower-bound. -/// -/// - From is a raw -#[implement(super::Map)] -#[inline] -pub fn raw_count_from<'a, P>( - self: &'a Arc, - from: &'a P, -) -> impl Future + Send + 'a -where - P: AsRef<[u8]> + ?Sized + Debug + Sync + 'a, -{ - self.raw_keys_from(from).count() -} + /// Count the number of entries in the map starting from a lower-bound. + /// + /// - From is a raw + #[inline] + pub fn raw_count_from<'a, P>( + self: &'a Arc, + from: &'a P, + ) -> impl Future + Send + 'a + where + P: AsRef<[u8]> + ?Sized + Debug + Sync + 'a, + { + self.raw_keys_from(from).count() + } -/// Count the number of entries in the map matching a prefix. -/// -/// - Prefix is structured key -#[implement(super::Map)] -#[inline] -pub fn count_prefix<'a, P>( - self: &'a Arc, - prefix: &P, -) -> impl Future + Send + 'a + use<'a, P> -where - P: Serialize + ?Sized + Debug + 'a, -{ - self.keys_prefix_raw(prefix).count() -} + /// Count the number of entries in the map matching a prefix. + /// + /// - Prefix is structured key + #[inline] + pub fn count_prefix<'a, P>( + self: &'a Arc, + prefix: &P, + ) -> impl Future + Send + 'a + use<'a, P> + where + P: Serialize + ?Sized + Debug + 'a, + { + self.keys_prefix_raw(prefix).count() + } -/// Count the number of entries in the map matching a prefix. -/// -/// - Prefix is raw -#[implement(super::Map)] -#[inline] -pub fn raw_count_prefix<'a, P>( - self: &'a Arc, - prefix: &'a P, -) -> impl Future + Send + 'a -where - P: AsRef<[u8]> + ?Sized + Debug + Sync + 'a, -{ - self.raw_keys_prefix(prefix).count() + /// Count the number of entries in the map matching a prefix. + /// + /// - Prefix is raw + #[inline] + pub fn raw_count_prefix<'a, P>( + self: &'a Arc, + prefix: &'a P, + ) -> impl Future + Send + 'a + where + P: AsRef<[u8]> + ?Sized + Debug + Sync + 'a, + { + self.raw_keys_prefix(prefix).count() + } } diff --git a/src/database/map/get.rs b/src/database/map/get.rs index 0971fb17e..c8407f7dd 100644 --- a/src/database/map/get.rs +++ b/src/database/map/get.rs @@ -1,6 +1,6 @@ use std::{convert::AsRef, fmt::Debug, sync::Arc}; -use conduwuit::{Err, Result, err, implement, utils::result::MapExpect}; +use conduwuit::{Err, Result, err, utils::result::MapExpect}; use futures::{Future, FutureExt, TryFutureExt, future::ready}; use rocksdb::{DBPinnableSlice, ReadOptions}; use tokio::task; @@ -10,74 +10,72 @@ use crate::{ util::{is_incomplete, map_err, or_else}, }; -/// Fetch a value from the database into cache, returning a reference-handle -/// asynchronously. The key is referenced directly to perform the query. -#[implement(super::Map)] -#[tracing::instrument(skip(self, key), fields(%self), level = "trace")] -pub fn get( - self: &Arc, - key: &K, -) -> impl Future>> + Send + use<'_, K> -where - K: AsRef<[u8]> + Debug + ?Sized, -{ - use crate::pool::Get; +impl super::Map { + /// Fetch a value from the database into cache, returning a reference-handle + /// asynchronously. The key is referenced directly to perform the query. + #[tracing::instrument(skip(self, key), fields(%self), level = "trace")] + pub fn get( + self: &Arc, + key: &K, + ) -> impl Future>> + Send + use<'_, K> + where + K: AsRef<[u8]> + Debug + ?Sized, + { + use crate::pool::Get; - let cached = self.get_cached(key); - if matches!(cached, Err(_) | Ok(Some(_))) { - return task::consume_budget() - .map(move |()| cached.map_expect("data found in cache")) - .boxed(); + let cached = self.get_cached(key); + if matches!(cached, Err(_) | Ok(Some(_))) { + return task::consume_budget() + .map(move |()| cached.map_expect("data found in cache")) + .boxed(); + } + + debug_assert!(matches!(cached, Ok(None)), "expected status Incomplete"); + let cmd = Get { + map: self.clone(), + key: [key.as_ref().into()].into(), + res: None, + }; + + self.db + .pool + .execute_get(cmd) + .and_then(|mut res| ready(res.remove(0))) + .boxed() } - debug_assert!(matches!(cached, Ok(None)), "expected status Incomplete"); - let cmd = Get { - map: self.clone(), - key: [key.as_ref().into()].into(), - res: None, - }; + /// Fetch a value from the cache without I/O. + #[tracing::instrument(skip(self, key), name = "cache", level = "trace")] + pub(crate) fn get_cached(&self, key: &K) -> Result>> + where + K: AsRef<[u8]> + Debug + ?Sized, + { + let res = self.get_blocking_opts(key, &self.cache_read_options); + cached_handle_from(res) + } - self.db - .pool - .execute_get(cmd) - .and_then(|mut res| ready(res.remove(0))) - .boxed() -} + /// Fetch a value from the database into cache, returning a + /// reference-handle. The key is referenced directly to perform the query. + /// This is a thread- blocking call. + #[tracing::instrument(skip(self, key), name = "blocking", level = "trace")] + pub fn get_blocking(&self, key: &K) -> Result> + where + K: AsRef<[u8]> + ?Sized, + { + let res = self.get_blocking_opts(key, &self.read_options); + handle_from(res) + } -/// Fetch a value from the cache without I/O. -#[implement(super::Map)] -#[tracing::instrument(skip(self, key), name = "cache", level = "trace")] -pub(crate) fn get_cached(&self, key: &K) -> Result>> -where - K: AsRef<[u8]> + Debug + ?Sized, -{ - let res = self.get_blocking_opts(key, &self.cache_read_options); - cached_handle_from(res) -} - -/// Fetch a value from the database into cache, returning a reference-handle. -/// The key is referenced directly to perform the query. This is a thread- -/// blocking call. -#[implement(super::Map)] -#[tracing::instrument(skip(self, key), name = "blocking", level = "trace")] -pub fn get_blocking(&self, key: &K) -> Result> -where - K: AsRef<[u8]> + ?Sized, -{ - let res = self.get_blocking_opts(key, &self.read_options); - handle_from(res) -} - -#[implement(super::Map)] -fn get_blocking_opts( - &self, - key: &K, - read_options: &ReadOptions, -) -> Result>, rocksdb::Error> -where - K: AsRef<[u8]> + ?Sized, -{ - self.db.db.get_pinned_cf_opt(&self.cf(), key, read_options) + fn get_blocking_opts( + &self, + key: &K, + read_options: &ReadOptions, + ) -> Result>, rocksdb::Error> + where + K: AsRef<[u8]> + ?Sized, + { + self.db.db.get_pinned_cf_opt(&self.cf(), key, read_options) + } } #[inline] diff --git a/src/database/map/get_batch.rs b/src/database/map/get_batch.rs index 20a5fb7bc..998c8f717 100644 --- a/src/database/map/get_batch.rs +++ b/src/database/map/get_batch.rs @@ -1,7 +1,7 @@ use std::{convert::AsRef, sync::Arc}; use conduwuit::{ - Result, implement, + Result, utils::{ IterStream, stream::{WidebandExt, automatic_amplification, automatic_width}, @@ -34,60 +34,59 @@ where } } -#[implement(super::Map)] -#[tracing::instrument(skip(self, keys), level = "trace")] -pub(crate) fn get_batch<'a, S, K>( - self: &'a Arc, - keys: S, -) -> impl Stream>> + Send + 'a -where - S: Stream + Send + 'a, - K: AsRef<[u8]> + Send + Sync + 'a, -{ - use crate::pool::Get; +impl super::Map { + #[tracing::instrument(skip(self, keys), level = "trace")] + pub(crate) fn get_batch<'a, S, K>( + self: &'a Arc, + keys: S, + ) -> impl Stream>> + Send + 'a + where + S: Stream + Send + 'a, + K: AsRef<[u8]> + Send + Sync + 'a, + { + use crate::pool::Get; - keys.ready_chunks(automatic_amplification()) - .widen_then(automatic_width(), |chunk| { - self.db.pool.execute_get(Get { - map: self.clone(), - key: chunk.iter().map(AsRef::as_ref).map(Into::into).collect(), - res: None, + keys.ready_chunks(automatic_amplification()) + .widen_then(automatic_width(), |chunk| { + self.db.pool.execute_get(Get { + map: self.clone(), + key: chunk.iter().map(AsRef::as_ref).map(Into::into).collect(), + res: None, + }) }) - }) - .map_ok(|results| results.into_iter().stream()) - .try_flatten() -} + .map_ok(|results| results.into_iter().stream()) + .try_flatten() + } -#[implement(super::Map)] -#[tracing::instrument(name = "batch_blocking", level = "trace", skip_all)] -pub(crate) fn get_batch_blocking<'a, I, K>( - &self, - keys: I, -) -> impl Iterator>> + Send + use<'_, I, K> -where - I: Iterator + ExactSizeIterator + Send, - K: AsRef<[u8]> + Send + ?Sized + Sync + 'a, -{ - self.get_batch_blocking_opts(keys, &self.read_options) - .map(handle_from) -} + #[tracing::instrument(name = "batch_blocking", level = "trace", skip_all)] + pub(crate) fn get_batch_blocking<'a, I, K>( + &self, + keys: I, + ) -> impl Iterator>> + Send + use<'_, I, K> + where + I: Iterator + ExactSizeIterator + Send, + K: AsRef<[u8]> + Send + ?Sized + Sync + 'a, + { + self.get_batch_blocking_opts(keys, &self.read_options) + .map(handle_from) + } -#[implement(super::Map)] -fn get_batch_blocking_opts<'a, I, K>( - &self, - keys: I, - read_options: &ReadOptions, -) -> impl Iterator>, rocksdb::Error>> + Send + use<'_, I, K> -where - I: Iterator + ExactSizeIterator + Send, - K: AsRef<[u8]> + Send + ?Sized + Sync + 'a, -{ - // Optimization can be `true` if key vector is pre-sorted **by the column - // comparator**. - const SORTED: bool = false; + fn get_batch_blocking_opts<'a, I, K>( + &self, + keys: I, + read_options: &ReadOptions, + ) -> impl Iterator>, rocksdb::Error>> + Send + use<'_, I, K> + where + I: Iterator + ExactSizeIterator + Send, + K: AsRef<[u8]> + Send + ?Sized + Sync + 'a, + { + // Optimization can be `true` if key vector is pre-sorted **by the column + // comparator**. + const SORTED: bool = false; - self.db - .db - .batched_multi_get_cf_opt(&self.cf(), keys, SORTED, read_options) - .into_iter() + self.db + .db + .batched_multi_get_cf_opt(&self.cf(), keys, SORTED, read_options) + .into_iter() + } } diff --git a/src/database/map/insert.rs b/src/database/map/insert.rs index 8d2d4fea4..80e8c448c 100644 --- a/src/database/map/insert.rs +++ b/src/database/map/insert.rs @@ -5,7 +5,7 @@ use std::{convert::AsRef, fmt::Debug, io::Write}; -use conduwuit::{arrayvec::ArrayVec, implement}; +use conduwuit::arrayvec::ArrayVec; use rocksdb::WriteBatchWithTransaction; use serde::Serialize; @@ -15,223 +15,212 @@ use crate::{ util::or_else, }; -/// Insert Key/Value -/// -/// - Key is serialized -/// - Val is serialized -#[implement(super::Map)] -#[inline] -pub fn put(&self, key: K, val: V) -where - K: Serialize + Debug, - V: Serialize, -{ - let mut key_buf = KeyBuf::new(); - let mut val_buf = ValBuf::new(); - self.bput(key, val, (&mut key_buf, &mut val_buf)); -} - -/// Insert Key/Value -/// -/// - Key is serialized -/// - Val is raw -#[implement(super::Map)] -#[inline] -pub fn put_raw(&self, key: K, val: V) -where - K: Serialize + Debug, - V: AsRef<[u8]>, -{ - let mut key_buf = KeyBuf::new(); - self.bput_raw(key, val, &mut key_buf); -} - -/// Insert Key/Value -/// -/// - Key is raw -/// - Val is serialized -#[implement(super::Map)] -#[inline] -pub fn raw_put(&self, key: K, val: V) -where - K: AsRef<[u8]>, - V: Serialize, -{ - let mut val_buf = ValBuf::new(); - self.raw_bput(key, val, &mut val_buf); -} - -/// Insert Key/Value -/// -/// - Key is serialized -/// - Val is serialized to stack-buffer -#[implement(super::Map)] -#[inline] -pub fn put_aput(&self, key: K, val: V) -where - K: Serialize + Debug, - V: Serialize, -{ - let mut key_buf = KeyBuf::new(); - let mut val_buf = ArrayVec::::new(); - self.bput(key, val, (&mut key_buf, &mut val_buf)); -} - -/// Insert Key/Value -/// -/// - Key is serialized to stack-buffer -/// - Val is serialized -#[implement(super::Map)] -#[inline] -pub fn aput_put(&self, key: K, val: V) -where - K: Serialize + Debug, - V: Serialize, -{ - let mut key_buf = ArrayVec::::new(); - let mut val_buf = ValBuf::new(); - self.bput(key, val, (&mut key_buf, &mut val_buf)); -} - -/// Insert Key/Value -/// -/// - Key is serialized to stack-buffer -/// - Val is serialized to stack-buffer -#[implement(super::Map)] -#[inline] -pub fn aput(&self, key: K, val: V) -where - K: Serialize + Debug, - V: Serialize, -{ - let mut key_buf = ArrayVec::::new(); - let mut val_buf = ArrayVec::::new(); - self.bput(key, val, (&mut key_buf, &mut val_buf)); -} - -/// Insert Key/Value -/// -/// - Key is serialized to stack-buffer -/// - Val is raw -#[implement(super::Map)] -#[inline] -pub fn aput_raw(&self, key: K, val: V) -where - K: Serialize + Debug, - V: AsRef<[u8]>, -{ - let mut key_buf = ArrayVec::::new(); - self.bput_raw(key, val, &mut key_buf); -} - -/// Insert Key/Value -/// -/// - Key is raw -/// - Val is serialized to stack-buffer -#[implement(super::Map)] -#[inline] -pub fn raw_aput(&self, key: K, val: V) -where - K: AsRef<[u8]>, - V: Serialize, -{ - let mut val_buf = ArrayVec::::new(); - self.raw_bput(key, val, &mut val_buf); -} - -/// Insert Key/Value -/// -/// - Key is serialized to supplied buffer -/// - Val is serialized to supplied buffer -#[implement(super::Map)] -pub fn bput(&self, key: K, val: V, mut buf: (Bk, Bv)) -where - K: Serialize + Debug, - V: Serialize, - Bk: Write + AsRef<[u8]>, - Bv: Write + AsRef<[u8]>, -{ - let val = ser::serialize(&mut buf.1, val).expect("failed to serialize insertion val"); - self.bput_raw(key, val, &mut buf.0); -} - -/// Insert Key/Value -/// -/// - Key is serialized to supplied buffer -/// - Val is raw -#[implement(super::Map)] -#[tracing::instrument(skip(self, val, buf), level = "trace")] -pub fn bput_raw(&self, key: K, val: V, mut buf: Bk) -where - K: Serialize + Debug, - V: AsRef<[u8]>, - Bk: Write + AsRef<[u8]>, -{ - let key = ser::serialize(&mut buf, key).expect("failed to serialize insertion key"); - self.insert(&key, val); -} - -/// Insert Key/Value -/// -/// - Key is raw -/// - Val is serialized to supplied buffer -#[implement(super::Map)] -pub fn raw_bput(&self, key: K, val: V, mut buf: Bv) -where - K: AsRef<[u8]>, - V: Serialize, - Bv: Write + AsRef<[u8]>, -{ - let val = ser::serialize(&mut buf, val).expect("failed to serialize insertion val"); - self.insert(&key, val); -} - -/// Insert Key/Value -/// -/// - Key is raw -/// - Val is raw -#[implement(super::Map)] -#[tracing::instrument(skip_all, fields(%self), level = "trace")] -pub fn insert(&self, key: &K, val: V) -where - K: AsRef<[u8]> + ?Sized, - V: AsRef<[u8]>, -{ - let write_options = &self.write_options; - self.db - .db - .put_cf_opt(&self.cf(), key, val, write_options) - .or_else(or_else) - .expect("database insert error"); - - if !self.db.corked() { - self.db.flush().expect("database flush error"); +impl super::Map { + /// Insert Key/Value + /// + /// - Key is serialized + /// - Val is serialized + #[inline] + pub fn put(&self, key: K, val: V) + where + K: Serialize + Debug, + V: Serialize, + { + let mut key_buf = KeyBuf::new(); + let mut val_buf = ValBuf::new(); + self.bput(key, val, (&mut key_buf, &mut val_buf)); } - self.watchers.wake(key.as_ref()); -} - -#[implement(super::Map)] -#[tracing::instrument(skip(self, iter), fields(%self), level = "trace")] -pub fn insert_batch<'a, I, K, V>(&'a self, iter: I) -where - I: Iterator + Send + Debug, - K: AsRef<[u8]> + Sized + Debug + 'a, - V: AsRef<[u8]> + Sized + 'a, -{ - let mut batch = WriteBatchWithTransaction::::default(); - for (key, val) in iter { - batch.put_cf(&self.cf(), key.as_ref(), val.as_ref()); + /// Insert Key/Value + /// + /// - Key is serialized + /// - Val is raw + #[inline] + pub fn put_raw(&self, key: K, val: V) + where + K: Serialize + Debug, + V: AsRef<[u8]>, + { + let mut key_buf = KeyBuf::new(); + self.bput_raw(key, val, &mut key_buf); } - let write_options = &self.write_options; - self.db - .db - .write_opt(&batch, write_options) - .or_else(or_else) - .expect("database insert batch error"); + /// Insert Key/Value + /// + /// - Key is raw + /// - Val is serialized + #[inline] + pub fn raw_put(&self, key: K, val: V) + where + K: AsRef<[u8]>, + V: Serialize, + { + let mut val_buf = ValBuf::new(); + self.raw_bput(key, val, &mut val_buf); + } - if !self.db.corked() { - self.db.flush().expect("database flush error"); + /// Insert Key/Value + /// + /// - Key is serialized + /// - Val is serialized to stack-buffer + #[inline] + pub fn put_aput(&self, key: K, val: V) + where + K: Serialize + Debug, + V: Serialize, + { + let mut key_buf = KeyBuf::new(); + let mut val_buf = ArrayVec::::new(); + self.bput(key, val, (&mut key_buf, &mut val_buf)); + } + + /// Insert Key/Value + /// + /// - Key is serialized to stack-buffer + /// - Val is serialized + #[inline] + pub fn aput_put(&self, key: K, val: V) + where + K: Serialize + Debug, + V: Serialize, + { + let mut key_buf = ArrayVec::::new(); + let mut val_buf = ValBuf::new(); + self.bput(key, val, (&mut key_buf, &mut val_buf)); + } + + /// Insert Key/Value + /// + /// - Key is serialized to stack-buffer + /// - Val is serialized to stack-buffer + #[inline] + pub fn aput(&self, key: K, val: V) + where + K: Serialize + Debug, + V: Serialize, + { + let mut key_buf = ArrayVec::::new(); + let mut val_buf = ArrayVec::::new(); + self.bput(key, val, (&mut key_buf, &mut val_buf)); + } + + /// Insert Key/Value + /// + /// - Key is serialized to stack-buffer + /// - Val is raw + #[inline] + pub fn aput_raw(&self, key: K, val: V) + where + K: Serialize + Debug, + V: AsRef<[u8]>, + { + let mut key_buf = ArrayVec::::new(); + self.bput_raw(key, val, &mut key_buf); + } + + /// Insert Key/Value + /// + /// - Key is raw + /// - Val is serialized to stack-buffer + #[inline] + pub fn raw_aput(&self, key: K, val: V) + where + K: AsRef<[u8]>, + V: Serialize, + { + let mut val_buf = ArrayVec::::new(); + self.raw_bput(key, val, &mut val_buf); + } + + /// Insert Key/Value + /// + /// - Key is serialized to supplied buffer + /// - Val is serialized to supplied buffer + pub fn bput(&self, key: K, val: V, mut buf: (Bk, Bv)) + where + K: Serialize + Debug, + V: Serialize, + Bk: Write + AsRef<[u8]>, + Bv: Write + AsRef<[u8]>, + { + let val = ser::serialize(&mut buf.1, val).expect("failed to serialize insertion val"); + self.bput_raw(key, val, &mut buf.0); + } + + /// Insert Key/Value + /// + /// - Key is serialized to supplied buffer + /// - Val is raw + #[tracing::instrument(skip(self, val, buf), level = "trace")] + pub fn bput_raw(&self, key: K, val: V, mut buf: Bk) + where + K: Serialize + Debug, + V: AsRef<[u8]>, + Bk: Write + AsRef<[u8]>, + { + let key = ser::serialize(&mut buf, key).expect("failed to serialize insertion key"); + self.insert(&key, val); + } + + /// Insert Key/Value + /// + /// - Key is raw + /// - Val is serialized to supplied buffer + pub fn raw_bput(&self, key: K, val: V, mut buf: Bv) + where + K: AsRef<[u8]>, + V: Serialize, + Bv: Write + AsRef<[u8]>, + { + let val = ser::serialize(&mut buf, val).expect("failed to serialize insertion val"); + self.insert(&key, val); + } + + /// Insert Key/Value + /// + /// - Key is raw + /// - Val is raw + #[tracing::instrument(skip_all, fields(%self), level = "trace")] + pub fn insert(&self, key: &K, val: V) + where + K: AsRef<[u8]> + ?Sized, + V: AsRef<[u8]>, + { + let write_options = &self.write_options; + self.db + .db + .put_cf_opt(&self.cf(), key, val, write_options) + .or_else(or_else) + .expect("database insert error"); + + if !self.db.corked() { + self.db.flush().expect("database flush error"); + } + + self.watchers.wake(key.as_ref()); + } + + #[tracing::instrument(skip(self, iter), fields(%self), level = "trace")] + pub fn insert_batch<'a, I, K, V>(&'a self, iter: I) + where + I: Iterator + Send + Debug, + K: AsRef<[u8]> + Sized + Debug + 'a, + V: AsRef<[u8]> + Sized + 'a, + { + let mut batch = WriteBatchWithTransaction::::default(); + for (key, val) in iter { + batch.put_cf(&self.cf(), key.as_ref(), val.as_ref()); + } + + let write_options = &self.write_options; + self.db + .db + .write_opt(&batch, write_options) + .or_else(or_else) + .expect("database insert batch error"); + + if !self.db.corked() { + self.db.flush().expect("database flush error"); + } } } diff --git a/src/database/map/keys_from.rs b/src/database/map/keys_from.rs index 11245f7b8..80bafd8db 100644 --- a/src/database/map/keys_from.rs +++ b/src/database/map/keys_from.rs @@ -11,73 +11,71 @@ use crate::{ stream, }; -#[implement(super::Map)] -pub fn keys_from<'a, K, P>( - self: &'a Arc, - from: &P, -) -> impl Stream>> + Send + use<'a, K, P> -where - P: Serialize + ?Sized + Debug, - K: Deserialize<'a> + Send, -{ - self.keys_from_raw(from).map(result_deserialize_key::) -} - -#[implement(super::Map)] -#[tracing::instrument(skip(self), level = "trace")] -pub fn keys_from_raw

( - self: &Arc, - from: &P, -) -> impl Stream>> + Send + use<'_, P> -where - P: Serialize + ?Sized + Debug, -{ - let key = serialize_key(from).expect("failed to serialize query key"); - self.raw_keys_from(&key) -} - -#[implement(super::Map)] -pub fn keys_raw_from<'a, K, P>( - self: &'a Arc, - from: &P, -) -> impl Stream>> + Send + use<'a, K, P> -where - P: AsRef<[u8]> + ?Sized + Debug + Sync, - K: Deserialize<'a> + Send, -{ - self.raw_keys_from(from).map(result_deserialize_key::) -} - -#[implement(super::Map)] -#[tracing::instrument(skip(self, from), fields(%self), level = "trace")] -pub fn raw_keys_from

( - self: &Arc, - from: &P, -) -> impl Stream>> + Send + use<'_, P> -where - P: AsRef<[u8]> + ?Sized + Debug, -{ - use crate::pool::Seek; - - let opts = super::iter_options_default(&self.db); - let state = stream::State::new(self, opts); - if is_cached(self, from) { - return stream::Keys::<'_>::from(state.init_fwd(from.as_ref().into())).boxed(); +impl super::Map { + pub fn keys_from<'a, K, P>( + self: &'a Arc, + from: &P, + ) -> impl Stream>> + Send + use<'a, K, P> + where + P: Serialize + ?Sized + Debug, + K: Deserialize<'a> + Send, + { + self.keys_from_raw(from).map(result_deserialize_key::) } - let seek = Seek { - map: self.clone(), - dir: Direction::Forward, - key: Some(from.as_ref().into()), - state: crate::pool::into_send_seek(state), - res: None, - }; + #[tracing::instrument(skip(self), level = "trace")] + pub fn keys_from_raw

( + self: &Arc, + from: &P, + ) -> impl Stream>> + Send + use<'_, P> + where + P: Serialize + ?Sized + Debug, + { + let key = serialize_key(from).expect("failed to serialize query key"); + self.raw_keys_from(&key) + } - self.db - .pool - .execute_iter(seek) - .ok_into::>() - .into_stream() - .try_flatten() - .boxed() + pub fn keys_raw_from<'a, K, P>( + self: &'a Arc, + from: &P, + ) -> impl Stream>> + Send + use<'a, K, P> + where + P: AsRef<[u8]> + ?Sized + Debug + Sync, + K: Deserialize<'a> + Send, + { + self.raw_keys_from(from).map(result_deserialize_key::) + } + + #[tracing::instrument(skip(self, from), fields(%self), level = "trace")] + pub fn raw_keys_from

( + self: &Arc, + from: &P, + ) -> impl Stream>> + Send + use<'_, P> + where + P: AsRef<[u8]> + ?Sized + Debug, + { + use crate::pool::Seek; + + let opts = super::iter_options_default(&self.db); + let state = stream::State::new(self, opts); + if is_cached(self, from) { + return stream::Keys::<'_>::from(state.init_fwd(from.as_ref().into())).boxed(); + } + + let seek = Seek { + map: self.clone(), + dir: Direction::Forward, + key: Some(from.as_ref().into()), + state: crate::pool::into_send_seek(state), + res: None, + }; + + self.db + .pool + .execute_iter(seek) + .ok_into::>() + .into_stream() + .try_flatten() + .boxed() + } } diff --git a/src/database/map/keys_prefix.rs b/src/database/map/keys_prefix.rs index e6a9457f5..9477ec06d 100644 --- a/src/database/map/keys_prefix.rs +++ b/src/database/map/keys_prefix.rs @@ -1,59 +1,57 @@ use std::{convert::AsRef, fmt::Debug, sync::Arc}; -use conduwuit::{Result, implement}; +use conduwuit::Result; use futures::{Stream, StreamExt, TryStreamExt, future}; use serde::{Deserialize, Serialize}; use crate::keyval::{Key, result_deserialize_key, serialize_key}; -#[implement(super::Map)] -pub fn keys_prefix<'a, K, P>( - self: &'a Arc, - prefix: &P, -) -> impl Stream>> + Send + use<'a, K, P> -where - P: Serialize + ?Sized + Debug, - K: Deserialize<'a> + Send, -{ - self.keys_prefix_raw(prefix) - .map(result_deserialize_key::) -} +impl super::Map { + pub fn keys_prefix<'a, K, P>( + self: &'a Arc, + prefix: &P, + ) -> impl Stream>> + Send + use<'a, K, P> + where + P: Serialize + ?Sized + Debug, + K: Deserialize<'a> + Send, + { + self.keys_prefix_raw(prefix) + .map(result_deserialize_key::) + } -#[implement(super::Map)] -#[tracing::instrument(skip(self), level = "trace")] -pub fn keys_prefix_raw

( - self: &Arc, - prefix: &P, -) -> impl Stream>> + Send + use<'_, P> -where - P: Serialize + ?Sized + Debug, -{ - let key = serialize_key(prefix).expect("failed to serialize query key"); - self.raw_keys_from(&key) - .try_take_while(move |k: &Key<'_>| future::ok(k.starts_with(&key))) -} + #[tracing::instrument(skip(self), level = "trace")] + pub fn keys_prefix_raw

( + self: &Arc, + prefix: &P, + ) -> impl Stream>> + Send + use<'_, P> + where + P: Serialize + ?Sized + Debug, + { + let key = serialize_key(prefix).expect("failed to serialize query key"); + self.raw_keys_from(&key) + .try_take_while(move |k: &Key<'_>| future::ok(k.starts_with(&key))) + } -#[implement(super::Map)] -pub fn keys_raw_prefix<'a, K, P>( - self: &'a Arc, - prefix: &'a P, -) -> impl Stream>> + Send + 'a -where - P: AsRef<[u8]> + ?Sized + Debug + Sync + 'a, - K: Deserialize<'a> + Send + 'a, -{ - self.raw_keys_prefix(prefix) - .map(result_deserialize_key::) -} + pub fn keys_raw_prefix<'a, K, P>( + self: &'a Arc, + prefix: &'a P, + ) -> impl Stream>> + Send + 'a + where + P: AsRef<[u8]> + ?Sized + Debug + Sync + 'a, + K: Deserialize<'a> + Send + 'a, + { + self.raw_keys_prefix(prefix) + .map(result_deserialize_key::) + } -#[implement(super::Map)] -pub fn raw_keys_prefix<'a, P>( - self: &'a Arc, - prefix: &'a P, -) -> impl Stream>> + Send + 'a -where - P: AsRef<[u8]> + ?Sized + Debug + Sync + 'a, -{ - self.raw_keys_from(prefix) - .try_take_while(|k: &Key<'_>| future::ok(k.starts_with(prefix.as_ref()))) + pub fn raw_keys_prefix<'a, P>( + self: &'a Arc, + prefix: &'a P, + ) -> impl Stream>> + Send + 'a + where + P: AsRef<[u8]> + ?Sized + Debug + Sync + 'a, + { + self.raw_keys_from(prefix) + .try_take_while(|k: &Key<'_>| future::ok(k.starts_with(prefix.as_ref()))) + } } diff --git a/src/database/map/qry.rs b/src/database/map/qry.rs index c6f13c0b7..79f95dade 100644 --- a/src/database/map/qry.rs +++ b/src/database/map/qry.rs @@ -1,56 +1,56 @@ use std::{convert::AsRef, fmt::Debug, io::Write, sync::Arc}; -use conduwuit::{Result, arrayvec::ArrayVec, implement}; +use conduwuit::{Result, arrayvec::ArrayVec}; use futures::Future; use serde::Serialize; use crate::{Handle, keyval::KeyBuf, ser}; -/// Fetch a value from the database into cache, returning a reference-handle -/// asynchronously. The key is serialized into an allocated buffer to perform -/// the query. -#[implement(super::Map)] -#[inline] -pub fn qry( - self: &Arc, - key: &K, -) -> impl Future>> + Send + use<'_, K> -where - K: Serialize + ?Sized + Debug, -{ - let mut buf = KeyBuf::new(); - self.bqry(key, &mut buf) -} +impl super::Map { + /// Fetch a value from the database into cache, returning a reference-handle + /// asynchronously. The key is serialized into an allocated buffer to + /// perform the query. + #[inline] + pub fn qry( + self: &Arc, + key: &K, + ) -> impl Future>> + Send + use<'_, K> + where + K: Serialize + ?Sized + Debug, + { + let mut buf = KeyBuf::new(); + self.bqry(key, &mut buf) + } -/// Fetch a value from the database into cache, returning a reference-handle -/// asynchronously. The key is serialized into a fixed-sized buffer to perform -/// the query. The maximum size is supplied as const generic parameter. -#[implement(super::Map)] -#[inline] -pub fn aqry( - self: &Arc, - key: &K, -) -> impl Future>> + Send + use<'_, MAX, K> -where - K: Serialize + ?Sized + Debug, -{ - let mut buf = ArrayVec::::new(); - self.bqry(key, &mut buf) -} + /// Fetch a value from the database into cache, returning a reference-handle + /// asynchronously. The key is serialized into a fixed-sized buffer to + /// perform the query. The maximum size is supplied as const generic + /// parameter. + #[inline] + pub fn aqry( + self: &Arc, + key: &K, + ) -> impl Future>> + Send + use<'_, MAX, K> + where + K: Serialize + ?Sized + Debug, + { + let mut buf = ArrayVec::::new(); + self.bqry(key, &mut buf) + } -/// Fetch a value from the database into cache, returning a reference-handle -/// asynchronously. The key is serialized into a user-supplied Writer. -#[implement(super::Map)] -#[tracing::instrument(skip(self, buf), level = "trace")] -pub fn bqry( - self: &Arc, - key: &K, - buf: &mut B, -) -> impl Future>> + Send + use<'_, K, B> -where - K: Serialize + ?Sized + Debug, - B: Write + AsRef<[u8]>, -{ - let key = ser::serialize(buf, key).expect("failed to serialize query key"); - self.get(key) + /// Fetch a value from the database into cache, returning a reference-handle + /// asynchronously. The key is serialized into a user-supplied Writer. + #[tracing::instrument(skip(self, buf), level = "trace")] + pub fn bqry( + self: &Arc, + key: &K, + buf: &mut B, + ) -> impl Future>> + Send + use<'_, K, B> + where + K: Serialize + ?Sized + Debug, + B: Write + AsRef<[u8]>, + { + let key = ser::serialize(buf, key).expect("failed to serialize query key"); + self.get(key) + } } diff --git a/src/database/map/qry_batch.rs b/src/database/map/qry_batch.rs index 9da546e6c..a1f38c4e1 100644 --- a/src/database/map/qry_batch.rs +++ b/src/database/map/qry_batch.rs @@ -32,30 +32,31 @@ where } } -#[implement(super::Map)] -#[tracing::instrument(skip(self, keys), level = "trace")] -pub(crate) fn qry_batch<'a, S, K>( - self: &'a Arc, - keys: S, -) -> impl Stream>> + Send + 'a -where - S: Stream + Send + 'a, - K: Serialize + Debug + 'a, -{ - use crate::pool::Get; +impl super::Map { + #[tracing::instrument(skip(self, keys), level = "trace")] + pub(crate) fn qry_batch<'a, S, K>( + self: &'a Arc, + keys: S, + ) -> impl Stream>> + Send + 'a + where + S: Stream + Send + 'a, + K: Serialize + Debug + 'a, + { + use crate::pool::Get; - keys.ready_chunks(automatic_amplification()) - .widen_then(automatic_width(), |chunk| { - let keys = chunk - .iter() - .map(ser::serialize_to::) - .map(|result| result.expect("failed to serialize query key")) - .collect(); + keys.ready_chunks(automatic_amplification()) + .widen_then(automatic_width(), |chunk| { + let keys = chunk + .iter() + .map(ser::serialize_to::) + .map(|result| result.expect("failed to serialize query key")) + .collect(); - self.db - .pool - .execute_get(Get { map: self.clone(), key: keys, res: None }) - }) - .map_ok(|results| results.into_iter().stream()) - .try_flatten() + self.db + .pool + .execute_get(Get { map: self.clone(), key: keys, res: None }) + }) + .map_ok(|results| results.into_iter().stream()) + .try_flatten() + } } diff --git a/src/database/map/remove.rs b/src/database/map/remove.rs index a7ae91336..11ec3143c 100644 --- a/src/database/map/remove.rs +++ b/src/database/map/remove.rs @@ -1,55 +1,53 @@ use std::{convert::AsRef, fmt::Debug, io::Write}; -use conduwuit::{arrayvec::ArrayVec, implement}; +use conduwuit::arrayvec::ArrayVec; use serde::Serialize; use crate::{keyval::KeyBuf, ser, util::or_else}; -#[implement(super::Map)] -#[inline] -pub fn del(&self, key: K) -where - K: Serialize + Debug, -{ - let mut buf = KeyBuf::new(); - self.bdel(key, &mut buf); -} +impl super::Map { + #[inline] + pub fn del(&self, key: K) + where + K: Serialize + Debug, + { + let mut buf = KeyBuf::new(); + self.bdel(key, &mut buf); + } -#[implement(super::Map)] -#[inline] -pub fn adel(&self, key: K) -where - K: Serialize + Debug, -{ - let mut buf = ArrayVec::::new(); - self.bdel(key, &mut buf); -} + #[inline] + pub fn adel(&self, key: K) + where + K: Serialize + Debug, + { + let mut buf = ArrayVec::::new(); + self.bdel(key, &mut buf); + } -#[implement(super::Map)] -#[tracing::instrument(skip(self, buf), level = "trace")] -pub fn bdel(&self, key: K, buf: &mut B) -where - K: Serialize + Debug, - B: Write + AsRef<[u8]>, -{ - let key = ser::serialize(buf, key).expect("failed to serialize deletion key"); - self.remove(key); -} + #[tracing::instrument(skip(self, buf), level = "trace")] + pub fn bdel(&self, key: K, buf: &mut B) + where + K: Serialize + Debug, + B: Write + AsRef<[u8]>, + { + let key = ser::serialize(buf, key).expect("failed to serialize deletion key"); + self.remove(key); + } -#[implement(super::Map)] -#[tracing::instrument(skip(self, key), fields(%self), level = "trace")] -pub fn remove(&self, key: &K) -where - K: AsRef<[u8]> + ?Sized + Debug, -{ - let write_options = &self.write_options; - self.db - .db - .delete_cf_opt(&self.cf(), key, write_options) - .or_else(or_else) - .expect("database remove error"); + #[tracing::instrument(skip(self, key), fields(%self), level = "trace")] + pub fn remove(&self, key: &K) + where + K: AsRef<[u8]> + ?Sized + Debug, + { + let write_options = &self.write_options; + self.db + .db + .delete_cf_opt(&self.cf(), key, write_options) + .or_else(or_else) + .expect("database remove error"); - if !self.db.corked() { - self.db.flush().expect("database flush error"); + if !self.db.corked() { + self.db.flush().expect("database flush error"); + } } } diff --git a/src/database/map/rev_keys.rs b/src/database/map/rev_keys.rs index 8f48a17e6..cb264fc12 100644 --- a/src/database/map/rev_keys.rs +++ b/src/database/map/rev_keys.rs @@ -1,6 +1,6 @@ use std::sync::Arc; -use conduwuit::{Result, implement}; +use conduwuit::Result; use futures::{FutureExt, Stream, StreamExt, TryFutureExt, TryStreamExt}; use rocksdb::Direction; use serde::Deserialize; @@ -9,43 +9,43 @@ use tokio::task; use super::rev_stream::is_cached; use crate::{keyval, keyval::Key, stream}; -#[implement(super::Map)] -pub fn rev_keys<'a, K>(self: &'a Arc) -> impl Stream>> + Send -where - K: Deserialize<'a> + Send, -{ - self.rev_raw_keys().map(keyval::result_deserialize_key::) -} - -#[implement(super::Map)] -#[tracing::instrument(skip(self), fields(%self), level = "trace")] -pub fn rev_raw_keys(self: &Arc) -> impl Stream>> + Send { - use crate::pool::Seek; - - let opts = super::iter_options_default(&self.db); - let state = stream::State::new(self, opts); - if is_cached(self) { - let state = state.init_rev(None); - return task::consume_budget() - .map(move |()| stream::KeysRev::<'_>::from(state)) - .into_stream() - .flatten() - .boxed(); +impl super::Map { + pub fn rev_keys<'a, K>(self: &'a Arc) -> impl Stream>> + Send + where + K: Deserialize<'a> + Send, + { + self.rev_raw_keys().map(keyval::result_deserialize_key::) } - let seek = Seek { - map: self.clone(), - dir: Direction::Reverse, - state: crate::pool::into_send_seek(state), - key: None, - res: None, - }; + #[tracing::instrument(skip(self), fields(%self), level = "trace")] + pub fn rev_raw_keys(self: &Arc) -> impl Stream>> + Send { + use crate::pool::Seek; - self.db - .pool - .execute_iter(seek) - .ok_into::>() - .into_stream() - .try_flatten() - .boxed() + let opts = super::iter_options_default(&self.db); + let state = stream::State::new(self, opts); + if is_cached(self) { + let state = state.init_rev(None); + return task::consume_budget() + .map(move |()| stream::KeysRev::<'_>::from(state)) + .into_stream() + .flatten() + .boxed(); + } + + let seek = Seek { + map: self.clone(), + dir: Direction::Reverse, + state: crate::pool::into_send_seek(state), + key: None, + res: None, + }; + + self.db + .pool + .execute_iter(seek) + .ok_into::>() + .into_stream() + .try_flatten() + .boxed() + } } diff --git a/src/database/map/rev_keys_from.rs b/src/database/map/rev_keys_from.rs index 021e3b924..c5dfe52a4 100644 --- a/src/database/map/rev_keys_from.rs +++ b/src/database/map/rev_keys_from.rs @@ -1,6 +1,6 @@ use std::{convert::AsRef, fmt::Debug, sync::Arc}; -use conduwuit::{Result, implement}; +use conduwuit::Result; use futures::{FutureExt, Stream, StreamExt, TryFutureExt, TryStreamExt}; use rocksdb::Direction; use serde::{Deserialize, Serialize}; @@ -11,75 +11,73 @@ use crate::{ stream, }; -#[implement(super::Map)] -pub fn rev_keys_from<'a, K, P>( - self: &'a Arc, - from: &P, -) -> impl Stream>> + Send + use<'a, K, P> -where - P: Serialize + ?Sized + Debug, - K: Deserialize<'a> + Send, -{ - self.rev_keys_from_raw(from) - .map(result_deserialize_key::) -} - -#[implement(super::Map)] -#[tracing::instrument(skip(self), level = "trace")] -pub fn rev_keys_from_raw

( - self: &Arc, - from: &P, -) -> impl Stream>> + Send + use<'_, P> -where - P: Serialize + ?Sized + Debug, -{ - let key = serialize_key(from).expect("failed to serialize query key"); - self.rev_raw_keys_from(&key) -} - -#[implement(super::Map)] -pub fn rev_keys_raw_from<'a, K, P>( - self: &'a Arc, - from: &P, -) -> impl Stream>> + Send + use<'a, K, P> -where - P: AsRef<[u8]> + ?Sized + Debug + Sync, - K: Deserialize<'a> + Send, -{ - self.rev_raw_keys_from(from) - .map(result_deserialize_key::) -} - -#[implement(super::Map)] -#[tracing::instrument(skip(self, from), fields(%self), level = "trace")] -pub fn rev_raw_keys_from

( - self: &Arc, - from: &P, -) -> impl Stream>> + Send + use<'_, P> -where - P: AsRef<[u8]> + ?Sized + Debug, -{ - use crate::pool::Seek; - - let opts = super::iter_options_default(&self.db); - let state = stream::State::new(self, opts); - if is_cached(self, from) { - return stream::KeysRev::<'_>::from(state.init_rev(from.as_ref().into())).boxed(); +impl super::Map { + pub fn rev_keys_from<'a, K, P>( + self: &'a Arc, + from: &P, + ) -> impl Stream>> + Send + use<'a, K, P> + where + P: Serialize + ?Sized + Debug, + K: Deserialize<'a> + Send, + { + self.rev_keys_from_raw(from) + .map(result_deserialize_key::) } - let seek = Seek { - map: self.clone(), - dir: Direction::Reverse, - key: Some(from.as_ref().into()), - state: crate::pool::into_send_seek(state), - res: None, - }; + #[tracing::instrument(skip(self), level = "trace")] + pub fn rev_keys_from_raw

( + self: &Arc, + from: &P, + ) -> impl Stream>> + Send + use<'_, P> + where + P: Serialize + ?Sized + Debug, + { + let key = serialize_key(from).expect("failed to serialize query key"); + self.rev_raw_keys_from(&key) + } - self.db - .pool - .execute_iter(seek) - .ok_into::>() - .into_stream() - .try_flatten() - .boxed() + pub fn rev_keys_raw_from<'a, K, P>( + self: &'a Arc, + from: &P, + ) -> impl Stream>> + Send + use<'a, K, P> + where + P: AsRef<[u8]> + ?Sized + Debug + Sync, + K: Deserialize<'a> + Send, + { + self.rev_raw_keys_from(from) + .map(result_deserialize_key::) + } + + #[tracing::instrument(skip(self, from), fields(%self), level = "trace")] + pub fn rev_raw_keys_from

( + self: &Arc, + from: &P, + ) -> impl Stream>> + Send + use<'_, P> + where + P: AsRef<[u8]> + ?Sized + Debug, + { + use crate::pool::Seek; + + let opts = super::iter_options_default(&self.db); + let state = stream::State::new(self, opts); + if is_cached(self, from) { + return stream::KeysRev::<'_>::from(state.init_rev(from.as_ref().into())).boxed(); + } + + let seek = Seek { + map: self.clone(), + dir: Direction::Reverse, + key: Some(from.as_ref().into()), + state: crate::pool::into_send_seek(state), + res: None, + }; + + self.db + .pool + .execute_iter(seek) + .ok_into::>() + .into_stream() + .try_flatten() + .boxed() + } } diff --git a/src/database/map/rev_keys_prefix.rs b/src/database/map/rev_keys_prefix.rs index 5b1459f58..d55493e81 100644 --- a/src/database/map/rev_keys_prefix.rs +++ b/src/database/map/rev_keys_prefix.rs @@ -6,54 +6,52 @@ use serde::{Deserialize, Serialize}; use crate::keyval::{Key, result_deserialize_key, serialize_key}; -#[implement(super::Map)] -pub fn rev_keys_prefix<'a, K, P>( - self: &'a Arc, - prefix: &P, -) -> impl Stream>> + Send + use<'a, K, P> -where - P: Serialize + ?Sized + Debug, - K: Deserialize<'a> + Send, -{ - self.rev_keys_prefix_raw(prefix) - .map(result_deserialize_key::) -} +impl super::Map { + pub fn rev_keys_prefix<'a, K, P>( + self: &'a Arc, + prefix: &P, + ) -> impl Stream>> + Send + use<'a, K, P> + where + P: Serialize + ?Sized + Debug, + K: Deserialize<'a> + Send, + { + self.rev_keys_prefix_raw(prefix) + .map(result_deserialize_key::) + } -#[implement(super::Map)] -#[tracing::instrument(skip(self), level = "trace")] -pub fn rev_keys_prefix_raw

( - self: &Arc, - prefix: &P, -) -> impl Stream>> + Send + use<'_, P> -where - P: Serialize + ?Sized + Debug, -{ - let key = serialize_key(prefix).expect("failed to serialize query key"); - self.rev_raw_keys_from(&key) - .try_take_while(move |k: &Key<'_>| future::ok(k.starts_with(&key))) -} + #[tracing::instrument(skip(self), level = "trace")] + pub fn rev_keys_prefix_raw

( + self: &Arc, + prefix: &P, + ) -> impl Stream>> + Send + use<'_, P> + where + P: Serialize + ?Sized + Debug, + { + let key = serialize_key(prefix).expect("failed to serialize query key"); + self.rev_raw_keys_from(&key) + .try_take_while(move |k: &Key<'_>| future::ok(k.starts_with(&key))) + } -#[implement(super::Map)] -pub fn rev_keys_raw_prefix<'a, K, P>( - self: &'a Arc, - prefix: &'a P, -) -> impl Stream>> + Send + 'a -where - P: AsRef<[u8]> + ?Sized + Debug + Sync + 'a, - K: Deserialize<'a> + Send + 'a, -{ - self.rev_raw_keys_prefix(prefix) - .map(result_deserialize_key::) -} + pub fn rev_keys_raw_prefix<'a, K, P>( + self: &'a Arc, + prefix: &'a P, + ) -> impl Stream>> + Send + 'a + where + P: AsRef<[u8]> + ?Sized + Debug + Sync + 'a, + K: Deserialize<'a> + Send + 'a, + { + self.rev_raw_keys_prefix(prefix) + .map(result_deserialize_key::) + } -#[implement(super::Map)] -pub fn rev_raw_keys_prefix<'a, P>( - self: &'a Arc, - prefix: &'a P, -) -> impl Stream>> + Send + 'a -where - P: AsRef<[u8]> + ?Sized + Debug + Sync + 'a, -{ - self.rev_raw_keys_from(prefix) - .try_take_while(|k: &Key<'_>| future::ok(k.starts_with(prefix.as_ref()))) + pub fn rev_raw_keys_prefix<'a, P>( + self: &'a Arc, + prefix: &'a P, + ) -> impl Stream>> + Send + 'a + where + P: AsRef<[u8]> + ?Sized + Debug + Sync + 'a, + { + self.rev_raw_keys_from(prefix) + .try_take_while(|k: &Key<'_>| future::ok(k.starts_with(prefix.as_ref()))) + } } diff --git a/src/database/map/rev_stream.rs b/src/database/map/rev_stream.rs index 92d7bac87..380205ed7 100644 --- a/src/database/map/rev_stream.rs +++ b/src/database/map/rev_stream.rs @@ -1,6 +1,6 @@ use std::sync::Arc; -use conduwuit::{Result, implement}; +use conduwuit::Result; use futures::{FutureExt, Stream, StreamExt, TryFutureExt, TryStreamExt}; use rocksdb::Direction; use serde::Deserialize; @@ -8,55 +8,55 @@ use tokio::task; use crate::{keyval, keyval::KeyVal, stream}; -/// Iterate key-value entries in the map from the end. -/// -/// - Result is deserialized -#[implement(super::Map)] -pub fn rev_stream<'a, K, V>( - self: &'a Arc, -) -> impl Stream>> + Send -where - K: Deserialize<'a> + Send, - V: Deserialize<'a> + Send, -{ - self.rev_raw_stream() - .map(keyval::result_deserialize::) -} - -/// Iterate key-value entries in the map from the end. -/// -/// - Result is raw -#[implement(super::Map)] -#[tracing::instrument(skip(self), fields(%self), level = "trace")] -pub fn rev_raw_stream(self: &Arc) -> impl Stream>> + Send { - use crate::pool::Seek; - - let opts = super::iter_options_default(&self.db); - let state = stream::State::new(self, opts); - if is_cached(self) { - let state = state.init_rev(None); - return task::consume_budget() - .map(move |()| stream::ItemsRev::<'_>::from(state)) - .into_stream() - .flatten() - .boxed(); +impl super::Map { + /// Iterate key-value entries in the map from the end. + /// + /// - Result is deserialized + pub fn rev_stream<'a, K, V>( + self: &'a Arc, + ) -> impl Stream>> + Send + where + K: Deserialize<'a> + Send, + V: Deserialize<'a> + Send, + { + self.rev_raw_stream() + .map(keyval::result_deserialize::) } - let seek = Seek { - map: self.clone(), - dir: Direction::Reverse, - state: crate::pool::into_send_seek(state), - key: None, - res: None, - }; + /// Iterate key-value entries in the map from the end. + /// + /// - Result is raw + #[tracing::instrument(skip(self), fields(%self), level = "trace")] + pub fn rev_raw_stream(self: &Arc) -> impl Stream>> + Send { + use crate::pool::Seek; - self.db - .pool - .execute_iter(seek) - .ok_into::>() - .into_stream() - .try_flatten() - .boxed() + let opts = super::iter_options_default(&self.db); + let state = stream::State::new(self, opts); + if is_cached(self) { + let state = state.init_rev(None); + return task::consume_budget() + .map(move |()| stream::ItemsRev::<'_>::from(state)) + .into_stream() + .flatten() + .boxed(); + } + + let seek = Seek { + map: self.clone(), + dir: Direction::Reverse, + state: crate::pool::into_send_seek(state), + key: None, + res: None, + }; + + self.db + .pool + .execute_iter(seek) + .ok_into::>() + .into_stream() + .try_flatten() + .boxed() + } } #[tracing::instrument( diff --git a/src/database/map/rev_stream_from.rs b/src/database/map/rev_stream_from.rs index 7fef1b35e..1ced851d5 100644 --- a/src/database/map/rev_stream_from.rs +++ b/src/database/map/rev_stream_from.rs @@ -1,6 +1,6 @@ use std::{convert::AsRef, fmt::Debug, sync::Arc}; -use conduwuit::{Result, implement}; +use conduwuit::Result; use futures::{FutureExt, Stream, StreamExt, TryFutureExt, TryStreamExt}; use rocksdb::Direction; use serde::{Deserialize, Serialize}; @@ -12,100 +12,98 @@ use crate::{ util::is_incomplete, }; -/// Iterate key-value entries in the map starting from upper-bound. -/// -/// - Query is serialized -/// - Result is deserialized -#[implement(super::Map)] -pub fn rev_stream_from<'a, K, V, P>( - self: &'a Arc, - from: &P, -) -> impl Stream>> + Send + use<'a, K, V, P> -where - P: Serialize + ?Sized + Debug, - K: Deserialize<'a> + Send, - V: Deserialize<'a> + Send, -{ - self.rev_stream_from_raw(from) - .map(result_deserialize::) -} - -/// Iterate key-value entries in the map starting from upper-bound. -/// -/// - Query is serialized -/// - Result is raw -#[implement(super::Map)] -#[tracing::instrument(skip(self), level = "trace")] -pub fn rev_stream_from_raw

( - self: &Arc, - from: &P, -) -> impl Stream>> + Send + use<'_, P> -where - P: Serialize + ?Sized + Debug, -{ - let key = serialize_key(from).expect("failed to serialize query key"); - self.rev_raw_stream_from(&key) -} - -/// Iterate key-value entries in the map starting from upper-bound. -/// -/// - Query is raw -/// - Result is deserialized -#[implement(super::Map)] -pub fn rev_stream_raw_from<'a, K, V, P>( - self: &'a Arc, - from: &P, -) -> impl Stream>> + Send + use<'a, K, V, P> -where - P: AsRef<[u8]> + ?Sized + Debug + Sync, - K: Deserialize<'a> + Send, - V: Deserialize<'a> + Send, -{ - self.rev_raw_stream_from(from) - .map(result_deserialize::) -} - -/// Iterate key-value entries in the map starting from upper-bound. -/// -/// - Query is raw -/// - Result is raw -#[implement(super::Map)] -#[tracing::instrument(skip(self, from), fields(%self), level = "trace")] -pub fn rev_raw_stream_from

( - self: &Arc, - from: &P, -) -> impl Stream>> + Send + use<'_, P> -where - P: AsRef<[u8]> + ?Sized + Debug, -{ - use crate::pool::Seek; - - let opts = super::iter_options_default(&self.db); - let state = stream::State::new(self, opts); - if is_cached(self, from) { - let state = state.init_rev(from.as_ref().into()); - return task::consume_budget() - .map(move |()| stream::ItemsRev::<'_>::from(state)) - .into_stream() - .flatten() - .boxed(); +impl super::Map { + /// Iterate key-value entries in the map starting from upper-bound. + /// + /// - Query is serialized + /// - Result is deserialized + pub fn rev_stream_from<'a, K, V, P>( + self: &'a Arc, + from: &P, + ) -> impl Stream>> + Send + use<'a, K, V, P> + where + P: Serialize + ?Sized + Debug, + K: Deserialize<'a> + Send, + V: Deserialize<'a> + Send, + { + self.rev_stream_from_raw(from) + .map(result_deserialize::) } - let seek = Seek { - map: self.clone(), - dir: Direction::Reverse, - key: Some(from.as_ref().into()), - state: crate::pool::into_send_seek(state), - res: None, - }; + /// Iterate key-value entries in the map starting from upper-bound. + /// + /// - Query is serialized + /// - Result is raw + #[tracing::instrument(skip(self), level = "trace")] + pub fn rev_stream_from_raw

( + self: &Arc, + from: &P, + ) -> impl Stream>> + Send + use<'_, P> + where + P: Serialize + ?Sized + Debug, + { + let key = serialize_key(from).expect("failed to serialize query key"); + self.rev_raw_stream_from(&key) + } - self.db - .pool - .execute_iter(seek) - .ok_into::>() - .into_stream() - .try_flatten() - .boxed() + /// Iterate key-value entries in the map starting from upper-bound. + /// + /// - Query is raw + /// - Result is deserialized + pub fn rev_stream_raw_from<'a, K, V, P>( + self: &'a Arc, + from: &P, + ) -> impl Stream>> + Send + use<'a, K, V, P> + where + P: AsRef<[u8]> + ?Sized + Debug + Sync, + K: Deserialize<'a> + Send, + V: Deserialize<'a> + Send, + { + self.rev_raw_stream_from(from) + .map(result_deserialize::) + } + + /// Iterate key-value entries in the map starting from upper-bound. + /// + /// - Query is raw + /// - Result is raw + #[tracing::instrument(skip(self, from), fields(%self), level = "trace")] + pub fn rev_raw_stream_from

( + self: &Arc, + from: &P, + ) -> impl Stream>> + Send + use<'_, P> + where + P: AsRef<[u8]> + ?Sized + Debug, + { + use crate::pool::Seek; + + let opts = super::iter_options_default(&self.db); + let state = stream::State::new(self, opts); + if is_cached(self, from) { + let state = state.init_rev(from.as_ref().into()); + return task::consume_budget() + .map(move |()| stream::ItemsRev::<'_>::from(state)) + .into_stream() + .flatten() + .boxed(); + } + + let seek = Seek { + map: self.clone(), + dir: Direction::Reverse, + key: Some(from.as_ref().into()), + state: crate::pool::into_send_seek(state), + res: None, + }; + + self.db + .pool + .execute_iter(seek) + .ok_into::>() + .into_stream() + .try_flatten() + .boxed() + } } #[tracing::instrument( diff --git a/src/database/map/rev_stream_prefix.rs b/src/database/map/rev_stream_prefix.rs index 70d4abf73..f042d5d71 100644 --- a/src/database/map/rev_stream_prefix.rs +++ b/src/database/map/rev_stream_prefix.rs @@ -1,77 +1,75 @@ use std::{convert::AsRef, fmt::Debug, sync::Arc}; -use conduwuit::{Result, implement}; +use conduwuit::Result; use futures::{Stream, StreamExt, TryStreamExt, future}; use serde::{Deserialize, Serialize}; use crate::keyval::{KeyVal, result_deserialize, serialize_key}; -/// Iterate key-value entries in the map where the key matches a prefix. -/// -/// - Query is serialized -/// - Result is deserialized -#[implement(super::Map)] -pub fn rev_stream_prefix<'a, K, V, P>( - self: &'a Arc, - prefix: &P, -) -> impl Stream>> + Send + use<'a, K, V, P> -where - P: Serialize + ?Sized + Debug, - K: Deserialize<'a> + Send, - V: Deserialize<'a> + Send, -{ - self.rev_stream_prefix_raw(prefix) - .map(result_deserialize::) -} +impl super::Map { + /// Iterate key-value entries in the map where the key matches a prefix. + /// + /// - Query is serialized + /// - Result is deserialized + pub fn rev_stream_prefix<'a, K, V, P>( + self: &'a Arc, + prefix: &P, + ) -> impl Stream>> + Send + use<'a, K, V, P> + where + P: Serialize + ?Sized + Debug, + K: Deserialize<'a> + Send, + V: Deserialize<'a> + Send, + { + self.rev_stream_prefix_raw(prefix) + .map(result_deserialize::) + } -/// Iterate key-value entries in the map where the key matches a prefix. -/// -/// - Query is serialized -/// - Result is raw -#[implement(super::Map)] -#[tracing::instrument(skip(self), level = "trace")] -pub fn rev_stream_prefix_raw

( - self: &Arc, - prefix: &P, -) -> impl Stream>> + Send + use<'_, P> -where - P: Serialize + ?Sized + Debug, -{ - let key = serialize_key(prefix).expect("failed to serialize query key"); - self.rev_raw_stream_from(&key) - .try_take_while(move |(k, _): &KeyVal<'_>| future::ok(k.starts_with(&key))) -} + /// Iterate key-value entries in the map where the key matches a prefix. + /// + /// - Query is serialized + /// - Result is raw + #[tracing::instrument(skip(self), level = "trace")] + pub fn rev_stream_prefix_raw

( + self: &Arc, + prefix: &P, + ) -> impl Stream>> + Send + use<'_, P> + where + P: Serialize + ?Sized + Debug, + { + let key = serialize_key(prefix).expect("failed to serialize query key"); + self.rev_raw_stream_from(&key) + .try_take_while(move |(k, _): &KeyVal<'_>| future::ok(k.starts_with(&key))) + } -/// Iterate key-value entries in the map where the key matches a prefix. -/// -/// - Query is raw -/// - Result is deserialized -#[implement(super::Map)] -pub fn rev_stream_raw_prefix<'a, K, V, P>( - self: &'a Arc, - prefix: &'a P, -) -> impl Stream>> + Send + 'a -where - P: AsRef<[u8]> + ?Sized + Debug + Sync + 'a, - K: Deserialize<'a> + Send + 'a, - V: Deserialize<'a> + Send + 'a, -{ - self.rev_raw_stream_prefix(prefix) - .map(result_deserialize::) -} + /// Iterate key-value entries in the map where the key matches a prefix. + /// + /// - Query is raw + /// - Result is deserialized + pub fn rev_stream_raw_prefix<'a, K, V, P>( + self: &'a Arc, + prefix: &'a P, + ) -> impl Stream>> + Send + 'a + where + P: AsRef<[u8]> + ?Sized + Debug + Sync + 'a, + K: Deserialize<'a> + Send + 'a, + V: Deserialize<'a> + Send + 'a, + { + self.rev_raw_stream_prefix(prefix) + .map(result_deserialize::) + } -/// Iterate key-value entries in the map where the key matches a prefix. -/// -/// - Query is raw -/// - Result is raw -#[implement(super::Map)] -pub fn rev_raw_stream_prefix<'a, P>( - self: &'a Arc, - prefix: &'a P, -) -> impl Stream>> + Send + 'a -where - P: AsRef<[u8]> + ?Sized + Debug + Sync + 'a, -{ - self.rev_raw_stream_from(prefix) - .try_take_while(|(k, _): &KeyVal<'_>| future::ok(k.starts_with(prefix.as_ref()))) + /// Iterate key-value entries in the map where the key matches a prefix. + /// + /// - Query is raw + /// - Result is raw + pub fn rev_raw_stream_prefix<'a, P>( + self: &'a Arc, + prefix: &'a P, + ) -> impl Stream>> + Send + 'a + where + P: AsRef<[u8]> + ?Sized + Debug + Sync + 'a, + { + self.rev_raw_stream_from(prefix) + .try_take_while(|(k, _): &KeyVal<'_>| future::ok(k.starts_with(prefix.as_ref()))) + } } diff --git a/src/database/map/stream.rs b/src/database/map/stream.rs index 736ab2685..b2da4743e 100644 --- a/src/database/map/stream.rs +++ b/src/database/map/stream.rs @@ -1,6 +1,6 @@ use std::sync::Arc; -use conduwuit::{Result, implement}; +use conduwuit::Result; use futures::{FutureExt, Stream, StreamExt, TryFutureExt, TryStreamExt}; use rocksdb::Direction; use serde::Deserialize; @@ -8,54 +8,54 @@ use tokio::task; use crate::{keyval, keyval::KeyVal, stream}; -/// Iterate key-value entries in the map from the beginning. -/// -/// - Result is deserialized -#[implement(super::Map)] -pub fn stream<'a, K, V>( - self: &'a Arc, -) -> impl Stream>> + Send -where - K: Deserialize<'a> + Send, - V: Deserialize<'a> + Send, -{ - self.raw_stream().map(keyval::result_deserialize::) -} - -/// Iterate key-value entries in the map from the beginning. -/// -/// - Result is raw -#[implement(super::Map)] -#[tracing::instrument(skip(self), fields(%self), level = "trace")] -pub fn raw_stream(self: &Arc) -> impl Stream>> + Send { - use crate::pool::Seek; - - let opts = super::iter_options_default(&self.db); - let state = stream::State::new(self, opts); - if is_cached(self) { - let state = state.init_fwd(None); - return task::consume_budget() - .map(move |()| stream::Items::<'_>::from(state)) - .into_stream() - .flatten() - .boxed(); +impl super::Map { + /// Iterate key-value entries in the map from the beginning. + /// + /// - Result is deserialized + pub fn stream<'a, K, V>( + self: &'a Arc, + ) -> impl Stream>> + Send + where + K: Deserialize<'a> + Send, + V: Deserialize<'a> + Send, + { + self.raw_stream().map(keyval::result_deserialize::) } - let seek = Seek { - map: self.clone(), - dir: Direction::Forward, - state: crate::pool::into_send_seek(state), - key: None, - res: None, - }; + /// Iterate key-value entries in the map from the beginning. + /// + /// - Result is raw + #[tracing::instrument(skip(self), fields(%self), level = "trace")] + pub fn raw_stream(self: &Arc) -> impl Stream>> + Send { + use crate::pool::Seek; - self.db - .pool - .execute_iter(seek) - .ok_into::>() - .into_stream() - .try_flatten() - .boxed() + let opts = super::iter_options_default(&self.db); + let state = stream::State::new(self, opts); + if is_cached(self) { + let state = state.init_fwd(None); + return task::consume_budget() + .map(move |()| stream::Items::<'_>::from(state)) + .into_stream() + .flatten() + .boxed(); + } + + let seek = Seek { + map: self.clone(), + dir: Direction::Forward, + state: crate::pool::into_send_seek(state), + key: None, + res: None, + }; + + self.db + .pool + .execute_iter(seek) + .ok_into::>() + .into_stream() + .try_flatten() + .boxed() + } } #[tracing::instrument( diff --git a/src/database/map/stream_from.rs b/src/database/map/stream_from.rs index 9acec1739..76398ba63 100644 --- a/src/database/map/stream_from.rs +++ b/src/database/map/stream_from.rs @@ -1,6 +1,6 @@ use std::{convert::AsRef, fmt::Debug, sync::Arc}; -use conduwuit::{Result, implement}; +use conduwuit::Result; use futures::{FutureExt, Stream, StreamExt, TryFutureExt, TryStreamExt}; use rocksdb::Direction; use serde::{Deserialize, Serialize}; @@ -11,105 +11,103 @@ use crate::{ stream, }; -/// Iterate key-value entries in the map starting from lower-bound. -/// -/// - Query is serialized -/// - Result is deserialized -#[implement(super::Map)] -pub fn stream_from<'a, K, V, P>( - self: &'a Arc, - from: &P, -) -> impl Stream>> + Send + use<'a, K, V, P> -where - P: Serialize + ?Sized + Debug, - K: Deserialize<'a> + Send, - V: Deserialize<'a> + Send, -{ - self.stream_from_raw(from).map(result_deserialize::) -} - -/// Iterate key-value entries in the map starting from lower-bound. -/// -/// - Query is serialized -/// - Result is raw -#[implement(super::Map)] -#[tracing::instrument(skip(self), level = "trace")] -pub fn stream_from_raw

( - self: &Arc, - from: &P, -) -> impl Stream>> + Send + use<'_, P> -where - P: Serialize + ?Sized + Debug, -{ - let key = serialize_key(from).expect("failed to serialize query key"); - self.raw_stream_from(&key) -} - -/// Iterate key-value entries in the map starting from lower-bound. -/// -/// - Query is raw -/// - Result is deserialized -#[implement(super::Map)] -pub fn stream_raw_from<'a, K, V, P>( - self: &'a Arc, - from: &P, -) -> impl Stream>> + Send + use<'a, K, V, P> -where - P: AsRef<[u8]> + ?Sized + Debug + Sync, - K: Deserialize<'a> + Send, - V: Deserialize<'a> + Send, -{ - self.raw_stream_from(from).map(result_deserialize::) -} - -/// Iterate key-value entries in the map starting from lower-bound. -/// -/// - Query is raw -/// - Result is raw -#[implement(super::Map)] -#[tracing::instrument(skip(self, from), fields(%self), level = "trace")] -pub fn raw_stream_from

( - self: &Arc, - from: &P, -) -> impl Stream>> + Send + use<'_, P> -where - P: AsRef<[u8]> + ?Sized + Debug, -{ - use crate::pool::Seek; - - let opts = super::iter_options_default(&self.db); - let state = stream::State::new(self, opts); - if is_cached(self, from) { - let state = state.init_fwd(from.as_ref().into()); - return task::consume_budget() - .map(move |()| stream::Items::<'_>::from(state)) - .into_stream() - .flatten() - .boxed(); +impl super::Map { + /// Iterate key-value entries in the map starting from lower-bound. + /// + /// - Query is serialized + /// - Result is deserialized + pub fn stream_from<'a, K, V, P>( + self: &'a Arc, + from: &P, + ) -> impl Stream>> + Send + use<'a, K, V, P> + where + P: Serialize + ?Sized + Debug, + K: Deserialize<'a> + Send, + V: Deserialize<'a> + Send, + { + self.stream_from_raw(from).map(result_deserialize::) } - let seek = Seek { - map: self.clone(), - dir: Direction::Forward, - key: Some(from.as_ref().into()), - state: crate::pool::into_send_seek(state), - res: None, - }; + /// Iterate key-value entries in the map starting from lower-bound. + /// + /// - Query is serialized + /// - Result is raw + #[tracing::instrument(skip(self), level = "trace")] + pub fn stream_from_raw

( + self: &Arc, + from: &P, + ) -> impl Stream>> + Send + use<'_, P> + where + P: Serialize + ?Sized + Debug, + { + let key = serialize_key(from).expect("failed to serialize query key"); + self.raw_stream_from(&key) + } - self.db - .pool - .execute_iter(seek) - .ok_into::>() - .into_stream() - .try_flatten() - .boxed() + /// Iterate key-value entries in the map starting from lower-bound. + /// + /// - Query is raw + /// - Result is deserialized + pub fn stream_raw_from<'a, K, V, P>( + self: &'a Arc, + from: &P, + ) -> impl Stream>> + Send + use<'a, K, V, P> + where + P: AsRef<[u8]> + ?Sized + Debug + Sync, + K: Deserialize<'a> + Send, + V: Deserialize<'a> + Send, + { + self.raw_stream_from(from).map(result_deserialize::) + } + + /// Iterate key-value entries in the map starting from lower-bound. + /// + /// - Query is raw + /// - Result is raw + #[tracing::instrument(skip(self, from), fields(%self), level = "trace")] + pub fn raw_stream_from

( + self: &Arc, + from: &P, + ) -> impl Stream>> + Send + use<'_, P> + where + P: AsRef<[u8]> + ?Sized + Debug, + { + use crate::pool::Seek; + + let opts = super::iter_options_default(&self.db); + let state = stream::State::new(self, opts); + if is_cached(self, from) { + let state = state.init_fwd(from.as_ref().into()); + return task::consume_budget() + .map(move |()| stream::Items::<'_>::from(state)) + .into_stream() + .flatten() + .boxed(); + } + + let seek = Seek { + map: self.clone(), + dir: Direction::Forward, + key: Some(from.as_ref().into()), + state: crate::pool::into_send_seek(state), + res: None, + }; + + self.db + .pool + .execute_iter(seek) + .ok_into::>() + .into_stream() + .try_flatten() + .boxed() + } } #[tracing::instrument( - name = "cached", - level = "trace", - skip(map, from), - fields(%map), + name = "cached", + level = "trace", + skip(map, from), + fields(%map), )] pub(super) fn is_cached

(map: &Arc, from: &P) -> bool where diff --git a/src/database/map/stream_prefix.rs b/src/database/map/stream_prefix.rs index 8210e1527..5b9e59d0f 100644 --- a/src/database/map/stream_prefix.rs +++ b/src/database/map/stream_prefix.rs @@ -1,77 +1,75 @@ use std::{convert::AsRef, fmt::Debug, sync::Arc}; -use conduwuit::{Result, implement}; +use conduwuit::Result; use futures::{Stream, StreamExt, TryStreamExt, future}; use serde::{Deserialize, Serialize}; use crate::keyval::{KeyVal, result_deserialize, serialize_key}; -/// Iterate key-value entries in the map where the key matches a prefix. -/// -/// - Query is serialized -/// - Result is deserialized -#[implement(super::Map)] -pub fn stream_prefix<'a, K, V, P>( - self: &'a Arc, - prefix: &P, -) -> impl Stream>> + Send + use<'a, K, V, P> -where - P: Serialize + ?Sized + Debug, - K: Deserialize<'a> + Send, - V: Deserialize<'a> + Send, -{ - self.stream_prefix_raw(prefix) - .map(result_deserialize::) -} +impl super::Map { + /// Iterate key-value entries in the map where the key matches a prefix. + /// + /// - Query is serialized + /// - Result is deserialized + pub fn stream_prefix<'a, K, V, P>( + self: &'a Arc, + prefix: &P, + ) -> impl Stream>> + Send + use<'a, K, V, P> + where + P: Serialize + ?Sized + Debug, + K: Deserialize<'a> + Send, + V: Deserialize<'a> + Send, + { + self.stream_prefix_raw(prefix) + .map(result_deserialize::) + } -/// Iterate key-value entries in the map where the key matches a prefix. -/// -/// - Query is serialized -/// - Result is raw -#[implement(super::Map)] -#[tracing::instrument(skip(self), level = "trace")] -pub fn stream_prefix_raw

( - self: &Arc, - prefix: &P, -) -> impl Stream>> + Send + use<'_, P> -where - P: Serialize + ?Sized + Debug, -{ - let key = serialize_key(prefix).expect("failed to serialize query key"); - self.raw_stream_from(&key) - .try_take_while(move |(k, _): &KeyVal<'_>| future::ok(k.starts_with(&key))) -} + /// Iterate key-value entries in the map where the key matches a prefix. + /// + /// - Query is serialized + /// - Result is raw + #[tracing::instrument(skip(self), level = "trace")] + pub fn stream_prefix_raw

( + self: &Arc, + prefix: &P, + ) -> impl Stream>> + Send + use<'_, P> + where + P: Serialize + ?Sized + Debug, + { + let key = serialize_key(prefix).expect("failed to serialize query key"); + self.raw_stream_from(&key) + .try_take_while(move |(k, _): &KeyVal<'_>| future::ok(k.starts_with(&key))) + } -/// Iterate key-value entries in the map where the key matches a prefix. -/// -/// - Query is raw -/// - Result is deserialized -#[implement(super::Map)] -pub fn stream_raw_prefix<'a, K, V, P>( - self: &'a Arc, - prefix: &'a P, -) -> impl Stream>> + Send + 'a -where - P: AsRef<[u8]> + ?Sized + Debug + Sync + 'a, - K: Deserialize<'a> + Send + 'a, - V: Deserialize<'a> + Send + 'a, -{ - self.raw_stream_prefix(prefix) - .map(result_deserialize::) -} + /// Iterate key-value entries in the map where the key matches a prefix. + /// + /// - Query is raw + /// - Result is deserialized + pub fn stream_raw_prefix<'a, K, V, P>( + self: &'a Arc, + prefix: &'a P, + ) -> impl Stream>> + Send + 'a + where + P: AsRef<[u8]> + ?Sized + Debug + Sync + 'a, + K: Deserialize<'a> + Send + 'a, + V: Deserialize<'a> + Send + 'a, + { + self.raw_stream_prefix(prefix) + .map(result_deserialize::) + } -/// Iterate key-value entries in the map where the key matches a prefix. -/// -/// - Query is raw -/// - Result is raw -#[implement(super::Map)] -pub fn raw_stream_prefix<'a, P>( - self: &'a Arc, - prefix: &'a P, -) -> impl Stream>> + Send + 'a -where - P: AsRef<[u8]> + ?Sized + Debug + Sync + 'a, -{ - self.raw_stream_from(prefix) - .try_take_while(|(k, _): &KeyVal<'_>| future::ok(k.starts_with(prefix.as_ref()))) + /// Iterate key-value entries in the map where the key matches a prefix. + /// + /// - Query is raw + /// - Result is raw + pub fn raw_stream_prefix<'a, P>( + self: &'a Arc, + prefix: &'a P, + ) -> impl Stream>> + Send + 'a + where + P: AsRef<[u8]> + ?Sized + Debug + Sync + 'a, + { + self.raw_stream_from(prefix) + .try_take_while(|(k, _): &KeyVal<'_>| future::ok(k.starts_with(prefix.as_ref()))) + } } From f1d3db0ea3378cdbf8a05f43c543ffdc5a0ff544 Mon Sep 17 00:00:00 2001 From: timedout Date: Fri, 26 Jun 2026 06:31:46 +0100 Subject: [PATCH 60/70] style: Eliminate `implement` --- src/admin/check/commands.rs | 30 +++--- src/core/matrix/pdu/redact.rs | 40 ++++---- src/core/matrix/pdu/unsigned.rs | 156 ++++++++++++++++---------------- 3 files changed, 113 insertions(+), 113 deletions(-) diff --git a/src/admin/check/commands.rs b/src/admin/check/commands.rs index e482d1a9b..96dd9e4b3 100644 --- a/src/admin/check/commands.rs +++ b/src/admin/check/commands.rs @@ -1,23 +1,23 @@ use conduwuit::Result; -use conduwuit_macros::implement; use futures::StreamExt; use crate::Context; -#[implement(Context, params = "<'_>")] -pub(super) async fn check_all_users(&self) -> Result { - let timer = tokio::time::Instant::now(); - let users = self.services.users.stream().collect::>().await; - let query_time = timer.elapsed(); +impl Context { + pub(super) async fn check_all_users(&self) -> Result { + let timer = tokio::time::Instant::now(); + let users = self.services.users.stream().collect::>().await; + let query_time = timer.elapsed(); - let total = users.len(); - let err_count = users.iter().filter(|_user| false).count(); - let ok_count = users.iter().filter(|_user| true).count(); + let total = users.len(); + let err_count = users.iter().filter(|_user| false).count(); + let ok_count = users.iter().filter(|_user| true).count(); - self.write_str(&format!( - "Database query completed in {query_time:?}:\n\n```\nTotal entries: \ - {total:?}\nFailure/Invalid user count: {err_count:?}\nSuccess/Valid user count: \ - {ok_count:?}\n```" - )) - .await + self.write_str(&format!( + "Database query completed in {query_time:?}:\n\n```\nTotal entries: \ + {total:?}\nFailure/Invalid user count: {err_count:?}\nSuccess/Valid user count: \ + {ok_count:?}\n```" + )) + .await + } } diff --git a/src/core/matrix/pdu/redact.rs b/src/core/matrix/pdu/redact.rs index 280c62622..e60dccdd5 100644 --- a/src/core/matrix/pdu/redact.rs +++ b/src/core/matrix/pdu/redact.rs @@ -1,32 +1,34 @@ use ruma::{RoomVersionId, canonical_json::redact_content_in_place}; use serde_json::{Value as JsonValue, json, value::to_raw_value}; -use crate::{Err, Result, err, implement}; +use crate::{Err, Result, err}; -#[implement(super::Pdu)] -pub fn redact(&mut self, room_version_id: &RoomVersionId, reason: JsonValue) -> Result { - let Some(rules) = room_version_id.rules() else { - return Err!("Cannot redact event for unknown room version {room_version_id}"); - }; +impl super::Pdu { + pub fn redact(&mut self, room_version_id: &RoomVersionId, reason: JsonValue) -> Result { + let Some(rules) = room_version_id.rules() else { + return Err!("Cannot redact event for unknown room version {room_version_id}"); + }; - self.unsigned = None; + self.unsigned = None; - let mut content = serde_json::from_str(self.content.get()) - .map_err(|e| err!(Request(BadJson("Failed to deserialize content into type: {e}"))))?; + let mut content = serde_json::from_str(self.content.get()).map_err(|e| { + err!(Request(BadJson("Failed to deserialize content into type: {e}"))) + })?; - redact_content_in_place(&mut content, &rules.redaction, self.kind.to_string()); + redact_content_in_place(&mut content, &rules.redaction, self.kind.to_string()); - let reason = serde_json::to_value(reason).expect("Failed to preserialize reason"); + let reason = serde_json::to_value(reason).expect("Failed to preserialize reason"); - let redacted_because = json!({ - "redacted_because": reason, - }); + let redacted_because = json!({ + "redacted_because": reason, + }); - self.unsigned = to_raw_value(&redacted_because) - .expect("Failed to serialize unsigned") - .into(); + self.unsigned = to_raw_value(&redacted_because) + .expect("Failed to serialize unsigned") + .into(); - self.content = to_raw_value(&content).expect("Failed to serialize content"); + self.content = to_raw_value(&content).expect("Failed to serialize content"); - Ok(()) + Ok(()) + } } diff --git a/src/core/matrix/pdu/unsigned.rs b/src/core/matrix/pdu/unsigned.rs index d3da10531..2db1d55af 100644 --- a/src/core/matrix/pdu/unsigned.rs +++ b/src/core/matrix/pdu/unsigned.rs @@ -4,85 +4,83 @@ use ruma::MilliSecondsSinceUnixEpoch; use serde_json::value::{RawValue as RawJsonValue, Value as JsonValue, to_raw_value}; use super::Pdu; -use crate::{Result, err, implement, result::LogErr}; +use crate::{Result, err, result::LogErr}; -/// Set the `unsigned` field of the PDU using only information in the PDU. -/// Some unsigned data is already set within the database (eg. prev events, -/// threads). Once this is done, other data must be calculated from the database -/// (eg. relations) This is for server-to-client events. -/// Backfill handles this itself. -#[implement(Pdu)] -pub fn set_unsigned(&mut self, user_id: Option<&ruma::UserId>) { - if Some(self.sender.borrow()) != user_id { - self.remove_transaction_id().log_err().ok(); +impl Pdu { + /// Set the `unsigned` field of the PDU using only information in the PDU. + /// Some unsigned data is already set within the database (eg. prev events, + /// threads). Once this is done, other data must be calculated from the + /// database (eg. relations) This is for server-to-client events. + /// Backfill handles this itself. + pub fn set_unsigned(&mut self, user_id: Option<&ruma::UserId>) { + if Some(self.sender.borrow()) != user_id { + self.remove_transaction_id().log_err().ok(); + } + self.add_age().log_err().ok(); + } + + pub fn remove_transaction_id(&mut self) -> Result { + use BTreeMap as Map; + + let Some(unsigned) = &self.unsigned else { + return Ok(()); + }; + + let mut unsigned: Map<&str, Box> = serde_json::from_str(unsigned.get()) + .map_err(|e| err!(Database("Invalid unsigned in pdu event: {e}")))?; + + unsigned.remove("transaction_id"); + self.unsigned = to_raw_value(&unsigned) + .map(Some) + .expect("unsigned is valid"); + + Ok(()) + } + + pub fn add_age(&mut self) -> Result { + use BTreeMap as Map; + + let mut unsigned: Map<&str, Box> = self + .unsigned + .as_deref() + .map(RawJsonValue::get) + .map_or_else(|| Ok(Map::new()), serde_json::from_str) + .map_err(|e| err!(Database("Invalid unsigned in pdu event: {e}")))?; + + // deliberately allowing for the possibility of negative age + let now: i128 = MilliSecondsSinceUnixEpoch::now().get().into(); + let then: i128 = self.origin_server_ts.into(); + let this_age = now.saturating_sub(then); + + unsigned.insert("age", to_raw_value(&this_age)?); + self.unsigned = Some(to_raw_value(&unsigned)?); + + Ok(()) + } + + pub fn add_relation(&mut self, name: &str, pdu: Option<&Pdu>) -> Result { + use serde_json::Map; + + let mut unsigned: Map = self + .unsigned + .as_deref() + .map(RawJsonValue::get) + .map_or_else(|| Ok(Map::new()), serde_json::from_str) + .map_err(|e| err!(Database("Invalid unsigned in pdu event: {e}")))?; + + let pdu = pdu + .map(serde_json::to_value) + .transpose()? + .unwrap_or_else(|| JsonValue::Object(Map::new())); + + unsigned + .entry("m.relations") + .or_insert(JsonValue::Object(Map::new())) + .as_object_mut() + .map(|object| object.insert(name.to_owned(), pdu)); + + self.unsigned = Some(to_raw_value(&unsigned)?); + + Ok(()) } - self.add_age().log_err().ok(); -} - -#[implement(Pdu)] -pub fn remove_transaction_id(&mut self) -> Result { - use BTreeMap as Map; - - let Some(unsigned) = &self.unsigned else { - return Ok(()); - }; - - let mut unsigned: Map<&str, Box> = serde_json::from_str(unsigned.get()) - .map_err(|e| err!(Database("Invalid unsigned in pdu event: {e}")))?; - - unsigned.remove("transaction_id"); - self.unsigned = to_raw_value(&unsigned) - .map(Some) - .expect("unsigned is valid"); - - Ok(()) -} - -#[implement(Pdu)] -pub fn add_age(&mut self) -> Result { - use BTreeMap as Map; - - let mut unsigned: Map<&str, Box> = self - .unsigned - .as_deref() - .map(RawJsonValue::get) - .map_or_else(|| Ok(Map::new()), serde_json::from_str) - .map_err(|e| err!(Database("Invalid unsigned in pdu event: {e}")))?; - - // deliberately allowing for the possibility of negative age - let now: i128 = MilliSecondsSinceUnixEpoch::now().get().into(); - let then: i128 = self.origin_server_ts.into(); - let this_age = now.saturating_sub(then); - - unsigned.insert("age", to_raw_value(&this_age)?); - self.unsigned = Some(to_raw_value(&unsigned)?); - - Ok(()) -} - -#[implement(Pdu)] -pub fn add_relation(&mut self, name: &str, pdu: Option<&Pdu>) -> Result { - use serde_json::Map; - - let mut unsigned: Map = self - .unsigned - .as_deref() - .map(RawJsonValue::get) - .map_or_else(|| Ok(Map::new()), serde_json::from_str) - .map_err(|e| err!(Database("Invalid unsigned in pdu event: {e}")))?; - - let pdu = pdu - .map(serde_json::to_value) - .transpose()? - .unwrap_or_else(|| JsonValue::Object(Map::new())); - - unsigned - .entry("m.relations") - .or_insert(JsonValue::Object(Map::new())) - .as_object_mut() - .map(|object| object.insert(name.to_owned(), pdu)); - - self.unsigned = Some(to_raw_value(&unsigned)?); - - Ok(()) } From 3eb4257ae872e3a706289f1afbabaf59c6e3bf9f Mon Sep 17 00:00:00 2001 From: timedout Date: Fri, 26 Jun 2026 06:32:26 +0100 Subject: [PATCH 61/70] style: Remove remaining `implement` imports --- src/core/mod.rs | 2 +- src/core/utils/mod.rs | 2 -- src/database/engine/open.rs | 2 +- src/database/map/compact.rs | 2 +- src/database/map/keys_from.rs | 2 +- src/database/map/qry_batch.rs | 2 +- src/database/map/rev_keys_prefix.rs | 2 +- 7 files changed, 6 insertions(+), 8 deletions(-) diff --git a/src/core/mod.rs b/src/core/mod.rs index bd597c369..b025b8b17 100644 --- a/src/core/mod.rs +++ b/src/core/mod.rs @@ -41,7 +41,7 @@ pub use info::{ pub use matrix::{Event, EventTypeExt, Pdu, PduCount, PduEvent, PduId, pdu, state_res}; pub use parking_lot::{Mutex as SyncMutex, RwLock as SyncRwLock}; pub use server::Server; -pub use utils::{implement, result, result::Result}; +pub use utils::{result, result::Result}; pub use crate as conduwuit_core; diff --git a/src/core/utils/mod.rs b/src/core/utils/mod.rs index 1d62e7508..12baee8f8 100644 --- a/src/core/utils/mod.rs +++ b/src/core/utils/mod.rs @@ -22,8 +22,6 @@ mod tests; pub mod time; pub mod with_lock; -pub use ::conduwuit_macros::implement; - pub use self::{ arrayvec::ArrayVecExt, bool::BoolExt, diff --git a/src/database/engine/open.rs b/src/database/engine/open.rs index 094ff8f1b..73d9d6c8a 100644 --- a/src/database/engine/open.rs +++ b/src/database/engine/open.rs @@ -4,7 +4,7 @@ use std::{ sync::{Arc, atomic::AtomicU32}, }; -use conduwuit::{Result, debug, implement, info, warn}; +use conduwuit::{Result, debug, info, warn}; use rocksdb::{ColumnFamilyDescriptor, Options}; use super::{ diff --git a/src/database/map/compact.rs b/src/database/map/compact.rs index ae6890ccd..9a5c0b718 100644 --- a/src/database/map/compact.rs +++ b/src/database/map/compact.rs @@ -1,4 +1,4 @@ -use conduwuit::{Err, Result, implement}; +use conduwuit::{Err, Result}; use rocksdb::{BottommostLevelCompaction, CompactOptions}; use crate::keyval::KeyBuf; diff --git a/src/database/map/keys_from.rs b/src/database/map/keys_from.rs index 80bafd8db..47b5d7e6b 100644 --- a/src/database/map/keys_from.rs +++ b/src/database/map/keys_from.rs @@ -1,6 +1,6 @@ use std::{convert::AsRef, fmt::Debug, sync::Arc}; -use conduwuit::{Result, implement}; +use conduwuit::Result; use futures::{FutureExt, Stream, StreamExt, TryFutureExt, TryStreamExt}; use rocksdb::Direction; use serde::{Deserialize, Serialize}; diff --git a/src/database/map/qry_batch.rs b/src/database/map/qry_batch.rs index a1f38c4e1..115798b56 100644 --- a/src/database/map/qry_batch.rs +++ b/src/database/map/qry_batch.rs @@ -1,7 +1,7 @@ use std::{fmt::Debug, sync::Arc}; use conduwuit::{ - Result, implement, + Result, utils::{ IterStream, stream::{WidebandExt, automatic_amplification, automatic_width}, diff --git a/src/database/map/rev_keys_prefix.rs b/src/database/map/rev_keys_prefix.rs index d55493e81..2cbc11129 100644 --- a/src/database/map/rev_keys_prefix.rs +++ b/src/database/map/rev_keys_prefix.rs @@ -1,6 +1,6 @@ use std::{convert::AsRef, fmt::Debug, sync::Arc}; -use conduwuit::{Result, implement}; +use conduwuit::Result; use futures::{Stream, StreamExt, TryStreamExt, future}; use serde::{Deserialize, Serialize}; From 0310b4b2c54453cbbe23273de8b4a83c6210eb53 Mon Sep 17 00:00:00 2001 From: timedout Date: Fri, 26 Jun 2026 06:33:53 +0100 Subject: [PATCH 62/70] style: Remove `implement` entirely --- src/macros/implement.rs | 35 ----------------------------------- src/macros/mod.rs | 6 ------ 2 files changed, 41 deletions(-) delete mode 100644 src/macros/implement.rs diff --git a/src/macros/implement.rs b/src/macros/implement.rs deleted file mode 100644 index 7acc12d29..000000000 --- a/src/macros/implement.rs +++ /dev/null @@ -1,35 +0,0 @@ -use proc_macro::{Span, TokenStream}; -use quote::quote; -use syn::{Error, ItemFn, Meta, Path}; -use utils::get_named_generics; - -use crate::{Result, utils}; - -pub(super) fn implement(item: ItemFn, args: &[Meta]) -> Result { - let generics = get_named_generics(args, "generics")?; - let receiver = get_receiver(args)?; - let params = get_named_generics(args, "params")?; - let input = item; - let out = quote! { - impl #generics #receiver #params { - #input - } - }; - - Ok(out.into()) -} - -fn get_receiver(args: &[Meta]) -> Result { - let receiver = &args.first().ok_or_else(|| { - Error::new(Span::call_site().into(), "Missing required argument to receiver") - })?; - - let Meta::Path(receiver) = receiver else { - return Err(Error::new( - Span::call_site().into(), - "First argument is not path to receiver", - )); - }; - - Ok(receiver.clone()) -} diff --git a/src/macros/mod.rs b/src/macros/mod.rs index 906188275..843cd4e82 100644 --- a/src/macros/mod.rs +++ b/src/macros/mod.rs @@ -2,7 +2,6 @@ mod admin; mod build_info; mod config; mod debug; -mod implement; mod refutable; mod utils; @@ -35,11 +34,6 @@ pub fn refutable(args: TokenStream, input: TokenStream) -> TokenStream { attribute_macro::(args, input, refutable::refutable) } -#[proc_macro_attribute] -pub fn implement(args: TokenStream, input: TokenStream) -> TokenStream { - attribute_macro::(args, input, implement::implement) -} - #[proc_macro_attribute] pub fn config_example_generator(args: TokenStream, input: TokenStream) -> TokenStream { attribute_macro::(args, input, config::example_generator) From 478560daaeb5ab095f2ca3b0c31f9ddf58c1dc9e Mon Sep 17 00:00:00 2001 From: timedout Date: Fri, 26 Jun 2026 06:36:04 +0100 Subject: [PATCH 63/70] style: Remove now dead code --- src/core/matrix/pdu/unsigned.rs | 2 +- src/macros/utils.rs | 19 ------------------- 2 files changed, 1 insertion(+), 20 deletions(-) diff --git a/src/core/matrix/pdu/unsigned.rs b/src/core/matrix/pdu/unsigned.rs index 2db1d55af..be293a408 100644 --- a/src/core/matrix/pdu/unsigned.rs +++ b/src/core/matrix/pdu/unsigned.rs @@ -58,7 +58,7 @@ impl Pdu { Ok(()) } - pub fn add_relation(&mut self, name: &str, pdu: Option<&Pdu>) -> Result { + pub fn add_relation(&mut self, name: &str, pdu: Option<&Self>) -> Result { use serde_json::Map; let mut unsigned: Map = self diff --git a/src/macros/utils.rs b/src/macros/utils.rs index a45e5ecc5..598f60ffa 100644 --- a/src/macros/utils.rs +++ b/src/macros/utils.rs @@ -44,25 +44,6 @@ pub(crate) fn legacy_is_cargo_build() -> bool { pub(crate) fn is_cargo_test() -> bool { std::env::args().any(|flag| flag == "--test") } -pub(crate) fn get_named_generics(args: &[Meta], name: &str) -> Result { - const DEFAULT: &str = "<>"; - - parse_str::(&get_named_string(args, name).unwrap_or_else(|| DEFAULT.to_owned())) -} - -pub(crate) fn get_named_string(args: &[Meta], name: &str) -> Option { - args.iter().find_map(|arg| { - let value = arg.require_name_value().ok()?; - let Expr::Lit(ref lit) = value.value else { - return None; - }; - let Lit::Str(ref str) = lit.lit else { - return None; - }; - value.path.is_ident(name).then_some(str.value()) - }) -} - #[must_use] pub(crate) fn camel_to_snake_string(s: &str) -> String { let mut output = String::with_capacity( From 6fdeadc356028c7366608b7d5272f5b7fc7a5066 Mon Sep 17 00:00:00 2001 From: timedout Date: Fri, 26 Jun 2026 07:20:20 +0100 Subject: [PATCH 64/70] style: Refactor admin crate, drop `admin_command` macro `admin_command` depended on `implement`, which is now gone. --- src/admin/appservice/commands.rs | 122 +- src/admin/check/commands.rs | 2 +- src/admin/debug/commands.rs | 2001 +++++++++++++------------- src/admin/debug/tester.rs | 51 +- src/admin/federation/commands.rs | 228 +-- src/admin/media/commands.rs | 670 ++++----- src/admin/mod.rs | 2 +- src/admin/query/account_data.rs | 86 +- src/admin/query/raw.rs | 484 ++++--- src/admin/query/resolver.rs | 95 +- src/admin/query/room_timeline.rs | 60 +- src/admin/query/short.rs | 32 +- src/admin/query/users.rs | 514 ++++--- src/admin/room/commands.rs | 144 +- src/admin/room/info.rs | 108 +- src/admin/room/moderation.rs | 703 ++++----- src/admin/server/commands.rs | 407 +++--- src/admin/token/commands.rs | 132 +- src/admin/user/commands.rs | 2313 +++++++++++++++--------------- src/macros/admin.rs | 11 +- src/macros/mod.rs | 5 - src/macros/utils.rs | 4 +- 22 files changed, 4068 insertions(+), 4106 deletions(-) diff --git a/src/admin/appservice/commands.rs b/src/admin/appservice/commands.rs index 2c1bb7c10..6afd9f468 100644 --- a/src/admin/appservice/commands.rs +++ b/src/admin/appservice/commands.rs @@ -1,80 +1,76 @@ use conduwuit::{Err, Result, checked}; use futures::{FutureExt, StreamExt, TryFutureExt}; -use crate::admin_command; +impl crate::Context<'_> { + pub(super) async fn register(&self) -> Result { + let body = &self.body; + let body_len = self.body.len(); + if body_len < 2 + || !body[0].trim().starts_with("```") + || body.last().unwrap_or(&"").trim() != "```" + { + return Err!("Expected code block in command body. Add --help for details."); + } -#[admin_command] -pub(super) async fn register(&self) -> Result { - let body = &self.body; - let body_len = self.body.len(); - if body_len < 2 - || !body[0].trim().starts_with("```") - || body.last().unwrap_or(&"").trim() != "```" - { - return Err!("Expected code block in command body. Add --help for details."); + let range = 1..checked!(body_len - 1)?; + let appservice_config_body = body[range].join("\n"); + let parsed_config = serde_saphyr::from_str(&appservice_config_body); + match parsed_config { + | Err(e) => return Err!("Could not parse appservice config as YAML: {e}"), + | Ok(registration) => match self + .services + .appservice + .register_appservice(®istration, &appservice_config_body) + .await + .map(|()| registration.id) + { + | Err(e) => return Err!("Failed to register appservice: {e}"), + | Ok(id) => write!(self, "Appservice registered with ID: {id}"), + }, + } + .await } - let range = 1..checked!(body_len - 1)?; - let appservice_config_body = body[range].join("\n"); - let parsed_config = serde_saphyr::from_str(&appservice_config_body); - match parsed_config { - | Err(e) => return Err!("Could not parse appservice config as YAML: {e}"), - | Ok(registration) => match self + pub(super) async fn unregister(&self, appservice_identifier: String) -> Result { + match self .services .appservice - .register_appservice(®istration, &appservice_config_body) + .unregister_appservice(&appservice_identifier) .await - .map(|()| registration.id) { - | Err(e) => return Err!("Failed to register appservice: {e}"), - | Ok(id) => write!(self, "Appservice registered with ID: {id}"), - }, - } - .await -} - -#[admin_command] -pub(super) async fn unregister(&self, appservice_identifier: String) -> Result { - match self - .services - .appservice - .unregister_appservice(&appservice_identifier) + | Err(e) => return Err!("Failed to unregister appservice: {e}"), + | Ok(()) => write!(self, "Appservice unregistered."), + } .await - { - | Err(e) => return Err!("Failed to unregister appservice: {e}"), - | Ok(()) => write!(self, "Appservice unregistered."), } - .await -} -#[admin_command] -pub(super) async fn show_appservice_config(&self, appservice_identifier: String) -> Result { - match self - .services - .appservice - .get_registration(&appservice_identifier) + pub(super) async fn show_appservice_config(&self, appservice_identifier: String) -> Result { + match self + .services + .appservice + .get_registration(&appservice_identifier) + .await + { + | None => return Err!("Appservice does not exist."), + | Some(config) => { + let config_str = serde_saphyr::to_string(&config)?; + write!(self, "Config for {appservice_identifier}:\n\n```yaml\n{config_str}\n```") + }, + } .await - { - | None => return Err!("Appservice does not exist."), - | Some(config) => { - let config_str = serde_saphyr::to_string(&config)?; - write!(self, "Config for {appservice_identifier}:\n\n```yaml\n{config_str}\n```") - }, } - .await -} -#[admin_command] -pub(super) async fn list_registered(&self) -> Result { - self.services - .appservice - .iter_ids() - .collect() - .map(Ok) - .and_then(|appservices: Vec<_>| { - let len = appservices.len(); - let list = appservices.join(", "); - write!(self, "Appservices ({len}): {list}") - }) - .await + pub(super) async fn list_registered(&self) -> Result { + self.services + .appservice + .iter_ids() + .collect() + .map(Ok) + .and_then(|appservices: Vec<_>| { + let len = appservices.len(); + let list = appservices.join(", "); + write!(self, "Appservices ({len}): {list}") + }) + .await + } } diff --git a/src/admin/check/commands.rs b/src/admin/check/commands.rs index 96dd9e4b3..14ef4f78f 100644 --- a/src/admin/check/commands.rs +++ b/src/admin/check/commands.rs @@ -3,7 +3,7 @@ use futures::StreamExt; use crate::Context; -impl Context { +impl Context<'_> { pub(super) async fn check_all_users(&self) -> Result { let timer = tokio::time::Instant::now(); let users = self.services.users.stream().collect::>().await; diff --git a/src/admin/debug/commands.rs b/src/admin/debug/commands.rs index 1aa9cb062..1fdc9509a 100644 --- a/src/admin/debug/commands.rs +++ b/src/admin/debug/commands.rs @@ -31,44 +31,6 @@ use service::rooms::{ }; use tracing_subscriber::EnvFilter; -use crate::admin_command; - -#[admin_command] -pub(super) async fn echo(&self, message: Vec) -> Result { - let message = message.join(" "); - self.write_str(&message).await -} - -#[admin_command] -pub(super) async fn get_auth_chain(&self, event_id: OwnedEventId) -> Result { - let Ok(event) = self.services.rooms.timeline.get_pdu_json(&event_id).await else { - return Err!("Event not found."); - }; - - let room_id_str = event - .get("room_id") - .and_then(CanonicalJsonValue::as_str) - .ok_or_else(|| err!(Database("Invalid event in database")))?; - - let room_id = <&RoomId>::try_from(room_id_str) - .map_err(|_| err!(Database("Invalid room id field in event in database")))?; - - let start = Instant::now(); - let count = self - .services - .rooms - .auth_chain - .event_ids_iter(room_id, once(event_id.as_ref())) - .ready_filter_map(Result::ok) - .count() - .await; - - let elapsed = start.elapsed(); - let out = format!("Loaded auth chain with length {count} in {elapsed:?}"); - - self.write_str(&out).await -} - #[derive(Clone, Copy, Eq, PartialEq)] enum NodeStatus { Normal(bool), @@ -115,1064 +77,1093 @@ fn render_node( Ok(()) } -#[admin_command] -pub(super) async fn show_auth_chain(&self, event_id: OwnedEventId) -> Result { - let node_status = async |event_id: &EventId, missing: bool| -> NodeStatus { - if self +impl crate::Context<'_> { + pub(super) async fn echo(&self, message: Vec) -> Result { + let message = message.join(" "); + self.write_str(&message).await + } + + pub(super) async fn get_auth_chain(&self, event_id: OwnedEventId) -> Result { + let Ok(event) = self.services.rooms.timeline.get_pdu_json(&event_id).await else { + return Err!("Event not found."); + }; + + let room_id_str = event + .get("room_id") + .and_then(CanonicalJsonValue::as_str) + .ok_or_else(|| err!(Database("Invalid event in database")))?; + + let room_id = <&RoomId>::try_from(room_id_str) + .map_err(|_| err!(Database("Invalid room id field in event in database")))?; + + let start = Instant::now(); + let count = self .services .rooms - .pdu_metadata - .is_event_rejected(event_id) - .await - { - NodeStatus::Rejected(missing) - } else if self - .services - .rooms - .pdu_metadata - .is_event_soft_failed(event_id) - .await - { - NodeStatus::SoftFailed(missing) - } else { - NodeStatus::Normal(missing) - } - }; + .auth_chain + .event_ids_iter(room_id, once(event_id.as_ref())) + .ready_filter_map(Result::ok) + .count() + .await; - let Ok(root) = self.services.rooms.timeline.get_pdu(&event_id).await else { - return Err!("Event not found."); - }; + let elapsed = start.elapsed(); + let out = format!("Loaded auth chain with length {count} in {elapsed:?}"); - let mut graph = String::from( - "```mermaid\n%% This is a mermaid graph. You can plug this output into\n\ + self.write_str(&out).await + } + + pub(super) async fn show_auth_chain(&self, event_id: OwnedEventId) -> Result { + let node_status = async |event_id: &EventId, missing: bool| -> NodeStatus { + if self + .services + .rooms + .pdu_metadata + .is_event_rejected(event_id) + .await + { + NodeStatus::Rejected(missing) + } else if self + .services + .rooms + .pdu_metadata + .is_event_soft_failed(event_id) + .await + { + NodeStatus::SoftFailed(missing) + } else { + NodeStatus::Normal(missing) + } + }; + + let Ok(root) = self.services.rooms.timeline.get_pdu(&event_id).await else { + return Err!("Event not found."); + }; + + let mut graph = String::from( + "```mermaid\n%% This is a mermaid graph. You can plug this output into\n\ %% https://mermaid.live/edit to visualise it on-the-fly.\nflowchart TD\n\ classDef rejected fill:#ffe5e5,stroke:#cc0000,stroke-width:2px,color:#000;\n\ classDef soft_failed fill:#fff6cc,stroke:#c9a400,stroke-width:2px,color:#000;\n" - ); + ); - let mut node_ids: HashMap = HashMap::new(); - let mut cached_events: HashMap = - HashMap::from([(event_id.clone(), root.clone())]); - let mut scheduled: HashSet = HashSet::from([event_id.clone()]); - let mut visited: HashSet = HashSet::new(); - let mut stack = vec![root]; - let mut next_node_id = 0_usize; + let mut node_ids: HashMap = HashMap::new(); + let mut cached_events: HashMap = + HashMap::from([(event_id.clone(), root.clone())]); + let mut scheduled: HashSet = HashSet::from([event_id.clone()]); + let mut visited: HashSet = HashSet::new(); + let mut stack = vec![root]; + let mut next_node_id = 0_usize; - let node_id_for = |event_id: &OwnedEventId, - node_ids: &mut HashMap, - next_node_id: &mut usize| { - node_ids - .entry(event_id.clone()) - .or_insert_with(|| { - let id = format!("n{}", *next_node_id); - *next_node_id = next_node_id.saturating_add(1); - id - }) - .clone() - }; - let node_name = |e: &PduEvent| { - if let Some(state_key) = e.state_key() { - format!("{},'{}'", e.event_type(), state_key) - } else { - format!("{}", e.event_type()) - } - }; - - while let Some(event) = stack.pop() { - let current_event_id = event.event_id().to_owned(); - if !visited.insert(current_event_id.clone()) { - continue; - } - - let current_node_id = node_id_for(¤t_event_id, &mut node_ids, &mut next_node_id); - let current_status = node_status(¤t_event_id, false).await; - - render_node( - &mut graph, - ¤t_node_id, - ¤t_event_id, - &node_name(&event), - current_status, - )?; - - let mut children = Vec::with_capacity(event.auth_events.len()); - for auth_event_id in event.auth_events().rev() { - let auth_event_id = auth_event_id.to_owned(); - let auth_node_id = node_id_for(&auth_event_id, &mut node_ids, &mut next_node_id); - writeln!(graph, "{current_node_id} --> {auth_node_id}")?; - - let first_seen = scheduled.insert(auth_event_id.clone()); - let auth_pdu = if let Some(auth_pdu) = cached_events.get(&auth_event_id) { - // NOTE: events might be referenced multiple times (like the create event) - // so this saves some cheeky db lookup time - Some(auth_pdu.clone()) - } else if first_seen { - match self.services.rooms.timeline.get_pdu(&auth_event_id).await { - | Ok(auth_event) => { - cached_events.insert(auth_event_id.clone(), auth_event.clone()); - Some(auth_event) - }, - | Err(_) => None, - } + let node_id_for = |event_id: &OwnedEventId, + node_ids: &mut HashMap, + next_node_id: &mut usize| { + node_ids + .entry(event_id.clone()) + .or_insert_with(|| { + let id = format!("n{}", *next_node_id); + *next_node_id = next_node_id.saturating_add(1); + id + }) + .clone() + }; + let node_name = |e: &PduEvent| { + if let Some(state_key) = e.state_key() { + format!("{},'{}'", e.event_type(), state_key) } else { - None - }; + format!("{}", e.event_type()) + } + }; - // NOTE: Depth is used as the primary sorting key here, even though it has no - // bearing on state resolution or anything. Timestamp is used as a - // tiebreaker, failing back to lexicographical comparison. - let (depth, ts) = auth_pdu - .as_ref() - .map_or((UInt::MAX, UInt::MAX), |pdu| (pdu.depth, pdu.origin_server_ts)); - - children.push(AuthChild { - node_id: auth_node_id, - event_id: auth_event_id, - depth, - ts, - first_seen, - pdu: auth_pdu, - }); - } - - children.sort_by(|a, b| { - a.depth - .cmp(&b.depth) - .then(a.ts.cmp(&b.ts)) - .then(a.event_id.as_str().cmp(b.event_id.as_str())) - }); - - for child in children.into_iter().rev() { - if !child.first_seen { + while let Some(event) = stack.pop() { + let current_event_id = event.event_id().to_owned(); + if !visited.insert(current_event_id.clone()) { continue; } - if let Some(child_pdu) = child.pdu { - // We have this PDU so will want to traverse it. - stack.push(child_pdu); - } else { - // We don't have this PDU locally so we can't traverse its auth events, - // but we can still render it as a node. - render_node( - &mut graph, - &child.node_id, - &child.event_id, - "", - node_status(&child.event_id, true).await, - )?; - } - } - } + let current_node_id = + node_id_for(¤t_event_id, &mut node_ids, &mut next_node_id); + let current_status = node_status(¤t_event_id, false).await; - graph.push_str("```\n"); - self.write_str(&graph).await -} + render_node( + &mut graph, + ¤t_node_id, + ¤t_event_id, + &node_name(&event), + current_status, + )?; -#[admin_command] -pub(super) async fn parse_pdu(&self) -> Result { - if self.body.len() < 2 - || !self.body[0].trim().starts_with("```") - || self.body.last().unwrap_or(&EMPTY).trim() != "```" - { - return Err!("Expected code block in command body. Add --help for details."); - } + let mut children = Vec::with_capacity(event.auth_events.len()); + for auth_event_id in event.auth_events().rev() { + let auth_event_id = auth_event_id.to_owned(); + let auth_node_id = node_id_for(&auth_event_id, &mut node_ids, &mut next_node_id); + writeln!(graph, "{current_node_id} --> {auth_node_id}")?; - let string = self.body[1..self.body.len().saturating_sub(1)].join("\n"); - let room_version_rules = RoomVersionId::V12.rules().unwrap(); - - match serde_json::from_str(&string) { - | Err(e) => return Err!("Invalid json in command body: {e}"), - | Ok(value) => match ruma::signatures::reference_hash(&value, &room_version_rules) { - | Err(e) => return Err!("Could not parse PDU JSON: {e:?}"), - | Ok(hash) => { - let event_id = EventId::parse(format!("${hash}")); - match serde_json::from_value::(serde_json::to_value(value)?) { - | Err(e) => return Err!("EventId: {event_id:?}\nCould not parse event: {e}"), - | Ok(pdu) => write!(self, "EventId: {event_id:?}\n{pdu:#?}"), - } - }, - }, - } - .await -} - -#[admin_command] -pub(super) async fn get_pdu(&self, event_id: OwnedEventId) -> Result { - let mut outlier = false; - let mut pdu_json = self - .services - .rooms - .timeline - .get_non_outlier_pdu_json(&event_id) - .await; - - if pdu_json.is_err() { - outlier = true; - pdu_json = self.services.rooms.timeline.get_pdu_json(&event_id).await; - } - let rejected = self - .services - .rooms - .pdu_metadata - .is_event_rejected(&event_id) - .await; - let soft_failed = self - .services - .rooms - .pdu_metadata - .is_event_soft_failed(&event_id) - .await; - - match pdu_json { - | Err(_) => return Err!("PDU not found locally."), - | Ok(json) => { - let text = serde_json::to_string_pretty(&json)?; - let msg = if rejected { - "Rejected PDU:" - } else if soft_failed { - "Soft-failed PDU:" - } else if outlier { - "Outlier PDU:" - } else { - "PDU:" - }; - write!(self, "{msg}\n```json\n{text}\n```") - }, - } - .await -} - -#[admin_command] -pub(super) async fn get_short_pdu( - &self, - shortroomid: ShortRoomId, - shorteventid: ShortEventId, -) -> Result { - let pdu_id: RawPduId = PduId { - shortroomid, - shorteventid: shorteventid.into(), - } - .into(); - - let pdu_json = self - .services - .rooms - .timeline - .get_pdu_json_from_id(&pdu_id) - .await; - - match pdu_json { - | Err(_) => return Err!("PDU not found locally."), - | Ok(json) => { - let json_text = serde_json::to_string_pretty(&json)?; - write!(self, "```json\n{json_text}\n```") - }, - } - .await -} - -#[admin_command] -pub(super) async fn get_remote_pdu_list(&self, server: OwnedServerName, force: bool) -> Result { - if !self.services.server.config.allow_federation { - return Err!("Federation is disabled on this homeserver.",); - } - - if server == self.services.globals.server_name() { - return Err!( - "Not allowed to send federation requests to ourselves. Please use `get-pdu` for \ - fetching local PDUs from the database.", - ); - } - - if self.body.len() < 2 - || !self.body[0].trim().starts_with("```") - || self.body.last().unwrap_or(&EMPTY).trim() != "```" - { - return Err!("Expected code block in command body. Add --help for details.",); - } - - let list = self - .body - .iter() - .collect::>() - .drain(1..self.body.len().saturating_sub(1)) - .filter_map(|pdu| EventId::parse(pdu).ok()) - .collect::>(); - - let mut failed_count: usize = 0; - let mut success_count: usize = 0; - - for event_id in list { - if force { - match self.get_remote_pdu(event_id.clone(), server.clone()).await { - | Err(e) => { - failed_count = failed_count.saturating_add(1); - self.services - .admin - .send_text(&format!("Failed to get remote PDU, ignoring error: {e}")) - .await; - - warn!("Failed to get remote PDU, ignoring error: {e}"); - }, - | _ => { - success_count = success_count.saturating_add(1); - }, - } - } else { - self.get_remote_pdu(event_id.clone(), server.clone()) - .await?; - success_count = success_count.saturating_add(1); - } - } - - let out = - format!("Fetched {success_count} remote PDUs successfully with {failed_count} failures"); - - self.write_str(&out).await -} - -#[admin_command] -pub(super) async fn get_remote_pdu( - &self, - event_id: OwnedEventId, - server: OwnedServerName, -) -> Result { - if !self.services.server.config.allow_federation { - return Err!("Federation is disabled on this homeserver."); - } - - if server == self.services.globals.server_name() { - return Err!( - "Not allowed to send federation requests to ourselves. Please use `get-pdu` for \ - fetching local PDUs.", - ); - } - - match self - .services - .sending - .send_federation_request( - &server, - ruma::api::federation::event::get_event::v1::Request::new(event_id.clone()), - ) - .await - { - | Err(e) => { - return Err!( - "Remote server did not have PDU or failed sending request to remote server: {e}" - ); - }, - | Ok(response) => { - let json: CanonicalJsonObject = - serde_json::from_str(response.pdu.get()).map_err(|e| { - warn!( - "Requested event ID {event_id} from server but failed to convert from \ - RawValue to CanonicalJsonObject (malformed event/response?): {e}" - ); - err!(Request(Unknown( - "Received response from server but failed to parse PDU" - ))) - })?; - - trace!("Attempting to parse PDU: {:?}", &response.pdu); - let _parsed_pdu = { - let parsed_result = self - .services - .rooms - .event_handler - .parse_incoming_pdu(&response.pdu) - .boxed() - .await; - - let (event_id, value, room_id) = match parsed_result { - | Ok(t) => t, - | Err(e) => { - warn!("Failed to parse PDU: {e}"); - info!("Full PDU: {:?}", &response.pdu); - return Err!("Failed to parse PDU remote server {server} sent us: {e}"); - }, + let first_seen = scheduled.insert(auth_event_id.clone()); + let auth_pdu = if let Some(auth_pdu) = cached_events.get(&auth_event_id) { + // NOTE: events might be referenced multiple times (like the create event) + // so this saves some cheeky db lookup time + Some(auth_pdu.clone()) + } else if first_seen { + match self.services.rooms.timeline.get_pdu(&auth_event_id).await { + | Ok(auth_event) => { + cached_events.insert(auth_event_id.clone(), auth_event.clone()); + Some(auth_event) + }, + | Err(_) => None, + } + } else { + None }; - vec![(event_id, value, room_id)] - }; + // NOTE: Depth is used as the primary sorting key here, even though it has no + // bearing on state resolution or anything. Timestamp is used as a + // tiebreaker, failing back to lexicographical comparison. + let (depth, ts) = auth_pdu + .as_ref() + .map_or((UInt::MAX, UInt::MAX), |pdu| (pdu.depth, pdu.origin_server_ts)); - let text = serde_json::to_string_pretty(&json)?; - let msg = "Got PDU from specified server:"; - write!(self, "{msg}. Event body:\n```json\n{text}\n```") - }, - } - .await -} + children.push(AuthChild { + node_id: auth_node_id, + event_id: auth_event_id, + depth, + ts, + first_seen, + pdu: auth_pdu, + }); + } -#[admin_command] -pub(super) async fn get_state_at(&self, event_id: OwnedEventId) -> Result { - self.bail_restricted()?; + children.sort_by(|a, b| { + a.depth + .cmp(&b.depth) + .then(a.ts.cmp(&b.ts)) + .then(a.event_id.as_str().cmp(b.event_id.as_str())) + }); - let shortstatehash = self - .services - .rooms - .state_accessor - .pdu_shortstatehash(&event_id) - .await?; + for child in children.into_iter().rev() { + if !child.first_seen { + continue; + } - let state_ids: Vec = self - .services - .rooms - .state_accessor - .state_full_ids(shortstatehash) - .map(at!(1)) - .collect() - .await; + if let Some(child_pdu) = child.pdu { + // We have this PDU so will want to traverse it. + stack.push(child_pdu); + } else { + // We don't have this PDU locally so we can't traverse its auth events, + // but we can still render it as a node. + render_node( + &mut graph, + &child.node_id, + &child.event_id, + "", + node_status(&child.event_id, true).await, + )?; + } + } + } - let pdus: Vec = state_ids - .iter() - .try_stream() - .and_then(|id| self.services.rooms.timeline.get_pdu_json(id)) - .try_collect() - .await?; - - let json = serde_json::to_string_pretty(&pdus).map_err(|e| { - err!(Database( - "Failed to convert room state events to pretty JSON, possible invalid room state \ - events in our database {e}", - )) - })?; - - let out = format!("```json\n{json}\n```"); - self.write_str(&out).await -} - -#[admin_command] -pub(super) async fn get_room_state(&self, room: OwnedRoomOrAliasId) -> Result { - self.bail_restricted()?; - - let room_id = self.services.rooms.alias.resolve(&room).await?; - let room_state: Vec> = self - .services - .rooms - .state_accessor - .room_state_full_pdus(&room_id) - .map_ok(Event::into_format) - .try_collect() - .await?; - - if room_state.is_empty() { - return Err!("Unable to find room state in our database (vector is empty)",); + graph.push_str("```\n"); + self.write_str(&graph).await } - let json = serde_json::to_string_pretty(&room_state).map_err(|e| { - err!(Database( - "Failed to convert room state events to pretty JSON, possible invalid room state \ - events in our database {e}", - )) - })?; + pub(super) async fn parse_pdu(&self) -> Result { + if self.body.len() < 2 + || !self.body[0].trim().starts_with("```") + || self.body.last().unwrap_or(&EMPTY).trim() != "```" + { + return Err!("Expected code block in command body. Add --help for details."); + } - let out = format!("```json\n{json}\n```"); - self.write_str(&out).await -} + let string = self.body[1..self.body.len().saturating_sub(1)].join("\n"); + let room_version_rules = RoomVersionId::V12.rules().unwrap(); -#[admin_command] -pub(super) async fn ping(&self, server: OwnedServerName) -> Result { - if server == self.services.globals.server_name() { - return Err!("Not allowed to send federation requests to ourselves."); - } - - let timer = tokio::time::Instant::now(); - - match self - .services - .sending - .send_unauthenticated_request( - &server, - ruma::api::federation::discovery::get_server_version::v1::Request::new(), - ) + match serde_json::from_str(&string) { + | Err(e) => return Err!("Invalid json in command body: {e}"), + | Ok(value) => match ruma::signatures::reference_hash(&value, &room_version_rules) { + | Err(e) => return Err!("Could not parse PDU JSON: {e:?}"), + | Ok(hash) => { + let event_id = EventId::parse(format!("${hash}")); + match serde_json::from_value::(serde_json::to_value(value)?) { + | Err(e) => + return Err!("EventId: {event_id:?}\nCould not parse event: {e}"), + | Ok(pdu) => write!(self, "EventId: {event_id:?}\n{pdu:#?}"), + } + }, + }, + } .await - { - | Err(e) => { - return Err!("Failed sending federation request to specified server:\n\n{e}"); - }, - | Ok(response) => { - let ping_time = timer.elapsed(); - let json_text_res = serde_json::to_string_pretty(&response.server); + } - let out = if let Ok(json) = json_text_res { - format!("Got response which took {ping_time:?} time:\n```json\n{json}\n```") + pub(super) async fn get_pdu(&self, event_id: OwnedEventId) -> Result { + let mut outlier = false; + let mut pdu_json = self + .services + .rooms + .timeline + .get_non_outlier_pdu_json(&event_id) + .await; + + if pdu_json.is_err() { + outlier = true; + pdu_json = self.services.rooms.timeline.get_pdu_json(&event_id).await; + } + let rejected = self + .services + .rooms + .pdu_metadata + .is_event_rejected(&event_id) + .await; + let soft_failed = self + .services + .rooms + .pdu_metadata + .is_event_soft_failed(&event_id) + .await; + + match pdu_json { + | Err(_) => return Err!("PDU not found locally."), + | Ok(json) => { + let text = serde_json::to_string_pretty(&json)?; + let msg = if rejected { + "Rejected PDU:" + } else if soft_failed { + "Soft-failed PDU:" + } else if outlier { + "Outlier PDU:" + } else { + "PDU:" + }; + write!(self, "{msg}\n```json\n{text}\n```") + }, + } + .await + } + + pub(super) async fn get_short_pdu( + &self, + shortroomid: ShortRoomId, + shorteventid: ShortEventId, + ) -> Result { + let pdu_id: RawPduId = PduId { + shortroomid, + shorteventid: shorteventid.into(), + } + .into(); + + let pdu_json = self + .services + .rooms + .timeline + .get_pdu_json_from_id(&pdu_id) + .await; + + match pdu_json { + | Err(_) => return Err!("PDU not found locally."), + | Ok(json) => { + let json_text = serde_json::to_string_pretty(&json)?; + write!(self, "```json\n{json_text}\n```") + }, + } + .await + } + + pub(super) async fn get_remote_pdu_list( + &self, + server: OwnedServerName, + force: bool, + ) -> Result { + if !self.services.server.config.allow_federation { + return Err!("Federation is disabled on this homeserver.",); + } + + if server == self.services.globals.server_name() { + return Err!( + "Not allowed to send federation requests to ourselves. Please use `get-pdu` for \ + fetching local PDUs from the database.", + ); + } + + if self.body.len() < 2 + || !self.body[0].trim().starts_with("```") + || self.body.last().unwrap_or(&EMPTY).trim() != "```" + { + return Err!("Expected code block in command body. Add --help for details.",); + } + + let list = self + .body + .iter() + .collect::>() + .drain(1..self.body.len().saturating_sub(1)) + .filter_map(|pdu| EventId::parse(pdu).ok()) + .collect::>(); + + let mut failed_count: usize = 0; + let mut success_count: usize = 0; + + for event_id in list { + if force { + match self.get_remote_pdu(event_id.clone(), server.clone()).await { + | Err(e) => { + failed_count = failed_count.saturating_add(1); + self.services + .admin + .send_text(&format!("Failed to get remote PDU, ignoring error: {e}")) + .await; + + warn!("Failed to get remote PDU, ignoring error: {e}"); + }, + | _ => { + success_count = success_count.saturating_add(1); + }, + } } else { - format!("Got non-JSON response which took {ping_time:?} time:\n{response:?}") - }; + self.get_remote_pdu(event_id.clone(), server.clone()) + .await?; + success_count = success_count.saturating_add(1); + } + } - write!(self, "{out}") - }, + let out = format!( + "Fetched {success_count} remote PDUs successfully with {failed_count} failures" + ); + + self.write_str(&out).await } - .await -} -#[admin_command] -pub(super) async fn force_device_list_updates(&self) -> Result { - // Force E2EE device list updates for all users - self.services - .users - .stream() - .for_each(async |user_id| self.services.users.mark_device_key_update(&user_id).await) - .await; + pub(super) async fn get_remote_pdu( + &self, + event_id: OwnedEventId, + server: OwnedServerName, + ) -> Result { + if !self.services.server.config.allow_federation { + return Err!("Federation is disabled on this homeserver."); + } - write!(self, "Marked all devices for all users as having new keys to update").await -} - -#[admin_command] -pub(super) async fn change_log_level(&self, filter: Option, reset: bool) -> Result { - let handles = &["console"]; - - if reset { - let old_filter_layer = match EnvFilter::try_new(&self.services.server.config.log) { - | Ok(s) => s, - | Err(e) => return Err!("Log level from config appears to be invalid now: {e}"), - }; + if server == self.services.globals.server_name() { + return Err!( + "Not allowed to send federation requests to ourselves. Please use `get-pdu` for \ + fetching local PDUs.", + ); + } match self .services - .server - .log - .reload - .reload(&old_filter_layer, Some(handles)) - { - | Err(e) => { - return Err!("Failed to modify and reload the global tracing log level: {e}"); - }, - | Ok(()) => { - let value = &self.services.server.config.log; - let out = format!("Successfully changed log level back to config value {value}"); - return self.write_str(&out).await; - }, - } - } - - if let Some(filter) = filter { - let new_filter_layer = match EnvFilter::try_new(filter) { - | Ok(s) => s, - | Err(e) => return Err!("Invalid log level filter specified: {e}"), - }; - - match self - .services - .server - .log - .reload - .reload(&new_filter_layer, Some(handles)) - { - | Ok(()) => { - return self.write_str("Successfully changed log level").await; - }, - | Err(e) => { - return Err!("Failed to modify and reload the global tracing log level: {e}"); - }, - } - } - - Err!("No log level was specified.") -} - -#[admin_command] -pub(super) async fn verify_json(&self) -> Result { - if self.body.len() < 2 - || !self.body[0].trim().starts_with("```") - || self.body.last().unwrap_or(&"").trim() != "```" - { - return Err!("Expected code block in command body. Add --help for details."); - } - - let string = self.body[1..self.body.len().checked_sub(1).unwrap()].join("\n"); - let room_version_rules = RoomVersionId::V12.rules().unwrap(); - - match serde_json::from_str::(&string) { - | Err(e) => return Err!("Invalid json: {e}"), - | Ok(value) => match self - .services - .server_keys - .verify_json(&value, &room_version_rules) + .sending + .send_federation_request( + &server, + ruma::api::federation::event::get_event::v1::Request::new(event_id.clone()), + ) .await { - | Err(e) => return Err!("Signature verification failed: {e}"), - | Ok(()) => write!(self, "Signature correct"), - }, - } - .await -} + | Err(e) => { + return Err!( + "Remote server did not have PDU or failed sending request to remote server: \ + {e}" + ); + }, + | Ok(response) => { + let json: CanonicalJsonObject = serde_json::from_str(response.pdu.get()) + .map_err(|e| { + warn!( + "Requested event ID {event_id} from server but failed to convert \ + from RawValue to CanonicalJsonObject (malformed event/response?): \ + {e}" + ); + err!(Request(Unknown( + "Received response from server but failed to parse PDU" + ))) + })?; -#[admin_command] -pub(super) async fn verify_pdu(&self, event_id: OwnedEventId) -> Result { - use ruma::signatures::Verified; + trace!("Attempting to parse PDU: {:?}", &response.pdu); + let _parsed_pdu = { + let parsed_result = self + .services + .rooms + .event_handler + .parse_incoming_pdu(&response.pdu) + .boxed() + .await; - let mut event = self.services.rooms.timeline.get_pdu_json(&event_id).await?; - let room_version_rules = RoomVersionId::V12.rules().unwrap(); + let (event_id, value, room_id) = match parsed_result { + | Ok(t) => t, + | Err(e) => { + warn!("Failed to parse PDU: {e}"); + info!("Full PDU: {:?}", &response.pdu); + return Err!( + "Failed to parse PDU remote server {server} sent us: {e}" + ); + }, + }; - event.remove("event_id"); - let msg = match self - .services - .server_keys - .verify_event(&event, &room_version_rules) + vec![(event_id, value, room_id)] + }; + + let text = serde_json::to_string_pretty(&json)?; + let msg = "Got PDU from specified server:"; + write!(self, "{msg}. Event body:\n```json\n{text}\n```") + }, + } .await - { - | Err(e) => return Err(e), - | Ok(Verified::Signatures) => "signatures OK, but content hash failed (redaction).", - | Ok(Verified::All) => "signatures and hashes OK.", - }; - - self.write_str(msg).await -} - -#[admin_command] -#[tracing::instrument(skip(self), level = "info")] -pub(super) async fn first_pdu_in_room(&self, room_id: OwnedRoomId) -> Result { - self.bail_restricted()?; - - if !self - .services - .rooms - .state_cache - .server_in_room(&self.services.server.name, &room_id) - .await - { - return Err!("We are not participating in the room / we don't know about the room ID.",); } - let first_pdu = self - .services - .rooms - .timeline - .first_pdu_in_room(&room_id) - .await - .map_err(|_| err!(Database("Failed to find the first PDU in database")))?; + pub(super) async fn get_state_at(&self, event_id: OwnedEventId) -> Result { + self.bail_restricted()?; - let out = format!("{first_pdu:?}"); - self.write_str(&out).await -} + let shortstatehash = self + .services + .rooms + .state_accessor + .pdu_shortstatehash(&event_id) + .await?; -#[admin_command] -#[tracing::instrument(skip(self), level = "info")] -pub(super) async fn latest_pdu_in_room(&self, room_id: OwnedRoomId) -> Result { - self.bail_restricted()?; + let state_ids: Vec = self + .services + .rooms + .state_accessor + .state_full_ids(shortstatehash) + .map(at!(1)) + .collect() + .await; - if !self - .services - .rooms - .state_cache - .server_in_room(&self.services.server.name, &room_id) - .await - { - return Err!("We are not participating in the room / we don't know about the room ID."); + let pdus: Vec = state_ids + .iter() + .try_stream() + .and_then(|id| self.services.rooms.timeline.get_pdu_json(id)) + .try_collect() + .await?; + + let json = serde_json::to_string_pretty(&pdus).map_err(|e| { + err!(Database( + "Failed to convert room state events to pretty JSON, possible invalid room state \ + events in our database {e}", + )) + })?; + + let out = format!("```json\n{json}\n```"); + self.write_str(&out).await } - let latest_pdu = self - .services - .rooms - .timeline - .latest_pdu_in_room(&room_id) - .await - .map_err(|_| err!(Database("Failed to find the latest PDU in database")))?; + pub(super) async fn get_room_state(&self, room: OwnedRoomOrAliasId) -> Result { + self.bail_restricted()?; - let out = format!("{latest_pdu:?}"); - self.write_str(&out).await -} + let room_id = self.services.rooms.alias.resolve(&room).await?; + let room_state: Vec> = self + .services + .rooms + .state_accessor + .room_state_full_pdus(&room_id) + .map_ok(Event::into_format) + .try_collect() + .await?; -#[admin_command] -#[tracing::instrument(skip(self), level = "info")] -pub(super) async fn force_set_room_state_from_server( - &self, - room_id: OwnedRoomId, - server_name: OwnedServerName, - at_event: Option, -) -> Result { - self.bail_restricted()?; + if room_state.is_empty() { + return Err!("Unable to find room state in our database (vector is empty)",); + } - if !self - .services - .rooms - .state_cache - .server_in_room(&self.services.server.name, &room_id) - .await - { - return Err!("We are not participating in the room / we don't know about the room ID."); + let json = serde_json::to_string_pretty(&room_state).map_err(|e| { + err!(Database( + "Failed to convert room state events to pretty JSON, possible invalid room \ + state events in our database {e}", + )) + })?; + + let out = format!("```json\n{json}\n```"); + self.write_str(&out).await } - let at_event_id = match at_event { - | Some(event_id) => event_id, - | None => self + pub(super) async fn ping(&self, server: OwnedServerName) -> Result { + if server == self.services.globals.server_name() { + return Err!("Not allowed to send federation requests to ourselves."); + } + + let timer = tokio::time::Instant::now(); + + match self + .services + .sending + .send_unauthenticated_request( + &server, + ruma::api::federation::discovery::get_server_version::v1::Request::new(), + ) + .await + { + | Err(e) => { + return Err!("Failed sending federation request to specified server:\n\n{e}"); + }, + | Ok(response) => { + let ping_time = timer.elapsed(); + let json_text_res = serde_json::to_string_pretty(&response.server); + + let out = if let Ok(json) = json_text_res { + format!("Got response which took {ping_time:?} time:\n```json\n{json}\n```") + } else { + format!("Got non-JSON response which took {ping_time:?} time:\n{response:?}") + }; + + write!(self, "{out}") + }, + } + .await + } + + pub(super) async fn force_device_list_updates(&self) -> Result { + // Force E2EE device list updates for all users + self.services + .users + .stream() + .for_each(async |user_id| self.services.users.mark_device_key_update(&user_id).await) + .await; + + write!(self, "Marked all devices for all users as having new keys to update").await + } + + pub(super) async fn change_log_level(&self, filter: Option, reset: bool) -> Result { + let handles = &["console"]; + + if reset { + let old_filter_layer = match EnvFilter::try_new(&self.services.server.config.log) { + | Ok(s) => s, + | Err(e) => return Err!("Log level from config appears to be invalid now: {e}"), + }; + + match self + .services + .server + .log + .reload + .reload(&old_filter_layer, Some(handles)) + { + | Err(e) => { + return Err!("Failed to modify and reload the global tracing log level: {e}"); + }, + | Ok(()) => { + let value = &self.services.server.config.log; + let out = + format!("Successfully changed log level back to config value {value}"); + return self.write_str(&out).await; + }, + } + } + + if let Some(filter) = filter { + let new_filter_layer = match EnvFilter::try_new(filter) { + | Ok(s) => s, + | Err(e) => return Err!("Invalid log level filter specified: {e}"), + }; + + match self + .services + .server + .log + .reload + .reload(&new_filter_layer, Some(handles)) + { + | Ok(()) => { + return self.write_str("Successfully changed log level").await; + }, + | Err(e) => { + return Err!("Failed to modify and reload the global tracing log level: {e}"); + }, + } + } + + Err!("No log level was specified.") + } + + pub(super) async fn verify_json(&self) -> Result { + if self.body.len() < 2 + || !self.body[0].trim().starts_with("```") + || self.body.last().unwrap_or(&"").trim() != "```" + { + return Err!("Expected code block in command body. Add --help for details."); + } + + let string = self.body[1..self.body.len().checked_sub(1).unwrap()].join("\n"); + let room_version_rules = RoomVersionId::V12.rules().unwrap(); + + match serde_json::from_str::(&string) { + | Err(e) => return Err!("Invalid json: {e}"), + | Ok(value) => match self + .services + .server_keys + .verify_json(&value, &room_version_rules) + .await + { + | Err(e) => return Err!("Signature verification failed: {e}"), + | Ok(()) => write!(self, "Signature correct"), + }, + } + .await + } + + pub(super) async fn verify_pdu(&self, event_id: OwnedEventId) -> Result { + use ruma::signatures::Verified; + + let mut event = self.services.rooms.timeline.get_pdu_json(&event_id).await?; + let room_version_rules = RoomVersionId::V12.rules().unwrap(); + + event.remove("event_id"); + let msg = match self + .services + .server_keys + .verify_event(&event, &room_version_rules) + .await + { + | Err(e) => return Err(e), + | Ok(Verified::Signatures) => "signatures OK, but content hash failed (redaction).", + | Ok(Verified::All) => "signatures and hashes OK.", + }; + + self.write_str(msg).await + } + + #[tracing::instrument(skip(self), level = "info")] + pub(super) async fn first_pdu_in_room(&self, room_id: OwnedRoomId) -> Result { + self.bail_restricted()?; + + if !self + .services + .rooms + .state_cache + .server_in_room(&self.services.server.name, &room_id) + .await + { + return Err!( + "We are not participating in the room / we don't know about the room ID.", + ); + } + + let first_pdu = self + .services + .rooms + .timeline + .first_pdu_in_room(&room_id) + .await + .map_err(|_| err!(Database("Failed to find the first PDU in database")))?; + + let out = format!("{first_pdu:?}"); + self.write_str(&out).await + } + + #[tracing::instrument(skip(self), level = "info")] + pub(super) async fn latest_pdu_in_room(&self, room_id: OwnedRoomId) -> Result { + self.bail_restricted()?; + + if !self + .services + .rooms + .state_cache + .server_in_room(&self.services.server.name, &room_id) + .await + { + return Err!( + "We are not participating in the room / we don't know about the room ID." + ); + } + + let latest_pdu = self .services .rooms .timeline .latest_pdu_in_room(&room_id) .await - .map_err(|_| err!(Database("Failed to find the latest PDU in database")))? - .event_id() - .to_owned(), - }; + .map_err(|_| err!(Database("Failed to find the latest PDU in database")))?; - let room_version = self.services.rooms.state.get_room_version(&room_id).await?; - let room_version_rules = room_version.rules().unwrap(); - - let mut state: HashMap = HashMap::new(); - - let remote_state_response = self - .services - .sending - .send_federation_request( - &server_name, - get_room_state::v1::Request::new(at_event_id, room_id.clone()), - ) - .await?; - - for pdu in remote_state_response.pdus.clone() { - match self - .services - .rooms - .event_handler - .parse_incoming_pdu(&pdu) - .await - { - | Ok(t) => t, - | Err(e) => { - warn!("Could not parse PDU, ignoring: {e}"); - continue; - }, - }; + let out = format!("{latest_pdu:?}"); + self.write_str(&out).await } - info!("Going through room_state response PDUs"); - for result in remote_state_response.pdus.iter().map(|pdu| { - self.services - .server_keys - .validate_and_add_event_id(pdu, &room_version_rules) - }) { - let Ok((event_id, value)) = result.await else { - continue; - }; + #[tracing::instrument(skip(self), level = "info")] + pub(super) async fn force_set_room_state_from_server( + &self, + room_id: OwnedRoomId, + server_name: OwnedServerName, + at_event: Option, + ) -> Result { + self.bail_restricted()?; - let pdu = PduEvent::from_id_val(&event_id, value.clone()).map_err(|e| { - debug_error!("Invalid PDU in fetching remote room state PDUs response: {value:#?}"); - err!(BadServerResponse(debug_error!("Invalid PDU in send_join response: {e:?}"))) - })?; - - self.services + if !self + .services .rooms - .outlier - .add_pdu_outlier(&event_id, &value); + .state_cache + .server_in_room(&self.services.server.name, &room_id) + .await + { + return Err!( + "We are not participating in the room / we don't know about the room ID." + ); + } - if let Some(state_key) = &pdu.state_key { - let shortstatekey = self + let at_event_id = match at_event { + | Some(event_id) => event_id, + | None => self .services .rooms - .short - .get_or_create_shortstatekey(&pdu.kind.to_string().into(), state_key) - .await; + .timeline + .latest_pdu_in_room(&room_id) + .await + .map_err(|_| err!(Database("Failed to find the latest PDU in database")))? + .event_id() + .to_owned(), + }; - state.insert(shortstatekey, pdu.event_id.clone()); + let room_version = self.services.rooms.state.get_room_version(&room_id).await?; + let room_version_rules = room_version.rules().unwrap(); + + let mut state: HashMap = HashMap::new(); + + let remote_state_response = self + .services + .sending + .send_federation_request( + &server_name, + get_room_state::v1::Request::new(at_event_id, room_id.clone()), + ) + .await?; + + for pdu in remote_state_response.pdus.clone() { + match self + .services + .rooms + .event_handler + .parse_incoming_pdu(&pdu) + .await + { + | Ok(t) => t, + | Err(e) => { + warn!("Could not parse PDU, ignoring: {e}"); + continue; + }, + }; + } + + info!("Going through room_state response PDUs"); + for result in remote_state_response.pdus.iter().map(|pdu| { + self.services + .server_keys + .validate_and_add_event_id(pdu, &room_version_rules) + }) { + let Ok((event_id, value)) = result.await else { + continue; + }; + + let pdu = PduEvent::from_id_val(&event_id, value.clone()).map_err(|e| { + debug_error!( + "Invalid PDU in fetching remote room state PDUs response: {value:#?}" + ); + err!(BadServerResponse(debug_error!("Invalid PDU in send_join response: {e:?}"))) + })?; + + self.services + .rooms + .outlier + .add_pdu_outlier(&event_id, &value); + + if let Some(state_key) = &pdu.state_key { + let shortstatekey = self + .services + .rooms + .short + .get_or_create_shortstatekey(&pdu.kind.to_string().into(), state_key) + .await; + + state.insert(shortstatekey, pdu.event_id.clone()); + self.services + .rooms + .pdu_metadata + .clear_pdu_markers(pdu.event_id()); + } + } + + info!("Going through auth_chain response"); + for result in remote_state_response.auth_chain.iter().map(|pdu| { + self.services + .server_keys + .validate_and_add_event_id(pdu, &room_version_rules) + }) { + let Ok((event_id, value)) = result.await else { + continue; + }; + + self.services + .rooms + .outlier + .add_pdu_outlier(&event_id, &value); self.services .rooms .pdu_metadata - .clear_pdu_markers(pdu.event_id()); + .clear_pdu_markers(&event_id); } - } - info!("Going through auth_chain response"); - for result in remote_state_response.auth_chain.iter().map(|pdu| { - self.services - .server_keys - .validate_and_add_event_id(pdu, &room_version_rules) - }) { - let Ok((event_id, value)) = result.await else { - continue; - }; - - self.services - .rooms - .outlier - .add_pdu_outlier(&event_id, &value); - self.services - .rooms - .pdu_metadata - .clear_pdu_markers(&event_id); - } - - info!("Resolving new room state"); - let new_room_state = self - .services - .rooms - .event_handler - .resolve_state(&room_id, &room_version_rules, state) - .await?; - - info!("Compressing new room state"); - let HashSetCompressStateEvent { - shortstatehash: short_state_hash, - added, - removed, - } = self - .services - .rooms - .state_compressor - .save_state(room_id.clone().as_ref(), new_room_state) - .await?; - - let state_lock = self.services.rooms.state.mutex.lock(&*room_id).await; - - info!("Forcing new room state"); - self.services - .rooms - .state - .force_state(room_id.clone().as_ref(), short_state_hash, added, removed, &state_lock) - .await?; - - info!("Updating joined counts for room"); - self.services - .rooms - .state_cache - .update_joined_count(&room_id) - .await; - - self.write_str("Successfully forced the room state from the requested remote server.") - .await -} - -#[admin_command] -pub(super) async fn get_signing_keys( - &self, - server_name: Option, - notary: Option, - query: bool, -) -> Result { - let server_name = server_name.unwrap_or_else(|| self.services.server.name.clone()); - - if let Some(notary) = notary { - let signing_keys = self + info!("Resolving new room state"); + let new_room_state = self .services - .server_keys - .notary_request(¬ary, &server_name) + .rooms + .event_handler + .resolve_state(&room_id, &room_version_rules, state) .await?; + info!("Compressing new room state"); + let HashSetCompressStateEvent { + shortstatehash: short_state_hash, + added, + removed, + } = self + .services + .rooms + .state_compressor + .save_state(room_id.clone().as_ref(), new_room_state) + .await?; + + let state_lock = self.services.rooms.state.mutex.lock(&*room_id).await; + + info!("Forcing new room state"); + self.services + .rooms + .state + .force_state(room_id.clone().as_ref(), short_state_hash, added, removed, &state_lock) + .await?; + + info!("Updating joined counts for room"); + self.services + .rooms + .state_cache + .update_joined_count(&room_id) + .await; + + self.write_str("Successfully forced the room state from the requested remote server.") + .await + } + + pub(super) async fn get_signing_keys( + &self, + server_name: Option, + notary: Option, + query: bool, + ) -> Result { + let server_name = server_name.unwrap_or_else(|| self.services.server.name.clone()); + + if let Some(notary) = notary { + let signing_keys = self + .services + .server_keys + .notary_request(¬ary, &server_name) + .await?; + + let out = format!("```rs\n{signing_keys:#?}\n```"); + return self.write_str(&out).await; + } + + let signing_keys = if query { + self.services + .server_keys + .server_request(&server_name) + .await? + } else { + self.services + .server_keys + .signing_keys_for(&server_name) + .await? + }; + let out = format!("```rs\n{signing_keys:#?}\n```"); - return self.write_str(&out).await; + self.write_str(&out).await } - let signing_keys = if query { - self.services + pub(super) async fn get_verify_keys(&self, server_name: Option) -> Result { + let server_name = server_name.unwrap_or_else(|| self.services.server.name.clone()); + + let keys = self + .services .server_keys - .server_request(&server_name) - .await? - } else { - self.services - .server_keys - .signing_keys_for(&server_name) - .await? - }; + .verify_keys_for(&server_name) + .await; - let out = format!("```rs\n{signing_keys:#?}\n```"); - self.write_str(&out).await -} + let mut out = String::new(); + writeln!(out, "| Key ID | Public Key |")?; + writeln!(out, "| --- | --- |")?; + for (key_id, key) in keys { + writeln!(out, "| {key_id} | {key:?} |")?; + } -#[admin_command] -pub(super) async fn get_verify_keys(&self, server_name: Option) -> Result { - let server_name = server_name.unwrap_or_else(|| self.services.server.name.clone()); - - let keys = self - .services - .server_keys - .verify_keys_for(&server_name) - .await; - - let mut out = String::new(); - writeln!(out, "| Key ID | Public Key |")?; - writeln!(out, "| --- | --- |")?; - for (key_id, key) in keys { - writeln!(out, "| {key_id} | {key:?} |")?; + self.write_str(&out).await } - self.write_str(&out).await -} + pub(super) async fn resolve_true_destination( + &self, + server_name: OwnedServerName, + no_cache: bool, + ) -> Result { + if !self.services.server.config.allow_federation { + return Err!("Federation is disabled on this homeserver.",); + } -#[admin_command] -pub(super) async fn resolve_true_destination( - &self, - server_name: OwnedServerName, - no_cache: bool, -) -> Result { - if !self.services.server.config.allow_federation { - return Err!("Federation is disabled on this homeserver.",); + if server_name == self.services.server.name { + return Err!( + "Not allowed to send federation requests to ourselves. Please use `get-pdu` for \ + fetching local PDUs.", + ); + } + + let actual = self + .services + .resolver + .resolve_actual_dest(&server_name, !no_cache) + .await?; + + let msg = format!("Destination: {}\nHostname URI: {}", actual.dest, actual.host); + self.write_str(&msg).await } - if server_name == self.services.server.name { - return Err!( - "Not allowed to send federation requests to ourselves. Please use `get-pdu` for \ - fetching local PDUs.", + pub(super) async fn memory_stats(&self, opts: Option) -> Result { + const OPTS: &str = "abcdefghijklmnopqrstuvwxyz"; + + let opts: String = OPTS + .chars() + .filter(|&c| { + let allow_any = opts.as_ref().is_some_and(|opts| opts == "*"); + + let allow = allow_any || opts.as_ref().is_some_and(|opts| opts.contains(c)); + + !allow + }) + .collect(); + + let stats = conduwuit::alloc::memory_stats(&opts).unwrap_or_default(); + + self.write_str("```\n").await?; + self.write_str(&stats).await?; + self.write_str("\n```").await?; + Ok(()) + } + + #[cfg(tokio_unstable)] + + pub(super) async fn runtime_metrics(&self) -> Result { + let out = self.services.server.metrics.runtime_metrics().map_or_else( + || "Runtime metrics are not available.".to_owned(), + |metrics| { + format!( + "```rs\nnum_workers: {}\nnum_alive_tasks: {}\nglobal_queue_depth: {}\n```", + metrics.num_workers(), + metrics.num_alive_tasks(), + metrics.global_queue_depth() + ) + }, ); + + self.write_str(&out).await } - let actual = self - .services - .resolver - .resolve_actual_dest(&server_name, !no_cache) - .await?; + #[cfg(not(tokio_unstable))] + pub(super) async fn runtime_metrics(&self) -> Result { + self.write_str("Runtime metrics require building with `tokio_unstable`.") + .await + } - let msg = format!("Destination: {}\nHostname URI: {}", actual.dest, actual.host); - self.write_str(&msg).await -} - -#[admin_command] -pub(super) async fn memory_stats(&self, opts: Option) -> Result { - const OPTS: &str = "abcdefghijklmnopqrstuvwxyz"; - - let opts: String = OPTS - .chars() - .filter(|&c| { - let allow_any = opts.as_ref().is_some_and(|opts| opts == "*"); - - let allow = allow_any || opts.as_ref().is_some_and(|opts| opts.contains(c)); - - !allow - }) - .collect(); - - let stats = conduwuit::alloc::memory_stats(&opts).unwrap_or_default(); - - self.write_str("```\n").await?; - self.write_str(&stats).await?; - self.write_str("\n```").await?; - Ok(()) -} - -#[cfg(tokio_unstable)] -#[admin_command] -pub(super) async fn runtime_metrics(&self) -> Result { - let out = self.services.server.metrics.runtime_metrics().map_or_else( - || "Runtime metrics are not available.".to_owned(), - |metrics| { - format!( - "```rs\nnum_workers: {}\nnum_alive_tasks: {}\nglobal_queue_depth: {}\n```", - metrics.num_workers(), - metrics.num_alive_tasks(), - metrics.global_queue_depth() - ) - }, - ); - - self.write_str(&out).await -} - -#[cfg(not(tokio_unstable))] -#[admin_command] -pub(super) async fn runtime_metrics(&self) -> Result { - self.write_str("Runtime metrics require building with `tokio_unstable`.") - .await -} - -#[cfg(tokio_unstable)] -#[admin_command] -pub(super) async fn runtime_interval(&self) -> Result { - let out = self.services.server.metrics.runtime_interval().map_or_else( - || "Runtime metrics are not available.".to_owned(), - |metrics| format!("```rs\n{metrics:#?}\n```"), - ); - - self.write_str(&out).await -} - -#[cfg(not(tokio_unstable))] -#[admin_command] -pub(super) async fn runtime_interval(&self) -> Result { - self.write_str("Runtime metrics require building with `tokio_unstable`.") - .await -} - -#[admin_command] -pub(super) async fn time(&self) -> Result { - let now = SystemTime::now(); - let now = utils::time::format(now, "%+"); - - self.write_str(&now).await -} - -#[admin_command] -pub(super) async fn database_stats( - &self, - property: Option, - map: Option, -) -> Result { - let map_name = map.as_ref().map_or(EMPTY, String::as_str); - let property = property.unwrap_or_else(|| "rocksdb.stats".to_owned()); - self.services - .db - .iter() - .filter(|&(&name, _)| map_name.is_empty() || map_name == name) - .try_stream() - .try_for_each(|(&name, map)| { - let res = map.property(&property).expect("invalid property"); - writeln!(self, "##### {name}:\n```\n{}\n```", res.trim()) - }) - .await -} - -#[admin_command] -pub(super) async fn database_files(&self, map: Option, level: Option) -> Result { - let mut files: Vec<_> = self.services.db.db.file_list().collect::>()?; - - files.sort_by_key(|f| f.name.clone()); - - writeln!(self, "| lev | sst | keys | dels | size | column |").await?; - writeln!(self, "| ---: | :--- | ---: | ---: | ---: | :--- |").await?; - files - .into_iter() - .filter(|file| { - map.as_deref() - .is_none_or(|map| map == file.column_family_name) - }) - .filter(|file| level.as_ref().is_none_or(|&level| level == file.level)) - .try_stream() - .try_for_each(|file| { - writeln!( - self, - "| {} | {:<13} | {:7}+ | {:4}- | {:9} | {} |", - file.level, - file.name, - file.num_entries, - file.num_deletions, - file.size, - file.column_family_name, - ) - }) - .await -} - -#[admin_command] -pub(super) async fn trim_memory(&self) -> Result { - conduwuit::alloc::trim(None)?; - - writeln!(self, "done").await -} - -#[admin_command] -pub(super) async fn send_test_email(&self) -> Result { - self.bail_restricted()?; - - let mailer = self.services.mailer.expect_mailer()?; - let Some(sender) = self.sender else { - return Err!("No sender user provided in context"); - }; - - let Some(email) = self - .services - .threepid - .get_email_for_localpart(sender.localpart()) - .await - else { - return Err!("{} has no associated email address", sender); - }; - - mailer - .send(Mailbox::new(None, email.clone()), service::mailer::messages::Test) - .await?; - - self.write_str(&format!("Test email successfully sent to {email}")) - .await?; - - Ok(()) + #[cfg(tokio_unstable)] + + pub(super) async fn runtime_interval(&self) -> Result { + let out = self.services.server.metrics.runtime_interval().map_or_else( + || "Runtime metrics are not available.".to_owned(), + |metrics| format!("```rs\n{metrics:#?}\n```"), + ); + + self.write_str(&out).await + } + + #[cfg(not(tokio_unstable))] + pub(super) async fn runtime_interval(&self) -> Result { + self.write_str("Runtime metrics require building with `tokio_unstable`.") + .await + } + + pub(super) async fn time(&self) -> Result { + let now = SystemTime::now(); + let now = utils::time::format(now, "%+"); + + self.write_str(&now).await + } + + pub(super) async fn database_stats( + &self, + property: Option, + map: Option, + ) -> Result { + let map_name = map.as_ref().map_or(EMPTY, String::as_str); + let property = property.unwrap_or_else(|| "rocksdb.stats".to_owned()); + self.services + .db + .iter() + .filter(|&(&name, _)| map_name.is_empty() || map_name == name) + .try_stream() + .try_for_each(|(&name, map)| { + let res = map.property(&property).expect("invalid property"); + writeln!(self, "##### {name}:\n```\n{}\n```", res.trim()) + }) + .await + } + + pub(super) async fn database_files(&self, map: Option, level: Option) -> Result { + let mut files: Vec<_> = self.services.db.db.file_list().collect::>()?; + + files.sort_by_key(|f| f.name.clone()); + + writeln!(self, "| lev | sst | keys | dels | size | column |").await?; + writeln!(self, "| ---: | :--- | ---: | ---: | ---: | :--- |").await?; + files + .into_iter() + .filter(|file| { + map.as_deref() + .is_none_or(|map| map == file.column_family_name) + }) + .filter(|file| level.as_ref().is_none_or(|&level| level == file.level)) + .try_stream() + .try_for_each(|file| { + writeln!( + self, + "| {} | {:<13} | {:7}+ | {:4}- | {:9} | {} |", + file.level, + file.name, + file.num_entries, + file.num_deletions, + file.size, + file.column_family_name, + ) + }) + .await + } + + pub(super) async fn trim_memory(&self) -> Result { + conduwuit::alloc::trim(None)?; + + writeln!(self, "done").await + } + + pub(super) async fn send_test_email(&self) -> Result { + self.bail_restricted()?; + + let mailer = self.services.mailer.expect_mailer()?; + let Some(sender) = self.sender else { + return Err!("No sender user provided in context"); + }; + + let Some(email) = self + .services + .threepid + .get_email_for_localpart(sender.localpart()) + .await + else { + return Err!("{} has no associated email address", sender); + }; + + mailer + .send(Mailbox::new(None, email.clone()), service::mailer::messages::Test) + .await?; + + self.write_str(&format!("Test email successfully sent to {email}")) + .await?; + + Ok(()) + } } diff --git a/src/admin/debug/tester.rs b/src/admin/debug/tester.rs index 75da382bc..80f590f4c 100644 --- a/src/admin/debug/tester.rs +++ b/src/admin/debug/tester.rs @@ -1,6 +1,6 @@ use conduwuit::{Err, Result}; -use crate::{admin_command, admin_command_dispatch}; +use crate::admin_command_dispatch; #[admin_command_dispatch] #[derive(Debug, clap::Subcommand)] @@ -11,37 +11,32 @@ pub enum TesterCommand { Timer, } -#[rustfmt::skip] -#[admin_command] -async fn panic(&self) -> Result { +impl crate::Context<'_> { + #[rustfmt::skip] + async fn panic(&self) -> Result { + panic!("panicked") + } - panic!("panicked") -} + #[rustfmt::skip] + async fn failure(&self) -> Result { + Err!("failed") + } -#[rustfmt::skip] -#[admin_command] -async fn failure(&self) -> Result { + #[inline(never)] + #[rustfmt::skip] + async fn tester(&self) -> Result { + self.write_str("Ok").await + } - Err!("failed") -} + #[inline(never)] + #[rustfmt::skip] + async fn timer(&self) -> Result { + let started = std::time::Instant::now(); + timed(self.body); -#[inline(never)] -#[rustfmt::skip] -#[admin_command] -async fn tester(&self) -> Result { - - self.write_str("Ok").await -} - -#[inline(never)] -#[rustfmt::skip] -#[admin_command] -async fn timer(&self) -> Result { - let started = std::time::Instant::now(); - timed(self.body); - - let elapsed = started.elapsed(); - self.write_str(&format!("completed in {elapsed:#?}")).await + let elapsed = started.elapsed(); + self.write_str(&format!("completed in {elapsed:#?}")).await + } } #[inline(never)] diff --git a/src/admin/federation/commands.rs b/src/admin/federation/commands.rs index bad1c403b..64a8e0dac 100644 --- a/src/admin/federation/commands.rs +++ b/src/admin/federation/commands.rs @@ -4,127 +4,131 @@ use conduwuit::{Err, Result, utils::response::LimitReadExt}; use futures::StreamExt; use ruma::{OwnedRoomId, OwnedServerName, OwnedUserId}; -use crate::{admin_command, get_room_info}; +use crate::get_room_info; -#[admin_command] -pub(super) async fn disable_room(&self, room_id: OwnedRoomId) -> Result { - self.bail_restricted()?; - self.services.rooms.metadata.disable_room(&room_id, true); - self.write_str("Room disabled.").await -} +impl crate::Context<'_> { + pub(super) async fn disable_room(&self, room_id: OwnedRoomId) -> Result { + self.bail_restricted()?; + self.services.rooms.metadata.disable_room(&room_id, true); + self.write_str("Room disabled.").await + } -#[admin_command] -pub(super) async fn enable_room(&self, room_id: OwnedRoomId) -> Result { - self.bail_restricted()?; - self.services.rooms.metadata.disable_room(&room_id, false); - self.write_str("Room enabled.").await -} + pub(super) async fn enable_room(&self, room_id: OwnedRoomId) -> Result { + self.bail_restricted()?; + self.services.rooms.metadata.disable_room(&room_id, false); + self.write_str("Room enabled.").await + } -#[admin_command] -pub(super) async fn incoming_federation(&self) -> Result { - let msg = { - let map = self + pub(super) async fn incoming_federation(&self) -> Result { + let msg = { + let map = self + .services + .rooms + .event_handler + .federation_handletime + .read(); + + let mut msg = format!( + "Handling {} incoming PDUs across {} active transactions:\n", + map.len(), + self.services.transactions.txn_active_handle_count() + ); + for (r, (e, i)) in map.iter() { + let elapsed = i.elapsed(); + writeln!( + msg, + "{} {}: {}m{}s", + r, + e, + elapsed.as_secs() / 60, + elapsed.as_secs() % 60 + )?; + } + msg + }; + + self.write_str(&msg).await + } + + pub(super) async fn fetch_support_well_known(&self, server_name: OwnedServerName) -> Result { + let response = self + .services + .client + .default + .get(format!("https://{server_name}/.well-known/matrix/support")) + .send() + .await?; + + let text = response + .limit_read_text( + self.services + .config + .max_request_size + .try_into() + .expect("u64 fits into usize"), + ) + .await?; + + if text.is_empty() { + return Err!("Response text/body is empty."); + } + + if text.len() > 1500 { + return Err!( + "Response text/body is over 1500 characters, assuming no support well-known.", + ); + } + + let json: serde_json::Value = match serde_json::from_str(&text) { + | Ok(json) => json, + | Err(_) => { + return Err!("Response text/body is not valid JSON.",); + }, + }; + + let pretty_json: String = match serde_json::to_string_pretty(&json) { + | Ok(json) => json, + | Err(_) => { + return Err!("Response text/body is not valid JSON.",); + }, + }; + + self.write_str(&format!("Got JSON response:\n\n```json\n{pretty_json}\n```")) + .await + } + + pub(super) async fn remote_user_in_rooms(&self, user_id: OwnedUserId) -> Result { + if user_id.server_name() == self.services.server.name { + return Err!( + "User belongs to our server, please use `list-joined-rooms` user admin command \ + instead.", + ); + } + + let mut rooms: Vec<(OwnedRoomId, u64, String)> = self .services .rooms - .event_handler - .federation_handletime - .read(); + .state_cache + .rooms_joined(&user_id) + .then(async |room_id| get_room_info(self.services, &room_id).await) + .collect() + .await; - let mut msg = format!( - "Handling {} incoming PDUs across {} active transactions:\n", - map.len(), - self.services.transactions.txn_active_handle_count() - ); - for (r, (e, i)) in map.iter() { - let elapsed = i.elapsed(); - writeln!(msg, "{} {}: {}m{}s", r, e, elapsed.as_secs() / 60, elapsed.as_secs() % 60)?; + if rooms.is_empty() { + return Err!("User is not in any rooms."); } - msg - }; - self.write_str(&msg).await -} + rooms.sort_by_key(|r| r.1); + rooms.reverse(); -#[admin_command] -pub(super) async fn fetch_support_well_known(&self, server_name: OwnedServerName) -> Result { - let response = self - .services - .client - .default - .get(format!("https://{server_name}/.well-known/matrix/support")) - .send() - .await?; + let num = rooms.len(); + let body = rooms + .iter() + .map(|(id, members, name)| format!("{id} | Members: {members} | Name: {name}")) + .collect::>() + .join("\n"); - let text = response - .limit_read_text( - self.services - .config - .max_request_size - .try_into() - .expect("u64 fits into usize"), - ) - .await?; - - if text.is_empty() { - return Err!("Response text/body is empty."); - } - - if text.len() > 1500 { - return Err!( - "Response text/body is over 1500 characters, assuming no support well-known.", - ); - } - - let json: serde_json::Value = match serde_json::from_str(&text) { - | Ok(json) => json, - | Err(_) => { - return Err!("Response text/body is not valid JSON.",); - }, - }; - - let pretty_json: String = match serde_json::to_string_pretty(&json) { - | Ok(json) => json, - | Err(_) => { - return Err!("Response text/body is not valid JSON.",); - }, - }; - - self.write_str(&format!("Got JSON response:\n\n```json\n{pretty_json}\n```")) - .await -} - -#[admin_command] -pub(super) async fn remote_user_in_rooms(&self, user_id: OwnedUserId) -> Result { - if user_id.server_name() == self.services.server.name { - return Err!( - "User belongs to our server, please use `list-joined-rooms` user admin command \ - instead.", - ); - } - - let mut rooms: Vec<(OwnedRoomId, u64, String)> = self - .services - .rooms - .state_cache - .rooms_joined(&user_id) - .then(async |room_id| get_room_info(self.services, &room_id).await) - .collect() - .await; - - if rooms.is_empty() { - return Err!("User is not in any rooms."); + self.write_str(&format!("Rooms {user_id} shares with us ({num}):\n```\n{body}\n```")) + .await } - - rooms.sort_by_key(|r| r.1); - rooms.reverse(); - - let num = rooms.len(); - let body = rooms - .iter() - .map(|(id, members, name)| format!("{id} | Members: {members} | Name: {name}")) - .collect::>() - .join("\n"); - - self.write_str(&format!("Rooms {user_id} shares with us ({num}):\n```\n{body}\n```")) - .await } diff --git a/src/admin/media/commands.rs b/src/admin/media/commands.rs index 6943d82d5..b2809ecee 100644 --- a/src/admin/media/commands.rs +++ b/src/admin/media/commands.rs @@ -9,401 +9,403 @@ use conduwuit_service::media::Dim; use ruma::{OwnedEventId, OwnedMxcUri, OwnedServerName}; use service::media::mxc::Mxc; -use crate::{admin_command, utils::parse_local_user_id}; +use crate::utils::parse_local_user_id; -#[admin_command] -pub(super) async fn delete( - &self, - mxc: Option, - event_id: Option, -) -> Result { - self.bail_restricted()?; +impl crate::Context<'_> { + pub(super) async fn delete( + &self, + mxc: Option, + event_id: Option, + ) -> Result { + self.bail_restricted()?; - if event_id.is_some() && mxc.is_some() { - return Err!("Please specify either an MXC or an event ID, not both.",); - } + if event_id.is_some() && mxc.is_some() { + return Err!("Please specify either an MXC or an event ID, not both.",); + } - if let Some(mxc) = mxc { - trace!("Got MXC URL: {mxc}"); - self.services - .media - .delete(&mxc.as_str().try_into()?) - .await?; + if let Some(mxc) = mxc { + trace!("Got MXC URL: {mxc}"); + self.services + .media + .delete(&mxc.as_str().try_into()?) + .await?; - return self - .write_str("Deleted the MXC from our database and on our filesystem.") - .await; - } + return self + .write_str("Deleted the MXC from our database and on our filesystem.") + .await; + } - if let Some(event_id) = event_id { - trace!("Got event ID to delete media from: {event_id}"); + if let Some(event_id) = event_id { + trace!("Got event ID to delete media from: {event_id}"); - let mut mxc_urls = Vec::with_capacity(4); + let mut mxc_urls = Vec::with_capacity(4); - // parsing the PDU for any MXC URLs begins here - match self.services.rooms.timeline.get_pdu_json(&event_id).await { - | Ok(event_json) => { - if let Some(content_key) = event_json.get("content") { - debug!("Event ID has \"content\"."); - let content_obj = content_key.as_object(); + // parsing the PDU for any MXC URLs begins here + match self.services.rooms.timeline.get_pdu_json(&event_id).await { + | Ok(event_json) => { + if let Some(content_key) = event_json.get("content") { + debug!("Event ID has \"content\"."); + let content_obj = content_key.as_object(); - if let Some(content) = content_obj { - // 1. attempts to parse the "url" key - debug!("Attempting to go into \"url\" key for main media file"); - if let Some(url) = content.get("url") { - debug!("Got a URL in the event ID {event_id}: {url}"); + if let Some(content) = content_obj { + // 1. attempts to parse the "url" key + debug!("Attempting to go into \"url\" key for main media file"); + if let Some(url) = content.get("url") { + debug!("Got a URL in the event ID {event_id}: {url}"); - if url.to_string().starts_with("\"mxc://") { - debug!("Pushing URL {url} to list of MXCs to delete"); - let final_url = url.to_string().replace('"', ""); - mxc_urls.push(final_url); - } else { - info!( - "Found a URL in the event ID {event_id} but did not start \ - with mxc://, ignoring" - ); - } - } - - // 2. attempts to parse the "info" key - debug!("Attempting to go into \"info\" key for thumbnails"); - if let Some(info_key) = content.get("info") { - debug!("Event ID has \"info\"."); - let info_obj = info_key.as_object(); - - if let Some(info) = info_obj { - if let Some(thumbnail_url) = info.get("thumbnail_url") { - debug!("Found a thumbnail_url in info key: {thumbnail_url}"); - - if thumbnail_url.to_string().starts_with("\"mxc://") { - debug!( - "Pushing thumbnail URL {thumbnail_url} to list of \ - MXCs to delete" - ); - let final_thumbnail_url = - thumbnail_url.to_string().replace('"', ""); - mxc_urls.push(final_thumbnail_url); - } else { - info!( - "Found a thumbnail URL in the event ID {event_id} \ - but did not start with mxc://, ignoring" - ); - } + if url.to_string().starts_with("\"mxc://") { + debug!("Pushing URL {url} to list of MXCs to delete"); + let final_url = url.to_string().replace('"', ""); + mxc_urls.push(final_url); } else { info!( - "No \"thumbnail_url\" key in \"info\" key, assuming no \ - thumbnails." + "Found a URL in the event ID {event_id} but did not \ + start with mxc://, ignoring" ); } } - } - // 3. attempts to parse the "file" key - debug!("Attempting to go into \"file\" key"); - if let Some(file_key) = content.get("file") { - debug!("Event ID has \"file\"."); - let file_obj = file_key.as_object(); + // 2. attempts to parse the "info" key + debug!("Attempting to go into \"info\" key for thumbnails"); + if let Some(info_key) = content.get("info") { + debug!("Event ID has \"info\"."); + let info_obj = info_key.as_object(); - if let Some(file) = file_obj { - if let Some(url) = file.get("url") { - debug!("Found url in file key: {url}"); + if let Some(info) = info_obj { + if let Some(thumbnail_url) = info.get("thumbnail_url") { + debug!( + "Found a thumbnail_url in info key: {thumbnail_url}" + ); - if url.to_string().starts_with("\"mxc://") { - debug!("Pushing URL {url} to list of MXCs to delete"); - let final_url = url.to_string().replace('"', ""); - mxc_urls.push(final_url); + if thumbnail_url.to_string().starts_with("\"mxc://") { + debug!( + "Pushing thumbnail URL {thumbnail_url} to list \ + of MXCs to delete" + ); + let final_thumbnail_url = + thumbnail_url.to_string().replace('"', ""); + mxc_urls.push(final_thumbnail_url); + } else { + info!( + "Found a thumbnail URL in the event ID \ + {event_id} but did not start with mxc://, \ + ignoring" + ); + } } else { - warn!( - "Found a URL in the event ID {event_id} but did not \ - start with mxc://, ignoring" + info!( + "No \"thumbnail_url\" key in \"info\" key, assuming \ + no thumbnails." ); } - } else { - error!("No \"url\" key in \"file\" key."); } } + + // 3. attempts to parse the "file" key + debug!("Attempting to go into \"file\" key"); + if let Some(file_key) = content.get("file") { + debug!("Event ID has \"file\"."); + let file_obj = file_key.as_object(); + + if let Some(file) = file_obj { + if let Some(url) = file.get("url") { + debug!("Found url in file key: {url}"); + + if url.to_string().starts_with("\"mxc://") { + debug!("Pushing URL {url} to list of MXCs to delete"); + let final_url = url.to_string().replace('"', ""); + mxc_urls.push(final_url); + } else { + warn!( + "Found a URL in the event ID {event_id} but did \ + not start with mxc://, ignoring" + ); + } + } else { + error!("No \"url\" key in \"file\" key."); + } + } + } + } else { + return Err!( + "Event ID does not have a \"content\" key or failed parsing the \ + event ID JSON.", + ); } } else { return Err!( - "Event ID does not have a \"content\" key or failed parsing the \ - event ID JSON.", + "Event ID does not have a \"content\" key, this is not a message or \ + an event type that contains media.", ); } - } else { - return Err!( - "Event ID does not have a \"content\" key, this is not a message or an \ - event type that contains media.", - ); + }, + | _ => { + return Err!("Event ID does not exist or is not known to us.",); + }, + } + + if mxc_urls.is_empty() { + return Err!("Parsed event ID but found no MXC URLs.",); + } + + let mut mxc_deletion_count: usize = 0; + + for mxc_url in mxc_urls { + match self + .services + .media + .delete(&mxc_url.as_str().try_into()?) + .await + { + | Ok(()) => { + debug_info!( + "Successfully deleted {mxc_url} from filesystem and database" + ); + mxc_deletion_count = mxc_deletion_count.saturating_add(1); + }, + | Err(e) => { + debug_warn!( + "Failed to delete {mxc_url}, ignoring error and skipping: {e}" + ); + continue; + }, } - }, - | _ => { - return Err!("Event ID does not exist or is not known to us.",); - }, + } + + return self + .write_str(&format!( + "Deleted {mxc_deletion_count} total MXCs from our database and the \ + filesystem from event ID {event_id}." + )) + .await; } - if mxc_urls.is_empty() { - return Err!("Parsed event ID but found no MXC URLs.",); + Err!( + "Please specify either an MXC using --mxc or an event ID using --event-id of the \ + message containing an image. See --help for details." + ) + } + + pub(super) async fn delete_list(&self) -> Result { + self.bail_restricted()?; + + if self.body.len() < 2 + || !self.body[0].trim().starts_with("```") + || self.body.last().unwrap_or(&"").trim() != "```" + { + return Err!("Expected code block in command body. Add --help for details.",); } + let mut failed_parsed_mxcs: usize = 0; + + let mxc_list = self + .body + .to_vec() + .drain(1..self.body.len().checked_sub(1).unwrap()) + .filter_map(|mxc_s| { + mxc_s + .try_into() + .inspect_err(|e| { + debug_warn!("Failed to parse user-provided MXC URI: {e}"); + failed_parsed_mxcs = failed_parsed_mxcs.saturating_add(1); + }) + .ok() + }) + .collect::>>(); + let mut mxc_deletion_count: usize = 0; - for mxc_url in mxc_urls { - match self - .services - .media - .delete(&mxc_url.as_str().try_into()?) - .await - { + for mxc in &mxc_list { + trace!(%failed_parsed_mxcs, %mxc_deletion_count, "Deleting MXC {mxc} in bulk"); + match self.services.media.delete(mxc).await { | Ok(()) => { - debug_info!("Successfully deleted {mxc_url} from filesystem and database"); + debug_info!("Successfully deleted {mxc} from filesystem and database"); mxc_deletion_count = mxc_deletion_count.saturating_add(1); }, | Err(e) => { - debug_warn!("Failed to delete {mxc_url}, ignoring error and skipping: {e}"); + debug_warn!("Failed to delete {mxc}, ignoring error and skipping: {e}"); continue; }, } } - return self - .write_str(&format!( - "Deleted {mxc_deletion_count} total MXCs from our database and the filesystem \ - from event ID {event_id}." - )) - .await; + self.write_str(&format!( + "Finished bulk MXC deletion, deleted {mxc_deletion_count} total MXCs from our \ + database and the filesystem. {failed_parsed_mxcs} MXCs failed to be parsed from \ + the database.", + )) + .await } - Err!( - "Please specify either an MXC using --mxc or an event ID using --event-id of the \ - message containing an image. See --help for details." - ) -} + pub(super) async fn delete_past_remote_media( + &self, + duration: String, + before: bool, + after: bool, + yes_i_want_to_delete_local_media: bool, + ) -> Result { + self.bail_restricted()?; -#[admin_command] -pub(super) async fn delete_list(&self) -> Result { - self.bail_restricted()?; - - if self.body.len() < 2 - || !self.body[0].trim().starts_with("```") - || self.body.last().unwrap_or(&"").trim() != "```" - { - return Err!("Expected code block in command body. Add --help for details.",); - } - - let mut failed_parsed_mxcs: usize = 0; - - let mxc_list = self - .body - .to_vec() - .drain(1..self.body.len().checked_sub(1).unwrap()) - .filter_map(|mxc_s| { - mxc_s - .try_into() - .inspect_err(|e| { - debug_warn!("Failed to parse user-provided MXC URI: {e}"); - failed_parsed_mxcs = failed_parsed_mxcs.saturating_add(1); - }) - .ok() - }) - .collect::>>(); - - let mut mxc_deletion_count: usize = 0; - - for mxc in &mxc_list { - trace!(%failed_parsed_mxcs, %mxc_deletion_count, "Deleting MXC {mxc} in bulk"); - match self.services.media.delete(mxc).await { - | Ok(()) => { - debug_info!("Successfully deleted {mxc} from filesystem and database"); - mxc_deletion_count = mxc_deletion_count.saturating_add(1); - }, - | Err(e) => { - debug_warn!("Failed to delete {mxc}, ignoring error and skipping: {e}"); - continue; - }, + if before && after { + return Err!("Please only pick one argument, --before or --after.",); } - } + assert!(!(before && after), "--before and --after should not be specified together"); - self.write_str(&format!( - "Finished bulk MXC deletion, deleted {mxc_deletion_count} total MXCs from our database \ - and the filesystem. {failed_parsed_mxcs} MXCs failed to be parsed from the database.", - )) - .await -} - -#[admin_command] -pub(super) async fn delete_past_remote_media( - &self, - duration: String, - before: bool, - after: bool, - yes_i_want_to_delete_local_media: bool, -) -> Result { - self.bail_restricted()?; - - if before && after { - return Err!("Please only pick one argument, --before or --after.",); - } - assert!(!(before && after), "--before and --after should not be specified together"); - - let direction = if after { - TimeDirection::After - } else { - TimeDirection::Before - }; - - let time_boundary = parse_timepoint_ago(&duration)?; - let deleted_count = self - .services - .media - .delete_all_media_within_timeframe( - time_boundary, - direction, - yes_i_want_to_delete_local_media, - ) - .await?; - - self.write_str(&format!("Deleted {deleted_count} total files.")) - .await -} - -#[admin_command] -pub(super) async fn delete_all_from_user(&self, username: String) -> Result { - let user_id = parse_local_user_id(self.services, &username)?; - - let deleted_count = self.services.media.delete_from_user(&user_id).await?; - - self.write_str(&format!("Deleted {deleted_count} total files.")) - .await -} - -#[admin_command] -pub(super) async fn delete_all_from_server( - &self, - server_name: OwnedServerName, - yes_i_want_to_delete_local_media: bool, -) -> Result { - self.bail_restricted()?; - - if server_name == self.services.globals.server_name() && !yes_i_want_to_delete_local_media { - return Err!("This command only works for remote media by default.",); - } - - let Ok(all_mxcs) = self - .services - .media - .get_all_mxcs() - .await - .inspect_err(|e| error!("Failed to get MXC URIs from our database: {e}")) - else { - return Err!("Failed to get MXC URIs from our database",); - }; - - let mut deleted_count: usize = 0; - - for mxc in all_mxcs { - let Ok(mxc_server_name) = mxc.server_name().inspect_err(|e| { - debug_warn!( - "Failed to parse MXC {mxc} server name from database, ignoring error and \ - skipping: {e}" - ); - }) else { - continue; + let direction = if after { + TimeDirection::After + } else { + TimeDirection::Before }; - if mxc_server_name != server_name - || (self.services.globals.server_is_ours(mxc_server_name) - && !yes_i_want_to_delete_local_media) + let time_boundary = parse_timepoint_ago(&duration)?; + let deleted_count = self + .services + .media + .delete_all_media_within_timeframe( + time_boundary, + direction, + yes_i_want_to_delete_local_media, + ) + .await?; + + self.write_str(&format!("Deleted {deleted_count} total files.")) + .await + } + + pub(super) async fn delete_all_from_user(&self, username: String) -> Result { + let user_id = parse_local_user_id(self.services, &username)?; + + let deleted_count = self.services.media.delete_from_user(&user_id).await?; + + self.write_str(&format!("Deleted {deleted_count} total files.")) + .await + } + + pub(super) async fn delete_all_from_server( + &self, + server_name: OwnedServerName, + yes_i_want_to_delete_local_media: bool, + ) -> Result { + self.bail_restricted()?; + + if server_name == self.services.globals.server_name() && !yes_i_want_to_delete_local_media { - trace!("skipping MXC URI {mxc}"); - continue; + return Err!("This command only works for remote media by default.",); } - let mxc: Mxc<'_> = mxc.as_str().try_into()?; + let Ok(all_mxcs) = self + .services + .media + .get_all_mxcs() + .await + .inspect_err(|e| error!("Failed to get MXC URIs from our database: {e}")) + else { + return Err!("Failed to get MXC URIs from our database",); + }; - match self.services.media.delete(&mxc).await { - | Ok(()) => { - deleted_count = deleted_count.saturating_add(1); - }, - | Err(e) => { - debug_warn!("Failed to delete {mxc}, ignoring error and skipping: {e}"); + let mut deleted_count: usize = 0; + + for mxc in all_mxcs { + let Ok(mxc_server_name) = mxc.server_name().inspect_err(|e| { + debug_warn!( + "Failed to parse MXC {mxc} server name from database, ignoring error and \ + skipping: {e}" + ); + }) else { continue; - }, + }; + + if mxc_server_name != server_name + || (self.services.globals.server_is_ours(mxc_server_name) + && !yes_i_want_to_delete_local_media) + { + trace!("skipping MXC URI {mxc}"); + continue; + } + + let mxc: Mxc<'_> = mxc.as_str().try_into()?; + + match self.services.media.delete(&mxc).await { + | Ok(()) => { + deleted_count = deleted_count.saturating_add(1); + }, + | Err(e) => { + debug_warn!("Failed to delete {mxc}, ignoring error and skipping: {e}"); + continue; + }, + } } + + self.write_str(&format!("Deleted {deleted_count} total files.")) + .await } - self.write_str(&format!("Deleted {deleted_count} total files.")) - .await -} + pub(super) async fn get_file_info(&self, mxc: OwnedMxcUri) -> Result { + let mxc: Mxc<'_> = mxc.as_str().try_into()?; + let metadata = self.services.media.get_metadata(&mxc).await; -#[admin_command] -pub(super) async fn get_file_info(&self, mxc: OwnedMxcUri) -> Result { - let mxc: Mxc<'_> = mxc.as_str().try_into()?; - let metadata = self.services.media.get_metadata(&mxc).await; - - self.write_str(&format!("```\n{metadata:#?}\n```")).await -} - -#[admin_command] -pub(super) async fn get_remote_file( - &self, - mxc: OwnedMxcUri, - server: Option, - timeout: u32, -) -> Result { - let mxc: Mxc<'_> = mxc.as_str().try_into()?; - let timeout = Duration::from_millis(timeout.into()); - let mut result = self - .services - .media - .fetch_remote_content(&mxc, None, server.as_deref(), timeout) - .await?; - - // Grab the length of the content before clearing it to not flood the output - let len = result.content.as_ref().expect("content").len(); - result.content.as_mut().expect("content").clear(); - - self.write_str(&format!("```\n{result:#?}\nreceived {len} bytes for file content.\n```")) - .await -} - -#[admin_command] -pub(super) async fn get_remote_thumbnail( - &self, - mxc: OwnedMxcUri, - server: Option, - timeout: u32, - width: u32, - height: u32, -) -> Result { - let mxc: Mxc<'_> = mxc.as_str().try_into()?; - let timeout = Duration::from_millis(timeout.into()); - let dim = Dim::new(width, height, None); - let mut result = self - .services - .media - .fetch_remote_thumbnail(&mxc, None, server.as_deref(), timeout, &dim) - .await?; - - // Grab the length of the content before clearing it to not flood the output - let len = result.content.as_ref().expect("content").len(); - result.content.as_mut().expect("content").clear(); - - self.write_str(&format!("```\n{result:#?}\nreceived {len} bytes for file content.\n```")) - .await -} - -#[admin_command] -pub(super) async fn delete_url_preview(&self, url: Option, all: bool) -> Result { - if all { - self.services.media.clear_url_previews().await; - - return self.write_str("Deleted all cached URL previews.").await; + self.write_str(&format!("```\n{metadata:#?}\n```")).await } - let url = url.expect("clap enforces url is required unless --all"); + pub(super) async fn get_remote_file( + &self, + mxc: OwnedMxcUri, + server: Option, + timeout: u32, + ) -> Result { + let mxc: Mxc<'_> = mxc.as_str().try_into()?; + let timeout = Duration::from_millis(timeout.into()); + let mut result = self + .services + .media + .fetch_remote_content(&mxc, None, server.as_deref(), timeout) + .await?; - self.services.media.remove_url_preview(&url).await?; + // Grab the length of the content before clearing it to not flood the output + let len = result.content.as_ref().expect("content").len(); + result.content.as_mut().expect("content").clear(); - self.write_str(&format!("Deleted cached URL preview for: {url}")) - .await + self.write_str(&format!("```\n{result:#?}\nreceived {len} bytes for file content.\n```")) + .await + } + + pub(super) async fn get_remote_thumbnail( + &self, + mxc: OwnedMxcUri, + server: Option, + timeout: u32, + width: u32, + height: u32, + ) -> Result { + let mxc: Mxc<'_> = mxc.as_str().try_into()?; + let timeout = Duration::from_millis(timeout.into()); + let dim = Dim::new(width, height, None); + let mut result = self + .services + .media + .fetch_remote_thumbnail(&mxc, None, server.as_deref(), timeout, &dim) + .await?; + + // Grab the length of the content before clearing it to not flood the output + let len = result.content.as_ref().expect("content").len(); + result.content.as_mut().expect("content").clear(); + + self.write_str(&format!("```\n{result:#?}\nreceived {len} bytes for file content.\n```")) + .await + } + + pub(super) async fn delete_url_preview(&self, url: Option, all: bool) -> Result { + if all { + self.services.media.clear_url_previews().await; + + return self.write_str("Deleted all cached URL previews.").await; + } + + let url = url.expect("clap enforces url is required unless --all"); + + self.services.media.remove_url_preview(&url).await?; + + self.write_str(&format!("Deleted cached URL preview for: {url}")) + .await + } } diff --git a/src/admin/mod.rs b/src/admin/mod.rs index 3c47f4855..1bc33691c 100644 --- a/src/admin/mod.rs +++ b/src/admin/mod.rs @@ -26,7 +26,7 @@ extern crate conduwuit_api as api; extern crate conduwuit_core as conduwuit; extern crate conduwuit_service as service; -pub(crate) use conduwuit_macros::{admin_command, admin_command_dispatch}; +pub(crate) use conduwuit_macros::admin_command_dispatch; pub(crate) use crate::{context::Context, utils::get_room_info}; diff --git a/src/admin/query/account_data.rs b/src/admin/query/account_data.rs index 746f72031..82e881e0b 100644 --- a/src/admin/query/account_data.rs +++ b/src/admin/query/account_data.rs @@ -4,7 +4,7 @@ use conduwuit_database::Deserialized as _; use futures::StreamExt; use ruma::{OwnedRoomId, OwnedUserId, exports::serde::Serialize}; -use crate::{admin_command, admin_command_dispatch}; +use crate::admin_command_dispatch; #[admin_command_dispatch] #[derive(Debug, Subcommand)] @@ -31,50 +31,50 @@ pub enum AccountDataCommand { }, } -#[admin_command] -async fn changes_since( - &self, - user_id: OwnedUserId, - since: u64, - room_id: Option, -) -> Result { - let timer = tokio::time::Instant::now(); - let results: Vec<_> = self - .services - .account_data - .changes_since(room_id.as_deref(), &user_id, Some(since), None) - .collect() - .await; - let query_time = timer.elapsed(); +impl crate::Context<'_> { + async fn changes_since( + &self, + user_id: OwnedUserId, + since: u64, + room_id: Option, + ) -> Result { + let timer = tokio::time::Instant::now(); + let results: Vec<_> = self + .services + .account_data + .changes_since(room_id.as_deref(), &user_id, Some(since), None) + .collect() + .await; + let query_time = timer.elapsed(); - self.write_str(&format!("Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```")) - .await -} + self.write_str(&format!("Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```")) + .await + } -#[admin_command] -async fn account_data_get( - &self, - user_id: OwnedUserId, - kind: String, - room_id: Option, -) -> Result { - let timer = tokio::time::Instant::now(); - let result = self - .services - .account_data - .get_raw(room_id.as_deref(), &user_id, &kind) - .await; - let query_time = timer.elapsed(); + async fn account_data_get( + &self, + user_id: OwnedUserId, + kind: String, + room_id: Option, + ) -> Result { + let timer = tokio::time::Instant::now(); + let result = self + .services + .account_data + .get_raw(room_id.as_deref(), &user_id, &kind) + .await; + let query_time = timer.elapsed(); - let json = serde_json::to_string_pretty(&match room_id { - | None => result - .deserialized::>()? - .serialize(serde_json::value::Serializer)?, - | Some(_) => result - .deserialized::>()? - .serialize(serde_json::value::Serializer)?, - })?; + let json = serde_json::to_string_pretty(&match room_id { + | None => result + .deserialized::>()? + .serialize(serde_json::value::Serializer)?, + | Some(_) => result + .deserialized::>()? + .serialize(serde_json::value::Serializer)?, + })?; - self.write_str(&format!("Query completed in {query_time:?}:\n\n```rs\n{json}\n```")) - .await + self.write_str(&format!("Query completed in {query_time:?}:\n\n```rs\n{json}\n```")) + .await + } } diff --git a/src/admin/query/raw.rs b/src/admin/query/raw.rs index 97579d844..557c6d844 100644 --- a/src/admin/query/raw.rs +++ b/src/admin/query/raw.rs @@ -13,7 +13,7 @@ use conduwuit_service::Services; use futures::{FutureExt, Stream, StreamExt, TryStreamExt}; use tokio::time::Instant; -use crate::{admin_command, admin_command_dispatch}; +use crate::admin_command_dispatch; #[admin_command_dispatch] #[derive(Debug, Subcommand)] @@ -159,279 +159,287 @@ pub enum RawCommand { }, } -#[admin_command] -pub(super) async fn compact( - &self, - map: Option>, - start: Option, - stop: Option, - from: Option, - into: Option, - parallelism: Option, - exhaustive: bool, -) -> Result { - use conduwuit_database::compact::Options; +impl crate::Context<'_> { + pub(super) async fn compact( + &self, + map: Option>, + start: Option, + stop: Option, + from: Option, + into: Option, + parallelism: Option, + exhaustive: bool, + ) -> Result { + use conduwuit_database::compact::Options; - let default_all_maps: Option<_> = map.is_none().then(|| { - self.services - .db - .keys() - .map(Deref::deref) - .map(ToOwned::to_owned) - }); + let default_all_maps: Option<_> = map.is_none().then(|| { + self.services + .db + .keys() + .map(Deref::deref) + .map(ToOwned::to_owned) + }); - let maps: Vec<_> = map - .unwrap_or_default() - .into_iter() - .chain(default_all_maps.into_iter().flatten()) - .map(|map| self.services.db.get(&map)) - .filter_map(Result::ok) - .cloned() - .collect(); + let maps: Vec<_> = map + .unwrap_or_default() + .into_iter() + .chain(default_all_maps.into_iter().flatten()) + .map(|map| self.services.db.get(&map)) + .filter_map(Result::ok) + .cloned() + .collect(); - if maps.is_empty() { - return Err!("--map argument invalid. not found in database"); + if maps.is_empty() { + return Err!("--map argument invalid. not found in database"); + } + + let range = ( + start.as_ref().map(String::as_bytes).map(Into::into), + stop.as_ref().map(String::as_bytes).map(Into::into), + ); + + let options = Options { + range, + level: (from, into), + exclusive: parallelism.is_some_and(is_zero!()), + exhaustive, + }; + + let runtime = self.services.server.runtime().clone(); + let parallelism = parallelism.unwrap_or(1); + let results = maps + .into_iter() + .try_stream::() + .paralleln_and_then(runtime, parallelism, move |map| { + map.compact_blocking(options.clone())?; + Ok(map.name().to_owned()) + }) + .collect::>(); + + let timer = Instant::now(); + let results = results.await; + let query_time = timer.elapsed(); + self.write_str(&format!("Jobs completed in {query_time:?}:\n\n```rs\n{results:#?}\n```")) + .await } - let range = ( - start.as_ref().map(String::as_bytes).map(Into::into), - stop.as_ref().map(String::as_bytes).map(Into::into), - ); + pub(super) async fn raw_count(&self, map: Option, prefix: Option) -> Result { + let prefix = prefix.as_deref().unwrap_or(EMPTY); - let options = Options { - range, - level: (from, into), - exclusive: parallelism.is_some_and(is_zero!()), - exhaustive, - }; + let timer = Instant::now(); + let count = with_maps_or(map.as_deref(), self.services) + .then(|map| map.raw_count_prefix(&prefix)) + .ready_fold(0_usize, usize::saturating_add) + .await; - let runtime = self.services.server.runtime().clone(); - let parallelism = parallelism.unwrap_or(1); - let results = maps - .into_iter() - .try_stream::() - .paralleln_and_then(runtime, parallelism, move |map| { - map.compact_blocking(options.clone())?; - Ok(map.name().to_owned()) - }) - .collect::>(); + let query_time = timer.elapsed(); + self.write_str(&format!("Query completed in {query_time:?}:\n\n```rs\n{count:#?}\n```")) + .await + } - let timer = Instant::now(); - let results = results.await; - let query_time = timer.elapsed(); - self.write_str(&format!("Jobs completed in {query_time:?}:\n\n```rs\n{results:#?}\n```")) - .await -} + pub(super) async fn raw_keys(&self, map: String, prefix: Option) -> Result { + writeln!(self, "```").boxed().await?; -#[admin_command] -pub(super) async fn raw_count(&self, map: Option, prefix: Option) -> Result { - let prefix = prefix.as_deref().unwrap_or(EMPTY); + let map = self.services.db.get(map.as_str())?; + let timer = Instant::now(); + prefix + .as_deref() + .map_or_else(|| map.raw_keys().boxed(), |prefix| map.raw_keys_prefix(prefix).boxed()) + .map_ok(String::from_utf8_lossy) + .try_for_each(|str| writeln!(self, "{str:?}")) + .boxed() + .await?; - let timer = Instant::now(); - let count = with_maps_or(map.as_deref(), self.services) - .then(|map| map.raw_count_prefix(&prefix)) - .ready_fold(0_usize, usize::saturating_add) - .await; + let query_time = timer.elapsed(); + self.write_str(&format!("\n```\n\nQuery completed in {query_time:?}")) + .await + } - let query_time = timer.elapsed(); - self.write_str(&format!("Query completed in {query_time:?}:\n\n```rs\n{count:#?}\n```")) - .await -} + pub(super) async fn raw_keys_sizes( + &self, + map: Option, + prefix: Option, + ) -> Result { + let prefix = prefix.as_deref().unwrap_or(EMPTY); -#[admin_command] -pub(super) async fn raw_keys(&self, map: String, prefix: Option) -> Result { - writeln!(self, "```").boxed().await?; + let timer = Instant::now(); + let result = with_maps_or(map.as_deref(), self.services) + .map(|map| map.raw_keys_prefix(&prefix)) + .flatten() + .ignore_err() + .map(<[u8]>::len) + .ready_fold_default(|mut map: BTreeMap<_, usize>, len| { + let entry = map.entry(len).or_default(); + *entry = entry.saturating_add(1); + map + }) + .await; - let map = self.services.db.get(map.as_str())?; - let timer = Instant::now(); - prefix - .as_deref() - .map_or_else(|| map.raw_keys().boxed(), |prefix| map.raw_keys_prefix(prefix).boxed()) - .map_ok(String::from_utf8_lossy) - .try_for_each(|str| writeln!(self, "{str:?}")) - .boxed() - .await?; + let query_time = timer.elapsed(); + self.write_str(&format!("```\n{result:#?}\n```\n\nQuery completed in {query_time:?}")) + .await + } - let query_time = timer.elapsed(); - self.write_str(&format!("\n```\n\nQuery completed in {query_time:?}")) - .await -} + pub(super) async fn raw_keys_total( + &self, + map: Option, + prefix: Option, + ) -> Result { + let prefix = prefix.as_deref().unwrap_or(EMPTY); -#[admin_command] -pub(super) async fn raw_keys_sizes(&self, map: Option, prefix: Option) -> Result { - let prefix = prefix.as_deref().unwrap_or(EMPTY); + let timer = Instant::now(); + let result = with_maps_or(map.as_deref(), self.services) + .map(|map| map.raw_keys_prefix(&prefix)) + .flatten() + .ignore_err() + .map(<[u8]>::len) + .ready_fold_default(|acc: usize, len| acc.saturating_add(len)) + .await; - let timer = Instant::now(); - let result = with_maps_or(map.as_deref(), self.services) - .map(|map| map.raw_keys_prefix(&prefix)) - .flatten() - .ignore_err() - .map(<[u8]>::len) - .ready_fold_default(|mut map: BTreeMap<_, usize>, len| { - let entry = map.entry(len).or_default(); - *entry = entry.saturating_add(1); - map - }) - .await; + let query_time = timer.elapsed(); + self.write_str(&format!("```\n{result:#?}\n\n```\n\nQuery completed in {query_time:?}")) + .await + } - let query_time = timer.elapsed(); - self.write_str(&format!("```\n{result:#?}\n```\n\nQuery completed in {query_time:?}")) - .await -} + pub(super) async fn raw_vals_sizes( + &self, + map: Option, + prefix: Option, + ) -> Result { + let prefix = prefix.as_deref().unwrap_or(EMPTY); -#[admin_command] -pub(super) async fn raw_keys_total(&self, map: Option, prefix: Option) -> Result { - let prefix = prefix.as_deref().unwrap_or(EMPTY); + let timer = Instant::now(); + let result = with_maps_or(map.as_deref(), self.services) + .map(|map| map.raw_stream_prefix(&prefix)) + .flatten() + .ignore_err() + .map(at!(1)) + .map(<[u8]>::len) + .ready_fold_default(|mut map: BTreeMap<_, usize>, len| { + let entry = map.entry(len).or_default(); + *entry = entry.saturating_add(1); + map + }) + .await; - let timer = Instant::now(); - let result = with_maps_or(map.as_deref(), self.services) - .map(|map| map.raw_keys_prefix(&prefix)) - .flatten() - .ignore_err() - .map(<[u8]>::len) - .ready_fold_default(|acc: usize, len| acc.saturating_add(len)) - .await; + let query_time = timer.elapsed(); + self.write_str(&format!("```\n{result:#?}\n```\n\nQuery completed in {query_time:?}")) + .await + } - let query_time = timer.elapsed(); - self.write_str(&format!("```\n{result:#?}\n\n```\n\nQuery completed in {query_time:?}")) - .await -} + pub(super) async fn raw_vals_total( + &self, + map: Option, + prefix: Option, + ) -> Result { + let prefix = prefix.as_deref().unwrap_or(EMPTY); -#[admin_command] -pub(super) async fn raw_vals_sizes(&self, map: Option, prefix: Option) -> Result { - let prefix = prefix.as_deref().unwrap_or(EMPTY); + let timer = Instant::now(); + let result = with_maps_or(map.as_deref(), self.services) + .map(|map| map.raw_stream_prefix(&prefix)) + .flatten() + .ignore_err() + .map(at!(1)) + .map(<[u8]>::len) + .ready_fold_default(|acc: usize, len| acc.saturating_add(len)) + .await; - let timer = Instant::now(); - let result = with_maps_or(map.as_deref(), self.services) - .map(|map| map.raw_stream_prefix(&prefix)) - .flatten() - .ignore_err() - .map(at!(1)) - .map(<[u8]>::len) - .ready_fold_default(|mut map: BTreeMap<_, usize>, len| { - let entry = map.entry(len).or_default(); - *entry = entry.saturating_add(1); - map - }) - .await; + let query_time = timer.elapsed(); + self.write_str(&format!("```\n{result:#?}\n\n```\n\nQuery completed in {query_time:?}")) + .await + } - let query_time = timer.elapsed(); - self.write_str(&format!("```\n{result:#?}\n```\n\nQuery completed in {query_time:?}")) - .await -} + pub(super) async fn raw_iter(&self, map: String, prefix: Option) -> Result { + writeln!(self, "```").await?; -#[admin_command] -pub(super) async fn raw_vals_total(&self, map: Option, prefix: Option) -> Result { - let prefix = prefix.as_deref().unwrap_or(EMPTY); + let map = self.services.db.get(&map)?; + let timer = Instant::now(); + prefix + .as_deref() + .map_or_else( + || map.raw_stream().boxed(), + |prefix| map.raw_stream_prefix(prefix).boxed(), + ) + .map_ok(apply!(2, String::from_utf8_lossy)) + .map_ok(apply!(2, Cow::into_owned)) + .try_for_each(|keyval| writeln!(self, "{keyval:?}")) + .boxed() + .await?; - let timer = Instant::now(); - let result = with_maps_or(map.as_deref(), self.services) - .map(|map| map.raw_stream_prefix(&prefix)) - .flatten() - .ignore_err() - .map(at!(1)) - .map(<[u8]>::len) - .ready_fold_default(|acc: usize, len| acc.saturating_add(len)) - .await; + let query_time = timer.elapsed(); + self.write_str(&format!("\n```\n\nQuery completed in {query_time:?}")) + .await + } - let query_time = timer.elapsed(); - self.write_str(&format!("```\n{result:#?}\n\n```\n\nQuery completed in {query_time:?}")) - .await -} + pub(super) async fn raw_keys_from( + &self, + map: String, + start: String, + limit: Option, + ) -> Result { + writeln!(self, "```").await?; -#[admin_command] -pub(super) async fn raw_iter(&self, map: String, prefix: Option) -> Result { - writeln!(self, "```").await?; + let map = self.services.db.get(&map)?; + let timer = Instant::now(); + map.raw_keys_from(&start) + .map_ok(String::from_utf8_lossy) + .take(limit.unwrap_or(usize::MAX)) + .try_for_each(|str| writeln!(self, "{str:?}")) + .boxed() + .await?; - let map = self.services.db.get(&map)?; - let timer = Instant::now(); - prefix - .as_deref() - .map_or_else(|| map.raw_stream().boxed(), |prefix| map.raw_stream_prefix(prefix).boxed()) - .map_ok(apply!(2, String::from_utf8_lossy)) - .map_ok(apply!(2, Cow::into_owned)) - .try_for_each(|keyval| writeln!(self, "{keyval:?}")) - .boxed() - .await?; + let query_time = timer.elapsed(); + self.write_str(&format!("\n```\n\nQuery completed in {query_time:?}")) + .await + } - let query_time = timer.elapsed(); - self.write_str(&format!("\n```\n\nQuery completed in {query_time:?}")) - .await -} + pub(super) async fn raw_iter_from( + &self, + map: String, + start: String, + limit: Option, + ) -> Result { + let map = self.services.db.get(&map)?; + let timer = Instant::now(); + let result = map + .raw_stream_from(&start) + .map_ok(apply!(2, String::from_utf8_lossy)) + .map_ok(apply!(2, Cow::into_owned)) + .take(limit.unwrap_or(usize::MAX)) + .try_collect::>() + .await?; -#[admin_command] -pub(super) async fn raw_keys_from( - &self, - map: String, - start: String, - limit: Option, -) -> Result { - writeln!(self, "```").await?; + let query_time = timer.elapsed(); + self.write_str(&format!("Query completed in {query_time:?}:\n\n```rs\n{result:#?}\n```")) + .await + } - let map = self.services.db.get(&map)?; - let timer = Instant::now(); - map.raw_keys_from(&start) - .map_ok(String::from_utf8_lossy) - .take(limit.unwrap_or(usize::MAX)) - .try_for_each(|str| writeln!(self, "{str:?}")) - .boxed() - .await?; + pub(super) async fn raw_del(&self, map: String, key: String) -> Result { + let map = self.services.db.get(&map)?; + let timer = Instant::now(); + map.remove(&key); - let query_time = timer.elapsed(); - self.write_str(&format!("\n```\n\nQuery completed in {query_time:?}")) - .await -} + let query_time = timer.elapsed(); + self.write_str(&format!("Operation completed in {query_time:?}")) + .await + } -#[admin_command] -pub(super) async fn raw_iter_from( - &self, - map: String, - start: String, - limit: Option, -) -> Result { - let map = self.services.db.get(&map)?; - let timer = Instant::now(); - let result = map - .raw_stream_from(&start) - .map_ok(apply!(2, String::from_utf8_lossy)) - .map_ok(apply!(2, Cow::into_owned)) - .take(limit.unwrap_or(usize::MAX)) - .try_collect::>() - .await?; + pub(super) async fn raw_get(&self, map: String, key: String) -> Result { + let map = self.services.db.get(&map)?; + let timer = Instant::now(); + let handle = map.get(&key).await?; - let query_time = timer.elapsed(); - self.write_str(&format!("Query completed in {query_time:?}:\n\n```rs\n{result:#?}\n```")) - .await -} + let query_time = timer.elapsed(); + let result = String::from_utf8_lossy(&handle); + self.write_str(&format!("Query completed in {query_time:?}:\n\n```rs\n{result:?}\n```")) + .await + } -#[admin_command] -pub(super) async fn raw_del(&self, map: String, key: String) -> Result { - let map = self.services.db.get(&map)?; - let timer = Instant::now(); - map.remove(&key); + pub(super) async fn raw_maps(&self) -> Result { + let list: Vec<_> = self.services.db.iter().map(at!(0)).copied().collect(); - let query_time = timer.elapsed(); - self.write_str(&format!("Operation completed in {query_time:?}")) - .await -} - -#[admin_command] -pub(super) async fn raw_get(&self, map: String, key: String) -> Result { - let map = self.services.db.get(&map)?; - let timer = Instant::now(); - let handle = map.get(&key).await?; - - let query_time = timer.elapsed(); - let result = String::from_utf8_lossy(&handle); - self.write_str(&format!("Query completed in {query_time:?}:\n\n```rs\n{result:?}\n```")) - .await -} - -#[admin_command] -pub(super) async fn raw_maps(&self) -> Result { - let list: Vec<_> = self.services.db.iter().map(at!(0)).copied().collect(); - - self.write_str(&format!("{list:#?}")).await + self.write_str(&format!("{list:#?}")).await + } } fn with_maps_or<'a>( diff --git a/src/admin/query/resolver.rs b/src/admin/query/resolver.rs index d651e5bff..7e5a4b40d 100644 --- a/src/admin/query/resolver.rs +++ b/src/admin/query/resolver.rs @@ -3,7 +3,7 @@ use conduwuit::{Err, Result, utils::time}; use futures::StreamExt; use ruma::OwnedServerName; -use crate::{admin_command, admin_command_dispatch}; +use crate::admin_command_dispatch; #[admin_command_dispatch] #[derive(Debug, Subcommand)] @@ -39,67 +39,68 @@ pub enum ResolverCommand { }, } -#[admin_command] -async fn destinations_cache(&self, server_name: Option) -> Result { - use service::resolver::cache::CachedDest; +impl crate::Context<'_> { + async fn destinations_cache(&self, server_name: Option) -> Result { + use service::resolver::cache::CachedDest; - writeln!(self, "| Server Name | Destination | Hostname | Expires |").await?; - writeln!(self, "| ----------- | ----------- | -------- | ------- |").await?; + writeln!(self, "| Server Name | Destination | Hostname | Expires |").await?; + writeln!(self, "| ----------- | ----------- | -------- | ------- |").await?; - let mut destinations = self.services.resolver.cache.destinations().boxed(); + let mut destinations = self.services.resolver.cache.destinations().boxed(); - while let Some((name, CachedDest { dest, host, expire })) = destinations.next().await { - if let Some(server_name) = server_name.as_ref() { - if name != *server_name { - continue; + while let Some((name, CachedDest { dest, host, expire })) = destinations.next().await { + if let Some(server_name) = server_name.as_ref() { + if name != *server_name { + continue; + } } + + let expire = time::format(expire, "%+"); + self.write_str(&format!("| {name} | {dest} | {host} | {expire} |\n")) + .await?; } - let expire = time::format(expire, "%+"); - self.write_str(&format!("| {name} | {dest} | {host} | {expire} |\n")) - .await?; + Ok(()) } - Ok(()) -} + async fn overrides_cache(&self, server_name: Option) -> Result { + use service::resolver::cache::CachedOverride; -#[admin_command] -async fn overrides_cache(&self, server_name: Option) -> Result { - use service::resolver::cache::CachedOverride; + writeln!(self, "| Server Name | IP | Port | Expires | Overriding |").await?; + writeln!(self, "| ----------- | --- | ----:| ------- | ---------- |").await?; - writeln!(self, "| Server Name | IP | Port | Expires | Overriding |").await?; - writeln!(self, "| ----------- | --- | ----:| ------- | ---------- |").await?; + let mut overrides = self.services.resolver.cache.overrides().boxed(); - let mut overrides = self.services.resolver.cache.overrides().boxed(); - - while let Some((name, CachedOverride { ips, port, expire, overriding })) = - overrides.next().await - { - if let Some(server_name) = server_name.as_ref() { - if name != *server_name { - continue; + while let Some((name, CachedOverride { ips, port, expire, overriding })) = + overrides.next().await + { + if let Some(server_name) = server_name.as_ref() { + if name != *server_name { + continue; + } } + + let expire = time::format(expire, "%+"); + self.write_str(&format!( + "| {name} | {ips:?} | {port} | {expire} | {overriding:?} |\n" + )) + .await?; } - let expire = time::format(expire, "%+"); - self.write_str(&format!("| {name} | {ips:?} | {port} | {expire} | {overriding:?} |\n")) - .await?; + Ok(()) } - Ok(()) -} - -#[admin_command] -async fn flush_cache(&self, name: Option, all: bool) -> Result { - if all { - self.services.resolver.cache.clear().await; - writeln!(self, "Resolver caches cleared!").await - } else if let Some(name) = name { - self.services.resolver.cache.del_destination(&name); - self.services.resolver.cache.del_override(&name); - self.write_str(&format!("Cleared {name} from resolver caches!")) - .await - } else { - Err!("Missing name. Supply a name or use --all to flush the whole cache.") + async fn flush_cache(&self, name: Option, all: bool) -> Result { + if all { + self.services.resolver.cache.clear().await; + writeln!(self, "Resolver caches cleared!").await + } else if let Some(name) = name { + self.services.resolver.cache.del_destination(&name); + self.services.resolver.cache.del_override(&name); + self.write_str(&format!("Cleared {name} from resolver caches!")) + .await + } else { + Err!("Missing name. Supply a name or use --all to flush the whole cache.") + } } } diff --git a/src/admin/query/room_timeline.rs b/src/admin/query/room_timeline.rs index 2ca46944e..2b2a9b8d4 100644 --- a/src/admin/query/room_timeline.rs +++ b/src/admin/query/room_timeline.rs @@ -3,7 +3,7 @@ use conduwuit::{PduCount, Result, utils::stream::TryTools}; use futures::TryStreamExt; use ruma::OwnedRoomOrAliasId; -use crate::{admin_command, admin_command_dispatch}; +use crate::admin_command_dispatch; #[admin_command_dispatch] #[derive(Debug, Subcommand)] @@ -23,39 +23,39 @@ pub enum RoomTimelineCommand { }, } -#[admin_command] -pub(super) async fn last(&self, room_id: OwnedRoomOrAliasId) -> Result { - let room_id = self.services.rooms.alias.resolve(&room_id).await?; +impl crate::Context<'_> { + pub(super) async fn last(&self, room_id: OwnedRoomOrAliasId) -> Result { + let room_id = self.services.rooms.alias.resolve(&room_id).await?; - let result = self - .services - .rooms - .timeline - .last_timeline_count(&room_id) - .await?; + let result = self + .services + .rooms + .timeline + .last_timeline_count(&room_id) + .await?; - self.write_str(&format!("{result:#?}")).await -} + self.write_str(&format!("{result:#?}")).await + } -#[admin_command] -pub(super) async fn pdus( - &self, - room_id: OwnedRoomOrAliasId, - from: Option, - limit: Option, -) -> Result { - let room_id = self.services.rooms.alias.resolve(&room_id).await?; + pub(super) async fn pdus( + &self, + room_id: OwnedRoomOrAliasId, + from: Option, + limit: Option, + ) -> Result { + let room_id = self.services.rooms.alias.resolve(&room_id).await?; - let from: Option = from.as_deref().map(str::parse).transpose()?; + let from: Option = from.as_deref().map(str::parse).transpose()?; - let result: Vec<_> = self - .services - .rooms - .timeline - .pdus_rev(&room_id, from) - .try_take(limit.unwrap_or(3)) - .try_collect() - .await?; + let result: Vec<_> = self + .services + .rooms + .timeline + .pdus_rev(&room_id, from) + .try_take(limit.unwrap_or(3)) + .try_collect() + .await?; - self.write_str(&format!("```\n{result:#?}\n```")).await + self.write_str(&format!("```\n{result:#?}\n```")).await + } } diff --git a/src/admin/query/short.rs b/src/admin/query/short.rs index 3ebfbbcf8..6bcd2d4c0 100644 --- a/src/admin/query/short.rs +++ b/src/admin/query/short.rs @@ -2,7 +2,7 @@ use clap::Subcommand; use conduwuit::Result; use ruma::{OwnedEventId, OwnedRoomOrAliasId}; -use crate::{admin_command, admin_command_dispatch}; +use crate::admin_command_dispatch; #[admin_command_dispatch] #[derive(Debug, Subcommand)] @@ -17,23 +17,23 @@ pub enum ShortCommand { }, } -#[admin_command] -pub(super) async fn short_event_id(&self, event_id: OwnedEventId) -> Result { - let shortid = self - .services - .rooms - .short - .get_shorteventid(&event_id) - .await?; +impl crate::Context<'_> { + pub(super) async fn short_event_id(&self, event_id: OwnedEventId) -> Result { + let shortid = self + .services + .rooms + .short + .get_shorteventid(&event_id) + .await?; - self.write_str(&format!("{shortid:#?}")).await -} + self.write_str(&format!("{shortid:#?}")).await + } -#[admin_command] -pub(super) async fn short_room_id(&self, room_id: OwnedRoomOrAliasId) -> Result { - let room_id = self.services.rooms.alias.resolve(&room_id).await?; + pub(super) async fn short_room_id(&self, room_id: OwnedRoomOrAliasId) -> Result { + let room_id = self.services.rooms.alias.resolve(&room_id).await?; - let shortid = self.services.rooms.short.get_shortroomid(&room_id).await?; + let shortid = self.services.rooms.short.get_shortroomid(&room_id).await?; - self.write_str(&format!("{shortid:#?}")).await + self.write_str(&format!("{shortid:#?}")).await + } } diff --git a/src/admin/query/users.rs b/src/admin/query/users.rs index dd49415f7..13fe9a888 100644 --- a/src/admin/query/users.rs +++ b/src/admin/query/users.rs @@ -3,7 +3,7 @@ use conduwuit::Result; use futures::stream::StreamExt; use ruma::{OwnedDeviceId, OwnedRoomId, OwnedUserId}; -use crate::{admin_command, admin_command_dispatch}; +use crate::admin_command_dispatch; #[admin_command_dispatch] #[derive(Debug, Subcommand)] @@ -92,263 +92,259 @@ pub enum UsersCommand { }, } -#[admin_command] -async fn get_shared_rooms(&self, user_a: OwnedUserId, user_b: OwnedUserId) -> Result { - let timer = tokio::time::Instant::now(); - let result: Vec<_> = self - .services - .rooms - .state_cache - .get_shared_rooms(&user_a, &user_b) - .collect() - .await; - let query_time = timer.elapsed(); +impl crate::Context<'_> { + async fn get_shared_rooms(&self, user_a: OwnedUserId, user_b: OwnedUserId) -> Result { + let timer = tokio::time::Instant::now(); + let result: Vec<_> = self + .services + .rooms + .state_cache + .get_shared_rooms(&user_a, &user_b) + .collect() + .await; + let query_time = timer.elapsed(); - self.write_str(&format!("Query completed in {query_time:?}:\n\n```rs\n{result:#?}\n```")) - .await -} - -#[admin_command] -async fn get_backup_session( - &self, - user_id: OwnedUserId, - version: String, - room_id: OwnedRoomId, - session_id: String, -) -> Result { - let timer = tokio::time::Instant::now(); - let result = self - .services - .key_backups - .get_session(&user_id, &version, &room_id, &session_id) - .await; - let query_time = timer.elapsed(); - - self.write_str(&format!("Query completed in {query_time:?}:\n\n```rs\n{result:#?}\n```")) - .await -} - -#[admin_command] -async fn get_room_backups( - &self, - user_id: OwnedUserId, - version: String, - room_id: OwnedRoomId, -) -> Result { - let timer = tokio::time::Instant::now(); - let result = self - .services - .key_backups - .get_room(&user_id, &version, &room_id) - .await; - let query_time = timer.elapsed(); - - self.write_str(&format!("Query completed in {query_time:?}:\n\n```rs\n{result:#?}\n```")) - .await -} - -#[admin_command] -async fn get_all_backups(&self, user_id: OwnedUserId, version: String) -> Result { - let timer = tokio::time::Instant::now(); - let result = self.services.key_backups.get_all(&user_id, &version).await; - let query_time = timer.elapsed(); - - self.write_str(&format!("Query completed in {query_time:?}:\n\n```rs\n{result:#?}\n```")) - .await -} - -#[admin_command] -async fn get_backup_algorithm(&self, user_id: OwnedUserId, version: String) -> Result { - let timer = tokio::time::Instant::now(); - let result = self - .services - .key_backups - .get_backup(&user_id, &version) - .await; - let query_time = timer.elapsed(); - - self.write_str(&format!("Query completed in {query_time:?}:\n\n```rs\n{result:#?}\n```")) - .await -} - -#[admin_command] -async fn get_latest_backup_version(&self, user_id: OwnedUserId) -> Result { - let timer = tokio::time::Instant::now(); - let result = self - .services - .key_backups - .get_latest_backup_version(&user_id) - .await; - let query_time = timer.elapsed(); - - self.write_str(&format!("Query completed in {query_time:?}:\n\n```rs\n{result:#?}\n```")) - .await -} - -#[admin_command] -async fn get_latest_backup(&self, user_id: OwnedUserId) -> Result { - let timer = tokio::time::Instant::now(); - let result = self.services.key_backups.get_latest_backup(&user_id).await; - let query_time = timer.elapsed(); - - self.write_str(&format!("Query completed in {query_time:?}:\n\n```rs\n{result:#?}\n```")) - .await -} - -#[admin_command] -async fn iter_users(&self) -> Result { - let timer = tokio::time::Instant::now(); - let result: Vec = self.services.users.stream().map(Into::into).collect().await; - - let query_time = timer.elapsed(); - - self.write_str(&format!("Query completed in {query_time:?}:\n\n```rs\n{result:#?}\n```")) - .await -} - -#[admin_command] -async fn iter_users2(&self) -> Result { - let timer = tokio::time::Instant::now(); - let result: Vec<_> = self.services.users.stream().collect().await; - let result: Vec<_> = result - .into_iter() - .map(|user_id| String::from_utf8_lossy(user_id.as_bytes()).into_owned()) - .collect(); - - let query_time = timer.elapsed(); - - self.write_str(&format!("Query completed in {query_time:?}:\n\n```rs\n{result:?}\n```")) - .await -} - -#[admin_command] -async fn count_users(&self) -> Result { - let timer = tokio::time::Instant::now(); - let result = self.services.users.count().await; - let query_time = timer.elapsed(); - - self.write_str(&format!("Query completed in {query_time:?}:\n\n```rs\n{result:#?}\n```")) - .await -} - -#[admin_command] -async fn list_devices(&self, user_id: OwnedUserId) -> Result { - let timer = tokio::time::Instant::now(); - let devices = self - .services - .users - .all_device_ids(&user_id) - .collect::>() - .await; - - let query_time = timer.elapsed(); - - self.write_str(&format!("Query completed in {query_time:?}:\n\n```rs\n{devices:#?}\n```")) - .await -} - -#[admin_command] -async fn list_devices_metadata(&self, user_id: OwnedUserId) -> Result { - let timer = tokio::time::Instant::now(); - let devices = self - .services - .users - .all_devices_metadata(&user_id) - .collect::>() - .await; - let query_time = timer.elapsed(); - - self.write_str(&format!("Query completed in {query_time:?}:\n\n```rs\n{devices:#?}\n```")) - .await -} - -#[admin_command] -async fn get_device_metadata(&self, user_id: OwnedUserId, device_id: OwnedDeviceId) -> Result { - let timer = tokio::time::Instant::now(); - let device = self - .services - .users - .get_device_metadata(&user_id, &device_id) - .await; - let query_time = timer.elapsed(); - - self.write_str(&format!("Query completed in {query_time:?}:\n\n```rs\n{device:#?}\n```")) - .await -} - -#[admin_command] -async fn get_devices_version(&self, user_id: OwnedUserId) -> Result { - let timer = tokio::time::Instant::now(); - let device = self.services.users.get_devicelist_version(&user_id).await; - let query_time = timer.elapsed(); - - self.write_str(&format!("Query completed in {query_time:?}:\n\n```rs\n{device:#?}\n```")) - .await -} - -#[admin_command] -async fn count_one_time_keys(&self, user_id: OwnedUserId, device_id: OwnedDeviceId) -> Result { - let timer = tokio::time::Instant::now(); - let result = self - .services - .users - .count_one_time_keys(&user_id, &device_id) - .await; - let query_time = timer.elapsed(); - - self.write_str(&format!("Query completed in {query_time:?}:\n\n```rs\n{result:#?}\n```")) - .await -} - -#[admin_command] -async fn get_device_keys(&self, user_id: OwnedUserId, device_id: OwnedDeviceId) -> Result { - let timer = tokio::time::Instant::now(); - let result = self - .services - .users - .get_device_keys(&user_id, &device_id) - .await; - let query_time = timer.elapsed(); - - self.write_str(&format!("Query completed in {query_time:?}:\n\n```rs\n{result:#?}\n```")) - .await -} - -#[admin_command] -async fn get_user_signing_key(&self, user_id: OwnedUserId) -> Result { - let timer = tokio::time::Instant::now(); - let result = self.services.users.get_user_signing_key(&user_id).await; - let query_time = timer.elapsed(); - - self.write_str(&format!("Query completed in {query_time:?}:\n\n```rs\n{result:#?}\n```")) - .await -} - -#[admin_command] -async fn get_master_key(&self, user_id: OwnedUserId) -> Result { - let timer = tokio::time::Instant::now(); - let result = self - .services - .users - .get_master_key(None, &user_id, &|_| true) - .await; - let query_time = timer.elapsed(); - - self.write_str(&format!("Query completed in {query_time:?}:\n\n```rs\n{result:#?}\n```")) - .await -} - -#[admin_command] -async fn get_to_device_events(&self, user_id: OwnedUserId, device_id: OwnedDeviceId) -> Result { - let timer = tokio::time::Instant::now(); - let result = self - .services - .users - .get_to_device_events(&user_id, &device_id, None, None) - .collect::>() - .await; - let query_time = timer.elapsed(); - - self.write_str(&format!("Query completed in {query_time:?}:\n\n```rs\n{result:#?}\n```")) - .await + self.write_str(&format!("Query completed in {query_time:?}:\n\n```rs\n{result:#?}\n```")) + .await + } + + async fn get_backup_session( + &self, + user_id: OwnedUserId, + version: String, + room_id: OwnedRoomId, + session_id: String, + ) -> Result { + let timer = tokio::time::Instant::now(); + let result = self + .services + .key_backups + .get_session(&user_id, &version, &room_id, &session_id) + .await; + let query_time = timer.elapsed(); + + self.write_str(&format!("Query completed in {query_time:?}:\n\n```rs\n{result:#?}\n```")) + .await + } + + async fn get_room_backups( + &self, + user_id: OwnedUserId, + version: String, + room_id: OwnedRoomId, + ) -> Result { + let timer = tokio::time::Instant::now(); + let result = self + .services + .key_backups + .get_room(&user_id, &version, &room_id) + .await; + let query_time = timer.elapsed(); + + self.write_str(&format!("Query completed in {query_time:?}:\n\n```rs\n{result:#?}\n```")) + .await + } + + async fn get_all_backups(&self, user_id: OwnedUserId, version: String) -> Result { + let timer = tokio::time::Instant::now(); + let result = self.services.key_backups.get_all(&user_id, &version).await; + let query_time = timer.elapsed(); + + self.write_str(&format!("Query completed in {query_time:?}:\n\n```rs\n{result:#?}\n```")) + .await + } + + async fn get_backup_algorithm(&self, user_id: OwnedUserId, version: String) -> Result { + let timer = tokio::time::Instant::now(); + let result = self + .services + .key_backups + .get_backup(&user_id, &version) + .await; + let query_time = timer.elapsed(); + + self.write_str(&format!("Query completed in {query_time:?}:\n\n```rs\n{result:#?}\n```")) + .await + } + + async fn get_latest_backup_version(&self, user_id: OwnedUserId) -> Result { + let timer = tokio::time::Instant::now(); + let result = self + .services + .key_backups + .get_latest_backup_version(&user_id) + .await; + let query_time = timer.elapsed(); + + self.write_str(&format!("Query completed in {query_time:?}:\n\n```rs\n{result:#?}\n```")) + .await + } + + async fn get_latest_backup(&self, user_id: OwnedUserId) -> Result { + let timer = tokio::time::Instant::now(); + let result = self.services.key_backups.get_latest_backup(&user_id).await; + let query_time = timer.elapsed(); + + self.write_str(&format!("Query completed in {query_time:?}:\n\n```rs\n{result:#?}\n```")) + .await + } + + async fn iter_users(&self) -> Result { + let timer = tokio::time::Instant::now(); + let result: Vec = + self.services.users.stream().map(Into::into).collect().await; + + let query_time = timer.elapsed(); + + self.write_str(&format!("Query completed in {query_time:?}:\n\n```rs\n{result:#?}\n```")) + .await + } + + async fn iter_users2(&self) -> Result { + let timer = tokio::time::Instant::now(); + let result: Vec<_> = self.services.users.stream().collect().await; + let result: Vec<_> = result + .into_iter() + .map(|user_id| String::from_utf8_lossy(user_id.as_bytes()).into_owned()) + .collect(); + + let query_time = timer.elapsed(); + + self.write_str(&format!("Query completed in {query_time:?}:\n\n```rs\n{result:?}\n```")) + .await + } + + async fn count_users(&self) -> Result { + let timer = tokio::time::Instant::now(); + let result = self.services.users.count().await; + let query_time = timer.elapsed(); + + self.write_str(&format!("Query completed in {query_time:?}:\n\n```rs\n{result:#?}\n```")) + .await + } + + async fn list_devices(&self, user_id: OwnedUserId) -> Result { + let timer = tokio::time::Instant::now(); + let devices = self + .services + .users + .all_device_ids(&user_id) + .collect::>() + .await; + + let query_time = timer.elapsed(); + + self.write_str(&format!("Query completed in {query_time:?}:\n\n```rs\n{devices:#?}\n```")) + .await + } + + async fn list_devices_metadata(&self, user_id: OwnedUserId) -> Result { + let timer = tokio::time::Instant::now(); + let devices = self + .services + .users + .all_devices_metadata(&user_id) + .collect::>() + .await; + let query_time = timer.elapsed(); + + self.write_str(&format!("Query completed in {query_time:?}:\n\n```rs\n{devices:#?}\n```")) + .await + } + + async fn get_device_metadata( + &self, + user_id: OwnedUserId, + device_id: OwnedDeviceId, + ) -> Result { + let timer = tokio::time::Instant::now(); + let device = self + .services + .users + .get_device_metadata(&user_id, &device_id) + .await; + let query_time = timer.elapsed(); + + self.write_str(&format!("Query completed in {query_time:?}:\n\n```rs\n{device:#?}\n```")) + .await + } + + async fn get_devices_version(&self, user_id: OwnedUserId) -> Result { + let timer = tokio::time::Instant::now(); + let device = self.services.users.get_devicelist_version(&user_id).await; + let query_time = timer.elapsed(); + + self.write_str(&format!("Query completed in {query_time:?}:\n\n```rs\n{device:#?}\n```")) + .await + } + + async fn count_one_time_keys( + &self, + user_id: OwnedUserId, + device_id: OwnedDeviceId, + ) -> Result { + let timer = tokio::time::Instant::now(); + let result = self + .services + .users + .count_one_time_keys(&user_id, &device_id) + .await; + let query_time = timer.elapsed(); + + self.write_str(&format!("Query completed in {query_time:?}:\n\n```rs\n{result:#?}\n```")) + .await + } + + async fn get_device_keys(&self, user_id: OwnedUserId, device_id: OwnedDeviceId) -> Result { + let timer = tokio::time::Instant::now(); + let result = self + .services + .users + .get_device_keys(&user_id, &device_id) + .await; + let query_time = timer.elapsed(); + + self.write_str(&format!("Query completed in {query_time:?}:\n\n```rs\n{result:#?}\n```")) + .await + } + + async fn get_user_signing_key(&self, user_id: OwnedUserId) -> Result { + let timer = tokio::time::Instant::now(); + let result = self.services.users.get_user_signing_key(&user_id).await; + let query_time = timer.elapsed(); + + self.write_str(&format!("Query completed in {query_time:?}:\n\n```rs\n{result:#?}\n```")) + .await + } + + async fn get_master_key(&self, user_id: OwnedUserId) -> Result { + let timer = tokio::time::Instant::now(); + let result = self + .services + .users + .get_master_key(None, &user_id, &|_| true) + .await; + let query_time = timer.elapsed(); + + self.write_str(&format!("Query completed in {query_time:?}:\n\n```rs\n{result:#?}\n```")) + .await + } + + async fn get_to_device_events( + &self, + user_id: OwnedUserId, + device_id: OwnedDeviceId, + ) -> Result { + let timer = tokio::time::Instant::now(); + let result = self + .services + .users + .get_to_device_events(&user_id, &device_id, None, None) + .collect::>() + .await; + let query_time = timer.elapsed(); + + self.write_str(&format!("Query completed in {query_time:?}:\n\n```rs\n{result:#?}\n```")) + .await + } } diff --git a/src/admin/room/commands.rs b/src/admin/room/commands.rs index 06061311c..0cee2ebcc 100644 --- a/src/admin/room/commands.rs +++ b/src/admin/room/commands.rs @@ -2,83 +2,83 @@ use conduwuit::{Err, Result}; use futures::StreamExt; use ruma::OwnedRoomId; -use crate::{PAGE_SIZE, admin_command, get_room_info}; +use crate::{PAGE_SIZE, get_room_info}; -#[allow(clippy::fn_params_excessive_bools)] -#[admin_command] -pub(super) async fn list_rooms( - &self, - page: Option, - exclude_disabled: bool, - exclude_banned: bool, - include_empty: bool, - no_details: bool, -) -> Result { - // TODO: i know there's a way to do this with clap, but i can't seem to find it - let page = page.unwrap_or(1); - let mut rooms = self - .services - .rooms - .metadata - .iter_ids() - .filter_map(|room_id| async move { - (!exclude_disabled || !self.services.rooms.metadata.is_disabled(&room_id).await) - .then_some(room_id) - }) - .filter_map(|room_id| async move { - (!exclude_banned || !self.services.rooms.metadata.is_banned(&room_id).await) - .then_some(room_id) - }) - .then(async |room_id| get_room_info(self.services, &room_id).await) - .then(|(room_id, total_members, name)| async move { - let local_members: Vec<_> = self - .services - .rooms - .state_cache - .active_local_users_in_room(&room_id) - .collect() - .await; - let local_members = local_members.len(); - (room_id, total_members, local_members, name) - }) - .filter_map(|(room_id, total_members, local_members, name)| async move { - (include_empty || local_members > 0).then_some((room_id, total_members, name)) - }) - .collect::>() - .await; +impl crate::Context<'_> { + #[allow(clippy::fn_params_excessive_bools)] + pub(super) async fn list_rooms( + &self, + page: Option, + exclude_disabled: bool, + exclude_banned: bool, + include_empty: bool, + no_details: bool, + ) -> Result { + // TODO: i know there's a way to do this with clap, but i can't seem to find it + let page = page.unwrap_or(1); + let mut rooms = self + .services + .rooms + .metadata + .iter_ids() + .filter_map(|room_id| async move { + (!exclude_disabled || !self.services.rooms.metadata.is_disabled(&room_id).await) + .then_some(room_id) + }) + .filter_map(|room_id| async move { + (!exclude_banned || !self.services.rooms.metadata.is_banned(&room_id).await) + .then_some(room_id) + }) + .then(async |room_id| get_room_info(self.services, &room_id).await) + .then(|(room_id, total_members, name)| async move { + let local_members: Vec<_> = self + .services + .rooms + .state_cache + .active_local_users_in_room(&room_id) + .collect() + .await; + let local_members = local_members.len(); + (room_id, total_members, local_members, name) + }) + .filter_map(|(room_id, total_members, local_members, name)| async move { + (include_empty || local_members > 0).then_some((room_id, total_members, name)) + }) + .collect::>() + .await; - rooms.sort_by_key(|r| r.1); - rooms.reverse(); + rooms.sort_by_key(|r| r.1); + rooms.reverse(); - let rooms = rooms - .into_iter() - .skip(page.saturating_sub(1).saturating_mul(PAGE_SIZE)) - .take(PAGE_SIZE) - .collect::>(); + let rooms = rooms + .into_iter() + .skip(page.saturating_sub(1).saturating_mul(PAGE_SIZE)) + .take(PAGE_SIZE) + .collect::>(); - if rooms.is_empty() { - return Err!("No more rooms."); + if rooms.is_empty() { + return Err!("No more rooms."); + } + + let body = rooms + .iter() + .map(|(id, members, name)| { + if no_details { + format!("{id}") + } else { + format!("{id}\tMembers: {members}\tName: {name}") + } + }) + .collect::>() + .join("\n"); + + self.write_str(&format!("Rooms ({}):\n```\n{body}\n```", rooms.len())) + .await } - let body = rooms - .iter() - .map(|(id, members, name)| { - if no_details { - format!("{id}") - } else { - format!("{id}\tMembers: {members}\tName: {name}") - } - }) - .collect::>() - .join("\n"); + pub(super) async fn exists(&self, room_id: OwnedRoomId) -> Result { + let result = self.services.rooms.metadata.exists(&room_id).await; - self.write_str(&format!("Rooms ({}):\n```\n{body}\n```", rooms.len())) - .await -} - -#[admin_command] -pub(super) async fn exists(&self, room_id: OwnedRoomId) -> Result { - let result = self.services.rooms.metadata.exists(&room_id).await; - - self.write_str(&format!("{result}")).await + self.write_str(&format!("{result}")).await + } } diff --git a/src/admin/room/info.rs b/src/admin/room/info.rs index cfb7a8a83..a035027c7 100644 --- a/src/admin/room/info.rs +++ b/src/admin/room/info.rs @@ -3,7 +3,7 @@ use conduwuit::{Err, Result, utils::ReadyExt}; use futures::StreamExt; use ruma::OwnedRoomId; -use crate::{admin_command, admin_command_dispatch}; +use crate::admin_command_dispatch; #[admin_command_dispatch] #[derive(Debug, Subcommand)] @@ -26,62 +26,62 @@ pub enum RoomInfoCommand { }, } -#[admin_command] -async fn list_joined_members(&self, room_id: OwnedRoomId, local_only: bool) -> Result { - let room_name = self - .services - .rooms - .state_accessor - .get_name(&room_id) - .await - .unwrap_or_else(|_| room_id.to_string()); +impl crate::Context<'_> { + async fn list_joined_members(&self, room_id: OwnedRoomId, local_only: bool) -> Result { + let room_name = self + .services + .rooms + .state_accessor + .get_name(&room_id) + .await + .unwrap_or_else(|_| room_id.to_string()); - let member_info: Vec<_> = self - .services - .rooms - .state_cache - .room_members(&room_id) - .ready_filter(|user_id| { - local_only - .then(|| self.services.globals.user_is_local(user_id)) - .unwrap_or(true) - }) - .filter_map(|user_id| async move { - Some(( - self.services - .users - .displayname(&user_id) - .await - .unwrap_or_else(|_| user_id.to_string()), - user_id, - )) - }) - .collect() - .await; + let member_info: Vec<_> = self + .services + .rooms + .state_cache + .room_members(&room_id) + .ready_filter(|user_id| { + local_only + .then(|| self.services.globals.user_is_local(user_id)) + .unwrap_or(true) + }) + .filter_map(|user_id| async move { + Some(( + self.services + .users + .displayname(&user_id) + .await + .unwrap_or_else(|_| user_id.to_string()), + user_id, + )) + }) + .collect() + .await; - let num = member_info.len(); - let body = member_info - .into_iter() - .map(|(displayname, mxid)| format!("{mxid} | {displayname}")) - .collect::>() - .join("\n"); + let num = member_info.len(); + let body = member_info + .into_iter() + .map(|(displayname, mxid)| format!("{mxid} | {displayname}")) + .collect::>() + .join("\n"); - self.write_str(&format!("{num} Members in Room \"{room_name}\":\n```\n{body}\n```")) - .await -} + self.write_str(&format!("{num} Members in Room \"{room_name}\":\n```\n{body}\n```")) + .await + } -#[admin_command] -async fn view_room_topic(&self, room_id: OwnedRoomId) -> Result { - let Ok(room_topic) = self - .services - .rooms - .state_accessor - .get_room_topic(&room_id) - .await - else { - return Err!("Room does not have a room topic set."); - }; + async fn view_room_topic(&self, room_id: OwnedRoomId) -> Result { + let Ok(room_topic) = self + .services + .rooms + .state_accessor + .get_room_topic(&room_id) + .await + else { + return Err!("Room does not have a room topic set."); + }; - self.write_str(&format!("Room topic:\n```\n{room_topic}\n```")) - .await + self.write_str(&format!("Room topic:\n```\n{room_topic}\n```")) + .await + } } diff --git a/src/admin/room/moderation.rs b/src/admin/room/moderation.rs index 3ae9d978e..7b6758c01 100644 --- a/src/admin/room/moderation.rs +++ b/src/admin/room/moderation.rs @@ -8,7 +8,7 @@ use conduwuit::{ use futures::{FutureExt, StreamExt}; use ruma::{OwnedRoomId, OwnedRoomOrAliasId, RoomAliasId, RoomId, RoomOrAliasId}; -use crate::{admin_command, admin_command_dispatch, get_room_info}; +use crate::{admin_command_dispatch, get_room_info}; #[admin_command_dispatch] #[derive(Debug, Subcommand)] @@ -45,238 +45,72 @@ pub enum RoomModerationCommand { }, } -#[admin_command] -async fn ban_room(&self, room: OwnedRoomOrAliasId) -> Result { - debug!("Got room alias or ID: {}", room); +impl crate::Context<'_> { + async fn ban_room(&self, room: OwnedRoomOrAliasId) -> Result { + debug!("Got room alias or ID: {}", room); - let admin_room_alias = &self.services.globals.admin_alias; + let admin_room_alias = &self.services.globals.admin_alias; - if let Ok(admin_room_id) = self.services.admin.get_admin_room().await { - if room.to_string().eq(&admin_room_id) || room.to_string().eq(admin_room_alias) { - return Err!("Not allowed to ban the admin room."); + if let Ok(admin_room_id) = self.services.admin.get_admin_room().await { + if room.to_string().eq(&admin_room_id) || room.to_string().eq(admin_room_alias) { + return Err!("Not allowed to ban the admin room."); + } } - } - let room_id = if room.is_room_id() { - let room_id = match RoomId::parse(&room) { - | Ok(room_id) => room_id, - | Err(e) => { - return Err!( - "Failed to parse room ID {room}. Please note that this requires a full room \ - ID (`!awIh6gGInaS5wLQJwa:example.com`) or a room alias \ - (`#roomalias:example.com`): {e}" - ); - }, + let room_id = if room.is_room_id() { + let room_id = match RoomId::parse(&room) { + | Ok(room_id) => room_id, + | Err(e) => { + return Err!( + "Failed to parse room ID {room}. Please note that this requires a full \ + room ID (`!awIh6gGInaS5wLQJwa:example.com`) or a room alias \ + (`#roomalias:example.com`): {e}" + ); + }, + }; + + debug!("Room specified is a room ID, banning room ID"); + + room_id.clone() + } else if room.is_room_alias_id() { + let room_alias = match RoomAliasId::parse(&room) { + | Ok(room_alias) => room_alias, + | Err(e) => { + return Err!( + "Failed to parse room ID {room}. Please note that this requires a full \ + room ID (`!awIh6gGInaS5wLQJwa:example.com`) or a room alias \ + (`#roomalias:example.com`): {e}" + ); + }, + }; + + debug!( + "Room specified is not a room ID, attempting to resolve room alias to a room ID \ + locally, if not using get_alias_helper to fetch room ID remotely" + ); + + match self.services.rooms.alias.resolve_alias(&room_alias).await { + | Ok((room_id, servers)) => { + debug!( + %room_id, + ?servers, + "Got federation response fetching room ID for room {room}" + ); + room_id + }, + | Err(e) => { + return Err!("Failed to resolve room alias {room} to a room ID: {e}"); + }, + } + } else { + return Err!( + "Room specified is not a room ID or room alias. Please note that this requires \ + a full room ID (`!awIh6gGInaS5wLQJwa:example.com`) or a room alias \ + (`#roomalias:example.com`)", + ); }; - debug!("Room specified is a room ID, banning room ID"); - - room_id.clone() - } else if room.is_room_alias_id() { - let room_alias = match RoomAliasId::parse(&room) { - | Ok(room_alias) => room_alias, - | Err(e) => { - return Err!( - "Failed to parse room ID {room}. Please note that this requires a full room \ - ID (`!awIh6gGInaS5wLQJwa:example.com`) or a room alias \ - (`#roomalias:example.com`): {e}" - ); - }, - }; - - debug!( - "Room specified is not a room ID, attempting to resolve room alias to a room ID \ - locally, if not using get_alias_helper to fetch room ID remotely" - ); - - match self.services.rooms.alias.resolve_alias(&room_alias).await { - | Ok((room_id, servers)) => { - debug!( - %room_id, - ?servers, - "Got federation response fetching room ID for room {room}" - ); - room_id - }, - | Err(e) => { - return Err!("Failed to resolve room alias {room} to a room ID: {e}"); - }, - } - } else { - return Err!( - "Room specified is not a room ID or room alias. Please note that this requires a \ - full room ID (`!awIh6gGInaS5wLQJwa:example.com`) or a room alias \ - (`#roomalias:example.com`)", - ); - }; - - info!("Making all users leave the room {room_id} and forgetting it"); - let mut users = self - .services - .rooms - .state_cache - .room_members(&room_id) - .ready_filter(|user| self.services.globals.user_is_local(user)) - .boxed(); - - while let Some(ref user_id) = users.next().await { - info!( - "Attempting leave for user {user_id} in room {room_id} (ignoring all errors, \ - evicting admins too)", - ); - - if let Err(e) = leave_room(self.services, user_id, &room_id, None) - .boxed() - .await - { - warn!("Failed to leave room: {e}"); - } - - self.services.rooms.state_cache.forget(&room_id, user_id); - } - - self.services - .rooms - .alias - .local_aliases_for_room(&room_id) - .for_each(|local_alias| async move { - self.services - .rooms - .alias - .remove_alias(&local_alias, &self.services.globals.server_user) - .await - .ok(); - }) - .await; - - self.services.rooms.directory.set_not_public(&room_id); // remove from the room directory - self.services.rooms.metadata.ban_room(&room_id, true); // prevent further joins - self.services.rooms.metadata.disable_room(&room_id, true); // disable federation - - self.write_str( - "Room banned, removed all our local users, and disabled incoming federation with room.", - ) - .await -} - -#[admin_command] -async fn ban_list_of_rooms(&self) -> Result { - if self.body.len() < 2 - || !self.body[0].trim().starts_with("```") - || self.body.last().unwrap_or(&"").trim() != "```" - { - return Err!("Expected code block in command body. Add --help for details.",); - } - - let rooms_s = self - .body - .to_vec() - .drain(1..self.body.len().saturating_sub(1)) - .collect::>(); - - let admin_room_alias = &self.services.globals.admin_alias; - - let mut room_ban_count: usize = 0; - let mut room_ids: Vec = Vec::new(); - - for &room in &rooms_s { - match <&RoomOrAliasId>::try_from(room) { - | Ok(room_alias_or_id) => { - if let Ok(admin_room_id) = self.services.admin.get_admin_room().await { - if room.to_owned().eq(&admin_room_id) || room.to_owned().eq(admin_room_alias) - { - warn!("User specified admin room in bulk ban list, ignoring"); - continue; - } - } - - if room_alias_or_id.is_room_id() { - let room_id = match RoomId::parse(room_alias_or_id) { - | Ok(room_id) => room_id, - | Err(e) => { - // ignore rooms we failed to parse - warn!( - "Error parsing room \"{room}\" during bulk room banning, \ - ignoring error and logging here: {e}" - ); - continue; - }, - }; - - room_ids.push(room_id.clone()); - } - - if room_alias_or_id.is_room_alias_id() { - match RoomAliasId::parse(room_alias_or_id) { - | Ok(room_alias) => { - let room_id = match self - .services - .rooms - .alias - .resolve_local_alias(&room_alias) - .await - { - | Ok(room_id) => room_id, - | _ => { - debug!( - "We don't have this room alias to a room ID locally, \ - attempting to fetch room ID over federation" - ); - - match self - .services - .rooms - .alias - .resolve_alias(&room_alias) - .await - { - | Ok((room_id, servers)) => { - debug!( - %room_id, - ?servers, - "Got federation response fetching room ID for \ - {room}", - ); - room_id - }, - | Err(e) => { - warn!( - "Failed to resolve room alias {room} to a room \ - ID: {e}" - ); - continue; - }, - } - }, - }; - - room_ids.push(room_id); - }, - | Err(e) => { - warn!( - "Error parsing room \"{room}\" during bulk room banning, \ - ignoring error and logging here: {e}" - ); - continue; - }, - } - } - }, - | Err(e) => { - warn!( - "Error parsing room \"{room}\" during bulk room banning, ignoring error and \ - logging here: {e}" - ); - continue; - }, - } - } - - for room_id in room_ids { - debug!("Banned {room_id} successfully"); - room_ban_count = room_ban_count.saturating_add(1); - - debug!("Making all users leave the room {room_id} and forgetting it"); + info!("Making all users leave the room {room_id} and forgetting it"); let mut users = self .services .rooms @@ -286,7 +120,7 @@ async fn ban_list_of_rooms(&self) -> Result { .boxed(); while let Some(ref user_id) = users.next().await { - debug!( + info!( "Attempting leave for user {user_id} in room {room_id} (ignoring all errors, \ evicting admins too)", ); @@ -301,7 +135,6 @@ async fn ban_list_of_rooms(&self) -> Result { self.services.rooms.state_cache.forget(&room_id, user_id); } - // remove any local aliases, ignore errors self.services .rooms .alias @@ -316,139 +149,307 @@ async fn ban_list_of_rooms(&self) -> Result { }) .await; - self.services.rooms.metadata.ban_room(&room_id, true); - // unpublish from room directory, ignore errors - self.services.rooms.directory.set_not_public(&room_id); - self.services.rooms.metadata.disable_room(&room_id, true); + self.services.rooms.directory.set_not_public(&room_id); // remove from the room directory + self.services.rooms.metadata.ban_room(&room_id, true); // prevent further joins + self.services.rooms.metadata.disable_room(&room_id, true); // disable federation + + self.write_str( + "Room banned, removed all our local users, and disabled incoming federation with \ + room.", + ) + .await } - self.write_str(&format!( - "Finished bulk room ban, banned {room_ban_count} total rooms, evicted all users, and \ - disabled incoming federation with the room." - )) - .await -} + async fn ban_list_of_rooms(&self) -> Result { + if self.body.len() < 2 + || !self.body[0].trim().starts_with("```") + || self.body.last().unwrap_or(&"").trim() != "```" + { + return Err!("Expected code block in command body. Add --help for details.",); + } -#[admin_command] -async fn unban_room(&self, room: OwnedRoomOrAliasId) -> Result { - let room_id = if room.is_room_id() { - let room_id = match RoomId::parse(&room) { - | Ok(room_id) => room_id, - | Err(e) => { - return Err!( - "Failed to parse room ID {room}. Please note that this requires a full room \ - ID (`!awIh6gGInaS5wLQJwa:example.com`) or a room alias \ - (`#roomalias:example.com`): {e}" + let rooms_s = self + .body + .to_vec() + .drain(1..self.body.len().saturating_sub(1)) + .collect::>(); + + let admin_room_alias = &self.services.globals.admin_alias; + + let mut room_ban_count: usize = 0; + let mut room_ids: Vec = Vec::new(); + + for &room in &rooms_s { + match <&RoomOrAliasId>::try_from(room) { + | Ok(room_alias_or_id) => { + if let Ok(admin_room_id) = self.services.admin.get_admin_room().await { + if room.to_owned().eq(&admin_room_id) + || room.to_owned().eq(admin_room_alias) + { + warn!("User specified admin room in bulk ban list, ignoring"); + continue; + } + } + + if room_alias_or_id.is_room_id() { + let room_id = match RoomId::parse(room_alias_or_id) { + | Ok(room_id) => room_id, + | Err(e) => { + // ignore rooms we failed to parse + warn!( + "Error parsing room \"{room}\" during bulk room banning, \ + ignoring error and logging here: {e}" + ); + continue; + }, + }; + + room_ids.push(room_id.clone()); + } + + if room_alias_or_id.is_room_alias_id() { + match RoomAliasId::parse(room_alias_or_id) { + | Ok(room_alias) => { + let room_id = match self + .services + .rooms + .alias + .resolve_local_alias(&room_alias) + .await + { + | Ok(room_id) => room_id, + | _ => { + debug!( + "We don't have this room alias to a room ID \ + locally, attempting to fetch room ID over \ + federation" + ); + + match self + .services + .rooms + .alias + .resolve_alias(&room_alias) + .await + { + | Ok((room_id, servers)) => { + debug!( + %room_id, + ?servers, + "Got federation response fetching room ID for \ + {room}", + ); + room_id + }, + | Err(e) => { + warn!( + "Failed to resolve room alias {room} to a \ + room ID: {e}" + ); + continue; + }, + } + }, + }; + + room_ids.push(room_id); + }, + | Err(e) => { + warn!( + "Error parsing room \"{room}\" during bulk room banning, \ + ignoring error and logging here: {e}" + ); + continue; + }, + } + } + }, + | Err(e) => { + warn!( + "Error parsing room \"{room}\" during bulk room banning, ignoring error \ + and logging here: {e}" + ); + continue; + }, + } + } + + for room_id in room_ids { + debug!("Banned {room_id} successfully"); + room_ban_count = room_ban_count.saturating_add(1); + + debug!("Making all users leave the room {room_id} and forgetting it"); + let mut users = self + .services + .rooms + .state_cache + .room_members(&room_id) + .ready_filter(|user| self.services.globals.user_is_local(user)) + .boxed(); + + while let Some(ref user_id) = users.next().await { + debug!( + "Attempting leave for user {user_id} in room {room_id} (ignoring all \ + errors, evicting admins too)", ); - }, + + if let Err(e) = leave_room(self.services, user_id, &room_id, None) + .boxed() + .await + { + warn!("Failed to leave room: {e}"); + } + + self.services.rooms.state_cache.forget(&room_id, user_id); + } + + // remove any local aliases, ignore errors + self.services + .rooms + .alias + .local_aliases_for_room(&room_id) + .for_each(|local_alias| async move { + self.services + .rooms + .alias + .remove_alias(&local_alias, &self.services.globals.server_user) + .await + .ok(); + }) + .await; + + self.services.rooms.metadata.ban_room(&room_id, true); + // unpublish from room directory, ignore errors + self.services.rooms.directory.set_not_public(&room_id); + self.services.rooms.metadata.disable_room(&room_id, true); + } + + self.write_str(&format!( + "Finished bulk room ban, banned {room_ban_count} total rooms, evicted all users, \ + and disabled incoming federation with the room." + )) + .await + } + + async fn unban_room(&self, room: OwnedRoomOrAliasId) -> Result { + let room_id = if room.is_room_id() { + let room_id = match RoomId::parse(&room) { + | Ok(room_id) => room_id, + | Err(e) => { + return Err!( + "Failed to parse room ID {room}. Please note that this requires a full \ + room ID (`!awIh6gGInaS5wLQJwa:example.com`) or a room alias \ + (`#roomalias:example.com`): {e}" + ); + }, + }; + + debug!("Room specified is a room ID, unbanning room ID"); + self.services.rooms.metadata.ban_room(&room_id, false); + + room_id.clone() + } else if room.is_room_alias_id() { + let room_alias = match RoomAliasId::parse(&room) { + | Ok(room_alias) => room_alias, + | Err(e) => { + return Err!( + "Failed to parse room ID {room}. Please note that this requires a full \ + room ID (`!awIh6gGInaS5wLQJwa:example.com`) or a room alias \ + (`#roomalias:example.com`): {e}" + ); + }, + }; + + debug!( + "Room specified is not a room ID, attempting to resolve room alias to a room ID \ + locally, if not using get_alias_helper to fetch room ID remotely" + ); + + let room_id = match self + .services + .rooms + .alias + .resolve_local_alias(&room_alias) + .await + { + | Ok(room_id) => room_id, + | _ => { + debug!( + "We don't have this room alias to a room ID locally, attempting to \ + fetch room ID over federation" + ); + + match self.services.rooms.alias.resolve_alias(&room_alias).await { + | Ok((room_id, servers)) => { + debug!( + %room_id, + ?servers, + "Got federation response fetching room ID for room {room}" + ); + room_id + }, + | Err(e) => { + return Err!("Failed to resolve room alias {room} to a room ID: {e}"); + }, + } + }, + }; + + self.services.rooms.metadata.ban_room(&room_id, false); + + room_id + } else { + return Err!( + "Room specified is not a room ID or room alias. Please note that this requires \ + a full room ID (`!awIh6gGInaS5wLQJwa:example.com`) or a room alias \ + (`#roomalias:example.com`)", + ); }; - debug!("Room specified is a room ID, unbanning room ID"); - self.services.rooms.metadata.ban_room(&room_id, false); + self.services.rooms.metadata.disable_room(&room_id, false); + self.write_str("Room unbanned and federation re-enabled.") + .await + } - room_id.clone() - } else if room.is_room_alias_id() { - let room_alias = match RoomAliasId::parse(&room) { - | Ok(room_alias) => room_alias, - | Err(e) => { - return Err!( - "Failed to parse room ID {room}. Please note that this requires a full room \ - ID (`!awIh6gGInaS5wLQJwa:example.com`) or a room alias \ - (`#roomalias:example.com`): {e}" - ); - }, - }; - - debug!( - "Room specified is not a room ID, attempting to resolve room alias to a room ID \ - locally, if not using get_alias_helper to fetch room ID remotely" - ); - - let room_id = match self + async fn list_banned_rooms(&self, no_details: bool) -> Result { + let room_ids: Vec = self .services .rooms - .alias - .resolve_local_alias(&room_alias) - .await - { - | Ok(room_id) => room_id, - | _ => { - debug!( - "We don't have this room alias to a room ID locally, attempting to fetch \ - room ID over federation" - ); + .metadata + .list_banned_rooms() + .map(Into::into) + .collect() + .await; - match self.services.rooms.alias.resolve_alias(&room_alias).await { - | Ok((room_id, servers)) => { - debug!( - %room_id, - ?servers, - "Got federation response fetching room ID for room {room}" - ); - room_id - }, - | Err(e) => { - return Err!("Failed to resolve room alias {room} to a room ID: {e}"); - }, + if room_ids.is_empty() { + return Err!("No rooms are banned."); + } + + let mut rooms = room_ids + .iter() + .stream() + .then(|room_id| get_room_info(self.services, room_id)) + .collect::>() + .await; + + rooms.sort_by_key(|r| r.1); + rooms.reverse(); + + let num = rooms.len(); + + let body = rooms + .iter() + .map(|(id, members, name)| { + if no_details { + format!("{id}") + } else { + format!("{id}\tMembers: {members}\tName: {name}") } - }, - }; + }) + .collect::>() + .join("\n"); - self.services.rooms.metadata.ban_room(&room_id, false); - - room_id - } else { - return Err!( - "Room specified is not a room ID or room alias. Please note that this requires a \ - full room ID (`!awIh6gGInaS5wLQJwa:example.com`) or a room alias \ - (`#roomalias:example.com`)", - ); - }; - - self.services.rooms.metadata.disable_room(&room_id, false); - self.write_str("Room unbanned and federation re-enabled.") - .await -} - -#[admin_command] -async fn list_banned_rooms(&self, no_details: bool) -> Result { - let room_ids: Vec = self - .services - .rooms - .metadata - .list_banned_rooms() - .map(Into::into) - .collect() - .await; - - if room_ids.is_empty() { - return Err!("No rooms are banned."); + self.write_str(&format!("Rooms Banned ({num}):\n```\n{body}\n```")) + .await } - - let mut rooms = room_ids - .iter() - .stream() - .then(|room_id| get_room_info(self.services, room_id)) - .collect::>() - .await; - - rooms.sort_by_key(|r| r.1); - rooms.reverse(); - - let num = rooms.len(); - - let body = rooms - .iter() - .map(|(id, members, name)| { - if no_details { - format!("{id}") - } else { - format!("{id}\tMembers: {members}\tName: {name}") - } - }) - .collect::>() - .join("\n"); - - self.write_str(&format!("Rooms Banned ({num}):\n```\n{body}\n```")) - .await } diff --git a/src/admin/server/commands.rs b/src/admin/server/commands.rs index 462c047d6..744dde43e 100644 --- a/src/admin/server/commands.rs +++ b/src/admin/server/commands.rs @@ -7,243 +7,230 @@ use conduwuit::{ }; use futures::TryStreamExt; -use crate::admin_command; +impl crate::Context<'_> { + pub(super) async fn uptime(&self) -> Result { + let elapsed = self + .services + .server + .started + .elapsed() + .expect("standard duration"); -#[admin_command] -pub(super) async fn uptime(&self) -> Result { - let elapsed = self - .services - .server - .started - .elapsed() - .expect("standard duration"); + let result = time::pretty(elapsed); + self.write_str(&format!("{result}.")).await + } - let result = time::pretty(elapsed); - self.write_str(&format!("{result}.")).await -} + pub(super) async fn show_config(&self) -> Result { + self.bail_restricted()?; -#[admin_command] -pub(super) async fn show_config(&self) -> Result { - self.bail_restricted()?; + self.write_str(&format!("{}", *self.services.server.config)) + .await + } - self.write_str(&format!("{}", *self.services.server.config)) - .await -} + pub(super) async fn reload_config(&self, path: Option) -> Result { + // The path argument is only what's optionally passed via the admin command, + // so we need to merge it with the existing paths if any were given at startup. + let mut paths = Vec::new(); -#[admin_command] -pub(super) async fn reload_config(&self, path: Option) -> Result { - // The path argument is only what's optionally passed via the admin command, - // so we need to merge it with the existing paths if any were given at startup. - let mut paths = Vec::new(); + // Add previously saved paths to the argument list + self.services + .config + .config_paths + .clone() + .unwrap_or_default() + .iter() + .for_each(|p| paths.push(p.to_owned())); - // Add previously saved paths to the argument list - self.services - .config - .config_paths - .clone() - .unwrap_or_default() - .iter() - .for_each(|p| paths.push(p.to_owned())); - - // If a path is given, and it's not already in the list, - // add it last, so that it overrides earlier files - if let Some(p) = path { - if !paths.contains(&p) { - paths.push(p); + // If a path is given, and it's not already in the list, + // add it last, so that it overrides earlier files + if let Some(p) = path { + if !paths.contains(&p) { + paths.push(p); + } } + + self.services.config.reload(&paths)?; + + self.write_str(&format!("Successfully reconfigured from paths: {paths:?}")) + .await } - self.services.config.reload(&paths)?; + pub(super) async fn memory_usage(&self) -> Result { + let services_usage = self.services.memory_usage().await?; + let database_usage = self.services.db.db.memory_usage()?; + let allocator_usage = conduwuit::alloc::memory_usage() + .map_or(String::new(), |s| format!("\nAllocator:\n{s}")); - self.write_str(&format!("Successfully reconfigured from paths: {paths:?}")) + self.write_str(&format!( + "Services:\n{services_usage}\nDatabase:\n{database_usage}{allocator_usage}", + )) .await -} - -#[admin_command] -pub(super) async fn memory_usage(&self) -> Result { - let services_usage = self.services.memory_usage().await?; - let database_usage = self.services.db.db.memory_usage()?; - let allocator_usage = - conduwuit::alloc::memory_usage().map_or(String::new(), |s| format!("\nAllocator:\n{s}")); - - self.write_str(&format!( - "Services:\n{services_usage}\nDatabase:\n{database_usage}{allocator_usage}", - )) - .await -} - -#[admin_command] -pub(super) async fn clear_caches(&self) -> Result { - self.services.clear_cache().await; - - self.write_str("Done.").await -} - -#[admin_command] -pub(super) async fn list_backups(&self) -> Result { - self.services - .db - .db - .backup_list()? - .try_stream() - .try_for_each(|result| writeln!(self, "{result}")) - .await -} - -#[admin_command] -pub(super) async fn backup_database(&self) -> Result { - self.bail_restricted()?; - - let db = Arc::clone(&self.services.db); - let result = self - .services - .server - .runtime() - .spawn_blocking(move || match db.db.backup() { - | Ok(()) => "Done".to_owned(), - | Err(e) => format!("Failed: {e}"), - }) - .await?; - - let count = self.services.db.db.backup_count()?; - self.write_str(&format!("{result}. Currently have {count} backups.")) - .await -} - -#[admin_command] -pub(super) async fn admin_notice(&self, message: Vec) -> Result { - let message = message.join(" "); - self.services.admin.send_text(&message).await; - - self.write_str("Notice was sent to #admins").await -} - -#[admin_command] -pub(super) async fn reload_mods(&self) -> Result { - self.bail_restricted()?; - - self.services.server.reload()?; - - self.write_str("Reloading server...").await -} - -#[admin_command] -#[cfg(unix)] -pub(super) async fn restart(&self, force: bool) -> Result { - use conduwuit::utils::sys::current_exe_deleted; - - if !force && current_exe_deleted() { - return Err!( - "The server cannot be restarted because the executable changed. If this is expected \ - use --force to override." - ); } - self.services.server.restart()?; + pub(super) async fn clear_caches(&self) -> Result { + self.services.clear_cache().await; - self.write_str("Restarting server...").await -} - -#[admin_command] -pub(super) async fn shutdown(&self) -> Result { - self.bail_restricted()?; - - warn!("shutdown command"); - self.services.server.shutdown()?; - - self.write_str("Shutting down server...").await -} - -#[admin_command] -pub(super) async fn list_features(&self) -> Result { - let mut enabled_features = conduwuit::info::introspection::ENABLED_FEATURES - .lock() - .expect("locked") - .values() - .flat_map(|f| f.iter()) - .collect::>(); - - enabled_features.sort_unstable(); - enabled_features.dedup(); - - let mut available_features = conduwuit::build_metadata::WORKSPACE_FEATURES - .iter() - .flat_map(|(_, f)| f.iter()) - .collect::>(); - - available_features.sort_unstable(); - available_features.dedup(); - - let mut features = String::new(); - - for feature in available_features { - let active = enabled_features.contains(&feature); - let emoji = if active { "✅" } else { "❌" }; - let remark = if active { "[enabled]" } else { "" }; - writeln!(features, "{emoji} {feature} {remark}")?; + self.write_str("Done.").await } - self.write_str(&features).await -} - -#[admin_command] -pub(super) async fn build_info(&self) -> Result { - use conduwuit::build_metadata::built; + pub(super) async fn list_backups(&self) -> Result { + self.services + .db + .db + .backup_list()? + .try_stream() + .try_for_each(|result| writeln!(self, "{result}")) + .await + } - let mut info = String::new(); + pub(super) async fn backup_database(&self) -> Result { + self.bail_restricted()?; + + let db = Arc::clone(&self.services.db); + let result = self + .services + .server + .runtime() + .spawn_blocking(move || match db.db.backup() { + | Ok(()) => "Done".to_owned(), + | Err(e) => format!("Failed: {e}"), + }) + .await?; + + let count = self.services.db.db.backup_count()?; + self.write_str(&format!("{result}. Currently have {count} backups.")) + .await + } - // Version information - writeln!(info, "# Build Information\n")?; - writeln!(info, "**Version:** {}", built::PKG_VERSION)?; - writeln!(info, "**Package:** {}", built::PKG_NAME)?; - writeln!(info, "**Description:** {}", built::PKG_DESCRIPTION)?; + pub(super) async fn admin_notice(&self, message: Vec) -> Result { + let message = message.join(" "); + self.services.admin.send_text(&message).await; - // Git information - writeln!(info, "\n## Git Information\n")?; - if let Some(hash) = conduwuit::build_metadata::GIT_COMMIT_HASH { - writeln!(info, "**Commit Hash:** {hash}")?; - } - if let Some(hash) = conduwuit::build_metadata::GIT_COMMIT_HASH_SHORT { - writeln!(info, "**Commit Hash (short):** {hash}")?; - } - if let Some(url) = conduwuit::build_metadata::GIT_REMOTE_WEB_URL { - writeln!(info, "**Repository:** {url}")?; + self.write_str("Notice was sent to #admins").await } - if let Some(url) = conduwuit::build_metadata::GIT_REMOTE_COMMIT_URL { - writeln!(info, "**Commit URL:** {url}")?; + + pub(super) async fn reload_mods(&self) -> Result { + self.bail_restricted()?; + + self.services.server.reload()?; + + self.write_str("Reloading server...").await } - // Build environment - writeln!(info, "\n## Build Environment\n")?; - writeln!(info, "**Profile:** {}", built::PROFILE)?; - writeln!(info, "**Optimization Level:** {}", built::OPT_LEVEL)?; - writeln!(info, "**Debug:** {}", built::DEBUG)?; - writeln!(info, "**Target:** {}", built::TARGET)?; - writeln!(info, "**Host:** {}", built::HOST)?; + #[cfg(unix)] + pub(super) async fn restart(&self, force: bool) -> Result { + use conduwuit::utils::sys::current_exe_deleted; - // Rust compiler information - writeln!(info, "\n## Compiler Information\n")?; - writeln!(info, "**Rustc Version:** {}", built::RUSTC_VERSION)?; - if !built::RUSTDOC_VERSION.is_empty() { - writeln!(info, "**Rustdoc Version:** {}", built::RUSTDOC_VERSION)?; + if !force && current_exe_deleted() { + return Err!( + "The server cannot be restarted because the executable changed. If this is \ + expected use --force to override." + ); + } + + self.services.server.restart()?; + + self.write_str("Restarting server...").await } - // Target configuration - writeln!(info, "\n## Target Configuration\n")?; - writeln!(info, "**Architecture:** {}", built::CFG_TARGET_ARCH)?; - writeln!(info, "**OS:** {}", built::CFG_OS)?; - writeln!(info, "**Family:** {}", built::CFG_FAMILY)?; - writeln!(info, "**Endianness:** {}", built::CFG_ENDIAN)?; - writeln!(info, "**Pointer Width:** {} bits", built::CFG_POINTER_WIDTH)?; - if !built::CFG_ENV.is_empty() { - writeln!(info, "**Environment:** {}", built::CFG_ENV)?; + pub(super) async fn shutdown(&self) -> Result { + self.bail_restricted()?; + + warn!("shutdown command"); + self.services.server.shutdown()?; + + self.write_str("Shutting down server...").await } - // CI information - if let Some(ci) = built::CI_PLATFORM { - writeln!(info, "\n## CI Platform\n")?; - writeln!(info, "**Platform:** {ci}")?; + pub(super) async fn list_features(&self) -> Result { + let mut enabled_features = conduwuit::info::introspection::ENABLED_FEATURES + .lock() + .expect("locked") + .values() + .flat_map(|f| f.iter()) + .collect::>(); + + enabled_features.sort_unstable(); + enabled_features.dedup(); + + let mut available_features = conduwuit::build_metadata::WORKSPACE_FEATURES + .iter() + .flat_map(|(_, f)| f.iter()) + .collect::>(); + + available_features.sort_unstable(); + available_features.dedup(); + + let mut features = String::new(); + + for feature in available_features { + let active = enabled_features.contains(&feature); + let emoji = if active { "✅" } else { "❌" }; + let remark = if active { "[enabled]" } else { "" }; + writeln!(features, "{emoji} {feature} {remark}")?; + } + + self.write_str(&features).await } - self.write_str(&info).await + pub(super) async fn build_info(&self) -> Result { + use conduwuit::build_metadata::built; + + let mut info = String::new(); + + // Version information + writeln!(info, "# Build Information\n")?; + writeln!(info, "**Version:** {}", built::PKG_VERSION)?; + writeln!(info, "**Package:** {}", built::PKG_NAME)?; + writeln!(info, "**Description:** {}", built::PKG_DESCRIPTION)?; + + // Git information + writeln!(info, "\n## Git Information\n")?; + if let Some(hash) = conduwuit::build_metadata::GIT_COMMIT_HASH { + writeln!(info, "**Commit Hash:** {hash}")?; + } + if let Some(hash) = conduwuit::build_metadata::GIT_COMMIT_HASH_SHORT { + writeln!(info, "**Commit Hash (short):** {hash}")?; + } + if let Some(url) = conduwuit::build_metadata::GIT_REMOTE_WEB_URL { + writeln!(info, "**Repository:** {url}")?; + } + if let Some(url) = conduwuit::build_metadata::GIT_REMOTE_COMMIT_URL { + writeln!(info, "**Commit URL:** {url}")?; + } + + // Build environment + writeln!(info, "\n## Build Environment\n")?; + writeln!(info, "**Profile:** {}", built::PROFILE)?; + writeln!(info, "**Optimization Level:** {}", built::OPT_LEVEL)?; + writeln!(info, "**Debug:** {}", built::DEBUG)?; + writeln!(info, "**Target:** {}", built::TARGET)?; + writeln!(info, "**Host:** {}", built::HOST)?; + + // Rust compiler information + writeln!(info, "\n## Compiler Information\n")?; + writeln!(info, "**Rustc Version:** {}", built::RUSTC_VERSION)?; + if !built::RUSTDOC_VERSION.is_empty() { + writeln!(info, "**Rustdoc Version:** {}", built::RUSTDOC_VERSION)?; + } + + // Target configuration + writeln!(info, "\n## Target Configuration\n")?; + writeln!(info, "**Architecture:** {}", built::CFG_TARGET_ARCH)?; + writeln!(info, "**OS:** {}", built::CFG_OS)?; + writeln!(info, "**Family:** {}", built::CFG_FAMILY)?; + writeln!(info, "**Endianness:** {}", built::CFG_ENDIAN)?; + writeln!(info, "**Pointer Width:** {} bits", built::CFG_POINTER_WIDTH)?; + if !built::CFG_ENV.is_empty() { + writeln!(info, "**Environment:** {}", built::CFG_ENV)?; + } + + // CI information + if let Some(ci) = built::CI_PLATFORM { + writeln!(info, "\n## CI Platform\n")?; + writeln!(info, "**Platform:** {ci}")?; + } + + self.write_str(&info).await + } } diff --git a/src/admin/token/commands.rs b/src/admin/token/commands.rs index 4ff74a200..d6890e2cc 100644 --- a/src/admin/token/commands.rs +++ b/src/admin/token/commands.rs @@ -1,76 +1,76 @@ use conduwuit::{Err, Result, utils}; -use conduwuit_macros::admin_command; use futures::StreamExt; use service::registration_tokens::TokenExpires; -#[admin_command] -pub(super) async fn issue_token(&self, expires: super::TokenExpires) -> Result { - let expires = { - if expires.immortal { - None - } else if let Some(max_uses) = expires.max_uses { - Some(TokenExpires::AfterUses(max_uses)) - } else if expires.once { - Some(TokenExpires::AfterUses(1)) - } else if let Some(max_age) = expires - .max_age - .as_deref() - .map(|max_age| utils::time::timepoint_from_now(utils::time::parse_duration(max_age)?)) - .transpose()? - { - Some(TokenExpires::AfterTime(max_age)) - } else { - unreachable!(); - } - }; +impl crate::Context<'_> { + pub(super) async fn issue_token(&self, expires: super::TokenExpires) -> Result { + let expires = { + if expires.immortal { + None + } else if let Some(max_uses) = expires.max_uses { + Some(TokenExpires::AfterUses(max_uses)) + } else if expires.once { + Some(TokenExpires::AfterUses(1)) + } else if let Some(max_age) = expires + .max_age + .as_deref() + .map(|max_age| { + utils::time::timepoint_from_now(utils::time::parse_duration(max_age)?) + }) + .transpose()? + { + Some(TokenExpires::AfterTime(max_age)) + } else { + unreachable!(); + } + }; - let (token, info) = self - .services - .registration_tokens - .issue_token(self.sender_or_service_user().into(), expires); + let (token, info) = self + .services + .registration_tokens + .issue_token(self.sender_or_service_user().into(), expires); - self.write_str(&format!( - "New registration token issued: `{token}`. {}.", - if let Some(expires) = info.expires { - format!("{expires}") - } else { - "Never expires".to_owned() - } - )) - .await -} - -#[admin_command] -pub(super) async fn revoke_token(&self, token: String) -> Result { - let Some(token) = self - .services - .registration_tokens - .validate_token(token) + self.write_str(&format!( + "New registration token issued: `{token}`. {}.", + if let Some(expires) = info.expires { + format!("{expires}") + } else { + "Never expires".to_owned() + } + )) .await - else { - return Err!("This token does not exist or has already expired."); - }; - - self.services.registration_tokens.revoke_token(token)?; - - self.write_str("Token revoked successfully.").await -} - -#[admin_command] -pub(super) async fn list_tokens(&self) -> Result { - let tokens: Vec<_> = self - .services - .registration_tokens - .iterate_tokens() - .collect() - .await; - - self.write_str(&format!("Found {} registration tokens:\n", tokens.len())) - .await?; - - for token in tokens { - self.write_str(&format!("- {token}\n")).await?; } - Ok(()) + pub(super) async fn revoke_token(&self, token: String) -> Result { + let Some(token) = self + .services + .registration_tokens + .validate_token(token) + .await + else { + return Err!("This token does not exist or has already expired."); + }; + + self.services.registration_tokens.revoke_token(token)?; + + self.write_str("Token revoked successfully.").await + } + + pub(super) async fn list_tokens(&self) -> Result { + let tokens: Vec<_> = self + .services + .registration_tokens + .iterate_tokens() + .collect() + .await; + + self.write_str(&format!("Found {} registration tokens:\n", tokens.len())) + .await?; + + for token in tokens { + self.write_str(&format!("- {token}\n")).await?; + } + + Ok(()) + } } diff --git a/src/admin/user/commands.rs b/src/admin/user/commands.rs index 6a1ab7d3f..66589b4b6 100644 --- a/src/admin/user/commands.rs +++ b/src/admin/user/commands.rs @@ -26,266 +26,1093 @@ use ruma::{ use service::users::HashedPassword; use crate::{ - admin_command, get_room_info, + get_room_info, utils::{parse_active_local_user_id, parse_local_user_id}, }; const AUTO_GEN_PASSWORD_LENGTH: usize = 25; const BULK_JOIN_REASON: &str = "Bulk force joining this room as initiated by the server admin."; -#[admin_command] -pub(super) async fn list_users(&self) -> Result { - let users: Vec<_> = self - .services - .users - .list_local_users() - .map(|id| id.as_str().to_owned()) - .collect() - .await; +impl crate::Context<'_> { + pub(super) async fn list_users(&self) -> Result { + let users: Vec<_> = self + .services + .users + .list_local_users() + .map(|id| id.as_str().to_owned()) + .collect() + .await; - let mut plain_msg = format!("Found {} local user account(s):\n```\n", users.len()); - plain_msg += users.join("\n").as_str(); - plain_msg += "\n```"; + let mut plain_msg = format!("Found {} local user account(s):\n```\n", users.len()); + plain_msg += users.join("\n").as_str(); + plain_msg += "\n```"; - self.write_str(&plain_msg).await -} - -#[admin_command] -pub(super) async fn create_user(&self, username: String, password: Option) -> Result { - // Validate user id - let user_id = parse_local_user_id(self.services, &username)?; - - if let Err(e) = user_id.validate_strict() { - if self.services.config.emergency_password.is_none() { - return Err!("Username {user_id} contains disallowed characters or spaces: {e}"); - } + self.write_str(&plain_msg).await } - if self.services.users.exists(&user_id).await { - return Err!("User {user_id} already exists"); - } + pub(super) async fn create_user(&self, username: String, password: Option) -> Result { + // Validate user id + let user_id = parse_local_user_id(self.services, &username)?; - let password = password.unwrap_or_else(|| utils::random_string(AUTO_GEN_PASSWORD_LENGTH)); - - // Create user - self.services - .users - .create(&user_id, Some(HashedPassword::new(&password)?)) - .await?; - - // Default to pretty displayname - let mut displayname = user_id.localpart().to_owned(); - - // If `new_user_displayname_suffix` is set, registration will push whatever - // content is set to the user's display name with a space before it - if !self - .services - .server - .config - .new_user_displayname_suffix - .is_empty() - { - write!(displayname, " {}", self.services.server.config.new_user_displayname_suffix)?; - } - - self.services - .users - .set_displayname(&user_id, Some(displayname)); - - // Initial account data - self.services - .account_data - .update( - None, - &user_id, - ruma::events::GlobalAccountDataEventType::PushRules - .to_string() - .into(), - &serde_json::to_value(ruma::events::push_rules::PushRulesEvent::new( - ruma::events::push_rules::PushRulesEventContent::new( - ruma::push::Ruleset::server_default(&user_id), - ), - )) - .unwrap(), - ) - .await?; - - if !self.services.server.config.auto_join_rooms.is_empty() { - for room in &self.services.server.config.auto_join_rooms { - let Ok(room_id) = self.services.rooms.alias.resolve(room).await else { - error!( - %user_id, - "Failed to resolve room alias to room ID when attempting to auto join {room}, skipping" - ); - continue; - }; - - if !self - .services - .rooms - .state_cache - .server_in_room(self.services.globals.server_name(), &room_id) - .await - { - warn!( - "Skipping room {room} to automatically join as we have never joined before." - ); - continue; + if let Err(e) = user_id.validate_strict() { + if self.services.config.emergency_password.is_none() { + return Err!("Username {user_id} contains disallowed characters or spaces: {e}"); } + } - if let Some(room_server_name) = room.server_name() { - match self + if self.services.users.exists(&user_id).await { + return Err!("User {user_id} already exists"); + } + + let password = password.unwrap_or_else(|| utils::random_string(AUTO_GEN_PASSWORD_LENGTH)); + + // Create user + self.services + .users + .create(&user_id, Some(HashedPassword::new(&password)?)) + .await?; + + // Default to pretty displayname + let mut displayname = user_id.localpart().to_owned(); + + // If `new_user_displayname_suffix` is set, registration will push whatever + // content is set to the user's display name with a space before it + if !self + .services + .server + .config + .new_user_displayname_suffix + .is_empty() + { + write!(displayname, " {}", self.services.server.config.new_user_displayname_suffix)?; + } + + self.services + .users + .set_displayname(&user_id, Some(displayname)); + + // Initial account data + self.services + .account_data + .update( + None, + &user_id, + ruma::events::GlobalAccountDataEventType::PushRules + .to_string() + .into(), + &serde_json::to_value(ruma::events::push_rules::PushRulesEvent::new( + ruma::events::push_rules::PushRulesEventContent::new( + ruma::push::Ruleset::server_default(&user_id), + ), + )) + .unwrap(), + ) + .await?; + + if !self.services.server.config.auto_join_rooms.is_empty() { + for room in &self.services.server.config.auto_join_rooms { + let Ok(room_id) = self.services.rooms.alias.resolve(room).await else { + error!( + %user_id, + "Failed to resolve room alias to room ID when attempting to auto join {room}, skipping" + ); + continue; + }; + + if !self .services .rooms - .membership - .join_room( - &user_id, - &room_id, - Some("Automatically joining this room upon registration".to_owned()), - &[ - self.services.globals.server_name().to_owned(), - room_server_name.to_owned(), - ], - ) + .state_cache + .server_in_room(self.services.globals.server_name(), &room_id) .await { - | Ok(_response) => { - info!("Automatically joined room {room} for user {user_id}"); - }, - | Err(e) => { - // don't return this error so we don't fail registrations - error!( - "Failed to automatically join room {room} for user {user_id}: {e}" - ); - self.services - .admin - .send_text(&format!( + warn!( + "Skipping room {room} to automatically join as we have never joined \ + before." + ); + continue; + } + + if let Some(room_server_name) = room.server_name() { + match self + .services + .rooms + .membership + .join_room( + &user_id, + &room_id, + Some("Automatically joining this room upon registration".to_owned()), + &[ + self.services.globals.server_name().to_owned(), + room_server_name.to_owned(), + ], + ) + .await + { + | Ok(_response) => { + info!("Automatically joined room {room} for user {user_id}"); + }, + | Err(e) => { + // don't return this error so we don't fail registrations + error!( "Failed to automatically join room {room} for user {user_id}: \ {e}" - )) - .await; - }, + ); + self.services + .admin + .send_text(&format!( + "Failed to automatically join room {room} for user \ + {user_id}: {e}" + )) + .await; + }, + } } } } - } - // we dont add a device since we're not the user, just the creator + // we dont add a device since we're not the user, just the creator - // Make the first user to register an administrator and disable first-run mode. - self.services.firstrun.empower_first_user(&user_id).await?; + // Make the first user to register an administrator and disable first-run mode. + self.services.firstrun.empower_first_user(&user_id).await?; - self.write_str(&format!("Created user with user_id: {user_id} and password: `{password}`")) + self.write_str(&format!( + "Created user with user_id: {user_id} and password: `{password}`" + )) .await -} - -#[admin_command] -pub(super) async fn deactivate(&self, no_leave_rooms: bool, user_id: String) -> Result { - // Validate user id - let user_id = parse_local_user_id(self.services, &user_id)?; - - // don't deactivate the server service account - if user_id == self.services.globals.server_user { - return Err!("Not allowed to deactivate the server service account.",); } - self.services.users.deactivate_account(&user_id).await?; + pub(super) async fn deactivate(&self, no_leave_rooms: bool, user_id: String) -> Result { + // Validate user id + let user_id = parse_local_user_id(self.services, &user_id)?; - if !no_leave_rooms { + // don't deactivate the server service account + if user_id == self.services.globals.server_user { + return Err!("Not allowed to deactivate the server service account.",); + } + + self.services.users.deactivate_account(&user_id).await?; + + if !no_leave_rooms { + self.services + .admin + .send_text(&format!("Making {user_id} leave all rooms after deactivation...")) + .await; + + let all_joined_rooms: Vec = self + .services + .rooms + .state_cache + .rooms_joined(&user_id) + .map(Into::into) + .collect() + .await; + + full_user_deactivate(self.services, &user_id, &all_joined_rooms) + .boxed() + .await?; + } + + self.write_str(&format!("User {user_id} has been deactivated")) + .await + } + + pub(super) async fn suspend(&self, user_id: String) -> Result { + self.bail_restricted()?; + let user_id = parse_local_user_id(self.services, &user_id)?; + + if user_id == self.services.globals.server_user { + return Err!("Not allowed to suspend the server service account.",); + } + + if !self.services.users.exists(&user_id).await { + return Err!("User {user_id} does not exist."); + } + if self.services.users.is_admin(&user_id).await { + return Err!("Admin users cannot be suspended."); + } + // TODO: Record the actual user that sent the suspension where possible self.services - .admin - .send_text(&format!("Making {user_id} leave all rooms after deactivation...")) + .users + .suspend_account(&user_id, self.sender_or_service_user()) .await; - let all_joined_rooms: Vec = self + self.write_str(&format!("User {user_id} has been suspended.")) + .await + } + + pub(super) async fn unsuspend(&self, user_id: String) -> Result { + self.bail_restricted()?; + let user_id = parse_local_user_id(self.services, &user_id)?; + + if user_id == self.services.globals.server_user { + return Err!("Not allowed to unsuspend the server service account.",); + } + + if !self.services.users.exists(&user_id).await { + return Err!("User {user_id} does not exist."); + } + self.services.users.unsuspend_account(&user_id).await; + + self.write_str(&format!("User {user_id} has been unsuspended.")) + .await + } + + pub(super) async fn reset_password( + &self, + logout: bool, + username: String, + password: Option, + ) -> Result { + let user_id = parse_local_user_id(self.services, &username)?; + + if user_id == self.services.globals.server_user { + return Err!( + "Not allowed to set the password for the server account. Please use the \ + emergency password config option.", + ); + } + + let new_password = + password.unwrap_or_else(|| utils::random_string(AUTO_GEN_PASSWORD_LENGTH)); + + self.services + .users + .set_password(&user_id, Some(HashedPassword::new(&new_password)?)); + + self.write_str(&format!( + "Successfully reset the password for user {user_id}: `{new_password}`" + )) + .await?; + + if logout { + self.services + .users + .all_device_ids(&user_id) + .for_each(async |device_id| { + self.services + .users + .remove_device(&user_id, &device_id) + .await; + }) + .await; + write!(self, "\nAll existing sessions have been logged out.").await?; + } + + Ok(()) + } + + pub(super) async fn issue_password_reset_link(&self, username: String) -> Result { + use conduwuit_service::password_reset::{PASSWORD_RESET_PATH, RESET_TOKEN_QUERY_PARAM}; + + self.bail_restricted()?; + + let mut reset_url = self + .services + .config + .get_client_domain() + .join(PASSWORD_RESET_PATH) + .unwrap(); + + let user_id = parse_local_user_id(self.services, &username)?; + let token = self.services.password_reset.issue_token(user_id).await?; + reset_url + .query_pairs_mut() + .append_pair(RESET_TOKEN_QUERY_PARAM, &token.token); + + self.write_str(&format!("Password reset link issued for {username}: {reset_url}")) + .await?; + + Ok(()) + } + + pub(super) async fn deactivate_all(&self, no_leave_rooms: bool, force: bool) -> Result { + if self.body.len() < 2 + || !self.body[0].trim().starts_with("```") + || self.body.last().unwrap_or(&"").trim() != "```" + { + return Err!("Expected code block in command body. Add --help for details.",); + } + + let usernames = self + .body + .to_vec() + .drain(1..self.body.len().saturating_sub(1)) + .collect::>(); + + let mut user_ids: Vec = Vec::with_capacity(usernames.len()); + let mut admins = Vec::new(); + + for username in usernames { + match parse_active_local_user_id(self.services, username).await { + | Err(e) => { + self.services + .admin + .send_text(&format!( + "{username} is not a valid username, skipping over: {e}" + )) + .await; + + continue; + }, + | Ok(user_id) => { + if self.services.users.is_admin(&user_id).await && !force { + self.services + .admin + .send_text(&format!( + "{username} is an admin and --force is not set, skipping over" + )) + .await; + + admins.push(username); + continue; + } + + // don't deactivate the server service account + if user_id == self.services.globals.server_user { + self.services + .admin + .send_text(&format!( + "{username} is the server service account, skipping over" + )) + .await; + + continue; + } + + user_ids.push(user_id); + }, + } + } + + let mut deactivation_count: usize = 0; + + for user_id in user_ids { + match self.services.users.deactivate_account(&user_id).await { + | Err(e) => { + self.services + .admin + .send_text(&format!("Failed deactivating user: {e}")) + .await; + }, + | Ok(()) => { + deactivation_count = deactivation_count.saturating_add(1); + if !no_leave_rooms { + info!( + "Forcing user {user_id} to leave all rooms apart of deactivate-all" + ); + let all_joined_rooms: Vec = self + .services + .rooms + .state_cache + .rooms_joined(&user_id) + .map(Into::into) + .collect() + .await; + + full_user_deactivate(self.services, &user_id, &all_joined_rooms) + .boxed() + .await?; + } + }, + } + } + + if admins.is_empty() { + write!(self, "Deactivated {deactivation_count} accounts.") + } else { + write!( + self, + "Deactivated {deactivation_count} accounts.\nSkipped admin accounts: {}. Use \ + --force to deactivate admin accounts", + admins.join(", ") + ) + } + .await + } + + pub(super) async fn list_invited_rooms(&self, user_id: String) -> Result { + // Validate user id + let user_id = parse_local_user_id(self.services, &user_id)?; + + let mut rooms: Vec<((OwnedRoomId, u64, String), Result)> = self + .services + .rooms + .state_cache + .rooms_invited(&user_id) + .then(async |(room_id, _)| { + let sender = self + .services + .rooms + .state_cache + .invite_sender(&user_id, &room_id) + .await; + (get_room_info(self.services, &room_id).await, sender) + }) + .collect() + .await; + + if rooms.is_empty() { + return Err!("User is not invited to any rooms."); + } + + rooms.sort_by_key(|r| r.0.1); + rooms.reverse(); + + let body = rooms + .iter() + .map(|((id, members, name), sender)| match sender { + | Ok(user_id) => + format!("{id}\tInviter: {user_id}\tMembers: {members}\tName: {name}"), + | Err(_) => format!("{id}\tMembers: {members}\tName: {name}"), + }) + .collect::>() + .join("\n"); + + self.write_str(&format!( + "Rooms {user_id} is Invited to ({}):\n```\n{body}\n```", + rooms.len() + )) + .await + } + + pub(super) async fn reject_all_invites(&self, user_id: String) -> Result { + let user_id = parse_local_user_id(self.services, &user_id)?; + + assert!( + self.services.globals.user_is_local(&user_id), + "Parsed user_id must be a local user" + ); + + let fails = self + .services + .rooms + .state_cache + .rooms_invited(&user_id) + .filter_map(async |(room_id, _)| { + match leave_room(self.services, &user_id, &room_id, None).await { + | Err(ref e) => { + warn!(%user_id, "Failed to leave {room_id}: {e}"); + Some(()) + }, + | Ok(()) => None, + } + }) + .count() + .await; + + if fails > 0 { + return Err!("{fails} invites could not be rejected"); + } + + self.write_str("Successfully rejected all invites.").await + } + + pub(super) async fn list_joined_rooms(&self, user_id: String) -> Result { + // Validate user id + let user_id = parse_local_user_id(self.services, &user_id)?; + + let mut rooms: Vec<(OwnedRoomId, u64, String)> = self .services .rooms .state_cache .rooms_joined(&user_id) - .map(Into::into) + .then(async |room_id| get_room_info(self.services, &room_id).await) .collect() .await; - full_user_deactivate(self.services, &user_id, &all_joined_rooms) + if rooms.is_empty() { + return Err!("User is not in any rooms."); + } + + rooms.sort_by_key(|r| r.1); + rooms.reverse(); + + let body = rooms + .iter() + .map(|(id, members, name)| format!("{id}\tMembers: {members}\tName: {name}")) + .collect::>() + .join("\n"); + + self.write_str(&format!("Rooms {user_id} Joined ({}):\n```\n{body}\n```", rooms.len())) + .await + } + + pub(super) async fn force_join_list_of_local_users( + &self, + room_id: OwnedRoomOrAliasId, + yes_i_want_to_do_this: bool, + ) -> Result { + if self.body.len() < 2 + || !self.body[0].trim().starts_with("```") + || self.body.last().unwrap_or(&"").trim() != "```" + { + return Err!("Expected code block in command body. Add --help for details.",); + } + + if !yes_i_want_to_do_this { + return Err!( + "You must pass the --yes-i-want-to-do-this-flag to ensure you really want to \ + force bulk join all specified local users.", + ); + } + + let server_admins = self.services.admin.get_admins().await; + + if server_admins.is_empty() { + return Err!("There are no admins set for this server."); + } + + let (room_id, servers) = self + .services + .rooms + .alias + .resolve_with_servers(&room_id, None) + .await?; + + if !self + .services + .rooms + .state_cache + .server_in_room(self.services.globals.server_name(), &room_id) + .await + { + return Err!("We are not joined in this room."); + } + + if !self + .services + .rooms + .state_cache + .room_members(&room_id) + .ready_any(|user_id| server_admins.contains(&user_id)) + .await + { + return Err!("There is not a single server admin in the room.",); + } + + let usernames = self + .body + .to_vec() + .drain(1..self.body.len().saturating_sub(1)) + .collect::>(); + + let mut user_ids: Vec = Vec::with_capacity(usernames.len()); + + for username in usernames { + match parse_active_local_user_id(self.services, username).await { + | Ok(user_id) => { + // don't make the server service account join + if user_id == self.services.globals.server_user { + self.services + .admin + .send_text(&format!( + "{username} is the server service account, skipping over" + )) + .await; + + continue; + } + + user_ids.push(user_id); + }, + | Err(e) => { + self.services + .admin + .send_text(&format!( + "{username} is not a valid username, skipping over: {e}" + )) + .await; + + continue; + }, + } + } + + let mut failed_joins: usize = 0; + let mut successful_joins: usize = 0; + + for user_id in user_ids { + match self + .services + .rooms + .membership + .join_room(&user_id, &room_id, Some(String::from(BULK_JOIN_REASON)), &servers) + .await + { + | Ok(_res) => { + successful_joins = successful_joins.saturating_add(1); + }, + | Err(e) => { + debug_warn!( + "Failed force joining {user_id} to {room_id} during bulk join: {e}" + ); + failed_joins = failed_joins.saturating_add(1); + }, + } + } + + self.write_str(&format!( + "{successful_joins} local users have been joined to {room_id}. {failed_joins} joins \ + failed.", + )) + .await + } + + pub(super) async fn force_join_all_local_users( + &self, + room_id: OwnedRoomOrAliasId, + yes_i_want_to_do_this: bool, + ) -> Result { + if !yes_i_want_to_do_this { + return Err!( + "You must pass the --yes-i-want-to-do-this-flag to ensure you really want to \ + force bulk join all local users.", + ); + } + + let server_admins = self.services.admin.get_admins().await; + + if server_admins.is_empty() { + return Err!("There are no admins set for this server."); + } + + let (room_id, servers) = self + .services + .rooms + .alias + .resolve_with_servers(&room_id, None) + .await?; + + if !self + .services + .rooms + .state_cache + .server_in_room(self.services.globals.server_name(), &room_id) + .await + { + return Err!("We are not joined in this room."); + } + + if !self + .services + .rooms + .state_cache + .room_members(&room_id) + .ready_any(|user_id| server_admins.contains(&user_id)) + .await + { + return Err!("There is not a single server admin in the room.",); + } + + let mut failed_joins: usize = 0; + let mut successful_joins: usize = 0; + + for user_id in &self + .services + .users + .list_local_users() + .collect::>() + .await + { + match self + .services + .rooms + .membership + .join_room(user_id, &room_id, Some(String::from(BULK_JOIN_REASON)), &servers) + .await + { + | Ok(_res) => { + successful_joins = successful_joins.saturating_add(1); + }, + | Err(e) => { + debug_warn!( + "Failed force joining {user_id} to {room_id} during bulk join: {e}" + ); + failed_joins = failed_joins.saturating_add(1); + }, + } + } + + self.write_str(&format!( + "{successful_joins} local users have been joined to {room_id}. {failed_joins} joins \ + failed.", + )) + .await + } + + pub(super) async fn force_join_room( + &self, + user_id: String, + room_id: OwnedRoomOrAliasId, + via: Option, + ) -> Result { + let user_id = parse_local_user_id(self.services, &user_id)?; + let (room_id, mut servers) = self + .services + .rooms + .alias + .resolve_with_servers(&room_id, None) + .await?; + if let Some(via) = via.map(ServerName::parse).transpose()? { + servers.retain(|n| *n != via); + servers.insert(0, via); + } + + assert!( + self.services.globals.user_is_local(&user_id), + "Parsed user_id must be a local user" + ); + self.services + .rooms + .membership + .join_room(&user_id, &room_id, None, &servers) + .await?; + + self.write_str(&format!("{user_id} has been joined to {room_id}.")) + .await + } + + pub(super) async fn force_leave_room( + &self, + user_id: String, + room_id: OwnedRoomOrAliasId, + ) -> Result { + let user_id = parse_local_user_id(self.services, &user_id)?; + let room_id = self.services.rooms.alias.resolve(&room_id).await?; + + assert!( + self.services.globals.user_is_local(&user_id), + "Parsed user_id must be a local user" + ); + + if !self + .services + .rooms + .state_cache + .is_joined(&user_id, &room_id) + .await + { + return Err!("{user_id} is not joined in the room"); + } + + leave_room(self.services, &user_id, &room_id, None) .boxed() .await?; + + self.write_str(&format!("{user_id} has left {room_id}.")) + .await } - self.write_str(&format!("User {user_id} has been deactivated")) - .await -} + pub(super) async fn force_demote( + &self, + user_id: String, + room_id: OwnedRoomOrAliasId, + ) -> Result { + let user_id = parse_local_user_id(self.services, &user_id)?; + let room_id = self.services.rooms.alias.resolve(&room_id).await?; -#[admin_command] -pub(super) async fn suspend(&self, user_id: String) -> Result { - self.bail_restricted()?; - let user_id = parse_local_user_id(self.services, &user_id)?; - - if user_id == self.services.globals.server_user { - return Err!("Not allowed to suspend the server service account.",); - } - - if !self.services.users.exists(&user_id).await { - return Err!("User {user_id} does not exist."); - } - if self.services.users.is_admin(&user_id).await { - return Err!("Admin users cannot be suspended."); - } - // TODO: Record the actual user that sent the suspension where possible - self.services - .users - .suspend_account(&user_id, self.sender_or_service_user()) - .await; - - self.write_str(&format!("User {user_id} has been suspended.")) - .await -} - -#[admin_command] -pub(super) async fn unsuspend(&self, user_id: String) -> Result { - self.bail_restricted()?; - let user_id = parse_local_user_id(self.services, &user_id)?; - - if user_id == self.services.globals.server_user { - return Err!("Not allowed to unsuspend the server service account.",); - } - - if !self.services.users.exists(&user_id).await { - return Err!("User {user_id} does not exist."); - } - self.services.users.unsuspend_account(&user_id).await; - - self.write_str(&format!("User {user_id} has been unsuspended.")) - .await -} - -#[admin_command] -pub(super) async fn reset_password( - &self, - logout: bool, - username: String, - password: Option, -) -> Result { - let user_id = parse_local_user_id(self.services, &username)?; - - if user_id == self.services.globals.server_user { - return Err!( - "Not allowed to set the password for the server account. Please use the emergency \ - password config option.", + assert!( + self.services.globals.user_is_local(&user_id), + "Parsed user_id must be a local user" ); + + let state_lock = self.services.rooms.state.mutex.lock(room_id.as_str()).await; + + let mut room_power_levels = self + .services + .rooms + .state_accessor + .get_room_power_levels(&room_id) + .await; + + let user_can_demote_self = + room_power_levels.user_can_change_user_power_level(&user_id, &user_id); + + if !user_can_demote_self { + return Err!("User is not allowed to modify their own power levels in the room.",); + } + + room_power_levels.users.remove(&user_id); + + let event_id = self + .services + .rooms + .timeline + .build_and_append_pdu( + PartialPdu::state( + String::new(), + &RoomPowerLevelsEventContent::try_from(room_power_levels) + .expect("PLs should be valid for room version"), + ), + &user_id, + Some(&room_id), + &state_lock, + ) + .await?; + + self.write_str(&format!( + "User {user_id} demoted themselves to the room default power level in {room_id} - \ + {event_id}" + )) + .await } - let new_password = password.unwrap_or_else(|| utils::random_string(AUTO_GEN_PASSWORD_LENGTH)); + pub(super) async fn make_user_admin(&self, user_id: String) -> Result { + let user_id = parse_local_user_id(self.services, &user_id)?; + assert!( + self.services.globals.user_is_local(&user_id), + "Parsed user_id must be a local user" + ); - self.services - .users - .set_password(&user_id, Some(HashedPassword::new(&new_password)?)); + self.services + .admin + .make_user_admin(&user_id) + .boxed() + .await?; - self.write_str(&format!( - "Successfully reset the password for user {user_id}: `{new_password}`" - )) - .await?; + self.write_str(&format!("{user_id} has been granted admin privileges.")) + .await + } - if logout { + pub(super) async fn put_room_tag( + &self, + user_id: String, + room_id: OwnedRoomId, + tag: String, + ) -> Result { + let user_id = parse_active_local_user_id(self.services, &user_id).await?; + + let mut tags_event = self + .services + .account_data + .get_room(&room_id, &user_id, RoomAccountDataEventType::Tag) + .await + .unwrap_or_else(|_| TagEvent::new(TagEventContent::new(BTreeMap::new()))); + + tags_event + .content + .tags + .insert(tag.clone().into(), TagInfo::new()); + + self.services + .account_data + .update( + Some(&room_id), + &user_id, + RoomAccountDataEventType::Tag, + &serde_json::to_value(tags_event).expect("to json value always works"), + ) + .await?; + + self.write_str(&format!( + "Successfully updated room account data for {user_id} and room {room_id} with tag \ + {tag}" + )) + .await + } + + pub(super) async fn delete_room_tag( + &self, + user_id: String, + room_id: OwnedRoomId, + tag: String, + ) -> Result { + let user_id = parse_active_local_user_id(self.services, &user_id).await?; + + let mut tags_event = self + .services + .account_data + .get_room(&room_id, &user_id, RoomAccountDataEventType::Tag) + .await + .unwrap_or_else(|_| TagEvent::new(TagEventContent::new(BTreeMap::new()))); + + tags_event.content.tags.remove(&tag.clone().into()); + + self.services + .account_data + .update( + Some(&room_id), + &user_id, + RoomAccountDataEventType::Tag, + &serde_json::to_value(tags_event).expect("to json value always works"), + ) + .await?; + + self.write_str(&format!( + "Successfully updated room account data for {user_id} and room {room_id}, deleting \ + room tag {tag}" + )) + .await + } + + pub(super) async fn get_room_tags(&self, user_id: String, room_id: OwnedRoomId) -> Result { + let user_id = parse_active_local_user_id(self.services, &user_id).await?; + + let tags_event = self + .services + .account_data + .get_room(&room_id, &user_id, RoomAccountDataEventType::Tag) + .await + .unwrap_or_else(|_| TagEvent::new(TagEventContent::new(BTreeMap::new()))); + + self.write_str(&format!("```\n{:#?}\n```", tags_event.content.tags)) + .await + } + + pub(super) async fn redact_event(&self, event_id: OwnedEventId) -> Result { + let Ok(event) = self + .services + .rooms + .timeline + .get_non_outlier_pdu(&event_id) + .await + else { + return Err!("Event does not exist in our database."); + }; + + if event.is_redacted() { + return Err!("Event is already redacted."); + } + + if !self.services.globals.user_is_local(event.sender()) { + return Err!("This command only works on local users."); + } + + let reason = format!( + "The administrator(s) of {} has redacted this user's message.", + self.services.globals.server_name() + ); + + let redaction_event_id = { + let state_lock = self + .services + .rooms + .state + .mutex + .lock(event.room_id_or_hash().as_str()) + .await; + + self.services + .rooms + .timeline + .build_and_append_pdu( + PartialPdu { + redacts: Some(event.event_id().to_owned()), + ..PartialPdu::timeline(&assign!(RoomRedactionEventContent::new_v1(), { + redacts: Some(event.event_id().to_owned()), + reason: Some(reason), + })) + }, + event.sender(), + Some(&event.room_id_or_hash()), + &state_lock, + ) + .await? + }; + + self.write_str(&format!( + "Successfully redacted event. Redaction event ID: {redaction_event_id}" + )) + .await + } + + pub(super) async fn force_leave_remote_room( + &self, + user_id: String, + room_id: OwnedRoomOrAliasId, + via: Option, + ) -> Result { + let user_id = parse_local_user_id(self.services, &user_id)?; + let (room_id, vias_raw) = self + .services + .rooms + .alias + .resolve_with_servers( + &room_id, + if let Some(v) = via.clone() { + Some(vec![ServerName::parse(v)?]) + } else { + None + }, + ) + .await?; + + assert!( + self.services.globals.user_is_local(&user_id), + "Parsed user_id must be a local user" + ); + let mut vias: HashSet = HashSet::new(); + if let Some(via) = via { + vias.insert(ServerName::parse(via)?); + } + for server in vias_raw { + vias.insert(server); + } + remote_leave_room(self.services, &user_id, &room_id, None, vias) + .boxed() + .await?; + + self.write_str(&format!("{user_id} successfully left {room_id} via remote server.")) + .await + } + + pub(super) async fn lock(&self, user_id: String) -> Result { + self.bail_restricted()?; + let user_id = parse_local_user_id(self.services, &user_id)?; + assert!( + self.services.globals.user_is_local(&user_id), + "Parsed user_id must be a local user" + ); + if user_id == self.services.globals.server_user { + return Err!("Not allowed to lock the server service account.",); + } + + if !self.services.users.exists(&user_id).await { + return Err!("User {user_id} does not exist."); + } + if self.services.users.is_admin(&user_id).await { + return Err!("Admin users cannot be locked."); + } + self.services + .users + .lock_account(&user_id, self.sender_or_service_user()) + .await; + + self.write_str(&format!("User {user_id} has been locked.")) + .await + } + + pub(super) async fn unlock(&self, user_id: String) -> Result { + self.bail_restricted()?; + let user_id = parse_local_user_id(self.services, &user_id)?; + assert!( + self.services.globals.user_is_local(&user_id), + "Parsed user_id must be a local user" + ); + self.services.users.unlock_account(&user_id).await; + + self.write_str(&format!("User {user_id} has been unlocked.")) + .await + } + + pub(super) async fn logout(&self, user_id: String) -> Result { + self.bail_restricted()?; + let user_id = parse_local_user_id(self.services, &user_id)?; + assert!( + self.services.globals.user_is_local(&user_id), + "Parsed user_id must be a local user" + ); + if user_id == self.services.globals.server_user { + return Err!("Not allowed to log out the server service account.",); + } + + if !self.services.users.exists(&user_id).await { + return Err!("User {user_id} does not exist."); + } + if self.services.users.is_admin(&user_id).await { + return Err!("You cannot forcefully log out admin users."); + } self.services .users .all_device_ids(&user_id) @@ -296,988 +1123,158 @@ pub(super) async fn reset_password( .await; }) .await; - write!(self, "\nAll existing sessions have been logged out.").await?; + self.write_str(&format!("User {user_id} has been logged out from all devices.")) + .await } - Ok(()) -} - -#[admin_command] -pub(super) async fn issue_password_reset_link(&self, username: String) -> Result { - use conduwuit_service::password_reset::{PASSWORD_RESET_PATH, RESET_TOKEN_QUERY_PARAM}; - - self.bail_restricted()?; - - let mut reset_url = self - .services - .config - .get_client_domain() - .join(PASSWORD_RESET_PATH) - .unwrap(); - - let user_id = parse_local_user_id(self.services, &username)?; - let token = self.services.password_reset.issue_token(user_id).await?; - reset_url - .query_pairs_mut() - .append_pair(RESET_TOKEN_QUERY_PARAM, &token.token); - - self.write_str(&format!("Password reset link issued for {username}: {reset_url}")) - .await?; - - Ok(()) -} - -#[admin_command] -pub(super) async fn deactivate_all(&self, no_leave_rooms: bool, force: bool) -> Result { - if self.body.len() < 2 - || !self.body[0].trim().starts_with("```") - || self.body.last().unwrap_or(&"").trim() != "```" - { - return Err!("Expected code block in command body. Add --help for details.",); - } - - let usernames = self - .body - .to_vec() - .drain(1..self.body.len().saturating_sub(1)) - .collect::>(); - - let mut user_ids: Vec = Vec::with_capacity(usernames.len()); - let mut admins = Vec::new(); - - for username in usernames { - match parse_active_local_user_id(self.services, username).await { - | Err(e) => { - self.services - .admin - .send_text(&format!("{username} is not a valid username, skipping over: {e}")) - .await; - - continue; - }, - | Ok(user_id) => { - if self.services.users.is_admin(&user_id).await && !force { - self.services - .admin - .send_text(&format!( - "{username} is an admin and --force is not set, skipping over" - )) - .await; - - admins.push(username); - continue; - } - - // don't deactivate the server service account - if user_id == self.services.globals.server_user { - self.services - .admin - .send_text(&format!( - "{username} is the server service account, skipping over" - )) - .await; - - continue; - } - - user_ids.push(user_id); - }, - } - } - - let mut deactivation_count: usize = 0; - - for user_id in user_ids { - match self.services.users.deactivate_account(&user_id).await { - | Err(e) => { - self.services - .admin - .send_text(&format!("Failed deactivating user: {e}")) - .await; - }, - | Ok(()) => { - deactivation_count = deactivation_count.saturating_add(1); - if !no_leave_rooms { - info!("Forcing user {user_id} to leave all rooms apart of deactivate-all"); - let all_joined_rooms: Vec = self - .services - .rooms - .state_cache - .rooms_joined(&user_id) - .map(Into::into) - .collect() - .await; - - full_user_deactivate(self.services, &user_id, &all_joined_rooms) - .boxed() - .await?; - } - }, - } - } - - if admins.is_empty() { - write!(self, "Deactivated {deactivation_count} accounts.") - } else { - write!( - self, - "Deactivated {deactivation_count} accounts.\nSkipped admin accounts: {}. Use \ - --force to deactivate admin accounts", - admins.join(", ") - ) - } - .await -} - -#[admin_command] -pub(super) async fn list_invited_rooms(&self, user_id: String) -> Result { - // Validate user id - let user_id = parse_local_user_id(self.services, &user_id)?; - - let mut rooms: Vec<((OwnedRoomId, u64, String), Result)> = self - .services - .rooms - .state_cache - .rooms_invited(&user_id) - .then(async |(room_id, _)| { - let sender = self - .services - .rooms - .state_cache - .invite_sender(&user_id, &room_id) - .await; - (get_room_info(self.services, &room_id).await, sender) - }) - .collect() - .await; - - if rooms.is_empty() { - return Err!("User is not invited to any rooms."); - } - - rooms.sort_by_key(|r| r.0.1); - rooms.reverse(); - - let body = rooms - .iter() - .map(|((id, members, name), sender)| match sender { - | Ok(user_id) => - format!("{id}\tInviter: {user_id}\tMembers: {members}\tName: {name}"), - | Err(_) => format!("{id}\tMembers: {members}\tName: {name}"), - }) - .collect::>() - .join("\n"); - - self.write_str(&format!("Rooms {user_id} is Invited to ({}):\n```\n{body}\n```", rooms.len())) - .await -} - -#[admin_command] -pub(super) async fn reject_all_invites(&self, user_id: String) -> Result { - let user_id = parse_local_user_id(self.services, &user_id)?; - - assert!( - self.services.globals.user_is_local(&user_id), - "Parsed user_id must be a local user" - ); - - let fails = self - .services - .rooms - .state_cache - .rooms_invited(&user_id) - .filter_map(async |(room_id, _)| { - match leave_room(self.services, &user_id, &room_id, None).await { - | Err(ref e) => { - warn!(%user_id, "Failed to leave {room_id}: {e}"); - Some(()) - }, - | Ok(()) => None, - } - }) - .count() - .await; - - if fails > 0 { - return Err!("{fails} invites could not be rejected"); - } - - self.write_str("Successfully rejected all invites.").await -} - -#[admin_command] -pub(super) async fn list_joined_rooms(&self, user_id: String) -> Result { - // Validate user id - let user_id = parse_local_user_id(self.services, &user_id)?; - - let mut rooms: Vec<(OwnedRoomId, u64, String)> = self - .services - .rooms - .state_cache - .rooms_joined(&user_id) - .then(async |room_id| get_room_info(self.services, &room_id).await) - .collect() - .await; - - if rooms.is_empty() { - return Err!("User is not in any rooms."); - } - - rooms.sort_by_key(|r| r.1); - rooms.reverse(); - - let body = rooms - .iter() - .map(|(id, members, name)| format!("{id}\tMembers: {members}\tName: {name}")) - .collect::>() - .join("\n"); - - self.write_str(&format!("Rooms {user_id} Joined ({}):\n```\n{body}\n```", rooms.len())) - .await -} - -#[admin_command] -pub(super) async fn force_join_list_of_local_users( - &self, - room_id: OwnedRoomOrAliasId, - yes_i_want_to_do_this: bool, -) -> Result { - if self.body.len() < 2 - || !self.body[0].trim().starts_with("```") - || self.body.last().unwrap_or(&"").trim() != "```" - { - return Err!("Expected code block in command body. Add --help for details.",); - } - - if !yes_i_want_to_do_this { - return Err!( - "You must pass the --yes-i-want-to-do-this-flag to ensure you really want to force \ - bulk join all specified local users.", + pub(super) async fn disable_login(&self, user_id: String) -> Result { + self.bail_restricted()?; + let user_id = parse_local_user_id(self.services, &user_id)?; + assert!( + self.services.globals.user_is_local(&user_id), + "Parsed user_id must be a local user" ); - } - - let server_admins = self.services.admin.get_admins().await; - - if server_admins.is_empty() { - return Err!("There are no admins set for this server."); - } - - let (room_id, servers) = self - .services - .rooms - .alias - .resolve_with_servers(&room_id, None) - .await?; - - if !self - .services - .rooms - .state_cache - .server_in_room(self.services.globals.server_name(), &room_id) - .await - { - return Err!("We are not joined in this room."); - } - - if !self - .services - .rooms - .state_cache - .room_members(&room_id) - .ready_any(|user_id| server_admins.contains(&user_id)) - .await - { - return Err!("There is not a single server admin in the room.",); - } - - let usernames = self - .body - .to_vec() - .drain(1..self.body.len().saturating_sub(1)) - .collect::>(); - - let mut user_ids: Vec = Vec::with_capacity(usernames.len()); - - for username in usernames { - match parse_active_local_user_id(self.services, username).await { - | Ok(user_id) => { - // don't make the server service account join - if user_id == self.services.globals.server_user { - self.services - .admin - .send_text(&format!( - "{username} is the server service account, skipping over" - )) - .await; - - continue; - } - - user_ids.push(user_id); - }, - | Err(e) => { - self.services - .admin - .send_text(&format!("{username} is not a valid username, skipping over: {e}")) - .await; - - continue; - }, + if user_id == self.services.globals.server_user { + return Err!("Not allowed to disable login for the server service account.",); } + + if !self.services.users.exists(&user_id).await { + return Err!("User {user_id} does not exist."); + } + if self.services.users.is_admin(&user_id).await { + return Err!("Admin users cannot have their login disallowed."); + } + self.services.users.disable_login(&user_id); + + self.write_str(&format!( + "{user_id} can no longer log in. Their existing sessions remain unaffected." + )) + .await } - let mut failed_joins: usize = 0; - let mut successful_joins: usize = 0; + pub(super) async fn enable_login(&self, user_id: String) -> Result { + self.bail_restricted()?; + let user_id = parse_local_user_id(self.services, &user_id)?; + assert!( + self.services.globals.user_is_local(&user_id), + "Parsed user_id must be a local user" + ); + if !self.services.users.exists(&user_id).await { + return Err!("User {user_id} does not exist."); + } + self.services.users.enable_login(&user_id); + + self.write_str(&format!("{user_id} can now log in.")).await + } + + pub(super) async fn get_email(&self, user_id: String) -> Result { + self.bail_restricted()?; + let user_id = parse_local_user_id(self.services, &user_id)?; - for user_id in user_ids { match self .services - .rooms - .membership - .join_room(&user_id, &room_id, Some(String::from(BULK_JOIN_REASON)), &servers) + .threepid + .get_email_for_localpart(user_id.localpart()) .await { - | Ok(_res) => { - successful_joins = successful_joins.saturating_add(1); - }, - | Err(e) => { - debug_warn!("Failed force joining {user_id} to {room_id} during bulk join: {e}"); - failed_joins = failed_joins.saturating_add(1); - }, + | Some(email) => + self.write_str(&format!("{user_id} has the associated email address {email}.")) + .await, + | None => + self.write_str(&format!("{user_id} has no associated email address.")) + .await, } } - self.write_str(&format!( - "{successful_joins} local users have been joined to {room_id}. {failed_joins} joins \ - failed.", - )) - .await -} + pub(super) async fn get_user_by_email(&self, email: String) -> Result { + self.bail_restricted()?; -#[admin_command] -pub(super) async fn force_join_all_local_users( - &self, - room_id: OwnedRoomOrAliasId, - yes_i_want_to_do_this: bool, -) -> Result { - if !yes_i_want_to_do_this { - return Err!( - "You must pass the --yes-i-want-to-do-this-flag to ensure you really want to force \ - bulk join all local users.", - ); - } + let Ok(email) = Address::try_from(email) else { + return Err!("Invalid email address."); + }; - let server_admins = self.services.admin.get_admins().await; + match self.services.threepid.get_localpart_for_email(&email).await { + | Some(localpart) => { + let user_id = UserId::parse(format!( + "@{localpart}:{}", + self.services.globals.server_name() + )) + .unwrap(); - if server_admins.is_empty() { - return Err!("There are no admins set for this server."); - } - - let (room_id, servers) = self - .services - .rooms - .alias - .resolve_with_servers(&room_id, None) - .await?; - - if !self - .services - .rooms - .state_cache - .server_in_room(self.services.globals.server_name(), &room_id) - .await - { - return Err!("We are not joined in this room."); - } - - if !self - .services - .rooms - .state_cache - .room_members(&room_id) - .ready_any(|user_id| server_admins.contains(&user_id)) - .await - { - return Err!("There is not a single server admin in the room.",); - } - - let mut failed_joins: usize = 0; - let mut successful_joins: usize = 0; - - for user_id in &self - .services - .users - .list_local_users() - .collect::>() - .await - { - match self - .services - .rooms - .membership - .join_room(user_id, &room_id, Some(String::from(BULK_JOIN_REASON)), &servers) - .await - { - | Ok(_res) => { - successful_joins = successful_joins.saturating_add(1); - }, - | Err(e) => { - debug_warn!("Failed force joining {user_id} to {room_id} during bulk join: {e}"); - failed_joins = failed_joins.saturating_add(1); + self.write_str(&format!("{email} belongs to {user_id}.")) + .await }, + | None => + self.write_str(&format!("No user has {email} as their email address.")) + .await, } } - self.write_str(&format!( - "{successful_joins} local users have been joined to {room_id}. {failed_joins} joins \ - failed.", - )) - .await -} + pub(super) async fn change_email(&self, user_id: String, email: Option) -> Result { + self.bail_restricted()?; -#[admin_command] -pub(super) async fn force_join_room( - &self, - user_id: String, - room_id: OwnedRoomOrAliasId, - via: Option, -) -> Result { - let user_id = parse_local_user_id(self.services, &user_id)?; - let (room_id, mut servers) = self - .services - .rooms - .alias - .resolve_with_servers(&room_id, None) - .await?; - if let Some(via) = via.map(ServerName::parse).transpose()? { - servers.retain(|n| *n != via); - servers.insert(0, via); - } + let user_id = parse_local_user_id(self.services, &user_id)?; + let Ok(new_email) = email.map(Address::try_from).transpose() else { + return Err!("Invalid email address."); + }; - assert!( - self.services.globals.user_is_local(&user_id), - "Parsed user_id must be a local user" - ); - self.services - .rooms - .membership - .join_room(&user_id, &room_id, None, &servers) - .await?; + if self.services.mailer.mailer().is_none() { + warn!("SMTP has not been configured on this server, emails cannot be sent."); + } - self.write_str(&format!("{user_id} has been joined to {room_id}.")) - .await -} - -#[admin_command] -pub(super) async fn force_leave_room( - &self, - user_id: String, - room_id: OwnedRoomOrAliasId, -) -> Result { - let user_id = parse_local_user_id(self.services, &user_id)?; - let room_id = self.services.rooms.alias.resolve(&room_id).await?; - - assert!( - self.services.globals.user_is_local(&user_id), - "Parsed user_id must be a local user" - ); - - if !self - .services - .rooms - .state_cache - .is_joined(&user_id, &room_id) - .await - { - return Err!("{user_id} is not joined in the room"); - } - - leave_room(self.services, &user_id, &room_id, None) - .boxed() - .await?; - - self.write_str(&format!("{user_id} has left {room_id}.")) - .await -} - -#[admin_command] -pub(super) async fn force_demote(&self, user_id: String, room_id: OwnedRoomOrAliasId) -> Result { - let user_id = parse_local_user_id(self.services, &user_id)?; - let room_id = self.services.rooms.alias.resolve(&room_id).await?; - - assert!( - self.services.globals.user_is_local(&user_id), - "Parsed user_id must be a local user" - ); - - let state_lock = self.services.rooms.state.mutex.lock(room_id.as_str()).await; - - let mut room_power_levels = self - .services - .rooms - .state_accessor - .get_room_power_levels(&room_id) - .await; - - let user_can_demote_self = - room_power_levels.user_can_change_user_power_level(&user_id, &user_id); - - if !user_can_demote_self { - return Err!("User is not allowed to modify their own power levels in the room.",); - } - - room_power_levels.users.remove(&user_id); - - let event_id = self - .services - .rooms - .timeline - .build_and_append_pdu( - PartialPdu::state( - String::new(), - &RoomPowerLevelsEventContent::try_from(room_power_levels) - .expect("PLs should be valid for room version"), - ), - &user_id, - Some(&room_id), - &state_lock, - ) - .await?; - - self.write_str(&format!( - "User {user_id} demoted themselves to the room default power level in {room_id} - \ - {event_id}" - )) - .await -} - -#[admin_command] -pub(super) async fn make_user_admin(&self, user_id: String) -> Result { - let user_id = parse_local_user_id(self.services, &user_id)?; - assert!( - self.services.globals.user_is_local(&user_id), - "Parsed user_id must be a local user" - ); - - self.services - .admin - .make_user_admin(&user_id) - .boxed() - .await?; - - self.write_str(&format!("{user_id} has been granted admin privileges.")) - .await -} - -#[admin_command] -pub(super) async fn put_room_tag( - &self, - user_id: String, - room_id: OwnedRoomId, - tag: String, -) -> Result { - let user_id = parse_active_local_user_id(self.services, &user_id).await?; - - let mut tags_event = self - .services - .account_data - .get_room(&room_id, &user_id, RoomAccountDataEventType::Tag) - .await - .unwrap_or_else(|_| TagEvent::new(TagEventContent::new(BTreeMap::new()))); - - tags_event - .content - .tags - .insert(tag.clone().into(), TagInfo::new()); - - self.services - .account_data - .update( - Some(&room_id), - &user_id, - RoomAccountDataEventType::Tag, - &serde_json::to_value(tags_event).expect("to json value always works"), - ) - .await?; - - self.write_str(&format!( - "Successfully updated room account data for {user_id} and room {room_id} with tag {tag}" - )) - .await -} - -#[admin_command] -pub(super) async fn delete_room_tag( - &self, - user_id: String, - room_id: OwnedRoomId, - tag: String, -) -> Result { - let user_id = parse_active_local_user_id(self.services, &user_id).await?; - - let mut tags_event = self - .services - .account_data - .get_room(&room_id, &user_id, RoomAccountDataEventType::Tag) - .await - .unwrap_or_else(|_| TagEvent::new(TagEventContent::new(BTreeMap::new()))); - - tags_event.content.tags.remove(&tag.clone().into()); - - self.services - .account_data - .update( - Some(&room_id), - &user_id, - RoomAccountDataEventType::Tag, - &serde_json::to_value(tags_event).expect("to json value always works"), - ) - .await?; - - self.write_str(&format!( - "Successfully updated room account data for {user_id} and room {room_id}, deleting room \ - tag {tag}" - )) - .await -} - -#[admin_command] -pub(super) async fn get_room_tags(&self, user_id: String, room_id: OwnedRoomId) -> Result { - let user_id = parse_active_local_user_id(self.services, &user_id).await?; - - let tags_event = self - .services - .account_data - .get_room(&room_id, &user_id, RoomAccountDataEventType::Tag) - .await - .unwrap_or_else(|_| TagEvent::new(TagEventContent::new(BTreeMap::new()))); - - self.write_str(&format!("```\n{:#?}\n```", tags_event.content.tags)) - .await -} - -#[admin_command] -pub(super) async fn redact_event(&self, event_id: OwnedEventId) -> Result { - let Ok(event) = self - .services - .rooms - .timeline - .get_non_outlier_pdu(&event_id) - .await - else { - return Err!("Event does not exist in our database."); - }; - - if event.is_redacted() { - return Err!("Event is already redacted."); - } - - if !self.services.globals.user_is_local(event.sender()) { - return Err!("This command only works on local users."); - } - - let reason = format!( - "The administrator(s) of {} has redacted this user's message.", - self.services.globals.server_name() - ); - - let redaction_event_id = { - let state_lock = self + let current_email = self .services - .rooms - .state - .mutex - .lock(event.room_id_or_hash().as_str()) + .threepid + .get_email_for_localpart(user_id.localpart()) .await; - self.services - .rooms - .timeline - .build_and_append_pdu( - PartialPdu { - redacts: Some(event.event_id().to_owned()), - ..PartialPdu::timeline(&assign!(RoomRedactionEventContent::new_v1(), { - redacts: Some(event.event_id().to_owned()), - reason: Some(reason), - })) - }, - event.sender(), - Some(&event.room_id_or_hash()), - &state_lock, - ) - .await? - }; + match (current_email, new_email) { + | (None, None) => + self.write_str(&format!( + "{user_id} already had no associated email. No changes have been made." + )) + .await, + | (current_email, Some(new_email)) => { + self.services + .threepid + .associate_localpart_email(user_id.localpart(), &new_email) + .await?; - self.write_str(&format!( - "Successfully redacted event. Redaction event ID: {redaction_event_id}" - )) - .await -} - -#[admin_command] -pub(super) async fn force_leave_remote_room( - &self, - user_id: String, - room_id: OwnedRoomOrAliasId, - via: Option, -) -> Result { - let user_id = parse_local_user_id(self.services, &user_id)?; - let (room_id, vias_raw) = self - .services - .rooms - .alias - .resolve_with_servers( - &room_id, - if let Some(v) = via.clone() { - Some(vec![ServerName::parse(v)?]) - } else { - None + if let Some(current_email) = current_email { + self.write_str(&format!( + "The associated email of {user_id} has been changed from \ + {current_email} to {new_email}." + )) + .await + } else { + self.write_str(&format!( + "{user_id} has been associated with the email {new_email}." + )) + .await + } }, - ) - .await?; + | (Some(current_email), None) => { + self.services + .threepid + .disassociate_localpart_email(user_id.localpart()) + .await; - assert!( - self.services.globals.user_is_local(&user_id), - "Parsed user_id must be a local user" - ); - let mut vias: HashSet = HashSet::new(); - if let Some(via) = via { - vias.insert(ServerName::parse(via)?); - } - for server in vias_raw { - vias.insert(server); - } - remote_leave_room(self.services, &user_id, &room_id, None, vias) - .boxed() - .await?; - - self.write_str(&format!("{user_id} successfully left {room_id} via remote server.")) - .await -} - -#[admin_command] -pub(super) async fn lock(&self, user_id: String) -> Result { - self.bail_restricted()?; - let user_id = parse_local_user_id(self.services, &user_id)?; - assert!( - self.services.globals.user_is_local(&user_id), - "Parsed user_id must be a local user" - ); - if user_id == self.services.globals.server_user { - return Err!("Not allowed to lock the server service account.",); - } - - if !self.services.users.exists(&user_id).await { - return Err!("User {user_id} does not exist."); - } - if self.services.users.is_admin(&user_id).await { - return Err!("Admin users cannot be locked."); - } - self.services - .users - .lock_account(&user_id, self.sender_or_service_user()) - .await; - - self.write_str(&format!("User {user_id} has been locked.")) - .await -} - -#[admin_command] -pub(super) async fn unlock(&self, user_id: String) -> Result { - self.bail_restricted()?; - let user_id = parse_local_user_id(self.services, &user_id)?; - assert!( - self.services.globals.user_is_local(&user_id), - "Parsed user_id must be a local user" - ); - self.services.users.unlock_account(&user_id).await; - - self.write_str(&format!("User {user_id} has been unlocked.")) - .await -} - -#[admin_command] -pub(super) async fn logout(&self, user_id: String) -> Result { - self.bail_restricted()?; - let user_id = parse_local_user_id(self.services, &user_id)?; - assert!( - self.services.globals.user_is_local(&user_id), - "Parsed user_id must be a local user" - ); - if user_id == self.services.globals.server_user { - return Err!("Not allowed to log out the server service account.",); - } - - if !self.services.users.exists(&user_id).await { - return Err!("User {user_id} does not exist."); - } - if self.services.users.is_admin(&user_id).await { - return Err!("You cannot forcefully log out admin users."); - } - self.services - .users - .all_device_ids(&user_id) - .for_each(async |device_id| { - self.services - .users - .remove_device(&user_id, &device_id) - .await; - }) - .await; - self.write_str(&format!("User {user_id} has been logged out from all devices.")) - .await -} - -#[admin_command] -pub(super) async fn disable_login(&self, user_id: String) -> Result { - self.bail_restricted()?; - let user_id = parse_local_user_id(self.services, &user_id)?; - assert!( - self.services.globals.user_is_local(&user_id), - "Parsed user_id must be a local user" - ); - if user_id == self.services.globals.server_user { - return Err!("Not allowed to disable login for the server service account.",); - } - - if !self.services.users.exists(&user_id).await { - return Err!("User {user_id} does not exist."); - } - if self.services.users.is_admin(&user_id).await { - return Err!("Admin users cannot have their login disallowed."); - } - self.services.users.disable_login(&user_id); - - self.write_str(&format!( - "{user_id} can no longer log in. Their existing sessions remain unaffected." - )) - .await -} - -#[admin_command] -pub(super) async fn enable_login(&self, user_id: String) -> Result { - self.bail_restricted()?; - let user_id = parse_local_user_id(self.services, &user_id)?; - assert!( - self.services.globals.user_is_local(&user_id), - "Parsed user_id must be a local user" - ); - if !self.services.users.exists(&user_id).await { - return Err!("User {user_id} does not exist."); - } - self.services.users.enable_login(&user_id); - - self.write_str(&format!("{user_id} can now log in.")).await -} - -#[admin_command] -pub(super) async fn get_email(&self, user_id: String) -> Result { - self.bail_restricted()?; - let user_id = parse_local_user_id(self.services, &user_id)?; - - match self - .services - .threepid - .get_email_for_localpart(user_id.localpart()) - .await - { - | Some(email) => - self.write_str(&format!("{user_id} has the associated email address {email}.")) - .await, - | None => - self.write_str(&format!("{user_id} has no associated email address.")) - .await, - } -} - -#[admin_command] -pub(super) async fn get_user_by_email(&self, email: String) -> Result { - self.bail_restricted()?; - - let Ok(email) = Address::try_from(email) else { - return Err!("Invalid email address."); - }; - - match self.services.threepid.get_localpart_for_email(&email).await { - | Some(localpart) => { - let user_id = - UserId::parse(format!("@{localpart}:{}", self.services.globals.server_name())) - .unwrap(); - - self.write_str(&format!("{email} belongs to {user_id}.")) - .await - }, - | None => - self.write_str(&format!("No user has {email} as their email address.")) - .await, - } -} - -#[admin_command] -pub(super) async fn change_email(&self, user_id: String, email: Option) -> Result { - self.bail_restricted()?; - - let user_id = parse_local_user_id(self.services, &user_id)?; - let Ok(new_email) = email.map(Address::try_from).transpose() else { - return Err!("Invalid email address."); - }; - - if self.services.mailer.mailer().is_none() { - warn!("SMTP has not been configured on this server, emails cannot be sent."); - } - - let current_email = self - .services - .threepid - .get_email_for_localpart(user_id.localpart()) - .await; - - match (current_email, new_email) { - | (None, None) => - self.write_str(&format!( - "{user_id} already had no associated email. No changes have been made." - )) - .await, - | (current_email, Some(new_email)) => { - self.services - .threepid - .associate_localpart_email(user_id.localpart(), &new_email) - .await?; - - if let Some(current_email) = current_email { self.write_str(&format!( - "The associated email of {user_id} has been changed from {current_email} to \ - {new_email}." + "The associated email of {user_id} has been removed (it was \ + {current_email})." )) .await - } else { - self.write_str(&format!( - "{user_id} has been associated with the email {new_email}." - )) - .await - } - }, - | (Some(current_email), None) => { - self.services - .threepid - .disassociate_localpart_email(user_id.localpart()) - .await; + }, + } + } - self.write_str(&format!( - "The associated email of {user_id} has been removed (it was {current_email})." - )) + pub(super) async fn reset_push_rules(&self, user_id: String) -> Result { + let user_id = parse_local_user_id(self.services, &user_id)?; + if !self.services.users.is_active(&user_id).await { + return Err!("User is not active."); + } + recreate_push_rules_and_return(self.services, &user_id).await?; + self.write_str("Reset user's push rules to the server default.") .await - }, } } - -#[admin_command] -pub(super) async fn reset_push_rules(&self, user_id: String) -> Result { - let user_id = parse_local_user_id(self.services, &user_id)?; - if !self.services.users.is_active(&user_id).await { - return Err!("User is not active."); - } - recreate_push_rules_and_return(self.services, &user_id).await?; - self.write_str("Reset user's push rules to the server default.") - .await -} diff --git a/src/macros/admin.rs b/src/macros/admin.rs index fe227b436..365d593c4 100644 --- a/src/macros/admin.rs +++ b/src/macros/admin.rs @@ -2,19 +2,10 @@ use itertools::Itertools; use proc_macro::{Span, TokenStream}; use proc_macro2::TokenStream as TokenStream2; use quote::{ToTokens, quote}; -use syn::{Attribute, Error, Fields, Ident, ItemEnum, ItemFn, Meta, Variant, parse_quote}; +use syn::{Error, Fields, Ident, ItemEnum, Meta, Variant}; use crate::{Result, utils::camel_to_snake_string}; -pub(super) fn command(mut item: ItemFn, _args: &[Meta]) -> Result { - let attr: Attribute = parse_quote! { - #[conduwuit_macros::implement(crate::Context, params = "<'_>")] - }; - - item.attrs.push(attr); - Ok(item.into_token_stream().into()) -} - pub(super) fn command_dispatch(item: ItemEnum, _args: &[Meta]) -> Result { let name = &item.ident; let arm: Vec = item.variants.iter().map(dispatch_arm).try_collect()?; diff --git a/src/macros/mod.rs b/src/macros/mod.rs index 843cd4e82..b533230b3 100644 --- a/src/macros/mod.rs +++ b/src/macros/mod.rs @@ -14,11 +14,6 @@ use syn::{ pub(crate) type Result = std::result::Result; -#[proc_macro_attribute] -pub fn admin_command(args: TokenStream, input: TokenStream) -> TokenStream { - attribute_macro::(args, input, admin::command) -} - #[proc_macro_attribute] pub fn admin_command_dispatch(args: TokenStream, input: TokenStream) -> TokenStream { attribute_macro::(args, input, admin::command_dispatch) diff --git a/src/macros/utils.rs b/src/macros/utils.rs index 598f60ffa..a345fc673 100644 --- a/src/macros/utils.rs +++ b/src/macros/utils.rs @@ -1,8 +1,6 @@ use std::collections::HashMap; -use syn::{Expr, ExprLit, Generics, Lit, Meta, MetaNameValue, parse_str}; - -use crate::Result; +use syn::{Expr, ExprLit, Lit, Meta, MetaNameValue}; pub(crate) fn get_simple_settings(args: &[Meta]) -> HashMap { args.iter().fold(HashMap::new(), |mut map, arg| { From 967e2cc54fb1accbef7fc7bf4e644903accc7cca Mon Sep 17 00:00:00 2001 From: timedout Date: Fri, 26 Jun 2026 07:53:25 +0100 Subject: [PATCH 65/70] refactor: Split the users service into several modules Also resolvers several lints, including needless async on create_user --- src/admin/user/commands.rs | 3 +- src/api/client/account/register.rs | 2 +- src/service/admin/create.rs | 2 +- src/service/appservice/mod.rs | 5 +- src/service/rooms/state_cache/update.rs | 2 +- src/service/users/account.rs | 393 +++++++ src/service/users/device.rs | 321 ++++++ src/service/users/filters.rs | 26 + src/service/users/keys.rs | 532 +++++++++ src/service/users/mod.rs | 1323 +---------------------- src/service/users/profile.rs | 94 ++ 11 files changed, 1381 insertions(+), 1322 deletions(-) create mode 100644 src/service/users/account.rs create mode 100644 src/service/users/device.rs create mode 100644 src/service/users/filters.rs create mode 100644 src/service/users/keys.rs create mode 100644 src/service/users/profile.rs diff --git a/src/admin/user/commands.rs b/src/admin/user/commands.rs index 66589b4b6..a7b270e5a 100644 --- a/src/admin/user/commands.rs +++ b/src/admin/user/commands.rs @@ -69,8 +69,7 @@ impl crate::Context<'_> { // Create user self.services .users - .create(&user_id, Some(HashedPassword::new(&password)?)) - .await?; + .create(&user_id, Some(HashedPassword::new(&password)?))?; // Default to pretty displayname let mut displayname = user_id.localpart().to_owned(); diff --git a/src/api/client/account/register.rs b/src/api/client/account/register.rs index 1f3d17311..37f4da73b 100644 --- a/src/api/client/account/register.rs +++ b/src/api/client/account/register.rs @@ -134,7 +134,7 @@ pub(crate) async fn register_route( }; // Create user - services.users.create(&user_id, password).await?; + services.users.create(&user_id, password)?; // Set an initial display name let mut displayname = user_id.localpart().to_owned(); diff --git a/src/service/admin/create.rs b/src/service/admin/create.rs index 50bb03ac8..46c838f9d 100644 --- a/src/service/admin/create.rs +++ b/src/service/admin/create.rs @@ -48,7 +48,7 @@ pub async fn create_admin_room(services: &Services) -> Result { // Create a user for the server let server_user = services.globals.server_user.as_ref(); - services.users.create(server_user, None).await?; + services.users.create(server_user, None)?; let mut create_content = if room_version_rules.authorization.use_room_create_sender { RoomCreateEventContent::new_v1(server_user.into()) diff --git a/src/service/appservice/mod.rs b/src/service/appservice/mod.rs index d492f3188..1784ea021 100644 --- a/src/service/appservice/mod.rs +++ b/src/service/appservice/mod.rs @@ -109,10 +109,7 @@ impl Service { )?; if !self.services.users.exists(&appservice_user_id).await { - self.services - .users - .create(&appservice_user_id, None) - .await?; + self.services.users.create(&appservice_user_id, None)?; } else if self .services .users diff --git a/src/service/rooms/state_cache/update.rs b/src/service/rooms/state_cache/update.rs index 20c6a700f..5249df182 100644 --- a/src/service/rooms/state_cache/update.rs +++ b/src/service/rooms/state_cache/update.rs @@ -38,7 +38,7 @@ impl super::Service { if !self.services.globals.user_is_local(user_id) && !self.services.users.exists(user_id).await { - self.services.users.create(user_id, None).await?; + self.services.users.create(user_id, None)?; } match &membership.membership { diff --git a/src/service/users/account.rs b/src/service/users/account.rs new file mode 100644 index 000000000..7992d0967 --- /dev/null +++ b/src/service/users/account.rs @@ -0,0 +1,393 @@ +use conduwuit::{ + Err, debug_error, debug_warn, err, trace, utils, + utils::{ReadyExt, stream::TryIgnore}, +}; +use database::{Deserialized, Json}; +use futures::{Stream, StreamExt, TryFutureExt}; +use ruma::{ + MilliSecondsSinceUnixEpoch, OwnedDeviceId, OwnedUserId, UserId, + events::{GlobalAccountDataEventType, ignored_user_list::IgnoredUserListEvent}, +}; +use ruminuwuity::invite_permission_config::{FilterLevel, InvitePermissionConfigEvent}; + +use crate::users::{HashedPassword, UserSuspension}; + +impl super::Service { + /// Returns true/false based on whether the recipient/receiving user has + /// ignored the sender. + pub async fn user_is_ignored(&self, sender_user: &UserId, recipient_user: &UserId) -> bool { + self.services + .account_data + .get_global(recipient_user, GlobalAccountDataEventType::IgnoredUserList) + .await + .is_ok_and(|ignored: IgnoredUserListEvent| { + ignored + .content + .ignored_users + .keys() + .any(|blocked_user| blocked_user == sender_user) + }) + } + + /// Returns the recipient's filter level for an invite from the sender. + /// + /// If the sender is ignored by the recipient, `Ignore` is returned. + /// Otherwise, the resulting value depends on their invite blocking or + /// invite filtering configuration. + pub async fn invite_filter_level( + &self, + sender_user: &UserId, + recipient_user: &UserId, + ) -> FilterLevel { + if self.user_is_ignored(sender_user, recipient_user).await { + FilterLevel::Ignore + } else { + let (stable, unstable) = tokio::join!( + self.services + .account_data + .get_global::( + recipient_user, + GlobalAccountDataEventType::InvitePermissionConfig + ), + self.services + .account_data + .get_global::( + recipient_user, + "org.matrix.msc4155.invite_permission_config".into() + ) // TODO: MSC4155 probably needs upstreaming to ruma at some point + ); + if stable.is_err() && unstable.is_err() { + return FilterLevel::Allow; + } + stable + .unwrap_or_else(|_| unstable.unwrap()) + .content + .user_filter_level(sender_user) + } + } + + /// Check if a user is an admin + #[inline] + pub async fn is_admin(&self, user_id: &UserId) -> bool { + self.services.admin.user_is_admin(user_id).await + } + + /// Create a new user account on this homeserver. Set the password to `None` + /// to create a non-local user. Non-local users with a password will return + /// an error. + #[inline] + pub fn create( + &self, + user_id: &UserId, + password: Option, + ) -> conduwuit::Result<()> { + if !self.services.globals.user_is_local(user_id) && password.is_some() { + return Err!("Cannot create a nonlocal user with a set password"); + } + + self.set_password(user_id, password); + + Ok(()) + } + + /// Deactivates an account, removing all of their device IDs and unsetting + /// their password. + pub async fn deactivate_account(&self, user_id: &UserId) -> conduwuit::Result<()> { + // Remove all associated devices + self.all_device_ids(user_id) + .for_each(async |device_id| self.remove_device(user_id, &device_id).await) + .await; + + // Set the password to "" to indicate a deactivated account. Hashes will never + // result in an empty string, so the user will not be able to log in again. + // Systems like changing the password without logging in should check if the + // account is deactivated. + self.set_password(user_id, None); + + // TODO: Unhook 3PID + Ok(()) + } + + /// Suspend account, placing it in a read-only state + pub async fn suspend_account(&self, user_id: &UserId, suspending_user: &UserId) { + self.db.userid_suspension.raw_put( + user_id, + Json(UserSuspension { + suspended: true, + suspended_at: MilliSecondsSinceUnixEpoch::now().get().into(), + suspended_by: suspending_user.to_string(), + }), + ); + } + + /// Unsuspend account, placing it in a read-write state + pub async fn unsuspend_account(&self, user_id: &UserId) { + self.db.userid_suspension.remove(user_id); + } + + /// Locks an account, preventing it being used until it is unlocked. + pub async fn lock_account(&self, user_id: &UserId, locking_user: &UserId) { + // NOTE: Locking is basically just suspension with a more severe effect, + // so we'll just re-use the suspension data structure to store the lock state. + let suspension = self + .db + .userid_lock + .get(user_id) + .await + .deserialized::() + .unwrap_or_else(|_| UserSuspension { + suspended: true, + suspended_at: MilliSecondsSinceUnixEpoch::now().get().into(), + suspended_by: locking_user.to_string(), + }); + + self.db.userid_lock.raw_put(user_id, Json(suspension)); + } + + /// Unlocks an account, allowing the user to log in and use it again. + pub async fn unlock_account(&self, user_id: &UserId) { self.db.userid_lock.remove(user_id); } + + /// Check if the provided user ID belongs to an existing (possibly + /// deactivated) account on this homeserver. + #[inline] + pub async fn exists(&self, user_id: &UserId) -> bool { + self.services.globals.user_is_local(user_id) + && self.db.userid_password.get(user_id).await.is_ok() + } + + /// Check if account is deactivated (has an empty password). Returns a + /// NotFound error if the user does not exist. + pub async fn is_deactivated(&self, user_id: &UserId) -> conduwuit::Result { + self.db + .userid_password + .get(user_id) + .map_ok(|val| val.is_empty()) + .map_err(|_| err!(Request(NotFound("User does not exist.")))) + .await + } + + /// Check if account is suspended. Returns false if the user does not exist. + pub async fn is_suspended(&self, user_id: &UserId) -> conduwuit::Result { + match self + .db + .userid_suspension + .get(user_id) + .await + .deserialized::() + { + | Ok(s) => Ok(s.suspended), + | Err(e) => + if e.is_not_found() { + Ok(false) + } else { + Err(e) + }, + } + } + + /// Returns true if the user is locked. Returns false if the user does not + /// exist or is not locked. + pub async fn is_locked(&self, user_id: &UserId) -> conduwuit::Result { + match self + .db + .userid_lock + .get(user_id) + .await + .deserialized::() + { + | Ok(s) => Ok(s.suspended), + | Err(e) => + if e.is_not_found() { + Ok(false) + } else { + Err(e) + }, + } + } + + /// Disables login for a user, preventing them from creating new devices, + /// but allows them to continue using their existing sessions unimpeded. + pub fn disable_login(&self, user_id: &UserId) { + self.db.userid_logindisabled.insert(user_id, ""); + } + + /// Re-enables login for a user, allowing them to create new devices again. + pub fn enable_login(&self, user_id: &UserId) { self.db.userid_logindisabled.remove(user_id); } + + /// Returns true if the target user's login is disabled. + pub async fn is_login_disabled(&self, user_id: &UserId) -> bool { + self.db + .userid_logindisabled + .exists(user_id.as_str()) + .await + .is_ok() + } + + /// Check if account is active (not deactivated) + pub async fn is_active(&self, user_id: &UserId) -> bool { + !self.is_deactivated(user_id).await.unwrap_or(true) + } + + /// Check if account is a local user, and is active (not deactivated) + pub async fn is_active_local(&self, user_id: &UserId) -> bool { + self.services.globals.user_is_local(user_id) && self.is_active(user_id).await + } + + /// Returns the number of users registered on this server, including + /// deactivated users. + #[inline] + pub async fn count(&self) -> usize { self.db.userid_password.count().await } + + /// Find out which user an access token belongs to. Will panic if the access + /// token is empty. + pub async fn find_from_token( + &self, + token: &str, + ) -> conduwuit::Result<(OwnedUserId, OwnedDeviceId)> { + assert!(!token.is_empty(), "Empty access token"); + self.db.token_userdeviceid.get(token).await.deserialized() + } + + /// Returns an iterator over all users on this homeserver. + pub fn stream(&self) -> impl Stream + Send { + self.db.userid_password.keys().ignore_err() + } + + /// Returns a list of active local users. + /// + /// A user account is considered `local` if the associated password is not + /// empty. + pub fn list_local_users(&self) -> impl Stream + Send + '_ { + self.db + .userid_password + .stream() + .ignore_err() + .ready_filter_map(|(u, p): (OwnedUserId, &[u8])| (!p.is_empty()).then_some(u)) + } + + /// Set a user's password. + pub fn set_password(&self, user_id: &UserId, password: Option) { + if let Some(hash) = password { + self.db.userid_password.insert(user_id, hash.0); + } else { + self.db.userid_password.insert(user_id, b""); + } + } + + /// Check a user's password. + pub async fn check_password( + &self, + user_id: &UserId, + password: &str, + ) -> conduwuit::Result { + let (hash, user_id): (String, OwnedUserId) = + if let Ok(hash) = self.db.userid_password.get(user_id).await.deserialized() { + (hash, user_id.to_owned()) + } else { + // We also check the lowercased version of the user ID to handle legacy user IDs + // better + let lowercase_user_id = UserId::parse(user_id.as_str().to_lowercase()).unwrap(); + + if let Ok(hash) = self + .db + .userid_password + .get(lowercase_user_id.as_str()) + .await + .deserialized() + { + (hash, lowercase_user_id) + } else { + return Err!(Request(Forbidden("This user cannot log in with a password."))); + } + }; + + if hash.is_empty() { + return Err!(Request(UserDeactivated("This user is deactivated"))); + } + + utils::hash::verify_password(password, &hash) + .inspect_err(|e| debug_error!("{e}")) + .map_err(|_| err!(Request(Forbidden("Invalid identifier or password."))))?; + + Ok(user_id) + } + + /// Creates an OpenID token, which can be used to prove that a user has + /// access to an account (primarily for integrations) + pub fn create_openid_token(&self, user_id: &UserId, token: &str) -> conduwuit::Result { + use std::num::Saturating as Sat; + + let expires_in = self.services.server.config.openid_token_ttl; + let expires_at = Sat(utils::millis_since_unix_epoch()) + Sat(expires_in) * Sat(1000); + + let mut value = expires_at.0.to_be_bytes().to_vec(); + value.extend_from_slice(user_id.as_bytes()); + + self.db + .openidtoken_expiresatuserid + .insert(token.as_bytes(), value.as_slice()); + + Ok(expires_in) + } + + /// Find out which user an OpenID access token belongs to. + pub async fn find_from_openid_token(&self, token: &str) -> conduwuit::Result { + let Ok(value) = self.db.openidtoken_expiresatuserid.get(token).await else { + return Err!(Request(Unauthorized("OpenID token is unrecognised"))); + }; + + let (expires_at_bytes, user_bytes) = value.split_at(0_u64.to_be_bytes().len()); + let expires_at = + u64::from_be_bytes(expires_at_bytes.try_into().map_err(|e| { + err!(Database("expires_at in openid_userid is invalid u64. {e}")) + })?); + + if expires_at < utils::millis_since_unix_epoch() { + debug_warn!("OpenID token is expired, removing"); + self.db.openidtoken_expiresatuserid.remove(token.as_bytes()); + + return Err!(Request(Unauthorized("OpenID token is expired"))); + } + + let user_string = utils::string_from_bytes(user_bytes) + .map_err(|e| err!(Database("User ID in openid_userid is invalid unicode. {e}")))?; + + OwnedUserId::try_from(user_string) + .map_err(|e| err!(Database("User ID in openid_userid is invalid. {e}"))) + } + + /// Creates a short-lived login token, which can be used to log in using the + /// `m.login.token` mechanism. + pub fn create_login_token(&self, user_id: &UserId, token: &str) -> u64 { + use std::num::Saturating as Sat; + + let expires_in = self.services.server.config.login_token_ttl; + let expires_at = Sat(utils::millis_since_unix_epoch()) + Sat(expires_in); + + let value = (expires_at.0, user_id); + self.db.logintoken_expiresatuserid.raw_put(token, value); + + expires_in + } + + /// Find out which user a login token belongs to. + /// Removes the token to prevent double-use attacks. + pub async fn find_from_login_token(&self, token: &str) -> conduwuit::Result { + let Ok(value) = self.db.logintoken_expiresatuserid.get(token).await else { + return Err!(Request(Forbidden("Login token is unrecognised"))); + }; + let (expires_at, user_id): (u64, OwnedUserId) = value.deserialized()?; + + if expires_at < utils::millis_since_unix_epoch() { + trace!(%user_id, ?token, "Removing expired login token"); + + self.db.logintoken_expiresatuserid.remove(token); + + return Err!(Request(Forbidden("Login token is expired"))); + } + + self.db.logintoken_expiresatuserid.remove(token); + + Ok(user_id) + } +} diff --git a/src/service/users/device.rs b/src/service/users/device.rs new file mode 100644 index 000000000..9c5ff4524 --- /dev/null +++ b/src/service/users/device.rs @@ -0,0 +1,321 @@ +use std::net::IpAddr; + +use conduwuit::{ + Err, utils, + utils::{ReadyExt, stream::TryIgnore}, +}; +use database::{Deserialized, Ignore, Interfix, Json}; +use futures::{Stream, StreamExt}; +use ruma::{ + DeviceId, MilliSecondsSinceUnixEpoch, OwnedDeviceId, OwnedUserId, UserId, + api::client::device::Device, events::AnyToDeviceEvent, serde::Raw, uint, +}; +use serde_json::json; + +use crate::users::increment; + +impl super::Service { + /// Adds a new device to a user. The user must exist, otherwise InvalidParam + /// is returned. + pub async fn create_device( + &self, + user_id: &UserId, + device_id: &DeviceId, + token: &str, + initial_device_display_name: Option, + client_ip: Option, + ) -> conduwuit::Result<()> { + if !self.exists(user_id).await { + return Err!(Request(InvalidParam(error!( + "Called create_device for non-existent user {user_id}" + )))); + } + + let key = (user_id, device_id); + let mut device = Device::new(device_id.into()); + device.display_name = initial_device_display_name; + device.last_seen_ip = client_ip; + device.last_seen_ts = Some(MilliSecondsSinceUnixEpoch::now()); + + increment(&self.db.userid_devicelistversion, user_id.as_bytes()); + self.db.userdeviceid_metadata.put(key, Json(device)); + self.set_token(user_id, device_id, token).await + } + + /// Removes a device from a user. + pub async fn remove_device(&self, user_id: &UserId, device_id: &DeviceId) { + // Remove dehydrated device if this is the dehydrated device + let _: conduwuit::Result<_> = self + .remove_dehydrated_device(user_id, Some(device_id)) + .await; + + let userdeviceid = (user_id, device_id); + + // Remove tokens + if let Ok(old_token) = self.db.userdeviceid_token.qry(&userdeviceid).await { + self.db.userdeviceid_token.del(userdeviceid); + self.db.token_userdeviceid.remove(&old_token); + } + + // Remove todevice events + let prefix = (user_id, device_id, Interfix); + self.db + .todeviceid_events + .keys_prefix_raw(&prefix) + .ignore_err() + .ready_for_each(|key| self.db.todeviceid_events.remove(key)) + .await; + + // TODO: Remove onetimekeys + + increment(&self.db.userid_devicelistversion, user_id.as_bytes()); + + self.db.userdeviceid_metadata.del(userdeviceid); + self.mark_device_key_update(user_id).await; + } + + /// Returns an iterator over all device ids of this user. + pub fn all_device_ids<'a>( + &'a self, + user_id: &'a UserId, + ) -> impl Stream + Send + 'a { + let prefix = (user_id, Interfix); + self.db + .userdeviceid_metadata + .keys_prefix(&prefix) + .ignore_err() + .map(|(_, device_id): (Ignore, OwnedDeviceId)| device_id) + } + + /// Gets the access token associated with a device. + pub async fn get_token( + &self, + user_id: &UserId, + device_id: &DeviceId, + ) -> conduwuit::Result { + let key = (user_id, device_id); + self.db.userdeviceid_token.qry(&key).await.deserialized() + } + + /// Generate a unique access token that doesn't collide with existing tokens + pub async fn generate_unique_token(&self) -> String { + loop { + let token = utils::random_string(32); + + // Check for collision with existing appservice and user tokens + let (appservice, usr) = tokio::join!( + self.services.appservice.find_from_token(&token), + self.db.token_userdeviceid.get(&token) + ); + if appservice.is_ok() || usr.is_ok() { + continue; + } + + return token; + } + } + + /// Replaces the access token of one device. + pub async fn set_token( + &self, + user_id: &UserId, + device_id: &DeviceId, + token: &str, + ) -> conduwuit::Result<()> { + let key = (user_id, device_id); + if self.db.userdeviceid_metadata.qry(&key).await.is_err() { + return Err!(Database(error!( + %user_id, + %device_id, + "User does not exist or device has no metadata." + ))); + } + + // Check for token collision with appservices + if self + .services + .appservice + .find_from_token(token) + .await + .is_ok() + { + return Err!(Request(InvalidParam( + "Token conflicts with an existing appservice token" + ))); + } + + // Remove old token + if let Ok(old_token) = self.db.userdeviceid_token.qry(&key).await { + self.db.token_userdeviceid.remove(&old_token); + // It will be removed from userdeviceid_token by the insert later + } + + // Assign token to user device combination + self.db.userdeviceid_token.put_raw(key, token); + self.db.token_userdeviceid.raw_put(token, key); + + Ok(()) + } + + /// Pushes a new to-device event into a device's inbox. + pub async fn add_to_device_event( + &self, + sender: &UserId, + target_user_id: &UserId, + target_device_id: &DeviceId, + event_type: &str, + content: serde_json::Value, + ) { + let count = self.services.globals.next_count().unwrap(); + + let key = (target_user_id, target_device_id, count); + self.db.todeviceid_events.put( + key, + Json(json!({ + "type": event_type, + "sender": sender, + "content": content, + })), + ); + } + + /// Gets all to-device events between the two counts. + pub fn get_to_device_events<'a>( + &'a self, + user_id: &'a UserId, + device_id: &'a DeviceId, + since: Option, + to: Option, + ) -> impl Stream)> + Send + 'a { + type Key = (OwnedUserId, OwnedDeviceId, u64); + + let from = (user_id, device_id, since.map_or(0, |since| since.saturating_add(1))); + + self.db + .todeviceid_events + .stream_from(&from) + .ignore_err() + .ready_take_while(move |((user_id_, device_id_, count), _): &(Key, _)| { + user_id == *user_id_ + && device_id == *device_id_ + && to.is_none_or(|to| *count <= to) + }) + .map(|((_, _, count), event)| (count, event)) + } + + /// Removes to-device events from the target device's inbox, until the given + /// count. + pub async fn remove_to_device_events( + &self, + user_id: &UserId, + device_id: &DeviceId, + until: Until, + ) where + Until: Into> + Send, + { + type Key = (OwnedUserId, OwnedDeviceId, u64); + + let until = until.into().unwrap_or(u64::MAX); + let from = (user_id, device_id, until); + self.db + .todeviceid_events + .rev_keys_from(&from) + .ignore_err() + .ready_take_while(move |(user_id_, device_id_, _): &Key| { + user_id == *user_id_ && device_id == *device_id_ + }) + .ready_for_each(|key: Key| { + self.db.todeviceid_events.del(key); + }) + .await; + } + + /// Updates device metadata and increments the device list version. + pub async fn update_device_metadata( + &self, + user_id: &UserId, + device_id: &DeviceId, + device: &Device, + ) -> conduwuit::Result<()> { + increment(&self.db.userid_devicelistversion, user_id.as_bytes()); + self.update_device_metadata_no_increment(user_id, device_id, device) + } + + /// Updates device metadata without incrementing the device list version. + /// This is namely used for updating the last_seen_ip and last_seen_ts + /// values, as those do not need a device list version bump due to them not + /// being relevant to other consumers. + fn update_device_metadata_no_increment( + &self, + user_id: &UserId, + device_id: &DeviceId, + device: &Device, + ) -> conduwuit::Result<()> { + let key = (user_id, device_id); + self.db.userdeviceid_metadata.put(key, Json(device)); + + Ok(()) + } + + /// Updates the last seen timestamp for a device. Silently does nothing if + /// the last update was less than 10 seconds ago, or the device does not + /// exist. + pub async fn update_device_last_seen( + &self, + user_id: &UserId, + device_id: Option<&DeviceId>, + ip: IpAddr, + ) { + let now = MilliSecondsSinceUnixEpoch::now(); + if let Some(device_id) = device_id { + if let Ok(mut device) = self.get_device_metadata(user_id, device_id).await { + device.last_seen_ip = Some(ip.to_string()); + // If the last update was less than 10 seconds ago, don't update the timestamp + if let Some(prev) = device.last_seen_ts { + if now.get().saturating_sub(prev.get()) < uint!(10_000) { + return; + } + } + device.last_seen_ts = Some(now); + + self.update_device_metadata_no_increment(user_id, device_id, &device) + .ok(); + } + } + } + + /// Get device metadata. + pub async fn get_device_metadata( + &self, + user_id: &UserId, + device_id: &DeviceId, + ) -> conduwuit::Result { + self.db + .userdeviceid_metadata + .qry(&(user_id, device_id)) + .await + .deserialized() + } + + /// Gets the most recent device list version for a user. + pub async fn get_devicelist_version(&self, user_id: &UserId) -> conduwuit::Result { + self.db + .userid_devicelistversion + .get(user_id) + .await + .deserialized() + } + + /// Gets metadata for all devices belonging to the target user. + pub fn all_devices_metadata<'a>( + &'a self, + user_id: &'a UserId, + ) -> impl Stream + Send + 'a { + let key = (user_id, Interfix); + self.db + .userdeviceid_metadata + .stream_prefix(&key) + .ignore_err() + .map(|(_, val): (Ignore, Device)| val) + } +} diff --git a/src/service/users/filters.rs b/src/service/users/filters.rs new file mode 100644 index 000000000..bbe7aff29 --- /dev/null +++ b/src/service/users/filters.rs @@ -0,0 +1,26 @@ +use conduwuit::utils; +use database::{Deserialized, Json}; +use ruma::{UserId, api::client::filter::FilterDefinition}; + +impl super::Service { + /// Creates a new sync filter. Returns the filter id. + pub fn create_filter(&self, user_id: &UserId, filter: &FilterDefinition) -> String { + let filter_id = utils::random_string(4); + + // TODO: filters should be de-duplicated and also not per-user + let key = (user_id, &filter_id); + self.db.userfilterid_filter.put(key, Json(filter)); + + filter_id + } + + /// Fetches a filter from a filter ID belonging to a user. + pub async fn get_filter( + &self, + user_id: &UserId, + filter_id: &str, + ) -> conduwuit::Result { + let key = (user_id, filter_id); + self.db.userfilterid_filter.qry(&key).await.deserialized() + } +} diff --git a/src/service/users/keys.rs b/src/service/users/keys.rs new file mode 100644 index 000000000..b6699f88c --- /dev/null +++ b/src/service/users/keys.rs @@ -0,0 +1,532 @@ +use std::collections::BTreeMap; + +use conduwuit::{ + Err, Error, Result, err, + utils::{ReadyExt, stream::TryIgnore, string::Unquoted}, +}; +use database::{Deserialized, Ignore, Json}; +use futures::{Stream, StreamExt, TryFutureExt}; +use ruma::{ + DeviceId, OneTimeKeyAlgorithm, OneTimeKeyId, OneTimeKeyName, OwnedKeyId, OwnedOneTimeKeyId, + OwnedUserId, RoomId, UInt, UserId, + api::error::ErrorKind, + encryption::{CrossSigningKey, DeviceKeys, OneTimeKey}, + serde::Raw, +}; + +use crate::users::{clean_signatures, parse_master_key, parse_user_signing_key}; + +impl super::Service { + /// Adds a single one-time key to a device. + pub async fn add_one_time_key( + &self, + user_id: &UserId, + device_id: &DeviceId, + one_time_key_key: &OneTimeKeyId, + one_time_key_value: &Raw, + ) -> Result { + // All devices have metadata + // Only existing devices should be able to call this, but we shouldn't assert + // either... + let key = (user_id, device_id); + if self.db.userdeviceid_metadata.qry(&key).await.is_err() { + return Err!(Database(error!( + %user_id, + %device_id, + "User does not exist or device has no metadata." + ))); + } + + let mut key = user_id.as_bytes().to_vec(); + key.push(0xFF); + key.extend_from_slice(device_id.as_bytes()); + key.push(0xFF); + // TODO: Use DeviceKeyId::to_string when it's available (and update everything, + // because there are no wrapping quotation marks anymore) + key.extend_from_slice( + serde_json::to_string(one_time_key_key) + .expect("DeviceKeyId::to_string always works") + .as_bytes(), + ); + + self.db + .onetimekeyid_onetimekeys + .raw_put(key, Json(one_time_key_value)); + + let count = self.services.globals.next_count().unwrap(); + self.db.userid_lastonetimekeyupdate.raw_put(user_id, count); + + Ok(()) + } + + /// Save a fallback key for the given user, device, and algorithm + /// This key will replace an existing fallback key. + pub async fn add_fallback_key( + &self, + user_id: &UserId, + device_id: &DeviceId, + fallback_key_id: &OneTimeKeyId, + fallback_key: &Raw, + used: bool, + ) -> Result { + // All devices have metadata + // Only existing devices should be able to call this, but we shouldn't assert + // either... + let key = (user_id, device_id); + if self.db.userdeviceid_metadata.qry(&key).await.is_err() { + return Err!(Database(error!( + %user_id, + %device_id, + "User does not exist or device has no metadata." + ))); + } + + // There is one fallback key slot per user, per device, per algorithm + // Therefore we use this as the DB key for this column + let db_key = (user_id, device_id, fallback_key_id.algorithm()); + + self.db + .fallbackkeyid_fallbackkey + .put(db_key, (used, fallback_key_id.as_str(), Json(fallback_key))); + + Ok(()) + } + + /// Returns the timestamp at when the last OTK update for the user was, or 0 + /// if the keys have never been updated. + pub async fn last_one_time_keys_update(&self, user_id: &UserId) -> u64 { + self.db + .userid_lastonetimekeyupdate + .get(user_id) + .await + .deserialized() + .unwrap_or(0) + } + + /// Consumes a one-time key belonging to the device of a given algorithm. If + /// no one-time keys are available, a fallback key is returned instead (if + /// available). If neither an OTK nor fallback key are available, NotFound + /// is returned. + pub async fn take_one_time_key( + &self, + user_id: &UserId, + device_id: &DeviceId, + key_algorithm: &OneTimeKeyAlgorithm, + ) -> Result<(OwnedKeyId, Raw)> { + let count = self.services.globals.next_count()?.to_be_bytes(); + self.db.userid_lastonetimekeyupdate.insert(user_id, count); + + let mut prefix = user_id.as_bytes().to_vec(); + prefix.push(0xFF); + prefix.extend_from_slice(device_id.as_bytes()); + prefix.push(0xFF); + prefix.push(b'"'); // Annoying quotation mark + prefix.extend_from_slice(key_algorithm.as_ref().as_bytes()); + prefix.push(b':'); + + let one_time_key = self + .db + .onetimekeyid_onetimekeys + .raw_stream_prefix(&prefix) + .ignore_err() + .next() + .await + .map(|(key, val)| { + self.db.onetimekeyid_onetimekeys.remove(key); + + let key = key + .rsplit(|&b| b == 0xFF) + .next() + .ok_or_else(|| err!(Database("OneTimeKeyId in db is invalid."))) + .unwrap(); + + let key = serde_json::from_slice(key) + .map_err(|e| err!(Database("OneTimeKeyId in db is invalid. {e}"))) + .unwrap(); + + let val = serde_json::from_slice(val) + .map_err(|e| err!(Database("OneTimeKeys in db are invalid. {e}"))) + .unwrap(); + + (key, val) + }); + + if let Some(result) = one_time_key { + return Ok(result); + } + + // No one-time key has been found. Look for a fallback key. + + let db_key = (user_id, device_id, key_algorithm); + + let fallback_key = self + .db + .fallbackkeyid_fallbackkey + .qry(&db_key) + .await + .ok() + .and_then(|handle| { + handle + .deserialized::<(bool, OwnedOneTimeKeyId, Raw)>() + .ok() + }); + + if let Some((used, fallback_key_id, fallback_key_value)) = fallback_key { + if !used { + // write the key to the database again to mark it as used + self.add_fallback_key( + user_id, + device_id, + &fallback_key_id, + &fallback_key_value, + true, + ) + .await?; + } + return Ok((fallback_key_id, fallback_key_value)); + } + + Err(err!(Request(NotFound("No one-time key or fallback key found")))) + } + + /// Returns the number of one-time keys the given device has. Does not count + /// fallback keys. + pub async fn count_one_time_keys( + &self, + user_id: &UserId, + device_id: &DeviceId, + ) -> BTreeMap { + type KeyVal<'a> = ((Ignore, Ignore, &'a Unquoted), Ignore); + + let mut algorithm_counts = BTreeMap::::new(); + let query = (user_id, device_id); + self.db + .onetimekeyid_onetimekeys + .stream_prefix(&query) + .ignore_err() + .ready_for_each(|((_, _, device_key_id), _): KeyVal<'_>| { + let one_time_key_id: &OneTimeKeyId = device_key_id + .as_str() + .try_into() + .expect("Invalid DeviceKeyID in database"); + + let count: &mut UInt = algorithm_counts + .entry(one_time_key_id.algorithm()) + .or_default(); + + *count = count.saturating_add(1_u32.into()); + }) + .await; + + algorithm_counts + } + + /// Returns a list of *unused* fallback key types. + pub async fn list_unused_fallback_key_types( + &self, + user_id: &UserId, + device_id: &DeviceId, + ) -> Vec { + type KeyVal = ((String, String, OneTimeKeyAlgorithm), (bool, String, Ignore)); + + let mut query = user_id.as_bytes().to_vec(); + query.push(0xFF); + query.extend_from_slice(device_id.as_bytes()); + query.push(0xFF); + + let mut unused_algorithms = Vec::new(); + + self.db + .fallbackkeyid_fallbackkey + .stream_prefix(&query) + .ignore_err() + .ready_for_each(|((_, _, fallback_key_algorithm), (used, ..)): KeyVal| { + if !used { + unused_algorithms.push(fallback_key_algorithm); + } + }) + .await; + + unused_algorithms + } + + /// Adds device identity keys to a device, overwriting existing ones if they + /// exist. Dispatches a device key update. + pub async fn add_device_keys( + &self, + user_id: &UserId, + device_id: &DeviceId, + device_keys: &Raw, + ) { + let key = (user_id, device_id); + + self.db.keyid_key.put(key, Json(device_keys)); + self.mark_device_key_update(user_id).await; + } + + /// Adds cross-signing keys for the given user. If notify is true, a device + /// key update is dispatched. + pub async fn add_cross_signing_keys( + &self, + user_id: &UserId, + master_key: &Option>, + self_signing_key: &Option>, + user_signing_key: &Option>, + notify: bool, + ) -> Result<()> { + let mut prefix = user_id.as_bytes().to_vec(); + prefix.push(0xFF); + + if let Some(master_key) = master_key { + let (master_key_key, _) = parse_master_key(user_id, master_key)?; + + self.db + .keyid_key + .insert(&master_key_key, master_key.json().get().as_bytes()); + + self.db + .userid_masterkeyid + .insert(user_id.as_bytes(), &master_key_key); + } + + // Self-signing key + if let Some(self_signing_key) = self_signing_key { + let mut self_signing_key_ids = self_signing_key + .deserialize() + .map_err(|e| err!(Request(InvalidParam("Invalid self signing key: {e:?}"))))? + .keys + .into_values(); + + let self_signing_key_id = self_signing_key_ids.next().ok_or(Error::BadRequest( + ErrorKind::InvalidParam, + "Self signing key contained no key.", + ))?; + + if self_signing_key_ids.next().is_some() { + return Err(Error::BadRequest( + ErrorKind::InvalidParam, + "Self signing key contained more than one key.", + )); + } + + let mut self_signing_key_key = prefix.clone(); + self_signing_key_key.extend_from_slice(self_signing_key_id.as_bytes()); + + self.db + .keyid_key + .insert(&self_signing_key_key, self_signing_key.json().get().as_bytes()); + + self.db + .userid_selfsigningkeyid + .insert(user_id.as_bytes(), &self_signing_key_key); + } + + // User-signing key + if let Some(user_signing_key) = user_signing_key { + let user_signing_key_id = parse_user_signing_key(user_signing_key)?; + + let user_signing_key_key = (user_id, &user_signing_key_id); + self.db + .keyid_key + .put_raw(user_signing_key_key, user_signing_key.json().get().as_bytes()); + + self.db + .userid_usersigningkeyid + .raw_put(user_id, user_signing_key_key); + } + + if notify { + self.mark_device_key_update(user_id).await; + } + + Ok(()) + } + + /// Appends a new signature to the given key ID. Dispatches a new device key + /// update. + pub async fn sign_key( + &self, + target_id: &UserId, + key_id: &str, + signature: (String, String), + sender_id: &UserId, + ) -> Result { + let key = (target_id, key_id); + + let mut cross_signing_key: serde_json::Value = self + .db + .keyid_key + .qry(&key) + .await + .map_err(|_| err!(Request(InvalidParam("Tried to sign nonexistent key"))))? + .deserialized() + .map_err(|e| err!(Database(debug_warn!("key in keyid_key is invalid: {e:?}"))))?; + + let signatures = cross_signing_key + .get_mut("signatures") + .ok_or_else(|| { + err!(Database(debug_warn!("key in keyid_key has no signatures field"))) + })? + .as_object_mut() + .ok_or_else(|| { + err!(Database(debug_warn!("key in keyid_key has invalid signatures field."))) + })? + .entry(sender_id.to_string()) + .or_insert_with(|| serde_json::Map::new().into()); + + signatures + .as_object_mut() + .ok_or_else(|| { + err!(Database(debug_warn!("signatures in keyid_key for a user is invalid."))) + })? + .insert(signature.0, signature.1.into()); + + let key = (target_id, key_id); + self.db.keyid_key.put(key, Json(cross_signing_key)); + + self.mark_device_key_update(target_id).await; + + Ok(()) + } + + /// Returns a stream of changed keys between the two counts. + #[inline] + pub fn keys_changed<'a>( + &'a self, + user_id: &'a UserId, + from: Option, + to: Option, + ) -> impl Stream + Send + 'a { + self.keys_changed_user_or_room(user_id.as_str(), from, to) + .map(|(user_id, ..)| user_id) + } + + /// Returns a stream of changed keys between the two counts in the given + /// room specifically. + #[inline] + pub fn room_keys_changed<'a>( + &'a self, + room_id: &'a RoomId, + from: Option, + to: Option, + ) -> impl Stream + Send + 'a { + self.keys_changed_user_or_room(room_id.as_str(), from, to) + } + + fn keys_changed_user_or_room<'a>( + &'a self, + user_or_room_id: &'a str, + from: Option, + to: Option, + ) -> impl Stream + Send + 'a { + type KeyVal<'a> = ((&'a str, u64), OwnedUserId); + + let from = from.unwrap_or(0); + let to = to.unwrap_or(u64::MAX); + let start = (user_or_room_id, from.saturating_add(1)); + self.db + .keychangeid_userid + .stream_from(&start) + .ignore_err() + .ready_take_while(move |((prefix, count), _): &KeyVal<'_>| { + *prefix == user_or_room_id && *count <= to + }) + .map(|((_, count), user_id): KeyVal<'_>| (user_id, count)) + } + + /// Marks that a user's device keys have been updated, so that other users + /// can be notified of the change. + pub async fn mark_device_key_update(&self, user_id: &UserId) { + let count = self.services.globals.next_count().unwrap(); + + self.services + .state_cache + .rooms_joined(user_id) + // Don't send key updates to unencrypted rooms + .filter_map(async |room_id| { + if self.services.state_accessor.is_encrypted_room(&room_id).await { + Some(room_id) + } else { + None + } + }) + .ready_for_each(|room_id| { + let key = (room_id, count); + self.db.keychangeid_userid.put_raw(key, user_id); + }) + .await; + + let key = (user_id, count); + self.db.keychangeid_userid.put_raw(key, user_id); + } + + /// Returns the device identity keys for a given device. + pub async fn get_device_keys<'a>( + &'a self, + user_id: &'a UserId, + device_id: &DeviceId, + ) -> Result> { + let key_id = (user_id, device_id); + self.db.keyid_key.qry(&key_id).await.deserialized() + } + + /// Gets a specific cross-signing key. + pub async fn get_key( + &self, + key_id: &[u8], + sender_user: Option<&UserId>, + user_id: &UserId, + allowed_signatures: &F, + ) -> Result> + where + F: Fn(&UserId) -> bool + Send + Sync, + { + let key: serde_json::Value = self.db.keyid_key.get(key_id).await.deserialized()?; + + let cleaned = clean_signatures(key, sender_user, user_id, allowed_signatures)?; + let raw_value = serde_json::value::to_raw_value(&cleaned)?; + Ok(Raw::from_json(raw_value)) + } + + /// Gets a user's master cross-signing key. + pub async fn get_master_key( + &self, + sender_user: Option<&UserId>, + user_id: &UserId, + allowed_signatures: &F, + ) -> Result> + where + F: Fn(&UserId) -> bool + Send + Sync, + { + let key_id = self.db.userid_masterkeyid.get(user_id).await?; + + self.get_key(&key_id, sender_user, user_id, allowed_signatures) + .await + } + + /// Gets a user's self-signing cross-signing key. + pub async fn get_self_signing_key( + &self, + sender_user: Option<&UserId>, + user_id: &UserId, + allowed_signatures: &F, + ) -> Result> + where + F: Fn(&UserId) -> bool + Send + Sync, + { + let key_id = self.db.userid_selfsigningkeyid.get(user_id).await?; + + self.get_key(&key_id, sender_user, user_id, allowed_signatures) + .await + } + + /// Gets a user's user-signing cross-signing key. + pub async fn get_user_signing_key(&self, user_id: &UserId) -> Result> { + self.db + .userid_usersigningkeyid + .get(user_id) + .and_then(|key_id| self.db.keyid_key.get(&*key_id)) + .await + .deserialized() + } +} diff --git a/src/service/users/mod.rs b/src/service/users/mod.rs index 429facfd1..58916b001 100644 --- a/src/service/users/mod.rs +++ b/src/service/users/mod.rs @@ -1,30 +1,19 @@ +pub(super) mod account; pub(super) mod dehydrated_device; +pub(super) mod device; +pub(super) mod filters; +pub(super) mod keys; +pub(super) mod profile; -use std::{collections::BTreeMap, mem, net::IpAddr, sync::Arc}; +use std::{mem, sync::Arc}; use conduwuit::{ - Err, Error, Result, Server, debug_error, debug_warn, err, trace, - utils::{self, ReadyExt, stream::TryIgnore, string::Unquoted}, + Err, Error, Result, Server, err, + utils::{self}, }; -use database::{Deserialized, Ignore, Interfix, Json, Map}; -use futures::{Stream, StreamExt, TryFutureExt}; -use ruma::{ - DeviceId, MilliSecondsSinceUnixEpoch, OneTimeKeyAlgorithm, OneTimeKeyId, OneTimeKeyName, - OwnedDeviceId, OwnedKeyId, OwnedMxcUri, OwnedOneTimeKeyId, OwnedUserId, RoomId, UInt, UserId, - api::{ - client::{device::Device, filter::FilterDefinition}, - error::ErrorKind, - }, - encryption::{CrossSigningKey, DeviceKeys, OneTimeKey}, - events::{ - AnyToDeviceEvent, GlobalAccountDataEventType, ignored_user_list::IgnoredUserListEvent, - }, - serde::Raw, - uint, -}; -use ruminuwuity::invite_permission_config::{FilterLevel, InvitePermissionConfigEvent}; +use database::Map; +use ruma::{UserId, api::error::ErrorKind, encryption::CrossSigningKey, serde::Raw}; use serde::{Deserialize, Serialize}; -use serde_json::json; use crate::{Dep, account_data, admin, appservice, globals, rooms}; @@ -138,1298 +127,6 @@ impl crate::Service for Service { fn name(&self) -> &str { crate::service::make_name(std::module_path!()) } } -impl Service { - /// Returns true/false based on whether the recipient/receiving user has - /// ignored the sender. - pub async fn user_is_ignored(&self, sender_user: &UserId, recipient_user: &UserId) -> bool { - self.services - .account_data - .get_global(recipient_user, GlobalAccountDataEventType::IgnoredUserList) - .await - .is_ok_and(|ignored: IgnoredUserListEvent| { - ignored - .content - .ignored_users - .keys() - .any(|blocked_user| blocked_user == sender_user) - }) - } - - /// Returns the recipient's filter level for an invite from the sender. - pub async fn invite_filter_level( - &self, - sender_user: &UserId, - recipient_user: &UserId, - ) -> FilterLevel { - if self.user_is_ignored(sender_user, recipient_user).await { - FilterLevel::Ignore - } else { - let (stable, unstable) = tokio::join!( - self.services - .account_data - .get_global::( - recipient_user, - GlobalAccountDataEventType::InvitePermissionConfig - ), - self.services - .account_data - .get_global::( - recipient_user, - "org.matrix.msc4155.invite_permission_config".into() - ) // TODO: MSC4155 probably needs upstreaming to ruma at some point - ); - if stable.is_err() && unstable.is_err() { - return FilterLevel::Allow; - } - stable - .unwrap_or_else(|_| unstable.unwrap()) - .content - .user_filter_level(sender_user) - } - } - - /// Check if a user is an admin - #[inline] - pub async fn is_admin(&self, user_id: &UserId) -> bool { - self.services.admin.user_is_admin(user_id).await - } - - /// Create a new user account on this homeserver. - #[inline] - pub async fn create(&self, user_id: &UserId, password: Option) -> Result<()> { - if !self.services.globals.user_is_local(user_id) && password.is_some() { - return Err!("Cannot create a nonlocal user with a set password"); - } - - self.set_password(user_id, password); - - Ok(()) - } - - // /// Create a new account for a local human or bot user. - // pub async fn create_local_account( - // &self, - // username: String, - // password: - // ) - - /// Deactivates an account, removing all of their device IDs and unsetting - /// their password. - pub async fn deactivate_account(&self, user_id: &UserId) -> Result<()> { - // Remove all associated devices - self.all_device_ids(user_id) - .for_each(async |device_id| self.remove_device(user_id, &device_id).await) - .await; - - // Set the password to "" to indicate a deactivated account. Hashes will never - // result in an empty string, so the user will not be able to log in again. - // Systems like changing the password without logging in should check if the - // account is deactivated. - self.set_password(user_id, None); - - // TODO: Unhook 3PID - Ok(()) - } - - /// Suspend account, placing it in a read-only state - pub async fn suspend_account(&self, user_id: &UserId, suspending_user: &UserId) { - self.db.userid_suspension.raw_put( - user_id, - Json(UserSuspension { - suspended: true, - suspended_at: MilliSecondsSinceUnixEpoch::now().get().into(), - suspended_by: suspending_user.to_string(), - }), - ); - } - - /// Unsuspend account, placing it in a read-write state - pub async fn unsuspend_account(&self, user_id: &UserId) { - self.db.userid_suspension.remove(user_id); - } - - /// Locks an account, preventing it being used until it is unlocked. - pub async fn lock_account(&self, user_id: &UserId, locking_user: &UserId) { - // NOTE: Locking is basically just suspension with a more severe effect, - // so we'll just re-use the suspension data structure to store the lock state. - let suspension = self - .db - .userid_lock - .get(user_id) - .await - .deserialized::() - .unwrap_or_else(|_| UserSuspension { - suspended: true, - suspended_at: MilliSecondsSinceUnixEpoch::now().get().into(), - suspended_by: locking_user.to_string(), - }); - - self.db.userid_lock.raw_put(user_id, Json(suspension)); - } - - /// Unlocks an account, allowing the user to log in and use it again. - pub async fn unlock_account(&self, user_id: &UserId) { self.db.userid_lock.remove(user_id); } - - /// Check if the provided user ID belongs to an existing (possibly - /// deactivated) account on this homeserver. - #[inline] - pub async fn exists(&self, user_id: &UserId) -> bool { - self.services.globals.user_is_local(user_id) - && self.db.userid_password.get(user_id).await.is_ok() - } - - /// Check if account is deactivated (has an empty password). Returns a - /// NotFound error if the user does not exist. - pub async fn is_deactivated(&self, user_id: &UserId) -> Result { - self.db - .userid_password - .get(user_id) - .map_ok(|val| val.is_empty()) - .map_err(|_| err!(Request(NotFound("User does not exist.")))) - .await - } - - /// Check if account is suspended. Returns false if the user does not exist. - pub async fn is_suspended(&self, user_id: &UserId) -> Result { - match self - .db - .userid_suspension - .get(user_id) - .await - .deserialized::() - { - | Ok(s) => Ok(s.suspended), - | Err(e) => - if e.is_not_found() { - Ok(false) - } else { - Err(e) - }, - } - } - - /// Returns true if the user is locked. Returns false if the user does not - /// exist or is not locked. - pub async fn is_locked(&self, user_id: &UserId) -> Result { - match self - .db - .userid_lock - .get(user_id) - .await - .deserialized::() - { - | Ok(s) => Ok(s.suspended), - | Err(e) => - if e.is_not_found() { - Ok(false) - } else { - Err(e) - }, - } - } - - /// Disables login for a user, preventing them from creating new devices, - /// but allows them to continue using their existing sessions unimpeded. - pub fn disable_login(&self, user_id: &UserId) { - self.db.userid_logindisabled.insert(user_id, ""); - } - - /// Re-enables login for a user, allowing them to create new devices again. - pub fn enable_login(&self, user_id: &UserId) { self.db.userid_logindisabled.remove(user_id); } - - /// Returns true if the target user's login is disabled. - pub async fn is_login_disabled(&self, user_id: &UserId) -> bool { - self.db - .userid_logindisabled - .exists(user_id.as_str()) - .await - .is_ok() - } - - /// Check if account is active (not deactivated) - pub async fn is_active(&self, user_id: &UserId) -> bool { - !self.is_deactivated(user_id).await.unwrap_or(true) - } - - /// Check if account is a local user, and is active (not deactivated) - pub async fn is_active_local(&self, user_id: &UserId) -> bool { - self.services.globals.user_is_local(user_id) && self.is_active(user_id).await - } - - /// Returns the number of users registered on this server, including - /// deactivated users. - #[inline] - pub async fn count(&self) -> usize { self.db.userid_password.count().await } - - /// Find out which user an access token belongs to. Will panic if the access - /// token is empty. - pub async fn find_from_token(&self, token: &str) -> Result<(OwnedUserId, OwnedDeviceId)> { - assert!(!token.is_empty(), "Empty access token"); - self.db.token_userdeviceid.get(token).await.deserialized() - } - - /// Returns an iterator over all users on this homeserver. - pub fn stream(&self) -> impl Stream + Send { - self.db.userid_password.keys().ignore_err() - } - - /// Returns a list of active local users. - /// - /// A user account is considered `local` if the associated password is not - /// empty. - pub fn list_local_users(&self) -> impl Stream + Send + '_ { - self.db - .userid_password - .stream() - .ignore_err() - .ready_filter_map(|(u, p): (OwnedUserId, &[u8])| (!p.is_empty()).then_some(u)) - } - - /// Set a user's password. - pub fn set_password(&self, user_id: &UserId, password: Option) { - if let Some(hash) = password { - self.db.userid_password.insert(user_id, hash.0); - } else { - self.db.userid_password.insert(user_id, b""); - } - } - - /// Check a user's password. - pub async fn check_password(&self, user_id: &UserId, password: &str) -> Result { - let (hash, user_id): (String, OwnedUserId) = - if let Ok(hash) = self.db.userid_password.get(user_id).await.deserialized() { - (hash, user_id.to_owned()) - } else { - // We also check the lowercased version of the user ID to handle legacy user IDs - // better - let lowercase_user_id = UserId::parse(user_id.as_str().to_lowercase()).unwrap(); - - if let Ok(hash) = self - .db - .userid_password - .get(lowercase_user_id.as_str()) - .await - .deserialized() - { - (hash, lowercase_user_id) - } else { - return Err!(Request(Forbidden("This user cannot log in with a password."))); - } - }; - - if hash.is_empty() { - return Err!(Request(UserDeactivated("This user is deactivated"))); - } - - utils::hash::verify_password(password, &hash) - .inspect_err(|e| debug_error!("{e}")) - .map_err(|_| err!(Request(Forbidden("Invalid identifier or password."))))?; - - Ok(user_id) - } - - /// Returns the displayname of a user on this homeserver. - pub async fn displayname(&self, user_id: &UserId) -> Result { - self.db.userid_displayname.get(user_id).await.deserialized() - } - - /// Sets a new displayname or removes it if displayname is None. You still - /// need to notify all rooms of this change. - pub fn set_displayname(&self, user_id: &UserId, displayname: Option) { - if let Some(displayname) = displayname { - self.db.userid_displayname.insert(user_id, displayname); - } else { - self.db.userid_displayname.remove(user_id); - } - } - - /// Get the `avatar_url` of a user. - pub async fn avatar_url(&self, user_id: &UserId) -> Result { - self.db.userid_avatarurl.get(user_id).await.deserialized() - } - - /// Sets a new avatar_url or removes it if avatar_url is None. - pub fn set_avatar_url(&self, user_id: &UserId, avatar_url: Option) { - match avatar_url { - | Some(avatar_url) => { - self.db.userid_avatarurl.insert(user_id, &avatar_url); - }, - | _ => { - self.db.userid_avatarurl.remove(user_id); - }, - } - } - - /// Adds a new device to a user. The user must exist, otherwise InvalidParam - /// is returned. - pub async fn create_device( - &self, - user_id: &UserId, - device_id: &DeviceId, - token: &str, - initial_device_display_name: Option, - client_ip: Option, - ) -> Result<()> { - if !self.exists(user_id).await { - return Err!(Request(InvalidParam(error!( - "Called create_device for non-existent user {user_id}" - )))); - } - - let key = (user_id, device_id); - let mut device = Device::new(device_id.into()); - device.display_name = initial_device_display_name; - device.last_seen_ip = client_ip; - device.last_seen_ts = Some(MilliSecondsSinceUnixEpoch::now()); - - increment(&self.db.userid_devicelistversion, user_id.as_bytes()); - self.db.userdeviceid_metadata.put(key, Json(device)); - self.set_token(user_id, device_id, token).await - } - - /// Removes a device from a user. - pub async fn remove_device(&self, user_id: &UserId, device_id: &DeviceId) { - // Remove dehydrated device if this is the dehydrated device - let _: Result<_> = self - .remove_dehydrated_device(user_id, Some(device_id)) - .await; - - let userdeviceid = (user_id, device_id); - - // Remove tokens - if let Ok(old_token) = self.db.userdeviceid_token.qry(&userdeviceid).await { - self.db.userdeviceid_token.del(userdeviceid); - self.db.token_userdeviceid.remove(&old_token); - } - - // Remove todevice events - let prefix = (user_id, device_id, Interfix); - self.db - .todeviceid_events - .keys_prefix_raw(&prefix) - .ignore_err() - .ready_for_each(|key| self.db.todeviceid_events.remove(key)) - .await; - - // TODO: Remove onetimekeys - - increment(&self.db.userid_devicelistversion, user_id.as_bytes()); - - self.db.userdeviceid_metadata.del(userdeviceid); - self.mark_device_key_update(user_id).await; - } - - /// Returns an iterator over all device ids of this user. - pub fn all_device_ids<'a>( - &'a self, - user_id: &'a UserId, - ) -> impl Stream + Send + 'a { - let prefix = (user_id, Interfix); - self.db - .userdeviceid_metadata - .keys_prefix(&prefix) - .ignore_err() - .map(|(_, device_id): (Ignore, OwnedDeviceId)| device_id) - } - - /// Gets the access token associated with a device. - pub async fn get_token(&self, user_id: &UserId, device_id: &DeviceId) -> Result { - let key = (user_id, device_id); - self.db.userdeviceid_token.qry(&key).await.deserialized() - } - - /// Generate a unique access token that doesn't collide with existing tokens - pub async fn generate_unique_token(&self) -> String { - loop { - let token = utils::random_string(32); - - // Check for collision with existing appservice and user tokens - let (appservice, usr) = tokio::join!( - self.services.appservice.find_from_token(&token), - self.db.token_userdeviceid.get(&token) - ); - if appservice.is_ok() || usr.is_ok() { - continue; - } - - return token; - } - } - - /// Replaces the access token of one device. - pub async fn set_token( - &self, - user_id: &UserId, - device_id: &DeviceId, - token: &str, - ) -> Result<()> { - let key = (user_id, device_id); - if self.db.userdeviceid_metadata.qry(&key).await.is_err() { - return Err!(Database(error!( - %user_id, - %device_id, - "User does not exist or device has no metadata." - ))); - } - - // Check for token collision with appservices - if self - .services - .appservice - .find_from_token(token) - .await - .is_ok() - { - return Err!(Request(InvalidParam( - "Token conflicts with an existing appservice token" - ))); - } - - // Remove old token - if let Ok(old_token) = self.db.userdeviceid_token.qry(&key).await { - self.db.token_userdeviceid.remove(&old_token); - // It will be removed from userdeviceid_token by the insert later - } - - // Assign token to user device combination - self.db.userdeviceid_token.put_raw(key, token); - self.db.token_userdeviceid.raw_put(token, key); - - Ok(()) - } - - /// Adds a single one-time key to a device. - pub async fn add_one_time_key( - &self, - user_id: &UserId, - device_id: &DeviceId, - one_time_key_key: &OneTimeKeyId, - one_time_key_value: &Raw, - ) -> Result { - // All devices have metadata - // Only existing devices should be able to call this, but we shouldn't assert - // either... - let key = (user_id, device_id); - if self.db.userdeviceid_metadata.qry(&key).await.is_err() { - return Err!(Database(error!( - %user_id, - %device_id, - "User does not exist or device has no metadata." - ))); - } - - let mut key = user_id.as_bytes().to_vec(); - key.push(0xFF); - key.extend_from_slice(device_id.as_bytes()); - key.push(0xFF); - // TODO: Use DeviceKeyId::to_string when it's available (and update everything, - // because there are no wrapping quotation marks anymore) - key.extend_from_slice( - serde_json::to_string(one_time_key_key) - .expect("DeviceKeyId::to_string always works") - .as_bytes(), - ); - - self.db - .onetimekeyid_onetimekeys - .raw_put(key, Json(one_time_key_value)); - - let count = self.services.globals.next_count().unwrap(); - self.db.userid_lastonetimekeyupdate.raw_put(user_id, count); - - Ok(()) - } - - /// Save a fallback key for the given user, device, and algorithm - /// This key will replace an existing fallback key. - pub async fn add_fallback_key( - &self, - user_id: &UserId, - device_id: &DeviceId, - fallback_key_id: &OneTimeKeyId, - fallback_key: &Raw, - used: bool, - ) -> Result { - // All devices have metadata - // Only existing devices should be able to call this, but we shouldn't assert - // either... - let key = (user_id, device_id); - if self.db.userdeviceid_metadata.qry(&key).await.is_err() { - return Err!(Database(error!( - %user_id, - %device_id, - "User does not exist or device has no metadata." - ))); - } - - // There is one fallback key slot per user, per device, per algorithm - // Therefore we use this as the DB key for this column - let db_key = (user_id, device_id, fallback_key_id.algorithm()); - - self.db - .fallbackkeyid_fallbackkey - .put(db_key, (used, fallback_key_id.as_str(), Json(fallback_key))); - - Ok(()) - } - - /// Returns the timestamp at when the last OTK update for the user was, or 0 - /// if the keys have never been updated. - pub async fn last_one_time_keys_update(&self, user_id: &UserId) -> u64 { - self.db - .userid_lastonetimekeyupdate - .get(user_id) - .await - .deserialized() - .unwrap_or(0) - } - - /// Consumes a one-time key belonging to the device of a given algorithm. If - /// no one-time keys are available, a fallback key is returned instead (if - /// available). If neither an OTK nor fallback key are available, NotFound - /// is returned. - pub async fn take_one_time_key( - &self, - user_id: &UserId, - device_id: &DeviceId, - key_algorithm: &OneTimeKeyAlgorithm, - ) -> Result<(OwnedKeyId, Raw)> { - let count = self.services.globals.next_count()?.to_be_bytes(); - self.db.userid_lastonetimekeyupdate.insert(user_id, count); - - let mut prefix = user_id.as_bytes().to_vec(); - prefix.push(0xFF); - prefix.extend_from_slice(device_id.as_bytes()); - prefix.push(0xFF); - prefix.push(b'"'); // Annoying quotation mark - prefix.extend_from_slice(key_algorithm.as_ref().as_bytes()); - prefix.push(b':'); - - let one_time_key = self - .db - .onetimekeyid_onetimekeys - .raw_stream_prefix(&prefix) - .ignore_err() - .next() - .await - .map(|(key, val)| { - self.db.onetimekeyid_onetimekeys.remove(key); - - let key = key - .rsplit(|&b| b == 0xFF) - .next() - .ok_or_else(|| err!(Database("OneTimeKeyId in db is invalid."))) - .unwrap(); - - let key = serde_json::from_slice(key) - .map_err(|e| err!(Database("OneTimeKeyId in db is invalid. {e}"))) - .unwrap(); - - let val = serde_json::from_slice(val) - .map_err(|e| err!(Database("OneTimeKeys in db are invalid. {e}"))) - .unwrap(); - - (key, val) - }); - - if let Some(result) = one_time_key { - return Ok(result); - } - - // No one-time key has been found. Look for a fallback key. - - let db_key = (user_id, device_id, key_algorithm); - - let fallback_key = self - .db - .fallbackkeyid_fallbackkey - .qry(&db_key) - .await - .ok() - .and_then(|handle| { - handle - .deserialized::<(bool, OwnedOneTimeKeyId, Raw)>() - .ok() - }); - - if let Some((used, fallback_key_id, fallback_key_value)) = fallback_key { - if !used { - // write the key to the database again to mark it as used - self.add_fallback_key( - user_id, - device_id, - &fallback_key_id, - &fallback_key_value, - true, - ) - .await?; - } - return Ok((fallback_key_id, fallback_key_value)); - } - - Err(err!(Request(NotFound("No one-time key or fallback key found")))) - } - - /// Returns the number of one-time keys the given device has. Does not count - /// fallback keys. - pub async fn count_one_time_keys( - &self, - user_id: &UserId, - device_id: &DeviceId, - ) -> BTreeMap { - type KeyVal<'a> = ((Ignore, Ignore, &'a Unquoted), Ignore); - - let mut algorithm_counts = BTreeMap::::new(); - let query = (user_id, device_id); - self.db - .onetimekeyid_onetimekeys - .stream_prefix(&query) - .ignore_err() - .ready_for_each(|((Ignore, Ignore, device_key_id), Ignore): KeyVal<'_>| { - let one_time_key_id: &OneTimeKeyId = device_key_id - .as_str() - .try_into() - .expect("Invalid DeviceKeyID in database"); - - let count: &mut UInt = algorithm_counts - .entry(one_time_key_id.algorithm()) - .or_default(); - - *count = count.saturating_add(1_u32.into()); - }) - .await; - - algorithm_counts - } - - /// Returns a list of *unused* fallback key types. - pub async fn list_unused_fallback_key_types( - &self, - user_id: &UserId, - device_id: &DeviceId, - ) -> Vec { - type KeyVal = ((String, String, OneTimeKeyAlgorithm), (bool, String, Ignore)); - - let mut query = user_id.as_bytes().to_vec(); - query.push(0xFF); - query.extend_from_slice(device_id.as_bytes()); - query.push(0xFF); - - let mut unused_algorithms = Vec::new(); - - self.db - .fallbackkeyid_fallbackkey - .stream_prefix(&query) - .ignore_err() - .ready_for_each(|((_, _, fallback_key_algorithm), (used, ..)): KeyVal| { - if !used { - unused_algorithms.push(fallback_key_algorithm); - } - }) - .await; - - unused_algorithms - } - - /// Adds device identity keys to a device, overwriting existing ones if they - /// exist. Dispatches a device key update. - pub async fn add_device_keys( - &self, - user_id: &UserId, - device_id: &DeviceId, - device_keys: &Raw, - ) { - let key = (user_id, device_id); - - self.db.keyid_key.put(key, Json(device_keys)); - self.mark_device_key_update(user_id).await; - } - - /// Adds cross-signing keys for the given user. If notify is true, a device - /// key update is dispatched. - pub async fn add_cross_signing_keys( - &self, - user_id: &UserId, - master_key: &Option>, - self_signing_key: &Option>, - user_signing_key: &Option>, - notify: bool, - ) -> Result<()> { - // TODO: Check signatures - let mut prefix = user_id.as_bytes().to_vec(); - prefix.push(0xFF); - - if let Some(master_key) = master_key { - let (master_key_key, _) = parse_master_key(user_id, master_key)?; - - self.db - .keyid_key - .insert(&master_key_key, master_key.json().get().as_bytes()); - - self.db - .userid_masterkeyid - .insert(user_id.as_bytes(), &master_key_key); - } - - // Self-signing key - if let Some(self_signing_key) = self_signing_key { - let mut self_signing_key_ids = self_signing_key - .deserialize() - .map_err(|e| err!(Request(InvalidParam("Invalid self signing key: {e:?}"))))? - .keys - .into_values(); - - let self_signing_key_id = self_signing_key_ids.next().ok_or(Error::BadRequest( - ErrorKind::InvalidParam, - "Self signing key contained no key.", - ))?; - - if self_signing_key_ids.next().is_some() { - return Err(Error::BadRequest( - ErrorKind::InvalidParam, - "Self signing key contained more than one key.", - )); - } - - let mut self_signing_key_key = prefix.clone(); - self_signing_key_key.extend_from_slice(self_signing_key_id.as_bytes()); - - self.db - .keyid_key - .insert(&self_signing_key_key, self_signing_key.json().get().as_bytes()); - - self.db - .userid_selfsigningkeyid - .insert(user_id.as_bytes(), &self_signing_key_key); - } - - // User-signing key - if let Some(user_signing_key) = user_signing_key { - let user_signing_key_id = parse_user_signing_key(user_signing_key)?; - - let user_signing_key_key = (user_id, &user_signing_key_id); - self.db - .keyid_key - .put_raw(user_signing_key_key, user_signing_key.json().get().as_bytes()); - - self.db - .userid_usersigningkeyid - .raw_put(user_id, user_signing_key_key); - } - - if notify { - self.mark_device_key_update(user_id).await; - } - - Ok(()) - } - - /// Appends a new signature to the given key ID. Dispatches a new device key - /// update. - pub async fn sign_key( - &self, - target_id: &UserId, - key_id: &str, - signature: (String, String), - sender_id: &UserId, - ) -> Result { - let key = (target_id, key_id); - - let mut cross_signing_key: serde_json::Value = self - .db - .keyid_key - .qry(&key) - .await - .map_err(|_| err!(Request(InvalidParam("Tried to sign nonexistent key"))))? - .deserialized() - .map_err(|e| err!(Database(debug_warn!("key in keyid_key is invalid: {e:?}"))))?; - - let signatures = cross_signing_key - .get_mut("signatures") - .ok_or_else(|| { - err!(Database(debug_warn!("key in keyid_key has no signatures field"))) - })? - .as_object_mut() - .ok_or_else(|| { - err!(Database(debug_warn!("key in keyid_key has invalid signatures field."))) - })? - .entry(sender_id.to_string()) - .or_insert_with(|| serde_json::Map::new().into()); - - signatures - .as_object_mut() - .ok_or_else(|| { - err!(Database(debug_warn!("signatures in keyid_key for a user is invalid."))) - })? - .insert(signature.0, signature.1.into()); - - let key = (target_id, key_id); - self.db.keyid_key.put(key, Json(cross_signing_key)); - - self.mark_device_key_update(target_id).await; - - Ok(()) - } - - /// Returns a stream of changed keys between the two counts. - #[inline] - pub fn keys_changed<'a>( - &'a self, - user_id: &'a UserId, - from: Option, - to: Option, - ) -> impl Stream + Send + 'a { - self.keys_changed_user_or_room(user_id.as_str(), from, to) - .map(|(user_id, ..)| user_id) - } - - /// Returns a stream of changed keys between the two counts in the given - /// room specifically. - #[inline] - pub fn room_keys_changed<'a>( - &'a self, - room_id: &'a RoomId, - from: Option, - to: Option, - ) -> impl Stream + Send + 'a { - self.keys_changed_user_or_room(room_id.as_str(), from, to) - } - - fn keys_changed_user_or_room<'a>( - &'a self, - user_or_room_id: &'a str, - from: Option, - to: Option, - ) -> impl Stream + Send + 'a { - type KeyVal<'a> = ((&'a str, u64), OwnedUserId); - - let from = from.unwrap_or(0); - let to = to.unwrap_or(u64::MAX); - let start = (user_or_room_id, from.saturating_add(1)); - self.db - .keychangeid_userid - .stream_from(&start) - .ignore_err() - .ready_take_while(move |((prefix, count), _): &KeyVal<'_>| { - *prefix == user_or_room_id && *count <= to - }) - .map(|((_, count), user_id): KeyVal<'_>| (user_id, count)) - } - - /// Marks that a user's device keys have been updated, so that other users - /// can be notified of the change. - pub async fn mark_device_key_update(&self, user_id: &UserId) { - let count = self.services.globals.next_count().unwrap(); - - self.services - .state_cache - .rooms_joined(user_id) - // Don't send key updates to unencrypted rooms - .filter_map(async |room_id| { - if self.services.state_accessor.is_encrypted_room(&room_id).await { - Some(room_id) - } else { - None - } - }) - .ready_for_each(|room_id| { - let key = (room_id, count); - self.db.keychangeid_userid.put_raw(key, user_id); - }) - .await; - - let key = (user_id, count); - self.db.keychangeid_userid.put_raw(key, user_id); - } - - /// Returns the device identity keys for a given device. - pub async fn get_device_keys<'a>( - &'a self, - user_id: &'a UserId, - device_id: &DeviceId, - ) -> Result> { - let key_id = (user_id, device_id); - self.db.keyid_key.qry(&key_id).await.deserialized() - } - - /// Gets a specific cross-signing key. - pub async fn get_key( - &self, - key_id: &[u8], - sender_user: Option<&UserId>, - user_id: &UserId, - allowed_signatures: &F, - ) -> Result> - where - F: Fn(&UserId) -> bool + Send + Sync, - { - let key: serde_json::Value = self.db.keyid_key.get(key_id).await.deserialized()?; - - let cleaned = clean_signatures(key, sender_user, user_id, allowed_signatures)?; - let raw_value = serde_json::value::to_raw_value(&cleaned)?; - Ok(Raw::from_json(raw_value)) - } - - /// Gets a user's master cross-signing key. - pub async fn get_master_key( - &self, - sender_user: Option<&UserId>, - user_id: &UserId, - allowed_signatures: &F, - ) -> Result> - where - F: Fn(&UserId) -> bool + Send + Sync, - { - let key_id = self.db.userid_masterkeyid.get(user_id).await?; - - self.get_key(&key_id, sender_user, user_id, allowed_signatures) - .await - } - - /// Gets a user's self-signing cross-signing key. - pub async fn get_self_signing_key( - &self, - sender_user: Option<&UserId>, - user_id: &UserId, - allowed_signatures: &F, - ) -> Result> - where - F: Fn(&UserId) -> bool + Send + Sync, - { - let key_id = self.db.userid_selfsigningkeyid.get(user_id).await?; - - self.get_key(&key_id, sender_user, user_id, allowed_signatures) - .await - } - - /// Gets a user's user-signing cross-signing key. - pub async fn get_user_signing_key(&self, user_id: &UserId) -> Result> { - self.db - .userid_usersigningkeyid - .get(user_id) - .and_then(|key_id| self.db.keyid_key.get(&*key_id)) - .await - .deserialized() - } - - /// Pushes a new to-device event into a device's inbox. - pub async fn add_to_device_event( - &self, - sender: &UserId, - target_user_id: &UserId, - target_device_id: &DeviceId, - event_type: &str, - content: serde_json::Value, - ) { - let count = self.services.globals.next_count().unwrap(); - - let key = (target_user_id, target_device_id, count); - self.db.todeviceid_events.put( - key, - Json(json!({ - "type": event_type, - "sender": sender, - "content": content, - })), - ); - } - - /// Gets all to-device events between the two counts. - pub fn get_to_device_events<'a>( - &'a self, - user_id: &'a UserId, - device_id: &'a DeviceId, - since: Option, - to: Option, - ) -> impl Stream)> + Send + 'a { - type Key = (OwnedUserId, OwnedDeviceId, u64); - - let from = (user_id, device_id, since.map_or(0, |since| since.saturating_add(1))); - - self.db - .todeviceid_events - .stream_from(&from) - .ignore_err() - .ready_take_while(move |((user_id_, device_id_, count), _): &(Key, _)| { - user_id == *user_id_ - && device_id == *device_id_ - && to.is_none_or(|to| *count <= to) - }) - .map(|((_, _, count), event)| (count, event)) - } - - /// Removes to-device events from the target device's inbox, until the given - /// count. - pub async fn remove_to_device_events( - &self, - user_id: &UserId, - device_id: &DeviceId, - until: Until, - ) where - Until: Into> + Send, - { - type Key = (OwnedUserId, OwnedDeviceId, u64); - - let until = until.into().unwrap_or(u64::MAX); - let from = (user_id, device_id, until); - self.db - .todeviceid_events - .rev_keys_from(&from) - .ignore_err() - .ready_take_while(move |(user_id_, device_id_, _): &Key| { - user_id == *user_id_ && device_id == *device_id_ - }) - .ready_for_each(|key: Key| { - self.db.todeviceid_events.del(key); - }) - .await; - } - - /// Updates device metadata and increments the device list version. - pub async fn update_device_metadata( - &self, - user_id: &UserId, - device_id: &DeviceId, - device: &Device, - ) -> Result<()> { - increment(&self.db.userid_devicelistversion, user_id.as_bytes()); - self.update_device_metadata_no_increment(user_id, device_id, device) - } - - /// Updates device metadata without incrementing the device list version. - /// This is namely used for updating the last_seen_ip and last_seen_ts - /// values, as those do not need a device list version bump due to them not - /// being relevant to other consumers. - fn update_device_metadata_no_increment( - &self, - user_id: &UserId, - device_id: &DeviceId, - device: &Device, - ) -> Result<()> { - let key = (user_id, device_id); - self.db.userdeviceid_metadata.put(key, Json(device)); - - Ok(()) - } - - /// Updates the last seen timestamp for a device. Silently does nothing if - /// the last update was less than 10 seconds ago, or the device does not - /// exist. - pub async fn update_device_last_seen( - &self, - user_id: &UserId, - device_id: Option<&DeviceId>, - ip: IpAddr, - ) { - let now = MilliSecondsSinceUnixEpoch::now(); - if let Some(device_id) = device_id { - if let Ok(mut device) = self.get_device_metadata(user_id, device_id).await { - device.last_seen_ip = Some(ip.to_string()); - // If the last update was less than 10 seconds ago, don't update the timestamp - if let Some(prev) = device.last_seen_ts { - if now.get().saturating_sub(prev.get()) < uint!(10_000) { - return; - } - } - device.last_seen_ts = Some(now); - - self.update_device_metadata_no_increment(user_id, device_id, &device) - .ok(); - } - } - } - - /// Get device metadata. - pub async fn get_device_metadata( - &self, - user_id: &UserId, - device_id: &DeviceId, - ) -> Result { - self.db - .userdeviceid_metadata - .qry(&(user_id, device_id)) - .await - .deserialized() - } - - /// Gets the most recent device list version for a user. - pub async fn get_devicelist_version(&self, user_id: &UserId) -> Result { - self.db - .userid_devicelistversion - .get(user_id) - .await - .deserialized() - } - - /// Gets metadata for all devices belonging to the target user. - pub fn all_devices_metadata<'a>( - &'a self, - user_id: &'a UserId, - ) -> impl Stream + Send + 'a { - let key = (user_id, Interfix); - self.db - .userdeviceid_metadata - .stream_prefix(&key) - .ignore_err() - .map(|(_, val): (Ignore, Device)| val) - } - - /// Creates a new sync filter. Returns the filter id. - pub fn create_filter(&self, user_id: &UserId, filter: &FilterDefinition) -> String { - let filter_id = utils::random_string(4); - - let key = (user_id, &filter_id); - self.db.userfilterid_filter.put(key, Json(filter)); - - filter_id - } - - /// Fetches a filter from a filter ID belonging to a user. - pub async fn get_filter( - &self, - user_id: &UserId, - filter_id: &str, - ) -> Result { - let key = (user_id, filter_id); - self.db.userfilterid_filter.qry(&key).await.deserialized() - } - - /// Creates an OpenID token, which can be used to prove that a user has - /// access to an account (primarily for integrations) - pub fn create_openid_token(&self, user_id: &UserId, token: &str) -> Result { - use std::num::Saturating as Sat; - - let expires_in = self.services.server.config.openid_token_ttl; - let expires_at = Sat(utils::millis_since_unix_epoch()) + Sat(expires_in) * Sat(1000); - - let mut value = expires_at.0.to_be_bytes().to_vec(); - value.extend_from_slice(user_id.as_bytes()); - - self.db - .openidtoken_expiresatuserid - .insert(token.as_bytes(), value.as_slice()); - - Ok(expires_in) - } - - /// Find out which user an OpenID access token belongs to. - pub async fn find_from_openid_token(&self, token: &str) -> Result { - let Ok(value) = self.db.openidtoken_expiresatuserid.get(token).await else { - return Err!(Request(Unauthorized("OpenID token is unrecognised"))); - }; - - let (expires_at_bytes, user_bytes) = value.split_at(0_u64.to_be_bytes().len()); - let expires_at = - u64::from_be_bytes(expires_at_bytes.try_into().map_err(|e| { - err!(Database("expires_at in openid_userid is invalid u64. {e}")) - })?); - - if expires_at < utils::millis_since_unix_epoch() { - debug_warn!("OpenID token is expired, removing"); - self.db.openidtoken_expiresatuserid.remove(token.as_bytes()); - - return Err!(Request(Unauthorized("OpenID token is expired"))); - } - - let user_string = utils::string_from_bytes(user_bytes) - .map_err(|e| err!(Database("User ID in openid_userid is invalid unicode. {e}")))?; - - OwnedUserId::try_from(user_string) - .map_err(|e| err!(Database("User ID in openid_userid is invalid. {e}"))) - } - - /// Creates a short-lived login token, which can be used to log in using the - /// `m.login.token` mechanism. - pub fn create_login_token(&self, user_id: &UserId, token: &str) -> u64 { - use std::num::Saturating as Sat; - - let expires_in = self.services.server.config.login_token_ttl; - let expires_at = Sat(utils::millis_since_unix_epoch()) + Sat(expires_in); - - let value = (expires_at.0, user_id); - self.db.logintoken_expiresatuserid.raw_put(token, value); - - expires_in - } - - /// Find out which user a login token belongs to. - /// Removes the token to prevent double-use attacks. - pub async fn find_from_login_token(&self, token: &str) -> Result { - let Ok(value) = self.db.logintoken_expiresatuserid.get(token).await else { - return Err!(Request(Forbidden("Login token is unrecognised"))); - }; - let (expires_at, user_id): (u64, OwnedUserId) = value.deserialized()?; - - if expires_at < utils::millis_since_unix_epoch() { - trace!(%user_id, ?token, "Removing expired login token"); - - self.db.logintoken_expiresatuserid.remove(token); - - return Err!(Request(Forbidden("Login token is expired"))); - } - - self.db.logintoken_expiresatuserid.remove(token); - - Ok(user_id) - } - - /// Gets a specific user profile key - pub async fn profile_key( - &self, - user_id: &UserId, - profile_key: &str, - ) -> Result { - let key = (user_id, profile_key); - self.db - .useridprofilekey_value - .qry(&key) - .await - .and_then(|handle| serde_json::from_slice(&handle).map_err(Into::into)) - } - - /// Gets all the user's profile keys and values in an iterator - pub fn all_profile_keys<'a>( - &'a self, - user_id: &'a UserId, - ) -> impl Stream + 'a + Send { - type KeyVal<'a> = ((Ignore, String), &'a [u8]); - - let prefix = (user_id, Interfix); - self.db - .useridprofilekey_value - .stream_prefix(&prefix) - .ignore_err() - .map(|((_, key), value): KeyVal<'_>| Ok((key, serde_json::from_slice(value)?))) - .ignore_err() - } - - /// Sets a new profile key value, removes the key if value is None - pub fn set_profile_key( - &self, - user_id: &UserId, - profile_key: &str, - profile_key_value: Option, - ) { - let key = (user_id, profile_key); - - if let Some(value) = profile_key_value { - self.db.useridprofilekey_value.put(key, Json(value)); - } else { - self.db.useridprofilekey_value.del(key); - } - } - - /// Clears all profile data for a user, including display name and avatar - /// url. - pub async fn clear_profile(&self, user_id: &UserId) { - self.set_displayname(user_id, None); - self.set_avatar_url(user_id, None); - self.all_profile_keys(user_id) - .ready_for_each(|(key, _)| self.set_profile_key(user_id, &key, None)) - .await; - } -} - pub fn parse_master_key( user_id: &UserId, master_key: &Raw, diff --git a/src/service/users/profile.rs b/src/service/users/profile.rs new file mode 100644 index 000000000..a4ed3ee3d --- /dev/null +++ b/src/service/users/profile.rs @@ -0,0 +1,94 @@ +use conduwuit::utils::{ReadyExt, stream::TryIgnore}; +use database::{Deserialized, Ignore, Interfix, Json}; +use futures::{Stream, StreamExt}; +use ruma::{OwnedMxcUri, UserId}; + +impl super::Service { + /// Returns the displayname of a user on this homeserver. + pub async fn displayname(&self, user_id: &UserId) -> conduwuit::Result { + self.db.userid_displayname.get(user_id).await.deserialized() + } + + /// Sets a new displayname or removes it if displayname is None. You still + /// need to notify all rooms of this change. + pub fn set_displayname(&self, user_id: &UserId, displayname: Option) { + if let Some(displayname) = displayname { + self.db.userid_displayname.insert(user_id, displayname); + } else { + self.db.userid_displayname.remove(user_id); + } + } + + /// Get the `avatar_url` of a user. + pub async fn avatar_url(&self, user_id: &UserId) -> conduwuit::Result { + self.db.userid_avatarurl.get(user_id).await.deserialized() + } + + /// Sets a new avatar_url or removes it if avatar_url is None. + pub fn set_avatar_url(&self, user_id: &UserId, avatar_url: Option) { + match avatar_url { + | Some(avatar_url) => { + self.db.userid_avatarurl.insert(user_id, &avatar_url); + }, + | _ => { + self.db.userid_avatarurl.remove(user_id); + }, + } + } + + /// Gets a specific user profile key + pub async fn profile_key( + &self, + user_id: &UserId, + profile_key: &str, + ) -> conduwuit::Result { + let key = (user_id, profile_key); + self.db + .useridprofilekey_value + .qry(&key) + .await + .and_then(|handle| serde_json::from_slice(&handle).map_err(Into::into)) + } + + /// Gets all the user's profile keys and values in an iterator + pub fn all_profile_keys<'a>( + &'a self, + user_id: &'a UserId, + ) -> impl Stream + 'a + Send { + type KeyVal<'a> = ((Ignore, String), &'a [u8]); + + let prefix = (user_id, Interfix); + self.db + .useridprofilekey_value + .stream_prefix(&prefix) + .ignore_err() + .map(|((_, key), value): KeyVal<'_>| Ok((key, serde_json::from_slice(value)?))) + .ignore_err() + } + + /// Sets a new profile key value, removes the key if value is None + pub fn set_profile_key( + &self, + user_id: &UserId, + profile_key: &str, + profile_key_value: Option, + ) { + let key = (user_id, profile_key); + + if let Some(value) = profile_key_value { + self.db.useridprofilekey_value.put(key, Json(value)); + } else { + self.db.useridprofilekey_value.del(key); + } + } + + /// Clears all profile data for a user, including display name and avatar + /// url. + pub async fn clear_profile(&self, user_id: &UserId) { + self.set_displayname(user_id, None); + self.set_avatar_url(user_id, None); + self.all_profile_keys(user_id) + .ready_for_each(|(key, _)| self.set_profile_key(user_id, &key, None)) + .await; + } +} From 124241238e05959a1a55537ea5b3e87a4108e449 Mon Sep 17 00:00:00 2001 From: timedout Date: Fri, 26 Jun 2026 07:58:13 +0100 Subject: [PATCH 66/70] refactor: Rename "synapse"-related federation methods to "slow" I won't even comment on how stupid that naming was. --- src/service/client/mod.rs | 4 ++-- src/service/federation/execute.rs | 6 +++--- src/service/rooms/membership/mod.rs | 2 +- src/service/sending/mod.rs | 7 ++----- 4 files changed, 8 insertions(+), 11 deletions(-) diff --git a/src/service/client/mod.rs b/src/service/client/mod.rs index 8492714da..24c839213 100644 --- a/src/service/client/mod.rs +++ b/src/service/client/mod.rs @@ -13,7 +13,7 @@ pub struct Service { pub extern_media: reqwest::Client, pub well_known: reqwest::Client, pub federation: reqwest::Client, - pub synapse: reqwest::Client, + pub federation_slow: reqwest::Client, pub sender: reqwest::Client, pub appservice: reqwest::Client, pub pusher: reqwest::Client, @@ -85,7 +85,7 @@ impl crate::Service for Service { .redirect(redirect::Policy::limited(3)) .build()?, - synapse: base(config)? + federation_slow: base(config)? .dns_resolver(resolver.resolver.hooked.clone()) .connect_timeout(Duration::from_secs(config.federation_conn_timeout)) .read_timeout(Duration::from_secs(config.federation_timeout.saturating_mul(6))) diff --git a/src/service/federation/execute.rs b/src/service/federation/execute.rs index ee588df7c..5a0d7a3dd 100644 --- a/src/service/federation/execute.rs +++ b/src/service/federation/execute.rs @@ -39,8 +39,8 @@ impl super::Service { } /// Like execute() but with a very large timeout - #[tracing::instrument(skip_all, name = "synapse", level = "debug")] - pub async fn execute_synapse<'i, T>( + #[tracing::instrument(skip_all, name = "request_slow", level = "debug")] + pub async fn execute_slow<'i, T>( &self, dest: &ServerName, request: T, @@ -52,7 +52,7 @@ impl super::Service { > + Debug + Send, { - let client = &self.services.client.synapse; + let client = &self.services.client.federation_slow; self.execute_signed(client, dest, request).await } diff --git a/src/service/rooms/membership/mod.rs b/src/service/rooms/membership/mod.rs index 0cbad3509..5ef56d1d5 100644 --- a/src/service/rooms/membership/mod.rs +++ b/src/service/rooms/membership/mod.rs @@ -394,7 +394,7 @@ impl Service { let send_join_response = match self .services .sending - .send_synapse_request(&remote_server, send_join_request) + .send_slow_federation_request(&remote_server, send_join_request) .await { | Ok(response) => response, diff --git a/src/service/sending/mod.rs b/src/service/sending/mod.rs index 026a7132f..b43098af3 100644 --- a/src/service/sending/mod.rs +++ b/src/service/sending/mod.rs @@ -311,7 +311,7 @@ impl Service { /// Like send_federation_request() but with a very large timeout #[inline] - pub async fn send_synapse_request<'i, T>( + pub async fn send_slow_federation_request<'i, T>( &self, dest: &ServerName, request: T, @@ -323,10 +323,7 @@ impl Service { > + Debug + Send, { - self.services - .federation - .execute_synapse(dest, request) - .await + self.services.federation.execute_slow(dest, request).await } /// Send an unauthenticated federation request with no X-Matrix header. From 064028689cf26f83416f527d3db29f438b4b0921 Mon Sep 17 00:00:00 2001 From: timedout Date: Fri, 26 Jun 2026 08:08:58 +0100 Subject: [PATCH 67/70] style: Add documentation to execute module of sender service --- src/service/federation/execute.rs | 40 ++++++++++++++++++++++++++----- 1 file changed, 34 insertions(+), 6 deletions(-) diff --git a/src/service/federation/execute.rs b/src/service/federation/execute.rs index 5a0d7a3dd..70bef2f0e 100644 --- a/src/service/federation/execute.rs +++ b/src/service/federation/execute.rs @@ -1,8 +1,9 @@ -use std::{borrow::Cow, fmt::Debug, mem}; +use std::{borrow::Cow, fmt::Debug, mem, time::Instant}; use bytes::Bytes; use conduwuit::{ - Err, Error, Result, debug, debug_error, debug_warn, err, trace, utils::response::LimitReadExt, + Err, Error, Result, debug, debug_error, debug_info, debug_warn, err, trace, + utils::response::LimitReadExt, }; use ipaddress::IPAddress; use reqwest::{Client, Method, Request, Response, Url}; @@ -20,7 +21,7 @@ use ruma::{ use crate::{SUPPORTED_VERSIONS, resolver::actual::ActualDest}; impl super::Service { - /// Sends a request to a federation server + /// Sends a signed request to a remote server over federation. #[tracing::instrument(skip_all, name = "request", level = "debug")] pub async fn execute<'i, T>( &self, @@ -38,7 +39,8 @@ impl super::Service { self.execute_signed(client, dest, request).await } - /// Like execute() but with a very large timeout + /// Sends a signed request to a remote server over federation, with a + /// significantly higher timeout to account for slow endpoints. #[tracing::instrument(skip_all, name = "request_slow", level = "debug")] pub async fn execute_slow<'i, T>( &self, @@ -56,6 +58,8 @@ impl super::Service { self.execute_signed(client, dest, request).await } + /// Sends an unsigned (unauthenticated) request to a remote server over + /// federation. pub async fn execute_unauthenticated<'i, T>( &self, dest: &ServerName, @@ -73,6 +77,8 @@ impl super::Service { self.execute_on(client, dest, request, ()).await } + /// Sends a signed request to a remote server over federation, given the + /// specific client. pub async fn execute_signed<'i, T>( &self, client: &Client, @@ -94,6 +100,10 @@ impl super::Service { self.execute_on(client, dest, request, authentication).await } + /// Prepares and sends a federation request with the given client. + /// If federation is disabled, an error is returned here. Likewise, if the + /// destination is forbidden. + /// Preparation includes resolving the server before sending the request. #[tracing::instrument( name = "fed", level = "info", @@ -133,6 +143,10 @@ impl super::Service { self.perform::(dest, &actual, request, client).await } + /// Actually sends the federation request, handling the response before + /// returning. + /// + /// Timing info and request logs are available when debug logs are enabled. async fn perform( &self, dest: &ServerName, @@ -146,8 +160,20 @@ impl super::Service { let url = request.url().clone(); let method = request.method().clone(); - debug!(%method, %url, "Sending request"); - match client.execute(request).await { + debug_info!( + "Sending request: {} matrix-federation://{dest}{}", + method.as_str(), + url.path(), + ); + let start = Instant::now(); + let response = client.execute(request).await; + debug_info!( + "Request finished in {:?}: {} matrix-federation://{dest}{}", + start.elapsed(), + method.as_str(), + url.path(), + ); + match response { | Ok(response) => self.handle_response::(dest, actual, &method, &url, response) .await, @@ -167,6 +193,8 @@ impl super::Service { Ok(()) } + /// Handles a successful response from a remote server, reading and + /// deserializing the response body (if any). async fn handle_response( &self, dest: &ServerName, From e4922814b9be9834d18bd1a78e1aedbc96c324a9 Mon Sep 17 00:00:00 2001 From: timedout Date: Fri, 26 Jun 2026 08:48:43 +0100 Subject: [PATCH 68/70] refactor: Add better documentation to sender service, add additional logging traces This is a surprise tool that will help us later! --- src/core/utils/time/exponential_backoff.rs | 9 ++ src/service/sending/antispam.rs | 26 +++-- src/service/sending/sender.rs | 116 +++++++++++++-------- 3 files changed, 99 insertions(+), 52 deletions(-) diff --git a/src/core/utils/time/exponential_backoff.rs b/src/core/utils/time/exponential_backoff.rs index 682c2592e..76f0be2b2 100644 --- a/src/core/utils/time/exponential_backoff.rs +++ b/src/core/utils/time/exponential_backoff.rs @@ -27,3 +27,12 @@ pub fn continue_exponential_backoff( let min = cmp::min(min, max); elapsed < min } + +/// Determines the minimum number of backoff seconds +#[must_use] +pub fn min_exp_backoff_duration(min: u64, max: u64, retries: u32) -> Duration { + let min = Duration::from_secs(min) + .saturating_mul(retries) + .saturating_mul(retries); + Duration::from_secs(max).min(min) +} diff --git a/src/service/sending/antispam.rs b/src/service/sending/antispam.rs index 9cc1b52c3..7aa1b248f 100644 --- a/src/service/sending/antispam.rs +++ b/src/service/sending/antispam.rs @@ -1,7 +1,7 @@ -use std::{borrow::Cow, fmt::Debug, mem}; +use std::{borrow::Cow, fmt::Debug, mem, time::Instant}; use bytes::BytesMut; -use conduwuit::{Err, Result, debug_error, err, utils, utils::response::LimitReadExt, warn}; +use conduwuit::{Err, Result, debug_info, err, utils, utils::response::LimitReadExt, warn}; use reqwest::Client; use ruma::api::{ IncomingResponse, OutgoingRequest, @@ -32,10 +32,20 @@ where .map(BytesMut::freeze); let reqwest_request = reqwest::Request::try_from(http_request)?; + let method = reqwest_request.method().clone(); + let url = reqwest_request.url().clone(); + debug_info!("Sending request to appservice: {} {}", method, url); + let start = Instant::now(); let mut response = client.execute(reqwest_request).await.map_err(|e| { - warn!("Could not send request to antispam: {e:?}"); - e + err!(BadServerResponse(error!(?e, "Failed to contact antispam service."))) })?; + debug_info!( + "Received response (HTTP {}) from antispam service in {:?}: {} {}", + response.status(), + start.elapsed().as_millis(), + method, + url, + ); // reqwest::Response -> http::Response conversion let status = response.status(); @@ -49,10 +59,14 @@ where .expect("http::response::Builder is usable"), ); - let body = response.limit_read(65535).await?; // TODO: handle timeout + let body = response.limit_read(65535).await.map_err(|e| { + err!(BadServerResponse(error!( + ?e, + "Failed to read response body from antispam service." + ))) + })?; // TODO: handle timeout if !status.is_success() { - debug_error!("Antispam response bytes: {:?}", utils::string_from_bytes(&body)); return match status { | http::StatusCode::FORBIDDEN => Err!(Request(Forbidden("Request was rejected by antispam service.",))), diff --git a/src/service/sending/sender.rs b/src/service/sending/sender.rs index 21d5c5547..0c44d66d9 100644 --- a/src/service/sending/sender.rs +++ b/src/service/sending/sender.rs @@ -1,6 +1,7 @@ use std::{ collections::{BTreeMap, HashMap, HashSet}, fmt::Debug, + future::ready, sync::{ Arc, atomic::{AtomicU64, AtomicUsize, Ordering}, @@ -9,7 +10,10 @@ use std::{ }; use base64::{Engine as _, engine::general_purpose::URL_SAFE_NO_PAD}; -use conduwuit::info; +use conduwuit::{ + debug_error, debug_info, debug_warn, info, + utils::time::exponential_backoff::min_exp_backoff_duration, +}; use conduwuit_core::{ Error, Event, Result, at, debug, err, error, result::LogErr, @@ -80,9 +84,11 @@ impl Service { let mut statuses: CurTransactionStatus = CurTransactionStatus::new(); let mut futures: SendingFutures<'_> = FuturesUnordered::new(); + info!(shard = id, "Running startup netburst"); self.startup_netburst(id, &mut futures, &mut statuses) .boxed() .await; + info!(shard = id, "Finished startup netburst"); self.work_loop(id, &mut futures, &mut statuses).await; @@ -127,6 +133,8 @@ impl Service { } } + /// Handles a response, calling the appropriate sub-handler depending on if + /// the request was successful or not. #[tracing::instrument(name = "response", level = "debug", skip_all)] async fn handle_response<'a>( &'a self, @@ -140,6 +148,7 @@ impl Service { } } + /// Handles a response error, incrementing the backoff factor. fn handle_response_err(dest: Destination, statuses: &mut CurTransactionStatus, e: &Error) { debug!(dest = ?dest, "{e:?}"); statuses.entry(dest).and_modify(|e| { @@ -154,11 +163,11 @@ impl Service { }); } - #[allow(clippy::needless_pass_by_ref_mut)] + /// Handles a successful response, queueing up more sends if necessary. async fn handle_response_ok<'a>( &'a self, dest: &Destination, - futures: &mut SendingFutures<'a>, + futures: &SendingFutures<'a>, statuses: &mut CurTransactionStatus, ) { let _cork = self.db.db.cork(); @@ -183,12 +192,14 @@ impl Service { } } - #[allow(clippy::needless_pass_by_ref_mut)] + /// Handles sending a new request by queuing up any new events. If there are + /// no new events to send the destination, it is removed from the status map + /// instead. #[tracing::instrument(name = "request", level = "debug", skip_all)] async fn handle_request<'a>( &'a self, msg: Msg, - futures: &mut SendingFutures<'a>, + futures: &SendingFutures<'a>, statuses: &mut CurTransactionStatus, ) { let iv = vec![(msg.queue_id, msg.event)]; @@ -201,6 +212,8 @@ impl Service { } } + /// Cleans up dangling futures by waiting for them to complete, aborting + /// early if the shutdown timeout is reached. #[tracing::instrument( name = "finish", level = "info", @@ -218,7 +231,7 @@ impl Service { let now = Instant::now(); let deadline = now.checked_add(timeout).unwrap_or(now); loop { - trace!("Waiting for {} requests to complete...", futures.len()); + info!("Waiting for {} requests to complete...", futures.len()); select! { () = sleep_until(deadline) => return, response = futures.next() => match response { @@ -230,6 +243,8 @@ impl Service { } } + /// Performs startup netburst, which sends any previously queued up but not + /// yet sent requests immediately. #[tracing::instrument( name = "netburst", level = "debug", @@ -255,7 +270,12 @@ impl Service { let entry = txns.entry(dest.clone()).or_default(); if self.server.config.startup_netburst_keep >= 0 && entry.len() >= keep { - warn!("Dropping unsent event {dest:?} {:?}", String::from_utf8_lossy(&key)); + warn!( + startup_netburst_keep = self.server.config.startup_netburst_keep, + queue_size = entry.len(), + "Dropping unsent event to {dest:?}: {:?}", + String::from_utf8_lossy(&key) + ); self.db.delete_active_request(&key); } else { entry.push(event); @@ -270,8 +290,9 @@ impl Service { } } + /// Selects any new events to send to the given destination. #[tracing::instrument( - name = "select",, + name = "select", level = "debug", skip_all, fields( @@ -285,7 +306,7 @@ impl Service { new_events: Vec, // Events we want to send: event and full key statuses: &mut CurTransactionStatus, ) -> Result>> { - let (allow, retry) = self.select_events_current(dest, statuses)?; + let (allow, retry) = self.should_attempt_send(dest, statuses)?; // Nothing can be done for this remote, bail out. if !allow { @@ -328,29 +349,34 @@ impl Service { Ok(Some(events)) } - fn select_events_current( + /// Determines if the destination should be retried, honouring exponential + /// backoff. + fn should_attempt_send( &self, dest: &Destination, statuses: &mut CurTransactionStatus, ) -> Result<(bool, bool)> { let (mut allow, mut retry) = (true, false); statuses - .entry(dest.clone()) // TODO: can we avoid cloning? + .entry(dest.clone()) .and_modify(|e| match e { - TransactionStatus::Failed(tries, time) => { + | TransactionStatus::Failed(tries, time) => { // Fail if a request has failed recently (exponential backoff) let min = self.server.config.sender_timeout; let max = self.server.config.sender_retry_backoff_limit; if continue_exponential_backoff_secs(min, max, time.elapsed(), *tries) && !matches!(dest, Destination::Appservice(_)) { + let retry_after = min_exp_backoff_duration(min, max, *tries); + debug_warn!("Not retrying destination for another {retry_after:?}"); allow = false; } else { retry = true; + debug_info!("Will retry destination ({} tries)", *tries); *e = TransactionStatus::Retrying(*tries); } }, - TransactionStatus::Running | TransactionStatus::Retrying(_) => { + | TransactionStatus::Running | TransactionStatus::Retrying(_) => { allow = false; // already running }, }) @@ -359,11 +385,8 @@ impl Service { Ok((allow, retry)) } - #[tracing::instrument( - name = "edus",, - level = "debug", - skip_all, - )] + /// Selects a number of EDUs to send to the target. + #[tracing::instrument(name = "edus", level = "debug", skip_all)] async fn select_edus(&self, server_name: &ServerName) -> Result<(EduVec, u64)> { // selection window let since = self.db.get_latest_educount(server_name).await; @@ -640,8 +663,14 @@ impl Service { Some(buf) } + /// Actually sends fetched events to the destination. + /// If the events vec is empty, a stub response is returned, and a debug + /// warning is logged. fn send_events(&self, dest: Destination, events: Vec) -> SendingFuture<'_> { - debug_assert!(!events.is_empty(), "sending empty transaction"); + if events.is_empty() { + debug_warn!(?dest, "Attempted to send empty transaction!"); + return std::future::ready(Ok(dest)).boxed(); + } match dest { | Destination::Federation(server) => self.send_events_dest_federation(server, events).boxed(), @@ -708,9 +737,6 @@ impl Service { let txn_id = &*URL_SAFE_NO_PAD.encode(txn_hash); - //debug_assert!(pdu_jsons.len() + edu_jsons.len() > 0, "sending empty - // transaction"); - let mut request = ruma::api::appservice::event::push_events::v1::Request::new(txn_id.into(), pdu_jsons); request.ephemeral = edu_jsons; @@ -843,12 +869,28 @@ impl Service { ); request.pdus = pdus; request.edus = edus; + let pdu_count = request.pdus.len(); + let edu_count = request.edus.len(); + debug_info!( + %txn_id, + pdus=pdu_count, + edus=edu_count, + "Sending transaction to remote" + ); + let start = Instant::now(); let result = self .services .federation .execute_signed(&self.services.client.sender, &server, request) .await; + debug_info!( + %txn_id, + pdus=pdu_count, + edus=edu_count, + elapsed=?start.elapsed(), + "Finished sending transaction" + ); for (event_id, result) in result.iter().flat_map(|resp| resp.pdus.iter()) { if let Err(e) = result { @@ -868,7 +910,8 @@ impl Service { } } - /// This does not return a full `Pdu` it is only to satisfy ruma's types. + /// Converts and sanitises an outgoing PDU object for federation + /// transmission. pub async fn convert_to_outgoing_federation_event( &self, mut pdu_json: CanonicalJsonObject, @@ -877,31 +920,12 @@ impl Service { .get_mut("unsigned") .and_then(|val| val.as_object_mut()) { + // TODO: remove all unsigned data over federation. unsigned.remove("transaction_id"); } - - // room v3 and above removed the "event_id" field from remote PDU format - if let Some(room_id) = pdu_json - .get("room_id") - .and_then(|val| RoomId::parse(val.as_str()?).ok()) - { - match self.services.state.get_room_version(&room_id).await { - | Ok(room_version_id) => match room_version_id { - | RoomVersionId::V1 | RoomVersionId::V2 => {}, - | _ => _ = pdu_json.remove("event_id"), - }, - | Err(_) => _ = pdu_json.remove("event_id"), - } - } else { - pdu_json.remove("event_id"); - } - - // TODO: another option would be to convert it to a canonical string to validate - // size and return a Result> - // serde_json::from_str::>( - // ruma::serde::to_canonical_json_string(pdu_json).expect("CanonicalJson is - // valid serde_json::Value"), ) - // .expect("Raw::from_value always works") + // We don't support any room versions that have the event_id in the PDU JSON, + // so we can safely remove it here. + pdu_json.remove("event_id"); to_raw_value(&pdu_json).expect("CanonicalJson is valid serde_json::Value") } From 2d65d4749818eb18b2fec1c6351c5c75dd657c8a Mon Sep 17 00:00:00 2001 From: timedout Date: Fri, 26 Jun 2026 16:12:21 +0100 Subject: [PATCH 69/70] fix: Load bearing mut refs --- src/service/sending/antispam.rs | 2 +- src/service/sending/mod.rs | 2 -- src/service/sending/sender.rs | 13 ++++++------- 3 files changed, 7 insertions(+), 10 deletions(-) diff --git a/src/service/sending/antispam.rs b/src/service/sending/antispam.rs index 7aa1b248f..580e9a067 100644 --- a/src/service/sending/antispam.rs +++ b/src/service/sending/antispam.rs @@ -1,7 +1,7 @@ use std::{borrow::Cow, fmt::Debug, mem, time::Instant}; use bytes::BytesMut; -use conduwuit::{Err, Result, debug_info, err, utils, utils::response::LimitReadExt, warn}; +use conduwuit::{Err, Result, debug_info, err, utils::response::LimitReadExt}; use reqwest::Client; use ruma::api::{ IncomingResponse, OutgoingRequest, diff --git a/src/service/sending/mod.rs b/src/service/sending/mod.rs index b43098af3..01a568d6a 100644 --- a/src/service/sending/mod.rs +++ b/src/service/sending/mod.rs @@ -53,7 +53,6 @@ pub struct Service { struct Services { client: Dep, globals: Dep, - state: Dep, state_cache: Dep, user: Dep, users: Dep, @@ -97,7 +96,6 @@ impl crate::Service for Service { services: Services { client: args.depend::("client"), globals: args.depend::("globals"), - state: args.depend::("rooms::state"), state_cache: args.depend::("rooms::state_cache"), user: args.depend::("rooms::user"), users: args.depend::("users"), diff --git a/src/service/sending/sender.rs b/src/service/sending/sender.rs index 0c44d66d9..c07139841 100644 --- a/src/service/sending/sender.rs +++ b/src/service/sending/sender.rs @@ -1,7 +1,6 @@ use std::{ collections::{BTreeMap, HashMap, HashSet}, fmt::Debug, - future::ready, sync::{ Arc, atomic::{AtomicU64, AtomicUsize, Ordering}, @@ -11,13 +10,11 @@ use std::{ use base64::{Engine as _, engine::general_purpose::URL_SAFE_NO_PAD}; use conduwuit::{ - debug_error, debug_info, debug_warn, info, - utils::time::exponential_backoff::min_exp_backoff_duration, + debug_info, debug_warn, info, utils::time::exponential_backoff::min_exp_backoff_duration, }; use conduwuit_core::{ Error, Event, Result, at, debug, err, error, result::LogErr, - trace, utils::{ ReadyExt, calculate_hash, continue_exponential_backoff_secs, future::TryExtExt, @@ -33,7 +30,7 @@ use futures::{ }; use ruma::{ CanonicalJsonObject, MilliSecondsSinceUnixEpoch, OwnedRoomId, OwnedServerName, OwnedUserId, - RoomId, RoomVersionId, ServerName, UInt, + RoomId, ServerName, UInt, api::{ appservice::event::push_events::v1::EphemeralData, federation::transactions::{ @@ -164,10 +161,11 @@ impl Service { } /// Handles a successful response, queueing up more sends if necessary. + #[allow(clippy::needless_pass_by_ref_mut)] async fn handle_response_ok<'a>( &'a self, dest: &Destination, - futures: &SendingFutures<'a>, + futures: &mut SendingFutures<'a>, statuses: &mut CurTransactionStatus, ) { let _cork = self.db.db.cork(); @@ -196,10 +194,11 @@ impl Service { /// no new events to send the destination, it is removed from the status map /// instead. #[tracing::instrument(name = "request", level = "debug", skip_all)] + #[allow(clippy::needless_pass_by_ref_mut)] async fn handle_request<'a>( &'a self, msg: Msg, - futures: &SendingFutures<'a>, + futures: &mut SendingFutures<'a>, statuses: &mut CurTransactionStatus, ) { let iv = vec![(msg.queue_id, msg.event)]; From 458e0b53ac9821bd1d8bc5198cc5c3cdb3e9b5a1 Mon Sep 17 00:00:00 2001 From: timedout Date: Sat, 27 Jun 2026 18:41:43 +0100 Subject: [PATCH 70/70] style: Reformat --- src/admin/debug/commands.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/admin/debug/commands.rs b/src/admin/debug/commands.rs index 1fdc9509a..d1e5c2544 100644 --- a/src/admin/debug/commands.rs +++ b/src/admin/debug/commands.rs @@ -534,9 +534,9 @@ impl crate::Context<'_> { let json = serde_json::to_string_pretty(&pdus).map_err(|e| { err!(Database( - "Failed to convert room state events to pretty JSON, possible invalid room state \ - events in our database {e}", - )) + "Failed to convert room state events to pretty JSON, possible invalid room \ + state events in our database {e}", + )) })?; let out = format!("```json\n{json}\n```");