add ignored user checks on /context and /event, misc cleanup

Signed-off-by: strawberry <strawberry@puppygock.gay>

.github/workflows/ci.yml

@@ -3,20 +3,14 @@ name: CI and Artifacts
 on:
     pull_request:
     push:
-        # documentation workflow deals with this or is not relevant for this workflow
         paths-ignore:
-          - '*.md'
-          - 'conduwuit-example.toml'
-          - 'book.toml'
           - '.gitlab-ci.yml'
           - '.gitignore'
           - 'renovate.json'
-          - 'docs/**'
           - 'debian/**'
           - 'docker/**'
         branches:
             - main
-            - change-ci-cache
         tags:
           - '*'
     # Allows you to run this workflow manually from the Actions tab
@@ -51,8 +45,11 @@ env:
     # Get error output from nix that we can actually use, and use our binary caches for the earlier CI steps
     NIX_CONFIG: |
       show-trace = true
-      extra-substituters = https://attic.kennel.juneis.dog/conduit https://attic.kennel.juneis.dog/conduwuit https://cache.lix.systems https://conduwuit.cachix.org
+      extra-substituters = https://attic.kennel.juneis.dog/conduwuit https://attic.kennel.juneis.dog/conduit https://cache.lix.systems https://conduwuit.cachix.org https://aseipp-nix-cache.freetls.fastly.net
       extra-trusted-public-keys = conduit:eEKoUwlQGDdYmAI/Q/0slVlegqh/QmAvQd7HBSm21Wk= conduwuit:BbycGUgTISsltcmH0qNjFR9dbrQNYgdIAcmViSGoVTE= cache.lix.systems:aBnZUw8zA7H35Cz2RyKFVs3H4PlGTLawyY5KRbvJR8o= conduwuit.cachix.org-1:MFRm6jcnfTf0jSAbmvLfhO3KBMt4px+1xaereWXp8Xg=
+      experimental-features = nix-command flakes
+      extra-experimental-features = nix-command flakes
+      accept-flake-config = true
     # complement uses libolm
     NIXPKGS_ALLOW_INSECURE: 1
 
@@ -63,12 +60,20 @@ permissions:
 jobs:
     tests:
         name: Test
-        runs-on: ubuntu-latest
+        runs-on: ubuntu-24.04
-        env:
-          CARGO_PROFILE: "test"
         steps:
-            - name: Free Disk Space (Ubuntu)
+            - name: Install liburing
-              uses: jlumbroso/free-disk-space@main
+              run: |
+                  sudo apt install liburing-dev -y
+
+            - name: Free up a bit of runner space
+              run: |
+                  set +o pipefail
+                  sudo docker image prune --all --force || true
+                  sudo apt purge -y 'php.*' '^mongodb-.*' '^mysql-.*' azure-cli google-cloud-cli google-chrome-stable firefox powershell microsoft-edge-stable || true
+                  sudo apt clean
+                  sudo rm -v -rf /usr/local/games /usr/local/sqlpackage /usr/local/share/powershell /usr/local/share/edge_driver /usr/local/share/gecko_driver /usr/local/share/chromium /usr/local/share/chromedriver-linux64 /usr/lib/google-cloud-sdk /usr/lib/jvm /usr/lib/mono /usr/lib/heroku
+                  set -o pipefail
 
             - name: Sync repository
               uses: actions/checkout@v4
@@ -85,13 +90,7 @@ jobs:
                     exit 1
                   fi
 
-            - uses: nixbuild/nix-quick-install-action@v28
+            - uses: nixbuild/nix-quick-install-action@master
-
-            - name: Enable Cachix binary cache
-              run: |
-                  nix profile install nixpkgs#cachix
-                  cachix use crane
-                  cachix use nix-community
 
             - name: Restore and cache Nix store
               uses: nix-community/cache-nix-action@v5.1.0
@@ -114,11 +113,20 @@ jobs:
                 # always save the cache
                 save-always: true
 
+            - name: Enable Cachix binary cache
+              run: |
+                  nix profile install nixpkgs#cachix
+                  cachix use crane
+                  cachix use nix-community
+
             - name: Apply Nix binary cache configuration
               run: |
                   sudo tee -a "${XDG_CONFIG_HOME:-$HOME/.config}/nix/nix.conf" > /dev/null <<EOF
-                  extra-substituters = https://attic.kennel.juneis.dog/conduit https://attic.kennel.juneis.dog/conduwuit https://cache.lix.systems https://conduwuit.cachix.org
+                  extra-substituters = https://attic.kennel.juneis.dog/conduwuit https://attic.kennel.juneis.dog/conduit https://cache.lix.systems https://conduwuit.cachix.org https://aseipp-nix-cache.freetls.fastly.net
                   extra-trusted-public-keys = conduit:eEKoUwlQGDdYmAI/Q/0slVlegqh/QmAvQd7HBSm21Wk= conduwuit:BbycGUgTISsltcmH0qNjFR9dbrQNYgdIAcmViSGoVTE= cache.lix.systems:aBnZUw8zA7H35Cz2RyKFVs3H4PlGTLawyY5KRbvJR8o= conduwuit.cachix.org-1:MFRm6jcnfTf0jSAbmvLfhO3KBMt4px+1xaereWXp8Xg=
+                  experimental-features = nix-command flakes
+                  extra-experimental-features = nix-command flakes
+                  accept-flake-config = true
                   EOF
 
             - name: Use alternative Nix binary caches if specified
@@ -132,29 +140,16 @@ jobs:
             - name: Prepare build environment
               run: |
                   echo 'source $HOME/.nix-profile/share/nix-direnv/direnvrc' > "$HOME/.direnvrc"
-                  nix profile install --impure --inputs-from . nixpkgs#direnv nixpkgs#nix-direnv
+                  nix profile install --inputs-from . nixpkgs#direnv nixpkgs#nix-direnv
                   direnv allow
-                  nix develop .#all-features --command true --impure
+                  nix develop .#all-features --command true
 
             - name: Cache CI dependencies
               run: |
-                  # attic nix binary cache server is very, very terribly flakey. nothing i can do to fix it other than retry multiple times here
+                  bin/nix-build-and-cache ci
-                  ATTEMPTS=3
+                  bin/nix-build-and-cache just '.#devShells.x86_64-linux.default'
-                  SUCCESS=false
+                  bin/nix-build-and-cache just '.#devShells.x86_64-linux.all-features'
-                  while (( ATTEMPTS-- > 0 ))
+                  bin/nix-build-and-cache just '.#devShells.x86_64-linux.dynamic'
-                  do
-                    bin/nix-build-and-cache ci
-                    if [[ $? == 0 ]]; then
-                      SUCCESS=true
-                      break
-                    else
-                      sleep 3
-                    fi
-                  done
-
-                  if [[ $SUCCESS == "false" ]]; then
-                    exit 1
-                  fi
 
             # use sccache for Rust
             - name: Run sccache-cache
@@ -167,10 +162,14 @@ jobs:
                 cache-all-crates: "true"
 
             - name: Run CI tests
+              env:
+                CARGO_PROFILE: "test"
               run: |
                   direnv exec . engage > >(tee -a test_output.log)
 
             - name: Run Complement tests
+              env:
+                CARGO_PROFILE: "test"
               run: |
                   # the nix devshell sets $COMPLEMENT_SRC, so "/dev/null" is no-op
                   direnv exec . bin/complement "/dev/null" complement_test_logs.jsonl complement_test_results.jsonl > >(tee -a test_output.log)
@@ -218,33 +217,24 @@ jobs:
                       echo '```' >> $GITHUB_STEP_SUMMARY
                   fi
 
-            - name: Run cargo clean test artifacts
+            - name: Run cargo clean test artifacts to free up space
               run: |
                   cargo clean --profile test
 
     build:
         name: Build
-        runs-on: ubuntu-latest
+        runs-on: ubuntu-24.04
         needs: tests
         strategy:
             matrix:
                 include:
-                    - target: aarch64-unknown-linux-musl
+                    - target: aarch64-linux-musl
-                    - target: x86_64-unknown-linux-musl
+                    - target: x86_64-linux-musl
         steps:
-            - name: Free Disk Space (Ubuntu)
-              uses: jlumbroso/free-disk-space@main
-
             - name: Sync repository
               uses: actions/checkout@v4
 
-            - uses: nixbuild/nix-quick-install-action@v28
+            - uses: nixbuild/nix-quick-install-action@master
-
-            - name: Enable Cachix binary cache
-              run: |
-                  nix profile install nixpkgs#cachix
-                  cachix use crane
-                  cachix use nix-community
 
             - name: Restore and cache Nix store
               uses: nix-community/cache-nix-action@v5.1.0
@@ -267,11 +257,20 @@ jobs:
                 # always save the cache
                 save-always: true
 
+            - name: Enable Cachix binary cache
+              run: |
+                  nix profile install nixpkgs#cachix
+                  cachix use crane
+                  cachix use nix-community
+
             - name: Apply Nix binary cache configuration
               run: |
                   sudo tee -a "${XDG_CONFIG_HOME:-$HOME/.config}/nix/nix.conf" > /dev/null <<EOF
-                  extra-substituters = https://attic.kennel.juneis.dog/conduit https://attic.kennel.juneis.dog/conduwuit https://cache.lix.systems https://conduwuit.cachix.org
+                  extra-substituters = https://attic.kennel.juneis.dog/conduwuit https://attic.kennel.juneis.dog/conduit https://cache.lix.systems https://conduwuit.cachix.org https://aseipp-nix-cache.freetls.fastly.net
                   extra-trusted-public-keys = conduit:eEKoUwlQGDdYmAI/Q/0slVlegqh/QmAvQd7HBSm21Wk= conduwuit:BbycGUgTISsltcmH0qNjFR9dbrQNYgdIAcmViSGoVTE= cache.lix.systems:aBnZUw8zA7H35Cz2RyKFVs3H4PlGTLawyY5KRbvJR8o= conduwuit.cachix.org-1:MFRm6jcnfTf0jSAbmvLfhO3KBMt4px+1xaereWXp8Xg=
+                  experimental-features = nix-command flakes
+                  extra-experimental-features = nix-command flakes
+                  accept-flake-config = true
                   EOF
 
             - name: Use alternative Nix binary caches if specified
@@ -301,26 +300,17 @@ jobs:
 
             - name: Build static ${{ matrix.target }}
               run: |
-                  CARGO_DEB_TARGET_TUPLE=$(echo ${{ matrix.target }} | grep -o -E '^([^-]*-){3}[^-]*')
+                  if [[ ${{ matrix.target }} == "x86_64-linux-musl" ]]
+                  then
+                    CARGO_DEB_TARGET_TUPLE="x86_64-unknown-linux-musl"
+                  elif [[ ${{ matrix.target }} == "aarch64-linux-musl" ]]
+                  then
+                    CARGO_DEB_TARGET_TUPLE="aarch64-unknown-linux-musl"
+                  fi
+
                   SOURCE_DATE_EPOCH=$(git log -1 --pretty=%ct)
 
-                  # attic nix binary cache server is very, very terribly flakey. nothing i can do to fix it other than retry multiple times here
+                  bin/nix-build-and-cache just .#static-${{ matrix.target }}-all-features
-                  ATTEMPTS=3
-                  SUCCESS=false
-                  while (( ATTEMPTS-- > 0 ))
-                  do
-                    bin/nix-build-and-cache just .#static-${{ matrix.target }}-all-features
-                    if [[ $? == 0 ]]; then
-                      SUCCESS=true
-                      break
-                    else
-                      sleep 3
-                    fi
-                  done
-
-                  if [[ $SUCCESS == "false" ]]; then
-                    exit 1
-                  fi
 
                   mkdir -v -p target/release/
                   mkdir -v -p target/$CARGO_DEB_TARGET_TUPLE/release/
@@ -341,26 +331,17 @@ jobs:
 
             - name: Build static debug ${{ matrix.target }}
               run: |
-                  CARGO_DEB_TARGET_TUPLE=$(echo ${{ matrix.target }} | grep -o -E '^([^-]*-){3}[^-]*')
+                  if [[ ${{ matrix.target }} == "x86_64-linux-musl" ]]
+                  then
+                    CARGO_DEB_TARGET_TUPLE="x86_64-unknown-linux-musl"
+                  elif [[ ${{ matrix.target }} == "aarch64-linux-musl" ]]
+                  then
+                    CARGO_DEB_TARGET_TUPLE="aarch64-unknown-linux-musl"
+                  fi
+
                   SOURCE_DATE_EPOCH=$(git log -1 --pretty=%ct)
 
-                  # attic nix binary cache server is very, very terribly flakey. nothing i can do to fix it other than retry multiple times here
+                  bin/nix-build-and-cache just .#static-${{ matrix.target }}-all-features-debug
-                  ATTEMPTS=3
-                  SUCCESS=false
-                  while (( ATTEMPTS-- > 0 ))
-                  do
-                    bin/nix-build-and-cache just .#static-${{ matrix.target }}-all-features-debug
-                    if [[ $? == 0 ]]; then
-                      SUCCESS=true
-                      break
-                    else
-                      sleep 3
-                    fi
-                  done
-
-                  if [[ $SUCCESS == "false" ]]; then
-                    exit 1
-                  fi
 
                   # > warning: dev profile is not supported and will be a hard error in the future. cargo-deb is for making releases, and it doesn't make sense to use it with dev profiles.
                   # so we need to coerce cargo-deb into thinking this is a release binary
@@ -423,45 +404,13 @@ jobs:
 
             - name: Build OCI image ${{ matrix.target }}
               run: |
-                  # attic nix binary cache server is very, very terribly flakey. nothing i can do to fix it other than retry multiple times here
+                  bin/nix-build-and-cache just .#oci-image-${{ matrix.target }}-all-features
-                  ATTEMPTS=3
-                  SUCCESS=false
-                  while (( ATTEMPTS-- > 0 ))
-                  do
-                    bin/nix-build-and-cache just .#oci-image-${{ matrix.target }}-all-features
-                    if [[ $? == 0 ]]; then
-                      SUCCESS=true
-                      break
-                    else
-                      sleep 3
-                    fi
-                  done
-
-                  if [[ $SUCCESS == "false" ]]; then
-                    exit 1
-                  fi
 
                   cp -v -f result oci-image-${{ matrix.target }}.tar.gz
 
             - name: Build debug OCI image ${{ matrix.target }}
               run: |
-                  # attic nix binary cache server is very, very terribly flakey. nothing i can do to fix it other than retry multiple times here
+                  bin/nix-build-and-cache just .#oci-image-${{ matrix.target }}-all-features-debug
-                  ATTEMPTS=3
-                  SUCCESS=false
-                  while (( ATTEMPTS-- > 0 ))
-                  do
-                    bin/nix-build-and-cache just .#oci-image-${{ matrix.target }}-all-features-debug
-                    if [[ $? == 0 ]]; then
-                      SUCCESS=true
-                      break
-                    else
-                      sleep 3
-                    fi
-                  done
-
-                  if [[ $SUCCESS == "false" ]]; then
-                    exit 1
-                  fi
 
                   cp -v -f result oci-image-${{ matrix.target }}-debug.tar.gz
 
@@ -481,9 +430,83 @@ jobs:
                   if-no-files-found: error
                   compression-level: 0
 
+    build_mac_binaries:
+        name: Build MacOS Binaries
+        strategy:
+          matrix:
+            os: [macos-latest, macos-13]
+        runs-on: ${{ matrix.os }}
+        steps:
+            - name: Sync repository
+              uses: actions/checkout@v4
+
+            - name: Tag comparison check
+              if: ${{ startsWith(github.ref, 'refs/tags/v') && !endsWith(github.ref, '-rc') }}
+              run: |
+                  # Tag mismatch with latest repo tag check to prevent potential downgrades
+                  LATEST_TAG=$(git describe --tags `git rev-list --tags --max-count=1`)
+                  if [ $LATEST_TAG != ${{ github.ref_name }} ]; then
+                    echo '# WARNING: Attempting to run this workflow for a tag that is not the latest repo tag. Aborting.'
+                    echo '# WARNING: Attempting to run this workflow for a tag that is not the latest repo tag. Aborting.' >> $GITHUB_STEP_SUMMARY
+                    exit 1
+                  fi
+
+            # use sccache for Rust
+            - name: Run sccache-cache
+              if: (github.event.pull_request.draft != true) && (vars.DOCKER_USERNAME != '') && (vars.GITLAB_USERNAME != '') && (vars.SCCACHE_ENDPOINT != '') && (github.event.pull_request.user.login != 'renovate[bot]')
+              uses: mozilla-actions/sccache-action@main
+
+            # use rust-cache
+            - uses: Swatinem/rust-cache@v2
+              with:
+                cache-all-crates: "true"
+
+            # Nix can't do portable macOS builds yet
+            - name: Build macOS x86_64 binary
+              if: ${{ matrix.os == 'macos-13' }}
+              run: |
+                  CONDUWUIT_VERSION_EXTRA="$(git rev-parse --short HEAD)" cargo build --release
+                  cp -v -f target/release/conduit conduwuit-macos-x86_64
+                  otool -L conduwuit-macos-x86_64
+
+            # quick smoke test of the x86_64 macOS binary
+            - name: Run x86_64 macOS release binary
+              if: ${{ matrix.os == 'macos-13' }}
+              run: |
+                  ./conduwuit-macos-x86_64 --version
+
+            - name: Build macOS arm64 binary
+              if: ${{ matrix.os == 'macos-latest' }}
+              run: |
+                  CONDUWUIT_VERSION_EXTRA="$(git rev-parse --short HEAD)" cargo build --release
+                  cp -v -f target/release/conduit conduwuit-macos-arm64
+                  otool -L conduwuit-macos-arm64
+
+            # quick smoke test of the arm64 macOS binary
+            - name: Run arm64 macOS release binary
+              if: ${{ matrix.os == 'macos-latest' }}
+              run: |
+                  ./conduwuit-macos-arm64 --version
+
+            - name: Upload macOS x86_64 binary
+              if: ${{ matrix.os == 'macos-13' }}
+              uses: actions/upload-artifact@v4
+              with:
+                  name: conduwuit-macos-x86_64
+                  path: conduwuit-macos-x86_64
+                  if-no-files-found: error
+
+            - name: Upload macOS arm64 binary
+              if: ${{ matrix.os == 'macos-latest' }}
+              uses: actions/upload-artifact@v4
+              with:
+                  name: conduwuit-macos-arm64
+                  path: conduwuit-macos-arm64
+                  if-no-files-found: error
+
     docker:
         name: Docker publish
-        runs-on: ubuntu-latest
+        runs-on: ubuntu-24.04
         needs: build
         if: (startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main' || (github.event.pull_request.draft != true)) && (vars.DOCKER_USERNAME != '') && (vars.GITLAB_USERNAME != '') && github.event.pull_request.user.login != 'renovate[bot]'
         env:
@@ -531,10 +554,10 @@ jobs:
 
             - name: Move OCI images into position
               run: |
-                  mv -v oci-image-x86_64-unknown-linux-musl/*.tar.gz oci-image-amd64.tar.gz
+                  mv -v oci-image-x86_64-linux-musl/*.tar.gz oci-image-amd64.tar.gz
-                  mv -v oci-image-aarch64-unknown-linux-musl/*.tar.gz oci-image-arm64v8.tar.gz
+                  mv -v oci-image-aarch64-linux-musl/*.tar.gz oci-image-arm64v8.tar.gz
-                  mv -v oci-image-x86_64-unknown-linux-musl-debug/*.tar.gz oci-image-amd64-debug.tar.gz
+                  mv -v oci-image-x86_64-linux-musl-debug/*.tar.gz oci-image-amd64-debug.tar.gz
-                  mv -v oci-image-aarch64-unknown-linux-musl-debug/*.tar.gz oci-image-arm64v8-debug.tar.gz
+                  mv -v oci-image-aarch64-linux-musl-debug/*.tar.gz oci-image-arm64v8-debug.tar.gz
 
             - name: Load and push amd64 image
               if: ${{ (vars.DOCKER_USERNAME != '') && (env.DOCKERHUB_TOKEN != '') }}

.github/workflows/documentation.yml

@@ -24,8 +24,11 @@ env:
   # Get error output from nix that we can actually use, and use our binary caches for the earlier CI steps
   NIX_CONFIG: |
     show-trace = true
-    extra-substituters = https://attic.kennel.juneis.dog/conduit https://attic.kennel.juneis.dog/conduwuit https://cache.lix.systems https://conduwuit.cachix.org
+    extra-substituters = extra-substituters = https://attic.kennel.juneis.dog/conduwuit https://attic.kennel.juneis.dog/conduit https://cache.lix.systems https://conduwuit.cachix.org https://aseipp-nix-cache.freetls.fastly.net
     extra-trusted-public-keys = conduit:eEKoUwlQGDdYmAI/Q/0slVlegqh/QmAvQd7HBSm21Wk= conduwuit:BbycGUgTISsltcmH0qNjFR9dbrQNYgdIAcmViSGoVTE= cache.lix.systems:aBnZUw8zA7H35Cz2RyKFVs3H4PlGTLawyY5KRbvJR8o= conduwuit.cachix.org-1:MFRm6jcnfTf0jSAbmvLfhO3KBMt4px+1xaereWXp8Xg=
+    experimental-features = nix-command flakes
+    extra-experimental-features = nix-command flakes
+    accept-flake-config = true
 
 # Allow only one concurrent deployment, skipping runs queued between the run in-progress and latest queued.
 # However, do NOT cancel in-progress runs as we want to allow these production deployments to complete.
@@ -36,7 +39,7 @@ concurrency:
 jobs:
   docs:
     name: Documentation and GitHub Pages
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
 
     permissions:
       pages: write
@@ -47,23 +50,23 @@ jobs:
       url: ${{ steps.deployment.outputs.page_url }}
 
     steps:
-      - name: Free Disk Space (Ubuntu)
+      - name: Free up a bit of runner space
-        uses: jlumbroso/free-disk-space@main
+        run: |
+            set +o pipefail
+            sudo docker image prune --all --force || true
+            sudo apt purge -y 'php.*' '^mongodb-.*' '^mysql-.*' azure-cli google-cloud-cli google-chrome-stable firefox powershell microsoft-edge-stable || true
+            sudo apt clean
+            sudo rm -v -rf /usr/local/games /usr/local/sqlpackage /usr/local/share/powershell /usr/local/share/edge_driver /usr/local/share/gecko_driver /usr/local/share/chromium /usr/local/share/chromedriver-linux64 /usr/lib/google-cloud-sdk /usr/lib/jvm /usr/lib/mono /usr/lib/heroku
+            set -o pipefail
 
       - name: Sync repository
         uses: actions/checkout@v4
 
       - name: Setup GitHub Pages
-        if: github.event_name != 'pull_request'
+        if: (startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main') && (github.event_name != 'pull_request')
         uses: actions/configure-pages@v5
 
-      - uses: nixbuild/nix-quick-install-action@v28
+      - uses: nixbuild/nix-quick-install-action@master
-
-      - name: Enable Cachix binary cache
-        run: |
-            nix profile install nixpkgs#cachix
-            cachix use crane
-            cachix use nix-community
 
       - name: Restore and cache Nix store
         uses: nix-community/cache-nix-action@v5.1.0
@@ -86,11 +89,20 @@ jobs:
           # always save the cache
           save-always: true
 
+      - name: Enable Cachix binary cache
+        run: |
+            nix profile install nixpkgs#cachix
+            cachix use crane
+            cachix use nix-community
+
       - name: Apply Nix binary cache configuration
         run: |
             sudo tee -a "${XDG_CONFIG_HOME:-$HOME/.config}/nix/nix.conf" > /dev/null <<EOF
-            extra-substituters = https://attic.kennel.juneis.dog/conduit https://attic.kennel.juneis.dog/conduwuit https://cache.lix.systems https://conduwuit.cachix.org
+            extra-substituters = https://attic.kennel.juneis.dog/conduwuit https://attic.kennel.juneis.dog/conduit https://cache.lix.systems https://conduwuit.cachix.org https://aseipp-nix-cache.freetls.fastly.net
             extra-trusted-public-keys = conduit:eEKoUwlQGDdYmAI/Q/0slVlegqh/QmAvQd7HBSm21Wk= conduwuit:BbycGUgTISsltcmH0qNjFR9dbrQNYgdIAcmViSGoVTE= cache.lix.systems:aBnZUw8zA7H35Cz2RyKFVs3H4PlGTLawyY5KRbvJR8o= conduwuit.cachix.org-1:MFRm6jcnfTf0jSAbmvLfhO3KBMt4px+1xaereWXp8Xg=
+            experimental-features = nix-command flakes
+            extra-experimental-features = nix-command flakes
+            accept-flake-config = true
             EOF
 
       - name: Use alternative Nix binary caches if specified
@@ -110,23 +122,7 @@ jobs:
 
       - name: Cache CI dependencies
         run: |
-            # attic nix binary cache server is very, very terribly flakey. nothing i can do to fix it other than retry multiple times here
+            bin/nix-build-and-cache ci
-            ATTEMPTS=3
-            SUCCESS=false
-            while (( ATTEMPTS-- > 0 ))
-            do
-              bin/nix-build-and-cache ci
-              if [[ $? == 0 ]]; then
-                SUCCESS=true
-                break
-              else
-                sleep 3
-              fi
-            done
-
-            if [[ $SUCCESS == "false" ]]; then
-              exit 1
-            fi
 
       - name: Run lychee and markdownlint
         run: |
@@ -135,23 +131,7 @@ jobs:
 
       - name: Build documentation (book)
         run: |
-            # attic nix binary cache server is very, very terribly flakey. nothing i can do to fix it other than retry multiple times here
+            bin/nix-build-and-cache just .#book
-            ATTEMPTS=3
-            SUCCESS=false
-            while (( ATTEMPTS-- > 0 ))
-            do
-              bin/nix-build-and-cache just .#book
-              if [[ $? == 0 ]]; then
-                SUCCESS=true
-                break
-              else
-                sleep 3
-              fi
-            done
-
-            if [[ $SUCCESS == "false" ]]; then
-              exit 1
-            fi
 
             cp -r --dereference result public
 
@@ -165,12 +145,12 @@ jobs:
           compression-level: 0
 
       - name: Upload generated documentation (book) as GitHub Pages artifact
-        if: github.event_name != 'pull_request'
+        if: (startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main') && (github.event_name != 'pull_request')
         uses: actions/upload-pages-artifact@v3
         with:
           path: public
 
       - name: Deploy to GitHub Pages
-        if: github.event_name != 'pull_request'
+        if: (startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main') && (github.event_name != 'pull_request')
         id: deployment
         uses: actions/deploy-pages@v4

.github/workflows/trivy.yml

@@ -1,42 +0,0 @@
-name: Trivy code and vulnerability scanning
-
-on:
-  pull_request:
-  push:
-    branches:
-      - main
-    tags:
-      - '*'
-  schedule:
-    - cron: '00 12 * * *'
-
-permissions:
-  contents: read
-
-jobs:
-  trivy-scan:
-    name: Trivy Scan
-    runs-on: ubuntu-latest
-    permissions:
-      contents: read
-      security-events: write
-      actions: read
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v4
-
-      - name: Run Trivy code and vulnerability scanner on repo
-        uses: aquasecurity/trivy-action@0.24.0
-        with:
-          scan-type: repo
-          format: sarif
-          output: trivy-results.sarif
-          severity: CRITICAL,HIGH,MEDIUM,LOW
-
-      - name: Run Trivy code and vulnerability scanner on filesystem
-        uses: aquasecurity/trivy-action@0.24.0
-        with:
-          scan-type: fs
-          format: sarif
-          output: trivy-results.sarif
-          severity: CRITICAL,HIGH,MEDIUM,LOW

.gitlab-ci.yml

@@ -10,6 +10,13 @@ variables:
   FF_USE_FASTZIP: true
   # Print progress reports for cache and artifact transfers
   TRANSFER_METER_FREQUENCY: 5s
+  NIX_CONFIG: |
+    show-trace = true
+    extra-substituters = https://attic.kennel.juneis.dog/conduit https://attic.kennel.juneis.dog/conduwuit https://cache.lix.systems https://conduwuit.cachix.org
+    extra-trusted-public-keys = conduit:eEKoUwlQGDdYmAI/Q/0slVlegqh/QmAvQd7HBSm21Wk= conduwuit:BbycGUgTISsltcmH0qNjFR9dbrQNYgdIAcmViSGoVTE= cache.lix.systems:aBnZUw8zA7H35Cz2RyKFVs3H4PlGTLawyY5KRbvJR8o= conduwuit.cachix.org-1:MFRm6jcnfTf0jSAbmvLfhO3KBMt4px+1xaereWXp8Xg=
+    experimental-features = nix-command flakes
+    extra-experimental-features = nix-command flakes
+    accept-flake-config = true
 
 # Avoid duplicate pipelines
 # See: https://docs.gitlab.com/ee/ci/yaml/workflow.html#switch-between-branch-pipelines-and-merge-request-pipelines
@@ -23,6 +30,9 @@ workflow:
 before_script:
   # Enable nix-command and flakes
   - if command -v nix > /dev/null; then echo "experimental-features = nix-command flakes" >> /etc/nix/nix.conf; fi
+  - if command -v nix > /dev/null; then echo "extra-experimental-features = nix-command flakes" >> /etc/nix/nix.conf; fi
+  # Accept flake config from "untrusted" users
+  - if command -v nix > /dev/null; then echo "accept-flake-config = true" >> /etc/nix/nix.conf; fi
 
   # Add conduwuit binary cache
   - if command -v nix > /dev/null; then echo "extra-substituters = https://attic.kennel.juneis.dog/conduwuit" >> /etc/nix/nix.conf; fi
@@ -47,6 +57,8 @@ before_script:
   - if command -v nix > /dev/null; then echo "extra-substituters = https://nix-community.cachix.org" >> /etc/nix/nix.conf; fi
   - if command -v nix > /dev/null; then echo "extra-trusted-public-keys = nix-community.cachix.org-1:mB9FSh9qf2dCimDSUo8Zy7bkq5CX+/rkCWyvRCYg3Fs=" >> /etc/nix/nix.conf; fi
 
+  - if command -v nix > /dev/null; then echo "extra-substituters = https://aseipp-nix-cache.freetls.fastly.net" >> /etc/nix/nix.conf; fi
+
   # Install direnv and nix-direnv
   - if command -v nix > /dev/null; then nix-env -iA nixpkgs.direnv nixpkgs.nix-direnv; fi
 
@@ -58,7 +70,7 @@ before_script:
 
 ci:
   stage: ci
-  image: nixos/nix:2.23.3
+  image: nixos/nix:2.24.9
   script:
     # Cache CI dependencies
     - ./bin/nix-build-and-cache ci
@@ -83,31 +95,31 @@ ci:
 
 artifacts:
   stage: artifacts
-  image: nixos/nix:2.23.3
+  image: nixos/nix:2.24.9
   script:
-    - ./bin/nix-build-and-cache just .#static-x86_64-unknown-linux-musl
+    - ./bin/nix-build-and-cache just .#static-x86_64-linux-musl
-    - cp result/bin/conduit x86_64-unknown-linux-musl
+    - cp result/bin/conduit x86_64-linux-musl
 
     - mkdir -p target/release
     - cp result/bin/conduit target/release
     - direnv exec . cargo deb --no-build --no-strip
-    - mv target/debian/*.deb x86_64-unknown-linux-musl.deb
+    - mv target/debian/*.deb x86_64-linux-musl.deb
 
     # Since the OCI image package is based on the binary package, this has the
     # fun side effect of uploading the normal binary too. Conduit users who are
     # deploying with Nix can leverage this fact by adding our binary cache to
     # their systems.
     #
-    # Note that although we have an `oci-image-x86_64-unknown-linux-musl`
+    # Note that although we have an `oci-image-x86_64-linux-musl`
     # output, we don't build it because it would be largely redundant to this
     # one since it's all containerized anyway.
     - ./bin/nix-build-and-cache just .#oci-image
     - cp result oci-image-amd64.tar.gz
 
-    - ./bin/nix-build-and-cache just .#static-aarch64-unknown-linux-musl
+    - ./bin/nix-build-and-cache just .#static-aarch64-linux-musl
-    - cp result/bin/conduit aarch64-unknown-linux-musl
+    - cp result/bin/conduit aarch64-linux-musl
 
-    - ./bin/nix-build-and-cache just .#oci-image-aarch64-unknown-linux-musl
+    - ./bin/nix-build-and-cache just .#oci-image-aarch64-linux-musl
     - cp result oci-image-arm64v8.tar.gz
 
     - ./bin/nix-build-and-cache just .#book
@@ -115,9 +127,9 @@ artifacts:
     - cp -r --dereference result public
   artifacts:
     paths:
-      - x86_64-unknown-linux-musl
+      - x86_64-linux-musl
-      - aarch64-unknown-linux-musl
+      - aarch64-linux-musl
-      - x86_64-unknown-linux-musl.deb
+      - x86_64-linux-musl.deb
       - oci-image-amd64.tar.gz
       - oci-image-arm64v8.tar.gz
       - public

CONTRIBUTING.md

@@ -1,7 +1,7 @@
 # Contributing guide
 
 This page is for about contributing to conduwuit. The
-[development](development.md) page may be of interest for you as well.
+[development](./development.md) page may be of interest for you as well.
 
 If you would like to work on an [issue][issues] that is not assigned, preferably
 ask in the Matrix room first at [#conduwuit:puppygock.gay][conduwuit-matrix],
@@ -67,7 +67,7 @@ failing from your changes, please review the logs (they are uploaded as
 artifacts) and determine if they're intended or not.
 
 If you'd like to run Complement locally using Nix, see the
-[testing](docs/development/testing.md) page.
+[testing](development/testing.md) page.
 
 [Sytest][sytest] support will come soon.
 
@@ -128,7 +128,11 @@ Direct all PRs/MRs to the `main` branch.
 
 By sending a pull request or patch, you are agreeing that your changes are
 allowed to be licenced under the Apache-2.0 licence and all of your conduct is
-in line with the Contributor's Covenant.
+in line with the Contributor's Covenant, and conduwuit's Code of Conduct.
+
+Contribution by users who violate either of these code of conducts will not have
+their contributions accepted. This includes users who have been banned from
+conduwuit Matrix rooms for Code of Conduct violations.
 
 [issues]: https://github.com/girlbossceo/conduwuit/issues
 [conduwuit-matrix]: https://matrix.to/#/#conduwuit:puppygock.gay

Cargo.toml

@@ -19,20 +19,30 @@ license = "Apache-2.0"
 # See also `rust-toolchain.toml`
 readme = "README.md"
 repository = "https://github.com/girlbossceo/conduwuit"
-rust-version = "1.80.1"
+rust-version = "1.83.0"
-version = "0.4.6"
+version = "0.5.0"
 
 [workspace.metadata.crane]
 name = "conduit"
 
 [workspace.dependencies.arrayvec]
 version = "0.7.4"
+features = ["serde"]
+
+[workspace.dependencies.smallvec]
+version = "1.13.2"
+features = [
+	"const_generics",
+	"const_new",
+	"serde",
+	"write",
+]
 
 [workspace.dependencies.const-str]
 version = "0.5.7"
 
 [workspace.dependencies.ctor]
-version = "0.2.8"
+version = "0.2.9"
 
 [workspace.dependencies.cargo_toml]
 version = "0.20"
@@ -45,20 +55,20 @@ default-features = false
 features = ["parse"]
 
 [workspace.dependencies.sanitize-filename]
-version = "0.5.0"
+version = "0.6.0"
 
 [workspace.dependencies.jsonwebtoken]
 version = "9.3.0"
+default-features = false
 
 [workspace.dependencies.base64]
 version = "0.22.1"
+default-features = false
 
 # used for TURN server authentication
 [workspace.dependencies.hmac]
 version = "0.12.1"
-
+default-features = false
-[workspace.dependencies.sha-1]
-version = "0.10.1"
 
 # used for checking if an IP is in specific subnets / CIDR ranges easier
 [workspace.dependencies.ipaddress]
@@ -69,19 +79,19 @@ version = "0.8.5"
 
 # Used for the http request / response body type for Ruma endpoints used with reqwest
 [workspace.dependencies.bytes]
-version = "1.7.1"
+version = "1.8.0"
 
 [workspace.dependencies.http-body-util]
-version = "0.1.1"
+version = "0.1.2"
 
 [workspace.dependencies.http]
 version = "1.1.0"
 
 [workspace.dependencies.regex]
-version = "1.10.6"
+version = "1.11.1"
 
 [workspace.dependencies.axum]
-version = "0.7.5"
+version = "0.7.9"
 default-features = false
 features = [
 	"form",
@@ -94,29 +104,28 @@ features = [
 ]
 
 [workspace.dependencies.axum-extra]
-version = "0.9.3"
+version = "0.9.6"
 default-features = false
 features = ["typed-header", "tracing"]
 
 [workspace.dependencies.axum-server]
 version = "0.7.1"
 default-features = false
-features = ["tls-rustls"]
 
 # to listen on both HTTP and HTTPS if listening on TLS dierctly from conduwuit for complement or sytest
 [workspace.dependencies.axum-server-dual-protocol]
 version = "0.7"
 
 [workspace.dependencies.axum-client-ip]
-version = "0.6.0"
+version = "0.6.1"
 
 [workspace.dependencies.tower]
-version = "0.5.0"
+version = "0.5.1"
 default-features = false
 features = ["util"]
 
 [workspace.dependencies.tower-http]
-version = "0.5.2"
+version = "0.6.2"
 default-features = false
 features = [
     "add-extension",
@@ -129,10 +138,12 @@ features = [
 ]
 
 [workspace.dependencies.rustls]
-version = "0.23.12"
+version = "0.23.16"
+default-features = false
+features = ["aws_lc_rs"]
 
 [workspace.dependencies.reqwest]
-version = "0.12.7"
+version = "0.12.9"
 default-features = false
 features = [
 	"rustls-tls-native-roots",
@@ -142,12 +153,12 @@ features = [
 ]
 
 [workspace.dependencies.serde]
-version = "1.0.204"
+version = "1.0.215"
 default-features = false
 features = ["rc"]
 
 [workspace.dependencies.serde_json]
-version = "1.0.124"
+version = "1.0.133"
 default-features = false
 features = ["raw_value"]
 
@@ -171,7 +182,7 @@ default-features = false
 
 # Used to generate thumbnails for images
 [workspace.dependencies.image]
-version = "0.25.1"
+version = "0.25.5"
 default-features = false
 features = [
 	"jpeg",
@@ -182,16 +193,18 @@ features = [
 
 # logging
 [workspace.dependencies.log]
-version = "0.4.21"
+version = "0.4.22"
 default-features = false
 [workspace.dependencies.tracing]
-version = "0.1.40"
+version = "0.1.41"
 default-features = false
 [workspace.dependencies.tracing-subscriber]
 version = "0.3.18"
-features = ["env-filter"]
+default-features = false
+features = ["env-filter", "std", "tracing", "tracing-log", "ansi", "fmt"]
 [workspace.dependencies.tracing-core]
-version = "0.1.32"
+version = "0.1.33"
+default-features = false
 
 # for URL previews
 [workspace.dependencies.webpage]
@@ -200,23 +213,25 @@ default-features = false
 
 # used for conduit's CLI and admin room command parsing
 [workspace.dependencies.clap]
-version = "4.5.15"
+version = "4.5.21"
 default-features = false
 features = [
-	"std",
 	"derive",
-	"help",
+	"env",
-	"usage",
 	"error-context",
+	"help",
+	"std",
 	"string",
+	"usage",
 ]
 
-[workspace.dependencies.futures-util]
+[workspace.dependencies.futures]
 version = "0.3.30"
 default-features = false
+features = ["std", "async-await"]
 
 [workspace.dependencies.tokio]
-version = "1.39.2"
+version = "1.41.1"
 default-features = false
 features = [
 	"fs",
@@ -237,7 +252,7 @@ version = "0.8.5"
 
 # Validating urls in config, was already a transitive dependency
 [workspace.dependencies.url]
-version = "2.5.0"
+version = "2.5.4"
 default-features = false
 features = ["serde"]
 
@@ -248,7 +263,7 @@ features = ["alloc", "std"]
 default-features = false
 
 [workspace.dependencies.hyper]
-version = "1.4.1"
+version = "1.5.1"
 default-features = false
 features = [
 	"server",
@@ -257,25 +272,24 @@ features = [
 ]
 
 [workspace.dependencies.hyper-util]
-version = "0.1.6"
+# hyper-util >=0.1.9 seems to have DNS issues
+version = "=0.1.8"
 default-features = false
 features = [
-	"client",
 	"server-auto",
 	"server-graceful",
-	"service",
 	"tokio",
 ]
 
 # to support multiple variations of setting a config option
 [workspace.dependencies.either]
-version = "1.11.0"
+version = "1.13.0"
 default-features = false
 features = ["serde"]
 
 # Used for reading the configuration from conduwuit.toml & environment variables
 [workspace.dependencies.figment]
-version = "0.10.18"
+version = "0.10.19"
 default-features = false
 features = ["env", "toml"]
 
@@ -285,11 +299,13 @@ default-features = false
 
 # Used for conduit::Error type
 [workspace.dependencies.thiserror]
-version = "1.0.63"
+version = "2.0.3"
+default-features = false
 
 # Used when hashing the state
 [workspace.dependencies.ring]
 version = "0.17.8"
+default-features = false
 
 # Used to make working with iterators easier, was already a transitive depdendency
 [workspace.dependencies.itertools]
@@ -300,12 +316,16 @@ version = "0.13.0"
 [workspace.dependencies.cyborgtime]
 version = "2.1.1"
 
-# used to replace the channels of the tokio runtime
+# used for MPSC channels
 [workspace.dependencies.loole]
-version = "0.3.1"
+version = "0.4.0"
+
+# used for MPMC channels
+[workspace.dependencies.async-channel]
+version = "2.3.1"
 
 [workspace.dependencies.async-trait]
-version = "0.1.81"
+version = "0.1.83"
 
 [workspace.dependencies.lru-cache]
 version = "0.1.2"
@@ -314,7 +334,7 @@ version = "0.1.2"
 [workspace.dependencies.ruma]
 git = "https://github.com/girlbossceo/ruwuma"
 #branch = "conduwuit-changes"
-rev = "d7ddcd036f81edb257ab9371f9cadd46444e8a90"
+rev = "1a550585bf025cce48ef8b734339245092bc986e"
 features = [
     "compat",
     "rand",
@@ -327,20 +347,26 @@ features = [
     "server-util",
     "unstable-exhaustive-types",
     "ring-compat",
+    "compat-upload-signatures",
     "identifiers-validation",
     "unstable-unspecified",
+    "unstable-msc2409",
     "unstable-msc2448",
     "unstable-msc2666",
     "unstable-msc2867",
     "unstable-msc2870",
     "unstable-msc3026",
     "unstable-msc3061",
+    "unstable-msc3245",
     "unstable-msc3266",
     "unstable-msc3381", # polls
     "unstable-msc3489", # beacon / live location
     "unstable-msc3575",
+    "unstable-msc4075",
     "unstable-msc4121",
     "unstable-msc4125",
+    "unstable-msc4186",
+    "unstable-msc4210", # remove legacy mentions
     "unstable-extensible-events",
 ]
 
@@ -356,9 +382,13 @@ features = [
 	"bzip2",
 ]
 
-# optional SHA256 media keys feature
 [workspace.dependencies.sha2]
 version = "0.10.8"
+default-features = false
+
+[workspace.dependencies.sha1]
+version = "0.10.6"
+default-features = false
 
 # optional opentelemetry, performance measurements, flamegraphs, etc for performance measurements and monitoring
 [workspace.dependencies.opentelemetry]
@@ -403,17 +433,17 @@ version = "0.34.0"
 # jemalloc usage
 [workspace.dependencies.tikv-jemalloc-sys]
 git = "https://github.com/girlbossceo/jemallocator"
-rev = "c32af15f3b440ae5e46c3404f78b19093bbd5294"
+rev = "d87938bfddc26377dd7fdf14bbcd345f3ab19442"
 default-features = false
 features = ["unprefixed_malloc_on_supported_platforms"]
 [workspace.dependencies.tikv-jemallocator]
 git = "https://github.com/girlbossceo/jemallocator"
-rev = "c32af15f3b440ae5e46c3404f78b19093bbd5294"
+rev = "d87938bfddc26377dd7fdf14bbcd345f3ab19442"
 default-features = false
 features = ["unprefixed_malloc_on_supported_platforms"]
 [workspace.dependencies.tikv-jemalloc-ctl]
 git = "https://github.com/girlbossceo/jemallocator"
-rev = "c32af15f3b440ae5e46c3404f78b19093bbd5294"
+rev = "d87938bfddc26377dd7fdf14bbcd345f3ab19442"
 default-features = false
 features = ["use_std"]
 
@@ -426,7 +456,8 @@ default-features = false
 features = ["resource"]
 
 [workspace.dependencies.sd-notify]
-version = "0.4.1"
+version = "0.4.3"
+default-features = false
 
 [workspace.dependencies.hardened_malloc-rs]
 version = "0.1.2"
@@ -442,23 +473,25 @@ version = "0.4.3"
 default-features = false
 
 [workspace.dependencies.termimad]
-version = "0.30.0"
+version = "0.31.1"
 default-features = false
 
 [workspace.dependencies.checked_ops]
 version = "0.1"
 
 [workspace.dependencies.syn]
-version = "2.0.72"
+version = "2.0.87"
 default-features = false
 features = ["full", "extra-traits"]
 
 [workspace.dependencies.quote]
-version = "1.0.36"
+version = "1.0.37"
 
 [workspace.dependencies.proc-macro2]
-version = "1.0.86"
+version = "1.0.89"
 
+[workspace.dependencies.bytesize]
+version = "1.3.0"
 
 #
 # Patches
@@ -469,22 +502,22 @@ version = "1.0.86"
 # https://github.com/girlbossceo/tracing/commit/b348dca742af641c47bc390261f60711c2af573c
 [patch.crates-io.tracing-subscriber]
 git = "https://github.com/girlbossceo/tracing"
-rev = "4d78a14a5e03f539b8c6b475aefa08bb14e4de91"
+rev = "ccc4fbd8238c2d5ba354e61ec17ac610af11401d"
 [patch.crates-io.tracing]
 git = "https://github.com/girlbossceo/tracing"
-rev = "4d78a14a5e03f539b8c6b475aefa08bb14e4de91"
+rev = "ccc4fbd8238c2d5ba354e61ec17ac610af11401d"
 [patch.crates-io.tracing-core]
 git = "https://github.com/girlbossceo/tracing"
-rev = "4d78a14a5e03f539b8c6b475aefa08bb14e4de91"
+rev = "ccc4fbd8238c2d5ba354e61ec17ac610af11401d"
 [patch.crates-io.tracing-log]
 git = "https://github.com/girlbossceo/tracing"
-rev = "4d78a14a5e03f539b8c6b475aefa08bb14e4de91"
+rev = "ccc4fbd8238c2d5ba354e61ec17ac610af11401d"
 
 # adds a tab completion callback: https://github.com/girlbossceo/rustyline-async/commit/de26100b0db03e419a3d8e1dd26895d170d1fe50
 # adds event for CTRL+\: https://github.com/girlbossceo/rustyline-async/commit/67d8c49aeac03a5ef4e818f663eaa94dd7bf339b
 [patch.crates-io.rustyline-async]
 git = "https://github.com/girlbossceo/rustyline-async"
-rev = "9654cc84e19241f6e19021eb8e677892656f5071"
+rev = "deaeb0694e2083f53d363b648da06e10fc13900c"
 
 #
 # Our crates
@@ -608,12 +641,11 @@ inherits = "release"
 # and can be raised if build times are tolerable.
 
 [profile.dev]
-debug = 1
+debug = "full"
 opt-level = 0
 panic = "unwind"
 debug-assertions = true
 incremental = true
-codegen-units = 64
 #rustflags = [
 #	'--cfg', 'conduit_mods',
 #	'-Ztime-passes',
@@ -655,7 +687,6 @@ incremental = false
 
 [profile.dev.package.conduit]
 inherits = "dev"
-incremental = false
 #rustflags = [
 #	'--cfg', 'conduit_mods',
 #	'-Ztime-passes',
@@ -715,12 +746,16 @@ opt-level = 'z'
 # primarily used for CI
 [profile.test]
 inherits = "dev"
+strip = false
+opt-level = 0
 codegen-units = 16
 incremental = false
 
 [profile.test.package.'*']
 inherits = "dev"
 debug = 0
+strip = false
+opt-level = 0
 codegen-units = 16
 incremental = false
 
@@ -763,6 +798,7 @@ unused-qualifications = "warn"
 #unused-results = "warn"                                             # TODO
 
 ## some sadness
+elided_named_lifetimes = "allow"                                     # TODO!
 let_underscore_drop = "allow"
 missing_docs = "allow"
 # cfgs cannot be limited to expected cfgs or their de facto non-transitive/opt-in use-case e.g.
@@ -808,6 +844,7 @@ significant_drop_tightening = { level = "allow", priority = 1 }      # TODO
 pedantic = { level = "warn", priority = -1 }
 
 ## some sadness
+too_long_first_doc_paragraph = { level = "allow", priority = 1 }
 doc_markdown = { level = "allow", priority = 1 }
 enum_glob_use = { level = "allow", priority = 1 }
 if_not_else = { level = "allow", priority = 1 }
@@ -819,6 +856,7 @@ missing_panics_doc = { level = "allow", priority = 1 }
 module_name_repetitions = { level = "allow", priority = 1 }
 no_effect_underscore_binding = { level = "allow", priority = 1 }
 similar_names = { level = "allow", priority = 1 }
+single_match_else = { level = "allow", priority = 1 }
 struct_field_names = { level = "allow", priority = 1 }
 unnecessary_wraps = { level = "allow", priority = 1 }
 unused_async = { level = "allow", priority = 1 }

README.md

@@ -1,15 +1,19 @@
 # conduwuit
 
-`main` / stable: [![CI and
+[![conduwuit main room](https://img.shields.io/matrix/conduwuit%3Apuppygock.gay?server_fqdn=matrix.transfem.dev&style=flat&logo=matrix&logoColor=%23f5b3ff&label=%23conduwuit%3Apuppygock.gay&color=%23f652ff)](https://matrix.to/#/#conduwuit:puppygock.gay) [![conduwuit space](https://img.shields.io/matrix/conduwuit-space%3Apuppygock.gay?server_fqdn=matrix.transfem.dev&style=flat&logo=matrix&logoColor=%23f5b3ff&label=%23conduwuit-space%3Apuppygock.gay&color=%23f652ff)](https://matrix.to/#/#conduwuit-space:puppygock.gay) [![CI and Artifacts](https://github.com/girlbossceo/conduwuit/actions/workflows/ci.yml/badge.svg?branch=main)](https://github.com/girlbossceo/conduwuit/actions/workflows/ci.yml)
-Artifacts](https://github.com/girlbossceo/conduwuit/actions/workflows/ci.yml/badge.svg?branch=main)](https://github.com/girlbossceo/conduwuit/actions/workflows/ci.yml)
 
-<!-- ANCHOR: catchphrase --> ### a very cool, featureful fork of
+<!-- ANCHOR: catchphrase -->
-[Conduit](https://conduit.rs/) <!-- ANCHOR_END: catchphrase -->
 
-Visit the [Conduwuit documentation](https://conduwuit.puppyirl.gay/) for more
+### a very cool, featureful fork of [Conduit](https://conduit.rs/)
-information.
 
-<!-- ANCHOR: body --> #### What is Matrix?
+<!-- ANCHOR_END: catchphrase -->
+
+Visit the [conduwuit documentation](https://conduwuit.puppyirl.gay/) for more
+information and how to deploy/setup conduwuit.
+
+<!-- ANCHOR: body -->
+
+#### What is Matrix?
 
 [Matrix](https://matrix.org) is an open network for secure and decentralized
 communication. Users from every Matrix homeserver can chat with users from all
@@ -18,9 +22,9 @@ to communicate with users outside of Matrix, like a community on Discord.
 
 #### What is the goal?
 
-An efficient Matrix homeserver that's easy to set up and just works. You can
+A high-performance and efficient Matrix homeserver that's easy to set up and
-install it on a mini-computer like the Raspberry Pi to host Matrix for your
+just works. You can install it on a mini-computer like the Raspberry Pi to
-family, friends or company.
+host Matrix for your family, friends or company.
 
 #### Can I try it out?
 
@@ -37,13 +41,36 @@ transfem.dev is also listed at
 
 #### What is the current status?
 
-conduwuit is a hard fork of Conduit which is in beta, meaning you can join and
+conduwuit is technically a hard fork of Conduit, which is in Beta. The Beta status
-participate in most Matrix rooms, but not all features are supported and you
+initially was inherited from Conduit, however overtime this Beta status is rapidly
-might run into bugs from time to time.
+becoming less and less relevant as our codebase significantly diverges more and more.
+
+conduwuit is quite stable and very usable as a daily driver and for a low-medium
+sized homeserver. There is still a lot of more work to be done, but it is in a far
+better place than the project was in early 2024.
+
+#### How is conduwuit funded? Is conduwuit sustainable?
+
+conduwuit has no external funding. This is made possible purely in my freetime with
+contributors, also in their free time, and only by user-curated donations.
+
+conduwuit has existed since around November 2023, but [only became more publicly known
+in March/April 2024](https://matrix.org/blog/2024/04/26/this-week-in-matrix-2024-04-26/#conduwuit-website)
+and we have no plans in stopping or slowing down any time soon!
+
+#### Can I migrate or switch from Conduit?
+
+conduwuit is a complete drop-in replacement for Conduit. As long as you are using RocksDB,
+the only "migration" you need to do is replace the binary or container image. There
+is no harm or additional steps required for using conduwuit. See the
+[Migrating from Conduit](https://conduwuit.puppyirl.gay/deploying/generic.html#migrating-from-conduit) section
+on the generic deploying guide.
 
 <!-- ANCHOR_END: body -->
 
-<!-- ANCHOR: footer --> #### Contact
+<!-- ANCHOR: footer -->
+
+#### Contact
 
 If you run into any question, feel free to
 
@@ -52,8 +79,12 @@ If you run into any question, feel free to
 
 #### Donate
 
+conduwuit development is purely made possible by myself and contributors. I do
+not get paid to work on this, and I work on it in my free time. Donations are
+heavily appreciated! 💜🥺
+
 - Liberapay: <https://liberapay.com/girlbossceo>
-- Ko-fi: <https://ko-fi.com/puppygock>
+- Ko-fi (note they take a fee): <https://ko-fi.com/puppygock>
 - GitHub Sponsors: <https://github.com/sponsors/girlbossceo>
 
 #### Logo
@@ -73,5 +104,6 @@ Both, but I prefer conduwuit.
 - git.girlcock.ceo: <https://git.girlcock.ceo/strawberry/conduwuit>
 - git.gay: <https://git.gay/june/conduwuit>
 - Codeberg: <https://codeberg.org/girlbossceo/conduwuit>
-- sourcehut: <https://git.sr.ht/~girlbossceo/conduwuit> <!-- ANCHOR_END: footer
+- sourcehut: <https://git.sr.ht/~girlbossceo/conduwuit>
--->
+
+<!-- ANCHOR_END: footer -->

arch/conduwuit.service

@@ -15,7 +15,7 @@ DevicePolicy=closed
 LockPersonality=yes
 MemoryDenyWriteExecute=yes
 NoNewPrivileges=yes
-ProcSubset=pid
+#ProcSubset=pid
 ProtectClock=yes
 ProtectControlGroups=yes
 ProtectHome=yes

bin/complement

@@ -15,10 +15,10 @@ LOG_FILE="$2"
 # A `.jsonl` file to write test results to
 RESULTS_FILE="$3"
 
-OCI_IMAGE="complement-conduit:main"
+OCI_IMAGE="complement-conduwuit:main"
 
 # Complement tests that are skipped due to flakiness/reliability issues
-SKIPPED_COMPLEMENT_TESTS='-skip=TestClientSpacesSummary.*|TestJoinFederatedRoomFromApplicationServiceBridgeUser.*|TestJumpToDateEndpoint.*'
+SKIPPED_COMPLEMENT_TESTS='-skip=TestClientSpacesSummary.*|TestJoinFederatedRoomFromApplicationServiceBridgeUser.*|TestJumpToDateEndpoint.*|TestUnbanViaInvite.*'
 
 # $COMPLEMENT_SRC needs to be a directory to Complement source code
 if [ -f "$COMPLEMENT_SRC" ]; then
@@ -34,7 +34,7 @@ toplevel="$(git rev-parse --show-toplevel)"
 
 pushd "$toplevel" > /dev/null
 
-bin/nix-build-and-cache just .#static-complement
+bin/nix-build-and-cache just .#linux-complement
 
 docker load < result
 popd > /dev/null

bin/nix-build-and-cache

@@ -26,7 +26,12 @@ just() {
         "$ATTIC_TOKEN"
 
     # Find all output paths of the installables and their build dependencies
-    readarray -t derivations < <(nix path-info --derivation "$@")
+    #readarray -t derivations < <(nix path-info --derivation "$@")
+    derivations=()
+    while IFS=$'\n' read derivation; do
+        derivations+=("$derivation")
+    done < <(nix path-info --derivation "$@")
+
     cache=()
     for derivation in "${derivations[@]}"; do
         cache+=(
@@ -34,6 +39,9 @@ just() {
         )
     done
 
+    withattic() {
+        nix shell --inputs-from "$toplevel" attic --command xargs attic push "$@" <<< "${cache[*]}"
+    }
     # Upload them to Attic (conduit store)
     #
     # Use `xargs` and a here-string because something would probably explode if
@@ -41,8 +49,7 @@ just() {
     # store paths include a newline in them.
     (
         IFS=$'\n'
-        nix shell --inputs-from "$toplevel" attic -c xargs \
+        withattic conduit || withattic conduit || withattic conduit || true
-            attic push conduit <<< "${cache[*]}"
     )
 
     # main "conduwuit" store
@@ -59,8 +66,7 @@ just() {
     # store paths include a newline in them.
     (
         IFS=$'\n'
-        nix shell --inputs-from "$toplevel" attic -c xargs \
+        withattic conduwuit || withattic conduwuit || withattic conduwuit || true
-            attic push conduwuit <<< "${cache[*]}"
 
         # push to cachix if available
         if [ "$CACHIX_AUTH_TOKEN" ]; then
@@ -76,8 +82,8 @@ ci() {
         --inputs-from "$toplevel"
 
         # Keep sorted
-        "$toplevel#devShells.x86_64-linux.default"
+        #"$toplevel#devShells.x86_64-linux.default"
-        "$toplevel#devShells.x86_64-linux.all-features"
+        #"$toplevel#devShells.x86_64-linux.all-features"
         attic#default
         cachix#default
         nixpkgs#direnv

clippy.toml

@@ -2,6 +2,14 @@ array-size-threshold = 4096
 cognitive-complexity-threshold = 94         # TODO reduce me ALARA
 excessive-nesting-threshold = 11            # TODO reduce me to 4 or 5
 future-size-threshold = 7745                # TODO reduce me ALARA
-stack-size-threshold = 144000               # reduce me ALARA
+stack-size-threshold = 196608               # reduce me ALARA
 too-many-lines-threshold = 700              # TODO reduce me to <= 100
 type-complexity-threshold = 250             # reduce me to ~200
+
+disallowed-macros = [
+	{ path = "log::error", reason = "use conduit_core::error" },
+	{ path = "log::warn", reason = "use conduit_core::warn" },
+	{ path = "log::info", reason = "use conduit_core::info" },
+	{ path = "log::debug", reason = "use conduit_core::debug" },
+	{ path = "log::trace", reason = "use conduit_core::trace" },
+]

debian/README.md

@@ -1,17 +1,22 @@
 # conduwuit for Debian
 
-Information about downloading and deploying the Debian package. This may also be referenced for other `apt`-based distros such as Ubuntu.
+Information about downloading and deploying the Debian package. This may also be
+referenced for other `apt`-based distros such as Ubuntu.
 
 ### Installation
 
-It is recommended to see the [generic deployment guide](../deploying/generic.md) for further information if needed as usage of the Debian package is generally related.
+It is recommended to see the [generic deployment guide](../deploying/generic.md)
+for further information if needed as usage of the Debian package is generally
+related.
 
 ### Configuration
 
-When installed, the example config is placed at `/etc/conduwuit/conduwuit.toml` as the default config. At the minimum, you will need to change your `server_name` here.
+When installed, the example config is placed at `/etc/conduwuit/conduwuit.toml`
+as the default config. The config mentions things required to be changed before
+starting.
 
-You can tweak more detailed settings by uncommenting and setting the config options
+You can tweak more detailed settings by uncommenting and setting the config
-in `/etc/conduwuit/conduwuit.toml`.
+options in `/etc/conduwuit/conduwuit.toml`.
 
 ### Running
 

debian/conduwuit.service

@@ -22,7 +22,7 @@ DevicePolicy=closed
 LockPersonality=yes
 MemoryDenyWriteExecute=yes
 NoNewPrivileges=yes
-ProcSubset=pid
+#ProcSubset=pid
 ProtectClock=yes
 ProtectControlGroups=yes
 ProtectHome=yes

deps/rust-rocksdb/Cargo.toml

@@ -27,7 +27,7 @@ malloc-usable-size = ["rust-rocksdb/malloc-usable-size"]
 
 [dependencies.rust-rocksdb]
 git = "https://github.com/girlbossceo/rust-rocksdb-zaidoon1"
-rev = "5383ca8173299066b516406e3a2cf945ead891cb"
+rev = "4bce1bb97d8be6f0d47245c99d465ca9cef33aad"
 #branch = "master"
 default-features = false
 

development.md

@@ -0,0 +1 @@
+docs/development.md

docs/SUMMARY.md

@@ -8,6 +8,7 @@
   - [Generic](deploying/generic.md)
   - [NixOS](deploying/nixos.md)
   - [Docker](deploying/docker.md)
+  - [Kubernetes](deploying/kubernetes.md)
   - [Arch Linux](deploying/arch-linux.md)
   - [Debian](deploying/debian.md)
   - [FreeBSD](deploying/freebsd.md)

docs/conduwuit_coc.md

@@ -9,7 +9,7 @@ environment for everyone. This Code of Conduct applies to all conduwuit spaces,
 including any further community rooms that reference this CoC. Here are our
 guidelines to help maintain the welcoming atmosphere that sets conduwuit apart.
 
-For the general foundational rules, please refer to the [Contributor's 
+For the general foundational rules, please refer to the [Contributor's
 Covenant](https://github.com/girlbossceo/conduwuit/blob/main/CODE_OF_CONDUCT.md).
 Below are additional guidelines specific to the conduwuit community.
 
@@ -90,4 +90,4 @@ comfortable doing that, then please send a DM to one of the moderators directly.
 
 Together, let’s build a community where everyone feels valued and respected.
 
-- The conduwuit Moderation Team
+— The conduwuit Moderation Team

docs/configuration.md

@@ -31,6 +31,22 @@ string. This does not apply to options that take booleans or numbers:
 - `--option log=\"debug\"` works ✅
 - `--option server_name='"example.com'"` works ✅
 
+## Execute commandline flag
+
+conduwuit supports running admin commands on startup using the commandline
+argument `--execute`. The most notable use for this is to create an admin user
+on first startup.
+
+The syntax of this is a standard admin command without the prefix such as
+`./conduwuit --execute "users create_user june"`
+
+An example output of a success is:
+```
+INFO conduit_service::admin::startup: Startup command #0 completed:
+Created user with user_id: @june:girlboss.ceo and password: `<redacted>`
+```
+
+This commandline argument can be paired with the `--option` flag.
 
 ## Environment variables
 

docs/contributing.md

@@ -1 +0,0 @@
-{{#include ../CONTRIBUTING.md}}

docs/contributing.md

@@ -0,0 +1 @@
+../CONTRIBUTING.md

docs/deploying/docker-compose.for-traefik.yml

@@ -1,40 +1,43 @@
 # conduwuit - Behind Traefik Reverse Proxy
 
 services:
-    homeserver:
+  homeserver:
-        ### If you already built the conduduwit image with 'docker build' or want to use the Docker Hub image,
+    ### If you already built the conduduwit image with 'docker build' or want to use the Docker Hub image,
-        ### then you are ready to go.
+    ### then you are ready to go.
-        image: girlbossceo/conduwuit:latest
+    image: girlbossceo/conduwuit:latest
-        restart: unless-stopped
+    restart: unless-stopped
-        volumes:
+    volumes:
-            - db:/var/lib/conduwuit
+      - db:/var/lib/conduwuit
-            #- ./conduwuit.toml:/etc/conduwuit.toml
+      #- ./conduwuit.toml:/etc/conduwuit.toml
-        networks:
+    networks:
-            - proxy
+      - proxy
-        environment:
+    environment:
-            CONDUWUIT_SERVER_NAME: your.server.name # EDIT THIS
+      CONDUWUIT_SERVER_NAME: your.server.name.example # EDIT THIS
-            CONDUWUIT_DATABASE_PATH: /var/lib/conduwuit
+      CONDUWUIT_DATABASE_PATH: /var/lib/conduwuit
-            CONDUWUIT_DATABASE_BACKEND: rocksdb
+      CONDUWUIT_PORT: 6167 # should match the loadbalancer traefik label
-            CONDUWUIT_PORT: 6167
+      CONDUWUIT_MAX_REQUEST_SIZE: 20000000 # in bytes, ~20 MB
-            CONDUWUIT_MAX_REQUEST_SIZE: 20_000_000 # in bytes, ~20 MB
+      CONDUWUIT_ALLOW_REGISTRATION: 'true'
-            CONDUWUIT_ALLOW_REGISTRATION: 'true'
+      CONDUWUIT_ALLOW_FEDERATION: 'true'
-            CONDUWUIT_ALLOW_FEDERATION: 'true'
+      CONDUWUIT_ALLOW_CHECK_FOR_UPDATES: 'true'
-            CONDUWUIT_ALLOW_CHECK_FOR_UPDATES: 'true'
+      CONDUWUIT_TRUSTED_SERVERS: '["matrix.org"]'
-            CONDUWUIT_TRUSTED_SERVERS: '["matrix.org"]'
+      #CONDUWUIT_LOG: warn,state_res=warn
-            #CONDUWUIT_LOG: warn,state_res=warn
+      CONDUWUIT_ADDRESS: 0.0.0.0
-            CONDUWUIT_ADDRESS: 0.0.0.0
+      #CONDUWUIT_CONFIG: '/etc/conduwuit.toml' # Uncomment if you mapped config toml above
-            #CONDUWUIT_CONFIG: '/etc/conduwuit.toml' # Uncomment if you mapped config toml above
+
-        #cpuset: "0-4" # Uncomment to limit to specific CPU cores
+      # We need some way to serve the client and server .well-known json. The simplest way is via the CONDUWUIT_WELL_KNOWN
+      # variable / config option, there are multiple ways to do this, e.g. in the conduwuit.toml file, and in a seperate
+      # see the override file for more information about delegation
+      CONDUWUIT_WELL_KNOWN: |
+        {
+        client=https://your.server.name.example,
+        server=your.server.name.example:443
+        }
+    #cpuset: "0-4" # Uncomment to limit to specific CPU cores
+    ulimits: # conduwuit uses quite a few file descriptors, and on some systems it defaults to 1024, so you can tell docker to increase it
+      nofile:
+        soft: 1048567
+        hard: 1048567
 
-    # We need some way to server the client and server .well-known json. The simplest way is to use a nginx container
-    # to serve those two as static files. If you want to use a different way, delete or comment the below service, here
-    # and in the docker compose override file.
-    well-known:
-        image: nginx:latest
-        restart: unless-stopped
-        volumes:
-            - ./nginx/matrix.conf:/etc/nginx/conf.d/matrix.conf # the config to serve the .well-known/matrix files
-            - ./nginx/www:/var/www/ # location of the client and server .well-known-files
     ### Uncomment if you want to use your own Element-Web App.
     ### Note: You need to provide a config.json for Element and you also need a second
     ###       Domain or Subdomain for the communication between Element and conduwuit
@@ -50,10 +53,12 @@ services:
     #         - homeserver
 
 volumes:
-    db:
+  db:
 
 networks:
-    # This is the network Traefik listens to, if your network has a different
+  # This is the network Traefik listens to, if your network has a different
-    # name, don't forget to change it here and in the docker-compose.override.yml
+  # name, don't forget to change it here and in the docker-compose.override.yml
-    proxy:
+  proxy:
-        external: true
+    external: true
+
+# vim: ts=2:sw=2:expandtab

docs/deploying/docker-compose.override.yml

@@ -1,44 +1,37 @@
 # conduwuit - Traefik Reverse Proxy Labels
 
 services:
-    homeserver:
+  homeserver:
-        labels:
+    labels:
-            - "traefik.enable=true"
+      - "traefik.enable=true"
-            - "traefik.docker.network=proxy"  # Change this to the name of your Traefik docker proxy network
+      - "traefik.docker.network=proxy"  # Change this to the name of your Traefik docker proxy network
 
-            - "traefik.http.routers.to-conduwuit.rule=Host(`<SUBDOMAIN>.<DOMAIN>`)"  # Change to the address on which conduwuit is hosted
+      - "traefik.http.routers.to-conduwuit.rule=Host(`<SUBDOMAIN>.<DOMAIN>`)"  # Change to the address on which conduwuit is hosted
-            - "traefik.http.routers.to-conduwuit.tls=true"
+      - "traefik.http.routers.to-conduwuit.tls=true"
-            - "traefik.http.routers.to-conduwuit.tls.certresolver=letsencrypt"
+      - "traefik.http.routers.to-conduwuit.tls.certresolver=letsencrypt"
-            - "traefik.http.routers.to-conduwuit.middlewares=cors-headers@docker"
+      - "traefik.http.routers.to-conduwuit.middlewares=cors-headers@docker"
+      - "traefik.http.services.to_conduwuit.loadbalancer.server.port=6167"
 
-            - "traefik.http.middlewares.cors-headers.headers.accessControlAllowOriginList=*"
+      - "traefik.http.middlewares.cors-headers.headers.accessControlAllowOriginList=*"
-            - "traefik.http.middlewares.cors-headers.headers.accessControlAllowHeaders=Origin, X-Requested-With, Content-Type, Accept, Authorization"
+      - "traefik.http.middlewares.cors-headers.headers.accessControlAllowHeaders=Origin, X-Requested-With, Content-Type, Accept, Authorization"
-            - "traefik.http.middlewares.cors-headers.headers.accessControlAllowMethods=GET, POST, PUT, DELETE, OPTIONS"
+      - "traefik.http.middlewares.cors-headers.headers.accessControlAllowMethods=GET, POST, PUT, DELETE, OPTIONS"
 
-    # We need some way to server the client and server .well-known json. The simplest way is to use a nginx container
+      # If you want to have your account on <DOMAIN>, but host conduwuit on a subdomain,
-    # to serve those two as static files. If you want to use a different way, delete or comment the below service, here
+      # you can let it only handle the well known file on that domain instead
-    # and in the docker compose file.
+      #- "traefik.http.routers.to-matrix-wellknown.rule=Host(`<DOMAIN>`) && PathPrefix(`/.well-known/matrix`)"
-    well-known:
+      #- "traefik.http.routers.to-matrix-wellknown.tls=true"
-        labels:
+      #- "traefik.http.routers.to-matrix-wellknown.tls.certresolver=letsencrypt"
-            - "traefik.enable=true"
+      #- "traefik.http.routers.to-matrix-wellknown.middlewares=cors-headers@docker"
-            - "traefik.docker.network=proxy"
 
-            - "traefik.http.routers.to-matrix-wellknown.rule=Host(`<SUBDOMAIN>.<DOMAIN>`) && PathPrefix(`/.well-known/matrix`)"
+  ### Uncomment this if you uncommented Element-Web App in the docker-compose.yml
-            - "traefik.http.routers.to-matrix-wellknown.tls=true"
+  # element-web:
-            - "traefik.http.routers.to-matrix-wellknown.tls.certresolver=letsencrypt"
+  #     labels:
-            - "traefik.http.routers.to-matrix-wellknown.middlewares=cors-headers@docker"
+  #         - "traefik.enable=true"
+  #         - "traefik.docker.network=proxy"  # Change this to the name of your Traefik docker proxy network
 
-            - "traefik.http.middlewares.cors-headers.headers.accessControlAllowOriginList=*"
+  #         - "traefik.http.routers.to-element-web.rule=Host(`<SUBDOMAIN>.<DOMAIN>`)"  # Change to the address on which Element-Web is hosted
-            - "traefik.http.middlewares.cors-headers.headers.accessControlAllowHeaders=Origin, X-Requested-With, Content-Type, Accept, Authorization"
+  #         - "traefik.http.routers.to-element-web.tls=true"
-            - "traefik.http.middlewares.cors-headers.headers.accessControlAllowMethods=GET, POST, PUT, DELETE, OPTIONS"
+  #         - "traefik.http.routers.to-element-web.tls.certresolver=letsencrypt"
 
+# vim: ts=2:sw=2:expandtab
 
-    ### Uncomment this if you uncommented Element-Web App in the docker-compose.yml
-    # element-web:
-    #     labels:
-    #         - "traefik.enable=true"
-    #         - "traefik.docker.network=proxy"  # Change this to the name of your Traefik docker proxy network
-
-    #         - "traefik.http.routers.to-element-web.rule=Host(`<SUBDOMAIN>.<DOMAIN>`)"  # Change to the address on which Element-Web is hosted
-    #         - "traefik.http.routers.to-element-web.tls=true"
-    #         - "traefik.http.routers.to-element-web.tls.certresolver=letsencrypt"

docs/deploying/docker-compose.with-caddy.yml

@@ -30,9 +30,8 @@ services:
         environment:
             CONDUWUIT_SERVER_NAME: example.com # EDIT THIS
             CONDUWUIT_DATABASE_PATH: /var/lib/conduwuit
-            CONDUWUIT_DATABASE_BACKEND: rocksdb
             CONDUWUIT_PORT: 6167
-            CONDUWUIT_MAX_REQUEST_SIZE: 20_000_000 # in bytes, ~20 MB
+            CONDUWUIT_MAX_REQUEST_SIZE: 20000000 # in bytes, ~20 MB
             CONDUWUIT_ALLOW_REGISTRATION: 'true'
             CONDUWUIT_ALLOW_FEDERATION: 'true'
             CONDUWUIT_ALLOW_CHECK_FOR_UPDATES: 'true'

docs/deploying/docker-compose.with-traefik.yml

@@ -1,42 +1,52 @@
 # conduwuit - Behind Traefik Reverse Proxy
 
 services:
-    homeserver:
+  homeserver:
-        ### If you already built the conduwuit image with 'docker build' or want to use the Docker Hub image,
+    ### If you already built the conduwuit image with 'docker build' or want to use the Docker Hub image,
-        ### then you are ready to go.
+    ### then you are ready to go.
-        image: girlbossceo/conduwuit:latest
+    image: girlbossceo/conduwuit:latest
-        restart: unless-stopped
+    restart: unless-stopped
-        volumes:
+    volumes:
-            - db:/srv/conduwuit/.local/share/conduwuit
+      - db:/var/lib/conduwuit
-            #- ./conduwuit.toml:/etc/conduwuit.toml
+      #- ./conduwuit.toml:/etc/conduwuit.toml
-        networks:
+    networks:
-            - proxy
+      - proxy
-        environment:
+    environment:
-            CONDUWUIT_SERVER_NAME: your.server.name # EDIT THIS
+      CONDUWUIT_SERVER_NAME: your.server.name.example # EDIT THIS
-            CONDUWUIT_TRUSTED_SERVERS: '["matrix.org"]'
+      CONDUWUIT_TRUSTED_SERVERS: '["matrix.org"]'
-            CONDUWUIT_ALLOW_REGISTRATION : 'true'
+      CONDUWUIT_ALLOW_REGISTRATION: 'false' # After setting a secure registration token, you can enable this
-            #CONDUWUIT_CONFIG: '/etc/conduwuit.toml' # Uncomment if you mapped config toml above
+      CONDUWUIT_REGISTRATION_TOKEN: "" # This is a token you can use to register on the server
-            ### Uncomment and change values as desired
+      #CONDUWUIT_REGISTRATION_TOKEN_FILE: "" # Alternatively you can configure a path to a token file to read
-            # CONDUWUIT_ADDRESS: 0.0.0.0
+      CONDUWUIT_ADDRESS: 0.0.0.0
-            # CONDUWUIT_PORT: 6167
+      CONDUWUIT_PORT: 6167 # you need to match this with the traefik load balancer label if you're want to change it
-            # CONDUWUIT_LOG: info  # default is: "warn,state_res=warn"
+      CONDUWUIT_DATABASE_PATH: /var/lib/conduwuit
-            # CONDUWUIT_ALLOW_JAEGER: 'false'
+      #CONDUWUIT_CONFIG: '/etc/conduit.toml' # Uncomment if you mapped config toml above
-            # CONDUWUIT_ALLOW_ENCRYPTION: 'true'
+      ### Uncomment and change values as desired, note that conduwuit has plenty of config options, so you should check out the example example config too
-            # CONDUWUIT_ALLOW_FEDERATION: 'true'
+      # Available levels are: error, warn, info, debug, trace - more info at: https://docs.rs/env_logger/*/env_logger/#enabling-logging
-            # CONDUWUIT_ALLOW_CHECK_FOR_UPDATES: 'true'
+      # CONDUWUIT_LOG: info  # default is: "warn,state_res=warn"
-            # CONDUWUIT_DATABASE_PATH: /srv/conduwuit/.local/share/conduwuit
+      # CONDUWUIT_ALLOW_ENCRYPTION: 'true'
-            # CONDUWUIT_WORKERS: 10
+      # CONDUWUIT_ALLOW_FEDERATION: 'true'
-            # CONDUWUIT_MAX_REQUEST_SIZE: 20000000  # in bytes, ~20 MB
+      # CONDUWUIT_ALLOW_CHECK_FOR_UPDATES: 'true'
+      # CONDUWUIT_ALLOW_INCOMING_PRESENCE: true
+      # CONDUWUIT_ALLOW_OUTGOING_PRESENCE: true
+      # CONDUWUIT_ALLOW_LOCAL_PRESENCE: true
+      # CONDUWUIT_WORKERS: 10
+      # CONDUWUIT_MAX_REQUEST_SIZE: 20000000  # in bytes, ~20 MB
+      # CONDUWUIT_NEW_USER_DISPLAYNAME_SUFFIX = "🏳<200d>⚧"
 
-    # We need some way to server the client and server .well-known json. The simplest way is to use a nginx container
+      # We need some way to serve the client and server .well-known json. The simplest way is via the CONDUWUIT_WELL_KNOWN
-    # to serve those two as static files. If you want to use a different way, delete or comment the below service, here
+      # variable / config option, there are multiple ways to do this, e.g. in the conduwuit.toml file, and in a seperate
-    # and in the docker compose override file.
+      # reverse proxy, but since you do not have a reverse proxy and following this guide, this example is included
-    well-known:
+      CONDUWUIT_WELL_KNOWN: |
-        image: nginx:latest
+        {
-        restart: unless-stopped
+          client=https://your.server.name.example,
-        volumes:
+          server=your.server.name.example:443
-            - ./nginx/matrix.conf:/etc/nginx/conf.d/matrix.conf # the config to serve the .well-known/matrix files
+        }
-            - ./nginx/www:/var/www/ # location of the client and server .well-known-files
+    #cpuset: "0-4" # Uncomment to limit to specific CPU cores
+    ulimits: # conduwuit uses quite a few file descriptors, and on some systems it defaults to 1024, so you can tell docker to increase it
+      nofile:
+        soft: 1048567
+        hard: 1048567
 
     ### Uncomment if you want to use your own Element-Web App.
     ### Note: You need to provide a config.json for Element and you also need a second
@@ -52,29 +62,79 @@ services:
     #     depends_on:
     #         - homeserver
 
-    traefik:
+  traefik:
-        image: "traefik:latest"
+    image: "traefik:latest"
-        container_name: "traefik"
+    container_name: "traefik"
-        restart: "unless-stopped"
+    restart: "unless-stopped"
-        ports:
+    ports:
-            - "80:80"
+      - "80:80"
-            - "443:443"
+      - "443:443"
-        volumes:
+    volumes:
-            - "/var/run/docker.sock:/var/run/docker.sock"
+      - "/var/run/docker.sock:/var/run/docker.sock:z"
-            # - "./traefik_config:/etc/traefik"
+      - "acme:/etc/traefik/acme"
-            - "acme:/etc/traefik/acme"
+      #- "./traefik_config:/etc/traefik:z"
-        labels:
+    labels:
-            - "traefik.enable=true"
+      - "traefik.enable=true"
 
-            # middleware redirect
+      # middleware redirect
-            - "traefik.http.middlewares.redirect-to-https.redirectscheme.scheme=https"
+      - "traefik.http.middlewares.redirect-to-https.redirectscheme.scheme=https"
-            # global redirect to https
+      # global redirect to https
-            - "traefik.http.routers.redirs.rule=hostregexp(`{host:.+}`)"
+      - "traefik.http.routers.redirs.rule=hostregexp(`{host:.+}`)"
-            - "traefik.http.routers.redirs.entrypoints=http"
+      - "traefik.http.routers.redirs.entrypoints=web"
-            - "traefik.http.routers.redirs.middlewares=redirect-to-https"
+      - "traefik.http.routers.redirs.middlewares=redirect-to-https"
 
-        networks:
+    configs:
-            - proxy
+      - source: dynamic.yml
+        target: /etc/traefik/dynamic.yml
+
+    environment:
+      TRAEFIK_LOG_LEVEL: DEBUG
+      TRAEFIK_ENTRYPOINTS_WEB: true
+      TRAEFIK_ENTRYPOINTS_WEB_ADDRESS: ":80"
+      TRAEFIK_ENTRYPOINTS_WEB_HTTP_REDIRECTIONS_ENTRYPOINT_TO: websecure
+
+      TRAEFIK_ENTRYPOINTS_WEBSECURE: true
+      TRAEFIK_ENTRYPOINTS_WEBSECURE_ADDRESS: ":443"
+      TRAEFIK_ENTRYPOINTS_WEBSECURE_HTTP_TLS_CERTRESOLVER: letsencrypt
+      #TRAEFIK_ENTRYPOINTS_WEBSECURE_HTTP_MIDDLEWARES: secureHeaders@file # if you want to enabled STS
+
+      TRAEFIK_CERTIFICATESRESOLVERS_LETSENCRYPT: true
+      TRAEFIK_CERTIFICATESRESOLVERS_LETSENCRYPT_ACME_EMAIL: # Set this to the email you want to receive certificate expiration emails for
+      TRAEFIK_CERTIFICATESRESOLVERS_LETSENCRYPT_ACME_KEYTYPE: EC384
+      TRAEFIK_CERTIFICATESRESOLVERS_LETSENCRYPT_ACME_HTTPCHALLENGE: true
+      TRAEFIK_CERTIFICATESRESOLVERS_LETSENCRYPT_ACME_HTTPCHALLENGE_ENTRYPOINT: web
+      TRAEFIK_CERTIFICATESRESOLVERS_LETSENCRYPT_ACME_STORAGE: "/etc/traefik/acme/acme.json"
+
+      TRAEFIK_PROVIDERS_DOCKER: true
+      TRAEFIK_PROVIDERS_DOCKER_ENDPOINT: "unix:///var/run/docker.sock"
+      TRAEFIK_PROVIDERS_DOCKER_EXPOSEDBYDEFAULT: false
+
+      TRAEFIK_PROVIDERS_FILE: true
+      TRAEFIK_PROVIDERS_FILE_FILENAME: "/etc/traefik/dynamic.yml"
+
+configs:
+  dynamic.yml:
+    content: |
+      # Optionally set STS headers, like in https://hstspreload.org
+      # http:
+      #   middlewares:
+      #     secureHeaders:
+      #       headers:
+      #         forceSTSHeader: true
+      #         stsIncludeSubdomains: true
+      #         stsPreload: true
+      #         stsSeconds: 31536000
+      tls:
+        options:
+          default:
+            cipherSuites:
+              - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384
+              - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384
+              - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256
+              - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256
+              - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305
+              - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305
+            minVersion: VersionTLS12
 
 volumes:
     db:
@@ -82,3 +142,5 @@ volumes:
 
 networks:
     proxy:
+
+# vim: ts=2:sw=2:expandtab

docs/deploying/docker-compose.yml

@@ -14,9 +14,8 @@ services:
         environment:
             CONDUWUIT_SERVER_NAME: your.server.name # EDIT THIS
             CONDUWUIT_DATABASE_PATH: /var/lib/conduwuit
-            CONDUWUIT_DATABASE_BACKEND: rocksdb
             CONDUWUIT_PORT: 6167
-            CONDUWUIT_MAX_REQUEST_SIZE: 20_000_000 # in bytes, ~20 MB
+            CONDUWUIT_MAX_REQUEST_SIZE: 20000000 # in bytes, ~20 MB
             CONDUWUIT_ALLOW_REGISTRATION: 'true'
             CONDUWUIT_ALLOW_FEDERATION: 'true'
             CONDUWUIT_ALLOW_CHECK_FOR_UPDATES: 'true'

docs/deploying/docker.md

@@ -9,22 +9,14 @@ from a registry.
 
 OCI images for conduwuit are available in the registries listed below.
 
-| Registry        | Image
+| Registry        | Image                                                           | Size                          | Notes                  |
-| Size                          | Notes                  | | --------------- |
+| --------------- | --------------------------------------------------------------- | ----------------------------- | ---------------------- |
---------------------------------------------------------------- |
+| GitHub Registry | [ghcr.io/girlbossceo/conduwuit:latest][gh] | ![Image Size][shield-latest]  | Stable latest tagged image.          |
------------------------------ | ---------------------- | | GitHub Registry |
+| GitLab Registry | [registry.gitlab.com/conduwuit/conduwuit:latest][gl] | ![Image Size][shield-latest]  | Stable latest tagged image.          |
-[ghcr.io/girlbossceo/conduwuit:latest][gh] | ![Image Size][shield-latest]  |
+| Docker Hub      | [docker.io/girlbossceo/conduwuit:latest][dh]             | ![Image Size][shield-latest]  | Stable latest tagged image.          |
-Stable tagged image.          | | GitLab Registry |
+| GitHub Registry | [ghcr.io/girlbossceo/conduwuit:main][gh]   | ![Image Size][shield-main]    | Stable main branch.   |
-[registry.gitlab.com/conduwuit/conduwuit:latest][gl] | ![Image
+| GitLab Registry | [registry.gitlab.com/conduwuit/conduwuit:main][gl]   | ![Image Size][shield-main]    | Stable main branch.   |
-Size][shield-latest]  | Stable tagged image.          | | Docker Hub      |
+| Docker Hub      | [docker.io/girlbossceo/conduwuit:main][dh]               | ![Image Size][shield-main]    | Stable main branch.   |
-[docker.io/girlbossceo/conduwuit:latest][dh]             | ![Image
-Size][shield-latest]  | Stable tagged image.          | | GitHub Registry |
-[ghcr.io/girlbossceo/conduwuit:main][gh]   | ![Image Size][shield-main]    |
-Stable main branch.   | | GitLab Registry |
-[registry.gitlab.com/conduwuit/conduwuit:main][gl]   | ![Image
-Size][shield-main]    | Stable main branch.   | | Docker Hub      |
-[docker.io/girlbossceo/conduwuit:main][dh]               | ![Image
-Size][shield-main]    | Stable main branch.   |
 
 [dh]: https://hub.docker.com/r/girlbossceo/conduwuit
 [gh]: https://github.com/girlbossceo/conduwuit/pkgs/container/conduwuit
@@ -34,7 +26,9 @@ Size][shield-main]    | Stable main branch.   |
 
 Use
 
-```bash docker image pull <link> ```
+```bash
+docker image pull $LINK
+```
 
 to pull it to your machine.
 
@@ -42,13 +36,12 @@ to pull it to your machine.
 
 When you have the image you can simply run it with
 
-```bash 
+```bash
-docker run -d -p 8448:6167 \ 
+docker run -d -p 8448:6167 \
-    -v db:/var/lib/conduwuit/ \ 
+    -v db:/var/lib/conduwuit/ \
-    -e CONDUWUIT_SERVER_NAME="your.server.name" \ 
+    -e CONDUWUIT_SERVER_NAME="your.server.name" \
-    -e CONDUWUIT_DATABASE_BACKEND="rocksdb" \ 
     -e CONDUWUIT_ALLOW_REGISTRATION=false \
-    --name conduit <link> 
+    --name conduit $LINK
 ```
 
 or you can use [docker compose](#docker-compose).
@@ -88,7 +81,9 @@ server.
 When picking the `caddy-docker-proxy` compose file, it's important to first
 create the `caddy` network before spinning up the containers:
 
-```bash docker network create caddy ```
+```bash
+docker network create caddy
+```
 
 After that, you can rename it so it matches `docker-compose.yml` and spin up the
 containers!
@@ -97,16 +92,28 @@ Additional info about deploying conduwuit can be found [here](generic.md).
 
 ### Build
 
-To build the conduwuit image with docker-compose, you first need to open and
+Official conduwuit images are built using Nix's
-modify the `docker-compose.yml` file. There you need to comment the `image:`
+[`buildLayeredImage`][nix-buildlayeredimage]. This ensures all OCI images are
-option and uncomment the `build:` option. Then call docker compose with:
+repeatable and reproducible by anyone, keeps the images lightweight, and can be
+built offline.
 
-```bash 
+This also ensures portability of our images because `buildLayeredImage` builds
-docker compose up
+OCI images, not Docker images, and works with other container software.
-```
 
-This will also start the container right afterwards, so if want it to run in
+The OCI images are OS-less with only a very minimal environment of the `tini`
-detached mode, you also should use the `-d` flag.
+init system, CA certificates, and the conduwuit binary. This does mean there is
+not a shell, but in theory you can get a shell by adding the necessary layers
+to the layered image. However it's very unlikely you will need a shell for any
+real troubleshooting.
+
+The flake file for the OCI image definition is at [`nix/pkgs/oci-image/default.nix`][oci-image-def].
+
+To build an OCI image using Nix, the following outputs can be built:
+- `nix build -L .#oci-image` (default features, x86_64 glibc)
+- `nix build -L .#oci-image-x86_64-linux-musl` (default features, x86_64 musl)
+- `nix build -L .#oci-image-aarch64-linux-musl` (default features, aarch64 musl)
+- `nix build -L .#oci-image-x86_64-linux-musl-all-features` (all features, x86_64 musl)
+- `nix build -L .#oci-image-aarch64-linux-musl-all-features` (all features, aarch64 musl)
 
 ### Run
 
@@ -141,3 +148,6 @@ those two files.
 ## Voice communication
 
 See the [TURN](../turn.md) page.
+
+[nix-buildlayeredimage]: https://ryantm.github.io/nixpkgs/builders/images/dockertools/#ssec-pkgs-dockerTools-buildLayeredImage
+[oci-image-def]: https://github.com/girlbossceo/conduwuit/blob/main/nix/pkgs/oci-image/default.nix

docs/deploying/freebsd.md

@@ -1,11 +1,5 @@
 # conduwuit for FreeBSD
 
-conduwuit at the moment does not provide FreeBSD builds. Building conduwuit on
+conduwuit at the moment does not provide FreeBSD builds or have FreeBSD packaging, however conduwuit does build and work on FreeBSD using the system-provided RocksDB.
-FreeBSD requires a specific environment variable to use the system prebuilt
-RocksDB library instead of rust-rocksdb / rust-librocksdb-sys which does *not*
-work and will cause a build error or coredump.
 
-Use the following environment variable: `ROCKSDB_LIB_DIR=/usr/local/lib`
+Contributions for getting conduwuit packaged are welcome.
-
-Such example commandline with it can be: `ROCKSDB_LIB_DIR=/usr/local/lib cargo
-build --release`

docs/deploying/generic.md

@@ -13,55 +13,95 @@ what you need.
 
 Prebuilt fully static musl binaries can be downloaded from the latest tagged
 release [here](https://github.com/girlbossceo/conduwuit/releases/latest) or
-`main` CI branch workflow artifact output. These also include Debian packages.
+`main` CI branch workflow artifact output. These also include Debian/Ubuntu packages.
+
 These binaries have jemalloc and io_uring statically linked and included with
-them.
+them, so no additional dynamic dependencies need to be installed.
 
 Alternatively, you may compile the binary yourself. We recommend using
-[Lix](https://lix.systems) to build conduwuit as this has the most guaranteed
+Nix (or [Lix](https://lix.systems)) to build conduwuit as this has the most guaranteed
-reproducibiltiy and easiest to get a build environment and output going.
+reproducibiltiy and easiest to get a build environment and output going. This also
+allows easy cross-compilation.
+
+You can run the `nix build -L .#static-x86_64-linux-musl-all-features` or
+`nix build -L .#static-aarch64-linux-musl-all-features` commands based
+on architecture to cross-compile the necessary static binary located at
+`result/bin/conduit`. This is reproducible with the static binaries produced in our CI.
 
 Otherwise, follow standard Rust project build guides (installing git and cloning
 the repo, getting the Rust toolchain via rustup, installing LLVM toolchain +
 libclang for RocksDB, installing liburing for io_uring and RocksDB, etc).
 
+## Migrating from Conduit
+
+As mentioned in the README, there is little to no steps needed to migrate
+from Conduit. As long as you are using the RocksDB database backend, just
+replace the binary / container image / etc.
+
+**Note**: If you are relying on Conduit's "automatic delegation" feature,
+this will **NOT** work on conduwuit and you must configure delegation manually.
+This is not a mistake and no support for this feature will be added.
+
+If you are using SQLite, you **MUST** migrate to RocksDB. You can use this
+tool to migrate from SQLite to RocksDB: <https://github.com/ShadowJonathan/conduit_toolbox/>
+
+See the `[global.well_known]` config section, or configure your web server
+appropriately to send the delegation responses.
+
 ## Adding a conduwuit user
 
 While conduwuit can run as any user it is better to use dedicated users for
 different services. This also allows you to make sure that the file permissions
 are correctly set up.
 
-In Debian or Fedora/RHEL, you can use this command to create a conduwuit user:
+In Debian, you can use this command to create a conduwuit user:
 
 ```bash
-sudo adduser --system conduwuit --group --disabled-login --no-create-home 
+sudo adduser --system conduwuit --group --disabled-login --no-create-home
 ```
 
-For distros without `adduser`:
+For distros without `adduser` (or where it's a symlink to `useradd`):
 
-```bash sudo useradd -r --shell /usr/bin/nologin --no-create-home conduwuit ```
+```bash
+sudo useradd -r --shell /usr/bin/nologin --no-create-home conduwuit
+```
 
 ## Forwarding ports in the firewall or the router
 
-conduwuit uses the ports 443 and 8448 both of which need to be open in the
+Matrix's default federation port is port 8448, and clients must be using port 443.
-firewall.
+If you would like to use only port 443, or a different port, you will need to setup
+delegation. conduwuit has config options for doing delegation, or you can configure
+your reverse proxy to manually serve the necessary JSON files to do delegation
+(see the `[global.well_known]` config section).
 
 If conduwuit runs behind a router or in a container and has a different public
 IP address than the host system these public ports need to be forwarded directly
 or indirectly to the port mentioned in the config.
 
+Note for NAT users; if you have trouble connecting to your server from the inside
+of your network, you need to research your router and see if it supports "NAT
+hairpinning" or "NAT loopback".
+
+If your router does not support this feature, you need to research doing local
+DNS overrides and force your Matrix DNS records to use your local IP internally.
+This can be done at the host level using `/etc/hosts`. If you need this to be
+on the network level, consider something like NextDNS or Pi-Hole.
+
 ## Setting up a systemd service
 
 The systemd unit for conduwuit can be found
 [here](../configuration/examples.md#example-systemd-unit-file). You may need to
 change the `ExecStart=` path to where you placed the conduwuit binary.
 
+On systems where rsyslog is used alongside journald (i.e. Red Hat-based distros and OpenSUSE), put `$EscapeControlCharactersOnReceive off` inside `/etc/rsyslog.conf` to allow color in logs.
+
 ## Creating the conduwuit configuration file
 
 Now we need to create the conduwuit's config file in
 `/etc/conduwuit/conduwuit.toml`. The example config can be found at
-[conduwuit-example.toml](../configuration/examples.md).**Please take a moment to
+[conduwuit-example.toml](../configuration/examples.md).
-read it. You need to change at least the server name.**
+
+**Please take a moment to read the config. You need to change at least the server name.**
 
 RocksDB is the only supported database backend.
 
@@ -71,53 +111,94 @@ If you are using a dedicated user for conduwuit, you will need to allow it to
 read the config. To do that you can run this:
 
 ```bash
-sudo chown -R root:root /etc/conduwuit sudo chmod -R 755 /etc/conduwuit
+sudo chown -R root:root /etc/conduwuit
+sudo chmod -R 755 /etc/conduwuit
 ```
 
 If you use the default database path you also need to run this:
 
-```bash 
+```bash
-sudo mkdir -p /var/lib/conduwuit/ sudo chown -R conduwuit:conduwuit
+sudo mkdir -p /var/lib/conduwuit/
-/var/lib/conduwuit/ sudo chmod 700 /var/lib/conduwuit/ 
+sudo chown -R conduwuit:conduwuit /var/lib/conduwuit/
+sudo chmod 700 /var/lib/conduwuit/
 ```
 
 ## Setting up the Reverse Proxy
 
 Refer to the documentation or various guides online of your chosen reverse proxy
-software. A [Caddy](https://caddyserver.com/) example will be provided as this
+software. There are many examples of basic Apache/Nginx reverse proxy setups
+out there.
+
+A [Caddy](https://caddyserver.com/) example will be provided as this
 is the recommended reverse proxy for new users and is very trivial to use
 (handles TLS, reverse proxy headers, etc transparently with proper defaults).
 
 Lighttpd is not supported as it seems to mess with the `X-Matrix` Authorization
-header, making federation non-functional. If using Apache, you need to use
+header, making federation non-functional. If a workaround is found, feel free to share to get it added to the documentation here.
-`nocanon` to prevent this.
+
+If using Apache, you need to use `nocanon` in your `ProxyPass` directive to prevent this (note that Apache isn't very good as a general reverse proxy and we discourage the usage of it if you can).
+
+If using Nginx, you need to give conduwuit the request URI using `$request_uri`, or like so:
+- `proxy_pass http://127.0.0.1:6167$request_uri;`
+- `proxy_pass http://127.0.0.1:6167;`
+
+Nginx users need to increase `client_max_body_size` (default is 1M) to match
+`max_request_size` defined in conduwuit.toml.
+
+You will need to reverse proxy everything under following routes:
+- `/_matrix/` - core Matrix C-S and S-S APIs
+- `/_conduwuit/` - ad-hoc conduwuit routes such as `/local_user_count` and
+`/server_version`
+
+You can optionally reverse proxy the following individual routes:
+- `/.well-known/matrix/client` and `/.well-known/matrix/server` if using
+conduwuit to perform delegation (see the `[global.well_known]` config section)
+- `/.well-known/matrix/support` if using conduwuit to send the homeserver admin
+contact and support page (formerly known as MSC1929)
+- `/` if you would like to see `hewwo from conduwuit woof!` at the root
+
+See the following spec pages for more details on these files:
+- [`/.well-known/matrix/server`](https://spec.matrix.org/latest/client-server-api/#getwell-knownmatrixserver)
+- [`/.well-known/matrix/client`](https://spec.matrix.org/latest/client-server-api/#getwell-knownmatrixclient)
+- [`/.well-known/matrix/support`](https://spec.matrix.org/latest/client-server-api/#getwell-knownmatrixsupport)
+
+Examples of delegation:
+- <https://puppygock.gay/.well-known/matrix/server>
+- <https://puppygock.gay/.well-known/matrix/client>
 
 ### Caddy
 
 Create `/etc/caddy/conf.d/conduwuit_caddyfile` and enter this (substitute for
 your server name).
 
-```caddy
+```caddyfile
-your.server.name, your.server.name:8448 { # TCP reverse_proxy
+your.server.name, your.server.name:8448 {
+    # TCP reverse_proxy
     127.0.0.1:6167
-    # UNIX socket 
+    # UNIX socket
-    #reverse_proxy unix//run/conduwuit/conduwuit.sock 
+    #reverse_proxy unix//run/conduwuit/conduwuit.sock
 }
 ```
 
 That's it! Just start and enable the service and you're set.
 
-```bash sudo systemctl enable --now caddy ```
+```bash
+sudo systemctl enable --now caddy
+```
 
 ## You're done
 
 Now you can start conduwuit with:
 
-```bash sudo systemctl start conduwuit ```
+```bash
+sudo systemctl start conduwuit
+```
 
 Set it to start automatically when your system boots with:
 
-```bash sudo systemctl enable conduwuit ```
+```bash
+sudo systemctl enable conduwuit
+```
 
 ## How do I know it works?
 
@@ -127,10 +208,15 @@ homeserver and try to register.
 You can also use these commands as a quick health check (replace
 `your.server.name`).
 
-```bash $ curl https://your.server.name/_conduwuit/server_version
+```bash
+curl https://your.server.name/_conduwuit/server_version
 
-# If using port 8448 $ curl
+# If using port 8448
-https://your.server.name:8448/_conduwuit/server_version ```
+curl https://your.server.name:8448/_conduwuit/server_version
+
+# If federation is enabled
+curl https://your.server.name:8448/_matrix/federation/v1/version
+```
 
 - To check if your server can talk with other homeservers, you can use the
 [Matrix Federation Tester](https://federationtester.matrix.org/). If you can

docs/deploying/kubernetes.md

@@ -0,0 +1,8 @@
+# conduwuit for Kubernetes
+
+conduwuit doesn't support horizontal scalability or distributed loading
+natively, however a community maintained Helm Chart is available here to run
+conduwuit on Kubernetes: <https://gitlab.cronce.io/charts/conduwuit>
+
+Should changes need to be made, please reach out to the maintainer in our
+Matrix room as this is not maintained/controlled by the conduwuit maintainers.

docs/deploying/nixos.md

@@ -1,38 +1,102 @@
 # conduwuit for NixOS
 
-conduwuit can be acquired by [Lix][lix] from various places:
+conduwuit can be acquired by Nix (or [Lix][lix]) from various places:
 
 * The `flake.nix` at the root of the repo
 * The `default.nix` at the root of the repo
 * From conduwuit's binary cache
 
+A community maintained NixOS package is available at [`conduwuit`](https://search.nixos.org/packages?channel=unstable&show=conduwuit&from=0&size=50&sort=relevance&type=packages&query=conduwuit)
+
+### Binary cache
+
 A binary cache for conduwuit that the CI/CD publishes to is available at the
 following places (both are the same just different names):
 
-``` https://attic.kennel.juneis.dog/conduit
+```
+https://attic.kennel.juneis.dog/conduit
 conduit:eEKoUwlQGDdYmAI/Q/0slVlegqh/QmAvQd7HBSm21Wk=
 
 https://attic.kennel.juneis.dog/conduwuit
-conduwuit:BbycGUgTISsltcmH0qNjFR9dbrQNYgdIAcmViSGoVTE= ```
+conduwuit:BbycGUgTISsltcmH0qNjFR9dbrQNYgdIAcmViSGoVTE=
+```
 
-The binary caches have been recreated recently due to attic issues. The old
+The binary caches were recreated some months ago due to attic issues. The old public
-public keys were:
+keys were:
 
-``` conduit:Isq8FGyEC6FOXH6nD+BOeAA+bKp6X6UIbupSlGEPuOg=
+```
+conduit:Isq8FGyEC6FOXH6nD+BOeAA+bKp6X6UIbupSlGEPuOg=
+conduwuit:lYPVh7o1hLu1idH4Xt2QHaRa49WRGSAqzcfFd94aOTw=
+```
 
-conduwuit:lYPVh7o1hLu1idH4Xt2QHaRa49WRGSAqzcfFd94aOTw= ```
 
-If specifying a URL in your flake, please use the GitHub remote:
+If specifying a Git remote URL in your flake, you can use any remotes that
-`github:girlbossceo/conduwuit`
+are specified on the README (the mirrors), such as the GitHub: `github:girlbossceo/conduwuit`
 
-The `flake.nix` and `default.nix` do not (currently) provide a NixOS module, so
+### NixOS module
-(for now) [`services.matrix-conduit`][module] from Nixpkgs should be used to
-configure conduwuit.
 
-If you want to run the latest code, you should get conduwuit from the
+The `flake.nix` and `default.nix` do not currently provide a NixOS module (contributions
-`flake.nix` or `default.nix` and set
+welcome!), so [`services.matrix-conduit`][module] from Nixpkgs can be used to configure
-[`services.matrix-conduit.package`][package] appropriately.
+conduwuit.
+
+### Conduit NixOS Config Module and SQLite
+
+Beware! The [`services.matrix-conduit`][module] module defaults to SQLite as a database backend.
+Conduwuit dropped SQLite support in favor of exclusively supporting the much faster RocksDB.
+Make sure that you are using the RocksDB backend before migrating!
+
+There is a [tool to  migrate a Conduit SQLite database to
+RocksDB](https://github.com/ShadowJonathan/conduit_toolbox/).
+
+If you want to run the latest code, you should get conduwuit from the `flake.nix`
+or `default.nix` and set [`services.matrix-conduit.package`][package]
+appropriately to use conduwuit instead of Conduit.
+
+### UNIX sockets
+
+Due to the lack of a conduwuit NixOS module, when using the `services.matrix-conduit` module
+a workaround like the one below is necessary to use UNIX sockets. This is because the UNIX
+socket option does not exist in Conduit, and the module forcibly sets the `address` and 
+`port` config options.
+
+```nix
+options.services.matrix-conduit.settings = lib.mkOption {
+  apply = old: old // (
+    if (old.global ? "unix_socket_path")
+    then { global = builtins.removeAttrs old.global [ "address" "port" ]; }
+    else {  }
+  );
+};
+
+```
+
+Additionally, the [`matrix-conduit` systemd unit][systemd-unit] in the module does not allow
+the `AF_UNIX` socket address family in their systemd unit's `RestrictAddressFamilies=` which
+disallows the namespace from accessing or creating UNIX sockets and has to be enabled like so:
+
+```nix
+systemd.services.conduit.serviceConfig.RestrictAddressFamilies = [ "AF_UNIX" ];
+```
+
+Even though those workarounds are feasible a conduwuit NixOS configuration module, developed and
+published by the community, would be appreciated.
+
+### jemalloc and hardened profile
+
+conduwuit uses jemalloc by default. This may interfere with the [`hardened.nix` profile][hardened.nix]
+due to them using `scudo` by default. You must either disable/hide `scudo` from conduwuit, or
+disable jemalloc like so:
+
+```nix
+let
+    conduwuit = pkgs.unstable.conduwuit.override {
+      enableJemalloc = false;
+    };
+in
+```
 
 [lix]: https://lix.systems/
 [module]: https://search.nixos.org/options?channel=unstable&query=services.matrix-conduit
 [package]: https://search.nixos.org/options?channel=unstable&query=services.matrix-conduit.package
+[hardened.nix]: https://github.com/NixOS/nixpkgs/blob/master/nixos/modules/profiles/hardened.nix#L22
+[systemd-unit]: https://github.com/NixOS/nixpkgs/blob/master/nixos/modules/services/matrix/conduit.nix#L132

docs/development.md

@@ -1,31 +1,95 @@
 # Development
 
 Information about developing the project. If you are only interested in using
-it, you can safely ignore this section. If you plan on contributing, see the
+it, you can safely ignore this page. If you plan on contributing, see the
-[contributor's guide](contributing.md).
+[contributor's guide](./contributing.md).
 
-## List of forked dependencies During conduwuit development, we have had to fork
+## conduwuit project layout
+
+conduwuit uses a collection of sub-crates, packages, or workspace members
+that indicate what each general area of code is for. All of the workspace
+members are under `src/`. The workspace definition is at the top level / root
+`Cargo.toml`.
+
+The crate names are generally self-explanatory:
+- `admin` is the admin room
+- `api` is the HTTP API, Matrix C-S and S-S endpoints, etc
+- `core` is core conduwuit functionality like config loading, error definitions,
+global utilities, logging infrastructure, etc
+- `database` is RocksDB methods, helpers, RocksDB config, and general database definitions,
+utilities, or functions
+- `macros` are conduwuit Rust [macros][macros] like general helper macros, logging
+and error handling macros, and [syn][syn] and [procedural macros][proc-macro]
+used for admin room commands and others
+- `main` is the "primary" sub-crate. This is where the `main()` function lives,
+tokio worker and async initialisation, Sentry initialisation, [clap][clap] init,
+and signal handling. If you are adding new [Rust features][features], they *must*
+go here.
+- `router` is the webserver and request handling bits, using axum, tower, tower-http,
+hyper, etc, and the [global server state][state] to access `services`.
+- `service` is the high-level database definitions and functions for data,
+outbound/sending code, and other business logic such as media fetching.
+
+It is highly unlikely you will ever need to add a new workspace member, but
+if you truly find yourself needing to, we recommend reaching out to us in
+the Matrix room for discussions about it beforehand.
+
+The primary inspiration for this design was apart of hot reloadable development,
+to support "conduwuit as a library" where specific parts can simply be swapped out.
+There is evidence Conduit wanted to go this route too as `axum` is technically an
+optional feature in Conduit, and can be compiled without the binary or axum library
+for handling inbound web requests; but it was never completed or worked.
+
+See the Rust documentation on [Workspaces][workspaces] for general questions
+and information on Cargo workspaces.
+
+## Adding compile-time [features][features]
+
+If you'd like to add a compile-time feature, you must first define it in
+the `main` workspace crate located in `src/main/Cargo.toml`. The feature must
+enable a feature in the other workspace crate(s) you intend to use it in. Then
+the said workspace crate(s) must define the feature there in its `Cargo.toml`.
+
+So, if this is adding a feature to the API such as `woof`, you define the feature
+in the `api` crate's `Cargo.toml` as `woof = []`. The feature definition in `main`'s
+`Cargo.toml` will be `woof = ["conduit-api/woof"]`.
+
+The rationale for this is due to Rust / Cargo not supporting
+["workspace level features"][9], we must make a choice of; either scattering
+features all over the workspace crates, making it difficult for anyone to add
+or remove default features; or define all the features in one central workspace
+crate that propagate down/up to the other workspace crates. It is a Cargo pitfall,
+and we'd like to see better developer UX in Rust's Workspaces.
+
+Additionally, the definition of one single place makes "feature collection" in our
+Nix flake a million times easier instead of collecting and deduping them all from
+searching in all the workspace crates' `Cargo.toml`s. Though we wouldn't need to
+do this if Rust supported workspace-level features to begin with.
+
+## List of forked dependencies
+
+During conduwuit development, we have had to fork
 some dependencies to support our use-cases in some areas. This ranges from
 things said upstream project won't accept for any reason, faster-paced
 development (unresponsive or slow upstream), conduwuit-specific usecases, or
 lack of time to upstream some things.
 
 - [ruma/ruma][1]: <https://github.com/girlbossceo/ruwuma> - various performance
-improvements, more features, faster-paced development, client/server interop
+improvements, more features, faster-paced development, better client/server interop
 hacks upstream won't accept, etc
 - [facebook/rocksdb][2]: <https://github.com/girlbossceo/rocksdb> - liburing
-build fixes, GCC build fix, and logging callback C API for Rust tracing
+build fixes and GCC debug build fix
-integration
 - [tikv/jemallocator][3]: <https://github.com/girlbossceo/jemallocator> - musl
-builds seem to be broken on upstream
+builds seem to be broken on upstream, fixes some broken/suspicious code in
+places, additional safety measures, and support redzones for Valgrind
 - [zyansheep/rustyline-async][4]:
 <https://github.com/girlbossceo/rustyline-async> - tab completion callback and
-`CTRL+\` signal quit event for CLI
+`CTRL+\` signal quit event for conduwuit console CLI
 - [rust-rocksdb/rust-rocksdb][5]:
-<https://github.com/girlbossceo/rust-rocksdb-zaidoon1> - [`@zaidoon1`'s][8] fork
+<https://github.com/girlbossceo/rust-rocksdb-zaidoon1> - [`@zaidoon1`][8]'s fork
-has quicker updates, more up to date dependencies. Our changes fix musl build
+has quicker updates, more up to date dependencies, etc. Our fork fixes musl build
-issues, Rust part of the logging callback C API, removes unnecessary `gtest`
+issues, removes unnecessary `gtest` include, and uses our RocksDB and jemallocator
-include, and uses our RocksDB and jemallocator
+forks.
 - [tokio-rs/tracing][6]: <https://github.com/girlbossceo/tracing> - Implements
 `Clone` for `EnvFilter` to support dynamically changing tracing envfilter's
 alongside other logging/metrics things
@@ -38,11 +102,17 @@ disable the default `release_max_log_level` feature, and set the `--cfg
 tokio_unstable` flag to enable experimental tokio APIs. A build might look like
 this:
 
-```bash RUSTFLAGS="--cfg tokio_unstable" cargo build \ --release \
+```bash
---no-default-features \
+RUSTFLAGS="--cfg tokio_unstable" cargo +nightly build \
---features=systemd,element_hacks,gzip_compression,brotli_compression,zstd_compression,tokio_console
+    --release \
+    --no-default-features \
+    --features=systemd,element_hacks,gzip_compression,brotli_compression,zstd_compression,tokio_console
 ```
 
+You will also need to enable the `tokio_console` config option in conduwuit when
+starting it. This was due to tokio-console causing gradual memory leak/usage
+if left enabled.
+
 [1]: https://github.com/ruma/ruma/
 [2]: https://github.com/facebook/rocksdb/
 [3]: https://github.com/tikv/jemallocator/
@@ -51,3 +121,11 @@ this:
 [6]: https://github.com/tokio-rs/tracing/
 [7]: https://docs.rs/tokio-console/latest/tokio_console/
 [8]: https://github.com/zaidoon1/
+[9]: https://github.com/rust-lang/cargo/issues/12162
+[workspaces]: https://doc.rust-lang.org/cargo/reference/workspaces.html
+[macros]: https://doc.rust-lang.org/book/ch19-06-macros.html
+[syn]: https://docs.rs/syn/latest/syn/
+[proc-macro]: https://doc.rust-lang.org/reference/procedural-macros.html
+[clap]: https://docs.rs/clap/latest/clap/
+[features]: https://doc.rust-lang.org/cargo/reference/features.html
+[state]: https://docs.rs/axum/latest/axum/extract/struct.State.html

docs/development/testing.md

@@ -5,8 +5,8 @@
 Have a look at [Complement's repository][complement] for an explanation of what
 it is.
 
-To test against Complement, with [Lix][lix] and direnv installed and set up, you
+To test against Complement, with Nix (or [Lix](https://lix.systems) and direnv
-can:
+installed and set up, you can:
 
 * Run `./bin/complement "$COMPLEMENT_SRC" ./path/to/logs.jsonl
 ./path/to/results.jsonl` to build a Complement image, run the tests, and output
@@ -18,6 +18,5 @@ Complement OCI image outputted to `result` (it's a `.tar.gz` file)
 output from the commit/revision you want to test (e.g. from main)
 [here][ci-workflows]
 
-[lix]: https://lix.systems/
 [ci-workflows]: https://github.com/girlbossceo/conduwuit/actions/workflows/ci.yml?query=event%3Apush+is%3Asuccess+actor%3Agirlbossceo
 [complement]: https://github.com/matrix-org/complement

docs/differences.md

@@ -241,8 +241,7 @@ both new users and power users
 - Fixed every single clippy (default lints) and rustc warnings, including some
 that were performance related or potential safety issues / unsoundness
 - Add a **lot** of other clippy and rustc lints and a rustfmt.toml file
-- Repo uses [Renovate](https://docs.renovatebot.com/),
+- Repo uses [Renovate](https://docs.renovatebot.com/) and keeps ALL
-[Trivy](https://github.com/aquasecurity/trivy-action), and keeps ALL
 dependencies as up to date as possible
 - Purge unmaintained/irrelevant/broken database backends (heed, sled, persy) and
 other unnecessary code or overhead

docs/troubleshooting.md

@@ -13,6 +13,17 @@
 > If there are things like Compose file issues or Dockerhub image issues, those
 > can still be mentioned as long as they're something we can fix.
 
+## conduwuit and Matrix issues
+
+#### Lost access to admin room
+
+You can reinvite yourself to the admin room through the following methods:
+- Use the `--execute "users make_user_admin <username>"` conduwuit binary
+argument once to invite yourslf to the admin room on startup
+- Use the conduwuit console/CLI to run the `users make_user_admin` command
+- Or specify the `emergency_password` config option to allow you to temporarily
+log into the server account (`@conduit`) from a web client
+
 ## General potential issues
 
 #### Potential DNS issues when using Docker
@@ -30,16 +41,17 @@ workarounds for this are:
 - Don't use Docker's default DNS setup and instead allow the container to use
 and communicate with your host's DNS servers (host's `/etc/resolv.conf`)
 
-## Rocksdb / database issues
+## RocksDB / database issues
 
 #### Direct IO
 
 Some filesystems may not like RocksDB using [Direct
 IO](https://github.com/facebook/rocksdb/wiki/Direct-IO). Direct IO is for
-non-buffered I/O which improves conduwuit performance, but at least FUSE is a
+non-buffered I/O which improves conduwuit performance and reduces system CPU
-filesystem potentially known to not like this. See the [example
+usage, but at least FUSE and possibly ZFS are filesystems potentially known
-config](configuration/examples.md) for disabling it if needed. Issues from
+to not like this. See the [example config](configuration/examples.md) for
-Direct IO on unsupported filesystems are usually shown as startup errors.
+disabling it if needed. Issues from Direct IO on unsupported filesystems are
+usually shown as startup errors.
 
 #### Database corruption
 
@@ -94,24 +106,46 @@ Various debug commands can be found in `!admin debug`.
 
 #### Debug/Trace log level
 
-conduwuit builds without debug or trace log levels by default for at least
+conduwuit builds without debug or trace log levels at compile time by default
-performance reasons. This may change in the future and/or binaries providing
+for substantial performance gains in CPU usage and improved compile times. If
-such configurations may be provided. If you need to access debug/trace log
+you need to access debug/trace log levels, you will need to build without the
-levels, you will need to build without the `release_max_log_level` feature.
+`release_max_log_level` feature or use our provided static debug binaries.
 
 #### Changing log level dynamically
 
 conduwuit supports changing the tracing log environment filter on-the-fly using
-the admin command `!admin debug change-log-level`. This accepts a string
+the admin command `!admin debug change-log-level <log env filter>`. This accepts
-**without quotes** the same format as the `log` config option.
+a string **without quotes** the same format as the `log` config option.
+
+Example: `!admin debug change-log-level debug`
+
+This can also accept complex filters such as:
+`!admin debug change-log-level info,conduit_service[{dest="example.com"}]=trace,ruma_state_res=trace`
+`!admin debug change-log-level info,conduit_service[{dest="example.com"}]=trace,conduit_service[send{dest="example.org"}]=trace`
+
+And to reset the log level to the one that was set at startup / last config
+load, simply pass the `--reset` flag.
+
+`!admin debug change-log-level --reset`
 
 #### Pinging servers
 
-conduwuit can ping other servers using `!admin debug ping`. This takes a server
+conduwuit can ping other servers using `!admin debug ping <server>`. This takes
-name and goes through the server discovery process and queries
+a server name and goes through the server discovery process and queries
 `/_matrix/federation/v1/version`. Errors are outputted.
 
+While it does measure the latency of the request, it is not indicative of
+server performance on either side as that endpoint is completely unauthenticated
+and simply fetches a string on a static JSON endpoint. It is very low cost both
+bandwidth and computationally.
+
 #### Allocator memory stats
 
-When using jemalloc with jemallocator's `stats` feature, you can see conduwuit's
+When using jemalloc with jemallocator's `stats` feature (`--enable-stats`), you
-jemalloc memory stats by using `!admin debug memory-stats`
+can see conduwuit's high-level allocator stats by using
+`!admin server memory-usage` at the bottom.
+
+If you are a developer, you can also view the raw jemalloc statistics with
+`!admin debug memory-stats`. Please note that this output is extremely large
+which may only be visible in the conduwuit console CLI due to PDU size limits,
+and is not easy for non-developers to understand.

docs/turn.md

@@ -21,6 +21,22 @@ These same values need to be set in conduwuit. See the [example
 config](configuration/examples.md) in the TURN section for configuring these and
 restart conduwuit after.
 
+`turn_secret` or a path to `turn_secret_file` must have a value of your
+coturn `static-auth-secret`, or use `turn_username` and `turn_password`
+if using legacy username:password TURN authentication (not preferred).
+
+`turn_uris` must be the list of TURN URIs you would like to send to the client.
+Typically you will just replace the example domain `example.turn.uri` with the
+`realm` you set from the example config.
+
+If you are using TURN over TLS, you can replace `turn:` with `turns:` in the
+`turn_uris` config option to instruct clients to attempt to connect to
+TURN over TLS. This is highly recommended.
+
+If you need unauthenticated access to the TURN URIs, or some clients may be
+having trouble, you can enable `turn_guest_access` in conduwuit which disables
+authentication for the TURN URI endpoint `/_matrix/client/v3/voip/turnServer`
+
 ### Run
 
 Run the [Coturn](https://hub.docker.com/r/coturn/coturn) image using

engage.toml

@@ -152,7 +152,7 @@ cargo clippy \
 [[task]]
 name = "lychee"
 group = "lints"
-script = "lychee --verbose --offline docs *.md --exclude development.md"
+script = "lychee --verbose --offline docs *.md --exclude development.md --exclude contributing.md --exclude testing.md"
 
 [[task]]
 name = "markdownlint"
@@ -188,6 +188,16 @@ cargo test \
     --color=always
 """
 
+# Checks if the generated example config differs from the checked in repo's
+# example config.
+[[task]]
+name = "example-config"
+group = "tests"
+depends = ["cargo/default"]
+script = """
+git diff --exit-code conduwuit-example.toml
+"""
+
 # Ensure that the flake's default output can build and run without crashing
 #
 # This is a dynamically-linked jemalloc build, which is a case not covered by

flake.lock

@@ -4,16 +4,16 @@
       "inputs": {
         "crane": "crane",
         "flake-compat": "flake-compat",
-        "flake-utils": "flake-utils",
+        "flake-parts": "flake-parts",
         "nixpkgs": "nixpkgs",
         "nixpkgs-stable": "nixpkgs-stable"
       },
       "locked": {
-        "lastModified": 1724226964,
+        "lastModified": 1729116596,
-        "narHash": "sha256-cltFh4su2vcFidxKp7LuEgX3ZGLfPy0DCdrQZ/QTe68=",
+        "narHash": "sha256-NnLMLIXGZtAscUF4dCShksuQ1nOGF6Y2dEeyj0rBbUg=",
         "owner": "zhaofengli",
         "repo": "attic",
-        "rev": "6d9aeaef0a067d664cb11bb7704f7ec373d47fb2",
+        "rev": "2b05b7d986cf6009b1c1ef7daa4961cd1a658782",
         "type": "github"
       },
       "original": {
@@ -28,14 +28,14 @@
         "devenv": "devenv",
         "flake-compat": "flake-compat_3",
         "git-hooks": "git-hooks",
-        "nixpkgs": "nixpkgs_3"
+        "nixpkgs": "nixpkgs_4"
       },
       "locked": {
-        "lastModified": 1724232775,
+        "lastModified": 1728672398,
-        "narHash": "sha256-6u2DycIEgrgNYlLxyGqdFVmBNiKIitnQKJ1pbRP5oko=",
+        "narHash": "sha256-KxuGSoVUFnQLB2ZcYODW7AVPAh9JqRlD5BrfsC/Q4qs=",
         "owner": "cachix",
         "repo": "cachix",
-        "rev": "03b6cb3f953097bff378fb8b9ea094bd091a4ec7",
+        "rev": "aac51f698309fd0f381149214b7eee213c66ef0a",
         "type": "github"
       },
       "original": {
@@ -53,12 +53,51 @@
           "devenv",
           "flake-compat"
         ],
+        "git-hooks": [
+          "cachix",
+          "devenv",
+          "pre-commit-hooks"
+        ],
         "nixpkgs": [
           "cachix",
           "devenv",
           "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1726520618,
+        "narHash": "sha256-jOsaBmJ/EtX5t/vbylCdS7pWYcKGmWOKg4QKUzKr6dA=",
+        "owner": "cachix",
+        "repo": "cachix",
+        "rev": "695525f9086542dfb09fde0871dbf4174abbf634",
+        "type": "github"
+      },
+      "original": {
+        "owner": "cachix",
+        "repo": "cachix",
+        "type": "github"
+      }
+    },
+    "cachix_3": {
+      "inputs": {
+        "devenv": "devenv_3",
+        "flake-compat": [
+          "cachix",
+          "devenv",
+          "cachix",
+          "devenv",
+          "flake-compat"
+        ],
+        "nixpkgs": [
+          "cachix",
+          "devenv",
+          "cachix",
+          "devenv",
+          "nixpkgs"
         ],
         "pre-commit-hooks": [
+          "cachix",
+          "devenv",
           "cachix",
           "devenv",
           "pre-commit-hooks"
@@ -81,11 +120,11 @@
     "complement": {
       "flake": false,
       "locked": {
-        "lastModified": 1722323564,
+        "lastModified": 1724347376,
-        "narHash": "sha256-6w6/N8walz4Ayc9zu7iySqJRmGFukhkaICLn4dweAcA=",
+        "narHash": "sha256-y0e/ULDJ92IhNQZsS/06g0s+AYZ82aJfrIO9qEse94c=",
         "owner": "matrix-org",
         "repo": "complement",
-        "rev": "6e4426a9e63233f9821a4d2382bfed145244183f",
+        "rev": "39733c1b2f8314800776748cc7164f9a34650686",
         "type": "github"
       },
       "original": {
@@ -117,17 +156,12 @@
       }
     },
     "crane_2": {
-      "inputs": {
-        "nixpkgs": [
-          "nixpkgs"
-        ]
-      },
       "locked": {
-        "lastModified": 1724006180,
+        "lastModified": 1729741221,
-        "narHash": "sha256-PVxPj0Ga2fMYMtcT9ARCthF+4U71YkOT7ZjgD/vf1Aw=",
+        "narHash": "sha256-8AHZZXs1lFkERfBY0C8cZGElSo33D/et7NKEpLRmvzo=",
         "owner": "ipetkov",
         "repo": "crane",
-        "rev": "7ce92819802bc583b7e82ebc08013a530f22209f",
+        "rev": "f235b656ee5b2bfd6d94c3bfd67896a575d4a6ed",
         "type": "github"
       },
       "original": {
@@ -144,7 +178,7 @@
           "cachix",
           "flake-compat"
         ],
-        "nix": "nix_2",
+        "nix": "nix_3",
         "nixpkgs": [
           "cachix",
           "nixpkgs"
@@ -154,6 +188,43 @@
           "git-hooks"
         ]
       },
+      "locked": {
+        "lastModified": 1727963652,
+        "narHash": "sha256-os0EDjn7QVXL6RtHNb9TrZLXVm2Tc5/nZKk3KpbTzd8=",
+        "owner": "cachix",
+        "repo": "devenv",
+        "rev": "cb0052e25dbcc8267b3026160dc73cddaac7d5fd",
+        "type": "github"
+      },
+      "original": {
+        "owner": "cachix",
+        "repo": "devenv",
+        "type": "github"
+      }
+    },
+    "devenv_2": {
+      "inputs": {
+        "cachix": "cachix_3",
+        "flake-compat": [
+          "cachix",
+          "devenv",
+          "cachix",
+          "flake-compat"
+        ],
+        "nix": "nix_2",
+        "nixpkgs": [
+          "cachix",
+          "devenv",
+          "cachix",
+          "nixpkgs"
+        ],
+        "pre-commit-hooks": [
+          "cachix",
+          "devenv",
+          "cachix",
+          "git-hooks"
+        ]
+      },
       "locked": {
         "lastModified": 1723156315,
         "narHash": "sha256-0JrfahRMJ37Rf1i0iOOn+8Z4CLvbcGNwa2ChOAVrp/8=",
@@ -168,9 +239,11 @@
         "type": "github"
       }
     },
-    "devenv_2": {
+    "devenv_3": {
       "inputs": {
         "flake-compat": [
+          "cachix",
+          "devenv",
           "cachix",
           "devenv",
           "cachix",
@@ -180,6 +253,8 @@
         "nixpkgs": "nixpkgs_2",
         "poetry2nix": "poetry2nix",
         "pre-commit-hooks": [
+          "cachix",
+          "devenv",
           "cachix",
           "devenv",
           "cachix",
@@ -209,11 +284,11 @@
         "rust-analyzer-src": "rust-analyzer-src"
       },
       "locked": {
-        "lastModified": 1724221791,
+        "lastModified": 1729751566,
-        "narHash": "sha256-mKX67QPnUybOopVph/LhOV1G/H4EvPxDIfSmbufrVdA=",
+        "narHash": "sha256-99u/hrgBdi8bxSXZc9ZbNkR5EL1htrkbd3lsbKzS60g=",
         "owner": "nix-community",
         "repo": "fenix",
-        "rev": "e88b38a5a3834e039d413a88f8150a75ef6453ef",
+        "rev": "f32a2d484091a6dc98220b1f4a2c2d60b7c97c64",
         "type": "github"
       },
       "original": {
@@ -288,27 +363,53 @@
         "type": "github"
       }
     },
-    "flake-utils": {
+    "flake-parts": {
       "inputs": {
-        "systems": "systems"
+        "nixpkgs-lib": [
+          "attic",
+          "nixpkgs"
+        ]
       },
       "locked": {
-        "lastModified": 1710146030,
+        "lastModified": 1722555600,
-        "narHash": "sha256-SZ5L6eA7HJ/nmkzGG7/ISclqe6oZdOZTNoesiInkXPQ=",
+        "narHash": "sha256-XOQkdLafnb/p9ij77byFQjDf5m5QYl9b2REiVClC+x4=",
-        "owner": "numtide",
+        "owner": "hercules-ci",
-        "repo": "flake-utils",
+        "repo": "flake-parts",
-        "rev": "b1d9ab70662946ef0850d488da1c9019f3a9752a",
+        "rev": "8471fe90ad337a8074e957b69ca4d0089218391d",
         "type": "github"
       },
       "original": {
-        "owner": "numtide",
+        "owner": "hercules-ci",
-        "repo": "flake-utils",
+        "repo": "flake-parts",
         "type": "github"
       }
     },
-    "flake-utils_2": {
+    "flake-parts_2": {
       "inputs": {
-        "systems": "systems_2"
+        "nixpkgs-lib": [
+          "cachix",
+          "devenv",
+          "nix",
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1712014858,
+        "narHash": "sha256-sB4SWl2lX95bExY2gMFG5HIzvva5AVMJd4Igm+GpZNw=",
+        "owner": "hercules-ci",
+        "repo": "flake-parts",
+        "rev": "9126214d0a59633752a136528f5f3b9aa8565b7d",
+        "type": "github"
+      },
+      "original": {
+        "owner": "hercules-ci",
+        "repo": "flake-parts",
+        "type": "github"
+      }
+    },
+    "flake-utils": {
+      "inputs": {
+        "systems": "systems"
       },
       "locked": {
         "lastModified": 1689068808,
@@ -324,9 +425,24 @@
         "type": "github"
       }
     },
+    "flake-utils_2": {
+      "locked": {
+        "lastModified": 1667395993,
+        "narHash": "sha256-nuEHfE/LcWyuSWnS8t12N1wc105Qtau+/OdUAjtQ0rA=",
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "rev": "5aed5285a952e0b949eb3ba02c12fa4fcfef535f",
+        "type": "github"
+      },
+      "original": {
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "type": "github"
+      }
+    },
     "flake-utils_3": {
       "inputs": {
-        "systems": "systems_3"
+        "systems": "systems_2"
       },
       "locked": {
         "lastModified": 1710146030,
@@ -357,11 +473,11 @@
         "nixpkgs-stable": "nixpkgs-stable_2"
       },
       "locked": {
-        "lastModified": 1723202784,
+        "lastModified": 1727854478,
-        "narHash": "sha256-qbhjc/NEGaDbyy0ucycubq4N3//gDFFH3DOmp1D3u1Q=",
+        "narHash": "sha256-/odH2nUMAwkMgOS2nG2z0exLQNJS4S2LfMW0teqU7co=",
         "owner": "cachix",
         "repo": "git-hooks.nix",
-        "rev": "c7012d0c18567c889b948781bc74a501e92275d1",
+        "rev": "5f58871c9657b5fc0a7f65670fe2ba99c26c1d79",
         "type": "github"
       },
       "original": {
@@ -392,14 +508,30 @@
         "type": "github"
       }
     },
+    "libgit2": {
+      "flake": false,
+      "locked": {
+        "lastModified": 1697646580,
+        "narHash": "sha256-oX4Z3S9WtJlwvj0uH9HlYcWv+x1hqp8mhXl7HsLu2f0=",
+        "owner": "libgit2",
+        "repo": "libgit2",
+        "rev": "45fd9ed7ae1a9b74b957ef4f337bc3c8b3df01b5",
+        "type": "github"
+      },
+      "original": {
+        "owner": "libgit2",
+        "repo": "libgit2",
+        "type": "github"
+      }
+    },
     "liburing": {
       "flake": false,
       "locked": {
-        "lastModified": 1724199144,
+        "lastModified": 1725659644,
-        "narHash": "sha256-MVjnwO6EbKzzSrU51dSseLarZ1fRp+6SagAf/nE/XZU=",
+        "narHash": "sha256-WjnpmopfvFoUbubIu9bki+Y6P4YXDfvnW4+72hniq3g=",
         "owner": "axboe",
         "repo": "liburing",
-        "rev": "2d4e799017d64cd2f8304503eef9064931bb3fbd",
+        "rev": "0fe5c09195c0918f89582dd6ff098a58a0bdf62a",
         "type": "github"
       },
       "original": {
@@ -417,6 +549,8 @@
           "devenv",
           "cachix",
           "devenv",
+          "cachix",
+          "devenv",
           "nixpkgs"
         ],
         "nixpkgs-regression": "nixpkgs-regression"
@@ -459,6 +593,8 @@
           "devenv",
           "cachix",
           "devenv",
+          "cachix",
+          "devenv",
           "poetry2nix",
           "nixpkgs"
         ]
@@ -480,11 +616,15 @@
     "nix_2": {
       "inputs": {
         "flake-compat": [
+          "cachix",
+          "devenv",
           "cachix",
           "devenv",
           "flake-compat"
         ],
         "nixpkgs": [
+          "cachix",
+          "devenv",
           "cachix",
           "devenv",
           "nixpkgs"
@@ -506,13 +646,42 @@
         "type": "github"
       }
     },
+    "nix_3": {
+      "inputs": {
+        "flake-compat": [
+          "cachix",
+          "devenv",
+          "flake-compat"
+        ],
+        "flake-parts": "flake-parts_2",
+        "libgit2": "libgit2",
+        "nixpkgs": "nixpkgs_3",
+        "nixpkgs-23-11": "nixpkgs-23-11",
+        "nixpkgs-regression": "nixpkgs-regression_3",
+        "pre-commit-hooks": "pre-commit-hooks"
+      },
+      "locked": {
+        "lastModified": 1727438425,
+        "narHash": "sha256-X8ES7I1cfNhR9oKp06F6ir4Np70WGZU5sfCOuNBEwMg=",
+        "owner": "domenkozar",
+        "repo": "nix",
+        "rev": "f6c5ae4c1b2e411e6b1e6a8181cc84363d6a7546",
+        "type": "github"
+      },
+      "original": {
+        "owner": "domenkozar",
+        "ref": "devenv-2.24",
+        "repo": "nix",
+        "type": "github"
+      }
+    },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1723827930,
+        "lastModified": 1726042813,
-        "narHash": "sha256-EU+W5F6y2CVNxGrGIMpY7nSVYq72WRChYxF4zpjx0y4=",
+        "narHash": "sha256-LnNKCCxnwgF+575y0pxUdlGZBO/ru1CtGHIqQVfvjlA=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "d4a7a4d0e066278bfb0d77bd2a7adde1c0ec9e3d",
+        "rev": "159be5db480d1df880a0135ca0bfed84c2f88353",
         "type": "github"
       },
       "original": {
@@ -522,6 +691,22 @@
         "type": "github"
       }
     },
+    "nixpkgs-23-11": {
+      "locked": {
+        "lastModified": 1717159533,
+        "narHash": "sha256-oamiKNfr2MS6yH64rUn99mIZjc45nGJlj9eGth/3Xuw=",
+        "owner": "NixOS",
+        "repo": "nixpkgs",
+        "rev": "a62e6edd6d5e1fa0329b8653c801147986f8d446",
+        "type": "github"
+      },
+      "original": {
+        "owner": "NixOS",
+        "repo": "nixpkgs",
+        "rev": "a62e6edd6d5e1fa0329b8653c801147986f8d446",
+        "type": "github"
+      }
+    },
     "nixpkgs-regression": {
       "locked": {
         "lastModified": 1643052045,
@@ -554,18 +739,34 @@
         "type": "github"
       }
     },
-    "nixpkgs-stable": {
+    "nixpkgs-regression_3": {
       "locked": {
-        "lastModified": 1720535198,
+        "lastModified": 1643052045,
-        "narHash": "sha256-zwVvxrdIzralnSbcpghA92tWu2DV2lwv89xZc8MTrbg=",
+        "narHash": "sha256-uGJ0VXIhWKGXxkeNnq4TvV3CIOkUJ3PAoLZ3HMzNVMw=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "205fd4226592cc83fd4c0885a3e4c9c400efabb5",
+        "rev": "215d4d0fd80ca5163643b03a33fde804a29cc1e2",
         "type": "github"
       },
       "original": {
         "owner": "NixOS",
-        "ref": "nixos-23.11",
+        "repo": "nixpkgs",
+        "rev": "215d4d0fd80ca5163643b03a33fde804a29cc1e2",
+        "type": "github"
+      }
+    },
+    "nixpkgs-stable": {
+      "locked": {
+        "lastModified": 1724316499,
+        "narHash": "sha256-Qb9MhKBUTCfWg/wqqaxt89Xfi6qTD3XpTzQ9eXi3JmE=",
+        "owner": "NixOS",
+        "repo": "nixpkgs",
+        "rev": "797f7dc49e0bc7fab4b57c021cdf68f595e47841",
+        "type": "github"
+      },
+      "original": {
+        "owner": "NixOS",
+        "ref": "nixos-24.05",
         "repo": "nixpkgs",
         "type": "github"
       }
@@ -604,27 +805,27 @@
     },
     "nixpkgs_3": {
       "locked": {
-        "lastModified": 1722813957,
+        "lastModified": 1717432640,
-        "narHash": "sha256-IAoYyYnED7P8zrBFMnmp7ydaJfwTnwcnqxUElC1I26Y=",
+        "narHash": "sha256-+f9c4/ZX5MWDOuB1rKoWj+lBNm0z0rs4CK47HBLxy1o=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "cb9a96f23c491c081b38eab96d22fa958043c9fa",
+        "rev": "88269ab3044128b7c2f4c7d68448b2fb50456870",
         "type": "github"
       },
       "original": {
         "owner": "NixOS",
-        "ref": "nixos-unstable",
+        "ref": "release-24.05",
         "repo": "nixpkgs",
         "type": "github"
       }
     },
     "nixpkgs_4": {
       "locked": {
-        "lastModified": 1724271409,
+        "lastModified": 1727802920,
-        "narHash": "sha256-z4nw9HxkaXEn+5OT8ljLVL2oataHvAzUQ1LEi8Fp+SY=",
+        "narHash": "sha256-HP89HZOT0ReIbI7IJZJQoJgxvB2Tn28V6XS3MNKnfLs=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "36a9aeaaa17a2d4348498275f9fe530cd4f9e519",
+        "rev": "27e30d177e57d912d614c88c622dcfdb2e6e6515",
         "type": "github"
       },
       "original": {
@@ -634,15 +835,33 @@
         "type": "github"
       }
     },
+    "nixpkgs_5": {
+      "locked": {
+        "lastModified": 1725534445,
+        "narHash": "sha256-Yd0FK9SkWy+ZPuNqUgmVPXokxDgMJoGuNpMEtkfcf84=",
+        "owner": "NixOS",
+        "repo": "nixpkgs",
+        "rev": "9bb1e7571aadf31ddb4af77fc64b2d59580f9a39",
+        "type": "github"
+      },
+      "original": {
+        "owner": "NixOS",
+        "ref": "nixpkgs-unstable",
+        "repo": "nixpkgs",
+        "type": "github"
+      }
+    },
     "poetry2nix": {
       "inputs": {
-        "flake-utils": "flake-utils_2",
+        "flake-utils": "flake-utils",
         "nix-github-actions": "nix-github-actions",
         "nixpkgs": [
           "cachix",
           "devenv",
           "cachix",
           "devenv",
+          "cachix",
+          "devenv",
           "nixpkgs"
         ]
       },
@@ -660,19 +879,59 @@
         "type": "github"
       }
     },
+    "pre-commit-hooks": {
+      "inputs": {
+        "flake-compat": [
+          "cachix",
+          "devenv",
+          "nix"
+        ],
+        "flake-utils": "flake-utils_2",
+        "gitignore": [
+          "cachix",
+          "devenv",
+          "nix"
+        ],
+        "nixpkgs": [
+          "cachix",
+          "devenv",
+          "nix",
+          "nixpkgs"
+        ],
+        "nixpkgs-stable": [
+          "cachix",
+          "devenv",
+          "nix",
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1712897695,
+        "narHash": "sha256-nMirxrGteNAl9sWiOhoN5tIHyjBbVi5e2tgZUgZlK3Y=",
+        "owner": "cachix",
+        "repo": "pre-commit-hooks.nix",
+        "rev": "40e6053ecb65fcbf12863338a6dcefb3f55f1bf8",
+        "type": "github"
+      },
+      "original": {
+        "owner": "cachix",
+        "repo": "pre-commit-hooks.nix",
+        "type": "github"
+      }
+    },
     "rocksdb": {
       "flake": false,
       "locked": {
-        "lastModified": 1724285323,
+        "lastModified": 1731690620,
-        "narHash": "sha256-k60kreKQ0v+bQ16yBd2SfLYpuNjMw2qoRmZL/S3k6CU=",
+        "narHash": "sha256-Xd4TJYqPERMJLXaGa6r6Ny1Wlw8Uy5Cyf/8q7nS58QM=",
         "owner": "girlbossceo",
         "repo": "rocksdb",
-        "rev": "5a67ad7ce46328578ee5587fb0c23faa03d14e67",
+        "rev": "292446aa2bc41699204d817a1e4b091679a886eb",
         "type": "github"
       },
       "original": {
         "owner": "girlbossceo",
-        "ref": "v9.5.2",
+        "ref": "v9.7.4",
         "repo": "rocksdb",
         "type": "github"
       }
@@ -688,18 +947,18 @@
         "flake-utils": "flake-utils_3",
         "liburing": "liburing",
         "nix-filter": "nix-filter",
-        "nixpkgs": "nixpkgs_4",
+        "nixpkgs": "nixpkgs_5",
         "rocksdb": "rocksdb"
       }
     },
     "rust-analyzer-src": {
       "flake": false,
       "locked": {
-        "lastModified": 1724153119,
+        "lastModified": 1729715509,
-        "narHash": "sha256-WxpvDJDttkINkXmUA/W5o11lwLPYhATAgu0QUAacZ2g=",
+        "narHash": "sha256-jUDN4e1kObbksb4sc+57NEeujBEDRdLCOu9wiE3RZdM=",
         "owner": "rust-lang",
         "repo": "rust-analyzer",
-        "rev": "3723e5910c14f0ffbd13de474b8a8fcc74db04ce",
+        "rev": "40492e15d49b89cf409e2c5536444131fac49429",
         "type": "github"
       },
       "original": {
@@ -738,21 +997,6 @@
         "repo": "default",
         "type": "github"
       }
-    },
-    "systems_3": {
-      "locked": {
-        "lastModified": 1681028828,
-        "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
-        "owner": "nix-systems",
-        "repo": "default",
-        "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
-        "type": "github"
-      },
-      "original": {
-        "owner": "nix-systems",
-        "repo": "default",
-        "type": "github"
-      }
     }
   },
   "root": "root",

flake.nix

@@ -3,13 +3,13 @@
     attic.url = "github:zhaofengli/attic?ref=main";
     cachix.url = "github:cachix/cachix?ref=master";
     complement = { url = "github:matrix-org/complement?ref=main"; flake = false; };
-    crane = { url = "github:ipetkov/crane?ref=master"; inputs.nixpkgs.follows = "nixpkgs"; };
+    crane = { url = "github:ipetkov/crane?ref=master"; };
     fenix = { url = "github:nix-community/fenix?ref=main"; inputs.nixpkgs.follows = "nixpkgs"; };
     flake-compat = { url = "github:edolstra/flake-compat?ref=master"; flake = false; };
     flake-utils.url = "github:numtide/flake-utils?ref=main";
     nix-filter.url = "github:numtide/nix-filter?ref=main";
-    nixpkgs.url = "github:NixOS/nixpkgs?ref=nixos-unstable";
+    nixpkgs.url = "github:NixOS/nixpkgs?ref=nixpkgs-unstable";
-    rocksdb = { url = "github:girlbossceo/rocksdb?ref=v9.5.2"; flake = false; };
+    rocksdb = { url = "github:girlbossceo/rocksdb?ref=v9.7.4"; flake = false; };
     liburing = { url = "github:axboe/liburing?ref=master"; flake = false; };
   };
 
@@ -27,7 +27,7 @@
         file = ./rust-toolchain.toml;
 
         # See also `rust-toolchain.toml`
-        sha256 = "sha256-3jVIIf5XPnUU1CRaTyAiO0XHVbJl12MSx3eucTXCjtE=";
+        sha256 = "sha256-s1RPtyvDGJaX/BisLT+ifVfuhDT1nZkZ1NcK8sbwELM=";
       };
 
       mkScope = pkgs: pkgs.lib.makeScope pkgs.newScope (self: {
@@ -38,7 +38,23 @@
         inherit inputs;
         main = self.callPackage ./nix/pkgs/main {};
         oci-image = self.callPackage ./nix/pkgs/oci-image {};
-        rocksdb = pkgs.rocksdb.overrideAttrs (old: {
+        tini = pkgs.tini.overrideAttrs {
+            # newer clang/gcc is unhappy with tini-static: <https://3.dog/~strawberry/pb/c8y4>
+            patches = [ (pkgs.fetchpatch {
+                url = "https://patch-diff.githubusercontent.com/raw/krallin/tini/pull/224.patch";
+                hash = "sha256-4bTfAhRyIT71VALhHY13hUgbjLEUyvgkIJMt3w9ag3k=";
+              })
+            ];
+        };
+        liburing = pkgs.liburing.overrideAttrs {
+          # Tests weren't building
+          outputs = [ "out" "dev" "man" ];
+          buildFlags = [ "library" ];
+          src = inputs.liburing;
+        };
+        rocksdb = (pkgs.rocksdb.override {
+          liburing = self.liburing;
+        }).overrideAttrs (old: {
           src = inputs.rocksdb;
           version = pkgs.lib.removePrefix
             "v"
@@ -76,18 +92,20 @@
           # preInstall hooks has stuff for messing with ldb/sst_dump which we dont need or use
           preInstall = "";
         });
-        # TODO: remove once https://github.com/NixOS/nixpkgs/pull/314945 is available
-        liburing = pkgs.liburing.overrideAttrs (old: {
-          # the configure script doesn't support these, and unconditionally
-          # builds both static and dynamic libraries.
-          configureFlags = pkgs.lib.subtractLists
-            [ "--enable-static" "--disable-shared" ]
-            old.configureFlags;
-        });
       });
 
       scopeHost = mkScope pkgsHost;
       scopeHostStatic = mkScope pkgsHostStatic;
+      scopeCrossLinux = mkScope pkgsHost.pkgsLinux.pkgsStatic;
+      mkCrossScope = crossSystem:
+        let pkgsCrossStatic = (import inputs.nixpkgs {
+          inherit system;
+          crossSystem = {
+           config = crossSystem;
+          };
+        }).pkgsStatic;
+         in
+        mkScope pkgsCrossStatic;
 
       mkDevShell = scope: scope.pkgs.mkShell {
         env = scope.main.env // {
@@ -119,6 +137,9 @@
           engage
           cargo-audit
 
+          # Required by hardened-malloc.rs dep
+          binutils
+
           # Needed for producing Debian packages
           cargo-deb
 
@@ -145,12 +166,21 @@
 
           # needed so we can get rid of gcc and other unused deps that bloat OCI images
           removeReferencesTo
-        ])
+        ]
+        # liburing is Linux-exclusive
+        ++ lib.optional stdenv.hostPlatform.isLinux liburing
+        # needed to build Rust applications on macOS
+        ++ lib.optionals stdenv.hostPlatform.isDarwin [
+            # https://github.com/NixOS/nixpkgs/issues/206242
+            # ld: library not found for -liconv
+            libiconv
+            # https://stackoverflow.com/questions/69869574/properly-adding-darwin-apple-sdk-to-a-nix-shell
+            # https://discourse.nixos.org/t/compile-a-rust-binary-on-macos-dbcrossbar/8612
+            pkgsBuildHost.darwin.apple_sdk.frameworks.Security
+            ])
         ++ scope.main.buildInputs
         ++ scope.main.propagatedBuildInputs
         ++ scope.main.nativeBuildInputs;
-
-        meta.broken = scope.main.meta.broken;
       };
     in
     {
@@ -224,6 +254,8 @@
 
         complement = scopeHost.complement;
         static-complement = scopeHostStatic.complement;
+        # macOS containers don't exist, so the complement images must be forced to linux
+        linux-complement = (mkCrossScope "${pkgsHost.hostPlatform.qemuArch}-linux-musl").complement;
       }
       //
       builtins.listToAttrs
@@ -232,14 +264,7 @@
             (crossSystem:
               let
                 binaryName = "static-${crossSystem}";
-                pkgsCrossStatic =
+                scopeCrossStatic = mkCrossScope crossSystem;
-                  (import inputs.nixpkgs {
-                    inherit system;
-                    crossSystem = {
-                      config = crossSystem;
-                    };
-                  }).pkgsStatic;
-                scopeCrossStatic = mkScope pkgsCrossStatic;
               in
               [
                 # An output for a statically-linked binary
@@ -369,11 +394,20 @@
                     };
                   };
                 }
+
+                # An output for a complement OCI image for the specified platform
+                {
+                  name = "complement-${crossSystem}";
+                  value = scopeCrossStatic.complement;
+                }
               ]
             )
             [
-              "x86_64-unknown-linux-musl"
+              #"x86_64-apple-darwin"
-              "aarch64-unknown-linux-musl"
+              #"aarch64-apple-darwin"
+              "x86_64-linux-gnu"
+              "x86_64-linux-musl"
+              "aarch64-linux-musl"
             ]
           )
         );

nix/pkgs/book/default.nix

@@ -14,8 +14,10 @@ stdenv.mkDerivation {
     include = [
       "book.toml"
       "conduwuit-example.toml"
+      "CODE_OF_CONDUCT.md"
       "CONTRIBUTING.md"
       "README.md"
+      "development.md"
       "debian/conduwuit.service"
       "debian/README.md"
       "arch/conduwuit.service"

nix/pkgs/complement/config.toml

@@ -16,6 +16,7 @@ url_preview_domain_contains_allowlist = ["*"]
 media_compat_file_link = false
 media_startup_check = false
 rocksdb_direct_io = false
+log_colors = false
 
 [global.tls]
 certs = "/certificate.crt"

nix/pkgs/complement/default.nix

@@ -18,6 +18,16 @@ let
     all_features = true;
     disable_release_max_log_level = true;
     disable_features = [
+        # no reason to use jemalloc for complement, just has compatibility/build issues
+        "jemalloc"
+        # console/CLI stuff isn't used or relevant for complement
+        "console"
+        "tokio_console"
+        # sentry telemetry isn't useful for complement, disabled by default anyways
+        "sentry_telemetry"
+        "perf_measurements"
+        # the containers don't use or need systemd signal support
+        "systemd"
         # this is non-functional on nix for some reason
         "hardened_malloc"
         # dont include experimental features
@@ -57,7 +67,7 @@ let
 in
 
 dockerTools.buildImage {
-  name = "complement-${main.pname}";
+  name = "complement-conduwuit";
   tag = "main";
 
   copyToRoot = buildEnv {
@@ -78,7 +88,7 @@ dockerTools.buildImage {
       "${lib.getExe start}"
     ];
 
-    Entrypoint = if !stdenv.isDarwin
+    Entrypoint = if !stdenv.hostPlatform.isDarwin
       # Use the `tini` init system so that signals (e.g. ctrl+c/SIGINT)
       # are handled as expected
       then [ "${lib.getExe' tini "tini"}" "--" ]
@@ -87,6 +97,7 @@ dockerTools.buildImage {
     Env = [
       "SSL_CERT_FILE=/complement/ca/ca.crt"
       "CONDUWUIT_CONFIG=${./config.toml}"
+      "RUST_BACKTRACE=full"
     ];
 
     ExposedPorts = {

nix/pkgs/main/cross-compilation-env.nix

@@ -1,5 +1,6 @@
 { lib
 , pkgsBuildHost
+, pkgsBuildTarget
 , rust
 , stdenv
 }:
@@ -13,12 +14,6 @@ lib.optionalAttrs stdenv.hostPlatform.isStatic {
     lib.concatStringsSep
       " "
       ([]
-        ++ lib.optionals
-          stdenv.targetPlatform.isx86_64
-          [ "-C" "target-cpu=x86-64-v2" ]
-        ++ lib.optionals
-          stdenv.targetPlatform.isAarch64
-          [ "-C" "target-cpu=cortex-a55" ] # cortex-a55 == ARMv8.2-a
         # This disables PIE for static builds, which isn't great in terms
         # of security. Unfortunately, my hand is forced because nixpkgs'
         # `libstdc++.a` is built without `-fPIE`, which precludes us from
@@ -41,7 +36,7 @@ lib.optionalAttrs stdenv.hostPlatform.isStatic {
             # including it here. Linkers are weird.
             (stdenv.hostPlatform.isAarch64 || stdenv.hostPlatform.isx86_64)
               && stdenv.hostPlatform.isStatic
-              && !stdenv.isDarwin
+              && !stdenv.hostPlatform.isDarwin
               && !stdenv.cc.bintools.isLLVM
           )
           [
@@ -58,11 +53,12 @@ lib.optionalAttrs stdenv.hostPlatform.isStatic {
 # even covers the case of build scripts that need native code compiled and
 # run on the build platform (I think).
 #
-# [0]: https://github.com/NixOS/nixpkgs/blob/5cdb38bb16c6d0a38779db14fcc766bc1b2394d6/pkgs/build-support/rust/lib/default.nix#L57-L80
+# [0]: https://github.com/NixOS/nixpkgs/blob/nixpkgs-unstable/pkgs/build-support/rust/lib/default.nix#L48-L68
 //
 (
   let
     inherit (rust.lib) envVars;
+    shouldUseLLD = platform: platform.isAarch64 && platform.isStatic && !stdenv.hostPlatform.isDarwin;
   in
   lib.optionalAttrs
     (stdenv.targetPlatform.rust.rustcTarget
@@ -70,23 +66,30 @@ lib.optionalAttrs stdenv.hostPlatform.isStatic {
     (
       let
         inherit (stdenv.targetPlatform.rust) cargoEnvVarTarget;
+        linkerForTarget = if shouldUseLLD stdenv.targetPlatform
+          && !stdenv.cc.bintools.isLLVM # whether stdenv's linker is lld already
+          then "${pkgsBuildTarget.llvmPackages.bintools}/bin/${stdenv.cc.targetPrefix}ld.lld"
+          else envVars.ccForTarget;
       in
       {
         "CC_${cargoEnvVarTarget}" = envVars.ccForTarget;
         "CXX_${cargoEnvVarTarget}" = envVars.cxxForTarget;
-        "CARGO_TARGET_${cargoEnvVarTarget}_LINKER" =
+        "CARGO_TARGET_${cargoEnvVarTarget}_LINKER" = linkerForTarget;
-          envVars.linkerForTarget;
       }
     )
   //
   (
     let
       inherit (stdenv.hostPlatform.rust) cargoEnvVarTarget rustcTarget;
+      linkerForHost = if shouldUseLLD stdenv.targetPlatform
+        && !stdenv.cc.bintools.isLLVM
+        then "${pkgsBuildHost.llvmPackages.bintools}/bin/${stdenv.cc.targetPrefix}ld.lld"
+        else envVars.ccForHost;
     in
     {
       "CC_${cargoEnvVarTarget}" = envVars.ccForHost;
       "CXX_${cargoEnvVarTarget}" = envVars.cxxForHost;
-      "CARGO_TARGET_${cargoEnvVarTarget}_LINKER" = envVars.linkerForHost;
+      "CARGO_TARGET_${cargoEnvVarTarget}_LINKER" = linkerForHost;
       CARGO_BUILD_TARGET = rustcTarget;
     }
   )
@@ -98,7 +101,7 @@ lib.optionalAttrs stdenv.hostPlatform.isStatic {
     {
       "CC_${cargoEnvVarTarget}" = envVars.ccForBuild;
       "CXX_${cargoEnvVarTarget}" = envVars.cxxForBuild;
-      "CARGO_TARGET_${cargoEnvVarTarget}_LINKER" = envVars.linkerForBuild;
+      "CARGO_TARGET_${cargoEnvVarTarget}_LINKER" = envVars.ccForBuild;
       HOST_CC = "${pkgsBuildHost.stdenv.cc}/bin/cc";
       HOST_CXX = "${pkgsBuildHost.stdenv.cc}/bin/c++";
     }

nix/pkgs/main/default.nix

@@ -6,6 +6,7 @@
 , libiconv
 , liburing
 , pkgsBuildHost
+, pkgsBuildTarget
 , rocksdb
 , removeReferencesTo
 , rust
@@ -40,7 +41,7 @@ features'' = lib.subtractLists disable_features' features';
 
 featureEnabled = feature : builtins.elem feature features'';
 
-enableLiburing = featureEnabled "io_uring" && !stdenv.isDarwin;
+enableLiburing = featureEnabled "io_uring" && !stdenv.hostPlatform.isDarwin;
 
 # This derivation will set the JEMALLOC_OVERRIDE variable, causing the
 # tikv-jemalloc-sys crate to use the nixpkgs jemalloc instead of building it's
@@ -72,35 +73,13 @@ buildDepsOnlyEnv =
       # jemalloc symbols are prefixed.
       #
       # [1]: https://github.com/tikv/jemallocator/blob/ab0676d77e81268cd09b059260c75b38dbef2d51/jemalloc-sys/src/env.rs#L17
-      enableJemalloc = featureEnabled "jemalloc" && !stdenv.isDarwin;
+      enableJemalloc = featureEnabled "jemalloc" && !stdenv.hostPlatform.isDarwin;
 
       # for some reason enableLiburing in nixpkgs rocksdb is default true
       # which breaks Darwin entirely
       enableLiburing = enableLiburing;
     }).overrideAttrs (old: {
-      # TODO: static rocksdb fails to build on darwin, also see <https://github.com/NixOS/nixpkgs/issues/320448>
-      # build log at <https://girlboss.ceo/~strawberry/pb/JjGH>
-      meta.broken = stdenv.hostPlatform.isStatic && stdenv.isDarwin;
-
       enableLiburing = enableLiburing;
-
-      sse42Support = stdenv.targetPlatform.isx86_64;
-
-      cmakeFlags = if stdenv.targetPlatform.isx86_64
-        then lib.subtractLists [ "-DPORTABLE=1" ] old.cmakeFlags
-        ++ lib.optionals stdenv.targetPlatform.isx86_64 [
-          "-DPORTABLE=x86-64-v2"
-          "-DUSE_SSE=1"
-          "-DHAVE_SSE=1"
-          "-DHAVE_SSE42=1"
-        ]
-        else if stdenv.targetPlatform.isAarch64
-        then lib.subtractLists [ "-DPORTABLE=1" ] old.cmakeFlags
-        ++ lib.optionals stdenv.targetPlatform.isAarch64 [
-          # cortex-a73 == ARMv8-A
-          "-DPORTABLE=armv8-a"
-        ]
-        else old.cmakeFlags;
     });
   in
   {
@@ -117,6 +96,7 @@ buildDepsOnlyEnv =
     inherit
       lib
       pkgsBuildHost
+      pkgsBuildTarget
       rust
       stdenv;
   });
@@ -127,11 +107,7 @@ buildPackageEnv = {
   # Only needed in static stdenv because these are transitive dependencies of rocksdb
   CARGO_BUILD_RUSTFLAGS = buildDepsOnlyEnv.CARGO_BUILD_RUSTFLAGS
     + lib.optionalString (enableLiburing && stdenv.hostPlatform.isStatic)
-      " -L${lib.getLib liburing}/lib -luring"
+      " -L${lib.getLib liburing}/lib -luring";
-    + lib.optionalString stdenv.targetPlatform.isx86_64
-      " -Ctarget-cpu=x86-64-v2"
-    + lib.optionalString stdenv.targetPlatform.isAarch64
-      " -Ctarget-cpu=cortex-a73"; # cortex-a73 == ARMv8-A
 };
 
 
@@ -159,7 +135,16 @@ commonAttrs = {
     dontStrip = profile == "dev" || profile == "test";
     dontPatchELF = profile == "dev" || profile == "test";
 
-    buildInputs = lib.optional (featureEnabled "jemalloc") rust-jemalloc-sys';
+    buildInputs = lib.optional (featureEnabled "jemalloc") rust-jemalloc-sys'
+    # needed to build Rust applications on macOS
+    ++ lib.optionals stdenv.hostPlatform.isDarwin [
+        # https://github.com/NixOS/nixpkgs/issues/206242
+        # ld: library not found for -liconv
+        libiconv
+        # https://stackoverflow.com/questions/69869574/properly-adding-darwin-apple-sdk-to-a-nix-shell
+        # https://discourse.nixos.org/t/compile-a-rust-binary-on-macos-dbcrossbar/8612
+        pkgsBuildHost.darwin.apple_sdk.frameworks.Security
+      ];
 
     nativeBuildInputs = [
       # bindgen needs the build platform's libclang. Apparently due to "splicing
@@ -176,8 +161,10 @@ commonAttrs = {
       # needed so we can get rid of gcc and other unused deps that bloat OCI images
       removeReferencesTo
   ]
-  ++ lib.optionals stdenv.isDarwin [
+  # needed to build Rust applications on macOS
+  ++ lib.optionals stdenv.hostPlatform.isDarwin [
       # https://github.com/NixOS/nixpkgs/issues/206242
+      # ld: library not found for -liconv
       libiconv
 
       # https://stackoverflow.com/questions/69869574/properly-adding-darwin-apple-sdk-to-a-nix-shell
@@ -189,7 +176,7 @@ commonAttrs = {
     #
     # <https://github.com/input-output-hk/haskell.nix/issues/829>
     postInstall = with pkgsBuildHost; ''
-        find "$out" -type f -exec remove-references-to -t ${stdenv.cc} -t ${gcc} -t ${libgcc} -t ${linuxHeaders} -t ${libidn2} -t ${libunistring} '{}' +
+        find "$out" -type f -exec remove-references-to -t ${stdenv.cc} -t ${gcc} -t ${libgcc} -t ${llvm} -t ${libllvm} -t ${rustc.unwrapped} -t ${rustc} -t ${libidn2} -t ${libunistring} '{}' +
     '';
  };
 in

nix/pkgs/oci-image/default.nix

@@ -14,9 +14,10 @@ dockerTools.buildLayeredImage {
   created = "@${toString inputs.self.lastModified}";
   contents = [
     dockerTools.caCertificates
+    main
   ];
   config = {
-    Entrypoint = if !stdenv.isDarwin
+    Entrypoint = if !stdenv.hostPlatform.isDarwin
       # Use the `tini` init system so that signals (e.g. ctrl+c/SIGINT)
       # are handled as expected
       then [ "${lib.getExe' tini "tini"}" "--" ]
@@ -24,5 +25,8 @@ dockerTools.buildLayeredImage {
     Cmd = [
       "${lib.getExe main}"
     ];
+    Env = [
+      "RUST_BACKTRACE=full"
+    ];
   };
 }

renovate.json

@@ -12,5 +12,15 @@
   "nix": {
     "enabled": true
   },
-  "labels": ["dependencies", "github_actions"]
+  "labels": [
+    "dependencies",
+    "github_actions"
+  ],
+  "ignoreDeps": [
+    "tikv-jemllocator",
+    "tikv-jemalloc-sys",
+    "tikv-jemalloc-ctl",
+    "opentelemetry-rust",
+    "tracing-opentelemetry"
+  ]
 }

rust-toolchain.toml

@@ -2,8 +2,6 @@
 #
 # Other files that need upkeep when this changes:
 #
-# * `.gitlab-ci.yml`
-# * `.github/workflows/ci.yml`
 # * `Cargo.toml`
 # * `flake.nix`
 #
@@ -11,13 +9,20 @@
 # If you're having trouble making the relevant changes, bug a maintainer.
 
 [toolchain]
-channel = "1.80.1"
+channel = "1.83.0"
+profile = "minimal"
 components = [
     # For rust-analyzer
     "rust-src",
+    "rust-analyzer",
+    # For CI and editors
+    "rustfmt",
+    "clippy",
 ]
 targets = [
+    #"x86_64-apple-darwin",
     "x86_64-unknown-linux-gnu",
     "x86_64-unknown-linux-musl",
     "aarch64-unknown-linux-musl",
+    #"aarch64-apple-darwin",
 ]

rustfmt.toml

@@ -1,28 +1,27 @@
-edition = "2021"
+array_width = 80
-
+chain_width = 60
+comment_width = 80
 condense_wildcard_suffixes = true
+edition = "2021"
+fn_call_width = 80
+fn_params_layout = "Compressed"
+fn_single_line = true
 format_code_in_doc_comments = true
 format_macro_bodies = true
 format_macro_matchers = true
 format_strings = true
-hex_literal_case = "Upper"
+group_imports = "StdExternalCrate"
-max_width = 120
-tab_spaces = 4
-array_width = 80
-comment_width = 80
-wrap_comments = true
-fn_params_layout = "Compressed"
-fn_call_width = 80
-fn_single_line = true
 hard_tabs = true
-match_block_trailing_comma = true
+hex_literal_case = "Upper"
 imports_granularity = "Crate"
+match_block_trailing_comma = true
+max_width = 120
+newline_style = "Unix"
 normalize_comments = false
 reorder_impl_items = true
 reorder_imports = true
-group_imports = "StdExternalCrate"
+tab_spaces = 4
-newline_style = "Unix"
 use_field_init_shorthand = true
 use_small_heuristics = "Off"
 use_try_shorthand = true
-chain_width = 60
+wrap_comments = true

src/admin/Cargo.toml

@@ -29,10 +29,11 @@ release_max_log_level = [
 clap.workspace = true
 conduit-api.workspace = true
 conduit-core.workspace = true
+conduit-database.workspace = true
 conduit-macros.workspace = true
 conduit-service.workspace = true
 const-str.workspace = true
-futures-util.workspace = true
+futures.workspace = true
 log.workspace = true
 ruma.workspace = true
 serde_json.workspace = true

src/admin/admin.rs

@@ -9,7 +9,7 @@ use crate::{
 };
 
 #[derive(Debug, Parser)]
-#[command(name = "admin", version = env!("CARGO_PKG_VERSION"))]
+#[command(name = "conduwuit", version = conduit::version())]
 pub(super) enum AdminCommand {
 	#[command(subcommand)]
 	/// - Commands for managing appservices

src/admin/appservice/commands.rs

@@ -11,19 +11,25 @@ pub(super) async fn register(&self) -> Result<RoomMessageEventContent> {
 		));
 	}
 
-	let appservice_config = self.body[1..self.body.len().checked_sub(1).unwrap()].join("\n");
+	let appservice_config_body = self.body[1..self.body.len().checked_sub(1).unwrap()].join("\n");
-	let parsed_config = serde_yaml::from_str::<Registration>(&appservice_config);
+	let parsed_config = serde_yaml::from_str::<Registration>(&appservice_config_body);
 	match parsed_config {
-		Ok(yaml) => match self.services.appservice.register_appservice(yaml).await {
+		Ok(registration) => match self
-			Ok(id) => Ok(RoomMessageEventContent::text_plain(format!(
+			.services
-				"Appservice registered with ID: {id}."
+			.appservice
+			.register_appservice(&registration, &appservice_config_body)
+			.await
+		{
+			Ok(()) => Ok(RoomMessageEventContent::text_plain(format!(
+				"Appservice registered with ID: {}",
+				registration.id
 			))),
 			Err(e) => Ok(RoomMessageEventContent::text_plain(format!(
 				"Failed to register appservice: {e}"
 			))),
 		},
 		Err(e) => Ok(RoomMessageEventContent::text_plain(format!(
-			"Could not parse appservice config: {e}"
+			"Could not parse appservice config as YAML: {e}"
 		))),
 	}
 }

src/admin/check/commands.rs

@@ -1,5 +1,6 @@
 use conduit::Result;
 use conduit_macros::implement;
+use futures::StreamExt;
 use ruma::events::room::message::RoomMessageEventContent;
 
 use crate::Command;
@@ -10,14 +11,12 @@ use crate::Command;
 #[implement(Command, params = "<'_>")]
 pub(super) async fn check_all_users(&self) -> Result<RoomMessageEventContent> {
 	let timer = tokio::time::Instant::now();
-	let results = self.services.users.db.iter();
+	let users = self.services.users.iter().collect::<Vec<_>>().await;
 	let query_time = timer.elapsed();
 
-	let users = results.collect::<Vec<_>>();
-
 	let total = users.len();
-	let err_count = users.iter().filter(|user| user.is_err()).count();
+	let err_count = users.iter().filter(|_user| false).count();
-	let ok_count = users.iter().filter(|user| user.is_ok()).count();
+	let ok_count = users.iter().filter(|_user| true).count();
 
 	let message = format!(
 		"Database query completed in {query_time:?}:\n\n```\nTotal entries: {total:?}\nFailure/Invalid user count: \

src/admin/debug/commands.rs

@@ -1,18 +1,19 @@
 use std::{
-	collections::{BTreeMap, HashMap},
+	collections::HashMap,
 	fmt::Write,
+	iter::once,
 	sync::Arc,
 	time::{Instant, SystemTime},
 };
 
-use api::client::validate_and_add_event_id;
+use conduit::{debug_error, err, info, trace, utils, utils::string::EMPTY, warn, Error, PduEvent, Result};
-use conduit::{debug, debug_error, err, info, trace, utils, warn, Error, PduEvent, Result};
+use futures::{FutureExt, StreamExt};
 use ruma::{
 	api::{client::error::ErrorKind, federation::event::get_room_state},
 	events::room::message::RoomMessageEventContent,
 	CanonicalJsonObject, EventId, OwnedRoomOrAliasId, RoomId, RoomVersionId, ServerName,
 };
-use tokio::sync::RwLock;
+use service::rooms::state_compressor::HashSetCompressStateEvent;
 use tracing_subscriber::EnvFilter;
 
 use crate::admin_command;
@@ -26,37 +27,39 @@ pub(super) async fn echo(&self, message: Vec<String>) -> Result<RoomMessageEvent
 
 #[admin_command]
 pub(super) async fn get_auth_chain(&self, event_id: Box<EventId>) -> Result<RoomMessageEventContent> {
-	let event_id = Arc::<EventId>::from(event_id);
+	let Ok(event) = self.services.rooms.timeline.get_pdu_json(&event_id).await else {
-	if let Some(event) = self.services.rooms.timeline.get_pdu_json(&event_id)? {
+		return Ok(RoomMessageEventContent::notice_plain("Event not found."));
-		let room_id_str = event
+	};
-			.get("room_id")
-			.and_then(|val| val.as_str())
-			.ok_or_else(|| Error::bad_database("Invalid event in database"))?;
 
-		let room_id = <&RoomId>::try_from(room_id_str)
+	let room_id_str = event
-			.map_err(|_| Error::bad_database("Invalid room id field in event in database"))?;
+		.get("room_id")
+		.and_then(|val| val.as_str())
+		.ok_or_else(|| Error::bad_database("Invalid event in database"))?;
 
-		let start = Instant::now();
+	let room_id = <&RoomId>::try_from(room_id_str)
-		let count = self
+		.map_err(|_| Error::bad_database("Invalid room id field in event in database"))?;
-			.services
-			.rooms
-			.auth_chain
-			.event_ids_iter(room_id, vec![event_id])
-			.await?
-			.count();
 
-		let elapsed = start.elapsed();
+	let start = Instant::now();
-		Ok(RoomMessageEventContent::text_plain(format!(
+	let count = self
-			"Loaded auth chain with length {count} in {elapsed:?}"
+		.services
-		)))
+		.rooms
-	} else {
+		.auth_chain
-		Ok(RoomMessageEventContent::text_plain("Event not found."))
+		.event_ids_iter(room_id, once(event_id.as_ref()))
-	}
+		.await?
+		.count()
+		.await;
+
+	let elapsed = start.elapsed();
+	Ok(RoomMessageEventContent::text_plain(format!(
+		"Loaded auth chain with length {count} in {elapsed:?}"
+	)))
 }
 
 #[admin_command]
 pub(super) async fn parse_pdu(&self) -> Result<RoomMessageEventContent> {
-	if self.body.len() < 2 || !self.body[0].trim().starts_with("```") || self.body.last().unwrap_or(&"").trim() != "```"
+	if self.body.len() < 2
+		|| !self.body[0].trim().starts_with("```")
+		|| self.body.last().unwrap_or(&EMPTY).trim() != "```"
 	{
 		return Ok(RoomMessageEventContent::text_plain(
 			"Expected code block in command body. Add --help for details.",
@@ -91,25 +94,28 @@ pub(super) async fn get_pdu(&self, event_id: Box<EventId>) -> Result<RoomMessage
 		.services
 		.rooms
 		.timeline
-		.get_non_outlier_pdu_json(&event_id)?;
+		.get_non_outlier_pdu_json(&event_id)
-	if pdu_json.is_none() {
+		.await;
+
+	if pdu_json.is_err() {
 		outlier = true;
-		pdu_json = self.services.rooms.timeline.get_pdu_json(&event_id)?;
+		pdu_json = self.services.rooms.timeline.get_pdu_json(&event_id).await;
 	}
+
 	match pdu_json {
-		Some(json) => {
+		Ok(json) => {
 			let json_text = serde_json::to_string_pretty(&json).expect("canonical json is valid json");
 			Ok(RoomMessageEventContent::notice_markdown(format!(
 				"{}\n```json\n{}\n```",
 				if outlier {
-					"Outlier PDU found in our database"
+					"Outlier (Rejected / Soft Failed) PDU found in our database"
 				} else {
 					"PDU found in our database"
 				},
 				json_text
 			)))
 		},
-		None => Ok(RoomMessageEventContent::text_plain("PDU not found locally.")),
+		Err(_) => Ok(RoomMessageEventContent::text_plain("PDU not found locally.")),
 	}
 }
 
@@ -130,7 +136,9 @@ pub(super) async fn get_remote_pdu_list(
 		));
 	}
 
-	if self.body.len() < 2 || !self.body[0].trim().starts_with("```") || self.body.last().unwrap_or(&"").trim() != "```"
+	if self.body.len() < 2
+		|| !self.body[0].trim().starts_with("```")
+		|| self.body.last().unwrap_or(&EMPTY).trim() != "```"
 	{
 		return Ok(RoomMessageEventContent::text_plain(
 			"Expected code block in command body. Add --help for details.",
@@ -157,7 +165,8 @@ pub(super) async fn get_remote_pdu_list(
 					.send_message(RoomMessageEventContent::text_plain(format!(
 						"Failed to get remote PDU, ignoring error: {e}"
 					)))
-					.await;
+					.await
+					.ok();
 				warn!("Failed to get remote PDU, ignoring error: {e}");
 			} else {
 				success_count = success_count.saturating_add(1);
@@ -196,6 +205,7 @@ pub(super) async fn get_remote_pdu(
 			&server,
 			ruma::api::federation::event::get_event::v1::Request {
 				event_id: event_id.clone().into(),
+				include_unredacted_content: None,
 			},
 		)
 		.await
@@ -210,12 +220,14 @@ pub(super) async fn get_remote_pdu(
 			})?;
 
 			trace!("Attempting to parse PDU: {:?}", &response.pdu);
-			let parsed_pdu = {
+			let _parsed_pdu = {
 				let parsed_result = self
 					.services
 					.rooms
 					.event_handler
-					.parse_incoming_pdu(&response.pdu);
+					.parse_incoming_pdu(&response.pdu)
+					.await;
+
 				let (event_id, value, room_id) = match parsed_result {
 					Ok(t) => t,
 					Err(e) => {
@@ -230,22 +242,12 @@ pub(super) async fn get_remote_pdu(
 				vec![(event_id, value, room_id)]
 			};
 
-			let pub_key_map = RwLock::new(BTreeMap::new());
-
-			debug!("Attempting to fetch homeserver signing keys for {server}");
-			self.services
-				.server_keys
-				.fetch_required_signing_keys(parsed_pdu.iter().map(|(_event_id, event, _room_id)| event), &pub_key_map)
-				.await
-				.unwrap_or_else(|e| {
-					warn!("Could not fetch all signatures for PDUs from {server}: {e:?}");
-				});
-
 			info!("Attempting to handle event ID {event_id} as backfilled PDU");
 			self.services
 				.rooms
 				.timeline
-				.backfill_pdu(&server, response.pdu, &pub_key_map)
+				.backfill_pdu(&server, response.pdu)
+				.boxed()
 				.await?;
 
 			let json_text = serde_json::to_string_pretty(&json).expect("canonical json is valid json");
@@ -264,15 +266,15 @@ pub(super) async fn get_remote_pdu(
 #[admin_command]
 pub(super) async fn get_room_state(&self, room: OwnedRoomOrAliasId) -> Result<RoomMessageEventContent> {
 	let room_id = self.services.rooms.alias.resolve(&room).await?;
-	let room_state = self
+	let room_state: Vec<_> = self
 		.services
 		.rooms
 		.state_accessor
 		.room_state_full(&room_id)
 		.await?
 		.values()
-		.map(|pdu| pdu.to_state_event())
+		.map(PduEvent::to_state_event)
-		.collect::<Vec<_>>();
+		.collect();
 
 	if room_state.is_empty() {
 		return Ok(RoomMessageEventContent::text_plain(
@@ -333,9 +335,12 @@ pub(super) async fn ping(&self, server: Box<ServerName>) -> Result<RoomMessageEv
 #[admin_command]
 pub(super) async fn force_device_list_updates(&self) -> Result<RoomMessageEventContent> {
 	// Force E2EE device list updates for all users
-	for user_id in self.services.users.iter().filter_map(Result::ok) {
+	self.services
-		self.services.users.mark_device_key_update(&user_id)?;
+		.users
-	}
+		.stream()
+		.for_each(|user_id| self.services.users.mark_device_key_update(user_id))
+		.await;
+
 	Ok(RoomMessageEventContent::text_plain(
 		"Marked all devices for all users as having new keys to update",
 	))
@@ -419,12 +424,10 @@ pub(super) async fn sign_json(&self) -> Result<RoomMessageEventContent> {
 	let string = self.body[1..self.body.len().checked_sub(1).unwrap()].join("\n");
 	match serde_json::from_str(&string) {
 		Ok(mut value) => {
-			ruma::signatures::sign_json(
+			self.services
-				self.services.globals.server_name().as_str(),
+				.server_keys
-				self.services.globals.keypair(),
+				.sign_json(&mut value)
-				&mut value,
+				.expect("our request json is what ruma expects");
-			)
-			.expect("our request json is what ruma expects");
 			let json_text = serde_json::to_string_pretty(&value).expect("canonical json is valid json");
 			Ok(RoomMessageEventContent::text_plain(json_text))
 		},
@@ -442,27 +445,31 @@ pub(super) async fn verify_json(&self) -> Result<RoomMessageEventContent> {
 	}
 
 	let string = self.body[1..self.body.len().checked_sub(1).unwrap()].join("\n");
-	match serde_json::from_str(&string) {
+	match serde_json::from_str::<CanonicalJsonObject>(&string) {
-		Ok(value) => {
+		Ok(value) => match self.services.server_keys.verify_json(&value, None).await {
-			let pub_key_map = RwLock::new(BTreeMap::new());
+			Ok(()) => Ok(RoomMessageEventContent::text_plain("Signature correct")),
-
+			Err(e) => Ok(RoomMessageEventContent::text_plain(format!(
-			self.services
+				"Signature verification failed: {e}"
-				.server_keys
+			))),
-				.fetch_required_signing_keys([&value], &pub_key_map)
-				.await?;
-
-			let pub_key_map = pub_key_map.read().await;
-			match ruma::signatures::verify_json(&pub_key_map, &value) {
-				Ok(()) => Ok(RoomMessageEventContent::text_plain("Signature correct")),
-				Err(e) => Ok(RoomMessageEventContent::text_plain(format!(
-					"Signature verification failed: {e}"
-				))),
-			}
 		},
 		Err(e) => Ok(RoomMessageEventContent::text_plain(format!("Invalid json: {e}"))),
 	}
 }
 
+#[admin_command]
+pub(super) async fn verify_pdu(&self, event_id: Box<EventId>) -> Result<RoomMessageEventContent> {
+	let mut event = self.services.rooms.timeline.get_pdu_json(&event_id).await?;
+
+	event.remove("event_id");
+	let msg = match self.services.server_keys.verify_event(&event, None).await {
+		Ok(ruma::signatures::Verified::Signatures) => "signatures OK, but content hash failed (redaction).",
+		Ok(ruma::signatures::Verified::All) => "signatures and hashes OK.",
+		Err(e) => return Err(e),
+	};
+
+	Ok(RoomMessageEventContent::notice_plain(msg))
+}
+
 #[admin_command]
 #[tracing::instrument(skip(self))]
 pub(super) async fn first_pdu_in_room(&self, room_id: Box<RoomId>) -> Result<RoomMessageEventContent> {
@@ -470,7 +477,8 @@ pub(super) async fn first_pdu_in_room(&self, room_id: Box<RoomId>) -> Result<Roo
 		.services
 		.rooms
 		.state_cache
-		.server_in_room(&self.services.globals.config.server_name, &room_id)?
+		.server_in_room(&self.services.globals.config.server_name, &room_id)
+		.await
 	{
 		return Ok(RoomMessageEventContent::text_plain(
 			"We are not participating in the room / we don't know about the room ID.",
@@ -481,8 +489,9 @@ pub(super) async fn first_pdu_in_room(&self, room_id: Box<RoomId>) -> Result<Roo
 		.services
 		.rooms
 		.timeline
-		.first_pdu_in_room(&room_id)?
+		.first_pdu_in_room(&room_id)
-		.ok_or_else(|| Error::bad_database("Failed to find the first PDU in database"))?;
+		.await
+		.map_err(|_| Error::bad_database("Failed to find the first PDU in database"))?;
 
 	Ok(RoomMessageEventContent::text_plain(format!("{first_pdu:?}")))
 }
@@ -494,7 +503,8 @@ pub(super) async fn latest_pdu_in_room(&self, room_id: Box<RoomId>) -> Result<Ro
 		.services
 		.rooms
 		.state_cache
-		.server_in_room(&self.services.globals.config.server_name, &room_id)?
+		.server_in_room(&self.services.globals.config.server_name, &room_id)
+		.await
 	{
 		return Ok(RoomMessageEventContent::text_plain(
 			"We are not participating in the room / we don't know about the room ID.",
@@ -505,8 +515,9 @@ pub(super) async fn latest_pdu_in_room(&self, room_id: Box<RoomId>) -> Result<Ro
 		.services
 		.rooms
 		.timeline
-		.latest_pdu_in_room(&room_id)?
+		.latest_pdu_in_room(&room_id)
-		.ok_or_else(|| Error::bad_database("Failed to find the latest PDU in database"))?;
+		.await
+		.map_err(|_| Error::bad_database("Failed to find the latest PDU in database"))?;
 
 	Ok(RoomMessageEventContent::text_plain(format!("{latest_pdu:?}")))
 }
@@ -520,7 +531,8 @@ pub(super) async fn force_set_room_state_from_server(
 		.services
 		.rooms
 		.state_cache
-		.server_in_room(&self.services.globals.config.server_name, &room_id)?
+		.server_in_room(&self.services.globals.config.server_name, &room_id)
+		.await
 	{
 		return Ok(RoomMessageEventContent::text_plain(
 			"We are not participating in the room / we don't know about the room ID.",
@@ -531,13 +543,13 @@ pub(super) async fn force_set_room_state_from_server(
 		.services
 		.rooms
 		.timeline
-		.latest_pdu_in_room(&room_id)?
+		.latest_pdu_in_room(&room_id)
-		.ok_or_else(|| Error::bad_database("Failed to find the latest PDU in database"))?;
+		.await
+		.map_err(|_| Error::bad_database("Failed to find the latest PDU in database"))?;
 
-	let room_version = self.services.rooms.state.get_room_version(&room_id)?;
+	let room_version = self.services.rooms.state.get_room_version(&room_id).await?;
 
 	let mut state: HashMap<u64, Arc<EventId>> = HashMap::new();
-	let pub_key_map = RwLock::new(BTreeMap::new());
 
 	let remote_state_response = self
 		.services
@@ -551,30 +563,28 @@ pub(super) async fn force_set_room_state_from_server(
 		)
 		.await?;
 
-	let mut events = Vec::with_capacity(remote_state_response.pdus.len());
-
 	for pdu in remote_state_response.pdus.clone() {
-		events.push(match self.services.rooms.event_handler.parse_incoming_pdu(&pdu) {
+		match self
+			.services
+			.rooms
+			.event_handler
+			.parse_incoming_pdu(&pdu)
+			.await
+		{
 			Ok(t) => t,
 			Err(e) => {
 				warn!("Could not parse PDU, ignoring: {e}");
 				continue;
 			},
-		});
+		};
 	}
 
-	info!("Fetching required signing keys for all the state events we got");
-	self.services
-		.server_keys
-		.fetch_required_signing_keys(events.iter().map(|(_event_id, event, _room_id)| event), &pub_key_map)
-		.await?;
-
 	info!("Going through room_state response PDUs");
-	for result in remote_state_response
+	for result in remote_state_response.pdus.iter().map(|pdu| {
-		.pdus
+		self.services
-		.iter()
+			.server_keys
-		.map(|pdu| validate_and_add_event_id(self.services, pdu, &room_version, &pub_key_map))
+			.validate_and_add_event_id(pdu, &room_version)
-	{
+	}) {
 		let Ok((event_id, value)) = result.await else {
 			continue;
 		};
@@ -587,23 +597,26 @@ pub(super) async fn force_set_room_state_from_server(
 		self.services
 			.rooms
 			.outlier
-			.add_pdu_outlier(&event_id, &value)?;
+			.add_pdu_outlier(&event_id, &value);
+
 		if let Some(state_key) = &pdu.state_key {
 			let shortstatekey = self
 				.services
 				.rooms
 				.short
-				.get_or_create_shortstatekey(&pdu.kind.to_string().into(), state_key)?;
+				.get_or_create_shortstatekey(&pdu.kind.to_string().into(), state_key)
+				.await;
+
 			state.insert(shortstatekey, pdu.event_id.clone());
 		}
 	}
 
 	info!("Going through auth_chain response");
-	for result in remote_state_response
+	for result in remote_state_response.auth_chain.iter().map(|pdu| {
-		.auth_chain
+		self.services
-		.iter()
+			.server_keys
-		.map(|pdu| validate_and_add_event_id(self.services, pdu, &room_version, &pub_key_map))
+			.validate_and_add_event_id(pdu, &room_version)
-	{
+	}) {
 		let Ok((event_id, value)) = result.await else {
 			continue;
 		};
@@ -611,7 +624,7 @@ pub(super) async fn force_set_room_state_from_server(
 		self.services
 			.rooms
 			.outlier
-			.add_pdu_outlier(&event_id, &value)?;
+			.add_pdu_outlier(&event_id, &value);
 	}
 
 	let new_room_state = self
@@ -622,17 +635,22 @@ pub(super) async fn force_set_room_state_from_server(
 		.await?;
 
 	info!("Forcing new room state");
-	let (short_state_hash, new, removed) = self
+	let HashSetCompressStateEvent {
+		shortstatehash: short_state_hash,
+		added,
+		removed,
+	} = self
 		.services
 		.rooms
 		.state_compressor
-		.save_state(room_id.clone().as_ref(), new_room_state)?;
+		.save_state(room_id.clone().as_ref(), new_room_state)
+		.await?;
 
 	let state_lock = self.services.rooms.state.mutex.lock(&room_id).await;
 	self.services
 		.rooms
 		.state
-		.force_state(room_id.clone().as_ref(), short_state_hash, new, removed, &state_lock)
+		.force_state(room_id.clone().as_ref(), short_state_hash, added, removed, &state_lock)
 		.await?;
 
 	info!(
@@ -642,7 +660,8 @@ pub(super) async fn force_set_room_state_from_server(
 	self.services
 		.rooms
 		.state_cache
-		.update_joined_count(&room_id)?;
+		.update_joined_count(&room_id)
+		.await;
 
 	drop(state_lock);
 
@@ -653,10 +672,33 @@ pub(super) async fn force_set_room_state_from_server(
 
 #[admin_command]
 pub(super) async fn get_signing_keys(
-	&self, server_name: Option<Box<ServerName>>, _cached: bool,
+	&self, server_name: Option<Box<ServerName>>, notary: Option<Box<ServerName>>, query: bool,
 ) -> Result<RoomMessageEventContent> {
 	let server_name = server_name.unwrap_or_else(|| self.services.server.config.server_name.clone().into());
-	let signing_keys = self.services.globals.signing_keys_for(&server_name)?;
+
+	if let Some(notary) = notary {
+		let signing_keys = self
+			.services
+			.server_keys
+			.notary_request(&notary, &server_name)
+			.await?;
+
+		return Ok(RoomMessageEventContent::notice_markdown(format!(
+			"```rs\n{signing_keys:#?}\n```"
+		)));
+	}
+
+	let signing_keys = if query {
+		self.services
+			.server_keys
+			.server_request(&server_name)
+			.await?
+	} else {
+		self.services
+			.server_keys
+			.signing_keys_for(&server_name)
+			.await?
+	};
 
 	Ok(RoomMessageEventContent::notice_markdown(format!(
 		"```rs\n{signing_keys:#?}\n```"
@@ -664,34 +706,20 @@ pub(super) async fn get_signing_keys(
 }
 
 #[admin_command]
-#[allow(dead_code)]
+pub(super) async fn get_verify_keys(&self, server_name: Option<Box<ServerName>>) -> Result<RoomMessageEventContent> {
-pub(super) async fn get_verify_keys(
-	&self, server_name: Option<Box<ServerName>>, cached: bool,
-) -> Result<RoomMessageEventContent> {
 	let server_name = server_name.unwrap_or_else(|| self.services.server.config.server_name.clone().into());
-	let mut out = String::new();
 
-	if cached {
-		writeln!(out, "| Key ID | VerifyKey |")?;
-		writeln!(out, "| --- | --- |")?;
-		for (key_id, verify_key) in self.services.globals.verify_keys_for(&server_name)? {
-			writeln!(out, "| {key_id} | {verify_key:?} |")?;
-		}
-
-		return Ok(RoomMessageEventContent::notice_markdown(out));
-	}
-
-	let signature_ids: Vec<String> = Vec::new();
 	let keys = self
 		.services
 		.server_keys
-		.fetch_signing_keys_for_server(&server_name, signature_ids)
+		.verify_keys_for(&server_name)
-		.await?;
+		.await;
 
+	let mut out = String::new();
 	writeln!(out, "| Key ID | Public Key |")?;
 	writeln!(out, "| --- | --- |")?;
 	for (key_id, key) in keys {
-		writeln!(out, "| {key_id} | {key} |")?;
+		writeln!(out, "| {key_id} | {key:?} |")?;
 	}
 
 	Ok(RoomMessageEventContent::notice_markdown(out))
@@ -814,10 +842,10 @@ pub(super) async fn database_stats(
 	&self, property: Option<String>, map: Option<String>,
 ) -> Result<RoomMessageEventContent> {
 	let property = property.unwrap_or_else(|| "rocksdb.stats".to_owned());
-	let map_name = map.as_ref().map_or(utils::string::EMPTY, String::as_str);
+	let map_name = map.as_ref().map_or(EMPTY, String::as_str);
 
 	let mut out = String::new();
-	for (name, map) in self.services.db.iter_maps() {
+	for (name, map) in self.services.db.iter() {
 		if !map_name.is_empty() && *map_name != *name {
 			continue;
 		}

src/admin/debug/mod.rs

@@ -80,8 +80,16 @@ pub(super) enum DebugCommand {
 	GetSigningKeys {
 		server_name: Option<Box<ServerName>>,
 
+		#[arg(long)]
+		notary: Option<Box<ServerName>>,
+
 		#[arg(short, long)]
-		cached: bool,
+		query: bool,
+	},
+
+	/// - Get and display signing keys from local cache or remote server.
+	GetVerifyKeys {
+		server_name: Option<Box<ServerName>>,
 	},
 
 	/// - Sends a federation request to the remote server's
@@ -119,6 +127,13 @@ pub(super) enum DebugCommand {
 	/// the command.
 	VerifyJson,
 
+	/// - Verify PDU
+	///
+	/// This re-verifies a PDU existing in the database found by ID.
+	VerifyPdu {
+		event_id: Box<EventId>,
+	},
+
 	/// - Prints the very first PDU in the specified room (typically
 	///   m.room.create)
 	FirstPduInRoom {

src/admin/federation/commands.rs

@@ -1,19 +1,20 @@
 use std::fmt::Write;
 
 use conduit::Result;
+use futures::StreamExt;
 use ruma::{events::room::message::RoomMessageEventContent, OwnedRoomId, RoomId, ServerName, UserId};
 
-use crate::{admin_command, escape_html, get_room_info};
+use crate::{admin_command, get_room_info};
 
 #[admin_command]
 pub(super) async fn disable_room(&self, room_id: Box<RoomId>) -> Result<RoomMessageEventContent> {
-	self.services.rooms.metadata.disable_room(&room_id, true)?;
+	self.services.rooms.metadata.disable_room(&room_id, true);
 	Ok(RoomMessageEventContent::text_plain("Room disabled."))
 }
 
 #[admin_command]
 pub(super) async fn enable_room(&self, room_id: Box<RoomId>) -> Result<RoomMessageEventContent> {
-	self.services.rooms.metadata.disable_room(&room_id, false)?;
+	self.services.rooms.metadata.disable_room(&room_id, false);
 	Ok(RoomMessageEventContent::text_plain("Room enabled."))
 }
 
@@ -85,7 +86,7 @@ pub(super) async fn remote_user_in_rooms(&self, user_id: Box<UserId>) -> Result<
 		));
 	}
 
-	if !self.services.users.exists(&user_id)? {
+	if !self.services.users.exists(&user_id).await {
 		return Ok(RoomMessageEventContent::text_plain(
 			"Remote user does not exist in our database.",
 		));
@@ -96,9 +97,9 @@ pub(super) async fn remote_user_in_rooms(&self, user_id: Box<UserId>) -> Result<
 		.rooms
 		.state_cache
 		.rooms_joined(&user_id)
-		.filter_map(Result::ok)
+		.then(|room_id| get_room_info(self.services, room_id))
-		.map(|room_id| get_room_info(self.services, &room_id))
+		.collect()
-		.collect();
+		.await;
 
 	if rooms.is_empty() {
 		return Ok(RoomMessageEventContent::text_plain("User is not in any rooms."));
@@ -107,33 +108,15 @@ pub(super) async fn remote_user_in_rooms(&self, user_id: Box<UserId>) -> Result<
 	rooms.sort_by_key(|r| r.1);
 	rooms.reverse();
 
-	let output_plain = format!(
+	let output = format!(
-		"Rooms {user_id} shares with us ({}):\n{}",
+		"Rooms {user_id} shares with us ({}):\n```\n{}\n```",
 		rooms.len(),
 		rooms
 			.iter()
-			.map(|(id, members, name)| format!("{id}\tMembers: {members}\tName: {name}"))
+			.map(|(id, members, name)| format!("{id} | Members: {members} | Name: {name}"))
 			.collect::<Vec<_>>()
 			.join("\n")
 	);
-	let output_html = format!(
-		"<table><caption>Rooms {user_id} shares with us \
-		 ({})</caption>\n<tr><th>id</th>\t<th>members</th>\t<th>name</th></tr>\n{}</table>",
-		rooms.len(),
-		rooms
-			.iter()
-			.fold(String::new(), |mut output, (id, members, name)| {
-				writeln!(
-					output,
-					"<tr><td>{}</td>\t<td>{}</td>\t<td>{}</td></tr>",
-					id,
-					members,
-					escape_html(name)
-				)
-				.expect("should be able to write to string buffer");
-				output
-			})
-	);
 
-	Ok(RoomMessageEventContent::text_html(output_plain, output_html))
+	Ok(RoomMessageEventContent::text_markdown(output))
 }

src/admin/media/commands.rs

@@ -1,6 +1,6 @@
 use std::time::Duration;
 
-use conduit::{debug, info, trace, utils::time::parse_timepoint_ago, warn, Result};
+use conduit::{debug, debug_info, debug_warn, error, info, trace, utils::time::parse_timepoint_ago, Result};
 use conduit_service::media::Dim;
 use ruma::{
 	events::room::message::RoomMessageEventContent, EventId, Mxc, MxcUri, OwnedMxcUri, OwnedServerName, ServerName,
@@ -19,7 +19,7 @@ pub(super) async fn delete(
 	}
 
 	if let Some(mxc) = mxc {
-		debug!("Got MXC URL: {mxc}");
+		trace!("Got MXC URL: {mxc}");
 		self.services
 			.media
 			.delete(&mxc.as_str().try_into()?)
@@ -28,14 +28,15 @@ pub(super) async fn delete(
 		return Ok(RoomMessageEventContent::text_plain(
 			"Deleted the MXC from our database and on our filesystem.",
 		));
-	} else if let Some(event_id) = event_id {
+	}
-		debug!("Got event ID to delete media from: {event_id}");
 
-		let mut mxc_urls = vec![];
+	if let Some(event_id) = event_id {
-		let mut mxc_deletion_count: usize = 0;
+		trace!("Got event ID to delete media from: {event_id}");
+
+		let mut mxc_urls = Vec::with_capacity(4);
 
 		// parsing the PDU for any MXC URLs begins here
-		if let Some(event_json) = self.services.rooms.timeline.get_pdu_json(&event_id)? {
+		if let Ok(event_json) = self.services.rooms.timeline.get_pdu_json(&event_id).await {
 			if let Some(content_key) = event_json.get("content") {
 				debug!("Event ID has \"content\".");
 				let content_obj = content_key.as_object();
@@ -124,18 +125,28 @@ pub(super) async fn delete(
 		}
 
 		if mxc_urls.is_empty() {
-			// we shouldn't get here (should have errored earlier) but just in case for
-			// whatever reason we do...
 			info!("Parsed event ID {event_id} but did not contain any MXC URLs.");
 			return Ok(RoomMessageEventContent::text_plain("Parsed event ID but found no MXC URLs."));
 		}
 
+		let mut mxc_deletion_count: usize = 0;
+
 		for mxc_url in mxc_urls {
-			self.services
+			match self
+				.services
 				.media
 				.delete(&mxc_url.as_str().try_into()?)
-				.await?;
+				.await
-			mxc_deletion_count = mxc_deletion_count.saturating_add(1);
+			{
+				Ok(()) => {
+					debug_info!("Successfully deleted {mxc_url} from filesystem and database");
+					mxc_deletion_count = mxc_deletion_count.saturating_add(1);
+				},
+				Err(e) => {
+					debug_warn!("Failed to delete {mxc_url}, ignoring error and skipping: {e}");
+					continue;
+				},
+			}
 		}
 
 		return Ok(RoomMessageEventContent::text_plain(format!(
@@ -158,34 +169,62 @@ pub(super) async fn delete_list(&self) -> Result<RoomMessageEventContent> {
 		));
 	}
 
+	let mut failed_parsed_mxcs: usize = 0;
+
 	let mxc_list = self
 		.body
 		.to_vec()
 		.drain(1..self.body.len().checked_sub(1).unwrap())
-		.collect::<Vec<_>>();
+		.filter_map(|mxc_s| {
+			mxc_s
+				.try_into()
+				.inspect_err(|e| {
+					debug_warn!("Failed to parse user-provided MXC URI: {e}");
+
+					failed_parsed_mxcs = failed_parsed_mxcs.saturating_add(1);
+				})
+				.ok()
+		})
+		.collect::<Vec<Mxc<'_>>>();
 
 	let mut mxc_deletion_count: usize = 0;
 
-	for mxc in mxc_list {
+	for mxc in &mxc_list {
-		debug!("Deleting MXC {mxc} in bulk");
+		trace!(%failed_parsed_mxcs, %mxc_deletion_count, "Deleting MXC {mxc} in bulk");
-		self.services.media.delete(&mxc.try_into()?).await?;
+		match self.services.media.delete(mxc).await {
-		mxc_deletion_count = mxc_deletion_count
+			Ok(()) => {
-			.checked_add(1)
+				debug_info!("Successfully deleted {mxc} from filesystem and database");
-			.expect("mxc_deletion_count should not get this high");
+				mxc_deletion_count = mxc_deletion_count.saturating_add(1);
+			},
+			Err(e) => {
+				debug_warn!("Failed to delete {mxc}, ignoring error and skipping: {e}");
+				continue;
+			},
+		}
 	}
 
 	Ok(RoomMessageEventContent::text_plain(format!(
-		"Finished bulk MXC deletion, deleted {mxc_deletion_count} total MXCs from our database and the filesystem.",
+		"Finished bulk MXC deletion, deleted {mxc_deletion_count} total MXCs from our database and the filesystem. \
+		 {failed_parsed_mxcs} MXCs failed to be parsed from the database.",
 	)))
 }
 
 #[admin_command]
-pub(super) async fn delete_past_remote_media(&self, duration: String, force: bool) -> Result<RoomMessageEventContent> {
+pub(super) async fn delete_past_remote_media(
+	&self, duration: String, before: bool, after: bool, yes_i_want_to_delete_local_media: bool,
+) -> Result<RoomMessageEventContent> {
+	if before && after {
+		return Ok(RoomMessageEventContent::text_plain(
+			"Please only pick one argument, --before or --after.",
+		));
+	}
+	assert!(!(before && after), "--before and --after should not be specified together");
+
 	let duration = parse_timepoint_ago(&duration)?;
 	let deleted_count = self
 		.services
 		.media
-		.delete_all_remote_media_at_after_time(duration, force)
+		.delete_all_remote_media_at_after_time(duration, before, after, yes_i_want_to_delete_local_media)
 		.await?;
 
 	Ok(RoomMessageEventContent::text_plain(format!(
@@ -194,14 +233,10 @@ pub(super) async fn delete_past_remote_media(&self, duration: String, force: boo
 }
 
 #[admin_command]
-pub(super) async fn delete_all_from_user(&self, username: String, force: bool) -> Result<RoomMessageEventContent> {
+pub(super) async fn delete_all_from_user(&self, username: String) -> Result<RoomMessageEventContent> {
 	let user_id = parse_local_user_id(self.services, &username)?;
 
-	let deleted_count = self
+	let deleted_count = self.services.media.delete_from_user(&user_id).await?;
-		.services
-		.media
-		.delete_from_user(&user_id, force)
-		.await?;
 
 	Ok(RoomMessageEventContent::text_plain(format!(
 		"Deleted {deleted_count} total files.",
@@ -210,34 +245,36 @@ pub(super) async fn delete_all_from_user(&self, username: String, force: bool) -
 
 #[admin_command]
 pub(super) async fn delete_all_from_server(
-	&self, server_name: Box<ServerName>, force: bool,
+	&self, server_name: Box<ServerName>, yes_i_want_to_delete_local_media: bool,
 ) -> Result<RoomMessageEventContent> {
-	if server_name == self.services.globals.server_name() {
+	if server_name == self.services.globals.server_name() && !yes_i_want_to_delete_local_media {
-		return Ok(RoomMessageEventContent::text_plain("This command only works for remote media."));
+		return Ok(RoomMessageEventContent::text_plain(
+			"This command only works for remote media by default.",
+		));
 	}
 
-	let Ok(all_mxcs) = self.services.media.get_all_mxcs().await else {
+	let Ok(all_mxcs) = self
+		.services
+		.media
+		.get_all_mxcs()
+		.await
+		.inspect_err(|e| error!("Failed to get MXC URIs from our database: {e}"))
+	else {
 		return Ok(RoomMessageEventContent::text_plain("Failed to get MXC URIs from our database"));
 	};
 
 	let mut deleted_count: usize = 0;
 
 	for mxc in all_mxcs {
-		let mxc_server_name = match mxc.server_name() {
+		let Ok(mxc_server_name) = mxc.server_name().inspect_err(|e| {
-			Ok(server_name) => server_name,
+			debug_warn!("Failed to parse MXC {mxc} server name from database, ignoring error and skipping: {e}");
-			Err(e) => {
+		}) else {
-				if force {
+			continue;
-					warn!("Failed to parse MXC {mxc} server name from database, ignoring error and skipping: {e}");
-					continue;
-				}
-
-				return Ok(RoomMessageEventContent::text_plain(format!(
-					"Failed to parse MXC {mxc} server name from database: {e}",
-				)));
-			},
 		};
 
-		if mxc_server_name != server_name || self.services.globals.server_is_ours(mxc_server_name) {
+		if mxc_server_name != server_name
+			|| (self.services.globals.server_is_ours(mxc_server_name) && !yes_i_want_to_delete_local_media)
+		{
 			trace!("skipping MXC URI {mxc}");
 			continue;
 		}
@@ -249,12 +286,8 @@ pub(super) async fn delete_all_from_server(
 				deleted_count = deleted_count.saturating_add(1);
 			},
 			Err(e) => {
-				if force {
+				debug_warn!("Failed to delete {mxc}, ignoring error and skipping: {e}");
-					warn!("Failed to delete {mxc}, ignoring error and skipping: {e}");
+				continue;
-					continue;
-				}
-
-				return Ok(RoomMessageEventContent::text_plain(format!("Failed to delete MXC {mxc}: {e}")));
 			},
 		}
 	}
@@ -267,7 +300,7 @@ pub(super) async fn delete_all_from_server(
 #[admin_command]
 pub(super) async fn get_file_info(&self, mxc: OwnedMxcUri) -> Result<RoomMessageEventContent> {
 	let mxc: Mxc<'_> = mxc.as_str().try_into()?;
-	let metadata = self.services.media.get_metadata(&mxc);
+	let metadata = self.services.media.get_metadata(&mxc).await;
 
 	Ok(RoomMessageEventContent::notice_markdown(format!("```\n{metadata:#?}\n```")))
 }

src/admin/media/mod.rs

@@ -10,7 +10,7 @@ use crate::admin_command_dispatch;
 #[derive(Debug, Subcommand)]
 pub(super) enum MediaCommand {
 	/// - Deletes a single media file from our database and on the filesystem
-	///   via a single MXC URL
+	///   via a single MXC URL or event ID (not redacted)
 	Delete {
 		/// The MXC URL to delete
 		#[arg(long)]
@@ -23,37 +23,44 @@ pub(super) enum MediaCommand {
 	},
 
 	/// - Deletes a codeblock list of MXC URLs from our database and on the
-	///   filesystem
+	///   filesystem. This will always ignore errors.
 	DeleteList,
 
-	/// - Deletes all remote media in the last X amount of time using filesystem
+	/// - Deletes all remote media in the last/after "X" time using filesystem
-	///   metadata first created at date.
+	///   metadata first created at date, or fallback to last modified date.
+	///   This will always ignore errors by default.
+	///
+	/// Synapse
 	DeletePastRemoteMedia {
-		/// - The duration (at or after), e.g. "5m" to delete all media in the
+		/// - The duration (at or after/before), e.g. "5m" to delete all media
-		///   past 5 minutes
+		///   in the past or up to 5 minutes
 		duration: String,
 
-		/// Continues deleting remote media if an undeletable object is found
+		#[arg(long, short)]
-		#[arg(short, long)]
+		before: bool,
-		force: bool,
+
+		#[arg(long, short)]
+		after: bool,
+
+		/// Long argument to delete local media
+		#[arg(long)]
+		yes_i_want_to_delete_local_media: bool,
 	},
 
-	/// - Deletes all the local media from a local user on our server
+	/// - Deletes all the local media from a local user on our server. This will
+	///   always ignore errors by default.
 	DeleteAllFromUser {
 		username: String,
-
-		/// Continues deleting media if an undeletable object is found
-		#[arg(short, long)]
-		force: bool,
 	},
 
-	/// - Deletes all remote media from the specified remote server
+	/// - Deletes all remote media from the specified remote server. This will
+	///   always ignore errors by default.
 	DeleteAllFromServer {
 		server_name: Box<ServerName>,
 
-		/// Continues deleting media if an undeletable object is found
+		/// Long argument to delete local media
-		#[arg(short, long)]
+		#[arg(long)]
-		force: bool,
+		yes_i_want_to_delete_local_media: bool,
 	},
 
 	GetFileInfo {
@@ -82,10 +89,10 @@ pub(super) enum MediaCommand {
 		#[arg(short, long, default_value("10000"))]
 		timeout: u32,
 
-		#[arg(short, long)]
+		#[arg(short, long, default_value("800"))]
 		width: u32,
 
-		#[arg(short, long)]
+		#[arg(short, long, default_value("800"))]
 		height: u32,
 	},
 }

src/admin/processor.rs

@@ -15,9 +15,9 @@ use conduit::{
 	},
 	trace,
 	utils::string::{collect_stream, common_prefix},
-	Error, Result,
+	warn, Error, Result,
 };
-use futures_util::future::FutureExt;
+use futures::future::FutureExt;
 use ruma::{
 	events::{
 		relation::InReplyTo,
@@ -114,7 +114,15 @@ async fn process(context: &Command<'_>, command: AdminCommand, args: &[String])
 
 fn capture_create(context: &Command<'_>) -> (Arc<Capture>, Arc<Mutex<String>>) {
 	let env_config = &context.services.server.config.admin_log_capture;
-	let env_filter = EnvFilter::try_new(env_config).unwrap_or_else(|_| "debug".into());
+	let env_filter = EnvFilter::try_new(env_config).unwrap_or_else(|e| {
+		warn!("admin_log_capture filter invalid: {e:?}");
+		cfg!(debug_assertions)
+			.then_some("debug")
+			.or(Some("info"))
+			.map(Into::into)
+			.expect("default capture EnvFilter")
+	});
+
 	let log_level = env_filter
 		.max_level_hint()
 		.and_then(LevelFilter::into_level)
@@ -149,10 +157,7 @@ fn parse<'a>(
 			let message = error
 				.to_string()
 				.replace("server.name", services.globals.server_name().as_str());
-			Err(reply(
+			Err(reply(RoomMessageEventContent::notice_plain(message), input.reply_id.as_deref()))
-				RoomMessageEventContent::notice_markdown(message),
-				input.reply_id.as_deref(),
-			))
 		},
 	}
 }

src/admin/query/account_data.rs

@@ -1,9 +1,7 @@
 use clap::Subcommand;
 use conduit::Result;
-use ruma::{
+use futures::StreamExt;
-	events::{room::message::RoomMessageEventContent, RoomAccountDataEventType},
+use ruma::{events::room::message::RoomMessageEventContent, RoomId, UserId};
-	RoomId, UserId,
-};
 
 use crate::Command;
 
@@ -25,7 +23,7 @@ pub(crate) enum AccountDataCommand {
 		/// Full user ID
 		user_id: Box<UserId>,
 		/// Account data event type
-		kind: RoomAccountDataEventType,
+		kind: String,
 		/// Optional room ID of the account data
 		room_id: Option<Box<RoomId>>,
 	},
@@ -42,9 +40,11 @@ pub(super) async fn process(subcommand: AccountDataCommand, context: &Command<'_
 			room_id,
 		} => {
 			let timer = tokio::time::Instant::now();
-			let results = services
+			let results: Vec<_> = services
 				.account_data
-				.changes_since(room_id.as_deref(), &user_id, since)?;
+				.changes_since(room_id.as_deref(), &user_id, since)
+				.collect()
+				.await;
 			let query_time = timer.elapsed();
 
 			Ok(RoomMessageEventContent::notice_markdown(format!(
@@ -59,7 +59,8 @@ pub(super) async fn process(subcommand: AccountDataCommand, context: &Command<'_
 			let timer = tokio::time::Instant::now();
 			let results = services
 				.account_data
-				.get(room_id.as_deref(), &user_id, kind)?;
+				.get_raw(room_id.as_deref(), &user_id, &kind)
+				.await;
 			let query_time = timer.elapsed();
 
 			Ok(RoomMessageEventContent::notice_markdown(format!(

src/admin/query/appservice.rs

@@ -26,10 +26,8 @@ pub(super) async fn process(subcommand: AppserviceCommand, context: &Command<'_>
 			appservice_id,
 		} => {
 			let timer = tokio::time::Instant::now();
-			let results = services
+			let results = services.appservice.get_registration(&appservice_id).await;
-				.appservice
+
-				.db
-				.get_registration(appservice_id.as_ref());
 			let query_time = timer.elapsed();
 
 			Ok(RoomMessageEventContent::notice_markdown(format!(
@@ -38,7 +36,7 @@ pub(super) async fn process(subcommand: AppserviceCommand, context: &Command<'_>
 		},
 		AppserviceCommand::All => {
 			let timer = tokio::time::Instant::now();
-			let results = services.appservice.all();
+			let results = services.appservice.all().await;
 			let query_time = timer.elapsed();
 
 			Ok(RoomMessageEventContent::notice_markdown(format!(

src/admin/query/globals.rs

@@ -13,8 +13,6 @@ pub(crate) enum GlobalsCommand {
 
 	LastCheckForUpdatesId,
 
-	LoadKeypair,
-
 	/// - This returns an empty `Ok(BTreeMap<..>)` when there are no keys found
 	///   for the server.
 	SigningKeysFor {
@@ -29,7 +27,7 @@ pub(super) async fn process(subcommand: GlobalsCommand, context: &Command<'_>) -
 	match subcommand {
 		GlobalsCommand::DatabaseVersion => {
 			let timer = tokio::time::Instant::now();
-			let results = services.globals.db.database_version();
+			let results = services.globals.db.database_version().await;
 			let query_time = timer.elapsed();
 
 			Ok(RoomMessageEventContent::notice_markdown(format!(
@@ -47,16 +45,7 @@ pub(super) async fn process(subcommand: GlobalsCommand, context: &Command<'_>) -
 		},
 		GlobalsCommand::LastCheckForUpdatesId => {
 			let timer = tokio::time::Instant::now();
-			let results = services.updates.last_check_for_updates_id();
+			let results = services.updates.last_check_for_updates_id().await;
-			let query_time = timer.elapsed();
-
-			Ok(RoomMessageEventContent::notice_markdown(format!(
-				"Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```"
-			)))
-		},
-		GlobalsCommand::LoadKeypair => {
-			let timer = tokio::time::Instant::now();
-			let results = services.globals.db.load_keypair();
 			let query_time = timer.elapsed();
 
 			Ok(RoomMessageEventContent::notice_markdown(format!(
@@ -67,7 +56,7 @@ pub(super) async fn process(subcommand: GlobalsCommand, context: &Command<'_>) -
 			origin,
 		} => {
 			let timer = tokio::time::Instant::now();
-			let results = services.globals.db.verify_keys_for(&origin);
+			let results = services.server_keys.verify_keys_for(&origin).await;
 			let query_time = timer.elapsed();
 
 			Ok(RoomMessageEventContent::notice_markdown(format!(

src/admin/query/mod.rs

@@ -2,6 +2,7 @@ mod account_data;
 mod appservice;
 mod globals;
 mod presence;
+mod pusher;
 mod resolver;
 mod room_alias;
 mod room_state_cache;
@@ -13,7 +14,7 @@ use conduit::Result;
 
 use self::{
 	account_data::AccountDataCommand, appservice::AppserviceCommand, globals::GlobalsCommand,
-	presence::PresenceCommand, resolver::ResolverCommand, room_alias::RoomAliasCommand,
+	presence::PresenceCommand, pusher::PusherCommand, resolver::ResolverCommand, room_alias::RoomAliasCommand,
 	room_state_cache::RoomStateCacheCommand, sending::SendingCommand, users::UsersCommand,
 };
 use crate::admin_command_dispatch;
@@ -57,4 +58,8 @@ pub(super) enum QueryCommand {
 	/// - resolver service
 	#[command(subcommand)]
 	Resolver(ResolverCommand),
+
+	/// - pusher service
+	#[command(subcommand)]
+	Pusher(PusherCommand),
 }

src/admin/query/presence.rs

@@ -1,5 +1,6 @@
 use clap::Subcommand;
 use conduit::Result;
+use futures::StreamExt;
 use ruma::{events::room::message::RoomMessageEventContent, UserId};
 
 use crate::Command;
@@ -30,7 +31,7 @@ pub(super) async fn process(subcommand: PresenceCommand, context: &Command<'_>)
 			user_id,
 		} => {
 			let timer = tokio::time::Instant::now();
-			let results = services.presence.db.get_presence(&user_id)?;
+			let results = services.presence.db.get_presence(&user_id).await;
 			let query_time = timer.elapsed();
 
 			Ok(RoomMessageEventContent::notice_markdown(format!(
@@ -41,12 +42,16 @@ pub(super) async fn process(subcommand: PresenceCommand, context: &Command<'_>)
 			since,
 		} => {
 			let timer = tokio::time::Instant::now();
-			let results = services.presence.db.presence_since(since);
+			let results: Vec<(_, _, _)> = services
-			let presence_since: Vec<(_, _, _)> = results.collect();
+				.presence
+				.presence_since(since)
+				.map(|(user_id, count, bytes)| (user_id.to_owned(), count, bytes.to_vec()))
+				.collect()
+				.await;
 			let query_time = timer.elapsed();
 
 			Ok(RoomMessageEventContent::notice_markdown(format!(
-				"Query completed in {query_time:?}:\n\n```rs\n{presence_since:#?}\n```"
+				"Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```"
 			)))
 		},
 	}

src/admin/query/pusher.rs

@@ -0,0 +1,32 @@
+use clap::Subcommand;
+use conduit::Result;
+use ruma::{events::room::message::RoomMessageEventContent, UserId};
+
+use crate::Command;
+
+#[derive(Debug, Subcommand)]
+pub(crate) enum PusherCommand {
+	/// - Returns all the pushers for the user.
+	GetPushers {
+		/// Full user ID
+		user_id: Box<UserId>,
+	},
+}
+
+pub(super) async fn process(subcommand: PusherCommand, context: &Command<'_>) -> Result<RoomMessageEventContent> {
+	let services = context.services;
+
+	match subcommand {
+		PusherCommand::GetPushers {
+			user_id,
+		} => {
+			let timer = tokio::time::Instant::now();
+			let results = services.pusher.get_pushers(&user_id).await;
+			let query_time = timer.elapsed();
+
+			Ok(RoomMessageEventContent::notice_markdown(format!(
+				"Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```"
+			)))
+		},
+	}
+}

src/admin/query/room_alias.rs

@@ -1,5 +1,6 @@
 use clap::Subcommand;
 use conduit::Result;
+use futures::StreamExt;
 use ruma::{events::room::message::RoomMessageEventContent, RoomAliasId, RoomId};
 
 use crate::Command;
@@ -31,7 +32,7 @@ pub(super) async fn process(subcommand: RoomAliasCommand, context: &Command<'_>)
 			alias,
 		} => {
 			let timer = tokio::time::Instant::now();
-			let results = services.rooms.alias.resolve_local_alias(&alias);
+			let results = services.rooms.alias.resolve_local_alias(&alias).await;
 			let query_time = timer.elapsed();
 
 			Ok(RoomMessageEventContent::notice_markdown(format!(
@@ -42,8 +43,13 @@ pub(super) async fn process(subcommand: RoomAliasCommand, context: &Command<'_>)
 			room_id,
 		} => {
 			let timer = tokio::time::Instant::now();
-			let results = services.rooms.alias.local_aliases_for_room(&room_id);
+			let aliases: Vec<_> = services
-			let aliases: Vec<_> = results.collect();
+				.rooms
+				.alias
+				.local_aliases_for_room(&room_id)
+				.map(ToOwned::to_owned)
+				.collect()
+				.await;
 			let query_time = timer.elapsed();
 
 			Ok(RoomMessageEventContent::notice_markdown(format!(
@@ -52,8 +58,13 @@ pub(super) async fn process(subcommand: RoomAliasCommand, context: &Command<'_>)
 		},
 		RoomAliasCommand::AllLocalAliases => {
 			let timer = tokio::time::Instant::now();
-			let results = services.rooms.alias.all_local_aliases();
+			let aliases = services
-			let aliases: Vec<_> = results.collect();
+				.rooms
+				.alias
+				.all_local_aliases()
+				.map(|(room_id, alias)| (room_id.to_owned(), alias.to_owned()))
+				.collect::<Vec<_>>()
+				.await;
 			let query_time = timer.elapsed();
 
 			Ok(RoomMessageEventContent::notice_markdown(format!(

src/admin/query/room_state_cache.rs

@@ -1,5 +1,6 @@
 use clap::Subcommand;
 use conduit::Result;
+use futures::StreamExt;
 use ruma::{events::room::message::RoomMessageEventContent, RoomId, ServerName, UserId};
 
 use crate::Command;
@@ -86,7 +87,11 @@ pub(super) async fn process(
 			room_id,
 		} => {
 			let timer = tokio::time::Instant::now();
-			let result = services.rooms.state_cache.server_in_room(&server, &room_id);
+			let result = services
+				.rooms
+				.state_cache
+				.server_in_room(&server, &room_id)
+				.await;
 			let query_time = timer.elapsed();
 
 			Ok(RoomMessageEventContent::notice_markdown(format!(
@@ -97,7 +102,13 @@ pub(super) async fn process(
 			room_id,
 		} => {
 			let timer = tokio::time::Instant::now();
-			let results: Result<Vec<_>> = services.rooms.state_cache.room_servers(&room_id).collect();
+			let results: Vec<_> = services
+				.rooms
+				.state_cache
+				.room_servers(&room_id)
+				.map(ToOwned::to_owned)
+				.collect()
+				.await;
 			let query_time = timer.elapsed();
 
 			Ok(RoomMessageEventContent::notice_markdown(format!(
@@ -108,7 +119,13 @@ pub(super) async fn process(
 			server,
 		} => {
 			let timer = tokio::time::Instant::now();
-			let results: Result<Vec<_>> = services.rooms.state_cache.server_rooms(&server).collect();
+			let results: Vec<_> = services
+				.rooms
+				.state_cache
+				.server_rooms(&server)
+				.map(ToOwned::to_owned)
+				.collect()
+				.await;
 			let query_time = timer.elapsed();
 
 			Ok(RoomMessageEventContent::notice_markdown(format!(
@@ -119,7 +136,13 @@ pub(super) async fn process(
 			room_id,
 		} => {
 			let timer = tokio::time::Instant::now();
-			let results: Result<Vec<_>> = services.rooms.state_cache.room_members(&room_id).collect();
+			let results: Vec<_> = services
+				.rooms
+				.state_cache
+				.room_members(&room_id)
+				.map(ToOwned::to_owned)
+				.collect()
+				.await;
 			let query_time = timer.elapsed();
 
 			Ok(RoomMessageEventContent::notice_markdown(format!(
@@ -134,7 +157,9 @@ pub(super) async fn process(
 				.rooms
 				.state_cache
 				.local_users_in_room(&room_id)
-				.collect();
+				.map(ToOwned::to_owned)
+				.collect()
+				.await;
 			let query_time = timer.elapsed();
 
 			Ok(RoomMessageEventContent::notice_markdown(format!(
@@ -149,7 +174,9 @@ pub(super) async fn process(
 				.rooms
 				.state_cache
 				.active_local_users_in_room(&room_id)
-				.collect();
+				.map(ToOwned::to_owned)
+				.collect()
+				.await;
 			let query_time = timer.elapsed();
 
 			Ok(RoomMessageEventContent::notice_markdown(format!(
@@ -160,7 +187,7 @@ pub(super) async fn process(
 			room_id,
 		} => {
 			let timer = tokio::time::Instant::now();
-			let results = services.rooms.state_cache.room_joined_count(&room_id);
+			let results = services.rooms.state_cache.room_joined_count(&room_id).await;
 			let query_time = timer.elapsed();
 
 			Ok(RoomMessageEventContent::notice_markdown(format!(
@@ -171,7 +198,11 @@ pub(super) async fn process(
 			room_id,
 		} => {
 			let timer = tokio::time::Instant::now();
-			let results = services.rooms.state_cache.room_invited_count(&room_id);
+			let results = services
+				.rooms
+				.state_cache
+				.room_invited_count(&room_id)
+				.await;
 			let query_time = timer.elapsed();
 
 			Ok(RoomMessageEventContent::notice_markdown(format!(
@@ -182,11 +213,13 @@ pub(super) async fn process(
 			room_id,
 		} => {
 			let timer = tokio::time::Instant::now();
-			let results: Result<Vec<_>> = services
+			let results: Vec<_> = services
 				.rooms
 				.state_cache
 				.room_useroncejoined(&room_id)
-				.collect();
+				.map(ToOwned::to_owned)
+				.collect()
+				.await;
 			let query_time = timer.elapsed();
 
 			Ok(RoomMessageEventContent::notice_markdown(format!(
@@ -197,11 +230,13 @@ pub(super) async fn process(
 			room_id,
 		} => {
 			let timer = tokio::time::Instant::now();
-			let results: Result<Vec<_>> = services
+			let results: Vec<_> = services
 				.rooms
 				.state_cache
 				.room_members_invited(&room_id)
-				.collect();
+				.map(ToOwned::to_owned)
+				.collect()
+				.await;
 			let query_time = timer.elapsed();
 
 			Ok(RoomMessageEventContent::notice_markdown(format!(
@@ -216,7 +251,8 @@ pub(super) async fn process(
 			let results = services
 				.rooms
 				.state_cache
-				.get_invite_count(&room_id, &user_id);
+				.get_invite_count(&room_id, &user_id)
+				.await;
 			let query_time = timer.elapsed();
 
 			Ok(RoomMessageEventContent::notice_markdown(format!(
@@ -231,7 +267,8 @@ pub(super) async fn process(
 			let results = services
 				.rooms
 				.state_cache
-				.get_left_count(&room_id, &user_id);
+				.get_left_count(&room_id, &user_id)
+				.await;
 			let query_time = timer.elapsed();
 
 			Ok(RoomMessageEventContent::notice_markdown(format!(
@@ -242,7 +279,13 @@ pub(super) async fn process(
 			user_id,
 		} => {
 			let timer = tokio::time::Instant::now();
-			let results: Result<Vec<_>> = services.rooms.state_cache.rooms_joined(&user_id).collect();
+			let results: Vec<_> = services
+				.rooms
+				.state_cache
+				.rooms_joined(&user_id)
+				.map(ToOwned::to_owned)
+				.collect()
+				.await;
 			let query_time = timer.elapsed();
 
 			Ok(RoomMessageEventContent::notice_markdown(format!(
@@ -253,7 +296,12 @@ pub(super) async fn process(
 			user_id,
 		} => {
 			let timer = tokio::time::Instant::now();
-			let results: Result<Vec<_>> = services.rooms.state_cache.rooms_invited(&user_id).collect();
+			let results: Vec<_> = services
+				.rooms
+				.state_cache
+				.rooms_invited(&user_id)
+				.collect()
+				.await;
 			let query_time = timer.elapsed();
 
 			Ok(RoomMessageEventContent::notice_markdown(format!(
@@ -264,7 +312,12 @@ pub(super) async fn process(
 			user_id,
 		} => {
 			let timer = tokio::time::Instant::now();
-			let results: Result<Vec<_>> = services.rooms.state_cache.rooms_left(&user_id).collect();
+			let results: Vec<_> = services
+				.rooms
+				.state_cache
+				.rooms_left(&user_id)
+				.collect()
+				.await;
 			let query_time = timer.elapsed();
 
 			Ok(RoomMessageEventContent::notice_markdown(format!(
@@ -276,7 +329,11 @@ pub(super) async fn process(
 			room_id,
 		} => {
 			let timer = tokio::time::Instant::now();
-			let results = services.rooms.state_cache.invite_state(&user_id, &room_id);
+			let results = services
+				.rooms
+				.state_cache
+				.invite_state(&user_id, &room_id)
+				.await;
 			let query_time = timer.elapsed();
 
 			Ok(RoomMessageEventContent::notice_markdown(format!(

src/admin/query/sending.rs

@@ -1,5 +1,6 @@
 use clap::Subcommand;
 use conduit::Result;
+use futures::StreamExt;
 use ruma::{events::room::message::RoomMessageEventContent, ServerName, UserId};
 use service::sending::Destination;
 
@@ -68,7 +69,7 @@ pub(super) async fn process(subcommand: SendingCommand, context: &Command<'_>) -
 		SendingCommand::ActiveRequests => {
 			let timer = tokio::time::Instant::now();
 			let results = services.sending.db.active_requests();
-			let active_requests: Result<Vec<(_, _, _)>> = results.collect();
+			let active_requests = results.collect::<Vec<_>>().await;
 			let query_time = timer.elapsed();
 
 			Ok(RoomMessageEventContent::notice_markdown(format!(
@@ -133,7 +134,7 @@ pub(super) async fn process(subcommand: SendingCommand, context: &Command<'_>) -
 				},
 			};
 
-			let queued_requests = results.collect::<Result<Vec<(_, _)>>>();
+			let queued_requests = results.collect::<Vec<_>>().await;
 			let query_time = timer.elapsed();
 
 			Ok(RoomMessageEventContent::notice_markdown(format!(
@@ -199,7 +200,7 @@ pub(super) async fn process(subcommand: SendingCommand, context: &Command<'_>) -
 				},
 			};
 
-			let active_requests = results.collect::<Result<Vec<(_, _)>>>();
+			let active_requests = results.collect::<Vec<_>>().await;
 			let query_time = timer.elapsed();
 
 			Ok(RoomMessageEventContent::notice_markdown(format!(
@@ -210,7 +211,7 @@ pub(super) async fn process(subcommand: SendingCommand, context: &Command<'_>) -
 			server_name,
 		} => {
 			let timer = tokio::time::Instant::now();
-			let results = services.sending.db.get_latest_educount(&server_name);
+			let results = services.sending.db.get_latest_educount(&server_name).await;
 			let query_time = timer.elapsed();
 
 			Ok(RoomMessageEventContent::notice_markdown(format!(

src/admin/query/users.rs

@@ -1,29 +1,344 @@
 use clap::Subcommand;
 use conduit::Result;
-use ruma::events::room::message::RoomMessageEventContent;
+use futures::stream::StreamExt;
+use ruma::{events::room::message::RoomMessageEventContent, OwnedDeviceId, OwnedRoomId, OwnedUserId};
 
-use crate::Command;
+use crate::{admin_command, admin_command_dispatch};
 
+#[admin_command_dispatch]
 #[derive(Debug, Subcommand)]
 /// All the getters and iterators from src/database/key_value/users.rs
 pub(crate) enum UsersCommand {
-	Iter,
+	CountUsers,
+
+	IterUsers,
+
+	PasswordHash {
+		user_id: OwnedUserId,
+	},
+
+	ListDevices {
+		user_id: OwnedUserId,
+	},
+
+	ListDevicesMetadata {
+		user_id: OwnedUserId,
+	},
+
+	GetDeviceMetadata {
+		user_id: OwnedUserId,
+		device_id: OwnedDeviceId,
+	},
+
+	GetDevicesVersion {
+		user_id: OwnedUserId,
+	},
+
+	CountOneTimeKeys {
+		user_id: OwnedUserId,
+		device_id: OwnedDeviceId,
+	},
+
+	GetDeviceKeys {
+		user_id: OwnedUserId,
+		device_id: OwnedDeviceId,
+	},
+
+	GetUserSigningKey {
+		user_id: OwnedUserId,
+	},
+
+	GetMasterKey {
+		user_id: OwnedUserId,
+	},
+
+	GetToDeviceEvents {
+		user_id: OwnedUserId,
+		device_id: OwnedDeviceId,
+	},
+
+	GetLatestBackup {
+		user_id: OwnedUserId,
+	},
+
+	GetLatestBackupVersion {
+		user_id: OwnedUserId,
+	},
+
+	GetBackupAlgorithm {
+		user_id: OwnedUserId,
+		version: String,
+	},
+
+	GetAllBackups {
+		user_id: OwnedUserId,
+		version: String,
+	},
+
+	GetRoomBackups {
+		user_id: OwnedUserId,
+		version: String,
+		room_id: OwnedRoomId,
+	},
+
+	GetBackupSession {
+		user_id: OwnedUserId,
+		version: String,
+		room_id: OwnedRoomId,
+		session_id: String,
+	},
 }
 
-/// All the getters and iterators in key_value/users.rs
+#[admin_command]
-pub(super) async fn process(subcommand: UsersCommand, context: &Command<'_>) -> Result<RoomMessageEventContent> {
+async fn get_backup_session(
-	let services = context.services;
+	&self, user_id: OwnedUserId, version: String, room_id: OwnedRoomId, session_id: String,
+) -> Result<RoomMessageEventContent> {
+	let timer = tokio::time::Instant::now();
+	let result = self
+		.services
+		.key_backups
+		.get_session(&user_id, &version, &room_id, &session_id)
+		.await;
+	let query_time = timer.elapsed();
 
-	match subcommand {
+	Ok(RoomMessageEventContent::notice_markdown(format!(
-		UsersCommand::Iter => {
+		"Query completed in {query_time:?}:\n\n```rs\n{result:#?}\n```"
-			let timer = tokio::time::Instant::now();
+	)))
-			let results = services.users.db.iter();
+}
-			let users = results.collect::<Vec<_>>();
+
-			let query_time = timer.elapsed();
+#[admin_command]
-
+async fn get_room_backups(
-			Ok(RoomMessageEventContent::notice_markdown(format!(
+	&self, user_id: OwnedUserId, version: String, room_id: OwnedRoomId,
-				"Query completed in {query_time:?}:\n\n```rs\n{users:#?}\n```"
+) -> Result<RoomMessageEventContent> {
-			)))
+	let timer = tokio::time::Instant::now();
-		},
+	let result = self
-	}
+		.services
+		.key_backups
+		.get_room(&user_id, &version, &room_id)
+		.await;
+	let query_time = timer.elapsed();
+
+	Ok(RoomMessageEventContent::notice_markdown(format!(
+		"Query completed in {query_time:?}:\n\n```rs\n{result:#?}\n```"
+	)))
+}
+
+#[admin_command]
+async fn get_all_backups(&self, user_id: OwnedUserId, version: String) -> Result<RoomMessageEventContent> {
+	let timer = tokio::time::Instant::now();
+	let result = self.services.key_backups.get_all(&user_id, &version).await;
+	let query_time = timer.elapsed();
+
+	Ok(RoomMessageEventContent::notice_markdown(format!(
+		"Query completed in {query_time:?}:\n\n```rs\n{result:#?}\n```"
+	)))
+}
+
+#[admin_command]
+async fn get_backup_algorithm(&self, user_id: OwnedUserId, version: String) -> Result<RoomMessageEventContent> {
+	let timer = tokio::time::Instant::now();
+	let result = self
+		.services
+		.key_backups
+		.get_backup(&user_id, &version)
+		.await;
+	let query_time = timer.elapsed();
+
+	Ok(RoomMessageEventContent::notice_markdown(format!(
+		"Query completed in {query_time:?}:\n\n```rs\n{result:#?}\n```"
+	)))
+}
+
+#[admin_command]
+async fn get_latest_backup_version(&self, user_id: OwnedUserId) -> Result<RoomMessageEventContent> {
+	let timer = tokio::time::Instant::now();
+	let result = self
+		.services
+		.key_backups
+		.get_latest_backup_version(&user_id)
+		.await;
+	let query_time = timer.elapsed();
+
+	Ok(RoomMessageEventContent::notice_markdown(format!(
+		"Query completed in {query_time:?}:\n\n```rs\n{result:#?}\n```"
+	)))
+}
+
+#[admin_command]
+async fn get_latest_backup(&self, user_id: OwnedUserId) -> Result<RoomMessageEventContent> {
+	let timer = tokio::time::Instant::now();
+	let result = self.services.key_backups.get_latest_backup(&user_id).await;
+	let query_time = timer.elapsed();
+
+	Ok(RoomMessageEventContent::notice_markdown(format!(
+		"Query completed in {query_time:?}:\n\n```rs\n{result:#?}\n```"
+	)))
+}
+
+#[admin_command]
+async fn iter_users(&self) -> Result<RoomMessageEventContent> {
+	let timer = tokio::time::Instant::now();
+	let result: Vec<OwnedUserId> = self.services.users.stream().map(Into::into).collect().await;
+
+	let query_time = timer.elapsed();
+
+	Ok(RoomMessageEventContent::notice_markdown(format!(
+		"Query completed in {query_time:?}:\n\n```rs\n{result:#?}\n```"
+	)))
+}
+
+#[admin_command]
+async fn count_users(&self) -> Result<RoomMessageEventContent> {
+	let timer = tokio::time::Instant::now();
+	let result = self.services.users.count().await;
+	let query_time = timer.elapsed();
+
+	Ok(RoomMessageEventContent::notice_markdown(format!(
+		"Query completed in {query_time:?}:\n\n```rs\n{result:#?}\n```"
+	)))
+}
+
+#[admin_command]
+async fn password_hash(&self, user_id: OwnedUserId) -> Result<RoomMessageEventContent> {
+	let timer = tokio::time::Instant::now();
+	let result = self.services.users.password_hash(&user_id).await;
+	let query_time = timer.elapsed();
+
+	Ok(RoomMessageEventContent::notice_markdown(format!(
+		"Query completed in {query_time:?}:\n\n```rs\n{result:#?}\n```"
+	)))
+}
+
+#[admin_command]
+async fn list_devices(&self, user_id: OwnedUserId) -> Result<RoomMessageEventContent> {
+	let timer = tokio::time::Instant::now();
+	let devices = self
+		.services
+		.users
+		.all_device_ids(&user_id)
+		.map(ToOwned::to_owned)
+		.collect::<Vec<_>>()
+		.await;
+
+	let query_time = timer.elapsed();
+
+	Ok(RoomMessageEventContent::notice_markdown(format!(
+		"Query completed in {query_time:?}:\n\n```rs\n{devices:#?}\n```"
+	)))
+}
+
+#[admin_command]
+async fn list_devices_metadata(&self, user_id: OwnedUserId) -> Result<RoomMessageEventContent> {
+	let timer = tokio::time::Instant::now();
+	let devices = self
+		.services
+		.users
+		.all_devices_metadata(&user_id)
+		.collect::<Vec<_>>()
+		.await;
+	let query_time = timer.elapsed();
+
+	Ok(RoomMessageEventContent::notice_markdown(format!(
+		"Query completed in {query_time:?}:\n\n```rs\n{devices:#?}\n```"
+	)))
+}
+
+#[admin_command]
+async fn get_device_metadata(&self, user_id: OwnedUserId, device_id: OwnedDeviceId) -> Result<RoomMessageEventContent> {
+	let timer = tokio::time::Instant::now();
+	let device = self
+		.services
+		.users
+		.get_device_metadata(&user_id, &device_id)
+		.await;
+	let query_time = timer.elapsed();
+
+	Ok(RoomMessageEventContent::notice_markdown(format!(
+		"Query completed in {query_time:?}:\n\n```rs\n{device:#?}\n```"
+	)))
+}
+
+#[admin_command]
+async fn get_devices_version(&self, user_id: OwnedUserId) -> Result<RoomMessageEventContent> {
+	let timer = tokio::time::Instant::now();
+	let device = self.services.users.get_devicelist_version(&user_id).await;
+	let query_time = timer.elapsed();
+
+	Ok(RoomMessageEventContent::notice_markdown(format!(
+		"Query completed in {query_time:?}:\n\n```rs\n{device:#?}\n```"
+	)))
+}
+
+#[admin_command]
+async fn count_one_time_keys(&self, user_id: OwnedUserId, device_id: OwnedDeviceId) -> Result<RoomMessageEventContent> {
+	let timer = tokio::time::Instant::now();
+	let result = self
+		.services
+		.users
+		.count_one_time_keys(&user_id, &device_id)
+		.await;
+	let query_time = timer.elapsed();
+
+	Ok(RoomMessageEventContent::notice_markdown(format!(
+		"Query completed in {query_time:?}:\n\n```rs\n{result:#?}\n```"
+	)))
+}
+
+#[admin_command]
+async fn get_device_keys(&self, user_id: OwnedUserId, device_id: OwnedDeviceId) -> Result<RoomMessageEventContent> {
+	let timer = tokio::time::Instant::now();
+	let result = self
+		.services
+		.users
+		.get_device_keys(&user_id, &device_id)
+		.await;
+	let query_time = timer.elapsed();
+
+	Ok(RoomMessageEventContent::notice_markdown(format!(
+		"Query completed in {query_time:?}:\n\n```rs\n{result:#?}\n```"
+	)))
+}
+
+#[admin_command]
+async fn get_user_signing_key(&self, user_id: OwnedUserId) -> Result<RoomMessageEventContent> {
+	let timer = tokio::time::Instant::now();
+	let result = self.services.users.get_user_signing_key(&user_id).await;
+	let query_time = timer.elapsed();
+
+	Ok(RoomMessageEventContent::notice_markdown(format!(
+		"Query completed in {query_time:?}:\n\n```rs\n{result:#?}\n```"
+	)))
+}
+
+#[admin_command]
+async fn get_master_key(&self, user_id: OwnedUserId) -> Result<RoomMessageEventContent> {
+	let timer = tokio::time::Instant::now();
+	let result = self
+		.services
+		.users
+		.get_master_key(None, &user_id, &|_| true)
+		.await;
+	let query_time = timer.elapsed();
+
+	Ok(RoomMessageEventContent::notice_markdown(format!(
+		"Query completed in {query_time:?}:\n\n```rs\n{result:#?}\n```"
+	)))
+}
+
+#[admin_command]
+async fn get_to_device_events(
+	&self, user_id: OwnedUserId, device_id: OwnedDeviceId,
+) -> Result<RoomMessageEventContent> {
+	let timer = tokio::time::Instant::now();
+	let result = self
+		.services
+		.users
+		.get_to_device_events(&user_id, &device_id)
+		.collect::<Vec<_>>()
+		.await;
+	let query_time = timer.elapsed();
+
+	Ok(RoomMessageEventContent::notice_markdown(format!(
+		"Query completed in {query_time:?}:\n\n```rs\n{result:#?}\n```"
+	)))
 }

src/admin/room/alias.rs

@@ -2,7 +2,8 @@ use std::fmt::Write;
 
 use clap::Subcommand;
 use conduit::Result;
-use ruma::{events::room::message::RoomMessageEventContent, RoomAliasId, RoomId};
+use futures::StreamExt;
+use ruma::{events::room::message::RoomMessageEventContent, OwnedRoomAliasId, OwnedRoomId, RoomAliasId, RoomId};
 
 use crate::{escape_html, Command};
 
@@ -66,8 +67,8 @@ pub(super) async fn process(command: RoomAliasCommand, context: &Command<'_>) ->
 					force,
 					room_id,
 					..
-				} => match (force, services.rooms.alias.resolve_local_alias(&room_alias)) {
+				} => match (force, services.rooms.alias.resolve_local_alias(&room_alias).await) {
-					(true, Ok(Some(id))) => match services
+					(true, Ok(id)) => match services
 						.rooms
 						.alias
 						.set_alias(&room_alias, &room_id, server_user)
@@ -77,10 +78,10 @@ pub(super) async fn process(command: RoomAliasCommand, context: &Command<'_>) ->
 						))),
 						Err(err) => Ok(RoomMessageEventContent::text_plain(format!("Failed to remove alias: {err}"))),
 					},
-					(false, Ok(Some(id))) => Ok(RoomMessageEventContent::text_plain(format!(
+					(false, Ok(id)) => Ok(RoomMessageEventContent::text_plain(format!(
 						"Refusing to overwrite in use alias for {id}, use -f or --force to overwrite"
 					))),
-					(_, Ok(None)) => match services
+					(_, Err(_)) => match services
 						.rooms
 						.alias
 						.set_alias(&room_alias, &room_id, server_user)
@@ -88,12 +89,11 @@ pub(super) async fn process(command: RoomAliasCommand, context: &Command<'_>) ->
 						Ok(()) => Ok(RoomMessageEventContent::text_plain("Successfully set alias")),
 						Err(err) => Ok(RoomMessageEventContent::text_plain(format!("Failed to remove alias: {err}"))),
 					},
-					(_, Err(err)) => Ok(RoomMessageEventContent::text_plain(format!("Unable to lookup alias: {err}"))),
 				},
 				RoomAliasCommand::Remove {
 					..
-				} => match services.rooms.alias.resolve_local_alias(&room_alias) {
+				} => match services.rooms.alias.resolve_local_alias(&room_alias).await {
-					Ok(Some(id)) => match services
+					Ok(id) => match services
 						.rooms
 						.alias
 						.remove_alias(&room_alias, server_user)
@@ -102,15 +102,13 @@ pub(super) async fn process(command: RoomAliasCommand, context: &Command<'_>) ->
 						Ok(()) => Ok(RoomMessageEventContent::text_plain(format!("Removed alias from {id}"))),
 						Err(err) => Ok(RoomMessageEventContent::text_plain(format!("Failed to remove alias: {err}"))),
 					},
-					Ok(None) => Ok(RoomMessageEventContent::text_plain("Alias isn't in use.")),
+					Err(_) => Ok(RoomMessageEventContent::text_plain("Alias isn't in use.")),
-					Err(err) => Ok(RoomMessageEventContent::text_plain(format!("Unable to lookup alias: {err}"))),
 				},
 				RoomAliasCommand::Which {
 					..
-				} => match services.rooms.alias.resolve_local_alias(&room_alias) {
+				} => match services.rooms.alias.resolve_local_alias(&room_alias).await {
-					Ok(Some(id)) => Ok(RoomMessageEventContent::text_plain(format!("Alias resolves to {id}"))),
+					Ok(id) => Ok(RoomMessageEventContent::text_plain(format!("Alias resolves to {id}"))),
-					Ok(None) => Ok(RoomMessageEventContent::text_plain("Alias isn't in use.")),
+					Err(_) => Ok(RoomMessageEventContent::text_plain("Alias isn't in use.")),
-					Err(err) => Ok(RoomMessageEventContent::text_plain(format!("Unable to lookup alias: {err}"))),
 				},
 				RoomAliasCommand::List {
 					..
@@ -121,67 +119,63 @@ pub(super) async fn process(command: RoomAliasCommand, context: &Command<'_>) ->
 			room_id,
 		} => {
 			if let Some(room_id) = room_id {
-				let aliases = services
+				let aliases: Vec<OwnedRoomAliasId> = services
 					.rooms
 					.alias
 					.local_aliases_for_room(&room_id)
-					.collect::<Result<Vec<_>, _>>();
+					.map(Into::into)
-				match aliases {
+					.collect()
-					Ok(aliases) => {
+					.await;
-						let plain_list = aliases.iter().fold(String::new(), |mut output, alias| {
-							writeln!(output, "- {alias}").expect("should be able to write to string buffer");
-							output
-						});
 
-						let html_list = aliases.iter().fold(String::new(), |mut output, alias| {
+				let plain_list = aliases.iter().fold(String::new(), |mut output, alias| {
-							writeln!(output, "<li>{}</li>", escape_html(alias.as_ref()))
+					writeln!(output, "- {alias}").expect("should be able to write to string buffer");
-								.expect("should be able to write to string buffer");
+					output
-							output
+				});
-						});
 
-						let plain = format!("Aliases for {room_id}:\n{plain_list}");
+				let html_list = aliases.iter().fold(String::new(), |mut output, alias| {
-						let html = format!("Aliases for {room_id}:\n<ul>{html_list}</ul>");
+					writeln!(output, "<li>{}</li>", escape_html(alias.as_ref()))
-						Ok(RoomMessageEventContent::text_html(plain, html))
+						.expect("should be able to write to string buffer");
-					},
+					output
-					Err(err) => Ok(RoomMessageEventContent::text_plain(format!("Unable to list aliases: {err}"))),
+				});
-				}
+
+				let plain = format!("Aliases for {room_id}:\n{plain_list}");
+				let html = format!("Aliases for {room_id}:\n<ul>{html_list}</ul>");
+				Ok(RoomMessageEventContent::text_html(plain, html))
 			} else {
 				let aliases = services
 					.rooms
 					.alias
 					.all_local_aliases()
-					.collect::<Result<Vec<_>, _>>();
+					.map(|(room_id, localpart)| (room_id.into(), localpart.into()))
-				match aliases {
+					.collect::<Vec<(OwnedRoomId, String)>>()
-					Ok(aliases) => {
+					.await;
-						let server_name = services.globals.server_name();
-						let plain_list = aliases
-							.iter()
-							.fold(String::new(), |mut output, (alias, id)| {
-								writeln!(output, "- `{alias}` -> #{id}:{server_name}")
-									.expect("should be able to write to string buffer");
-								output
-							});
 
-						let html_list = aliases
+				let server_name = services.globals.server_name();
-							.iter()
+				let plain_list = aliases
-							.fold(String::new(), |mut output, (alias, id)| {
+					.iter()
-								writeln!(
+					.fold(String::new(), |mut output, (alias, id)| {
-									output,
+						writeln!(output, "- `{alias}` -> #{id}:{server_name}")
-									"<li><code>{}</code> -> #{}:{}</li>",
+							.expect("should be able to write to string buffer");
-									escape_html(alias.as_ref()),
+						output
-									escape_html(id.as_ref()),
+					});
-									server_name
-								)
-								.expect("should be able to write to string buffer");
-								output
-							});
 
-						let plain = format!("Aliases:\n{plain_list}");
+				let html_list = aliases
-						let html = format!("Aliases:\n<ul>{html_list}</ul>");
+					.iter()
-						Ok(RoomMessageEventContent::text_html(plain, html))
+					.fold(String::new(), |mut output, (alias, id)| {
-					},
+						writeln!(
-					Err(e) => Ok(RoomMessageEventContent::text_plain(format!("Unable to list room aliases: {e}"))),
+							output,
-				}
+							"<li><code>{}</code> -> #{}:{}</li>",
+							escape_html(alias.as_ref()),
+							escape_html(id),
+							server_name
+						)
+						.expect("should be able to write to string buffer");
+						output
+					});
+
+				let plain = format!("Aliases:\n{plain_list}");
+				let html = format!("Aliases:\n<ul>{html_list}</ul>");
+				Ok(RoomMessageEventContent::text_html(plain, html))
 			}
 		},
 	}

src/admin/room/commands.rs

@@ -1,13 +1,12 @@
-use std::fmt::Write;
-
 use conduit::Result;
-use ruma::events::room::message::RoomMessageEventContent;
+use futures::StreamExt;
+use ruma::{events::room::message::RoomMessageEventContent, OwnedRoomId};
 
-use crate::{admin_command, escape_html, get_room_info, PAGE_SIZE};
+use crate::{admin_command, get_room_info, PAGE_SIZE};
 
 #[admin_command]
 pub(super) async fn list_rooms(
-	&self, page: Option<usize>, exclude_disabled: bool, exclude_banned: bool,
+	&self, page: Option<usize>, exclude_disabled: bool, exclude_banned: bool, no_details: bool,
 ) -> Result<RoomMessageEventContent> {
 	// TODO: i know there's a way to do this with clap, but i can't seem to find it
 	let page = page.unwrap_or(1);
@@ -16,37 +15,16 @@ pub(super) async fn list_rooms(
 		.rooms
 		.metadata
 		.iter_ids()
-		.filter_map(|room_id| {
+		.filter_map(|room_id| async move {
-			room_id
+			(!exclude_disabled || !self.services.rooms.metadata.is_disabled(room_id).await).then_some(room_id)
-				.ok()
-				.filter(|room_id| {
-					if exclude_disabled
-						&& self
-							.services
-							.rooms
-							.metadata
-							.is_disabled(room_id)
-							.unwrap_or(false)
-					{
-						return false;
-					}
-
-					if exclude_banned
-						&& self
-							.services
-							.rooms
-							.metadata
-							.is_banned(room_id)
-							.unwrap_or(false)
-					{
-						return false;
-					}
-
-					true
-				})
-				.map(|room_id| get_room_info(self.services, &room_id))
 		})
-		.collect::<Vec<_>>();
+		.filter_map(|room_id| async move {
+			(!exclude_banned || !self.services.rooms.metadata.is_banned(room_id).await).then_some(room_id)
+		})
+		.then(|room_id| get_room_info(self.services, room_id))
+		.collect::<Vec<_>>()
+		.await;
+
 	rooms.sort_by_key(|r| r.1);
 	rooms.reverse();
 
@@ -61,29 +39,25 @@ pub(super) async fn list_rooms(
 	};
 
 	let output_plain = format!(
-		"Rooms:\n{}",
+		"Rooms ({}):\n```\n{}\n```",
+		rooms.len(),
 		rooms
 			.iter()
-			.map(|(id, members, name)| format!("{id}\tMembers: {members}\tName: {name}"))
+			.map(|(id, members, name)| if no_details {
+				format!("{id}")
+			} else {
+				format!("{id}\tMembers: {members}\tName: {name}")
+			})
 			.collect::<Vec<_>>()
 			.join("\n")
 	);
-	let output_html = format!(
+
-		"<table><caption>Room list - page \
+	Ok(RoomMessageEventContent::notice_markdown(output_plain))
-		 {page}</caption>\n<tr><th>id</th>\t<th>members</th>\t<th>name</th></tr>\n{}</table>",
+}
-		rooms
+
-			.iter()
+#[admin_command]
-			.fold(String::new(), |mut output, (id, members, name)| {
+pub(super) async fn exists(&self, room_id: OwnedRoomId) -> Result<RoomMessageEventContent> {
-				writeln!(
+	let result = self.services.rooms.metadata.exists(&room_id).await;
-					output,
+
-					"<tr><td>{}</td>\t<td>{}</td>\t<td>{}</td></tr>",
+	Ok(RoomMessageEventContent::notice_markdown(format!("{result}")))
-					escape_html(id.as_ref()),
-					members,
-					escape_html(name)
-				)
-				.expect("should be able to write to string buffer");
-				output
-			})
-	);
-	Ok(RoomMessageEventContent::text_html(output_plain, output_html))
 }

src/admin/room/directory.rs

@@ -1,10 +1,9 @@
-use std::fmt::Write;
-
 use clap::Subcommand;
 use conduit::Result;
-use ruma::{events::room::message::RoomMessageEventContent, OwnedRoomId, RoomId};
+use futures::StreamExt;
+use ruma::{events::room::message::RoomMessageEventContent, RoomId};
 
-use crate::{escape_html, get_room_info, Command, PAGE_SIZE};
+use crate::{get_room_info, Command, PAGE_SIZE};
 
 #[derive(Debug, Subcommand)]
 pub(crate) enum RoomDirectoryCommand {
@@ -31,67 +30,51 @@ pub(super) async fn process(command: RoomDirectoryCommand, context: &Command<'_>
 	match command {
 		RoomDirectoryCommand::Publish {
 			room_id,
-		} => match services.rooms.directory.set_public(&room_id) {
+		} => {
-			Ok(()) => Ok(RoomMessageEventContent::text_plain("Room published")),
+			services.rooms.directory.set_public(&room_id);
-			Err(err) => Ok(RoomMessageEventContent::text_plain(format!("Unable to update room: {err}"))),
+			Ok(RoomMessageEventContent::notice_plain("Room published"))
 		},
 		RoomDirectoryCommand::Unpublish {
 			room_id,
-		} => match services.rooms.directory.set_not_public(&room_id) {
+		} => {
-			Ok(()) => Ok(RoomMessageEventContent::text_plain("Room unpublished")),
+			services.rooms.directory.set_not_public(&room_id);
-			Err(err) => Ok(RoomMessageEventContent::text_plain(format!("Unable to update room: {err}"))),
+			Ok(RoomMessageEventContent::notice_plain("Room unpublished"))
 		},
 		RoomDirectoryCommand::List {
 			page,
 		} => {
 			// TODO: i know there's a way to do this with clap, but i can't seem to find it
 			let page = page.unwrap_or(1);
-			let mut rooms = services
+			let mut rooms: Vec<_> = services
 				.rooms
 				.directory
 				.public_rooms()
-				.filter_map(Result::ok)
+				.then(|room_id| get_room_info(services, room_id))
-				.map(|id: OwnedRoomId| get_room_info(services, &id))
+				.collect()
-				.collect::<Vec<_>>();
+				.await;
+
 			rooms.sort_by_key(|r| r.1);
 			rooms.reverse();
 
-			let rooms = rooms
+			let rooms: Vec<_> = rooms
 				.into_iter()
 				.skip(page.saturating_sub(1).saturating_mul(PAGE_SIZE))
 				.take(PAGE_SIZE)
-				.collect::<Vec<_>>();
+				.collect();
 
 			if rooms.is_empty() {
 				return Ok(RoomMessageEventContent::text_plain("No more rooms."));
 			};
 
-			let output_plain = format!(
+			let output = format!(
-				"Rooms:\n{}",
+				"Rooms (page {page}):\n```\n{}\n```",
 				rooms
 					.iter()
-					.map(|(id, members, name)| format!("{id}\tMembers: {members}\tName: {name}"))
+					.map(|(id, members, name)| format!("{id} | Members: {members} | Name: {name}"))
 					.collect::<Vec<_>>()
 					.join("\n")
 			);
-			let output_html = format!(
+			Ok(RoomMessageEventContent::text_markdown(output))
-				"<table><caption>Room directory - page \
-				 {page}</caption>\n<tr><th>id</th>\t<th>members</th>\t<th>name</th></tr>\n{}</table>",
-				rooms
-					.iter()
-					.fold(String::new(), |mut output, (id, members, name)| {
-						writeln!(
-							output,
-							"<tr><td>{}</td>\t<td>{}</td>\t<td>{}</td></tr>",
-							escape_html(id.as_ref()),
-							members,
-							escape_html(name.as_ref())
-						)
-						.expect("should be able to write to string buffer");
-						output
-					})
-			);
-			Ok(RoomMessageEventContent::text_html(output_plain, output_html))
 		},
 	}
 }

src/admin/room/info.rs

@@ -1,5 +1,6 @@
 use clap::Subcommand;
-use conduit::Result;
+use conduit::{utils::ReadyExt, Result};
+use futures::StreamExt;
 use ruma::{events::room::message::RoomMessageEventContent, RoomId};
 
 use crate::{admin_command, admin_command_dispatch};
@@ -10,6 +11,10 @@ pub(crate) enum RoomInfoCommand {
 	/// - List joined members in a room
 	ListJoinedMembers {
 		room_id: Box<RoomId>,
+
+		/// Lists only our local users in the specified room
+		#[arg(long)]
+		local_only: bool,
 	},
 
 	/// - Displays room topic
@@ -22,44 +27,46 @@ pub(crate) enum RoomInfoCommand {
 }
 
 #[admin_command]
-async fn list_joined_members(&self, room_id: Box<RoomId>) -> Result<RoomMessageEventContent> {
+async fn list_joined_members(&self, room_id: Box<RoomId>, local_only: bool) -> Result<RoomMessageEventContent> {
 	let room_name = self
 		.services
 		.rooms
 		.state_accessor
 		.get_name(&room_id)
-		.ok()
+		.await
-		.flatten()
+		.unwrap_or_else(|_| room_id.to_string());
-		.unwrap_or_else(|| room_id.to_string());
 
-	let members = self
+	let member_info: Vec<_> = self
 		.services
 		.rooms
 		.state_cache
 		.room_members(&room_id)
-		.filter_map(Result::ok);
+		.ready_filter(|user_id| {
-
+			local_only
-	let member_info = members
+				.then(|| self.services.globals.user_is_local(user_id))
-		.into_iter()
+				.unwrap_or(true)
-		.map(|user_id| {
+		})
-			(
+		.map(ToOwned::to_owned)
-				user_id.clone(),
+		.filter_map(|user_id| async move {
+			Some((
 				self.services
 					.users
 					.displayname(&user_id)
-					.unwrap_or(None)
+					.await
-					.unwrap_or_else(|| user_id.to_string()),
+					.unwrap_or_else(|_| user_id.to_string()),
-			)
+				user_id,
+			))
 		})
-		.collect::<Vec<_>>();
+		.collect()
+		.await;
 
 	let output_plain = format!(
 		"{} Members in Room \"{}\":\n```\n{}\n```",
 		member_info.len(),
 		room_name,
 		member_info
-			.iter()
+			.into_iter()
-			.map(|(mxid, displayname)| format!("{mxid} | {displayname}"))
+			.map(|(displayname, mxid)| format!("{mxid} | {displayname}"))
 			.collect::<Vec<_>>()
 			.join("\n")
 	);
@@ -69,11 +76,12 @@ async fn list_joined_members(&self, room_id: Box<RoomId>) -> Result<RoomMessageE
 
 #[admin_command]
 async fn view_room_topic(&self, room_id: Box<RoomId>) -> Result<RoomMessageEventContent> {
-	let Some(room_topic) = self
+	let Ok(room_topic) = self
 		.services
 		.rooms
 		.state_accessor
-		.get_room_topic(&room_id)?
+		.get_room_topic(&room_id)
+		.await
 	else {
 		return Ok(RoomMessageEventContent::text_plain("Room does not have a room topic set."));
 	};

src/admin/room/mod.rs

@@ -6,6 +6,7 @@ mod moderation;
 
 use clap::Subcommand;
 use conduit::Result;
+use ruma::OwnedRoomId;
 
 use self::{
 	alias::RoomAliasCommand, directory::RoomDirectoryCommand, info::RoomInfoCommand, moderation::RoomModerationCommand,
@@ -27,6 +28,11 @@ pub(super) enum RoomCommand {
 		/// Excludes rooms that we have banned
 		#[arg(long)]
 		exclude_banned: bool,
+
+		#[arg(long)]
+		/// Whether to only output room IDs without supplementary room
+		/// information
+		no_details: bool,
 	},
 
 	#[command(subcommand)]
@@ -44,4 +50,9 @@ pub(super) enum RoomCommand {
 	#[command(subcommand)]
 	/// - Manage the room directory
 	Directory(RoomDirectoryCommand),
+
+	/// - Check if we know about a room
+	Exists {
+		room_id: OwnedRoomId,
+	},
 }

src/admin/room/moderation.rs

@@ -1,6 +1,11 @@
 use api::client::leave_room;
 use clap::Subcommand;
-use conduit::{debug, error, info, warn, Result};
+use conduit::{
+	debug, error, info,
+	utils::{IterStream, ReadyExt},
+	warn, Result,
+};
+use futures::StreamExt;
 use ruma::{events::room::message::RoomMessageEventContent, OwnedRoomId, RoomAliasId, RoomId, RoomOrAliasId};
 
 use crate::{admin_command, admin_command_dispatch, get_room_info};
@@ -76,7 +81,7 @@ async fn ban_room(
 
 	let admin_room_alias = &self.services.globals.admin_alias;
 
-	if let Some(admin_room_id) = self.services.admin.get_admin_room()? {
+	if let Ok(admin_room_id) = self.services.admin.get_admin_room().await {
 		if room.to_string().eq(&admin_room_id) || room.to_string().eq(admin_room_alias) {
 			return Ok(RoomMessageEventContent::text_plain("Not allowed to ban the admin room."));
 		}
@@ -95,7 +100,7 @@ async fn ban_room(
 
 		debug!("Room specified is a room ID, banning room ID");
 
-		self.services.rooms.metadata.ban_room(&room_id, true)?;
+		self.services.rooms.metadata.ban_room(&room_id, true);
 
 		room_id
 	} else if room.is_room_alias_id() {
@@ -114,7 +119,13 @@ async fn ban_room(
 			 get_alias_helper to fetch room ID remotely"
 		);
 
-		let room_id = if let Some(room_id) = self.services.rooms.alias.resolve_local_alias(&room_alias)? {
+		let room_id = if let Ok(room_id) = self
+			.services
+			.rooms
+			.alias
+			.resolve_local_alias(&room_alias)
+			.await
+		{
 			room_id
 		} else {
 			debug!("We don't have this room alias to a room ID locally, attempting to fetch room ID over federation");
@@ -138,7 +149,7 @@ async fn ban_room(
 			}
 		};
 
-		self.services.rooms.metadata.ban_room(&room_id, true)?;
+		self.services.rooms.metadata.ban_room(&room_id, true);
 
 		room_id
 	} else {
@@ -150,56 +161,40 @@ async fn ban_room(
 
 	debug!("Making all users leave the room {}", &room);
 	if force {
-		for local_user in self
+		let mut users = self
 			.services
 			.rooms
 			.state_cache
 			.room_members(&room_id)
-			.filter_map(|user| {
+			.ready_filter(|user| self.services.globals.user_is_local(user))
-				user.ok().filter(|local_user| {
+			.boxed();
-					self.services.globals.user_is_local(local_user)
+
-								// additional wrapped check here is to avoid adding remote users
+		while let Some(local_user) = users.next().await {
-								// who are in the admin room to the list of local users (would
-								// fail auth check)
-								&& (self.services.globals.user_is_local(local_user)
-									// since this is a force operation, assume user is an admin
-									// if somehow this fails
-									&& self.services
-										.users
-										.is_admin(local_user)
-										.unwrap_or(true))
-				})
-			}) {
 			debug!(
-				"Attempting leave for user {} in room {} (forced, ignoring all errors, evicting admins too)",
+				"Attempting leave for user {local_user} in room {room_id} (forced, ignoring all errors, evicting \
-				&local_user, &room_id
+				 admins too)",
 			);
 
-			if let Err(e) = leave_room(self.services, &local_user, &room_id, None).await {
+			if let Err(e) = leave_room(self.services, local_user, &room_id, None).await {
 				warn!(%e, "Failed to leave room");
 			}
 		}
 	} else {
-		for local_user in self
+		let mut users = self
 			.services
 			.rooms
 			.state_cache
 			.room_members(&room_id)
-			.filter_map(|user| {
+			.ready_filter(|user| self.services.globals.user_is_local(user))
-				user.ok().filter(|local_user| {
+			.boxed();
-					local_user.server_name() == self.services.globals.server_name()
+
-								// additional wrapped check here is to avoid adding remote users
+		while let Some(local_user) = users.next().await {
-								// who are in the admin room to the list of local users (would fail auth check)
+			if self.services.users.is_admin(local_user).await {
-								&& (local_user.server_name()
+				continue;
-									== self.services.globals.server_name()
+			}
-									&& !self.services
+
-										.users
-										.is_admin(local_user)
-										.unwrap_or(false))
-				})
-			}) {
 			debug!("Attempting leave for user {} in room {}", &local_user, &room_id);
-			if let Err(e) = leave_room(self.services, &local_user, &room_id, None).await {
+			if let Err(e) = leave_room(self.services, local_user, &room_id, None).await {
 				error!(
 					"Error attempting to make local user {} leave room {} during room banning: {}",
 					&local_user, &room_id, e
@@ -214,12 +209,14 @@ async fn ban_room(
 	}
 
 	// remove any local aliases, ignore errors
-	for ref local_alias in self
+	for local_alias in &self
 		.services
 		.rooms
 		.alias
 		.local_aliases_for_room(&room_id)
-		.filter_map(Result::ok)
+		.map(ToOwned::to_owned)
+		.collect::<Vec<_>>()
+		.await
 	{
 		_ = self
 			.services
@@ -230,10 +227,10 @@ async fn ban_room(
 	}
 
 	// unpublish from room directory, ignore errors
-	_ = self.services.rooms.directory.set_not_public(&room_id);
+	self.services.rooms.directory.set_not_public(&room_id);
 
 	if disable_federation {
-		self.services.rooms.metadata.disable_room(&room_id, true)?;
+		self.services.rooms.metadata.disable_room(&room_id, true);
 		return Ok(RoomMessageEventContent::text_plain(
 			"Room banned, removed all our local users, and disabled incoming federation with room.",
 		));
@@ -268,7 +265,7 @@ async fn ban_list_of_rooms(&self, force: bool, disable_federation: bool) -> Resu
 	for &room in &rooms_s {
 		match <&RoomOrAliasId>::try_from(room) {
 			Ok(room_alias_or_id) => {
-				if let Some(admin_room_id) = self.services.admin.get_admin_room()? {
+				if let Ok(admin_room_id) = self.services.admin.get_admin_room().await {
 					if room.to_owned().eq(&admin_room_id) || room.to_owned().eq(admin_room_alias) {
 						info!("User specified admin room in bulk ban list, ignoring");
 						continue;
@@ -300,43 +297,48 @@ async fn ban_list_of_rooms(&self, force: bool, disable_federation: bool) -> Resu
 				if room_alias_or_id.is_room_alias_id() {
 					match RoomAliasId::parse(room_alias_or_id) {
 						Ok(room_alias) => {
-							let room_id =
+							let room_id = if let Ok(room_id) = self
-								if let Some(room_id) = self.services.rooms.alias.resolve_local_alias(&room_alias)? {
+								.services
-									room_id
+								.rooms
-								} else {
+								.alias
-									debug!(
+								.resolve_local_alias(&room_alias)
-										"We don't have this room alias to a room ID locally, attempting to fetch room \
+								.await
-										 ID over federation"
+							{
-									);
+								room_id
+							} else {
+								debug!(
+									"We don't have this room alias to a room ID locally, attempting to fetch room ID \
+									 over federation"
+								);
 
-									match self
+								match self
-										.services
+									.services
-										.rooms
+									.rooms
-										.alias
+									.alias
-										.resolve_alias(&room_alias, None)
+									.resolve_alias(&room_alias, None)
-										.await
+									.await
-									{
+								{
-										Ok((room_id, servers)) => {
+									Ok((room_id, servers)) => {
-											debug!(
+										debug!(
-												?room_id,
+											?room_id,
-												?servers,
+											?servers,
-												"Got federation response fetching room ID for {room}",
+											"Got federation response fetching room ID for {room}",
-											);
+										);
-											room_id
+										room_id
-										},
+									},
-										Err(e) => {
+									Err(e) => {
-											// don't fail if force blocking
+										// don't fail if force blocking
-											if force {
+										if force {
-												warn!("Failed to resolve room alias {room} to a room ID: {e}");
+											warn!("Failed to resolve room alias {room} to a room ID: {e}");
-												continue;
+											continue;
-											}
+										}
 
-											return Ok(RoomMessageEventContent::text_plain(format!(
+										return Ok(RoomMessageEventContent::text_plain(format!(
-												"Failed to resolve room alias {room} to a room ID: {e}"
+											"Failed to resolve room alias {room} to a room ID: {e}"
-											)));
+										)));
-										},
+									},
-									}
+								}
-								};
+							};
 
 							room_ids.push(room_id);
 						},
@@ -374,74 +376,52 @@ async fn ban_list_of_rooms(&self, force: bool, disable_federation: bool) -> Resu
 	}
 
 	for room_id in room_ids {
-		if self
+		self.services.rooms.metadata.ban_room(&room_id, true);
-			.services
+
-			.rooms
+		debug!("Banned {room_id} successfully");
-			.metadata
+		room_ban_count = room_ban_count.saturating_add(1);
-			.ban_room(&room_id, true)
-			.is_ok()
-		{
-			debug!("Banned {room_id} successfully");
-			room_ban_count = room_ban_count.saturating_add(1);
-		}
 
 		debug!("Making all users leave the room {}", &room_id);
 		if force {
-			for local_user in self
+			let mut users = self
 				.services
 				.rooms
 				.state_cache
 				.room_members(&room_id)
-				.filter_map(|user| {
+				.ready_filter(|user| self.services.globals.user_is_local(user))
-					user.ok().filter(|local_user| {
+				.boxed();
-						local_user.server_name() == self.services.globals.server_name()
+
-										// additional wrapped check here is to avoid adding remote
+			while let Some(local_user) = users.next().await {
-										// users who are in the admin room to the list of local
-										// users (would fail auth check)
-										&& (local_user.server_name()
-											== self.services.globals.server_name()
-											// since this is a force operation, assume user is an
-											// admin if somehow this fails
-											&& self.services
-												.users
-												.is_admin(local_user)
-												.unwrap_or(true))
-					})
-				}) {
 				debug!(
-					"Attempting leave for user {} in room {} (forced, ignoring all errors, evicting admins too)",
+					"Attempting leave for user {local_user} in room {room_id} (forced, ignoring all errors, evicting \
-					&local_user, room_id
+					 admins too)",
 				);
-				if let Err(e) = leave_room(self.services, &local_user, &room_id, None).await {
+
+				if let Err(e) = leave_room(self.services, local_user, &room_id, None).await {
 					warn!(%e, "Failed to leave room");
 				}
 			}
 		} else {
-			for local_user in self
+			let mut users = self
 				.services
 				.rooms
 				.state_cache
 				.room_members(&room_id)
-				.filter_map(|user| {
+				.ready_filter(|user| self.services.globals.user_is_local(user))
-					user.ok().filter(|local_user| {
+				.boxed();
-						local_user.server_name() == self.services.globals.server_name()
+
-										// additional wrapped check here is to avoid adding remote
+			while let Some(local_user) = users.next().await {
-										// users who are in the admin room to the list of local
+				if self.services.users.is_admin(local_user).await {
-										// users (would fail auth check)
+					continue;
-										&& (local_user.server_name()
+				}
-											== self.services.globals.server_name()
+
-											&& !self.services
+				debug!("Attempting leave for user {local_user} in room {room_id}");
-												.users
+				if let Err(e) = leave_room(self.services, local_user, &room_id, None).await {
-												.is_admin(local_user)
-												.unwrap_or(false))
-					})
-				}) {
-				debug!("Attempting leave for user {} in room {}", &local_user, &room_id);
-				if let Err(e) = leave_room(self.services, &local_user, &room_id, None).await {
 					error!(
-						"Error attempting to make local user {} leave room {} during bulk room banning: {}",
+						"Error attempting to make local user {local_user} leave room {room_id} during bulk room \
-						&local_user, &room_id, e
+						 banning: {e}",
 					);
+
 					return Ok(RoomMessageEventContent::text_plain(format!(
 						"Error attempting to make local user {} leave room {} during room banning (room is still \
 						 banned but not removing any more users and not banning any more rooms): {}\nIf you would \
@@ -453,26 +433,26 @@ async fn ban_list_of_rooms(&self, force: bool, disable_federation: bool) -> Resu
 		}
 
 		// remove any local aliases, ignore errors
-		for ref local_alias in self
+		self.services
-			.services
 			.rooms
 			.alias
 			.local_aliases_for_room(&room_id)
-			.filter_map(Result::ok)
+			.map(ToOwned::to_owned)
-		{
+			.for_each(|local_alias| async move {
-			_ = self
+				self.services
-				.services
+					.rooms
-				.rooms
+					.alias
-				.alias
+					.remove_alias(&local_alias, &self.services.globals.server_user)
-				.remove_alias(local_alias, &self.services.globals.server_user)
+					.await
-				.await;
+					.ok();
-		}
+			})
+			.await;
 
 		// unpublish from room directory, ignore errors
-		_ = self.services.rooms.directory.set_not_public(&room_id);
+		self.services.rooms.directory.set_not_public(&room_id);
 
 		if disable_federation {
-			self.services.rooms.metadata.disable_room(&room_id, true)?;
+			self.services.rooms.metadata.disable_room(&room_id, true);
 		}
 	}
 
@@ -503,7 +483,7 @@ async fn unban_room(&self, enable_federation: bool, room: Box<RoomOrAliasId>) ->
 
 		debug!("Room specified is a room ID, unbanning room ID");
 
-		self.services.rooms.metadata.ban_room(&room_id, false)?;
+		self.services.rooms.metadata.ban_room(&room_id, false);
 
 		room_id
 	} else if room.is_room_alias_id() {
@@ -522,7 +502,13 @@ async fn unban_room(&self, enable_federation: bool, room: Box<RoomOrAliasId>) ->
 			 get_alias_helper to fetch room ID remotely"
 		);
 
-		let room_id = if let Some(room_id) = self.services.rooms.alias.resolve_local_alias(&room_alias)? {
+		let room_id = if let Ok(room_id) = self
+			.services
+			.rooms
+			.alias
+			.resolve_local_alias(&room_alias)
+			.await
+		{
 			room_id
 		} else {
 			debug!("We don't have this room alias to a room ID locally, attempting to fetch room ID over federation");
@@ -546,7 +532,7 @@ async fn unban_room(&self, enable_federation: bool, room: Box<RoomOrAliasId>) ->
 			}
 		};
 
-		self.services.rooms.metadata.ban_room(&room_id, false)?;
+		self.services.rooms.metadata.ban_room(&room_id, false);
 
 		room_id
 	} else {
@@ -557,7 +543,7 @@ async fn unban_room(&self, enable_federation: bool, room: Box<RoomOrAliasId>) ->
 	};
 
 	if enable_federation {
-		self.services.rooms.metadata.disable_room(&room_id, false)?;
+		self.services.rooms.metadata.disable_room(&room_id, false);
 		return Ok(RoomMessageEventContent::text_plain("Room unbanned."));
 	}
 
@@ -569,45 +555,42 @@ async fn unban_room(&self, enable_federation: bool, room: Box<RoomOrAliasId>) ->
 
 #[admin_command]
 async fn list_banned_rooms(&self, no_details: bool) -> Result<RoomMessageEventContent> {
-	let rooms = self
+	let room_ids: Vec<OwnedRoomId> = self
 		.services
 		.rooms
 		.metadata
 		.list_banned_rooms()
-		.collect::<Result<Vec<_>, _>>();
+		.map(Into::into)
+		.collect()
+		.await;
 
-	match rooms {
+	if room_ids.is_empty() {
-		Ok(room_ids) => {
+		return Ok(RoomMessageEventContent::text_plain("No rooms are banned."));
-			if room_ids.is_empty() {
-				return Ok(RoomMessageEventContent::text_plain("No rooms are banned."));
-			}
-
-			let mut rooms = room_ids
-				.into_iter()
-				.map(|room_id| get_room_info(self.services, &room_id))
-				.collect::<Vec<_>>();
-			rooms.sort_by_key(|r| r.1);
-			rooms.reverse();
-
-			let output_plain = format!(
-				"Rooms Banned ({}):\n```\n{}\n```",
-				rooms.len(),
-				rooms
-					.iter()
-					.map(|(id, members, name)| if no_details {
-						format!("{id}")
-					} else {
-						format!("{id}\tMembers: {members}\tName: {name}")
-					})
-					.collect::<Vec<_>>()
-					.join("\n")
-			);
-
-			Ok(RoomMessageEventContent::notice_markdown(output_plain))
-		},
-		Err(e) => {
-			error!("Failed to list banned rooms: {e}");
-			Ok(RoomMessageEventContent::text_plain(format!("Unable to list banned rooms: {e}")))
-		},
 	}
+
+	let mut rooms = room_ids
+		.iter()
+		.stream()
+		.then(|room_id| get_room_info(self.services, room_id))
+		.collect::<Vec<_>>()
+		.await;
+
+	rooms.sort_by_key(|r| r.1);
+	rooms.reverse();
+
+	let output_plain = format!(
+		"Rooms Banned ({}):\n```\n{}\n```",
+		rooms.len(),
+		rooms
+			.iter()
+			.map(|(id, members, name)| if no_details {
+				format!("{id}")
+			} else {
+				format!("{id}\tMembers: {members}\tName: {name}")
+			})
+			.collect::<Vec<_>>()
+			.join("\n")
+	);
+
+	Ok(RoomMessageEventContent::notice_markdown(output_plain))
 }

src/admin/server/commands.rs

@@ -21,7 +21,10 @@ pub(super) async fn uptime(&self) -> Result<RoomMessageEventContent> {
 #[admin_command]
 pub(super) async fn show_config(&self) -> Result<RoomMessageEventContent> {
 	// Construct and send the response
-	Ok(RoomMessageEventContent::text_plain(format!("{}", self.services.globals.config)))
+	Ok(RoomMessageEventContent::text_markdown(format!(
+		"```\n{}\n```",
+		self.services.globals.config
+	)))
 }
 
 #[admin_command]
@@ -104,7 +107,7 @@ pub(super) async fn backup_database(&self) -> Result<RoomMessageEventContent> {
 		.runtime()
 		.spawn_blocking(move || match globals.db.backup() {
 			Ok(()) => String::new(),
-			Err(e) => (*e).to_string(),
+			Err(e) => e.to_string(),
 		})
 		.await?;
 

src/admin/user/commands.rs

@@ -1,36 +1,49 @@
 use std::{collections::BTreeMap, fmt::Write as _};
 
-use api::client::{join_room_by_id_helper, leave_all_rooms, update_avatar_url, update_displayname};
+use api::client::{full_user_deactivate, join_room_by_id_helper, leave_room};
-use conduit::{error, info, utils, warn, PduBuilder, Result};
+use conduit::{
+	debug_warn, error, info, is_equal_to,
+	utils::{self, ReadyExt},
+	warn, PduBuilder, Result,
+};
+use conduit_api::client::{leave_all_rooms, update_avatar_url, update_displayname};
+use futures::StreamExt;
 use ruma::{
 	events::{
-		room::{message::RoomMessageEventContent, redaction::RoomRedactionEventContent},
+		room::{
+			message::RoomMessageEventContent,
+			power_levels::{RoomPowerLevels, RoomPowerLevelsEventContent},
+			redaction::RoomRedactionEventContent,
+		},
 		tag::{TagEvent, TagEventContent, TagInfo},
-		RoomAccountDataEventType, TimelineEventType,
+		RoomAccountDataEventType, StateEventType,
 	},
-	EventId, OwnedRoomId, OwnedRoomOrAliasId, OwnedUserId, RoomId,
+	EventId, OwnedRoomId, OwnedRoomOrAliasId, OwnedUserId, RoomId, UserId,
 };
-use serde_json::value::to_raw_value;
 
 use crate::{
-	admin_command, escape_html, get_room_info,
+	admin_command, get_room_info,
 	utils::{parse_active_local_user_id, parse_local_user_id},
 };
 
 const AUTO_GEN_PASSWORD_LENGTH: usize = 25;
+const BULK_JOIN_REASON: &str = "Bulk force joining this room as initiated by the server admin.";
 
 #[admin_command]
 pub(super) async fn list_users(&self) -> Result<RoomMessageEventContent> {
-	match self.services.users.list_local_users() {
+	let users = self
-		Ok(users) => {
+		.services
-			let mut plain_msg = format!("Found {} local user account(s):\n```\n", users.len());
+		.users
-			plain_msg += users.join("\n").as_str();
+		.list_local_users()
-			plain_msg += "\n```";
+		.map(ToString::to_string)
+		.collect::<Vec<_>>()
+		.await;
 
-			Ok(RoomMessageEventContent::notice_markdown(plain_msg))
+	let mut plain_msg = format!("Found {} local user account(s):\n```\n", users.len());
-		},
+	plain_msg += users.join("\n").as_str();
-		Err(e) => Ok(RoomMessageEventContent::text_plain(e.to_string())),
+	plain_msg += "\n```";
-	}
+
+	Ok(RoomMessageEventContent::notice_markdown(plain_msg))
 }
 
 #[admin_command]
@@ -38,7 +51,7 @@ pub(super) async fn create_user(&self, username: String, password: Option<String
 	// Validate user id
 	let user_id = parse_local_user_id(self.services, &username)?;
 
-	if self.services.users.exists(&user_id)? {
+	if self.services.users.exists(&user_id).await {
 		return Ok(RoomMessageEventContent::text_plain(format!("Userid {user_id} already exists")));
 	}
 
@@ -73,44 +86,53 @@ pub(super) async fn create_user(&self, username: String, password: Option<String
 
 	self.services
 		.users
-		.set_displayname(&user_id, Some(displayname))
+		.set_displayname(&user_id, Some(displayname));
-		.await?;
 
 	// Initial account data
-	self.services.account_data.update(
+	self.services
-		None,
+		.account_data
-		&user_id,
+		.update(
-		ruma::events::GlobalAccountDataEventType::PushRules
+			None,
-			.to_string()
+			&user_id,
-			.into(),
+			ruma::events::GlobalAccountDataEventType::PushRules
-		&serde_json::to_value(ruma::events::push_rules::PushRulesEvent {
+				.to_string()
-			content: ruma::events::push_rules::PushRulesEventContent {
+				.into(),
-				global: ruma::push::Ruleset::server_default(&user_id),
+			&serde_json::to_value(ruma::events::push_rules::PushRulesEvent {
-			},
+				content: ruma::events::push_rules::PushRulesEventContent {
-		})
+					global: ruma::push::Ruleset::server_default(&user_id),
-		.expect("to json value always works"),
+				},
-	)?;
+			})
+			.expect("to json value always works"),
+		)
+		.await?;
 
 	if !self.services.globals.config.auto_join_rooms.is_empty() {
 		for room in &self.services.globals.config.auto_join_rooms {
+			let Ok(room_id) = self.services.rooms.alias.resolve(room).await else {
+				error!(%user_id, "Failed to resolve room alias to room ID when attempting to auto join {room}, skipping");
+				continue;
+			};
+
 			if !self
 				.services
 				.rooms
 				.state_cache
-				.server_in_room(self.services.globals.server_name(), room)?
+				.server_in_room(self.services.globals.server_name(), &room_id)
+				.await
 			{
 				warn!("Skipping room {room} to automatically join as we have never joined before.");
 				continue;
 			}
 
-			if let Some(room_id_server_name) = room.server_name() {
+			if let Some(room_server_name) = room.server_name() {
 				match join_room_by_id_helper(
 					self.services,
 					&user_id,
-					room,
+					&room_id,
 					Some("Automatically joining this room upon registration".to_owned()),
-					&[room_id_server_name.to_owned(), self.services.globals.server_name().to_owned()],
+					&[self.services.globals.server_name().to_owned(), room_server_name.to_owned()],
 					None,
+					&None,
 				)
 				.await
 				{
@@ -118,6 +140,13 @@ pub(super) async fn create_user(&self, username: String, password: Option<String
 						info!("Automatically joined room {room} for user {user_id}");
 					},
 					Err(e) => {
+						self.services
+							.admin
+							.send_message(RoomMessageEventContent::text_plain(format!(
+								"Failed to automatically join room {room} for user {user_id}: {e}"
+							)))
+							.await
+							.ok();
 						// don't return this error so we don't fail registrations
 						error!("Failed to automatically join room {room} for user {user_id}: {e}");
 					},
@@ -128,6 +157,23 @@ pub(super) async fn create_user(&self, username: String, password: Option<String
 
 	// we dont add a device since we're not the user, just the creator
 
+	// if this account creation is from the CLI / --execute, invite the first user
+	// to admin room
+	if let Ok(admin_room) = self.services.admin.get_admin_room().await {
+		if self
+			.services
+			.rooms
+			.state_cache
+			.room_joined_count(&admin_room)
+			.await
+			.is_ok_and(is_equal_to!(1))
+		{
+			self.services.admin.make_user_admin(&user_id).await?;
+
+			warn!("Granting {user_id} admin privileges as the first user");
+		}
+	}
+
 	// Inhibit login does not work for guests
 	Ok(RoomMessageEventContent::text_plain(format!(
 		"Created user with user_id: {user_id} and password: `{password}`"
@@ -146,7 +192,7 @@ pub(super) async fn deactivate(&self, no_leave_rooms: bool, user_id: String) ->
 		));
 	}
 
-	self.services.users.deactivate_account(&user_id)?;
+	self.services.users.deactivate_account(&user_id).await?;
 
 	if !no_leave_rooms {
 		self.services
@@ -154,17 +200,21 @@ pub(super) async fn deactivate(&self, no_leave_rooms: bool, user_id: String) ->
 			.send_message(RoomMessageEventContent::text_plain(format!(
 				"Making {user_id} leave all rooms after deactivation..."
 			)))
-			.await;
+			.await
+			.ok();
 
 		let all_joined_rooms: Vec<OwnedRoomId> = self
 			.services
 			.rooms
 			.state_cache
 			.rooms_joined(&user_id)
-			.filter_map(Result::ok)
+			.map(Into::into)
-			.collect();
+			.collect()
-		update_displayname(self.services, user_id.clone(), None, all_joined_rooms.clone()).await?;
+			.await;
-		update_avatar_url(self.services, user_id.clone(), None, None, all_joined_rooms).await?;
+
+		full_user_deactivate(self.services, &user_id, &all_joined_rooms).await?;
+		update_displayname(self.services, &user_id, None, &all_joined_rooms).await?;
+		update_avatar_url(self.services, &user_id, None, None, &all_joined_rooms).await?;
 		leave_all_rooms(self.services, &user_id).await;
 	}
 
@@ -218,15 +268,16 @@ pub(super) async fn deactivate_all(&self, no_leave_rooms: bool, force: bool) ->
 	let mut admins = Vec::new();
 
 	for username in usernames {
-		match parse_active_local_user_id(self.services, username) {
+		match parse_active_local_user_id(self.services, username).await {
 			Ok(user_id) => {
-				if self.services.users.is_admin(&user_id)? && !force {
+				if self.services.users.is_admin(&user_id).await && !force {
 					self.services
 						.admin
 						.send_message(RoomMessageEventContent::text_plain(format!(
 							"{username} is an admin and --force is not set, skipping over"
 						)))
-						.await;
+						.await
+						.ok();
 					admins.push(username);
 					continue;
 				}
@@ -238,7 +289,8 @@ pub(super) async fn deactivate_all(&self, no_leave_rooms: bool, force: bool) ->
 						.send_message(RoomMessageEventContent::text_plain(format!(
 							"{username} is the server service account, skipping over"
 						)))
-						.await;
+						.await
+						.ok();
 					continue;
 				}
 
@@ -250,7 +302,8 @@ pub(super) async fn deactivate_all(&self, no_leave_rooms: bool, force: bool) ->
 					.send_message(RoomMessageEventContent::text_plain(format!(
 						"{username} is not a valid username, skipping over: {e}"
 					)))
-					.await;
+					.await
+					.ok();
 				continue;
 			},
 		}
@@ -259,7 +312,7 @@ pub(super) async fn deactivate_all(&self, no_leave_rooms: bool, force: bool) ->
 	let mut deactivation_count: usize = 0;
 
 	for user_id in user_ids {
-		match self.services.users.deactivate_account(&user_id) {
+		match self.services.users.deactivate_account(&user_id).await {
 			Ok(()) => {
 				deactivation_count = deactivation_count.saturating_add(1);
 				if !no_leave_rooms {
@@ -269,10 +322,17 @@ pub(super) async fn deactivate_all(&self, no_leave_rooms: bool, force: bool) ->
 						.rooms
 						.state_cache
 						.rooms_joined(&user_id)
-						.filter_map(Result::ok)
+						.map(Into::into)
-						.collect();
+						.collect()
-					update_displayname(self.services, user_id.clone(), None, all_joined_rooms.clone()).await?;
+						.await;
-					update_avatar_url(self.services, user_id.clone(), None, None, all_joined_rooms).await?;
+
+					full_user_deactivate(self.services, &user_id, &all_joined_rooms).await?;
+					update_displayname(self.services, &user_id, None, &all_joined_rooms)
+						.await
+						.ok();
+					update_avatar_url(self.services, &user_id, None, None, &all_joined_rooms)
+						.await
+						.ok();
 					leave_all_rooms(self.services, &user_id).await;
 				}
 			},
@@ -280,7 +340,8 @@ pub(super) async fn deactivate_all(&self, no_leave_rooms: bool, force: bool) ->
 				self.services
 					.admin
 					.send_message(RoomMessageEventContent::text_plain(format!("Failed deactivating user: {e}")))
-					.await;
+					.await
+					.ok();
 			},
 		}
 	}
@@ -308,9 +369,9 @@ pub(super) async fn list_joined_rooms(&self, user_id: String) -> Result<RoomMess
 		.rooms
 		.state_cache
 		.rooms_joined(&user_id)
-		.filter_map(Result::ok)
+		.then(|room_id| get_room_info(self.services, room_id))
-		.map(|room_id| get_room_info(self.services, &room_id))
+		.collect()
-		.collect();
+		.await;
 
 	if rooms.is_empty() {
 		return Ok(RoomMessageEventContent::text_plain("User is not in any rooms."));
@@ -320,7 +381,7 @@ pub(super) async fn list_joined_rooms(&self, user_id: String) -> Result<RoomMess
 	rooms.reverse();
 
 	let output_plain = format!(
-		"Rooms {user_id} Joined ({}):\n{}",
+		"Rooms {user_id} Joined ({}):\n```\n{}\n```",
 		rooms.len(),
 		rooms
 			.iter()
@@ -329,31 +390,259 @@ pub(super) async fn list_joined_rooms(&self, user_id: String) -> Result<RoomMess
 			.join("\n")
 	);
 
-	let output_html = format!(
+	Ok(RoomMessageEventContent::notice_markdown(output_plain))
-		"<table><caption>Rooms {user_id} Joined \
+}
-		 ({})</caption>\n<tr><th>id</th>\t<th>members</th>\t<th>name</th></tr>\n{}</table>",
-		rooms.len(),
-		rooms
-			.iter()
-			.fold(String::new(), |mut output, (id, members, name)| {
-				writeln!(
-					output,
-					"<tr><td>{}</td>\t<td>{}</td>\t<td>{}</td></tr>",
-					escape_html(id.as_ref()),
-					members,
-					escape_html(name)
-				)
-				.unwrap();
-				output
-			})
-	);
 
-	Ok(RoomMessageEventContent::text_html(output_plain, output_html))
+#[admin_command]
+pub(super) async fn force_join_list_of_local_users(
+	&self, room_id: OwnedRoomOrAliasId, yes_i_want_to_do_this: bool,
+) -> Result<RoomMessageEventContent> {
+	if self.body.len() < 2 || !self.body[0].trim().starts_with("```") || self.body.last().unwrap_or(&"").trim() != "```"
+	{
+		return Ok(RoomMessageEventContent::text_plain(
+			"Expected code block in command body. Add --help for details.",
+		));
+	}
+
+	if !yes_i_want_to_do_this {
+		return Ok(RoomMessageEventContent::notice_markdown(
+			"You must pass the --yes-i-want-to-do-this-flag to ensure you really want to force bulk join all \
+			 specified local users.",
+		));
+	}
+
+	let Ok(admin_room) = self.services.admin.get_admin_room().await else {
+		return Ok(RoomMessageEventContent::notice_markdown(
+			"There is not an admin room to check for server admins.",
+		));
+	};
+
+	let (room_id, servers) = self
+		.services
+		.rooms
+		.alias
+		.resolve_with_servers(&room_id, None)
+		.await?;
+
+	if !self
+		.services
+		.rooms
+		.state_cache
+		.server_in_room(self.services.globals.server_name(), &room_id)
+		.await
+	{
+		return Ok(RoomMessageEventContent::notice_markdown("We are not joined in this room."));
+	}
+
+	let server_admins: Vec<_> = self
+		.services
+		.rooms
+		.state_cache
+		.active_local_users_in_room(&admin_room)
+		.map(ToOwned::to_owned)
+		.collect()
+		.await;
+
+	if !self
+		.services
+		.rooms
+		.state_cache
+		.room_members(&room_id)
+		.ready_any(|user_id| server_admins.contains(&user_id.to_owned()))
+		.await
+	{
+		return Ok(RoomMessageEventContent::notice_markdown(
+			"There is not a single server admin in the room.",
+		));
+	}
+
+	let usernames = self
+		.body
+		.to_vec()
+		.drain(1..self.body.len().saturating_sub(1))
+		.collect::<Vec<_>>();
+
+	let mut user_ids: Vec<OwnedUserId> = Vec::with_capacity(usernames.len());
+
+	for username in usernames {
+		match parse_active_local_user_id(self.services, username).await {
+			Ok(user_id) => {
+				// don't make the server service account join
+				if user_id == self.services.globals.server_user {
+					self.services
+						.admin
+						.send_message(RoomMessageEventContent::text_plain(format!(
+							"{username} is the server service account, skipping over"
+						)))
+						.await
+						.ok();
+					continue;
+				}
+
+				user_ids.push(user_id);
+			},
+			Err(e) => {
+				self.services
+					.admin
+					.send_message(RoomMessageEventContent::text_plain(format!(
+						"{username} is not a valid username, skipping over: {e}"
+					)))
+					.await
+					.ok();
+				continue;
+			},
+		}
+	}
+
+	let mut failed_joins: usize = 0;
+	let mut successful_joins: usize = 0;
+
+	for user_id in user_ids {
+		match join_room_by_id_helper(
+			self.services,
+			&user_id,
+			&room_id,
+			Some(String::from(BULK_JOIN_REASON)),
+			&servers,
+			None,
+			&None,
+		)
+		.await
+		{
+			Ok(_res) => {
+				successful_joins = successful_joins.saturating_add(1);
+			},
+			Err(e) => {
+				debug_warn!("Failed force joining {user_id} to {room_id} during bulk join: {e}");
+				failed_joins = failed_joins.saturating_add(1);
+			},
+		};
+	}
+
+	Ok(RoomMessageEventContent::notice_markdown(format!(
+		"{successful_joins} local users have been joined to {room_id}. {failed_joins} joins failed.",
+	)))
+}
+
+#[admin_command]
+pub(super) async fn force_join_all_local_users(
+	&self, room_id: OwnedRoomOrAliasId, yes_i_want_to_do_this: bool,
+) -> Result<RoomMessageEventContent> {
+	if !yes_i_want_to_do_this {
+		return Ok(RoomMessageEventContent::notice_markdown(
+			"You must pass the --yes-i-want-to-do-this-flag to ensure you really want to force bulk join all local \
+			 users.",
+		));
+	}
+
+	let Ok(admin_room) = self.services.admin.get_admin_room().await else {
+		return Ok(RoomMessageEventContent::notice_markdown(
+			"There is not an admin room to check for server admins.",
+		));
+	};
+
+	let (room_id, servers) = self
+		.services
+		.rooms
+		.alias
+		.resolve_with_servers(&room_id, None)
+		.await?;
+
+	if !self
+		.services
+		.rooms
+		.state_cache
+		.server_in_room(self.services.globals.server_name(), &room_id)
+		.await
+	{
+		return Ok(RoomMessageEventContent::notice_markdown("We are not joined in this room."));
+	}
+
+	let server_admins: Vec<_> = self
+		.services
+		.rooms
+		.state_cache
+		.active_local_users_in_room(&admin_room)
+		.map(ToOwned::to_owned)
+		.collect()
+		.await;
+
+	if !self
+		.services
+		.rooms
+		.state_cache
+		.room_members(&room_id)
+		.ready_any(|user_id| server_admins.contains(&user_id.to_owned()))
+		.await
+	{
+		return Ok(RoomMessageEventContent::notice_markdown(
+			"There is not a single server admin in the room.",
+		));
+	}
+
+	let mut failed_joins: usize = 0;
+	let mut successful_joins: usize = 0;
+
+	for user_id in &self
+		.services
+		.users
+		.list_local_users()
+		.map(UserId::to_owned)
+		.collect::<Vec<_>>()
+		.await
+	{
+		match join_room_by_id_helper(
+			self.services,
+			user_id,
+			&room_id,
+			Some(String::from(BULK_JOIN_REASON)),
+			&servers,
+			None,
+			&None,
+		)
+		.await
+		{
+			Ok(_res) => {
+				successful_joins = successful_joins.saturating_add(1);
+			},
+			Err(e) => {
+				debug_warn!("Failed force joining {user_id} to {room_id} during bulk join: {e}");
+				failed_joins = failed_joins.saturating_add(1);
+			},
+		};
+	}
+
+	Ok(RoomMessageEventContent::notice_markdown(format!(
+		"{successful_joins} local users have been joined to {room_id}. {failed_joins} joins failed.",
+	)))
 }
 
 #[admin_command]
 pub(super) async fn force_join_room(
 	&self, user_id: String, room_id: OwnedRoomOrAliasId,
+) -> Result<RoomMessageEventContent> {
+	let user_id = parse_local_user_id(self.services, &user_id)?;
+	let (room_id, servers) = self
+		.services
+		.rooms
+		.alias
+		.resolve_with_servers(&room_id, None)
+		.await?;
+
+	assert!(
+		self.services.globals.user_is_local(&user_id),
+		"Parsed user_id must be a local user"
+	);
+	join_room_by_id_helper(self.services, &user_id, &room_id, None, &servers, None, &None).await?;
+
+	Ok(RoomMessageEventContent::notice_markdown(format!(
+		"{user_id} has been joined to {room_id}.",
+	)))
+}
+
+#[admin_command]
+pub(super) async fn force_leave_room(
+	&self, user_id: String, room_id: OwnedRoomOrAliasId,
 ) -> Result<RoomMessageEventContent> {
 	let user_id = parse_local_user_id(self.services, &user_id)?;
 	let room_id = self.services.rooms.alias.resolve(&room_id).await?;
@@ -362,30 +651,82 @@ pub(super) async fn force_join_room(
 		self.services.globals.user_is_local(&user_id),
 		"Parsed user_id must be a local user"
 	);
-	join_room_by_id_helper(self.services, &user_id, &room_id, None, &[], None).await?;
+	leave_room(self.services, &user_id, &room_id, None).await?;
 
 	Ok(RoomMessageEventContent::notice_markdown(format!(
-		"{user_id} has been joined to {room_id}.",
+		"{user_id} has left {room_id}.",
+	)))
+}
+
+#[admin_command]
+pub(super) async fn force_demote(
+	&self, user_id: String, room_id: OwnedRoomOrAliasId,
+) -> Result<RoomMessageEventContent> {
+	let user_id = parse_local_user_id(self.services, &user_id)?;
+	let room_id = self.services.rooms.alias.resolve(&room_id).await?;
+
+	assert!(
+		self.services.globals.user_is_local(&user_id),
+		"Parsed user_id must be a local user"
+	);
+
+	let state_lock = self.services.rooms.state.mutex.lock(&room_id).await;
+
+	let room_power_levels = self
+		.services
+		.rooms
+		.state_accessor
+		.room_state_get_content::<RoomPowerLevelsEventContent>(&room_id, &StateEventType::RoomPowerLevels, "")
+		.await
+		.ok();
+
+	let user_can_demote_self = room_power_levels
+		.as_ref()
+		.is_some_and(|power_levels_content| {
+			RoomPowerLevels::from(power_levels_content.clone()).user_can_change_user_power_level(&user_id, &user_id)
+		}) || self
+		.services
+		.rooms
+		.state_accessor
+		.room_state_get(&room_id, &StateEventType::RoomCreate, "")
+		.await
+		.is_ok_and(|event| event.sender == user_id);
+
+	if !user_can_demote_self {
+		return Ok(RoomMessageEventContent::notice_markdown(
+			"User is not allowed to modify their own power levels in the room.",
+		));
+	}
+
+	let mut power_levels_content = room_power_levels.unwrap_or_default();
+	power_levels_content.users.remove(&user_id);
+
+	let event_id = self
+		.services
+		.rooms
+		.timeline
+		.build_and_append_pdu(
+			PduBuilder::state(String::new(), &power_levels_content),
+			&user_id,
+			&room_id,
+			&state_lock,
+		)
+		.await?;
+
+	Ok(RoomMessageEventContent::notice_markdown(format!(
+		"User {user_id} demoted themselves to the room default power level in {room_id} - {event_id}"
 	)))
 }
 
 #[admin_command]
 pub(super) async fn make_user_admin(&self, user_id: String) -> Result<RoomMessageEventContent> {
 	let user_id = parse_local_user_id(self.services, &user_id)?;
-	let displayname = self
-		.services
-		.users
-		.displayname(&user_id)?
-		.unwrap_or_else(|| user_id.to_string());
 
 	assert!(
 		self.services.globals.user_is_local(&user_id),
 		"Parsed user_id must be a local user"
 	);
-	self.services
+	self.services.admin.make_user_admin(&user_id).await?;
-		.admin
-		.make_user_admin(&user_id, displayname)
-		.await?;
 
 	Ok(RoomMessageEventContent::notice_markdown(format!(
 		"{user_id} has been granted admin privileges.",
@@ -396,33 +737,33 @@ pub(super) async fn make_user_admin(&self, user_id: String) -> Result<RoomMessag
 pub(super) async fn put_room_tag(
 	&self, user_id: String, room_id: Box<RoomId>, tag: String,
 ) -> Result<RoomMessageEventContent> {
-	let user_id = parse_active_local_user_id(self.services, &user_id)?;
+	let user_id = parse_active_local_user_id(self.services, &user_id).await?;
 
-	let event = self
+	let mut tags_event = self
 		.services
 		.account_data
-		.get(Some(&room_id), &user_id, RoomAccountDataEventType::Tag)?;
+		.get_room(&room_id, &user_id, RoomAccountDataEventType::Tag)
-
+		.await
-	let mut tags_event = event.map_or_else(
+		.unwrap_or(TagEvent {
-		|| TagEvent {
 			content: TagEventContent {
 				tags: BTreeMap::new(),
 			},
-		},
+		});
-		|e| serde_json::from_str(e.get()).expect("Bad account data in database for user {user_id}"),
-	);
 
 	tags_event
 		.content
 		.tags
 		.insert(tag.clone().into(), TagInfo::new());
 
-	self.services.account_data.update(
+	self.services
-		Some(&room_id),
+		.account_data
-		&user_id,
+		.update(
-		RoomAccountDataEventType::Tag,
+			Some(&room_id),
-		&serde_json::to_value(tags_event).expect("to json value always works"),
+			&user_id,
-	)?;
+			RoomAccountDataEventType::Tag,
+			&serde_json::to_value(tags_event).expect("to json value always works"),
+		)
+		.await?;
 
 	Ok(RoomMessageEventContent::text_plain(format!(
 		"Successfully updated room account data for {user_id} and room {room_id} with tag {tag}"
@@ -433,30 +774,30 @@ pub(super) async fn put_room_tag(
 pub(super) async fn delete_room_tag(
 	&self, user_id: String, room_id: Box<RoomId>, tag: String,
 ) -> Result<RoomMessageEventContent> {
-	let user_id = parse_active_local_user_id(self.services, &user_id)?;
+	let user_id = parse_active_local_user_id(self.services, &user_id).await?;
 
-	let event = self
+	let mut tags_event = self
 		.services
 		.account_data
-		.get(Some(&room_id), &user_id, RoomAccountDataEventType::Tag)?;
+		.get_room(&room_id, &user_id, RoomAccountDataEventType::Tag)
-
+		.await
-	let mut tags_event = event.map_or_else(
+		.unwrap_or(TagEvent {
-		|| TagEvent {
 			content: TagEventContent {
 				tags: BTreeMap::new(),
 			},
-		},
+		});
-		|e| serde_json::from_str(e.get()).expect("Bad account data in database for user {user_id}"),
-	);
 
 	tags_event.content.tags.remove(&tag.clone().into());
 
-	self.services.account_data.update(
+	self.services
-		Some(&room_id),
+		.account_data
-		&user_id,
+		.update(
-		RoomAccountDataEventType::Tag,
+			Some(&room_id),
-		&serde_json::to_value(tags_event).expect("to json value always works"),
+			&user_id,
-	)?;
+			RoomAccountDataEventType::Tag,
+			&serde_json::to_value(tags_event).expect("to json value always works"),
+		)
+		.await?;
 
 	Ok(RoomMessageEventContent::text_plain(format!(
 		"Successfully updated room account data for {user_id} and room {room_id}, deleting room tag {tag}"
@@ -465,21 +806,18 @@ pub(super) async fn delete_room_tag(
 
 #[admin_command]
 pub(super) async fn get_room_tags(&self, user_id: String, room_id: Box<RoomId>) -> Result<RoomMessageEventContent> {
-	let user_id = parse_active_local_user_id(self.services, &user_id)?;
+	let user_id = parse_active_local_user_id(self.services, &user_id).await?;
 
-	let event = self
+	let tags_event = self
 		.services
 		.account_data
-		.get(Some(&room_id), &user_id, RoomAccountDataEventType::Tag)?;
+		.get_room(&room_id, &user_id, RoomAccountDataEventType::Tag)
-
+		.await
-	let tags_event = event.map_or_else(
+		.unwrap_or(TagEvent {
-		|| TagEvent {
 			content: TagEventContent {
 				tags: BTreeMap::new(),
 			},
-		},
+		});
-		|e| serde_json::from_str(e.get()).expect("Bad account data in database for user {user_id}"),
-	);
 
 	Ok(RoomMessageEventContent::notice_markdown(format!(
 		"```\n{:#?}\n```",
@@ -489,11 +827,12 @@ pub(super) async fn get_room_tags(&self, user_id: String, room_id: Box<RoomId>)
 
 #[admin_command]
 pub(super) async fn redact_event(&self, event_id: Box<EventId>) -> Result<RoomMessageEventContent> {
-	let Some(event) = self
+	let Ok(event) = self
 		.services
 		.rooms
 		.timeline
-		.get_non_outlier_pdu(&event_id)?
+		.get_non_outlier_pdu(&event_id)
+		.await
 	else {
 		return Ok(RoomMessageEventContent::text_plain("Event does not exist in our database."));
 	};
@@ -522,16 +861,11 @@ pub(super) async fn redact_event(&self, event_id: Box<EventId>) -> Result<RoomMe
 		.timeline
 		.build_and_append_pdu(
 			PduBuilder {
-				event_type: TimelineEventType::RoomRedaction,
+				redacts: Some(event.event_id.clone()),
-				content: to_raw_value(&RoomRedactionEventContent {
+				..PduBuilder::timeline(&RoomRedactionEventContent {
 					redacts: Some(event.event_id.clone().into()),
 					reason: Some(reason),
 				})
-				.expect("event is valid, we just created it"),
-				unsigned: None,
-				state_key: None,
-				redacts: Some(event.event_id),
-				timestamp: None,
 			},
 			&sender_user,
 			&room_id,

src/admin/user/mod.rs

@@ -73,6 +73,19 @@ pub(super) enum UserCommand {
 		room_id: OwnedRoomOrAliasId,
 	},
 
+	/// - Manually leave a local user from a room.
+	ForceLeaveRoom {
+		user_id: String,
+		room_id: OwnedRoomOrAliasId,
+	},
+
+	/// - Forces the specified user to drop their power levels to the room
+	///   default, if their permissions allow and the auth check permits
+	ForceDemote {
+		user_id: String,
+		room_id: OwnedRoomOrAliasId,
+	},
+
 	/// - Grant server-admin privileges to a user.
 	MakeUserAdmin {
 		user_id: String,
@@ -111,4 +124,31 @@ pub(super) enum UserCommand {
 	RedactEvent {
 		event_id: Box<EventId>,
 	},
+
+	/// - Force joins a specified list of local users to join the specified
+	///   room.
+	///
+	/// Specify a codeblock of usernames.
+	///
+	/// At least 1 server admin must be in the room to reduce abuse.
+	///
+	/// Requires the `--yes-i-want-to-do-this` flag.
+	ForceJoinListOfLocalUsers {
+		room_id: OwnedRoomOrAliasId,
+
+		#[arg(long)]
+		yes_i_want_to_do_this: bool,
+	},
+
+	/// - Force joins all local users to the specified room.
+	///
+	/// At least 1 server admin must be in the room to reduce abuse.
+	///
+	/// Requires the `--yes-i-want-to-do-this` flag.
+	ForceJoinAllLocalUsers {
+		room_id: OwnedRoomOrAliasId,
+
+		#[arg(long)]
+		yes_i_want_to_do_this: bool,
+	},
 }

src/admin/utils.rs

@@ -8,23 +8,21 @@ pub(crate) fn escape_html(s: &str) -> String {
 		.replace('>', ">")
 }
 
-pub(crate) fn get_room_info(services: &Services, id: &RoomId) -> (OwnedRoomId, u64, String) {
+pub(crate) async fn get_room_info(services: &Services, room_id: &RoomId) -> (OwnedRoomId, u64, String) {
 	(
-		id.into(),
+		room_id.into(),
 		services
 			.rooms
 			.state_cache
-			.room_joined_count(id)
+			.room_joined_count(room_id)
-			.ok()
+			.await
-			.flatten()
 			.unwrap_or(0),
 		services
 			.rooms
 			.state_accessor
-			.get_name(id)
+			.get_name(room_id)
-			.ok()
+			.await
-			.flatten()
+			.unwrap_or_else(|_| room_id.to_string()),
-			.unwrap_or_else(|| id.to_string()),
 	)
 }
 
@@ -46,14 +44,14 @@ pub(crate) fn parse_local_user_id(services: &Services, user_id: &str) -> Result<
 }
 
 /// Parses user ID that is an active (not guest or deactivated) local user
-pub(crate) fn parse_active_local_user_id(services: &Services, user_id: &str) -> Result<OwnedUserId> {
+pub(crate) async fn parse_active_local_user_id(services: &Services, user_id: &str) -> Result<OwnedUserId> {
 	let user_id = parse_local_user_id(services, user_id)?;
 
-	if !services.users.exists(&user_id)? {
+	if !services.users.exists(&user_id).await {
 		return Err!("User {user_id:?} does not exist on this server.");
 	}
 
-	if services.users.is_deactivated(&user_id)? {
+	if services.users.is_deactivated(&user_id).await? {
 		return Err!("User {user_id:?} is deactivated.");
 	}
 

src/api/Cargo.toml

@@ -45,7 +45,7 @@ conduit-core.workspace = true
 conduit-database.workspace = true
 conduit-service.workspace = true
 const-str.workspace = true
-futures-util.workspace = true
+futures.workspace = true
 hmac.workspace = true
 http.workspace = true
 http-body-util.workspace = true
@@ -59,7 +59,7 @@ ruma.workspace = true
 serde_html_form.workspace = true
 serde_json.workspace = true
 serde.workspace = true
-sha-1.workspace = true
+sha1.workspace = true
 tokio.workspace = true
 tracing.workspace = true
 

src/api/client/account.rs

@@ -2,7 +2,8 @@ use std::fmt::Write;
 
 use axum::extract::State;
 use axum_client_ip::InsecureClientIp;
-use conduit::{debug_info, error, info, utils, warn, Error, Result};
+use conduit::{debug_info, error, info, is_equal_to, utils, utils::ReadyExt, warn, Error, PduBuilder, Result};
+use futures::{FutureExt, StreamExt};
 use register::RegistrationKind;
 use ruma::{
 	api::client::{
@@ -15,9 +16,16 @@ use ruma::{
 		error::ErrorKind,
 		uiaa::{AuthFlow, AuthType, UiaaInfo},
 	},
-	events::{room::message::RoomMessageEventContent, GlobalAccountDataEventType},
+	events::{
+		room::{
+			message::RoomMessageEventContent,
+			power_levels::{RoomPowerLevels, RoomPowerLevelsEventContent},
+		},
+		GlobalAccountDataEventType, StateEventType,
+	},
 	push, OwnedRoomId, UserId,
 };
+use service::Services;
 
 use super::{join_room_by_id_helper, DEVICE_ID_LENGTH, SESSION_ID_LENGTH, TOKEN_LENGTH};
 use crate::Ruma;
@@ -40,14 +48,30 @@ pub(crate) async fn get_register_available_route(
 	State(services): State<crate::State>, InsecureClientIp(client): InsecureClientIp,
 	body: Ruma<get_username_availability::v3::Request>,
 ) -> Result<get_username_availability::v3::Response> {
+	// workaround for https://github.com/matrix-org/matrix-appservice-irc/issues/1780 due to inactivity of fixing the issue
+	let is_matrix_appservice_irc = body.appservice_info.as_ref().is_some_and(|appservice| {
+		appservice.registration.id == "irc"
+			|| appservice.registration.id.contains("matrix-appservice-irc")
+			|| appservice.registration.id.contains("matrix_appservice_irc")
+	});
+
+	// don't force the username lowercase if it's from matrix-appservice-irc
+	let body_username = if is_matrix_appservice_irc {
+		body.username.clone()
+	} else {
+		body.username.to_lowercase()
+	};
+
 	// Validate user id
-	let user_id = UserId::parse_with_server_name(body.username.to_lowercase(), services.globals.server_name())
+	let user_id = UserId::parse_with_server_name(body_username, services.globals.server_name())
 		.ok()
-		.filter(|user_id| !user_id.is_historical() && services.globals.user_is_local(user_id))
+		.filter(|user_id| {
+			(!user_id.is_historical() || is_matrix_appservice_irc) && services.globals.user_is_local(user_id)
+		})
 		.ok_or(Error::BadRequest(ErrorKind::InvalidUsername, "Username is invalid."))?;
 
 	// Check if username is creative enough
-	if services.users.exists(&user_id)? {
+	if services.users.exists(&user_id).await {
 		return Err(Error::BadRequest(ErrorKind::UserInUse, "Desired user ID is already taken."));
 	}
 
@@ -92,8 +116,8 @@ pub(crate) async fn register_route(
 	if !services.globals.allow_registration() && body.appservice_info.is_none() {
 		info!(
 			"Registration disabled and request not from known appservice, rejecting registration attempt for username \
-			 {:?}",
+			 \"{}\"",
-			body.username
+			body.username.as_deref().unwrap_or("")
 		);
 		return Err(Error::BadRequest(ErrorKind::forbidden(), "Registration has been disabled."));
 	}
@@ -102,12 +126,12 @@ pub(crate) async fn register_route(
 
 	if is_guest
 		&& (!services.globals.allow_guest_registration()
-			|| (services.globals.allow_registration() && services.globals.config.registration_token.is_some()))
+			|| (services.globals.allow_registration() && services.globals.registration_token.is_some()))
 	{
 		info!(
 			"Guest registration disabled / registration enabled with token configured, rejecting guest registration \
-			 attempt, initial device name: {:?}",
+			 attempt, initial device name: \"{}\"",
-			body.initial_device_display_name
+			body.initial_device_display_name.as_deref().unwrap_or("")
 		);
 		return Err(Error::BadRequest(
 			ErrorKind::GuestAccessForbidden,
@@ -117,24 +141,39 @@ pub(crate) async fn register_route(
 
 	// forbid guests from registering if there is not a real admin user yet. give
 	// generic user error.
-	if is_guest && services.users.count()? < 2 {
+	if is_guest && services.users.count().await < 2 {
 		warn!(
 			"Guest account attempted to register before a real admin user has been registered, rejecting \
-			 registration. Guest's initial device name: {:?}",
+			 registration. Guest's initial device name: \"{}\"",
-			body.initial_device_display_name
+			body.initial_device_display_name.as_deref().unwrap_or("")
 		);
 		return Err(Error::BadRequest(ErrorKind::forbidden(), "Registration temporarily disabled."));
 	}
 
 	let user_id = match (&body.username, is_guest) {
 		(Some(username), false) => {
-			let proposed_user_id =
+			// workaround for https://github.com/matrix-org/matrix-appservice-irc/issues/1780 due to inactivity of fixing the issue
-				UserId::parse_with_server_name(username.to_lowercase(), services.globals.server_name())
+			let is_matrix_appservice_irc = body.appservice_info.as_ref().is_some_and(|appservice| {
-					.ok()
+				appservice.registration.id == "irc"
-					.filter(|user_id| !user_id.is_historical() && services.globals.user_is_local(user_id))
+					|| appservice.registration.id.contains("matrix-appservice-irc")
-					.ok_or(Error::BadRequest(ErrorKind::InvalidUsername, "Username is invalid."))?;
+					|| appservice.registration.id.contains("matrix_appservice_irc")
+			});
 
-			if services.users.exists(&proposed_user_id)? {
+			// don't force the username lowercase if it's from matrix-appservice-irc
+			let body_username = if is_matrix_appservice_irc {
+				username.clone()
+			} else {
+				username.to_lowercase()
+			};
+
+			let proposed_user_id = UserId::parse_with_server_name(body_username, services.globals.server_name())
+				.ok()
+				.filter(|user_id| {
+					(!user_id.is_historical() || is_matrix_appservice_irc) && services.globals.user_is_local(user_id)
+				})
+				.ok_or(Error::BadRequest(ErrorKind::InvalidUsername, "Username is invalid."))?;
+
+			if services.users.exists(&proposed_user_id).await {
 				return Err(Error::BadRequest(ErrorKind::UserInUse, "Desired user ID is already taken."));
 			}
 
@@ -154,7 +193,7 @@ pub(crate) async fn register_route(
 				services.globals.server_name(),
 			)
 			.unwrap();
-			if !services.users.exists(&proposed_user_id)? {
+			if !services.users.exists(&proposed_user_id).await {
 				break proposed_user_id;
 			}
 		},
@@ -174,7 +213,7 @@ pub(crate) async fn register_route(
 
 	// UIAA
 	let mut uiaainfo;
-	let skip_auth = if services.globals.config.registration_token.is_some() {
+	let skip_auth = if services.globals.registration_token.is_some() {
 		// Registration token required
 		uiaainfo = UiaaInfo {
 			flows: vec![AuthFlow {
@@ -202,12 +241,15 @@ pub(crate) async fn register_route(
 
 	if !skip_auth {
 		if let Some(auth) = &body.auth {
-			let (worked, uiaainfo) = services.uiaa.try_auth(
+			let (worked, uiaainfo) = services
-				&UserId::parse_with_server_name("", services.globals.server_name()).expect("we know this is valid"),
+				.uiaa
-				"".into(),
+				.try_auth(
-				auth,
+					&UserId::parse_with_server_name("", services.globals.server_name()).expect("we know this is valid"),
-				&uiaainfo,
+					"".into(),
-			)?;
+					auth,
+					&uiaainfo,
+				)
+				.await?;
 			if !worked {
 				return Err(Error::Uiaa(uiaainfo));
 			}
@@ -219,7 +261,7 @@ pub(crate) async fn register_route(
 				"".into(),
 				&uiaainfo,
 				&json,
-			)?;
+			);
 			return Err(Error::Uiaa(uiaainfo));
 		} else {
 			return Err(Error::BadRequest(ErrorKind::NotJson, "Not json."));
@@ -247,21 +289,23 @@ pub(crate) async fn register_route(
 
 	services
 		.users
-		.set_displayname(&user_id, Some(displayname.clone()))
+		.set_displayname(&user_id, Some(displayname.clone()));
-		.await?;
 
 	// Initial account data
-	services.account_data.update(
+	services
-		None,
+		.account_data
-		&user_id,
+		.update(
-		GlobalAccountDataEventType::PushRules.to_string().into(),
+			None,
-		&serde_json::to_value(ruma::events::push_rules::PushRulesEvent {
+			&user_id,
-			content: ruma::events::push_rules::PushRulesEventContent {
+			GlobalAccountDataEventType::PushRules.to_string().into(),
-				global: push::Ruleset::server_default(&user_id),
+			&serde_json::to_value(ruma::events::push_rules::PushRulesEvent {
-			},
+				content: ruma::events::push_rules::PushRulesEventContent {
-		})
+					global: push::Ruleset::server_default(&user_id),
-		.expect("to json always works"),
+				},
-	)?;
+			})
+			.expect("to json always works"),
+		)
+		.await?;
 
 	// Inhibit login does not work for guests
 	if !is_guest && body.inhibit_login {
@@ -286,73 +330,95 @@ pub(crate) async fn register_route(
 	let token = utils::random_string(TOKEN_LENGTH);
 
 	// Create device for this account
-	services.users.create_device(
+	services
-		&user_id,
+		.users
-		&device_id,
+		.create_device(
-		&token,
+			&user_id,
-		body.initial_device_display_name.clone(),
+			&device_id,
-		Some(client.to_string()),
+			&token,
-	)?;
+			body.initial_device_display_name.clone(),
+			Some(client.to_string()),
+		)
+		.await?;
 
 	debug_info!(%user_id, %device_id, "User account was created");
 
+	let device_display_name = body.initial_device_display_name.as_deref().unwrap_or("");
+
 	// log in conduit admin channel if a non-guest user registered
 	if body.appservice_info.is_none() && !is_guest {
-		info!("New user \"{user_id}\" registered on this server.");
+		if !device_display_name.is_empty() {
-		services
+			info!(
-			.admin
+				"New user \"{user_id}\" registered on this server with device display name: \"{device_display_name}\""
-			.send_message(RoomMessageEventContent::notice_plain(format!(
+			);
-				"New user \"{user_id}\" registered on this server from IP {client}."
+
-			)))
+			if services.globals.config.admin_room_notices {
-			.await;
+				services
+					.admin
+					.send_message(RoomMessageEventContent::notice_plain(format!(
+						"New user \"{user_id}\" registered on this server from IP {client} and device display name \
+						 \"{device_display_name}\""
+					)))
+					.await
+					.ok();
+			}
+		} else {
+			info!("New user \"{user_id}\" registered on this server.");
+
+			if services.globals.config.admin_room_notices {
+				services
+					.admin
+					.send_message(RoomMessageEventContent::notice_plain(format!(
+						"New user \"{user_id}\" registered on this server from IP {client}"
+					)))
+					.await
+					.ok();
+			}
+		}
 	}
 
 	// log in conduit admin channel if a guest registered
 	if body.appservice_info.is_none() && is_guest && services.globals.log_guest_registrations() {
 		info!("New guest user \"{user_id}\" registered on this server.");
 
-		if let Some(device_display_name) = &body.initial_device_display_name {
+		if !device_display_name.is_empty() {
-			if body
+			if services.globals.config.admin_room_notices {
-				.initial_device_display_name
-				.as_ref()
-				.is_some_and(|device_display_name| !device_display_name.is_empty())
-			{
 				services
 					.admin
 					.send_message(RoomMessageEventContent::notice_plain(format!(
-						"Guest user \"{user_id}\" with device display name `{device_display_name}` registered on this \
+						"Guest user \"{user_id}\" with device display name \"{device_display_name}\" registered on \
-						 server from IP {client}."
+						 this server from IP {client}"
 					)))
-					.await;
+					.await
-			} else {
+					.ok();
+			}
+		} else {
+			#[allow(clippy::collapsible_else_if)]
+			if services.globals.config.admin_room_notices {
 				services
 					.admin
 					.send_message(RoomMessageEventContent::notice_plain(format!(
 						"Guest user \"{user_id}\" with no device display name registered on this server from IP \
-						 {client}.",
+						 {client}",
 					)))
-					.await;
+					.await
+					.ok();
 			}
-		} else {
-			services
-				.admin
-				.send_message(RoomMessageEventContent::notice_plain(format!(
-					"Guest user \"{user_id}\" with no device display name registered on this server from IP {client}.",
-				)))
-				.await;
 		}
 	}
 
 	// If this is the first real user, grant them admin privileges except for guest
 	// users Note: the server user, @conduit:servername, is generated first
 	if !is_guest {
-		if let Some(admin_room) = services.admin.get_admin_room()? {
+		if let Ok(admin_room) = services.admin.get_admin_room().await {
-			if services.rooms.state_cache.room_joined_count(&admin_room)? == Some(1) {
+			if services
-				services
+				.rooms
-					.admin
+				.state_cache
-					.make_user_admin(&user_id, displayname)
+				.room_joined_count(&admin_room)
-					.await?;
+				.await
-
+				.is_ok_and(is_equal_to!(1))
+			{
+				services.admin.make_user_admin(&user_id).await?;
 				warn!("Granting {user_id} admin privileges as the first user");
 			}
 		}
@@ -363,24 +429,32 @@ pub(crate) async fn register_route(
 		&& (services.globals.allow_guests_auto_join_rooms() || !is_guest)
 	{
 		for room in &services.globals.config.auto_join_rooms {
+			let Ok(room_id) = services.rooms.alias.resolve(room).await else {
+				error!("Failed to resolve room alias to room ID when attempting to auto join {room}, skipping");
+				continue;
+			};
+
 			if !services
 				.rooms
 				.state_cache
-				.server_in_room(services.globals.server_name(), room)?
+				.server_in_room(services.globals.server_name(), &room_id)
+				.await
 			{
 				warn!("Skipping room {room} to automatically join as we have never joined before.");
 				continue;
 			}
 
-			if let Some(room_id_server_name) = room.server_name() {
+			if let Some(room_server_name) = room.server_name() {
 				if let Err(e) = join_room_by_id_helper(
 					&services,
 					&user_id,
-					room,
+					&room_id,
 					Some("Automatically joining this room upon registration".to_owned()),
-					&[room_id_server_name.to_owned(), services.globals.server_name().to_owned()],
+					&[services.globals.server_name().to_owned(), room_server_name.to_owned()],
 					None,
+					&body.appservice_info,
 				)
+				.boxed()
 				.await
 				{
 					// don't return this error so we don't fail registrations
@@ -444,16 +518,20 @@ pub(crate) async fn change_password_route(
 	if let Some(auth) = &body.auth {
 		let (worked, uiaainfo) = services
 			.uiaa
-			.try_auth(sender_user, sender_device, auth, &uiaainfo)?;
+			.try_auth(sender_user, sender_device, auth, &uiaainfo)
+			.await?;
+
 		if !worked {
 			return Err(Error::Uiaa(uiaainfo));
 		}
-	// Success!
+
+		// Success!
 	} else if let Some(json) = body.json_body {
 		uiaainfo.session = Some(utils::random_string(SESSION_ID_LENGTH));
 		services
 			.uiaa
-			.create(sender_user, sender_device, &uiaainfo, &json)?;
+			.create(sender_user, sender_device, &uiaainfo, &json);
+
 		return Err(Error::Uiaa(uiaainfo));
 	} else {
 		return Err(Error::BadRequest(ErrorKind::NotJson, "Not json."));
@@ -465,23 +543,25 @@ pub(crate) async fn change_password_route(
 
 	if body.logout_devices {
 		// Logout all devices except the current one
-		for id in services
+		services
 			.users
 			.all_device_ids(sender_user)
-			.filter_map(Result::ok)
+			.ready_filter(|id| id != sender_device)
-			.filter(|id| id != sender_device)
+			.for_each(|id| services.users.remove_device(sender_user, id))
-		{
+			.await;
-			services.users.remove_device(sender_user, &id)?;
-		}
 	}
 
 	info!("User {sender_user} changed their password.");
-	services
+
-		.admin
+	if services.globals.config.admin_room_notices {
-		.send_message(RoomMessageEventContent::notice_plain(format!(
+		services
-			"User {sender_user} changed their password."
+			.admin
-		)))
+			.send_message(RoomMessageEventContent::notice_plain(format!(
-		.await;
+				"User {sender_user} changed their password."
+			)))
+			.await
+			.ok();
+	}
 
 	Ok(change_password::v3::Response {})
 }
@@ -500,7 +580,7 @@ pub(crate) async fn whoami_route(
 	Ok(whoami::v3::Response {
 		user_id: sender_user.clone(),
 		device_id,
-		is_guest: services.users.is_deactivated(sender_user)? && body.appservice_info.is_none(),
+		is_guest: services.users.is_deactivated(sender_user).await? && body.appservice_info.is_none(),
 	})
 }
 
@@ -541,7 +621,9 @@ pub(crate) async fn deactivate_route(
 	if let Some(auth) = &body.auth {
 		let (worked, uiaainfo) = services
 			.uiaa
-			.try_auth(sender_user, sender_device, auth, &uiaainfo)?;
+			.try_auth(sender_user, sender_device, auth, &uiaainfo)
+			.await?;
+
 		if !worked {
 			return Err(Error::Uiaa(uiaainfo));
 		}
@@ -550,35 +632,38 @@ pub(crate) async fn deactivate_route(
 		uiaainfo.session = Some(utils::random_string(SESSION_ID_LENGTH));
 		services
 			.uiaa
-			.create(sender_user, sender_device, &uiaainfo, &json)?;
+			.create(sender_user, sender_device, &uiaainfo, &json);
+
 		return Err(Error::Uiaa(uiaainfo));
 	} else {
 		return Err(Error::BadRequest(ErrorKind::NotJson, "Not json."));
 	}
 
-	// Remove devices and mark account as deactivated
-	services.users.deactivate_account(sender_user)?;
-
 	// Remove profile pictures and display name
 	let all_joined_rooms: Vec<OwnedRoomId> = services
 		.rooms
 		.state_cache
 		.rooms_joined(sender_user)
-		.filter_map(Result::ok)
+		.map(Into::into)
-		.collect();
+		.collect()
-	super::update_displayname(&services, sender_user.clone(), None, all_joined_rooms.clone()).await?;
+		.await;
-	super::update_avatar_url(&services, sender_user.clone(), None, None, all_joined_rooms).await?;
 
-	// Make the user leave all rooms before deactivation
+	super::update_displayname(&services, sender_user, None, &all_joined_rooms).await?;
-	super::leave_all_rooms(&services, sender_user).await;
+	super::update_avatar_url(&services, sender_user, None, None, &all_joined_rooms).await?;
+
+	full_user_deactivate(&services, sender_user, &all_joined_rooms).await?;
 
 	info!("User {sender_user} deactivated their account.");
-	services
+
-		.admin
+	if services.globals.config.admin_room_notices {
-		.send_message(RoomMessageEventContent::notice_plain(format!(
+		services
-			"User {sender_user} deactivated their account."
+			.admin
-		)))
+			.send_message(RoomMessageEventContent::notice_plain(format!(
-		.await;
+				"User {sender_user} deactivated their account."
+			)))
+			.await
+			.ok();
+	}
 
 	Ok(deactivate::v3::Response {
 		id_server_unbind_result: ThirdPartyIdRemovalStatus::NoSupport,
@@ -637,7 +722,7 @@ pub(crate) async fn request_3pid_management_token_via_msisdn_route(
 pub(crate) async fn check_registration_token_validity(
 	State(services): State<crate::State>, body: Ruma<check_registration_token_validity::v1::Request>,
 ) -> Result<check_registration_token_validity::v1::Response> {
-	let Some(reg_token) = services.globals.config.registration_token.clone() else {
+	let Some(reg_token) = services.globals.registration_token.clone() else {
 		return Err(Error::BadRequest(
 			ErrorKind::forbidden(),
 			"Server does not allow token registration.",
@@ -648,3 +733,72 @@ pub(crate) async fn check_registration_token_validity(
 		valid: reg_token == body.token,
 	})
 }
+
+/// Runs through all the deactivation steps:
+///
+/// - Mark as deactivated
+/// - Removing display name
+/// - Removing avatar URL and blurhash
+/// - Removing all profile data
+/// - Leaving all rooms (and forgets all of them)
+pub async fn full_user_deactivate(
+	services: &Services, user_id: &UserId, all_joined_rooms: &[OwnedRoomId],
+) -> Result<()> {
+	services.users.deactivate_account(user_id).await?;
+	super::update_displayname(services, user_id, None, all_joined_rooms).await?;
+	super::update_avatar_url(services, user_id, None, None, all_joined_rooms).await?;
+
+	services
+		.users
+		.all_profile_keys(user_id)
+		.ready_for_each(|(profile_key, _)| services.users.set_profile_key(user_id, &profile_key, None))
+		.await;
+
+	for room_id in all_joined_rooms {
+		let state_lock = services.rooms.state.mutex.lock(room_id).await;
+
+		let room_power_levels = services
+			.rooms
+			.state_accessor
+			.room_state_get_content::<RoomPowerLevelsEventContent>(room_id, &StateEventType::RoomPowerLevels, "")
+			.await
+			.ok();
+
+		let user_can_demote_self = room_power_levels
+			.as_ref()
+			.is_some_and(|power_levels_content| {
+				RoomPowerLevels::from(power_levels_content.clone()).user_can_change_user_power_level(user_id, user_id)
+			}) || services
+			.rooms
+			.state_accessor
+			.room_state_get(room_id, &StateEventType::RoomCreate, "")
+			.await
+			.is_ok_and(|event| event.sender == user_id);
+
+		if user_can_demote_self {
+			let mut power_levels_content = room_power_levels.unwrap_or_default();
+			power_levels_content.users.remove(user_id);
+
+			// ignore errors so deactivation doesn't fail
+			if let Err(e) = services
+				.rooms
+				.timeline
+				.build_and_append_pdu(
+					PduBuilder::state(String::new(), &power_levels_content),
+					user_id,
+					room_id,
+					&state_lock,
+				)
+				.await
+			{
+				warn!(%room_id, %user_id, "Failed to demote user's own power level: {e}");
+			} else {
+				info!("Demoted {user_id} in {room_id} as part of account deactivation");
+			}
+		}
+	}
+
+	super::leave_all_rooms(services, user_id).await;
+
+	Ok(())
+}

src/api/client/alias.rs

@@ -1,11 +1,9 @@
 use axum::extract::State;
-use conduit::{debug, Error, Result};
+use conduit::{debug, Err, Result};
+use futures::StreamExt;
 use rand::seq::SliceRandom;
 use ruma::{
-	api::client::{
+	api::client::alias::{create_alias, delete_alias, get_alias},
-		alias::{create_alias, delete_alias, get_alias},
-		error::ErrorKind,
-	},
 	OwnedServerName, RoomAliasId, RoomId,
 };
 use service::Services;
@@ -33,16 +31,17 @@ pub(crate) async fn create_alias_route(
 		.forbidden_alias_names()
 		.is_match(body.room_alias.alias())
 	{
-		return Err(Error::BadRequest(ErrorKind::forbidden(), "Room alias is forbidden."));
+		return Err!(Request(Forbidden("Room alias is forbidden.")));
 	}
 
 	if services
 		.rooms
 		.alias
-		.resolve_local_alias(&body.room_alias)?
+		.resolve_local_alias(&body.room_alias)
-		.is_some()
+		.await
+		.is_ok()
 	{
-		return Err(Error::Conflict("Alias already exists."));
+		return Err!(Conflict("Alias already exists."));
 	}
 
 	services
@@ -87,39 +86,32 @@ pub(crate) async fn get_alias_route(
 	State(services): State<crate::State>, body: Ruma<get_alias::v3::Request>,
 ) -> Result<get_alias::v3::Response> {
 	let room_alias = body.body.room_alias;
-	let servers = None;
 
-	let Ok((room_id, pre_servers)) = services
+	let Ok((room_id, servers)) = services.rooms.alias.resolve_alias(&room_alias, None).await else {
-		.rooms
+		return Err!(Request(NotFound("Room with alias not found.")));
-		.alias
-		.resolve_alias(&room_alias, servers.as_ref())
-		.await
-	else {
-		return Err(Error::BadRequest(ErrorKind::NotFound, "Room with alias not found."));
 	};
 
-	let servers = room_available_servers(&services, &room_id, &room_alias, &pre_servers);
+	let servers = room_available_servers(&services, &room_id, &room_alias, servers).await;
 	debug!(?room_alias, ?room_id, "available servers: {servers:?}");
 
 	Ok(get_alias::v3::Response::new(room_id, servers))
 }
 
-fn room_available_servers(
+async fn room_available_servers(
-	services: &Services, room_id: &RoomId, room_alias: &RoomAliasId, pre_servers: &Option<Vec<OwnedServerName>>,
+	services: &Services, room_id: &RoomId, room_alias: &RoomAliasId, pre_servers: Vec<OwnedServerName>,
 ) -> Vec<OwnedServerName> {
 	// find active servers in room state cache to suggest
 	let mut servers: Vec<OwnedServerName> = services
 		.rooms
 		.state_cache
 		.room_servers(room_id)
-		.filter_map(Result::ok)
+		.map(ToOwned::to_owned)
-		.collect();
+		.collect()
+		.await;
 
 	// push any servers we want in the list already (e.g. responded remote alias
 	// servers, room alias server itself)
-	if let Some(pre_servers) = pre_servers {
+	servers.extend(pre_servers);
-		servers.extend(pre_servers.clone());
-	};
 
 	servers.sort_unstable();
 	servers.dedup();

src/api/client/appservice.rs

@@ -0,0 +1,47 @@
+use axum::extract::State;
+use conduit::{err, Err, Result};
+use ruma::api::{appservice::ping, client::appservice::request_ping};
+
+use crate::Ruma;
+
+/// # `POST /_matrix/client/v1/appservice/{appserviceId}/ping`
+///
+/// Ask the homeserver to ping the application service to ensure the connection
+/// works.
+pub(crate) async fn appservice_ping(
+	State(services): State<crate::State>, body: Ruma<request_ping::v1::Request>,
+) -> Result<request_ping::v1::Response> {
+	let appservice_info = body
+		.appservice_info
+		.as_ref()
+		.ok_or_else(|| err!(Request(Forbidden("This endpoint can only be called by appservices."))))?;
+
+	if body.appservice_id != appservice_info.registration.id {
+		return Err!(Request(Forbidden(
+			"Appservices can only ping themselves (wrong appservice ID)."
+		)));
+	}
+
+	if appservice_info.registration.url.is_none() {
+		return Err!(Request(UrlNotSet(
+			"Appservice does not have a URL set, there is nothing to ping."
+		)));
+	}
+
+	let timer = tokio::time::Instant::now();
+
+	let _response = services
+		.sending
+		.send_appservice_request(
+			appservice_info.registration.clone(),
+			ping::send_ping::v1::Request {
+				transaction_id: body.transaction_id.clone(),
+			},
+		)
+		.await?
+		.expect("We already validated if an appservice URL exists above");
+
+	Ok(request_ping::v1::Response {
+		duration: timer.elapsed(),
+	})
+}

src/api/client/backup.rs

@@ -1,18 +1,16 @@
 use axum::extract::State;
+use conduit::{err, Err};
 use ruma::{
-	api::client::{
+	api::client::backup::{
-		backup::{
+		add_backup_keys, add_backup_keys_for_room, add_backup_keys_for_session, create_backup_version,
-			add_backup_keys, add_backup_keys_for_room, add_backup_keys_for_session, create_backup_version,
+		delete_backup_keys, delete_backup_keys_for_room, delete_backup_keys_for_session, delete_backup_version,
-			delete_backup_keys, delete_backup_keys_for_room, delete_backup_keys_for_session, delete_backup_version,
+		get_backup_info, get_backup_keys, get_backup_keys_for_room, get_backup_keys_for_session,
-			get_backup_info, get_backup_keys, get_backup_keys_for_room, get_backup_keys_for_session,
+		get_latest_backup_info, update_backup_version,
-			get_latest_backup_info, update_backup_version,
-		},
-		error::ErrorKind,
 	},
 	UInt,
 };
 
-use crate::{Error, Result, Ruma};
+use crate::{Result, Ruma};
 
 /// # `POST /_matrix/client/r0/room_keys/version`
 ///
@@ -20,10 +18,9 @@ use crate::{Error, Result, Ruma};
 pub(crate) async fn create_backup_version_route(
 	State(services): State<crate::State>, body: Ruma<create_backup_version::v3::Request>,
 ) -> Result<create_backup_version::v3::Response> {
-	let sender_user = body.sender_user.as_ref().expect("user is authenticated");
 	let version = services
 		.key_backups
-		.create_backup(sender_user, &body.algorithm)?;
+		.create_backup(body.sender_user(), &body.algorithm)?;
 
 	Ok(create_backup_version::v3::Response {
 		version,
@@ -37,10 +34,10 @@ pub(crate) async fn create_backup_version_route(
 pub(crate) async fn update_backup_version_route(
 	State(services): State<crate::State>, body: Ruma<update_backup_version::v3::Request>,
 ) -> Result<update_backup_version::v3::Response> {
-	let sender_user = body.sender_user.as_ref().expect("user is authenticated");
 	services
 		.key_backups
-		.update_backup(sender_user, &body.version, &body.algorithm)?;
+		.update_backup(body.sender_user(), &body.version, &body.algorithm)
+		.await?;
 
 	Ok(update_backup_version::v3::Response {})
 }
@@ -51,18 +48,25 @@ pub(crate) async fn update_backup_version_route(
 pub(crate) async fn get_latest_backup_info_route(
 	State(services): State<crate::State>, body: Ruma<get_latest_backup_info::v3::Request>,
 ) -> Result<get_latest_backup_info::v3::Response> {
-	let sender_user = body.sender_user.as_ref().expect("user is authenticated");
-
 	let (version, algorithm) = services
 		.key_backups
-		.get_latest_backup(sender_user)?
+		.get_latest_backup(body.sender_user())
-		.ok_or_else(|| Error::BadRequest(ErrorKind::NotFound, "Key backup does not exist."))?;
+		.await
+		.map_err(|_| err!(Request(NotFound("Key backup does not exist."))))?;
 
 	Ok(get_latest_backup_info::v3::Response {
 		algorithm,
-		count: (UInt::try_from(services.key_backups.count_keys(sender_user, &version)?)
+		count: (UInt::try_from(
-			.expect("user backup keys count should not be that high")),
+			services
-		etag: services.key_backups.get_etag(sender_user, &version)?,
+				.key_backups
+				.count_keys(body.sender_user(), &version)
+				.await,
+		)
+		.expect("user backup keys count should not be that high")),
+		etag: services
+			.key_backups
+			.get_etag(body.sender_user(), &version)
+			.await,
 		version,
 	})
 }
@@ -73,21 +77,23 @@ pub(crate) async fn get_latest_backup_info_route(
 pub(crate) async fn get_backup_info_route(
 	State(services): State<crate::State>, body: Ruma<get_backup_info::v3::Request>,
 ) -> Result<get_backup_info::v3::Response> {
-	let sender_user = body.sender_user.as_ref().expect("user is authenticated");
 	let algorithm = services
 		.key_backups
-		.get_backup(sender_user, &body.version)?
+		.get_backup(body.sender_user(), &body.version)
-		.ok_or_else(|| Error::BadRequest(ErrorKind::NotFound, "Key backup does not exist."))?;
+		.await
+		.map_err(|_| err!(Request(NotFound("Key backup does not exist at version {:?}", body.version))))?;
 
 	Ok(get_backup_info::v3::Response {
 		algorithm,
-		count: (UInt::try_from(
+		count: services
-			services
+			.key_backups
-				.key_backups
+			.count_keys(body.sender_user(), &body.version)
-				.count_keys(sender_user, &body.version)?,
+			.await
-		)
+			.try_into()?,
-		.expect("user backup keys count should not be that high")),
+		etag: services
-		etag: services.key_backups.get_etag(sender_user, &body.version)?,
+			.key_backups
+			.get_etag(body.sender_user(), &body.version)
+			.await,
 		version: body.version.clone(),
 	})
 }
@@ -101,11 +107,10 @@ pub(crate) async fn get_backup_info_route(
 pub(crate) async fn delete_backup_version_route(
 	State(services): State<crate::State>, body: Ruma<delete_backup_version::v3::Request>,
 ) -> Result<delete_backup_version::v3::Response> {
-	let sender_user = body.sender_user.as_ref().expect("user is authenticated");
-
 	services
 		.key_backups
-		.delete_backup(sender_user, &body.version)?;
+		.delete_backup(body.sender_user(), &body.version)
+		.await;
 
 	Ok(delete_backup_version::v3::Response {})
 }
@@ -121,36 +126,36 @@ pub(crate) async fn delete_backup_version_route(
 pub(crate) async fn add_backup_keys_route(
 	State(services): State<crate::State>, body: Ruma<add_backup_keys::v3::Request>,
 ) -> Result<add_backup_keys::v3::Response> {
-	let sender_user = body.sender_user.as_ref().expect("user is authenticated");
+	if services
-
+		.key_backups
-	if Some(&body.version)
+		.get_latest_backup_version(body.sender_user())
-		!= services
+		.await
-			.key_backups
+		.is_ok_and(|version| version != body.version)
-			.get_latest_backup_version(sender_user)?
-			.as_ref()
 	{
-		return Err(Error::BadRequest(
+		return Err!(Request(InvalidParam(
-			ErrorKind::InvalidParam,
+			"You may only manipulate the most recently created version of the backup."
-			"You may only manipulate the most recently created version of the backup.",
+		)));
-		));
 	}
 
 	for (room_id, room) in &body.rooms {
 		for (session_id, key_data) in &room.sessions {
 			services
 				.key_backups
-				.add_key(sender_user, &body.version, room_id, session_id, key_data)?;
+				.add_key(body.sender_user(), &body.version, room_id, session_id, key_data)
+				.await?;
 		}
 	}
 
 	Ok(add_backup_keys::v3::Response {
-		count: (UInt::try_from(
+		count: services
-			services
+			.key_backups
-				.key_backups
+			.count_keys(body.sender_user(), &body.version)
-				.count_keys(sender_user, &body.version)?,
+			.await
-		)
+			.try_into()?,
-		.expect("user backup keys count should not be that high")),
+		etag: services
-		etag: services.key_backups.get_etag(sender_user, &body.version)?,
+			.key_backups
+			.get_etag(body.sender_user(), &body.version)
+			.await,
 	})
 }
 
@@ -165,34 +170,34 @@ pub(crate) async fn add_backup_keys_route(
 pub(crate) async fn add_backup_keys_for_room_route(
 	State(services): State<crate::State>, body: Ruma<add_backup_keys_for_room::v3::Request>,
 ) -> Result<add_backup_keys_for_room::v3::Response> {
-	let sender_user = body.sender_user.as_ref().expect("user is authenticated");
+	if services
-
+		.key_backups
-	if Some(&body.version)
+		.get_latest_backup_version(body.sender_user())
-		!= services
+		.await
-			.key_backups
+		.is_ok_and(|version| version != body.version)
-			.get_latest_backup_version(sender_user)?
-			.as_ref()
 	{
-		return Err(Error::BadRequest(
+		return Err!(Request(InvalidParam(
-			ErrorKind::InvalidParam,
+			"You may only manipulate the most recently created version of the backup."
-			"You may only manipulate the most recently created version of the backup.",
+		)));
-		));
 	}
 
 	for (session_id, key_data) in &body.sessions {
 		services
 			.key_backups
-			.add_key(sender_user, &body.version, &body.room_id, session_id, key_data)?;
+			.add_key(body.sender_user(), &body.version, &body.room_id, session_id, key_data)
+			.await?;
 	}
 
 	Ok(add_backup_keys_for_room::v3::Response {
-		count: (UInt::try_from(
+		count: services
-			services
+			.key_backups
-				.key_backups
+			.count_keys(body.sender_user(), &body.version)
-				.count_keys(sender_user, &body.version)?,
+			.await
-		)
+			.try_into()?,
-		.expect("user backup keys count should not be that high")),
+		etag: services
-		etag: services.key_backups.get_etag(sender_user, &body.version)?,
+			.key_backups
+			.get_etag(body.sender_user(), &body.version)
+			.await,
 	})
 }
 
@@ -207,32 +212,38 @@ pub(crate) async fn add_backup_keys_for_room_route(
 pub(crate) async fn add_backup_keys_for_session_route(
 	State(services): State<crate::State>, body: Ruma<add_backup_keys_for_session::v3::Request>,
 ) -> Result<add_backup_keys_for_session::v3::Response> {
-	let sender_user = body.sender_user.as_ref().expect("user is authenticated");
+	if services
-
+		.key_backups
-	if Some(&body.version)
+		.get_latest_backup_version(body.sender_user())
-		!= services
+		.await
-			.key_backups
+		.is_ok_and(|version| version != body.version)
-			.get_latest_backup_version(sender_user)?
-			.as_ref()
 	{
-		return Err(Error::BadRequest(
+		return Err!(Request(InvalidParam(
-			ErrorKind::InvalidParam,
+			"You may only manipulate the most recently created version of the backup."
-			"You may only manipulate the most recently created version of the backup.",
+		)));
-		));
 	}
 
 	services
 		.key_backups
-		.add_key(sender_user, &body.version, &body.room_id, &body.session_id, &body.session_data)?;
+		.add_key(
+			body.sender_user(),
+			&body.version,
+			&body.room_id,
+			&body.session_id,
+			&body.session_data,
+		)
+		.await?;
 
 	Ok(add_backup_keys_for_session::v3::Response {
-		count: (UInt::try_from(
+		count: services
-			services
+			.key_backups
-				.key_backups
+			.count_keys(body.sender_user(), &body.version)
-				.count_keys(sender_user, &body.version)?,
+			.await
-		)
+			.try_into()?,
-		.expect("user backup keys count should not be that high")),
+		etag: services
-		etag: services.key_backups.get_etag(sender_user, &body.version)?,
+			.key_backups
+			.get_etag(body.sender_user(), &body.version)
+			.await,
 	})
 }
 
@@ -242,9 +253,10 @@ pub(crate) async fn add_backup_keys_for_session_route(
 pub(crate) async fn get_backup_keys_route(
 	State(services): State<crate::State>, body: Ruma<get_backup_keys::v3::Request>,
 ) -> Result<get_backup_keys::v3::Response> {
-	let sender_user = body.sender_user.as_ref().expect("user is authenticated");
+	let rooms = services
-
+		.key_backups
-	let rooms = services.key_backups.get_all(sender_user, &body.version)?;
+		.get_all(body.sender_user(), &body.version)
+		.await;
 
 	Ok(get_backup_keys::v3::Response {
 		rooms,
@@ -257,11 +269,10 @@ pub(crate) async fn get_backup_keys_route(
 pub(crate) async fn get_backup_keys_for_room_route(
 	State(services): State<crate::State>, body: Ruma<get_backup_keys_for_room::v3::Request>,
 ) -> Result<get_backup_keys_for_room::v3::Response> {
-	let sender_user = body.sender_user.as_ref().expect("user is authenticated");
-
 	let sessions = services
 		.key_backups
-		.get_room(sender_user, &body.version, &body.room_id)?;
+		.get_room(body.sender_user(), &body.version, &body.room_id)
+		.await;
 
 	Ok(get_backup_keys_for_room::v3::Response {
 		sessions,
@@ -274,12 +285,11 @@ pub(crate) async fn get_backup_keys_for_room_route(
 pub(crate) async fn get_backup_keys_for_session_route(
 	State(services): State<crate::State>, body: Ruma<get_backup_keys_for_session::v3::Request>,
 ) -> Result<get_backup_keys_for_session::v3::Response> {
-	let sender_user = body.sender_user.as_ref().expect("user is authenticated");
-
 	let key_data = services
 		.key_backups
-		.get_session(sender_user, &body.version, &body.room_id, &body.session_id)?
+		.get_session(body.sender_user(), &body.version, &body.room_id, &body.session_id)
-		.ok_or_else(|| Error::BadRequest(ErrorKind::NotFound, "Backup key not found for this user's session."))?;
+		.await
+		.map_err(|_| err!(Request(NotFound(debug_error!("Backup key not found for this user's session.")))))?;
 
 	Ok(get_backup_keys_for_session::v3::Response {
 		key_data,
@@ -292,20 +302,21 @@ pub(crate) async fn get_backup_keys_for_session_route(
 pub(crate) async fn delete_backup_keys_route(
 	State(services): State<crate::State>, body: Ruma<delete_backup_keys::v3::Request>,
 ) -> Result<delete_backup_keys::v3::Response> {
-	let sender_user = body.sender_user.as_ref().expect("user is authenticated");
-
 	services
 		.key_backups
-		.delete_all_keys(sender_user, &body.version)?;
+		.delete_all_keys(body.sender_user(), &body.version)
+		.await;
 
 	Ok(delete_backup_keys::v3::Response {
-		count: (UInt::try_from(
+		count: services
-			services
+			.key_backups
-				.key_backups
+			.count_keys(body.sender_user(), &body.version)
-				.count_keys(sender_user, &body.version)?,
+			.await
-		)
+			.try_into()?,
-		.expect("user backup keys count should not be that high")),
+		etag: services
-		etag: services.key_backups.get_etag(sender_user, &body.version)?,
+			.key_backups
+			.get_etag(body.sender_user(), &body.version)
+			.await,
 	})
 }
 
@@ -315,20 +326,21 @@ pub(crate) async fn delete_backup_keys_route(
 pub(crate) async fn delete_backup_keys_for_room_route(
 	State(services): State<crate::State>, body: Ruma<delete_backup_keys_for_room::v3::Request>,
 ) -> Result<delete_backup_keys_for_room::v3::Response> {
-	let sender_user = body.sender_user.as_ref().expect("user is authenticated");
-
 	services
 		.key_backups
-		.delete_room_keys(sender_user, &body.version, &body.room_id)?;
+		.delete_room_keys(body.sender_user(), &body.version, &body.room_id)
+		.await;
 
 	Ok(delete_backup_keys_for_room::v3::Response {
-		count: (UInt::try_from(
+		count: services
-			services
+			.key_backups
-				.key_backups
+			.count_keys(body.sender_user(), &body.version)
-				.count_keys(sender_user, &body.version)?,
+			.await
-		)
+			.try_into()?,
-		.expect("user backup keys count should not be that high")),
+		etag: services
-		etag: services.key_backups.get_etag(sender_user, &body.version)?,
+			.key_backups
+			.get_etag(body.sender_user(), &body.version)
+			.await,
 	})
 }
 
@@ -338,19 +350,20 @@ pub(crate) async fn delete_backup_keys_for_room_route(
 pub(crate) async fn delete_backup_keys_for_session_route(
 	State(services): State<crate::State>, body: Ruma<delete_backup_keys_for_session::v3::Request>,
 ) -> Result<delete_backup_keys_for_session::v3::Response> {
-	let sender_user = body.sender_user.as_ref().expect("user is authenticated");
-
 	services
 		.key_backups
-		.delete_room_key(sender_user, &body.version, &body.room_id, &body.session_id)?;
+		.delete_room_key(body.sender_user(), &body.version, &body.room_id, &body.session_id)
+		.await;
 
 	Ok(delete_backup_keys_for_session::v3::Response {
-		count: (UInt::try_from(
+		count: services
-			services
+			.key_backups
-				.key_backups
+			.count_keys(body.sender_user(), &body.version)
-				.count_keys(sender_user, &body.version)?,
+			.await
-		)
+			.try_into()?,
-		.expect("user backup keys count should not be that high")),
+		etag: services
-		etag: services.key_backups.get_etag(sender_user, &body.version)?,
+			.key_backups
+			.get_etag(body.sender_user(), &body.version)
+			.await,
 	})
 }

src/api/client/capabilities.rs

@@ -1,9 +1,14 @@
 use std::collections::BTreeMap;
 
 use axum::extract::State;
-use ruma::api::client::discovery::get_capabilities::{
+use ruma::{
-	self, Capabilities, RoomVersionStability, RoomVersionsCapability, ThirdPartyIdChangesCapability,
+	api::client::discovery::get_capabilities::{
+		self, Capabilities, GetLoginTokenCapability, RoomVersionStability, RoomVersionsCapability,
+		ThirdPartyIdChangesCapability,
+	},
+	RoomVersionId,
 };
+use serde_json::json;
 
 use crate::{Result, Ruma};
 
@@ -14,13 +19,19 @@ use crate::{Result, Ruma};
 pub(crate) async fn get_capabilities_route(
 	State(services): State<crate::State>, _body: Ruma<get_capabilities::v3::Request>,
 ) -> Result<get_capabilities::v3::Response> {
-	let mut available = BTreeMap::new();
+	let available: BTreeMap<RoomVersionId, RoomVersionStability> = services
-	for room_version in &services.globals.unstable_room_versions {
+		.globals
-		available.insert(room_version.clone(), RoomVersionStability::Unstable);
+		.unstable_room_versions
-	}
+		.iter()
-	for room_version in &services.globals.stable_room_versions {
+		.map(|unstable_room_version| (unstable_room_version.clone(), RoomVersionStability::Unstable))
-		available.insert(room_version.clone(), RoomVersionStability::Stable);
+		.chain(
-	}
+			services
+				.globals
+				.stable_room_versions
+				.iter()
+				.map(|stable_room_version| (stable_room_version.clone(), RoomVersionStability::Stable)),
+		)
+		.collect();
 
 	let mut capabilities = Capabilities::default();
 	capabilities.room_versions = RoomVersionsCapability {
@@ -28,11 +39,21 @@ pub(crate) async fn get_capabilities_route(
 		available,
 	};
 
-	// conduit does not implement 3PID stuff
+	// we do not implement 3PID stuff
 	capabilities.thirdparty_id_changes = ThirdPartyIdChangesCapability {
 		enabled: false,
 	};
 
+	// we dont support generating tokens yet
+	capabilities.get_login_token = GetLoginTokenCapability {
+		enabled: false,
+	};
+
+	// MSC4133 capability
+	capabilities
+		.set("uk.tcpip.msc4133.profile_fields", json!({"enabled": true}))
+		.expect("this is valid JSON we created");
+
 	Ok(get_capabilities::v3::Response {
 		capabilities,
 	})

src/api/client/config.rs

@@ -1,4 +1,5 @@
 use axum::extract::State;
+use conduit::err;
 use ruma::{
 	api::client::{
 		config::{get_global_account_data, get_room_account_data, set_global_account_data, set_room_account_data},
@@ -22,10 +23,11 @@ pub(crate) async fn set_global_account_data_route(
 	set_account_data(
 		&services,
 		None,
-		&body.sender_user,
+		body.sender_user.as_ref(),
 		&body.event_type.to_string(),
 		body.data.json(),
-	)?;
+	)
+	.await?;
 
 	Ok(set_global_account_data::v3::Response {})
 }
@@ -39,10 +41,11 @@ pub(crate) async fn set_room_account_data_route(
 	set_account_data(
 		&services,
 		Some(&body.room_id),
-		&body.sender_user,
+		body.sender_user.as_ref(),
 		&body.event_type.to_string(),
 		body.data.json(),
-	)?;
+	)
+	.await?;
 
 	Ok(set_room_account_data::v3::Response {})
 }
@@ -55,17 +58,14 @@ pub(crate) async fn get_global_account_data_route(
 ) -> Result<get_global_account_data::v3::Response> {
 	let sender_user = body.sender_user.as_ref().expect("user is authenticated");
 
-	let event: Box<RawJsonValue> = services
+	let account_data: ExtractGlobalEventContent = services
 		.account_data
-		.get(None, sender_user, body.event_type.to_string().into())?
+		.get_global(sender_user, body.event_type.clone())
-		.ok_or_else(|| Error::BadRequest(ErrorKind::NotFound, "Data not found."))?;
+		.await
-
+		.map_err(|_| err!(Request(NotFound("Data not found."))))?;
-	let account_data = serde_json::from_str::<ExtractGlobalEventContent>(event.get())
-		.map_err(|_| Error::bad_database("Invalid account data event in db."))?
-		.content;
 
 	Ok(get_global_account_data::v3::Response {
-		account_data,
+		account_data: account_data.content,
 	})
 }
 
@@ -77,22 +77,19 @@ pub(crate) async fn get_room_account_data_route(
 ) -> Result<get_room_account_data::v3::Response> {
 	let sender_user = body.sender_user.as_ref().expect("user is authenticated");
 
-	let event: Box<RawJsonValue> = services
+	let account_data: ExtractRoomEventContent = services
 		.account_data
-		.get(Some(&body.room_id), sender_user, body.event_type.clone())?
+		.get_room(&body.room_id, sender_user, body.event_type.clone())
-		.ok_or_else(|| Error::BadRequest(ErrorKind::NotFound, "Data not found."))?;
+		.await
-
+		.map_err(|_| err!(Request(NotFound("Data not found."))))?;
-	let account_data = serde_json::from_str::<ExtractRoomEventContent>(event.get())
-		.map_err(|_| Error::bad_database("Invalid account data event in db."))?
-		.content;
 
 	Ok(get_room_account_data::v3::Response {
-		account_data,
+		account_data: account_data.content,
 	})
 }
 
-fn set_account_data(
+async fn set_account_data(
-	services: &Services, room_id: Option<&RoomId>, sender_user: &Option<OwnedUserId>, event_type: &str,
+	services: &Services, room_id: Option<&RoomId>, sender_user: Option<&OwnedUserId>, event_type: &str,
 	data: &RawJsonValue,
 ) -> Result<()> {
 	let sender_user = sender_user.as_ref().expect("user is authenticated");
@@ -100,15 +97,18 @@ fn set_account_data(
 	let data: serde_json::Value =
 		serde_json::from_str(data.get()).map_err(|_| Error::BadRequest(ErrorKind::BadJson, "Data is invalid."))?;
 
-	services.account_data.update(
+	services
-		room_id,
+		.account_data
-		sender_user,
+		.update(
-		event_type.into(),
+			room_id,
-		&json!({
+			sender_user,
-			"type": event_type,
+			event_type.into(),
-			"content": data,
+			&json!({
-		}),
+				"type": event_type,
-	)?;
+				"content": data,
+			}),
+		)
+		.await?;
 
 	Ok(())
 }

src/api/client/context.rs

@@ -1,15 +1,31 @@
-use std::collections::HashSet;
+use std::iter::once;
 
 use axum::extract::State;
-use ruma::{
+use conduit::{
-	api::client::{context::get_context, error::ErrorKind, filter::LazyLoadOptions},
+	at, err, ref_at,
-	events::StateEventType,
+	utils::{
+		future::TryExtExt,
+		stream::{BroadbandExt, ReadyExt, WidebandExt},
+		IterStream,
+	},
+	Err, Result,
+};
+use futures::{join, try_join, FutureExt, StreamExt, TryFutureExt};
+use ruma::{
+	api::client::{context::get_context, filter::LazyLoadOptions},
+	events::StateEventType,
+	OwnedEventId, UserId,
 };
-use tracing::error;
 
-use crate::{Error, Result, Ruma};
+use crate::{
+	client::message::{event_filter, ignored_filter, update_lazy, visibility_filter, LazySet},
+	Ruma,
+};
 
-/// # `GET /_matrix/client/r0/rooms/{roomId}/context`
+const LIMIT_MAX: usize = 100;
+const LIMIT_DEFAULT: usize = 10;
+
+/// # `GET /_matrix/client/r0/rooms/{roomId}/context/{eventId}`
 ///
 /// Allows loading room history around an event.
 ///
@@ -18,186 +34,172 @@ use crate::{Error, Result, Ruma};
 pub(crate) async fn get_context_route(
 	State(services): State<crate::State>, body: Ruma<get_context::v3::Request>,
 ) -> Result<get_context::v3::Response> {
-	let sender_user = body.sender_user.as_ref().expect("user is authenticated");
+	let filter = &body.filter;
-	let sender_device = body.sender_device.as_ref().expect("user is authenticated");
+	let sender = body.sender();
+	let (sender_user, _) = sender;
+	let room_id = &body.room_id;
+
+	// Use limit or else 10, with maximum 100
+	let limit: usize = body
+		.limit
+		.try_into()
+		.unwrap_or(LIMIT_DEFAULT)
+		.min(LIMIT_MAX);
 
 	// some clients, at least element, seem to require knowledge of redundant
 	// members for "inline" profiles on the timeline to work properly
-	let (lazy_load_enabled, lazy_load_send_redundant) = match &body.filter.lazy_load_options {
+	let lazy_load_enabled = matches!(filter.lazy_load_options, LazyLoadOptions::Enabled { .. });
-		LazyLoadOptions::Enabled {
-			include_redundant_members,
-		} => (true, *include_redundant_members),
-		LazyLoadOptions::Disabled => (false, cfg!(feature = "element_hacks")),
-	};
 
-	let mut lazy_loaded = HashSet::new();
+	let lazy_load_redundant = if let LazyLoadOptions::Enabled {
+		include_redundant_members,
+	} = filter.lazy_load_options
+	{
+		include_redundant_members
+	} else {
+		false
+	};
 
 	let base_token = services
 		.rooms
 		.timeline
-		.get_pdu_count(&body.event_id)?
+		.get_pdu_count(&body.event_id)
-		.ok_or(Error::BadRequest(ErrorKind::NotFound, "Base event id not found."))?;
+		.map_err(|_| err!(Request(NotFound("Event not found."))));
 
 	let base_event = services
 		.rooms
 		.timeline
-		.get_pdu(&body.event_id)?
+		.get_pdu(&body.event_id)
-		.ok_or(Error::BadRequest(ErrorKind::NotFound, "Base event not found."))?;
+		.map_err(|_| err!(Request(NotFound("Base event not found."))));
 
-	let room_id = base_event.room_id.clone();
+	let visible = services
-
-	if !services
 		.rooms
 		.state_accessor
-		.user_can_see_event(sender_user, &room_id, &body.event_id)?
+		.user_can_see_event(sender_user, &body.room_id, &body.event_id)
-	{
+		.map(Ok);
-		return Err(Error::BadRequest(
+
-			ErrorKind::forbidden(),
+	let (base_token, base_event, visible) = try_join!(base_token, base_event, visible)?;
-			"You don't have permission to view this event.",
+
-		));
+	if base_event.room_id != body.room_id || base_event.event_id != body.event_id {
+		return Err!(Request(NotFound("Base event not found.")));
 	}
 
-	if !services.rooms.lazy_loading.lazy_load_was_sent_before(
+	if !visible
-		sender_user,
+		|| ignored_filter(&services, (base_token, base_event.clone()), sender_user)
-		sender_device,
+			.await
-		&room_id,
+			.is_none()
-		&base_event.sender,
-	)? || lazy_load_send_redundant
 	{
-		lazy_loaded.insert(base_event.sender.as_str().to_owned());
+		return Err!(Request(Forbidden("You don't have permission to view this event.")));
 	}
 
-	// Use limit or else 10, with maximum 100
+	let events_before = services
-	let limit = usize::try_from(body.limit).unwrap_or(10).min(100);
-
-	let base_event = base_event.to_room_event();
-
-	let events_before: Vec<_> = services
 		.rooms
 		.timeline
-		.pdus_until(sender_user, &room_id, base_token)?
+		.pdus_rev(Some(sender_user), room_id, Some(base_token));
+
+	let events_after = services
+		.rooms
+		.timeline
+		.pdus(Some(sender_user), room_id, Some(base_token));
+
+	let (events_before, events_after) = try_join!(events_before, events_after)?;
+
+	let events_before = events_before
+		.ready_filter_map(|item| event_filter(item, filter))
+		.wide_filter_map(|item| ignored_filter(&services, item, sender_user))
+		.wide_filter_map(|item| visibility_filter(&services, item, sender_user))
 		.take(limit / 2)
-		.filter_map(Result::ok) // Remove buggy events
-		.filter(|(_, pdu)| {
-			services
-				.rooms
-				.state_accessor
-				.user_can_see_event(sender_user, &room_id, &pdu.event_id)
-				.unwrap_or(false)
-		})
 		.collect();
 
-	for (_, event) in &events_before {
+	let events_after = events_after
-		if !services.rooms.lazy_loading.lazy_load_was_sent_before(
+		.ready_filter_map(|item| event_filter(item, filter))
-			sender_user,
+		.wide_filter_map(|item| ignored_filter(&services, item, sender_user))
-			sender_device,
+		.wide_filter_map(|item| visibility_filter(&services, item, sender_user))
-			&room_id,
+		.take(limit / 2)
-			&event.sender,
+		.collect();
-		)? || lazy_load_send_redundant
-		{
-			lazy_loaded.insert(event.sender.as_str().to_owned());
-		}
-	}
 
-	let start_token = events_before
+	let (events_before, events_after): (Vec<_>, Vec<_>) = join!(events_before, events_after);
+
+	let state_at = events_after
 		.last()
-		.map_or_else(|| base_token.stringify(), |(count, _)| count.stringify());
+		.map(ref_at!(1))
-
+		.map_or(body.event_id.as_ref(), |e| e.event_id.as_ref());
-	let events_before: Vec<_> = events_before
-		.into_iter()
-		.map(|(_, pdu)| pdu.to_room_event())
-		.collect();
-
-	let events_after: Vec<_> = services
-		.rooms
-		.timeline
-		.pdus_after(sender_user, &room_id, base_token)?
-		.take(limit / 2)
-		.filter_map(Result::ok) // Remove buggy events
-		.filter(|(_, pdu)| {
-			services
-				.rooms
-				.state_accessor
-				.user_can_see_event(sender_user, &room_id, &pdu.event_id)
-				.unwrap_or(false)
-		})
-		.collect();
-
-	for (_, event) in &events_after {
-		if !services.rooms.lazy_loading.lazy_load_was_sent_before(
-			sender_user,
-			sender_device,
-			&room_id,
-			&event.sender,
-		)? || lazy_load_send_redundant
-		{
-			lazy_loaded.insert(event.sender.as_str().to_owned());
-		}
-	}
-
-	let shortstatehash = services
-		.rooms
-		.state_accessor
-		.pdu_shortstatehash(
-			events_after
-				.last()
-				.map_or(&*body.event_id, |(_, e)| &*e.event_id),
-		)?
-		.map_or(
-			services
-				.rooms
-				.state
-				.get_room_shortstatehash(&room_id)?
-				.expect("All rooms have state"),
-			|hash| hash,
-		);
 
 	let state_ids = services
 		.rooms
 		.state_accessor
-		.state_full_ids(shortstatehash)
+		.pdu_shortstatehash(state_at)
+		.or_else(|_| services.rooms.state.get_room_shortstatehash(room_id))
+		.and_then(|shortstatehash| services.rooms.state_accessor.state_full_ids(shortstatehash))
+		.map_err(|e| err!(Database("State not found: {e}")))
 		.await?;
 
-	let end_token = events_after
+	let lazy = once(&(base_token, base_event.clone()))
-		.last()
+		.chain(events_before.iter())
-		.map_or_else(|| base_token.stringify(), |(count, _)| count.stringify());
+		.chain(events_after.iter())
+		.stream()
+		.fold(LazySet::new(), |lazy, item| {
+			update_lazy(&services, room_id, sender, lazy, item, lazy_load_redundant)
+		})
+		.await;
 
-	let events_after: Vec<_> = events_after
+	let lazy = &lazy;
-		.into_iter()
+	let state: Vec<_> = state_ids
-		.map(|(_, pdu)| pdu.to_room_event())
+		.iter()
-		.collect();
+		.stream()
+		.broad_filter_map(|(shortstatekey, event_id)| {
+			services
+				.rooms
+				.short
+				.get_statekey_from_short(*shortstatekey)
+				.map_ok(move |(event_type, state_key)| (event_type, state_key, event_id))
+				.ok()
+		})
+		.ready_filter_map(|(event_type, state_key, event_id)| {
+			if !lazy_load_enabled || event_type != StateEventType::RoomMember {
+				return Some(event_id);
+			}
 
-	let mut state = Vec::with_capacity(state_ids.len());
+			state_key
-
+				.as_str()
-	for (shortstatekey, id) in state_ids {
+				.try_into()
-		let (event_type, state_key) = services
+				.ok()
-			.rooms
+				.filter(|&user_id: &&UserId| lazy.contains(user_id))
-			.short
+				.map(|_| event_id)
-			.get_statekey_from_short(shortstatekey)?;
+		})
-
+		.broad_filter_map(|event_id: &OwnedEventId| services.rooms.timeline.get_pdu(event_id).ok())
-		if event_type != StateEventType::RoomMember {
+		.map(|pdu| pdu.to_state_event())
-			let Some(pdu) = services.rooms.timeline.get_pdu(&id)? else {
+		.collect()
-				error!("Pdu in state not found: {}", id);
+		.await;
-				continue;
-			};
-
-			state.push(pdu.to_state_event());
-		} else if !lazy_load_enabled || lazy_loaded.contains(&state_key) {
-			let Some(pdu) = services.rooms.timeline.get_pdu(&id)? else {
-				error!("Pdu in state not found: {}", id);
-				continue;
-			};
-
-			state.push(pdu.to_state_event());
-		}
-	}
 
 	Ok(get_context::v3::Response {
-		start: Some(start_token),
+		event: Some(base_event.to_room_event()),
-		end: Some(end_token),
+
-		events_before,
+		start: events_before
-		event: Some(base_event),
+			.last()
-		events_after,
+			.map(at!(0))
+			.or(Some(base_token))
+			.as_ref()
+			.map(ToString::to_string),
+
+		end: events_after
+			.last()
+			.map(at!(0))
+			.or(Some(base_token))
+			.as_ref()
+			.map(ToString::to_string),
+
+		events_before: events_before
+			.into_iter()
+			.map(at!(1))
+			.map(|pdu| pdu.to_room_event())
+			.collect(),
+
+		events_after: events_after
+			.into_iter()
+			.map(at!(1))
+			.map(|pdu| pdu.to_room_event())
+			.collect(),
+
 		state,
 	})
 }

src/api/client/device.rs

@@ -1,8 +1,14 @@
 use axum::extract::State;
-use ruma::api::client::{
+use axum_client_ip::InsecureClientIp;
-	device::{self, delete_device, delete_devices, get_device, get_devices, update_device},
+use conduit::{err, Err};
-	error::ErrorKind,
+use futures::StreamExt;
-	uiaa::{AuthFlow, AuthType, UiaaInfo},
+use ruma::{
+	api::client::{
+		device::{self, delete_device, delete_devices, get_device, get_devices, update_device},
+		error::ErrorKind,
+		uiaa::{AuthFlow, AuthType, UiaaInfo},
+	},
+	MilliSecondsSinceUnixEpoch,
 };
 
 use super::SESSION_ID_LENGTH;
@@ -19,8 +25,8 @@ pub(crate) async fn get_devices_route(
 	let devices: Vec<device::Device> = services
 		.users
 		.all_devices_metadata(sender_user)
-		.filter_map(Result::ok) // Filter out buggy devices
+		.collect()
-		.collect();
+		.await;
 
 	Ok(get_devices::v3::Response {
 		devices,
@@ -37,8 +43,9 @@ pub(crate) async fn get_device_route(
 
 	let device = services
 		.users
-		.get_device_metadata(sender_user, &body.body.device_id)?
+		.get_device_metadata(sender_user, &body.body.device_id)
-		.ok_or(Error::BadRequest(ErrorKind::NotFound, "Device not found."))?;
+		.await
+		.map_err(|_| err!(Request(NotFound("Device not found."))))?;
 
 	Ok(get_device::v3::Response {
 		device,
@@ -48,21 +55,29 @@ pub(crate) async fn get_device_route(
 /// # `PUT /_matrix/client/r0/devices/{deviceId}`
 ///
 /// Updates the metadata on a given device of the sender user.
+#[tracing::instrument(skip_all, fields(%client), name = "update_device")]
 pub(crate) async fn update_device_route(
-	State(services): State<crate::State>, body: Ruma<update_device::v3::Request>,
+	State(services): State<crate::State>, InsecureClientIp(client): InsecureClientIp,
+	body: Ruma<update_device::v3::Request>,
 ) -> Result<update_device::v3::Response> {
 	let sender_user = body.sender_user.as_ref().expect("user is authenticated");
 
 	let mut device = services
 		.users
-		.get_device_metadata(sender_user, &body.device_id)?
+		.get_device_metadata(sender_user, &body.device_id)
-		.ok_or(Error::BadRequest(ErrorKind::NotFound, "Device not found."))?;
+		.await
+		.map_err(|_| err!(Request(NotFound("Device not found."))))?;
 
 	device.display_name.clone_from(&body.display_name);
+	device.last_seen_ip.clone_from(&Some(client.to_string()));
+	device
+		.last_seen_ts
+		.clone_from(&Some(MilliSecondsSinceUnixEpoch::now()));
 
 	services
 		.users
-		.update_device_metadata(sender_user, &body.device_id, &device)?;
+		.update_device_metadata(sender_user, &body.device_id, &device)
+		.await?;
 
 	Ok(update_device::v3::Response {})
 }
@@ -97,22 +112,28 @@ pub(crate) async fn delete_device_route(
 	if let Some(auth) = &body.auth {
 		let (worked, uiaainfo) = services
 			.uiaa
-			.try_auth(sender_user, sender_device, auth, &uiaainfo)?;
+			.try_auth(sender_user, sender_device, auth, &uiaainfo)
+			.await?;
+
 		if !worked {
-			return Err(Error::Uiaa(uiaainfo));
+			return Err!(Uiaa(uiaainfo));
 		}
 	// Success!
 	} else if let Some(json) = body.json_body {
 		uiaainfo.session = Some(utils::random_string(SESSION_ID_LENGTH));
 		services
 			.uiaa
-			.create(sender_user, sender_device, &uiaainfo, &json)?;
+			.create(sender_user, sender_device, &uiaainfo, &json);
-		return Err(Error::Uiaa(uiaainfo));
+
+		return Err!(Uiaa(uiaainfo));
 	} else {
-		return Err(Error::BadRequest(ErrorKind::NotJson, "Not json."));
+		return Err!(Request(NotJson("Not json.")));
 	}
 
-	services.users.remove_device(sender_user, &body.device_id)?;
+	services
+		.users
+		.remove_device(sender_user, &body.device_id)
+		.await;
 
 	Ok(delete_device::v3::Response {})
 }
@@ -149,7 +170,9 @@ pub(crate) async fn delete_devices_route(
 	if let Some(auth) = &body.auth {
 		let (worked, uiaainfo) = services
 			.uiaa
-			.try_auth(sender_user, sender_device, auth, &uiaainfo)?;
+			.try_auth(sender_user, sender_device, auth, &uiaainfo)
+			.await?;
+
 		if !worked {
 			return Err(Error::Uiaa(uiaainfo));
 		}
@@ -158,14 +181,15 @@ pub(crate) async fn delete_devices_route(
 		uiaainfo.session = Some(utils::random_string(SESSION_ID_LENGTH));
 		services
 			.uiaa
-			.create(sender_user, sender_device, &uiaainfo, &json)?;
+			.create(sender_user, sender_device, &uiaainfo, &json);
+
 		return Err(Error::Uiaa(uiaainfo));
 	} else {
 		return Err(Error::BadRequest(ErrorKind::NotJson, "Not json."));
 	}
 
 	for device_id in &body.devices {
-		services.users.remove_device(sender_user, device_id)?;
+		services.users.remove_device(sender_user, device_id).await;
 	}
 
 	Ok(delete_devices::v3::Response {})

src/api/client/directory.rs

@@ -1,6 +1,7 @@
 use axum::extract::State;
 use axum_client_ip::InsecureClientIp;
-use conduit::{err, info, warn, Error, Result};
+use conduit::{info, warn, Err, Error, Result};
+use futures::{StreamExt, TryFutureExt};
 use ruma::{
 	api::{
 		client::{
@@ -18,7 +19,7 @@ use ruma::{
 		},
 		StateEventType,
 	},
-	uint, RoomId, ServerName, UInt, UserId,
+	uint, OwnedRoomId, RoomId, ServerName, UInt, UserId,
 };
 use service::Services;
 
@@ -36,14 +37,12 @@ pub(crate) async fn get_public_rooms_filtered_route(
 ) -> Result<get_public_rooms_filtered::v3::Response> {
 	if let Some(server) = &body.server {
 		if services
-			.globals
+			.server
-			.forbidden_remote_room_directory_server_names()
+			.config
+			.forbidden_remote_room_directory_server_names
 			.contains(server)
 		{
-			return Err(Error::BadRequest(
+			return Err!(Request(Forbidden("Server is banned on this homeserver.")));
-				ErrorKind::forbidden(),
-				"Server is banned on this homeserver.",
-			));
 		}
 	}
 
@@ -76,14 +75,12 @@ pub(crate) async fn get_public_rooms_route(
 ) -> Result<get_public_rooms::v3::Response> {
 	if let Some(server) = &body.server {
 		if services
-			.globals
+			.server
-			.forbidden_remote_room_directory_server_names()
+			.config
+			.forbidden_remote_room_directory_server_names
 			.contains(server)
 		{
-			return Err(Error::BadRequest(
+			return Err!(Request(Forbidden("Server is banned on this homeserver.")));
-				ErrorKind::forbidden(),
-				"Server is banned on this homeserver.",
-			));
 		}
 	}
 
@@ -119,12 +116,22 @@ pub(crate) async fn set_room_visibility_route(
 ) -> Result<set_room_visibility::v3::Response> {
 	let sender_user = body.sender_user.as_ref().expect("user is authenticated");
 
-	if !services.rooms.metadata.exists(&body.room_id)? {
+	if !services.rooms.metadata.exists(&body.room_id).await {
 		// Return 404 if the room doesn't exist
 		return Err(Error::BadRequest(ErrorKind::NotFound, "Room not found"));
 	}
 
-	if !user_can_publish_room(&services, sender_user, &body.room_id)? {
+	if services
+		.users
+		.is_deactivated(sender_user)
+		.await
+		.unwrap_or(false)
+		&& body.appservice_info.is_none()
+	{
+		return Err!(Request(Forbidden("Guests cannot publish to room directories")));
+	}
+
+	if !user_can_publish_room(&services, sender_user, &body.room_id).await? {
 		return Err(Error::BadRequest(
 			ErrorKind::forbidden(),
 			"User is not allowed to publish this room",
@@ -133,23 +140,44 @@ pub(crate) async fn set_room_visibility_route(
 
 	match &body.visibility {
 		room::Visibility::Public => {
-			if services.globals.config.lockdown_public_room_directory && !services.users.is_admin(sender_user)? {
+			if services.globals.config.lockdown_public_room_directory
+				&& !services.users.is_admin(sender_user).await
+				&& body.appservice_info.is_none()
+			{
 				info!(
 					"Non-admin user {sender_user} tried to publish {0} to the room directory while \
 					 \"lockdown_public_room_directory\" is enabled",
 					body.room_id
 				);
 
+				if services.globals.config.admin_room_notices {
+					services
+						.admin
+						.send_text(&format!(
+							"Non-admin user {sender_user} tried to publish {0} to the room directory while \
+							 \"lockdown_public_room_directory\" is enabled",
+							body.room_id
+						))
+						.await;
+				}
+
 				return Err(Error::BadRequest(
 					ErrorKind::forbidden(),
 					"Publishing rooms to the room directory is not allowed",
 				));
 			}
 
-			services.rooms.directory.set_public(&body.room_id)?;
+			services.rooms.directory.set_public(&body.room_id);
-			info!("{sender_user} made {0} public", body.room_id);
+
+			if services.globals.config.admin_room_notices {
+				services
+					.admin
+					.send_text(&format!("{sender_user} made {} public to the room directory", body.room_id))
+					.await;
+			}
+			info!("{sender_user} made {0} public to the room directory", body.room_id);
 		},
-		room::Visibility::Private => services.rooms.directory.set_not_public(&body.room_id)?,
+		room::Visibility::Private => services.rooms.directory.set_not_public(&body.room_id),
 		_ => {
 			return Err(Error::BadRequest(
 				ErrorKind::InvalidParam,
@@ -167,13 +195,13 @@ pub(crate) async fn set_room_visibility_route(
 pub(crate) async fn get_room_visibility_route(
 	State(services): State<crate::State>, body: Ruma<get_room_visibility::v3::Request>,
 ) -> Result<get_room_visibility::v3::Response> {
-	if !services.rooms.metadata.exists(&body.room_id)? {
+	if !services.rooms.metadata.exists(&body.room_id).await {
 		// Return 404 if the room doesn't exist
 		return Err(Error::BadRequest(ErrorKind::NotFound, "Room not found"));
 	}
 
 	Ok(get_room_visibility::v3::Response {
-		visibility: if services.rooms.directory.is_public_room(&body.room_id)? {
+		visibility: if services.rooms.directory.is_public_room(&body.room_id).await {
 			room::Visibility::Public
 		} else {
 			room::Visibility::Private
@@ -232,101 +260,41 @@ pub(crate) async fn get_public_rooms_filtered_helper(
 		}
 	}
 
-	let mut all_rooms: Vec<_> = services
+	let mut all_rooms: Vec<PublicRoomsChunk> = services
 		.rooms
 		.directory
 		.public_rooms()
-		.map(|room_id| {
+		.map(ToOwned::to_owned)
-			let room_id = room_id?;
+		.then(|room_id| public_rooms_chunk(services, room_id))
-
+		.filter_map(|chunk| async move {
-			let chunk = PublicRoomsChunk {
-				canonical_alias: services
-					.rooms
-					.state_accessor
-					.get_canonical_alias(&room_id)?,
-				name: services.rooms.state_accessor.get_name(&room_id)?,
-				num_joined_members: services
-					.rooms
-					.state_cache
-					.room_joined_count(&room_id)?
-					.unwrap_or_else(|| {
-						warn!("Room {} has no member count", room_id);
-						0
-					})
-					.try_into()
-					.expect("user count should not be that big"),
-				topic: services
-					.rooms
-					.state_accessor
-					.get_room_topic(&room_id)
-					.unwrap_or(None),
-				world_readable: services.rooms.state_accessor.is_world_readable(&room_id)?,
-				guest_can_join: services
-					.rooms
-					.state_accessor
-					.guest_can_join(&room_id)?,
-				avatar_url: services
-					.rooms
-					.state_accessor
-					.get_avatar(&room_id)?
-					.into_option()
-					.unwrap_or_default()
-					.url,
-				join_rule: services
-					.rooms
-					.state_accessor
-					.room_state_get(&room_id, &StateEventType::RoomJoinRules, "")?
-					.map(|s| {
-						serde_json::from_str(s.content.get())
-							.map(|c: RoomJoinRulesEventContent| match c.join_rule {
-								JoinRule::Public => Some(PublicRoomJoinRule::Public),
-								JoinRule::Knock => Some(PublicRoomJoinRule::Knock),
-								_ => None,
-							})
-							.map_err(|e| {
-								err!(Database(error!("Invalid room join rule event in database: {e}")))
-							})
-					})
-					.transpose()?
-					.flatten()
-					.ok_or_else(|| Error::bad_database("Missing room join rule event for room."))?,
-				room_type: services
-					.rooms
-					.state_accessor
-					.get_room_type(&room_id)?,
-				room_id,
-			};
-			Ok(chunk)
-		})
-		.filter_map(|r: Result<_>| r.ok()) // Filter out buggy rooms
-		.filter(|chunk| {
 			if let Some(query) = filter.generic_search_term.as_ref().map(|q| q.to_lowercase()) {
 				if let Some(name) = &chunk.name {
 					if name.as_str().to_lowercase().contains(&query) {
-						return true;
+						return Some(chunk);
 					}
 				}
 
 				if let Some(topic) = &chunk.topic {
 					if topic.to_lowercase().contains(&query) {
-						return true;
+						return Some(chunk);
 					}
 				}
 
 				if let Some(canonical_alias) = &chunk.canonical_alias {
 					if canonical_alias.as_str().to_lowercase().contains(&query) {
-						return true;
+						return Some(chunk);
 					}
 				}
 
-				false
+				return None;
-			} else {
-				// No search term
-				true
 			}
+
+			// No search term
+			Some(chunk)
 		})
 		// We need to collect all, so we can sort by member count
-		.collect();
+		.collect()
+		.await;
 
 	all_rooms.sort_by(|l, r| r.num_joined_members.cmp(&l.num_joined_members));
 
@@ -369,22 +337,23 @@ pub(crate) async fn get_public_rooms_filtered_helper(
 
 /// Check whether the user can publish to the room directory via power levels of
 /// room history visibility event or room creator
-fn user_can_publish_room(services: &Services, user_id: &UserId, room_id: &RoomId) -> Result<bool> {
+async fn user_can_publish_room(services: &Services, user_id: &UserId, room_id: &RoomId) -> Result<bool> {
-	if let Some(event) = services
+	if let Ok(event) = services
 		.rooms
 		.state_accessor
-		.room_state_get(room_id, &StateEventType::RoomPowerLevels, "")?
+		.room_state_get(room_id, &StateEventType::RoomPowerLevels, "")
+		.await
 	{
 		serde_json::from_str(event.content.get())
 			.map_err(|_| Error::bad_database("Invalid event content for m.room.power_levels"))
 			.map(|content: RoomPowerLevelsEventContent| {
 				RoomPowerLevels::from(content).user_can_send_state(user_id, StateEventType::RoomHistoryVisibility)
 			})
-	} else if let Some(event) =
+	} else if let Ok(event) = services
-		services
+		.rooms
-			.rooms
+		.state_accessor
-			.state_accessor
+		.room_state_get(room_id, &StateEventType::RoomCreate, "")
-			.room_state_get(room_id, &StateEventType::RoomCreate, "")?
+		.await
 	{
 		Ok(event.sender == user_id)
 	} else {
@@ -394,3 +363,62 @@ fn user_can_publish_room(services: &Services, user_id: &UserId, room_id: &RoomId
 		));
 	}
 }
+
+async fn public_rooms_chunk(services: &Services, room_id: OwnedRoomId) -> PublicRoomsChunk {
+	PublicRoomsChunk {
+		canonical_alias: services
+			.rooms
+			.state_accessor
+			.get_canonical_alias(&room_id)
+			.await
+			.ok(),
+		name: services.rooms.state_accessor.get_name(&room_id).await.ok(),
+		num_joined_members: services
+			.rooms
+			.state_cache
+			.room_joined_count(&room_id)
+			.await
+			.unwrap_or(0)
+			.try_into()
+			.expect("joined count overflows ruma UInt"),
+		topic: services
+			.rooms
+			.state_accessor
+			.get_room_topic(&room_id)
+			.await
+			.ok(),
+		world_readable: services
+			.rooms
+			.state_accessor
+			.is_world_readable(&room_id)
+			.await,
+		guest_can_join: services.rooms.state_accessor.guest_can_join(&room_id).await,
+		avatar_url: services
+			.rooms
+			.state_accessor
+			.get_avatar(&room_id)
+			.await
+			.into_option()
+			.unwrap_or_default()
+			.url,
+		join_rule: services
+			.rooms
+			.state_accessor
+			.room_state_get_content(&room_id, &StateEventType::RoomJoinRules, "")
+			.map_ok(|c: RoomJoinRulesEventContent| match c.join_rule {
+				JoinRule::Public => PublicRoomJoinRule::Public,
+				JoinRule::Knock => "knock".into(),
+				JoinRule::KnockRestricted(_) => "knock_restricted".into(),
+				_ => "invite".into(),
+			})
+			.await
+			.unwrap_or_default(),
+		room_type: services
+			.rooms
+			.state_accessor
+			.get_room_type(&room_id)
+			.await
+			.ok(),
+		room_id,
+	}
+}

src/api/client/filter.rs

@@ -1,10 +1,8 @@
 use axum::extract::State;
-use ruma::api::client::{
+use conduit::err;
-	error::ErrorKind,
+use ruma::api::client::filter::{create_filter, get_filter};
-	filter::{create_filter, get_filter},
-};
 
-use crate::{Error, Result, Ruma};
+use crate::{Result, Ruma};
 
 /// # `GET /_matrix/client/r0/user/{userId}/filter/{filterId}`
 ///
@@ -15,11 +13,13 @@ pub(crate) async fn get_filter_route(
 	State(services): State<crate::State>, body: Ruma<get_filter::v3::Request>,
 ) -> Result<get_filter::v3::Response> {
 	let sender_user = body.sender_user.as_ref().expect("user is authenticated");
-	let Some(filter) = services.users.get_filter(sender_user, &body.filter_id)? else {
-		return Err(Error::BadRequest(ErrorKind::NotFound, "Filter not found."));
-	};
 
-	Ok(get_filter::v3::Response::new(filter))
+	services
+		.users
+		.get_filter(sender_user, &body.filter_id)
+		.await
+		.map(get_filter::v3::Response::new)
+		.map_err(|_| err!(Request(NotFound("Filter not found."))))
 }
 
 /// # `PUT /_matrix/client/r0/user/{userId}/filter`
@@ -29,7 +29,8 @@ pub(crate) async fn create_filter_route(
 	State(services): State<crate::State>, body: Ruma<create_filter::v3::Request>,
 ) -> Result<create_filter::v3::Response> {
 	let sender_user = body.sender_user.as_ref().expect("user is authenticated");
-	Ok(create_filter::v3::Response::new(
+
-		services.users.create_filter(sender_user, &body.filter)?,
+	let filter_id = services.users.create_filter(sender_user, &body.filter);
-	))
+
+	Ok(create_filter::v3::Response::new(filter_id))
 }

src/api/client/keys.rs

@@ -4,8 +4,8 @@ use std::{
 };
 
 use axum::extract::State;
-use conduit::{utils, utils::math::continue_exponential_backoff_secs, Err, Error, Result};
+use conduit::{err, utils, utils::math::continue_exponential_backoff_secs, Err, Error, Result};
-use futures_util::{stream::FuturesUnordered, StreamExt};
+use futures::{stream::FuturesUnordered, StreamExt};
 use ruma::{
 	api::{
 		client::{
@@ -16,12 +16,15 @@ use ruma::{
 		federation,
 	},
 	serde::Raw,
-	DeviceKeyAlgorithm, OwnedDeviceId, OwnedUserId, UserId,
+	OneTimeKeyAlgorithm, OwnedDeviceId, OwnedUserId, UserId,
 };
 use serde_json::json;
 
 use super::SESSION_ID_LENGTH;
-use crate::{service::Services, Ruma};
+use crate::{
+	service::{users::parse_master_key, Services},
+	Ruma,
+};
 
 /// # `POST /_matrix/client/r0/keys/upload`
 ///
@@ -33,13 +36,13 @@ use crate::{service::Services, Ruma};
 pub(crate) async fn upload_keys_route(
 	State(services): State<crate::State>, body: Ruma<upload_keys::v3::Request>,
 ) -> Result<upload_keys::v3::Response> {
-	let sender_user = body.sender_user.as_ref().expect("user is authenticated");
+	let (sender_user, sender_device) = body.sender();
-	let sender_device = body.sender_device.as_ref().expect("user is authenticated");
 
-	for (key_key, key_value) in &body.one_time_keys {
+	for (key_id, one_time_key) in &body.one_time_keys {
 		services
 			.users
-			.add_one_time_key(sender_user, sender_device, key_key, key_value)?;
+			.add_one_time_key(sender_user, sender_device, key_id, one_time_key)
+			.await?;
 	}
 
 	if let Some(device_keys) = &body.device_keys {
@@ -47,19 +50,22 @@ pub(crate) async fn upload_keys_route(
 		// This check is needed to assure that signatures are kept
 		if services
 			.users
-			.get_device_keys(sender_user, sender_device)?
+			.get_device_keys(sender_user, sender_device)
-			.is_none()
+			.await
+			.is_err()
 		{
 			services
 				.users
-				.add_device_keys(sender_user, sender_device, device_keys)?;
+				.add_device_keys(sender_user, sender_device, device_keys)
+				.await;
 		}
 	}
 
 	Ok(upload_keys::v3::Response {
 		one_time_key_counts: services
 			.users
-			.count_one_time_keys(sender_user, sender_device)?,
+			.count_one_time_keys(sender_user, sender_device)
+			.await,
 	})
 }
 
@@ -120,7 +126,9 @@ pub(crate) async fn upload_signing_keys_route(
 	if let Some(auth) = &body.auth {
 		let (worked, uiaainfo) = services
 			.uiaa
-			.try_auth(sender_user, sender_device, auth, &uiaainfo)?;
+			.try_auth(sender_user, sender_device, auth, &uiaainfo)
+			.await?;
+
 		if !worked {
 			return Err(Error::Uiaa(uiaainfo));
 		}
@@ -129,20 +137,24 @@ pub(crate) async fn upload_signing_keys_route(
 		uiaainfo.session = Some(utils::random_string(SESSION_ID_LENGTH));
 		services
 			.uiaa
-			.create(sender_user, sender_device, &uiaainfo, &json)?;
+			.create(sender_user, sender_device, &uiaainfo, &json);
+
 		return Err(Error::Uiaa(uiaainfo));
 	} else {
 		return Err(Error::BadRequest(ErrorKind::NotJson, "Not json."));
 	}
 
 	if let Some(master_key) = &body.master_key {
-		services.users.add_cross_signing_keys(
+		services
-			sender_user,
+			.users
-			master_key,
+			.add_cross_signing_keys(
-			&body.self_signing_key,
+				sender_user,
-			&body.user_signing_key,
+				master_key,
-			true, // notify so that other users see the new keys
+				&body.self_signing_key,
-		)?;
+				&body.user_signing_key,
+				true, // notify so that other users see the new keys
+			)
+			.await?;
 	}
 
 	Ok(upload_signing_keys::v3::Response {})
@@ -179,9 +191,11 @@ pub(crate) async fn upload_signatures_route(
 						.ok_or(Error::BadRequest(ErrorKind::InvalidParam, "Invalid signature value."))?
 						.to_owned(),
 				);
+
 				services
 					.users
-					.sign_key(user_id, key_id, signature, sender_user)?;
+					.sign_key(user_id, key_id, signature, sender_user)
+					.await?;
 			}
 		}
 	}
@@ -204,56 +218,52 @@ pub(crate) async fn get_key_changes_route(
 
 	let mut device_list_updates = HashSet::new();
 
+	let from = body
+		.from
+		.parse()
+		.map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "Invalid `from`."))?;
+
+	let to = body
+		.to
+		.parse()
+		.map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "Invalid `to`."))?;
+
 	device_list_updates.extend(
 		services
 			.users
-			.keys_changed(
+			.keys_changed(sender_user, from, Some(to))
-				sender_user.as_str(),
+			.map(ToOwned::to_owned)
-				body.from
+			.collect::<Vec<_>>()
-					.parse()
+			.await,
-					.map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "Invalid `from`."))?,
-				Some(
-					body.to
-						.parse()
-						.map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "Invalid `to`."))?,
-				),
-			)
-			.filter_map(Result::ok),
 	);
 
-	for room_id in services
+	let mut rooms_joined = services.rooms.state_cache.rooms_joined(sender_user).boxed();
-		.rooms
+
-		.state_cache
+	while let Some(room_id) = rooms_joined.next().await {
-		.rooms_joined(sender_user)
-		.filter_map(Result::ok)
-	{
 		device_list_updates.extend(
 			services
 				.users
-				.keys_changed(
+				.room_keys_changed(room_id, from, Some(to))
-					room_id.as_ref(),
+				.map(|(user_id, _)| user_id)
-					body.from
+				.map(ToOwned::to_owned)
-						.parse()
+				.collect::<Vec<_>>()
-						.map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "Invalid `from`."))?,
+				.await,
-					Some(
-						body.to
-							.parse()
-							.map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "Invalid `to`."))?,
-					),
-				)
-				.filter_map(Result::ok),
 		);
 	}
+
 	Ok(get_key_changes::v3::Response {
 		changed: device_list_updates.into_iter().collect(),
 		left: Vec::new(), // TODO
 	})
 }
 
-pub(crate) async fn get_keys_helper<F: Fn(&UserId) -> bool + Send>(
+pub(crate) async fn get_keys_helper<F>(
 	services: &Services, sender_user: Option<&UserId>, device_keys_input: &BTreeMap<OwnedUserId, Vec<OwnedDeviceId>>,
 	allowed_signatures: F, include_display_names: bool,
-) -> Result<get_keys::v3::Response> {
+) -> Result<get_keys::v3::Response>
+where
+	F: Fn(&UserId) -> bool + Send + Sync,
+{
 	let mut master_keys = BTreeMap::new();
 	let mut self_signing_keys = BTreeMap::new();
 	let mut user_signing_keys = BTreeMap::new();
@@ -274,56 +284,60 @@ pub(crate) async fn get_keys_helper<F: Fn(&UserId) -> bool + Send>(
 
 		if device_ids.is_empty() {
 			let mut container = BTreeMap::new();
-			for device_id in services.users.all_device_ids(user_id) {
+			let mut devices = services.users.all_device_ids(user_id).boxed();
-				let device_id = device_id?;
+
-				if let Some(mut keys) = services.users.get_device_keys(user_id, &device_id)? {
+			while let Some(device_id) = devices.next().await {
+				if let Ok(mut keys) = services.users.get_device_keys(user_id, device_id).await {
 					let metadata = services
 						.users
-						.get_device_metadata(user_id, &device_id)?
+						.get_device_metadata(user_id, device_id)
-						.ok_or_else(|| Error::bad_database("all_device_keys contained nonexistent device."))?;
+						.await
+						.map_err(|_| err!(Database("all_device_keys contained nonexistent device.")))?;
 
 					add_unsigned_device_display_name(&mut keys, metadata, include_display_names)
-						.map_err(|_| Error::bad_database("invalid device keys in database"))?;
+						.map_err(|_| err!(Database("invalid device keys in database")))?;
 
-					container.insert(device_id, keys);
+					container.insert(device_id.to_owned(), keys);
 				}
 			}
+
 			device_keys.insert(user_id.to_owned(), container);
 		} else {
 			for device_id in device_ids {
 				let mut container = BTreeMap::new();
-				if let Some(mut keys) = services.users.get_device_keys(user_id, device_id)? {
+				if let Ok(mut keys) = services.users.get_device_keys(user_id, device_id).await {
 					let metadata = services
 						.users
-						.get_device_metadata(user_id, device_id)?
+						.get_device_metadata(user_id, device_id)
-						.ok_or(Error::BadRequest(
+						.await
-							ErrorKind::InvalidParam,
+						.map_err(|_| err!(Request(InvalidParam("Tried to get keys for nonexistent device."))))?;
-							"Tried to get keys for nonexistent device.",
-						))?;
 
 					add_unsigned_device_display_name(&mut keys, metadata, include_display_names)
-						.map_err(|_| Error::bad_database("invalid device keys in database"))?;
+						.map_err(|_| err!(Database("invalid device keys in database")))?;
+
 					container.insert(device_id.to_owned(), keys);
 				}
+
 				device_keys.insert(user_id.to_owned(), container);
 			}
 		}
 
-		if let Some(master_key) = services
+		if let Ok(master_key) = services
 			.users
-			.get_master_key(sender_user, user_id, &allowed_signatures)?
+			.get_master_key(sender_user, user_id, &allowed_signatures)
+			.await
 		{
 			master_keys.insert(user_id.to_owned(), master_key);
 		}
-		if let Some(self_signing_key) =
+		if let Ok(self_signing_key) = services
-			services
+			.users
-				.users
+			.get_self_signing_key(sender_user, user_id, &allowed_signatures)
-				.get_self_signing_key(sender_user, user_id, &allowed_signatures)?
+			.await
 		{
 			self_signing_keys.insert(user_id.to_owned(), self_signing_key);
 		}
 		if Some(user_id) == sender_user {
-			if let Some(user_signing_key) = services.users.get_user_signing_key(user_id)? {
+			if let Ok(user_signing_key) = services.users.get_user_signing_key(user_id).await {
 				user_signing_keys.insert(user_id.to_owned(), user_signing_key);
 			}
 		}
@@ -385,24 +399,27 @@ pub(crate) async fn get_keys_helper<F: Fn(&UserId) -> bool + Send>(
 
 	while let Some((server, response)) = futures.next().await {
 		if let Ok(Ok(response)) = response {
-			for (user, masterkey) in response.master_keys {
+			for (user, master_key) in response.master_keys {
-				let (master_key_id, mut master_key) = services.users.parse_master_key(&user, &masterkey)?;
+				let (master_key_id, mut master_key) = parse_master_key(&user, &master_key)?;
 
-				if let Some(our_master_key) =
+				if let Ok(our_master_key) = services
-					services
+					.users
-						.users
+					.get_key(&master_key_id, sender_user, &user, &allowed_signatures)
-						.get_key(&master_key_id, sender_user, &user, &allowed_signatures)?
+					.await
 				{
-					let (_, our_master_key) = services.users.parse_master_key(&user, &our_master_key)?;
+					let (_, mut our_master_key) = parse_master_key(&user, &our_master_key)?;
-					master_key.signatures.extend(our_master_key.signatures);
+					master_key.signatures.append(&mut our_master_key.signatures);
 				}
 				let json = serde_json::to_value(master_key).expect("to_value always works");
 				let raw = serde_json::from_value(json).expect("Raw::from_value always works");
-				services.users.add_cross_signing_keys(
+				services
-					&user, &raw, &None, &None,
+					.users
-					false, /* Dont notify. A notification would trigger another key request resulting in an
+					.add_cross_signing_keys(
-					       * endless loop */
+						&user, &raw, &None, &None,
-				)?;
+						false, /* Dont notify. A notification would trigger another key request resulting in an
+						       * endless loop */
+					)
+					.await?;
 				master_keys.insert(user.clone(), raw);
 			}
 
@@ -449,7 +466,7 @@ fn add_unsigned_device_display_name(
 }
 
 pub(crate) async fn claim_keys_helper(
-	services: &Services, one_time_keys_input: &BTreeMap<OwnedUserId, BTreeMap<OwnedDeviceId, DeviceKeyAlgorithm>>,
+	services: &Services, one_time_keys_input: &BTreeMap<OwnedUserId, BTreeMap<OwnedDeviceId, OneTimeKeyAlgorithm>>,
 ) -> Result<claim_keys::v3::Response> {
 	let mut one_time_keys = BTreeMap::new();
 
@@ -465,9 +482,10 @@ pub(crate) async fn claim_keys_helper(
 
 		let mut container = BTreeMap::new();
 		for (device_id, key_algorithm) in map {
-			if let Some(one_time_keys) = services
+			if let Ok(one_time_keys) = services
 				.users
-				.take_one_time_key(user_id, device_id, key_algorithm)?
+				.take_one_time_key(user_id, device_id, key_algorithm)
+				.await
 			{
 				let mut c = BTreeMap::new();
 				c.insert(one_time_keys.0, one_time_keys.1);

src/api/client/media.rs

@@ -11,6 +11,7 @@ use conduit_service::{
 	media::{Dim, FileMeta, CACHE_CONTROL_IMMUTABLE, CORP_CROSS_ORIGIN, MXC_LENGTH},
 	Services,
 };
+use reqwest::Url;
 use ruma::{
 	api::client::{
 		authenticated_media::{
@@ -165,23 +166,33 @@ pub(crate) async fn get_media_preview_route(
 	let sender_user = body.sender_user.as_ref().expect("user is authenticated");
 
 	let url = &body.url;
-	if !services.media.url_preview_allowed(url) {
+	let url = Url::parse(&body.url).map_err(|e| {
+		err!(Request(InvalidParam(
+			debug_warn!(%sender_user, %url, "Requested URL is not valid: {e}")
+		)))
+	})?;
+
+	if !services.media.url_preview_allowed(&url) {
 		return Err!(Request(Forbidden(
 			debug_warn!(%sender_user, %url, "URL is not allowed to be previewed")
 		)));
 	}
 
-	let preview = services.media.get_url_preview(url).await.map_err(|error| {
+	let preview = services
-		err!(Request(Unknown(
+		.media
-			debug_error!(%sender_user, %url, ?error, "Failed to fetch URL preview.")
+		.get_url_preview(&url)
-		)))
+		.await
-	})?;
+		.map_err(|error| {
+			err!(Request(Unknown(
+				debug_error!(%sender_user, %url, "Failed to fetch URL preview: {error}")
+			)))
+		})?;
 
 	serde_json::value::to_raw_value(&preview)
 		.map(get_media_preview::v1::Response::from_raw_value)
 		.map_err(|error| {
 			err!(Request(Unknown(
-				debug_error!(%sender_user, %url, ?error, "Failed to parse URL preview.")
+				debug_error!(%sender_user, %url, "Failed to parse URL preview: {error}")
 			)))
 		})
 }

src/api/client/media_legacy.rs

@@ -8,6 +8,7 @@ use conduit::{
 	Err, Result,
 };
 use conduit_service::media::{Dim, FileMeta, CACHE_CONTROL_IMMUTABLE, CORP_CROSS_ORIGIN};
+use reqwest::Url;
 use ruma::{
 	api::client::media::{
 		create_content, get_content, get_content_as_filename, get_content_thumbnail, get_media_config,
@@ -55,25 +56,31 @@ pub(crate) async fn get_media_preview_legacy_route(
 	let sender_user = body.sender_user.as_ref().expect("user is authenticated");
 
 	let url = &body.url;
-	if !services.media.url_preview_allowed(url) {
+	let url = Url::parse(&body.url).map_err(|e| {
+		err!(Request(InvalidParam(
+			debug_warn!(%sender_user, %url, "Requested URL is not valid: {e}")
+		)))
+	})?;
+
+	if !services.media.url_preview_allowed(&url) {
 		return Err!(Request(Forbidden(
 			debug_warn!(%sender_user, %url, "URL is not allowed to be previewed")
 		)));
 	}
 
-	let preview = services.media.get_url_preview(url).await.map_err(|e| {
+	let preview = services.media.get_url_preview(&url).await.map_err(|e| {
 		err!(Request(Unknown(
 			debug_error!(%sender_user, %url, "Failed to fetch a URL preview: {e}")
 		)))
 	})?;
 
-	let res = serde_json::value::to_raw_value(&preview).map_err(|e| {
+	serde_json::value::to_raw_value(&preview)
-		err!(Request(Unknown(
+		.map(get_media_preview::v3::Response::from_raw_value)
-			debug_error!(%sender_user, %url, "Failed to parse a URL preview: {e}")
+		.map_err(|error| {
-		)))
+			err!(Request(Unknown(
-	})?;
+				debug_error!(%sender_user, %url, "Failed to parse URL preview: {error}")
-
+			)))
-	Ok(get_media_preview::v3::Response::from_raw_value(res))
+		})
 }
 
 /// # `GET /_matrix/media/v1/preview_url`

src/api/client/message.rs

@@ -1,109 +1,53 @@
-use std::collections::{BTreeMap, HashSet};
+use std::collections::HashSet;
 
 use axum::extract::State;
-use conduit::PduCount;
+use conduit::{
-use ruma::{
+	at, is_equal_to,
-	api::client::{
+	utils::{
-		error::ErrorKind,
+		result::{FlatOk, LogErr},
-		filter::{RoomEventFilter, UrlFilter},
+		stream::{BroadbandExt, WidebandExt},
-		message::{get_message_events, send_message_event},
+		IterStream, ReadyExt,
 	},
-	events::{MessageLikeEventType, StateEventType},
+	Event, PduCount, Result,
-	RoomId, UserId,
 };
-use serde_json::{from_str, Value};
+use futures::{FutureExt, StreamExt};
-
+use ruma::{
-use crate::{
+	api::{
-	service::{pdu::PduBuilder, Services},
+		client::{filter::RoomEventFilter, message::get_message_events},
-	utils, Error, PduEvent, Result, Ruma,
+		Direction,
+	},
+	events::{AnyStateEvent, StateEventType, TimelineEventType, TimelineEventType::*},
+	serde::Raw,
+	DeviceId, OwnedUserId, RoomId, UserId,
 };
+use service::{rooms::timeline::PdusIterItem, Services};
 
-/// # `PUT /_matrix/client/v3/rooms/{roomId}/send/{eventType}/{txnId}`
+use crate::Ruma;
-///
-/// Send a message event into the room.
-///
-/// - Is a NOOP if the txn id was already used before and returns the same event
-///   id again
-/// - The only requirement for the content is that it has to be valid json
-/// - Tries to send the event into the room, auth rules will determine if it is
-///   allowed
-pub(crate) async fn send_message_event_route(
-	State(services): State<crate::State>, body: Ruma<send_message_event::v3::Request>,
-) -> Result<send_message_event::v3::Response> {
-	let sender_user = body.sender_user.as_ref().expect("user is authenticated");
-	let sender_device = body.sender_device.as_deref();
 
-	let state_lock = services.rooms.state.mutex.lock(&body.room_id).await;
+pub(crate) type LazySet = HashSet<OwnedUserId>;
 
-	// Forbid m.room.encrypted if encryption is disabled
+/// list of safe and common non-state events to ignore if the user is ignored
-	if MessageLikeEventType::RoomEncrypted == body.event_type && !services.globals.allow_encryption() {
+const IGNORED_MESSAGE_TYPES: &[TimelineEventType; 16] = &[
-		return Err(Error::BadRequest(ErrorKind::forbidden(), "Encryption has been disabled"));
+	RoomMessage,
-	}
+	Sticker,
+	CallInvite,
+	CallNotify,
+	RoomEncrypted,
+	Image,
+	File,
+	Audio,
+	Voice,
+	Video,
+	UnstablePollStart,
+	PollStart,
+	KeyVerificationStart,
+	Reaction,
+	Emote,
+	Location,
+];
 
-	if body.event_type == MessageLikeEventType::CallInvite && services.rooms.directory.is_public_room(&body.room_id)? {
+const LIMIT_MAX: usize = 100;
-		return Err(Error::BadRequest(
+const LIMIT_DEFAULT: usize = 10;
-			ErrorKind::forbidden(),
-			"Room call invites are not allowed in public rooms",
-		));
-	}
-
-	// Check if this is a new transaction id
-	if let Some(response) = services
-		.transaction_ids
-		.existing_txnid(sender_user, sender_device, &body.txn_id)?
-	{
-		// The client might have sent a txnid of the /sendToDevice endpoint
-		// This txnid has no response associated with it
-		if response.is_empty() {
-			return Err(Error::BadRequest(
-				ErrorKind::InvalidParam,
-				"Tried to use txn id already used for an incompatible endpoint.",
-			));
-		}
-
-		let event_id = utils::string_from_bytes(&response)
-			.map_err(|_| Error::bad_database("Invalid txnid bytes in database."))?
-			.try_into()
-			.map_err(|_| Error::bad_database("Invalid event id in txnid data."))?;
-		return Ok(send_message_event::v3::Response {
-			event_id,
-		});
-	}
-
-	let mut unsigned = BTreeMap::new();
-	unsigned.insert("transaction_id".to_owned(), body.txn_id.to_string().into());
-
-	let event_id = services
-		.rooms
-		.timeline
-		.build_and_append_pdu(
-			PduBuilder {
-				event_type: body.event_type.to_string().into(),
-				content: from_str(body.body.body.json().get())
-					.map_err(|_| Error::BadRequest(ErrorKind::BadJson, "Invalid JSON body."))?,
-				unsigned: Some(unsigned),
-				state_key: None,
-				redacts: None,
-				timestamp: if body.appservice_info.is_some() {
-					body.timestamp
-				} else {
-					None
-				},
-			},
-			sender_user,
-			&body.room_id,
-			&state_lock,
-		)
-		.await?;
-
-	services
-		.transaction_ids
-		.add_txnid(sender_user, sender_device, &body.txn_id, event_id.as_bytes())?;
-
-	drop(state_lock);
-
-	Ok(send_message_event::v3::Response::new((*event_id).to_owned()))
-}
 
 /// # `GET /_matrix/client/r0/rooms/{roomId}/messages`
 ///
@@ -114,169 +58,184 @@ pub(crate) async fn send_message_event_route(
 pub(crate) async fn get_message_events_route(
 	State(services): State<crate::State>, body: Ruma<get_message_events::v3::Request>,
 ) -> Result<get_message_events::v3::Response> {
-	let sender_user = body.sender_user.as_ref().expect("user is authenticated");
+	let sender = body.sender();
-	let sender_device = body.sender_device.as_ref().expect("user is authenticated");
+	let (sender_user, sender_device) = sender;
+	let room_id = &body.room_id;
+	let filter = &body.filter;
 
-	let from = match body.from.clone() {
+	let from: PduCount = body
-		Some(from) => PduCount::try_from_string(&from)?,
+		.from
-		None => match body.dir {
+		.as_deref()
-			ruma::api::Direction::Forward => PduCount::min(),
+		.map(str::parse)
-			ruma::api::Direction::Backward => PduCount::max(),
+		.transpose()?
-		},
+		.unwrap_or_else(|| match body.dir {
-	};
+			Direction::Forward => PduCount::min(),
+			Direction::Backward => PduCount::max(),
+		});
 
-	let to = body
+	let to: Option<PduCount> = body.to.as_deref().map(str::parse).flat_ok();
-		.to
+
-		.as_ref()
+	let limit: usize = body
-		.and_then(|t| PduCount::try_from_string(t).ok());
+		.limit
+		.try_into()
+		.unwrap_or(LIMIT_DEFAULT)
+		.min(LIMIT_MAX);
 
 	services
 		.rooms
 		.lazy_loading
-		.lazy_load_confirm_delivery(sender_user, sender_device, &body.room_id, from)
+		.lazy_load_confirm_delivery(sender_user, sender_device, room_id, from);
-		.await?;
 
-	let limit = usize::try_from(body.limit).unwrap_or(10).min(100);
+	if matches!(body.dir, Direction::Backward) {
-
+		services
-	let next_token;
+			.rooms
-
+			.timeline
-	let mut resp = get_message_events::v3::Response::new();
+			.backfill_if_required(room_id, from)
-
+			.boxed()
-	let mut lazy_loaded = HashSet::new();
+			.await
-
+			.log_err()
-	match body.dir {
+			.ok();
-		ruma::api::Direction::Forward => {
-			let events_after: Vec<_> = services
-				.rooms
-				.timeline
-				.pdus_after(sender_user, &body.room_id, from)?
-				.filter_map(Result::ok) // Filter out buggy events
-				.filter(|(_, pdu)| { contains_url_filter(pdu, &body.filter) && visibility_filter(&services, pdu, sender_user, &body.room_id)
-
-				})
-				.take_while(|&(k, _)| Some(k) != to) // Stop at `to`
-				.take(limit)
-				.collect();
-
-			for (_, event) in &events_after {
-				/* TODO: Remove the not "element_hacks" check when these are resolved:
-				 * https://github.com/vector-im/element-android/issues/3417
-				 * https://github.com/vector-im/element-web/issues/21034
-				 */
-				if !cfg!(feature = "element_hacks")
-					&& !services.rooms.lazy_loading.lazy_load_was_sent_before(
-						sender_user,
-						sender_device,
-						&body.room_id,
-						&event.sender,
-					)? {
-					lazy_loaded.insert(event.sender.clone());
-				}
-
-				lazy_loaded.insert(event.sender.clone());
-			}
-
-			next_token = events_after.last().map(|(count, _)| count).copied();
-
-			let events_after: Vec<_> = events_after
-				.into_iter()
-				.map(|(_, pdu)| pdu.to_room_event())
-				.collect();
-
-			resp.start = from.stringify();
-			resp.end = next_token.map(|count| count.stringify());
-			resp.chunk = events_after;
-		},
-		ruma::api::Direction::Backward => {
-			services
-				.rooms
-				.timeline
-				.backfill_if_required(&body.room_id, from)
-				.await?;
-			let events_before: Vec<_> = services
-				.rooms
-				.timeline
-				.pdus_until(sender_user, &body.room_id, from)?
-				.filter_map(Result::ok) // Filter out buggy events
-				.filter(|(_, pdu)| {contains_url_filter(pdu, &body.filter) && visibility_filter(&services, pdu, sender_user, &body.room_id)})
-				.take_while(|&(k, _)| Some(k) != to) // Stop at `to`
-				.take(limit)
-				.collect();
-
-			for (_, event) in &events_before {
-				/* TODO: Remove the not "element_hacks" check when these are resolved:
-				 * https://github.com/vector-im/element-android/issues/3417
-				 * https://github.com/vector-im/element-web/issues/21034
-				 */
-				if !cfg!(feature = "element_hacks")
-					&& !services.rooms.lazy_loading.lazy_load_was_sent_before(
-						sender_user,
-						sender_device,
-						&body.room_id,
-						&event.sender,
-					)? {
-					lazy_loaded.insert(event.sender.clone());
-				}
-
-				lazy_loaded.insert(event.sender.clone());
-			}
-
-			next_token = events_before.last().map(|(count, _)| count).copied();
-
-			let events_before: Vec<_> = events_before
-				.into_iter()
-				.map(|(_, pdu)| pdu.to_room_event())
-				.collect();
-
-			resp.start = from.stringify();
-			resp.end = next_token.map(|count| count.stringify());
-			resp.chunk = events_before;
-		},
 	}
 
-	resp.state = Vec::new();
+	let it = match body.dir {
-	for ll_id in &lazy_loaded {
+		Direction::Forward => services
-		if let Some(member_event) =
+			.rooms
-			services
+			.timeline
-				.rooms
+			.pdus(Some(sender_user), room_id, Some(from))
-				.state_accessor
+			.await?
-				.room_state_get(&body.room_id, &StateEventType::RoomMember, ll_id.as_str())?
+			.boxed(),
-		{
+
-			resp.state.push(member_event.to_state_event());
+		Direction::Backward => services
-		}
+			.rooms
-	}
+			.timeline
+			.pdus_rev(Some(sender_user), room_id, Some(from))
+			.await?
+			.boxed(),
+	};
+
+	let events: Vec<_> = it
+		.ready_take_while(|(count, _)| Some(*count) != to)
+		.ready_filter_map(|item| event_filter(item, filter))
+		.wide_filter_map(|item| ignored_filter(&services, item, sender_user))
+		.wide_filter_map(|item| visibility_filter(&services, item, sender_user))
+		.take(limit)
+		.collect()
+		.await;
+
+	let lazy = events
+		.iter()
+		.stream()
+		.fold(LazySet::new(), |lazy, item| {
+			update_lazy(&services, room_id, sender, lazy, item, false)
+		})
+		.await;
+
+	let state = lazy
+		.iter()
+		.stream()
+		.broad_filter_map(|user_id| get_member_event(&services, room_id, user_id))
+		.collect()
+		.await;
+
+	let next_token = events.last().map(at!(0));
 
-	// remove the feature check when we are sure clients like element can handle it
 	if !cfg!(feature = "element_hacks") {
 		if let Some(next_token) = next_token {
 			services
 				.rooms
 				.lazy_loading
-				.lazy_load_mark_sent(sender_user, sender_device, &body.room_id, lazy_loaded, next_token)
+				.lazy_load_mark_sent(sender_user, sender_device, room_id, lazy, next_token);
-				.await;
 		}
 	}
 
-	Ok(resp)
+	let chunk = events
+		.into_iter()
+		.map(at!(1))
+		.map(|pdu| pdu.to_room_event())
+		.collect();
+
+	Ok(get_message_events::v3::Response {
+		start: from.to_string(),
+		end: next_token.as_ref().map(ToString::to_string),
+		chunk,
+		state,
+	})
 }
 
-fn visibility_filter(services: &Services, pdu: &PduEvent, user_id: &UserId, room_id: &RoomId) -> bool {
+async fn get_member_event(services: &Services, room_id: &RoomId, user_id: &UserId) -> Option<Raw<AnyStateEvent>> {
 	services
 		.rooms
 		.state_accessor
-		.user_can_see_event(user_id, room_id, &pdu.event_id)
+		.room_state_get(room_id, &StateEventType::RoomMember, user_id.as_str())
-		.unwrap_or(false)
+		.await
+		.map(|member_event| member_event.to_state_event())
+		.ok()
 }
 
-fn contains_url_filter(pdu: &PduEvent, filter: &RoomEventFilter) -> bool {
+pub(crate) async fn update_lazy(
-	if filter.url_filter.is_none() {
+	services: &Services, room_id: &RoomId, sender: (&UserId, &DeviceId), mut lazy: LazySet, item: &PdusIterItem,
-		return true;
+	force: bool,
+) -> LazySet {
+	let (_, event) = &item;
+	let (sender_user, sender_device) = sender;
+
+	/* TODO: Remove the not "element_hacks" check when these are resolved:
+	 * https://github.com/vector-im/element-android/issues/3417
+	 * https://github.com/vector-im/element-web/issues/21034
+	 */
+	if force || cfg!(features = "element_hacks") {
+		lazy.insert(event.sender().into());
+		return lazy;
 	}
 
-	let content: Value = from_str(pdu.content.get()).unwrap();
+	if lazy.contains(event.sender()) {
-	match filter.url_filter {
+		return lazy;
-		Some(UrlFilter::EventsWithoutUrl) => !content["url"].is_string(),
-		Some(UrlFilter::EventsWithUrl) => content["url"].is_string(),
-		None => true,
 	}
+
+	if !services
+		.rooms
+		.lazy_loading
+		.lazy_load_was_sent_before(sender_user, sender_device, room_id, event.sender())
+		.await
+	{
+		lazy.insert(event.sender().into());
+	}
+
+	lazy
+}
+
+pub(crate) async fn ignored_filter(services: &Services, item: PdusIterItem, user_id: &UserId) -> Option<PdusIterItem> {
+	let (_, pdu) = &item;
+
+	// exclude Synapse's dummy events from bloating up response bodies. clients
+	// don't need to see this.
+	if pdu.kind.to_cow_str() == "org.matrix.dummy_event" {
+		return None;
+	}
+
+	if IGNORED_MESSAGE_TYPES.iter().any(is_equal_to!(&pdu.kind))
+		&& services.users.user_is_ignored(&pdu.sender, user_id).await
+	{
+		return None;
+	}
+
+	Some(item)
+}
+
+pub(crate) async fn visibility_filter(
+	services: &Services, item: PdusIterItem, user_id: &UserId,
+) -> Option<PdusIterItem> {
+	let (_, pdu) = &item;
+
+	services
+		.rooms
+		.state_accessor
+		.user_can_see_event(user_id, &pdu.room_id, &pdu.event_id)
+		.await
+		.then_some(item)
+}
+
+pub(crate) fn event_filter(item: PdusIterItem, filter: &RoomEventFilter) -> Option<PdusIterItem> {
+	let (_, pdu) = &item;
+	pdu.matches(filter).then_some(item)
 }

src/api/client/mod.rs

@@ -1,5 +1,6 @@
 pub(super) mod account;
 pub(super) mod alias;
+pub(super) mod appservice;
 pub(super) mod backup;
 pub(super) mod capabilities;
 pub(super) mod config;
@@ -22,6 +23,7 @@ pub(super) mod relations;
 pub(super) mod report;
 pub(super) mod room;
 pub(super) mod search;
+pub(super) mod send;
 pub(super) mod session;
 pub(super) mod space;
 pub(super) mod state;
@@ -35,9 +37,12 @@ pub(super) mod unstable;
 pub(super) mod unversioned;
 pub(super) mod user_directory;
 pub(super) mod voip;
+pub(super) mod well_known;
 
+pub use account::full_user_deactivate;
 pub(super) use account::*;
 pub(super) use alias::*;
+pub(super) use appservice::*;
 pub(super) use backup::*;
 pub(super) use capabilities::*;
 pub(super) use config::*;
@@ -49,7 +54,7 @@ pub(super) use keys::*;
 pub(super) use media::*;
 pub(super) use media_legacy::*;
 pub(super) use membership::*;
-pub use membership::{join_room_by_id_helper, leave_all_rooms, leave_room, validate_and_add_event_id};
+pub use membership::{join_room_by_id_helper, leave_all_rooms, leave_room};
 pub(super) use message::*;
 pub(super) use openid::*;
 pub(super) use presence::*;
@@ -62,6 +67,7 @@ pub(super) use relations::*;
 pub(super) use report::*;
 pub(super) use room::*;
 pub(super) use search::*;
+pub(super) use send::*;
 pub(super) use session::*;
 pub(super) use space::*;
 pub(super) use state::*;
@@ -75,6 +81,7 @@ pub(super) use unstable::*;
 pub(super) use unversioned::*;
 pub(super) use user_directory::*;
 pub(super) use voip::*;
+pub(super) use well_known::*;
 
 /// generated device ID length
 const DEVICE_ID_LENGTH: usize = 10;

src/api/client/presence.rs

@@ -28,7 +28,8 @@ pub(crate) async fn set_presence_route(
 
 	services
 		.presence
-		.set_presence(sender_user, &body.presence, None, None, body.status_msg.clone())?;
+		.set_presence(sender_user, &body.presence, None, None, body.status_msg.clone())
+		.await?;
 
 	Ok(set_presence::v3::Response {})
 }
@@ -49,14 +50,15 @@ pub(crate) async fn get_presence_route(
 
 	let mut presence_event = None;
 
-	for _room_id in services
+	let has_shared_rooms = services
 		.rooms
-		.user
+		.state_cache
-		.get_shared_rooms(vec![sender_user.clone(), body.user_id.clone()])?
+		.user_sees_user(sender_user, &body.user_id)
-	{
+		.await;
-		if let Some(presence) = services.presence.get_presence(&body.user_id)? {
+
+	if has_shared_rooms {
+		if let Ok(presence) = services.presence.get_presence(&body.user_id).await {
 			presence_event = Some(presence);
-			break;
 		}
 	}
 

src/api/client/profile.rs

@@ -1,5 +1,10 @@
 use axum::extract::State;
-use conduit::{pdu::PduBuilder, warn, Error, Result};
+use conduit::{
+	pdu::PduBuilder,
+	utils::{stream::TryIgnore, IterStream},
+	warn, Err, Error, Result,
+};
+use futures::{StreamExt, TryStreamExt};
 use ruma::{
 	api::{
 		client::{
@@ -8,11 +13,10 @@ use ruma::{
 		},
 		federation,
 	},
-	events::{room::member::RoomMemberEventContent, StateEventType, TimelineEventType},
+	events::{room::member::RoomMemberEventContent, StateEventType},
 	presence::PresenceState,
-	OwnedMxcUri, OwnedRoomId, OwnedUserId,
+	OwnedMxcUri, OwnedRoomId, UserId,
 };
-use serde_json::value::to_raw_value;
 use service::Services;
 
 use crate::Ruma;
@@ -26,20 +30,27 @@ pub(crate) async fn set_displayname_route(
 	State(services): State<crate::State>, body: Ruma<set_display_name::v3::Request>,
 ) -> Result<set_display_name::v3::Response> {
 	let sender_user = body.sender_user.as_ref().expect("user is authenticated");
+
+	if *sender_user != body.user_id && body.appservice_info.is_none() {
+		return Err!(Request(Forbidden("You cannot update the profile of another user")));
+	}
+
 	let all_joined_rooms: Vec<OwnedRoomId> = services
 		.rooms
 		.state_cache
-		.rooms_joined(sender_user)
+		.rooms_joined(&body.user_id)
-		.filter_map(Result::ok)
+		.map(ToOwned::to_owned)
-		.collect();
+		.collect()
+		.await;
 
-	update_displayname(&services, sender_user.clone(), body.displayname.clone(), all_joined_rooms).await?;
+	update_displayname(&services, &body.user_id, body.displayname.clone(), &all_joined_rooms).await?;
 
 	if services.globals.allow_local_presence() {
 		// Presence update
 		services
 			.presence
-			.ping_presence(sender_user, &PresenceState::Online)?;
+			.ping_presence(&body.user_id, &PresenceState::Online)
+			.await?;
 	}
 
 	Ok(set_display_name::v3::Response {})
@@ -67,22 +78,19 @@ pub(crate) async fn get_displayname_route(
 			)
 			.await
 		{
-			if !services.users.exists(&body.user_id)? {
+			if !services.users.exists(&body.user_id).await {
 				services.users.create(&body.user_id, None)?;
 			}
 
 			services
 				.users
-				.set_displayname(&body.user_id, response.displayname.clone())
+				.set_displayname(&body.user_id, response.displayname.clone());
-				.await?;
 			services
 				.users
-				.set_avatar_url(&body.user_id, response.avatar_url.clone())
+				.set_avatar_url(&body.user_id, response.avatar_url.clone());
-				.await?;
 			services
 				.users
-				.set_blurhash(&body.user_id, response.blurhash.clone())
+				.set_blurhash(&body.user_id, response.blurhash.clone());
-				.await?;
 
 			return Ok(get_display_name::v3::Response {
 				displayname: response.displayname,
@@ -90,14 +98,14 @@ pub(crate) async fn get_displayname_route(
 		}
 	}
 
-	if !services.users.exists(&body.user_id)? {
+	if !services.users.exists(&body.user_id).await {
 		// Return 404 if this user doesn't exist and we couldn't fetch it over
 		// federation
 		return Err(Error::BadRequest(ErrorKind::NotFound, "Profile was not found."));
 	}
 
 	Ok(get_display_name::v3::Response {
-		displayname: services.users.displayname(&body.user_id)?,
+		displayname: services.users.displayname(&body.user_id).await.ok(),
 	})
 }
 
@@ -110,19 +118,25 @@ pub(crate) async fn set_avatar_url_route(
 	State(services): State<crate::State>, body: Ruma<set_avatar_url::v3::Request>,
 ) -> Result<set_avatar_url::v3::Response> {
 	let sender_user = body.sender_user.as_ref().expect("user is authenticated");
+
+	if *sender_user != body.user_id && body.appservice_info.is_none() {
+		return Err!(Request(Forbidden("You cannot update the profile of another user")));
+	}
+
 	let all_joined_rooms: Vec<OwnedRoomId> = services
 		.rooms
 		.state_cache
-		.rooms_joined(sender_user)
+		.rooms_joined(&body.user_id)
-		.filter_map(Result::ok)
+		.map(ToOwned::to_owned)
-		.collect();
+		.collect()
+		.await;
 
 	update_avatar_url(
 		&services,
-		sender_user.clone(),
+		&body.user_id,
 		body.avatar_url.clone(),
 		body.blurhash.clone(),
-		all_joined_rooms,
+		&all_joined_rooms,
 	)
 	.await?;
 
@@ -130,7 +144,9 @@ pub(crate) async fn set_avatar_url_route(
 		// Presence update
 		services
 			.presence
-			.ping_presence(sender_user, &PresenceState::Online)?;
+			.ping_presence(&body.user_id, &PresenceState::Online)
+			.await
+			.ok();
 	}
 
 	Ok(set_avatar_url::v3::Response {})
@@ -158,22 +174,21 @@ pub(crate) async fn get_avatar_url_route(
 			)
 			.await
 		{
-			if !services.users.exists(&body.user_id)? {
+			if !services.users.exists(&body.user_id).await {
 				services.users.create(&body.user_id, None)?;
 			}
 
 			services
 				.users
-				.set_displayname(&body.user_id, response.displayname.clone())
+				.set_displayname(&body.user_id, response.displayname.clone());
-				.await?;
+
 			services
 				.users
-				.set_avatar_url(&body.user_id, response.avatar_url.clone())
+				.set_avatar_url(&body.user_id, response.avatar_url.clone());
-				.await?;
+
 			services
 				.users
-				.set_blurhash(&body.user_id, response.blurhash.clone())
+				.set_blurhash(&body.user_id, response.blurhash.clone());
-				.await?;
 
 			return Ok(get_avatar_url::v3::Response {
 				avatar_url: response.avatar_url,
@@ -182,21 +197,21 @@ pub(crate) async fn get_avatar_url_route(
 		}
 	}
 
-	if !services.users.exists(&body.user_id)? {
+	if !services.users.exists(&body.user_id).await {
 		// Return 404 if this user doesn't exist and we couldn't fetch it over
 		// federation
 		return Err(Error::BadRequest(ErrorKind::NotFound, "Profile was not found."));
 	}
 
 	Ok(get_avatar_url::v3::Response {
-		avatar_url: services.users.avatar_url(&body.user_id)?,
+		avatar_url: services.users.avatar_url(&body.user_id).await.ok(),
-		blurhash: services.users.blurhash(&body.user_id)?,
+		blurhash: services.users.blurhash(&body.user_id).await.ok(),
 	})
 }
 
 /// # `GET /_matrix/client/v3/profile/{userId}`
 ///
-/// Returns the displayname, avatar_url and blurhash of the user.
+/// Returns the displayname, avatar_url, blurhash, and tz of the user.
 ///
 /// - If user is on another server and we do not have a local copy already,
 ///   fetch profile over federation.
@@ -216,153 +231,144 @@ pub(crate) async fn get_profile_route(
 			)
 			.await
 		{
-			if !services.users.exists(&body.user_id)? {
+			if !services.users.exists(&body.user_id).await {
 				services.users.create(&body.user_id, None)?;
 			}
 
 			services
 				.users
-				.set_displayname(&body.user_id, response.displayname.clone())
+				.set_displayname(&body.user_id, response.displayname.clone());
-				.await?;
+
 			services
 				.users
-				.set_avatar_url(&body.user_id, response.avatar_url.clone())
+				.set_avatar_url(&body.user_id, response.avatar_url.clone());
-				.await?;
+
 			services
 				.users
-				.set_blurhash(&body.user_id, response.blurhash.clone())
+				.set_blurhash(&body.user_id, response.blurhash.clone());
-				.await?;
+
+			services
+				.users
+				.set_timezone(&body.user_id, response.tz.clone());
+
+			for (profile_key, profile_key_value) in &response.custom_profile_fields {
+				services
+					.users
+					.set_profile_key(&body.user_id, profile_key, Some(profile_key_value.clone()));
+			}
 
 			return Ok(get_profile::v3::Response {
 				displayname: response.displayname,
 				avatar_url: response.avatar_url,
 				blurhash: response.blurhash,
+				tz: response.tz,
+				custom_profile_fields: response.custom_profile_fields,
 			});
 		}
 	}
 
-	if !services.users.exists(&body.user_id)? {
+	if !services.users.exists(&body.user_id).await {
 		// Return 404 if this user doesn't exist and we couldn't fetch it over
 		// federation
 		return Err(Error::BadRequest(ErrorKind::NotFound, "Profile was not found."));
 	}
 
 	Ok(get_profile::v3::Response {
-		avatar_url: services.users.avatar_url(&body.user_id)?,
+		avatar_url: services.users.avatar_url(&body.user_id).await.ok(),
-		blurhash: services.users.blurhash(&body.user_id)?,
+		blurhash: services.users.blurhash(&body.user_id).await.ok(),
-		displayname: services.users.displayname(&body.user_id)?,
+		displayname: services.users.displayname(&body.user_id).await.ok(),
+		tz: services.users.timezone(&body.user_id).await.ok(),
+		custom_profile_fields: services
+			.users
+			.all_profile_keys(&body.user_id)
+			.collect()
+			.await,
 	})
 }
 
 pub async fn update_displayname(
-	services: &Services, user_id: OwnedUserId, displayname: Option<String>, all_joined_rooms: Vec<OwnedRoomId>,
+	services: &Services, user_id: &UserId, displayname: Option<String>, all_joined_rooms: &[OwnedRoomId],
 ) -> Result<()> {
-	let current_display_name = services.users.displayname(&user_id).unwrap_or_default();
+	let current_display_name = services.users.displayname(user_id).await.ok();
 
 	if displayname == current_display_name {
 		return Ok(());
 	}
 
-	services
+	services.users.set_displayname(user_id, displayname.clone());
-		.users
-		.set_displayname(&user_id, displayname.clone())
-		.await?;
 
 	// Send a new join membership event into all joined rooms
-	let all_joined_rooms: Vec<_> = all_joined_rooms
+	let mut joined_rooms = Vec::new();
-		.iter()
+	for room_id in all_joined_rooms {
-		.map(|room_id| {
+		let Ok(content) = services
-			Ok::<_, Error>((
+			.rooms
-				PduBuilder {
+			.state_accessor
-					event_type: TimelineEventType::RoomMember,
+			.room_state_get_content(room_id, &StateEventType::RoomMember, user_id.as_str())
-					content: to_raw_value(&RoomMemberEventContent {
+			.await
-						displayname: displayname.clone(),
+		else {
-						join_authorized_via_users_server: None,
+			continue;
-						..serde_json::from_str(
+		};
-							services
-								.rooms
-								.state_accessor
-								.room_state_get(room_id, &StateEventType::RoomMember, user_id.as_str())?
-								.ok_or_else(|| {
-									Error::bad_database("Tried to send display name update for user not in the room.")
-								})?
-								.content
-								.get(),
-						)
-						.map_err(|_| Error::bad_database("Database contains invalid PDU."))?
-					})
-					.expect("event is valid, we just created it"),
-					unsigned: None,
-					state_key: Some(user_id.to_string()),
-					redacts: None,
-					timestamp: None,
-				},
-				room_id,
-			))
-		})
-		.filter_map(Result::ok)
-		.collect();
 
-	update_all_rooms(services, all_joined_rooms, user_id).await;
+		let pdu = PduBuilder::state(
+			user_id.to_string(),
+			&RoomMemberEventContent {
+				displayname: displayname.clone(),
+				join_authorized_via_users_server: None,
+				..content
+			},
+		);
+
+		joined_rooms.push((pdu, room_id));
+	}
+
+	update_all_rooms(services, joined_rooms, user_id).await;
 
 	Ok(())
 }
 
 pub async fn update_avatar_url(
-	services: &Services, user_id: OwnedUserId, avatar_url: Option<OwnedMxcUri>, blurhash: Option<String>,
+	services: &Services, user_id: &UserId, avatar_url: Option<OwnedMxcUri>, blurhash: Option<String>,
-	all_joined_rooms: Vec<OwnedRoomId>,
+	all_joined_rooms: &[OwnedRoomId],
 ) -> Result<()> {
-	let current_avatar_url = services.users.avatar_url(&user_id).unwrap_or_default();
+	let current_avatar_url = services.users.avatar_url(user_id).await.ok();
-	let current_blurhash = services.users.blurhash(&user_id).unwrap_or_default();
+	let current_blurhash = services.users.blurhash(user_id).await.ok();
 
 	if current_avatar_url == avatar_url && current_blurhash == blurhash {
 		return Ok(());
 	}
 
-	services
+	services.users.set_avatar_url(user_id, avatar_url.clone());
-		.users
+
-		.set_avatar_url(&user_id, avatar_url.clone())
+	services.users.set_blurhash(user_id, blurhash.clone());
-		.await?;
-	services
-		.users
-		.set_blurhash(&user_id, blurhash.clone())
-		.await?;
 
 	// Send a new join membership event into all joined rooms
+	let avatar_url = &avatar_url;
+	let blurhash = &blurhash;
 	let all_joined_rooms: Vec<_> = all_joined_rooms
 		.iter()
-		.map(|room_id| {
+		.try_stream()
-			Ok::<_, Error>((
+		.and_then(|room_id: &OwnedRoomId| async move {
-				PduBuilder {
+			let content = services
-					event_type: TimelineEventType::RoomMember,
+				.rooms
-					content: to_raw_value(&RoomMemberEventContent {
+				.state_accessor
-						avatar_url: avatar_url.clone(),
+				.room_state_get_content(room_id, &StateEventType::RoomMember, user_id.as_str())
-						blurhash: blurhash.clone(),
+				.await?;
-						join_authorized_via_users_server: None,
+
-						..serde_json::from_str(
+			let pdu = PduBuilder::state(
-							services
+				user_id.to_string(),
-								.rooms
+				&RoomMemberEventContent {
-								.state_accessor
+					avatar_url: avatar_url.clone(),
-								.room_state_get(room_id, &StateEventType::RoomMember, user_id.as_str())?
+					blurhash: blurhash.clone(),
-								.ok_or_else(|| {
+					join_authorized_via_users_server: None,
-									Error::bad_database("Tried to send avatar URL update for user not in the room.")
+					..content
-								})?
-								.content
-								.get(),
-						)
-						.map_err(|_| Error::bad_database("Database contains invalid PDU."))?
-					})
-					.expect("event is valid, we just created it"),
-					unsigned: None,
-					state_key: Some(user_id.to_string()),
-					redacts: None,
-					timestamp: None,
 				},
-				room_id,
+			);
-			))
+
+			Ok((pdu, room_id))
 		})
-		.filter_map(Result::ok)
+		.ignore_err()
-		.collect();
+		.collect()
+		.await;
 
 	update_all_rooms(services, all_joined_rooms, user_id).await;
 
@@ -370,14 +376,14 @@ pub async fn update_avatar_url(
 }
 
 pub async fn update_all_rooms(
-	services: &Services, all_joined_rooms: Vec<(PduBuilder, &OwnedRoomId)>, user_id: OwnedUserId,
+	services: &Services, all_joined_rooms: Vec<(PduBuilder, &OwnedRoomId)>, user_id: &UserId,
 ) {
 	for (pdu_builder, room_id) in all_joined_rooms {
 		let state_lock = services.rooms.state.mutex.lock(room_id).await;
 		if let Err(e) = services
 			.rooms
 			.timeline
-			.build_and_append_pdu(pdu_builder, &user_id, room_id, &state_lock)
+			.build_and_append_pdu(pdu_builder, user_id, room_id, &state_lock)
 			.await
 		{
 			warn!(%user_id, %room_id, %e, "Failed to update/send new profile join membership update in room");

src/api/client/push.rs

@@ -1,19 +1,19 @@
 use axum::extract::State;
-use conduit::err;
+use conduit::{err, Err};
 use ruma::{
 	api::client::{
 		error::ErrorKind,
 		push::{
 			delete_pushrule, get_pushers, get_pushrule, get_pushrule_actions, get_pushrule_enabled, get_pushrules_all,
-			set_pusher, set_pushrule, set_pushrule_actions, set_pushrule_enabled, RuleScope,
+			get_pushrules_global_scope, set_pusher, set_pushrule, set_pushrule_actions, set_pushrule_enabled,
 		},
 	},
 	events::{
 		push_rules::{PushRulesEvent, PushRulesEventContent},
 		GlobalAccountDataEventType,
 	},
-	push::{InsertPushRuleError, RemovePushRuleError, Ruleset},
+	push::{InsertPushRuleError, PredefinedContentRuleId, PredefinedOverrideRuleId, RemovePushRuleError, Ruleset},
-	CanonicalJsonObject,
+	CanonicalJsonObject, CanonicalJsonValue,
 };
 use service::Services;
 
@@ -27,48 +27,113 @@ pub(crate) async fn get_pushrules_all_route(
 ) -> Result<get_pushrules_all::v3::Response> {
 	let sender_user = body.sender_user.as_ref().expect("user is authenticated");
 
-	let global_ruleset: Ruleset;
+	let Some(content_value) = services
-
+		.account_data
-	let Ok(event) =
+		.get_global::<CanonicalJsonObject>(sender_user, GlobalAccountDataEventType::PushRules)
-		services
+		.await
-			.account_data
+		.ok()
-			.get(None, sender_user, GlobalAccountDataEventType::PushRules.to_string().into())
+		.and_then(|event| event.get("content").cloned())
+		.filter(CanonicalJsonValue::is_object)
 	else {
-		// push rules event doesn't exist, create it and return default
-		return recreate_push_rules_and_return(&services, sender_user);
-	};
-
-	if let Some(event) = event {
-		let value = serde_json::from_str::<CanonicalJsonObject>(event.get())
-			.map_err(|e| err!(Database(warn!("Invalid push rules account data event in database: {e}"))))?;
-
-		let Some(content_value) = value.get("content") else {
-			// user somehow has a push rule event with no content key, recreate it and
-			// return server default silently
-			return recreate_push_rules_and_return(&services, sender_user);
-		};
-
-		if content_value.to_string().is_empty() {
-			// user somehow has a push rule event with empty content, recreate it and return
-			// server default silently
-			return recreate_push_rules_and_return(&services, sender_user);
-		}
-
-		let account_data_content = serde_json::from_value::<PushRulesEventContent>(content_value.clone().into())
-			.map_err(|e| err!(Database(warn!("Invalid push rules account data event in database: {e}"))))?;
-
-		global_ruleset = account_data_content.global;
-	} else {
 		// user somehow has non-existent push rule event. recreate it and return server
 		// default silently
-		return recreate_push_rules_and_return(&services, sender_user);
+		return recreate_push_rules_and_return(&services, sender_user).await;
-	}
+	};
+
+	let account_data_content = serde_json::from_value::<PushRulesEventContent>(content_value.into())
+		.map_err(|e| err!(Database(warn!("Invalid push rules account data event in database: {e}"))))?;
+
+	let mut global_ruleset = account_data_content.global;
+
+	// remove old deprecated mentions push rules as per MSC4210
+	#[allow(deprecated)]
+	{
+		use ruma::push::RuleKind::*;
+
+		global_ruleset
+			.remove(Override, PredefinedOverrideRuleId::ContainsDisplayName)
+			.ok();
+		global_ruleset
+			.remove(Override, PredefinedOverrideRuleId::RoomNotif)
+			.ok();
+
+		global_ruleset
+			.remove(Content, PredefinedContentRuleId::ContainsUserName)
+			.ok();
+	};
 
 	Ok(get_pushrules_all::v3::Response {
 		global: global_ruleset,
 	})
 }
 
+/// # `GET /_matrix/client/r0/pushrules/global/`
+///
+/// Retrieves the push rules event for this user.
+///
+/// This appears to be the exact same as `GET /_matrix/client/r0/pushrules/`.
+pub(crate) async fn get_pushrules_global_route(
+	State(services): State<crate::State>, body: Ruma<get_pushrules_global_scope::v3::Request>,
+) -> Result<get_pushrules_global_scope::v3::Response> {
+	let sender_user = body.sender_user.as_ref().expect("user is authenticated");
+
+	let Some(content_value) = services
+		.account_data
+		.get_global::<CanonicalJsonObject>(sender_user, GlobalAccountDataEventType::PushRules)
+		.await
+		.ok()
+		.and_then(|event| event.get("content").cloned())
+		.filter(CanonicalJsonValue::is_object)
+	else {
+		// user somehow has non-existent push rule event. recreate it and return server
+		// default silently
+		services
+			.account_data
+			.update(
+				None,
+				sender_user,
+				GlobalAccountDataEventType::PushRules.to_string().into(),
+				&serde_json::to_value(PushRulesEvent {
+					content: PushRulesEventContent {
+						global: Ruleset::server_default(sender_user),
+					},
+				})
+				.expect("to json always works"),
+			)
+			.await?;
+
+		return Ok(get_pushrules_global_scope::v3::Response {
+			global: Ruleset::server_default(sender_user),
+		});
+	};
+
+	let account_data_content = serde_json::from_value::<PushRulesEventContent>(content_value.into())
+		.map_err(|e| err!(Database(warn!("Invalid push rules account data event in database: {e}"))))?;
+
+	let mut global_ruleset = account_data_content.global;
+
+	// remove old deprecated mentions push rules as per MSC4210
+	#[allow(deprecated)]
+	{
+		use ruma::push::RuleKind::*;
+
+		global_ruleset
+			.remove(Override, PredefinedOverrideRuleId::ContainsDisplayName)
+			.ok();
+		global_ruleset
+			.remove(Override, PredefinedOverrideRuleId::RoomNotif)
+			.ok();
+
+		global_ruleset
+			.remove(Content, PredefinedContentRuleId::ContainsUserName)
+			.ok();
+	};
+
+	Ok(get_pushrules_global_scope::v3::Response {
+		global: global_ruleset,
+	})
+}
+
 /// # `GET /_matrix/client/r0/pushrules/{scope}/{kind}/{ruleId}`
 ///
 /// Retrieves a single specified push rule for this user.
@@ -77,16 +142,23 @@ pub(crate) async fn get_pushrule_route(
 ) -> Result<get_pushrule::v3::Response> {
 	let sender_user = body.sender_user.as_ref().expect("user is authenticated");
 
-	let event = services
+	// remove old deprecated mentions push rules as per MSC4210
+	#[allow(deprecated)]
+	if body.rule_id.as_str() == PredefinedContentRuleId::ContainsUserName.as_str()
+		|| body.rule_id.as_str() == PredefinedOverrideRuleId::ContainsDisplayName.as_str()
+		|| body.rule_id.as_str() == PredefinedOverrideRuleId::RoomNotif.as_str()
+	{
+		return Err!(Request(NotFound("Push rule not found.")));
+	}
+
+	let event: PushRulesEvent = services
 		.account_data
-		.get(None, sender_user, GlobalAccountDataEventType::PushRules.to_string().into())?
+		.get_global(sender_user, GlobalAccountDataEventType::PushRules)
-		.ok_or(Error::BadRequest(ErrorKind::NotFound, "PushRules event not found."))?;
+		.await
+		.map_err(|_| err!(Request(NotFound("PushRules event not found."))))?;
 
-	let account_data = serde_json::from_str::<PushRulesEvent>(event.get())
+	let rule = event
-		.map_err(|_| Error::bad_database("Invalid account data event in db."))?
+		.content
-		.content;
-
-	let rule = account_data
 		.global
 		.get(body.kind.clone(), &body.rule_id)
 		.map(Into::into);
@@ -100,7 +172,7 @@ pub(crate) async fn get_pushrule_route(
 	}
 }
 
-/// # `PUT /_matrix/client/r0/pushrules/{scope}/{kind}/{ruleId}`
+/// # `PUT /_matrix/client/r0/pushrules/global/{kind}/{ruleId}`
 ///
 /// Creates a single specified push rule for this user.
 pub(crate) async fn set_pushrule_route(
@@ -109,20 +181,11 @@ pub(crate) async fn set_pushrule_route(
 	let sender_user = body.sender_user.as_ref().expect("user is authenticated");
 	let body = body.body;
 
-	if body.scope != RuleScope::Global {
+	let mut account_data: PushRulesEvent = services
-		return Err(Error::BadRequest(
-			ErrorKind::InvalidParam,
-			"Scopes other than 'global' are not supported.",
-		));
-	}
-
-	let event = services
 		.account_data
-		.get(None, sender_user, GlobalAccountDataEventType::PushRules.to_string().into())?
+		.get_global(sender_user, GlobalAccountDataEventType::PushRules)
-		.ok_or(Error::BadRequest(ErrorKind::NotFound, "PushRules event not found."))?;
+		.await
-
+		.map_err(|_| err!(Request(NotFound("PushRules event not found."))))?;
-	let mut account_data = serde_json::from_str::<PushRulesEvent>(event.get())
-		.map_err(|_| Error::bad_database("Invalid account data event in db."))?;
 
 	if let Err(error) =
 		account_data
@@ -155,17 +218,20 @@ pub(crate) async fn set_pushrule_route(
 		return Err(err);
 	}
 
-	services.account_data.update(
+	services
-		None,
+		.account_data
-		sender_user,
+		.update(
-		GlobalAccountDataEventType::PushRules.to_string().into(),
+			None,
-		&serde_json::to_value(account_data).expect("to json value always works"),
+			sender_user,
-	)?;
+			GlobalAccountDataEventType::PushRules.to_string().into(),
+			&serde_json::to_value(account_data).expect("to json value always works"),
+		)
+		.await?;
 
 	Ok(set_pushrule::v3::Response {})
 }
 
-/// # `GET /_matrix/client/r0/pushrules/{scope}/{kind}/{ruleId}/actions`
+/// # `GET /_matrix/client/r0/pushrules/global/{kind}/{ruleId}/actions`
 ///
 /// Gets the actions of a single specified push rule for this user.
 pub(crate) async fn get_pushrule_actions_route(
@@ -173,34 +239,34 @@ pub(crate) async fn get_pushrule_actions_route(
 ) -> Result<get_pushrule_actions::v3::Response> {
 	let sender_user = body.sender_user.as_ref().expect("user is authenticated");
 
-	if body.scope != RuleScope::Global {
+	// remove old deprecated mentions push rules as per MSC4210
-		return Err(Error::BadRequest(
+	#[allow(deprecated)]
-			ErrorKind::InvalidParam,
+	if body.rule_id.as_str() == PredefinedContentRuleId::ContainsUserName.as_str()
-			"Scopes other than 'global' are not supported.",
+		|| body.rule_id.as_str() == PredefinedOverrideRuleId::ContainsDisplayName.as_str()
-		));
+		|| body.rule_id.as_str() == PredefinedOverrideRuleId::RoomNotif.as_str()
+	{
+		return Err!(Request(NotFound("Push rule not found.")));
 	}
 
-	let event = services
+	let event: PushRulesEvent = services
 		.account_data
-		.get(None, sender_user, GlobalAccountDataEventType::PushRules.to_string().into())?
+		.get_global(sender_user, GlobalAccountDataEventType::PushRules)
-		.ok_or(Error::BadRequest(ErrorKind::NotFound, "PushRules event not found."))?;
+		.await
+		.map_err(|_| err!(Request(NotFound("PushRules event not found."))))?;
 
-	let account_data = serde_json::from_str::<PushRulesEvent>(event.get())
+	let actions = event
-		.map_err(|_| Error::bad_database("Invalid account data event in db."))?
+		.content
-		.content;
+		.global
-
-	let global = account_data.global;
-	let actions = global
 		.get(body.kind.clone(), &body.rule_id)
 		.map(|rule| rule.actions().to_owned())
-		.ok_or(Error::BadRequest(ErrorKind::NotFound, "Push rule not found."))?;
+		.ok_or(err!(Request(NotFound("Push rule not found."))))?;
 
 	Ok(get_pushrule_actions::v3::Response {
 		actions,
 	})
 }
 
-/// # `PUT /_matrix/client/r0/pushrules/{scope}/{kind}/{ruleId}/actions`
+/// # `PUT /_matrix/client/r0/pushrules/global/{kind}/{ruleId}/actions`
 ///
 /// Sets the actions of a single specified push rule for this user.
 pub(crate) async fn set_pushrule_actions_route(
@@ -208,20 +274,11 @@ pub(crate) async fn set_pushrule_actions_route(
 ) -> Result<set_pushrule_actions::v3::Response> {
 	let sender_user = body.sender_user.as_ref().expect("user is authenticated");
 
-	if body.scope != RuleScope::Global {
+	let mut account_data: PushRulesEvent = services
-		return Err(Error::BadRequest(
-			ErrorKind::InvalidParam,
-			"Scopes other than 'global' are not supported.",
-		));
-	}
-
-	let event = services
 		.account_data
-		.get(None, sender_user, GlobalAccountDataEventType::PushRules.to_string().into())?
+		.get_global(sender_user, GlobalAccountDataEventType::PushRules)
-		.ok_or(Error::BadRequest(ErrorKind::NotFound, "PushRules event not found."))?;
+		.await
-
+		.map_err(|_| err!(Request(NotFound("PushRules event not found."))))?;
-	let mut account_data = serde_json::from_str::<PushRulesEvent>(event.get())
-		.map_err(|_| Error::bad_database("Invalid account data event in db."))?;
 
 	if account_data
 		.content
@@ -232,17 +289,20 @@ pub(crate) async fn set_pushrule_actions_route(
 		return Err(Error::BadRequest(ErrorKind::NotFound, "Push rule not found."));
 	}
 
-	services.account_data.update(
+	services
-		None,
+		.account_data
-		sender_user,
+		.update(
-		GlobalAccountDataEventType::PushRules.to_string().into(),
+			None,
-		&serde_json::to_value(account_data).expect("to json value always works"),
+			sender_user,
-	)?;
+			GlobalAccountDataEventType::PushRules.to_string().into(),
+			&serde_json::to_value(account_data).expect("to json value always works"),
+		)
+		.await?;
 
 	Ok(set_pushrule_actions::v3::Response {})
 }
 
-/// # `GET /_matrix/client/r0/pushrules/{scope}/{kind}/{ruleId}/enabled`
+/// # `GET /_matrix/client/r0/pushrules/global/{kind}/{ruleId}/enabled`
 ///
 /// Gets the enabled status of a single specified push rule for this user.
 pub(crate) async fn get_pushrule_enabled_route(
@@ -250,33 +310,36 @@ pub(crate) async fn get_pushrule_enabled_route(
 ) -> Result<get_pushrule_enabled::v3::Response> {
 	let sender_user = body.sender_user.as_ref().expect("user is authenticated");
 
-	if body.scope != RuleScope::Global {
+	// remove old deprecated mentions push rules as per MSC4210
-		return Err(Error::BadRequest(
+	#[allow(deprecated)]
-			ErrorKind::InvalidParam,
+	if body.rule_id.as_str() == PredefinedContentRuleId::ContainsUserName.as_str()
-			"Scopes other than 'global' are not supported.",
+		|| body.rule_id.as_str() == PredefinedOverrideRuleId::ContainsDisplayName.as_str()
-		));
+		|| body.rule_id.as_str() == PredefinedOverrideRuleId::RoomNotif.as_str()
+	{
+		return Ok(get_pushrule_enabled::v3::Response {
+			enabled: false,
+		});
 	}
 
-	let event = services
+	let event: PushRulesEvent = services
 		.account_data
-		.get(None, sender_user, GlobalAccountDataEventType::PushRules.to_string().into())?
+		.get_global(sender_user, GlobalAccountDataEventType::PushRules)
-		.ok_or(Error::BadRequest(ErrorKind::NotFound, "PushRules event not found."))?;
+		.await
+		.map_err(|_| err!(Request(NotFound("PushRules event not found."))))?;
 
-	let account_data = serde_json::from_str::<PushRulesEvent>(event.get())
+	let enabled = event
-		.map_err(|_| Error::bad_database("Invalid account data event in db."))?;
+		.content
-
+		.global
-	let global = account_data.content.global;
-	let enabled = global
 		.get(body.kind.clone(), &body.rule_id)
 		.map(ruma::push::AnyPushRuleRef::enabled)
-		.ok_or(Error::BadRequest(ErrorKind::NotFound, "Push rule not found."))?;
+		.ok_or(err!(Request(NotFound("Push rule not found."))))?;
 
 	Ok(get_pushrule_enabled::v3::Response {
 		enabled,
 	})
 }
 
-/// # `PUT /_matrix/client/r0/pushrules/{scope}/{kind}/{ruleId}/enabled`
+/// # `PUT /_matrix/client/r0/pushrules/global/{kind}/{ruleId}/enabled`
 ///
 /// Sets the enabled status of a single specified push rule for this user.
 pub(crate) async fn set_pushrule_enabled_route(
@@ -284,20 +347,11 @@ pub(crate) async fn set_pushrule_enabled_route(
 ) -> Result<set_pushrule_enabled::v3::Response> {
 	let sender_user = body.sender_user.as_ref().expect("user is authenticated");
 
-	if body.scope != RuleScope::Global {
+	let mut account_data: PushRulesEvent = services
-		return Err(Error::BadRequest(
-			ErrorKind::InvalidParam,
-			"Scopes other than 'global' are not supported.",
-		));
-	}
-
-	let event = services
 		.account_data
-		.get(None, sender_user, GlobalAccountDataEventType::PushRules.to_string().into())?
+		.get_global(sender_user, GlobalAccountDataEventType::PushRules)
-		.ok_or(Error::BadRequest(ErrorKind::NotFound, "PushRules event not found."))?;
+		.await
-
+		.map_err(|_| err!(Request(NotFound("PushRules event not found."))))?;
-	let mut account_data = serde_json::from_str::<PushRulesEvent>(event.get())
-		.map_err(|_| Error::bad_database("Invalid account data event in db."))?;
 
 	if account_data
 		.content
@@ -308,17 +362,20 @@ pub(crate) async fn set_pushrule_enabled_route(
 		return Err(Error::BadRequest(ErrorKind::NotFound, "Push rule not found."));
 	}
 
-	services.account_data.update(
+	services
-		None,
+		.account_data
-		sender_user,
+		.update(
-		GlobalAccountDataEventType::PushRules.to_string().into(),
+			None,
-		&serde_json::to_value(account_data).expect("to json value always works"),
+			sender_user,
-	)?;
+			GlobalAccountDataEventType::PushRules.to_string().into(),
+			&serde_json::to_value(account_data).expect("to json value always works"),
+		)
+		.await?;
 
 	Ok(set_pushrule_enabled::v3::Response {})
 }
 
-/// # `DELETE /_matrix/client/r0/pushrules/{scope}/{kind}/{ruleId}`
+/// # `DELETE /_matrix/client/r0/pushrules/global/{kind}/{ruleId}`
 ///
 /// Deletes a single specified push rule for this user.
 pub(crate) async fn delete_pushrule_route(
@@ -326,20 +383,11 @@ pub(crate) async fn delete_pushrule_route(
 ) -> Result<delete_pushrule::v3::Response> {
 	let sender_user = body.sender_user.as_ref().expect("user is authenticated");
 
-	if body.scope != RuleScope::Global {
+	let mut account_data: PushRulesEvent = services
-		return Err(Error::BadRequest(
-			ErrorKind::InvalidParam,
-			"Scopes other than 'global' are not supported.",
-		));
-	}
-
-	let event = services
 		.account_data
-		.get(None, sender_user, GlobalAccountDataEventType::PushRules.to_string().into())?
+		.get_global(sender_user, GlobalAccountDataEventType::PushRules)
-		.ok_or(Error::BadRequest(ErrorKind::NotFound, "PushRules event not found."))?;
+		.await
-
+		.map_err(|_| err!(Request(NotFound("PushRules event not found."))))?;
-	let mut account_data = serde_json::from_str::<PushRulesEvent>(event.get())
-		.map_err(|_| Error::bad_database("Invalid account data event in db."))?;
 
 	if let Err(error) = account_data
 		.content
@@ -357,12 +405,15 @@ pub(crate) async fn delete_pushrule_route(
 		return Err(err);
 	}
 
-	services.account_data.update(
+	services
-		None,
+		.account_data
-		sender_user,
+		.update(
-		GlobalAccountDataEventType::PushRules.to_string().into(),
+			None,
-		&serde_json::to_value(account_data).expect("to json value always works"),
+			sender_user,
-	)?;
+			GlobalAccountDataEventType::PushRules.to_string().into(),
+			&serde_json::to_value(account_data).expect("to json value always works"),
+		)
+		.await?;
 
 	Ok(delete_pushrule::v3::Response {})
 }
@@ -376,7 +427,7 @@ pub(crate) async fn get_pushers_route(
 	let sender_user = body.sender_user.as_ref().expect("user is authenticated");
 
 	Ok(get_pushers::v3::Response {
-		pushers: services.pusher.get_pushers(sender_user)?,
+		pushers: services.pusher.get_pushers(sender_user).await,
 	})
 }
 
@@ -390,27 +441,33 @@ pub(crate) async fn set_pushers_route(
 ) -> Result<set_pusher::v3::Response> {
 	let sender_user = body.sender_user.as_ref().expect("user is authenticated");
 
-	services.pusher.set_pusher(sender_user, &body.action)?;
+	services
+		.pusher
+		.set_pusher(sender_user, &body.action)
+		.await?;
 
-	Ok(set_pusher::v3::Response::default())
+	Ok(set_pusher::v3::Response::new())
 }
 
 /// user somehow has bad push rules, these must always exist per spec.
 /// so recreate it and return server default silently
-fn recreate_push_rules_and_return(
+async fn recreate_push_rules_and_return(
 	services: &Services, sender_user: &ruma::UserId,
 ) -> Result<get_pushrules_all::v3::Response> {
-	services.account_data.update(
+	services
-		None,
+		.account_data
-		sender_user,
+		.update(
-		GlobalAccountDataEventType::PushRules.to_string().into(),
+			None,
-		&serde_json::to_value(PushRulesEvent {
+			sender_user,
-			content: PushRulesEventContent {
+			GlobalAccountDataEventType::PushRules.to_string().into(),
-				global: Ruleset::server_default(sender_user),
+			&serde_json::to_value(PushRulesEvent {
-			},
+				content: PushRulesEventContent {
-		})
+					global: Ruleset::server_default(sender_user),
-		.expect("to json always works"),
+				},
-	)?;
+			})
+			.expect("to json always works"),
+		)
+		.await?;
 
 	Ok(get_pushrules_all::v3::Response {
 		global: Ruleset::server_default(sender_user),

src/api/client/read_marker.rs

@@ -31,27 +31,32 @@ pub(crate) async fn set_read_marker_route(
 				event_id: fully_read.clone(),
 			},
 		};
-		services.account_data.update(
+		services
-			Some(&body.room_id),
+			.account_data
-			sender_user,
+			.update(
-			RoomAccountDataEventType::FullyRead,
+				Some(&body.room_id),
-			&serde_json::to_value(fully_read_event).expect("to json value always works"),
+				sender_user,
-		)?;
+				RoomAccountDataEventType::FullyRead,
+				&serde_json::to_value(fully_read_event).expect("to json value always works"),
+			)
+			.await?;
 	}
 
 	if body.private_read_receipt.is_some() || body.read_receipt.is_some() {
 		services
 			.rooms
 			.user
-			.reset_notification_counts(sender_user, &body.room_id)?;
+			.reset_notification_counts(sender_user, &body.room_id);
 	}
 
 	if let Some(event) = &body.private_read_receipt {
 		let count = services
 			.rooms
 			.timeline
-			.get_pdu_count(event)?
+			.get_pdu_count(event)
-			.ok_or(Error::BadRequest(ErrorKind::InvalidParam, "Event does not exist."))?;
+			.await
+			.map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "Event does not exist."))?;
+
 		let count = match count {
 			PduCount::Backfilled(_) => {
 				return Err(Error::BadRequest(
@@ -64,7 +69,7 @@ pub(crate) async fn set_read_marker_route(
 		services
 			.rooms
 			.read_receipt
-			.private_read_set(&body.room_id, sender_user, count)?;
+			.private_read_set(&body.room_id, sender_user, count);
 	}
 
 	if let Some(event) = &body.read_receipt {
@@ -83,14 +88,18 @@ pub(crate) async fn set_read_marker_route(
 		let mut receipt_content = BTreeMap::new();
 		receipt_content.insert(event.to_owned(), receipts);
 
-		services.rooms.read_receipt.readreceipt_update(
+		services
-			sender_user,
+			.rooms
-			&body.room_id,
+			.read_receipt
-			&ruma::events::receipt::ReceiptEvent {
+			.readreceipt_update(
-				content: ruma::events::receipt::ReceiptEventContent(receipt_content),
+				sender_user,
-				room_id: body.room_id.clone(),
+				&body.room_id,
-			},
+				&ruma::events::receipt::ReceiptEvent {
-		)?;
+					content: ruma::events::receipt::ReceiptEventContent(receipt_content),
+					room_id: body.room_id.clone(),
+				},
+			)
+			.await;
 	}
 
 	Ok(set_read_marker::v3::Response {})
@@ -111,7 +120,7 @@ pub(crate) async fn create_receipt_route(
 		services
 			.rooms
 			.user
-			.reset_notification_counts(sender_user, &body.room_id)?;
+			.reset_notification_counts(sender_user, &body.room_id);
 	}
 
 	match body.receipt_type {
@@ -121,12 +130,15 @@ pub(crate) async fn create_receipt_route(
 					event_id: body.event_id.clone(),
 				},
 			};
-			services.account_data.update(
+			services
-				Some(&body.room_id),
+				.account_data
-				sender_user,
+				.update(
-				RoomAccountDataEventType::FullyRead,
+					Some(&body.room_id),
-				&serde_json::to_value(fully_read_event).expect("to json value always works"),
+					sender_user,
-			)?;
+					RoomAccountDataEventType::FullyRead,
+					&serde_json::to_value(fully_read_event).expect("to json value always works"),
+				)
+				.await?;
 		},
 		create_receipt::v3::ReceiptType::Read => {
 			let mut user_receipts = BTreeMap::new();
@@ -143,21 +155,27 @@ pub(crate) async fn create_receipt_route(
 			let mut receipt_content = BTreeMap::new();
 			receipt_content.insert(body.event_id.clone(), receipts);
 
-			services.rooms.read_receipt.readreceipt_update(
+			services
-				sender_user,
+				.rooms
-				&body.room_id,
+				.read_receipt
-				&ruma::events::receipt::ReceiptEvent {
+				.readreceipt_update(
-					content: ruma::events::receipt::ReceiptEventContent(receipt_content),
+					sender_user,
-					room_id: body.room_id.clone(),
+					&body.room_id,
-				},
+					&ruma::events::receipt::ReceiptEvent {
-			)?;
+						content: ruma::events::receipt::ReceiptEventContent(receipt_content),
+						room_id: body.room_id.clone(),
+					},
+				)
+				.await;
 		},
 		create_receipt::v3::ReceiptType::ReadPrivate => {
 			let count = services
 				.rooms
 				.timeline
-				.get_pdu_count(&body.event_id)?
+				.get_pdu_count(&body.event_id)
-				.ok_or(Error::BadRequest(ErrorKind::InvalidParam, "Event does not exist."))?;
+				.await
+				.map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "Event does not exist."))?;
+
 			let count = match count {
 				PduCount::Backfilled(_) => {
 					return Err(Error::BadRequest(
@@ -170,7 +188,7 @@ pub(crate) async fn create_receipt_route(
 			services
 				.rooms
 				.read_receipt
-				.private_read_set(&body.room_id, sender_user, count)?;
+				.private_read_set(&body.room_id, sender_user, count);
 		},
 		_ => return Err(Error::bad_database("Unsupported receipt type")),
 	}