misc nix CI fixes that might speed it up a bit

Signed-off-by: June Clementine Strawberry <strawberry@puppygock.gay>
Signed-off-by: strawberry <strawberry@puppygock.gay>

.gitea/PULL_REQUEST_TEMPLATE.md

@@ -1,8 +0,0 @@
-
-<!-- Please describe your changes here -->
-
------------------------------------------------------------------------------
-
-- [ ] I ran `cargo fmt`, `cargo clippy`, and `cargo test`
-- [ ] I agree to release my code and all other changes of this MR under the Apache-2.0 license
-

.gitea/workflows/ci.yml

@@ -1,264 +0,0 @@
-name: CI and Artifacts
-
-on:
-    pull_request:
-    push:
-        # documentation workflow deals with this or is not relevant for this workflow
-        paths-ignore:
-          - '*.md'
-          - 'conduwuit-example.toml'
-          - 'book.toml'
-          - '.gitlab-ci.yml'
-          - '.gitignore'
-          - 'renovate.json'
-          - 'docs/**'
-          - 'debian/**'
-          - 'docker/**'
-        branches:
-            - main
-        tags:
-          - '*'
-    # Allows you to run this workflow manually from the Actions tab
-    #workflow_dispatch:
-
-#concurrency:
-#    group: ${{ gitea.head_ref || gitea.ref_name }}
-#    cancel-in-progress: true
-
-env:
-    # Required to make some things output color
-    TERM: ansi
-    # Publishing to my nix binary cache
-    ATTIC_TOKEN: ${{ secrets.ATTIC_TOKEN }}
-    # conduwuit.cachix.org
-    CACHIX_AUTH_TOKEN: ${{ secrets.CACHIX_AUTH_TOKEN }}
-    # Just in case incremental is still being set to true, speeds up CI
-    CARGO_INCREMENTAL: 0
-    # Custom nix binary cache if fork is being used
-    ATTIC_ENDPOINT: ${{ vars.ATTIC_ENDPOINT }}
-    ATTIC_PUBLIC_KEY: ${{ vars.ATTIC_PUBLIC_KEY }}
-    # Get error output from nix that we can actually use
-    NIX_CONFIG: show-trace = true
-
-#permissions:
-#    packages: write
-#    contents: read
-
-jobs:
-    tests:
-        name: Test
-        runs-on: ubuntu-latest
-        steps:
-            - name: Sync repository
-              uses: https://github.com/actions/checkout@v4
-
-            - name: Tag comparison check
-              if: startsWith(gitea.ref, 'refs/tags/v')
-              run: |
-                  # Tag mismatch with latest repo tag check to prevent potential downgrades
-                  LATEST_TAG=$(git describe --tags `git rev-list --tags --max-count=1`)
-
-                  if [ $LATEST_TAG != ${{ gitea.ref_name }} ]; then
-                    echo '# WARNING: Attempting to run this workflow for a tag that is not the latest repo tag. Aborting.'
-                    echo '# WARNING: Attempting to run this workflow for a tag that is not the latest repo tag. Aborting.' >> $GITHUB_STEP_SUMMARY
-                    exit 1
-                  fi
-
-            - name: Install Nix
-              uses: https://github.com/DeterminateSystems/nix-installer-action@main
-              with:
-                diagnostic-endpoint: ""
-                extra-conf: |
-                  experimental-features = nix-command flakes
-                  accept-flake-config = true
-
-            - name: Enable Cachix binary cache
-              run: |
-                  nix profile install nixpkgs#cachix
-                  cachix use crane
-                  cachix use nix-community
-
-            - name: Configure Magic Nix Cache
-              uses: https://github.com/DeterminateSystems/magic-nix-cache-action@main
-              with:
-                diagnostic-endpoint: ""
-                upstream-cache: "https://attic.kennel.juneis.dog/conduwuit"
-
-            - name: Apply Nix binary cache configuration
-              run: |
-                  sudo tee -a /etc/nix/nix.conf > /dev/null <<EOF
-                  extra-substituters = https://attic.kennel.juneis.dog/conduit https://attic.kennel.juneis.dog/conduwuit https://cache.lix.systems https://conduwuit.cachix.org
-                  extra-trusted-public-keys = conduit:eEKoUwlQGDdYmAI/Q/0slVlegqh/QmAvQd7HBSm21Wk= conduwuit:BbycGUgTISsltcmH0qNjFR9dbrQNYgdIAcmViSGoVTE= cache.lix.systems:aBnZUw8zA7H35Cz2RyKFVs3H4PlGTLawyY5KRbvJR8o= conduwuit.cachix.org-1:MFRm6jcnfTf0jSAbmvLfhO3KBMt4px+1xaereWXp8Xg=
-                  EOF
-
-            - name: Use alternative Nix binary caches if specified
-              if: ${{ (env.ATTIC_ENDPOINT != '') && (env.ATTIC_PUBLIC_KEY != '') }}
-              run: |
-                  sudo tee -a /etc/nix/nix.conf > /dev/null <<EOF
-                  extra-substituters = ${{ env.ATTIC_ENDPOINT }}
-                  extra-trusted-public-keys = ${{ env.ATTIC_PUBLIC_KEY }}
-                  EOF
-
-            - name: Prepare build environment
-              run: |
-                  echo 'source $HOME/.nix-profile/share/nix-direnv/direnvrc' > "$HOME/.direnvrc"
-                  nix profile install --impure --inputs-from . nixpkgs#direnv nixpkgs#nix-direnv
-                  direnv allow
-                  nix develop .#all-features --command true
-
-            - name: Cache CI dependencies
-              run: |
-                  bin/nix-build-and-cache ci
-
-            - name: Run CI tests
-              run: |
-                  direnv exec . engage > >(tee -a test_output.log)
-
-            - name: Sync Complement repository
-              uses: https://github.com/actions/checkout@v4
-              with:
-                repository: 'matrix-org/complement'
-                path: complement_src
-
-            - name: Run Complement tests
-              run: |
-                  direnv exec . bin/complement 'complement_src' 'complement_test_logs.jsonl' 'complement_test_results.jsonl'
-                  cp -v -f result complement_oci_image.tar.gz
-
-            - name: Upload Complement OCI image
-              uses: https://github.com/actions/upload-artifact@v4
-              with:
-                name: complement_oci_image.tar.gz
-                path: complement_oci_image.tar.gz
-                if-no-files-found: error
-
-            - name: Upload Complement logs
-              uses: https://github.com/actions/upload-artifact@v4
-              with:
-                name: complement_test_logs.jsonl
-                path: complement_test_logs.jsonl
-                if-no-files-found: error
-
-            - name: Upload Complement results
-              uses: https://github.com/actions/upload-artifact@v4
-              with:
-                name: complement_test_results.jsonl
-                path: complement_test_results.jsonl
-                if-no-files-found: error
-
-            - name: Diff Complement results with checked-in repo results
-              run: |
-                  diff -u --color=always tests/test_results/complement/test_results.jsonl complement_test_results.jsonl > >(tee -a complement_test_output.log)
-                  echo '# Complement diff results' >> $GITHUB_STEP_SUMMARY
-                  echo '```diff' >> $GITHUB_STEP_SUMMARY
-                  tail -n 100 complement_test_output.log | sed 's/\x1b\[[0-9;]*m//g' >> $GITHUB_STEP_SUMMARY
-                  echo '```' >> $GITHUB_STEP_SUMMARY
-
-            - name: Update Job Summary
-              if: success() || failure()
-              run: |
-                  if [ ${{ job.status }} == 'success' ]; then
-                      echo '# ✅ completed suwuccessfully' >> $GITHUB_STEP_SUMMARY
-                  else
-                      echo '```' >> $GITHUB_STEP_SUMMARY
-                      tail -n 40 test_output.log | sed 's/\x1b\[[0-9;]*m//g' >> $GITHUB_STEP_SUMMARY
-                      echo '```' >> $GITHUB_STEP_SUMMARY
-                  fi
-
-    build:
-        name: Build
-        runs-on: ubuntu-latest
-        needs: tests
-        strategy:
-            matrix:
-                include:
-                    - target: aarch64-unknown-linux-musl
-                    - target: x86_64-unknown-linux-musl
-        steps:
-            - name: Sync repository
-              uses: https://github.com/actions/checkout@v4
-
-            - name: Install Nix
-              uses: https://github.com/DeterminateSystems/nix-installer-action@main
-              with:
-                diagnostic-endpoint: ""
-                extra-conf: |
-                  experimental-features = nix-command flakes
-                  accept-flake-config = true
-
-            - name: Install and enable Cachix binary cache
-              run: |
-                  nix profile install nixpkgs#cachix
-                  cachix use crane
-                  cachix use nix-community
-
-            - name: Configure Magic Nix Cache
-              uses: https://github.com/DeterminateSystems/magic-nix-cache-action@main
-              with:
-                diagnostic-endpoint: ""
-                upstream-cache: "https://attic.kennel.juneis.dog/conduwuit"
-
-            - name: Apply Nix binary cache configuration
-              run: |
-                  sudo tee -a /etc/nix/nix.conf > /dev/null <<EOF
-                  extra-substituters = https://attic.kennel.juneis.dog/conduit https://attic.kennel.juneis.dog/conduwuit https://cache.lix.systems https://conduwuit.cachix.org
-                  extra-trusted-public-keys = conduit:eEKoUwlQGDdYmAI/Q/0slVlegqh/QmAvQd7HBSm21Wk= conduwuit:BbycGUgTISsltcmH0qNjFR9dbrQNYgdIAcmViSGoVTE= cache.lix.systems:aBnZUw8zA7H35Cz2RyKFVs3H4PlGTLawyY5KRbvJR8o= conduwuit.cachix.org-1:MFRm6jcnfTf0jSAbmvLfhO3KBMt4px+1xaereWXp8Xg=
-                  EOF
-
-            - name: Use alternative Nix binary caches if specified
-              if: ${{ (env.ATTIC_ENDPOINT != '') && (env.ATTIC_PUBLIC_KEY != '') }}
-              run: |
-                  sudo tee -a /etc/nix/nix.conf > /dev/null <<EOF
-                  extra-substituters = ${{ env.ATTIC_ENDPOINT }}
-                  extra-trusted-public-keys = ${{ env.ATTIC_PUBLIC_KEY }}
-                  EOF
-
-            - name: Prepare build environment
-              run: |
-                  echo 'source $HOME/.nix-profile/share/nix-direnv/direnvrc' > "$HOME/.direnvrc"
-                  nix profile install --impure --inputs-from . nixpkgs#direnv nixpkgs#nix-direnv
-                  direnv allow
-                  nix develop .#all-features --command true
-
-            - name: Build static ${{ matrix.target }}
-              run: |
-                  CARGO_DEB_TARGET_TUPLE=$(echo ${{ matrix.target }} | grep -o -E '^([^-]*-){3}[^-]*')
-                  SOURCE_DATE_EPOCH=$(git log -1 --pretty=%ct)
-
-                  bin/nix-build-and-cache just .#static-${{ matrix.target }}
-                  mkdir -v -p target/release/
-                  mkdir -v -p target/$CARGO_DEB_TARGET_TUPLE/release/
-                  cp -v -f result/bin/conduit target/release/conduwuit
-                  cp -v -f result/bin/conduit target/$CARGO_DEB_TARGET_TUPLE/release/conduwuit
-                  # -p conduit is the main crate name
-                  direnv exec . cargo deb --verbose --no-build --no-strip -p conduit --target=$CARGO_DEB_TARGET_TUPLE --output target/release/${{ matrix.target }}.deb
-                  mv -v target/release/conduwuit static-${{ matrix.target }}
-                  mv -v target/release/${{ matrix.target }}.deb ${{ matrix.target }}.deb
-
-            - name: Upload static-${{ matrix.target }}
-              uses: https://github.com/actions/upload-artifact@v4
-              with:
-                  name: static-${{ matrix.target }}
-                  path: static-${{ matrix.target }}
-                  if-no-files-found: error
-
-            - name: Upload deb ${{ matrix.target }}
-              uses: https://github.com/actions/upload-artifact@v4
-              with:
-                  name: deb-${{ matrix.target }}
-                  path: ${{ matrix.target }}.deb
-                  if-no-files-found: error
-                  compression-level: 0
-
-            - name: Build OCI image ${{ matrix.target }}
-              run: |
-                  bin/nix-build-and-cache just .#oci-image-${{ matrix.target }}
-                  cp -v -f result oci-image-${{ matrix.target }}.tar.gz
-
-            - name: Upload OCI image ${{ matrix.target }}
-              uses: https://github.com/actions/upload-artifact@v4
-              with:
-                  name: oci-image-${{ matrix.target }}
-                  path: oci-image-${{ matrix.target }}.tar.gz
-                  if-no-files-found: error
-                  compression-level: 0

.github/workflows/ci.yml

@@ -22,8 +22,8 @@ concurrency:
 
 env:
     # sccache only on main repo
-    SCCACHE_GHA_ENABLED: "${{ (github.event.pull_request.draft != true) && (vars.DOCKER_USERNAME != '') && (vars.GITLAB_USERNAME != '') && (vars.SCCACHE_ENDPOINT != '') && (github.event.pull_request.user.login != 'renovate[bot]') && 'true' || 'false' }}"
-    RUSTC_WRAPPER: "${{ (github.event.pull_request.draft != true) && (vars.DOCKER_USERNAME != '') && (vars.GITLAB_USERNAME != '') && (vars.SCCACHE_ENDPOINT != '') && (github.event.pull_request.user.login != 'renovate[bot]') && 'sccache' || '' }}"
+    SCCACHE_GHA_ENABLED: "${{ !startsWith(github.ref, 'refs/tags/') && (github.event.pull_request.draft != true) && (vars.DOCKER_USERNAME != '') && (vars.GITLAB_USERNAME != '') && (vars.SCCACHE_ENDPOINT != '') && (github.event.pull_request.user.login != 'renovate[bot]') && 'true' || 'false' }}"
+    RUSTC_WRAPPER: "${{ !startsWith(github.ref, 'refs/tags/') && (github.event.pull_request.draft != true) && (vars.DOCKER_USERNAME != '') && (vars.GITLAB_USERNAME != '') && (vars.SCCACHE_ENDPOINT != '') && (github.event.pull_request.user.login != 'renovate[bot]') && 'sccache' || '' }}"
     SCCACHE_BUCKET: "${{ (github.event.pull_request.draft != true) && (vars.DOCKER_USERNAME != '') && (vars.GITLAB_USERNAME != '') && (vars.SCCACHE_ENDPOINT != '') && (github.event.pull_request.user.login != 'renovate[bot]') && 'sccache' || '' }}"
     SCCACHE_S3_USE_SSL: ${{ vars.SCCACHE_S3_USE_SSL }}
     SCCACHE_REGION: ${{ vars.SCCACHE_REGION }}
@@ -45,16 +45,14 @@ env:
     # Get error output from nix that we can actually use, and use our binary caches for the earlier CI steps
     NIX_CONFIG: |
       show-trace = true
-      extra-substituters = https://attic.kennel.juneis.dog/conduwuit https://attic.kennel.juneis.dog/conduit https://cache.lix.systems https://conduwuit.cachix.org https://aseipp-nix-cache.freetls.fastly.net
-      extra-trusted-public-keys = conduit:eEKoUwlQGDdYmAI/Q/0slVlegqh/QmAvQd7HBSm21Wk= conduwuit:BbycGUgTISsltcmH0qNjFR9dbrQNYgdIAcmViSGoVTE= cache.lix.systems:aBnZUw8zA7H35Cz2RyKFVs3H4PlGTLawyY5KRbvJR8o= conduwuit.cachix.org-1:MFRm6jcnfTf0jSAbmvLfhO3KBMt4px+1xaereWXp8Xg=
+      extra-substituters = https://attic.kennel.juneis.dog/conduwuit https://attic.kennel.juneis.dog/conduit https://conduwuit.cachix.org https://aseipp-nix-cache.freetls.fastly.net
+      extra-trusted-public-keys = conduit:eEKoUwlQGDdYmAI/Q/0slVlegqh/QmAvQd7HBSm21Wk= conduwuit:BbycGUgTISsltcmH0qNjFR9dbrQNYgdIAcmViSGoVTE= conduwuit.cachix.org-1:MFRm6jcnfTf0jSAbmvLfhO3KBMt4px+1xaereWXp8Xg=
       experimental-features = nix-command flakes
       extra-experimental-features = nix-command flakes
       accept-flake-config = true
-    # complement uses libolm
-    NIXPKGS_ALLOW_INSECURE: 1
     WEB_UPLOAD_SSH_USERNAME: ${{ secrets.WEB_UPLOAD_SSH_USERNAME }}
-    GH_SHA: ${{ github.sha }}
     GH_REF_NAME: ${{ github.ref_name }}
+    WEBSERVER_DIR_NAME: ${{ (github.head_ref != '' && format('merge-{0}-{1}', github.event.number, github.event.pull_request.user.login)) || github.ref_name }}-${{ github.sha }}
 
 permissions: {}
 
@@ -87,11 +85,13 @@ jobs:
                   END
 
                   echo "Checking connection"
-                  ssh -q website "echo test"
+                  ssh -q website "echo test" || ssh -q website "echo test"
 
                   echo "Creating commit rev directory on web server"
-                  ssh -q website "rm -rf /var/www/girlboss.ceo/~strawberry/conduwuit/ci-bins/$GITHUB_SHA/"
-                  ssh -q website "mkdir -v /var/www/girlboss.ceo/~strawberry/conduwuit/ci-bins/$GITHUB_SHA/"
+                  ssh -q website "rm -rf /var/www/girlboss.ceo/~strawberry/conduwuit/ci-bins/${WEBSERVER_DIR_NAME}/" || ssh -q website "rm -rf /var/www/girlboss.ceo/~strawberry/conduwuit/ci-bins/${WEBSERVER_DIR_NAME}/" || true
+                  ssh -q website "mkdir -v /var/www/girlboss.ceo/~strawberry/conduwuit/ci-bins/${WEBSERVER_DIR_NAME}/" || ssh -q website "mkdir -v /var/www/girlboss.ceo/~strawberry/conduwuit/ci-bins/${WEBSERVER_DIR_NAME}/" || true
+
+                  echo "SSH_WEBSITE=1" >> "$GITHUB_ENV"
 
             - name: Install liburing
               run: |
@@ -126,6 +126,9 @@ jobs:
             - uses: nixbuild/nix-quick-install-action@master
 
             - name: Restore and cache Nix store
+              # we want a fresh-state when we do releases/tags to avoid potential cache poisoning attacks impacting
+              # releases and tags
+              if: ${{ !startsWith(github.ref, 'refs/tags/') }}
               uses: nix-community/cache-nix-action@v5.1.0
               with:
                 # restore and save a cache using this key
@@ -155,7 +158,7 @@ jobs:
             - name: Apply Nix binary cache configuration
               run: |
                   sudo tee -a "${XDG_CONFIG_HOME:-$HOME/.config}/nix/nix.conf" > /dev/null <<EOF
-                  extra-substituters = https://attic.kennel.juneis.dog/conduwuit https://attic.kennel.juneis.dog/conduit https://cache.lix.systems https://conduwuit.cachix.org https://aseipp-nix-cache.freetls.fastly.net
+                  extra-substituters = https://attic.kennel.juneis.dog/conduwuit https://attic.kennel.juneis.dog/conduit https://conduwuit.cachix.org https://aseipp-nix-cache.freetls.fastly.net
                   extra-trusted-public-keys = conduit:eEKoUwlQGDdYmAI/Q/0slVlegqh/QmAvQd7HBSm21Wk= conduwuit:BbycGUgTISsltcmH0qNjFR9dbrQNYgdIAcmViSGoVTE= cache.lix.systems:aBnZUw8zA7H35Cz2RyKFVs3H4PlGTLawyY5KRbvJR8o= conduwuit.cachix.org-1:MFRm6jcnfTf0jSAbmvLfhO3KBMt4px+1xaereWXp8Xg=
                   experimental-features = nix-command flakes
                   extra-experimental-features = nix-command flakes
@@ -186,11 +189,16 @@ jobs:
 
             # use sccache for Rust
             - name: Run sccache-cache
-              if: (env.SCCACHE_GHA_ENABLED == 'true')
+              # we want a fresh-state when we do releases/tags to avoid potential cache poisoning attacks impacting
+              # releases and tags
+              if: ${{ (env.SCCACHE_GHA_ENABLED == 'true') && !startsWith(github.ref, 'refs/tags/') }}
               uses: mozilla-actions/sccache-action@main
 
             # use rust-cache
             - uses: Swatinem/rust-cache@v2
+              # we want a fresh-state when we do releases/tags to avoid potential cache poisoning attacks impacting
+              # releases and tags
+              if: ${{ !startsWith(github.ref, 'refs/tags/') }}
               with:
                 cache-all-crates: "true"
                 cache-on-failure: "true"
@@ -306,11 +314,16 @@ jobs:
                   END
 
                   echo "Checking connection"
-                  ssh -q website "echo test"
+                  ssh -q website "echo test" || ssh -q website "echo test"
+
+                  echo "SSH_WEBSITE=1" >> "$GITHUB_ENV"
 
             - uses: nixbuild/nix-quick-install-action@master
 
             - name: Restore and cache Nix store
+              # we want a fresh-state when we do releases/tags to avoid potential cache poisoning attacks impacting
+              # releases and tags
+              if: ${{ !startsWith(github.ref, 'refs/tags/') }}
               uses: nix-community/cache-nix-action@v5.1.0
               with:
                 # restore and save a cache using this key
@@ -340,7 +353,7 @@ jobs:
             - name: Apply Nix binary cache configuration
               run: |
                   sudo tee -a "${XDG_CONFIG_HOME:-$HOME/.config}/nix/nix.conf" > /dev/null <<EOF
-                  extra-substituters = https://attic.kennel.juneis.dog/conduwuit https://attic.kennel.juneis.dog/conduit https://cache.lix.systems https://conduwuit.cachix.org https://aseipp-nix-cache.freetls.fastly.net
+                  extra-substituters = https://attic.kennel.juneis.dog/conduwuit https://attic.kennel.juneis.dog/conduit https://conduwuit.cachix.org https://aseipp-nix-cache.freetls.fastly.net
                   extra-trusted-public-keys = conduit:eEKoUwlQGDdYmAI/Q/0slVlegqh/QmAvQd7HBSm21Wk= conduwuit:BbycGUgTISsltcmH0qNjFR9dbrQNYgdIAcmViSGoVTE= cache.lix.systems:aBnZUw8zA7H35Cz2RyKFVs3H4PlGTLawyY5KRbvJR8o= conduwuit.cachix.org-1:MFRm6jcnfTf0jSAbmvLfhO3KBMt4px+1xaereWXp8Xg=
                   experimental-features = nix-command flakes
                   extra-experimental-features = nix-command flakes
@@ -364,11 +377,16 @@ jobs:
 
             # use sccache for Rust
             - name: Run sccache-cache
-              if: (env.SCCACHE_GHA_ENABLED == 'true')
+              # we want a fresh-state when we do releases/tags to avoid potential cache poisoning attacks impacting
+              # releases and tags
+              if: ${{ (env.SCCACHE_GHA_ENABLED == 'true') && !startsWith(github.ref, 'refs/tags/') }}
               uses: mozilla-actions/sccache-action@main
 
             # use rust-cache
             - uses: Swatinem/rust-cache@v2
+              # we want a fresh-state when we do releases/tags to avoid potential cache poisoning attacks impacting
+              # releases and tags
+              if: ${{ !startsWith(github.ref, 'refs/tags/') }}
               with:
                 cache-all-crates: "true"
                 cache-on-failure: "true"
@@ -491,29 +509,29 @@ jobs:
             - name: Upload static-x86_64-linux-musl-all-features-x86_64-haswell-optimised to webserver
               if: ${{ matrix.target == 'x86_64-linux-musl' }}
               run: |
-                  if [ ! -z $WEB_UPLOAD_SSH_USERNAME ]; then
-                    scp static-x86_64-linux-musl-x86_64-haswell-optimised website:/var/www/girlboss.ceo/~strawberry/conduwuit/ci-bins/${GH_SHA}/static-x86_64-linux-musl-x86_64-haswell-optimised
+                  if [ ! -z $SSH_WEBSITE ]; then
+                    chmod +x static-x86_64-linux-musl-x86_64-haswell-optimised
+                    scp static-x86_64-linux-musl-x86_64-haswell-optimised website:/var/www/girlboss.ceo/~strawberry/conduwuit/ci-bins/${WEBSERVER_DIR_NAME}/static-x86_64-linux-musl-x86_64-haswell-optimised
                   fi
 
             - name: Upload static-${{ matrix.target }}-all-features to webserver
-              if: (startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main' || (github.event.pull_request.draft != true)) && (env.web_upload_ssh_private_key != '') && github.event.pull_request.user.login != 'renovate[bot]'
               run: |
-                  if [ ! -z $WEB_UPLOAD_SSH_USERNAME ]; then
-                    scp static-${{ matrix.target }} website:/var/www/girlboss.ceo/~strawberry/conduwuit/ci-bins/${GH_SHA}/static-${{ matrix.target }}
+                  if [ ! -z $SSH_WEBSITE ]; then
+                    chmod +x static-${{ matrix.target }}
+                    scp static-${{ matrix.target }} website:/var/www/girlboss.ceo/~strawberry/conduwuit/ci-bins/${WEBSERVER_DIR_NAME}/static-${{ matrix.target }}
                   fi
 
             - name: Upload static deb x86_64-linux-musl-all-features-x86_64-haswell-optimised to webserver
               if: ${{ matrix.target == 'x86_64-linux-musl' }}
               run: |
-                  if [ ! -z $WEB_UPLOAD_SSH_USERNAME ]; then
-                    scp x86_64-linux-musl-x86_64-haswell-optimised.deb website:/var/www/girlboss.ceo/~strawberry/conduwuit/ci-bins/${GH_SHA}/x86_64-linux-musl-x86_64-haswell-optimised.deb
+                  if [ ! -z $SSH_WEBSITE ]; then
+                    scp x86_64-linux-musl-x86_64-haswell-optimised.deb website:/var/www/girlboss.ceo/~strawberry/conduwuit/ci-bins/${WEBSERVER_DIR_NAME}/x86_64-linux-musl-x86_64-haswell-optimised.deb
                   fi
 
             - name: Upload static deb ${{ matrix.target }}-all-features to webserver
-              if: (startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main' || (github.event.pull_request.draft != true)) && (env.web_upload_ssh_private_key != '') && github.event.pull_request.user.login != 'renovate[bot]'
               run: |
-                  if [ ! -z $WEB_UPLOAD_SSH_USERNAME ]; then
-                    scp ${{ matrix.target }}.deb website:/var/www/girlboss.ceo/~strawberry/conduwuit/ci-bins/${GH_SHA}/${{ matrix.target }}.deb
+                  if [ ! -z $SSH_WEBSITE ]; then
+                    scp ${{ matrix.target }}.deb website:/var/www/girlboss.ceo/~strawberry/conduwuit/ci-bins/${WEBSERVER_DIR_NAME}/${{ matrix.target }}.deb
                   fi
 
             - name: Upload static-${{ matrix.target }}-debug-all-features to GitHub
@@ -532,17 +550,15 @@ jobs:
                   compression-level: 0
 
             - name: Upload static-${{ matrix.target }}-debug-all-features to webserver
-              if: (startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main' || (github.event.pull_request.draft != true)) && (env.web_upload_ssh_private_key != '') && github.event.pull_request.user.login != 'renovate[bot]'
               run: |
-                  if [ ! -z $WEB_UPLOAD_SSH_USERNAME ]; then
-                    scp static-${{ matrix.target }}-debug website:/var/www/girlboss.ceo/~strawberry/conduwuit/ci-bins/${GH_SHA}/static-${{ matrix.target }}-debug
+                  if [ ! -z $SSH_WEBSITE ]; then
+                    scp static-${{ matrix.target }}-debug website:/var/www/girlboss.ceo/~strawberry/conduwuit/ci-bins/${WEBSERVER_DIR_NAME}/static-${{ matrix.target }}-debug
                   fi
 
             - name: Upload static deb ${{ matrix.target }}-debug-all-features to webserver
-              if: (startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main' || (github.event.pull_request.draft != true)) && (env.web_upload_ssh_private_key != '') && github.event.pull_request.user.login != 'renovate[bot]'
               run: |
-                  if [ ! -z $WEB_UPLOAD_SSH_USERNAME ]; then
-                    scp ${{ matrix.target }}-debug.deb website:/var/www/girlboss.ceo/~strawberry/conduwuit/ci-bins/${GH_SHA}/${{ matrix.target }}-debug.deb
+                  if [ ! -z $SSH_WEBSITE ]; then
+                    scp ${{ matrix.target }}-debug.deb website:/var/www/girlboss.ceo/~strawberry/conduwuit/ci-bins/${WEBSERVER_DIR_NAME}/${{ matrix.target }}-debug.deb
                   fi
 
             - name: Build OCI image ${{ matrix.target }}-all-features
@@ -564,6 +580,14 @@ jobs:
 
                   cp -v -f result oci-image-${{ matrix.target }}-debug.tar.gz
 
+            - name: Upload OCI image x86_64-linux-musl-all-features-x86_64-haswell-optimised to GitHub
+              if: ${{ matrix.target == 'x86_64-linux-musl' }}
+              uses: actions/upload-artifact@v4
+              with:
+                  name: oci-image-x86_64-linux-musl-all-features-x86_64-haswell-optimised
+                  path: oci-image-x86_64-linux-musl-all-features-x86_64-haswell-optimised.tar.gz
+                  if-no-files-found: error
+                  compression-level: 0
             - name: Upload OCI image ${{ matrix.target }}-all-features to GitHub
               uses: actions/upload-artifact@v4
               with:
@@ -583,22 +607,20 @@ jobs:
             - name: Upload OCI image x86_64-linux-musl-all-features-x86_64-haswell-optimised.tar.gz to webserver
               if: ${{ matrix.target == 'x86_64-linux-musl' }}
               run: |
-                  if [ ! -z $WEB_UPLOAD_SSH_USERNAME ]; then
-                    scp oci-image-x86_64-linux-musl-all-features-x86_64-haswell-optimised.tar.gz website:/var/www/girlboss.ceo/~strawberry/conduwuit/ci-bins/${GH_SHA}/oci-image-x86_64-linux-musl-all-features-x86_64-haswell-optimised.tar.gz
+                  if [ ! -z $SSH_WEBSITE ]; then
+                    scp oci-image-x86_64-linux-musl-all-features-x86_64-haswell-optimised.tar.gz website:/var/www/girlboss.ceo/~strawberry/conduwuit/ci-bins/${WEBSERVER_DIR_NAME}/oci-image-x86_64-linux-musl-all-features-x86_64-haswell-optimised.tar.gz
                   fi
 
             - name: Upload OCI image ${{ matrix.target }}-all-features to webserver
-              if: (startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main' || (github.event.pull_request.draft != true)) && (env.web_upload_ssh_private_key != '') && github.event.pull_request.user.login != 'renovate[bot]'
               run: |
-                  if [ ! -z $WEB_UPLOAD_SSH_USERNAME ]; then
-                    scp oci-image-${{ matrix.target }}.tar.gz website:/var/www/girlboss.ceo/~strawberry/conduwuit/ci-bins/${GH_SHA}/oci-image-${{ matrix.target }}.tar.gz
+                  if [ ! -z $SSH_WEBSITE ]; then
+                    scp oci-image-${{ matrix.target }}.tar.gz website:/var/www/girlboss.ceo/~strawberry/conduwuit/ci-bins/${WEBSERVER_DIR_NAME}/oci-image-${{ matrix.target }}.tar.gz
                   fi
 
             - name: Upload OCI image ${{ matrix.target }}-debug-all-features to webserver
-              if: (startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main' || (github.event.pull_request.draft != true)) && (env.web_upload_ssh_private_key != '') && github.event.pull_request.user.login != 'renovate[bot]'
               run: |
-                  if [ ! -z $WEB_UPLOAD_SSH_USERNAME ]; then
-                    scp oci-image-${{ matrix.target }}-debug.tar.gz website:/var/www/girlboss.ceo/~strawberry/conduwuit/ci-bins/${GH_SHA}/oci-image-${{ matrix.target }}-debug.tar.gz
+                  if [ ! -z $SSH_WEBSITE ]; then
+                    scp oci-image-${{ matrix.target }}-debug.tar.gz website:/var/www/girlboss.ceo/~strawberry/conduwuit/ci-bins/${WEBSERVER_DIR_NAME}/oci-image-${{ matrix.target }}-debug.tar.gz
                   fi
 
     build_mac_binaries:
@@ -637,7 +659,9 @@ jobs:
                   END
 
                   echo "Checking connection"
-                  ssh -q website "echo test"
+                  ssh -q website "echo test" || ssh -q website "echo test"
+
+                  echo "SSH_WEBSITE=1" >> "$GITHUB_ENV"
 
             - name: Tag comparison check
               if: ${{ startsWith(github.ref, 'refs/tags/v') && !endsWith(github.ref, '-rc') }}
@@ -653,7 +677,9 @@ jobs:
 
             # use sccache for Rust
             - name: Run sccache-cache
-              if: (env.SCCACHE_GHA_ENABLED == 'true')
+              # we want a fresh-state when we do releases/tags to avoid potential cache poisoning attacks impacting
+              # releases and tags
+              if: ${{ (env.SCCACHE_GHA_ENABLED == 'true') && !startsWith(github.ref, 'refs/tags/') }}
               uses: mozilla-actions/sccache-action@main
 
             # use rust-cache
@@ -667,7 +693,7 @@ jobs:
             - name: Build macOS x86_64 binary
               if: ${{ matrix.os == 'macos-13' }}
               run: |
-                  CONDUWUIT_VERSION_EXTRA="$(git rev-parse --short ${{ github.sha }})" cargo build --release
+                  CONDUWUIT_VERSION_EXTRA="$(git rev-parse --short ${{ github.sha }})" cargo build --release --locked --features=perf_measurements,sentry_telemetry,direct_tls
                   cp -v -f target/release/conduwuit conduwuit-macos-x86_64
                   otool -L conduwuit-macos-x86_64
 
@@ -675,12 +701,13 @@ jobs:
             - name: Run x86_64 macOS release binary
               if: ${{ matrix.os == 'macos-13' }}
               run: |
+                  ./conduwuit-macos-x86_64 --help
                   ./conduwuit-macos-x86_64 --version
 
             - name: Build macOS arm64 binary
               if: ${{ matrix.os == 'macos-latest' }}
               run: |
-                  CONDUWUIT_VERSION_EXTRA="$(git rev-parse --short ${{ github.sha }})" cargo build --release
+                  CONDUWUIT_VERSION_EXTRA="$(git rev-parse --short ${{ github.sha }})" cargo build --release --locked --features=perf_measurements,sentry_telemetry,direct_tls
                   cp -v -f target/release/conduwuit conduwuit-macos-arm64
                   otool -L conduwuit-macos-arm64
 
@@ -688,20 +715,23 @@ jobs:
             - name: Run arm64 macOS release binary
               if: ${{ matrix.os == 'macos-latest' }}
               run: |
+                  ./conduwuit-macos-arm64 --help
                   ./conduwuit-macos-arm64 --version
 
             - name: Upload macOS x86_64 binary to webserver
               if: ${{ matrix.os == 'macos-13' }}
               run: |
-                  if [ ! -z $WEB_UPLOAD_SSH_USERNAME ]; then
-                    scp conduwuit-macos-x86_64 website:/var/www/girlboss.ceo/~strawberry/conduwuit/ci-bins/${GH_SHA}/conduwuit-macos-x86_64
+                  if [ ! -z $SSH_WEBSITE ]; then
+                    chmod +x conduwuit-macos-x86_64
+                    scp conduwuit-macos-x86_64 website:/var/www/girlboss.ceo/~strawberry/conduwuit/ci-bins/${WEBSERVER_DIR_NAME}/conduwuit-macos-x86_64
                   fi
 
             - name: Upload macOS arm64 binary to webserver
               if: ${{ matrix.os == 'macos-latest' }}
               run: |
-                  if [ ! -z $WEB_UPLOAD_SSH_USERNAME ]; then
-                    scp conduwuit-macos-arm64 website:/var/www/girlboss.ceo/~strawberry/conduwuit/ci-bins/${GH_SHA}/conduwuit-macos-arm64
+                  if [ ! -z $SSH_WEBSITE ]; then
+                    chmod +x conduwuit-macos-arm64
+                    scp conduwuit-macos-arm64 website:/var/www/girlboss.ceo/~strawberry/conduwuit/ci-bins/${WEBSERVER_DIR_NAME}/conduwuit-macos-arm64
                   fi
 
             - name: Upload macOS x86_64 binary
@@ -733,24 +763,17 @@ jobs:
     docker:
         name: Docker publish
         runs-on: ubuntu-24.04
-        needs: [build, variables]
+        needs: [build, variables, tests]
         permissions:
           packages: write
           contents: read
         if: (startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main' || (github.event.pull_request.draft != true)) && github.event.pull_request.user.login != 'renovate[bot]'
         env:
-            DOCKER_ARM64: docker.io/${{ needs.variables.outputs.github_repository }}:${{ (github.head_ref != '' && format('merge-{0}-{1}', github.event.number, github.event.pull_request.user.login)) || github.ref_name }}-${{ github.sha }}-arm64v8
-            DOCKER_AMD64: docker.io/${{ needs.variables.outputs.github_repository }}:${{ (github.head_ref != '' && format('merge-{0}-{1}', github.event.number, github.event.pull_request.user.login)) || github.ref_name }}-${{ github.sha }}-amd64
-            DOCKER_TAG: docker.io/${{ needs.variables.outputs.github_repository }}:${{ (github.head_ref != '' && format('merge-{0}-{1}', github.event.number, github.event.pull_request.user.login)) || github.ref_name }}-${{ github.sha }}
-            DOCKER_BRANCH: docker.io/${{ needs.variables.outputs.github_repository }}:${{ (startsWith(github.ref, 'refs/tags/v') && !endsWith(github.ref, '-rc') && 'latest') || (github.head_ref != '' && format('merge-{0}-{1}', github.event.number, github.event.pull_request.user.login)) || github.ref_name }}
-            GHCR_ARM64: ghcr.io/${{ needs.variables.outputs.github_repository }}:${{ (github.head_ref != '' && format('merge-{0}-{1}', github.event.number, github.event.pull_request.user.login)) || github.ref_name }}-${{ github.sha }}-arm64v8
-            GHCR_AMD64: ghcr.io/${{ needs.variables.outputs.github_repository }}:${{ (github.head_ref != '' && format('merge-{0}-{1}', github.event.number, github.event.pull_request.user.login)) || github.ref_name }}-${{ github.sha }}-amd64
-            GHCR_TAG: ghcr.io/${{ needs.variables.outputs.github_repository }}:${{ (github.head_ref != '' && format('merge-{0}-{1}', github.event.number, github.event.pull_request.user.login)) || github.ref_name }}-${{ github.sha }}
-            GHCR_BRANCH: ghcr.io/${{ needs.variables.outputs.github_repository }}:${{ (startsWith(github.ref, 'refs/tags/v') && !endsWith(github.ref, '-rc') && 'latest') || (github.head_ref != '' && format('merge-{0}-{1}', github.event.number, github.event.pull_request.user.login)) || github.ref_name }}
-            GLCR_ARM64: registry.gitlab.com/conduwuit/conduwuit:${{ (github.head_ref != '' && format('merge-{0}-{1}', github.event.number, github.event.pull_request.user.login)) || github.ref_name }}-${{ github.sha }}-arm64v8
-            GLCR_AMD64: registry.gitlab.com/conduwuit/conduwuit:${{ (github.head_ref != '' && format('merge-{0}-{1}', github.event.number, github.event.pull_request.user.login)) || github.ref_name }}-${{ github.sha }}-amd64
-            GLCR_TAG: registry.gitlab.com/conduwuit/conduwuit:${{ (github.head_ref != '' && format('merge-{0}-{1}', github.event.number, github.event.pull_request.user.login)) || github.ref_name }}-${{ github.sha }}
-            GLCR_BRANCH: registry.gitlab.com/conduwuit/conduwuit:${{ (startsWith(github.ref, 'refs/tags/v') && !endsWith(github.ref, '-rc') && 'latest') || (github.head_ref != '' && format('merge-{0}-{1}', github.event.number, github.event.pull_request.user.login)) || github.ref_name }}
+            DOCKER_HUB_REPO: docker.io/${{ needs.variables.outputs.github_repository }}
+            GHCR_REPO: ghcr.io/${{ needs.variables.outputs.github_repository }}
+            GLCR_REPO: registry.gitlab.com/conduwuit/conduwuit
+            UNIQUE_TAG: ${{ (github.head_ref != '' && format('merge-{0}-{1}', github.event.number, github.event.pull_request.user.login)) || github.ref_name }}-${{ github.sha }}
+            BRANCH_TAG: ${{ (startsWith(github.ref, 'refs/tags/v') && !endsWith(github.ref, '-rc') && 'latest') || (github.head_ref != '' && format('merge-{0}-{1}', github.event.number, github.event.pull_request.user.login)) || github.ref_name }}
 
             DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
             GITLAB_TOKEN: ${{ secrets.GITLAB_TOKEN }}
@@ -784,143 +807,187 @@ jobs:
 
             - name: Move OCI images into position
               run: |
+                  mv -v oci-image-x86_64-linux-musl-all-features-x86_64-haswell-optimised/*.tar.gz oci-image-amd64-haswell-optimised.tar.gz
                   mv -v oci-image-x86_64-linux-musl/*.tar.gz oci-image-amd64.tar.gz
                   mv -v oci-image-aarch64-linux-musl/*.tar.gz oci-image-arm64v8.tar.gz
                   mv -v oci-image-x86_64-linux-musl-debug/*.tar.gz oci-image-amd64-debug.tar.gz
                   mv -v oci-image-aarch64-linux-musl-debug/*.tar.gz oci-image-arm64v8-debug.tar.gz
 
+            - name: Load and push amd64 haswell image
+              run: |
+                  docker load -i oci-image-amd64-haswell-optimised.tar.gz
+                  if [ ! -z $DOCKERHUB_TOKEN ]; then
+                    docker tag $(docker images -q conduwuit:main) ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-haswell
+                    docker push ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-haswell
+                  fi
+                  if [ $GHCR_ENABLED = "true" ]; then
+                    docker tag $(docker images -q conduwuit:main) ${GHCR_REPO}:${UNIQUE_TAG}-haswell
+                    docker push ${GHCR_REPO}:${UNIQUE_TAG}-haswell
+                  fi
+                  if [ ! -z $GITLAB_TOKEN ]; then
+                    docker tag $(docker images -q conduwuit:main) ${GLCR_REPO}:${UNIQUE_TAG}-haswell
+                    docker push ${GLCR_REPO}:${UNIQUE_TAG}-haswell
+                  fi
+
             - name: Load and push amd64 image
               run: |
                   docker load -i oci-image-amd64.tar.gz
                   if [ ! -z $DOCKERHUB_TOKEN ]; then
-                    docker tag $(docker images -q conduwuit:main) ${DOCKER_AMD64}
-                    docker push ${DOCKER_AMD64}
+                    docker tag $(docker images -q conduwuit:main) ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-amd64
+                    docker push ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-amd64
                   fi
                   if [ $GHCR_ENABLED = "true" ]; then
-                    docker tag $(docker images -q conduwuit:main) ${GHCR_AMD64}
-                    docker push ${GHCR_AMD64}
+                    docker tag $(docker images -q conduwuit:main) ${GHCR_REPO}:${UNIQUE_TAG}-amd64
+                    docker push ${GHCR_REPO}:${UNIQUE_TAG}-amd64
                   fi
                   if [ ! -z $GITLAB_TOKEN ]; then
-                    docker tag $(docker images -q conduwuit:main) ${GLCR_AMD64}
-                    docker push ${GLCR_AMD64}
+                    docker tag $(docker images -q conduwuit:main) ${GLCR_REPO}:${UNIQUE_TAG}-amd64
+                    docker push ${GLCR_REPO}:${UNIQUE_TAG}-amd64
                   fi
 
             - name: Load and push arm64 image
               run: |
                   docker load -i oci-image-arm64v8.tar.gz
                   if [ ! -z $DOCKERHUB_TOKEN ]; then
-                    docker tag $(docker images -q conduwuit:main) ${DOCKER_ARM64}
-                    docker push ${DOCKER_ARM64}
+                    docker tag $(docker images -q conduwuit:main) ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-arm64v8
+                    docker push ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-arm64v8
                   fi
                   if [ $GHCR_ENABLED = "true" ]; then
-                    docker tag $(docker images -q conduwuit:main) ${GHCR_ARM64}
-                    docker push ${GHCR_ARM64}
+                    docker tag $(docker images -q conduwuit:main) ${GHCR_REPO}:${UNIQUE_TAG}-arm64v8
+                    docker push ${GHCR_REPO}:${UNIQUE_TAG}-arm64v8
                   fi
                   if [ ! -z $GITLAB_TOKEN ]; then
-                    docker tag $(docker images -q conduwuit:main) ${GLCR_ARM64}
-                    docker push ${GLCR_ARM64}
+                    docker tag $(docker images -q conduwuit:main) ${GLCR_REPO}:${UNIQUE_TAG}-arm64v8
+                    docker push ${GLCR_REPO}:${UNIQUE_TAG}-arm64v8
                   fi
 
             - name: Load and push amd64 debug image
               run: |
                   docker load -i oci-image-amd64-debug.tar.gz
                   if [ ! -z $DOCKERHUB_TOKEN ]; then
-                    docker tag $(docker images -q conduwuit:main) ${DOCKER_AMD64}-debug
-                    docker push ${DOCKER_AMD64}-debug
+                    docker tag $(docker images -q conduwuit:main) ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-amd64-debug
+                    docker push ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-amd64-debug
                   fi
                   if [ $GHCR_ENABLED = "true" ]; then
-                    docker tag $(docker images -q conduwuit:main) ${GHCR_AMD64}-debug
-                    docker push ${GHCR_AMD64}-debug
+                    docker tag $(docker images -q conduwuit:main) ${GHCR_REPO}:${UNIQUE_TAG}-amd64-debug
+                    docker push ${GHCR_REPO}:${UNIQUE_TAG}-amd64-debug
                   fi
                   if [ ! -z $GITLAB_TOKEN ]; then
-                    docker tag $(docker images -q conduwuit:main) ${GLCR_AMD64}-debug
-                    docker push ${GLCR_AMD64}-debug
+                    docker tag $(docker images -q conduwuit:main) ${GLCR_REPO}:${UNIQUE_TAG}-amd64-debug
+                    docker push ${GLCR_REPO}:${UNIQUE_TAG}-amd64-debug
                   fi
 
             - name: Load and push arm64 debug image
               run: |
                   docker load -i oci-image-arm64v8-debug.tar.gz
                   if [ ! -z $DOCKERHUB_TOKEN ]; then
-                    docker tag $(docker images -q conduwuit:main) ${DOCKER_ARM64}-debug
-                    docker push ${DOCKER_ARM64}-debug
+                    docker tag $(docker images -q conduwuit:main) ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-arm64v8-debug
+                    docker push ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-arm64v8-debug
                   fi
                   if [ $GHCR_ENABLED = "true" ]; then
-                    docker tag $(docker images -q conduwuit:main) ${GHCR_ARM64}-debug
-                    docker push ${GHCR_ARM64}-debug
+                    docker tag $(docker images -q conduwuit:main) ${GHCR_REPO}:${UNIQUE_TAG}-arm64v8-debug
+                    docker push ${GHCR_REPO}:${UNIQUE_TAG}-arm64v8-debug
                   fi
                   if [ ! -z $GITLAB_TOKEN ]; then
-                    docker tag $(docker images -q conduwuit:main) ${GLCR_ARM64}-debug
-                    docker push ${GLCR_ARM64}-debug
+                    docker tag $(docker images -q conduwuit:main) ${GLCR_REPO}:${UNIQUE_TAG}-arm64v8-debug
+                    docker push ${GLCR_REPO}:${UNIQUE_TAG}-arm64v8-debug
+                  fi
+
+            - name: Create Docker haswell manifests
+              run: |
+                  # Dockerhub Container Registry
+                  if [ ! -z $DOCKERHUB_TOKEN ]; then
+                    docker manifest create ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-haswell --amend ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-haswell
+                    docker manifest create ${DOCKER_HUB_REPO}:${BRANCH_TAG}-haswell --amend ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-haswell
+                  fi
+                  # GitHub Container Registry
+                  if [ $GHCR_ENABLED = "true" ]; then
+                    docker manifest create ${GHCR_REPO}:${UNIQUE_TAG}-haswell --amend ${GHCR_REPO}:${UNIQUE_TAG}-haswell
+                    docker manifest create ${GHCR_REPO}:${BRANCH_TAG}-haswell --amend ${GHCR_REPO}:${UNIQUE_TAG}-haswell
+                  fi
+                  # GitLab Container Registry
+                  if [ ! -z $GITLAB_TOKEN ]; then
+                    docker manifest create ${GLCR_REPO}:${UNIQUE_TAG}-haswell --amend ${GLCR_REPO}:${UNIQUE_TAG}-haswell
+                    docker manifest create ${GLCR_REPO}:${BRANCH_TAG}-haswell --amend ${GLCR_REPO}:${UNIQUE_TAG}-haswell
                   fi
 
             - name: Create Docker combined manifests
               run: |
                   # Dockerhub Container Registry
                   if [ ! -z $DOCKERHUB_TOKEN ]; then
-                    docker manifest create ${DOCKER_TAG} --amend ${DOCKER_ARM64} --amend ${DOCKER_AMD64}
-                    docker manifest create ${DOCKER_BRANCH} --amend ${DOCKER_ARM64} --amend ${DOCKER_AMD64}
+                    docker manifest create ${DOCKER_HUB_REPO}:${UNIQUE_TAG} --amend ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-arm64v8 --amend ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-amd64
+                    docker manifest create ${DOCKER_HUB_REPO}:${BRANCH_TAG} --amend ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-arm64v8 --amend ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-amd64
                   fi
                   # GitHub Container Registry
                   if [ $GHCR_ENABLED = "true" ]; then
-                    docker manifest create ${GHCR_TAG} --amend ${GHCR_ARM64} --amend ${GHCR_AMD64}
-                    docker manifest create ${GHCR_BRANCH} --amend ${GHCR_ARM64} --amend ${GHCR_AMD64}
+                    docker manifest create ${GHCR_REPO}:${UNIQUE_TAG} --amend ${GHCR_REPO}:${UNIQUE_TAG}-arm64v8 --amend ${GHCR_REPO}:${UNIQUE_TAG}-amd64
+                    docker manifest create ${GHCR_REPO}:${BRANCH_TAG} --amend ${GHCR_REPO}:${UNIQUE_TAG}-arm64v8 --amend ${GHCR_REPO}:${UNIQUE_TAG}-amd64
                   fi
                   # GitLab Container Registry
                   if [ ! -z $GITLAB_TOKEN ]; then
-                    docker manifest create ${GLCR_TAG} --amend ${GLCR_ARM64} --amend ${GLCR_AMD64}
-                    docker manifest create ${GLCR_BRANCH} --amend ${GLCR_ARM64} --amend ${GLCR_AMD64}
+                    docker manifest create ${GLCR_REPO}:${UNIQUE_TAG} --amend ${GLCR_REPO}:${UNIQUE_TAG}-arm64v8 --amend ${GLCR_REPO}:${UNIQUE_TAG}-amd64
+                    docker manifest create ${GLCR_REPO}:${BRANCH_TAG} --amend ${GLCR_REPO}:${UNIQUE_TAG}-arm64v8 --amend ${GLCR_REPO}:${UNIQUE_TAG}-amd64
                   fi
 
             - name: Create Docker combined debug manifests
               run: |
                   # Dockerhub Container Registry
                   if [ ! -z $DOCKERHUB_TOKEN ]; then
-                    docker manifest create ${DOCKER_TAG}-debug --amend ${DOCKER_ARM64}-debug --amend ${DOCKER_AMD64}-debug
-                    docker manifest create ${DOCKER_BRANCH}-debug --amend ${DOCKER_ARM64}-debug --amend ${DOCKER_AMD64}-debug
+                    docker manifest create ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-debug --amend ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-arm64v8-debug --amend ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-amd64-debug
+                    docker manifest create ${DOCKER_HUB_REPO}:${BRANCH_TAG}-debug --amend ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-arm64v8-debug --amend ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-amd64-debug
                   fi
                   # GitHub Container Registry
                   if [ $GHCR_ENABLED = "true" ]; then
-                    docker manifest create ${GHCR_TAG}-debug --amend ${GHCR_ARM64}-debug --amend ${GHCR_AMD64}-debug
-                    docker manifest create ${GHCR_BRANCH}-debug --amend ${GHCR_ARM64}-debug --amend ${GHCR_AMD64}-debug
+                    docker manifest create ${GHCR_REPO}:${UNIQUE_TAG}-debug --amend ${GHCR_REPO}:${UNIQUE_TAG}-arm64v8-debug --amend ${GHCR_REPO}:${UNIQUE_TAG}-amd64-debug
+                    docker manifest create ${GHCR_REPO}:${BRANCH_TAG}-debug --amend ${GHCR_REPO}:${UNIQUE_TAG}-arm64v8-debug --amend ${GHCR_REPO}:${UNIQUE_TAG}-amd64-debug
                   fi
                   # GitLab Container Registry
                   if [ ! -z $GITLAB_TOKEN ]; then
-                    docker manifest create ${GLCR_TAG}-debug --amend ${GLCR_ARM64}-debug --amend ${GLCR_AMD64}-debug
-                    docker manifest create ${GLCR_BRANCH}-debug --amend ${GLCR_ARM64}-debug --amend ${GLCR_AMD64}-debug
+                    docker manifest create ${GLCR_REPO}:${UNIQUE_TAG}-debug --amend ${GLCR_REPO}:${UNIQUE_TAG}-arm64v8-debug --amend ${GLCR_REPO}:${UNIQUE_TAG}-amd64-debug
+                    docker manifest create ${GLCR_REPO}:${BRANCH_TAG}-debug --amend ${GLCR_REPO}:${UNIQUE_TAG}-arm64v8-debug --amend ${GLCR_REPO}:${UNIQUE_TAG}-amd64-debug
                   fi
 
             - name: Push manifests to Docker registries
               run: |
                   if [ ! -z $DOCKERHUB_TOKEN ]; then
-                    docker manifest push ${DOCKER_TAG}
-                    docker manifest push ${DOCKER_BRANCH}
-                    docker manifest push ${DOCKER_TAG}-debug
-                    docker manifest push ${DOCKER_BRANCH}-debug
+                    docker manifest push ${DOCKER_HUB_REPO}:${UNIQUE_TAG}
+                    docker manifest push ${DOCKER_HUB_REPO}:${BRANCH_TAG}
+                    docker manifest push ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-debug
+                    docker manifest push ${DOCKER_HUB_REPO}:${BRANCH_TAG}-debug
+                    docker manifest push ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-haswell
+                    docker manifest push ${DOCKER_HUB_REPO}:${BRANCH_TAG}-haswell
                   fi
                   if [ $GHCR_ENABLED = "true" ]; then
-                    docker manifest push ${GHCR_TAG}
-                    docker manifest push ${GHCR_BRANCH}
-                    docker manifest push ${GHCR_TAG}-debug
-                    docker manifest push ${GHCR_BRANCH}-debug
+                    docker manifest push ${GHCR_REPO}:${UNIQUE_TAG}
+                    docker manifest push ${GHCR_REPO}:${BRANCH_TAG}
+                    docker manifest push ${GHCR_REPO}:${UNIQUE_TAG}-debug
+                    docker manifest push ${GHCR_REPO}:${BRANCH_TAG}-debug
+                    docker manifest push ${GHCR_REPO}:${UNIQUE_TAG}-haswell
+                    docker manifest push ${GHCR_REPO}:${BRANCH_TAG}-haswell
                   fi
                   if [ ! -z $GITLAB_TOKEN ]; then
-                    docker manifest push ${GLCR_TAG}
-                    docker manifest push ${GLCR_BRANCH}
-                    docker manifest push ${GLCR_TAG}-debug
-                    docker manifest push ${GLCR_BRANCH}-debug
+                    docker manifest push ${GLCR_REPO}:${UNIQUE_TAG}
+                    docker manifest push ${GLCR_REPO}:${BRANCH_TAG}
+                    docker manifest push ${GLCR_REPO}:${UNIQUE_TAG}-debug
+                    docker manifest push ${GLCR_REPO}:${BRANCH_TAG}-debug
+                    docker manifest push ${GLCR_REPO}:${UNIQUE_TAG}-haswell
+                    docker manifest push ${GLCR_REPO}:${BRANCH_TAG}-haswell
                   fi
 
             - name: Add Image Links to Job Summary
               run: |
                   if [ ! -z $DOCKERHUB_TOKEN ]; then
-                    echo "- \`docker pull ${DOCKER_TAG}\`" >> $GITHUB_STEP_SUMMARY
-                    echo "- \`docker pull ${DOCKER_TAG}-debug\`" >> $GITHUB_STEP_SUMMARY
+                    echo "- \`docker pull ${DOCKER_HUB_REPO}:${UNIQUE_TAG}\`" >> $GITHUB_STEP_SUMMARY
+                    echo "- \`docker pull ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-debug\`" >> $GITHUB_STEP_SUMMARY
+                    echo "- \`docker pull ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-haswell\`" >> $GITHUB_STEP_SUMMARY
                   fi
                   if [ $GHCR_ENABLED = "true" ]; then
-                    echo "- \`docker pull ${GHCR_TAG}\`" >> $GITHUB_STEP_SUMMARY
-                    echo "- \`docker pull ${GHCR_TAG}-debug\`" >> $GITHUB_STEP_SUMMARY
+                    echo "- \`docker pull ${GHCR_REPO}:${UNIQUE_TAG}\`" >> $GITHUB_STEP_SUMMARY
+                    echo "- \`docker pull ${GHCR_REPO}:${UNIQUE_TAG}-debug\`" >> $GITHUB_STEP_SUMMARY
+                    echo "- \`docker pull ${GHCR_REPO}:${UNIQUE_TAG}-haswell\`" >> $GITHUB_STEP_SUMMARY
                   fi
                   if [ ! -z $GITLAB_TOKEN ]; then
-                    echo "- \`docker pull ${GLCR_TAG}\`" >> $GITHUB_STEP_SUMMARY
-                    echo "- \`docker pull ${GLCR_TAG}-debug\`" >> $GITHUB_STEP_SUMMARY
+                    echo "- \`docker pull ${GLCR_REPO}:${UNIQUE_TAG}\`" >> $GITHUB_STEP_SUMMARY
+                    echo "- \`docker pull ${GLCR_REPO}:${UNIQUE_TAG}-debug\`" >> $GITHUB_STEP_SUMMARY
+                    echo "- \`docker pull ${GLCR_REPO}:${UNIQUE_TAG}-haswell\`" >> $GITHUB_STEP_SUMMARY
                   fi

.github/workflows/docker-hub-description.yml

@@ -0,0 +1,41 @@
+name: Update Docker Hub Description
+
+on:
+  push:
+    branches:
+      - main
+    paths:
+      - README.md
+      - .github/workflows/docker-hub-description.yml
+
+  workflow_dispatch:
+
+jobs:
+  dockerHubDescription:
+    runs-on: ubuntu-latest
+    if: ${{ (startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main' || (github.event.pull_request.draft != true)) && github.event.pull_request.user.login != 'renovate[bot]' && (vars.DOCKER_USERNAME != '') }}
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        persist-credentials: false
+
+    - name: Setting variables
+      uses: actions/github-script@v7
+      id: var
+      with:
+        script: |
+          const githubRepo = '${{ github.repository }}'.toLowerCase()
+          const repoId = githubRepo.split('/')[1]
+          
+          core.setOutput('github_repository', githubRepo)
+          const dockerRepo = '${{ vars.DOCKER_USERNAME }}'.toLowerCase() + '/' + repoId
+          core.setOutput('docker_repo', dockerRepo)
+
+    - name: Docker Hub Description
+      uses: peter-evans/dockerhub-description@v4
+      with:
+        username: ${{ vars.DOCKER_USERNAME }}
+        password: ${{ secrets.DOCKERHUB_TOKEN }}
+        repository: ${{ steps.var.outputs.docker_repo }}
+        short-description: ${{ github.event.repository.description }}
+        enable-url-completion: true

.github/workflows/documentation.yml

@@ -24,7 +24,7 @@ env:
   # Get error output from nix that we can actually use, and use our binary caches for the earlier CI steps
   NIX_CONFIG: |
     show-trace = true
-    extra-substituters = extra-substituters = https://attic.kennel.juneis.dog/conduwuit https://attic.kennel.juneis.dog/conduit https://cache.lix.systems https://conduwuit.cachix.org https://aseipp-nix-cache.freetls.fastly.net
+    extra-substituters = extra-substituters = https://attic.kennel.juneis.dog/conduwuit https://attic.kennel.juneis.dog/conduit https://conduwuit.cachix.org https://aseipp-nix-cache.freetls.fastly.net
     extra-trusted-public-keys = conduit:eEKoUwlQGDdYmAI/Q/0slVlegqh/QmAvQd7HBSm21Wk= conduwuit:BbycGUgTISsltcmH0qNjFR9dbrQNYgdIAcmViSGoVTE= cache.lix.systems:aBnZUw8zA7H35Cz2RyKFVs3H4PlGTLawyY5KRbvJR8o= conduwuit.cachix.org-1:MFRm6jcnfTf0jSAbmvLfhO3KBMt4px+1xaereWXp8Xg=
     experimental-features = nix-command flakes
     extra-experimental-features = nix-command flakes
@@ -73,6 +73,9 @@ jobs:
       - uses: nixbuild/nix-quick-install-action@master
 
       - name: Restore and cache Nix store
+        # we want a fresh-state when we do releases/tags to avoid potential cache poisoning attacks impacting
+        # releases and tags
+        if: ${{ !startsWith(github.ref, 'refs/tags/') }}
         uses: nix-community/cache-nix-action@v5.1.0
         with:
           # restore and save a cache using this key
@@ -102,7 +105,7 @@ jobs:
       - name: Apply Nix binary cache configuration
         run: |
             sudo tee -a "${XDG_CONFIG_HOME:-$HOME/.config}/nix/nix.conf" > /dev/null <<EOF
-            extra-substituters = https://attic.kennel.juneis.dog/conduwuit https://attic.kennel.juneis.dog/conduit https://cache.lix.systems https://conduwuit.cachix.org https://aseipp-nix-cache.freetls.fastly.net
+            extra-substituters = https://attic.kennel.juneis.dog/conduwuit https://attic.kennel.juneis.dog/conduit https://conduwuit.cachix.org https://aseipp-nix-cache.freetls.fastly.net
             extra-trusted-public-keys = conduit:eEKoUwlQGDdYmAI/Q/0slVlegqh/QmAvQd7HBSm21Wk= conduwuit:BbycGUgTISsltcmH0qNjFR9dbrQNYgdIAcmViSGoVTE= cache.lix.systems:aBnZUw8zA7H35Cz2RyKFVs3H4PlGTLawyY5KRbvJR8o= conduwuit.cachix.org-1:MFRm6jcnfTf0jSAbmvLfhO3KBMt4px+1xaereWXp8Xg=
             experimental-features = nix-command flakes
             extra-experimental-features = nix-command flakes

.gitlab-ci.yml

@@ -12,8 +12,8 @@ variables:
   TRANSFER_METER_FREQUENCY: 5s
   NIX_CONFIG: |
     show-trace = true
-    extra-substituters = https://attic.kennel.juneis.dog/conduit https://attic.kennel.juneis.dog/conduwuit https://cache.lix.systems https://conduwuit.cachix.org
-    extra-trusted-public-keys = conduit:eEKoUwlQGDdYmAI/Q/0slVlegqh/QmAvQd7HBSm21Wk= conduwuit:BbycGUgTISsltcmH0qNjFR9dbrQNYgdIAcmViSGoVTE= cache.lix.systems:aBnZUw8zA7H35Cz2RyKFVs3H4PlGTLawyY5KRbvJR8o= conduwuit.cachix.org-1:MFRm6jcnfTf0jSAbmvLfhO3KBMt4px+1xaereWXp8Xg=
+    extra-substituters = https://attic.kennel.juneis.dog/conduit https://attic.kennel.juneis.dog/conduwuit https://conduwuit.cachix.org
+    extra-trusted-public-keys = conduit:eEKoUwlQGDdYmAI/Q/0slVlegqh/QmAvQd7HBSm21Wk= conduwuit:BbycGUgTISsltcmH0qNjFR9dbrQNYgdIAcmViSGoVTE= conduwuit.cachix.org-1:MFRm6jcnfTf0jSAbmvLfhO3KBMt4px+1xaereWXp8Xg=
     experimental-features = nix-command flakes
     extra-experimental-features = nix-command flakes
     accept-flake-config = true
@@ -45,10 +45,6 @@ before_script:
   - if command -v nix > /dev/null && [ -n "$ATTIC_ENDPOINT" ]; then echo "extra-substituters = $ATTIC_ENDPOINT" >> /etc/nix/nix.conf; fi
   - if command -v nix > /dev/null && [ -n "$ATTIC_PUBLIC_KEY" ]; then echo "extra-trusted-public-keys = $ATTIC_PUBLIC_KEY" >> /etc/nix/nix.conf; fi
 
-  # Add Lix binary cache
-  - if command -v nix > /dev/null; then echo "extra-substituters = https://cache.lix.systems" >> /etc/nix/nix.conf; fi
-  - if command -v nix > /dev/null; then echo "extra-trusted-public-keys = cache.lix.systems:aBnZUw8zA7H35Cz2RyKFVs3H4PlGTLawyY5KRbvJR8o=" >> /etc/nix/nix.conf; fi
-
   # Add crane binary cache
   - if command -v nix > /dev/null; then echo "extra-substituters = https://crane.cachix.org" >> /etc/nix/nix.conf; fi
   - if command -v nix > /dev/null; then echo "extra-trusted-public-keys = crane.cachix.org-1:8Scfpmn9w+hGdXH/Q9tTLiYAE/2dnJYRJP7kl80GuRk=" >> /etc/nix/nix.conf; fi

Cargo.toml

@@ -7,19 +7,20 @@ default-members = ["src/*"]
 
 [workspace.package]
 authors = [
-    "strawberry <strawberry@puppygock.gay>",
-    "timokoesters <timo@koesters.xyz>",
+    "June Clementine Strawberry <june@girlboss.ceo>",
+    "strawberry <strawberry@puppygock.gay>", # woof
+    "Jason Volk <jason@zemos.net>",
 ]
 categories = ["network-programming"]
-description = "a very cool fork of Conduit, a Matrix homeserver written in Rust"
+description = "a very cool Matrix chat homeserver written in Rust"
 edition = "2021"
 homepage = "https://conduwuit.puppyirl.gay/"
-keywords = ["chat", "matrix", "server", "uwu"]
+keywords = ["chat", "matrix", "networking", "server", "uwu"]
 license = "Apache-2.0"
 # See also `rust-toolchain.toml`
 readme = "README.md"
 repository = "https://github.com/girlbossceo/conduwuit"
-rust-version = "1.83.0"
+rust-version = "1.84.0"
 version = "0.5.0"
 
 [workspace.metadata.crane]
@@ -58,10 +59,6 @@ features = ["parse"]
 [workspace.dependencies.sanitize-filename]
 version = "0.6.0"
 
-[workspace.dependencies.jsonwebtoken]
-version = "9.3.0"
-default-features = false
-
 [workspace.dependencies.base64]
 version = "0.22.1"
 default-features = false
@@ -336,7 +333,7 @@ version = "0.1.2"
 [workspace.dependencies.ruma]
 git = "https://github.com/girlbossceo/ruwuma"
 #branch = "conduwuit-changes"
-rev = "c4f55b39900b33b2d443dd12a6a2dab50961fdfb"
+rev = "b560338b2a50dbf61ecfe80808b9b095ad4cec00"
 features = [
     "compat",
     "rand",
@@ -435,17 +432,23 @@ version = "0.35.0"
 # jemalloc usage
 [workspace.dependencies.tikv-jemalloc-sys]
 git = "https://github.com/girlbossceo/jemallocator"
-rev = "d87938bfddc26377dd7fdf14bbcd345f3ab19442"
+rev = "82af58d6a13ddd5dcdc7d4e91eae3b63292995b8"
 default-features = false
-features = ["unprefixed_malloc_on_supported_platforms"]
+features = [
+	"background_threads_runtime_support",
+	"unprefixed_malloc_on_supported_platforms",
+]
 [workspace.dependencies.tikv-jemallocator]
 git = "https://github.com/girlbossceo/jemallocator"
-rev = "d87938bfddc26377dd7fdf14bbcd345f3ab19442"
+rev = "82af58d6a13ddd5dcdc7d4e91eae3b63292995b8"
 default-features = false
-features = ["unprefixed_malloc_on_supported_platforms"]
+features = [
+	"background_threads_runtime_support",
+	"unprefixed_malloc_on_supported_platforms",
+]
 [workspace.dependencies.tikv-jemalloc-ctl]
 git = "https://github.com/girlbossceo/jemallocator"
-rev = "d87938bfddc26377dd7fdf14bbcd345f3ab19442"
+rev = "82af58d6a13ddd5dcdc7d4e91eae3b63292995b8"
 default-features = false
 features = ["use_std"]
 
@@ -504,6 +507,14 @@ version = "0.2"
 [workspace.dependencies.num-traits]
 version = "0.2"
 
+[workspace.dependencies.minicbor]
+version = "0.25.1"
+features = ["std"]
+
+[workspace.dependencies.minicbor-serde]
+version = "0.3.2"
+features = ["std"]
+
 #
 # Patches
 #
@@ -513,16 +524,16 @@ version = "0.2"
 # https://github.com/girlbossceo/tracing/commit/b348dca742af641c47bc390261f60711c2af573c
 [patch.crates-io.tracing-subscriber]
 git = "https://github.com/girlbossceo/tracing"
-rev = "ccc4fbd8238c2d5ba354e61ec17ac610af11401d"
+rev = "05825066a6d0e9ad6b80dcf29457eb179ff4768c"
 [patch.crates-io.tracing]
 git = "https://github.com/girlbossceo/tracing"
-rev = "ccc4fbd8238c2d5ba354e61ec17ac610af11401d"
+rev = "05825066a6d0e9ad6b80dcf29457eb179ff4768c"
 [patch.crates-io.tracing-core]
 git = "https://github.com/girlbossceo/tracing"
-rev = "ccc4fbd8238c2d5ba354e61ec17ac610af11401d"
+rev = "05825066a6d0e9ad6b80dcf29457eb179ff4768c"
 [patch.crates-io.tracing-log]
 git = "https://github.com/girlbossceo/tracing"
-rev = "ccc4fbd8238c2d5ba354e61ec17ac610af11401d"
+rev = "05825066a6d0e9ad6b80dcf29457eb179ff4768c"
 
 # adds a tab completion callback: https://github.com/girlbossceo/rustyline-async/commit/de26100b0db03e419a3d8e1dd26895d170d1fe50
 # adds event for CTRL+\: https://github.com/girlbossceo/rustyline-async/commit/67d8c49aeac03a5ef4e818f663eaa94dd7bf339b
@@ -873,6 +884,7 @@ enum_glob_use = { level = "allow", priority = 1 }
 if_not_else = { level = "allow", priority = 1 }
 if_then_some_else_none = { level = "allow", priority = 1 }
 inline_always = { level = "allow", priority = 1 }
+match_bool = { level = "allow", priority = 1 }
 missing_docs_in_private_items = { level = "allow", priority = 1 }
 missing_errors_doc = { level = "allow", priority = 1 }
 missing_panics_doc = { level = "allow", priority = 1 }

README.md

@@ -4,7 +4,7 @@
 
 <!-- ANCHOR: catchphrase -->
 
-### a very cool, featureful fork of [Conduit](https://conduit.rs/)
+### a very cool [Matrix](https://matrix.org/) chat homeserver written in Rust
 
 <!-- ANCHOR_END: catchphrase -->
 
@@ -15,16 +15,15 @@ information and how to deploy/setup conduwuit.
 
 #### What is Matrix?
 
-[Matrix](https://matrix.org) is an open network for secure and decentralized
-communication. Users from every Matrix homeserver can chat with users from all
-other Matrix servers. You can even use bridges (also called Matrix Appservices)
-to communicate with users outside of Matrix, like a community on Discord.
+[Matrix](https://matrix.org) is an open, federated, and extensible network for
+decentralised communication. Users from any Matrix homeserver can chat with users from all
+other homeservers over federation. Matrix is designed to be extensible and built on top of.
+You can even use bridges such as Matrix Appservices to communicate with users outside of Matrix, like a community on Discord.
 
 #### What is the goal?
 
-A high-performance and efficient Matrix homeserver that's easy to set up and
-just works. You can install it on a mini-computer like the Raspberry Pi to
-host Matrix for your family, friends or company.
+A high-performance, efficient, low-cost, and featureful Matrix homeserver that's
+easy to set up and just works with minimal configuration needed.
 
 #### Can I try it out?
 
@@ -37,17 +36,22 @@ homeserver". This means there are rules, so please read the rules:
 [https://transfem.dev/homeserver_rules.txt](https://transfem.dev/homeserver_rules.txt)
 
 transfem.dev is also listed at
-[servers.joinmatrix.org](https://servers.joinmatrix.org/)
+[servers.joinmatrix.org](https://servers.joinmatrix.org/), which is a list of
+popular public Matrix homeservers, including some others that run conduwuit.
 
 #### What is the current status?
 
-conduwuit is technically a hard fork of Conduit, which is in Beta. The Beta status
-initially was inherited from Conduit, however overtime this Beta status is rapidly
-becoming less and less relevant as our codebase significantly diverges more and more.
+conduwuit is technically a hard fork of [Conduit](https://conduit.rs/), which is in beta.
+The beta status initially was inherited from Conduit, however the huge amount of
+codebase divergance, changes, fixes, and improvements have effectively made this
+beta status not entirely applicable to us anymore.
 
-conduwuit is quite stable and very usable as a daily driver and for a low-medium
-sized homeserver. There is still a lot of more work to be done, but it is in a far
-better place than the project was in early 2024.
+conduwuit is very stable based on our rapidly growing userbase, has lots of features that users
+expect, and very usable as a daily driver for small, medium, and upper-end medium sized homeservers.
+
+A lot of critical stability and performance issues have been fixed, and a lot of
+necessary groundwork has finished; making this project way better than it was
+back in the start at ~early 2024.
 
 #### How is conduwuit funded? Is conduwuit sustainable?
 
@@ -72,16 +76,37 @@ Conduit like before. If you are truly finding yourself wanting to migrate back
 to Conduit, we would appreciate all your feedback and if we can assist with
 any issues or concerns.
 
+#### Can I migrate from Synapse or Dendrite?
+
+Currently there is no known way to seamlessly migrate all user data from the old
+homeserver to conduwuit. However it is perfectly acceptable to replace the old
+homeserver software with conduwuit using the same server name and there will not
+be any issues with federation.
+
+There is an interest in developing a built-in seamless user data migration
+method into conduwuit, however there is no concrete ETA or timeline for this.
+
+
 <!-- ANCHOR_END: body -->
 
 <!-- ANCHOR: footer -->
 
 #### Contact
 
-If you run into any question, feel free to
+[`#conduwuit:puppygock.gay`](https://matrix.to/#/#conduwuit:puppygock.gay)
+is the official project Matrix room. You can get support here, ask questions or
+concerns, get assistance setting up conduwuit, etc.
 
-- Ask us in `#conduwuit:puppygock.gay` on Matrix
-- [Open an issue on GitHub](https://github.com/girlbossceo/conduwuit/issues/new)
+This room should stay relevant and focused on conduwuit. An offtopic general
+chatter room can be found there as well.
+
+Please keep the issue trackers focused on bug reports and enhancement requests.
+General support is extremely difficult to be offered over an issue tracker, and
+simple questions should be asked directly in an interactive platform like our
+Matrix room above as they can turn into a relevant discussion and/or may not be
+simple to answer. If you're not sure, just ask in the Matrix room.
+
+If you have a bug or feature to request: [Open an issue on GitHub](https://github.com/girlbossceo/conduwuit/issues/new)
 
 #### Donate
 
@@ -89,9 +114,11 @@ conduwuit development is purely made possible by myself and contributors. I do
 not get paid to work on this, and I work on it in my free time. Donations are
 heavily appreciated! 💜🥺
 
-- Liberapay: <https://liberapay.com/girlbossceo>
-- Ko-fi (note they take a fee): <https://ko-fi.com/puppygock>
-- GitHub Sponsors: <https://github.com/sponsors/girlbossceo>
+- Liberapay (preferred): <https://liberapay.com/girlbossceo>
+- GitHub Sponsors (preferred): <https://github.com/sponsors/girlbossceo>
+- Ko-fi: <https://ko-fi.com/puppygock>
+
+I do not and will not accept cryptocurrency donations, including things related.
 
 #### Logo
 
@@ -105,11 +132,15 @@ Both, but I prefer conduwuit.
 
 #### Mirrors of conduwuit
 
+If GitHub is unavailable in your country, or has poor connectivity, conduwuit's
+source code is mirrored onto the following additional platforms I maintain:
+
 - GitHub: <https://github.com/girlbossceo/conduwuit>
 - GitLab: <https://gitlab.com/conduwuit/conduwuit>
 - git.girlcock.ceo: <https://git.girlcock.ceo/strawberry/conduwuit>
 - git.gay: <https://git.gay/june/conduwuit>
-- Codeberg: <https://codeberg.org/girlbossceo/conduwuit>
+- mau.dev: <https://mau.dev/june/conduwuit>
+- Codeberg: <https://codeberg.org/arf/conduwuit>
 - sourcehut: <https://git.sr.ht/~girlbossceo/conduwuit>
 
 <!-- ANCHOR_END: footer -->

bin/complement

@@ -34,7 +34,9 @@ toplevel="$(git rev-parse --show-toplevel)"
 
 pushd "$toplevel" > /dev/null
 
-bin/nix-build-and-cache just .#linux-complement
+#bin/nix-build-and-cache just .#linux-complement
+bin/nix-build-and-cache just .#complement
+#nom build .#complement
 
 docker load < result
 popd > /dev/null

conduwuit-example.toml

@@ -100,20 +100,6 @@
 #
 #database_backups_to_keep = 1
 
-# Set this to any float value in megabytes for conduwuit to tell the
-# database engine that this much memory is available for database-related
-# caches.
-#
-# May be useful if you have significant memory to spare to increase
-# performance.
-#
-# Similar to the individual LRU caches, this is scaled up with your CPU
-# core count.
-#
-# This defaults to 128.0 + (64.0 * CPU core count).
-#
-#db_cache_capacity_mb = varies by system
-
 # Text which will be added to the end of the user's displayname upon
 # registration with a space before the text. In Conduit, this was the
 # lightning bolt emoji.
@@ -149,6 +135,34 @@
 #
 #cache_capacity_modifier = 1.0
 
+# Set this to any float value in megabytes for conduwuit to tell the
+# database engine that this much memory is available for database read
+# caches.
+#
+# May be useful if you have significant memory to spare to increase
+# performance.
+#
+# Similar to the individual LRU caches, this is scaled up with your CPU
+# core count.
+#
+# This defaults to 128.0 + (64.0 * CPU core count).
+#
+#db_cache_capacity_mb = varies by system
+
+# Set this to any float value in megabytes for conduwuit to tell the
+# database engine that this much memory is available for database write
+# caches.
+#
+# May be useful if you have significant memory to spare to increase
+# performance.
+#
+# Similar to the individual LRU caches, this is scaled up with your CPU
+# core count.
+#
+# This defaults to 48.0 + (4.0 * CPU core count).
+#
+#db_write_buffer_capacity_mb = varies by system
+
 # This item is undocumented. Please contribute documentation for it.
 #
 #pdu_cache_capacity = varies by system
@@ -375,13 +389,16 @@
 #
 #allow_registration = false
 
-# This item is undocumented. Please contribute documentation for it.
+# Enabling this setting opens registration to anyone without restrictions.
+# This makes your server vulnerable to abuse
 #
 #yes_i_am_very_very_sure_i_want_an_open_registration_server_prone_to_abuse = false
 
 # A static registration token that new users will have to provide when
 # creating an account. If unset and `allow_registration` is true,
-# registration is open without any condition.
+# you must set
+# `yes_i_am_very_very_sure_i_want_an_open_registration_server_prone_to_abuse`
+# to true to allow open registration without any conditions.
 #
 # YOU NEED TO EDIT THIS OR USE registration_token_file.
 #
@@ -549,10 +566,6 @@
 #
 #proxy = "none"
 
-# This item is undocumented. Please contribute documentation for it.
-#
-#jwt_secret =
-
 # Servers listed here will be used to gather public keys of other servers
 # (notary trusted key servers).
 #
@@ -635,6 +648,22 @@
 #
 #openid_token_ttl = 3600
 
+# Allow an existing session to mint a login token for another client.
+# This requires interactive authentication, but has security ramifications
+# as a malicious client could use the mechanism to spawn more than one
+# session.
+# Enabled by default.
+#
+#login_via_existing_session = true
+
+# Login token expiration/TTL in milliseconds.
+#
+# These are short-lived tokens for the m.login.token endpoint.
+# This is used to allow existing sessions to create new sessions.
+# see login_via_existing_session.
+#
+#login_token_ttl = 120000
+
 # Static TURN username to provide the client if not using a shared secret
 # ("turn_secret"), It is recommended to use a shared secret over static
 # credentials.
@@ -792,6 +821,9 @@
 # magic number and translated to the library's default compression level
 # as they all differ. See their `kDefaultCompressionLevel`.
 #
+# Note when using the default value we may override it with a setting
+# tailored specifically conduwuit.
+#
 #rocksdb_compression_level = 32767
 
 # Level of compression the specified compression algorithm for the
@@ -805,6 +837,9 @@
 # less likely for this data to be used. Research your chosen compression
 # algorithm.
 #
+# Note when using the default value we may override it with a setting
+# tailored specifically conduwuit.
+#
 #rocksdb_bottommost_compression_level = 32767
 
 # Whether to enable RocksDB's "bottommost_compression".
@@ -816,7 +851,7 @@
 #
 # See https://github.com/facebook/rocksdb/wiki/Compression for more details.
 #
-#rocksdb_bottommost_compression = false
+#rocksdb_bottommost_compression = true
 
 # Database recovery mode (for RocksDB WAL corruption).
 #
@@ -862,6 +897,13 @@
 #
 #rocksdb_paranoid_file_checks = false
 
+# Enables or disables checksum verification in rocksdb at runtime.
+# Checksums are usually hardware accelerated with low overhead; they are
+# enabled in rocksdb by default. Older or slower platforms may see gains
+# from disabling.
+#
+#rocksdb_checksums = true
+
 # Database repair mode (for RocksDB SST corruption).
 #
 # Use this option when the server reports corruption while running or
@@ -1320,6 +1362,13 @@
 #
 #admin_execute_errors_ignore = false
 
+# List of admin commands to execute on SIGUSR2.
+#
+# Similar to admin_execute, but these commands are executed when the
+# server receives SIGUSR2 on supporting platforms.
+#
+#admin_signal_execute = []
+
 # Controls the max log level for admin command log captures (logs
 # generated from running admin commands). Defaults to "info" on release
 # builds, else "debug" on debug builds.
@@ -1452,7 +1501,7 @@
 # responsiveness for many users at the cost of throughput for each.
 #
 # Setting this value to 0.0 causes the stream width to be fixed at the
-# value of stream_width_default. The default is 1.0 to match the
+# value of stream_width_default. The default scale is 1.0 to match the
 # capabilities detected for the system.
 #
 #stream_width_scale = 1.0
@@ -1477,6 +1526,16 @@
 #
 #sender_workers = 0
 
+# Enables listener sockets; can be set to false to disable listening. This
+# option is intended for developer/diagnostic purposes only.
+#
+#listening = true
+
+# Enables configuration reload when the server receives SIGUSR1 on
+# supporting platforms.
+#
+#config_reload_signal = true
+
 [global.tls]
 
 # Path to a valid TLS certificate file.

debian/postrm

@@ -10,21 +10,33 @@ CONDUWUIT_DATABASE_PATH_SYMLINK=/var/lib/matrix-conduit
 case $1 in
   purge)
     # Remove debconf changes from the db
-    db_purge
+    #db_purge
 
     # Per https://www.debian.org/doc/debian-policy/ch-files.html#behavior
     # "configuration files must be preserved when the package is removed, and
     #  only deleted when the package is purged."
+
+    #
+
     if [ -d "$CONDUWUIT_CONFIG_PATH" ]; then
-      rm -v -r "$CONDUWUIT_CONFIG_PATH"
+      if test -L "$CONDUWUIT_CONFIG_PATH"; then
+        echo "Deleting conduwuit configuration files"
+        rm -v -r "$CONDUWUIT_CONFIG_PATH"
+      fi
     fi
 
     if [ -d "$CONDUWUIT_DATABASE_PATH" ]; then
-      rm -v -r "$CONDUWUIT_DATABASE_PATH"
+      if test -L "$CONDUWUIT_DATABASE_PATH"; then
+        echo "Deleting conduwuit database directory"
+        rm -r "$CONDUWUIT_DATABASE_PATH"
+      fi
     fi
 
     if [ -d "$CONDUWUIT_DATABASE_PATH_SYMLINK" ]; then
-      rm -v -r "$CONDUWUIT_DATABASE_PATH_SYMLINK"
+      if test -L "$CONDUWUIT_DATABASE_SYMLINK"; then
+        echo "Removing matrix-conduit symlink"
+        rm -r "$CONDUWUIT_DATABASE_PATH_SYMLINK"
+      fi
     fi
     ;;
 esac

deps/rust-rocksdb/Cargo.toml

@@ -27,7 +27,7 @@ malloc-usable-size = ["rust-rocksdb/malloc-usable-size"]
 
 [dependencies.rust-rocksdb]
 git = "https://github.com/girlbossceo/rust-rocksdb-zaidoon1"
-rev = "123d6302fed23fc706344becb2f19623265a83f8"
+rev = "1f032427d3a0e7b0f13c04b4e34712bd8610291b"
 #branch = "master"
 default-features = false
 

docs/deploying/docker-compose.for-traefik.yml

@@ -17,6 +17,8 @@ services:
       CONDUWUIT_PORT: 6167 # should match the loadbalancer traefik label
       CONDUWUIT_MAX_REQUEST_SIZE: 20000000 # in bytes, ~20 MB
       CONDUWUIT_ALLOW_REGISTRATION: 'true'
+      CONDUWUIT_REGISTRATION_TOKEN: 'YOUR_TOKEN' # A registration token is required when registration is allowed.
+      #CONDUWUIT_YES_I_AM_VERY_VERY_SURE_I_WANT_AN_OPEN_REGISTRATION_SERVER_PRONE_TO_ABUSE: 'true'
       CONDUWUIT_ALLOW_FEDERATION: 'true'
       CONDUWUIT_ALLOW_CHECK_FOR_UPDATES: 'true'
       CONDUWUIT_TRUSTED_SERVERS: '["matrix.org"]'

docs/deploying/docker-compose.with-caddy.yml

@@ -33,6 +33,8 @@ services:
             CONDUWUIT_PORT: 6167
             CONDUWUIT_MAX_REQUEST_SIZE: 20000000 # in bytes, ~20 MB
             CONDUWUIT_ALLOW_REGISTRATION: 'true'
+            CONDUWUIT_REGISTRATION_TOKEN: 'YOUR_TOKEN' # A registration token is required when registration is allowed.
+            #CONDUWUIT_YES_I_AM_VERY_VERY_SURE_I_WANT_AN_OPEN_REGISTRATION_SERVER_PRONE_TO_ABUSE: 'true'
             CONDUWUIT_ALLOW_FEDERATION: 'true'
             CONDUWUIT_ALLOW_CHECK_FOR_UPDATES: 'true'
             CONDUWUIT_TRUSTED_SERVERS: '["matrix.org"]'

docs/deploying/docker-compose.yml

@@ -17,6 +17,8 @@ services:
             CONDUWUIT_PORT: 6167
             CONDUWUIT_MAX_REQUEST_SIZE: 20000000 # in bytes, ~20 MB
             CONDUWUIT_ALLOW_REGISTRATION: 'true'
+            CONDUWUIT_REGISTRATION_TOKEN: 'YOUR_TOKEN' # A registration token is required when registration is allowed.
+            #CONDUWUIT_YES_I_AM_VERY_VERY_SURE_I_WANT_AN_OPEN_REGISTRATION_SERVER_PRONE_TO_ABUSE: 'true'
             CONDUWUIT_ALLOW_FEDERATION: 'true'
             CONDUWUIT_ALLOW_CHECK_FOR_UPDATES: 'true'
             CONDUWUIT_TRUSTED_SERVERS: '["matrix.org"]'

engage.toml

@@ -97,6 +97,7 @@ env DIRENV_DEVSHELL=all-features \
 name = "clippy/default"
 group = "lints"
 script = """
+direnv exec . \
 cargo clippy \
     --workspace \
     --profile test \
@@ -126,6 +127,7 @@ env DIRENV_DEVSHELL=all-features \
 name = "clippy/jemalloc"
 group = "lints"
 script = """
+direnv exec . \
 cargo clippy \
     --workspace \
     --profile test \
@@ -179,13 +181,15 @@ env DIRENV_DEVSHELL=all-features \
 name = "cargo/default"
 group = "tests"
 script = """
-cargo test \
-    --workspace \
-    --profile test \
-    --all-targets \
-    --color=always \
-    -- \
-    --color=always
+env DIRENV_DEVSHELL=default \
+    direnv exec . \
+    cargo test \
+        --workspace \
+        --profile test \
+        --all-targets \
+        --color=always \
+        -- \
+        --color=always
 """
 
 # Checks if the generated example config differs from the checked in repo's

flake.lock

@@ -32,11 +32,11 @@
         "nixpkgs": "nixpkgs_4"
       },
       "locked": {
-        "lastModified": 1733424942,
-        "narHash": "sha256-5t7Sl6EkOaoP4FvzLmH7HFDbdl9SizmLh53RjDQCbWQ=",
+        "lastModified": 1737621947,
+        "narHash": "sha256-8HFvG7fvIFbgtaYAY2628Tb89fA55nPm2jSiNs0/Cws=",
         "owner": "cachix",
         "repo": "cachix",
-        "rev": "8b6b0e4694b9aa78b2ea4c93bff6e1a222dc7e4a",
+        "rev": "f65a3cd5e339c223471e64c051434616e18cc4f5",
         "type": "github"
       },
       "original": {
@@ -117,11 +117,11 @@
     },
     "crane_2": {
       "locked": {
-        "lastModified": 1734808813,
-        "narHash": "sha256-3aH/0Y6ajIlfy7j52FGZ+s4icVX0oHhqBzRdlOeztqg=",
+        "lastModified": 1737689766,
+        "narHash": "sha256-ivVXYaYlShxYoKfSo5+y5930qMKKJ8CLcAoIBPQfJ6s=",
         "owner": "ipetkov",
         "repo": "crane",
-        "rev": "72e2d02dbac80c8c86bf6bf3e785536acf8ee926",
+        "rev": "6fe74265bbb6d016d663b1091f015e2976c4a527",
         "type": "github"
       },
       "original": {
@@ -170,11 +170,11 @@
         "rust-analyzer-src": "rust-analyzer-src"
       },
       "locked": {
-        "lastModified": 1735799625,
-        "narHash": "sha256-lFadwWDvVIub11bwfZhsh2WUByf9LOi6yjsSUMmE0xk=",
+        "lastModified": 1737786656,
+        "narHash": "sha256-ubCW9Jy7ZUOF354bWxTgLDpVnTvIpNr6qR4H/j7I0oo=",
         "owner": "nix-community",
         "repo": "fenix",
-        "rev": "a9d84a1545814910cb4ab0515ed6921e8b07ee95",
+        "rev": "2f721f527886f801403f389a9cabafda8f1e3b7f",
         "type": "github"
       },
       "original": {
@@ -364,11 +364,11 @@
     "liburing": {
       "flake": false,
       "locked": {
-        "lastModified": 1733603756,
-        "narHash": "sha256-eTKnZDZ1Ex++v+BI0DBcUBmCXAO/tE8hxK9MiyztZkU=",
+        "lastModified": 1737600516,
+        "narHash": "sha256-EKyLQ3pbcjoU5jH5atge59F4fzuhTsb6yalUj6Ve2t8=",
         "owner": "axboe",
         "repo": "liburing",
-        "rev": "c3d5d6270cd5ed48d817fc1e8e95f7c8b222f2ff",
+        "rev": "6c509e2b0c881a13b83b259a221bf15fc9b3f681",
         "type": "github"
       },
       "original": {
@@ -550,11 +550,11 @@
     },
     "nixpkgs_5": {
       "locked": {
-        "lastModified": 1735685343,
-        "narHash": "sha256-h1CpBzdJDNtSUb5QMyfFHKHocTTky+4McgQEBQBM+xA=",
+        "lastModified": 1737717945,
+        "narHash": "sha256-ET91TMkab3PmOZnqiJQYOtSGvSTvGeHoegAv4zcTefM=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "81934660d6e9ea54d2f0cdee821e8533b10c221a",
+        "rev": "ecd26a469ac56357fd333946a99086e992452b6a",
         "type": "github"
       },
       "original": {
@@ -567,11 +567,11 @@
     "rocksdb": {
       "flake": false,
       "locked": {
-        "lastModified": 1734469478,
-        "narHash": "sha256-IcQ4N8xADYal79K+ONmNq4RLlIwdgUqgrVzgNgiIaG8=",
+        "lastModified": 1737828695,
+        "narHash": "sha256-8Ev6zzhNPU798JNvU27a7gj5X+6SDG3jBweUkQ59DbA=",
         "owner": "girlbossceo",
         "repo": "rocksdb",
-        "rev": "8b4808e7de2fbb5d119d8d72cdca76d8ab84bc47",
+        "rev": "a4d9230dcc9d03be428b9a728133f8f646c0065c",
         "type": "github"
       },
       "original": {
@@ -599,11 +599,11 @@
     "rust-analyzer-src": {
       "flake": false,
       "locked": {
-        "lastModified": 1735742096,
-        "narHash": "sha256-q3a80h8Jf8wfmPURUgRR46nQCB3I5fhZ+/swulTF5HY=",
+        "lastModified": 1737728869,
+        "narHash": "sha256-U4pl3Hi0lT6GP4ecN3q9wdD2sdaKMbmD/5NJ1NdJ9AM=",
         "owner": "rust-lang",
         "repo": "rust-analyzer",
-        "rev": "7e639ee3dda6ed9cecc79d41f6d38235121e483d",
+        "rev": "6e4c29f7ce18cea7d3d31237a4661ab932eab636",
         "type": "github"
       },
       "original": {

flake.nix

@@ -26,7 +26,7 @@
         file = ./rust-toolchain.toml;
 
         # See also `rust-toolchain.toml`
-        sha256 = "sha256-s1RPtyvDGJaX/BisLT+ifVfuhDT1nZkZ1NcK8sbwELM=";
+        sha256 = "sha256-lMLAupxng4Fd9F1oDw8gx+qA0RuF7ou7xhNU8wgs0PU=";
       };
 
       mkScope = pkgs: pkgs.lib.makeScope pkgs.newScope (self: {
@@ -169,10 +169,10 @@
 
           # used for rust caching in CI to speed it up
           sccache
-
-          # needed so we can get rid of gcc and other unused deps that bloat OCI images
-          removeReferencesTo
         ]
+        # valgrind is unavailable in static contexts
+        # used for CI and complement
+        ++ (if !stdenv.hostPlatform.isStatic then [ "valgrind" ] else [])
         # liburing is Linux-exclusive
         ++ lib.optional stdenv.hostPlatform.isLinux liburing
         # needed to build Rust applications on macOS
@@ -191,27 +191,59 @@
     in
     {
       packages = {
-        default = scopeHost.main;
-        default-debug = scopeHost.main.override {
-            profile = "dev";
-            # debug build users expect full logs
-            disable_release_max_log_level = true;
-        };
-        default-test = scopeHost.main.override {
-            profile = "test";
-            disable_release_max_log_level = true;
-        };
-        all-features = scopeHost.main.override {
-            all_features = true;
+        default = scopeHost.main.override {
             disable_features = [
-                # this is non-functional on nix for some reason
-                "hardened_malloc"
                 # dont include experimental features
                 "experimental"
                 # jemalloc profiling/stats features are expensive and shouldn't
                 # be expected on non-debug builds.
                 "jemalloc_prof"
                 "jemalloc_stats"
+                # this is non-functional on nix for some reason
+                "hardened_malloc"
+                # conduwuit_mods is a development-only hot reload feature
+                "conduwuit_mods"
+            ];
+        };
+        default-debug = scopeHost.main.override {
+            profile = "dev";
+            # debug build users expect full logs
+            disable_release_max_log_level = true;
+            disable_features = [
+                # dont include experimental features
+                "experimental"
+                # this is non-functional on nix for some reason
+                "hardened_malloc"
+                # conduwuit_mods is a development-only hot reload feature
+                "conduwuit_mods"
+            ];
+        };
+        # just a test profile used for things like CI and complement
+        default-test = scopeHost.main.override {
+            profile = "test";
+            disable_release_max_log_level = true;
+            disable_features = [
+                # dont include experimental features
+                "experimental"
+                # this is non-functional on nix for some reason
+                "hardened_malloc"
+                # conduwuit_mods is a development-only hot reload feature
+                "conduwuit_mods"
+            ];
+        };
+        all-features = scopeHost.main.override {
+            all_features = true;
+            disable_features = [
+                # dont include experimental features
+                "experimental"
+                # jemalloc profiling/stats features are expensive and shouldn't
+                # be expected on non-debug builds.
+                "jemalloc_prof"
+                "jemalloc_stats"
+                # this is non-functional on nix for some reason
+                "hardened_malloc"
+                # conduwuit_mods is a development-only hot reload feature
+                "conduwuit_mods"
             ];
         };
         all-features-debug = scopeHost.main.override {
@@ -220,10 +252,12 @@
             # debug build users expect full logs
             disable_release_max_log_level = true;
             disable_features = [
-                # this is non-functional on nix for some reason
-                "hardened_malloc"
                 # dont include experimental features
                 "experimental"
+                # this is non-functional on nix for some reason
+                "hardened_malloc"
+                # conduwuit_mods is a development-only hot reload feature
+                "conduwuit_mods"
             ];
         };
         hmalloc = scopeHost.main.override { features = ["hardened_malloc"]; };
@@ -233,14 +267,16 @@
           main = scopeHost.main.override {
             all_features = true;
             disable_features = [
-                # this is non-functional on nix for some reason
-                "hardened_malloc"
                 # dont include experimental features
                 "experimental"
                 # jemalloc profiling/stats features are expensive and shouldn't
                 # be expected on non-debug builds.
                 "jemalloc_prof"
                 "jemalloc_stats"
+                # this is non-functional on nix for some reason
+                "hardened_malloc"
+                # conduwuit_mods is a development-only hot reload feature
+                "conduwuit_mods"
             ];
           };
         };
@@ -251,10 +287,12 @@
             # debug build users expect full logs
             disable_release_max_log_level = true;
             disable_features = [
-                # this is non-functional on nix for some reason
-                "hardened_malloc"
                 # dont include experimental features
                 "experimental"
+                # this is non-functional on nix for some reason
+                "hardened_malloc"
+                # conduwuit_mods is a development-only hot reload feature
+                "conduwuit_mods"
             ];
           };
         };
@@ -313,6 +351,14 @@
                   value = scopeCrossStatic.main.override {
                     profile = "test";
                     disable_release_max_log_level = true;
+                    disable_features = [
+                        # dont include experimental features
+                        "experimental"
+                        # this is non-functional on nix for some reason
+                        "hardened_malloc"
+                        # conduwuit_mods is a development-only hot reload feature
+                        "conduwuit_mods"
+                    ];
                   };
                 }
 
@@ -322,14 +368,16 @@
                   value = scopeCrossStatic.main.override {
                     all_features = true;
                     disable_features = [
-                        # this is non-functional on nix for some reason
-                        "hardened_malloc"
                         # dont include experimental features
                         "experimental"
                         # jemalloc profiling/stats features are expensive and shouldn't
                         # be expected on non-debug builds.
                         "jemalloc_prof"
                         "jemalloc_stats"
+                        # this is non-functional on nix for some reason
+                        "hardened_malloc"
+                        # conduwuit_mods is a development-only hot reload feature
+                        "conduwuit_mods"
                     ];
                   };
                 }
@@ -341,14 +389,16 @@
                   value = scopeCrossStatic.main.override {
                     all_features = true;
                     disable_features = [
-                        # this is non-functional on nix for some reason
-                        "hardened_malloc"
                         # dont include experimental features
                         "experimental"
                         # jemalloc profiling/stats features are expensive and shouldn't
                         # be expected on non-debug builds.
                         "jemalloc_prof"
                         "jemalloc_stats"
+                        # this is non-functional on nix for some reason
+                        "hardened_malloc"
+                        # conduwuit_mods is a development-only hot reload feature
+                        "conduwuit_mods"
                     ];
                     x86_64_haswell_target_optimised = (if (crossSystem == "x86_64-linux-gnu" || crossSystem == "x86_64-linux-musl") then true else false);
                   };
@@ -363,10 +413,12 @@
                     # debug build users expect full logs
                     disable_release_max_log_level = true;
                     disable_features = [
-                        # this is non-functional on nix for some reason
-                        "hardened_malloc"
                         # dont include experimental features
                         "experimental"
+                        # this is non-functional on nix for some reason
+                        "hardened_malloc"
+                        # conduwuit_mods is a development-only hot reload feature
+                        "conduwuit_mods"
                     ];
                   };
                 }
@@ -415,14 +467,16 @@
                     main = scopeCrossStatic.main.override {
                       all_features = true;
                       disable_features = [
-                          # this is non-functional on nix for some reason
-                          "hardened_malloc"
-                          # dont include experimental features
-                          "experimental"
-                          # jemalloc profiling/stats features are expensive and shouldn't
-                          # be expected on non-debug builds.
-                          "jemalloc_prof"
-                          "jemalloc_stats"
+                        # dont include experimental features
+                        "experimental"
+                        # jemalloc profiling/stats features are expensive and shouldn't
+                        # be expected on non-debug builds.
+                        "jemalloc_prof"
+                        "jemalloc_stats"
+                        # this is non-functional on nix for some reason
+                        "hardened_malloc"
+                        # conduwuit_mods is a development-only hot reload feature
+                        "conduwuit_mods"
                       ];
                     };
                   };
@@ -436,14 +490,16 @@
                     main = scopeCrossStatic.main.override {
                       all_features = true;
                       disable_features = [
-                          # this is non-functional on nix for some reason
-                          "hardened_malloc"
-                          # dont include experimental features
-                          "experimental"
-                          # jemalloc profiling/stats features are expensive and shouldn't
-                          # be expected on non-debug builds.
-                          "jemalloc_prof"
-                          "jemalloc_stats"
+                        # dont include experimental features
+                        "experimental"
+                        # jemalloc profiling/stats features are expensive and shouldn't
+                        # be expected on non-debug builds.
+                        "jemalloc_prof"
+                        "jemalloc_stats"
+                        # this is non-functional on nix for some reason
+                        "hardened_malloc"
+                        # conduwuit_mods is a development-only hot reload feature
+                        "conduwuit_mods"
                       ];
                       x86_64_haswell_target_optimised = (if (crossSystem == "x86_64-linux-gnu" || crossSystem == "x86_64-linux-musl") then true else false);
                     };
@@ -460,10 +516,12 @@
                       # debug build users expect full logs
                       disable_release_max_log_level = true;
                       disable_features = [
-                          # this is non-functional on nix for some reason
-                          "hardened_malloc"
-                          # dont include experimental features
-                          "experimental"
+                        # dont include experimental features
+                        "experimental"
+                        # this is non-functional on nix for some reason
+                        "hardened_malloc"
+                        # conduwuit_mods is a development-only hot reload feature
+                        "conduwuit_mods"
                       ];
                     };
                   };
@@ -502,14 +560,16 @@
           main = prev.main.override {
             all_features = true;
             disable_features = [
-                # this is non-functional on nix for some reason
-                "hardened_malloc"
                 # dont include experimental features
                 "experimental"
                 # jemalloc profiling/stats features are expensive and shouldn't
                 # be expected on non-debug builds.
                 "jemalloc_prof"
                 "jemalloc_stats"
+                # this is non-functional on nix for some reason
+                "hardened_malloc"
+                # conduwuit_mods is a development-only hot reload feature
+                "conduwuit_mods"
             ];
         };
         }));

nix/pkgs/complement/config.toml

@@ -17,19 +17,30 @@ ip_range_denylist = []
 url_preview_domain_contains_allowlist = ["*"]
 url_preview_domain_explicit_denylist = ["*"]
 media_compat_file_link = false
-media_startup_check = false
-prune_missing_media = false
+media_startup_check = true
+prune_missing_media = true
 log_colors = false
 admin_room_notices = false
 allow_check_for_updates = false
-allow_unstable_room_versions = true
 rocksdb_log_level = "debug"
 rocksdb_max_log_files = 1
 rocksdb_recovery_mode = 0
 rocksdb_paranoid_file_checks = true
 log_guest_registrations = false
 allow_legacy_media = true
-startup_netburst = false
+startup_netburst = true
+
+# valgrind makes things so slow
+dns_timeout = 60
+dns_attempts = 20
+request_conn_timeout = 60
+request_timeout = 120
+well_known_conn_timeout = 60
+well_known_timeout = 60
+federation_idle_timeout = 300
+sender_timeout = 300
+sender_idle_timeout = 300
+sender_retry_backoff_limit = 300
 
 [global.tls]
 certs = "/certificate.crt"

nix/pkgs/complement/default.nix

@@ -9,17 +9,22 @@
 , openssl
 , stdenv
 , tini
+, valgrind
 , writeShellScriptBin
 }:
 
 let
   main' = main.override {
     profile = "test";
+    #profile = "release-debuginfo";
     all_features = true;
     disable_release_max_log_level = true;
     disable_features = [
-        # no reason to use jemalloc for complement, just has compatibility/build issues
         "jemalloc"
+        "jemalloc_stats"
+        "jemalloc_prof"
+        "jemalloc_conf"
+        "io_uring"
         # console/CLI stuff isn't used or relevant for complement
         "console"
         "tokio_console"
@@ -27,13 +32,28 @@ let
         "sentry_telemetry"
         "perf_measurements"
         # the containers don't use or need systemd signal support
-        "systemd"
+        #"systemd"
         # this is non-functional on nix for some reason
         "hardened_malloc"
         # dont include experimental features
         "experimental"
+        # compression isn't needed for complement
+        "brotli_compression"
+        "gzip_compression"
+        "zstd_compression"
+        # complement doesn't need hot reloading
+        "conduwuit_mods"
+        # complement doesn't have URL preview media tests
+        "url_preview"
     ];
   };
+        # TODO: figure out why a suspicious amounnt of complement tests fail with valgrind only under complement.
+        # maybe issue with direct TLS mode?
+        #${lib.getExe' valgrind "valgrind"} \
+        #--leak-check=no \
+        #--undef-value-errors=no \
+        #--exit-on-first-error=yes \
+        #--error-exitcode=1 \
 
   start = writeShellScriptBin "start" ''
     set -euxo pipefail

nix/pkgs/main/default.nix

@@ -15,7 +15,19 @@
 # Options (keep sorted)
 , all_features ? false
 , default_features ? true
-, disable_features ? []
+# default list of disabled features
+, disable_features ? [
+  # dont include experimental features
+  "experimental"
+  # jemalloc profiling/stats features are expensive and shouldn't
+  # be expected on non-debug builds.
+  "jemalloc_prof"
+  "jemalloc_stats"
+  # this is non-functional on nix for some reason
+  "hardened_malloc"
+  # conduwuit_mods is a development-only hot reload feature
+  "conduwuit_mods"
+]
 , disable_release_max_log_level ? false
 , features ? []
 , profile ? "release"
@@ -70,7 +82,7 @@ rust-jemalloc-sys' = (rust-jemalloc-sys.override {
 buildDepsOnlyEnv =
   let
     rocksdb' = (rocksdb.override {
-      jemalloc = rust-jemalloc-sys';
+      jemalloc = lib.optional (featureEnabled "jemalloc") rust-jemalloc-sys';
       # rocksdb fails to build with prefixed jemalloc, which is required on
       # darwin due to [1]. In this case, fall back to building rocksdb with
       # libc malloc. This should not cause conflicts, because all of the
@@ -91,6 +103,11 @@ buildDepsOnlyEnv =
       ++ [ "-DPORTABLE=haswell" ]) else ([ "-DPORTABLE=1" ])
       )
       ++ old.cmakeFlags;
+      # outputs has "tools" which we dont need or use
+      outputs = [ "out" ];
+
+      # preInstall hooks has stuff for messing with ldb/sst_dump which we dont need or use
+      preInstall = "";
     });
   in
   {
@@ -144,6 +161,19 @@ commonAttrs = {
       ];
     };
 
+    # This is redundant with CI
+    doCheck = false;
+
+    cargoTestCommand = "cargo test --locked ";
+    cargoExtraArgs = "--no-default-features --locked "
+      + lib.optionalString
+        (features'' != [])
+        "--features " + (builtins.concatStringsSep "," features'');
+    cargoTestExtraArgs = "--no-default-features --locked "
+      + lib.optionalString
+        (features'' != [])
+        "--features " + (builtins.concatStringsSep "," features'');
+
     dontStrip = profile == "dev" || profile == "test";
     dontPatchELF = profile == "dev" || profile == "test";
 
@@ -169,9 +199,6 @@ commonAttrs = {
       # differing values for `NIX_CFLAGS_COMPILE`, which contributes to spurious
       # rebuilds of bindgen and its depedents.
       jq
-
-      # needed so we can get rid of gcc and other unused deps that bloat OCI images
-      removeReferencesTo
   ]
   # needed to build Rust applications on macOS
   ++ lib.optionals stdenv.hostPlatform.isDarwin [
@@ -183,13 +210,6 @@ commonAttrs = {
       # https://discourse.nixos.org/t/compile-a-rust-binary-on-macos-dbcrossbar/8612
       pkgsBuildHost.darwin.apple_sdk.frameworks.Security
     ];
-
-    # for some reason gcc and other weird deps are added to OCI images and bloats it up
-    #
-    # <https://github.com/input-output-hk/haskell.nix/issues/829>
-    postInstall = with pkgsBuildHost; ''
-        find "$out" -type f -exec remove-references-to -t ${stdenv.cc} -t ${gcc} -t ${llvm} -t ${rustc.unwrapped} -t ${rustc} '{}' +
-    '';
  };
 in
 
@@ -198,15 +218,18 @@ craneLib.buildPackage ( commonAttrs // {
     env = buildDepsOnlyEnv;
   });
 
-  cargoExtraArgs = "--no-default-features "
+  # This is redundant with CI
+  doCheck = false;
+
+  cargoTestCommand = "cargo test --locked ";
+  cargoExtraArgs = "--no-default-features --locked "
+    + lib.optionalString
+      (features'' != [])
+      "--features " + (builtins.concatStringsSep "," features'');
+  cargoTestExtraArgs = "--no-default-features --locked "
     + lib.optionalString
       (features'' != [])
       "--features " + (builtins.concatStringsSep "," features'');
-
-  # This is redundant with CI
-  cargoTestCommand = "";
-  cargoCheckCommand = "";
-  doCheck = false;
 
   env = buildPackageEnv;
 

nix/pkgs/oci-image/default.nix

@@ -28,5 +28,18 @@ dockerTools.buildLayeredImage {
     Env = [
       "RUST_BACKTRACE=full"
     ];
+    Labels = {
+      "org.opencontainers.image.authors" = "June Clementine Strawberry <june@girlboss.ceo> and Jason Volk
+      <jason@zemos.net>";
+      "org.opencontainers.image.created" ="@${toString inputs.self.lastModified}";
+      "org.opencontainers.image.description" = "a very cool Matrix chat homeserver written in Rust";
+      "org.opencontainers.image.documentation" = "https://conduwuit.puppyirl.gay/";
+      "org.opencontainers.image.licenses" = "Apache-2.0";
+      "org.opencontainers.image.revision" = inputs.self.rev or inputs.self.dirtyRev or "";
+      "org.opencontainers.image.title" = main.pname;
+      "org.opencontainers.image.url" = "https://conduwuit.puppyirl.gay/";
+      "org.opencontainers.image.vendor" = "girlbossceo";
+      "org.opencontainers.image.version" = main.version;
+    };
   };
 }

rust-toolchain.toml

@@ -9,7 +9,7 @@
 # If you're having trouble making the relevant changes, bug a maintainer.
 
 [toolchain]
-channel = "1.83.0"
+channel = "1.84.0"
 profile = "minimal"
 components = [
     # For rust-analyzer

src/admin/admin.rs

@@ -1,6 +1,5 @@
 use clap::Parser;
 use conduwuit::Result;
-use ruma::events::room::message::RoomMessageEventContent;
 
 use crate::{
 	appservice, appservice::AppserviceCommand, check, check::CheckCommand, command::Command,
@@ -50,13 +49,10 @@ pub(super) enum AdminCommand {
 }
 
 #[tracing::instrument(skip_all, name = "command")]
-pub(super) async fn process(
-	command: AdminCommand,
-	context: &Command<'_>,
-) -> Result<RoomMessageEventContent> {
+pub(super) async fn process(command: AdminCommand, context: &Command<'_>) -> Result {
 	use AdminCommand::*;
 
-	Ok(match command {
+	match command {
 		| Appservices(command) => appservice::process(command, context).await?,
 		| Media(command) => media::process(command, context).await?,
 		| Users(command) => user::process(command, context).await?,
@@ -66,5 +62,7 @@ pub(super) async fn process(
 		| Debug(command) => debug::process(command, context).await?,
 		| Query(command) => query::process(command, context).await?,
 		| Check(command) => check::process(command, context).await?,
-	})
+	};
+
+	Ok(())
 }

src/admin/check/mod.rs

@@ -2,20 +2,11 @@ mod commands;
 
 use clap::Subcommand;
 use conduwuit::Result;
-use ruma::events::room::message::RoomMessageEventContent;
 
-use crate::Command;
+use crate::admin_command_dispatch;
 
+#[admin_command_dispatch]
 #[derive(Debug, Subcommand)]
 pub(super) enum CheckCommand {
-	AllUsers,
-}
-
-pub(super) async fn process(
-	command: CheckCommand,
-	context: &Command<'_>,
-) -> Result<RoomMessageEventContent> {
-	Ok(match command {
-		| CheckCommand::AllUsers => context.check_all_users().await?,
-	})
+	CheckAllUsers,
 }

src/admin/command.rs

@@ -1,6 +1,12 @@
-use std::time::SystemTime;
+use std::{fmt, time::SystemTime};
 
+use conduwuit::Result;
 use conduwuit_service::Services;
+use futures::{
+	io::{AsyncWriteExt, BufWriter},
+	lock::Mutex,
+	Future, FutureExt,
+};
 use ruma::EventId;
 
 pub(crate) struct Command<'a> {
@@ -8,4 +14,26 @@ pub(crate) struct Command<'a> {
 	pub(crate) body: &'a [&'a str],
 	pub(crate) timer: SystemTime,
 	pub(crate) reply_id: Option<&'a EventId>,
+	pub(crate) output: Mutex<BufWriter<Vec<u8>>>,
+}
+
+impl Command<'_> {
+	pub(crate) fn write_fmt(
+		&self,
+		arguments: fmt::Arguments<'_>,
+	) -> impl Future<Output = Result> + Send + '_ {
+		let buf = format!("{arguments}");
+		self.output.lock().then(|mut output| async move {
+			output.write_all(buf.as_bytes()).await.map_err(Into::into)
+		})
+	}
+
+	pub(crate) fn write_str<'a>(
+		&'a self,
+		s: &'a str,
+	) -> impl Future<Output = Result> + Send + 'a {
+		self.output.lock().then(move |mut output| async move {
+			output.write_all(s.as_bytes()).await.map_err(Into::into)
+		})
+	}
 }

src/admin/debug/commands.rs

@@ -6,16 +6,20 @@ use std::{
 };
 
 use conduwuit::{
-	debug_error, err, info, trace, utils, utils::string::EMPTY, warn, Error, PduEvent, Result,
+	debug_error, err, info, trace, utils, utils::string::EMPTY, warn, Error, PduEvent, PduId,
+	RawPduId, Result,
 };
-use futures::{FutureExt, StreamExt};
+use futures::{FutureExt, StreamExt, TryStreamExt};
 use ruma::{
 	api::{client::error::ErrorKind, federation::event::get_room_state},
 	events::room::message::RoomMessageEventContent,
 	CanonicalJsonObject, EventId, OwnedEventId, OwnedRoomOrAliasId, RoomId, RoomVersionId,
 	ServerName,
 };
-use service::rooms::state_compressor::HashSetCompressStateEvent;
+use service::rooms::{
+	short::{ShortEventId, ShortRoomId},
+	state_compressor::HashSetCompressStateEvent,
+};
 use tracing_subscriber::EnvFilter;
 
 use crate::admin_command;
@@ -131,13 +135,42 @@ pub(super) async fn get_pdu(&self, event_id: Box<EventId>) -> Result<RoomMessage
 	}
 }
 
+#[admin_command]
+pub(super) async fn get_short_pdu(
+	&self,
+	shortroomid: ShortRoomId,
+	shorteventid: ShortEventId,
+) -> Result<RoomMessageEventContent> {
+	let pdu_id: RawPduId = PduId {
+		shortroomid,
+		shorteventid: shorteventid.into(),
+	}
+	.into();
+
+	let pdu_json = self
+		.services
+		.rooms
+		.timeline
+		.get_pdu_json_from_id(&pdu_id)
+		.await;
+
+	match pdu_json {
+		| Ok(json) => {
+			let json_text =
+				serde_json::to_string_pretty(&json).expect("canonical json is valid json");
+			Ok(RoomMessageEventContent::notice_markdown(format!("```json\n{json_text}\n```",)))
+		},
+		| Err(_) => Ok(RoomMessageEventContent::text_plain("PDU not found locally.")),
+	}
+}
+
 #[admin_command]
 pub(super) async fn get_remote_pdu_list(
 	&self,
 	server: Box<ServerName>,
 	force: bool,
 ) -> Result<RoomMessageEventContent> {
-	if !self.services.globals.config.allow_federation {
+	if !self.services.server.config.allow_federation {
 		return Ok(RoomMessageEventContent::text_plain(
 			"Federation is disabled on this homeserver.",
 		));
@@ -202,7 +235,7 @@ pub(super) async fn get_remote_pdu(
 	event_id: Box<EventId>,
 	server: Box<ServerName>,
 ) -> Result<RoomMessageEventContent> {
-	if !self.services.globals.config.allow_federation {
+	if !self.services.server.config.allow_federation {
 		return Ok(RoomMessageEventContent::text_plain(
 			"Federation is disabled on this homeserver.",
 		));
@@ -294,11 +327,10 @@ pub(super) async fn get_room_state(
 		.services
 		.rooms
 		.state_accessor
-		.room_state_full(&room_id)
-		.await?
-		.values()
-		.map(PduEvent::to_state_event)
-		.collect();
+		.room_state_full_pdus(&room_id)
+		.map_ok(PduEvent::into_state_event)
+		.try_collect()
+		.await?;
 
 	if room_state.is_empty() {
 		return Ok(RoomMessageEventContent::text_plain(
@@ -386,7 +418,7 @@ pub(super) async fn change_log_level(
 	let handles = &["console"];
 
 	if reset {
-		let old_filter_layer = match EnvFilter::try_new(&self.services.globals.config.log) {
+		let old_filter_layer = match EnvFilter::try_new(&self.services.server.config.log) {
 			| Ok(s) => s,
 			| Err(e) => {
 				return Ok(RoomMessageEventContent::text_plain(format!(
@@ -405,7 +437,7 @@ pub(super) async fn change_log_level(
 			| Ok(()) => {
 				return Ok(RoomMessageEventContent::text_plain(format!(
 					"Successfully changed log level back to config value {}",
-					self.services.globals.config.log
+					self.services.server.config.log
 				)));
 			},
 			| Err(e) => {
@@ -521,7 +553,7 @@ pub(super) async fn first_pdu_in_room(
 		.services
 		.rooms
 		.state_cache
-		.server_in_room(&self.services.globals.config.server_name, &room_id)
+		.server_in_room(&self.services.server.name, &room_id)
 		.await
 	{
 		return Ok(RoomMessageEventContent::text_plain(
@@ -550,7 +582,7 @@ pub(super) async fn latest_pdu_in_room(
 		.services
 		.rooms
 		.state_cache
-		.server_in_room(&self.services.globals.config.server_name, &room_id)
+		.server_in_room(&self.services.server.name, &room_id)
 		.await
 	{
 		return Ok(RoomMessageEventContent::text_plain(
@@ -580,7 +612,7 @@ pub(super) async fn force_set_room_state_from_server(
 		.services
 		.rooms
 		.state_cache
-		.server_in_room(&self.services.globals.config.server_name, &room_id)
+		.server_in_room(&self.services.server.name, &room_id)
 		.await
 	{
 		return Ok(RoomMessageEventContent::text_plain(
@@ -692,7 +724,7 @@ pub(super) async fn force_set_room_state_from_server(
 		.save_state(room_id.clone().as_ref(), new_room_state)
 		.await?;
 
-	let state_lock = self.services.rooms.state.mutex.lock(&room_id).await;
+	let state_lock = self.services.rooms.state.mutex.lock(&*room_id).await;
 	self.services
 		.rooms
 		.state
@@ -723,8 +755,7 @@ pub(super) async fn get_signing_keys(
 	notary: Option<Box<ServerName>>,
 	query: bool,
 ) -> Result<RoomMessageEventContent> {
-	let server_name =
-		server_name.unwrap_or_else(|| self.services.server.config.server_name.clone().into());
+	let server_name = server_name.unwrap_or_else(|| self.services.server.name.clone().into());
 
 	if let Some(notary) = notary {
 		let signing_keys = self
@@ -760,8 +791,7 @@ pub(super) async fn get_verify_keys(
 	&self,
 	server_name: Option<Box<ServerName>>,
 ) -> Result<RoomMessageEventContent> {
-	let server_name =
-		server_name.unwrap_or_else(|| self.services.server.config.server_name.clone().into());
+	let server_name = server_name.unwrap_or_else(|| self.services.server.name.clone().into());
 
 	let keys = self
 		.services
@@ -785,13 +815,13 @@ pub(super) async fn resolve_true_destination(
 	server_name: Box<ServerName>,
 	no_cache: bool,
 ) -> Result<RoomMessageEventContent> {
-	if !self.services.globals.config.allow_federation {
+	if !self.services.server.config.allow_federation {
 		return Ok(RoomMessageEventContent::text_plain(
 			"Federation is disabled on this homeserver.",
 		));
 	}
 
-	if server_name == self.services.globals.config.server_name {
+	if server_name == self.services.server.name {
 		return Ok(RoomMessageEventContent::text_plain(
 			"Not allowed to send federation requests to ourselves. Please use `get-pdu` for \
 			 fetching local PDUs.",
@@ -810,19 +840,27 @@ pub(super) async fn resolve_true_destination(
 }
 
 #[admin_command]
-pub(super) async fn memory_stats(&self) -> Result<RoomMessageEventContent> {
-	let html_body = conduwuit::alloc::memory_stats();
+pub(super) async fn memory_stats(&self, opts: Option<String>) -> Result<RoomMessageEventContent> {
+	const OPTS: &str = "abcdefghijklmnopqrstuvwxyz";
 
-	if html_body.is_none() {
-		return Ok(RoomMessageEventContent::text_plain(
-			"malloc stats are not supported on your compiled malloc.",
-		));
-	}
+	let opts: String = OPTS
+		.chars()
+		.filter(|&c| {
+			let allow_any = opts.as_ref().is_some_and(|opts| opts == "*");
 
-	Ok(RoomMessageEventContent::text_html(
-		"This command's output can only be viewed by clients that render HTML.".to_owned(),
-		html_body.expect("string result"),
-	))
+			let allow = allow_any || opts.as_ref().is_some_and(|opts| opts.contains(c));
+
+			!allow
+		})
+		.collect();
+
+	let stats = conduwuit::alloc::memory_stats(&opts).unwrap_or_default();
+
+	self.write_str("```\n").await?;
+	self.write_str(&stats).await?;
+	self.write_str("\n```").await?;
+
+	Ok(RoomMessageEventContent::text_plain(""))
 }
 
 #[cfg(tokio_unstable)]
@@ -895,7 +933,7 @@ pub(super) async fn list_dependencies(&self, names: bool) -> Result<RoomMessageE
 		} else {
 			String::new()
 		};
-		writeln!(out, "{name} | {version} | {feats}")?;
+		writeln!(out, "| {name} | {version} | {feats} |")?;
 	}
 
 	Ok(RoomMessageEventContent::notice_markdown(out))
@@ -911,8 +949,8 @@ pub(super) async fn database_stats(
 	let map_name = map.as_ref().map_or(EMPTY, String::as_str);
 
 	let mut out = String::new();
-	for (name, map) in self.services.db.iter() {
-		if !map_name.is_empty() && *map_name != *name {
+	for (&name, map) in self.services.db.iter() {
+		if !map_name.is_empty() && map_name != name {
 			continue;
 		}
 
@@ -923,3 +961,12 @@ pub(super) async fn database_stats(
 
 	Ok(RoomMessageEventContent::notice_markdown(out))
 }
+
+#[admin_command]
+pub(super) async fn trim_memory(&self) -> Result<RoomMessageEventContent> {
+	conduwuit::alloc::trim(None)?;
+
+	writeln!(self, "done").await?;
+
+	Ok(RoomMessageEventContent::notice_plain(""))
+}

src/admin/debug/mod.rs

@@ -4,6 +4,7 @@ pub(crate) mod tester;
 use clap::Subcommand;
 use conduwuit::Result;
 use ruma::{EventId, OwnedRoomOrAliasId, RoomId, ServerName};
+use service::rooms::short::{ShortEventId, ShortRoomId};
 
 use self::tester::TesterCommand;
 use crate::admin_command_dispatch;
@@ -31,12 +32,21 @@ pub(super) enum DebugCommand {
 	/// the command.
 	ParsePdu,
 
-	/// - Retrieve and print a PDU by ID from the conduwuit database
+	/// - Retrieve and print a PDU by EventID from the conduwuit database
 	GetPdu {
 		/// An event ID (a $ followed by the base64 reference hash)
 		event_id: Box<EventId>,
 	},
 
+	/// - Retrieve and print a PDU by PduId from the conduwuit database
+	GetShortPdu {
+		/// Shortroomid integer
+		shortroomid: ShortRoomId,
+
+		/// Shorteventid integer
+		shorteventid: ShortEventId,
+	},
+
 	/// - Attempts to retrieve a PDU from a remote server. Inserts it into our
 	///   database/timeline if found and we do not have this PDU already
 	///   (following normal event auth rules, handles it as an incoming PDU).
@@ -181,7 +191,13 @@ pub(super) enum DebugCommand {
 	},
 
 	/// - Print extended memory usage
-	MemoryStats,
+	///
+	/// Optional argument is a character mask (a sequence of characters in any
+	/// order) which enable additional extended statistics. Known characters are
+	/// "abdeglmx". For convenience, a '*' will enable everything.
+	MemoryStats {
+		opts: Option<String>,
+	},
 
 	/// - Print general tokio runtime metric totals.
 	RuntimeMetrics,
@@ -207,6 +223,9 @@ pub(super) enum DebugCommand {
 		map: Option<String>,
 	},
 
+	/// - Trim memory usage
+	TrimMemory,
+
 	/// - Developer test stubs
 	#[command(subcommand)]
 	#[allow(non_snake_case)]

src/admin/debug/tester.rs

@@ -31,7 +31,7 @@ async fn failure(&self) -> Result<RoomMessageEventContent> {
 #[admin_command]
 async fn tester(&self) -> Result<RoomMessageEventContent> {
 
-	Ok(RoomMessageEventContent::notice_plain("completed"))
+	Ok(RoomMessageEventContent::notice_plain("legacy"))
 }
 
 #[inline(never)]

src/admin/federation/commands.rs

@@ -92,7 +92,7 @@ pub(super) async fn remote_user_in_rooms(
 	&self,
 	user_id: Box<UserId>,
 ) -> Result<RoomMessageEventContent> {
-	if user_id.server_name() == self.services.globals.config.server_name {
+	if user_id.server_name() == self.services.server.name {
 		return Ok(RoomMessageEventContent::text_plain(
 			"User belongs to our server, please use `list-joined-rooms` user admin command \
 			 instead.",

src/admin/mod.rs

@@ -1,6 +1,7 @@
 #![recursion_limit = "192"]
 #![allow(clippy::wildcard_imports)]
 #![allow(clippy::enum_glob_use)]
+#![allow(clippy::too_many_arguments)]
 
 pub(crate) mod admin;
 pub(crate) mod command;

src/admin/processor.rs

@@ -1,5 +1,6 @@
 use std::{
 	fmt::Write,
+	mem::take,
 	panic::AssertUnwindSafe,
 	sync::{Arc, Mutex},
 	time::SystemTime,
@@ -17,7 +18,7 @@ use conduwuit::{
 	utils::string::{collect_stream, common_prefix},
 	warn, Error, Result,
 };
-use futures::future::FutureExt;
+use futures::{future::FutureExt, io::BufWriter, AsyncWriteExt};
 use ruma::{
 	events::{
 		relation::InReplyTo,
@@ -62,9 +63,32 @@ async fn process_command(services: Arc<Services>, input: &CommandInput) -> Proce
 		body: &body,
 		timer: SystemTime::now(),
 		reply_id: input.reply_id.as_deref(),
+		output: BufWriter::new(Vec::new()).into(),
 	};
 
-	process(&context, command, &args).await
+	let (result, mut logs) = process(&context, command, &args).await;
+
+	let output = &mut context.output.lock().await;
+	output.flush().await.expect("final flush of output stream");
+
+	let output =
+		String::from_utf8(take(output.get_mut())).expect("invalid utf8 in command output stream");
+
+	match result {
+		| Ok(()) if logs.is_empty() =>
+			Ok(Some(reply(RoomMessageEventContent::notice_markdown(output), context.reply_id))),
+
+		| Ok(()) => {
+			logs.write_str(output.as_str()).expect("output buffer");
+			Ok(Some(reply(RoomMessageEventContent::notice_markdown(logs), context.reply_id)))
+		},
+		| Err(error) => {
+			write!(&mut logs, "Command failed with error:\n```\n{error:#?}\n```")
+				.expect("output buffer");
+
+			Err(reply(RoomMessageEventContent::notice_markdown(logs), context.reply_id))
+		},
+	}
 }
 
 fn handle_panic(error: &Error, command: &CommandInput) -> ProcessorResult {
@@ -81,7 +105,7 @@ async fn process(
 	context: &Command<'_>,
 	command: AdminCommand,
 	args: &[String],
-) -> ProcessorResult {
+) -> (Result, String) {
 	let (capture, logs) = capture_create(context);
 
 	let capture_scope = capture.start();
@@ -104,18 +128,7 @@ async fn process(
 	}
 	drop(logs);
 
-	match result {
-		| Ok(content) => {
-			write!(&mut output, "{0}", content.body())
-				.expect("failed to format command result to output buffer");
-			Ok(Some(reply(RoomMessageEventContent::notice_markdown(output), context.reply_id)))
-		},
-		| Err(error) => {
-			write!(&mut output, "Command failed with error:\n```\n{error:#?}\n```")
-				.expect("failed to format command result to output");
-			Err(reply(RoomMessageEventContent::notice_markdown(output), context.reply_id))
-		},
-	}
+	(result, output)
 }
 
 fn capture_create(context: &Command<'_>) -> (Arc<Capture>, Arc<Mutex<String>>) {

src/admin/query/account_data.rs

@@ -3,8 +3,9 @@ use conduwuit::Result;
 use futures::StreamExt;
 use ruma::{events::room::message::RoomMessageEventContent, RoomId, UserId};
 
-use crate::Command;
+use crate::{admin_command, admin_command_dispatch};
 
+#[admin_command_dispatch]
 #[derive(Debug, Subcommand)]
 /// All the getters and iterators from src/database/key_value/account_data.rs
 pub(crate) enum AccountDataCommand {
@@ -19,7 +20,7 @@ pub(crate) enum AccountDataCommand {
 	},
 
 	/// - Searches the account data for a specific kind.
-	Get {
+	AccountDataGet {
 		/// Full user ID
 		user_id: Box<UserId>,
 		/// Account data event type
@@ -29,38 +30,43 @@ pub(crate) enum AccountDataCommand {
 	},
 }
 
-/// All the getters and iterators from src/database/key_value/account_data.rs
-pub(super) async fn process(
-	subcommand: AccountDataCommand,
-	context: &Command<'_>,
+#[admin_command]
+async fn changes_since(
+	&self,
+	user_id: Box<UserId>,
+	since: u64,
+	room_id: Option<Box<RoomId>>,
 ) -> Result<RoomMessageEventContent> {
-	let services = context.services;
+	let timer = tokio::time::Instant::now();
+	let results: Vec<_> = self
+		.services
+		.account_data
+		.changes_since(room_id.as_deref(), &user_id, since)
+		.collect()
+		.await;
+	let query_time = timer.elapsed();
 
-	match subcommand {
-		| AccountDataCommand::ChangesSince { user_id, since, room_id } => {
-			let timer = tokio::time::Instant::now();
-			let results: Vec<_> = services
-				.account_data
-				.changes_since(room_id.as_deref(), &user_id, since)
-				.collect()
-				.await;
-			let query_time = timer.elapsed();
-
-			Ok(RoomMessageEventContent::notice_markdown(format!(
-				"Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```"
-			)))
-		},
-		| AccountDataCommand::Get { user_id, kind, room_id } => {
-			let timer = tokio::time::Instant::now();
-			let results = services
-				.account_data
-				.get_raw(room_id.as_deref(), &user_id, &kind)
-				.await;
-			let query_time = timer.elapsed();
-
-			Ok(RoomMessageEventContent::notice_markdown(format!(
-				"Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```"
-			)))
-		},
-	}
+	Ok(RoomMessageEventContent::notice_markdown(format!(
+		"Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```"
+	)))
+}
+
+#[admin_command]
+async fn account_data_get(
+	&self,
+	user_id: Box<UserId>,
+	kind: String,
+	room_id: Option<Box<RoomId>>,
+) -> Result<RoomMessageEventContent> {
+	let timer = tokio::time::Instant::now();
+	let results = self
+		.services
+		.account_data
+		.get_raw(room_id.as_deref(), &user_id, &kind)
+		.await;
+	let query_time = timer.elapsed();
+
+	Ok(RoomMessageEventContent::notice_markdown(format!(
+		"Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```"
+	)))
 }

src/admin/query/appservice.rs

@@ -1,6 +1,5 @@
 use clap::Subcommand;
 use conduwuit::Result;
-use ruma::events::room::message::RoomMessageEventContent;
 
 use crate::Command;
 
@@ -18,10 +17,7 @@ pub(crate) enum AppserviceCommand {
 }
 
 /// All the getters and iterators from src/database/key_value/appservice.rs
-pub(super) async fn process(
-	subcommand: AppserviceCommand,
-	context: &Command<'_>,
-) -> Result<RoomMessageEventContent> {
+pub(super) async fn process(subcommand: AppserviceCommand, context: &Command<'_>) -> Result {
 	let services = context.services;
 
 	match subcommand {
@@ -31,18 +27,15 @@ pub(super) async fn process(
 
 			let query_time = timer.elapsed();
 
-			Ok(RoomMessageEventContent::notice_markdown(format!(
-				"Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```"
-			)))
+			write!(context, "Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```")
 		},
 		| AppserviceCommand::All => {
 			let timer = tokio::time::Instant::now();
 			let results = services.appservice.all().await;
 			let query_time = timer.elapsed();
 
-			Ok(RoomMessageEventContent::notice_markdown(format!(
-				"Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```"
-			)))
+			write!(context, "Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```")
 		},
 	}
+	.await
 }

src/admin/query/globals.rs

@@ -1,6 +1,6 @@
 use clap::Subcommand;
 use conduwuit::Result;
-use ruma::{events::room::message::RoomMessageEventContent, ServerName};
+use ruma::ServerName;
 
 use crate::Command;
 
@@ -21,10 +21,7 @@ pub(crate) enum GlobalsCommand {
 }
 
 /// All the getters and iterators from src/database/key_value/globals.rs
-pub(super) async fn process(
-	subcommand: GlobalsCommand,
-	context: &Command<'_>,
-) -> Result<RoomMessageEventContent> {
+pub(super) async fn process(subcommand: GlobalsCommand, context: &Command<'_>) -> Result {
 	let services = context.services;
 
 	match subcommand {
@@ -33,36 +30,29 @@ pub(super) async fn process(
 			let results = services.globals.db.database_version().await;
 			let query_time = timer.elapsed();
 
-			Ok(RoomMessageEventContent::notice_markdown(format!(
-				"Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```"
-			)))
+			write!(context, "Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```")
 		},
 		| GlobalsCommand::CurrentCount => {
 			let timer = tokio::time::Instant::now();
 			let results = services.globals.db.current_count();
 			let query_time = timer.elapsed();
 
-			Ok(RoomMessageEventContent::notice_markdown(format!(
-				"Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```"
-			)))
+			write!(context, "Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```")
 		},
 		| GlobalsCommand::LastCheckForUpdatesId => {
 			let timer = tokio::time::Instant::now();
 			let results = services.updates.last_check_for_updates_id().await;
 			let query_time = timer.elapsed();
 
-			Ok(RoomMessageEventContent::notice_markdown(format!(
-				"Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```"
-			)))
+			write!(context, "Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```")
 		},
 		| GlobalsCommand::SigningKeysFor { origin } => {
 			let timer = tokio::time::Instant::now();
 			let results = services.server_keys.verify_keys_for(&origin).await;
 			let query_time = timer.elapsed();
 
-			Ok(RoomMessageEventContent::notice_markdown(format!(
-				"Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```"
-			)))
+			write!(context, "Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```")
 		},
 	}
+	.await
 }

src/admin/query/mod.rs

@@ -3,10 +3,13 @@ mod appservice;
 mod globals;
 mod presence;
 mod pusher;
+mod raw;
 mod resolver;
 mod room_alias;
 mod room_state_cache;
+mod room_timeline;
 mod sending;
+mod short;
 mod users;
 
 use clap::Subcommand;
@@ -14,9 +17,10 @@ use conduwuit::Result;
 
 use self::{
 	account_data::AccountDataCommand, appservice::AppserviceCommand, globals::GlobalsCommand,
-	presence::PresenceCommand, pusher::PusherCommand, resolver::ResolverCommand,
+	presence::PresenceCommand, pusher::PusherCommand, raw::RawCommand, resolver::ResolverCommand,
 	room_alias::RoomAliasCommand, room_state_cache::RoomStateCacheCommand,
-	sending::SendingCommand, users::UsersCommand,
+	room_timeline::RoomTimelineCommand, sending::SendingCommand, short::ShortCommand,
+	users::UsersCommand,
 };
 use crate::admin_command_dispatch;
 
@@ -44,6 +48,10 @@ pub(super) enum QueryCommand {
 	#[command(subcommand)]
 	RoomStateCache(RoomStateCacheCommand),
 
+	/// - rooms/timeline iterators and getters
+	#[command(subcommand)]
+	RoomTimeline(RoomTimelineCommand),
+
 	/// - globals.rs iterators and getters
 	#[command(subcommand)]
 	Globals(GlobalsCommand),
@@ -63,4 +71,12 @@ pub(super) enum QueryCommand {
 	/// - pusher service
 	#[command(subcommand)]
 	Pusher(PusherCommand),
+
+	/// - short service
+	#[command(subcommand)]
+	Short(ShortCommand),
+
+	/// - raw service
+	#[command(subcommand)]
+	Raw(RawCommand),
 }

src/admin/query/presence.rs

@@ -1,7 +1,7 @@
 use clap::Subcommand;
 use conduwuit::Result;
 use futures::StreamExt;
-use ruma::{events::room::message::RoomMessageEventContent, UserId};
+use ruma::UserId;
 
 use crate::Command;
 
@@ -23,21 +23,16 @@ pub(crate) enum PresenceCommand {
 }
 
 /// All the getters and iterators in key_value/presence.rs
-pub(super) async fn process(
-	subcommand: PresenceCommand,
-	context: &Command<'_>,
-) -> Result<RoomMessageEventContent> {
+pub(super) async fn process(subcommand: PresenceCommand, context: &Command<'_>) -> Result {
 	let services = context.services;
 
 	match subcommand {
 		| PresenceCommand::GetPresence { user_id } => {
 			let timer = tokio::time::Instant::now();
-			let results = services.presence.db.get_presence(&user_id).await;
+			let results = services.presence.get_presence(&user_id).await;
 			let query_time = timer.elapsed();
 
-			Ok(RoomMessageEventContent::notice_markdown(format!(
-				"Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```"
-			)))
+			write!(context, "Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```")
 		},
 		| PresenceCommand::PresenceSince { since } => {
 			let timer = tokio::time::Instant::now();
@@ -49,9 +44,8 @@ pub(super) async fn process(
 				.await;
 			let query_time = timer.elapsed();
 
-			Ok(RoomMessageEventContent::notice_markdown(format!(
-				"Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```"
-			)))
+			write!(context, "Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```")
 		},
 	}
+	.await
 }

src/admin/query/pusher.rs

@@ -1,6 +1,6 @@
 use clap::Subcommand;
 use conduwuit::Result;
-use ruma::{events::room::message::RoomMessageEventContent, UserId};
+use ruma::UserId;
 
 use crate::Command;
 
@@ -13,10 +13,7 @@ pub(crate) enum PusherCommand {
 	},
 }
 
-pub(super) async fn process(
-	subcommand: PusherCommand,
-	context: &Command<'_>,
-) -> Result<RoomMessageEventContent> {
+pub(super) async fn process(subcommand: PusherCommand, context: &Command<'_>) -> Result {
 	let services = context.services;
 
 	match subcommand {
@@ -25,9 +22,8 @@ pub(super) async fn process(
 			let results = services.pusher.get_pushers(&user_id).await;
 			let query_time = timer.elapsed();
 
-			Ok(RoomMessageEventContent::notice_markdown(format!(
-				"Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```"
-			)))
+			write!(context, "Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```")
 		},
 	}
+	.await
 }

src/admin/query/raw.rs

@@ -0,0 +1,576 @@
+use std::{borrow::Cow, collections::BTreeMap, ops::Deref};
+
+use clap::Subcommand;
+use conduwuit::{
+	apply, at, is_zero,
+	utils::{
+		stream::{ReadyExt, TryIgnore, TryParallelExt},
+		string::EMPTY,
+		IterStream,
+	},
+	Err, Result,
+};
+use futures::{FutureExt, StreamExt, TryStreamExt};
+use ruma::events::room::message::RoomMessageEventContent;
+use tokio::time::Instant;
+
+use crate::{admin_command, admin_command_dispatch};
+
+#[admin_command_dispatch]
+#[derive(Debug, Subcommand)]
+#[allow(clippy::enum_variant_names)]
+/// Query tables from database
+pub(crate) enum RawCommand {
+	/// - List database maps
+	RawMaps,
+
+	/// - Raw database query
+	RawGet {
+		/// Map name
+		map: String,
+
+		/// Key
+		key: String,
+	},
+
+	/// - Raw database delete (for string keys)
+	RawDel {
+		/// Map name
+		map: String,
+
+		/// Key
+		key: String,
+	},
+
+	/// - Raw database keys iteration
+	RawKeys {
+		/// Map name
+		map: String,
+
+		/// Key prefix
+		prefix: Option<String>,
+	},
+
+	/// - Raw database key size breakdown
+	RawKeysSizes {
+		/// Map name
+		map: Option<String>,
+
+		/// Key prefix
+		prefix: Option<String>,
+	},
+
+	/// - Raw database keys total bytes
+	RawKeysTotal {
+		/// Map name
+		map: Option<String>,
+
+		/// Key prefix
+		prefix: Option<String>,
+	},
+
+	/// - Raw database values size breakdown
+	RawValsSizes {
+		/// Map name
+		map: Option<String>,
+
+		/// Key prefix
+		prefix: Option<String>,
+	},
+
+	/// - Raw database values total bytes
+	RawValsTotal {
+		/// Map name
+		map: Option<String>,
+
+		/// Key prefix
+		prefix: Option<String>,
+	},
+
+	/// - Raw database items iteration
+	RawIter {
+		/// Map name
+		map: String,
+
+		/// Key prefix
+		prefix: Option<String>,
+	},
+
+	/// - Raw database keys iteration
+	RawKeysFrom {
+		/// Map name
+		map: String,
+
+		/// Lower-bound
+		start: String,
+
+		/// Limit
+		#[arg(short, long)]
+		limit: Option<usize>,
+	},
+
+	/// - Raw database items iteration
+	RawIterFrom {
+		/// Map name
+		map: String,
+
+		/// Lower-bound
+		start: String,
+
+		/// Limit
+		#[arg(short, long)]
+		limit: Option<usize>,
+	},
+
+	/// - Raw database record count
+	RawCount {
+		/// Map name
+		map: Option<String>,
+
+		/// Key prefix
+		prefix: Option<String>,
+	},
+
+	/// - Compact database
+	Compact {
+		#[arg(short, long, alias("column"))]
+		map: Option<Vec<String>>,
+
+		#[arg(long)]
+		start: Option<String>,
+
+		#[arg(long)]
+		stop: Option<String>,
+
+		#[arg(long)]
+		from: Option<usize>,
+
+		#[arg(long)]
+		into: Option<usize>,
+
+		/// There is one compaction job per column; then this controls how many
+		/// columns are compacted in parallel. If zero, one compaction job is
+		/// still run at a time here, but in exclusive-mode blocking any other
+		/// automatic compaction jobs until complete.
+		#[arg(long)]
+		parallelism: Option<usize>,
+
+		#[arg(long, default_value("false"))]
+		exhaustive: bool,
+	},
+}
+
+#[admin_command]
+pub(super) async fn compact(
+	&self,
+	map: Option<Vec<String>>,
+	start: Option<String>,
+	stop: Option<String>,
+	from: Option<usize>,
+	into: Option<usize>,
+	parallelism: Option<usize>,
+	exhaustive: bool,
+) -> Result<RoomMessageEventContent> {
+	use conduwuit_database::compact::Options;
+
+	let default_all_maps = map
+		.is_none()
+		.then(|| {
+			self.services
+				.db
+				.keys()
+				.map(Deref::deref)
+				.map(ToOwned::to_owned)
+		})
+		.into_iter()
+		.flatten();
+
+	let maps: Vec<_> = map
+		.unwrap_or_default()
+		.into_iter()
+		.chain(default_all_maps)
+		.map(|map| self.services.db.get(&map))
+		.filter_map(Result::ok)
+		.cloned()
+		.collect();
+
+	if maps.is_empty() {
+		return Err!("--map argument invalid. not found in database");
+	}
+
+	let range = (
+		start.as_ref().map(String::as_bytes).map(Into::into),
+		stop.as_ref().map(String::as_bytes).map(Into::into),
+	);
+
+	let options = Options {
+		range,
+		level: (from, into),
+		exclusive: parallelism.is_some_and(is_zero!()),
+		exhaustive,
+	};
+
+	let runtime = self.services.server.runtime().clone();
+	let parallelism = parallelism.unwrap_or(1);
+	let results = maps
+		.into_iter()
+		.try_stream()
+		.paralleln_and_then(runtime, parallelism, move |map| {
+			map.compact_blocking(options.clone())?;
+			Ok(map.name().to_owned())
+		})
+		.collect::<Vec<_>>();
+
+	let timer = Instant::now();
+	let results = results.await;
+	let query_time = timer.elapsed();
+	self.write_str(&format!("Jobs completed in {query_time:?}:\n\n```rs\n{results:#?}\n```"))
+		.await?;
+
+	Ok(RoomMessageEventContent::text_plain(""))
+}
+
+#[admin_command]
+pub(super) async fn raw_count(
+	&self,
+	map: Option<String>,
+	prefix: Option<String>,
+) -> Result<RoomMessageEventContent> {
+	let prefix = prefix.as_deref().unwrap_or(EMPTY);
+
+	let default_all_maps = map
+		.is_none()
+		.then(|| self.services.db.keys().map(Deref::deref))
+		.into_iter()
+		.flatten();
+
+	let maps: Vec<_> = map
+		.iter()
+		.map(String::as_str)
+		.chain(default_all_maps)
+		.map(|map| self.services.db.get(map))
+		.filter_map(Result::ok)
+		.cloned()
+		.collect();
+
+	let timer = Instant::now();
+	let count = maps
+		.iter()
+		.stream()
+		.then(|map| map.raw_count_prefix(&prefix))
+		.ready_fold(0_usize, usize::saturating_add)
+		.await;
+
+	let query_time = timer.elapsed();
+	self.write_str(&format!("Query completed in {query_time:?}:\n\n```rs\n{count:#?}\n```"))
+		.await?;
+
+	Ok(RoomMessageEventContent::text_plain(""))
+}
+
+#[admin_command]
+pub(super) async fn raw_keys(
+	&self,
+	map: String,
+	prefix: Option<String>,
+) -> Result<RoomMessageEventContent> {
+	writeln!(self, "```").boxed().await?;
+
+	let map = self.services.db.get(map.as_str())?;
+	let timer = Instant::now();
+	prefix
+		.as_deref()
+		.map_or_else(|| map.raw_keys().boxed(), |prefix| map.raw_keys_prefix(prefix).boxed())
+		.map_ok(String::from_utf8_lossy)
+		.try_for_each(|str| writeln!(self, "{str:?}"))
+		.boxed()
+		.await?;
+
+	let query_time = timer.elapsed();
+	let out = format!("\n```\n\nQuery completed in {query_time:?}");
+	self.write_str(out.as_str()).await?;
+
+	Ok(RoomMessageEventContent::text_plain(""))
+}
+
+#[admin_command]
+pub(super) async fn raw_keys_sizes(
+	&self,
+	map: Option<String>,
+	prefix: Option<String>,
+) -> Result<RoomMessageEventContent> {
+	let prefix = prefix.as_deref().unwrap_or(EMPTY);
+
+	let default_all_maps = map
+		.is_none()
+		.then(|| self.services.db.keys().map(Deref::deref))
+		.into_iter()
+		.flatten();
+
+	let maps: Vec<_> = map
+		.iter()
+		.map(String::as_str)
+		.chain(default_all_maps)
+		.map(|map| self.services.db.get(map))
+		.filter_map(Result::ok)
+		.cloned()
+		.collect();
+
+	let timer = Instant::now();
+	let result = maps
+		.iter()
+		.stream()
+		.map(|map| map.raw_keys_prefix(&prefix))
+		.flatten()
+		.ignore_err()
+		.map(<[u8]>::len)
+		.ready_fold_default(|mut map: BTreeMap<_, usize>, len| {
+			let entry = map.entry(len).or_default();
+			*entry = entry.saturating_add(1);
+			map
+		})
+		.await;
+
+	let query_time = timer.elapsed();
+	let result = format!("```\n{result:#?}\n```\n\nQuery completed in {query_time:?}");
+	self.write_str(result.as_str()).await?;
+
+	Ok(RoomMessageEventContent::text_plain(""))
+}
+
+#[admin_command]
+pub(super) async fn raw_keys_total(
+	&self,
+	map: Option<String>,
+	prefix: Option<String>,
+) -> Result<RoomMessageEventContent> {
+	let prefix = prefix.as_deref().unwrap_or(EMPTY);
+
+	let default_all_maps = map
+		.is_none()
+		.then(|| self.services.db.keys().map(Deref::deref))
+		.into_iter()
+		.flatten();
+
+	let maps: Vec<_> = map
+		.iter()
+		.map(String::as_str)
+		.chain(default_all_maps)
+		.map(|map| self.services.db.get(map))
+		.filter_map(Result::ok)
+		.cloned()
+		.collect();
+
+	let timer = Instant::now();
+	let result = maps
+		.iter()
+		.stream()
+		.map(|map| map.raw_keys_prefix(&prefix))
+		.flatten()
+		.ignore_err()
+		.map(<[u8]>::len)
+		.ready_fold_default(|acc: usize, len| acc.saturating_add(len))
+		.await;
+
+	let query_time = timer.elapsed();
+
+	self.write_str(&format!("```\n{result:#?}\n\n```\n\nQuery completed in {query_time:?}"))
+		.await?;
+
+	Ok(RoomMessageEventContent::text_plain(""))
+}
+
+#[admin_command]
+pub(super) async fn raw_vals_sizes(
+	&self,
+	map: Option<String>,
+	prefix: Option<String>,
+) -> Result<RoomMessageEventContent> {
+	let prefix = prefix.as_deref().unwrap_or(EMPTY);
+
+	let default_all_maps = map
+		.is_none()
+		.then(|| self.services.db.keys().map(Deref::deref))
+		.into_iter()
+		.flatten();
+
+	let maps: Vec<_> = map
+		.iter()
+		.map(String::as_str)
+		.chain(default_all_maps)
+		.map(|map| self.services.db.get(map))
+		.filter_map(Result::ok)
+		.cloned()
+		.collect();
+
+	let timer = Instant::now();
+	let result = maps
+		.iter()
+		.stream()
+		.map(|map| map.raw_stream_prefix(&prefix))
+		.flatten()
+		.ignore_err()
+		.map(at!(1))
+		.map(<[u8]>::len)
+		.ready_fold_default(|mut map: BTreeMap<_, usize>, len| {
+			let entry = map.entry(len).or_default();
+			*entry = entry.saturating_add(1);
+			map
+		})
+		.await;
+
+	let query_time = timer.elapsed();
+	let result = format!("```\n{result:#?}\n```\n\nQuery completed in {query_time:?}");
+	self.write_str(result.as_str()).await?;
+
+	Ok(RoomMessageEventContent::text_plain(""))
+}
+
+#[admin_command]
+pub(super) async fn raw_vals_total(
+	&self,
+	map: Option<String>,
+	prefix: Option<String>,
+) -> Result<RoomMessageEventContent> {
+	let prefix = prefix.as_deref().unwrap_or(EMPTY);
+
+	let default_all_maps = map
+		.is_none()
+		.then(|| self.services.db.keys().map(Deref::deref))
+		.into_iter()
+		.flatten();
+
+	let maps: Vec<_> = map
+		.iter()
+		.map(String::as_str)
+		.chain(default_all_maps)
+		.map(|map| self.services.db.get(map))
+		.filter_map(Result::ok)
+		.cloned()
+		.collect();
+
+	let timer = Instant::now();
+	let result = maps
+		.iter()
+		.stream()
+		.map(|map| map.raw_stream_prefix(&prefix))
+		.flatten()
+		.ignore_err()
+		.map(at!(1))
+		.map(<[u8]>::len)
+		.ready_fold_default(|acc: usize, len| acc.saturating_add(len))
+		.await;
+
+	let query_time = timer.elapsed();
+
+	self.write_str(&format!("```\n{result:#?}\n\n```\n\nQuery completed in {query_time:?}"))
+		.await?;
+
+	Ok(RoomMessageEventContent::text_plain(""))
+}
+
+#[admin_command]
+pub(super) async fn raw_iter(
+	&self,
+	map: String,
+	prefix: Option<String>,
+) -> Result<RoomMessageEventContent> {
+	writeln!(self, "```").await?;
+
+	let map = self.services.db.get(&map)?;
+	let timer = Instant::now();
+	prefix
+		.as_deref()
+		.map_or_else(|| map.raw_stream().boxed(), |prefix| map.raw_stream_prefix(prefix).boxed())
+		.map_ok(apply!(2, String::from_utf8_lossy))
+		.map_ok(apply!(2, Cow::into_owned))
+		.try_for_each(|keyval| writeln!(self, "{keyval:?}"))
+		.boxed()
+		.await?;
+
+	let query_time = timer.elapsed();
+	self.write_str(&format!("\n```\n\nQuery completed in {query_time:?}"))
+		.await?;
+
+	Ok(RoomMessageEventContent::text_plain(""))
+}
+
+#[admin_command]
+pub(super) async fn raw_keys_from(
+	&self,
+	map: String,
+	start: String,
+	limit: Option<usize>,
+) -> Result<RoomMessageEventContent> {
+	writeln!(self, "```").await?;
+
+	let map = self.services.db.get(&map)?;
+	let timer = Instant::now();
+	map.raw_keys_from(&start)
+		.map_ok(String::from_utf8_lossy)
+		.take(limit.unwrap_or(usize::MAX))
+		.try_for_each(|str| writeln!(self, "{str:?}"))
+		.boxed()
+		.await?;
+
+	let query_time = timer.elapsed();
+	self.write_str(&format!("\n```\n\nQuery completed in {query_time:?}"))
+		.await?;
+
+	Ok(RoomMessageEventContent::text_plain(""))
+}
+
+#[admin_command]
+pub(super) async fn raw_iter_from(
+	&self,
+	map: String,
+	start: String,
+	limit: Option<usize>,
+) -> Result<RoomMessageEventContent> {
+	let map = self.services.db.get(&map)?;
+	let timer = Instant::now();
+	let result = map
+		.raw_stream_from(&start)
+		.map_ok(apply!(2, String::from_utf8_lossy))
+		.map_ok(apply!(2, Cow::into_owned))
+		.take(limit.unwrap_or(usize::MAX))
+		.try_collect::<Vec<(String, String)>>()
+		.await?;
+
+	let query_time = timer.elapsed();
+	Ok(RoomMessageEventContent::notice_markdown(format!(
+		"Query completed in {query_time:?}:\n\n```rs\n{result:#?}\n```"
+	)))
+}
+
+#[admin_command]
+pub(super) async fn raw_del(&self, map: String, key: String) -> Result<RoomMessageEventContent> {
+	let map = self.services.db.get(&map)?;
+	let timer = Instant::now();
+	map.remove(&key);
+	let query_time = timer.elapsed();
+
+	Ok(RoomMessageEventContent::notice_markdown(format!(
+		"Operation completed in {query_time:?}"
+	)))
+}
+
+#[admin_command]
+pub(super) async fn raw_get(&self, map: String, key: String) -> Result<RoomMessageEventContent> {
+	let map = self.services.db.get(&map)?;
+	let timer = Instant::now();
+	let handle = map.get(&key).await?;
+	let query_time = timer.elapsed();
+	let result = String::from_utf8_lossy(&handle);
+
+	Ok(RoomMessageEventContent::notice_markdown(format!(
+		"Query completed in {query_time:?}:\n\n```rs\n{result:?}\n```"
+	)))
+}
+
+#[admin_command]
+pub(super) async fn raw_maps(&self) -> Result<RoomMessageEventContent> {
+	let list: Vec<_> = self.services.db.iter().map(at!(0)).copied().collect();
+
+	Ok(RoomMessageEventContent::notice_markdown(format!("{list:#?}")))
+}

src/admin/query/resolver.rs

@@ -1,7 +1,6 @@
-use std::fmt::Write;
-
 use clap::Subcommand;
 use conduwuit::{utils::time, Result};
+use futures::StreamExt;
 use ruma::{events::room::message::RoomMessageEventContent, OwnedServerName};
 
 use crate::{admin_command, admin_command_dispatch};
@@ -28,56 +27,48 @@ async fn destinations_cache(
 ) -> Result<RoomMessageEventContent> {
 	use service::resolver::cache::CachedDest;
 
-	let mut out = String::new();
-	writeln!(out, "| Server Name | Destination | Hostname | Expires |")?;
-	writeln!(out, "| ----------- | ----------- | -------- | ------- |")?;
-	let row = |(name, &CachedDest { ref dest, ref host, expire })| {
+	writeln!(self, "| Server Name | Destination | Hostname | Expires |").await?;
+	writeln!(self, "| ----------- | ----------- | -------- | ------- |").await?;
+
+	let mut destinations = self.services.resolver.cache.destinations().boxed();
+
+	while let Some((name, CachedDest { dest, host, expire })) = destinations.next().await {
+		if let Some(server_name) = server_name.as_ref() {
+			if name != server_name {
+				continue;
+			}
+		}
+
 		let expire = time::format(expire, "%+");
-		writeln!(out, "| {name} | {dest} | {host} | {expire} |").expect("wrote line");
-	};
-
-	let map = self
-		.services
-		.resolver
-		.cache
-		.destinations
-		.read()
-		.expect("locked");
-
-	if let Some(server_name) = server_name.as_ref() {
-		map.get_key_value(server_name).map(row);
-	} else {
-		map.iter().for_each(row);
+		self.write_str(&format!("| {name} | {dest} | {host} | {expire} |\n"))
+			.await?;
 	}
 
-	Ok(RoomMessageEventContent::notice_markdown(out))
+	Ok(RoomMessageEventContent::notice_plain(""))
 }
 
 #[admin_command]
 async fn overrides_cache(&self, server_name: Option<String>) -> Result<RoomMessageEventContent> {
 	use service::resolver::cache::CachedOverride;
 
-	let mut out = String::new();
-	writeln!(out, "| Server Name | IP  | Port | Expires |")?;
-	writeln!(out, "| ----------- | --- | ----:| ------- |")?;
-	let row = |(name, &CachedOverride { ref ips, port, expire })| {
+	writeln!(self, "| Server Name | IP  | Port | Expires | Overriding |").await?;
+	writeln!(self, "| ----------- | --- | ----:| ------- | ---------- |").await?;
+
+	let mut overrides = self.services.resolver.cache.overrides().boxed();
+
+	while let Some((name, CachedOverride { ips, port, expire, overriding })) =
+		overrides.next().await
+	{
+		if let Some(server_name) = server_name.as_ref() {
+			if name != server_name {
+				continue;
+			}
+		}
+
 		let expire = time::format(expire, "%+");
-		writeln!(out, "| {name} | {ips:?} | {port} | {expire} |").expect("wrote line");
-	};
-
-	let map = self
-		.services
-		.resolver
-		.cache
-		.overrides
-		.read()
-		.expect("locked");
-
-	if let Some(server_name) = server_name.as_ref() {
-		map.get_key_value(server_name).map(row);
-	} else {
-		map.iter().for_each(row);
+		self.write_str(&format!("| {name} | {ips:?} | {port} | {expire} | {overriding:?} |\n"))
+			.await?;
 	}
 
-	Ok(RoomMessageEventContent::notice_markdown(out))
+	Ok(RoomMessageEventContent::notice_plain(""))
 }

src/admin/query/room_alias.rs

@@ -1,7 +1,7 @@
 use clap::Subcommand;
 use conduwuit::Result;
 use futures::StreamExt;
-use ruma::{events::room::message::RoomMessageEventContent, RoomAliasId, RoomId};
+use ruma::{RoomAliasId, RoomId};
 
 use crate::Command;
 
@@ -24,10 +24,7 @@ pub(crate) enum RoomAliasCommand {
 }
 
 /// All the getters and iterators in src/database/key_value/rooms/alias.rs
-pub(super) async fn process(
-	subcommand: RoomAliasCommand,
-	context: &Command<'_>,
-) -> Result<RoomMessageEventContent> {
+pub(super) async fn process(subcommand: RoomAliasCommand, context: &Command<'_>) -> Result {
 	let services = context.services;
 
 	match subcommand {
@@ -36,9 +33,7 @@ pub(super) async fn process(
 			let results = services.rooms.alias.resolve_local_alias(&alias).await;
 			let query_time = timer.elapsed();
 
-			Ok(RoomMessageEventContent::notice_markdown(format!(
-				"Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```"
-			)))
+			write!(context, "Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```")
 		},
 		| RoomAliasCommand::LocalAliasesForRoom { room_id } => {
 			let timer = tokio::time::Instant::now();
@@ -51,9 +46,7 @@ pub(super) async fn process(
 				.await;
 			let query_time = timer.elapsed();
 
-			Ok(RoomMessageEventContent::notice_markdown(format!(
-				"Query completed in {query_time:?}:\n\n```rs\n{aliases:#?}\n```"
-			)))
+			write!(context, "Query completed in {query_time:?}:\n\n```rs\n{aliases:#?}\n```")
 		},
 		| RoomAliasCommand::AllLocalAliases => {
 			let timer = tokio::time::Instant::now();
@@ -66,9 +59,8 @@ pub(super) async fn process(
 				.await;
 			let query_time = timer.elapsed();
 
-			Ok(RoomMessageEventContent::notice_markdown(format!(
-				"Query completed in {query_time:?}:\n\n```rs\n{aliases:#?}\n```"
-			)))
+			write!(context, "Query completed in {query_time:?}:\n\n```rs\n{aliases:#?}\n```")
 		},
 	}
+	.await
 }

src/admin/query/room_state_cache.rs

@@ -1,5 +1,5 @@
 use clap::Subcommand;
-use conduwuit::Result;
+use conduwuit::{Error, Result};
 use futures::StreamExt;
 use ruma::{events::room::message::RoomMessageEventContent, RoomId, ServerName, UserId};
 
@@ -76,13 +76,10 @@ pub(crate) enum RoomStateCacheCommand {
 	},
 }
 
-pub(super) async fn process(
-	subcommand: RoomStateCacheCommand,
-	context: &Command<'_>,
-) -> Result<RoomMessageEventContent> {
+pub(super) async fn process(subcommand: RoomStateCacheCommand, context: &Command<'_>) -> Result {
 	let services = context.services;
 
-	match subcommand {
+	let c = match subcommand {
 		| RoomStateCacheCommand::ServerInRoom { server, room_id } => {
 			let timer = tokio::time::Instant::now();
 			let result = services
@@ -92,7 +89,7 @@ pub(super) async fn process(
 				.await;
 			let query_time = timer.elapsed();
 
-			Ok(RoomMessageEventContent::notice_markdown(format!(
+			Result::<_, Error>::Ok(RoomMessageEventContent::notice_markdown(format!(
 				"Query completed in {query_time:?}:\n\n```rs\n{result:#?}\n```"
 			)))
 		},
@@ -107,7 +104,7 @@ pub(super) async fn process(
 				.await;
 			let query_time = timer.elapsed();
 
-			Ok(RoomMessageEventContent::notice_markdown(format!(
+			Result::<_, Error>::Ok(RoomMessageEventContent::notice_markdown(format!(
 				"Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```"
 			)))
 		},
@@ -122,7 +119,7 @@ pub(super) async fn process(
 				.await;
 			let query_time = timer.elapsed();
 
-			Ok(RoomMessageEventContent::notice_markdown(format!(
+			Result::<_, Error>::Ok(RoomMessageEventContent::notice_markdown(format!(
 				"Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```"
 			)))
 		},
@@ -137,7 +134,7 @@ pub(super) async fn process(
 				.await;
 			let query_time = timer.elapsed();
 
-			Ok(RoomMessageEventContent::notice_markdown(format!(
+			Result::<_, Error>::Ok(RoomMessageEventContent::notice_markdown(format!(
 				"Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```"
 			)))
 		},
@@ -152,7 +149,7 @@ pub(super) async fn process(
 				.await;
 			let query_time = timer.elapsed();
 
-			Ok(RoomMessageEventContent::notice_markdown(format!(
+			Result::<_, Error>::Ok(RoomMessageEventContent::notice_markdown(format!(
 				"Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```"
 			)))
 		},
@@ -167,7 +164,7 @@ pub(super) async fn process(
 				.await;
 			let query_time = timer.elapsed();
 
-			Ok(RoomMessageEventContent::notice_markdown(format!(
+			Result::<_, Error>::Ok(RoomMessageEventContent::notice_markdown(format!(
 				"Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```"
 			)))
 		},
@@ -176,7 +173,7 @@ pub(super) async fn process(
 			let results = services.rooms.state_cache.room_joined_count(&room_id).await;
 			let query_time = timer.elapsed();
 
-			Ok(RoomMessageEventContent::notice_markdown(format!(
+			Result::<_, Error>::Ok(RoomMessageEventContent::notice_markdown(format!(
 				"Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```"
 			)))
 		},
@@ -189,7 +186,7 @@ pub(super) async fn process(
 				.await;
 			let query_time = timer.elapsed();
 
-			Ok(RoomMessageEventContent::notice_markdown(format!(
+			Result::<_, Error>::Ok(RoomMessageEventContent::notice_markdown(format!(
 				"Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```"
 			)))
 		},
@@ -204,7 +201,7 @@ pub(super) async fn process(
 				.await;
 			let query_time = timer.elapsed();
 
-			Ok(RoomMessageEventContent::notice_markdown(format!(
+			Result::<_, Error>::Ok(RoomMessageEventContent::notice_markdown(format!(
 				"Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```"
 			)))
 		},
@@ -219,7 +216,7 @@ pub(super) async fn process(
 				.await;
 			let query_time = timer.elapsed();
 
-			Ok(RoomMessageEventContent::notice_markdown(format!(
+			Result::<_, Error>::Ok(RoomMessageEventContent::notice_markdown(format!(
 				"Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```"
 			)))
 		},
@@ -232,7 +229,7 @@ pub(super) async fn process(
 				.await;
 			let query_time = timer.elapsed();
 
-			Ok(RoomMessageEventContent::notice_markdown(format!(
+			Result::<_, Error>::Ok(RoomMessageEventContent::notice_markdown(format!(
 				"Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```"
 			)))
 		},
@@ -245,7 +242,7 @@ pub(super) async fn process(
 				.await;
 			let query_time = timer.elapsed();
 
-			Ok(RoomMessageEventContent::notice_markdown(format!(
+			Result::<_, Error>::Ok(RoomMessageEventContent::notice_markdown(format!(
 				"Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```"
 			)))
 		},
@@ -260,7 +257,7 @@ pub(super) async fn process(
 				.await;
 			let query_time = timer.elapsed();
 
-			Ok(RoomMessageEventContent::notice_markdown(format!(
+			Result::<_, Error>::Ok(RoomMessageEventContent::notice_markdown(format!(
 				"Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```"
 			)))
 		},
@@ -274,7 +271,7 @@ pub(super) async fn process(
 				.await;
 			let query_time = timer.elapsed();
 
-			Ok(RoomMessageEventContent::notice_markdown(format!(
+			Result::<_, Error>::Ok(RoomMessageEventContent::notice_markdown(format!(
 				"Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```"
 			)))
 		},
@@ -288,7 +285,7 @@ pub(super) async fn process(
 				.await;
 			let query_time = timer.elapsed();
 
-			Ok(RoomMessageEventContent::notice_markdown(format!(
+			Result::<_, Error>::Ok(RoomMessageEventContent::notice_markdown(format!(
 				"Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```"
 			)))
 		},
@@ -301,9 +298,13 @@ pub(super) async fn process(
 				.await;
 			let query_time = timer.elapsed();
 
-			Ok(RoomMessageEventContent::notice_markdown(format!(
+			Result::<_, Error>::Ok(RoomMessageEventContent::notice_markdown(format!(
 				"Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```"
 			)))
 		},
-	}
+	}?;
+
+	context.write_str(c.body()).await?;
+
+	Ok(())
 }

src/admin/query/room_timeline.rs

@@ -0,0 +1,61 @@
+use clap::Subcommand;
+use conduwuit::{utils::stream::TryTools, PduCount, Result};
+use futures::TryStreamExt;
+use ruma::{events::room::message::RoomMessageEventContent, OwnedRoomOrAliasId};
+
+use crate::{admin_command, admin_command_dispatch};
+
+#[admin_command_dispatch]
+#[derive(Debug, Subcommand)]
+/// Query tables from database
+pub(crate) enum RoomTimelineCommand {
+	Pdus {
+		room_id: OwnedRoomOrAliasId,
+
+		from: Option<String>,
+
+		#[arg(short, long)]
+		limit: Option<usize>,
+	},
+
+	Last {
+		room_id: OwnedRoomOrAliasId,
+	},
+}
+
+#[admin_command]
+pub(super) async fn last(&self, room_id: OwnedRoomOrAliasId) -> Result<RoomMessageEventContent> {
+	let room_id = self.services.rooms.alias.resolve(&room_id).await?;
+
+	let result = self
+		.services
+		.rooms
+		.timeline
+		.last_timeline_count(None, &room_id)
+		.await?;
+
+	Ok(RoomMessageEventContent::notice_markdown(format!("{result:#?}")))
+}
+
+#[admin_command]
+pub(super) async fn pdus(
+	&self,
+	room_id: OwnedRoomOrAliasId,
+	from: Option<String>,
+	limit: Option<usize>,
+) -> Result<RoomMessageEventContent> {
+	let room_id = self.services.rooms.alias.resolve(&room_id).await?;
+
+	let from: Option<PduCount> = from.as_deref().map(str::parse).transpose()?;
+
+	let result: Vec<_> = self
+		.services
+		.rooms
+		.timeline
+		.pdus_rev(None, &room_id, from)
+		.try_take(limit.unwrap_or(3))
+		.try_collect()
+		.await?;
+
+	Ok(RoomMessageEventContent::notice_markdown(format!("{result:#?}")))
+}

src/admin/query/sending.rs

@@ -62,7 +62,14 @@ pub(crate) enum SendingCommand {
 }
 
 /// All the getters and iterators in key_value/sending.rs
-pub(super) async fn process(
+pub(super) async fn process(subcommand: SendingCommand, context: &Command<'_>) -> Result {
+	let c = reprocess(subcommand, context).await?;
+	context.write_str(c.body()).await?;
+	Ok(())
+}
+
+/// All the getters and iterators in key_value/sending.rs
+pub(super) async fn reprocess(
 	subcommand: SendingCommand,
 	context: &Command<'_>,
 ) -> Result<RoomMessageEventContent> {

src/admin/query/short.rs

@@ -0,0 +1,45 @@
+use clap::Subcommand;
+use conduwuit::Result;
+use ruma::{events::room::message::RoomMessageEventContent, OwnedEventId, OwnedRoomOrAliasId};
+
+use crate::{admin_command, admin_command_dispatch};
+
+#[admin_command_dispatch]
+#[derive(Debug, Subcommand)]
+/// Query tables from database
+pub(crate) enum ShortCommand {
+	ShortEventId {
+		event_id: OwnedEventId,
+	},
+
+	ShortRoomId {
+		room_id: OwnedRoomOrAliasId,
+	},
+}
+
+#[admin_command]
+pub(super) async fn short_event_id(
+	&self,
+	event_id: OwnedEventId,
+) -> Result<RoomMessageEventContent> {
+	let shortid = self
+		.services
+		.rooms
+		.short
+		.get_shorteventid(&event_id)
+		.await?;
+
+	Ok(RoomMessageEventContent::notice_markdown(format!("{shortid:#?}")))
+}
+
+#[admin_command]
+pub(super) async fn short_room_id(
+	&self,
+	room_id: OwnedRoomOrAliasId,
+) -> Result<RoomMessageEventContent> {
+	let room_id = self.services.rooms.alias.resolve(&room_id).await?;
+
+	let shortid = self.services.rooms.short.get_shortroomid(&room_id).await?;
+
+	Ok(RoomMessageEventContent::notice_markdown(format!("{shortid:#?}")))
+}

src/admin/query/users.rs

@@ -15,6 +15,8 @@ pub(crate) enum UsersCommand {
 
 	IterUsers,
 
+	IterUsers2,
+
 	PasswordHash {
 		user_id: OwnedUserId,
 	},
@@ -89,6 +91,33 @@ pub(crate) enum UsersCommand {
 		room_id: OwnedRoomId,
 		session_id: String,
 	},
+
+	GetSharedRooms {
+		user_a: OwnedUserId,
+		user_b: OwnedUserId,
+	},
+}
+
+#[admin_command]
+async fn get_shared_rooms(
+	&self,
+	user_a: OwnedUserId,
+	user_b: OwnedUserId,
+) -> Result<RoomMessageEventContent> {
+	let timer = tokio::time::Instant::now();
+	let result: Vec<_> = self
+		.services
+		.rooms
+		.state_cache
+		.get_shared_rooms(&user_a, &user_b)
+		.map(ToOwned::to_owned)
+		.collect()
+		.await;
+	let query_time = timer.elapsed();
+
+	Ok(RoomMessageEventContent::notice_markdown(format!(
+		"Query completed in {query_time:?}:\n\n```rs\n{result:#?}\n```"
+	)))
 }
 
 #[admin_command]
@@ -207,6 +236,23 @@ async fn iter_users(&self) -> Result<RoomMessageEventContent> {
 	)))
 }
 
+#[admin_command]
+async fn iter_users2(&self) -> Result<RoomMessageEventContent> {
+	let timer = tokio::time::Instant::now();
+	let result: Vec<_> = self.services.users.stream().collect().await;
+	let result: Vec<_> = result
+		.into_iter()
+		.map(ruma::UserId::as_bytes)
+		.map(String::from_utf8_lossy)
+		.collect();
+
+	let query_time = timer.elapsed();
+
+	Ok(RoomMessageEventContent::notice_markdown(format!(
+		"Query completed in {query_time:?}:\n\n```rs\n{result:?}\n```"
+	)))
+}
+
 #[admin_command]
 async fn count_users(&self) -> Result<RoomMessageEventContent> {
 	let timer = tokio::time::Instant::now();

src/admin/room/alias.rs

@@ -44,7 +44,14 @@ pub(crate) enum RoomAliasCommand {
 	},
 }
 
-pub(super) async fn process(
+pub(super) async fn process(command: RoomAliasCommand, context: &Command<'_>) -> Result {
+	let c = reprocess(command, context).await?;
+	context.write_str(c.body()).await?;
+
+	Ok(())
+}
+
+pub(super) async fn reprocess(
 	command: RoomAliasCommand,
 	context: &Command<'_>,
 ) -> Result<RoomMessageEventContent> {

src/admin/room/directory.rs

@@ -25,7 +25,13 @@ pub(crate) enum RoomDirectoryCommand {
 	},
 }
 
-pub(super) async fn process(
+pub(super) async fn process(command: RoomDirectoryCommand, context: &Command<'_>) -> Result {
+	let c = reprocess(command, context).await?;
+	context.write_str(c.body()).await?;
+	Ok(())
+}
+
+pub(super) async fn reprocess(
 	command: RoomDirectoryCommand,
 	context: &Command<'_>,
 ) -> Result<RoomMessageEventContent> {

src/admin/server/commands.rs

@@ -1,4 +1,4 @@
-use std::{fmt::Write, sync::Arc};
+use std::{fmt::Write, path::PathBuf, sync::Arc};
 
 use conduwuit::{info, utils::time, warn, Err, Result};
 use ruma::events::room::message::RoomMessageEventContent;
@@ -22,11 +22,22 @@ pub(super) async fn uptime(&self) -> Result<RoomMessageEventContent> {
 pub(super) async fn show_config(&self) -> Result<RoomMessageEventContent> {
 	// Construct and send the response
 	Ok(RoomMessageEventContent::text_markdown(format!(
-		"```\n{}\n```",
-		self.services.globals.config
+		"{}",
+		*self.services.server.config
 	)))
 }
 
+#[admin_command]
+pub(super) async fn reload_config(
+	&self,
+	path: Option<PathBuf>,
+) -> Result<RoomMessageEventContent> {
+	let path = path.as_deref().into_iter();
+	self.services.config.reload(path)?;
+
+	Ok(RoomMessageEventContent::text_plain("Successfully reconfigured."))
+}
+
 #[admin_command]
 pub(super) async fn list_features(
 	&self,

src/admin/server/mod.rs

@@ -1,5 +1,7 @@
 mod commands;
 
+use std::path::PathBuf;
+
 use clap::Subcommand;
 use conduwuit::Result;
 
@@ -14,6 +16,11 @@ pub(super) enum ServerCommand {
 	/// - Show configuration values
 	ShowConfig,
 
+	/// - Reload configuration values
+	ReloadConfig {
+		path: Option<PathBuf>,
+	},
+
 	/// - List the features built into the server
 	ListFeatures {
 		#[arg(short, long)]

src/admin/user/commands.rs

@@ -31,19 +31,21 @@ const BULK_JOIN_REASON: &str = "Bulk force joining this room as initiated by the
 
 #[admin_command]
 pub(super) async fn list_users(&self) -> Result<RoomMessageEventContent> {
-	let users = self
+	let users: Vec<_> = self
 		.services
 		.users
 		.list_local_users()
 		.map(ToString::to_string)
-		.collect::<Vec<_>>()
+		.collect()
 		.await;
 
 	let mut plain_msg = format!("Found {} local user account(s):\n```\n", users.len());
 	plain_msg += users.join("\n").as_str();
 	plain_msg += "\n```";
 
-	Ok(RoomMessageEventContent::notice_markdown(plain_msg))
+	self.write_str(plain_msg.as_str()).await?;
+
+	Ok(RoomMessageEventContent::text_plain(""))
 }
 
 #[admin_command]
@@ -81,12 +83,12 @@ pub(super) async fn create_user(
 	// content is set to the user's display name with a space before it
 	if !self
 		.services
-		.globals
+		.server
 		.config
 		.new_user_displayname_suffix
 		.is_empty()
 	{
-		write!(displayname, " {}", self.services.globals.config.new_user_displayname_suffix)
+		write!(displayname, " {}", self.services.server.config.new_user_displayname_suffix)
 			.expect("should be able to write to string buffer");
 	}
 
@@ -112,8 +114,8 @@ pub(super) async fn create_user(
 		)
 		.await?;
 
-	if !self.services.globals.config.auto_join_rooms.is_empty() {
-		for room in &self.services.globals.config.auto_join_rooms {
+	if !self.services.server.config.auto_join_rooms.is_empty() {
+		for room in &self.services.server.config.auto_join_rooms {
 			let Ok(room_id) = self.services.rooms.alias.resolve(room).await else {
 				error!(%user_id, "Failed to resolve room alias to room ID when attempting to auto join {room}, skipping");
 				continue;
@@ -912,29 +914,30 @@ pub(super) async fn redact_event(
 		self.services.globals.server_name()
 	);
 
-	let state_lock = self.services.rooms.state.mutex.lock(&room_id).await;
+	let redaction_event_id = {
+		let state_lock = self.services.rooms.state.mutex.lock(&room_id).await;
 
-	let redaction_event_id = self
-		.services
-		.rooms
-		.timeline
-		.build_and_append_pdu(
-			PduBuilder {
-				redacts: Some(event.event_id.clone()),
-				..PduBuilder::timeline(&RoomRedactionEventContent {
+		self.services
+			.rooms
+			.timeline
+			.build_and_append_pdu(
+				PduBuilder {
 					redacts: Some(event.event_id.clone()),
-					reason: Some(reason),
-				})
-			},
-			&sender_user,
-			&room_id,
-			&state_lock,
-		)
-		.await?;
+					..PduBuilder::timeline(&RoomRedactionEventContent {
+						redacts: Some(event.event_id.clone()),
+						reason: Some(reason),
+					})
+				},
+				&sender_user,
+				&room_id,
+				&state_lock,
+			)
+			.await?
+	};
 
-	drop(state_lock);
+	let out = format!("Successfully redacted event. Redaction event ID: {redaction_event_id}");
 
-	Ok(RoomMessageEventContent::text_plain(format!(
-		"Successfully redacted event. Redaction event ID: {redaction_event_id}"
-	)))
+	self.write_str(out.as_str()).await?;
+
+	Ok(RoomMessageEventContent::text_plain(""))
 }

src/api/Cargo.toml

@@ -50,7 +50,7 @@ http.workspace = true
 http-body-util.workspace = true
 hyper.workspace = true
 ipaddress.workspace = true
-jsonwebtoken.workspace = true
+itertools.workspace = true
 log.workspace = true
 rand.workspace = true
 reqwest.workspace = true

src/api/client/account.rs

@@ -299,7 +299,7 @@ pub(crate) async fn register_route(
 	if !services.globals.new_user_displayname_suffix().is_empty()
 		&& body.appservice_info.is_none()
 	{
-		write!(displayname, " {}", services.globals.config.new_user_displayname_suffix)
+		write!(displayname, " {}", services.server.config.new_user_displayname_suffix)
 			.expect("should be able to write to string buffer");
 	}
 
@@ -365,7 +365,7 @@ pub(crate) async fn register_route(
 				 \"{device_display_name}\""
 			);
 
-			if services.globals.config.admin_room_notices {
+			if services.server.config.admin_room_notices {
 				services
 					.admin
 					.send_message(RoomMessageEventContent::notice_plain(format!(
@@ -378,7 +378,7 @@ pub(crate) async fn register_route(
 		} else {
 			info!("New user \"{user_id}\" registered on this server.");
 
-			if services.globals.config.admin_room_notices {
+			if services.server.config.admin_room_notices {
 				services
 					.admin
 					.send_message(RoomMessageEventContent::notice_plain(format!(
@@ -395,7 +395,7 @@ pub(crate) async fn register_route(
 		info!("New guest user \"{user_id}\" registered on this server.");
 
 		if !device_display_name.is_empty() {
-			if services.globals.config.admin_room_notices {
+			if services.server.config.admin_room_notices {
 				services
 					.admin
 					.send_message(RoomMessageEventContent::notice_plain(format!(
@@ -407,7 +407,7 @@ pub(crate) async fn register_route(
 			}
 		} else {
 			#[allow(clippy::collapsible_else_if)]
-			if services.globals.config.admin_room_notices {
+			if services.server.config.admin_room_notices {
 				services
 					.admin
 					.send_message(RoomMessageEventContent::notice_plain(format!(
@@ -438,10 +438,10 @@ pub(crate) async fn register_route(
 	}
 
 	if body.appservice_info.is_none()
-		&& !services.globals.config.auto_join_rooms.is_empty()
+		&& !services.server.config.auto_join_rooms.is_empty()
 		&& (services.globals.allow_guests_auto_join_rooms() || !is_guest)
 	{
-		for room in &services.globals.config.auto_join_rooms {
+		for room in &services.server.config.auto_join_rooms {
 			let Ok(room_id) = services.rooms.alias.resolve(room).await else {
 				error!(
 					"Failed to resolve room alias to room ID when attempting to auto join \
@@ -570,7 +570,7 @@ pub(crate) async fn change_password_route(
 
 	info!("User {sender_user} changed their password.");
 
-	if services.globals.config.admin_room_notices {
+	if services.server.config.admin_room_notices {
 		services
 			.admin
 			.send_message(RoomMessageEventContent::notice_plain(format!(
@@ -673,7 +673,7 @@ pub(crate) async fn deactivate_route(
 
 	info!("User {sender_user} deactivated their account.");
 
-	if services.globals.config.admin_room_notices {
+	if services.server.config.admin_room_notices {
 		services
 			.admin
 			.send_message(RoomMessageEventContent::notice_plain(format!(

src/api/client/capabilities.rs

@@ -1,6 +1,7 @@
 use std::collections::BTreeMap;
 
 use axum::extract::State;
+use conduwuit::{Result, Server};
 use ruma::{
 	api::client::discovery::get_capabilities::{
 		self, Capabilities, GetLoginTokenCapability, RoomVersionStability,
@@ -10,7 +11,7 @@ use ruma::{
 };
 use serde_json::json;
 
-use crate::{Result, Ruma};
+use crate::Ruma;
 
 /// # `GET /_matrix/client/v3/capabilities`
 ///
@@ -21,7 +22,7 @@ pub(crate) async fn get_capabilities_route(
 	_body: Ruma<get_capabilities::v3::Request>,
 ) -> Result<get_capabilities::v3::Response> {
 	let available: BTreeMap<RoomVersionId, RoomVersionStability> =
-		services.server.available_room_versions().collect();
+		Server::available_room_versions().collect();
 
 	let mut capabilities = Capabilities::default();
 	capabilities.room_versions = RoomVersionsCapability {
@@ -32,8 +33,9 @@ pub(crate) async fn get_capabilities_route(
 	// we do not implement 3PID stuff
 	capabilities.thirdparty_id_changes = ThirdPartyIdChangesCapability { enabled: false };
 
-	// we dont support generating tokens yet
-	capabilities.get_login_token = GetLoginTokenCapability { enabled: false };
+	capabilities.get_login_token = GetLoginTokenCapability {
+		enabled: services.server.config.login_via_existing_session,
+	};
 
 	// MSC4133 capability
 	capabilities

src/api/client/context.rs

@@ -1,24 +1,22 @@
-use std::iter::once;
-
 use axum::extract::State;
 use conduwuit::{
 	at, err, ref_at,
 	utils::{
 		future::TryExtExt,
-		stream::{BroadbandExt, ReadyExt, WidebandExt},
+		stream::{BroadbandExt, ReadyExt, TryIgnore, WidebandExt},
 		IterStream,
 	},
-	Err, Result,
+	Err, PduEvent, Result,
 };
-use futures::{join, try_join, FutureExt, StreamExt, TryFutureExt};
-use ruma::{
-	api::client::{context::get_context, filter::LazyLoadOptions},
-	events::StateEventType,
-	OwnedEventId, UserId,
+use futures::{
+	future::{join, join3, try_join3, OptionFuture},
+	FutureExt, StreamExt, TryFutureExt, TryStreamExt,
 };
+use ruma::{api::client::context::get_context, events::StateEventType, OwnedEventId, UserId};
+use service::rooms::{lazy_loading, lazy_loading::Options, short::ShortStateKey};
 
 use crate::{
-	client::message::{event_filter, ignored_filter, update_lazy, visibility_filter, LazySet},
+	client::message::{event_filter, ignored_filter, lazy_loading_witness, visibility_filter},
 	Ruma,
 };
 
@@ -35,10 +33,10 @@ pub(crate) async fn get_context_route(
 	State(services): State<crate::State>,
 	body: Ruma<get_context::v3::Request>,
 ) -> Result<get_context::v3::Response> {
-	let filter = &body.filter;
 	let sender = body.sender();
-	let (sender_user, _) = sender;
+	let (sender_user, sender_device) = sender;
 	let room_id = &body.room_id;
+	let filter = &body.filter;
 
 	// Use limit or else 10, with maximum 100
 	let limit: usize = body
@@ -47,25 +45,13 @@ pub(crate) async fn get_context_route(
 		.unwrap_or(LIMIT_DEFAULT)
 		.min(LIMIT_MAX);
 
-	// some clients, at least element, seem to require knowledge of redundant
-	// members for "inline" profiles on the timeline to work properly
-	let lazy_load_enabled = matches!(filter.lazy_load_options, LazyLoadOptions::Enabled { .. });
-
-	let lazy_load_redundant = if let LazyLoadOptions::Enabled { include_redundant_members } =
-		filter.lazy_load_options
-	{
-		include_redundant_members
-	} else {
-		false
-	};
-
-	let base_token = services
+	let base_id = services
 		.rooms
 		.timeline
-		.get_pdu_count(&body.event_id)
+		.get_pdu_id(&body.event_id)
 		.map_err(|_| err!(Request(NotFound("Event not found."))));
 
-	let base_event = services
+	let base_pdu = services
 		.rooms
 		.timeline
 		.get_pdu(&body.event_id)
@@ -77,117 +63,133 @@ pub(crate) async fn get_context_route(
 		.user_can_see_event(sender_user, &body.room_id, &body.event_id)
 		.map(Ok);
 
-	let (base_token, base_event, visible) = try_join!(base_token, base_event, visible)?;
+	let (base_id, base_pdu, visible) = try_join3(base_id, base_pdu, visible).await?;
 
-	if base_event.room_id != body.room_id || base_event.event_id != body.event_id {
+	if base_pdu.room_id != body.room_id || base_pdu.event_id != body.event_id {
 		return Err!(Request(NotFound("Base event not found.")));
 	}
 
-	if !visible
-		|| ignored_filter(&services, (base_token, base_event.clone()), sender_user)
-			.await
-			.is_none()
-	{
+	if !visible {
 		return Err!(Request(Forbidden("You don't have permission to view this event.")));
 	}
 
-	let events_before =
-		services
-			.rooms
-			.timeline
-			.pdus_rev(Some(sender_user), room_id, Some(base_token));
+	let base_count = base_id.pdu_count();
+
+	let base_event = ignored_filter(&services, (base_count, base_pdu), sender_user);
+
+	let events_before = services
+		.rooms
+		.timeline
+		.pdus_rev(Some(sender_user), room_id, Some(base_count))
+		.ignore_err()
+		.ready_filter_map(|item| event_filter(item, filter))
+		.wide_filter_map(|item| ignored_filter(&services, item, sender_user))
+		.wide_filter_map(|item| visibility_filter(&services, item, sender_user))
+		.take(limit / 2)
+		.collect();
 
 	let events_after = services
 		.rooms
 		.timeline
-		.pdus(Some(sender_user), room_id, Some(base_token));
-
-	let (events_before, events_after) = try_join!(events_before, events_after)?;
-
-	let events_before = events_before
+		.pdus(Some(sender_user), room_id, Some(base_count))
+		.ignore_err()
 		.ready_filter_map(|item| event_filter(item, filter))
 		.wide_filter_map(|item| ignored_filter(&services, item, sender_user))
 		.wide_filter_map(|item| visibility_filter(&services, item, sender_user))
 		.take(limit / 2)
 		.collect();
 
-	let events_after = events_after
-		.ready_filter_map(|item| event_filter(item, filter))
-		.wide_filter_map(|item| ignored_filter(&services, item, sender_user))
-		.wide_filter_map(|item| visibility_filter(&services, item, sender_user))
-		.take(limit / 2)
-		.collect();
+	let (base_event, events_before, events_after): (_, Vec<_>, Vec<_>) =
+		join3(base_event, events_before, events_after).await;
 
-	let (events_before, events_after): (Vec<_>, Vec<_>) = join!(events_before, events_after);
+	let lazy_loading_context = lazy_loading::Context {
+		user_id: sender_user,
+		device_id: sender_device,
+		room_id,
+		token: Some(base_count.into_unsigned()),
+		options: Some(&filter.lazy_load_options),
+	};
+
+	let lazy_loading_witnessed: OptionFuture<_> = filter
+		.lazy_load_options
+		.is_enabled()
+		.then_some(
+			base_event
+				.iter()
+				.chain(events_before.iter())
+				.chain(events_after.iter()),
+		)
+		.map(|witnessed| lazy_loading_witness(&services, &lazy_loading_context, witnessed))
+		.into();
 
 	let state_at = events_after
 		.last()
 		.map(ref_at!(1))
-		.map_or(body.event_id.as_ref(), |e| e.event_id.as_ref());
+		.map_or(body.event_id.as_ref(), |pdu| pdu.event_id.as_ref());
 
 	let state_ids = services
 		.rooms
 		.state_accessor
 		.pdu_shortstatehash(state_at)
 		.or_else(|_| services.rooms.state.get_room_shortstatehash(room_id))
-		.and_then(|shortstatehash| services.rooms.state_accessor.state_full_ids(shortstatehash))
-		.map_err(|e| err!(Database("State not found: {e}")))
-		.await?;
-
-	let lazy = once(&(base_token, base_event.clone()))
-		.chain(events_before.iter())
-		.chain(events_after.iter())
-		.stream()
-		.fold(LazySet::new(), |lazy, item| {
-			update_lazy(&services, room_id, sender, lazy, item, lazy_load_redundant)
-		})
-		.await;
-
-	let lazy = &lazy;
-	let state: Vec<_> = state_ids
-		.iter()
-		.stream()
-		.broad_filter_map(|(shortstatekey, event_id)| {
+		.map_ok(|shortstatehash| {
 			services
 				.rooms
-				.short
-				.get_statekey_from_short(*shortstatekey)
-				.map_ok(move |(event_type, state_key)| (event_type, state_key, event_id))
-				.ok()
+				.state_accessor
+				.state_full_ids(shortstatehash)
+				.map(Ok)
 		})
-		.ready_filter_map(|(event_type, state_key, event_id)| {
-			if !lazy_load_enabled || event_type != StateEventType::RoomMember {
-				return Some(event_id);
+		.map_err(|e| err!(Database("State not found: {e}")))
+		.try_flatten_stream()
+		.try_collect()
+		.boxed();
+
+	let (lazy_loading_witnessed, state_ids) = join(lazy_loading_witnessed, state_ids).await;
+
+	let state_ids: Vec<(ShortStateKey, OwnedEventId)> = state_ids?;
+	let shortstatekeys = state_ids.iter().map(at!(0)).stream();
+	let shorteventids = state_ids.iter().map(ref_at!(1)).stream();
+	let lazy_loading_witnessed = lazy_loading_witnessed.unwrap_or_default();
+	let state: Vec<_> = services
+		.rooms
+		.short
+		.multi_get_statekey_from_short(shortstatekeys)
+		.zip(shorteventids)
+		.ready_filter_map(|item| Some((item.0.ok()?, item.1)))
+		.ready_filter_map(|((event_type, state_key), event_id)| {
+			if filter.lazy_load_options.is_enabled()
+				&& event_type == StateEventType::RoomMember
+				&& state_key
+					.as_str()
+					.try_into()
+					.is_ok_and(|user_id: &UserId| !lazy_loading_witnessed.contains(user_id))
+			{
+				return None;
 			}
 
-			state_key
-				.as_str()
-				.try_into()
-				.ok()
-				.filter(|&user_id: &&UserId| lazy.contains(user_id))
-				.map(|_| event_id)
+			Some(event_id)
 		})
 		.broad_filter_map(|event_id: &OwnedEventId| {
-			services.rooms.timeline.get_pdu(event_id).ok()
+			services.rooms.timeline.get_pdu(event_id.as_ref()).ok()
 		})
-		.map(|pdu| pdu.to_state_event())
+		.map(PduEvent::into_state_event)
 		.collect()
 		.await;
 
 	Ok(get_context::v3::Response {
-		event: Some(base_event.to_room_event()),
+		event: base_event.map(at!(1)).as_ref().map(PduEvent::to_room_event),
 
 		start: events_before
 			.last()
 			.map(at!(0))
-			.or(Some(base_token))
+			.or(Some(base_count))
 			.as_ref()
 			.map(ToString::to_string),
 
 		end: events_after
 			.last()
 			.map(at!(0))
-			.or(Some(base_token))
+			.or(Some(base_count))
 			.as_ref()
 			.map(ToString::to_string),
 

src/api/client/directory.rs

@@ -152,7 +152,7 @@ pub(crate) async fn set_room_visibility_route(
 
 	match &body.visibility {
 		| room::Visibility::Public => {
-			if services.globals.config.lockdown_public_room_directory
+			if services.server.config.lockdown_public_room_directory
 				&& !services.users.is_admin(sender_user).await
 				&& body.appservice_info.is_none()
 			{
@@ -162,7 +162,7 @@ pub(crate) async fn set_room_visibility_route(
 					body.room_id
 				);
 
-				if services.globals.config.admin_room_notices {
+				if services.server.config.admin_room_notices {
 					services
 						.admin
 						.send_text(&format!(
@@ -181,7 +181,7 @@ pub(crate) async fn set_room_visibility_route(
 
 			services.rooms.directory.set_public(&body.room_id);
 
-			if services.globals.config.admin_room_notices {
+			if services.server.config.admin_room_notices {
 				services
 					.admin
 					.send_text(&format!(

src/api/client/media.rs

@@ -31,7 +31,7 @@ pub(crate) async fn get_media_config_route(
 	_body: Ruma<get_media_config::v1::Request>,
 ) -> Result<get_media_config::v1::Response> {
 	Ok(get_media_config::v1::Response {
-		upload_size: ruma_from_usize(services.globals.config.max_request_size),
+		upload_size: ruma_from_usize(services.server.config.max_request_size),
 	})
 }
 

src/api/client/media_legacy.rs

@@ -27,7 +27,7 @@ pub(crate) async fn get_media_config_legacy_route(
 	_body: Ruma<get_media_config::v3::Request>,
 ) -> Result<get_media_config::v3::Response> {
 	Ok(get_media_config::v3::Response {
-		upload_size: ruma_from_usize(services.globals.config.max_request_size),
+		upload_size: ruma_from_usize(services.server.config.max_request_size),
 	})
 }
 
@@ -50,7 +50,7 @@ pub(crate) async fn get_media_config_legacy_legacy_route(
 /// # `GET /_matrix/media/v3/preview_url`
 ///
 /// Returns URL preview.
-#[tracing::instrument(skip_all, fields(%client), name = "url_preview_legacy")]
+#[tracing::instrument(skip_all, fields(%client), name = "url_preview_legacy", level = "debug")]
 pub(crate) async fn get_media_preview_legacy_route(
 	State(services): State<crate::State>,
 	InsecureClientIp(client): InsecureClientIp,
@@ -131,7 +131,7 @@ pub(crate) async fn create_content_legacy_route(
 /// - Only redirects if `allow_redirect` is true
 /// - Uses client-provided `timeout_ms` if available, else defaults to 20
 ///   seconds
-#[tracing::instrument(skip_all, fields(%client), name = "media_get_legacy")]
+#[tracing::instrument(skip_all, fields(%client), name = "media_get_legacy", level = "debug")]
 pub(crate) async fn get_content_legacy_route(
 	State(services): State<crate::State>,
 	InsecureClientIp(client): InsecureClientIp,
@@ -197,7 +197,7 @@ pub(crate) async fn get_content_legacy_route(
 /// - Only redirects if `allow_redirect` is true
 /// - Uses client-provided `timeout_ms` if available, else defaults to 20
 ///   seconds
-#[tracing::instrument(skip_all, fields(%client), name = "media_get_legacy")]
+#[tracing::instrument(skip_all, fields(%client), name = "media_get_legacy", level = "debug")]
 pub(crate) async fn get_content_legacy_legacy_route(
 	State(services): State<crate::State>,
 	InsecureClientIp(client): InsecureClientIp,
@@ -216,7 +216,7 @@ pub(crate) async fn get_content_legacy_legacy_route(
 /// - Only redirects if `allow_redirect` is true
 /// - Uses client-provided `timeout_ms` if available, else defaults to 20
 ///   seconds
-#[tracing::instrument(skip_all, fields(%client), name = "media_get_legacy")]
+#[tracing::instrument(skip_all, fields(%client), name = "media_get_legacy", level = "debug")]
 pub(crate) async fn get_content_as_filename_legacy_route(
 	State(services): State<crate::State>,
 	InsecureClientIp(client): InsecureClientIp,
@@ -303,7 +303,7 @@ pub(crate) async fn get_content_as_filename_legacy_legacy_route(
 /// - Only redirects if `allow_redirect` is true
 /// - Uses client-provided `timeout_ms` if available, else defaults to 20
 ///   seconds
-#[tracing::instrument(skip_all, fields(%client), name = "media_thumbnail_get_legacy")]
+#[tracing::instrument(skip_all, fields(%client), name = "media_thumbnail_get_legacy", level = "debug")]
 pub(crate) async fn get_content_thumbnail_legacy_route(
 	State(services): State<crate::State>,
 	InsecureClientIp(client): InsecureClientIp,

src/api/client/membership.rs

@@ -1,4 +1,5 @@
 use std::{
+	borrow::Borrow,
 	collections::{BTreeMap, HashMap, HashSet},
 	net::IpAddr,
 	sync::Arc,
@@ -7,18 +8,19 @@ use std::{
 use axum::extract::State;
 use axum_client_ip::InsecureClientIp;
 use conduwuit::{
-	debug, debug_info, debug_warn, err, info,
-	pdu::{self, gen_event_id_canonical_json, PduBuilder},
+	at, debug, debug_info, debug_warn, err, info,
+	pdu::{gen_event_id_canonical_json, PduBuilder},
 	result::FlatOk,
 	trace,
 	utils::{self, shuffle, IterStream, ReadyExt},
 	warn, Err, PduEvent, Result,
 };
-use futures::{join, FutureExt, StreamExt};
+use futures::{join, FutureExt, StreamExt, TryFutureExt};
 use ruma::{
 	api::{
 		client::{
 			error::ErrorKind,
+			knock::knock_room,
 			membership::{
 				ban_user, forget_room, get_member_events, invite_user, join_room_by_id,
 				join_room_by_id_or_alias,
@@ -37,11 +39,12 @@ use ruma::{
 		},
 		StateEventType,
 	},
-	state_res, CanonicalJsonObject, CanonicalJsonValue, OwnedRoomId, OwnedServerName,
-	OwnedUserId, RoomId, RoomVersionId, ServerName, UserId,
+	state_res, CanonicalJsonObject, CanonicalJsonValue, OwnedEventId, OwnedRoomId,
+	OwnedServerName, OwnedUserId, RoomId, RoomVersionId, ServerName, UserId,
 };
 use service::{
 	appservice::RegistrationInfo,
+	pdu::gen_event_id,
 	rooms::{state::RoomMutexGuard, state_compressor::HashSetCompressStateEvent},
 	Services,
 };
@@ -68,7 +71,7 @@ async fn banned_room_check(
 	if let Some(room_id) = room_id {
 		if services.rooms.metadata.is_banned(room_id).await
 			|| services
-				.globals
+				.server
 				.config
 				.forbidden_remote_server_names
 				.contains(&room_id.server_name().unwrap().to_owned())
@@ -78,12 +81,12 @@ async fn banned_room_check(
 				 attempted to join a banned room or banned room server name: {room_id}"
 			);
 
-			if services.globals.config.auto_deactivate_banned_room_attempts {
+			if services.server.config.auto_deactivate_banned_room_attempts {
 				warn!(
 					"Automatically deactivating user {user_id} due to attempted banned room join"
 				);
 
-				if services.globals.config.admin_room_notices {
+				if services.server.config.admin_room_notices {
 					services
 						.admin
 						.send_message(RoomMessageEventContent::text_plain(format!(
@@ -109,7 +112,7 @@ async fn banned_room_check(
 		}
 	} else if let Some(server_name) = server_name {
 		if services
-			.globals
+			.server
 			.config
 			.forbidden_remote_server_names
 			.contains(&server_name.to_owned())
@@ -119,12 +122,12 @@ async fn banned_room_check(
 				 name {server_name} that is globally forbidden. Rejecting.",
 			);
 
-			if services.globals.config.auto_deactivate_banned_room_attempts {
+			if services.server.config.auto_deactivate_banned_room_attempts {
 				warn!(
 					"Automatically deactivating user {user_id} due to attempted banned room join"
 				);
 
-				if services.globals.config.admin_room_notices {
+				if services.server.config.admin_room_notices {
 					services
 						.admin
 						.send_message(RoomMessageEventContent::text_plain(format!(
@@ -348,6 +351,116 @@ pub(crate) async fn join_room_by_id_or_alias_route(
 	Ok(join_room_by_id_or_alias::v3::Response { room_id: join_room_response.room_id })
 }
 
+/// # `POST /_matrix/client/*/knock/{roomIdOrAlias}`
+///
+/// Tries to knock the room to ask permission to join for the sender user.
+#[tracing::instrument(skip_all, fields(%client), name = "knock")]
+pub(crate) async fn knock_room_route(
+	State(services): State<crate::State>,
+	InsecureClientIp(client): InsecureClientIp,
+	body: Ruma<knock_room::v3::Request>,
+) -> Result<knock_room::v3::Response> {
+	let sender_user = body.sender_user.as_ref().expect("user is authenticated");
+	let body = body.body;
+
+	let (servers, room_id) = match OwnedRoomId::try_from(body.room_id_or_alias) {
+		| Ok(room_id) => {
+			banned_room_check(
+				&services,
+				sender_user,
+				Some(&room_id),
+				room_id.server_name(),
+				client,
+			)
+			.await?;
+
+			let mut servers = body.via.clone();
+			servers.extend(
+				services
+					.rooms
+					.state_cache
+					.servers_invite_via(&room_id)
+					.map(ToOwned::to_owned)
+					.collect::<Vec<_>>()
+					.await,
+			);
+
+			servers.extend(
+				services
+					.rooms
+					.state_cache
+					.invite_state(sender_user, &room_id)
+					.await
+					.unwrap_or_default()
+					.iter()
+					.filter_map(|event| event.get_field("sender").ok().flatten())
+					.filter_map(|sender: &str| UserId::parse(sender).ok())
+					.map(|user| user.server_name().to_owned()),
+			);
+
+			if let Some(server) = room_id.server_name() {
+				servers.push(server.to_owned());
+			}
+
+			servers.sort_unstable();
+			servers.dedup();
+			shuffle(&mut servers);
+
+			(servers, room_id)
+		},
+		| Err(room_alias) => {
+			let (room_id, mut servers) = services
+				.rooms
+				.alias
+				.resolve_alias(&room_alias, Some(body.via.clone()))
+				.await?;
+
+			banned_room_check(
+				&services,
+				sender_user,
+				Some(&room_id),
+				Some(room_alias.server_name()),
+				client,
+			)
+			.await?;
+
+			let addl_via_servers = services
+				.rooms
+				.state_cache
+				.servers_invite_via(&room_id)
+				.map(ToOwned::to_owned);
+
+			let addl_state_servers = services
+				.rooms
+				.state_cache
+				.invite_state(sender_user, &room_id)
+				.await
+				.unwrap_or_default();
+
+			let mut addl_servers: Vec<_> = addl_state_servers
+				.iter()
+				.map(|event| event.get_field("sender"))
+				.filter_map(FlatOk::flat_ok)
+				.map(|user: &UserId| user.server_name().to_owned())
+				.stream()
+				.chain(addl_via_servers)
+				.collect()
+				.await;
+
+			addl_servers.sort_unstable();
+			addl_servers.dedup();
+			shuffle(&mut addl_servers);
+			servers.append(&mut addl_servers);
+
+			(servers, room_id)
+		},
+	};
+
+	knock_room_by_id_helper(&services, sender_user, &room_id, body.reason.clone(), &servers)
+		.boxed()
+		.await
+}
+
 /// # `POST /_matrix/client/v3/rooms/{roomId}/leave`
 ///
 /// Tries to leave the sender user from a room.
@@ -403,6 +516,17 @@ pub(crate) async fn invite_user_route(
 			)));
 		}
 
+		if let Ok(target_user_membership) = services
+			.rooms
+			.state_accessor
+			.get_member(&body.room_id, user_id)
+			.await
+		{
+			if target_user_membership.membership == MembershipState::Ban {
+				return Err!(Request(Forbidden("User is banned from this room.")));
+			}
+		}
+
 		if recipient_ignored_by_sender {
 			// silently drop the invite to the recipient if they've been ignored by the
 			// sender, pretend it worked
@@ -439,6 +563,16 @@ pub(crate) async fn kick_user_route(
 		return Ok(kick_user::v3::Response::new());
 	};
 
+	if !matches!(
+		event.membership,
+		MembershipState::Invite | MembershipState::Knock | MembershipState::Join,
+	) {
+		return Err!(Request(Forbidden(
+			"Cannot kick a user who is not apart of the room (current membership: {})",
+			event.membership
+		)));
+	}
+
 	services
 		.rooms
 		.timeline
@@ -527,7 +661,7 @@ pub(crate) async fn unban_user_route(
 
 	if current_member_content.membership != MembershipState::Ban {
 		return Err!(Request(Forbidden(
-			"Cannot ban a user who is not banned (current membership: {})",
+			"Cannot unban a user who is not banned (current membership: {})",
 			current_member_content.membership
 		)));
 	}
@@ -631,11 +765,12 @@ pub(crate) async fn get_member_events_route(
 			.rooms
 			.state_accessor
 			.room_state_full(&body.room_id)
-			.await?
-			.iter()
-			.filter(|(key, _)| key.0 == StateEventType::RoomMember)
-			.map(|(_, pdu)| pdu.to_member_event())
-			.collect(),
+			.ready_filter_map(Result::ok)
+			.ready_filter(|((ty, _), _)| *ty == StateEventType::RoomMember)
+			.map(at!(1))
+			.map(PduEvent::into_member_event)
+			.collect()
+			.await,
 	})
 }
 
@@ -852,7 +987,7 @@ async fn join_room_by_id_helper_remote(
 		.hash_and_sign_event(&mut join_event_stub, &room_version_id)?;
 
 	// Generate event id
-	let event_id = pdu::gen_event_id(&join_event_stub, &room_version_id)?;
+	let event_id = gen_event_id(&join_event_stub, &room_version_id)?;
 
 	// Add event_id back
 	join_event_stub
@@ -1020,7 +1155,7 @@ async fn join_room_by_id_helper_remote(
 	};
 
 	let auth_check = state_res::event_auth::auth_check(
-		&state_res::RoomVersion::new(&room_version_id).expect("room version is supported"),
+		&state_res::RoomVersion::new(&room_version_id)?,
 		&parsed_join_pdu,
 		None, // TODO: third party invite
 		|k, s| state_fetch(k, s.to_owned()),
@@ -1033,10 +1168,10 @@ async fn join_room_by_id_helper_remote(
 	}
 
 	info!("Compressing state from send_join");
-	let compressed = state
-		.iter()
-		.stream()
-		.then(|(&k, id)| services.rooms.state_compressor.compress_state_event(k, id))
+	let compressed: HashSet<_> = services
+		.rooms
+		.state_compressor
+		.compress_state_events(state.iter().map(|(ssk, eid)| (ssk, eid.borrow())))
 		.collect()
 		.await;
 
@@ -1272,7 +1407,7 @@ async fn join_room_by_id_helper_local(
 		.hash_and_sign_event(&mut join_event_stub, &room_version_id)?;
 
 	// Generate event id
-	let event_id = pdu::gen_event_id(&join_event_stub, &room_version_id)?;
+	let event_id = gen_event_id(&join_event_stub, &room_version_id)?;
 
 	// Add event_id back
 	join_event_stub
@@ -1314,6 +1449,7 @@ async fn join_room_by_id_helper_local(
 			.rooms
 			.event_handler
 			.handle_incoming_pdu(&remote_server, room_id, &signed_event_id, signed_value, true)
+			.boxed()
 			.await?;
 	} else {
 		return Err(error);
@@ -1381,6 +1517,7 @@ async fn make_join_request(
 				);
 				make_join_response_and_server =
 					Err!(BadServerResponse("No server available to assist in joining."));
+
 				return make_join_response_and_server;
 			}
 		}
@@ -1491,6 +1628,7 @@ pub(crate) async fn invite_helper(
 			.rooms
 			.event_handler
 			.handle_incoming_pdu(&origin, room_id, &event_id, value, true)
+			.boxed()
 			.await?
 			.ok_or_else(|| {
 				err!(Request(InvalidParam("Could not accept incoming PDU as timeline event.")))
@@ -1557,7 +1695,7 @@ pub async fn leave_all_rooms(services: &Services, user_id: &UserId) {
 	for room_id in all_rooms {
 		// ignore errors
 		if let Err(e) = leave_room(services, user_id, &room_id, None).await {
-			warn!(%room_id, %user_id, %e, "Failed to leave room");
+			warn!(%user_id, "Failed to leave {room_id} remotely: {e}");
 		}
 
 		services.rooms.state_cache.forget(&room_id, user_id);
@@ -1570,14 +1708,15 @@ pub async fn leave_room(
 	room_id: &RoomId,
 	reason: Option<String>,
 ) -> Result<()> {
-	//use conduwuit::utils::stream::OptionStream;
-	use futures::TryFutureExt;
-
-	// Ask a remote server if we don't have this room
+	// Ask a remote server if we don't have this room and are not knocking on it
 	if !services
 		.rooms
 		.state_cache
 		.server_in_room(services.globals.server_name(), room_id)
+		.await && !services
+		.rooms
+		.state_cache
+		.is_knocked(user_id, room_id)
 		.await
 	{
 		if let Err(e) = remote_leave_room(services, user_id, room_id).await {
@@ -1589,7 +1728,8 @@ pub async fn leave_room(
 			.rooms
 			.state_cache
 			.invite_state(user_id, room_id)
-			.map_err(|_| services.rooms.state_cache.left_state(user_id, room_id))
+			.or_else(|_| services.rooms.state_cache.knock_state(user_id, room_id))
+			.or_else(|_| services.rooms.state_cache.left_state(user_id, room_id))
 			.await
 			.ok();
 
@@ -1671,13 +1811,6 @@ async fn remote_leave_room(
 	let mut make_leave_response_and_server =
 		Err!(BadServerResponse("No server available to assist in leaving."));
 
-	let invite_state = services
-		.rooms
-		.state_cache
-		.invite_state(user_id, room_id)
-		.await
-		.map_err(|_| err!(Request(BadState("User is not invited."))))?;
-
 	let mut servers: HashSet<OwnedServerName> = services
 		.rooms
 		.state_cache
@@ -1686,13 +1819,39 @@ async fn remote_leave_room(
 		.collect()
 		.await;
 
-	servers.extend(
-		invite_state
-			.iter()
-			.filter_map(|event| event.get_field("sender").ok().flatten())
-			.filter_map(|sender: &str| UserId::parse(sender).ok())
-			.map(|user| user.server_name().to_owned()),
-	);
+	if let Ok(invite_state) = services
+		.rooms
+		.state_cache
+		.invite_state(user_id, room_id)
+		.await
+	{
+		servers.extend(
+			invite_state
+				.iter()
+				.filter_map(|event| event.get_field("sender").ok().flatten())
+				.filter_map(|sender: &str| UserId::parse(sender).ok())
+				.map(|user| user.server_name().to_owned()),
+		);
+	} else if let Ok(knock_state) = services
+		.rooms
+		.state_cache
+		.knock_state(user_id, room_id)
+		.await
+	{
+		servers.extend(
+			knock_state
+				.iter()
+				.filter_map(|event| event.get_field("sender").ok().flatten())
+				.filter_map(|sender: &str| UserId::parse(sender).ok())
+				.filter_map(|sender| {
+					if !services.globals.user_is_local(sender) {
+						Some(sender.server_name().to_owned())
+					} else {
+						None
+					}
+				}),
+		);
+	}
 
 	if let Some(room_id_server_name) = room_id.server_name() {
 		servers.insert(room_id_server_name.to_owned());
@@ -1767,7 +1926,7 @@ async fn remote_leave_room(
 		.hash_and_sign_event(&mut leave_event_stub, &room_version_id)?;
 
 	// Generate event id
-	let event_id = pdu::gen_event_id(&leave_event_stub, &room_version_id)?;
+	let event_id = gen_event_id(&leave_event_stub, &room_version_id)?;
 
 	// Add event_id back
 	leave_event_stub
@@ -1793,3 +1952,514 @@ async fn remote_leave_room(
 
 	Ok(())
 }
+
+async fn knock_room_by_id_helper(
+	services: &Services,
+	sender_user: &UserId,
+	room_id: &RoomId,
+	reason: Option<String>,
+	servers: &[OwnedServerName],
+) -> Result<knock_room::v3::Response> {
+	let state_lock = services.rooms.state.mutex.lock(room_id).await;
+
+	if services
+		.rooms
+		.state_cache
+		.is_invited(sender_user, room_id)
+		.await
+	{
+		debug_warn!("{sender_user} is already invited in {room_id} but attempted to knock");
+		return Err!(Request(Forbidden(
+			"You cannot knock on a room you are already invited/accepted to."
+		)));
+	}
+
+	if services
+		.rooms
+		.state_cache
+		.is_joined(sender_user, room_id)
+		.await
+	{
+		debug_warn!("{sender_user} is already joined in {room_id} but attempted to knock");
+		return Err!(Request(Forbidden("You cannot knock on a room you are already joined in.")));
+	}
+
+	if services
+		.rooms
+		.state_cache
+		.is_knocked(sender_user, room_id)
+		.await
+	{
+		debug_warn!("{sender_user} is already knocked in {room_id}");
+		return Ok(knock_room::v3::Response { room_id: room_id.into() });
+	}
+
+	if let Ok(membership) = services
+		.rooms
+		.state_accessor
+		.get_member(room_id, sender_user)
+		.await
+	{
+		if membership.membership == MembershipState::Ban {
+			debug_warn!("{sender_user} is banned from {room_id} but attempted to knock");
+			return Err!(Request(Forbidden("You cannot knock on a room you are banned from.")));
+		}
+	}
+
+	let server_in_room = services
+		.rooms
+		.state_cache
+		.server_in_room(services.globals.server_name(), room_id)
+		.await;
+
+	let local_knock = server_in_room
+		|| servers.is_empty()
+		|| (servers.len() == 1 && services.globals.server_is_ours(&servers[0]));
+
+	if local_knock {
+		knock_room_helper_local(services, sender_user, room_id, reason, servers, state_lock)
+			.boxed()
+			.await?;
+	} else {
+		knock_room_helper_remote(services, sender_user, room_id, reason, servers, state_lock)
+			.boxed()
+			.await?;
+	}
+
+	Ok(knock_room::v3::Response::new(room_id.to_owned()))
+}
+
+async fn knock_room_helper_local(
+	services: &Services,
+	sender_user: &UserId,
+	room_id: &RoomId,
+	reason: Option<String>,
+	servers: &[OwnedServerName],
+	state_lock: RoomMutexGuard,
+) -> Result {
+	debug_info!("We can knock locally");
+
+	let room_version_id = services.rooms.state.get_room_version(room_id).await?;
+
+	if matches!(
+		room_version_id,
+		RoomVersionId::V1
+			| RoomVersionId::V2
+			| RoomVersionId::V3
+			| RoomVersionId::V4
+			| RoomVersionId::V5
+			| RoomVersionId::V6
+	) {
+		return Err!(Request(Forbidden("This room does not support knocking.")));
+	}
+
+	let content = RoomMemberEventContent {
+		displayname: services.users.displayname(sender_user).await.ok(),
+		avatar_url: services.users.avatar_url(sender_user).await.ok(),
+		blurhash: services.users.blurhash(sender_user).await.ok(),
+		reason: reason.clone(),
+		..RoomMemberEventContent::new(MembershipState::Knock)
+	};
+
+	// Try normal knock first
+	let Err(error) = services
+		.rooms
+		.timeline
+		.build_and_append_pdu(
+			PduBuilder::state(sender_user.to_string(), &content),
+			sender_user,
+			room_id,
+			&state_lock,
+		)
+		.await
+	else {
+		return Ok(());
+	};
+
+	if servers.is_empty() || (servers.len() == 1 && services.globals.server_is_ours(&servers[0]))
+	{
+		return Err(error);
+	}
+
+	warn!("We couldn't do the knock locally, maybe federation can help to satisfy the knock");
+
+	let (make_knock_response, remote_server) =
+		make_knock_request(services, sender_user, room_id, servers).await?;
+
+	info!("make_knock finished");
+
+	let room_version_id = make_knock_response.room_version;
+
+	if !services.server.supported_room_version(&room_version_id) {
+		return Err!(BadServerResponse(
+			"Remote room version {room_version_id} is not supported by conduwuit"
+		));
+	}
+
+	let mut knock_event_stub = serde_json::from_str::<CanonicalJsonObject>(
+		make_knock_response.event.get(),
+	)
+	.map_err(|e| {
+		err!(BadServerResponse("Invalid make_knock event json received from server: {e:?}"))
+	})?;
+
+	knock_event_stub.insert(
+		"origin".to_owned(),
+		CanonicalJsonValue::String(services.globals.server_name().as_str().to_owned()),
+	);
+	knock_event_stub.insert(
+		"origin_server_ts".to_owned(),
+		CanonicalJsonValue::Integer(
+			utils::millis_since_unix_epoch()
+				.try_into()
+				.expect("Timestamp is valid js_int value"),
+		),
+	);
+	knock_event_stub.insert(
+		"content".to_owned(),
+		to_canonical_value(RoomMemberEventContent {
+			displayname: services.users.displayname(sender_user).await.ok(),
+			avatar_url: services.users.avatar_url(sender_user).await.ok(),
+			blurhash: services.users.blurhash(sender_user).await.ok(),
+			reason,
+			..RoomMemberEventContent::new(MembershipState::Knock)
+		})
+		.expect("event is valid, we just created it"),
+	);
+
+	// In order to create a compatible ref hash (EventID) the `hashes` field needs
+	// to be present
+	services
+		.server_keys
+		.hash_and_sign_event(&mut knock_event_stub, &room_version_id)?;
+
+	// Generate event id
+	let event_id = gen_event_id(&knock_event_stub, &room_version_id)?;
+
+	// Add event_id
+	knock_event_stub
+		.insert("event_id".to_owned(), CanonicalJsonValue::String(event_id.clone().into()));
+
+	// It has enough fields to be called a proper event now
+	let knock_event = knock_event_stub;
+
+	info!("Asking {remote_server} for send_knock in room {room_id}");
+	let send_knock_request = federation::knock::send_knock::v1::Request {
+		room_id: room_id.to_owned(),
+		event_id: event_id.clone(),
+		pdu: services
+			.sending
+			.convert_to_outgoing_federation_event(knock_event.clone())
+			.await,
+	};
+
+	let send_knock_response = services
+		.sending
+		.send_federation_request(&remote_server, send_knock_request)
+		.await?;
+
+	info!("send_knock finished");
+
+	services
+		.rooms
+		.short
+		.get_or_create_shortroomid(room_id)
+		.await;
+
+	info!("Parsing knock event");
+
+	let parsed_knock_pdu = PduEvent::from_id_val(&event_id, knock_event.clone())
+		.map_err(|e| err!(BadServerResponse("Invalid knock event PDU: {e:?}")))?;
+
+	info!("Updating membership locally to knock state with provided stripped state events");
+	services
+		.rooms
+		.state_cache
+		.update_membership(
+			room_id,
+			sender_user,
+			parsed_knock_pdu
+				.get_content::<RoomMemberEventContent>()
+				.expect("we just created this"),
+			sender_user,
+			Some(send_knock_response.knock_room_state),
+			None,
+			false,
+		)
+		.await?;
+
+	info!("Appending room knock event locally");
+	services
+		.rooms
+		.timeline
+		.append_pdu(
+			&parsed_knock_pdu,
+			knock_event,
+			vec![(*parsed_knock_pdu.event_id).to_owned()],
+			&state_lock,
+		)
+		.await?;
+
+	Ok(())
+}
+
+async fn knock_room_helper_remote(
+	services: &Services,
+	sender_user: &UserId,
+	room_id: &RoomId,
+	reason: Option<String>,
+	servers: &[OwnedServerName],
+	state_lock: RoomMutexGuard,
+) -> Result {
+	info!("Knocking {room_id} over federation.");
+
+	let (make_knock_response, remote_server) =
+		make_knock_request(services, sender_user, room_id, servers).await?;
+
+	info!("make_knock finished");
+
+	let room_version_id = make_knock_response.room_version;
+
+	if !services.server.supported_room_version(&room_version_id) {
+		return Err!(BadServerResponse(
+			"Remote room version {room_version_id} is not supported by conduwuit"
+		));
+	}
+
+	let mut knock_event_stub: CanonicalJsonObject =
+		serde_json::from_str(make_knock_response.event.get()).map_err(|e| {
+			err!(BadServerResponse("Invalid make_knock event json received from server: {e:?}"))
+		})?;
+
+	knock_event_stub.insert(
+		"origin".to_owned(),
+		CanonicalJsonValue::String(services.globals.server_name().as_str().to_owned()),
+	);
+	knock_event_stub.insert(
+		"origin_server_ts".to_owned(),
+		CanonicalJsonValue::Integer(
+			utils::millis_since_unix_epoch()
+				.try_into()
+				.expect("Timestamp is valid js_int value"),
+		),
+	);
+	knock_event_stub.insert(
+		"content".to_owned(),
+		to_canonical_value(RoomMemberEventContent {
+			displayname: services.users.displayname(sender_user).await.ok(),
+			avatar_url: services.users.avatar_url(sender_user).await.ok(),
+			blurhash: services.users.blurhash(sender_user).await.ok(),
+			reason,
+			..RoomMemberEventContent::new(MembershipState::Knock)
+		})
+		.expect("event is valid, we just created it"),
+	);
+
+	// In order to create a compatible ref hash (EventID) the `hashes` field needs
+	// to be present
+	services
+		.server_keys
+		.hash_and_sign_event(&mut knock_event_stub, &room_version_id)?;
+
+	// Generate event id
+	let event_id = gen_event_id(&knock_event_stub, &room_version_id)?;
+
+	// Add event_id
+	knock_event_stub
+		.insert("event_id".to_owned(), CanonicalJsonValue::String(event_id.clone().into()));
+
+	// It has enough fields to be called a proper event now
+	let knock_event = knock_event_stub;
+
+	info!("Asking {remote_server} for send_knock in room {room_id}");
+	let send_knock_request = federation::knock::send_knock::v1::Request {
+		room_id: room_id.to_owned(),
+		event_id: event_id.clone(),
+		pdu: services
+			.sending
+			.convert_to_outgoing_federation_event(knock_event.clone())
+			.await,
+	};
+
+	let send_knock_response = services
+		.sending
+		.send_federation_request(&remote_server, send_knock_request)
+		.await?;
+
+	info!("send_knock finished");
+
+	services
+		.rooms
+		.short
+		.get_or_create_shortroomid(room_id)
+		.await;
+
+	info!("Parsing knock event");
+	let parsed_knock_pdu = PduEvent::from_id_val(&event_id, knock_event.clone())
+		.map_err(|e| err!(BadServerResponse("Invalid knock event PDU: {e:?}")))?;
+
+	info!("Going through send_knock response knock state events");
+	let state = send_knock_response
+		.knock_room_state
+		.iter()
+		.map(|event| serde_json::from_str::<CanonicalJsonObject>(event.clone().into_json().get()))
+		.filter_map(Result::ok);
+
+	let mut state_map: HashMap<u64, OwnedEventId> = HashMap::new();
+
+	for event in state {
+		let Some(state_key) = event.get("state_key") else {
+			debug_warn!("send_knock stripped state event missing state_key: {event:?}");
+			continue;
+		};
+		let Some(event_type) = event.get("type") else {
+			debug_warn!("send_knock stripped state event missing event type: {event:?}");
+			continue;
+		};
+
+		let Ok(state_key) = serde_json::from_value::<String>(state_key.clone().into()) else {
+			debug_warn!("send_knock stripped state event has invalid state_key: {event:?}");
+			continue;
+		};
+		let Ok(event_type) = serde_json::from_value::<StateEventType>(event_type.clone().into())
+		else {
+			debug_warn!("send_knock stripped state event has invalid event type: {event:?}");
+			continue;
+		};
+
+		let event_id = gen_event_id(&event, &room_version_id)?;
+		let shortstatekey = services
+			.rooms
+			.short
+			.get_or_create_shortstatekey(&event_type, &state_key)
+			.await;
+
+		services.rooms.outlier.add_pdu_outlier(&event_id, &event);
+		state_map.insert(shortstatekey, event_id.clone());
+	}
+
+	info!("Compressing state from send_knock");
+	let compressed: HashSet<_> = services
+		.rooms
+		.state_compressor
+		.compress_state_events(state_map.iter().map(|(ssk, eid)| (ssk, eid.borrow())))
+		.collect()
+		.await;
+
+	debug!("Saving compressed state");
+	let HashSetCompressStateEvent {
+		shortstatehash: statehash_before_knock,
+		added,
+		removed,
+	} = services
+		.rooms
+		.state_compressor
+		.save_state(room_id, Arc::new(compressed))
+		.await?;
+
+	debug!("Forcing state for new room");
+	services
+		.rooms
+		.state
+		.force_state(room_id, statehash_before_knock, added, removed, &state_lock)
+		.await?;
+
+	let statehash_after_knock = services
+		.rooms
+		.state
+		.append_to_state(&parsed_knock_pdu)
+		.await?;
+
+	info!("Updating membership locally to knock state with provided stripped state events");
+	services
+		.rooms
+		.state_cache
+		.update_membership(
+			room_id,
+			sender_user,
+			parsed_knock_pdu
+				.get_content::<RoomMemberEventContent>()
+				.expect("we just created this"),
+			sender_user,
+			Some(send_knock_response.knock_room_state),
+			None,
+			false,
+		)
+		.await?;
+
+	info!("Appending room knock event locally");
+	services
+		.rooms
+		.timeline
+		.append_pdu(
+			&parsed_knock_pdu,
+			knock_event,
+			vec![(*parsed_knock_pdu.event_id).to_owned()],
+			&state_lock,
+		)
+		.await?;
+
+	info!("Setting final room state for new room");
+	// We set the room state after inserting the pdu, so that we never have a moment
+	// in time where events in the current room state do not exist
+	services
+		.rooms
+		.state
+		.set_room_state(room_id, statehash_after_knock, &state_lock);
+
+	Ok(())
+}
+
+async fn make_knock_request(
+	services: &Services,
+	sender_user: &UserId,
+	room_id: &RoomId,
+	servers: &[OwnedServerName],
+) -> Result<(federation::knock::create_knock_event_template::v1::Response, OwnedServerName)> {
+	let mut make_knock_response_and_server =
+		Err!(BadServerResponse("No server available to assist in knocking."));
+
+	let mut make_knock_counter: usize = 0;
+
+	for remote_server in servers {
+		if services.globals.server_is_ours(remote_server) {
+			continue;
+		}
+
+		info!("Asking {remote_server} for make_knock ({make_knock_counter})");
+
+		let make_knock_response = services
+			.sending
+			.send_federation_request(
+				remote_server,
+				federation::knock::create_knock_event_template::v1::Request {
+					room_id: room_id.to_owned(),
+					user_id: sender_user.to_owned(),
+					ver: services.server.supported_room_versions().collect(),
+				},
+			)
+			.await;
+
+		trace!("make_knock response: {make_knock_response:?}");
+		make_knock_counter = make_knock_counter.saturating_add(1);
+
+		make_knock_response_and_server = make_knock_response.map(|r| (r, remote_server.clone()));
+
+		if make_knock_response_and_server.is_ok() {
+			break;
+		}
+
+		if make_knock_counter > 40 {
+			warn!(
+				"50 servers failed to provide valid make_knock response, assuming no server can \
+				 assist in knocking."
+			);
+			make_knock_response_and_server =
+				Err!(BadServerResponse("No server available to assist in knocking."));
+
+			return make_knock_response_and_server;
+		}
+	}
+
+	make_knock_response_and_server
+}

src/api/client/message.rs

@@ -1,16 +1,14 @@
-use std::collections::HashSet;
-
 use axum::extract::State;
 use conduwuit::{
 	at, is_equal_to,
 	utils::{
 		result::{FlatOk, LogErr},
-		stream::{BroadbandExt, WidebandExt},
+		stream::{BroadbandExt, TryIgnore, WidebandExt},
 		IterStream, ReadyExt,
 	},
-	Event, PduCount, Result,
+	Event, PduCount, PduEvent, Result,
 };
-use futures::{FutureExt, StreamExt};
+use futures::{future::OptionFuture, pin_mut, FutureExt, StreamExt, TryFutureExt};
 use ruma::{
 	api::{
 		client::{filter::RoomEventFilter, message::get_message_events},
@@ -18,14 +16,19 @@ use ruma::{
 	},
 	events::{AnyStateEvent, StateEventType, TimelineEventType, TimelineEventType::*},
 	serde::Raw,
-	DeviceId, OwnedUserId, RoomId, UserId,
+	RoomId, UserId,
+};
+use service::{
+	rooms::{
+		lazy_loading,
+		lazy_loading::{Options, Witness},
+		timeline::PdusIterItem,
+	},
+	Services,
 };
-use service::{rooms::timeline::PdusIterItem, Services};
 
 use crate::Ruma;
 
-pub(crate) type LazySet = HashSet<OwnedUserId>;
-
 /// list of safe and common non-state events to ignore if the user is ignored
 const IGNORED_MESSAGE_TYPES: &[TimelineEventType; 17] = &[
 	Audio,
@@ -84,13 +87,6 @@ pub(crate) async fn get_message_events_route(
 		.unwrap_or(LIMIT_DEFAULT)
 		.min(LIMIT_MAX);
 
-	services.rooms.lazy_loading.lazy_load_confirm_delivery(
-		sender_user,
-		sender_device,
-		room_id,
-		from,
-	);
-
 	if matches!(body.dir, Direction::Backward) {
 		services
 			.rooms
@@ -107,14 +103,14 @@ pub(crate) async fn get_message_events_route(
 			.rooms
 			.timeline
 			.pdus(Some(sender_user), room_id, Some(from))
-			.await?
+			.ignore_err()
 			.boxed(),
 
 		| Direction::Backward => services
 			.rooms
 			.timeline
 			.pdus_rev(Some(sender_user), room_id, Some(from))
-			.await?
+			.ignore_err()
 			.boxed(),
 	};
 
@@ -127,35 +123,34 @@ pub(crate) async fn get_message_events_route(
 		.collect()
 		.await;
 
-	let lazy = events
-		.iter()
-		.stream()
-		.fold(LazySet::new(), |lazy, item| {
-			update_lazy(&services, room_id, sender, lazy, item, false)
-		})
-		.await;
+	let lazy_loading_context = lazy_loading::Context {
+		user_id: sender_user,
+		device_id: sender_device,
+		room_id,
+		token: Some(from.into_unsigned()),
+		options: Some(&filter.lazy_load_options),
+	};
 
-	let state = lazy
-		.iter()
-		.stream()
-		.broad_filter_map(|user_id| get_member_event(&services, room_id, user_id))
+	let witness: OptionFuture<_> = filter
+		.lazy_load_options
+		.is_enabled()
+		.then(|| lazy_loading_witness(&services, &lazy_loading_context, events.iter()))
+		.into();
+
+	let state = witness
+		.map(Option::into_iter)
+		.map(|option| option.flat_map(Witness::into_iter))
+		.map(IterStream::stream)
+		.into_stream()
+		.flatten()
+		.broad_filter_map(|user_id| async move {
+			get_member_event(&services, room_id, &user_id).await
+		})
 		.collect()
 		.await;
 
 	let next_token = events.last().map(at!(0));
 
-	if !cfg!(feature = "element_hacks") {
-		if let Some(next_token) = next_token {
-			services.rooms.lazy_loading.lazy_load_mark_sent(
-				sender_user,
-				sender_device,
-				room_id,
-				lazy,
-				next_token,
-			);
-		}
-	}
-
 	let chunk = events
 		.into_iter()
 		.map(at!(1))
@@ -170,6 +165,52 @@ pub(crate) async fn get_message_events_route(
 	})
 }
 
+pub(crate) async fn lazy_loading_witness<'a, I>(
+	services: &Services,
+	lazy_loading_context: &lazy_loading::Context<'_>,
+	events: I,
+) -> Witness
+where
+	I: Iterator<Item = &'a PdusIterItem> + Clone + Send,
+{
+	let oldest = events
+		.clone()
+		.map(|(count, _)| count)
+		.copied()
+		.min()
+		.unwrap_or_else(PduCount::max);
+
+	let newest = events
+		.clone()
+		.map(|(count, _)| count)
+		.copied()
+		.max()
+		.unwrap_or_else(PduCount::max);
+
+	let receipts = services
+		.rooms
+		.read_receipt
+		.readreceipts_since(lazy_loading_context.room_id, oldest.into_unsigned());
+
+	pin_mut!(receipts);
+	let witness: Witness = events
+		.stream()
+		.map(|(_, pdu)| pdu.sender.clone())
+		.chain(
+			receipts
+				.ready_take_while(|(_, c, _)| *c <= newest.into_unsigned())
+				.map(|(user_id, ..)| user_id.to_owned()),
+		)
+		.collect()
+		.await;
+
+	services
+		.rooms
+		.lazy_loading
+		.witness_retain(witness, lazy_loading_context)
+		.await
+}
+
 async fn get_member_event(
 	services: &Services,
 	room_id: &RoomId,
@@ -179,47 +220,11 @@ async fn get_member_event(
 		.rooms
 		.state_accessor
 		.room_state_get(room_id, &StateEventType::RoomMember, user_id.as_str())
+		.map_ok(PduEvent::into_state_event)
 		.await
-		.map(|member_event| member_event.to_state_event())
 		.ok()
 }
 
-pub(crate) async fn update_lazy(
-	services: &Services,
-	room_id: &RoomId,
-	sender: (&UserId, &DeviceId),
-	mut lazy: LazySet,
-	item: &PdusIterItem,
-	force: bool,
-) -> LazySet {
-	let (_, event) = &item;
-	let (sender_user, sender_device) = sender;
-
-	/* TODO: Remove the "element_hacks" check when these are resolved:
-	 * https://github.com/vector-im/element-android/issues/3417
-	 * https://github.com/vector-im/element-web/issues/21034
-	 */
-	if force || cfg!(features = "element_hacks") {
-		lazy.insert(event.sender().into());
-		return lazy;
-	}
-
-	if lazy.contains(event.sender()) {
-		return lazy;
-	}
-
-	if !services
-		.rooms
-		.lazy_loading
-		.lazy_load_was_sent_before(sender_user, sender_device, room_id, event.sender())
-		.await
-	{
-		lazy.insert(event.sender().into());
-	}
-
-	lazy
-}
-
 pub(crate) async fn ignored_filter(
 	services: &Services,
 	item: PdusIterItem,

src/api/client/openid.rs

@@ -37,7 +37,7 @@ pub(crate) async fn create_openid_token_route(
 	Ok(account::request_openid_token::v3::Response {
 		access_token,
 		token_type: TokenType::Bearer,
-		matrix_server_name: services.globals.config.server_name.clone(),
+		matrix_server_name: services.server.name.clone(),
 		expires_in: Duration::from_secs(expires_in),
 	})
 }

src/api/client/presence.rs

@@ -82,14 +82,19 @@ pub(crate) async fn get_presence_route(
 			presence.content.status_msg
 		};
 
+		let last_active_ago = match presence.content.currently_active {
+			| Some(true) => None,
+			| _ => presence
+				.content
+				.last_active_ago
+				.map(|millis| Duration::from_millis(millis.into())),
+		};
+
 		Ok(get_presence::v3::Response {
 			// TODO: Should ruma just use the presenceeventcontent type here?
 			status_msg,
 			currently_active: presence.content.currently_active,
-			last_active_ago: presence
-				.content
-				.last_active_ago
-				.map(|millis| Duration::from_millis(millis.into())),
+			last_active_ago,
 			presence: presence.content.presence,
 		})
 	} else {

src/api/client/report.rs

@@ -50,7 +50,7 @@ pub(crate) async fn report_room_route(
 	if !services
 		.rooms
 		.state_cache
-		.server_in_room(&services.globals.config.server_name, &body.room_id)
+		.server_in_room(&services.server.name, &body.room_id)
 		.await
 	{
 		return Err!(Request(NotFound(

src/api/client/room/create.rs

@@ -71,7 +71,7 @@ pub(crate) async fn create_room_route(
 	let room_id: OwnedRoomId = if let Some(custom_room_id) = &body.room_id {
 		custom_room_id_check(&services, custom_room_id)?
 	} else {
-		RoomId::new(&services.globals.config.server_name)
+		RoomId::new(&services.server.name)
 	};
 
 	// check if room ID doesn't already exist instead of erroring on auth check
@@ -83,7 +83,7 @@ pub(crate) async fn create_room_route(
 	}
 
 	if body.visibility == room::Visibility::Public
-		&& services.globals.config.lockdown_public_room_directory
+		&& services.server.config.lockdown_public_room_directory
 		&& !services.users.is_admin(sender_user).await
 		&& body.appservice_info.is_none()
 	{
@@ -93,7 +93,7 @@ pub(crate) async fn create_room_route(
 			&room_id
 		);
 
-		if services.globals.config.admin_room_notices {
+		if services.server.config.admin_room_notices {
 			services
 				.admin
 				.send_text(&format!(
@@ -450,7 +450,7 @@ pub(crate) async fn create_room_route(
 	if body.visibility == room::Visibility::Public {
 		services.rooms.directory.set_public(&room_id);
 
-		if services.globals.config.admin_room_notices {
+		if services.server.config.admin_room_notices {
 			services
 				.admin
 				.send_text(&format!(

src/api/client/room/initial_sync.rs

@@ -1,6 +1,10 @@
 use axum::extract::State;
-use conduwuit::{at, utils::BoolExt, Err, Result};
-use futures::StreamExt;
+use conduwuit::{
+	at,
+	utils::{stream::TryTools, BoolExt},
+	Err, PduEvent, Result,
+};
+use futures::TryStreamExt;
 use ruma::api::client::room::initial_sync::v3::{PaginationChunk, Request, Response};
 
 use crate::Ruma;
@@ -27,19 +31,17 @@ pub(crate) async fn room_initial_sync_route(
 		.rooms
 		.timeline
 		.pdus_rev(None, room_id, None)
-		.await?
-		.take(limit)
-		.collect()
-		.await;
+		.try_take(limit)
+		.try_collect()
+		.await?;
 
 	let state: Vec<_> = services
 		.rooms
 		.state_accessor
 		.room_state_full_pdus(room_id)
-		.await?
-		.into_iter()
-		.map(|pdu| pdu.to_state_event())
-		.collect();
+		.map_ok(PduEvent::into_state_event)
+		.try_collect()
+		.await?;
 
 	let messages = PaginationChunk {
 		start: events.last().map(at!(0)).as_ref().map(ToString::to_string),

src/api/client/search.rs

@@ -7,7 +7,7 @@ use conduwuit::{
 	utils::{stream::ReadyExt, IterStream},
 	Err, PduEvent, Result,
 };
-use futures::{future::OptionFuture, FutureExt, StreamExt, TryFutureExt};
+use futures::{future::OptionFuture, FutureExt, StreamExt, TryFutureExt, TryStreamExt};
 use ruma::{
 	api::client::search::search_events::{
 		self,
@@ -181,15 +181,15 @@ async fn category_room_events(
 }
 
 async fn procure_room_state(services: &Services, room_id: &RoomId) -> Result<RoomState> {
-	let state_map = services
+	let state = services
 		.rooms
 		.state_accessor
-		.room_state_full(room_id)
+		.room_state_full_pdus(room_id)
+		.map_ok(PduEvent::into_state_event)
+		.try_collect()
 		.await?;
 
-	let state_events = state_map.values().map(PduEvent::to_state_event).collect();
-
-	Ok(state_events)
+	Ok(state)
 }
 
 async fn check_room_visible(

src/api/client/session.rs

@@ -1,3 +1,5 @@
+use std::time::Duration;
+
 use axum::extract::State;
 use axum_client_ip::InsecureClientIp;
 use conduwuit::{debug, err, info, utils::ReadyExt, warn, Err};
@@ -6,9 +8,10 @@ use ruma::{
 	api::client::{
 		error::ErrorKind,
 		session::{
+			get_login_token,
 			get_login_types::{
 				self,
-				v3::{ApplicationServiceLoginType, PasswordLoginType},
+				v3::{ApplicationServiceLoginType, PasswordLoginType, TokenLoginType},
 			},
 			login::{
 				self,
@@ -16,33 +19,31 @@ use ruma::{
 			},
 			logout, logout_all,
 		},
-		uiaa::UserIdentifier,
+		uiaa,
 	},
 	OwnedUserId, UserId,
 };
-use serde::Deserialize;
+use service::uiaa::SESSION_ID_LENGTH;
 
 use super::{DEVICE_ID_LENGTH, TOKEN_LENGTH};
 use crate::{utils, utils::hash, Error, Result, Ruma};
 
-#[derive(Debug, Deserialize)]
-struct Claims {
-	sub: String,
-	//exp: usize,
-}
-
 /// # `GET /_matrix/client/v3/login`
 ///
 /// Get the supported login types of this server. One of these should be used as
 /// the `type` field when logging in.
 #[tracing::instrument(skip_all, fields(%client), name = "login")]
 pub(crate) async fn get_login_types_route(
+	State(services): State<crate::State>,
 	InsecureClientIp(client): InsecureClientIp,
 	_body: Ruma<get_login_types::v3::Request>,
 ) -> Result<get_login_types::v3::Response> {
 	Ok(get_login_types::v3::Response::new(vec![
 		get_login_types::v3::LoginType::Password(PasswordLoginType::default()),
 		get_login_types::v3::LoginType::ApplicationService(ApplicationServiceLoginType::default()),
+		get_login_types::v3::LoginType::Token(TokenLoginType {
+			get_login_token: services.server.config.login_via_existing_session,
+		}),
 	]))
 }
 
@@ -77,7 +78,9 @@ pub(crate) async fn login_route(
 			..
 		}) => {
 			debug!("Got password login type");
-			let user_id = if let Some(UserIdentifier::UserIdOrLocalpart(user_id)) = identifier {
+			let user_id = if let Some(uiaa::UserIdentifier::UserIdOrLocalpart(user_id)) =
+				identifier
+			{
 				UserId::parse_with_server_name(
 					user_id.to_lowercase(),
 					services.globals.server_name(),
@@ -108,32 +111,10 @@ pub(crate) async fn login_route(
 		},
 		| login::v3::LoginInfo::Token(login::v3::Token { token }) => {
 			debug!("Got token login type");
-			if let Some(jwt_decoding_key) = services.globals.jwt_decoding_key() {
-				let token = jsonwebtoken::decode::<Claims>(
-					token,
-					jwt_decoding_key,
-					&jsonwebtoken::Validation::default(),
-				)
-				.map_err(|e| {
-					warn!("Failed to parse JWT token from user logging in: {e}");
-					Error::BadRequest(ErrorKind::InvalidUsername, "Token is invalid.")
-				})?;
-
-				let username = token.claims.sub.to_lowercase();
-
-				UserId::parse_with_server_name(username, services.globals.server_name()).map_err(
-					|e| {
-						err!(Request(InvalidUsername(debug_error!(
-							?e,
-							"Failed to parse login username"
-						))))
-					},
-				)?
-			} else {
-				return Err!(Request(Unknown(
-					"Token login is not supported (server has no jwt decoding key)."
-				)));
+			if !services.server.config.login_via_existing_session {
+				return Err!(Request(Unknown("Token login is not enabled.")));
 			}
+			services.users.find_from_login_token(token).await?
 		},
 		#[allow(deprecated)]
 		| login::v3::LoginInfo::ApplicationService(login::v3::ApplicationService {
@@ -141,21 +122,22 @@ pub(crate) async fn login_route(
 			user,
 		}) => {
 			debug!("Got appservice login type");
-			let user_id = if let Some(UserIdentifier::UserIdOrLocalpart(user_id)) = identifier {
-				UserId::parse_with_server_name(
-					user_id.to_lowercase(),
-					services.globals.server_name(),
-				)
-			} else if let Some(user) = user {
-				OwnedUserId::parse(user)
-			} else {
-				warn!("Bad login type: {:?}", &body.login_info);
-				return Err(Error::BadRequest(ErrorKind::forbidden(), "Bad login type."));
-			}
-			.map_err(|e| {
-				warn!("Failed to parse username from appservice logging in: {e}");
-				Error::BadRequest(ErrorKind::InvalidUsername, "Username is invalid.")
-			})?;
+			let user_id =
+				if let Some(uiaa::UserIdentifier::UserIdOrLocalpart(user_id)) = identifier {
+					UserId::parse_with_server_name(
+						user_id.to_lowercase(),
+						services.globals.server_name(),
+					)
+				} else if let Some(user) = user {
+					OwnedUserId::parse(user)
+				} else {
+					warn!("Bad login type: {:?}", &body.login_info);
+					return Err(Error::BadRequest(ErrorKind::forbidden(), "Bad login type."));
+				}
+				.map_err(|e| {
+					warn!("Failed to parse username from appservice logging in: {e}");
+					Error::BadRequest(ErrorKind::InvalidUsername, "Username is invalid.")
+				})?;
 
 			if let Some(ref info) = body.appservice_info {
 				if !info.is_user_match(&user_id) {
@@ -247,6 +229,67 @@ pub(crate) async fn login_route(
 	})
 }
 
+/// # `POST /_matrix/client/v1/login/get_token`
+///
+/// Allows a logged-in user to get a short-lived token which can be used
+/// to log in with the m.login.token flow.
+///
+/// <https://spec.matrix.org/v1.13/client-server-api/#post_matrixclientv1loginget_token>
+#[tracing::instrument(skip_all, fields(%client), name = "login_token")]
+pub(crate) async fn login_token_route(
+	State(services): State<crate::State>,
+	InsecureClientIp(client): InsecureClientIp,
+	body: Ruma<get_login_token::v1::Request>,
+) -> Result<get_login_token::v1::Response> {
+	if !services.server.config.login_via_existing_session {
+		return Err!(Request(Forbidden("Login via an existing session is not enabled")));
+	}
+
+	let sender_user = body.sender_user();
+	let sender_device = body.sender_device();
+
+	// This route SHOULD have UIA
+	// TODO: How do we make only UIA sessions that have not been used before valid?
+
+	let mut uiaainfo = uiaa::UiaaInfo {
+		flows: vec![uiaa::AuthFlow { stages: vec![uiaa::AuthType::Password] }],
+		completed: Vec::new(),
+		params: Box::default(),
+		session: None,
+		auth_error: None,
+	};
+
+	if let Some(auth) = &body.auth {
+		let (worked, uiaainfo) = services
+			.uiaa
+			.try_auth(sender_user, sender_device, auth, &uiaainfo)
+			.await?;
+
+		if !worked {
+			return Err(Error::Uiaa(uiaainfo));
+		}
+
+		// Success!
+	} else if let Some(json) = body.json_body.as_ref() {
+		uiaainfo.session = Some(utils::random_string(SESSION_ID_LENGTH));
+		services
+			.uiaa
+			.create(sender_user, sender_device, &uiaainfo, json);
+
+		return Err(Error::Uiaa(uiaainfo));
+	} else {
+		return Err!(Request(NotJson("No JSON body was sent when required.")));
+	}
+
+	let login_token = utils::random_string(TOKEN_LENGTH);
+	let expires_in = services.users.create_login_token(sender_user, &login_token);
+
+	Ok(get_login_token::v1::Response {
+		expires_in: Duration::from_millis(expires_in),
+		login_token,
+	})
+}
+
 /// # `POST /_matrix/client/v3/logout`
 ///
 /// Log out the current device.

src/api/client/state.rs

@@ -1,5 +1,6 @@
 use axum::extract::State;
 use conduwuit::{err, pdu::PduBuilder, utils::BoolExt, Err, PduEvent, Result};
+use futures::TryStreamExt;
 use ruma::{
 	api::client::state::{get_state_events, get_state_events_for_key, send_state_event},
 	events::{
@@ -82,11 +83,10 @@ pub(crate) async fn get_state_events_route(
 		room_state: services
 			.rooms
 			.state_accessor
-			.room_state_full(&body.room_id)
-			.await?
-			.values()
-			.map(PduEvent::to_state_event)
-			.collect(),
+			.room_state_full_pdus(&body.room_id)
+			.map_ok(PduEvent::into_state_event)
+			.try_collect()
+			.await?,
 	})
 }
 
@@ -133,7 +133,7 @@ pub(crate) async fn get_state_events_for_key_route(
 
 	Ok(get_state_events_for_key::v3::Response {
 		content: event_format.or(|| event.get_content_as_value()),
-		event: event_format.then(|| event.to_state_event_value()),
+		event: event_format.then(|| event.into_state_event_value()),
 	})
 }
 

src/api/client/sync/mod.rs

@@ -1,16 +1,31 @@
 mod v3;
 mod v4;
+mod v5;
 
 use conduwuit::{
-	utils::stream::{BroadbandExt, ReadyExt},
+	utils::{
+		stream::{BroadbandExt, ReadyExt, TryIgnore},
+		IterStream,
+	},
 	PduCount,
 };
-use futures::StreamExt;
-use ruma::{RoomId, UserId};
+use futures::{pin_mut, StreamExt};
+use ruma::{
+	directory::RoomTypeFilter,
+	events::TimelineEventType::{
+		self, Beacon, CallInvite, PollStart, RoomEncrypted, RoomMessage, Sticker,
+	},
+	RoomId, UserId,
+};
 
-pub(crate) use self::{v3::sync_events_route, v4::sync_events_v4_route};
+pub(crate) use self::{
+	v3::sync_events_route, v4::sync_events_v4_route, v5::sync_events_v5_route,
+};
 use crate::{service::Services, Error, PduEvent, Result};
 
+pub(crate) const DEFAULT_BUMP_TYPES: &[TimelineEventType; 6] =
+	&[CallInvite, PollStart, Beacon, RoomEncrypted, RoomMessage, Sticker];
+
 async fn load_timeline(
 	services: &Services,
 	sender_user: &UserId,
@@ -29,23 +44,19 @@ async fn load_timeline(
 		return Ok((Vec::new(), false));
 	}
 
-	let mut non_timeline_pdus = services
+	let non_timeline_pdus = services
 		.rooms
 		.timeline
 		.pdus_rev(Some(sender_user), room_id, None)
-		.await?
+		.ignore_err()
 		.ready_skip_while(|&(pducount, _)| pducount > next_batch.unwrap_or_else(PduCount::max))
 		.ready_take_while(|&(pducount, _)| pducount > roomsincecount);
 
 	// Take the last events for the timeline
-	let timeline_pdus: Vec<_> = non_timeline_pdus
-		.by_ref()
-		.take(limit)
-		.collect::<Vec<_>>()
-		.await
-		.into_iter()
-		.rev()
-		.collect();
+	pin_mut!(non_timeline_pdus);
+	let timeline_pdus: Vec<_> = non_timeline_pdus.by_ref().take(limit).collect().await;
+
+	let timeline_pdus: Vec<_> = timeline_pdus.into_iter().rev().collect();
 
 	// They /sync response doesn't always return all messages, so we say the output
 	// is limited unless there are events in non_timeline_pdus
@@ -73,3 +84,33 @@ async fn share_encrypted_room(
 		})
 		.await
 }
+
+pub(crate) async fn filter_rooms<'a>(
+	services: &Services,
+	rooms: &[&'a RoomId],
+	filter: &[RoomTypeFilter],
+	negate: bool,
+) -> Vec<&'a RoomId> {
+	rooms
+		.iter()
+		.stream()
+		.filter_map(|r| async move {
+			let room_type = services.rooms.state_accessor.get_room_type(r).await;
+
+			if room_type.as_ref().is_err_and(|e| !e.is_not_found()) {
+				return None;
+			}
+
+			let room_type_filter = RoomTypeFilter::from(room_type.ok());
+
+			let include = if negate {
+				!filter.contains(&room_type_filter)
+			} else {
+				filter.is_empty() || filter.contains(&room_type_filter)
+			};
+
+			include.then_some(r)
+		})
+		.collect()
+		.await
+}

src/api/client/sync/v3.rs

@@ -6,9 +6,9 @@ use std::{
 
 use axum::extract::State;
 use conduwuit::{
-	at, err, error, extract_variant, is_equal_to, is_false,
+	at, err, error, extract_variant, is_equal_to,
 	pdu::EventHash,
-	result::{FlatOk, LogDebugErr},
+	result::FlatOk,
 	utils::{
 		self,
 		future::OptionExt,
@@ -19,22 +19,26 @@ use conduwuit::{
 	Error, PduCount, PduEvent, Result,
 };
 use conduwuit_service::{
-	rooms::short::{ShortStateHash, ShortStateKey},
+	rooms::{
+		lazy_loading,
+		lazy_loading::{Options, Witness},
+		short::ShortStateHash,
+	},
 	Services,
 };
 use futures::{
-	future::{join, join3, join4, join5, try_join, try_join3, OptionFuture},
-	FutureExt, StreamExt, TryFutureExt,
+	future::{join, join3, join4, join5, try_join, try_join4, OptionFuture},
+	FutureExt, StreamExt, TryFutureExt, TryStreamExt,
 };
 use ruma::{
 	api::client::{
-		filter::{FilterDefinition, LazyLoadOptions},
+		filter::FilterDefinition,
 		sync::sync_events::{
 			self,
 			v3::{
 				Ephemeral, Filter, GlobalAccountData, InviteState, InvitedRoom, JoinedRoom,
-				LeftRoom, Presence, RoomAccountData, RoomSummary, Rooms, State as RoomState,
-				Timeline, ToDevice,
+				KnockState, KnockedRoom, LeftRoom, Presence, RoomAccountData, RoomSummary, Rooms,
+				State as RoomState, Timeline, ToDevice,
 			},
 			DeviceLists, UnreadNotificationsCount,
 		},
@@ -124,10 +128,42 @@ pub(crate) async fn sync_events_route(
 	// Setup watchers, so if there's no response, we can wait for them
 	let watcher = services.sync.watch(sender_user, sender_device);
 
-	let next_batch = services.globals.current_count()?;
-	let next_batch_string = next_batch.to_string();
+	let response = build_sync_events(&services, &body).await?;
+	if body.body.full_state
+		|| !(response.rooms.is_empty()
+			&& response.presence.is_empty()
+			&& response.account_data.is_empty()
+			&& response.device_lists.is_empty()
+			&& response.to_device.is_empty())
+	{
+		return Ok(response);
+	}
 
-	// Load filter
+	// Hang a few seconds so requests are not spammed
+	// Stop hanging if new info arrives
+	let default = Duration::from_secs(30);
+	let duration = cmp::min(body.body.timeout.unwrap_or(default), default);
+	_ = tokio::time::timeout(duration, watcher).await;
+
+	// Retry returning data
+	build_sync_events(&services, &body).await
+}
+
+pub(crate) async fn build_sync_events(
+	services: &Services,
+	body: &Ruma<sync_events::v3::Request>,
+) -> Result<sync_events::v3::Response, RumaResponse<UiaaResponse>> {
+	let (sender_user, sender_device) = body.sender();
+
+	let next_batch = services.globals.current_count()?;
+	let since = body
+		.body
+		.since
+		.as_ref()
+		.and_then(|string| string.parse().ok())
+		.unwrap_or(0);
+
+	let full_state = body.body.full_state;
 	let filter = match body.body.filter.as_ref() {
 		| None => FilterDefinition::default(),
 		| Some(Filter::FilterDefinition(ref filter)) => filter.clone(),
@@ -138,24 +174,6 @@ pub(crate) async fn sync_events_route(
 			.unwrap_or_default(),
 	};
 
-	// some clients, at least element, seem to require knowledge of redundant
-	// members for "inline" profiles on the timeline to work properly
-	let (lazy_load_enabled, lazy_load_send_redundant) = match filter.room.state.lazy_load_options
-	{
-		| LazyLoadOptions::Enabled { include_redundant_members } =>
-			(true, include_redundant_members),
-		| LazyLoadOptions::Disabled => (false, cfg!(feature = "element_hacks")),
-	};
-
-	let full_state = body.body.full_state;
-
-	let since = body
-		.body
-		.since
-		.as_ref()
-		.and_then(|string| string.parse().ok())
-		.unwrap_or(0);
-
 	let joined_rooms = services
 		.rooms
 		.state_cache
@@ -163,15 +181,14 @@ pub(crate) async fn sync_events_route(
 		.map(ToOwned::to_owned)
 		.broad_filter_map(|room_id| {
 			load_joined_room(
-				&services,
+				services,
 				sender_user,
 				sender_device,
 				room_id.clone(),
 				since,
 				next_batch,
-				lazy_load_enabled,
-				lazy_load_send_redundant,
 				full_state,
+				&filter,
 			)
 			.map_ok(move |(joined_room, dlu, jeu)| (room_id, joined_room, dlu, jeu))
 			.ok()
@@ -196,13 +213,13 @@ pub(crate) async fn sync_events_route(
 		.rooms_left(sender_user)
 		.broad_filter_map(|(room_id, _)| {
 			handle_left_room(
-				&services,
+				services,
 				since,
 				room_id.clone(),
 				sender_user,
-				&next_batch_string,
+				next_batch,
 				full_state,
-				lazy_load_enabled,
+				&filter,
 			)
 			.map_ok(move |left_room| (room_id, left_room))
 			.ok()
@@ -215,10 +232,6 @@ pub(crate) async fn sync_events_route(
 		.state_cache
 		.rooms_invited(sender_user)
 		.fold_default(|mut invited_rooms: BTreeMap<_, _>, (room_id, invite_state)| async move {
-			// Get and drop the lock to wait for remaining operations to finish
-			let insert_lock = services.rooms.timeline.mutex_insert.lock(&room_id).await;
-			drop(insert_lock);
-
 			let invite_count = services
 				.rooms
 				.state_cache
@@ -239,10 +252,35 @@ pub(crate) async fn sync_events_route(
 			invited_rooms
 		});
 
+	let knocked_rooms = services
+		.rooms
+		.state_cache
+		.rooms_knocked(sender_user)
+		.fold_default(|mut knocked_rooms: BTreeMap<_, _>, (room_id, knock_state)| async move {
+			let knock_count = services
+				.rooms
+				.state_cache
+				.get_knock_count(&room_id, sender_user)
+				.await
+				.ok();
+
+			// Knocked before last sync
+			if Some(since) >= knock_count {
+				return knocked_rooms;
+			}
+
+			let knocked_room = KnockedRoom {
+				knock_state: KnockState { events: knock_state },
+			};
+
+			knocked_rooms.insert(room_id, knocked_room);
+			knocked_rooms
+		});
+
 	let presence_updates: OptionFuture<_> = services
 		.globals
 		.allow_local_presence()
-		.then(|| process_presence_updates(&services, since, sender_user))
+		.then(|| process_presence_updates(services, since, sender_user))
 		.into();
 
 	let account_data = services
@@ -273,7 +311,7 @@ pub(crate) async fn sync_events_route(
 			.users
 			.remove_to_device_events(sender_user, sender_device, since);
 
-	let rooms = join3(joined_rooms, left_rooms, invited_rooms);
+	let rooms = join4(joined_rooms, left_rooms, invited_rooms, knocked_rooms);
 	let ephemeral = join3(remove_to_device_events, to_device_events, presence_updates);
 	let top = join5(account_data, ephemeral, device_one_time_keys_count, keys_changed, rooms)
 		.boxed()
@@ -281,7 +319,7 @@ pub(crate) async fn sync_events_route(
 
 	let (account_data, ephemeral, device_one_time_keys_count, keys_changed, rooms) = top;
 	let ((), to_device_events, presence_updates) = ephemeral;
-	let (joined_rooms, left_rooms, invited_rooms) = rooms;
+	let (joined_rooms, left_rooms, invited_rooms, knocked_rooms) = rooms;
 	let (joined_rooms, mut device_list_updates, left_encrypted_users) = joined_rooms;
 	device_list_updates.extend(keys_changed);
 
@@ -292,7 +330,7 @@ pub(crate) async fn sync_events_route(
 		.stream()
 		.broad_filter_map(|user_id| async move {
 			let no_shared_encrypted_room =
-				!share_encrypted_room(&services, sender_user, &user_id, None).await;
+				!share_encrypted_room(services, sender_user, &user_id, None).await;
 			no_shared_encrypted_room.then_some(user_id)
 		})
 		.ready_fold(HashSet::new(), |mut device_list_left, user_id| {
@@ -310,7 +348,7 @@ pub(crate) async fn sync_events_route(
 		device_one_time_keys_count,
 		// Fallback keys are not yet supported
 		device_unused_fallback_key_types: None,
-		next_batch: next_batch_string,
+		next_batch: next_batch.to_string(),
 		presence: Presence {
 			events: presence_updates
 				.unwrap_or_default()
@@ -322,26 +360,11 @@ pub(crate) async fn sync_events_route(
 			leave: left_rooms,
 			join: joined_rooms,
 			invite: invited_rooms,
-			knock: BTreeMap::new(), // TODO
+			knock: knocked_rooms,
 		},
 		to_device: ToDevice { events: to_device_events },
 	};
 
-	// TODO: Retry the endpoint instead of returning
-	if !full_state
-		&& response.rooms.is_empty()
-		&& response.presence.is_empty()
-		&& response.account_data.is_empty()
-		&& response.device_lists.is_empty()
-		&& response.to_device.is_empty()
-	{
-		// Hang a few seconds so requests are not spammed
-		// Stop hanging if new info arrives
-		let default = Duration::from_secs(30);
-		let duration = cmp::min(body.body.timeout.unwrap_or(default), default);
-		_ = tokio::time::timeout(duration, watcher).await;
-	}
-
 	Ok(response)
 }
 
@@ -370,7 +393,13 @@ async fn process_presence_updates(
 		.ready_fold(PresenceUpdates::new(), |mut updates, (user_id, event)| {
 			match updates.entry(user_id.into()) {
 				| Entry::Vacant(slot) => {
-					slot.insert(event);
+					let mut new_event = event;
+					new_event.content.last_active_ago = match new_event.content.currently_active {
+						| Some(true) => None,
+						| _ => new_event.content.last_active_ago,
+					};
+
+					slot.insert(new_event);
 				},
 				| Entry::Occupied(mut slot) => {
 					let curr_event = slot.get_mut();
@@ -382,8 +411,6 @@ async fn process_presence_updates(
 					curr_content.status_msg = new_content
 						.status_msg
 						.or_else(|| curr_content.status_msg.take());
-					curr_content.last_active_ago =
-						new_content.last_active_ago.or(curr_content.last_active_ago);
 					curr_content.displayname = new_content
 						.displayname
 						.or_else(|| curr_content.displayname.take());
@@ -393,6 +420,10 @@ async fn process_presence_updates(
 					curr_content.currently_active = new_content
 						.currently_active
 						.or(curr_content.currently_active);
+					curr_content.last_active_ago = match curr_content.currently_active {
+						| Some(true) => None,
+						| _ => new_content.last_active_ago.or(curr_content.last_active_ago),
+					};
 				},
 			};
 
@@ -408,7 +439,6 @@ async fn process_presence_updates(
 	fields(
 		room_id = %room_id,
 		full = %full_state,
-		ll = %lazy_load_enabled,
 	),
 )]
 #[allow(clippy::too_many_arguments)]
@@ -417,14 +447,10 @@ async fn handle_left_room(
 	since: u64,
 	ref room_id: OwnedRoomId,
 	sender_user: &UserId,
-	next_batch_string: &str,
+	next_batch: u64,
 	full_state: bool,
-	lazy_load_enabled: bool,
+	filter: &FilterDefinition,
 ) -> Result<Option<LeftRoom>> {
-	// Get and drop the lock to wait for remaining operations to finish
-	let insert_lock = services.rooms.timeline.mutex_insert.lock(room_id).await;
-	drop(insert_lock);
-
 	let left_count = services
 		.rooms
 		.state_cache
@@ -466,7 +492,7 @@ async fn handle_left_room(
 			account_data: RoomAccountData { events: Vec::new() },
 			timeline: Timeline {
 				limited: false,
-				prev_batch: Some(next_batch_string.to_owned()),
+				prev_batch: Some(next_batch.to_string()),
 				events: Vec::new(),
 			},
 			state: RoomState {
@@ -477,16 +503,20 @@ async fn handle_left_room(
 
 	let mut left_state_events = Vec::new();
 
-	let since_shortstatehash = services
-		.rooms
-		.user
-		.get_token_shortstatehash(room_id, since)
-		.await;
+	let since_shortstatehash = services.rooms.user.get_token_shortstatehash(room_id, since);
 
-	let since_state_ids = match since_shortstatehash {
-		| Ok(s) => services.rooms.state_accessor.state_full_ids(s).await?,
-		| Err(_) => HashMap::new(),
-	};
+	let since_state_ids: HashMap<_, OwnedEventId> = since_shortstatehash
+		.map_ok(|since_shortstatehash| {
+			services
+				.rooms
+				.state_accessor
+				.state_full_ids(since_shortstatehash)
+				.map(Ok)
+		})
+		.try_flatten_stream()
+		.try_collect()
+		.await
+		.unwrap_or_default();
 
 	let Ok(left_event_id): Result<OwnedEventId> = services
 		.rooms
@@ -508,11 +538,12 @@ async fn handle_left_room(
 		return Ok(None);
 	};
 
-	let mut left_state_ids = services
+	let mut left_state_ids: HashMap<_, _> = services
 		.rooms
 		.state_accessor
 		.state_full_ids(left_shortstatehash)
-		.await?;
+		.collect()
+		.await;
 
 	let leave_shortstatekey = services
 		.rooms
@@ -530,28 +561,32 @@ async fn handle_left_room(
 				.get_statekey_from_short(shortstatekey)
 				.await?;
 
-			// TODO: Delete "element_hacks" when this is resolved: https://github.com/vector-im/element-web/issues/22565
-			if !lazy_load_enabled
-				|| event_type != StateEventType::RoomMember
-				|| full_state
-				|| (cfg!(feature = "element_hacks") && *sender_user == state_key)
+			if filter.room.state.lazy_load_options.is_enabled()
+				&& event_type == StateEventType::RoomMember
+				&& !full_state
+				&& state_key
+					.as_str()
+					.try_into()
+					.is_ok_and(|user_id: &UserId| sender_user != user_id)
 			{
-				let Ok(pdu) = services.rooms.timeline.get_pdu(&event_id).await else {
-					error!("Pdu in state not found: {event_id}");
-					continue;
-				};
-
-				left_state_events.push(pdu.to_sync_state_event());
+				continue;
 			}
+
+			let Ok(pdu) = services.rooms.timeline.get_pdu(&event_id).await else {
+				error!("Pdu in state not found: {event_id}");
+				continue;
+			};
+
+			left_state_events.push(pdu.to_sync_state_event());
 		}
 	}
 
 	Ok(Some(LeftRoom {
 		account_data: RoomAccountData { events: Vec::new() },
 		timeline: Timeline {
-			limited: true, /* TODO: support left timeline events so we dont need to set this to
-			                * true */
-			prev_batch: Some(next_batch_string.to_owned()),
+			// TODO: support left timeline events so we dont need to set limited to true
+			limited: true,
+			prev_batch: Some(next_batch.to_string()),
 			events: Vec::new(), // and so we dont need to set this to empty vec
 		},
 		state: RoomState { events: left_state_events },
@@ -574,15 +609,9 @@ async fn load_joined_room(
 	ref room_id: OwnedRoomId,
 	since: u64,
 	next_batch: u64,
-	lazy_load_enabled: bool,
-	lazy_load_send_redundant: bool,
 	full_state: bool,
+	filter: &FilterDefinition,
 ) -> Result<(JoinedRoom, HashSet<OwnedUserId>, HashSet<OwnedUserId>)> {
-	// Get and drop the lock to wait for remaining operations to finish
-	// This will make sure the we have all events until next_batch
-	let insert_lock = services.rooms.timeline.mutex_insert.lock(room_id).await;
-	drop(insert_lock);
-
 	let sincecount = PduCount::Normal(since);
 	let next_batchcount = PduCount::Normal(next_batch);
 
@@ -608,17 +637,26 @@ async fn load_joined_room(
 		10_usize,
 	);
 
-	let (current_shortstatehash, since_shortstatehash, timeline) =
-		try_join3(current_shortstatehash, since_shortstatehash, timeline).await?;
+	let receipt_events = services
+		.rooms
+		.read_receipt
+		.readreceipts_since(room_id, since)
+		.filter_map(|(read_user, _, edu)| async move {
+			services
+				.users
+				.user_is_ignored(read_user, sender_user)
+				.await
+				.or_some((read_user.to_owned(), edu))
+		})
+		.collect::<HashMap<OwnedUserId, Raw<AnySyncEphemeralRoomEvent>>>()
+		.map(Ok);
+
+	let (current_shortstatehash, since_shortstatehash, timeline, receipt_events) =
+		try_join4(current_shortstatehash, since_shortstatehash, timeline, receipt_events)
+			.boxed()
+			.await?;
 
 	let (timeline_pdus, limited) = timeline;
-	let timeline_users =
-		timeline_pdus
-			.iter()
-			.fold(HashSet::new(), |mut timeline_users, (_, event)| {
-				timeline_users.insert(event.sender.as_str().to_owned());
-				timeline_users
-			});
 
 	let last_notification_read: OptionFuture<_> = timeline_pdus
 		.is_empty()
@@ -630,21 +668,68 @@ async fn load_joined_room(
 		})
 		.into();
 
-	let send_notification_counts = last_notification_read
-		.is_none_or(|&count| count > since)
-		.await;
-
-	services.rooms.lazy_loading.lazy_load_confirm_delivery(
-		sender_user,
-		sender_device,
-		room_id,
-		sincecount,
-	);
-
 	let no_state_changes = timeline_pdus.is_empty()
 		&& (since_shortstatehash.is_none()
 			|| since_shortstatehash.is_some_and(is_equal_to!(current_shortstatehash)));
 
+	let since_sender_member: OptionFuture<_> = since_shortstatehash
+		.map(|short| {
+			services
+				.rooms
+				.state_accessor
+				.state_get_content(short, &StateEventType::RoomMember, sender_user.as_str())
+				.ok()
+		})
+		.into();
+
+	let joined_since_last_sync =
+		since_sender_member
+			.await
+			.flatten()
+			.is_none_or(|content: RoomMemberEventContent| {
+				content.membership != MembershipState::Join
+			});
+
+	let lazy_loading_enabled = filter.room.state.lazy_load_options.is_enabled()
+		|| filter.room.timeline.lazy_load_options.is_enabled();
+
+	let generate_witness =
+		lazy_loading_enabled && (since_shortstatehash.is_none() || joined_since_last_sync);
+
+	let lazy_reset = lazy_loading_enabled && since_shortstatehash.is_none();
+
+	let lazy_loading_context = &lazy_loading::Context {
+		user_id: sender_user,
+		device_id: sender_device,
+		room_id,
+		token: None,
+		options: Some(&filter.room.state.lazy_load_options),
+	};
+
+	// Reset lazy loading because this is an initial sync
+	let lazy_load_reset: OptionFuture<_> = lazy_reset
+		.then(|| services.rooms.lazy_loading.reset(lazy_loading_context))
+		.into();
+
+	lazy_load_reset.await;
+	let witness: Option<Witness> = generate_witness.then(|| {
+		timeline_pdus
+			.iter()
+			.map(|(_, pdu)| pdu.sender.clone())
+			.chain(receipt_events.keys().cloned())
+			.collect()
+	});
+
+	let witness: OptionFuture<_> = witness
+		.map(|witness| {
+			services
+				.rooms
+				.lazy_loading
+				.witness_retain(witness, lazy_loading_context)
+		})
+		.into();
+
+	let witness = witness.await;
 	let mut device_list_updates = HashSet::<OwnedUserId>::new();
 	let mut left_encrypted_users = HashSet::<OwnedUserId>::new();
 	let StateChanges {
@@ -659,19 +744,17 @@ async fn load_joined_room(
 		calculate_state_changes(
 			services,
 			sender_user,
-			sender_device,
 			room_id,
-			next_batchcount,
-			lazy_load_enabled,
-			lazy_load_send_redundant,
 			full_state,
+			filter,
 			&mut device_list_updates,
 			&mut left_encrypted_users,
 			since_shortstatehash,
 			current_shortstatehash,
-			&timeline_pdus,
-			&timeline_users,
+			joined_since_last_sync,
+			witness.as_ref(),
 		)
+		.boxed()
 		.await?
 	};
 
@@ -696,19 +779,6 @@ async fn load_joined_room(
 		.map(|(_, pdu)| pdu.to_sync_room_event())
 		.collect();
 
-	let receipt_events = services
-		.rooms
-		.read_receipt
-		.readreceipts_since(room_id, since)
-		.filter_map(|(read_user, _, edu)| async move {
-			services
-				.users
-				.user_is_ignored(read_user, sender_user)
-				.await
-				.or_some((read_user.to_owned(), edu))
-		})
-		.collect::<HashMap<OwnedUserId, Raw<AnySyncEphemeralRoomEvent>>>();
-
 	let typing_events = services
 		.rooms
 		.typing
@@ -728,6 +798,10 @@ async fn load_joined_room(
 		})
 		.unwrap_or(Vec::new());
 
+	let send_notification_counts = last_notification_read
+		.is_none_or(|&count| count > since)
+		.await;
+
 	let notification_count: OptionFuture<_> = send_notification_counts
 		.then(|| {
 			services
@@ -750,14 +824,14 @@ async fn load_joined_room(
 		})
 		.into();
 
-	let events = join4(room_events, account_data_events, receipt_events, typing_events);
+	let events = join3(room_events, account_data_events, typing_events);
 	let unread_notifications = join(notification_count, highlight_count);
 	let (unread_notifications, events, device_updates) =
 		join3(unread_notifications, events, device_updates)
 			.boxed()
 			.await;
 
-	let (room_events, account_data_events, receipt_events, typing_events) = events;
+	let (room_events, account_data_events, typing_events) = events;
 	let (notification_count, highlight_count) = unread_notifications;
 
 	device_list_updates.extend(device_updates);
@@ -834,7 +908,6 @@ async fn load_joined_room(
 	skip_all,
 	fields(
 	    full = %full_state,
-	    ll = ?(lazy_load_enabled, lazy_load_send_redundant),
 	    cs = %current_shortstatehash,
 	    ss = ?since_shortstatehash,
     )
@@ -843,64 +916,38 @@ async fn load_joined_room(
 async fn calculate_state_changes(
 	services: &Services,
 	sender_user: &UserId,
-	sender_device: &DeviceId,
 	room_id: &RoomId,
-	next_batchcount: PduCount,
-	lazy_load_enabled: bool,
-	lazy_load_send_redundant: bool,
 	full_state: bool,
+	filter: &FilterDefinition,
 	device_list_updates: &mut HashSet<OwnedUserId>,
 	left_encrypted_users: &mut HashSet<OwnedUserId>,
 	since_shortstatehash: Option<ShortStateHash>,
 	current_shortstatehash: ShortStateHash,
-	timeline_pdus: &Vec<(PduCount, PduEvent)>,
-	timeline_users: &HashSet<String>,
+	joined_since_last_sync: bool,
+	witness: Option<&Witness>,
 ) -> Result<StateChanges> {
-	let since_sender_member: OptionFuture<_> = since_shortstatehash
-		.map(|short| {
-			services
-				.rooms
-				.state_accessor
-				.state_get_content(short, &StateEventType::RoomMember, sender_user.as_str())
-				.ok()
-		})
-		.into();
-
-	let joined_since_last_sync =
-		since_sender_member
-			.await
-			.flatten()
-			.is_none_or(|content: RoomMemberEventContent| {
-				content.membership != MembershipState::Join
-			});
-
 	if since_shortstatehash.is_none() || joined_since_last_sync {
 		calculate_state_initial(
 			services,
 			sender_user,
-			sender_device,
 			room_id,
-			next_batchcount,
-			lazy_load_enabled,
 			full_state,
+			filter,
 			current_shortstatehash,
-			timeline_users,
+			witness,
 		)
 		.await
 	} else {
 		calculate_state_incremental(
 			services,
 			sender_user,
-			sender_device,
 			room_id,
-			next_batchcount,
-			lazy_load_send_redundant,
 			full_state,
+			filter,
 			device_list_updates,
 			left_encrypted_users,
 			since_shortstatehash,
 			current_shortstatehash,
-			timeline_pdus,
 			joined_since_last_sync,
 		)
 		.await
@@ -912,87 +959,53 @@ async fn calculate_state_changes(
 async fn calculate_state_initial(
 	services: &Services,
 	sender_user: &UserId,
-	sender_device: &DeviceId,
 	room_id: &RoomId,
-	next_batchcount: PduCount,
-	lazy_load_enabled: bool,
 	full_state: bool,
+	filter: &FilterDefinition,
 	current_shortstatehash: ShortStateHash,
-	timeline_users: &HashSet<String>,
+	witness: Option<&Witness>,
 ) -> Result<StateChanges> {
-	// Probably since = 0, we will do an initial sync
-	let state = services
+	let (shortstatekeys, event_ids): (Vec<_>, Vec<_>) = services
 		.rooms
 		.state_accessor
 		.state_full_ids(current_shortstatehash)
-		.await?
-		.into_iter()
-		.stream()
-		.broad_filter_map(|(shortstatekey, event_id): (ShortStateKey, OwnedEventId)| {
-			services
-				.rooms
-				.short
-				.get_statekey_from_short(shortstatekey)
-				.map_ok(move |(event_type, state_key)| ((event_type, state_key), event_id))
-				.ok()
-		})
-		.fold((Vec::new(), HashSet::new()), |a, item| async move {
-			let (mut state_events, mut lazy_loaded) = a;
-			let ((event_type, state_key), event_id) = item;
+		.unzip()
+		.await;
 
-			if event_type != StateEventType::RoomMember {
-				let Ok(pdu) = services.rooms.timeline.get_pdu(&event_id).await else {
-					error!("Pdu in state not found: {event_id}");
-					return (state_events, lazy_loaded);
-				};
+	let state_events = services
+		.rooms
+		.short
+		.multi_get_statekey_from_short(shortstatekeys.into_iter().stream())
+		.zip(event_ids.into_iter().stream())
+		.ready_filter_map(|item| Some((item.0.ok()?, item.1)))
+		.ready_filter_map(|((event_type, state_key), event_id)| {
+			let lazy_load_enabled = filter.room.state.lazy_load_options.is_enabled()
+				|| filter.room.timeline.lazy_load_options.is_enabled();
 
-				state_events.push(pdu);
-				return (state_events, lazy_loaded);
+			if lazy_load_enabled
+				&& event_type == StateEventType::RoomMember
+				&& !full_state
+				&& state_key.as_str().try_into().is_ok_and(|user_id: &UserId| {
+					sender_user != user_id
+						&& witness.is_some_and(|witness| !witness.contains(user_id))
+				}) {
+				return None;
 			}
 
-			// TODO: Delete "element_hacks" when this is resolved: https://github.com/vector-im/element-web/issues/22565
-			if !lazy_load_enabled
-				|| full_state
-				|| timeline_users.contains(&state_key)
-				|| (cfg!(feature = "element_hacks") && *sender_user == state_key)
-			{
-				let Ok(pdu) = services.rooms.timeline.get_pdu(&event_id).await else {
-					error!("Pdu in state not found: {event_id}");
-					return (state_events, lazy_loaded);
-				};
-
-				// This check is in case a bad user ID made it into the database
-				if let Ok(uid) = OwnedUserId::parse(&state_key) {
-					lazy_loaded.insert(uid);
-				}
-
-				state_events.push(pdu);
-			}
-
-			(state_events, lazy_loaded)
+			Some(event_id)
 		})
+		.broad_filter_map(|event_id: OwnedEventId| async move {
+			services.rooms.timeline.get_pdu(&event_id).await.ok()
+		})
+		.collect()
 		.map(Ok);
 
 	let counts = calculate_counts(services, room_id, sender_user);
-	let ((joined_member_count, invited_member_count, heroes), (state_events, lazy_loaded)) =
-		try_join(counts, state).boxed().await?;
-
-	// Reset lazy loading because this is an initial sync
-	services
-		.rooms
-		.lazy_loading
-		.lazy_load_reset(sender_user, sender_device, room_id)
-		.await;
+	let ((joined_member_count, invited_member_count, heroes), state_events) =
+		try_join(counts, state_events).boxed().await?;
 
 	// The state_events above should contain all timeline_users, let's mark them as
 	// lazy loaded.
-	services.rooms.lazy_loading.lazy_load_mark_sent(
-		sender_user,
-		sender_device,
-		room_id,
-		lazy_loaded,
-		next_batchcount,
-	);
 
 	Ok(StateChanges {
 		heroes,
@@ -1008,16 +1021,13 @@ async fn calculate_state_initial(
 async fn calculate_state_incremental(
 	services: &Services,
 	sender_user: &UserId,
-	sender_device: &DeviceId,
 	room_id: &RoomId,
-	next_batchcount: PduCount,
-	lazy_load_send_redundant: bool,
 	full_state: bool,
+	_filter: &FilterDefinition,
 	device_list_updates: &mut HashSet<OwnedUserId>,
 	left_encrypted_users: &mut HashSet<OwnedUserId>,
 	since_shortstatehash: Option<ShortStateHash>,
 	current_shortstatehash: ShortStateHash,
-	timeline_pdus: &Vec<(PduCount, PduEvent)>,
 	joined_since_last_sync: bool,
 ) -> Result<StateChanges> {
 	// Incremental /sync
@@ -1030,17 +1040,19 @@ async fn calculate_state_incremental(
 		let current_state_ids = services
 			.rooms
 			.state_accessor
-			.state_full_ids(current_shortstatehash);
+			.state_full_ids(current_shortstatehash)
+			.collect();
 
 		let since_state_ids = services
 			.rooms
 			.state_accessor
-			.state_full_ids(since_shortstatehash);
+			.state_full_ids(since_shortstatehash)
+			.collect();
 
 		let (current_state_ids, since_state_ids): (
 			HashMap<_, OwnedEventId>,
 			HashMap<_, OwnedEventId>,
-		) = try_join(current_state_ids, since_state_ids).await?;
+		) = join(current_state_ids, since_state_ids).await;
 
 		current_state_ids
 			.iter()
@@ -1130,76 +1142,12 @@ async fn calculate_state_incremental(
 		(None, None, None)
 	};
 
-	let mut state_events = delta_state_events;
-
-	// Mark all member events we're returning as lazy-loaded
-	let mut lazy_loaded = state_events
-		.iter()
-		.filter(|pdu| pdu.kind == RoomMember)
-		.filter_map(|pdu| {
-			pdu.state_key
-				.clone()
-				.map(TryInto::try_into)
-				.map(LogDebugErr::log_debug_err)
-				.flat_ok()
-		})
-		.fold(HashSet::new(), |mut lazy_loaded, user_id| {
-			lazy_loaded.insert(user_id);
-			lazy_loaded
-		});
-
-	// Fetch contextual member state events for events from the timeline, and
-	// mark them as lazy-loaded as well.
-	for (_, event) in timeline_pdus {
-		if lazy_loaded.contains(&event.sender) {
-			continue;
-		}
-
-		let sent_before: OptionFuture<_> = (!lazy_load_send_redundant)
-			.then(|| {
-				services.rooms.lazy_loading.lazy_load_was_sent_before(
-					sender_user,
-					sender_device,
-					room_id,
-					&event.sender,
-				)
-			})
-			.into();
-
-		let member_event: OptionFuture<_> = sent_before
-			.await
-			.is_none_or(is_false!())
-			.then(|| {
-				services.rooms.state_accessor.room_state_get(
-					room_id,
-					&StateEventType::RoomMember,
-					event.sender.as_str(),
-				)
-			})
-			.into();
-
-		let Some(Ok(member_event)) = member_event.await else {
-			continue;
-		};
-
-		lazy_loaded.insert(event.sender.clone());
-		state_events.push(member_event);
-	}
-
-	services.rooms.lazy_loading.lazy_load_mark_sent(
-		sender_user,
-		sender_device,
-		room_id,
-		lazy_loaded,
-		next_batchcount,
-	);
-
 	Ok(StateChanges {
 		heroes,
 		joined_member_count,
 		invited_member_count,
 		joined_since_last_sync,
-		state_events,
+		state_events: delta_state_events,
 	})
 }
 
@@ -1223,7 +1171,7 @@ async fn calculate_counts(
 	let (joined_member_count, invited_member_count) =
 		join(joined_member_count, invited_member_count).await;
 
-	let small_room = joined_member_count.saturating_add(invited_member_count) > 5;
+	let small_room = joined_member_count.saturating_add(invited_member_count) <= 5;
 
 	let heroes: OptionFuture<_> = small_room
 		.then(|| calculate_heroes(services, room_id, sender_user))

src/api/client/sync/v4.rs

@@ -23,24 +23,23 @@ use ruma::{
 			DeviceLists, UnreadNotificationsCount,
 		},
 	},
-	directory::RoomTypeFilter,
 	events::{
 		room::member::{MembershipState, RoomMemberEventContent},
 		AnyRawAccountDataEvent, AnySyncEphemeralRoomEvent, StateEventType,
-		TimelineEventType::{self, *},
+		TimelineEventType::*,
 	},
 	serde::Raw,
-	uint, MilliSecondsSinceUnixEpoch, OwnedEventId, OwnedRoomId, OwnedUserId, UInt,
+	uint, MilliSecondsSinceUnixEpoch, OwnedEventId, OwnedRoomId, OwnedUserId, RoomId, UInt,
 };
-use service::{rooms::read_receipt::pack_receipts, Services};
+use service::rooms::read_receipt::pack_receipts;
 
 use super::{load_timeline, share_encrypted_room};
-use crate::{client::ignored_filter, Ruma};
+use crate::{
+	client::{filter_rooms, ignored_filter, sync::v5::TodoRooms, DEFAULT_BUMP_TYPES},
+	Ruma,
+};
 
-const SINGLE_CONNECTION_SYNC: &str = "single_connection_sync";
-
-const DEFAULT_BUMP_TYPES: &[TimelineEventType; 6] =
-	&[CallInvite, PollStart, Beacon, RoomEncrypted, RoomMessage, Sticker];
+pub(crate) const SINGLE_CONNECTION_SYNC: &str = "single_connection_sync";
 
 /// POST `/_matrix/client/unstable/org.matrix.msc3575/sync`
 ///
@@ -113,12 +112,27 @@ pub(crate) async fn sync_events_v4_route(
 		.collect()
 		.await;
 
-	let all_rooms = all_joined_rooms
+	let all_knocked_rooms: Vec<_> = services
+		.rooms
+		.state_cache
+		.rooms_knocked(sender_user)
+		.map(|r| r.0)
+		.collect()
+		.await;
+
+	let all_invited_rooms: Vec<&RoomId> = all_invited_rooms.iter().map(AsRef::as_ref).collect();
+	let all_knocked_rooms: Vec<&RoomId> = all_knocked_rooms.iter().map(AsRef::as_ref).collect();
+
+	let all_rooms: Vec<&RoomId> = all_joined_rooms
 		.iter()
-		.chain(all_invited_rooms.iter())
-		.map(Clone::clone)
+		.map(AsRef::as_ref)
+		.chain(all_invited_rooms.iter().map(AsRef::as_ref))
+		.chain(all_knocked_rooms.iter().map(AsRef::as_ref))
 		.collect();
 
+	let all_joined_rooms = all_joined_rooms.iter().map(AsRef::as_ref).collect();
+	let all_invited_rooms = all_invited_rooms.iter().map(AsRef::as_ref).collect();
+
 	if body.extensions.to_device.enabled.unwrap_or(false) {
 		services
 			.users
@@ -171,6 +185,7 @@ pub(crate) async fn sync_events_v4_route(
 		);
 
 		for room_id in &all_joined_rooms {
+			let room_id: &&RoomId = room_id;
 			let Ok(current_shortstatehash) =
 				services.rooms.state.get_room_shortstatehash(room_id).await
 			else {
@@ -226,13 +241,15 @@ pub(crate) async fn sync_events_v4_route(
 						.rooms
 						.state_accessor
 						.state_full_ids(current_shortstatehash)
-						.await?;
+						.collect()
+						.await;
 
-					let since_state_ids = services
+					let since_state_ids: HashMap<_, _> = services
 						.rooms
 						.state_accessor
 						.state_full_ids(since_shortstatehash)
-						.await?;
+						.collect()
+						.await;
 
 					for (key, id) in current_state_ids {
 						if since_state_ids.get(&key) != Some(&id) {
@@ -323,7 +340,7 @@ pub(crate) async fn sync_events_v4_route(
 	}
 
 	let mut lists = BTreeMap::new();
-	let mut todo_rooms = BTreeMap::new(); // and required state
+	let mut todo_rooms: TodoRooms = BTreeMap::new(); // and required state
 
 	for (list_id, list) in &body.lists {
 		let active_rooms = match list.filters.clone().and_then(|f| f.is_invite) {
@@ -344,7 +361,7 @@ pub(crate) async fn sync_events_v4_route(
 			| None => active_rooms,
 		};
 
-		let mut new_known_rooms = BTreeSet::new();
+		let mut new_known_rooms: BTreeSet<OwnedRoomId> = BTreeSet::new();
 
 		let ranges = list.ranges.clone();
 		lists.insert(list_id.clone(), sync_events::v4::SyncList {
@@ -366,9 +383,9 @@ pub(crate) async fn sync_events_v4_route(
 						Vec::new()
 					};
 
-					new_known_rooms.extend(room_ids.iter().cloned());
+					new_known_rooms.extend(room_ids.clone().into_iter().map(ToOwned::to_owned));
 					for room_id in &room_ids {
-						let todo_room = todo_rooms.entry(room_id.clone()).or_insert((
+						let todo_room = todo_rooms.entry((*room_id).to_owned()).or_insert((
 							BTreeSet::new(),
 							0_usize,
 							u64::MAX,
@@ -390,7 +407,7 @@ pub(crate) async fn sync_events_v4_route(
 						todo_room.2 = todo_room.2.min(
 							known_rooms
 								.get(list_id.as_str())
-								.and_then(|k| k.get(room_id))
+								.and_then(|k| k.get(*room_id))
 								.copied()
 								.unwrap_or(0),
 						);
@@ -399,7 +416,7 @@ pub(crate) async fn sync_events_v4_route(
 						op: SlidingOp::Sync,
 						range: Some(r),
 						index: None,
-						room_ids,
+						room_ids: room_ids.into_iter().map(ToOwned::to_owned).collect(),
 						room_id: None,
 					}
 				})
@@ -409,8 +426,8 @@ pub(crate) async fn sync_events_v4_route(
 
 		if let Some(conn_id) = &body.conn_id {
 			services.sync.update_sync_known_rooms(
-				sender_user.clone(),
-				sender_device.clone(),
+				sender_user,
+				&sender_device,
 				conn_id.clone(),
 				list_id.clone(),
 				new_known_rooms,
@@ -455,8 +472,8 @@ pub(crate) async fn sync_events_v4_route(
 
 	if let Some(conn_id) = &body.conn_id {
 		services.sync.update_sync_known_rooms(
-			sender_user.clone(),
-			sender_device.clone(),
+			sender_user,
+			&sender_device,
 			conn_id.clone(),
 			"subscriptions".to_owned(),
 			known_subscription_rooms,
@@ -480,7 +497,8 @@ pub(crate) async fn sync_events_v4_route(
 		let mut timestamp: Option<_> = None;
 		let mut invite_state = None;
 		let (timeline_pdus, limited);
-		if all_invited_rooms.contains(room_id) {
+		let new_room_id: &RoomId = (*room_id).as_ref();
+		if all_invited_rooms.contains(&new_room_id) {
 			// TODO: figure out a timestamp we can use for remote invites
 			invite_state = services
 				.rooms
@@ -510,7 +528,7 @@ pub(crate) async fn sync_events_v4_route(
 		}
 
 		account_data.rooms.insert(
-			room_id.clone(),
+			room_id.to_owned(),
 			services
 				.account_data
 				.changes_since(Some(room_id), sender_user, *roomsince)
@@ -740,10 +758,9 @@ pub(crate) async fn sync_events_v4_route(
 		});
 	}
 
-	if rooms
-		.iter()
-		.all(|(_, r)| r.timeline.is_empty() && r.required_state.is_empty())
-	{
+	if rooms.iter().all(|(id, r)| {
+		r.timeline.is_empty() && r.required_state.is_empty() && !receipts.rooms.contains_key(id)
+	}) {
 		// Hang a few seconds so requests are not spammed
 		// Stop hanging if new info arrives
 		let default = Duration::from_secs(30);
@@ -789,33 +806,3 @@ pub(crate) async fn sync_events_v4_route(
 		delta_token: None,
 	})
 }
-
-async fn filter_rooms(
-	services: &Services,
-	rooms: &[OwnedRoomId],
-	filter: &[RoomTypeFilter],
-	negate: bool,
-) -> Vec<OwnedRoomId> {
-	rooms
-		.iter()
-		.stream()
-		.filter_map(|r| async move {
-			let room_type = services.rooms.state_accessor.get_room_type(r).await;
-
-			if room_type.as_ref().is_err_and(|e| !e.is_not_found()) {
-				return None;
-			}
-
-			let room_type_filter = RoomTypeFilter::from(room_type.ok());
-
-			let include = if negate {
-				!filter.contains(&room_type_filter)
-			} else {
-				filter.is_empty() || filter.contains(&room_type_filter)
-			};
-
-			include.then_some(r.to_owned())
-		})
-		.collect()
-		.await
-}

src/api/client/sync/v5.rs

@@ -0,0 +1,888 @@
+use std::{
+	cmp::{self, Ordering},
+	collections::{BTreeMap, BTreeSet, HashMap, HashSet},
+	time::Duration,
+};
+
+use axum::extract::State;
+use conduwuit::{
+	debug, error, extract_variant, trace,
+	utils::{
+		math::{ruma_from_usize, usize_from_ruma},
+		BoolExt, IterStream, ReadyExt, TryFutureExtExt,
+	},
+	warn, Error, Result,
+};
+use futures::{FutureExt, StreamExt, TryFutureExt};
+use ruma::{
+	api::client::{
+		error::ErrorKind,
+		sync::sync_events::{self, DeviceLists, UnreadNotificationsCount},
+	},
+	events::{
+		room::member::{MembershipState, RoomMemberEventContent},
+		AnyRawAccountDataEvent, AnySyncEphemeralRoomEvent, StateEventType, TimelineEventType,
+	},
+	serde::Raw,
+	state_res::TypeStateKey,
+	uint, DeviceId, OwnedEventId, OwnedRoomId, OwnedUserId, RoomId, UInt, UserId,
+};
+use service::{rooms::read_receipt::pack_receipts, PduCount};
+
+use super::{filter_rooms, share_encrypted_room};
+use crate::{
+	client::{ignored_filter, sync::load_timeline, DEFAULT_BUMP_TYPES},
+	Ruma,
+};
+
+type SyncInfo<'a> = (&'a UserId, &'a DeviceId, u64, &'a sync_events::v5::Request);
+
+/// `POST /_matrix/client/unstable/org.matrix.simplified_msc3575/sync`
+/// ([MSC4186])
+///
+/// A simplified version of sliding sync ([MSC3575]).
+///
+/// Get all new events in a sliding window of rooms since the last sync or a
+/// given point in time.
+///
+/// [MSC3575]: https://github.com/matrix-org/matrix-spec-proposals/pull/3575
+/// [MSC4186]: https://github.com/matrix-org/matrix-spec-proposals/pull/4186
+pub(crate) async fn sync_events_v5_route(
+	State(services): State<crate::State>,
+	body: Ruma<sync_events::v5::Request>,
+) -> Result<sync_events::v5::Response> {
+	debug_assert!(DEFAULT_BUMP_TYPES.is_sorted(), "DEFAULT_BUMP_TYPES is not sorted");
+	let sender_user = body.sender_user.as_ref().expect("user is authenticated");
+	let sender_device = body.sender_device.as_ref().expect("user is authenticated");
+	let mut body = body.body;
+
+	// Setup watchers, so if there's no response, we can wait for them
+	let watcher = services.sync.watch(sender_user, sender_device);
+
+	let next_batch = services.globals.next_count()?;
+
+	let conn_id = body.conn_id.clone();
+
+	let globalsince = body
+		.pos
+		.as_ref()
+		.and_then(|string| string.parse().ok())
+		.unwrap_or(0);
+
+	if globalsince != 0
+		&& !services.sync.snake_connection_cached(
+			sender_user.clone(),
+			sender_device.clone(),
+			conn_id.clone(),
+		) {
+		debug!("Restarting sync stream because it was gone from the database");
+		return Err(Error::Request(
+			ErrorKind::UnknownPos,
+			"Connection data lost since last time".into(),
+			http::StatusCode::BAD_REQUEST,
+		));
+	}
+
+	// Client / User requested an initial sync
+	if globalsince == 0 {
+		services.sync.forget_snake_sync_connection(
+			sender_user.clone(),
+			sender_device.clone(),
+			conn_id.clone(),
+		);
+	}
+
+	// Get sticky parameters from cache
+	let known_rooms = services.sync.update_snake_sync_request_with_cache(
+		sender_user.clone(),
+		sender_device.clone(),
+		&mut body,
+	);
+
+	let all_joined_rooms: Vec<_> = services
+		.rooms
+		.state_cache
+		.rooms_joined(sender_user)
+		.map(ToOwned::to_owned)
+		.collect()
+		.await;
+
+	let all_invited_rooms: Vec<_> = services
+		.rooms
+		.state_cache
+		.rooms_invited(sender_user)
+		.map(|r| r.0)
+		.collect()
+		.await;
+
+	let all_knocked_rooms: Vec<_> = services
+		.rooms
+		.state_cache
+		.rooms_knocked(sender_user)
+		.map(|r| r.0)
+		.collect()
+		.await;
+
+	let all_rooms: Vec<&RoomId> = all_joined_rooms
+		.iter()
+		.map(AsRef::as_ref)
+		.chain(all_invited_rooms.iter().map(AsRef::as_ref))
+		.chain(all_knocked_rooms.iter().map(AsRef::as_ref))
+		.collect();
+
+	let all_joined_rooms = all_joined_rooms.iter().map(AsRef::as_ref).collect();
+	let all_invited_rooms = all_invited_rooms.iter().map(AsRef::as_ref).collect();
+
+	let pos = next_batch.clone().to_string();
+
+	let mut todo_rooms: TodoRooms = BTreeMap::new();
+
+	let sync_info: SyncInfo<'_> = (sender_user, sender_device, globalsince, &body);
+	let mut response = sync_events::v5::Response {
+		txn_id: body.txn_id.clone(),
+		pos,
+		lists: BTreeMap::new(),
+		rooms: BTreeMap::new(),
+		extensions: sync_events::v5::response::Extensions {
+			account_data: collect_account_data(services, sync_info).await,
+			e2ee: collect_e2ee(services, sync_info, &all_joined_rooms).await?,
+			to_device: collect_to_device(services, sync_info, next_batch).await,
+			receipts: collect_receipts(services).await,
+			typing: sync_events::v5::response::Typing::default(),
+		},
+	};
+
+	handle_lists(
+		services,
+		sync_info,
+		&all_invited_rooms,
+		&all_joined_rooms,
+		&all_rooms,
+		&mut todo_rooms,
+		&known_rooms,
+		&mut response,
+	)
+	.await;
+
+	fetch_subscriptions(services, sync_info, &known_rooms, &mut todo_rooms).await;
+
+	response.rooms = process_rooms(
+		services,
+		sender_user,
+		next_batch,
+		&all_invited_rooms,
+		&todo_rooms,
+		&mut response,
+		&body,
+	)
+	.await?;
+
+	if response.rooms.iter().all(|(id, r)| {
+		r.timeline.is_empty()
+			&& r.required_state.is_empty()
+			&& !response.extensions.receipts.rooms.contains_key(id)
+	}) && response
+		.extensions
+		.to_device
+		.clone()
+		.is_none_or(|to| to.events.is_empty())
+	{
+		// Hang a few seconds so requests are not spammed
+		// Stop hanging if new info arrives
+		let default = Duration::from_secs(30);
+		let duration = cmp::min(body.timeout.unwrap_or(default), default);
+		_ = tokio::time::timeout(duration, watcher).await;
+	}
+
+	trace!(
+		rooms=?response.rooms.len(),
+		account_data=?response.extensions.account_data.rooms.len(),
+		receipts=?response.extensions.receipts.rooms.len(),
+		"responding to request with"
+	);
+	Ok(response)
+}
+
+type KnownRooms = BTreeMap<String, BTreeMap<OwnedRoomId, u64>>;
+pub(crate) type TodoRooms = BTreeMap<OwnedRoomId, (BTreeSet<TypeStateKey>, usize, u64)>;
+
+async fn fetch_subscriptions(
+	services: crate::State,
+	(sender_user, sender_device, globalsince, body): SyncInfo<'_>,
+	known_rooms: &KnownRooms,
+	todo_rooms: &mut TodoRooms,
+) {
+	let mut known_subscription_rooms = BTreeSet::new();
+	for (room_id, room) in &body.room_subscriptions {
+		if !services.rooms.metadata.exists(room_id).await {
+			continue;
+		}
+		let todo_room =
+			todo_rooms
+				.entry(room_id.clone())
+				.or_insert((BTreeSet::new(), 0_usize, u64::MAX));
+
+		let limit: UInt = room.timeline_limit;
+
+		todo_room.0.extend(room.required_state.iter().cloned());
+		todo_room.1 = todo_room.1.max(usize_from_ruma(limit));
+		// 0 means unknown because it got out of date
+		todo_room.2 = todo_room.2.min(
+			known_rooms
+				.get("subscriptions")
+				.and_then(|k| k.get(room_id))
+				.copied()
+				.unwrap_or(0),
+		);
+		known_subscription_rooms.insert(room_id.clone());
+	}
+	// where this went (protomsc says it was removed)
+	//for r in body.unsubscribe_rooms {
+	//	known_subscription_rooms.remove(&r);
+	//	body.room_subscriptions.remove(&r);
+	//}
+
+	if let Some(conn_id) = &body.conn_id {
+		services.sync.update_snake_sync_known_rooms(
+			sender_user,
+			sender_device,
+			conn_id.clone(),
+			"subscriptions".to_owned(),
+			known_subscription_rooms,
+			globalsince,
+		);
+	}
+}
+
+#[allow(clippy::too_many_arguments)]
+async fn handle_lists<'a>(
+	services: crate::State,
+	(sender_user, sender_device, globalsince, body): SyncInfo<'_>,
+	all_invited_rooms: &Vec<&'a RoomId>,
+	all_joined_rooms: &Vec<&'a RoomId>,
+	all_rooms: &Vec<&'a RoomId>,
+	todo_rooms: &'a mut TodoRooms,
+	known_rooms: &'a KnownRooms,
+	response: &'_ mut sync_events::v5::Response,
+) -> KnownRooms {
+	for (list_id, list) in &body.lists {
+		let active_rooms = match list.filters.clone().and_then(|f| f.is_invite) {
+			| Some(true) => all_invited_rooms,
+			| Some(false) => all_joined_rooms,
+			| None => all_rooms,
+		};
+
+		let active_rooms = match list.filters.clone().map(|f| f.not_room_types) {
+			| Some(filter) if filter.is_empty() => active_rooms,
+			| Some(value) => &filter_rooms(&services, active_rooms, &value, true).await,
+			| None => active_rooms,
+		};
+
+		let mut new_known_rooms: BTreeSet<OwnedRoomId> = BTreeSet::new();
+
+		let ranges = list.ranges.clone();
+
+		for mut range in ranges {
+			range.0 = uint!(0);
+			range.1 = range
+				.1
+				.clamp(range.0, UInt::try_from(active_rooms.len()).unwrap_or(UInt::MAX));
+
+			let room_ids =
+				active_rooms[usize_from_ruma(range.0)..usize_from_ruma(range.1)].to_vec();
+
+			let new_rooms: BTreeSet<OwnedRoomId> =
+				room_ids.clone().into_iter().map(From::from).collect();
+			new_known_rooms.extend(new_rooms);
+			//new_known_rooms.extend(room_ids..cloned());
+			for room_id in room_ids {
+				let todo_room = todo_rooms.entry(room_id.to_owned()).or_insert((
+					BTreeSet::new(),
+					0_usize,
+					u64::MAX,
+				));
+
+				let limit: usize = usize_from_ruma(list.room_details.timeline_limit).min(100);
+
+				todo_room
+					.0
+					.extend(list.room_details.required_state.iter().cloned());
+
+				todo_room.1 = todo_room.1.max(limit);
+				// 0 means unknown because it got out of date
+				todo_room.2 = todo_room.2.min(
+					known_rooms
+						.get(list_id.as_str())
+						.and_then(|k| k.get(room_id))
+						.copied()
+						.unwrap_or(0),
+				);
+			}
+		}
+		response
+			.lists
+			.insert(list_id.clone(), sync_events::v5::response::List {
+				count: ruma_from_usize(active_rooms.len()),
+			});
+
+		if let Some(conn_id) = &body.conn_id {
+			services.sync.update_snake_sync_known_rooms(
+				sender_user,
+				sender_device,
+				conn_id.clone(),
+				list_id.clone(),
+				new_known_rooms,
+				globalsince,
+			);
+		}
+	}
+	BTreeMap::default()
+}
+
+async fn process_rooms(
+	services: crate::State,
+	sender_user: &UserId,
+	next_batch: u64,
+	all_invited_rooms: &[&RoomId],
+	todo_rooms: &TodoRooms,
+	response: &mut sync_events::v5::Response,
+	body: &sync_events::v5::Request,
+) -> Result<BTreeMap<OwnedRoomId, sync_events::v5::response::Room>> {
+	let mut rooms = BTreeMap::new();
+	for (room_id, (required_state_request, timeline_limit, roomsince)) in todo_rooms {
+		let roomsincecount = PduCount::Normal(*roomsince);
+
+		let mut timestamp: Option<_> = None;
+		let mut invite_state = None;
+		let (timeline_pdus, limited);
+		let new_room_id: &RoomId = (*room_id).as_ref();
+		if all_invited_rooms.contains(&new_room_id) {
+			// TODO: figure out a timestamp we can use for remote invites
+			invite_state = services
+				.rooms
+				.state_cache
+				.invite_state(sender_user, room_id)
+				.await
+				.ok();
+
+			(timeline_pdus, limited) = (Vec::new(), true);
+		} else {
+			(timeline_pdus, limited) = match load_timeline(
+				&services,
+				sender_user,
+				room_id,
+				roomsincecount,
+				Some(PduCount::from(next_batch)),
+				*timeline_limit,
+			)
+			.await
+			{
+				| Ok(value) => value,
+				| Err(err) => {
+					warn!("Encountered missing timeline in {}, error {}", room_id, err);
+					continue;
+				},
+			};
+		}
+
+		if body.extensions.account_data.enabled == Some(true) {
+			response.extensions.account_data.rooms.insert(
+				room_id.to_owned(),
+				services
+					.account_data
+					.changes_since(Some(room_id), sender_user, *roomsince)
+					.ready_filter_map(|e| extract_variant!(e, AnyRawAccountDataEvent::Room))
+					.collect()
+					.await,
+			);
+		}
+
+		let last_privateread_update = services
+			.rooms
+			.read_receipt
+			.last_privateread_update(sender_user, room_id)
+			.await > *roomsince;
+
+		let private_read_event = if last_privateread_update {
+			services
+				.rooms
+				.read_receipt
+				.private_read_get(room_id, sender_user)
+				.await
+				.ok()
+		} else {
+			None
+		};
+
+		let mut receipts: Vec<Raw<AnySyncEphemeralRoomEvent>> = services
+			.rooms
+			.read_receipt
+			.readreceipts_since(room_id, *roomsince)
+			.filter_map(|(read_user, _ts, v)| async move {
+				services
+					.users
+					.user_is_ignored(read_user, sender_user)
+					.await
+					.or_some(v)
+			})
+			.collect()
+			.await;
+
+		if let Some(private_read_event) = private_read_event {
+			receipts.push(private_read_event);
+		}
+
+		let receipt_size = receipts.len();
+
+		if receipt_size > 0 {
+			response
+				.extensions
+				.receipts
+				.rooms
+				.insert(room_id.clone(), pack_receipts(Box::new(receipts.into_iter())));
+		}
+
+		if roomsince != &0
+			&& timeline_pdus.is_empty()
+			&& response
+				.extensions
+				.account_data
+				.rooms
+				.get(room_id)
+				.is_none_or(Vec::is_empty)
+			&& receipt_size == 0
+		{
+			continue;
+		}
+
+		let prev_batch = timeline_pdus
+			.first()
+			.map_or(Ok::<_, Error>(None), |(pdu_count, _)| {
+				Ok(Some(match pdu_count {
+					| PduCount::Backfilled(_) => {
+						error!("timeline in backfill state?!");
+						"0".to_owned()
+					},
+					| PduCount::Normal(c) => c.to_string(),
+				}))
+			})?
+			.or_else(|| {
+				if roomsince != &0 {
+					Some(roomsince.to_string())
+				} else {
+					None
+				}
+			});
+
+		let room_events: Vec<_> = timeline_pdus
+			.iter()
+			.stream()
+			.filter_map(|item| ignored_filter(&services, item.clone(), sender_user))
+			.map(|(_, pdu)| pdu.to_sync_room_event())
+			.collect()
+			.await;
+
+		for (_, pdu) in timeline_pdus {
+			let ts = pdu.origin_server_ts;
+			if DEFAULT_BUMP_TYPES.binary_search(&pdu.kind).is_ok()
+				&& timestamp.is_none_or(|time| time <= ts)
+			{
+				timestamp = Some(ts);
+			}
+		}
+
+		let required_state = required_state_request
+			.iter()
+			.stream()
+			.filter_map(|state| async move {
+				services
+					.rooms
+					.state_accessor
+					.room_state_get(room_id, &state.0, &state.1)
+					.await
+					.map(|s| s.to_sync_state_event())
+					.ok()
+			})
+			.collect()
+			.await;
+
+		// Heroes
+		let heroes: Vec<_> = services
+			.rooms
+			.state_cache
+			.room_members(room_id)
+			.ready_filter(|member| *member != sender_user)
+			.filter_map(|user_id| {
+				services
+					.rooms
+					.state_accessor
+					.get_member(room_id, user_id)
+					.map_ok(|memberevent| sync_events::v5::response::Hero {
+						user_id: user_id.into(),
+						name: memberevent.displayname,
+						avatar: memberevent.avatar_url,
+					})
+					.ok()
+			})
+			.take(5)
+			.collect()
+			.await;
+
+		let name = match heroes.len().cmp(&(1_usize)) {
+			| Ordering::Greater => {
+				let firsts = heroes[1..]
+					.iter()
+					.map(|h| h.name.clone().unwrap_or_else(|| h.user_id.to_string()))
+					.collect::<Vec<_>>()
+					.join(", ");
+
+				let last = heroes[0]
+					.name
+					.clone()
+					.unwrap_or_else(|| heroes[0].user_id.to_string());
+
+				Some(format!("{firsts} and {last}"))
+			},
+			| Ordering::Equal => Some(
+				heroes[0]
+					.name
+					.clone()
+					.unwrap_or_else(|| heroes[0].user_id.to_string()),
+			),
+			| Ordering::Less => None,
+		};
+
+		let heroes_avatar = if heroes.len() == 1 {
+			heroes[0].avatar.clone()
+		} else {
+			None
+		};
+
+		rooms.insert(room_id.clone(), sync_events::v5::response::Room {
+			name: services
+				.rooms
+				.state_accessor
+				.get_name(room_id)
+				.await
+				.ok()
+				.or(name),
+			avatar: if let Some(heroes_avatar) = heroes_avatar {
+				ruma::JsOption::Some(heroes_avatar)
+			} else {
+				match services.rooms.state_accessor.get_avatar(room_id).await {
+					| ruma::JsOption::Some(avatar) => ruma::JsOption::from_option(avatar.url),
+					| ruma::JsOption::Null => ruma::JsOption::Null,
+					| ruma::JsOption::Undefined => ruma::JsOption::Undefined,
+				}
+			},
+			initial: Some(roomsince == &0),
+			is_dm: None,
+			invite_state,
+			unread_notifications: UnreadNotificationsCount {
+				highlight_count: Some(
+					services
+						.rooms
+						.user
+						.highlight_count(sender_user, room_id)
+						.await
+						.try_into()
+						.expect("notification count can't go that high"),
+				),
+				notification_count: Some(
+					services
+						.rooms
+						.user
+						.notification_count(sender_user, room_id)
+						.await
+						.try_into()
+						.expect("notification count can't go that high"),
+				),
+			},
+			timeline: room_events,
+			required_state,
+			prev_batch,
+			limited,
+			joined_count: Some(
+				services
+					.rooms
+					.state_cache
+					.room_joined_count(room_id)
+					.await
+					.unwrap_or(0)
+					.try_into()
+					.unwrap_or_else(|_| uint!(0)),
+			),
+			invited_count: Some(
+				services
+					.rooms
+					.state_cache
+					.room_invited_count(room_id)
+					.await
+					.unwrap_or(0)
+					.try_into()
+					.unwrap_or_else(|_| uint!(0)),
+			),
+			num_live: None, // Count events in timeline greater than global sync counter
+			bump_stamp: timestamp,
+			heroes: Some(heroes),
+		});
+	}
+	Ok(rooms)
+}
+async fn collect_account_data(
+	services: crate::State,
+	(sender_user, _, globalsince, body): (&UserId, &DeviceId, u64, &sync_events::v5::Request),
+) -> sync_events::v5::response::AccountData {
+	let mut account_data = sync_events::v5::response::AccountData {
+		global: Vec::new(),
+		rooms: BTreeMap::new(),
+	};
+
+	if !body.extensions.account_data.enabled.unwrap_or(false) {
+		return sync_events::v5::response::AccountData::default();
+	}
+
+	account_data.global = services
+		.account_data
+		.changes_since(None, sender_user, globalsince)
+		.ready_filter_map(|e| extract_variant!(e, AnyRawAccountDataEvent::Global))
+		.collect()
+		.await;
+
+	if let Some(rooms) = &body.extensions.account_data.rooms {
+		for room in rooms {
+			account_data.rooms.insert(
+				room.clone(),
+				services
+					.account_data
+					.changes_since(Some(room), sender_user, globalsince)
+					.ready_filter_map(|e| extract_variant!(e, AnyRawAccountDataEvent::Room))
+					.collect()
+					.await,
+			);
+		}
+	}
+
+	account_data
+}
+
+async fn collect_e2ee<'a>(
+	services: crate::State,
+	(sender_user, sender_device, globalsince, body): (
+		&UserId,
+		&DeviceId,
+		u64,
+		&sync_events::v5::Request,
+	),
+	all_joined_rooms: &'a Vec<&'a RoomId>,
+) -> Result<sync_events::v5::response::E2EE> {
+	if !body.extensions.e2ee.enabled.unwrap_or(false) {
+		return Ok(sync_events::v5::response::E2EE::default());
+	}
+	let mut left_encrypted_users = HashSet::new(); // Users that have left any encrypted rooms the sender was in
+	let mut device_list_changes = HashSet::new();
+	let mut device_list_left = HashSet::new();
+	// Look for device list updates of this account
+	device_list_changes.extend(
+		services
+			.users
+			.keys_changed(sender_user, globalsince, None)
+			.map(ToOwned::to_owned)
+			.collect::<Vec<_>>()
+			.await,
+	);
+
+	for room_id in all_joined_rooms {
+		let Ok(current_shortstatehash) =
+			services.rooms.state.get_room_shortstatehash(room_id).await
+		else {
+			error!("Room {room_id} has no state");
+			continue;
+		};
+
+		let since_shortstatehash = services
+			.rooms
+			.user
+			.get_token_shortstatehash(room_id, globalsince)
+			.await
+			.ok();
+
+		let encrypted_room = services
+			.rooms
+			.state_accessor
+			.state_get(current_shortstatehash, &StateEventType::RoomEncryption, "")
+			.await
+			.is_ok();
+
+		if let Some(since_shortstatehash) = since_shortstatehash {
+			// Skip if there are only timeline changes
+			if since_shortstatehash == current_shortstatehash {
+				continue;
+			}
+
+			let since_encryption = services
+				.rooms
+				.state_accessor
+				.state_get(since_shortstatehash, &StateEventType::RoomEncryption, "")
+				.await;
+
+			let since_sender_member: Option<RoomMemberEventContent> = services
+				.rooms
+				.state_accessor
+				.state_get_content(
+					since_shortstatehash,
+					&StateEventType::RoomMember,
+					sender_user.as_str(),
+				)
+				.ok()
+				.await;
+
+			let joined_since_last_sync = since_sender_member
+				.as_ref()
+				.is_none_or(|member| member.membership != MembershipState::Join);
+
+			let new_encrypted_room = encrypted_room && since_encryption.is_err();
+
+			if encrypted_room {
+				let current_state_ids: HashMap<_, OwnedEventId> = services
+					.rooms
+					.state_accessor
+					.state_full_ids(current_shortstatehash)
+					.collect()
+					.await;
+
+				let since_state_ids: HashMap<_, _> = services
+					.rooms
+					.state_accessor
+					.state_full_ids(since_shortstatehash)
+					.collect()
+					.await;
+
+				for (key, id) in current_state_ids {
+					if since_state_ids.get(&key) != Some(&id) {
+						let Ok(pdu) = services.rooms.timeline.get_pdu(&id).await else {
+							error!("Pdu in state not found: {id}");
+							continue;
+						};
+						if pdu.kind == TimelineEventType::RoomMember {
+							if let Some(state_key) = &pdu.state_key {
+								let user_id =
+									OwnedUserId::parse(state_key.clone()).map_err(|_| {
+										Error::bad_database("Invalid UserId in member PDU.")
+									})?;
+
+								if user_id == *sender_user {
+									continue;
+								}
+
+								let content: RoomMemberEventContent = pdu.get_content()?;
+								match content.membership {
+									| MembershipState::Join => {
+										// A new user joined an encrypted room
+										if !share_encrypted_room(
+											&services,
+											sender_user,
+											&user_id,
+											Some(room_id),
+										)
+										.await
+										{
+											device_list_changes.insert(user_id);
+										}
+									},
+									| MembershipState::Leave => {
+										// Write down users that have left encrypted rooms we
+										// are in
+										left_encrypted_users.insert(user_id);
+									},
+									| _ => {},
+								}
+							}
+						}
+					}
+				}
+				if joined_since_last_sync || new_encrypted_room {
+					// If the user is in a new encrypted room, give them all joined users
+					device_list_changes.extend(
+						services
+						.rooms
+						.state_cache
+						.room_members(room_id)
+						// Don't send key updates from the sender to the sender
+						.ready_filter(|user_id| sender_user != *user_id)
+						// Only send keys if the sender doesn't share an encrypted room with the target
+						// already
+						.filter_map(|user_id| {
+							share_encrypted_room(&services, sender_user, user_id, Some(room_id))
+								.map(|res| res.or_some(user_id.to_owned()))
+						})
+						.collect::<Vec<_>>()
+						.await,
+					);
+				}
+			}
+		}
+		// Look for device list updates in this room
+		device_list_changes.extend(
+			services
+				.users
+				.room_keys_changed(room_id, globalsince, None)
+				.map(|(user_id, _)| user_id)
+				.map(ToOwned::to_owned)
+				.collect::<Vec<_>>()
+				.await,
+		);
+	}
+
+	for user_id in left_encrypted_users {
+		let dont_share_encrypted_room =
+			!share_encrypted_room(&services, sender_user, &user_id, None).await;
+
+		// If the user doesn't share an encrypted room with the target anymore, we need
+		// to tell them
+		if dont_share_encrypted_room {
+			device_list_left.insert(user_id);
+		}
+	}
+
+	Ok(sync_events::v5::response::E2EE {
+		device_lists: DeviceLists {
+			changed: device_list_changes.into_iter().collect(),
+			left: device_list_left.into_iter().collect(),
+		},
+		device_one_time_keys_count: services
+			.users
+			.count_one_time_keys(sender_user, sender_device)
+			.await,
+		device_unused_fallback_key_types: None,
+	})
+}
+
+async fn collect_to_device(
+	services: crate::State,
+	(sender_user, sender_device, globalsince, body): SyncInfo<'_>,
+	next_batch: u64,
+) -> Option<sync_events::v5::response::ToDevice> {
+	if !body.extensions.to_device.enabled.unwrap_or(false) {
+		return None;
+	}
+
+	services
+		.users
+		.remove_to_device_events(sender_user, sender_device, globalsince)
+		.await;
+
+	Some(sync_events::v5::response::ToDevice {
+		next_batch: next_batch.to_string(),
+		events: services
+			.users
+			.get_to_device_events(sender_user, sender_device)
+			.collect()
+			.await,
+	})
+}
+
+async fn collect_receipts(_services: crate::State) -> sync_events::v5::response::Receipts {
+	sync_events::v5::response::Receipts { rooms: BTreeMap::new() }
+	// TODO: get explicitly requested read receipts
+}

src/api/client/to_device.rs

@@ -10,6 +10,7 @@ use ruma::{
 	},
 	to_device::DeviceIdOrAllDevices,
 };
+use service::sending::EduBuf;
 
 use crate::Ruma;
 
@@ -42,18 +43,21 @@ pub(crate) async fn send_event_to_device_route(
 				messages.insert(target_user_id.clone(), map);
 				let count = services.globals.next_count()?;
 
-				services.sending.send_edu_server(
-					target_user_id.server_name(),
-					serde_json::to_vec(&federation::transactions::edu::Edu::DirectToDevice(
-						DirectDeviceContent {
-							sender: sender_user.clone(),
-							ev_type: body.event_type.clone(),
-							message_id: count.to_string().into(),
-							messages,
-						},
-					))
-					.expect("DirectToDevice EDU can be serialized"),
-				)?;
+				let mut buf = EduBuf::new();
+				serde_json::to_writer(
+					&mut buf,
+					&federation::transactions::edu::Edu::DirectToDevice(DirectDeviceContent {
+						sender: sender_user.clone(),
+						ev_type: body.event_type.clone(),
+						message_id: count.to_string().into(),
+						messages,
+					}),
+				)
+				.expect("DirectToDevice EDU can be serialized");
+
+				services
+					.sending
+					.send_edu_server(target_user_id.server_name(), buf)?;
 
 				continue;
 			}

src/api/client/typing.rs

@@ -1,5 +1,5 @@
 use axum::extract::State;
-use conduwuit::Err;
+use conduwuit::{utils::math::Tried, Err};
 use ruma::api::client::typing::create_typing_event;
 
 use crate::{utils, Result, Ruma};
@@ -31,17 +31,15 @@ pub(crate) async fn create_typing_event_route(
 		let duration = utils::clamp(
 			duration.as_millis().try_into().unwrap_or(u64::MAX),
 			services
-				.globals
+				.server
 				.config
 				.typing_client_timeout_min_s
-				.checked_mul(1000)
-				.unwrap(),
+				.try_mul(1000)?,
 			services
-				.globals
+				.server
 				.config
 				.typing_client_timeout_max_s
-				.checked_mul(1000)
-				.unwrap(),
+				.try_mul(1000)?,
 		);
 		services
 			.rooms

src/api/client/unversioned.rs

@@ -52,6 +52,7 @@ pub(crate) async fn get_supported_versions_route(
 			("org.matrix.msc4180".to_owned(), true), /* stable flag for 3916 (https://github.com/matrix-org/matrix-spec-proposals/pull/4180) */
 			("uk.tcpip.msc4133".to_owned(), true), /* Extending User Profile API with Key:Value Pairs (https://github.com/matrix-org/matrix-spec-proposals/pull/4133) */
 			("us.cloke.msc4175".to_owned(), true), /* Profile field for user time zone (https://github.com/matrix-org/matrix-spec-proposals/pull/4175) */
+			("org.matrix.simplified_msc3575".to_owned(), true), /* Simplified Sliding sync (https://github.com/matrix-org/matrix-spec-proposals/pull/4186) */
 		]),
 	};
 

src/api/client/voip.rs

@@ -38,7 +38,7 @@ pub(crate) async fn turn_server_route(
 		let user = body.sender_user.unwrap_or_else(|| {
 			UserId::parse_with_server_name(
 				utils::random_string(RANDOM_USER_ID_LENGTH).to_lowercase(),
-				&services.globals.config.server_name,
+				&services.server.name,
 			)
 			.unwrap()
 		});

src/api/router.rs

@@ -34,6 +34,7 @@ pub fn build(router: Router<State>, server: &Server) -> Router<State> {
 		.ruma_route(&client::register_route)
 		.ruma_route(&client::get_login_types_route)
 		.ruma_route(&client::login_route)
+		.ruma_route(&client::login_token_route)
 		.ruma_route(&client::whoami_route)
 		.ruma_route(&client::logout_route)
 		.ruma_route(&client::logout_all_route)
@@ -99,6 +100,7 @@ pub fn build(router: Router<State>, server: &Server) -> Router<State> {
 		.ruma_route(&client::join_room_by_id_route)
 		.ruma_route(&client::join_room_by_id_or_alias_route)
 		.ruma_route(&client::joined_members_route)
+		.ruma_route(&client::knock_room_route)
 		.ruma_route(&client::leave_room_route)
 		.ruma_route(&client::forget_room_route)
 		.ruma_route(&client::joined_rooms_route)
@@ -144,6 +146,7 @@ pub fn build(router: Router<State>, server: &Server) -> Router<State> {
 		)
 		.ruma_route(&client::sync_events_route)
 		.ruma_route(&client::sync_events_v4_route)
+		.ruma_route(&client::sync_events_v5_route)
 		.ruma_route(&client::get_context_route)
 		.ruma_route(&client::get_message_events_route)
 		.ruma_route(&client::search_events_route)
@@ -204,8 +207,10 @@ pub fn build(router: Router<State>, server: &Server) -> Router<State> {
 			.ruma_route(&server::get_room_state_route)
 			.ruma_route(&server::get_room_state_ids_route)
 			.ruma_route(&server::create_leave_event_template_route)
+			.ruma_route(&server::create_knock_event_template_route)
 			.ruma_route(&server::create_leave_event_v1_route)
 			.ruma_route(&server::create_leave_event_v2_route)
+			.ruma_route(&server::create_knock_event_v1_route)
 			.ruma_route(&server::create_join_event_template_route)
 			.ruma_route(&server::create_join_event_v1_route)
 			.ruma_route(&server::create_join_event_v2_route)

src/api/router/auth.rs

@@ -71,7 +71,7 @@ pub(super) async fn auth(
 		match metadata {
 			| &get_public_rooms::v3::Request::METADATA => {
 				if !services
-					.globals
+					.server
 					.config
 					.allow_public_room_directory_without_auth
 				{
@@ -94,7 +94,7 @@ pub(super) async fn auth(
 			| &get_display_name::v3::Request::METADATA
 			| &get_avatar_url::v3::Request::METADATA
 			| &get_timezone_key::unstable::Request::METADATA => {
-				if services.globals.config.require_auth_for_profile_requests {
+				if services.server.config.require_auth_for_profile_requests {
 					match token {
 						| Token::Appservice(_) | Token::User(_) => {
 							// we should have validated the token above
@@ -127,7 +127,7 @@ pub(super) async fn auth(
 		}),
 		| (AuthScheme::AccessToken, Token::None) => match metadata {
 			| &get_turn_server_info::v3::Request::METADATA => {
-				if services.globals.config.turn_allow_guests {
+				if services.server.config.turn_allow_guests {
 					Ok(Auth {
 						origin: None,
 						sender_user: None,

src/api/router/request.rs

@@ -32,7 +32,7 @@ pub(super) async fn from(
 	let query = serde_html_form::from_str(query)
 		.map_err(|e| err!(Request(Unknown("Failed to read query parameters: {e}"))))?;
 
-	let max_body_size = services.globals.config.max_request_size;
+	let max_body_size = services.server.config.max_request_size;
 
 	let body = axum::body::to_bytes(body, max_body_size)
 		.await

src/api/server/backfill.rs

@@ -2,10 +2,10 @@ use std::cmp;
 
 use axum::extract::State;
 use conduwuit::{
-	utils::{IterStream, ReadyExt},
+	utils::{stream::TryTools, IterStream, ReadyExt},
 	PduCount, Result,
 };
-use futures::{FutureExt, StreamExt};
+use futures::{FutureExt, StreamExt, TryStreamExt};
 use ruma::{api::federation::backfill::get_backfill, uint, MilliSecondsSinceUnixEpoch};
 
 use super::AccessCheck;
@@ -57,26 +57,30 @@ pub(crate) async fn get_backfill_route(
 			.rooms
 			.timeline
 			.pdus_rev(None, &body.room_id, Some(from.saturating_add(1)))
-			.await?
-			.take(limit)
-			.filter_map(|(_, pdu)| async move {
-				services
+			.try_take(limit)
+			.try_filter_map(|(_, pdu)| async move {
+				Ok(services
 					.rooms
 					.state_accessor
 					.server_can_see_event(body.origin(), &pdu.room_id, &pdu.event_id)
 					.await
-					.then_some(pdu)
+					.then_some(pdu))
 			})
-			.filter_map(|pdu| async move {
-				services
+			.try_filter_map(|pdu| async move {
+				Ok(services
 					.rooms
 					.timeline
 					.get_pdu_json(&pdu.event_id)
 					.await
-					.ok()
+					.ok())
 			})
-			.then(|pdu| services.sending.convert_to_outgoing_federation_event(pdu))
-			.collect()
-			.await,
+			.and_then(|pdu| {
+				services
+					.sending
+					.convert_to_outgoing_federation_event(pdu)
+					.map(Ok)
+			})
+			.try_collect()
+			.await?,
 	})
 }

src/api/server/invite.rs

@@ -6,8 +6,9 @@ use ruma::{
 	api::{client::error::ErrorKind, federation::membership::create_invite},
 	events::room::member::{MembershipState, RoomMemberEventContent},
 	serde::JsonObject,
-	CanonicalJsonValue, OwnedEventId, OwnedUserId, UserId,
+	CanonicalJsonValue, OwnedUserId, UserId,
 };
+use service::pdu::gen_event_id;
 
 use crate::Ruma;
 
@@ -36,7 +37,7 @@ pub(crate) async fn create_invite_route(
 
 	if let Some(server) = body.room_id.server_name() {
 		if services
-			.globals
+			.server
 			.config
 			.forbidden_remote_server_names
 			.contains(&server.to_owned())
@@ -46,7 +47,7 @@ pub(crate) async fn create_invite_route(
 	}
 
 	if services
-		.globals
+		.server
 		.config
 		.forbidden_remote_server_names
 		.contains(body.origin())
@@ -86,12 +87,7 @@ pub(crate) async fn create_invite_route(
 		.map_err(|e| err!(Request(InvalidParam("Failed to sign event: {e}"))))?;
 
 	// Generate event id
-	let event_id = OwnedEventId::parse(format!(
-		"${}",
-		ruma::signatures::reference_hash(&signed_event, &body.room_version)
-			.expect("ruma can calculate reference hashes")
-	))
-	.expect("ruma's reference hashes are valid event ids");
+	let event_id = gen_event_id(&signed_event, &body.room_version)?;
 
 	// Add event_id back
 	signed_event.insert("event_id".to_owned(), CanonicalJsonValue::String(event_id.to_string()));
@@ -115,12 +111,12 @@ pub(crate) async fn create_invite_route(
 	let mut invite_state = body.invite_room_state.clone();
 
 	let mut event: JsonObject = serde_json::from_str(body.event.get())
-		.map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "Invalid invite event bytes."))?;
+		.map_err(|e| err!(Request(BadJson("Invalid invite event PDU: {e}"))))?;
 
 	event.insert("event_id".to_owned(), "$placeholder".into());
 
 	let pdu: PduEvent = serde_json::from_value(event.into())
-		.map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "Invalid invite event."))?;
+		.map_err(|e| err!(Request(BadJson("Invalid invite event PDU: {e}"))))?;
 
 	invite_state.push(pdu.to_stripped_state_event());
 

src/api/server/make_join.rs

@@ -42,7 +42,7 @@ pub(crate) async fn create_join_event_template_route(
 		.await?;
 
 	if services
-		.globals
+		.server
 		.config
 		.forbidden_remote_server_names
 		.contains(body.origin())
@@ -59,7 +59,7 @@ pub(crate) async fn create_join_event_template_route(
 
 	if let Some(server) = body.room_id.server_name() {
 		if services
-			.globals
+			.server
 			.config
 			.forbidden_remote_server_names
 			.contains(&server.to_owned())

src/api/server/make_knock.rs

@@ -1,5 +1,5 @@
 use axum::extract::State;
-use conduwuit::Err;
+use conduwuit::{debug_warn, Err};
 use ruma::{
 	api::{client::error::ErrorKind, federation::knock::create_knock_event_template},
 	events::room::member::{MembershipState, RoomMemberEventContent},
@@ -15,7 +15,8 @@ use crate::{service::pdu::PduBuilder, Error, Result, Ruma};
 ///
 /// Creates a knock template.
 pub(crate) async fn create_knock_event_template_route(
-	State(services): State<crate::State>, body: Ruma<create_knock_event_template::v1::Request>,
+	State(services): State<crate::State>,
+	body: Ruma<create_knock_event_template::v1::Request>,
 ) -> Result<create_knock_event_template::v1::Response> {
 	if !services.rooms.metadata.exists(&body.room_id).await {
 		return Err!(Request(NotFound("Room is unknown to this server.")));
@@ -33,14 +34,14 @@ pub(crate) async fn create_knock_event_template_route(
 		.await?;
 
 	if services
-		.globals
+		.server
 		.config
 		.forbidden_remote_server_names
 		.contains(body.origin())
 	{
 		warn!(
-			"Server {} for remote user {} tried knocking room ID {} which has a server name that is globally \
-			 forbidden. Rejecting.",
+			"Server {} for remote user {} tried knocking room ID {} which has a server name \
+			 that is globally forbidden. Rejecting.",
 			body.origin(),
 			&body.user_id,
 			&body.room_id,
@@ -50,7 +51,7 @@ pub(crate) async fn create_knock_event_template_route(
 
 	if let Some(server) = body.room_id.server_name() {
 		if services
-			.globals
+			.server
 			.config
 			.forbidden_remote_server_names
 			.contains(&server.to_owned())
@@ -63,29 +64,44 @@ pub(crate) async fn create_knock_event_template_route(
 
 	if matches!(room_version_id, V1 | V2 | V3 | V4 | V5 | V6) {
 		return Err(Error::BadRequest(
-			ErrorKind::IncompatibleRoomVersion {
-				room_version: room_version_id,
-			},
+			ErrorKind::IncompatibleRoomVersion { room_version: room_version_id },
 			"Room version does not support knocking.",
 		));
 	}
 
 	if !body.ver.contains(&room_version_id) {
 		return Err(Error::BadRequest(
-			ErrorKind::IncompatibleRoomVersion {
-				room_version: room_version_id,
-			},
+			ErrorKind::IncompatibleRoomVersion { room_version: room_version_id },
 			"Your homeserver does not support the features required to knock on this room.",
 		));
 	}
 
 	let state_lock = services.rooms.state.mutex.lock(&body.room_id).await;
 
+	if let Ok(membership) = services
+		.rooms
+		.state_accessor
+		.get_member(&body.room_id, &body.user_id)
+		.await
+	{
+		if membership.membership == MembershipState::Ban {
+			debug_warn!(
+				"Remote user {} is banned from {} but attempted to knock",
+				&body.user_id,
+				&body.room_id
+			);
+			return Err!(Request(Forbidden("You cannot knock on a room you are banned from.")));
+		}
+	}
+
 	let (_pdu, mut pdu_json) = services
 		.rooms
 		.timeline
 		.create_hash_and_sign_event(
-			PduBuilder::state(body.user_id.to_string(), &RoomMemberEventContent::new(MembershipState::Knock)),
+			PduBuilder::state(
+				body.user_id.to_string(),
+				&RoomMemberEventContent::new(MembershipState::Knock),
+			),
 			&body.user_id,
 			&body.room_id,
 			&state_lock,

src/api/server/make_leave.rs

@@ -9,7 +9,7 @@ use serde_json::value::to_raw_value;
 use super::make_join::maybe_strip_event_id;
 use crate::{service::pdu::PduBuilder, Ruma};
 
-/// # `PUT /_matrix/federation/v1/make_leave/{roomId}/{eventId}`
+/// # `GET /_matrix/federation/v1/make_leave/{roomId}/{eventId}`
 ///
 /// Creates a leave template.
 pub(crate) async fn create_leave_event_template_route(
@@ -21,7 +21,9 @@ pub(crate) async fn create_leave_event_template_route(
 	}
 
 	if body.user_id.server_name() != body.origin() {
-		return Err!(Request(BadJson("Not allowed to leave on behalf of another server/user.")));
+		return Err!(Request(Forbidden(
+			"Not allowed to leave on behalf of another server/user."
+		)));
 	}
 
 	// ACL check origin

src/api/server/media.rs

@@ -14,7 +14,12 @@ use crate::Ruma;
 /// # `GET /_matrix/federation/v1/media/download/{mediaId}`
 ///
 /// Load media from our server.
-#[tracing::instrument(skip_all, fields(%client), name = "media_get")]
+#[tracing::instrument(
+	name = "media_get",
+	level = "debug",
+	skip_all,
+	fields(%client)
+)]
 pub(crate) async fn get_content_route(
 	State(services): State<crate::State>,
 	InsecureClientIp(client): InsecureClientIp,
@@ -51,7 +56,12 @@ pub(crate) async fn get_content_route(
 /// # `GET /_matrix/federation/v1/media/thumbnail/{mediaId}`
 ///
 /// Load media thumbnail from our server.
-#[tracing::instrument(skip_all, fields(%client), name = "media_thumbnail_get")]
+#[tracing::instrument(
+	name = "media_thumbnail_get",
+	level = "debug",
+	skip_all,
+	fields(%client)
+)]
 pub(crate) async fn get_content_thumbnail_route(
 	State(services): State<crate::State>,
 	InsecureClientIp(client): InsecureClientIp,

src/api/server/mod.rs

@@ -6,6 +6,7 @@ pub(super) mod hierarchy;
 pub(super) mod invite;
 pub(super) mod key;
 pub(super) mod make_join;
+pub(super) mod make_knock;
 pub(super) mod make_leave;
 pub(super) mod media;
 pub(super) mod openid;
@@ -13,6 +14,7 @@ pub(super) mod publicrooms;
 pub(super) mod query;
 pub(super) mod send;
 pub(super) mod send_join;
+pub(super) mod send_knock;
 pub(super) mod send_leave;
 pub(super) mod state;
 pub(super) mod state_ids;
@@ -28,6 +30,7 @@ pub(super) use hierarchy::*;
 pub(super) use invite::*;
 pub(super) use key::*;
 pub(super) use make_join::*;
+pub(super) use make_knock::*;
 pub(super) use make_leave::*;
 pub(super) use media::*;
 pub(super) use openid::*;
@@ -35,6 +38,7 @@ pub(super) use publicrooms::*;
 pub(super) use query::*;
 pub(super) use send::*;
 pub(super) use send_join::*;
+pub(super) use send_knock::*;
 pub(super) use send_leave::*;
 pub(super) use state::*;
 pub(super) use state_ids::*;

src/api/server/publicrooms.rs

@@ -13,14 +13,14 @@ use crate::{Error, Result, Ruma};
 /// # `POST /_matrix/federation/v1/publicRooms`
 ///
 /// Lists the public rooms on this server.
-#[tracing::instrument(skip_all, fields(%client), name = "publicrooms")]
+#[tracing::instrument(name = "publicrooms", level = "debug", skip_all, fields(%client))]
 pub(crate) async fn get_public_rooms_filtered_route(
 	State(services): State<crate::State>,
 	InsecureClientIp(client): InsecureClientIp,
 	body: Ruma<get_public_rooms_filtered::v1::Request>,
 ) -> Result<get_public_rooms_filtered::v1::Response> {
 	if !services
-		.globals
+		.server
 		.config
 		.allow_public_room_directory_over_federation
 	{
@@ -51,7 +51,7 @@ pub(crate) async fn get_public_rooms_filtered_route(
 /// # `GET /_matrix/federation/v1/publicRooms`
 ///
 /// Lists the public rooms on this server.
-#[tracing::instrument(skip_all, fields(%client), "publicrooms")]
+#[tracing::instrument(name = "publicrooms", level = "debug", skip_all, fields(%client))]
 pub(crate) async fn get_public_rooms_route(
 	State(services): State<crate::State>,
 	InsecureClientIp(client): InsecureClientIp,

src/api/server/query.rs

@@ -63,7 +63,7 @@ pub(crate) async fn get_profile_information_route(
 	body: Ruma<get_profile_information::v1::Request>,
 ) -> Result<get_profile_information::v1::Response> {
 	if !services
-		.globals
+		.server
 		.config
 		.allow_inbound_profile_lookup_federation_requests
 	{

src/api/server/send.rs

@@ -3,17 +3,27 @@ use std::{collections::BTreeMap, net::IpAddr, time::Instant};
 use axum::extract::State;
 use axum_client_ip::InsecureClientIp;
 use conduwuit::{
-	debug, debug_warn, err, error, result::LogErr, trace, utils::ReadyExt, warn, Err, Error,
-	Result,
+	debug,
+	debug::INFO_SPAN_LEVEL,
+	debug_warn, err, error,
+	result::LogErr,
+	trace,
+	utils::{
+		stream::{automatic_width, BroadbandExt, TryBroadbandExt},
+		IterStream, ReadyExt,
+	},
+	warn, Err, Error, Result,
 };
-use futures::{FutureExt, StreamExt};
+use futures::{FutureExt, Stream, StreamExt, TryFutureExt, TryStreamExt};
+use itertools::Itertools;
 use ruma::{
 	api::{
 		client::error::ErrorKind,
 		federation::transactions::{
 			edu::{
 				DeviceListUpdateContent, DirectDeviceContent, Edu, PresenceContent,
-				ReceiptContent, SigningKeyUpdateContent, TypingContent,
+				PresenceUpdate, ReceiptContent, ReceiptData, ReceiptMap, SigningKeyUpdateContent,
+				TypingContent,
 			},
 			send_transaction_message,
 		},
@@ -21,25 +31,34 @@ use ruma::{
 	events::receipt::{ReceiptEvent, ReceiptEventContent, ReceiptType},
 	serde::Raw,
 	to_device::DeviceIdOrAllDevices,
-	OwnedEventId, ServerName,
+	CanonicalJsonObject, OwnedEventId, OwnedRoomId, OwnedUserId, RoomId, ServerName, UserId,
 };
-use serde_json::value::RawValue as RawJsonValue;
 use service::{
 	sending::{EDU_LIMIT, PDU_LIMIT},
 	Services,
 };
+use utils::millis_since_unix_epoch;
 
 use crate::{
 	utils::{self},
 	Ruma,
 };
 
-type ResolvedMap = BTreeMap<OwnedEventId, Result<()>>;
+type ResolvedMap = BTreeMap<OwnedEventId, Result>;
+type Pdu = (OwnedRoomId, OwnedEventId, CanonicalJsonObject);
 
 /// # `PUT /_matrix/federation/v1/send/{txnId}`
 ///
 /// Push EDUs and PDUs to this server.
-#[tracing::instrument(skip_all, fields(%client, origin = body.origin().as_str()), name = "send")]
+#[tracing::instrument(
+	name = "txn",
+	level = INFO_SPAN_LEVEL,
+	skip_all,
+	fields(
+		%client,
+		origin = body.origin().as_str()
+	),
+)]
 pub(crate) async fn send_transaction_message_route(
 	State(services): State<crate::State>,
 	InsecureClientIp(client): InsecureClientIp,
@@ -65,90 +84,41 @@ pub(crate) async fn send_transaction_message_route(
 
 	let txn_start_time = Instant::now();
 	trace!(
-		pdus = ?body.pdus.len(),
-		edus = ?body.edus.len(),
+		pdus = body.pdus.len(),
+		edus = body.edus.len(),
 		elapsed = ?txn_start_time.elapsed(),
 		id = ?body.transaction_id,
 		origin =?body.origin(),
 		"Starting txn",
 	);
 
-	let resolved_map =
-		handle_pdus(&services, &client, &body.pdus, body.origin(), &txn_start_time)
-			.boxed()
-			.await?;
+	let pdus = body
+		.pdus
+		.iter()
+		.stream()
+		.broad_then(|pdu| services.rooms.event_handler.parse_incoming_pdu(pdu))
+		.inspect_err(|e| debug_warn!("Could not parse PDU: {e}"))
+		.ready_filter_map(Result::ok);
 
-	handle_edus(&services, &client, &body.edus, body.origin())
-		.boxed()
-		.await;
+	let edus = body
+		.edus
+		.iter()
+		.map(|edu| edu.json().get())
+		.map(serde_json::from_str)
+		.filter_map(Result::ok)
+		.stream();
+
+	let results = handle(&services, &client, body.origin(), txn_start_time, pdus, edus).await?;
 
 	debug!(
-		pdus = ?body.pdus.len(),
-		edus = ?body.edus.len(),
+		pdus = body.pdus.len(),
+		edus = body.edus.len(),
 		elapsed = ?txn_start_time.elapsed(),
 		id = ?body.transaction_id,
 		origin =?body.origin(),
 		"Finished txn",
 	);
-
-	Ok(send_transaction_message::v1::Response {
-		pdus: resolved_map
-			.into_iter()
-			.map(|(e, r)| (e, r.map_err(error::sanitized_message)))
-			.collect(),
-	})
-}
-
-async fn handle_pdus(
-	services: &Services,
-	_client: &IpAddr,
-	pdus: &[Box<RawJsonValue>],
-	origin: &ServerName,
-	txn_start_time: &Instant,
-) -> Result<ResolvedMap> {
-	let mut parsed_pdus = Vec::with_capacity(pdus.len());
-	for pdu in pdus {
-		parsed_pdus.push(match services.rooms.event_handler.parse_incoming_pdu(pdu).await {
-			| Ok(t) => t,
-			| Err(e) => {
-				debug_warn!("Could not parse PDU: {e}");
-				continue;
-			},
-		});
-
-		// We do not add the event_id field to the pdu here because of signature
-		// and hashes checks
-	}
-
-	let mut resolved_map = BTreeMap::new();
-	for (event_id, value, room_id) in parsed_pdus {
-		services.server.check_running()?;
-		let pdu_start_time = Instant::now();
-		let mutex_lock = services
-			.rooms
-			.event_handler
-			.mutex_federation
-			.lock(&room_id)
-			.await;
-
-		let result = services
-			.rooms
-			.event_handler
-			.handle_incoming_pdu(origin, &room_id, &event_id, value, true)
-			.await
-			.map(|_| ());
-
-		drop(mutex_lock);
-		debug!(
-			pdu_elapsed = ?pdu_start_time.elapsed(),
-			txn_elapsed = ?txn_start_time.elapsed(),
-			"Finished PDU {event_id}",
-		);
-
-		resolved_map.insert(event_id, result);
-	}
-
-	for (id, result) in &resolved_map {
+	for (id, result) in &results {
 		if let Err(e) = result {
 			if matches!(e, Error::BadRequest(ErrorKind::NotFound, _)) {
 				warn!("Incoming PDU failed {id}: {e:?}");
@@ -156,39 +126,117 @@ async fn handle_pdus(
 		}
 	}
 
-	Ok(resolved_map)
+	Ok(send_transaction_message::v1::Response {
+		pdus: results
+			.into_iter()
+			.map(|(e, r)| (e, r.map_err(error::sanitized_message)))
+			.collect(),
+	})
 }
 
-async fn handle_edus(
+async fn handle(
 	services: &Services,
 	client: &IpAddr,
-	edus: &[Raw<Edu>],
 	origin: &ServerName,
-) {
-	for edu in edus
-		.iter()
-		.filter_map(|edu| serde_json::from_str::<Edu>(edu.json().get()).ok())
-	{
-		match edu {
-			| Edu::Presence(presence) => {
-				handle_edu_presence(services, client, origin, presence).await;
-			},
-			| Edu::Receipt(receipt) =>
-				handle_edu_receipt(services, client, origin, receipt).await,
-			| Edu::Typing(typing) => handle_edu_typing(services, client, origin, typing).await,
-			| Edu::DeviceListUpdate(content) => {
-				handle_edu_device_list_update(services, client, origin, content).await;
-			},
-			| Edu::DirectToDevice(content) => {
-				handle_edu_direct_to_device(services, client, origin, content).await;
-			},
-			| Edu::SigningKeyUpdate(content) => {
-				handle_edu_signing_key_update(services, client, origin, content).await;
-			},
-			| Edu::_Custom(ref _custom) => {
-				debug_warn!(?edus, "received custom/unknown EDU");
-			},
-		}
+	started: Instant,
+	pdus: impl Stream<Item = Pdu> + Send,
+	edus: impl Stream<Item = Edu> + Send,
+) -> Result<ResolvedMap> {
+	// group pdus by room
+	let pdus = pdus
+		.collect()
+		.map(|mut pdus: Vec<_>| {
+			pdus.sort_by(|(room_a, ..), (room_b, ..)| room_a.cmp(room_b));
+			pdus.into_iter()
+				.into_grouping_map_by(|(room_id, ..)| room_id.clone())
+				.collect()
+		})
+		.await;
+
+	// we can evaluate rooms concurrently
+	let results: ResolvedMap = pdus
+		.into_iter()
+		.try_stream()
+		.broad_and_then(|(room_id, pdus): (_, Vec<_>)| {
+			handle_room(services, client, origin, started, room_id, pdus.into_iter())
+				.map_ok(Vec::into_iter)
+				.map_ok(IterStream::try_stream)
+		})
+		.try_flatten()
+		.try_collect()
+		.boxed()
+		.await?;
+
+	// evaluate edus after pdus, at least for now.
+	edus.for_each_concurrent(automatic_width(), |edu| handle_edu(services, client, origin, edu))
+		.boxed()
+		.await;
+
+	Ok(results)
+}
+
+async fn handle_room(
+	services: &Services,
+	_client: &IpAddr,
+	origin: &ServerName,
+	txn_start_time: Instant,
+	room_id: OwnedRoomId,
+	pdus: impl Iterator<Item = Pdu> + Send,
+) -> Result<Vec<(OwnedEventId, Result)>> {
+	let _room_lock = services
+		.rooms
+		.event_handler
+		.mutex_federation
+		.lock(&room_id)
+		.await;
+
+	let room_id = &room_id;
+	pdus.try_stream()
+		.and_then(|(_, event_id, value)| async move {
+			services.server.check_running()?;
+			let pdu_start_time = Instant::now();
+			let result = services
+				.rooms
+				.event_handler
+				.handle_incoming_pdu(origin, room_id, &event_id, value, true)
+				.await
+				.map(|_| ());
+
+			debug!(
+				pdu_elapsed = ?pdu_start_time.elapsed(),
+				txn_elapsed = ?txn_start_time.elapsed(),
+				"Finished PDU {event_id}",
+			);
+
+			Ok((event_id, result))
+		})
+		.try_collect()
+		.await
+}
+
+async fn handle_edu(services: &Services, client: &IpAddr, origin: &ServerName, edu: Edu) {
+	match edu {
+		| Edu::Presence(presence) if services.server.config.allow_incoming_presence =>
+			handle_edu_presence(services, client, origin, presence).await,
+
+		| Edu::Receipt(receipt) if services.server.config.allow_incoming_read_receipts =>
+			handle_edu_receipt(services, client, origin, receipt).await,
+
+		| Edu::Typing(typing) if services.server.config.allow_incoming_typing =>
+			handle_edu_typing(services, client, origin, typing).await,
+
+		| Edu::DeviceListUpdate(content) =>
+			handle_edu_device_list_update(services, client, origin, content).await,
+
+		| Edu::DirectToDevice(content) =>
+			handle_edu_direct_to_device(services, client, origin, content).await,
+
+		| Edu::SigningKeyUpdate(content) =>
+			handle_edu_signing_key_update(services, client, origin, content).await,
+
+		| Edu::_Custom(ref _custom) => debug_warn!(?edu, "received custom/unknown EDU"),
+
+		| _ => trace!(?edu, "skipped"),
 	}
 }
 
@@ -198,32 +246,41 @@ async fn handle_edu_presence(
 	origin: &ServerName,
 	presence: PresenceContent,
 ) {
-	if !services.globals.allow_incoming_presence() {
+	presence
+		.push
+		.into_iter()
+		.stream()
+		.for_each_concurrent(automatic_width(), |update| {
+			handle_edu_presence_update(services, origin, update)
+		})
+		.await;
+}
+
+async fn handle_edu_presence_update(
+	services: &Services,
+	origin: &ServerName,
+	update: PresenceUpdate,
+) {
+	if update.user_id.server_name() != origin {
+		debug_warn!(
+			%update.user_id, %origin,
+			"received presence EDU for user not belonging to origin"
+		);
 		return;
 	}
 
-	for update in presence.push {
-		if update.user_id.server_name() != origin {
-			debug_warn!(
-				%update.user_id, %origin,
-				"received presence EDU for user not belonging to origin"
-			);
-			continue;
-		}
-
-		services
-			.presence
-			.set_presence(
-				&update.user_id,
-				&update.presence,
-				Some(update.currently_active),
-				Some(update.last_active_ago),
-				update.status_msg.clone(),
-			)
-			.await
-			.log_err()
-			.ok();
-	}
+	services
+		.presence
+		.set_presence(
+			&update.user_id,
+			&update.presence,
+			Some(update.currently_active),
+			Some(update.last_active_ago),
+			update.status_msg.clone(),
+		)
+		.await
+		.log_err()
+		.ok();
 }
 
 async fn handle_edu_receipt(
@@ -232,66 +289,94 @@ async fn handle_edu_receipt(
 	origin: &ServerName,
 	receipt: ReceiptContent,
 ) {
-	if !services.globals.allow_incoming_read_receipts() {
+	receipt
+		.receipts
+		.into_iter()
+		.stream()
+		.for_each_concurrent(automatic_width(), |(room_id, room_updates)| {
+			handle_edu_receipt_room(services, origin, room_id, room_updates)
+		})
+		.await;
+}
+
+async fn handle_edu_receipt_room(
+	services: &Services,
+	origin: &ServerName,
+	room_id: OwnedRoomId,
+	room_updates: ReceiptMap,
+) {
+	if services
+		.rooms
+		.event_handler
+		.acl_check(origin, &room_id)
+		.await
+		.is_err()
+	{
+		debug_warn!(
+			%origin, %room_id,
+			"received read receipt EDU from ACL'd server"
+		);
 		return;
 	}
 
-	for (room_id, room_updates) in receipt.receipts {
-		if services
-			.rooms
-			.event_handler
-			.acl_check(origin, &room_id)
-			.await
-			.is_err()
-		{
-			debug_warn!(
-				%origin, %room_id,
-				"received read receipt EDU from ACL'd server"
-			);
-			continue;
-		}
+	let room_id = &room_id;
+	room_updates
+		.read
+		.into_iter()
+		.stream()
+		.for_each_concurrent(automatic_width(), |(user_id, user_updates)| async move {
+			handle_edu_receipt_room_user(services, origin, room_id, &user_id, user_updates).await;
+		})
+		.await;
+}
 
-		for (user_id, user_updates) in room_updates.read {
-			if user_id.server_name() != origin {
-				debug_warn!(
-					%user_id, %origin,
-					"received read receipt EDU for user not belonging to origin"
-				);
-				continue;
-			}
-
-			if services
-				.rooms
-				.state_cache
-				.room_members(&room_id)
-				.ready_any(|member| member.server_name() == user_id.server_name())
-				.await
-			{
-				for event_id in &user_updates.event_ids {
-					let user_receipts =
-						BTreeMap::from([(user_id.clone(), user_updates.data.clone())]);
-					let receipts = BTreeMap::from([(ReceiptType::Read, user_receipts)]);
-					let receipt_content = BTreeMap::from([(event_id.to_owned(), receipts)]);
-					let event = ReceiptEvent {
-						content: ReceiptEventContent(receipt_content),
-						room_id: room_id.clone(),
-					};
-
-					services
-						.rooms
-						.read_receipt
-						.readreceipt_update(&user_id, &room_id, &event)
-						.await;
-				}
-			} else {
-				debug_warn!(
-					%user_id, %room_id, %origin,
-					"received read receipt EDU from server who does not have a member in the room",
-				);
-				continue;
-			}
-		}
+async fn handle_edu_receipt_room_user(
+	services: &Services,
+	origin: &ServerName,
+	room_id: &RoomId,
+	user_id: &UserId,
+	user_updates: ReceiptData,
+) {
+	if user_id.server_name() != origin {
+		debug_warn!(
+			%user_id, %origin,
+			"received read receipt EDU for user not belonging to origin"
+		);
+		return;
 	}
+
+	if !services
+		.rooms
+		.state_cache
+		.server_in_room(origin, room_id)
+		.await
+	{
+		debug_warn!(
+			%user_id, %room_id, %origin,
+			"received read receipt EDU from server who does not have a member in the room",
+		);
+		return;
+	}
+
+	let data = &user_updates.data;
+	user_updates
+		.event_ids
+		.into_iter()
+		.stream()
+		.for_each_concurrent(automatic_width(), |event_id| async move {
+			let user_data = [(user_id.to_owned(), data.clone())];
+			let receipts = [(ReceiptType::Read, BTreeMap::from(user_data))];
+			let content = [(event_id.clone(), BTreeMap::from(receipts))];
+			services
+				.rooms
+				.read_receipt
+				.readreceipt_update(user_id, room_id, &ReceiptEvent {
+					content: ReceiptEventContent(content.into()),
+					room_id: room_id.to_owned(),
+				})
+				.await;
+		})
+		.await;
 }
 
 async fn handle_edu_typing(
@@ -300,10 +385,6 @@ async fn handle_edu_typing(
 	origin: &ServerName,
 	typing: TypingContent,
 ) {
-	if !services.globals.config.allow_incoming_typing {
-		return;
-	}
-
 	if typing.user_id.server_name() != origin {
 		debug_warn!(
 			%typing.user_id, %origin,
@@ -326,41 +407,38 @@ async fn handle_edu_typing(
 		return;
 	}
 
-	if services
+	if !services
 		.rooms
 		.state_cache
 		.is_joined(&typing.user_id, &typing.room_id)
 		.await
 	{
-		if typing.typing {
-			let timeout = utils::millis_since_unix_epoch().saturating_add(
-				services
-					.globals
-					.config
-					.typing_federation_timeout_s
-					.saturating_mul(1000),
-			);
-			services
-				.rooms
-				.typing
-				.typing_add(&typing.user_id, &typing.room_id, timeout)
-				.await
-				.log_err()
-				.ok();
-		} else {
-			services
-				.rooms
-				.typing
-				.typing_remove(&typing.user_id, &typing.room_id)
-				.await
-				.log_err()
-				.ok();
-		}
-	} else {
 		debug_warn!(
 			%typing.user_id, %typing.room_id, %origin,
 			"received typing EDU for user not in room"
 		);
+		return;
+	}
+
+	if typing.typing {
+		let secs = services.server.config.typing_federation_timeout_s;
+		let timeout = millis_since_unix_epoch().saturating_add(secs.saturating_mul(1000));
+
+		services
+			.rooms
+			.typing
+			.typing_add(&typing.user_id, &typing.room_id, timeout)
+			.await
+			.log_err()
+			.ok();
+	} else {
+		services
+			.rooms
+			.typing
+			.typing_remove(&typing.user_id, &typing.room_id)
+			.await
+			.log_err()
+			.ok();
 	}
 }
 
@@ -389,7 +467,12 @@ async fn handle_edu_direct_to_device(
 	origin: &ServerName,
 	content: DirectDeviceContent,
 ) {
-	let DirectDeviceContent { sender, ev_type, message_id, messages } = content;
+	let DirectDeviceContent {
+		ref sender,
+		ref ev_type,
+		ref message_id,
+		messages,
+	} = content;
 
 	if sender.server_name() != origin {
 		debug_warn!(
@@ -402,60 +485,88 @@ async fn handle_edu_direct_to_device(
 	// Check if this is a new transaction id
 	if services
 		.transaction_ids
-		.existing_txnid(&sender, None, &message_id)
+		.existing_txnid(sender, None, message_id)
 		.await
 		.is_ok()
 	{
 		return;
 	}
 
-	for (target_user_id, map) in &messages {
-		for (target_device_id_maybe, event) in map {
-			let Ok(event) = event.deserialize_as().map_err(|e| {
-				err!(Request(InvalidParam(error!("To-Device event is invalid: {e}"))))
-			}) else {
-				continue;
-			};
-
-			let ev_type = ev_type.to_string();
-			match target_device_id_maybe {
-				| DeviceIdOrAllDevices::DeviceId(target_device_id) => {
-					services
-						.users
-						.add_to_device_event(
-							&sender,
-							target_user_id,
-							target_device_id,
-							&ev_type,
-							event,
-						)
-						.await;
-				},
-
-				| DeviceIdOrAllDevices::AllDevices => {
-					let (sender, ev_type, event) = (&sender, &ev_type, &event);
-					services
-						.users
-						.all_device_ids(target_user_id)
-						.for_each(|target_device_id| {
-							services.users.add_to_device_event(
-								sender,
-								target_user_id,
-								target_device_id,
-								ev_type,
-								event.clone(),
-							)
-						})
-						.await;
-				},
-			}
-		}
-	}
+	// process messages concurrently for different users
+	let ev_type = ev_type.to_string();
+	messages
+		.into_iter()
+		.stream()
+		.for_each_concurrent(automatic_width(), |(target_user_id, map)| {
+			handle_edu_direct_to_device_user(services, target_user_id, sender, &ev_type, map)
+		})
+		.await;
 
 	// Save transaction id with empty data
 	services
 		.transaction_ids
-		.add_txnid(&sender, None, &message_id, &[]);
+		.add_txnid(sender, None, message_id, &[]);
+}
+
+async fn handle_edu_direct_to_device_user<Event: Send + Sync>(
+	services: &Services,
+	target_user_id: OwnedUserId,
+	sender: &UserId,
+	ev_type: &str,
+	map: BTreeMap<DeviceIdOrAllDevices, Raw<Event>>,
+) {
+	for (target_device_id_maybe, event) in map {
+		let Ok(event) = event
+			.deserialize_as()
+			.map_err(|e| err!(Request(InvalidParam(error!("To-Device event is invalid: {e}")))))
+		else {
+			continue;
+		};
+
+		handle_edu_direct_to_device_event(
+			services,
+			&target_user_id,
+			sender,
+			target_device_id_maybe,
+			ev_type,
+			event,
+		)
+		.await;
+	}
+}
+
+async fn handle_edu_direct_to_device_event(
+	services: &Services,
+	target_user_id: &UserId,
+	sender: &UserId,
+	target_device_id_maybe: DeviceIdOrAllDevices,
+	ev_type: &str,
+	event: serde_json::Value,
+) {
+	match target_device_id_maybe {
+		| DeviceIdOrAllDevices::DeviceId(ref target_device_id) => {
+			services
+				.users
+				.add_to_device_event(sender, target_user_id, target_device_id, ev_type, event)
+				.await;
+		},
+
+		| DeviceIdOrAllDevices::AllDevices => {
+			services
+				.users
+				.all_device_ids(target_user_id)
+				.for_each(|target_device_id| {
+					services.users.add_to_device_event(
+						sender,
+						target_user_id,
+						target_device_id,
+						ev_type,
+						event.clone(),
+					)
+				})
+				.await;
+		},
+	}
 }
 
 async fn handle_edu_signing_key_update(

src/api/server/send_join.rs

@@ -1,10 +1,10 @@
 #![allow(deprecated)]
 
-use std::{borrow::Borrow, collections::HashMap};
+use std::borrow::Borrow;
 
 use axum::extract::State;
 use conduwuit::{
-	err,
+	at, err,
 	pdu::gen_event_id_canonical_json,
 	utils::stream::{IterStream, TryBroadbandExt},
 	warn, Err, Result,
@@ -186,14 +186,13 @@ async fn create_join_event(
 		.map_err(|e| err!(Request(InvalidParam(warn!("Failed to sign send_join event: {e}")))))?;
 
 	let origin: OwnedServerName = serde_json::from_value(
-		serde_json::to_value(
-			value
-				.get("origin")
-				.ok_or_else(|| err!(Request(BadJson("Event missing origin property."))))?,
-		)
-		.expect("CanonicalJson is valid json value"),
+		value
+			.get("origin")
+			.ok_or_else(|| err!(Request(BadJson("Event does not have an origin server name."))))?
+			.clone()
+			.into(),
 	)
-	.map_err(|e| err!(Request(BadJson(warn!("origin field is not a valid server name: {e}")))))?;
+	.map_err(|e| err!(Request(BadJson("Event has an invalid origin server name: {e}"))))?;
 
 	let mutex_lock = services
 		.rooms
@@ -212,14 +211,16 @@ async fn create_join_event(
 
 	drop(mutex_lock);
 
-	let state_ids: HashMap<_, OwnedEventId> = services
+	let state_ids: Vec<OwnedEventId> = services
 		.rooms
 		.state_accessor
 		.state_full_ids(shortstatehash)
-		.await?;
+		.map(at!(1))
+		.collect()
+		.await;
 
 	let state = state_ids
-		.values()
+		.iter()
 		.try_stream()
 		.broad_and_then(|event_id| services.rooms.timeline.get_pdu_json(event_id))
 		.broad_and_then(|pdu| {
@@ -232,7 +233,7 @@ async fn create_join_event(
 		.boxed()
 		.await?;
 
-	let starting_events = state_ids.values().map(Borrow::borrow);
+	let starting_events = state_ids.iter().map(Borrow::borrow);
 	let auth_chain = services
 		.rooms
 		.auth_chain
@@ -269,7 +270,7 @@ pub(crate) async fn create_join_event_v1_route(
 	body: Ruma<create_join_event::v1::Request>,
 ) -> Result<create_join_event::v1::Response> {
 	if services
-		.globals
+		.server
 		.config
 		.forbidden_remote_server_names
 		.contains(body.origin())
@@ -285,7 +286,7 @@ pub(crate) async fn create_join_event_v1_route(
 
 	if let Some(server) = body.room_id.server_name() {
 		if services
-			.globals
+			.server
 			.config
 			.forbidden_remote_server_names
 			.contains(&server.to_owned())
@@ -317,7 +318,7 @@ pub(crate) async fn create_join_event_v2_route(
 	body: Ruma<create_join_event::v2::Request>,
 ) -> Result<create_join_event::v2::Response> {
 	if services
-		.globals
+		.server
 		.config
 		.forbidden_remote_server_names
 		.contains(body.origin())
@@ -327,7 +328,7 @@ pub(crate) async fn create_join_event_v2_route(
 
 	if let Some(server) = body.room_id.server_name() {
 		if services
-			.globals
+			.server
 			.config
 			.forbidden_remote_server_names
 			.contains(&server.to_owned())

src/api/server/send_knock.rs

@@ -1,7 +1,8 @@
 use axum::extract::State;
-use conduwuit::{err, pdu::gen_event_id_canonical_json, warn, Err, Error, PduEvent, Result};
+use conduwuit::{err, pdu::gen_event_id_canonical_json, warn, Err, PduEvent, Result};
+use futures::FutureExt;
 use ruma::{
-	api::{client::error::ErrorKind, federation::knock::send_knock},
+	api::federation::knock::send_knock,
 	events::{
 		room::member::{MembershipState, RoomMemberEventContent},
 		StateEventType,
@@ -17,16 +18,18 @@ use crate::Ruma;
 ///
 /// Submits a signed knock event.
 pub(crate) async fn create_knock_event_v1_route(
-	State(services): State<crate::State>, body: Ruma<send_knock::v1::Request>,
+	State(services): State<crate::State>,
+	body: Ruma<send_knock::v1::Request>,
 ) -> Result<send_knock::v1::Response> {
 	if services
-		.globals
+		.server
 		.config
 		.forbidden_remote_server_names
 		.contains(body.origin())
 	{
 		warn!(
-			"Server {} tried knocking room ID {} who has a server name that is globally forbidden. Rejecting.",
+			"Server {} tried knocking room ID {} who has a server name that is globally \
+			 forbidden. Rejecting.",
 			body.origin(),
 			&body.room_id,
 		);
@@ -35,13 +38,14 @@ pub(crate) async fn create_knock_event_v1_route(
 
 	if let Some(server) = body.room_id.server_name() {
 		if services
-			.globals
+			.server
 			.config
 			.forbidden_remote_server_names
 			.contains(&server.to_owned())
 		{
 			warn!(
-				"Server {} tried knocking room ID {} which has a server name that is globally forbidden. Rejecting.",
+				"Server {} tried knocking room ID {} which has a server name that is globally \
+				 forbidden. Rejecting.",
 				body.origin(),
 				&body.room_id,
 			);
@@ -50,7 +54,7 @@ pub(crate) async fn create_knock_event_v1_route(
 	}
 
 	if !services.rooms.metadata.exists(&body.room_id).await {
-		return Err(Error::BadRequest(ErrorKind::NotFound, "Room is unknown to this server."));
+		return Err!(Request(NotFound("Room is unknown to this server.")));
 	}
 
 	// ACL check origin server
@@ -74,44 +78,42 @@ pub(crate) async fn create_knock_event_v1_route(
 	let event_type: StateEventType = serde_json::from_value(
 		value
 			.get("type")
-			.ok_or_else(|| Error::BadRequest(ErrorKind::InvalidParam, "Event missing type property."))?
+			.ok_or_else(|| err!(Request(InvalidParam("Event has no event type."))))?
 			.clone()
 			.into(),
 	)
-	.map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "Event has invalid event type."))?;
+	.map_err(|e| err!(Request(InvalidParam("Event has invalid event type: {e}"))))?;
 
 	if event_type != StateEventType::RoomMember {
-		return Err(Error::BadRequest(
-			ErrorKind::InvalidParam,
+		return Err!(Request(InvalidParam(
 			"Not allowed to send non-membership state event to knock endpoint.",
-		));
+		)));
 	}
 
 	let content: RoomMemberEventContent = serde_json::from_value(
 		value
 			.get("content")
-			.ok_or_else(|| Error::BadRequest(ErrorKind::InvalidParam, "Event missing content property"))?
+			.ok_or_else(|| err!(Request(InvalidParam("Membership event has no content"))))?
 			.clone()
 			.into(),
 	)
-	.map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "Event content is empty or invalid"))?;
+	.map_err(|e| err!(Request(InvalidParam("Event has invalid membership content: {e}"))))?;
 
 	if content.membership != MembershipState::Knock {
-		return Err(Error::BadRequest(
-			ErrorKind::InvalidParam,
-			"Not allowed to send a non-knock membership event to knock endpoint.",
-		));
+		return Err!(Request(InvalidParam(
+			"Not allowed to send a non-knock membership event to knock endpoint."
+		)));
 	}
 
 	// ACL check sender server name
 	let sender: OwnedUserId = serde_json::from_value(
 		value
 			.get("sender")
-			.ok_or_else(|| Error::BadRequest(ErrorKind::InvalidParam, "Event missing sender property."))?
+			.ok_or_else(|| err!(Request(InvalidParam("Event has no sender user ID."))))?
 			.clone()
 			.into(),
 	)
-	.map_err(|_| Error::BadRequest(ErrorKind::BadJson, "sender is not a valid user ID."))?;
+	.map_err(|e| err!(Request(BadJson("Event sender is not a valid user ID: {e}"))))?;
 
 	services
 		.rooms
@@ -127,36 +129,32 @@ pub(crate) async fn create_knock_event_v1_route(
 	let state_key: OwnedUserId = serde_json::from_value(
 		value
 			.get("state_key")
-			.ok_or_else(|| Error::BadRequest(ErrorKind::InvalidParam, "Event missing state_key property."))?
+			.ok_or_else(|| err!(Request(InvalidParam("Event does not have a state_key"))))?
 			.clone()
 			.into(),
 	)
-	.map_err(|_| Error::BadRequest(ErrorKind::BadJson, "state_key is invalid or not a user ID."))?;
+	.map_err(|e| err!(Request(BadJson("Event does not have a valid state_key: {e}"))))?;
 
 	if state_key != sender {
-		return Err(Error::BadRequest(
-			ErrorKind::InvalidParam,
-			"State key does not match sender user",
-		));
+		return Err!(Request(InvalidParam("state_key does not match sender user of event.")));
 	};
 
 	let origin: OwnedServerName = serde_json::from_value(
-		serde_json::to_value(
-			value
-				.get("origin")
-				.ok_or_else(|| Error::BadRequest(ErrorKind::InvalidParam, "Event missing origin property."))?,
-		)
-		.expect("CanonicalJson is valid json value"),
+		value
+			.get("origin")
+			.ok_or_else(|| err!(Request(BadJson("Event does not have an origin server name."))))?
+			.clone()
+			.into(),
 	)
-	.map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "origin is not a server name."))?;
+	.map_err(|e| err!(Request(BadJson("Event has an invalid origin server name: {e}"))))?;
 
 	let mut event: JsonObject = serde_json::from_str(body.pdu.get())
-		.map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "Invalid knock event PDU."))?;
+		.map_err(|e| err!(Request(InvalidParam("Invalid knock event PDU: {e}"))))?;
 
 	event.insert("event_id".to_owned(), "$placeholder".into());
 
 	let pdu: PduEvent = serde_json::from_value(event.into())
-		.map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "Invalid knock event PDU."))?;
+		.map_err(|e| err!(Request(InvalidParam("Invalid knock event PDU: {e}"))))?;
 
 	let mutex_lock = services
 		.rooms
@@ -169,19 +167,18 @@ pub(crate) async fn create_knock_event_v1_route(
 		.rooms
 		.event_handler
 		.handle_incoming_pdu(&origin, &body.room_id, &event_id, value.clone(), true)
+		.boxed()
 		.await?
 		.ok_or_else(|| err!(Request(InvalidParam("Could not accept as timeline event."))))?;
 
 	drop(mutex_lock);
 
-	let knock_room_state = services.rooms.state.summary_stripped(&pdu).await;
-
 	services
 		.sending
 		.send_pdu_room(&body.room_id, &pdu_id)
 		.await?;
 
-	Ok(send_knock::v1::Response {
-		knock_room_state,
-	})
+	let knock_room_state = services.rooms.state.summary_stripped(&pdu).await;
+
+	Ok(send_knock::v1::Response { knock_room_state })
 }

src/api/server/state.rs

@@ -1,7 +1,7 @@
 use std::{borrow::Borrow, iter::once};
 
 use axum::extract::State;
-use conduwuit::{err, result::LogErr, utils::IterStream, Result};
+use conduwuit::{at, err, utils::IterStream, Result};
 use futures::{FutureExt, StreamExt, TryStreamExt};
 use ruma::{api::federation::event::get_room_state, OwnedEventId};
 
@@ -35,11 +35,9 @@ pub(crate) async fn get_room_state_route(
 		.rooms
 		.state_accessor
 		.state_full_ids(shortstatehash)
-		.await
-		.log_err()
-		.map_err(|_| err!(Request(NotFound("PDU state IDs not found."))))?
-		.into_values()
-		.collect();
+		.map(at!(1))
+		.collect()
+		.await;
 
 	let pdus = state_ids
 		.iter()

src/api/server/state_ids.rs

@@ -1,7 +1,7 @@
 use std::{borrow::Borrow, iter::once};
 
 use axum::extract::State;
-use conduwuit::{err, Result};
+use conduwuit::{at, err, Result};
 use futures::StreamExt;
 use ruma::{api::federation::event::get_room_state_ids, OwnedEventId};
 
@@ -36,10 +36,9 @@ pub(crate) async fn get_room_state_ids_route(
 		.rooms
 		.state_accessor
 		.state_full_ids(shortstatehash)
-		.await
-		.map_err(|_| err!(Request(NotFound("State ids not found"))))?
-		.into_values()
-		.collect();
+		.map(at!(1))
+		.collect()
+		.await;
 
 	let auth_chain_ids = services
 		.rooms

src/api/server/utils.rs

@@ -1,6 +1,6 @@
 use conduwuit::{implement, is_false, Err, Result};
 use conduwuit_service::Services;
-use futures::{future::OptionFuture, join, FutureExt};
+use futures::{future::OptionFuture, join, FutureExt, StreamExt};
 use ruma::{EventId, RoomId, ServerName};
 
 pub(super) struct AccessCheck<'a> {
@@ -31,6 +31,15 @@ pub(super) async fn check(&self) -> Result {
 		.state_cache
 		.server_in_room(self.origin, self.room_id);
 
+	// if any user on our homeserver is trying to knock this room, we'll need to
+	// acknowledge bans or leaves
+	let user_is_knocking = self
+		.services
+		.rooms
+		.state_cache
+		.room_members_knocked(self.room_id)
+		.count();
+
 	let server_can_see: OptionFuture<_> = self
 		.event_id
 		.map(|event_id| {
@@ -42,14 +51,14 @@ pub(super) async fn check(&self) -> Result {
 		})
 		.into();
 
-	let (world_readable, server_in_room, server_can_see, acl_check) =
-		join!(world_readable, server_in_room, server_can_see, acl_check);
+	let (world_readable, server_in_room, server_can_see, acl_check, user_is_knocking) =
+		join!(world_readable, server_in_room, server_can_see, acl_check, user_is_knocking);
 
 	if !acl_check {
 		return Err!(Request(Forbidden("Server access denied.")));
 	}
 
-	if !world_readable && !server_in_room {
+	if !world_readable && !server_in_room && user_is_knocking == 0 {
 		return Err!(Request(Forbidden("Server is not in room.")));
 	}
 

src/core/Cargo.toml

@@ -36,6 +36,7 @@ jemalloc_stats = [
     "tikv-jemalloc-ctl/stats",
     "tikv-jemallocator/stats",
 ]
+jemalloc_conf = []
 hardened_malloc = [
 	"dep:hardened_malloc-rs"
 ]
@@ -50,6 +51,9 @@ zstd_compression = [
 ]
 perf_measurements = []
 sentry_telemetry = []
+conduwuit_mods = [
+    "dep:libloading"
+]
 
 [dependencies]
 argon2.workspace = true
@@ -71,11 +75,11 @@ figment.workspace = true
 futures.workspace = true
 http-body-util.workspace = true
 http.workspace = true
-image.workspace = true
 ipaddress.workspace = true
 itertools.workspace = true
 libc.workspace = true
 libloading.workspace = true
+libloading.optional = true
 log.workspace = true
 num-traits.workspace = true
 rand.workspace = true

src/core/alloc/default.rs

@@ -1,8 +1,11 @@
 //! Default allocator with no special features
 
+/// Always returns Ok
+pub fn trim<I: Into<Option<usize>>>(_: I) -> crate::Result { Ok(()) }
+
 /// Always returns None
 #[must_use]
-pub fn memory_stats() -> Option<String> { None }
+pub fn memory_stats(_opts: &str) -> Option<String> { None }
 
 /// Always returns None
 #[must_use]

src/core/alloc/hardened.rs

@@ -3,11 +3,13 @@
 #[global_allocator]
 static HMALLOC: hardened_malloc_rs::HardenedMalloc = hardened_malloc_rs::HardenedMalloc;
 
-#[must_use]
-//TODO: get usage
-pub fn memory_usage() -> Option<string> { None }
+pub fn trim<I: Into<Option<usize>>>(_: I) -> crate::Result { Ok(()) }
 
 #[must_use]
-pub fn memory_stats() -> Option<String> {
+//TODO: get usage
+pub fn memory_usage() -> Option<String> { None }
+
+#[must_use]
+pub fn memory_stats(_opts: &str) -> Option<String> {
 	Some("Extended statistics are not available from hardened_malloc.".to_owned())
 }

src/core/alloc/je.rs

@@ -1,18 +1,65 @@
 //! jemalloc allocator
 
-use std::ffi::{c_char, c_void};
+use std::{
+	cell::OnceCell,
+	ffi::{c_char, c_void, CStr},
+	fmt::Debug,
+	sync::RwLock,
+};
 
+use arrayvec::ArrayVec;
+use const_str::concat_bytes;
+use tikv_jemalloc_ctl as mallctl;
 use tikv_jemalloc_sys as ffi;
 use tikv_jemallocator as jemalloc;
 
+use crate::{
+	err, is_equal_to, is_nonzero,
+	utils::{math, math::Tried},
+	Result,
+};
+
+#[cfg(feature = "jemalloc_conf")]
+#[unsafe(no_mangle)]
+pub static malloc_conf: &[u8] = concat_bytes!(
+	"lg_extent_max_active_fit:4",
+	",oversize_threshold:16777216",
+	",tcache_max:2097152",
+	",dirty_decay_ms:16000",
+	",muzzy_decay_ms:144000",
+	",percpu_arena:percpu",
+	",metadata_thp:always",
+	",background_thread:true",
+	",max_background_threads:-1",
+	MALLOC_CONF_PROF,
+	0
+);
+
+#[cfg(all(feature = "jemalloc_conf", feature = "jemalloc_prof"))]
+const MALLOC_CONF_PROF: &str = ",prof_active:false";
+#[cfg(all(feature = "jemalloc_conf", not(feature = "jemalloc_prof")))]
+const MALLOC_CONF_PROF: &str = "";
+
 #[global_allocator]
 static JEMALLOC: jemalloc::Jemalloc = jemalloc::Jemalloc;
+static CONTROL: RwLock<()> = RwLock::new(());
+
+type Name = ArrayVec<u8, NAME_MAX>;
+type Key = ArrayVec<usize, KEY_SEGS>;
+
+const NAME_MAX: usize = 128;
+const KEY_SEGS: usize = 8;
+
+#[crate::ctor]
+fn _static_initialization() {
+	acq_epoch().expect("pre-initialization of jemalloc failed");
+	acq_epoch().expect("pre-initialization of jemalloc failed");
+}
 
 #[must_use]
 #[cfg(feature = "jemalloc_stats")]
 pub fn memory_usage() -> Option<String> {
 	use mallctl::stats;
-	use tikv_jemalloc_ctl as mallctl;
 
 	let mibs = |input: Result<usize, mallctl::Error>| {
 		let input = input.unwrap_or_default();
@@ -22,6 +69,9 @@ pub fn memory_usage() -> Option<String> {
 		kibs / 1024.0
 	};
 
+	// Acquire the epoch; ensure latest stats are pulled in
+	acq_epoch().ok()?;
+
 	let allocated = mibs(stats::allocated::read());
 	let active = mibs(stats::active::read());
 	let mapped = mibs(stats::mapped::read());
@@ -39,34 +89,293 @@ pub fn memory_usage() -> Option<String> {
 #[cfg(not(feature = "jemalloc_stats"))]
 pub fn memory_usage() -> Option<String> { None }
 
-#[must_use]
-pub fn memory_stats() -> Option<String> {
-	const MAX_LENGTH: usize = 65536 - 4096;
+pub fn memory_stats(opts: &str) -> Option<String> {
+	const MAX_LENGTH: usize = 1_048_576;
 
-	let opts_s = "d";
 	let mut str = String::new();
-
 	let opaque = std::ptr::from_mut(&mut str).cast::<c_void>();
-	let opts_p: *const c_char = std::ffi::CString::new(opts_s)
+	let opts_p: *const c_char = std::ffi::CString::new(opts)
 		.expect("cstring")
 		.into_raw()
 		.cast_const();
 
+	// Acquire the epoch; ensure latest stats are pulled in
+	acq_epoch().ok()?;
+
 	// SAFETY: calls malloc_stats_print() with our string instance which must remain
 	// in this frame. https://docs.rs/tikv-jemalloc-sys/latest/tikv_jemalloc_sys/fn.malloc_stats_print.html
 	unsafe { ffi::malloc_stats_print(Some(malloc_stats_cb), opaque, opts_p) };
 
 	str.truncate(MAX_LENGTH);
-	Some(format!("<pre><code>{str}</code></pre>"))
+
+	Some(str)
 }
 
 unsafe extern "C" fn malloc_stats_cb(opaque: *mut c_void, msg: *const c_char) {
 	// SAFETY: we have to trust the opaque points to our String
-	let res: &mut String = unsafe { opaque.cast::<String>().as_mut().unwrap() };
+	let res: &mut String = unsafe {
+		opaque
+			.cast::<String>()
+			.as_mut()
+			.expect("failed to cast void* to &mut String")
+	};
 
 	// SAFETY: we have to trust the string is null terminated.
-	let msg = unsafe { std::ffi::CStr::from_ptr(msg) };
+	let msg = unsafe { CStr::from_ptr(msg) };
 
 	let msg = String::from_utf8_lossy(msg.to_bytes());
 	res.push_str(msg.as_ref());
 }
+
+macro_rules! mallctl {
+	($name:expr) => {{
+		thread_local! {
+			static KEY: OnceCell<Key> = OnceCell::default();
+		};
+
+		KEY.with(|once| {
+			once.get_or_init(move || key($name).expect("failed to translate name into mib key"))
+				.clone()
+		})
+	}};
+}
+
+pub mod this_thread {
+	use super::{is_nonzero, key, math, Debug, Key, OnceCell, Result};
+
+	thread_local! {
+		static ALLOCATED_BYTES: OnceCell<&'static u64> = const { OnceCell::new() };
+		static DEALLOCATED_BYTES: OnceCell<&'static u64> = const { OnceCell::new() };
+	}
+
+	pub fn trim() -> Result { decay().and_then(|()| purge()) }
+
+	pub fn purge() -> Result { notify(mallctl!("arena.0.purge")) }
+
+	pub fn decay() -> Result { notify(mallctl!("arena.0.decay")) }
+
+	pub fn idle() -> Result { super::notify(&mallctl!("thread.idle")) }
+
+	pub fn flush() -> Result { super::notify(&mallctl!("thread.tcache.flush")) }
+
+	pub fn set_muzzy_decay(decay_ms: isize) -> Result<isize> {
+		set(mallctl!("arena.0.muzzy_decay_ms"), decay_ms)
+	}
+
+	pub fn get_muzzy_decay() -> Result<isize> { get(mallctl!("arena.0.muzzy_decay_ms")) }
+
+	pub fn set_dirty_decay(decay_ms: isize) -> Result<isize> {
+		set(mallctl!("arena.0.dirty_decay_ms"), decay_ms)
+	}
+
+	pub fn get_dirty_decay() -> Result<isize> { get(mallctl!("arena.0.dirty_decay_ms")) }
+
+	pub fn cache_enable(enable: bool) -> Result<bool> {
+		super::set::<u8>(&mallctl!("thread.tcache.enabled"), enable.into()).map(is_nonzero!())
+	}
+
+	pub fn is_cache_enabled() -> Result<bool> {
+		super::get::<u8>(&mallctl!("thread.tcache.enabled")).map(is_nonzero!())
+	}
+
+	pub fn set_arena(id: usize) -> Result<usize> {
+		super::set::<u32>(&mallctl!("thread.arena"), id.try_into()?).and_then(math::try_into)
+	}
+
+	pub fn arena_id() -> Result<usize> {
+		super::get::<u32>(&mallctl!("thread.arena")).and_then(math::try_into)
+	}
+
+	pub fn prof_enable(enable: bool) -> Result<bool> {
+		super::set::<u8>(&mallctl!("thread.prof.active"), enable.into()).map(is_nonzero!())
+	}
+
+	pub fn is_prof_enabled() -> Result<bool> {
+		super::get::<u8>(&mallctl!("thread.prof.active")).map(is_nonzero!())
+	}
+
+	pub fn reset_peak() -> Result { super::notify(&mallctl!("thread.peak.reset")) }
+
+	pub fn peak() -> Result<u64> { super::get(&mallctl!("thread.peak.read")) }
+
+	#[inline]
+	#[must_use]
+	pub fn allocated() -> u64 {
+		*ALLOCATED_BYTES.with(|once| init_tls_cell(once, "thread.allocatedp"))
+	}
+
+	#[inline]
+	#[must_use]
+	pub fn deallocated() -> u64 {
+		*DEALLOCATED_BYTES.with(|once| init_tls_cell(once, "thread.deallocatedp"))
+	}
+
+	fn notify(key: Key) -> Result { super::notify_by_arena(Some(arena_id()?), key) }
+
+	fn set<T>(key: Key, val: T) -> Result<T>
+	where
+		T: Copy + Debug,
+	{
+		super::set_by_arena(Some(arena_id()?), key, val)
+	}
+
+	fn get<T>(key: Key) -> Result<T>
+	where
+		T: Copy + Debug,
+	{
+		super::get_by_arena(Some(arena_id()?), key)
+	}
+
+	fn init_tls_cell(cell: &OnceCell<&'static u64>, name: &str) -> &'static u64 {
+		cell.get_or_init(|| {
+			let ptr: *const u64 = super::get(&mallctl!(name)).expect("failed to obtain pointer");
+
+			// SAFETY: ptr points directly to the internal state of jemalloc for this thread
+			unsafe { ptr.as_ref() }.expect("pointer must not be null")
+		})
+	}
+}
+
+pub fn stats_reset() -> Result { notify(&mallctl!("stats.mutexes.reset")) }
+
+pub fn prof_reset() -> Result { notify(&mallctl!("prof.reset")) }
+
+pub fn prof_enable(enable: bool) -> Result<bool> {
+	set::<u8>(&mallctl!("prof.active"), enable.into()).map(is_nonzero!())
+}
+
+pub fn is_prof_enabled() -> Result<bool> {
+	get::<u8>(&mallctl!("prof.active")).map(is_nonzero!())
+}
+
+pub fn trim<I: Into<Option<usize>> + Copy>(arena: I) -> Result {
+	decay(arena).and_then(|()| purge(arena))
+}
+
+pub fn purge<I: Into<Option<usize>>>(arena: I) -> Result {
+	notify_by_arena(arena.into(), mallctl!("arena.4096.purge"))
+}
+
+pub fn decay<I: Into<Option<usize>>>(arena: I) -> Result {
+	notify_by_arena(arena.into(), mallctl!("arena.4096.decay"))
+}
+
+pub fn set_muzzy_decay<I: Into<Option<usize>>>(arena: I, decay_ms: isize) -> Result<isize> {
+	if let Some(arena) = arena.into() {
+		set_by_arena(Some(arena), mallctl!("arena.4096.muzzy_decay_ms"), decay_ms)
+	} else {
+		set(&mallctl!("arenas.muzzy_decay_ms"), decay_ms)
+	}
+}
+
+pub fn set_dirty_decay<I: Into<Option<usize>>>(arena: I, decay_ms: isize) -> Result<isize> {
+	if let Some(arena) = arena.into() {
+		set_by_arena(Some(arena), mallctl!("arena.4096.dirty_decay_ms"), decay_ms)
+	} else {
+		set(&mallctl!("arenas.dirty_decay_ms"), decay_ms)
+	}
+}
+
+#[inline]
+#[must_use]
+pub fn is_affine_arena() -> bool { is_percpu_arena() || is_phycpu_arena() }
+
+#[inline]
+#[must_use]
+pub fn is_percpu_arena() -> bool { percpu_arenas().is_ok_and(is_equal_to!("percpu")) }
+
+#[inline]
+#[must_use]
+pub fn is_phycpu_arena() -> bool { percpu_arenas().is_ok_and(is_equal_to!("phycpu")) }
+
+pub fn percpu_arenas() -> Result<&'static str> {
+	let ptr = get::<*const c_char>(&mallctl!("opt.percpu_arena"))?;
+	//SAFETY: ptr points to a null-terminated string returned for opt.percpu_arena.
+	let cstr = unsafe { CStr::from_ptr(ptr) };
+	cstr.to_str().map_err(Into::into)
+}
+
+pub fn arenas() -> Result<usize> {
+	get::<u32>(&mallctl!("arenas.narenas")).and_then(math::try_into)
+}
+
+pub fn inc_epoch() -> Result<u64> { xchg(&mallctl!("epoch"), 1_u64) }
+
+pub fn acq_epoch() -> Result<u64> { xchg(&mallctl!("epoch"), 0_u64) }
+
+fn notify_by_arena(id: Option<usize>, mut key: Key) -> Result {
+	key[1] = id.unwrap_or(4096);
+	notify(&key)
+}
+
+fn set_by_arena<T>(id: Option<usize>, mut key: Key, val: T) -> Result<T>
+where
+	T: Copy + Debug,
+{
+	key[1] = id.unwrap_or(4096);
+	set(&key, val)
+}
+
+fn get_by_arena<T>(id: Option<usize>, mut key: Key) -> Result<T>
+where
+	T: Copy + Debug,
+{
+	key[1] = id.unwrap_or(4096);
+	get(&key)
+}
+
+fn notify(key: &Key) -> Result { xchg(key, ()) }
+
+fn set<T>(key: &Key, val: T) -> Result<T>
+where
+	T: Copy + Debug,
+{
+	let _lock = CONTROL.write()?;
+	let res = xchg(key, val)?;
+	inc_epoch()?;
+
+	Ok(res)
+}
+
+fn get<T>(key: &Key) -> Result<T>
+where
+	T: Copy + Debug,
+{
+	acq_epoch()?;
+	acq_epoch()?;
+
+	// SAFETY: T must be perfectly valid to receive value.
+	unsafe { mallctl::raw::read_mib(key.as_slice()) }.map_err(map_err)
+}
+
+fn xchg<T>(key: &Key, val: T) -> Result<T>
+where
+	T: Copy + Debug,
+{
+	// SAFETY: T must be the exact expected type.
+	unsafe { mallctl::raw::update_mib(key.as_slice(), val) }.map_err(map_err)
+}
+
+fn key(name: &str) -> Result<Key> {
+	// tikv asserts the output buffer length is tight to the number of required mibs
+	// so we slice that down here.
+	let segs = name.chars().filter(is_equal_to!(&'.')).count().try_add(1)?;
+
+	let name = self::name(name)?;
+	let mut buf = [0_usize; KEY_SEGS];
+	mallctl::raw::name_to_mib(name.as_slice(), &mut buf[0..segs])
+		.map_err(map_err)
+		.map(move |()| buf.into_iter().take(segs).collect())
+}
+
+fn name(name: &str) -> Result<Name> {
+	let mut buf = Name::new();
+	buf.try_extend_from_slice(name.as_bytes())?;
+	buf.try_extend_from_slice(b"\0")?;
+
+	Ok(buf)
+}
+
+fn map_err(error: tikv_jemalloc_ctl::Error) -> crate::Error {
+	err!("mallctl: {}", error.to_string())
+}

src/core/alloc/mod.rs

@@ -4,7 +4,7 @@
 #[cfg(all(not(target_env = "msvc"), feature = "jemalloc"))]
 pub mod je;
 #[cfg(all(not(target_env = "msvc"), feature = "jemalloc"))]
-pub use je::{memory_stats, memory_usage};
+pub use je::{memory_stats, memory_usage, trim};
 
 #[cfg(all(not(target_env = "msvc"), feature = "hardened_malloc", not(feature = "jemalloc")))]
 pub mod hardened;
@@ -13,7 +13,7 @@ pub mod hardened;
 	feature = "hardened_malloc",
 	not(feature = "jemalloc")
 ))]
-pub use hardened::{memory_stats, memory_usage};
+pub use hardened::{memory_stats, memory_usage, trim};
 
 #[cfg(any(
 	target_env = "msvc",
@@ -24,4 +24,4 @@ pub mod default;
 	target_env = "msvc",
 	all(not(feature = "hardened_malloc"), not(feature = "jemalloc"))
 ))]
-pub use default::{memory_stats, memory_usage};
+pub use default::{memory_stats, memory_usage, trim};