add almost complete valgrind support to complement

Signed-off-by: June Clementine Strawberry <strawberry@puppygock.gay>

.gitea/PULL_REQUEST_TEMPLATE.md

@@ -1,8 +0,0 @@
-
-<!-- Please describe your changes here -->
-
------------------------------------------------------------------------------
-
-- [ ] I ran `cargo fmt`, `cargo clippy`, and `cargo test`
-- [ ] I agree to release my code and all other changes of this MR under the Apache-2.0 license
-

.gitea/workflows/ci.yml

@@ -1,264 +0,0 @@
-name: CI and Artifacts
-
-on:
-    pull_request:
-    push:
-        # documentation workflow deals with this or is not relevant for this workflow
-        paths-ignore:
-          - '*.md'
-          - 'conduwuit-example.toml'
-          - 'book.toml'
-          - '.gitlab-ci.yml'
-          - '.gitignore'
-          - 'renovate.json'
-          - 'docs/**'
-          - 'debian/**'
-          - 'docker/**'
-        branches:
-            - main
-        tags:
-          - '*'
-    # Allows you to run this workflow manually from the Actions tab
-    #workflow_dispatch:
-
-#concurrency:
-#    group: ${{ gitea.head_ref || gitea.ref_name }}
-#    cancel-in-progress: true
-
-env:
-    # Required to make some things output color
-    TERM: ansi
-    # Publishing to my nix binary cache
-    ATTIC_TOKEN: ${{ secrets.ATTIC_TOKEN }}
-    # conduwuit.cachix.org
-    CACHIX_AUTH_TOKEN: ${{ secrets.CACHIX_AUTH_TOKEN }}
-    # Just in case incremental is still being set to true, speeds up CI
-    CARGO_INCREMENTAL: 0
-    # Custom nix binary cache if fork is being used
-    ATTIC_ENDPOINT: ${{ vars.ATTIC_ENDPOINT }}
-    ATTIC_PUBLIC_KEY: ${{ vars.ATTIC_PUBLIC_KEY }}
-    # Get error output from nix that we can actually use
-    NIX_CONFIG: show-trace = true
-
-#permissions:
-#    packages: write
-#    contents: read
-
-jobs:
-    tests:
-        name: Test
-        runs-on: ubuntu-latest
-        steps:
-            - name: Sync repository
-              uses: https://github.com/actions/checkout@v4
-
-            - name: Tag comparison check
-              if: startsWith(gitea.ref, 'refs/tags/v')
-              run: |
-                  # Tag mismatch with latest repo tag check to prevent potential downgrades
-                  LATEST_TAG=$(git describe --tags `git rev-list --tags --max-count=1`)
-
-                  if [ $LATEST_TAG != ${{ gitea.ref_name }} ]; then
-                    echo '# WARNING: Attempting to run this workflow for a tag that is not the latest repo tag. Aborting.'
-                    echo '# WARNING: Attempting to run this workflow for a tag that is not the latest repo tag. Aborting.' >> $GITHUB_STEP_SUMMARY
-                    exit 1
-                  fi
-
-            - name: Install Nix
-              uses: https://github.com/DeterminateSystems/nix-installer-action@main
-              with:
-                diagnostic-endpoint: ""
-                extra-conf: |
-                  experimental-features = nix-command flakes
-                  accept-flake-config = true
-
-            - name: Enable Cachix binary cache
-              run: |
-                  nix profile install nixpkgs#cachix
-                  cachix use crane
-                  cachix use nix-community
-
-            - name: Configure Magic Nix Cache
-              uses: https://github.com/DeterminateSystems/magic-nix-cache-action@main
-              with:
-                diagnostic-endpoint: ""
-                upstream-cache: "https://attic.kennel.juneis.dog/conduwuit"
-
-            - name: Apply Nix binary cache configuration
-              run: |
-                  sudo tee -a /etc/nix/nix.conf > /dev/null <<EOF
-                  extra-substituters = https://attic.kennel.juneis.dog/conduit https://attic.kennel.juneis.dog/conduwuit https://cache.lix.systems https://conduwuit.cachix.org
-                  extra-trusted-public-keys = conduit:eEKoUwlQGDdYmAI/Q/0slVlegqh/QmAvQd7HBSm21Wk= conduwuit:BbycGUgTISsltcmH0qNjFR9dbrQNYgdIAcmViSGoVTE= cache.lix.systems:aBnZUw8zA7H35Cz2RyKFVs3H4PlGTLawyY5KRbvJR8o= conduwuit.cachix.org-1:MFRm6jcnfTf0jSAbmvLfhO3KBMt4px+1xaereWXp8Xg=
-                  EOF
-
-            - name: Use alternative Nix binary caches if specified
-              if: ${{ (env.ATTIC_ENDPOINT != '') && (env.ATTIC_PUBLIC_KEY != '') }}
-              run: |
-                  sudo tee -a /etc/nix/nix.conf > /dev/null <<EOF
-                  extra-substituters = ${{ env.ATTIC_ENDPOINT }}
-                  extra-trusted-public-keys = ${{ env.ATTIC_PUBLIC_KEY }}
-                  EOF
-
-            - name: Prepare build environment
-              run: |
-                  echo 'source $HOME/.nix-profile/share/nix-direnv/direnvrc' > "$HOME/.direnvrc"
-                  nix profile install --impure --inputs-from . nixpkgs#direnv nixpkgs#nix-direnv
-                  direnv allow
-                  nix develop .#all-features --command true
-
-            - name: Cache CI dependencies
-              run: |
-                  bin/nix-build-and-cache ci
-
-            - name: Run CI tests
-              run: |
-                  direnv exec . engage > >(tee -a test_output.log)
-
-            - name: Sync Complement repository
-              uses: https://github.com/actions/checkout@v4
-              with:
-                repository: 'matrix-org/complement'
-                path: complement_src
-
-            - name: Run Complement tests
-              run: |
-                  direnv exec . bin/complement 'complement_src' 'complement_test_logs.jsonl' 'complement_test_results.jsonl'
-                  cp -v -f result complement_oci_image.tar.gz
-
-            - name: Upload Complement OCI image
-              uses: https://github.com/actions/upload-artifact@v4
-              with:
-                name: complement_oci_image.tar.gz
-                path: complement_oci_image.tar.gz
-                if-no-files-found: error
-
-            - name: Upload Complement logs
-              uses: https://github.com/actions/upload-artifact@v4
-              with:
-                name: complement_test_logs.jsonl
-                path: complement_test_logs.jsonl
-                if-no-files-found: error
-
-            - name: Upload Complement results
-              uses: https://github.com/actions/upload-artifact@v4
-              with:
-                name: complement_test_results.jsonl
-                path: complement_test_results.jsonl
-                if-no-files-found: error
-
-            - name: Diff Complement results with checked-in repo results
-              run: |
-                  diff -u --color=always tests/test_results/complement/test_results.jsonl complement_test_results.jsonl > >(tee -a complement_test_output.log)
-                  echo '# Complement diff results' >> $GITHUB_STEP_SUMMARY
-                  echo '```diff' >> $GITHUB_STEP_SUMMARY
-                  tail -n 100 complement_test_output.log | sed 's/\x1b\[[0-9;]*m//g' >> $GITHUB_STEP_SUMMARY
-                  echo '```' >> $GITHUB_STEP_SUMMARY
-
-            - name: Update Job Summary
-              if: success() || failure()
-              run: |
-                  if [ ${{ job.status }} == 'success' ]; then
-                      echo '# ✅ completed suwuccessfully' >> $GITHUB_STEP_SUMMARY
-                  else
-                      echo '```' >> $GITHUB_STEP_SUMMARY
-                      tail -n 40 test_output.log | sed 's/\x1b\[[0-9;]*m//g' >> $GITHUB_STEP_SUMMARY
-                      echo '```' >> $GITHUB_STEP_SUMMARY
-                  fi
-
-    build:
-        name: Build
-        runs-on: ubuntu-latest
-        needs: tests
-        strategy:
-            matrix:
-                include:
-                    - target: aarch64-unknown-linux-musl
-                    - target: x86_64-unknown-linux-musl
-        steps:
-            - name: Sync repository
-              uses: https://github.com/actions/checkout@v4
-
-            - name: Install Nix
-              uses: https://github.com/DeterminateSystems/nix-installer-action@main
-              with:
-                diagnostic-endpoint: ""
-                extra-conf: |
-                  experimental-features = nix-command flakes
-                  accept-flake-config = true
-
-            - name: Install and enable Cachix binary cache
-              run: |
-                  nix profile install nixpkgs#cachix
-                  cachix use crane
-                  cachix use nix-community
-
-            - name: Configure Magic Nix Cache
-              uses: https://github.com/DeterminateSystems/magic-nix-cache-action@main
-              with:
-                diagnostic-endpoint: ""
-                upstream-cache: "https://attic.kennel.juneis.dog/conduwuit"
-
-            - name: Apply Nix binary cache configuration
-              run: |
-                  sudo tee -a /etc/nix/nix.conf > /dev/null <<EOF
-                  extra-substituters = https://attic.kennel.juneis.dog/conduit https://attic.kennel.juneis.dog/conduwuit https://cache.lix.systems https://conduwuit.cachix.org
-                  extra-trusted-public-keys = conduit:eEKoUwlQGDdYmAI/Q/0slVlegqh/QmAvQd7HBSm21Wk= conduwuit:BbycGUgTISsltcmH0qNjFR9dbrQNYgdIAcmViSGoVTE= cache.lix.systems:aBnZUw8zA7H35Cz2RyKFVs3H4PlGTLawyY5KRbvJR8o= conduwuit.cachix.org-1:MFRm6jcnfTf0jSAbmvLfhO3KBMt4px+1xaereWXp8Xg=
-                  EOF
-
-            - name: Use alternative Nix binary caches if specified
-              if: ${{ (env.ATTIC_ENDPOINT != '') && (env.ATTIC_PUBLIC_KEY != '') }}
-              run: |
-                  sudo tee -a /etc/nix/nix.conf > /dev/null <<EOF
-                  extra-substituters = ${{ env.ATTIC_ENDPOINT }}
-                  extra-trusted-public-keys = ${{ env.ATTIC_PUBLIC_KEY }}
-                  EOF
-
-            - name: Prepare build environment
-              run: |
-                  echo 'source $HOME/.nix-profile/share/nix-direnv/direnvrc' > "$HOME/.direnvrc"
-                  nix profile install --impure --inputs-from . nixpkgs#direnv nixpkgs#nix-direnv
-                  direnv allow
-                  nix develop .#all-features --command true
-
-            - name: Build static ${{ matrix.target }}
-              run: |
-                  CARGO_DEB_TARGET_TUPLE=$(echo ${{ matrix.target }} | grep -o -E '^([^-]*-){3}[^-]*')
-                  SOURCE_DATE_EPOCH=$(git log -1 --pretty=%ct)
-
-                  bin/nix-build-and-cache just .#static-${{ matrix.target }}
-                  mkdir -v -p target/release/
-                  mkdir -v -p target/$CARGO_DEB_TARGET_TUPLE/release/
-                  cp -v -f result/bin/conduit target/release/conduwuit
-                  cp -v -f result/bin/conduit target/$CARGO_DEB_TARGET_TUPLE/release/conduwuit
-                  # -p conduit is the main crate name
-                  direnv exec . cargo deb --verbose --no-build --no-strip -p conduit --target=$CARGO_DEB_TARGET_TUPLE --output target/release/${{ matrix.target }}.deb
-                  mv -v target/release/conduwuit static-${{ matrix.target }}
-                  mv -v target/release/${{ matrix.target }}.deb ${{ matrix.target }}.deb
-
-            - name: Upload static-${{ matrix.target }}
-              uses: https://github.com/actions/upload-artifact@v4
-              with:
-                  name: static-${{ matrix.target }}
-                  path: static-${{ matrix.target }}
-                  if-no-files-found: error
-
-            - name: Upload deb ${{ matrix.target }}
-              uses: https://github.com/actions/upload-artifact@v4
-              with:
-                  name: deb-${{ matrix.target }}
-                  path: ${{ matrix.target }}.deb
-                  if-no-files-found: error
-                  compression-level: 0
-
-            - name: Build OCI image ${{ matrix.target }}
-              run: |
-                  bin/nix-build-and-cache just .#oci-image-${{ matrix.target }}
-                  cp -v -f result oci-image-${{ matrix.target }}.tar.gz
-
-            - name: Upload OCI image ${{ matrix.target }}
-              uses: https://github.com/actions/upload-artifact@v4
-              with:
-                  name: oci-image-${{ matrix.target }}
-                  path: oci-image-${{ matrix.target }}.tar.gz
-                  if-no-files-found: error
-                  compression-level: 0

.github/workflows/ci.yml

@@ -22,8 +22,8 @@ concurrency:
 
 env:
     # sccache only on main repo
-    SCCACHE_GHA_ENABLED: "${{ (github.event.pull_request.draft != true) && (vars.DOCKER_USERNAME != '') && (vars.GITLAB_USERNAME != '') && (vars.SCCACHE_ENDPOINT != '') && (github.event.pull_request.user.login != 'renovate[bot]') && 'true' || 'false' }}"
-    RUSTC_WRAPPER: "${{ (github.event.pull_request.draft != true) && (vars.DOCKER_USERNAME != '') && (vars.GITLAB_USERNAME != '') && (vars.SCCACHE_ENDPOINT != '') && (github.event.pull_request.user.login != 'renovate[bot]') && 'sccache' || '' }}"
+    SCCACHE_GHA_ENABLED: "${{ !startsWith(github.ref, 'refs/tags/') && (github.event.pull_request.draft != true) && (vars.DOCKER_USERNAME != '') && (vars.GITLAB_USERNAME != '') && (vars.SCCACHE_ENDPOINT != '') && (github.event.pull_request.user.login != 'renovate[bot]') && 'true' || 'false' }}"
+    RUSTC_WRAPPER: "${{ !startsWith(github.ref, 'refs/tags/') && (github.event.pull_request.draft != true) && (vars.DOCKER_USERNAME != '') && (vars.GITLAB_USERNAME != '') && (vars.SCCACHE_ENDPOINT != '') && (github.event.pull_request.user.login != 'renovate[bot]') && 'sccache' || '' }}"
     SCCACHE_BUCKET: "${{ (github.event.pull_request.draft != true) && (vars.DOCKER_USERNAME != '') && (vars.GITLAB_USERNAME != '') && (vars.SCCACHE_ENDPOINT != '') && (github.event.pull_request.user.login != 'renovate[bot]') && 'sccache' || '' }}"
     SCCACHE_S3_USE_SSL: ${{ vars.SCCACHE_S3_USE_SSL }}
     SCCACHE_REGION: ${{ vars.SCCACHE_REGION }}
@@ -45,16 +45,14 @@ env:
     # Get error output from nix that we can actually use, and use our binary caches for the earlier CI steps
     NIX_CONFIG: |
       show-trace = true
-      extra-substituters = https://attic.kennel.juneis.dog/conduwuit https://attic.kennel.juneis.dog/conduit https://cache.lix.systems https://conduwuit.cachix.org https://aseipp-nix-cache.freetls.fastly.net
-      extra-trusted-public-keys = conduit:eEKoUwlQGDdYmAI/Q/0slVlegqh/QmAvQd7HBSm21Wk= conduwuit:BbycGUgTISsltcmH0qNjFR9dbrQNYgdIAcmViSGoVTE= cache.lix.systems:aBnZUw8zA7H35Cz2RyKFVs3H4PlGTLawyY5KRbvJR8o= conduwuit.cachix.org-1:MFRm6jcnfTf0jSAbmvLfhO3KBMt4px+1xaereWXp8Xg=
+      extra-substituters = https://attic.kennel.juneis.dog/conduwuit https://attic.kennel.juneis.dog/conduit https://conduwuit.cachix.org https://aseipp-nix-cache.freetls.fastly.net
+      extra-trusted-public-keys = conduit:eEKoUwlQGDdYmAI/Q/0slVlegqh/QmAvQd7HBSm21Wk= conduwuit:BbycGUgTISsltcmH0qNjFR9dbrQNYgdIAcmViSGoVTE= conduwuit.cachix.org-1:MFRm6jcnfTf0jSAbmvLfhO3KBMt4px+1xaereWXp8Xg=
       experimental-features = nix-command flakes
       extra-experimental-features = nix-command flakes
       accept-flake-config = true
-    # complement uses libolm
-    NIXPKGS_ALLOW_INSECURE: 1
     WEB_UPLOAD_SSH_USERNAME: ${{ secrets.WEB_UPLOAD_SSH_USERNAME }}
-    GH_SHA: ${{ github.sha }}
     GH_REF_NAME: ${{ github.ref_name }}
+    WEBSERVER_DIR_NAME: ${{ (github.head_ref != '' && format('merge-{0}-{1}', github.event.number, github.event.pull_request.user.login)) || github.ref_name }}-${{ github.sha }}
 
 permissions: {}
 
@@ -87,11 +85,13 @@ jobs:
                   END
 
                   echo "Checking connection"
-                  ssh -q website "echo test"
+                  ssh -q website "echo test" || ssh -q website "echo test"
 
                   echo "Creating commit rev directory on web server"
-                  ssh -q website "rm -rf /var/www/girlboss.ceo/~strawberry/conduwuit/ci-bins/$GITHUB_SHA/"
-                  ssh -q website "mkdir -v /var/www/girlboss.ceo/~strawberry/conduwuit/ci-bins/$GITHUB_SHA/"
+                  ssh -q website "rm -rf /var/www/girlboss.ceo/~strawberry/conduwuit/ci-bins/${WEBSERVER_DIR_NAME}/" || ssh -q website "rm -rf /var/www/girlboss.ceo/~strawberry/conduwuit/ci-bins/${WEBSERVER_DIR_NAME}/"
+                  ssh -q website "mkdir -v /var/www/girlboss.ceo/~strawberry/conduwuit/ci-bins/${WEBSERVER_DIR_NAME}/" || ssh -q website "mkdir -v /var/www/girlboss.ceo/~strawberry/conduwuit/ci-bins/${WEBSERVER_DIR_NAME}/"
+
+                  echo "SSH_WEBSITE=1" >> "$GITHUB_ENV"
 
             - name: Install liburing
               run: |
@@ -126,6 +126,9 @@ jobs:
             - uses: nixbuild/nix-quick-install-action@master
 
             - name: Restore and cache Nix store
+              # we want a fresh-state when we do releases/tags to avoid potential cache poisoning attacks impacting
+              # releases and tags
+              if: ${{ !startsWith(github.ref, 'refs/tags/') }}
               uses: nix-community/cache-nix-action@v5.1.0
               with:
                 # restore and save a cache using this key
@@ -155,7 +158,7 @@ jobs:
             - name: Apply Nix binary cache configuration
               run: |
                   sudo tee -a "${XDG_CONFIG_HOME:-$HOME/.config}/nix/nix.conf" > /dev/null <<EOF
-                  extra-substituters = https://attic.kennel.juneis.dog/conduwuit https://attic.kennel.juneis.dog/conduit https://cache.lix.systems https://conduwuit.cachix.org https://aseipp-nix-cache.freetls.fastly.net
+                  extra-substituters = https://attic.kennel.juneis.dog/conduwuit https://attic.kennel.juneis.dog/conduit https://conduwuit.cachix.org https://aseipp-nix-cache.freetls.fastly.net
                   extra-trusted-public-keys = conduit:eEKoUwlQGDdYmAI/Q/0slVlegqh/QmAvQd7HBSm21Wk= conduwuit:BbycGUgTISsltcmH0qNjFR9dbrQNYgdIAcmViSGoVTE= cache.lix.systems:aBnZUw8zA7H35Cz2RyKFVs3H4PlGTLawyY5KRbvJR8o= conduwuit.cachix.org-1:MFRm6jcnfTf0jSAbmvLfhO3KBMt4px+1xaereWXp8Xg=
                   experimental-features = nix-command flakes
                   extra-experimental-features = nix-command flakes
@@ -186,11 +189,16 @@ jobs:
 
             # use sccache for Rust
             - name: Run sccache-cache
-              if: (env.SCCACHE_GHA_ENABLED == 'true')
+              # we want a fresh-state when we do releases/tags to avoid potential cache poisoning attacks impacting
+              # releases and tags
+              if: ${{ (env.SCCACHE_GHA_ENABLED == 'true') && !startsWith(github.ref, 'refs/tags/') }}
               uses: mozilla-actions/sccache-action@main
 
             # use rust-cache
             - uses: Swatinem/rust-cache@v2
+              # we want a fresh-state when we do releases/tags to avoid potential cache poisoning attacks impacting
+              # releases and tags
+              if: ${{ !startsWith(github.ref, 'refs/tags/') }}
               with:
                 cache-all-crates: "true"
                 cache-on-failure: "true"
@@ -306,11 +314,16 @@ jobs:
                   END
 
                   echo "Checking connection"
-                  ssh -q website "echo test"
+                  ssh -q website "echo test" || ssh -q website "echo test"
+
+                  echo "SSH_WEBSITE=1" >> "$GITHUB_ENV"
 
             - uses: nixbuild/nix-quick-install-action@master
 
             - name: Restore and cache Nix store
+              # we want a fresh-state when we do releases/tags to avoid potential cache poisoning attacks impacting
+              # releases and tags
+              if: ${{ !startsWith(github.ref, 'refs/tags/') }}
               uses: nix-community/cache-nix-action@v5.1.0
               with:
                 # restore and save a cache using this key
@@ -340,7 +353,7 @@ jobs:
             - name: Apply Nix binary cache configuration
               run: |
                   sudo tee -a "${XDG_CONFIG_HOME:-$HOME/.config}/nix/nix.conf" > /dev/null <<EOF
-                  extra-substituters = https://attic.kennel.juneis.dog/conduwuit https://attic.kennel.juneis.dog/conduit https://cache.lix.systems https://conduwuit.cachix.org https://aseipp-nix-cache.freetls.fastly.net
+                  extra-substituters = https://attic.kennel.juneis.dog/conduwuit https://attic.kennel.juneis.dog/conduit https://conduwuit.cachix.org https://aseipp-nix-cache.freetls.fastly.net
                   extra-trusted-public-keys = conduit:eEKoUwlQGDdYmAI/Q/0slVlegqh/QmAvQd7HBSm21Wk= conduwuit:BbycGUgTISsltcmH0qNjFR9dbrQNYgdIAcmViSGoVTE= cache.lix.systems:aBnZUw8zA7H35Cz2RyKFVs3H4PlGTLawyY5KRbvJR8o= conduwuit.cachix.org-1:MFRm6jcnfTf0jSAbmvLfhO3KBMt4px+1xaereWXp8Xg=
                   experimental-features = nix-command flakes
                   extra-experimental-features = nix-command flakes
@@ -364,11 +377,16 @@ jobs:
 
             # use sccache for Rust
             - name: Run sccache-cache
-              if: (env.SCCACHE_GHA_ENABLED == 'true')
+              # we want a fresh-state when we do releases/tags to avoid potential cache poisoning attacks impacting
+              # releases and tags
+              if: ${{ (env.SCCACHE_GHA_ENABLED == 'true') && !startsWith(github.ref, 'refs/tags/') }}
               uses: mozilla-actions/sccache-action@main
 
             # use rust-cache
             - uses: Swatinem/rust-cache@v2
+              # we want a fresh-state when we do releases/tags to avoid potential cache poisoning attacks impacting
+              # releases and tags
+              if: ${{ !startsWith(github.ref, 'refs/tags/') }}
               with:
                 cache-all-crates: "true"
                 cache-on-failure: "true"
@@ -491,29 +509,29 @@ jobs:
             - name: Upload static-x86_64-linux-musl-all-features-x86_64-haswell-optimised to webserver
               if: ${{ matrix.target == 'x86_64-linux-musl' }}
               run: |
-                  if [ ! -z $WEB_UPLOAD_SSH_USERNAME ]; then
-                    scp static-x86_64-linux-musl-x86_64-haswell-optimised website:/var/www/girlboss.ceo/~strawberry/conduwuit/ci-bins/${GH_SHA}/static-x86_64-linux-musl-x86_64-haswell-optimised
+                  if [ ! -z $SSH_WEBSITE ]; then
+                    chmod +x static-x86_64-linux-musl-x86_64-haswell-optimised
+                    scp static-x86_64-linux-musl-x86_64-haswell-optimised website:/var/www/girlboss.ceo/~strawberry/conduwuit/ci-bins/${WEBSERVER_DIR_NAME}/static-x86_64-linux-musl-x86_64-haswell-optimised
                   fi
 
             - name: Upload static-${{ matrix.target }}-all-features to webserver
-              if: (startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main' || (github.event.pull_request.draft != true)) && (env.web_upload_ssh_private_key != '') && github.event.pull_request.user.login != 'renovate[bot]'
               run: |
-                  if [ ! -z $WEB_UPLOAD_SSH_USERNAME ]; then
-                    scp static-${{ matrix.target }} website:/var/www/girlboss.ceo/~strawberry/conduwuit/ci-bins/${GH_SHA}/static-${{ matrix.target }}
+                  if [ ! -z $SSH_WEBSITE ]; then
+                    chmod +x static-${{ matrix.target }}
+                    scp static-${{ matrix.target }} website:/var/www/girlboss.ceo/~strawberry/conduwuit/ci-bins/${WEBSERVER_DIR_NAME}/static-${{ matrix.target }}
                   fi
 
             - name: Upload static deb x86_64-linux-musl-all-features-x86_64-haswell-optimised to webserver
               if: ${{ matrix.target == 'x86_64-linux-musl' }}
               run: |
-                  if [ ! -z $WEB_UPLOAD_SSH_USERNAME ]; then
-                    scp x86_64-linux-musl-x86_64-haswell-optimised.deb website:/var/www/girlboss.ceo/~strawberry/conduwuit/ci-bins/${GH_SHA}/x86_64-linux-musl-x86_64-haswell-optimised.deb
+                  if [ ! -z $SSH_WEBSITE ]; then
+                    scp x86_64-linux-musl-x86_64-haswell-optimised.deb website:/var/www/girlboss.ceo/~strawberry/conduwuit/ci-bins/${WEBSERVER_DIR_NAME}/x86_64-linux-musl-x86_64-haswell-optimised.deb
                   fi
 
             - name: Upload static deb ${{ matrix.target }}-all-features to webserver
-              if: (startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main' || (github.event.pull_request.draft != true)) && (env.web_upload_ssh_private_key != '') && github.event.pull_request.user.login != 'renovate[bot]'
               run: |
-                  if [ ! -z $WEB_UPLOAD_SSH_USERNAME ]; then
-                    scp ${{ matrix.target }}.deb website:/var/www/girlboss.ceo/~strawberry/conduwuit/ci-bins/${GH_SHA}/${{ matrix.target }}.deb
+                  if [ ! -z $SSH_WEBSITE ]; then
+                    scp ${{ matrix.target }}.deb website:/var/www/girlboss.ceo/~strawberry/conduwuit/ci-bins/${WEBSERVER_DIR_NAME}/${{ matrix.target }}.deb
                   fi
 
             - name: Upload static-${{ matrix.target }}-debug-all-features to GitHub
@@ -532,17 +550,15 @@ jobs:
                   compression-level: 0
 
             - name: Upload static-${{ matrix.target }}-debug-all-features to webserver
-              if: (startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main' || (github.event.pull_request.draft != true)) && (env.web_upload_ssh_private_key != '') && github.event.pull_request.user.login != 'renovate[bot]'
               run: |
-                  if [ ! -z $WEB_UPLOAD_SSH_USERNAME ]; then
-                    scp static-${{ matrix.target }}-debug website:/var/www/girlboss.ceo/~strawberry/conduwuit/ci-bins/${GH_SHA}/static-${{ matrix.target }}-debug
+                  if [ ! -z $SSH_WEBSITE ]; then
+                    scp static-${{ matrix.target }}-debug website:/var/www/girlboss.ceo/~strawberry/conduwuit/ci-bins/${WEBSERVER_DIR_NAME}/static-${{ matrix.target }}-debug
                   fi
 
             - name: Upload static deb ${{ matrix.target }}-debug-all-features to webserver
-              if: (startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main' || (github.event.pull_request.draft != true)) && (env.web_upload_ssh_private_key != '') && github.event.pull_request.user.login != 'renovate[bot]'
               run: |
-                  if [ ! -z $WEB_UPLOAD_SSH_USERNAME ]; then
-                    scp ${{ matrix.target }}-debug.deb website:/var/www/girlboss.ceo/~strawberry/conduwuit/ci-bins/${GH_SHA}/${{ matrix.target }}-debug.deb
+                  if [ ! -z $SSH_WEBSITE ]; then
+                    scp ${{ matrix.target }}-debug.deb website:/var/www/girlboss.ceo/~strawberry/conduwuit/ci-bins/${WEBSERVER_DIR_NAME}/${{ matrix.target }}-debug.deb
                   fi
 
             - name: Build OCI image ${{ matrix.target }}-all-features
@@ -564,6 +580,14 @@ jobs:
 
                   cp -v -f result oci-image-${{ matrix.target }}-debug.tar.gz
 
+            - name: Upload OCI image x86_64-linux-musl-all-features-x86_64-haswell-optimised to GitHub
+              if: ${{ matrix.target == 'x86_64-linux-musl' }}
+              uses: actions/upload-artifact@v4
+              with:
+                  name: oci-image-x86_64-linux-musl-all-features-x86_64-haswell-optimised
+                  path: oci-image-x86_64-linux-musl-all-features-x86_64-haswell-optimised.tar.gz
+                  if-no-files-found: error
+                  compression-level: 0
             - name: Upload OCI image ${{ matrix.target }}-all-features to GitHub
               uses: actions/upload-artifact@v4
               with:
@@ -583,22 +607,20 @@ jobs:
             - name: Upload OCI image x86_64-linux-musl-all-features-x86_64-haswell-optimised.tar.gz to webserver
               if: ${{ matrix.target == 'x86_64-linux-musl' }}
               run: |
-                  if [ ! -z $WEB_UPLOAD_SSH_USERNAME ]; then
-                    scp oci-image-x86_64-linux-musl-all-features-x86_64-haswell-optimised.tar.gz website:/var/www/girlboss.ceo/~strawberry/conduwuit/ci-bins/${GH_SHA}/oci-image-x86_64-linux-musl-all-features-x86_64-haswell-optimised.tar.gz
+                  if [ ! -z $SSH_WEBSITE ]; then
+                    scp oci-image-x86_64-linux-musl-all-features-x86_64-haswell-optimised.tar.gz website:/var/www/girlboss.ceo/~strawberry/conduwuit/ci-bins/${WEBSERVER_DIR_NAME}/oci-image-x86_64-linux-musl-all-features-x86_64-haswell-optimised.tar.gz
                   fi
 
             - name: Upload OCI image ${{ matrix.target }}-all-features to webserver
-              if: (startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main' || (github.event.pull_request.draft != true)) && (env.web_upload_ssh_private_key != '') && github.event.pull_request.user.login != 'renovate[bot]'
               run: |
-                  if [ ! -z $WEB_UPLOAD_SSH_USERNAME ]; then
-                    scp oci-image-${{ matrix.target }}.tar.gz website:/var/www/girlboss.ceo/~strawberry/conduwuit/ci-bins/${GH_SHA}/oci-image-${{ matrix.target }}.tar.gz
+                  if [ ! -z $SSH_WEBSITE ]; then
+                    scp oci-image-${{ matrix.target }}.tar.gz website:/var/www/girlboss.ceo/~strawberry/conduwuit/ci-bins/${WEBSERVER_DIR_NAME}/oci-image-${{ matrix.target }}.tar.gz
                   fi
 
             - name: Upload OCI image ${{ matrix.target }}-debug-all-features to webserver
-              if: (startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main' || (github.event.pull_request.draft != true)) && (env.web_upload_ssh_private_key != '') && github.event.pull_request.user.login != 'renovate[bot]'
               run: |
-                  if [ ! -z $WEB_UPLOAD_SSH_USERNAME ]; then
-                    scp oci-image-${{ matrix.target }}-debug.tar.gz website:/var/www/girlboss.ceo/~strawberry/conduwuit/ci-bins/${GH_SHA}/oci-image-${{ matrix.target }}-debug.tar.gz
+                  if [ ! -z $SSH_WEBSITE ]; then
+                    scp oci-image-${{ matrix.target }}-debug.tar.gz website:/var/www/girlboss.ceo/~strawberry/conduwuit/ci-bins/${WEBSERVER_DIR_NAME}/oci-image-${{ matrix.target }}-debug.tar.gz
                   fi
 
     build_mac_binaries:
@@ -637,7 +659,9 @@ jobs:
                   END
 
                   echo "Checking connection"
-                  ssh -q website "echo test"
+                  ssh -q website "echo test" || ssh -q website "echo test"
+
+                  echo "SSH_WEBSITE=1" >> "$GITHUB_ENV"
 
             - name: Tag comparison check
               if: ${{ startsWith(github.ref, 'refs/tags/v') && !endsWith(github.ref, '-rc') }}
@@ -653,7 +677,9 @@ jobs:
 
             # use sccache for Rust
             - name: Run sccache-cache
-              if: (env.SCCACHE_GHA_ENABLED == 'true')
+              # we want a fresh-state when we do releases/tags to avoid potential cache poisoning attacks impacting
+              # releases and tags
+              if: ${{ (env.SCCACHE_GHA_ENABLED == 'true') && !startsWith(github.ref, 'refs/tags/') }}
               uses: mozilla-actions/sccache-action@main
 
             # use rust-cache
@@ -667,7 +693,7 @@ jobs:
             - name: Build macOS x86_64 binary
               if: ${{ matrix.os == 'macos-13' }}
               run: |
-                  CONDUWUIT_VERSION_EXTRA="$(git rev-parse --short ${{ github.sha }})" cargo build --release
+                  CONDUWUIT_VERSION_EXTRA="$(git rev-parse --short ${{ github.sha }})" cargo build --release --locked --features=perf_measurements,sentry_telemetry,direct_tls
                   cp -v -f target/release/conduwuit conduwuit-macos-x86_64
                   otool -L conduwuit-macos-x86_64
 
@@ -675,12 +701,13 @@ jobs:
             - name: Run x86_64 macOS release binary
               if: ${{ matrix.os == 'macos-13' }}
               run: |
+                  ./conduwuit-macos-x86_64 --help
                   ./conduwuit-macos-x86_64 --version
 
             - name: Build macOS arm64 binary
               if: ${{ matrix.os == 'macos-latest' }}
               run: |
-                  CONDUWUIT_VERSION_EXTRA="$(git rev-parse --short ${{ github.sha }})" cargo build --release
+                  CONDUWUIT_VERSION_EXTRA="$(git rev-parse --short ${{ github.sha }})" cargo build --release --locked --features=perf_measurements,sentry_telemetry,direct_tls
                   cp -v -f target/release/conduwuit conduwuit-macos-arm64
                   otool -L conduwuit-macos-arm64
 
@@ -688,20 +715,23 @@ jobs:
             - name: Run arm64 macOS release binary
               if: ${{ matrix.os == 'macos-latest' }}
               run: |
+                  ./conduwuit-macos-arm64 --help
                   ./conduwuit-macos-arm64 --version
 
             - name: Upload macOS x86_64 binary to webserver
               if: ${{ matrix.os == 'macos-13' }}
               run: |
-                  if [ ! -z $WEB_UPLOAD_SSH_USERNAME ]; then
-                    scp conduwuit-macos-x86_64 website:/var/www/girlboss.ceo/~strawberry/conduwuit/ci-bins/${GH_SHA}/conduwuit-macos-x86_64
+                  if [ ! -z $SSH_WEBSITE ]; then
+                    chmod +x conduwuit-macos-x86_64
+                    scp conduwuit-macos-x86_64 website:/var/www/girlboss.ceo/~strawberry/conduwuit/ci-bins/${WEBSERVER_DIR_NAME}/conduwuit-macos-x86_64
                   fi
 
             - name: Upload macOS arm64 binary to webserver
               if: ${{ matrix.os == 'macos-latest' }}
               run: |
-                  if [ ! -z $WEB_UPLOAD_SSH_USERNAME ]; then
-                    scp conduwuit-macos-arm64 website:/var/www/girlboss.ceo/~strawberry/conduwuit/ci-bins/${GH_SHA}/conduwuit-macos-arm64
+                  if [ ! -z $SSH_WEBSITE ]; then
+                    chmod +x conduwuit-macos-arm64
+                    scp conduwuit-macos-arm64 website:/var/www/girlboss.ceo/~strawberry/conduwuit/ci-bins/${WEBSERVER_DIR_NAME}/conduwuit-macos-arm64
                   fi
 
             - name: Upload macOS x86_64 binary
@@ -733,24 +763,17 @@ jobs:
     docker:
         name: Docker publish
         runs-on: ubuntu-24.04
-        needs: [build, variables]
+        needs: [build, variables, tests]
         permissions:
           packages: write
           contents: read
         if: (startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main' || (github.event.pull_request.draft != true)) && github.event.pull_request.user.login != 'renovate[bot]'
         env:
-            DOCKER_ARM64: docker.io/${{ needs.variables.outputs.github_repository }}:${{ (github.head_ref != '' && format('merge-{0}-{1}', github.event.number, github.event.pull_request.user.login)) || github.ref_name }}-${{ github.sha }}-arm64v8
-            DOCKER_AMD64: docker.io/${{ needs.variables.outputs.github_repository }}:${{ (github.head_ref != '' && format('merge-{0}-{1}', github.event.number, github.event.pull_request.user.login)) || github.ref_name }}-${{ github.sha }}-amd64
-            DOCKER_TAG: docker.io/${{ needs.variables.outputs.github_repository }}:${{ (github.head_ref != '' && format('merge-{0}-{1}', github.event.number, github.event.pull_request.user.login)) || github.ref_name }}-${{ github.sha }}
-            DOCKER_BRANCH: docker.io/${{ needs.variables.outputs.github_repository }}:${{ (startsWith(github.ref, 'refs/tags/v') && !endsWith(github.ref, '-rc') && 'latest') || (github.head_ref != '' && format('merge-{0}-{1}', github.event.number, github.event.pull_request.user.login)) || github.ref_name }}
-            GHCR_ARM64: ghcr.io/${{ needs.variables.outputs.github_repository }}:${{ (github.head_ref != '' && format('merge-{0}-{1}', github.event.number, github.event.pull_request.user.login)) || github.ref_name }}-${{ github.sha }}-arm64v8
-            GHCR_AMD64: ghcr.io/${{ needs.variables.outputs.github_repository }}:${{ (github.head_ref != '' && format('merge-{0}-{1}', github.event.number, github.event.pull_request.user.login)) || github.ref_name }}-${{ github.sha }}-amd64
-            GHCR_TAG: ghcr.io/${{ needs.variables.outputs.github_repository }}:${{ (github.head_ref != '' && format('merge-{0}-{1}', github.event.number, github.event.pull_request.user.login)) || github.ref_name }}-${{ github.sha }}
-            GHCR_BRANCH: ghcr.io/${{ needs.variables.outputs.github_repository }}:${{ (startsWith(github.ref, 'refs/tags/v') && !endsWith(github.ref, '-rc') && 'latest') || (github.head_ref != '' && format('merge-{0}-{1}', github.event.number, github.event.pull_request.user.login)) || github.ref_name }}
-            GLCR_ARM64: registry.gitlab.com/conduwuit/conduwuit:${{ (github.head_ref != '' && format('merge-{0}-{1}', github.event.number, github.event.pull_request.user.login)) || github.ref_name }}-${{ github.sha }}-arm64v8
-            GLCR_AMD64: registry.gitlab.com/conduwuit/conduwuit:${{ (github.head_ref != '' && format('merge-{0}-{1}', github.event.number, github.event.pull_request.user.login)) || github.ref_name }}-${{ github.sha }}-amd64
-            GLCR_TAG: registry.gitlab.com/conduwuit/conduwuit:${{ (github.head_ref != '' && format('merge-{0}-{1}', github.event.number, github.event.pull_request.user.login)) || github.ref_name }}-${{ github.sha }}
-            GLCR_BRANCH: registry.gitlab.com/conduwuit/conduwuit:${{ (startsWith(github.ref, 'refs/tags/v') && !endsWith(github.ref, '-rc') && 'latest') || (github.head_ref != '' && format('merge-{0}-{1}', github.event.number, github.event.pull_request.user.login)) || github.ref_name }}
+            DOCKER_HUB_REPO: docker.io/${{ needs.variables.outputs.github_repository }}
+            GHCR_REPO: ghcr.io/${{ needs.variables.outputs.github_repository }}
+            GLCR_REPO: registry.gitlab.com/conduwuit/conduwuit
+            UNIQUE_TAG: ${{ (github.head_ref != '' && format('merge-{0}-{1}', github.event.number, github.event.pull_request.user.login)) || github.ref_name }}-${{ github.sha }}
+            BRANCH_TAG: ${{ (startsWith(github.ref, 'refs/tags/v') && !endsWith(github.ref, '-rc') && 'latest') || (github.head_ref != '' && format('merge-{0}-{1}', github.event.number, github.event.pull_request.user.login)) || github.ref_name }}
 
             DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
             GITLAB_TOKEN: ${{ secrets.GITLAB_TOKEN }}
@@ -784,143 +807,187 @@ jobs:
 
             - name: Move OCI images into position
               run: |
+                  mv -v oci-image-x86_64-linux-musl-all-features-x86_64-haswell-optimised/*.tar.gz oci-image-amd64-haswell-optimised.tar.gz
                   mv -v oci-image-x86_64-linux-musl/*.tar.gz oci-image-amd64.tar.gz
                   mv -v oci-image-aarch64-linux-musl/*.tar.gz oci-image-arm64v8.tar.gz
                   mv -v oci-image-x86_64-linux-musl-debug/*.tar.gz oci-image-amd64-debug.tar.gz
                   mv -v oci-image-aarch64-linux-musl-debug/*.tar.gz oci-image-arm64v8-debug.tar.gz
 
+            - name: Load and push amd64 haswell image
+              run: |
+                  docker load -i oci-image-amd64-haswell-optimised.tar.gz
+                  if [ ! -z $DOCKERHUB_TOKEN ]; then
+                    docker tag $(docker images -q conduwuit:main) ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-haswell
+                    docker push ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-haswell
+                  fi
+                  if [ $GHCR_ENABLED = "true" ]; then
+                    docker tag $(docker images -q conduwuit:main) ${GHCR_REPO}:${UNIQUE_TAG}-haswell
+                    docker push ${GHCR_REPO}:${UNIQUE_TAG}-haswell
+                  fi
+                  if [ ! -z $GITLAB_TOKEN ]; then
+                    docker tag $(docker images -q conduwuit:main) ${GLCR_REPO}:${UNIQUE_TAG}-haswell
+                    docker push ${GLCR_REPO}:${UNIQUE_TAG}-haswell
+                  fi
+
             - name: Load and push amd64 image
               run: |
                   docker load -i oci-image-amd64.tar.gz
                   if [ ! -z $DOCKERHUB_TOKEN ]; then
-                    docker tag $(docker images -q conduwuit:main) ${DOCKER_AMD64}
-                    docker push ${DOCKER_AMD64}
+                    docker tag $(docker images -q conduwuit:main) ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-amd64
+                    docker push ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-amd64
                   fi
                   if [ $GHCR_ENABLED = "true" ]; then
-                    docker tag $(docker images -q conduwuit:main) ${GHCR_AMD64}
-                    docker push ${GHCR_AMD64}
+                    docker tag $(docker images -q conduwuit:main) ${GHCR_REPO}:${UNIQUE_TAG}-amd64
+                    docker push ${GHCR_REPO}:${UNIQUE_TAG}-amd64
                   fi
                   if [ ! -z $GITLAB_TOKEN ]; then
-                    docker tag $(docker images -q conduwuit:main) ${GLCR_AMD64}
-                    docker push ${GLCR_AMD64}
+                    docker tag $(docker images -q conduwuit:main) ${GLCR_REPO}:${UNIQUE_TAG}-amd64
+                    docker push ${GLCR_REPO}:${UNIQUE_TAG}-amd64
                   fi
 
             - name: Load and push arm64 image
               run: |
                   docker load -i oci-image-arm64v8.tar.gz
                   if [ ! -z $DOCKERHUB_TOKEN ]; then
-                    docker tag $(docker images -q conduwuit:main) ${DOCKER_ARM64}
-                    docker push ${DOCKER_ARM64}
+                    docker tag $(docker images -q conduwuit:main) ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-arm64v8
+                    docker push ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-arm64v8
                   fi
                   if [ $GHCR_ENABLED = "true" ]; then
-                    docker tag $(docker images -q conduwuit:main) ${GHCR_ARM64}
-                    docker push ${GHCR_ARM64}
+                    docker tag $(docker images -q conduwuit:main) ${GHCR_REPO}:${UNIQUE_TAG}-arm64v8
+                    docker push ${GHCR_REPO}:${UNIQUE_TAG}-arm64v8
                   fi
                   if [ ! -z $GITLAB_TOKEN ]; then
-                    docker tag $(docker images -q conduwuit:main) ${GLCR_ARM64}
-                    docker push ${GLCR_ARM64}
+                    docker tag $(docker images -q conduwuit:main) ${GLCR_REPO}:${UNIQUE_TAG}-arm64v8
+                    docker push ${GLCR_REPO}:${UNIQUE_TAG}-arm64v8
                   fi
 
             - name: Load and push amd64 debug image
               run: |
                   docker load -i oci-image-amd64-debug.tar.gz
                   if [ ! -z $DOCKERHUB_TOKEN ]; then
-                    docker tag $(docker images -q conduwuit:main) ${DOCKER_AMD64}-debug
-                    docker push ${DOCKER_AMD64}-debug
+                    docker tag $(docker images -q conduwuit:main) ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-amd64-debug
+                    docker push ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-amd64-debug
                   fi
                   if [ $GHCR_ENABLED = "true" ]; then
-                    docker tag $(docker images -q conduwuit:main) ${GHCR_AMD64}-debug
-                    docker push ${GHCR_AMD64}-debug
+                    docker tag $(docker images -q conduwuit:main) ${GHCR_REPO}:${UNIQUE_TAG}-amd64-debug
+                    docker push ${GHCR_REPO}:${UNIQUE_TAG}-amd64-debug
                   fi
                   if [ ! -z $GITLAB_TOKEN ]; then
-                    docker tag $(docker images -q conduwuit:main) ${GLCR_AMD64}-debug
-                    docker push ${GLCR_AMD64}-debug
+                    docker tag $(docker images -q conduwuit:main) ${GLCR_REPO}:${UNIQUE_TAG}-amd64-debug
+                    docker push ${GLCR_REPO}:${UNIQUE_TAG}-amd64-debug
                   fi
 
             - name: Load and push arm64 debug image
               run: |
                   docker load -i oci-image-arm64v8-debug.tar.gz
                   if [ ! -z $DOCKERHUB_TOKEN ]; then
-                    docker tag $(docker images -q conduwuit:main) ${DOCKER_ARM64}-debug
-                    docker push ${DOCKER_ARM64}-debug
+                    docker tag $(docker images -q conduwuit:main) ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-arm64v8-debug
+                    docker push ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-arm64v8-debug
                   fi
                   if [ $GHCR_ENABLED = "true" ]; then
-                    docker tag $(docker images -q conduwuit:main) ${GHCR_ARM64}-debug
-                    docker push ${GHCR_ARM64}-debug
+                    docker tag $(docker images -q conduwuit:main) ${GHCR_REPO}:${UNIQUE_TAG}-arm64v8-debug
+                    docker push ${GHCR_REPO}:${UNIQUE_TAG}-arm64v8-debug
                   fi
                   if [ ! -z $GITLAB_TOKEN ]; then
-                    docker tag $(docker images -q conduwuit:main) ${GLCR_ARM64}-debug
-                    docker push ${GLCR_ARM64}-debug
+                    docker tag $(docker images -q conduwuit:main) ${GLCR_REPO}:${UNIQUE_TAG}-arm64v8-debug
+                    docker push ${GLCR_REPO}:${UNIQUE_TAG}-arm64v8-debug
+                  fi
+
+            - name: Create Docker haswell manifests
+              run: |
+                  # Dockerhub Container Registry
+                  if [ ! -z $DOCKERHUB_TOKEN ]; then
+                    docker manifest create ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-haswell --amend ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-haswell
+                    docker manifest create ${DOCKER_HUB_REPO}:${BRANCH_TAG}-haswell --amend ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-haswell
+                  fi
+                  # GitHub Container Registry
+                  if [ $GHCR_ENABLED = "true" ]; then
+                    docker manifest create ${GHCR_REPO}:${UNIQUE_TAG}-haswell --amend ${GHCR_REPO}:${UNIQUE_TAG}-haswell
+                    docker manifest create ${GHCR_REPO}:${BRANCH_TAG}-haswell --amend ${GHCR_REPO}:${UNIQUE_TAG}-haswell
+                  fi
+                  # GitLab Container Registry
+                  if [ ! -z $GITLAB_TOKEN ]; then
+                    docker manifest create ${GLCR_REPO}:${UNIQUE_TAG}-haswell --amend ${GLCR_REPO}:${UNIQUE_TAG}-haswell
+                    docker manifest create ${GLCR_REPO}:${BRANCH_TAG}-haswell --amend ${GLCR_REPO}:${UNIQUE_TAG}-haswell
                   fi
 
             - name: Create Docker combined manifests
               run: |
                   # Dockerhub Container Registry
                   if [ ! -z $DOCKERHUB_TOKEN ]; then
-                    docker manifest create ${DOCKER_TAG} --amend ${DOCKER_ARM64} --amend ${DOCKER_AMD64}
-                    docker manifest create ${DOCKER_BRANCH} --amend ${DOCKER_ARM64} --amend ${DOCKER_AMD64}
+                    docker manifest create ${DOCKER_HUB_REPO}:${UNIQUE_TAG} --amend ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-arm64v8 --amend ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-amd64
+                    docker manifest create ${DOCKER_HUB_REPO}:${BRANCH_TAG} --amend ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-arm64v8 --amend ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-amd64
                   fi
                   # GitHub Container Registry
                   if [ $GHCR_ENABLED = "true" ]; then
-                    docker manifest create ${GHCR_TAG} --amend ${GHCR_ARM64} --amend ${GHCR_AMD64}
-                    docker manifest create ${GHCR_BRANCH} --amend ${GHCR_ARM64} --amend ${GHCR_AMD64}
+                    docker manifest create ${GHCR_REPO}:${UNIQUE_TAG} --amend ${GHCR_REPO}:${UNIQUE_TAG}-arm64v8 --amend ${GHCR_REPO}:${UNIQUE_TAG}-amd64
+                    docker manifest create ${GHCR_REPO}:${BRANCH_TAG} --amend ${GHCR_REPO}:${UNIQUE_TAG}-arm64v8 --amend ${GHCR_REPO}:${UNIQUE_TAG}-amd64
                   fi
                   # GitLab Container Registry
                   if [ ! -z $GITLAB_TOKEN ]; then
-                    docker manifest create ${GLCR_TAG} --amend ${GLCR_ARM64} --amend ${GLCR_AMD64}
-                    docker manifest create ${GLCR_BRANCH} --amend ${GLCR_ARM64} --amend ${GLCR_AMD64}
+                    docker manifest create ${GLCR_REPO}:${UNIQUE_TAG} --amend ${GLCR_REPO}:${UNIQUE_TAG}-arm64v8 --amend ${GLCR_REPO}:${UNIQUE_TAG}-amd64
+                    docker manifest create ${GLCR_REPO}:${BRANCH_TAG} --amend ${GLCR_REPO}:${UNIQUE_TAG}-arm64v8 --amend ${GLCR_REPO}:${UNIQUE_TAG}-amd64
                   fi
 
             - name: Create Docker combined debug manifests
               run: |
                   # Dockerhub Container Registry
                   if [ ! -z $DOCKERHUB_TOKEN ]; then
-                    docker manifest create ${DOCKER_TAG}-debug --amend ${DOCKER_ARM64}-debug --amend ${DOCKER_AMD64}-debug
-                    docker manifest create ${DOCKER_BRANCH}-debug --amend ${DOCKER_ARM64}-debug --amend ${DOCKER_AMD64}-debug
+                    docker manifest create ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-debug --amend ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-arm64v8-debug --amend ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-amd64-debug
+                    docker manifest create ${DOCKER_HUB_REPO}:${BRANCH_TAG}-debug --amend ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-arm64v8-debug --amend ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-amd64-debug
                   fi
                   # GitHub Container Registry
                   if [ $GHCR_ENABLED = "true" ]; then
-                    docker manifest create ${GHCR_TAG}-debug --amend ${GHCR_ARM64}-debug --amend ${GHCR_AMD64}-debug
-                    docker manifest create ${GHCR_BRANCH}-debug --amend ${GHCR_ARM64}-debug --amend ${GHCR_AMD64}-debug
+                    docker manifest create ${GHCR_REPO}:${UNIQUE_TAG}-debug --amend ${GHCR_REPO}:${UNIQUE_TAG}-arm64v8-debug --amend ${GHCR_REPO}:${UNIQUE_TAG}-amd64-debug
+                    docker manifest create ${GHCR_REPO}:${BRANCH_TAG}-debug --amend ${GHCR_REPO}:${UNIQUE_TAG}-arm64v8-debug --amend ${GHCR_REPO}:${UNIQUE_TAG}-amd64-debug
                   fi
                   # GitLab Container Registry
                   if [ ! -z $GITLAB_TOKEN ]; then
-                    docker manifest create ${GLCR_TAG}-debug --amend ${GLCR_ARM64}-debug --amend ${GLCR_AMD64}-debug
-                    docker manifest create ${GLCR_BRANCH}-debug --amend ${GLCR_ARM64}-debug --amend ${GLCR_AMD64}-debug
+                    docker manifest create ${GLCR_REPO}:${UNIQUE_TAG}-debug --amend ${GLCR_REPO}:${UNIQUE_TAG}-arm64v8-debug --amend ${GLCR_REPO}:${UNIQUE_TAG}-amd64-debug
+                    docker manifest create ${GLCR_REPO}:${BRANCH_TAG}-debug --amend ${GLCR_REPO}:${UNIQUE_TAG}-arm64v8-debug --amend ${GLCR_REPO}:${UNIQUE_TAG}-amd64-debug
                   fi
 
             - name: Push manifests to Docker registries
               run: |
                   if [ ! -z $DOCKERHUB_TOKEN ]; then
-                    docker manifest push ${DOCKER_TAG}
-                    docker manifest push ${DOCKER_BRANCH}
-                    docker manifest push ${DOCKER_TAG}-debug
-                    docker manifest push ${DOCKER_BRANCH}-debug
+                    docker manifest push ${DOCKER_HUB_REPO}:${UNIQUE_TAG}
+                    docker manifest push ${DOCKER_HUB_REPO}:${BRANCH_TAG}
+                    docker manifest push ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-debug
+                    docker manifest push ${DOCKER_HUB_REPO}:${BRANCH_TAG}-debug
+                    docker manifest push ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-haswell
+                    docker manifest push ${DOCKER_HUB_REPO}:${BRANCH_TAG}-haswell
                   fi
                   if [ $GHCR_ENABLED = "true" ]; then
-                    docker manifest push ${GHCR_TAG}
-                    docker manifest push ${GHCR_BRANCH}
-                    docker manifest push ${GHCR_TAG}-debug
-                    docker manifest push ${GHCR_BRANCH}-debug
+                    docker manifest push ${GHCR_REPO}:${UNIQUE_TAG}
+                    docker manifest push ${GHCR_REPO}:${BRANCH_TAG}
+                    docker manifest push ${GHCR_REPO}:${UNIQUE_TAG}-debug
+                    docker manifest push ${GHCR_REPO}:${BRANCH_TAG}-debug
+                    docker manifest push ${GHCR_REPO}:${UNIQUE_TAG}-haswell
+                    docker manifest push ${GHCR_REPO}:${BRANCH_TAG}-haswell
                   fi
                   if [ ! -z $GITLAB_TOKEN ]; then
-                    docker manifest push ${GLCR_TAG}
-                    docker manifest push ${GLCR_BRANCH}
-                    docker manifest push ${GLCR_TAG}-debug
-                    docker manifest push ${GLCR_BRANCH}-debug
+                    docker manifest push ${GLCR_REPO}:${UNIQUE_TAG}
+                    docker manifest push ${GLCR_REPO}:${BRANCH_TAG}
+                    docker manifest push ${GLCR_REPO}:${UNIQUE_TAG}-debug
+                    docker manifest push ${GLCR_REPO}:${BRANCH_TAG}-debug
+                    docker manifest push ${GLCR_REPO}:${UNIQUE_TAG}-haswell
+                    docker manifest push ${GLCR_REPO}:${BRANCH_TAG}-haswell
                   fi
 
             - name: Add Image Links to Job Summary
               run: |
                   if [ ! -z $DOCKERHUB_TOKEN ]; then
-                    echo "- \`docker pull ${DOCKER_TAG}\`" >> $GITHUB_STEP_SUMMARY
-                    echo "- \`docker pull ${DOCKER_TAG}-debug\`" >> $GITHUB_STEP_SUMMARY
+                    echo "- \`docker pull ${DOCKER_HUB_REPO}:${UNIQUE_TAG}\`" >> $GITHUB_STEP_SUMMARY
+                    echo "- \`docker pull ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-debug\`" >> $GITHUB_STEP_SUMMARY
+                    echo "- \`docker pull ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-haswell\`" >> $GITHUB_STEP_SUMMARY
                   fi
                   if [ $GHCR_ENABLED = "true" ]; then
-                    echo "- \`docker pull ${GHCR_TAG}\`" >> $GITHUB_STEP_SUMMARY
-                    echo "- \`docker pull ${GHCR_TAG}-debug\`" >> $GITHUB_STEP_SUMMARY
+                    echo "- \`docker pull ${GHCR_REPO}:${UNIQUE_TAG}\`" >> $GITHUB_STEP_SUMMARY
+                    echo "- \`docker pull ${GHCR_REPO}:${UNIQUE_TAG}-debug\`" >> $GITHUB_STEP_SUMMARY
+                    echo "- \`docker pull ${GHCR_REPO}:${UNIQUE_TAG}-haswell\`" >> $GITHUB_STEP_SUMMARY
                   fi
                   if [ ! -z $GITLAB_TOKEN ]; then
-                    echo "- \`docker pull ${GLCR_TAG}\`" >> $GITHUB_STEP_SUMMARY
-                    echo "- \`docker pull ${GLCR_TAG}-debug\`" >> $GITHUB_STEP_SUMMARY
+                    echo "- \`docker pull ${GLCR_REPO}:${UNIQUE_TAG}\`" >> $GITHUB_STEP_SUMMARY
+                    echo "- \`docker pull ${GLCR_REPO}:${UNIQUE_TAG}-debug\`" >> $GITHUB_STEP_SUMMARY
+                    echo "- \`docker pull ${GLCR_REPO}:${UNIQUE_TAG}-haswell\`" >> $GITHUB_STEP_SUMMARY
                   fi

.github/workflows/docker-hub-description.yml

@@ -0,0 +1,41 @@
+name: Update Docker Hub Description
+
+on:
+  push:
+    branches:
+      - main
+    paths:
+      - README.md
+      - .github/workflows/docker-hub-description.yml
+
+  workflow_dispatch:
+
+jobs:
+  dockerHubDescription:
+    runs-on: ubuntu-latest
+    if: ${{ (startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main' || (github.event.pull_request.draft != true)) && github.event.pull_request.user.login != 'renovate[bot]' && (vars.DOCKER_USERNAME != '') }}
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        persist-credentials: false
+
+    - name: Setting variables
+      uses: actions/github-script@v7
+      id: var
+      with:
+        script: |
+          const githubRepo = '${{ github.repository }}'.toLowerCase()
+          const repoId = githubRepo.split('/')[1]
+          
+          core.setOutput('github_repository', githubRepo)
+          const dockerRepo = '${{ vars.DOCKER_USERNAME }}'.toLowerCase() + '/' + repoId
+          core.setOutput('docker_repo', dockerRepo)
+
+    - name: Docker Hub Description
+      uses: peter-evans/dockerhub-description@v4
+      with:
+        username: ${{ vars.DOCKER_USERNAME }}
+        password: ${{ secrets.DOCKERHUB_TOKEN }}
+        repository: ${{ steps.var.outputs.docker_repo }}
+        short-description: ${{ github.event.repository.description }}
+        enable-url-completion: true

.github/workflows/documentation.yml

@@ -24,7 +24,7 @@ env:
   # Get error output from nix that we can actually use, and use our binary caches for the earlier CI steps
   NIX_CONFIG: |
     show-trace = true
-    extra-substituters = extra-substituters = https://attic.kennel.juneis.dog/conduwuit https://attic.kennel.juneis.dog/conduit https://cache.lix.systems https://conduwuit.cachix.org https://aseipp-nix-cache.freetls.fastly.net
+    extra-substituters = extra-substituters = https://attic.kennel.juneis.dog/conduwuit https://attic.kennel.juneis.dog/conduit https://conduwuit.cachix.org https://aseipp-nix-cache.freetls.fastly.net
     extra-trusted-public-keys = conduit:eEKoUwlQGDdYmAI/Q/0slVlegqh/QmAvQd7HBSm21Wk= conduwuit:BbycGUgTISsltcmH0qNjFR9dbrQNYgdIAcmViSGoVTE= cache.lix.systems:aBnZUw8zA7H35Cz2RyKFVs3H4PlGTLawyY5KRbvJR8o= conduwuit.cachix.org-1:MFRm6jcnfTf0jSAbmvLfhO3KBMt4px+1xaereWXp8Xg=
     experimental-features = nix-command flakes
     extra-experimental-features = nix-command flakes
@@ -73,6 +73,9 @@ jobs:
       - uses: nixbuild/nix-quick-install-action@master
 
       - name: Restore and cache Nix store
+        # we want a fresh-state when we do releases/tags to avoid potential cache poisoning attacks impacting
+        # releases and tags
+        if: ${{ !startsWith(github.ref, 'refs/tags/') }}
         uses: nix-community/cache-nix-action@v5.1.0
         with:
           # restore and save a cache using this key
@@ -102,7 +105,7 @@ jobs:
       - name: Apply Nix binary cache configuration
         run: |
             sudo tee -a "${XDG_CONFIG_HOME:-$HOME/.config}/nix/nix.conf" > /dev/null <<EOF
-            extra-substituters = https://attic.kennel.juneis.dog/conduwuit https://attic.kennel.juneis.dog/conduit https://cache.lix.systems https://conduwuit.cachix.org https://aseipp-nix-cache.freetls.fastly.net
+            extra-substituters = https://attic.kennel.juneis.dog/conduwuit https://attic.kennel.juneis.dog/conduit https://conduwuit.cachix.org https://aseipp-nix-cache.freetls.fastly.net
             extra-trusted-public-keys = conduit:eEKoUwlQGDdYmAI/Q/0slVlegqh/QmAvQd7HBSm21Wk= conduwuit:BbycGUgTISsltcmH0qNjFR9dbrQNYgdIAcmViSGoVTE= cache.lix.systems:aBnZUw8zA7H35Cz2RyKFVs3H4PlGTLawyY5KRbvJR8o= conduwuit.cachix.org-1:MFRm6jcnfTf0jSAbmvLfhO3KBMt4px+1xaereWXp8Xg=
             experimental-features = nix-command flakes
             extra-experimental-features = nix-command flakes

.gitlab-ci.yml

@@ -12,8 +12,8 @@ variables:
   TRANSFER_METER_FREQUENCY: 5s
   NIX_CONFIG: |
     show-trace = true
-    extra-substituters = https://attic.kennel.juneis.dog/conduit https://attic.kennel.juneis.dog/conduwuit https://cache.lix.systems https://conduwuit.cachix.org
-    extra-trusted-public-keys = conduit:eEKoUwlQGDdYmAI/Q/0slVlegqh/QmAvQd7HBSm21Wk= conduwuit:BbycGUgTISsltcmH0qNjFR9dbrQNYgdIAcmViSGoVTE= cache.lix.systems:aBnZUw8zA7H35Cz2RyKFVs3H4PlGTLawyY5KRbvJR8o= conduwuit.cachix.org-1:MFRm6jcnfTf0jSAbmvLfhO3KBMt4px+1xaereWXp8Xg=
+    extra-substituters = https://attic.kennel.juneis.dog/conduit https://attic.kennel.juneis.dog/conduwuit https://conduwuit.cachix.org
+    extra-trusted-public-keys = conduit:eEKoUwlQGDdYmAI/Q/0slVlegqh/QmAvQd7HBSm21Wk= conduwuit:BbycGUgTISsltcmH0qNjFR9dbrQNYgdIAcmViSGoVTE= conduwuit.cachix.org-1:MFRm6jcnfTf0jSAbmvLfhO3KBMt4px+1xaereWXp8Xg=
     experimental-features = nix-command flakes
     extra-experimental-features = nix-command flakes
     accept-flake-config = true
@@ -45,10 +45,6 @@ before_script:
   - if command -v nix > /dev/null && [ -n "$ATTIC_ENDPOINT" ]; then echo "extra-substituters = $ATTIC_ENDPOINT" >> /etc/nix/nix.conf; fi
   - if command -v nix > /dev/null && [ -n "$ATTIC_PUBLIC_KEY" ]; then echo "extra-trusted-public-keys = $ATTIC_PUBLIC_KEY" >> /etc/nix/nix.conf; fi
 
-  # Add Lix binary cache
-  - if command -v nix > /dev/null; then echo "extra-substituters = https://cache.lix.systems" >> /etc/nix/nix.conf; fi
-  - if command -v nix > /dev/null; then echo "extra-trusted-public-keys = cache.lix.systems:aBnZUw8zA7H35Cz2RyKFVs3H4PlGTLawyY5KRbvJR8o=" >> /etc/nix/nix.conf; fi
-
   # Add crane binary cache
   - if command -v nix > /dev/null; then echo "extra-substituters = https://crane.cachix.org" >> /etc/nix/nix.conf; fi
   - if command -v nix > /dev/null; then echo "extra-trusted-public-keys = crane.cachix.org-1:8Scfpmn9w+hGdXH/Q9tTLiYAE/2dnJYRJP7kl80GuRk=" >> /etc/nix/nix.conf; fi

Cargo.toml

@@ -7,19 +7,20 @@ default-members = ["src/*"]
 
 [workspace.package]
 authors = [
-    "strawberry <strawberry@puppygock.gay>",
-    "timokoesters <timo@koesters.xyz>",
+    "June Clementine Strawberry <june@girlboss.ceo>",
+    "strawberry <strawberry@puppygock.gay>", # woof
+    "Jason Volk <jason@zemos.net>",
 ]
 categories = ["network-programming"]
-description = "a very cool fork of Conduit, a Matrix homeserver written in Rust"
+description = "a very cool Matrix chat homeserver written in Rust"
 edition = "2021"
 homepage = "https://conduwuit.puppyirl.gay/"
-keywords = ["chat", "matrix", "server", "uwu"]
+keywords = ["chat", "matrix", "networking", "server", "uwu"]
 license = "Apache-2.0"
 # See also `rust-toolchain.toml`
 readme = "README.md"
 repository = "https://github.com/girlbossceo/conduwuit"
-rust-version = "1.83.0"
+rust-version = "1.84.0"
 version = "0.5.0"
 
 [workspace.metadata.crane]
@@ -58,10 +59,6 @@ features = ["parse"]
 [workspace.dependencies.sanitize-filename]
 version = "0.6.0"
 
-[workspace.dependencies.jsonwebtoken]
-version = "9.3.0"
-default-features = false
-
 [workspace.dependencies.base64]
 version = "0.22.1"
 default-features = false
@@ -336,7 +333,7 @@ version = "0.1.2"
 [workspace.dependencies.ruma]
 git = "https://github.com/girlbossceo/ruwuma"
 #branch = "conduwuit-changes"
-rev = "c4f55b39900b33b2d443dd12a6a2dab50961fdfb"
+rev = "b560338b2a50dbf61ecfe80808b9b095ad4cec00"
 features = [
     "compat",
     "rand",
@@ -435,17 +432,23 @@ version = "0.35.0"
 # jemalloc usage
 [workspace.dependencies.tikv-jemalloc-sys]
 git = "https://github.com/girlbossceo/jemallocator"
-rev = "d87938bfddc26377dd7fdf14bbcd345f3ab19442"
+rev = "82af58d6a13ddd5dcdc7d4e91eae3b63292995b8"
 default-features = false
-features = ["unprefixed_malloc_on_supported_platforms"]
+features = [
+	"background_threads_runtime_support",
+	"unprefixed_malloc_on_supported_platforms",
+]
 [workspace.dependencies.tikv-jemallocator]
 git = "https://github.com/girlbossceo/jemallocator"
-rev = "d87938bfddc26377dd7fdf14bbcd345f3ab19442"
+rev = "82af58d6a13ddd5dcdc7d4e91eae3b63292995b8"
 default-features = false
-features = ["unprefixed_malloc_on_supported_platforms"]
+features = [
+	"background_threads_runtime_support",
+	"unprefixed_malloc_on_supported_platforms",
+]
 [workspace.dependencies.tikv-jemalloc-ctl]
 git = "https://github.com/girlbossceo/jemallocator"
-rev = "d87938bfddc26377dd7fdf14bbcd345f3ab19442"
+rev = "82af58d6a13ddd5dcdc7d4e91eae3b63292995b8"
 default-features = false
 features = ["use_std"]
 
@@ -504,6 +507,14 @@ version = "0.2"
 [workspace.dependencies.num-traits]
 version = "0.2"
 
+[workspace.dependencies.minicbor]
+version = "0.25.1"
+features = ["std"]
+
+[workspace.dependencies.minicbor-serde]
+version = "0.3.2"
+features = ["std"]
+
 #
 # Patches
 #
@@ -513,16 +524,16 @@ version = "0.2"
 # https://github.com/girlbossceo/tracing/commit/b348dca742af641c47bc390261f60711c2af573c
 [patch.crates-io.tracing-subscriber]
 git = "https://github.com/girlbossceo/tracing"
-rev = "ccc4fbd8238c2d5ba354e61ec17ac610af11401d"
+rev = "05825066a6d0e9ad6b80dcf29457eb179ff4768c"
 [patch.crates-io.tracing]
 git = "https://github.com/girlbossceo/tracing"
-rev = "ccc4fbd8238c2d5ba354e61ec17ac610af11401d"
+rev = "05825066a6d0e9ad6b80dcf29457eb179ff4768c"
 [patch.crates-io.tracing-core]
 git = "https://github.com/girlbossceo/tracing"
-rev = "ccc4fbd8238c2d5ba354e61ec17ac610af11401d"
+rev = "05825066a6d0e9ad6b80dcf29457eb179ff4768c"
 [patch.crates-io.tracing-log]
 git = "https://github.com/girlbossceo/tracing"
-rev = "ccc4fbd8238c2d5ba354e61ec17ac610af11401d"
+rev = "05825066a6d0e9ad6b80dcf29457eb179ff4768c"
 
 # adds a tab completion callback: https://github.com/girlbossceo/rustyline-async/commit/de26100b0db03e419a3d8e1dd26895d170d1fe50
 # adds event for CTRL+\: https://github.com/girlbossceo/rustyline-async/commit/67d8c49aeac03a5ef4e818f663eaa94dd7bf339b
@@ -873,6 +884,7 @@ enum_glob_use = { level = "allow", priority = 1 }
 if_not_else = { level = "allow", priority = 1 }
 if_then_some_else_none = { level = "allow", priority = 1 }
 inline_always = { level = "allow", priority = 1 }
+match_bool = { level = "allow", priority = 1 }
 missing_docs_in_private_items = { level = "allow", priority = 1 }
 missing_errors_doc = { level = "allow", priority = 1 }
 missing_panics_doc = { level = "allow", priority = 1 }

README.md

@@ -4,7 +4,7 @@
 
 <!-- ANCHOR: catchphrase -->
 
-### a very cool, featureful fork of [Conduit](https://conduit.rs/)
+### a very cool [Matrix](https://matrix.org/) chat homeserver written in Rust
 
 <!-- ANCHOR_END: catchphrase -->
 
@@ -15,16 +15,15 @@ information and how to deploy/setup conduwuit.
 
 #### What is Matrix?
 
-[Matrix](https://matrix.org) is an open network for secure and decentralized
-communication. Users from every Matrix homeserver can chat with users from all
-other Matrix servers. You can even use bridges (also called Matrix Appservices)
-to communicate with users outside of Matrix, like a community on Discord.
+[Matrix](https://matrix.org) is an open, federated, and extensible network for
+decentralised communication. Users from any Matrix homeserver can chat with users from all
+other homeservers over federation. Matrix is designed to be extensible and built on top of.
+You can even use bridges such as Matrix Appservices to communicate with users outside of Matrix, like a community on Discord.
 
 #### What is the goal?
 
-A high-performance and efficient Matrix homeserver that's easy to set up and
-just works. You can install it on a mini-computer like the Raspberry Pi to
-host Matrix for your family, friends or company.
+A high-performance, efficient, low-cost, and featureful Matrix homeserver that's
+easy to set up and just works with minimal configuration needed.
 
 #### Can I try it out?
 
@@ -37,17 +36,22 @@ homeserver". This means there are rules, so please read the rules:
 [https://transfem.dev/homeserver_rules.txt](https://transfem.dev/homeserver_rules.txt)
 
 transfem.dev is also listed at
-[servers.joinmatrix.org](https://servers.joinmatrix.org/)
+[servers.joinmatrix.org](https://servers.joinmatrix.org/), which is a list of
+popular public Matrix homeservers, including some others that run conduwuit.
 
 #### What is the current status?
 
-conduwuit is technically a hard fork of Conduit, which is in Beta. The Beta status
-initially was inherited from Conduit, however overtime this Beta status is rapidly
-becoming less and less relevant as our codebase significantly diverges more and more.
+conduwuit is technically a hard fork of [Conduit](https://conduit.rs/), which is in beta.
+The beta status initially was inherited from Conduit, however the huge amount of
+codebase divergance, changes, fixes, and improvements have effectively made this
+beta status not entirely applicable to us anymore.
 
-conduwuit is quite stable and very usable as a daily driver and for a low-medium
-sized homeserver. There is still a lot of more work to be done, but it is in a far
-better place than the project was in early 2024.
+conduwuit is very stable based on our rapidly growing userbase, has lots of features that users
+expect, and very usable as a daily driver for small, medium, and upper-end medium sized homeservers.
+
+A lot of critical stability and performance issues have been fixed, and a lot of
+necessary groundwork has finished; making this project way better than it was
+back in the start at ~early 2024.
 
 #### How is conduwuit funded? Is conduwuit sustainable?
 
@@ -72,16 +76,37 @@ Conduit like before. If you are truly finding yourself wanting to migrate back
 to Conduit, we would appreciate all your feedback and if we can assist with
 any issues or concerns.
 
+#### Can I migrate from Synapse or Dendrite?
+
+Currently there is no known way to seamlessly migrate all user data from the old
+homeserver to conduwuit. However it is perfectly acceptable to replace the old
+homeserver software with conduwuit using the same server name and there will not
+be any issues with federation.
+
+There is an interest in developing a built-in seamless user data migration
+method into conduwuit, however there is no concrete ETA or timeline for this.
+
+
 <!-- ANCHOR_END: body -->
 
 <!-- ANCHOR: footer -->
 
 #### Contact
 
-If you run into any question, feel free to
+[`#conduwuit:puppygock.gay`](https://matrix.to/#/#conduwuit:puppygock.gay)
+is the official project Matrix room. You can get support here, ask questions or
+concerns, get assistance setting up conduwuit, etc.
 
-- Ask us in `#conduwuit:puppygock.gay` on Matrix
-- [Open an issue on GitHub](https://github.com/girlbossceo/conduwuit/issues/new)
+This room should stay relevant and focused on conduwuit. An offtopic general
+chatter room can be found there as well.
+
+Please keep the issue trackers focused on bug reports and enhancement requests.
+General support is extremely difficult to be offered over an issue tracker, and
+simple questions should be asked directly in an interactive platform like our
+Matrix room above as they can turn into a relevant discussion and/or may not be
+simple to answer. If you're not sure, just ask in the Matrix room.
+
+If you have a bug or feature to request: [Open an issue on GitHub](https://github.com/girlbossceo/conduwuit/issues/new)
 
 #### Donate
 
@@ -89,9 +114,11 @@ conduwuit development is purely made possible by myself and contributors. I do
 not get paid to work on this, and I work on it in my free time. Donations are
 heavily appreciated! 💜🥺
 
-- Liberapay: <https://liberapay.com/girlbossceo>
-- Ko-fi (note they take a fee): <https://ko-fi.com/puppygock>
-- GitHub Sponsors: <https://github.com/sponsors/girlbossceo>
+- Liberapay (preferred): <https://liberapay.com/girlbossceo>
+- GitHub Sponsors (preferred): <https://github.com/sponsors/girlbossceo>
+- Ko-fi: <https://ko-fi.com/puppygock>
+
+I do not and will not accept cryptocurrency donations, including things related.
 
 #### Logo
 
@@ -105,11 +132,15 @@ Both, but I prefer conduwuit.
 
 #### Mirrors of conduwuit
 
+If GitHub is unavailable in your country, or has poor connectivity, conduwuit's
+source code is mirrored onto the following additional platforms I maintain:
+
 - GitHub: <https://github.com/girlbossceo/conduwuit>
 - GitLab: <https://gitlab.com/conduwuit/conduwuit>
 - git.girlcock.ceo: <https://git.girlcock.ceo/strawberry/conduwuit>
 - git.gay: <https://git.gay/june/conduwuit>
-- Codeberg: <https://codeberg.org/girlbossceo/conduwuit>
+- mau.dev: <https://mau.dev/june/conduwuit>
+- Codeberg: <https://codeberg.org/arf/conduwuit>
 - sourcehut: <https://git.sr.ht/~girlbossceo/conduwuit>
 
 <!-- ANCHOR_END: footer -->

bin/complement

@@ -34,7 +34,9 @@ toplevel="$(git rev-parse --show-toplevel)"
 
 pushd "$toplevel" > /dev/null
 
-bin/nix-build-and-cache just .#linux-complement
+#bin/nix-build-and-cache just .#linux-complement
+bin/nix-build-and-cache just .#complement
+#nom build .#complement
 
 docker load < result
 popd > /dev/null

conduwuit-example.toml

@@ -100,20 +100,6 @@
 #
 #database_backups_to_keep = 1
 
-# Set this to any float value in megabytes for conduwuit to tell the
-# database engine that this much memory is available for database-related
-# caches.
-#
-# May be useful if you have significant memory to spare to increase
-# performance.
-#
-# Similar to the individual LRU caches, this is scaled up with your CPU
-# core count.
-#
-# This defaults to 128.0 + (64.0 * CPU core count).
-#
-#db_cache_capacity_mb = varies by system
-
 # Text which will be added to the end of the user's displayname upon
 # registration with a space before the text. In Conduit, this was the
 # lightning bolt emoji.
@@ -149,6 +135,34 @@
 #
 #cache_capacity_modifier = 1.0
 
+# Set this to any float value in megabytes for conduwuit to tell the
+# database engine that this much memory is available for database read
+# caches.
+#
+# May be useful if you have significant memory to spare to increase
+# performance.
+#
+# Similar to the individual LRU caches, this is scaled up with your CPU
+# core count.
+#
+# This defaults to 128.0 + (64.0 * CPU core count).
+#
+#db_cache_capacity_mb = varies by system
+
+# Set this to any float value in megabytes for conduwuit to tell the
+# database engine that this much memory is available for database write
+# caches.
+#
+# May be useful if you have significant memory to spare to increase
+# performance.
+#
+# Similar to the individual LRU caches, this is scaled up with your CPU
+# core count.
+#
+# This defaults to 48.0 + (4.0 * CPU core count).
+#
+#db_write_buffer_capacity_mb = varies by system
+
 # This item is undocumented. Please contribute documentation for it.
 #
 #pdu_cache_capacity = varies by system
@@ -375,13 +389,16 @@
 #
 #allow_registration = false
 
-# This item is undocumented. Please contribute documentation for it.
+# Enabling this setting opens registration to anyone without restrictions.
+# This makes your server vulnerable to abuse
 #
 #yes_i_am_very_very_sure_i_want_an_open_registration_server_prone_to_abuse = false
 
 # A static registration token that new users will have to provide when
 # creating an account. If unset and `allow_registration` is true,
-# registration is open without any condition.
+# you must set
+# `yes_i_am_very_very_sure_i_want_an_open_registration_server_prone_to_abuse`
+# to true to allow open registration without any conditions.
 #
 # YOU NEED TO EDIT THIS OR USE registration_token_file.
 #
@@ -549,10 +566,6 @@
 #
 #proxy = "none"
 
-# This item is undocumented. Please contribute documentation for it.
-#
-#jwt_secret =
-
 # Servers listed here will be used to gather public keys of other servers
 # (notary trusted key servers).
 #
@@ -635,6 +648,22 @@
 #
 #openid_token_ttl = 3600
 
+# Allow an existing session to mint a login token for another client.
+# This requires interactive authentication, but has security ramifications
+# as a malicious client could use the mechanism to spawn more than one
+# session.
+# Enabled by default.
+#
+#login_via_existing_session = true
+
+# Login token expiration/TTL in milliseconds.
+#
+# These are short-lived tokens for the m.login.token endpoint.
+# This is used to allow existing sessions to create new sessions.
+# see login_via_existing_session.
+#
+#login_token_ttl = 120000
+
 # Static TURN username to provide the client if not using a shared secret
 # ("turn_secret"), It is recommended to use a shared secret over static
 # credentials.
@@ -792,6 +821,9 @@
 # magic number and translated to the library's default compression level
 # as they all differ. See their `kDefaultCompressionLevel`.
 #
+# Note when using the default value we may override it with a setting
+# tailored specifically conduwuit.
+#
 #rocksdb_compression_level = 32767
 
 # Level of compression the specified compression algorithm for the
@@ -805,6 +837,9 @@
 # less likely for this data to be used. Research your chosen compression
 # algorithm.
 #
+# Note when using the default value we may override it with a setting
+# tailored specifically conduwuit.
+#
 #rocksdb_bottommost_compression_level = 32767
 
 # Whether to enable RocksDB's "bottommost_compression".
@@ -816,7 +851,7 @@
 #
 # See https://github.com/facebook/rocksdb/wiki/Compression for more details.
 #
-#rocksdb_bottommost_compression = false
+#rocksdb_bottommost_compression = true
 
 # Database recovery mode (for RocksDB WAL corruption).
 #
@@ -1452,7 +1487,7 @@
 # responsiveness for many users at the cost of throughput for each.
 #
 # Setting this value to 0.0 causes the stream width to be fixed at the
-# value of stream_width_default. The default is 1.0 to match the
+# value of stream_width_default. The default scale is 1.0 to match the
 # capabilities detected for the system.
 #
 #stream_width_scale = 1.0
@@ -1477,6 +1512,11 @@
 #
 #sender_workers = 0
 
+# Enables listener sockets; can be set to false to disable listening. This
+# option is intended for developer/diagnostic purposes only.
+#
+#listening = true
+
 [global.tls]
 
 # Path to a valid TLS certificate file.

debian/postrm

@@ -10,21 +10,33 @@ CONDUWUIT_DATABASE_PATH_SYMLINK=/var/lib/matrix-conduit
 case $1 in
   purge)
     # Remove debconf changes from the db
-    db_purge
+    #db_purge
 
     # Per https://www.debian.org/doc/debian-policy/ch-files.html#behavior
     # "configuration files must be preserved when the package is removed, and
     #  only deleted when the package is purged."
+
+    #
+
     if [ -d "$CONDUWUIT_CONFIG_PATH" ]; then
-      rm -v -r "$CONDUWUIT_CONFIG_PATH"
+      if test -L "$CONDUWUIT_CONFIG_PATH"; then
+        echo "Deleting conduwuit configuration files"
+        rm -v -r "$CONDUWUIT_CONFIG_PATH"
+      fi
     fi
 
     if [ -d "$CONDUWUIT_DATABASE_PATH" ]; then
-      rm -v -r "$CONDUWUIT_DATABASE_PATH"
+      if test -L "$CONDUWUIT_DATABASE_PATH"; then
+        echo "Deleting conduwuit database directory"
+        rm -r "$CONDUWUIT_DATABASE_PATH"
+      fi
     fi
 
     if [ -d "$CONDUWUIT_DATABASE_PATH_SYMLINK" ]; then
-      rm -v -r "$CONDUWUIT_DATABASE_PATH_SYMLINK"
+      if test -L "$CONDUWUIT_DATABASE_SYMLINK"; then
+        echo "Removing matrix-conduit symlink"
+        rm -r "$CONDUWUIT_DATABASE_PATH_SYMLINK"
+      fi
     fi
     ;;
 esac

deps/rust-rocksdb/Cargo.toml

@@ -27,7 +27,7 @@ malloc-usable-size = ["rust-rocksdb/malloc-usable-size"]
 
 [dependencies.rust-rocksdb]
 git = "https://github.com/girlbossceo/rust-rocksdb-zaidoon1"
-rev = "123d6302fed23fc706344becb2f19623265a83f8"
+rev = "1f032427d3a0e7b0f13c04b4e34712bd8610291b"
 #branch = "master"
 default-features = false
 

docs/deploying/docker-compose.for-traefik.yml

@@ -17,6 +17,8 @@ services:
       CONDUWUIT_PORT: 6167 # should match the loadbalancer traefik label
       CONDUWUIT_MAX_REQUEST_SIZE: 20000000 # in bytes, ~20 MB
       CONDUWUIT_ALLOW_REGISTRATION: 'true'
+      CONDUWUIT_REGISTRATION_TOKEN: 'YOUR_TOKEN' # A registration token is required when registration is allowed.
+      #CONDUWUIT_YES_I_AM_VERY_VERY_SURE_I_WANT_AN_OPEN_REGISTRATION_SERVER_PRONE_TO_ABUSE: 'true'
       CONDUWUIT_ALLOW_FEDERATION: 'true'
       CONDUWUIT_ALLOW_CHECK_FOR_UPDATES: 'true'
       CONDUWUIT_TRUSTED_SERVERS: '["matrix.org"]'

docs/deploying/docker-compose.with-caddy.yml

@@ -33,6 +33,8 @@ services:
             CONDUWUIT_PORT: 6167
             CONDUWUIT_MAX_REQUEST_SIZE: 20000000 # in bytes, ~20 MB
             CONDUWUIT_ALLOW_REGISTRATION: 'true'
+            CONDUWUIT_REGISTRATION_TOKEN: 'YOUR_TOKEN' # A registration token is required when registration is allowed.
+            #CONDUWUIT_YES_I_AM_VERY_VERY_SURE_I_WANT_AN_OPEN_REGISTRATION_SERVER_PRONE_TO_ABUSE: 'true'
             CONDUWUIT_ALLOW_FEDERATION: 'true'
             CONDUWUIT_ALLOW_CHECK_FOR_UPDATES: 'true'
             CONDUWUIT_TRUSTED_SERVERS: '["matrix.org"]'

docs/deploying/docker-compose.yml

@@ -17,6 +17,8 @@ services:
             CONDUWUIT_PORT: 6167
             CONDUWUIT_MAX_REQUEST_SIZE: 20000000 # in bytes, ~20 MB
             CONDUWUIT_ALLOW_REGISTRATION: 'true'
+            CONDUWUIT_REGISTRATION_TOKEN: 'YOUR_TOKEN' # A registration token is required when registration is allowed.
+            #CONDUWUIT_YES_I_AM_VERY_VERY_SURE_I_WANT_AN_OPEN_REGISTRATION_SERVER_PRONE_TO_ABUSE: 'true'
             CONDUWUIT_ALLOW_FEDERATION: 'true'
             CONDUWUIT_ALLOW_CHECK_FOR_UPDATES: 'true'
             CONDUWUIT_TRUSTED_SERVERS: '["matrix.org"]'

engage.toml

@@ -97,6 +97,7 @@ env DIRENV_DEVSHELL=all-features \
 name = "clippy/default"
 group = "lints"
 script = """
+direnv exec . \
 cargo clippy \
     --workspace \
     --profile test \
@@ -126,6 +127,7 @@ env DIRENV_DEVSHELL=all-features \
 name = "clippy/jemalloc"
 group = "lints"
 script = """
+direnv exec . \
 cargo clippy \
     --workspace \
     --profile test \
@@ -179,13 +181,15 @@ env DIRENV_DEVSHELL=all-features \
 name = "cargo/default"
 group = "tests"
 script = """
-cargo test \
-    --workspace \
-    --profile test \
-    --all-targets \
-    --color=always \
-    -- \
-    --color=always
+env DIRENV_DEVSHELL=default \
+    direnv exec . \
+    cargo test \
+        --workspace \
+        --profile test \
+        --all-targets \
+        --color=always \
+        -- \
+        --color=always
 """
 
 # Checks if the generated example config differs from the checked in repo's

flake.lock

@@ -32,11 +32,11 @@
         "nixpkgs": "nixpkgs_4"
       },
       "locked": {
-        "lastModified": 1733424942,
-        "narHash": "sha256-5t7Sl6EkOaoP4FvzLmH7HFDbdl9SizmLh53RjDQCbWQ=",
+        "lastModified": 1737621947,
+        "narHash": "sha256-8HFvG7fvIFbgtaYAY2628Tb89fA55nPm2jSiNs0/Cws=",
         "owner": "cachix",
         "repo": "cachix",
-        "rev": "8b6b0e4694b9aa78b2ea4c93bff6e1a222dc7e4a",
+        "rev": "f65a3cd5e339c223471e64c051434616e18cc4f5",
         "type": "github"
       },
       "original": {
@@ -117,11 +117,11 @@
     },
     "crane_2": {
       "locked": {
-        "lastModified": 1734808813,
-        "narHash": "sha256-3aH/0Y6ajIlfy7j52FGZ+s4icVX0oHhqBzRdlOeztqg=",
+        "lastModified": 1737689766,
+        "narHash": "sha256-ivVXYaYlShxYoKfSo5+y5930qMKKJ8CLcAoIBPQfJ6s=",
         "owner": "ipetkov",
         "repo": "crane",
-        "rev": "72e2d02dbac80c8c86bf6bf3e785536acf8ee926",
+        "rev": "6fe74265bbb6d016d663b1091f015e2976c4a527",
         "type": "github"
       },
       "original": {
@@ -170,11 +170,11 @@
         "rust-analyzer-src": "rust-analyzer-src"
       },
       "locked": {
-        "lastModified": 1735799625,
-        "narHash": "sha256-lFadwWDvVIub11bwfZhsh2WUByf9LOi6yjsSUMmE0xk=",
+        "lastModified": 1737786656,
+        "narHash": "sha256-ubCW9Jy7ZUOF354bWxTgLDpVnTvIpNr6qR4H/j7I0oo=",
         "owner": "nix-community",
         "repo": "fenix",
-        "rev": "a9d84a1545814910cb4ab0515ed6921e8b07ee95",
+        "rev": "2f721f527886f801403f389a9cabafda8f1e3b7f",
         "type": "github"
       },
       "original": {
@@ -364,11 +364,11 @@
     "liburing": {
       "flake": false,
       "locked": {
-        "lastModified": 1733603756,
-        "narHash": "sha256-eTKnZDZ1Ex++v+BI0DBcUBmCXAO/tE8hxK9MiyztZkU=",
+        "lastModified": 1737600516,
+        "narHash": "sha256-EKyLQ3pbcjoU5jH5atge59F4fzuhTsb6yalUj6Ve2t8=",
         "owner": "axboe",
         "repo": "liburing",
-        "rev": "c3d5d6270cd5ed48d817fc1e8e95f7c8b222f2ff",
+        "rev": "6c509e2b0c881a13b83b259a221bf15fc9b3f681",
         "type": "github"
       },
       "original": {
@@ -550,11 +550,11 @@
     },
     "nixpkgs_5": {
       "locked": {
-        "lastModified": 1735685343,
-        "narHash": "sha256-h1CpBzdJDNtSUb5QMyfFHKHocTTky+4McgQEBQBM+xA=",
+        "lastModified": 1737717945,
+        "narHash": "sha256-ET91TMkab3PmOZnqiJQYOtSGvSTvGeHoegAv4zcTefM=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "81934660d6e9ea54d2f0cdee821e8533b10c221a",
+        "rev": "ecd26a469ac56357fd333946a99086e992452b6a",
         "type": "github"
       },
       "original": {
@@ -567,11 +567,11 @@
     "rocksdb": {
       "flake": false,
       "locked": {
-        "lastModified": 1734469478,
-        "narHash": "sha256-IcQ4N8xADYal79K+ONmNq4RLlIwdgUqgrVzgNgiIaG8=",
+        "lastModified": 1737828695,
+        "narHash": "sha256-8Ev6zzhNPU798JNvU27a7gj5X+6SDG3jBweUkQ59DbA=",
         "owner": "girlbossceo",
         "repo": "rocksdb",
-        "rev": "8b4808e7de2fbb5d119d8d72cdca76d8ab84bc47",
+        "rev": "a4d9230dcc9d03be428b9a728133f8f646c0065c",
         "type": "github"
       },
       "original": {
@@ -599,11 +599,11 @@
     "rust-analyzer-src": {
       "flake": false,
       "locked": {
-        "lastModified": 1735742096,
-        "narHash": "sha256-q3a80h8Jf8wfmPURUgRR46nQCB3I5fhZ+/swulTF5HY=",
+        "lastModified": 1737728869,
+        "narHash": "sha256-U4pl3Hi0lT6GP4ecN3q9wdD2sdaKMbmD/5NJ1NdJ9AM=",
         "owner": "rust-lang",
         "repo": "rust-analyzer",
-        "rev": "7e639ee3dda6ed9cecc79d41f6d38235121e483d",
+        "rev": "6e4c29f7ce18cea7d3d31237a4661ab932eab636",
         "type": "github"
       },
       "original": {

flake.nix

@@ -26,7 +26,7 @@
         file = ./rust-toolchain.toml;
 
         # See also `rust-toolchain.toml`
-        sha256 = "sha256-s1RPtyvDGJaX/BisLT+ifVfuhDT1nZkZ1NcK8sbwELM=";
+        sha256 = "sha256-lMLAupxng4Fd9F1oDw8gx+qA0RuF7ou7xhNU8wgs0PU=";
       };
 
       mkScope = pkgs: pkgs.lib.makeScope pkgs.newScope (self: {
@@ -169,10 +169,10 @@
 
           # used for rust caching in CI to speed it up
           sccache
-
-          # needed so we can get rid of gcc and other unused deps that bloat OCI images
-          removeReferencesTo
         ]
+        # valgrind is unavailable in static contexts
+        # used for CI and complement
+        ++ (if !stdenv.hostPlatform.isStatic then [ "valgrind" ] else [])
         # liburing is Linux-exclusive
         ++ lib.optional stdenv.hostPlatform.isLinux liburing
         # needed to build Rust applications on macOS
@@ -191,27 +191,59 @@
     in
     {
       packages = {
-        default = scopeHost.main;
-        default-debug = scopeHost.main.override {
-            profile = "dev";
-            # debug build users expect full logs
-            disable_release_max_log_level = true;
-        };
-        default-test = scopeHost.main.override {
-            profile = "test";
-            disable_release_max_log_level = true;
-        };
-        all-features = scopeHost.main.override {
-            all_features = true;
+        default = scopeHost.main.override {
             disable_features = [
-                # this is non-functional on nix for some reason
-                "hardened_malloc"
                 # dont include experimental features
                 "experimental"
                 # jemalloc profiling/stats features are expensive and shouldn't
                 # be expected on non-debug builds.
                 "jemalloc_prof"
                 "jemalloc_stats"
+                # this is non-functional on nix for some reason
+                "hardened_malloc"
+                # conduwuit_mods is a development-only hot reload feature
+                "conduwuit_mods"
+            ];
+        };
+        default-debug = scopeHost.main.override {
+            profile = "dev";
+            # debug build users expect full logs
+            disable_release_max_log_level = true;
+            disable_features = [
+                # dont include experimental features
+                "experimental"
+                # this is non-functional on nix for some reason
+                "hardened_malloc"
+                # conduwuit_mods is a development-only hot reload feature
+                "conduwuit_mods"
+            ];
+        };
+        # just a test profile used for things like CI and complement
+        default-test = scopeHost.main.override {
+            profile = "test";
+            disable_release_max_log_level = true;
+            disable_features = [
+                # dont include experimental features
+                "experimental"
+                # this is non-functional on nix for some reason
+                "hardened_malloc"
+                # conduwuit_mods is a development-only hot reload feature
+                "conduwuit_mods"
+            ];
+        };
+        all-features = scopeHost.main.override {
+            all_features = true;
+            disable_features = [
+                # dont include experimental features
+                "experimental"
+                # jemalloc profiling/stats features are expensive and shouldn't
+                # be expected on non-debug builds.
+                "jemalloc_prof"
+                "jemalloc_stats"
+                # this is non-functional on nix for some reason
+                "hardened_malloc"
+                # conduwuit_mods is a development-only hot reload feature
+                "conduwuit_mods"
             ];
         };
         all-features-debug = scopeHost.main.override {
@@ -220,10 +252,12 @@
             # debug build users expect full logs
             disable_release_max_log_level = true;
             disable_features = [
-                # this is non-functional on nix for some reason
-                "hardened_malloc"
                 # dont include experimental features
                 "experimental"
+                # this is non-functional on nix for some reason
+                "hardened_malloc"
+                # conduwuit_mods is a development-only hot reload feature
+                "conduwuit_mods"
             ];
         };
         hmalloc = scopeHost.main.override { features = ["hardened_malloc"]; };
@@ -233,14 +267,16 @@
           main = scopeHost.main.override {
             all_features = true;
             disable_features = [
-                # this is non-functional on nix for some reason
-                "hardened_malloc"
                 # dont include experimental features
                 "experimental"
                 # jemalloc profiling/stats features are expensive and shouldn't
                 # be expected on non-debug builds.
                 "jemalloc_prof"
                 "jemalloc_stats"
+                # this is non-functional on nix for some reason
+                "hardened_malloc"
+                # conduwuit_mods is a development-only hot reload feature
+                "conduwuit_mods"
             ];
           };
         };
@@ -251,10 +287,12 @@
             # debug build users expect full logs
             disable_release_max_log_level = true;
             disable_features = [
-                # this is non-functional on nix for some reason
-                "hardened_malloc"
                 # dont include experimental features
                 "experimental"
+                # this is non-functional on nix for some reason
+                "hardened_malloc"
+                # conduwuit_mods is a development-only hot reload feature
+                "conduwuit_mods"
             ];
           };
         };
@@ -313,6 +351,14 @@
                   value = scopeCrossStatic.main.override {
                     profile = "test";
                     disable_release_max_log_level = true;
+                    disable_features = [
+                        # dont include experimental features
+                        "experimental"
+                        # this is non-functional on nix for some reason
+                        "hardened_malloc"
+                        # conduwuit_mods is a development-only hot reload feature
+                        "conduwuit_mods"
+                    ];
                   };
                 }
 
@@ -322,14 +368,16 @@
                   value = scopeCrossStatic.main.override {
                     all_features = true;
                     disable_features = [
-                        # this is non-functional on nix for some reason
-                        "hardened_malloc"
                         # dont include experimental features
                         "experimental"
                         # jemalloc profiling/stats features are expensive and shouldn't
                         # be expected on non-debug builds.
                         "jemalloc_prof"
                         "jemalloc_stats"
+                        # this is non-functional on nix for some reason
+                        "hardened_malloc"
+                        # conduwuit_mods is a development-only hot reload feature
+                        "conduwuit_mods"
                     ];
                   };
                 }
@@ -341,14 +389,16 @@
                   value = scopeCrossStatic.main.override {
                     all_features = true;
                     disable_features = [
-                        # this is non-functional on nix for some reason
-                        "hardened_malloc"
                         # dont include experimental features
                         "experimental"
                         # jemalloc profiling/stats features are expensive and shouldn't
                         # be expected on non-debug builds.
                         "jemalloc_prof"
                         "jemalloc_stats"
+                        # this is non-functional on nix for some reason
+                        "hardened_malloc"
+                        # conduwuit_mods is a development-only hot reload feature
+                        "conduwuit_mods"
                     ];
                     x86_64_haswell_target_optimised = (if (crossSystem == "x86_64-linux-gnu" || crossSystem == "x86_64-linux-musl") then true else false);
                   };
@@ -363,10 +413,12 @@
                     # debug build users expect full logs
                     disable_release_max_log_level = true;
                     disable_features = [
-                        # this is non-functional on nix for some reason
-                        "hardened_malloc"
                         # dont include experimental features
                         "experimental"
+                        # this is non-functional on nix for some reason
+                        "hardened_malloc"
+                        # conduwuit_mods is a development-only hot reload feature
+                        "conduwuit_mods"
                     ];
                   };
                 }
@@ -415,14 +467,16 @@
                     main = scopeCrossStatic.main.override {
                       all_features = true;
                       disable_features = [
-                          # this is non-functional on nix for some reason
-                          "hardened_malloc"
-                          # dont include experimental features
-                          "experimental"
-                          # jemalloc profiling/stats features are expensive and shouldn't
-                          # be expected on non-debug builds.
-                          "jemalloc_prof"
-                          "jemalloc_stats"
+                        # dont include experimental features
+                        "experimental"
+                        # jemalloc profiling/stats features are expensive and shouldn't
+                        # be expected on non-debug builds.
+                        "jemalloc_prof"
+                        "jemalloc_stats"
+                        # this is non-functional on nix for some reason
+                        "hardened_malloc"
+                        # conduwuit_mods is a development-only hot reload feature
+                        "conduwuit_mods"
                       ];
                     };
                   };
@@ -436,14 +490,16 @@
                     main = scopeCrossStatic.main.override {
                       all_features = true;
                       disable_features = [
-                          # this is non-functional on nix for some reason
-                          "hardened_malloc"
-                          # dont include experimental features
-                          "experimental"
-                          # jemalloc profiling/stats features are expensive and shouldn't
-                          # be expected on non-debug builds.
-                          "jemalloc_prof"
-                          "jemalloc_stats"
+                        # dont include experimental features
+                        "experimental"
+                        # jemalloc profiling/stats features are expensive and shouldn't
+                        # be expected on non-debug builds.
+                        "jemalloc_prof"
+                        "jemalloc_stats"
+                        # this is non-functional on nix for some reason
+                        "hardened_malloc"
+                        # conduwuit_mods is a development-only hot reload feature
+                        "conduwuit_mods"
                       ];
                       x86_64_haswell_target_optimised = (if (crossSystem == "x86_64-linux-gnu" || crossSystem == "x86_64-linux-musl") then true else false);
                     };
@@ -460,10 +516,12 @@
                       # debug build users expect full logs
                       disable_release_max_log_level = true;
                       disable_features = [
-                          # this is non-functional on nix for some reason
-                          "hardened_malloc"
-                          # dont include experimental features
-                          "experimental"
+                        # dont include experimental features
+                        "experimental"
+                        # this is non-functional on nix for some reason
+                        "hardened_malloc"
+                        # conduwuit_mods is a development-only hot reload feature
+                        "conduwuit_mods"
                       ];
                     };
                   };
@@ -502,14 +560,16 @@
           main = prev.main.override {
             all_features = true;
             disable_features = [
-                # this is non-functional on nix for some reason
-                "hardened_malloc"
                 # dont include experimental features
                 "experimental"
                 # jemalloc profiling/stats features are expensive and shouldn't
                 # be expected on non-debug builds.
                 "jemalloc_prof"
                 "jemalloc_stats"
+                # this is non-functional on nix for some reason
+                "hardened_malloc"
+                # conduwuit_mods is a development-only hot reload feature
+                "conduwuit_mods"
             ];
         };
         }));

nix/pkgs/complement/config.toml

@@ -17,19 +17,30 @@ ip_range_denylist = []
 url_preview_domain_contains_allowlist = ["*"]
 url_preview_domain_explicit_denylist = ["*"]
 media_compat_file_link = false
-media_startup_check = false
-prune_missing_media = false
+media_startup_check = true
+prune_missing_media = true
 log_colors = false
 admin_room_notices = false
 allow_check_for_updates = false
-allow_unstable_room_versions = true
 rocksdb_log_level = "debug"
 rocksdb_max_log_files = 1
 rocksdb_recovery_mode = 0
 rocksdb_paranoid_file_checks = true
 log_guest_registrations = false
 allow_legacy_media = true
-startup_netburst = false
+startup_netburst = true
+
+# valgrind makes things so slow
+dns_timeout = 60
+dns_attempts = 20
+request_conn_timeout = 60
+request_timeout = 120
+well_known_conn_timeout = 60
+well_known_timeout = 60
+federation_idle_timeout = 300
+sender_timeout = 300
+sender_idle_timeout = 300
+sender_retry_backoff_limit = 300
 
 [global.tls]
 certs = "/certificate.crt"

nix/pkgs/complement/default.nix

@@ -9,17 +9,22 @@
 , openssl
 , stdenv
 , tini
+, valgrind
 , writeShellScriptBin
 }:
 
 let
   main' = main.override {
-    profile = "test";
+    #profile = "test";
+    profile = "release-debuginfo";
     all_features = true;
     disable_release_max_log_level = true;
     disable_features = [
-        # no reason to use jemalloc for complement, just has compatibility/build issues
         "jemalloc"
+        "jemalloc_stats"
+        "jemalloc_prof"
+        "jemalloc_conf"
+        "io_uring"
         # console/CLI stuff isn't used or relevant for complement
         "console"
         "tokio_console"
@@ -27,15 +32,65 @@ let
         "sentry_telemetry"
         "perf_measurements"
         # the containers don't use or need systemd signal support
-        "systemd"
+        #"systemd"
         # this is non-functional on nix for some reason
         "hardened_malloc"
         # dont include experimental features
         "experimental"
+        # compression isn't needed for complement
+        "brotli_compression"
+        "gzip_compression"
+        "zstd_compression"
+        # complement doesn't need hot reloading
+        "conduwuit_mods"
+        # complement doesn't have URL preview media tests
+        "url_preview"
     ];
   };
+        # TODO: figure out why a suspicious amounnt of complement tests fail with valgrind only under complement.
+        # maybe issue with direct TLS mode?
+        #${lib.getExe' valgrind "valgrind"} \
+        #--leak-check=no \
+        #--undef-value-errors=no \
+        #--exit-on-first-error=yes \
+        #--error-exitcode=1 \
 
-  start = writeShellScriptBin "start" ''
+  # valgrind only works in non-static ocntexts
+  start = if !stdenv.hostPlatform.isStatic then writeShellScriptBin "start" ''
+    set -euxo pipefail
+
+    ${lib.getExe openssl} genrsa -out private_key.key 2048
+    ${lib.getExe openssl} req \
+      -new \
+      -sha256 \
+      -key private_key.key \
+      -subj "/C=US/ST=CA/O=MyOrg, Inc./CN=$SERVER_NAME" \
+      -out signing_request.csr
+    cp ${./v3.ext} v3.ext
+    echo "DNS.1 = $SERVER_NAME" >> v3.ext
+    echo "IP.1 = $(${lib.getExe gawk} 'END{print $1}' /etc/hosts)" \
+      >> v3.ext
+    ${lib.getExe openssl} x509 \
+      -req \
+      -extfile v3.ext \
+      -in signing_request.csr \
+      -CA /complement/ca/ca.crt \
+      -CAkey /complement/ca/ca.key \
+      -CAcreateserial \
+      -out certificate.crt \
+      -days 1 \
+      -sha256
+
+    ${lib.getExe' coreutils "env"} \
+      CONDUWUIT_SERVER_NAME="$SERVER_NAME" \
+      TMPDIR="/" \
+        ${lib.getExe' valgrind "valgrind"} \
+          --leak-check=no \
+          --undef-value-errors=no \
+          --exit-on-first-error=yes \
+          --error-exitcode=1 \
+            ${lib.getExe main'}
+  '' else writeShellScriptBin "start" ''
     set -euxo pipefail
 
     ${lib.getExe openssl} genrsa -out private_key.key 2048
@@ -80,6 +135,7 @@ dockerTools.buildImage {
       coreutils
       main'
       start
+      valgrind
     ];
   };
 

nix/pkgs/main/default.nix

@@ -15,7 +15,19 @@
 # Options (keep sorted)
 , all_features ? false
 , default_features ? true
-, disable_features ? []
+# default list of disabled features
+, disable_features ? [
+  # dont include experimental features
+  "experimental"
+  # jemalloc profiling/stats features are expensive and shouldn't
+  # be expected on non-debug builds.
+  "jemalloc_prof"
+  "jemalloc_stats"
+  # this is non-functional on nix for some reason
+  "hardened_malloc"
+  # conduwuit_mods is a development-only hot reload feature
+  "conduwuit_mods"
+]
 , disable_release_max_log_level ? false
 , features ? []
 , profile ? "release"
@@ -70,7 +82,7 @@ rust-jemalloc-sys' = (rust-jemalloc-sys.override {
 buildDepsOnlyEnv =
   let
     rocksdb' = (rocksdb.override {
-      jemalloc = rust-jemalloc-sys';
+      jemalloc = lib.optional (featureEnabled "jemalloc") rust-jemalloc-sys';
       # rocksdb fails to build with prefixed jemalloc, which is required on
       # darwin due to [1]. In this case, fall back to building rocksdb with
       # libc malloc. This should not cause conflicts, because all of the
@@ -91,6 +103,11 @@ buildDepsOnlyEnv =
       ++ [ "-DPORTABLE=haswell" ]) else ([ "-DPORTABLE=1" ])
       )
       ++ old.cmakeFlags;
+      # outputs has "tools" which we dont need or use
+      outputs = [ "out" ];
+
+      # preInstall hooks has stuff for messing with ldb/sst_dump which we dont need or use
+      preInstall = "";
     });
   in
   {
@@ -144,6 +161,19 @@ commonAttrs = {
       ];
     };
 
+    # This is redundant with CI
+    doCheck = false;
+
+    cargoTestCommand = "cargo test --locked ";
+    cargoExtraArgs = "--no-default-features --locked "
+      + lib.optionalString
+        (features'' != [])
+        "--features " + (builtins.concatStringsSep "," features'');
+    cargoTestExtraArgs = "--no-default-features --locked "
+      + lib.optionalString
+        (features'' != [])
+        "--features " + (builtins.concatStringsSep "," features'');
+
     dontStrip = profile == "dev" || profile == "test";
     dontPatchELF = profile == "dev" || profile == "test";
 
@@ -169,9 +199,6 @@ commonAttrs = {
       # differing values for `NIX_CFLAGS_COMPILE`, which contributes to spurious
       # rebuilds of bindgen and its depedents.
       jq
-
-      # needed so we can get rid of gcc and other unused deps that bloat OCI images
-      removeReferencesTo
   ]
   # needed to build Rust applications on macOS
   ++ lib.optionals stdenv.hostPlatform.isDarwin [
@@ -183,13 +210,6 @@ commonAttrs = {
       # https://discourse.nixos.org/t/compile-a-rust-binary-on-macos-dbcrossbar/8612
       pkgsBuildHost.darwin.apple_sdk.frameworks.Security
     ];
-
-    # for some reason gcc and other weird deps are added to OCI images and bloats it up
-    #
-    # <https://github.com/input-output-hk/haskell.nix/issues/829>
-    postInstall = with pkgsBuildHost; ''
-        find "$out" -type f -exec remove-references-to -t ${stdenv.cc} -t ${gcc} -t ${llvm} -t ${rustc.unwrapped} -t ${rustc} '{}' +
-    '';
  };
 in
 
@@ -198,15 +218,18 @@ craneLib.buildPackage ( commonAttrs // {
     env = buildDepsOnlyEnv;
   });
 
-  cargoExtraArgs = "--no-default-features "
+  # This is redundant with CI
+  doCheck = false;
+
+  cargoTestCommand = "cargo test --locked ";
+  cargoExtraArgs = "--no-default-features --locked "
+    + lib.optionalString
+      (features'' != [])
+      "--features " + (builtins.concatStringsSep "," features'');
+  cargoTestExtraArgs = "--no-default-features --locked "
     + lib.optionalString
       (features'' != [])
       "--features " + (builtins.concatStringsSep "," features'');
-
-  # This is redundant with CI
-  cargoTestCommand = "";
-  cargoCheckCommand = "";
-  doCheck = false;
 
   env = buildPackageEnv;
 

nix/pkgs/oci-image/default.nix

@@ -28,5 +28,18 @@ dockerTools.buildLayeredImage {
     Env = [
       "RUST_BACKTRACE=full"
     ];
+    Labels = {
+      "org.opencontainers.image.authors" = "June Clementine Strawberry <june@girlboss.ceo> and Jason Volk
+      <jason@zemos.net>";
+      "org.opencontainers.image.created" ="@${toString inputs.self.lastModified}";
+      "org.opencontainers.image.description" = "a very cool Matrix chat homeserver written in Rust";
+      "org.opencontainers.image.documentation" = "https://conduwuit.puppyirl.gay/";
+      "org.opencontainers.image.licenses" = "Apache-2.0";
+      "org.opencontainers.image.revision" = inputs.self.rev or inputs.self.dirtyRev or "";
+      "org.opencontainers.image.title" = main.pname;
+      "org.opencontainers.image.url" = "https://conduwuit.puppyirl.gay/";
+      "org.opencontainers.image.vendor" = "girlbossceo";
+      "org.opencontainers.image.version" = main.version;
+    };
   };
 }

rust-toolchain.toml

@@ -9,7 +9,7 @@
 # If you're having trouble making the relevant changes, bug a maintainer.
 
 [toolchain]
-channel = "1.83.0"
+channel = "1.84.0"
 profile = "minimal"
 components = [
     # For rust-analyzer

src/admin/admin.rs

@@ -1,6 +1,5 @@
 use clap::Parser;
 use conduwuit::Result;
-use ruma::events::room::message::RoomMessageEventContent;
 
 use crate::{
 	appservice, appservice::AppserviceCommand, check, check::CheckCommand, command::Command,
@@ -50,13 +49,10 @@ pub(super) enum AdminCommand {
 }
 
 #[tracing::instrument(skip_all, name = "command")]
-pub(super) async fn process(
-	command: AdminCommand,
-	context: &Command<'_>,
-) -> Result<RoomMessageEventContent> {
+pub(super) async fn process(command: AdminCommand, context: &Command<'_>) -> Result {
 	use AdminCommand::*;
 
-	Ok(match command {
+	match command {
 		| Appservices(command) => appservice::process(command, context).await?,
 		| Media(command) => media::process(command, context).await?,
 		| Users(command) => user::process(command, context).await?,
@@ -66,5 +62,7 @@ pub(super) async fn process(
 		| Debug(command) => debug::process(command, context).await?,
 		| Query(command) => query::process(command, context).await?,
 		| Check(command) => check::process(command, context).await?,
-	})
+	};
+
+	Ok(())
 }

src/admin/check/mod.rs

@@ -2,20 +2,11 @@ mod commands;
 
 use clap::Subcommand;
 use conduwuit::Result;
-use ruma::events::room::message::RoomMessageEventContent;
 
-use crate::Command;
+use crate::admin_command_dispatch;
 
+#[admin_command_dispatch]
 #[derive(Debug, Subcommand)]
 pub(super) enum CheckCommand {
-	AllUsers,
-}
-
-pub(super) async fn process(
-	command: CheckCommand,
-	context: &Command<'_>,
-) -> Result<RoomMessageEventContent> {
-	Ok(match command {
-		| CheckCommand::AllUsers => context.check_all_users().await?,
-	})
+	CheckAllUsers,
 }

src/admin/command.rs

@@ -1,6 +1,12 @@
-use std::time::SystemTime;
+use std::{fmt, time::SystemTime};
 
+use conduwuit::Result;
 use conduwuit_service::Services;
+use futures::{
+	io::{AsyncWriteExt, BufWriter},
+	lock::Mutex,
+	Future, FutureExt,
+};
 use ruma::EventId;
 
 pub(crate) struct Command<'a> {
@@ -8,4 +14,26 @@ pub(crate) struct Command<'a> {
 	pub(crate) body: &'a [&'a str],
 	pub(crate) timer: SystemTime,
 	pub(crate) reply_id: Option<&'a EventId>,
+	pub(crate) output: Mutex<BufWriter<Vec<u8>>>,
+}
+
+impl Command<'_> {
+	pub(crate) fn write_fmt(
+		&self,
+		arguments: fmt::Arguments<'_>,
+	) -> impl Future<Output = Result> + Send + '_ {
+		let buf = format!("{arguments}");
+		self.output.lock().then(|mut output| async move {
+			output.write_all(buf.as_bytes()).await.map_err(Into::into)
+		})
+	}
+
+	pub(crate) fn write_str<'a>(
+		&'a self,
+		s: &'a str,
+	) -> impl Future<Output = Result> + Send + 'a {
+		self.output.lock().then(move |mut output| async move {
+			output.write_all(s.as_bytes()).await.map_err(Into::into)
+		})
+	}
 }

src/admin/debug/commands.rs

@@ -6,7 +6,8 @@ use std::{
 };
 
 use conduwuit::{
-	debug_error, err, info, trace, utils, utils::string::EMPTY, warn, Error, PduEvent, Result,
+	debug_error, err, info, trace, utils, utils::string::EMPTY, warn, Error, PduEvent, PduId,
+	RawPduId, Result,
 };
 use futures::{FutureExt, StreamExt};
 use ruma::{
@@ -15,7 +16,10 @@ use ruma::{
 	CanonicalJsonObject, EventId, OwnedEventId, OwnedRoomOrAliasId, RoomId, RoomVersionId,
 	ServerName,
 };
-use service::rooms::state_compressor::HashSetCompressStateEvent;
+use service::rooms::{
+	short::{ShortEventId, ShortRoomId},
+	state_compressor::HashSetCompressStateEvent,
+};
 use tracing_subscriber::EnvFilter;
 
 use crate::admin_command;
@@ -131,13 +135,42 @@ pub(super) async fn get_pdu(&self, event_id: Box<EventId>) -> Result<RoomMessage
 	}
 }
 
+#[admin_command]
+pub(super) async fn get_short_pdu(
+	&self,
+	shortroomid: ShortRoomId,
+	shorteventid: ShortEventId,
+) -> Result<RoomMessageEventContent> {
+	let pdu_id: RawPduId = PduId {
+		shortroomid,
+		shorteventid: shorteventid.into(),
+	}
+	.into();
+
+	let pdu_json = self
+		.services
+		.rooms
+		.timeline
+		.get_pdu_json_from_id(&pdu_id)
+		.await;
+
+	match pdu_json {
+		| Ok(json) => {
+			let json_text =
+				serde_json::to_string_pretty(&json).expect("canonical json is valid json");
+			Ok(RoomMessageEventContent::notice_markdown(format!("```json\n{json_text}\n```",)))
+		},
+		| Err(_) => Ok(RoomMessageEventContent::text_plain("PDU not found locally.")),
+	}
+}
+
 #[admin_command]
 pub(super) async fn get_remote_pdu_list(
 	&self,
 	server: Box<ServerName>,
 	force: bool,
 ) -> Result<RoomMessageEventContent> {
-	if !self.services.globals.config.allow_federation {
+	if !self.services.server.config.allow_federation {
 		return Ok(RoomMessageEventContent::text_plain(
 			"Federation is disabled on this homeserver.",
 		));
@@ -202,7 +235,7 @@ pub(super) async fn get_remote_pdu(
 	event_id: Box<EventId>,
 	server: Box<ServerName>,
 ) -> Result<RoomMessageEventContent> {
-	if !self.services.globals.config.allow_federation {
+	if !self.services.server.config.allow_federation {
 		return Ok(RoomMessageEventContent::text_plain(
 			"Federation is disabled on this homeserver.",
 		));
@@ -386,7 +419,7 @@ pub(super) async fn change_log_level(
 	let handles = &["console"];
 
 	if reset {
-		let old_filter_layer = match EnvFilter::try_new(&self.services.globals.config.log) {
+		let old_filter_layer = match EnvFilter::try_new(&self.services.server.config.log) {
 			| Ok(s) => s,
 			| Err(e) => {
 				return Ok(RoomMessageEventContent::text_plain(format!(
@@ -405,7 +438,7 @@ pub(super) async fn change_log_level(
 			| Ok(()) => {
 				return Ok(RoomMessageEventContent::text_plain(format!(
 					"Successfully changed log level back to config value {}",
-					self.services.globals.config.log
+					self.services.server.config.log
 				)));
 			},
 			| Err(e) => {
@@ -521,7 +554,7 @@ pub(super) async fn first_pdu_in_room(
 		.services
 		.rooms
 		.state_cache
-		.server_in_room(&self.services.globals.config.server_name, &room_id)
+		.server_in_room(&self.services.server.config.server_name, &room_id)
 		.await
 	{
 		return Ok(RoomMessageEventContent::text_plain(
@@ -550,7 +583,7 @@ pub(super) async fn latest_pdu_in_room(
 		.services
 		.rooms
 		.state_cache
-		.server_in_room(&self.services.globals.config.server_name, &room_id)
+		.server_in_room(&self.services.server.config.server_name, &room_id)
 		.await
 	{
 		return Ok(RoomMessageEventContent::text_plain(
@@ -580,7 +613,7 @@ pub(super) async fn force_set_room_state_from_server(
 		.services
 		.rooms
 		.state_cache
-		.server_in_room(&self.services.globals.config.server_name, &room_id)
+		.server_in_room(&self.services.server.config.server_name, &room_id)
 		.await
 	{
 		return Ok(RoomMessageEventContent::text_plain(
@@ -692,7 +725,7 @@ pub(super) async fn force_set_room_state_from_server(
 		.save_state(room_id.clone().as_ref(), new_room_state)
 		.await?;
 
-	let state_lock = self.services.rooms.state.mutex.lock(&room_id).await;
+	let state_lock = self.services.rooms.state.mutex.lock(&*room_id).await;
 	self.services
 		.rooms
 		.state
@@ -785,13 +818,13 @@ pub(super) async fn resolve_true_destination(
 	server_name: Box<ServerName>,
 	no_cache: bool,
 ) -> Result<RoomMessageEventContent> {
-	if !self.services.globals.config.allow_federation {
+	if !self.services.server.config.allow_federation {
 		return Ok(RoomMessageEventContent::text_plain(
 			"Federation is disabled on this homeserver.",
 		));
 	}
 
-	if server_name == self.services.globals.config.server_name {
+	if server_name == self.services.server.config.server_name {
 		return Ok(RoomMessageEventContent::text_plain(
 			"Not allowed to send federation requests to ourselves. Please use `get-pdu` for \
 			 fetching local PDUs.",
@@ -810,19 +843,27 @@ pub(super) async fn resolve_true_destination(
 }
 
 #[admin_command]
-pub(super) async fn memory_stats(&self) -> Result<RoomMessageEventContent> {
-	let html_body = conduwuit::alloc::memory_stats();
+pub(super) async fn memory_stats(&self, opts: Option<String>) -> Result<RoomMessageEventContent> {
+	const OPTS: &str = "abcdefghijklmnopqrstuvwxyz";
 
-	if html_body.is_none() {
-		return Ok(RoomMessageEventContent::text_plain(
-			"malloc stats are not supported on your compiled malloc.",
-		));
-	}
+	let opts: String = OPTS
+		.chars()
+		.filter(|&c| {
+			let allow_any = opts.as_ref().is_some_and(|opts| opts == "*");
 
-	Ok(RoomMessageEventContent::text_html(
-		"This command's output can only be viewed by clients that render HTML.".to_owned(),
-		html_body.expect("string result"),
-	))
+			let allow = allow_any || opts.as_ref().is_some_and(|opts| opts.contains(c));
+
+			!allow
+		})
+		.collect();
+
+	let stats = conduwuit::alloc::memory_stats(&opts).unwrap_or_default();
+
+	self.write_str("```\n").await?;
+	self.write_str(&stats).await?;
+	self.write_str("\n```").await?;
+
+	Ok(RoomMessageEventContent::text_plain(""))
 }
 
 #[cfg(tokio_unstable)]
@@ -895,7 +936,7 @@ pub(super) async fn list_dependencies(&self, names: bool) -> Result<RoomMessageE
 		} else {
 			String::new()
 		};
-		writeln!(out, "{name} | {version} | {feats}")?;
+		writeln!(out, "| {name} | {version} | {feats} |")?;
 	}
 
 	Ok(RoomMessageEventContent::notice_markdown(out))
@@ -911,8 +952,8 @@ pub(super) async fn database_stats(
 	let map_name = map.as_ref().map_or(EMPTY, String::as_str);
 
 	let mut out = String::new();
-	for (name, map) in self.services.db.iter() {
-		if !map_name.is_empty() && *map_name != *name {
+	for (&name, map) in self.services.db.iter() {
+		if !map_name.is_empty() && map_name != name {
 			continue;
 		}
 
@@ -923,3 +964,12 @@ pub(super) async fn database_stats(
 
 	Ok(RoomMessageEventContent::notice_markdown(out))
 }
+
+#[admin_command]
+pub(super) async fn trim_memory(&self) -> Result<RoomMessageEventContent> {
+	conduwuit::alloc::trim(None)?;
+
+	writeln!(self, "done").await?;
+
+	Ok(RoomMessageEventContent::notice_plain(""))
+}

src/admin/debug/mod.rs

@@ -4,6 +4,7 @@ pub(crate) mod tester;
 use clap::Subcommand;
 use conduwuit::Result;
 use ruma::{EventId, OwnedRoomOrAliasId, RoomId, ServerName};
+use service::rooms::short::{ShortEventId, ShortRoomId};
 
 use self::tester::TesterCommand;
 use crate::admin_command_dispatch;
@@ -31,12 +32,21 @@ pub(super) enum DebugCommand {
 	/// the command.
 	ParsePdu,
 
-	/// - Retrieve and print a PDU by ID from the conduwuit database
+	/// - Retrieve and print a PDU by EventID from the conduwuit database
 	GetPdu {
 		/// An event ID (a $ followed by the base64 reference hash)
 		event_id: Box<EventId>,
 	},
 
+	/// - Retrieve and print a PDU by PduId from the conduwuit database
+	GetShortPdu {
+		/// Shortroomid integer
+		shortroomid: ShortRoomId,
+
+		/// Shorteventid integer
+		shorteventid: ShortEventId,
+	},
+
 	/// - Attempts to retrieve a PDU from a remote server. Inserts it into our
 	///   database/timeline if found and we do not have this PDU already
 	///   (following normal event auth rules, handles it as an incoming PDU).
@@ -181,7 +191,13 @@ pub(super) enum DebugCommand {
 	},
 
 	/// - Print extended memory usage
-	MemoryStats,
+	///
+	/// Optional argument is a character mask (a sequence of characters in any
+	/// order) which enable additional extended statistics. Known characters are
+	/// "abdeglmx". For convenience, a '*' will enable everything.
+	MemoryStats {
+		opts: Option<String>,
+	},
 
 	/// - Print general tokio runtime metric totals.
 	RuntimeMetrics,
@@ -207,6 +223,9 @@ pub(super) enum DebugCommand {
 		map: Option<String>,
 	},
 
+	/// - Trim memory usage
+	TrimMemory,
+
 	/// - Developer test stubs
 	#[command(subcommand)]
 	#[allow(non_snake_case)]

src/admin/debug/tester.rs

@@ -31,7 +31,7 @@ async fn failure(&self) -> Result<RoomMessageEventContent> {
 #[admin_command]
 async fn tester(&self) -> Result<RoomMessageEventContent> {
 
-	Ok(RoomMessageEventContent::notice_plain("completed"))
+	Ok(RoomMessageEventContent::notice_plain("legacy"))
 }
 
 #[inline(never)]

src/admin/federation/commands.rs

@@ -92,7 +92,7 @@ pub(super) async fn remote_user_in_rooms(
 	&self,
 	user_id: Box<UserId>,
 ) -> Result<RoomMessageEventContent> {
-	if user_id.server_name() == self.services.globals.config.server_name {
+	if user_id.server_name() == self.services.server.config.server_name {
 		return Ok(RoomMessageEventContent::text_plain(
 			"User belongs to our server, please use `list-joined-rooms` user admin command \
 			 instead.",

src/admin/mod.rs

@@ -1,6 +1,7 @@
 #![recursion_limit = "192"]
 #![allow(clippy::wildcard_imports)]
 #![allow(clippy::enum_glob_use)]
+#![allow(clippy::too_many_arguments)]
 
 pub(crate) mod admin;
 pub(crate) mod command;

src/admin/processor.rs

@@ -1,5 +1,6 @@
 use std::{
 	fmt::Write,
+	mem::take,
 	panic::AssertUnwindSafe,
 	sync::{Arc, Mutex},
 	time::SystemTime,
@@ -17,7 +18,7 @@ use conduwuit::{
 	utils::string::{collect_stream, common_prefix},
 	warn, Error, Result,
 };
-use futures::future::FutureExt;
+use futures::{future::FutureExt, io::BufWriter, AsyncWriteExt};
 use ruma::{
 	events::{
 		relation::InReplyTo,
@@ -62,9 +63,32 @@ async fn process_command(services: Arc<Services>, input: &CommandInput) -> Proce
 		body: &body,
 		timer: SystemTime::now(),
 		reply_id: input.reply_id.as_deref(),
+		output: BufWriter::new(Vec::new()).into(),
 	};
 
-	process(&context, command, &args).await
+	let (result, mut logs) = process(&context, command, &args).await;
+
+	let output = &mut context.output.lock().await;
+	output.flush().await.expect("final flush of output stream");
+
+	let output =
+		String::from_utf8(take(output.get_mut())).expect("invalid utf8 in command output stream");
+
+	match result {
+		| Ok(()) if logs.is_empty() =>
+			Ok(Some(reply(RoomMessageEventContent::notice_markdown(output), context.reply_id))),
+
+		| Ok(()) => {
+			logs.write_str(output.as_str()).expect("output buffer");
+			Ok(Some(reply(RoomMessageEventContent::notice_markdown(logs), context.reply_id)))
+		},
+		| Err(error) => {
+			write!(&mut logs, "Command failed with error:\n```\n{error:#?}\n```")
+				.expect("output buffer");
+
+			Err(reply(RoomMessageEventContent::notice_markdown(logs), context.reply_id))
+		},
+	}
 }
 
 fn handle_panic(error: &Error, command: &CommandInput) -> ProcessorResult {
@@ -81,7 +105,7 @@ async fn process(
 	context: &Command<'_>,
 	command: AdminCommand,
 	args: &[String],
-) -> ProcessorResult {
+) -> (Result, String) {
 	let (capture, logs) = capture_create(context);
 
 	let capture_scope = capture.start();
@@ -104,18 +128,7 @@ async fn process(
 	}
 	drop(logs);
 
-	match result {
-		| Ok(content) => {
-			write!(&mut output, "{0}", content.body())
-				.expect("failed to format command result to output buffer");
-			Ok(Some(reply(RoomMessageEventContent::notice_markdown(output), context.reply_id)))
-		},
-		| Err(error) => {
-			write!(&mut output, "Command failed with error:\n```\n{error:#?}\n```")
-				.expect("failed to format command result to output");
-			Err(reply(RoomMessageEventContent::notice_markdown(output), context.reply_id))
-		},
-	}
+	(result, output)
 }
 
 fn capture_create(context: &Command<'_>) -> (Arc<Capture>, Arc<Mutex<String>>) {

src/admin/query/account_data.rs

@@ -3,8 +3,9 @@ use conduwuit::Result;
 use futures::StreamExt;
 use ruma::{events::room::message::RoomMessageEventContent, RoomId, UserId};
 
-use crate::Command;
+use crate::{admin_command, admin_command_dispatch};
 
+#[admin_command_dispatch]
 #[derive(Debug, Subcommand)]
 /// All the getters and iterators from src/database/key_value/account_data.rs
 pub(crate) enum AccountDataCommand {
@@ -19,7 +20,7 @@ pub(crate) enum AccountDataCommand {
 	},
 
 	/// - Searches the account data for a specific kind.
-	Get {
+	AccountDataGet {
 		/// Full user ID
 		user_id: Box<UserId>,
 		/// Account data event type
@@ -29,38 +30,43 @@ pub(crate) enum AccountDataCommand {
 	},
 }
 
-/// All the getters and iterators from src/database/key_value/account_data.rs
-pub(super) async fn process(
-	subcommand: AccountDataCommand,
-	context: &Command<'_>,
+#[admin_command]
+async fn changes_since(
+	&self,
+	user_id: Box<UserId>,
+	since: u64,
+	room_id: Option<Box<RoomId>>,
 ) -> Result<RoomMessageEventContent> {
-	let services = context.services;
+	let timer = tokio::time::Instant::now();
+	let results: Vec<_> = self
+		.services
+		.account_data
+		.changes_since(room_id.as_deref(), &user_id, since)
+		.collect()
+		.await;
+	let query_time = timer.elapsed();
 
-	match subcommand {
-		| AccountDataCommand::ChangesSince { user_id, since, room_id } => {
-			let timer = tokio::time::Instant::now();
-			let results: Vec<_> = services
-				.account_data
-				.changes_since(room_id.as_deref(), &user_id, since)
-				.collect()
-				.await;
-			let query_time = timer.elapsed();
-
-			Ok(RoomMessageEventContent::notice_markdown(format!(
-				"Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```"
-			)))
-		},
-		| AccountDataCommand::Get { user_id, kind, room_id } => {
-			let timer = tokio::time::Instant::now();
-			let results = services
-				.account_data
-				.get_raw(room_id.as_deref(), &user_id, &kind)
-				.await;
-			let query_time = timer.elapsed();
-
-			Ok(RoomMessageEventContent::notice_markdown(format!(
-				"Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```"
-			)))
-		},
-	}
+	Ok(RoomMessageEventContent::notice_markdown(format!(
+		"Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```"
+	)))
+}
+
+#[admin_command]
+async fn account_data_get(
+	&self,
+	user_id: Box<UserId>,
+	kind: String,
+	room_id: Option<Box<RoomId>>,
+) -> Result<RoomMessageEventContent> {
+	let timer = tokio::time::Instant::now();
+	let results = self
+		.services
+		.account_data
+		.get_raw(room_id.as_deref(), &user_id, &kind)
+		.await;
+	let query_time = timer.elapsed();
+
+	Ok(RoomMessageEventContent::notice_markdown(format!(
+		"Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```"
+	)))
 }

src/admin/query/appservice.rs

@@ -1,6 +1,5 @@
 use clap::Subcommand;
 use conduwuit::Result;
-use ruma::events::room::message::RoomMessageEventContent;
 
 use crate::Command;
 
@@ -18,10 +17,7 @@ pub(crate) enum AppserviceCommand {
 }
 
 /// All the getters and iterators from src/database/key_value/appservice.rs
-pub(super) async fn process(
-	subcommand: AppserviceCommand,
-	context: &Command<'_>,
-) -> Result<RoomMessageEventContent> {
+pub(super) async fn process(subcommand: AppserviceCommand, context: &Command<'_>) -> Result {
 	let services = context.services;
 
 	match subcommand {
@@ -31,18 +27,15 @@ pub(super) async fn process(
 
 			let query_time = timer.elapsed();
 
-			Ok(RoomMessageEventContent::notice_markdown(format!(
-				"Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```"
-			)))
+			write!(context, "Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```")
 		},
 		| AppserviceCommand::All => {
 			let timer = tokio::time::Instant::now();
 			let results = services.appservice.all().await;
 			let query_time = timer.elapsed();
 
-			Ok(RoomMessageEventContent::notice_markdown(format!(
-				"Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```"
-			)))
+			write!(context, "Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```")
 		},
 	}
+	.await
 }

src/admin/query/globals.rs

@@ -1,6 +1,6 @@
 use clap::Subcommand;
 use conduwuit::Result;
-use ruma::{events::room::message::RoomMessageEventContent, ServerName};
+use ruma::ServerName;
 
 use crate::Command;
 
@@ -21,10 +21,7 @@ pub(crate) enum GlobalsCommand {
 }
 
 /// All the getters and iterators from src/database/key_value/globals.rs
-pub(super) async fn process(
-	subcommand: GlobalsCommand,
-	context: &Command<'_>,
-) -> Result<RoomMessageEventContent> {
+pub(super) async fn process(subcommand: GlobalsCommand, context: &Command<'_>) -> Result {
 	let services = context.services;
 
 	match subcommand {
@@ -33,36 +30,29 @@ pub(super) async fn process(
 			let results = services.globals.db.database_version().await;
 			let query_time = timer.elapsed();
 
-			Ok(RoomMessageEventContent::notice_markdown(format!(
-				"Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```"
-			)))
+			write!(context, "Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```")
 		},
 		| GlobalsCommand::CurrentCount => {
 			let timer = tokio::time::Instant::now();
 			let results = services.globals.db.current_count();
 			let query_time = timer.elapsed();
 
-			Ok(RoomMessageEventContent::notice_markdown(format!(
-				"Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```"
-			)))
+			write!(context, "Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```")
 		},
 		| GlobalsCommand::LastCheckForUpdatesId => {
 			let timer = tokio::time::Instant::now();
 			let results = services.updates.last_check_for_updates_id().await;
 			let query_time = timer.elapsed();
 
-			Ok(RoomMessageEventContent::notice_markdown(format!(
-				"Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```"
-			)))
+			write!(context, "Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```")
 		},
 		| GlobalsCommand::SigningKeysFor { origin } => {
 			let timer = tokio::time::Instant::now();
 			let results = services.server_keys.verify_keys_for(&origin).await;
 			let query_time = timer.elapsed();
 
-			Ok(RoomMessageEventContent::notice_markdown(format!(
-				"Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```"
-			)))
+			write!(context, "Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```")
 		},
 	}
+	.await
 }

src/admin/query/mod.rs

@@ -3,10 +3,13 @@ mod appservice;
 mod globals;
 mod presence;
 mod pusher;
+mod raw;
 mod resolver;
 mod room_alias;
 mod room_state_cache;
+mod room_timeline;
 mod sending;
+mod short;
 mod users;
 
 use clap::Subcommand;
@@ -14,9 +17,10 @@ use conduwuit::Result;
 
 use self::{
 	account_data::AccountDataCommand, appservice::AppserviceCommand, globals::GlobalsCommand,
-	presence::PresenceCommand, pusher::PusherCommand, resolver::ResolverCommand,
+	presence::PresenceCommand, pusher::PusherCommand, raw::RawCommand, resolver::ResolverCommand,
 	room_alias::RoomAliasCommand, room_state_cache::RoomStateCacheCommand,
-	sending::SendingCommand, users::UsersCommand,
+	room_timeline::RoomTimelineCommand, sending::SendingCommand, short::ShortCommand,
+	users::UsersCommand,
 };
 use crate::admin_command_dispatch;
 
@@ -44,6 +48,10 @@ pub(super) enum QueryCommand {
 	#[command(subcommand)]
 	RoomStateCache(RoomStateCacheCommand),
 
+	/// - rooms/timeline iterators and getters
+	#[command(subcommand)]
+	RoomTimeline(RoomTimelineCommand),
+
 	/// - globals.rs iterators and getters
 	#[command(subcommand)]
 	Globals(GlobalsCommand),
@@ -63,4 +71,12 @@ pub(super) enum QueryCommand {
 	/// - pusher service
 	#[command(subcommand)]
 	Pusher(PusherCommand),
+
+	/// - short service
+	#[command(subcommand)]
+	Short(ShortCommand),
+
+	/// - raw service
+	#[command(subcommand)]
+	Raw(RawCommand),
 }

src/admin/query/presence.rs

@@ -1,7 +1,7 @@
 use clap::Subcommand;
 use conduwuit::Result;
 use futures::StreamExt;
-use ruma::{events::room::message::RoomMessageEventContent, UserId};
+use ruma::UserId;
 
 use crate::Command;
 
@@ -23,21 +23,16 @@ pub(crate) enum PresenceCommand {
 }
 
 /// All the getters and iterators in key_value/presence.rs
-pub(super) async fn process(
-	subcommand: PresenceCommand,
-	context: &Command<'_>,
-) -> Result<RoomMessageEventContent> {
+pub(super) async fn process(subcommand: PresenceCommand, context: &Command<'_>) -> Result {
 	let services = context.services;
 
 	match subcommand {
 		| PresenceCommand::GetPresence { user_id } => {
 			let timer = tokio::time::Instant::now();
-			let results = services.presence.db.get_presence(&user_id).await;
+			let results = services.presence.get_presence(&user_id).await;
 			let query_time = timer.elapsed();
 
-			Ok(RoomMessageEventContent::notice_markdown(format!(
-				"Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```"
-			)))
+			write!(context, "Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```")
 		},
 		| PresenceCommand::PresenceSince { since } => {
 			let timer = tokio::time::Instant::now();
@@ -49,9 +44,8 @@ pub(super) async fn process(
 				.await;
 			let query_time = timer.elapsed();
 
-			Ok(RoomMessageEventContent::notice_markdown(format!(
-				"Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```"
-			)))
+			write!(context, "Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```")
 		},
 	}
+	.await
 }

src/admin/query/pusher.rs

@@ -1,6 +1,6 @@
 use clap::Subcommand;
 use conduwuit::Result;
-use ruma::{events::room::message::RoomMessageEventContent, UserId};
+use ruma::UserId;
 
 use crate::Command;
 
@@ -13,10 +13,7 @@ pub(crate) enum PusherCommand {
 	},
 }
 
-pub(super) async fn process(
-	subcommand: PusherCommand,
-	context: &Command<'_>,
-) -> Result<RoomMessageEventContent> {
+pub(super) async fn process(subcommand: PusherCommand, context: &Command<'_>) -> Result {
 	let services = context.services;
 
 	match subcommand {
@@ -25,9 +22,8 @@ pub(super) async fn process(
 			let results = services.pusher.get_pushers(&user_id).await;
 			let query_time = timer.elapsed();
 
-			Ok(RoomMessageEventContent::notice_markdown(format!(
-				"Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```"
-			)))
+			write!(context, "Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```")
 		},
 	}
+	.await
 }

src/admin/query/raw.rs

@@ -0,0 +1,576 @@
+use std::{borrow::Cow, collections::BTreeMap, ops::Deref};
+
+use clap::Subcommand;
+use conduwuit::{
+	apply, at, is_zero,
+	utils::{
+		stream::{ReadyExt, TryIgnore, TryParallelExt},
+		string::EMPTY,
+		IterStream,
+	},
+	Err, Result,
+};
+use futures::{FutureExt, StreamExt, TryStreamExt};
+use ruma::events::room::message::RoomMessageEventContent;
+use tokio::time::Instant;
+
+use crate::{admin_command, admin_command_dispatch};
+
+#[admin_command_dispatch]
+#[derive(Debug, Subcommand)]
+#[allow(clippy::enum_variant_names)]
+/// Query tables from database
+pub(crate) enum RawCommand {
+	/// - List database maps
+	RawMaps,
+
+	/// - Raw database query
+	RawGet {
+		/// Map name
+		map: String,
+
+		/// Key
+		key: String,
+	},
+
+	/// - Raw database delete (for string keys)
+	RawDel {
+		/// Map name
+		map: String,
+
+		/// Key
+		key: String,
+	},
+
+	/// - Raw database keys iteration
+	RawKeys {
+		/// Map name
+		map: String,
+
+		/// Key prefix
+		prefix: Option<String>,
+	},
+
+	/// - Raw database key size breakdown
+	RawKeysSizes {
+		/// Map name
+		map: Option<String>,
+
+		/// Key prefix
+		prefix: Option<String>,
+	},
+
+	/// - Raw database keys total bytes
+	RawKeysTotal {
+		/// Map name
+		map: Option<String>,
+
+		/// Key prefix
+		prefix: Option<String>,
+	},
+
+	/// - Raw database values size breakdown
+	RawValsSizes {
+		/// Map name
+		map: Option<String>,
+
+		/// Key prefix
+		prefix: Option<String>,
+	},
+
+	/// - Raw database values total bytes
+	RawValsTotal {
+		/// Map name
+		map: Option<String>,
+
+		/// Key prefix
+		prefix: Option<String>,
+	},
+
+	/// - Raw database items iteration
+	RawIter {
+		/// Map name
+		map: String,
+
+		/// Key prefix
+		prefix: Option<String>,
+	},
+
+	/// - Raw database keys iteration
+	RawKeysFrom {
+		/// Map name
+		map: String,
+
+		/// Lower-bound
+		start: String,
+
+		/// Limit
+		#[arg(short, long)]
+		limit: Option<usize>,
+	},
+
+	/// - Raw database items iteration
+	RawIterFrom {
+		/// Map name
+		map: String,
+
+		/// Lower-bound
+		start: String,
+
+		/// Limit
+		#[arg(short, long)]
+		limit: Option<usize>,
+	},
+
+	/// - Raw database record count
+	RawCount {
+		/// Map name
+		map: Option<String>,
+
+		/// Key prefix
+		prefix: Option<String>,
+	},
+
+	/// - Compact database
+	Compact {
+		#[arg(short, long, alias("column"))]
+		map: Option<Vec<String>>,
+
+		#[arg(long)]
+		start: Option<String>,
+
+		#[arg(long)]
+		stop: Option<String>,
+
+		#[arg(long)]
+		from: Option<usize>,
+
+		#[arg(long)]
+		into: Option<usize>,
+
+		/// There is one compaction job per column; then this controls how many
+		/// columns are compacted in parallel. If zero, one compaction job is
+		/// still run at a time here, but in exclusive-mode blocking any other
+		/// automatic compaction jobs until complete.
+		#[arg(long)]
+		parallelism: Option<usize>,
+
+		#[arg(long, default_value("false"))]
+		exhaustive: bool,
+	},
+}
+
+#[admin_command]
+pub(super) async fn compact(
+	&self,
+	map: Option<Vec<String>>,
+	start: Option<String>,
+	stop: Option<String>,
+	from: Option<usize>,
+	into: Option<usize>,
+	parallelism: Option<usize>,
+	exhaustive: bool,
+) -> Result<RoomMessageEventContent> {
+	use conduwuit_database::compact::Options;
+
+	let default_all_maps = map
+		.is_none()
+		.then(|| {
+			self.services
+				.db
+				.keys()
+				.map(Deref::deref)
+				.map(ToOwned::to_owned)
+		})
+		.into_iter()
+		.flatten();
+
+	let maps: Vec<_> = map
+		.unwrap_or_default()
+		.into_iter()
+		.chain(default_all_maps)
+		.map(|map| self.services.db.get(&map))
+		.filter_map(Result::ok)
+		.cloned()
+		.collect();
+
+	if maps.is_empty() {
+		return Err!("--map argument invalid. not found in database");
+	}
+
+	let range = (
+		start.as_ref().map(String::as_bytes).map(Into::into),
+		stop.as_ref().map(String::as_bytes).map(Into::into),
+	);
+
+	let options = Options {
+		range,
+		level: (from, into),
+		exclusive: parallelism.is_some_and(is_zero!()),
+		exhaustive,
+	};
+
+	let runtime = self.services.server.runtime().clone();
+	let parallelism = parallelism.unwrap_or(1);
+	let results = maps
+		.into_iter()
+		.try_stream()
+		.paralleln_and_then(runtime, parallelism, move |map| {
+			map.compact_blocking(options.clone())?;
+			Ok(map.name().to_owned())
+		})
+		.collect::<Vec<_>>();
+
+	let timer = Instant::now();
+	let results = results.await;
+	let query_time = timer.elapsed();
+	self.write_str(&format!("Jobs completed in {query_time:?}:\n\n```rs\n{results:#?}\n```"))
+		.await?;
+
+	Ok(RoomMessageEventContent::text_plain(""))
+}
+
+#[admin_command]
+pub(super) async fn raw_count(
+	&self,
+	map: Option<String>,
+	prefix: Option<String>,
+) -> Result<RoomMessageEventContent> {
+	let prefix = prefix.as_deref().unwrap_or(EMPTY);
+
+	let default_all_maps = map
+		.is_none()
+		.then(|| self.services.db.keys().map(Deref::deref))
+		.into_iter()
+		.flatten();
+
+	let maps: Vec<_> = map
+		.iter()
+		.map(String::as_str)
+		.chain(default_all_maps)
+		.map(|map| self.services.db.get(map))
+		.filter_map(Result::ok)
+		.cloned()
+		.collect();
+
+	let timer = Instant::now();
+	let count = maps
+		.iter()
+		.stream()
+		.then(|map| map.raw_count_prefix(&prefix))
+		.ready_fold(0_usize, usize::saturating_add)
+		.await;
+
+	let query_time = timer.elapsed();
+	self.write_str(&format!("Query completed in {query_time:?}:\n\n```rs\n{count:#?}\n```"))
+		.await?;
+
+	Ok(RoomMessageEventContent::text_plain(""))
+}
+
+#[admin_command]
+pub(super) async fn raw_keys(
+	&self,
+	map: String,
+	prefix: Option<String>,
+) -> Result<RoomMessageEventContent> {
+	writeln!(self, "```").boxed().await?;
+
+	let map = self.services.db.get(map.as_str())?;
+	let timer = Instant::now();
+	prefix
+		.as_deref()
+		.map_or_else(|| map.raw_keys().boxed(), |prefix| map.raw_keys_prefix(prefix).boxed())
+		.map_ok(String::from_utf8_lossy)
+		.try_for_each(|str| writeln!(self, "{str:?}"))
+		.boxed()
+		.await?;
+
+	let query_time = timer.elapsed();
+	let out = format!("\n```\n\nQuery completed in {query_time:?}");
+	self.write_str(out.as_str()).await?;
+
+	Ok(RoomMessageEventContent::text_plain(""))
+}
+
+#[admin_command]
+pub(super) async fn raw_keys_sizes(
+	&self,
+	map: Option<String>,
+	prefix: Option<String>,
+) -> Result<RoomMessageEventContent> {
+	let prefix = prefix.as_deref().unwrap_or(EMPTY);
+
+	let default_all_maps = map
+		.is_none()
+		.then(|| self.services.db.keys().map(Deref::deref))
+		.into_iter()
+		.flatten();
+
+	let maps: Vec<_> = map
+		.iter()
+		.map(String::as_str)
+		.chain(default_all_maps)
+		.map(|map| self.services.db.get(map))
+		.filter_map(Result::ok)
+		.cloned()
+		.collect();
+
+	let timer = Instant::now();
+	let result = maps
+		.iter()
+		.stream()
+		.map(|map| map.raw_keys_prefix(&prefix))
+		.flatten()
+		.ignore_err()
+		.map(<[u8]>::len)
+		.ready_fold_default(|mut map: BTreeMap<_, usize>, len| {
+			let entry = map.entry(len).or_default();
+			*entry = entry.saturating_add(1);
+			map
+		})
+		.await;
+
+	let query_time = timer.elapsed();
+	let result = format!("```\n{result:#?}\n```\n\nQuery completed in {query_time:?}");
+	self.write_str(result.as_str()).await?;
+
+	Ok(RoomMessageEventContent::text_plain(""))
+}
+
+#[admin_command]
+pub(super) async fn raw_keys_total(
+	&self,
+	map: Option<String>,
+	prefix: Option<String>,
+) -> Result<RoomMessageEventContent> {
+	let prefix = prefix.as_deref().unwrap_or(EMPTY);
+
+	let default_all_maps = map
+		.is_none()
+		.then(|| self.services.db.keys().map(Deref::deref))
+		.into_iter()
+		.flatten();
+
+	let maps: Vec<_> = map
+		.iter()
+		.map(String::as_str)
+		.chain(default_all_maps)
+		.map(|map| self.services.db.get(map))
+		.filter_map(Result::ok)
+		.cloned()
+		.collect();
+
+	let timer = Instant::now();
+	let result = maps
+		.iter()
+		.stream()
+		.map(|map| map.raw_keys_prefix(&prefix))
+		.flatten()
+		.ignore_err()
+		.map(<[u8]>::len)
+		.ready_fold_default(|acc: usize, len| acc.saturating_add(len))
+		.await;
+
+	let query_time = timer.elapsed();
+
+	self.write_str(&format!("```\n{result:#?}\n\n```\n\nQuery completed in {query_time:?}"))
+		.await?;
+
+	Ok(RoomMessageEventContent::text_plain(""))
+}
+
+#[admin_command]
+pub(super) async fn raw_vals_sizes(
+	&self,
+	map: Option<String>,
+	prefix: Option<String>,
+) -> Result<RoomMessageEventContent> {
+	let prefix = prefix.as_deref().unwrap_or(EMPTY);
+
+	let default_all_maps = map
+		.is_none()
+		.then(|| self.services.db.keys().map(Deref::deref))
+		.into_iter()
+		.flatten();
+
+	let maps: Vec<_> = map
+		.iter()
+		.map(String::as_str)
+		.chain(default_all_maps)
+		.map(|map| self.services.db.get(map))
+		.filter_map(Result::ok)
+		.cloned()
+		.collect();
+
+	let timer = Instant::now();
+	let result = maps
+		.iter()
+		.stream()
+		.map(|map| map.raw_stream_prefix(&prefix))
+		.flatten()
+		.ignore_err()
+		.map(at!(1))
+		.map(<[u8]>::len)
+		.ready_fold_default(|mut map: BTreeMap<_, usize>, len| {
+			let entry = map.entry(len).or_default();
+			*entry = entry.saturating_add(1);
+			map
+		})
+		.await;
+
+	let query_time = timer.elapsed();
+	let result = format!("```\n{result:#?}\n```\n\nQuery completed in {query_time:?}");
+	self.write_str(result.as_str()).await?;
+
+	Ok(RoomMessageEventContent::text_plain(""))
+}
+
+#[admin_command]
+pub(super) async fn raw_vals_total(
+	&self,
+	map: Option<String>,
+	prefix: Option<String>,
+) -> Result<RoomMessageEventContent> {
+	let prefix = prefix.as_deref().unwrap_or(EMPTY);
+
+	let default_all_maps = map
+		.is_none()
+		.then(|| self.services.db.keys().map(Deref::deref))
+		.into_iter()
+		.flatten();
+
+	let maps: Vec<_> = map
+		.iter()
+		.map(String::as_str)
+		.chain(default_all_maps)
+		.map(|map| self.services.db.get(map))
+		.filter_map(Result::ok)
+		.cloned()
+		.collect();
+
+	let timer = Instant::now();
+	let result = maps
+		.iter()
+		.stream()
+		.map(|map| map.raw_stream_prefix(&prefix))
+		.flatten()
+		.ignore_err()
+		.map(at!(1))
+		.map(<[u8]>::len)
+		.ready_fold_default(|acc: usize, len| acc.saturating_add(len))
+		.await;
+
+	let query_time = timer.elapsed();
+
+	self.write_str(&format!("```\n{result:#?}\n\n```\n\nQuery completed in {query_time:?}"))
+		.await?;
+
+	Ok(RoomMessageEventContent::text_plain(""))
+}
+
+#[admin_command]
+pub(super) async fn raw_iter(
+	&self,
+	map: String,
+	prefix: Option<String>,
+) -> Result<RoomMessageEventContent> {
+	writeln!(self, "```").await?;
+
+	let map = self.services.db.get(&map)?;
+	let timer = Instant::now();
+	prefix
+		.as_deref()
+		.map_or_else(|| map.raw_stream().boxed(), |prefix| map.raw_stream_prefix(prefix).boxed())
+		.map_ok(apply!(2, String::from_utf8_lossy))
+		.map_ok(apply!(2, Cow::into_owned))
+		.try_for_each(|keyval| writeln!(self, "{keyval:?}"))
+		.boxed()
+		.await?;
+
+	let query_time = timer.elapsed();
+	self.write_str(&format!("\n```\n\nQuery completed in {query_time:?}"))
+		.await?;
+
+	Ok(RoomMessageEventContent::text_plain(""))
+}
+
+#[admin_command]
+pub(super) async fn raw_keys_from(
+	&self,
+	map: String,
+	start: String,
+	limit: Option<usize>,
+) -> Result<RoomMessageEventContent> {
+	writeln!(self, "```").await?;
+
+	let map = self.services.db.get(&map)?;
+	let timer = Instant::now();
+	map.raw_keys_from(&start)
+		.map_ok(String::from_utf8_lossy)
+		.take(limit.unwrap_or(usize::MAX))
+		.try_for_each(|str| writeln!(self, "{str:?}"))
+		.boxed()
+		.await?;
+
+	let query_time = timer.elapsed();
+	self.write_str(&format!("\n```\n\nQuery completed in {query_time:?}"))
+		.await?;
+
+	Ok(RoomMessageEventContent::text_plain(""))
+}
+
+#[admin_command]
+pub(super) async fn raw_iter_from(
+	&self,
+	map: String,
+	start: String,
+	limit: Option<usize>,
+) -> Result<RoomMessageEventContent> {
+	let map = self.services.db.get(&map)?;
+	let timer = Instant::now();
+	let result = map
+		.raw_stream_from(&start)
+		.map_ok(apply!(2, String::from_utf8_lossy))
+		.map_ok(apply!(2, Cow::into_owned))
+		.take(limit.unwrap_or(usize::MAX))
+		.try_collect::<Vec<(String, String)>>()
+		.await?;
+
+	let query_time = timer.elapsed();
+	Ok(RoomMessageEventContent::notice_markdown(format!(
+		"Query completed in {query_time:?}:\n\n```rs\n{result:#?}\n```"
+	)))
+}
+
+#[admin_command]
+pub(super) async fn raw_del(&self, map: String, key: String) -> Result<RoomMessageEventContent> {
+	let map = self.services.db.get(&map)?;
+	let timer = Instant::now();
+	map.remove(&key);
+	let query_time = timer.elapsed();
+
+	Ok(RoomMessageEventContent::notice_markdown(format!(
+		"Operation completed in {query_time:?}"
+	)))
+}
+
+#[admin_command]
+pub(super) async fn raw_get(&self, map: String, key: String) -> Result<RoomMessageEventContent> {
+	let map = self.services.db.get(&map)?;
+	let timer = Instant::now();
+	let handle = map.get(&key).await?;
+	let query_time = timer.elapsed();
+	let result = String::from_utf8_lossy(&handle);
+
+	Ok(RoomMessageEventContent::notice_markdown(format!(
+		"Query completed in {query_time:?}:\n\n```rs\n{result:?}\n```"
+	)))
+}
+
+#[admin_command]
+pub(super) async fn raw_maps(&self) -> Result<RoomMessageEventContent> {
+	let list: Vec<_> = self.services.db.iter().map(at!(0)).copied().collect();
+
+	Ok(RoomMessageEventContent::notice_markdown(format!("{list:#?}")))
+}

src/admin/query/resolver.rs

@@ -1,7 +1,6 @@
-use std::fmt::Write;
-
 use clap::Subcommand;
 use conduwuit::{utils::time, Result};
+use futures::StreamExt;
 use ruma::{events::room::message::RoomMessageEventContent, OwnedServerName};
 
 use crate::{admin_command, admin_command_dispatch};
@@ -28,56 +27,48 @@ async fn destinations_cache(
 ) -> Result<RoomMessageEventContent> {
 	use service::resolver::cache::CachedDest;
 
-	let mut out = String::new();
-	writeln!(out, "| Server Name | Destination | Hostname | Expires |")?;
-	writeln!(out, "| ----------- | ----------- | -------- | ------- |")?;
-	let row = |(name, &CachedDest { ref dest, ref host, expire })| {
+	writeln!(self, "| Server Name | Destination | Hostname | Expires |").await?;
+	writeln!(self, "| ----------- | ----------- | -------- | ------- |").await?;
+
+	let mut destinations = self.services.resolver.cache.destinations().boxed();
+
+	while let Some((name, CachedDest { dest, host, expire })) = destinations.next().await {
+		if let Some(server_name) = server_name.as_ref() {
+			if name != server_name {
+				continue;
+			}
+		}
+
 		let expire = time::format(expire, "%+");
-		writeln!(out, "| {name} | {dest} | {host} | {expire} |").expect("wrote line");
-	};
-
-	let map = self
-		.services
-		.resolver
-		.cache
-		.destinations
-		.read()
-		.expect("locked");
-
-	if let Some(server_name) = server_name.as_ref() {
-		map.get_key_value(server_name).map(row);
-	} else {
-		map.iter().for_each(row);
+		self.write_str(&format!("| {name} | {dest} | {host} | {expire} |\n"))
+			.await?;
 	}
 
-	Ok(RoomMessageEventContent::notice_markdown(out))
+	Ok(RoomMessageEventContent::notice_plain(""))
 }
 
 #[admin_command]
 async fn overrides_cache(&self, server_name: Option<String>) -> Result<RoomMessageEventContent> {
 	use service::resolver::cache::CachedOverride;
 
-	let mut out = String::new();
-	writeln!(out, "| Server Name | IP  | Port | Expires |")?;
-	writeln!(out, "| ----------- | --- | ----:| ------- |")?;
-	let row = |(name, &CachedOverride { ref ips, port, expire })| {
+	writeln!(self, "| Server Name | IP  | Port | Expires | Overriding |").await?;
+	writeln!(self, "| ----------- | --- | ----:| ------- | ---------- |").await?;
+
+	let mut overrides = self.services.resolver.cache.overrides().boxed();
+
+	while let Some((name, CachedOverride { ips, port, expire, overriding })) =
+		overrides.next().await
+	{
+		if let Some(server_name) = server_name.as_ref() {
+			if name != server_name {
+				continue;
+			}
+		}
+
 		let expire = time::format(expire, "%+");
-		writeln!(out, "| {name} | {ips:?} | {port} | {expire} |").expect("wrote line");
-	};
-
-	let map = self
-		.services
-		.resolver
-		.cache
-		.overrides
-		.read()
-		.expect("locked");
-
-	if let Some(server_name) = server_name.as_ref() {
-		map.get_key_value(server_name).map(row);
-	} else {
-		map.iter().for_each(row);
+		self.write_str(&format!("| {name} | {ips:?} | {port} | {expire} | {overriding:?} |\n"))
+			.await?;
 	}
 
-	Ok(RoomMessageEventContent::notice_markdown(out))
+	Ok(RoomMessageEventContent::notice_plain(""))
 }

src/admin/query/room_alias.rs

@@ -1,7 +1,7 @@
 use clap::Subcommand;
 use conduwuit::Result;
 use futures::StreamExt;
-use ruma::{events::room::message::RoomMessageEventContent, RoomAliasId, RoomId};
+use ruma::{RoomAliasId, RoomId};
 
 use crate::Command;
 
@@ -24,10 +24,7 @@ pub(crate) enum RoomAliasCommand {
 }
 
 /// All the getters and iterators in src/database/key_value/rooms/alias.rs
-pub(super) async fn process(
-	subcommand: RoomAliasCommand,
-	context: &Command<'_>,
-) -> Result<RoomMessageEventContent> {
+pub(super) async fn process(subcommand: RoomAliasCommand, context: &Command<'_>) -> Result {
 	let services = context.services;
 
 	match subcommand {
@@ -36,9 +33,7 @@ pub(super) async fn process(
 			let results = services.rooms.alias.resolve_local_alias(&alias).await;
 			let query_time = timer.elapsed();
 
-			Ok(RoomMessageEventContent::notice_markdown(format!(
-				"Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```"
-			)))
+			write!(context, "Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```")
 		},
 		| RoomAliasCommand::LocalAliasesForRoom { room_id } => {
 			let timer = tokio::time::Instant::now();
@@ -51,9 +46,7 @@ pub(super) async fn process(
 				.await;
 			let query_time = timer.elapsed();
 
-			Ok(RoomMessageEventContent::notice_markdown(format!(
-				"Query completed in {query_time:?}:\n\n```rs\n{aliases:#?}\n```"
-			)))
+			write!(context, "Query completed in {query_time:?}:\n\n```rs\n{aliases:#?}\n```")
 		},
 		| RoomAliasCommand::AllLocalAliases => {
 			let timer = tokio::time::Instant::now();
@@ -66,9 +59,8 @@ pub(super) async fn process(
 				.await;
 			let query_time = timer.elapsed();
 
-			Ok(RoomMessageEventContent::notice_markdown(format!(
-				"Query completed in {query_time:?}:\n\n```rs\n{aliases:#?}\n```"
-			)))
+			write!(context, "Query completed in {query_time:?}:\n\n```rs\n{aliases:#?}\n```")
 		},
 	}
+	.await
 }

src/admin/query/room_state_cache.rs

@@ -1,5 +1,5 @@
 use clap::Subcommand;
-use conduwuit::Result;
+use conduwuit::{Error, Result};
 use futures::StreamExt;
 use ruma::{events::room::message::RoomMessageEventContent, RoomId, ServerName, UserId};
 
@@ -76,13 +76,10 @@ pub(crate) enum RoomStateCacheCommand {
 	},
 }
 
-pub(super) async fn process(
-	subcommand: RoomStateCacheCommand,
-	context: &Command<'_>,
-) -> Result<RoomMessageEventContent> {
+pub(super) async fn process(subcommand: RoomStateCacheCommand, context: &Command<'_>) -> Result {
 	let services = context.services;
 
-	match subcommand {
+	let c = match subcommand {
 		| RoomStateCacheCommand::ServerInRoom { server, room_id } => {
 			let timer = tokio::time::Instant::now();
 			let result = services
@@ -92,7 +89,7 @@ pub(super) async fn process(
 				.await;
 			let query_time = timer.elapsed();
 
-			Ok(RoomMessageEventContent::notice_markdown(format!(
+			Result::<_, Error>::Ok(RoomMessageEventContent::notice_markdown(format!(
 				"Query completed in {query_time:?}:\n\n```rs\n{result:#?}\n```"
 			)))
 		},
@@ -107,7 +104,7 @@ pub(super) async fn process(
 				.await;
 			let query_time = timer.elapsed();
 
-			Ok(RoomMessageEventContent::notice_markdown(format!(
+			Result::<_, Error>::Ok(RoomMessageEventContent::notice_markdown(format!(
 				"Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```"
 			)))
 		},
@@ -122,7 +119,7 @@ pub(super) async fn process(
 				.await;
 			let query_time = timer.elapsed();
 
-			Ok(RoomMessageEventContent::notice_markdown(format!(
+			Result::<_, Error>::Ok(RoomMessageEventContent::notice_markdown(format!(
 				"Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```"
 			)))
 		},
@@ -137,7 +134,7 @@ pub(super) async fn process(
 				.await;
 			let query_time = timer.elapsed();
 
-			Ok(RoomMessageEventContent::notice_markdown(format!(
+			Result::<_, Error>::Ok(RoomMessageEventContent::notice_markdown(format!(
 				"Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```"
 			)))
 		},
@@ -152,7 +149,7 @@ pub(super) async fn process(
 				.await;
 			let query_time = timer.elapsed();
 
-			Ok(RoomMessageEventContent::notice_markdown(format!(
+			Result::<_, Error>::Ok(RoomMessageEventContent::notice_markdown(format!(
 				"Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```"
 			)))
 		},
@@ -167,7 +164,7 @@ pub(super) async fn process(
 				.await;
 			let query_time = timer.elapsed();
 
-			Ok(RoomMessageEventContent::notice_markdown(format!(
+			Result::<_, Error>::Ok(RoomMessageEventContent::notice_markdown(format!(
 				"Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```"
 			)))
 		},
@@ -176,7 +173,7 @@ pub(super) async fn process(
 			let results = services.rooms.state_cache.room_joined_count(&room_id).await;
 			let query_time = timer.elapsed();
 
-			Ok(RoomMessageEventContent::notice_markdown(format!(
+			Result::<_, Error>::Ok(RoomMessageEventContent::notice_markdown(format!(
 				"Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```"
 			)))
 		},
@@ -189,7 +186,7 @@ pub(super) async fn process(
 				.await;
 			let query_time = timer.elapsed();
 
-			Ok(RoomMessageEventContent::notice_markdown(format!(
+			Result::<_, Error>::Ok(RoomMessageEventContent::notice_markdown(format!(
 				"Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```"
 			)))
 		},
@@ -204,7 +201,7 @@ pub(super) async fn process(
 				.await;
 			let query_time = timer.elapsed();
 
-			Ok(RoomMessageEventContent::notice_markdown(format!(
+			Result::<_, Error>::Ok(RoomMessageEventContent::notice_markdown(format!(
 				"Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```"
 			)))
 		},
@@ -219,7 +216,7 @@ pub(super) async fn process(
 				.await;
 			let query_time = timer.elapsed();
 
-			Ok(RoomMessageEventContent::notice_markdown(format!(
+			Result::<_, Error>::Ok(RoomMessageEventContent::notice_markdown(format!(
 				"Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```"
 			)))
 		},
@@ -232,7 +229,7 @@ pub(super) async fn process(
 				.await;
 			let query_time = timer.elapsed();
 
-			Ok(RoomMessageEventContent::notice_markdown(format!(
+			Result::<_, Error>::Ok(RoomMessageEventContent::notice_markdown(format!(
 				"Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```"
 			)))
 		},
@@ -245,7 +242,7 @@ pub(super) async fn process(
 				.await;
 			let query_time = timer.elapsed();
 
-			Ok(RoomMessageEventContent::notice_markdown(format!(
+			Result::<_, Error>::Ok(RoomMessageEventContent::notice_markdown(format!(
 				"Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```"
 			)))
 		},
@@ -260,7 +257,7 @@ pub(super) async fn process(
 				.await;
 			let query_time = timer.elapsed();
 
-			Ok(RoomMessageEventContent::notice_markdown(format!(
+			Result::<_, Error>::Ok(RoomMessageEventContent::notice_markdown(format!(
 				"Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```"
 			)))
 		},
@@ -274,7 +271,7 @@ pub(super) async fn process(
 				.await;
 			let query_time = timer.elapsed();
 
-			Ok(RoomMessageEventContent::notice_markdown(format!(
+			Result::<_, Error>::Ok(RoomMessageEventContent::notice_markdown(format!(
 				"Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```"
 			)))
 		},
@@ -288,7 +285,7 @@ pub(super) async fn process(
 				.await;
 			let query_time = timer.elapsed();
 
-			Ok(RoomMessageEventContent::notice_markdown(format!(
+			Result::<_, Error>::Ok(RoomMessageEventContent::notice_markdown(format!(
 				"Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```"
 			)))
 		},
@@ -301,9 +298,13 @@ pub(super) async fn process(
 				.await;
 			let query_time = timer.elapsed();
 
-			Ok(RoomMessageEventContent::notice_markdown(format!(
+			Result::<_, Error>::Ok(RoomMessageEventContent::notice_markdown(format!(
 				"Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```"
 			)))
 		},
-	}
+	}?;
+
+	context.write_str(c.body()).await?;
+
+	Ok(())
 }

src/admin/query/room_timeline.rs

@@ -0,0 +1,61 @@
+use clap::Subcommand;
+use conduwuit::{utils::stream::TryTools, PduCount, Result};
+use futures::TryStreamExt;
+use ruma::{events::room::message::RoomMessageEventContent, OwnedRoomOrAliasId};
+
+use crate::{admin_command, admin_command_dispatch};
+
+#[admin_command_dispatch]
+#[derive(Debug, Subcommand)]
+/// Query tables from database
+pub(crate) enum RoomTimelineCommand {
+	Pdus {
+		room_id: OwnedRoomOrAliasId,
+
+		from: Option<String>,
+
+		#[arg(short, long)]
+		limit: Option<usize>,
+	},
+
+	Last {
+		room_id: OwnedRoomOrAliasId,
+	},
+}
+
+#[admin_command]
+pub(super) async fn last(&self, room_id: OwnedRoomOrAliasId) -> Result<RoomMessageEventContent> {
+	let room_id = self.services.rooms.alias.resolve(&room_id).await?;
+
+	let result = self
+		.services
+		.rooms
+		.timeline
+		.last_timeline_count(None, &room_id)
+		.await?;
+
+	Ok(RoomMessageEventContent::notice_markdown(format!("{result:#?}")))
+}
+
+#[admin_command]
+pub(super) async fn pdus(
+	&self,
+	room_id: OwnedRoomOrAliasId,
+	from: Option<String>,
+	limit: Option<usize>,
+) -> Result<RoomMessageEventContent> {
+	let room_id = self.services.rooms.alias.resolve(&room_id).await?;
+
+	let from: Option<PduCount> = from.as_deref().map(str::parse).transpose()?;
+
+	let result: Vec<_> = self
+		.services
+		.rooms
+		.timeline
+		.pdus_rev(None, &room_id, from)
+		.try_take(limit.unwrap_or(3))
+		.try_collect()
+		.await?;
+
+	Ok(RoomMessageEventContent::notice_markdown(format!("{result:#?}")))
+}

src/admin/query/sending.rs

@@ -62,7 +62,14 @@ pub(crate) enum SendingCommand {
 }
 
 /// All the getters and iterators in key_value/sending.rs
-pub(super) async fn process(
+pub(super) async fn process(subcommand: SendingCommand, context: &Command<'_>) -> Result {
+	let c = reprocess(subcommand, context).await?;
+	context.write_str(c.body()).await?;
+	Ok(())
+}
+
+/// All the getters and iterators in key_value/sending.rs
+pub(super) async fn reprocess(
 	subcommand: SendingCommand,
 	context: &Command<'_>,
 ) -> Result<RoomMessageEventContent> {

src/admin/query/short.rs

@@ -0,0 +1,45 @@
+use clap::Subcommand;
+use conduwuit::Result;
+use ruma::{events::room::message::RoomMessageEventContent, OwnedEventId, OwnedRoomOrAliasId};
+
+use crate::{admin_command, admin_command_dispatch};
+
+#[admin_command_dispatch]
+#[derive(Debug, Subcommand)]
+/// Query tables from database
+pub(crate) enum ShortCommand {
+	ShortEventId {
+		event_id: OwnedEventId,
+	},
+
+	ShortRoomId {
+		room_id: OwnedRoomOrAliasId,
+	},
+}
+
+#[admin_command]
+pub(super) async fn short_event_id(
+	&self,
+	event_id: OwnedEventId,
+) -> Result<RoomMessageEventContent> {
+	let shortid = self
+		.services
+		.rooms
+		.short
+		.get_shorteventid(&event_id)
+		.await?;
+
+	Ok(RoomMessageEventContent::notice_markdown(format!("{shortid:#?}")))
+}
+
+#[admin_command]
+pub(super) async fn short_room_id(
+	&self,
+	room_id: OwnedRoomOrAliasId,
+) -> Result<RoomMessageEventContent> {
+	let room_id = self.services.rooms.alias.resolve(&room_id).await?;
+
+	let shortid = self.services.rooms.short.get_shortroomid(&room_id).await?;
+
+	Ok(RoomMessageEventContent::notice_markdown(format!("{shortid:#?}")))
+}

src/admin/query/users.rs

@@ -15,6 +15,8 @@ pub(crate) enum UsersCommand {
 
 	IterUsers,
 
+	IterUsers2,
+
 	PasswordHash {
 		user_id: OwnedUserId,
 	},
@@ -89,6 +91,33 @@ pub(crate) enum UsersCommand {
 		room_id: OwnedRoomId,
 		session_id: String,
 	},
+
+	GetSharedRooms {
+		user_a: OwnedUserId,
+		user_b: OwnedUserId,
+	},
+}
+
+#[admin_command]
+async fn get_shared_rooms(
+	&self,
+	user_a: OwnedUserId,
+	user_b: OwnedUserId,
+) -> Result<RoomMessageEventContent> {
+	let timer = tokio::time::Instant::now();
+	let result: Vec<_> = self
+		.services
+		.rooms
+		.state_cache
+		.get_shared_rooms(&user_a, &user_b)
+		.map(ToOwned::to_owned)
+		.collect()
+		.await;
+	let query_time = timer.elapsed();
+
+	Ok(RoomMessageEventContent::notice_markdown(format!(
+		"Query completed in {query_time:?}:\n\n```rs\n{result:#?}\n```"
+	)))
 }
 
 #[admin_command]
@@ -207,6 +236,23 @@ async fn iter_users(&self) -> Result<RoomMessageEventContent> {
 	)))
 }
 
+#[admin_command]
+async fn iter_users2(&self) -> Result<RoomMessageEventContent> {
+	let timer = tokio::time::Instant::now();
+	let result: Vec<_> = self.services.users.stream().collect().await;
+	let result: Vec<_> = result
+		.into_iter()
+		.map(ruma::UserId::as_bytes)
+		.map(String::from_utf8_lossy)
+		.collect();
+
+	let query_time = timer.elapsed();
+
+	Ok(RoomMessageEventContent::notice_markdown(format!(
+		"Query completed in {query_time:?}:\n\n```rs\n{result:?}\n```"
+	)))
+}
+
 #[admin_command]
 async fn count_users(&self) -> Result<RoomMessageEventContent> {
 	let timer = tokio::time::Instant::now();

src/admin/room/alias.rs

@@ -44,7 +44,14 @@ pub(crate) enum RoomAliasCommand {
 	},
 }
 
-pub(super) async fn process(
+pub(super) async fn process(command: RoomAliasCommand, context: &Command<'_>) -> Result {
+	let c = reprocess(command, context).await?;
+	context.write_str(c.body()).await?;
+
+	Ok(())
+}
+
+pub(super) async fn reprocess(
 	command: RoomAliasCommand,
 	context: &Command<'_>,
 ) -> Result<RoomMessageEventContent> {

src/admin/room/directory.rs

@@ -25,7 +25,13 @@ pub(crate) enum RoomDirectoryCommand {
 	},
 }
 
-pub(super) async fn process(
+pub(super) async fn process(command: RoomDirectoryCommand, context: &Command<'_>) -> Result {
+	let c = reprocess(command, context).await?;
+	context.write_str(c.body()).await?;
+	Ok(())
+}
+
+pub(super) async fn reprocess(
 	command: RoomDirectoryCommand,
 	context: &Command<'_>,
 ) -> Result<RoomMessageEventContent> {

src/admin/server/commands.rs

@@ -1,6 +1,6 @@
-use std::{fmt::Write, sync::Arc};
+use std::{fmt::Write, path::PathBuf, sync::Arc};
 
-use conduwuit::{info, utils::time, warn, Err, Result};
+use conduwuit::{info, utils::time, warn, Config, Err, Result};
 use ruma::events::room::message::RoomMessageEventContent;
 
 use crate::admin_command;
@@ -22,11 +22,27 @@ pub(super) async fn uptime(&self) -> Result<RoomMessageEventContent> {
 pub(super) async fn show_config(&self) -> Result<RoomMessageEventContent> {
 	// Construct and send the response
 	Ok(RoomMessageEventContent::text_markdown(format!(
-		"```\n{}\n```",
-		self.services.globals.config
+		"{}",
+		*self.services.server.config
 	)))
 }
 
+#[admin_command]
+pub(super) async fn reload_config(
+	&self,
+	path: Option<PathBuf>,
+) -> Result<RoomMessageEventContent> {
+	let path = path.as_deref().into_iter();
+	let config = Config::load(path).and_then(|raw| Config::new(&raw))?;
+	if config.server_name != self.services.server.config.server_name {
+		return Err!("You can't change the server name.");
+	}
+
+	let _old = self.services.server.config.update(config)?;
+
+	Ok(RoomMessageEventContent::text_plain("Successfully reconfigured."))
+}
+
 #[admin_command]
 pub(super) async fn list_features(
 	&self,

src/admin/server/mod.rs

@@ -1,5 +1,7 @@
 mod commands;
 
+use std::path::PathBuf;
+
 use clap::Subcommand;
 use conduwuit::Result;
 
@@ -14,6 +16,11 @@ pub(super) enum ServerCommand {
 	/// - Show configuration values
 	ShowConfig,
 
+	/// - Reload configuration values
+	ReloadConfig {
+		path: Option<PathBuf>,
+	},
+
 	/// - List the features built into the server
 	ListFeatures {
 		#[arg(short, long)]

src/admin/user/commands.rs

@@ -31,19 +31,21 @@ const BULK_JOIN_REASON: &str = "Bulk force joining this room as initiated by the
 
 #[admin_command]
 pub(super) async fn list_users(&self) -> Result<RoomMessageEventContent> {
-	let users = self
+	let users: Vec<_> = self
 		.services
 		.users
 		.list_local_users()
 		.map(ToString::to_string)
-		.collect::<Vec<_>>()
+		.collect()
 		.await;
 
 	let mut plain_msg = format!("Found {} local user account(s):\n```\n", users.len());
 	plain_msg += users.join("\n").as_str();
 	plain_msg += "\n```";
 
-	Ok(RoomMessageEventContent::notice_markdown(plain_msg))
+	self.write_str(plain_msg.as_str()).await?;
+
+	Ok(RoomMessageEventContent::text_plain(""))
 }
 
 #[admin_command]
@@ -81,12 +83,12 @@ pub(super) async fn create_user(
 	// content is set to the user's display name with a space before it
 	if !self
 		.services
-		.globals
+		.server
 		.config
 		.new_user_displayname_suffix
 		.is_empty()
 	{
-		write!(displayname, " {}", self.services.globals.config.new_user_displayname_suffix)
+		write!(displayname, " {}", self.services.server.config.new_user_displayname_suffix)
 			.expect("should be able to write to string buffer");
 	}
 
@@ -112,8 +114,8 @@ pub(super) async fn create_user(
 		)
 		.await?;
 
-	if !self.services.globals.config.auto_join_rooms.is_empty() {
-		for room in &self.services.globals.config.auto_join_rooms {
+	if !self.services.server.config.auto_join_rooms.is_empty() {
+		for room in &self.services.server.config.auto_join_rooms {
 			let Ok(room_id) = self.services.rooms.alias.resolve(room).await else {
 				error!(%user_id, "Failed to resolve room alias to room ID when attempting to auto join {room}, skipping");
 				continue;
@@ -912,29 +914,30 @@ pub(super) async fn redact_event(
 		self.services.globals.server_name()
 	);
 
-	let state_lock = self.services.rooms.state.mutex.lock(&room_id).await;
+	let redaction_event_id = {
+		let state_lock = self.services.rooms.state.mutex.lock(&room_id).await;
 
-	let redaction_event_id = self
-		.services
-		.rooms
-		.timeline
-		.build_and_append_pdu(
-			PduBuilder {
-				redacts: Some(event.event_id.clone()),
-				..PduBuilder::timeline(&RoomRedactionEventContent {
+		self.services
+			.rooms
+			.timeline
+			.build_and_append_pdu(
+				PduBuilder {
 					redacts: Some(event.event_id.clone()),
-					reason: Some(reason),
-				})
-			},
-			&sender_user,
-			&room_id,
-			&state_lock,
-		)
-		.await?;
+					..PduBuilder::timeline(&RoomRedactionEventContent {
+						redacts: Some(event.event_id.clone()),
+						reason: Some(reason),
+					})
+				},
+				&sender_user,
+				&room_id,
+				&state_lock,
+			)
+			.await?
+	};
 
-	drop(state_lock);
+	let out = format!("Successfully redacted event. Redaction event ID: {redaction_event_id}");
 
-	Ok(RoomMessageEventContent::text_plain(format!(
-		"Successfully redacted event. Redaction event ID: {redaction_event_id}"
-	)))
+	self.write_str(out.as_str()).await?;
+
+	Ok(RoomMessageEventContent::text_plain(""))
 }

src/api/Cargo.toml

@@ -50,7 +50,6 @@ http.workspace = true
 http-body-util.workspace = true
 hyper.workspace = true
 ipaddress.workspace = true
-jsonwebtoken.workspace = true
 log.workspace = true
 rand.workspace = true
 reqwest.workspace = true

src/api/client/account.rs

@@ -299,7 +299,7 @@ pub(crate) async fn register_route(
 	if !services.globals.new_user_displayname_suffix().is_empty()
 		&& body.appservice_info.is_none()
 	{
-		write!(displayname, " {}", services.globals.config.new_user_displayname_suffix)
+		write!(displayname, " {}", services.server.config.new_user_displayname_suffix)
 			.expect("should be able to write to string buffer");
 	}
 
@@ -365,7 +365,7 @@ pub(crate) async fn register_route(
 				 \"{device_display_name}\""
 			);
 
-			if services.globals.config.admin_room_notices {
+			if services.server.config.admin_room_notices {
 				services
 					.admin
 					.send_message(RoomMessageEventContent::notice_plain(format!(
@@ -378,7 +378,7 @@ pub(crate) async fn register_route(
 		} else {
 			info!("New user \"{user_id}\" registered on this server.");
 
-			if services.globals.config.admin_room_notices {
+			if services.server.config.admin_room_notices {
 				services
 					.admin
 					.send_message(RoomMessageEventContent::notice_plain(format!(
@@ -395,7 +395,7 @@ pub(crate) async fn register_route(
 		info!("New guest user \"{user_id}\" registered on this server.");
 
 		if !device_display_name.is_empty() {
-			if services.globals.config.admin_room_notices {
+			if services.server.config.admin_room_notices {
 				services
 					.admin
 					.send_message(RoomMessageEventContent::notice_plain(format!(
@@ -407,7 +407,7 @@ pub(crate) async fn register_route(
 			}
 		} else {
 			#[allow(clippy::collapsible_else_if)]
-			if services.globals.config.admin_room_notices {
+			if services.server.config.admin_room_notices {
 				services
 					.admin
 					.send_message(RoomMessageEventContent::notice_plain(format!(
@@ -438,10 +438,10 @@ pub(crate) async fn register_route(
 	}
 
 	if body.appservice_info.is_none()
-		&& !services.globals.config.auto_join_rooms.is_empty()
+		&& !services.server.config.auto_join_rooms.is_empty()
 		&& (services.globals.allow_guests_auto_join_rooms() || !is_guest)
 	{
-		for room in &services.globals.config.auto_join_rooms {
+		for room in &services.server.config.auto_join_rooms {
 			let Ok(room_id) = services.rooms.alias.resolve(room).await else {
 				error!(
 					"Failed to resolve room alias to room ID when attempting to auto join \
@@ -570,7 +570,7 @@ pub(crate) async fn change_password_route(
 
 	info!("User {sender_user} changed their password.");
 
-	if services.globals.config.admin_room_notices {
+	if services.server.config.admin_room_notices {
 		services
 			.admin
 			.send_message(RoomMessageEventContent::notice_plain(format!(
@@ -673,7 +673,7 @@ pub(crate) async fn deactivate_route(
 
 	info!("User {sender_user} deactivated their account.");
 
-	if services.globals.config.admin_room_notices {
+	if services.server.config.admin_room_notices {
 		services
 			.admin
 			.send_message(RoomMessageEventContent::notice_plain(format!(

src/api/client/capabilities.rs

@@ -1,6 +1,7 @@
 use std::collections::BTreeMap;
 
 use axum::extract::State;
+use conduwuit::{Result, Server};
 use ruma::{
 	api::client::discovery::get_capabilities::{
 		self, Capabilities, GetLoginTokenCapability, RoomVersionStability,
@@ -10,7 +11,7 @@ use ruma::{
 };
 use serde_json::json;
 
-use crate::{Result, Ruma};
+use crate::Ruma;
 
 /// # `GET /_matrix/client/v3/capabilities`
 ///
@@ -21,7 +22,7 @@ pub(crate) async fn get_capabilities_route(
 	_body: Ruma<get_capabilities::v3::Request>,
 ) -> Result<get_capabilities::v3::Response> {
 	let available: BTreeMap<RoomVersionId, RoomVersionStability> =
-		services.server.available_room_versions().collect();
+		Server::available_room_versions().collect();
 
 	let mut capabilities = Capabilities::default();
 	capabilities.room_versions = RoomVersionsCapability {
@@ -32,8 +33,9 @@ pub(crate) async fn get_capabilities_route(
 	// we do not implement 3PID stuff
 	capabilities.thirdparty_id_changes = ThirdPartyIdChangesCapability { enabled: false };
 
-	// we dont support generating tokens yet
-	capabilities.get_login_token = GetLoginTokenCapability { enabled: false };
+	capabilities.get_login_token = GetLoginTokenCapability {
+		enabled: services.server.config.login_via_existing_session,
+	};
 
 	// MSC4133 capability
 	capabilities

src/api/client/context.rs

@@ -1,24 +1,22 @@
-use std::iter::once;
-
 use axum::extract::State;
 use conduwuit::{
-	at, err, ref_at,
+	at, deref_at, err, ref_at,
 	utils::{
 		future::TryExtExt,
-		stream::{BroadbandExt, ReadyExt, WidebandExt},
+		stream::{BroadbandExt, ReadyExt, TryIgnore, WidebandExt},
 		IterStream,
 	},
-	Err, Result,
+	Err, PduEvent, Result,
 };
-use futures::{join, try_join, FutureExt, StreamExt, TryFutureExt};
-use ruma::{
-	api::client::{context::get_context, filter::LazyLoadOptions},
-	events::StateEventType,
-	OwnedEventId, UserId,
+use futures::{
+	future::{join, join3, try_join3, OptionFuture},
+	FutureExt, StreamExt, TryFutureExt,
 };
+use ruma::{api::client::context::get_context, events::StateEventType, OwnedEventId, UserId};
+use service::rooms::{lazy_loading, lazy_loading::Options};
 
 use crate::{
-	client::message::{event_filter, ignored_filter, update_lazy, visibility_filter, LazySet},
+	client::message::{event_filter, ignored_filter, lazy_loading_witness, visibility_filter},
 	Ruma,
 };
 
@@ -35,10 +33,10 @@ pub(crate) async fn get_context_route(
 	State(services): State<crate::State>,
 	body: Ruma<get_context::v3::Request>,
 ) -> Result<get_context::v3::Response> {
-	let filter = &body.filter;
 	let sender = body.sender();
-	let (sender_user, _) = sender;
+	let (sender_user, sender_device) = sender;
 	let room_id = &body.room_id;
+	let filter = &body.filter;
 
 	// Use limit or else 10, with maximum 100
 	let limit: usize = body
@@ -47,25 +45,13 @@ pub(crate) async fn get_context_route(
 		.unwrap_or(LIMIT_DEFAULT)
 		.min(LIMIT_MAX);
 
-	// some clients, at least element, seem to require knowledge of redundant
-	// members for "inline" profiles on the timeline to work properly
-	let lazy_load_enabled = matches!(filter.lazy_load_options, LazyLoadOptions::Enabled { .. });
-
-	let lazy_load_redundant = if let LazyLoadOptions::Enabled { include_redundant_members } =
-		filter.lazy_load_options
-	{
-		include_redundant_members
-	} else {
-		false
-	};
-
-	let base_token = services
+	let base_id = services
 		.rooms
 		.timeline
-		.get_pdu_count(&body.event_id)
+		.get_pdu_id(&body.event_id)
 		.map_err(|_| err!(Request(NotFound("Event not found."))));
 
-	let base_event = services
+	let base_pdu = services
 		.rooms
 		.timeline
 		.get_pdu(&body.event_id)
@@ -77,53 +63,69 @@ pub(crate) async fn get_context_route(
 		.user_can_see_event(sender_user, &body.room_id, &body.event_id)
 		.map(Ok);
 
-	let (base_token, base_event, visible) = try_join!(base_token, base_event, visible)?;
+	let (base_id, base_pdu, visible) = try_join3(base_id, base_pdu, visible).await?;
 
-	if base_event.room_id != body.room_id || base_event.event_id != body.event_id {
+	if base_pdu.room_id != body.room_id || base_pdu.event_id != body.event_id {
 		return Err!(Request(NotFound("Base event not found.")));
 	}
 
-	if !visible
-		|| ignored_filter(&services, (base_token, base_event.clone()), sender_user)
-			.await
-			.is_none()
-	{
+	if !visible {
 		return Err!(Request(Forbidden("You don't have permission to view this event.")));
 	}
 
-	let events_before =
-		services
-			.rooms
-			.timeline
-			.pdus_rev(Some(sender_user), room_id, Some(base_token));
+	let base_count = base_id.pdu_count();
+
+	let base_event = ignored_filter(&services, (base_count, base_pdu), sender_user);
+
+	let events_before = services
+		.rooms
+		.timeline
+		.pdus_rev(Some(sender_user), room_id, Some(base_count))
+		.ignore_err()
+		.ready_filter_map(|item| event_filter(item, filter))
+		.wide_filter_map(|item| ignored_filter(&services, item, sender_user))
+		.wide_filter_map(|item| visibility_filter(&services, item, sender_user))
+		.take(limit / 2)
+		.collect();
 
 	let events_after = services
 		.rooms
 		.timeline
-		.pdus(Some(sender_user), room_id, Some(base_token));
-
-	let (events_before, events_after) = try_join!(events_before, events_after)?;
-
-	let events_before = events_before
+		.pdus(Some(sender_user), room_id, Some(base_count))
+		.ignore_err()
 		.ready_filter_map(|item| event_filter(item, filter))
 		.wide_filter_map(|item| ignored_filter(&services, item, sender_user))
 		.wide_filter_map(|item| visibility_filter(&services, item, sender_user))
 		.take(limit / 2)
 		.collect();
 
-	let events_after = events_after
-		.ready_filter_map(|item| event_filter(item, filter))
-		.wide_filter_map(|item| ignored_filter(&services, item, sender_user))
-		.wide_filter_map(|item| visibility_filter(&services, item, sender_user))
-		.take(limit / 2)
-		.collect();
+	let (base_event, events_before, events_after): (_, Vec<_>, Vec<_>) =
+		join3(base_event, events_before, events_after).await;
 
-	let (events_before, events_after): (Vec<_>, Vec<_>) = join!(events_before, events_after);
+	let lazy_loading_context = lazy_loading::Context {
+		user_id: sender_user,
+		device_id: sender_device,
+		room_id,
+		token: Some(base_count.into_unsigned()),
+		options: Some(&filter.lazy_load_options),
+	};
+
+	let lazy_loading_witnessed: OptionFuture<_> = filter
+		.lazy_load_options
+		.is_enabled()
+		.then_some(
+			base_event
+				.iter()
+				.chain(events_before.iter())
+				.chain(events_after.iter()),
+		)
+		.map(|witnessed| lazy_loading_witness(&services, &lazy_loading_context, witnessed))
+		.into();
 
 	let state_at = events_after
 		.last()
 		.map(ref_at!(1))
-		.map_or(body.event_id.as_ref(), |e| e.event_id.as_ref());
+		.map_or(body.event_id.as_ref(), |pdu| pdu.event_id.as_ref());
 
 	let state_ids = services
 		.rooms
@@ -132,40 +134,32 @@ pub(crate) async fn get_context_route(
 		.or_else(|_| services.rooms.state.get_room_shortstatehash(room_id))
 		.and_then(|shortstatehash| services.rooms.state_accessor.state_full_ids(shortstatehash))
 		.map_err(|e| err!(Database("State not found: {e}")))
-		.await?;
+		.boxed();
 
-	let lazy = once(&(base_token, base_event.clone()))
-		.chain(events_before.iter())
-		.chain(events_after.iter())
-		.stream()
-		.fold(LazySet::new(), |lazy, item| {
-			update_lazy(&services, room_id, sender, lazy, item, lazy_load_redundant)
-		})
-		.await;
+	let (lazy_loading_witnessed, state_ids) = join(lazy_loading_witnessed, state_ids).await;
 
-	let lazy = &lazy;
-	let state: Vec<_> = state_ids
-		.iter()
-		.stream()
-		.broad_filter_map(|(shortstatekey, event_id)| {
-			services
-				.rooms
-				.short
-				.get_statekey_from_short(*shortstatekey)
-				.map_ok(move |(event_type, state_key)| (event_type, state_key, event_id))
-				.ok()
-		})
-		.ready_filter_map(|(event_type, state_key, event_id)| {
-			if !lazy_load_enabled || event_type != StateEventType::RoomMember {
-				return Some(event_id);
+	let state_ids = state_ids?;
+	let lazy_loading_witnessed = lazy_loading_witnessed.unwrap_or_default();
+	let shortstatekeys = state_ids.iter().stream().map(deref_at!(0));
+
+	let state: Vec<_> = services
+		.rooms
+		.short
+		.multi_get_statekey_from_short(shortstatekeys)
+		.zip(state_ids.iter().stream().map(at!(1)))
+		.ready_filter_map(|item| Some((item.0.ok()?, item.1)))
+		.ready_filter_map(|((event_type, state_key), event_id)| {
+			if filter.lazy_load_options.is_enabled()
+				&& event_type == StateEventType::RoomMember
+				&& state_key
+					.as_str()
+					.try_into()
+					.is_ok_and(|user_id: &UserId| !lazy_loading_witnessed.contains(user_id))
+			{
+				return None;
 			}
 
-			state_key
-				.as_str()
-				.try_into()
-				.ok()
-				.filter(|&user_id: &&UserId| lazy.contains(user_id))
-				.map(|_| event_id)
+			Some(event_id)
 		})
 		.broad_filter_map(|event_id: &OwnedEventId| {
 			services.rooms.timeline.get_pdu(event_id).ok()
@@ -175,19 +169,19 @@ pub(crate) async fn get_context_route(
 		.await;
 
 	Ok(get_context::v3::Response {
-		event: Some(base_event.to_room_event()),
+		event: base_event.map(at!(1)).as_ref().map(PduEvent::to_room_event),
 
 		start: events_before
 			.last()
 			.map(at!(0))
-			.or(Some(base_token))
+			.or(Some(base_count))
 			.as_ref()
 			.map(ToString::to_string),
 
 		end: events_after
 			.last()
 			.map(at!(0))
-			.or(Some(base_token))
+			.or(Some(base_count))
 			.as_ref()
 			.map(ToString::to_string),
 

src/api/client/directory.rs

@@ -152,7 +152,7 @@ pub(crate) async fn set_room_visibility_route(
 
 	match &body.visibility {
 		| room::Visibility::Public => {
-			if services.globals.config.lockdown_public_room_directory
+			if services.server.config.lockdown_public_room_directory
 				&& !services.users.is_admin(sender_user).await
 				&& body.appservice_info.is_none()
 			{
@@ -162,7 +162,7 @@ pub(crate) async fn set_room_visibility_route(
 					body.room_id
 				);
 
-				if services.globals.config.admin_room_notices {
+				if services.server.config.admin_room_notices {
 					services
 						.admin
 						.send_text(&format!(
@@ -181,7 +181,7 @@ pub(crate) async fn set_room_visibility_route(
 
 			services.rooms.directory.set_public(&body.room_id);
 
-			if services.globals.config.admin_room_notices {
+			if services.server.config.admin_room_notices {
 				services
 					.admin
 					.send_text(&format!(

src/api/client/media.rs

@@ -31,7 +31,7 @@ pub(crate) async fn get_media_config_route(
 	_body: Ruma<get_media_config::v1::Request>,
 ) -> Result<get_media_config::v1::Response> {
 	Ok(get_media_config::v1::Response {
-		upload_size: ruma_from_usize(services.globals.config.max_request_size),
+		upload_size: ruma_from_usize(services.server.config.max_request_size),
 	})
 }
 

src/api/client/media_legacy.rs

@@ -27,7 +27,7 @@ pub(crate) async fn get_media_config_legacy_route(
 	_body: Ruma<get_media_config::v3::Request>,
 ) -> Result<get_media_config::v3::Response> {
 	Ok(get_media_config::v3::Response {
-		upload_size: ruma_from_usize(services.globals.config.max_request_size),
+		upload_size: ruma_from_usize(services.server.config.max_request_size),
 	})
 }
 
@@ -50,7 +50,7 @@ pub(crate) async fn get_media_config_legacy_legacy_route(
 /// # `GET /_matrix/media/v3/preview_url`
 ///
 /// Returns URL preview.
-#[tracing::instrument(skip_all, fields(%client), name = "url_preview_legacy")]
+#[tracing::instrument(skip_all, fields(%client), name = "url_preview_legacy", level = "debug")]
 pub(crate) async fn get_media_preview_legacy_route(
 	State(services): State<crate::State>,
 	InsecureClientIp(client): InsecureClientIp,
@@ -131,7 +131,7 @@ pub(crate) async fn create_content_legacy_route(
 /// - Only redirects if `allow_redirect` is true
 /// - Uses client-provided `timeout_ms` if available, else defaults to 20
 ///   seconds
-#[tracing::instrument(skip_all, fields(%client), name = "media_get_legacy")]
+#[tracing::instrument(skip_all, fields(%client), name = "media_get_legacy", level = "debug")]
 pub(crate) async fn get_content_legacy_route(
 	State(services): State<crate::State>,
 	InsecureClientIp(client): InsecureClientIp,
@@ -197,7 +197,7 @@ pub(crate) async fn get_content_legacy_route(
 /// - Only redirects if `allow_redirect` is true
 /// - Uses client-provided `timeout_ms` if available, else defaults to 20
 ///   seconds
-#[tracing::instrument(skip_all, fields(%client), name = "media_get_legacy")]
+#[tracing::instrument(skip_all, fields(%client), name = "media_get_legacy", level = "debug")]
 pub(crate) async fn get_content_legacy_legacy_route(
 	State(services): State<crate::State>,
 	InsecureClientIp(client): InsecureClientIp,
@@ -216,7 +216,7 @@ pub(crate) async fn get_content_legacy_legacy_route(
 /// - Only redirects if `allow_redirect` is true
 /// - Uses client-provided `timeout_ms` if available, else defaults to 20
 ///   seconds
-#[tracing::instrument(skip_all, fields(%client), name = "media_get_legacy")]
+#[tracing::instrument(skip_all, fields(%client), name = "media_get_legacy", level = "debug")]
 pub(crate) async fn get_content_as_filename_legacy_route(
 	State(services): State<crate::State>,
 	InsecureClientIp(client): InsecureClientIp,
@@ -303,7 +303,7 @@ pub(crate) async fn get_content_as_filename_legacy_legacy_route(
 /// - Only redirects if `allow_redirect` is true
 /// - Uses client-provided `timeout_ms` if available, else defaults to 20
 ///   seconds
-#[tracing::instrument(skip_all, fields(%client), name = "media_thumbnail_get_legacy")]
+#[tracing::instrument(skip_all, fields(%client), name = "media_thumbnail_get_legacy", level = "debug")]
 pub(crate) async fn get_content_thumbnail_legacy_route(
 	State(services): State<crate::State>,
 	InsecureClientIp(client): InsecureClientIp,

src/api/client/membership.rs

@@ -1,4 +1,5 @@
 use std::{
+	borrow::Borrow,
 	collections::{BTreeMap, HashMap, HashSet},
 	net::IpAddr,
 	sync::Arc,
@@ -8,7 +9,7 @@ use axum::extract::State;
 use axum_client_ip::InsecureClientIp;
 use conduwuit::{
 	debug, debug_info, debug_warn, err, info,
-	pdu::{self, gen_event_id_canonical_json, PduBuilder},
+	pdu::{gen_event_id_canonical_json, PduBuilder},
 	result::FlatOk,
 	trace,
 	utils::{self, shuffle, IterStream, ReadyExt},
@@ -19,6 +20,7 @@ use ruma::{
 	api::{
 		client::{
 			error::ErrorKind,
+			knock::knock_room,
 			membership::{
 				ban_user, forget_room, get_member_events, invite_user, join_room_by_id,
 				join_room_by_id_or_alias,
@@ -37,11 +39,12 @@ use ruma::{
 		},
 		StateEventType,
 	},
-	state_res, CanonicalJsonObject, CanonicalJsonValue, OwnedRoomId, OwnedServerName,
-	OwnedUserId, RoomId, RoomVersionId, ServerName, UserId,
+	state_res, CanonicalJsonObject, CanonicalJsonValue, OwnedEventId, OwnedRoomId,
+	OwnedServerName, OwnedUserId, RoomId, RoomVersionId, ServerName, UserId,
 };
 use service::{
 	appservice::RegistrationInfo,
+	pdu::gen_event_id,
 	rooms::{state::RoomMutexGuard, state_compressor::HashSetCompressStateEvent},
 	Services,
 };
@@ -68,7 +71,7 @@ async fn banned_room_check(
 	if let Some(room_id) = room_id {
 		if services.rooms.metadata.is_banned(room_id).await
 			|| services
-				.globals
+				.server
 				.config
 				.forbidden_remote_server_names
 				.contains(&room_id.server_name().unwrap().to_owned())
@@ -78,12 +81,12 @@ async fn banned_room_check(
 				 attempted to join a banned room or banned room server name: {room_id}"
 			);
 
-			if services.globals.config.auto_deactivate_banned_room_attempts {
+			if services.server.config.auto_deactivate_banned_room_attempts {
 				warn!(
 					"Automatically deactivating user {user_id} due to attempted banned room join"
 				);
 
-				if services.globals.config.admin_room_notices {
+				if services.server.config.admin_room_notices {
 					services
 						.admin
 						.send_message(RoomMessageEventContent::text_plain(format!(
@@ -109,7 +112,7 @@ async fn banned_room_check(
 		}
 	} else if let Some(server_name) = server_name {
 		if services
-			.globals
+			.server
 			.config
 			.forbidden_remote_server_names
 			.contains(&server_name.to_owned())
@@ -119,12 +122,12 @@ async fn banned_room_check(
 				 name {server_name} that is globally forbidden. Rejecting.",
 			);
 
-			if services.globals.config.auto_deactivate_banned_room_attempts {
+			if services.server.config.auto_deactivate_banned_room_attempts {
 				warn!(
 					"Automatically deactivating user {user_id} due to attempted banned room join"
 				);
 
-				if services.globals.config.admin_room_notices {
+				if services.server.config.admin_room_notices {
 					services
 						.admin
 						.send_message(RoomMessageEventContent::text_plain(format!(
@@ -348,6 +351,116 @@ pub(crate) async fn join_room_by_id_or_alias_route(
 	Ok(join_room_by_id_or_alias::v3::Response { room_id: join_room_response.room_id })
 }
 
+/// # `POST /_matrix/client/*/knock/{roomIdOrAlias}`
+///
+/// Tries to knock the room to ask permission to join for the sender user.
+#[tracing::instrument(skip_all, fields(%client), name = "knock")]
+pub(crate) async fn knock_room_route(
+	State(services): State<crate::State>,
+	InsecureClientIp(client): InsecureClientIp,
+	body: Ruma<knock_room::v3::Request>,
+) -> Result<knock_room::v3::Response> {
+	let sender_user = body.sender_user.as_ref().expect("user is authenticated");
+	let body = body.body;
+
+	let (servers, room_id) = match OwnedRoomId::try_from(body.room_id_or_alias) {
+		| Ok(room_id) => {
+			banned_room_check(
+				&services,
+				sender_user,
+				Some(&room_id),
+				room_id.server_name(),
+				client,
+			)
+			.await?;
+
+			let mut servers = body.via.clone();
+			servers.extend(
+				services
+					.rooms
+					.state_cache
+					.servers_invite_via(&room_id)
+					.map(ToOwned::to_owned)
+					.collect::<Vec<_>>()
+					.await,
+			);
+
+			servers.extend(
+				services
+					.rooms
+					.state_cache
+					.invite_state(sender_user, &room_id)
+					.await
+					.unwrap_or_default()
+					.iter()
+					.filter_map(|event| event.get_field("sender").ok().flatten())
+					.filter_map(|sender: &str| UserId::parse(sender).ok())
+					.map(|user| user.server_name().to_owned()),
+			);
+
+			if let Some(server) = room_id.server_name() {
+				servers.push(server.to_owned());
+			}
+
+			servers.sort_unstable();
+			servers.dedup();
+			shuffle(&mut servers);
+
+			(servers, room_id)
+		},
+		| Err(room_alias) => {
+			let (room_id, mut servers) = services
+				.rooms
+				.alias
+				.resolve_alias(&room_alias, Some(body.via.clone()))
+				.await?;
+
+			banned_room_check(
+				&services,
+				sender_user,
+				Some(&room_id),
+				Some(room_alias.server_name()),
+				client,
+			)
+			.await?;
+
+			let addl_via_servers = services
+				.rooms
+				.state_cache
+				.servers_invite_via(&room_id)
+				.map(ToOwned::to_owned);
+
+			let addl_state_servers = services
+				.rooms
+				.state_cache
+				.invite_state(sender_user, &room_id)
+				.await
+				.unwrap_or_default();
+
+			let mut addl_servers: Vec<_> = addl_state_servers
+				.iter()
+				.map(|event| event.get_field("sender"))
+				.filter_map(FlatOk::flat_ok)
+				.map(|user: &UserId| user.server_name().to_owned())
+				.stream()
+				.chain(addl_via_servers)
+				.collect()
+				.await;
+
+			addl_servers.sort_unstable();
+			addl_servers.dedup();
+			shuffle(&mut addl_servers);
+			servers.append(&mut addl_servers);
+
+			(servers, room_id)
+		},
+	};
+
+	knock_room_by_id_helper(&services, sender_user, &room_id, body.reason.clone(), &servers)
+		.boxed()
+		.await
+}
+
 /// # `POST /_matrix/client/v3/rooms/{roomId}/leave`
 ///
 /// Tries to leave the sender user from a room.
@@ -403,6 +516,17 @@ pub(crate) async fn invite_user_route(
 			)));
 		}
 
+		if let Ok(target_user_membership) = services
+			.rooms
+			.state_accessor
+			.get_member(&body.room_id, user_id)
+			.await
+		{
+			if target_user_membership.membership == MembershipState::Ban {
+				return Err!(Request(Forbidden("User is banned from this room.")));
+			}
+		}
+
 		if recipient_ignored_by_sender {
 			// silently drop the invite to the recipient if they've been ignored by the
 			// sender, pretend it worked
@@ -439,6 +563,16 @@ pub(crate) async fn kick_user_route(
 		return Ok(kick_user::v3::Response::new());
 	};
 
+	if !matches!(
+		event.membership,
+		MembershipState::Invite | MembershipState::Knock | MembershipState::Join,
+	) {
+		return Err!(Request(Forbidden(
+			"Cannot kick a user who is not apart of the room (current membership: {})",
+			event.membership
+		)));
+	}
+
 	services
 		.rooms
 		.timeline
@@ -527,7 +661,7 @@ pub(crate) async fn unban_user_route(
 
 	if current_member_content.membership != MembershipState::Ban {
 		return Err!(Request(Forbidden(
-			"Cannot ban a user who is not banned (current membership: {})",
+			"Cannot unban a user who is not banned (current membership: {})",
 			current_member_content.membership
 		)));
 	}
@@ -852,7 +986,7 @@ async fn join_room_by_id_helper_remote(
 		.hash_and_sign_event(&mut join_event_stub, &room_version_id)?;
 
 	// Generate event id
-	let event_id = pdu::gen_event_id(&join_event_stub, &room_version_id)?;
+	let event_id = gen_event_id(&join_event_stub, &room_version_id)?;
 
 	// Add event_id back
 	join_event_stub
@@ -1020,7 +1154,7 @@ async fn join_room_by_id_helper_remote(
 	};
 
 	let auth_check = state_res::event_auth::auth_check(
-		&state_res::RoomVersion::new(&room_version_id).expect("room version is supported"),
+		&state_res::RoomVersion::new(&room_version_id)?,
 		&parsed_join_pdu,
 		None, // TODO: third party invite
 		|k, s| state_fetch(k, s.to_owned()),
@@ -1033,10 +1167,10 @@ async fn join_room_by_id_helper_remote(
 	}
 
 	info!("Compressing state from send_join");
-	let compressed = state
-		.iter()
-		.stream()
-		.then(|(&k, id)| services.rooms.state_compressor.compress_state_event(k, id))
+	let compressed: HashSet<_> = services
+		.rooms
+		.state_compressor
+		.compress_state_events(state.iter().map(|(ssk, eid)| (ssk, eid.borrow())))
 		.collect()
 		.await;
 
@@ -1272,7 +1406,7 @@ async fn join_room_by_id_helper_local(
 		.hash_and_sign_event(&mut join_event_stub, &room_version_id)?;
 
 	// Generate event id
-	let event_id = pdu::gen_event_id(&join_event_stub, &room_version_id)?;
+	let event_id = gen_event_id(&join_event_stub, &room_version_id)?;
 
 	// Add event_id back
 	join_event_stub
@@ -1314,6 +1448,7 @@ async fn join_room_by_id_helper_local(
 			.rooms
 			.event_handler
 			.handle_incoming_pdu(&remote_server, room_id, &signed_event_id, signed_value, true)
+			.boxed()
 			.await?;
 	} else {
 		return Err(error);
@@ -1381,6 +1516,7 @@ async fn make_join_request(
 				);
 				make_join_response_and_server =
 					Err!(BadServerResponse("No server available to assist in joining."));
+
 				return make_join_response_and_server;
 			}
 		}
@@ -1491,6 +1627,7 @@ pub(crate) async fn invite_helper(
 			.rooms
 			.event_handler
 			.handle_incoming_pdu(&origin, room_id, &event_id, value, true)
+			.boxed()
 			.await?
 			.ok_or_else(|| {
 				err!(Request(InvalidParam("Could not accept incoming PDU as timeline event.")))
@@ -1557,7 +1694,7 @@ pub async fn leave_all_rooms(services: &Services, user_id: &UserId) {
 	for room_id in all_rooms {
 		// ignore errors
 		if let Err(e) = leave_room(services, user_id, &room_id, None).await {
-			warn!(%room_id, %user_id, %e, "Failed to leave room");
+			warn!(%user_id, "Failed to leave {room_id} remotely: {e}");
 		}
 
 		services.rooms.state_cache.forget(&room_id, user_id);
@@ -1573,11 +1710,15 @@ pub async fn leave_room(
 	//use conduwuit::utils::stream::OptionStream;
 	use futures::TryFutureExt;
 
-	// Ask a remote server if we don't have this room
+	// Ask a remote server if we don't have this room and are not knocking on it
 	if !services
 		.rooms
 		.state_cache
 		.server_in_room(services.globals.server_name(), room_id)
+		.await && !services
+		.rooms
+		.state_cache
+		.is_knocked(user_id, room_id)
 		.await
 	{
 		if let Err(e) = remote_leave_room(services, user_id, room_id).await {
@@ -1589,7 +1730,8 @@ pub async fn leave_room(
 			.rooms
 			.state_cache
 			.invite_state(user_id, room_id)
-			.map_err(|_| services.rooms.state_cache.left_state(user_id, room_id))
+			.or_else(|_| services.rooms.state_cache.knock_state(user_id, room_id))
+			.or_else(|_| services.rooms.state_cache.left_state(user_id, room_id))
 			.await
 			.ok();
 
@@ -1671,13 +1813,6 @@ async fn remote_leave_room(
 	let mut make_leave_response_and_server =
 		Err!(BadServerResponse("No server available to assist in leaving."));
 
-	let invite_state = services
-		.rooms
-		.state_cache
-		.invite_state(user_id, room_id)
-		.await
-		.map_err(|_| err!(Request(BadState("User is not invited."))))?;
-
 	let mut servers: HashSet<OwnedServerName> = services
 		.rooms
 		.state_cache
@@ -1686,13 +1821,39 @@ async fn remote_leave_room(
 		.collect()
 		.await;
 
-	servers.extend(
-		invite_state
-			.iter()
-			.filter_map(|event| event.get_field("sender").ok().flatten())
-			.filter_map(|sender: &str| UserId::parse(sender).ok())
-			.map(|user| user.server_name().to_owned()),
-	);
+	if let Ok(invite_state) = services
+		.rooms
+		.state_cache
+		.invite_state(user_id, room_id)
+		.await
+	{
+		servers.extend(
+			invite_state
+				.iter()
+				.filter_map(|event| event.get_field("sender").ok().flatten())
+				.filter_map(|sender: &str| UserId::parse(sender).ok())
+				.map(|user| user.server_name().to_owned()),
+		);
+	} else if let Ok(knock_state) = services
+		.rooms
+		.state_cache
+		.knock_state(user_id, room_id)
+		.await
+	{
+		servers.extend(
+			knock_state
+				.iter()
+				.filter_map(|event| event.get_field("sender").ok().flatten())
+				.filter_map(|sender: &str| UserId::parse(sender).ok())
+				.filter_map(|sender| {
+					if !services.globals.user_is_local(sender) {
+						Some(sender.server_name().to_owned())
+					} else {
+						None
+					}
+				}),
+		);
+	}
 
 	if let Some(room_id_server_name) = room_id.server_name() {
 		servers.insert(room_id_server_name.to_owned());
@@ -1767,7 +1928,7 @@ async fn remote_leave_room(
 		.hash_and_sign_event(&mut leave_event_stub, &room_version_id)?;
 
 	// Generate event id
-	let event_id = pdu::gen_event_id(&leave_event_stub, &room_version_id)?;
+	let event_id = gen_event_id(&leave_event_stub, &room_version_id)?;
 
 	// Add event_id back
 	leave_event_stub
@@ -1793,3 +1954,514 @@ async fn remote_leave_room(
 
 	Ok(())
 }
+
+async fn knock_room_by_id_helper(
+	services: &Services,
+	sender_user: &UserId,
+	room_id: &RoomId,
+	reason: Option<String>,
+	servers: &[OwnedServerName],
+) -> Result<knock_room::v3::Response> {
+	let state_lock = services.rooms.state.mutex.lock(room_id).await;
+
+	if services
+		.rooms
+		.state_cache
+		.is_invited(sender_user, room_id)
+		.await
+	{
+		debug_warn!("{sender_user} is already invited in {room_id} but attempted to knock");
+		return Err!(Request(Forbidden(
+			"You cannot knock on a room you are already invited/accepted to."
+		)));
+	}
+
+	if services
+		.rooms
+		.state_cache
+		.is_joined(sender_user, room_id)
+		.await
+	{
+		debug_warn!("{sender_user} is already joined in {room_id} but attempted to knock");
+		return Err!(Request(Forbidden("You cannot knock on a room you are already joined in.")));
+	}
+
+	if services
+		.rooms
+		.state_cache
+		.is_knocked(sender_user, room_id)
+		.await
+	{
+		debug_warn!("{sender_user} is already knocked in {room_id}");
+		return Ok(knock_room::v3::Response { room_id: room_id.into() });
+	}
+
+	if let Ok(membership) = services
+		.rooms
+		.state_accessor
+		.get_member(room_id, sender_user)
+		.await
+	{
+		if membership.membership == MembershipState::Ban {
+			debug_warn!("{sender_user} is banned from {room_id} but attempted to knock");
+			return Err!(Request(Forbidden("You cannot knock on a room you are banned from.")));
+		}
+	}
+
+	let server_in_room = services
+		.rooms
+		.state_cache
+		.server_in_room(services.globals.server_name(), room_id)
+		.await;
+
+	let local_knock = server_in_room
+		|| servers.is_empty()
+		|| (servers.len() == 1 && services.globals.server_is_ours(&servers[0]));
+
+	if local_knock {
+		knock_room_helper_local(services, sender_user, room_id, reason, servers, state_lock)
+			.boxed()
+			.await?;
+	} else {
+		knock_room_helper_remote(services, sender_user, room_id, reason, servers, state_lock)
+			.boxed()
+			.await?;
+	}
+
+	Ok(knock_room::v3::Response::new(room_id.to_owned()))
+}
+
+async fn knock_room_helper_local(
+	services: &Services,
+	sender_user: &UserId,
+	room_id: &RoomId,
+	reason: Option<String>,
+	servers: &[OwnedServerName],
+	state_lock: RoomMutexGuard,
+) -> Result {
+	debug_info!("We can knock locally");
+
+	let room_version_id = services.rooms.state.get_room_version(room_id).await?;
+
+	if matches!(
+		room_version_id,
+		RoomVersionId::V1
+			| RoomVersionId::V2
+			| RoomVersionId::V3
+			| RoomVersionId::V4
+			| RoomVersionId::V5
+			| RoomVersionId::V6
+	) {
+		return Err!(Request(Forbidden("This room does not support knocking.")));
+	}
+
+	let content = RoomMemberEventContent {
+		displayname: services.users.displayname(sender_user).await.ok(),
+		avatar_url: services.users.avatar_url(sender_user).await.ok(),
+		blurhash: services.users.blurhash(sender_user).await.ok(),
+		reason: reason.clone(),
+		..RoomMemberEventContent::new(MembershipState::Knock)
+	};
+
+	// Try normal knock first
+	let Err(error) = services
+		.rooms
+		.timeline
+		.build_and_append_pdu(
+			PduBuilder::state(sender_user.to_string(), &content),
+			sender_user,
+			room_id,
+			&state_lock,
+		)
+		.await
+	else {
+		return Ok(());
+	};
+
+	if servers.is_empty() || (servers.len() == 1 && services.globals.server_is_ours(&servers[0]))
+	{
+		return Err(error);
+	}
+
+	warn!("We couldn't do the knock locally, maybe federation can help to satisfy the knock");
+
+	let (make_knock_response, remote_server) =
+		make_knock_request(services, sender_user, room_id, servers).await?;
+
+	info!("make_knock finished");
+
+	let room_version_id = make_knock_response.room_version;
+
+	if !services.server.supported_room_version(&room_version_id) {
+		return Err!(BadServerResponse(
+			"Remote room version {room_version_id} is not supported by conduwuit"
+		));
+	}
+
+	let mut knock_event_stub = serde_json::from_str::<CanonicalJsonObject>(
+		make_knock_response.event.get(),
+	)
+	.map_err(|e| {
+		err!(BadServerResponse("Invalid make_knock event json received from server: {e:?}"))
+	})?;
+
+	knock_event_stub.insert(
+		"origin".to_owned(),
+		CanonicalJsonValue::String(services.globals.server_name().as_str().to_owned()),
+	);
+	knock_event_stub.insert(
+		"origin_server_ts".to_owned(),
+		CanonicalJsonValue::Integer(
+			utils::millis_since_unix_epoch()
+				.try_into()
+				.expect("Timestamp is valid js_int value"),
+		),
+	);
+	knock_event_stub.insert(
+		"content".to_owned(),
+		to_canonical_value(RoomMemberEventContent {
+			displayname: services.users.displayname(sender_user).await.ok(),
+			avatar_url: services.users.avatar_url(sender_user).await.ok(),
+			blurhash: services.users.blurhash(sender_user).await.ok(),
+			reason,
+			..RoomMemberEventContent::new(MembershipState::Knock)
+		})
+		.expect("event is valid, we just created it"),
+	);
+
+	// In order to create a compatible ref hash (EventID) the `hashes` field needs
+	// to be present
+	services
+		.server_keys
+		.hash_and_sign_event(&mut knock_event_stub, &room_version_id)?;
+
+	// Generate event id
+	let event_id = gen_event_id(&knock_event_stub, &room_version_id)?;
+
+	// Add event_id
+	knock_event_stub
+		.insert("event_id".to_owned(), CanonicalJsonValue::String(event_id.clone().into()));
+
+	// It has enough fields to be called a proper event now
+	let knock_event = knock_event_stub;
+
+	info!("Asking {remote_server} for send_knock in room {room_id}");
+	let send_knock_request = federation::knock::send_knock::v1::Request {
+		room_id: room_id.to_owned(),
+		event_id: event_id.clone(),
+		pdu: services
+			.sending
+			.convert_to_outgoing_federation_event(knock_event.clone())
+			.await,
+	};
+
+	let send_knock_response = services
+		.sending
+		.send_federation_request(&remote_server, send_knock_request)
+		.await?;
+
+	info!("send_knock finished");
+
+	services
+		.rooms
+		.short
+		.get_or_create_shortroomid(room_id)
+		.await;
+
+	info!("Parsing knock event");
+
+	let parsed_knock_pdu = PduEvent::from_id_val(&event_id, knock_event.clone())
+		.map_err(|e| err!(BadServerResponse("Invalid knock event PDU: {e:?}")))?;
+
+	info!("Updating membership locally to knock state with provided stripped state events");
+	services
+		.rooms
+		.state_cache
+		.update_membership(
+			room_id,
+			sender_user,
+			parsed_knock_pdu
+				.get_content::<RoomMemberEventContent>()
+				.expect("we just created this"),
+			sender_user,
+			Some(send_knock_response.knock_room_state),
+			None,
+			false,
+		)
+		.await?;
+
+	info!("Appending room knock event locally");
+	services
+		.rooms
+		.timeline
+		.append_pdu(
+			&parsed_knock_pdu,
+			knock_event,
+			vec![(*parsed_knock_pdu.event_id).to_owned()],
+			&state_lock,
+		)
+		.await?;
+
+	Ok(())
+}
+
+async fn knock_room_helper_remote(
+	services: &Services,
+	sender_user: &UserId,
+	room_id: &RoomId,
+	reason: Option<String>,
+	servers: &[OwnedServerName],
+	state_lock: RoomMutexGuard,
+) -> Result {
+	info!("Knocking {room_id} over federation.");
+
+	let (make_knock_response, remote_server) =
+		make_knock_request(services, sender_user, room_id, servers).await?;
+
+	info!("make_knock finished");
+
+	let room_version_id = make_knock_response.room_version;
+
+	if !services.server.supported_room_version(&room_version_id) {
+		return Err!(BadServerResponse(
+			"Remote room version {room_version_id} is not supported by conduwuit"
+		));
+	}
+
+	let mut knock_event_stub: CanonicalJsonObject =
+		serde_json::from_str(make_knock_response.event.get()).map_err(|e| {
+			err!(BadServerResponse("Invalid make_knock event json received from server: {e:?}"))
+		})?;
+
+	knock_event_stub.insert(
+		"origin".to_owned(),
+		CanonicalJsonValue::String(services.globals.server_name().as_str().to_owned()),
+	);
+	knock_event_stub.insert(
+		"origin_server_ts".to_owned(),
+		CanonicalJsonValue::Integer(
+			utils::millis_since_unix_epoch()
+				.try_into()
+				.expect("Timestamp is valid js_int value"),
+		),
+	);
+	knock_event_stub.insert(
+		"content".to_owned(),
+		to_canonical_value(RoomMemberEventContent {
+			displayname: services.users.displayname(sender_user).await.ok(),
+			avatar_url: services.users.avatar_url(sender_user).await.ok(),
+			blurhash: services.users.blurhash(sender_user).await.ok(),
+			reason,
+			..RoomMemberEventContent::new(MembershipState::Knock)
+		})
+		.expect("event is valid, we just created it"),
+	);
+
+	// In order to create a compatible ref hash (EventID) the `hashes` field needs
+	// to be present
+	services
+		.server_keys
+		.hash_and_sign_event(&mut knock_event_stub, &room_version_id)?;
+
+	// Generate event id
+	let event_id = gen_event_id(&knock_event_stub, &room_version_id)?;
+
+	// Add event_id
+	knock_event_stub
+		.insert("event_id".to_owned(), CanonicalJsonValue::String(event_id.clone().into()));
+
+	// It has enough fields to be called a proper event now
+	let knock_event = knock_event_stub;
+
+	info!("Asking {remote_server} for send_knock in room {room_id}");
+	let send_knock_request = federation::knock::send_knock::v1::Request {
+		room_id: room_id.to_owned(),
+		event_id: event_id.clone(),
+		pdu: services
+			.sending
+			.convert_to_outgoing_federation_event(knock_event.clone())
+			.await,
+	};
+
+	let send_knock_response = services
+		.sending
+		.send_federation_request(&remote_server, send_knock_request)
+		.await?;
+
+	info!("send_knock finished");
+
+	services
+		.rooms
+		.short
+		.get_or_create_shortroomid(room_id)
+		.await;
+
+	info!("Parsing knock event");
+	let parsed_knock_pdu = PduEvent::from_id_val(&event_id, knock_event.clone())
+		.map_err(|e| err!(BadServerResponse("Invalid knock event PDU: {e:?}")))?;
+
+	info!("Going through send_knock response knock state events");
+	let state = send_knock_response
+		.knock_room_state
+		.iter()
+		.map(|event| serde_json::from_str::<CanonicalJsonObject>(event.clone().into_json().get()))
+		.filter_map(Result::ok);
+
+	let mut state_map: HashMap<u64, OwnedEventId> = HashMap::new();
+
+	for event in state {
+		let Some(state_key) = event.get("state_key") else {
+			debug_warn!("send_knock stripped state event missing state_key: {event:?}");
+			continue;
+		};
+		let Some(event_type) = event.get("type") else {
+			debug_warn!("send_knock stripped state event missing event type: {event:?}");
+			continue;
+		};
+
+		let Ok(state_key) = serde_json::from_value::<String>(state_key.clone().into()) else {
+			debug_warn!("send_knock stripped state event has invalid state_key: {event:?}");
+			continue;
+		};
+		let Ok(event_type) = serde_json::from_value::<StateEventType>(event_type.clone().into())
+		else {
+			debug_warn!("send_knock stripped state event has invalid event type: {event:?}");
+			continue;
+		};
+
+		let event_id = gen_event_id(&event, &room_version_id)?;
+		let shortstatekey = services
+			.rooms
+			.short
+			.get_or_create_shortstatekey(&event_type, &state_key)
+			.await;
+
+		services.rooms.outlier.add_pdu_outlier(&event_id, &event);
+		state_map.insert(shortstatekey, event_id.clone());
+	}
+
+	info!("Compressing state from send_knock");
+	let compressed: HashSet<_> = services
+		.rooms
+		.state_compressor
+		.compress_state_events(state_map.iter().map(|(ssk, eid)| (ssk, eid.borrow())))
+		.collect()
+		.await;
+
+	debug!("Saving compressed state");
+	let HashSetCompressStateEvent {
+		shortstatehash: statehash_before_knock,
+		added,
+		removed,
+	} = services
+		.rooms
+		.state_compressor
+		.save_state(room_id, Arc::new(compressed))
+		.await?;
+
+	debug!("Forcing state for new room");
+	services
+		.rooms
+		.state
+		.force_state(room_id, statehash_before_knock, added, removed, &state_lock)
+		.await?;
+
+	let statehash_after_knock = services
+		.rooms
+		.state
+		.append_to_state(&parsed_knock_pdu)
+		.await?;
+
+	info!("Updating membership locally to knock state with provided stripped state events");
+	services
+		.rooms
+		.state_cache
+		.update_membership(
+			room_id,
+			sender_user,
+			parsed_knock_pdu
+				.get_content::<RoomMemberEventContent>()
+				.expect("we just created this"),
+			sender_user,
+			Some(send_knock_response.knock_room_state),
+			None,
+			false,
+		)
+		.await?;
+
+	info!("Appending room knock event locally");
+	services
+		.rooms
+		.timeline
+		.append_pdu(
+			&parsed_knock_pdu,
+			knock_event,
+			vec![(*parsed_knock_pdu.event_id).to_owned()],
+			&state_lock,
+		)
+		.await?;
+
+	info!("Setting final room state for new room");
+	// We set the room state after inserting the pdu, so that we never have a moment
+	// in time where events in the current room state do not exist
+	services
+		.rooms
+		.state
+		.set_room_state(room_id, statehash_after_knock, &state_lock);
+
+	Ok(())
+}
+
+async fn make_knock_request(
+	services: &Services,
+	sender_user: &UserId,
+	room_id: &RoomId,
+	servers: &[OwnedServerName],
+) -> Result<(federation::knock::create_knock_event_template::v1::Response, OwnedServerName)> {
+	let mut make_knock_response_and_server =
+		Err!(BadServerResponse("No server available to assist in knocking."));
+
+	let mut make_knock_counter: usize = 0;
+
+	for remote_server in servers {
+		if services.globals.server_is_ours(remote_server) {
+			continue;
+		}
+
+		info!("Asking {remote_server} for make_knock ({make_knock_counter})");
+
+		let make_knock_response = services
+			.sending
+			.send_federation_request(
+				remote_server,
+				federation::knock::create_knock_event_template::v1::Request {
+					room_id: room_id.to_owned(),
+					user_id: sender_user.to_owned(),
+					ver: services.server.supported_room_versions().collect(),
+				},
+			)
+			.await;
+
+		trace!("make_knock response: {make_knock_response:?}");
+		make_knock_counter = make_knock_counter.saturating_add(1);
+
+		make_knock_response_and_server = make_knock_response.map(|r| (r, remote_server.clone()));
+
+		if make_knock_response_and_server.is_ok() {
+			break;
+		}
+
+		if make_knock_counter > 40 {
+			warn!(
+				"50 servers failed to provide valid make_knock response, assuming no server can \
+				 assist in knocking."
+			);
+			make_knock_response_and_server =
+				Err!(BadServerResponse("No server available to assist in knocking."));
+
+			return make_knock_response_and_server;
+		}
+	}
+
+	make_knock_response_and_server
+}

src/api/client/message.rs

@@ -1,16 +1,14 @@
-use std::collections::HashSet;
-
 use axum::extract::State;
 use conduwuit::{
 	at, is_equal_to,
 	utils::{
 		result::{FlatOk, LogErr},
-		stream::{BroadbandExt, WidebandExt},
+		stream::{BroadbandExt, TryIgnore, WidebandExt},
 		IterStream, ReadyExt,
 	},
 	Event, PduCount, Result,
 };
-use futures::{FutureExt, StreamExt};
+use futures::{future::OptionFuture, pin_mut, FutureExt, StreamExt};
 use ruma::{
 	api::{
 		client::{filter::RoomEventFilter, message::get_message_events},
@@ -18,14 +16,19 @@ use ruma::{
 	},
 	events::{AnyStateEvent, StateEventType, TimelineEventType, TimelineEventType::*},
 	serde::Raw,
-	DeviceId, OwnedUserId, RoomId, UserId,
+	RoomId, UserId,
+};
+use service::{
+	rooms::{
+		lazy_loading,
+		lazy_loading::{Options, Witness},
+		timeline::PdusIterItem,
+	},
+	Services,
 };
-use service::{rooms::timeline::PdusIterItem, Services};
 
 use crate::Ruma;
 
-pub(crate) type LazySet = HashSet<OwnedUserId>;
-
 /// list of safe and common non-state events to ignore if the user is ignored
 const IGNORED_MESSAGE_TYPES: &[TimelineEventType; 17] = &[
 	Audio,
@@ -84,13 +87,6 @@ pub(crate) async fn get_message_events_route(
 		.unwrap_or(LIMIT_DEFAULT)
 		.min(LIMIT_MAX);
 
-	services.rooms.lazy_loading.lazy_load_confirm_delivery(
-		sender_user,
-		sender_device,
-		room_id,
-		from,
-	);
-
 	if matches!(body.dir, Direction::Backward) {
 		services
 			.rooms
@@ -107,14 +103,14 @@ pub(crate) async fn get_message_events_route(
 			.rooms
 			.timeline
 			.pdus(Some(sender_user), room_id, Some(from))
-			.await?
+			.ignore_err()
 			.boxed(),
 
 		| Direction::Backward => services
 			.rooms
 			.timeline
 			.pdus_rev(Some(sender_user), room_id, Some(from))
-			.await?
+			.ignore_err()
 			.boxed(),
 	};
 
@@ -127,35 +123,34 @@ pub(crate) async fn get_message_events_route(
 		.collect()
 		.await;
 
-	let lazy = events
-		.iter()
-		.stream()
-		.fold(LazySet::new(), |lazy, item| {
-			update_lazy(&services, room_id, sender, lazy, item, false)
-		})
-		.await;
+	let lazy_loading_context = lazy_loading::Context {
+		user_id: sender_user,
+		device_id: sender_device,
+		room_id,
+		token: Some(from.into_unsigned()),
+		options: Some(&filter.lazy_load_options),
+	};
 
-	let state = lazy
-		.iter()
-		.stream()
-		.broad_filter_map(|user_id| get_member_event(&services, room_id, user_id))
+	let witness: OptionFuture<_> = filter
+		.lazy_load_options
+		.is_enabled()
+		.then(|| lazy_loading_witness(&services, &lazy_loading_context, events.iter()))
+		.into();
+
+	let state = witness
+		.map(Option::into_iter)
+		.map(|option| option.flat_map(Witness::into_iter))
+		.map(IterStream::stream)
+		.into_stream()
+		.flatten()
+		.broad_filter_map(|user_id| async move {
+			get_member_event(&services, room_id, &user_id).await
+		})
 		.collect()
 		.await;
 
 	let next_token = events.last().map(at!(0));
 
-	if !cfg!(feature = "element_hacks") {
-		if let Some(next_token) = next_token {
-			services.rooms.lazy_loading.lazy_load_mark_sent(
-				sender_user,
-				sender_device,
-				room_id,
-				lazy,
-				next_token,
-			);
-		}
-	}
-
 	let chunk = events
 		.into_iter()
 		.map(at!(1))
@@ -170,6 +165,52 @@ pub(crate) async fn get_message_events_route(
 	})
 }
 
+pub(crate) async fn lazy_loading_witness<'a, I>(
+	services: &Services,
+	lazy_loading_context: &lazy_loading::Context<'_>,
+	events: I,
+) -> Witness
+where
+	I: Iterator<Item = &'a PdusIterItem> + Clone + Send,
+{
+	let oldest = events
+		.clone()
+		.map(|(count, _)| count)
+		.copied()
+		.min()
+		.unwrap_or_else(PduCount::max);
+
+	let newest = events
+		.clone()
+		.map(|(count, _)| count)
+		.copied()
+		.max()
+		.unwrap_or_else(PduCount::max);
+
+	let receipts = services
+		.rooms
+		.read_receipt
+		.readreceipts_since(lazy_loading_context.room_id, oldest.into_unsigned());
+
+	pin_mut!(receipts);
+	let witness: Witness = events
+		.stream()
+		.map(|(_, pdu)| pdu.sender.clone())
+		.chain(
+			receipts
+				.ready_take_while(|(_, c, _)| *c <= newest.into_unsigned())
+				.map(|(user_id, ..)| user_id.to_owned()),
+		)
+		.collect()
+		.await;
+
+	services
+		.rooms
+		.lazy_loading
+		.witness_retain(witness, lazy_loading_context)
+		.await
+}
+
 async fn get_member_event(
 	services: &Services,
 	room_id: &RoomId,
@@ -184,42 +225,6 @@ async fn get_member_event(
 		.ok()
 }
 
-pub(crate) async fn update_lazy(
-	services: &Services,
-	room_id: &RoomId,
-	sender: (&UserId, &DeviceId),
-	mut lazy: LazySet,
-	item: &PdusIterItem,
-	force: bool,
-) -> LazySet {
-	let (_, event) = &item;
-	let (sender_user, sender_device) = sender;
-
-	/* TODO: Remove the "element_hacks" check when these are resolved:
-	 * https://github.com/vector-im/element-android/issues/3417
-	 * https://github.com/vector-im/element-web/issues/21034
-	 */
-	if force || cfg!(features = "element_hacks") {
-		lazy.insert(event.sender().into());
-		return lazy;
-	}
-
-	if lazy.contains(event.sender()) {
-		return lazy;
-	}
-
-	if !services
-		.rooms
-		.lazy_loading
-		.lazy_load_was_sent_before(sender_user, sender_device, room_id, event.sender())
-		.await
-	{
-		lazy.insert(event.sender().into());
-	}
-
-	lazy
-}
-
 pub(crate) async fn ignored_filter(
 	services: &Services,
 	item: PdusIterItem,

src/api/client/openid.rs

@@ -37,7 +37,7 @@ pub(crate) async fn create_openid_token_route(
 	Ok(account::request_openid_token::v3::Response {
 		access_token,
 		token_type: TokenType::Bearer,
-		matrix_server_name: services.globals.config.server_name.clone(),
+		matrix_server_name: services.server.config.server_name.clone(),
 		expires_in: Duration::from_secs(expires_in),
 	})
 }

src/api/client/presence.rs

@@ -82,14 +82,19 @@ pub(crate) async fn get_presence_route(
 			presence.content.status_msg
 		};
 
+		let last_active_ago = match presence.content.currently_active {
+			| Some(true) => None,
+			| _ => presence
+				.content
+				.last_active_ago
+				.map(|millis| Duration::from_millis(millis.into())),
+		};
+
 		Ok(get_presence::v3::Response {
 			// TODO: Should ruma just use the presenceeventcontent type here?
 			status_msg,
 			currently_active: presence.content.currently_active,
-			last_active_ago: presence
-				.content
-				.last_active_ago
-				.map(|millis| Duration::from_millis(millis.into())),
+			last_active_ago,
 			presence: presence.content.presence,
 		})
 	} else {

src/api/client/report.rs

@@ -50,7 +50,7 @@ pub(crate) async fn report_room_route(
 	if !services
 		.rooms
 		.state_cache
-		.server_in_room(&services.globals.config.server_name, &body.room_id)
+		.server_in_room(&services.server.config.server_name, &body.room_id)
 		.await
 	{
 		return Err!(Request(NotFound(

src/api/client/room/create.rs

@@ -71,7 +71,7 @@ pub(crate) async fn create_room_route(
 	let room_id: OwnedRoomId = if let Some(custom_room_id) = &body.room_id {
 		custom_room_id_check(&services, custom_room_id)?
 	} else {
-		RoomId::new(&services.globals.config.server_name)
+		RoomId::new(&services.server.config.server_name)
 	};
 
 	// check if room ID doesn't already exist instead of erroring on auth check
@@ -83,7 +83,7 @@ pub(crate) async fn create_room_route(
 	}
 
 	if body.visibility == room::Visibility::Public
-		&& services.globals.config.lockdown_public_room_directory
+		&& services.server.config.lockdown_public_room_directory
 		&& !services.users.is_admin(sender_user).await
 		&& body.appservice_info.is_none()
 	{
@@ -93,7 +93,7 @@ pub(crate) async fn create_room_route(
 			&room_id
 		);
 
-		if services.globals.config.admin_room_notices {
+		if services.server.config.admin_room_notices {
 			services
 				.admin
 				.send_text(&format!(
@@ -450,7 +450,7 @@ pub(crate) async fn create_room_route(
 	if body.visibility == room::Visibility::Public {
 		services.rooms.directory.set_public(&room_id);
 
-		if services.globals.config.admin_room_notices {
+		if services.server.config.admin_room_notices {
 			services
 				.admin
 				.send_text(&format!(

src/api/client/room/initial_sync.rs

@@ -1,6 +1,10 @@
 use axum::extract::State;
-use conduwuit::{at, utils::BoolExt, Err, Result};
-use futures::StreamExt;
+use conduwuit::{
+	at,
+	utils::{stream::TryTools, BoolExt},
+	Err, Result,
+};
+use futures::TryStreamExt;
 use ruma::api::client::room::initial_sync::v3::{PaginationChunk, Request, Response};
 
 use crate::Ruma;
@@ -27,10 +31,9 @@ pub(crate) async fn room_initial_sync_route(
 		.rooms
 		.timeline
 		.pdus_rev(None, room_id, None)
-		.await?
-		.take(limit)
-		.collect()
-		.await;
+		.try_take(limit)
+		.try_collect()
+		.await?;
 
 	let state: Vec<_> = services
 		.rooms

src/api/client/session.rs

@@ -1,3 +1,5 @@
+use std::time::Duration;
+
 use axum::extract::State;
 use axum_client_ip::InsecureClientIp;
 use conduwuit::{debug, err, info, utils::ReadyExt, warn, Err};
@@ -6,9 +8,10 @@ use ruma::{
 	api::client::{
 		error::ErrorKind,
 		session::{
+			get_login_token,
 			get_login_types::{
 				self,
-				v3::{ApplicationServiceLoginType, PasswordLoginType},
+				v3::{ApplicationServiceLoginType, PasswordLoginType, TokenLoginType},
 			},
 			login::{
 				self,
@@ -16,33 +19,31 @@ use ruma::{
 			},
 			logout, logout_all,
 		},
-		uiaa::UserIdentifier,
+		uiaa,
 	},
 	OwnedUserId, UserId,
 };
-use serde::Deserialize;
+use service::uiaa::SESSION_ID_LENGTH;
 
 use super::{DEVICE_ID_LENGTH, TOKEN_LENGTH};
 use crate::{utils, utils::hash, Error, Result, Ruma};
 
-#[derive(Debug, Deserialize)]
-struct Claims {
-	sub: String,
-	//exp: usize,
-}
-
 /// # `GET /_matrix/client/v3/login`
 ///
 /// Get the supported login types of this server. One of these should be used as
 /// the `type` field when logging in.
 #[tracing::instrument(skip_all, fields(%client), name = "login")]
 pub(crate) async fn get_login_types_route(
+	State(services): State<crate::State>,
 	InsecureClientIp(client): InsecureClientIp,
 	_body: Ruma<get_login_types::v3::Request>,
 ) -> Result<get_login_types::v3::Response> {
 	Ok(get_login_types::v3::Response::new(vec![
 		get_login_types::v3::LoginType::Password(PasswordLoginType::default()),
 		get_login_types::v3::LoginType::ApplicationService(ApplicationServiceLoginType::default()),
+		get_login_types::v3::LoginType::Token(TokenLoginType {
+			get_login_token: services.server.config.login_via_existing_session,
+		}),
 	]))
 }
 
@@ -77,7 +78,9 @@ pub(crate) async fn login_route(
 			..
 		}) => {
 			debug!("Got password login type");
-			let user_id = if let Some(UserIdentifier::UserIdOrLocalpart(user_id)) = identifier {
+			let user_id = if let Some(uiaa::UserIdentifier::UserIdOrLocalpart(user_id)) =
+				identifier
+			{
 				UserId::parse_with_server_name(
 					user_id.to_lowercase(),
 					services.globals.server_name(),
@@ -108,32 +111,10 @@ pub(crate) async fn login_route(
 		},
 		| login::v3::LoginInfo::Token(login::v3::Token { token }) => {
 			debug!("Got token login type");
-			if let Some(jwt_decoding_key) = services.globals.jwt_decoding_key() {
-				let token = jsonwebtoken::decode::<Claims>(
-					token,
-					jwt_decoding_key,
-					&jsonwebtoken::Validation::default(),
-				)
-				.map_err(|e| {
-					warn!("Failed to parse JWT token from user logging in: {e}");
-					Error::BadRequest(ErrorKind::InvalidUsername, "Token is invalid.")
-				})?;
-
-				let username = token.claims.sub.to_lowercase();
-
-				UserId::parse_with_server_name(username, services.globals.server_name()).map_err(
-					|e| {
-						err!(Request(InvalidUsername(debug_error!(
-							?e,
-							"Failed to parse login username"
-						))))
-					},
-				)?
-			} else {
-				return Err!(Request(Unknown(
-					"Token login is not supported (server has no jwt decoding key)."
-				)));
+			if !services.server.config.login_via_existing_session {
+				return Err!(Request(Unknown("Token login is not enabled.")));
 			}
+			services.users.find_from_login_token(token).await?
 		},
 		#[allow(deprecated)]
 		| login::v3::LoginInfo::ApplicationService(login::v3::ApplicationService {
@@ -141,21 +122,22 @@ pub(crate) async fn login_route(
 			user,
 		}) => {
 			debug!("Got appservice login type");
-			let user_id = if let Some(UserIdentifier::UserIdOrLocalpart(user_id)) = identifier {
-				UserId::parse_with_server_name(
-					user_id.to_lowercase(),
-					services.globals.server_name(),
-				)
-			} else if let Some(user) = user {
-				OwnedUserId::parse(user)
-			} else {
-				warn!("Bad login type: {:?}", &body.login_info);
-				return Err(Error::BadRequest(ErrorKind::forbidden(), "Bad login type."));
-			}
-			.map_err(|e| {
-				warn!("Failed to parse username from appservice logging in: {e}");
-				Error::BadRequest(ErrorKind::InvalidUsername, "Username is invalid.")
-			})?;
+			let user_id =
+				if let Some(uiaa::UserIdentifier::UserIdOrLocalpart(user_id)) = identifier {
+					UserId::parse_with_server_name(
+						user_id.to_lowercase(),
+						services.globals.server_name(),
+					)
+				} else if let Some(user) = user {
+					OwnedUserId::parse(user)
+				} else {
+					warn!("Bad login type: {:?}", &body.login_info);
+					return Err(Error::BadRequest(ErrorKind::forbidden(), "Bad login type."));
+				}
+				.map_err(|e| {
+					warn!("Failed to parse username from appservice logging in: {e}");
+					Error::BadRequest(ErrorKind::InvalidUsername, "Username is invalid.")
+				})?;
 
 			if let Some(ref info) = body.appservice_info {
 				if !info.is_user_match(&user_id) {
@@ -247,6 +229,67 @@ pub(crate) async fn login_route(
 	})
 }
 
+/// # `POST /_matrix/client/v1/login/get_token`
+///
+/// Allows a logged-in user to get a short-lived token which can be used
+/// to log in with the m.login.token flow.
+///
+/// <https://spec.matrix.org/v1.13/client-server-api/#post_matrixclientv1loginget_token>
+#[tracing::instrument(skip_all, fields(%client), name = "login_token")]
+pub(crate) async fn login_token_route(
+	State(services): State<crate::State>,
+	InsecureClientIp(client): InsecureClientIp,
+	body: Ruma<get_login_token::v1::Request>,
+) -> Result<get_login_token::v1::Response> {
+	if !services.server.config.login_via_existing_session {
+		return Err!(Request(Forbidden("Login via an existing session is not enabled")));
+	}
+
+	let sender_user = body.sender_user();
+	let sender_device = body.sender_device();
+
+	// This route SHOULD have UIA
+	// TODO: How do we make only UIA sessions that have not been used before valid?
+
+	let mut uiaainfo = uiaa::UiaaInfo {
+		flows: vec![uiaa::AuthFlow { stages: vec![uiaa::AuthType::Password] }],
+		completed: Vec::new(),
+		params: Box::default(),
+		session: None,
+		auth_error: None,
+	};
+
+	if let Some(auth) = &body.auth {
+		let (worked, uiaainfo) = services
+			.uiaa
+			.try_auth(sender_user, sender_device, auth, &uiaainfo)
+			.await?;
+
+		if !worked {
+			return Err(Error::Uiaa(uiaainfo));
+		}
+
+		// Success!
+	} else if let Some(json) = body.json_body.as_ref() {
+		uiaainfo.session = Some(utils::random_string(SESSION_ID_LENGTH));
+		services
+			.uiaa
+			.create(sender_user, sender_device, &uiaainfo, json);
+
+		return Err(Error::Uiaa(uiaainfo));
+	} else {
+		return Err!(Request(NotJson("No JSON body was sent when required.")));
+	}
+
+	let login_token = utils::random_string(TOKEN_LENGTH);
+	let expires_in = services.users.create_login_token(sender_user, &login_token);
+
+	Ok(get_login_token::v1::Response {
+		expires_in: Duration::from_millis(expires_in),
+		login_token,
+	})
+}
+
 /// # `POST /_matrix/client/v3/logout`
 ///
 /// Log out the current device.

src/api/client/sync/mod.rs

@@ -1,16 +1,31 @@
 mod v3;
 mod v4;
+mod v5;
 
 use conduwuit::{
-	utils::stream::{BroadbandExt, ReadyExt},
+	utils::{
+		stream::{BroadbandExt, ReadyExt, TryIgnore},
+		IterStream,
+	},
 	PduCount,
 };
-use futures::StreamExt;
-use ruma::{RoomId, UserId};
+use futures::{pin_mut, StreamExt};
+use ruma::{
+	directory::RoomTypeFilter,
+	events::TimelineEventType::{
+		self, Beacon, CallInvite, PollStart, RoomEncrypted, RoomMessage, Sticker,
+	},
+	RoomId, UserId,
+};
 
-pub(crate) use self::{v3::sync_events_route, v4::sync_events_v4_route};
+pub(crate) use self::{
+	v3::sync_events_route, v4::sync_events_v4_route, v5::sync_events_v5_route,
+};
 use crate::{service::Services, Error, PduEvent, Result};
 
+pub(crate) const DEFAULT_BUMP_TYPES: &[TimelineEventType; 6] =
+	&[CallInvite, PollStart, Beacon, RoomEncrypted, RoomMessage, Sticker];
+
 async fn load_timeline(
 	services: &Services,
 	sender_user: &UserId,
@@ -29,23 +44,19 @@ async fn load_timeline(
 		return Ok((Vec::new(), false));
 	}
 
-	let mut non_timeline_pdus = services
+	let non_timeline_pdus = services
 		.rooms
 		.timeline
 		.pdus_rev(Some(sender_user), room_id, None)
-		.await?
+		.ignore_err()
 		.ready_skip_while(|&(pducount, _)| pducount > next_batch.unwrap_or_else(PduCount::max))
 		.ready_take_while(|&(pducount, _)| pducount > roomsincecount);
 
 	// Take the last events for the timeline
-	let timeline_pdus: Vec<_> = non_timeline_pdus
-		.by_ref()
-		.take(limit)
-		.collect::<Vec<_>>()
-		.await
-		.into_iter()
-		.rev()
-		.collect();
+	pin_mut!(non_timeline_pdus);
+	let timeline_pdus: Vec<_> = non_timeline_pdus.by_ref().take(limit).collect().await;
+
+	let timeline_pdus: Vec<_> = timeline_pdus.into_iter().rev().collect();
 
 	// They /sync response doesn't always return all messages, so we say the output
 	// is limited unless there are events in non_timeline_pdus
@@ -73,3 +84,33 @@ async fn share_encrypted_room(
 		})
 		.await
 }
+
+pub(crate) async fn filter_rooms<'a>(
+	services: &Services,
+	rooms: &[&'a RoomId],
+	filter: &[RoomTypeFilter],
+	negate: bool,
+) -> Vec<&'a RoomId> {
+	rooms
+		.iter()
+		.stream()
+		.filter_map(|r| async move {
+			let room_type = services.rooms.state_accessor.get_room_type(r).await;
+
+			if room_type.as_ref().is_err_and(|e| !e.is_not_found()) {
+				return None;
+			}
+
+			let room_type_filter = RoomTypeFilter::from(room_type.ok());
+
+			let include = if negate {
+				!filter.contains(&room_type_filter)
+			} else {
+				filter.is_empty() || filter.contains(&room_type_filter)
+			};
+
+			include.then_some(r)
+		})
+		.collect()
+		.await
+}

src/api/client/sync/v3.rs

@@ -6,9 +6,9 @@ use std::{
 
 use axum::extract::State;
 use conduwuit::{
-	at, err, error, extract_variant, is_equal_to, is_false,
+	at, err, error, extract_variant, is_equal_to,
 	pdu::EventHash,
-	result::{FlatOk, LogDebugErr},
+	result::FlatOk,
 	utils::{
 		self,
 		future::OptionExt,
@@ -19,22 +19,26 @@ use conduwuit::{
 	Error, PduCount, PduEvent, Result,
 };
 use conduwuit_service::{
-	rooms::short::{ShortStateHash, ShortStateKey},
+	rooms::{
+		lazy_loading,
+		lazy_loading::{Options, Witness},
+		short::ShortStateHash,
+	},
 	Services,
 };
 use futures::{
-	future::{join, join3, join4, join5, try_join, try_join3, OptionFuture},
+	future::{join, join3, join4, join5, try_join, try_join4, OptionFuture},
 	FutureExt, StreamExt, TryFutureExt,
 };
 use ruma::{
 	api::client::{
-		filter::{FilterDefinition, LazyLoadOptions},
+		filter::FilterDefinition,
 		sync::sync_events::{
 			self,
 			v3::{
 				Ephemeral, Filter, GlobalAccountData, InviteState, InvitedRoom, JoinedRoom,
-				LeftRoom, Presence, RoomAccountData, RoomSummary, Rooms, State as RoomState,
-				Timeline, ToDevice,
+				KnockState, KnockedRoom, LeftRoom, Presence, RoomAccountData, RoomSummary, Rooms,
+				State as RoomState, Timeline, ToDevice,
 			},
 			DeviceLists, UnreadNotificationsCount,
 		},
@@ -124,10 +128,42 @@ pub(crate) async fn sync_events_route(
 	// Setup watchers, so if there's no response, we can wait for them
 	let watcher = services.sync.watch(sender_user, sender_device);
 
-	let next_batch = services.globals.current_count()?;
-	let next_batch_string = next_batch.to_string();
+	let response = build_sync_events(&services, &body).await?;
+	if body.body.full_state
+		|| !(response.rooms.is_empty()
+			&& response.presence.is_empty()
+			&& response.account_data.is_empty()
+			&& response.device_lists.is_empty()
+			&& response.to_device.is_empty())
+	{
+		return Ok(response);
+	}
 
-	// Load filter
+	// Hang a few seconds so requests are not spammed
+	// Stop hanging if new info arrives
+	let default = Duration::from_secs(30);
+	let duration = cmp::min(body.body.timeout.unwrap_or(default), default);
+	_ = tokio::time::timeout(duration, watcher).await;
+
+	// Retry returning data
+	build_sync_events(&services, &body).await
+}
+
+pub(crate) async fn build_sync_events(
+	services: &Services,
+	body: &Ruma<sync_events::v3::Request>,
+) -> Result<sync_events::v3::Response, RumaResponse<UiaaResponse>> {
+	let (sender_user, sender_device) = body.sender();
+
+	let next_batch = services.globals.current_count()?;
+	let since = body
+		.body
+		.since
+		.as_ref()
+		.and_then(|string| string.parse().ok())
+		.unwrap_or(0);
+
+	let full_state = body.body.full_state;
 	let filter = match body.body.filter.as_ref() {
 		| None => FilterDefinition::default(),
 		| Some(Filter::FilterDefinition(ref filter)) => filter.clone(),
@@ -138,24 +174,6 @@ pub(crate) async fn sync_events_route(
 			.unwrap_or_default(),
 	};
 
-	// some clients, at least element, seem to require knowledge of redundant
-	// members for "inline" profiles on the timeline to work properly
-	let (lazy_load_enabled, lazy_load_send_redundant) = match filter.room.state.lazy_load_options
-	{
-		| LazyLoadOptions::Enabled { include_redundant_members } =>
-			(true, include_redundant_members),
-		| LazyLoadOptions::Disabled => (false, cfg!(feature = "element_hacks")),
-	};
-
-	let full_state = body.body.full_state;
-
-	let since = body
-		.body
-		.since
-		.as_ref()
-		.and_then(|string| string.parse().ok())
-		.unwrap_or(0);
-
 	let joined_rooms = services
 		.rooms
 		.state_cache
@@ -163,15 +181,14 @@ pub(crate) async fn sync_events_route(
 		.map(ToOwned::to_owned)
 		.broad_filter_map(|room_id| {
 			load_joined_room(
-				&services,
+				services,
 				sender_user,
 				sender_device,
 				room_id.clone(),
 				since,
 				next_batch,
-				lazy_load_enabled,
-				lazy_load_send_redundant,
 				full_state,
+				&filter,
 			)
 			.map_ok(move |(joined_room, dlu, jeu)| (room_id, joined_room, dlu, jeu))
 			.ok()
@@ -196,13 +213,13 @@ pub(crate) async fn sync_events_route(
 		.rooms_left(sender_user)
 		.broad_filter_map(|(room_id, _)| {
 			handle_left_room(
-				&services,
+				services,
 				since,
 				room_id.clone(),
 				sender_user,
-				&next_batch_string,
+				next_batch,
 				full_state,
-				lazy_load_enabled,
+				&filter,
 			)
 			.map_ok(move |left_room| (room_id, left_room))
 			.ok()
@@ -215,10 +232,6 @@ pub(crate) async fn sync_events_route(
 		.state_cache
 		.rooms_invited(sender_user)
 		.fold_default(|mut invited_rooms: BTreeMap<_, _>, (room_id, invite_state)| async move {
-			// Get and drop the lock to wait for remaining operations to finish
-			let insert_lock = services.rooms.timeline.mutex_insert.lock(&room_id).await;
-			drop(insert_lock);
-
 			let invite_count = services
 				.rooms
 				.state_cache
@@ -239,10 +252,35 @@ pub(crate) async fn sync_events_route(
 			invited_rooms
 		});
 
+	let knocked_rooms = services
+		.rooms
+		.state_cache
+		.rooms_knocked(sender_user)
+		.fold_default(|mut knocked_rooms: BTreeMap<_, _>, (room_id, knock_state)| async move {
+			let knock_count = services
+				.rooms
+				.state_cache
+				.get_knock_count(&room_id, sender_user)
+				.await
+				.ok();
+
+			// Knocked before last sync
+			if Some(since) >= knock_count {
+				return knocked_rooms;
+			}
+
+			let knocked_room = KnockedRoom {
+				knock_state: KnockState { events: knock_state },
+			};
+
+			knocked_rooms.insert(room_id, knocked_room);
+			knocked_rooms
+		});
+
 	let presence_updates: OptionFuture<_> = services
 		.globals
 		.allow_local_presence()
-		.then(|| process_presence_updates(&services, since, sender_user))
+		.then(|| process_presence_updates(services, since, sender_user))
 		.into();
 
 	let account_data = services
@@ -273,7 +311,7 @@ pub(crate) async fn sync_events_route(
 			.users
 			.remove_to_device_events(sender_user, sender_device, since);
 
-	let rooms = join3(joined_rooms, left_rooms, invited_rooms);
+	let rooms = join4(joined_rooms, left_rooms, invited_rooms, knocked_rooms);
 	let ephemeral = join3(remove_to_device_events, to_device_events, presence_updates);
 	let top = join5(account_data, ephemeral, device_one_time_keys_count, keys_changed, rooms)
 		.boxed()
@@ -281,7 +319,7 @@ pub(crate) async fn sync_events_route(
 
 	let (account_data, ephemeral, device_one_time_keys_count, keys_changed, rooms) = top;
 	let ((), to_device_events, presence_updates) = ephemeral;
-	let (joined_rooms, left_rooms, invited_rooms) = rooms;
+	let (joined_rooms, left_rooms, invited_rooms, knocked_rooms) = rooms;
 	let (joined_rooms, mut device_list_updates, left_encrypted_users) = joined_rooms;
 	device_list_updates.extend(keys_changed);
 
@@ -292,7 +330,7 @@ pub(crate) async fn sync_events_route(
 		.stream()
 		.broad_filter_map(|user_id| async move {
 			let no_shared_encrypted_room =
-				!share_encrypted_room(&services, sender_user, &user_id, None).await;
+				!share_encrypted_room(services, sender_user, &user_id, None).await;
 			no_shared_encrypted_room.then_some(user_id)
 		})
 		.ready_fold(HashSet::new(), |mut device_list_left, user_id| {
@@ -310,7 +348,7 @@ pub(crate) async fn sync_events_route(
 		device_one_time_keys_count,
 		// Fallback keys are not yet supported
 		device_unused_fallback_key_types: None,
-		next_batch: next_batch_string,
+		next_batch: next_batch.to_string(),
 		presence: Presence {
 			events: presence_updates
 				.unwrap_or_default()
@@ -322,26 +360,11 @@ pub(crate) async fn sync_events_route(
 			leave: left_rooms,
 			join: joined_rooms,
 			invite: invited_rooms,
-			knock: BTreeMap::new(), // TODO
+			knock: knocked_rooms,
 		},
 		to_device: ToDevice { events: to_device_events },
 	};
 
-	// TODO: Retry the endpoint instead of returning
-	if !full_state
-		&& response.rooms.is_empty()
-		&& response.presence.is_empty()
-		&& response.account_data.is_empty()
-		&& response.device_lists.is_empty()
-		&& response.to_device.is_empty()
-	{
-		// Hang a few seconds so requests are not spammed
-		// Stop hanging if new info arrives
-		let default = Duration::from_secs(30);
-		let duration = cmp::min(body.body.timeout.unwrap_or(default), default);
-		_ = tokio::time::timeout(duration, watcher).await;
-	}
-
 	Ok(response)
 }
 
@@ -370,7 +393,13 @@ async fn process_presence_updates(
 		.ready_fold(PresenceUpdates::new(), |mut updates, (user_id, event)| {
 			match updates.entry(user_id.into()) {
 				| Entry::Vacant(slot) => {
-					slot.insert(event);
+					let mut new_event = event;
+					new_event.content.last_active_ago = match new_event.content.currently_active {
+						| Some(true) => None,
+						| _ => new_event.content.last_active_ago,
+					};
+
+					slot.insert(new_event);
 				},
 				| Entry::Occupied(mut slot) => {
 					let curr_event = slot.get_mut();
@@ -382,8 +411,6 @@ async fn process_presence_updates(
 					curr_content.status_msg = new_content
 						.status_msg
 						.or_else(|| curr_content.status_msg.take());
-					curr_content.last_active_ago =
-						new_content.last_active_ago.or(curr_content.last_active_ago);
 					curr_content.displayname = new_content
 						.displayname
 						.or_else(|| curr_content.displayname.take());
@@ -393,6 +420,10 @@ async fn process_presence_updates(
 					curr_content.currently_active = new_content
 						.currently_active
 						.or(curr_content.currently_active);
+					curr_content.last_active_ago = match curr_content.currently_active {
+						| Some(true) => None,
+						| _ => new_content.last_active_ago.or(curr_content.last_active_ago),
+					};
 				},
 			};
 
@@ -408,7 +439,6 @@ async fn process_presence_updates(
 	fields(
 		room_id = %room_id,
 		full = %full_state,
-		ll = %lazy_load_enabled,
 	),
 )]
 #[allow(clippy::too_many_arguments)]
@@ -417,14 +447,10 @@ async fn handle_left_room(
 	since: u64,
 	ref room_id: OwnedRoomId,
 	sender_user: &UserId,
-	next_batch_string: &str,
+	next_batch: u64,
 	full_state: bool,
-	lazy_load_enabled: bool,
+	filter: &FilterDefinition,
 ) -> Result<Option<LeftRoom>> {
-	// Get and drop the lock to wait for remaining operations to finish
-	let insert_lock = services.rooms.timeline.mutex_insert.lock(room_id).await;
-	drop(insert_lock);
-
 	let left_count = services
 		.rooms
 		.state_cache
@@ -466,7 +492,7 @@ async fn handle_left_room(
 			account_data: RoomAccountData { events: Vec::new() },
 			timeline: Timeline {
 				limited: false,
-				prev_batch: Some(next_batch_string.to_owned()),
+				prev_batch: Some(next_batch.to_string()),
 				events: Vec::new(),
 			},
 			state: RoomState {
@@ -530,28 +556,32 @@ async fn handle_left_room(
 				.get_statekey_from_short(shortstatekey)
 				.await?;
 
-			// TODO: Delete "element_hacks" when this is resolved: https://github.com/vector-im/element-web/issues/22565
-			if !lazy_load_enabled
-				|| event_type != StateEventType::RoomMember
-				|| full_state
-				|| (cfg!(feature = "element_hacks") && *sender_user == state_key)
+			if filter.room.state.lazy_load_options.is_enabled()
+				&& event_type == StateEventType::RoomMember
+				&& !full_state
+				&& state_key
+					.as_str()
+					.try_into()
+					.is_ok_and(|user_id: &UserId| sender_user != user_id)
 			{
-				let Ok(pdu) = services.rooms.timeline.get_pdu(&event_id).await else {
-					error!("Pdu in state not found: {event_id}");
-					continue;
-				};
-
-				left_state_events.push(pdu.to_sync_state_event());
+				continue;
 			}
+
+			let Ok(pdu) = services.rooms.timeline.get_pdu(&event_id).await else {
+				error!("Pdu in state not found: {event_id}");
+				continue;
+			};
+
+			left_state_events.push(pdu.to_sync_state_event());
 		}
 	}
 
 	Ok(Some(LeftRoom {
 		account_data: RoomAccountData { events: Vec::new() },
 		timeline: Timeline {
-			limited: true, /* TODO: support left timeline events so we dont need to set this to
-			                * true */
-			prev_batch: Some(next_batch_string.to_owned()),
+			// TODO: support left timeline events so we dont need to set limited to true
+			limited: true,
+			prev_batch: Some(next_batch.to_string()),
 			events: Vec::new(), // and so we dont need to set this to empty vec
 		},
 		state: RoomState { events: left_state_events },
@@ -574,15 +604,9 @@ async fn load_joined_room(
 	ref room_id: OwnedRoomId,
 	since: u64,
 	next_batch: u64,
-	lazy_load_enabled: bool,
-	lazy_load_send_redundant: bool,
 	full_state: bool,
+	filter: &FilterDefinition,
 ) -> Result<(JoinedRoom, HashSet<OwnedUserId>, HashSet<OwnedUserId>)> {
-	// Get and drop the lock to wait for remaining operations to finish
-	// This will make sure the we have all events until next_batch
-	let insert_lock = services.rooms.timeline.mutex_insert.lock(room_id).await;
-	drop(insert_lock);
-
 	let sincecount = PduCount::Normal(since);
 	let next_batchcount = PduCount::Normal(next_batch);
 
@@ -608,17 +632,26 @@ async fn load_joined_room(
 		10_usize,
 	);
 
-	let (current_shortstatehash, since_shortstatehash, timeline) =
-		try_join3(current_shortstatehash, since_shortstatehash, timeline).await?;
+	let receipt_events = services
+		.rooms
+		.read_receipt
+		.readreceipts_since(room_id, since)
+		.filter_map(|(read_user, _, edu)| async move {
+			services
+				.users
+				.user_is_ignored(read_user, sender_user)
+				.await
+				.or_some((read_user.to_owned(), edu))
+		})
+		.collect::<HashMap<OwnedUserId, Raw<AnySyncEphemeralRoomEvent>>>()
+		.map(Ok);
+
+	let (current_shortstatehash, since_shortstatehash, timeline, receipt_events) =
+		try_join4(current_shortstatehash, since_shortstatehash, timeline, receipt_events)
+			.boxed()
+			.await?;
 
 	let (timeline_pdus, limited) = timeline;
-	let timeline_users =
-		timeline_pdus
-			.iter()
-			.fold(HashSet::new(), |mut timeline_users, (_, event)| {
-				timeline_users.insert(event.sender.as_str().to_owned());
-				timeline_users
-			});
 
 	let last_notification_read: OptionFuture<_> = timeline_pdus
 		.is_empty()
@@ -630,21 +663,68 @@ async fn load_joined_room(
 		})
 		.into();
 
-	let send_notification_counts = last_notification_read
-		.is_none_or(|&count| count > since)
-		.await;
-
-	services.rooms.lazy_loading.lazy_load_confirm_delivery(
-		sender_user,
-		sender_device,
-		room_id,
-		sincecount,
-	);
-
 	let no_state_changes = timeline_pdus.is_empty()
 		&& (since_shortstatehash.is_none()
 			|| since_shortstatehash.is_some_and(is_equal_to!(current_shortstatehash)));
 
+	let since_sender_member: OptionFuture<_> = since_shortstatehash
+		.map(|short| {
+			services
+				.rooms
+				.state_accessor
+				.state_get_content(short, &StateEventType::RoomMember, sender_user.as_str())
+				.ok()
+		})
+		.into();
+
+	let joined_since_last_sync =
+		since_sender_member
+			.await
+			.flatten()
+			.is_none_or(|content: RoomMemberEventContent| {
+				content.membership != MembershipState::Join
+			});
+
+	let lazy_loading_enabled = filter.room.state.lazy_load_options.is_enabled()
+		|| filter.room.timeline.lazy_load_options.is_enabled();
+
+	let generate_witness =
+		lazy_loading_enabled && (since_shortstatehash.is_none() || joined_since_last_sync);
+
+	let lazy_reset = lazy_loading_enabled && since_shortstatehash.is_none();
+
+	let lazy_loading_context = &lazy_loading::Context {
+		user_id: sender_user,
+		device_id: sender_device,
+		room_id,
+		token: None,
+		options: Some(&filter.room.state.lazy_load_options),
+	};
+
+	// Reset lazy loading because this is an initial sync
+	let lazy_load_reset: OptionFuture<_> = lazy_reset
+		.then(|| services.rooms.lazy_loading.reset(lazy_loading_context))
+		.into();
+
+	lazy_load_reset.await;
+	let witness: Option<Witness> = generate_witness.then(|| {
+		timeline_pdus
+			.iter()
+			.map(|(_, pdu)| pdu.sender.clone())
+			.chain(receipt_events.keys().cloned())
+			.collect()
+	});
+
+	let witness: OptionFuture<_> = witness
+		.map(|witness| {
+			services
+				.rooms
+				.lazy_loading
+				.witness_retain(witness, lazy_loading_context)
+		})
+		.into();
+
+	let witness = witness.await;
 	let mut device_list_updates = HashSet::<OwnedUserId>::new();
 	let mut left_encrypted_users = HashSet::<OwnedUserId>::new();
 	let StateChanges {
@@ -659,19 +739,17 @@ async fn load_joined_room(
 		calculate_state_changes(
 			services,
 			sender_user,
-			sender_device,
 			room_id,
-			next_batchcount,
-			lazy_load_enabled,
-			lazy_load_send_redundant,
 			full_state,
+			filter,
 			&mut device_list_updates,
 			&mut left_encrypted_users,
 			since_shortstatehash,
 			current_shortstatehash,
-			&timeline_pdus,
-			&timeline_users,
+			joined_since_last_sync,
+			witness.as_ref(),
 		)
+		.boxed()
 		.await?
 	};
 
@@ -696,19 +774,6 @@ async fn load_joined_room(
 		.map(|(_, pdu)| pdu.to_sync_room_event())
 		.collect();
 
-	let receipt_events = services
-		.rooms
-		.read_receipt
-		.readreceipts_since(room_id, since)
-		.filter_map(|(read_user, _, edu)| async move {
-			services
-				.users
-				.user_is_ignored(read_user, sender_user)
-				.await
-				.or_some((read_user.to_owned(), edu))
-		})
-		.collect::<HashMap<OwnedUserId, Raw<AnySyncEphemeralRoomEvent>>>();
-
 	let typing_events = services
 		.rooms
 		.typing
@@ -728,6 +793,10 @@ async fn load_joined_room(
 		})
 		.unwrap_or(Vec::new());
 
+	let send_notification_counts = last_notification_read
+		.is_none_or(|&count| count > since)
+		.await;
+
 	let notification_count: OptionFuture<_> = send_notification_counts
 		.then(|| {
 			services
@@ -750,14 +819,14 @@ async fn load_joined_room(
 		})
 		.into();
 
-	let events = join4(room_events, account_data_events, receipt_events, typing_events);
+	let events = join3(room_events, account_data_events, typing_events);
 	let unread_notifications = join(notification_count, highlight_count);
 	let (unread_notifications, events, device_updates) =
 		join3(unread_notifications, events, device_updates)
 			.boxed()
 			.await;
 
-	let (room_events, account_data_events, receipt_events, typing_events) = events;
+	let (room_events, account_data_events, typing_events) = events;
 	let (notification_count, highlight_count) = unread_notifications;
 
 	device_list_updates.extend(device_updates);
@@ -834,7 +903,6 @@ async fn load_joined_room(
 	skip_all,
 	fields(
 	    full = %full_state,
-	    ll = ?(lazy_load_enabled, lazy_load_send_redundant),
 	    cs = %current_shortstatehash,
 	    ss = ?since_shortstatehash,
     )
@@ -843,64 +911,38 @@ async fn load_joined_room(
 async fn calculate_state_changes(
 	services: &Services,
 	sender_user: &UserId,
-	sender_device: &DeviceId,
 	room_id: &RoomId,
-	next_batchcount: PduCount,
-	lazy_load_enabled: bool,
-	lazy_load_send_redundant: bool,
 	full_state: bool,
+	filter: &FilterDefinition,
 	device_list_updates: &mut HashSet<OwnedUserId>,
 	left_encrypted_users: &mut HashSet<OwnedUserId>,
 	since_shortstatehash: Option<ShortStateHash>,
 	current_shortstatehash: ShortStateHash,
-	timeline_pdus: &Vec<(PduCount, PduEvent)>,
-	timeline_users: &HashSet<String>,
+	joined_since_last_sync: bool,
+	witness: Option<&Witness>,
 ) -> Result<StateChanges> {
-	let since_sender_member: OptionFuture<_> = since_shortstatehash
-		.map(|short| {
-			services
-				.rooms
-				.state_accessor
-				.state_get_content(short, &StateEventType::RoomMember, sender_user.as_str())
-				.ok()
-		})
-		.into();
-
-	let joined_since_last_sync =
-		since_sender_member
-			.await
-			.flatten()
-			.is_none_or(|content: RoomMemberEventContent| {
-				content.membership != MembershipState::Join
-			});
-
 	if since_shortstatehash.is_none() || joined_since_last_sync {
 		calculate_state_initial(
 			services,
 			sender_user,
-			sender_device,
 			room_id,
-			next_batchcount,
-			lazy_load_enabled,
 			full_state,
+			filter,
 			current_shortstatehash,
-			timeline_users,
+			witness,
 		)
 		.await
 	} else {
 		calculate_state_incremental(
 			services,
 			sender_user,
-			sender_device,
 			room_id,
-			next_batchcount,
-			lazy_load_send_redundant,
 			full_state,
+			filter,
 			device_list_updates,
 			left_encrypted_users,
 			since_shortstatehash,
 			current_shortstatehash,
-			timeline_pdus,
 			joined_since_last_sync,
 		)
 		.await
@@ -912,87 +954,54 @@ async fn calculate_state_changes(
 async fn calculate_state_initial(
 	services: &Services,
 	sender_user: &UserId,
-	sender_device: &DeviceId,
 	room_id: &RoomId,
-	next_batchcount: PduCount,
-	lazy_load_enabled: bool,
 	full_state: bool,
+	filter: &FilterDefinition,
 	current_shortstatehash: ShortStateHash,
-	timeline_users: &HashSet<String>,
+	witness: Option<&Witness>,
 ) -> Result<StateChanges> {
-	// Probably since = 0, we will do an initial sync
-	let state = services
+	let state_events = services
 		.rooms
 		.state_accessor
 		.state_full_ids(current_shortstatehash)
-		.await?
-		.into_iter()
-		.stream()
-		.broad_filter_map(|(shortstatekey, event_id): (ShortStateKey, OwnedEventId)| {
-			services
-				.rooms
-				.short
-				.get_statekey_from_short(shortstatekey)
-				.map_ok(move |(event_type, state_key)| ((event_type, state_key), event_id))
-				.ok()
-		})
-		.fold((Vec::new(), HashSet::new()), |a, item| async move {
-			let (mut state_events, mut lazy_loaded) = a;
-			let ((event_type, state_key), event_id) = item;
+		.await?;
 
-			if event_type != StateEventType::RoomMember {
-				let Ok(pdu) = services.rooms.timeline.get_pdu(&event_id).await else {
-					error!("Pdu in state not found: {event_id}");
-					return (state_events, lazy_loaded);
-				};
+	let shortstatekeys = state_events.keys().copied().stream();
 
-				state_events.push(pdu);
-				return (state_events, lazy_loaded);
+	let state_events = services
+		.rooms
+		.short
+		.multi_get_statekey_from_short(shortstatekeys)
+		.zip(state_events.values().cloned().stream())
+		.ready_filter_map(|item| Some((item.0.ok()?, item.1)))
+		.ready_filter_map(|((event_type, state_key), event_id)| {
+			let lazy_load_enabled = filter.room.state.lazy_load_options.is_enabled()
+				|| filter.room.timeline.lazy_load_options.is_enabled();
+
+			if lazy_load_enabled
+				&& event_type == StateEventType::RoomMember
+				&& !full_state
+				&& state_key.as_str().try_into().is_ok_and(|user_id: &UserId| {
+					sender_user != user_id
+						&& witness.is_some_and(|witness| !witness.contains(user_id))
+				}) {
+				return None;
 			}
 
-			// TODO: Delete "element_hacks" when this is resolved: https://github.com/vector-im/element-web/issues/22565
-			if !lazy_load_enabled
-				|| full_state
-				|| timeline_users.contains(&state_key)
-				|| (cfg!(feature = "element_hacks") && *sender_user == state_key)
-			{
-				let Ok(pdu) = services.rooms.timeline.get_pdu(&event_id).await else {
-					error!("Pdu in state not found: {event_id}");
-					return (state_events, lazy_loaded);
-				};
-
-				// This check is in case a bad user ID made it into the database
-				if let Ok(uid) = OwnedUserId::parse(&state_key) {
-					lazy_loaded.insert(uid);
-				}
-
-				state_events.push(pdu);
-			}
-
-			(state_events, lazy_loaded)
+			Some(event_id)
 		})
+		.broad_filter_map(|event_id: OwnedEventId| async move {
+			services.rooms.timeline.get_pdu(&event_id).await.ok()
+		})
+		.collect()
 		.map(Ok);
 
 	let counts = calculate_counts(services, room_id, sender_user);
-	let ((joined_member_count, invited_member_count, heroes), (state_events, lazy_loaded)) =
-		try_join(counts, state).boxed().await?;
-
-	// Reset lazy loading because this is an initial sync
-	services
-		.rooms
-		.lazy_loading
-		.lazy_load_reset(sender_user, sender_device, room_id)
-		.await;
+	let ((joined_member_count, invited_member_count, heroes), state_events) =
+		try_join(counts, state_events).boxed().await?;
 
 	// The state_events above should contain all timeline_users, let's mark them as
 	// lazy loaded.
-	services.rooms.lazy_loading.lazy_load_mark_sent(
-		sender_user,
-		sender_device,
-		room_id,
-		lazy_loaded,
-		next_batchcount,
-	);
 
 	Ok(StateChanges {
 		heroes,
@@ -1008,16 +1017,13 @@ async fn calculate_state_initial(
 async fn calculate_state_incremental(
 	services: &Services,
 	sender_user: &UserId,
-	sender_device: &DeviceId,
 	room_id: &RoomId,
-	next_batchcount: PduCount,
-	lazy_load_send_redundant: bool,
 	full_state: bool,
+	_filter: &FilterDefinition,
 	device_list_updates: &mut HashSet<OwnedUserId>,
 	left_encrypted_users: &mut HashSet<OwnedUserId>,
 	since_shortstatehash: Option<ShortStateHash>,
 	current_shortstatehash: ShortStateHash,
-	timeline_pdus: &Vec<(PduCount, PduEvent)>,
 	joined_since_last_sync: bool,
 ) -> Result<StateChanges> {
 	// Incremental /sync
@@ -1130,76 +1136,12 @@ async fn calculate_state_incremental(
 		(None, None, None)
 	};
 
-	let mut state_events = delta_state_events;
-
-	// Mark all member events we're returning as lazy-loaded
-	let mut lazy_loaded = state_events
-		.iter()
-		.filter(|pdu| pdu.kind == RoomMember)
-		.filter_map(|pdu| {
-			pdu.state_key
-				.clone()
-				.map(TryInto::try_into)
-				.map(LogDebugErr::log_debug_err)
-				.flat_ok()
-		})
-		.fold(HashSet::new(), |mut lazy_loaded, user_id| {
-			lazy_loaded.insert(user_id);
-			lazy_loaded
-		});
-
-	// Fetch contextual member state events for events from the timeline, and
-	// mark them as lazy-loaded as well.
-	for (_, event) in timeline_pdus {
-		if lazy_loaded.contains(&event.sender) {
-			continue;
-		}
-
-		let sent_before: OptionFuture<_> = (!lazy_load_send_redundant)
-			.then(|| {
-				services.rooms.lazy_loading.lazy_load_was_sent_before(
-					sender_user,
-					sender_device,
-					room_id,
-					&event.sender,
-				)
-			})
-			.into();
-
-		let member_event: OptionFuture<_> = sent_before
-			.await
-			.is_none_or(is_false!())
-			.then(|| {
-				services.rooms.state_accessor.room_state_get(
-					room_id,
-					&StateEventType::RoomMember,
-					event.sender.as_str(),
-				)
-			})
-			.into();
-
-		let Some(Ok(member_event)) = member_event.await else {
-			continue;
-		};
-
-		lazy_loaded.insert(event.sender.clone());
-		state_events.push(member_event);
-	}
-
-	services.rooms.lazy_loading.lazy_load_mark_sent(
-		sender_user,
-		sender_device,
-		room_id,
-		lazy_loaded,
-		next_batchcount,
-	);
-
 	Ok(StateChanges {
 		heroes,
 		joined_member_count,
 		invited_member_count,
 		joined_since_last_sync,
-		state_events,
+		state_events: delta_state_events,
 	})
 }
 
@@ -1223,7 +1165,7 @@ async fn calculate_counts(
 	let (joined_member_count, invited_member_count) =
 		join(joined_member_count, invited_member_count).await;
 
-	let small_room = joined_member_count.saturating_add(invited_member_count) > 5;
+	let small_room = joined_member_count.saturating_add(invited_member_count) <= 5;
 
 	let heroes: OptionFuture<_> = small_room
 		.then(|| calculate_heroes(services, room_id, sender_user))

src/api/client/sync/v4.rs

@@ -23,24 +23,23 @@ use ruma::{
 			DeviceLists, UnreadNotificationsCount,
 		},
 	},
-	directory::RoomTypeFilter,
 	events::{
 		room::member::{MembershipState, RoomMemberEventContent},
 		AnyRawAccountDataEvent, AnySyncEphemeralRoomEvent, StateEventType,
-		TimelineEventType::{self, *},
+		TimelineEventType::*,
 	},
 	serde::Raw,
-	uint, MilliSecondsSinceUnixEpoch, OwnedEventId, OwnedRoomId, OwnedUserId, UInt,
+	uint, MilliSecondsSinceUnixEpoch, OwnedEventId, OwnedRoomId, OwnedUserId, RoomId, UInt,
 };
-use service::{rooms::read_receipt::pack_receipts, Services};
+use service::rooms::read_receipt::pack_receipts;
 
 use super::{load_timeline, share_encrypted_room};
-use crate::{client::ignored_filter, Ruma};
+use crate::{
+	client::{filter_rooms, ignored_filter, sync::v5::TodoRooms, DEFAULT_BUMP_TYPES},
+	Ruma,
+};
 
-const SINGLE_CONNECTION_SYNC: &str = "single_connection_sync";
-
-const DEFAULT_BUMP_TYPES: &[TimelineEventType; 6] =
-	&[CallInvite, PollStart, Beacon, RoomEncrypted, RoomMessage, Sticker];
+pub(crate) const SINGLE_CONNECTION_SYNC: &str = "single_connection_sync";
 
 /// POST `/_matrix/client/unstable/org.matrix.msc3575/sync`
 ///
@@ -113,12 +112,27 @@ pub(crate) async fn sync_events_v4_route(
 		.collect()
 		.await;
 
-	let all_rooms = all_joined_rooms
+	let all_knocked_rooms: Vec<_> = services
+		.rooms
+		.state_cache
+		.rooms_knocked(sender_user)
+		.map(|r| r.0)
+		.collect()
+		.await;
+
+	let all_invited_rooms: Vec<&RoomId> = all_invited_rooms.iter().map(AsRef::as_ref).collect();
+	let all_knocked_rooms: Vec<&RoomId> = all_knocked_rooms.iter().map(AsRef::as_ref).collect();
+
+	let all_rooms: Vec<&RoomId> = all_joined_rooms
 		.iter()
-		.chain(all_invited_rooms.iter())
-		.map(Clone::clone)
+		.map(AsRef::as_ref)
+		.chain(all_invited_rooms.iter().map(AsRef::as_ref))
+		.chain(all_knocked_rooms.iter().map(AsRef::as_ref))
 		.collect();
 
+	let all_joined_rooms = all_joined_rooms.iter().map(AsRef::as_ref).collect();
+	let all_invited_rooms = all_invited_rooms.iter().map(AsRef::as_ref).collect();
+
 	if body.extensions.to_device.enabled.unwrap_or(false) {
 		services
 			.users
@@ -171,6 +185,7 @@ pub(crate) async fn sync_events_v4_route(
 		);
 
 		for room_id in &all_joined_rooms {
+			let room_id: &&RoomId = room_id;
 			let Ok(current_shortstatehash) =
 				services.rooms.state.get_room_shortstatehash(room_id).await
 			else {
@@ -323,7 +338,7 @@ pub(crate) async fn sync_events_v4_route(
 	}
 
 	let mut lists = BTreeMap::new();
-	let mut todo_rooms = BTreeMap::new(); // and required state
+	let mut todo_rooms: TodoRooms = BTreeMap::new(); // and required state
 
 	for (list_id, list) in &body.lists {
 		let active_rooms = match list.filters.clone().and_then(|f| f.is_invite) {
@@ -344,7 +359,7 @@ pub(crate) async fn sync_events_v4_route(
 			| None => active_rooms,
 		};
 
-		let mut new_known_rooms = BTreeSet::new();
+		let mut new_known_rooms: BTreeSet<OwnedRoomId> = BTreeSet::new();
 
 		let ranges = list.ranges.clone();
 		lists.insert(list_id.clone(), sync_events::v4::SyncList {
@@ -366,9 +381,9 @@ pub(crate) async fn sync_events_v4_route(
 						Vec::new()
 					};
 
-					new_known_rooms.extend(room_ids.iter().cloned());
+					new_known_rooms.extend(room_ids.clone().into_iter().map(ToOwned::to_owned));
 					for room_id in &room_ids {
-						let todo_room = todo_rooms.entry(room_id.clone()).or_insert((
+						let todo_room = todo_rooms.entry((*room_id).to_owned()).or_insert((
 							BTreeSet::new(),
 							0_usize,
 							u64::MAX,
@@ -390,7 +405,7 @@ pub(crate) async fn sync_events_v4_route(
 						todo_room.2 = todo_room.2.min(
 							known_rooms
 								.get(list_id.as_str())
-								.and_then(|k| k.get(room_id))
+								.and_then(|k| k.get(*room_id))
 								.copied()
 								.unwrap_or(0),
 						);
@@ -399,7 +414,7 @@ pub(crate) async fn sync_events_v4_route(
 						op: SlidingOp::Sync,
 						range: Some(r),
 						index: None,
-						room_ids,
+						room_ids: room_ids.into_iter().map(ToOwned::to_owned).collect(),
 						room_id: None,
 					}
 				})
@@ -409,8 +424,8 @@ pub(crate) async fn sync_events_v4_route(
 
 		if let Some(conn_id) = &body.conn_id {
 			services.sync.update_sync_known_rooms(
-				sender_user.clone(),
-				sender_device.clone(),
+				sender_user,
+				&sender_device,
 				conn_id.clone(),
 				list_id.clone(),
 				new_known_rooms,
@@ -455,8 +470,8 @@ pub(crate) async fn sync_events_v4_route(
 
 	if let Some(conn_id) = &body.conn_id {
 		services.sync.update_sync_known_rooms(
-			sender_user.clone(),
-			sender_device.clone(),
+			sender_user,
+			&sender_device,
 			conn_id.clone(),
 			"subscriptions".to_owned(),
 			known_subscription_rooms,
@@ -480,7 +495,8 @@ pub(crate) async fn sync_events_v4_route(
 		let mut timestamp: Option<_> = None;
 		let mut invite_state = None;
 		let (timeline_pdus, limited);
-		if all_invited_rooms.contains(room_id) {
+		let new_room_id: &RoomId = (*room_id).as_ref();
+		if all_invited_rooms.contains(&new_room_id) {
 			// TODO: figure out a timestamp we can use for remote invites
 			invite_state = services
 				.rooms
@@ -510,7 +526,7 @@ pub(crate) async fn sync_events_v4_route(
 		}
 
 		account_data.rooms.insert(
-			room_id.clone(),
+			room_id.to_owned(),
 			services
 				.account_data
 				.changes_since(Some(room_id), sender_user, *roomsince)
@@ -740,10 +756,9 @@ pub(crate) async fn sync_events_v4_route(
 		});
 	}
 
-	if rooms
-		.iter()
-		.all(|(_, r)| r.timeline.is_empty() && r.required_state.is_empty())
-	{
+	if rooms.iter().all(|(id, r)| {
+		r.timeline.is_empty() && r.required_state.is_empty() && !receipts.rooms.contains_key(id)
+	}) {
 		// Hang a few seconds so requests are not spammed
 		// Stop hanging if new info arrives
 		let default = Duration::from_secs(30);
@@ -789,33 +804,3 @@ pub(crate) async fn sync_events_v4_route(
 		delta_token: None,
 	})
 }
-
-async fn filter_rooms(
-	services: &Services,
-	rooms: &[OwnedRoomId],
-	filter: &[RoomTypeFilter],
-	negate: bool,
-) -> Vec<OwnedRoomId> {
-	rooms
-		.iter()
-		.stream()
-		.filter_map(|r| async move {
-			let room_type = services.rooms.state_accessor.get_room_type(r).await;
-
-			if room_type.as_ref().is_err_and(|e| !e.is_not_found()) {
-				return None;
-			}
-
-			let room_type_filter = RoomTypeFilter::from(room_type.ok());
-
-			let include = if negate {
-				!filter.contains(&room_type_filter)
-			} else {
-				filter.is_empty() || filter.contains(&room_type_filter)
-			};
-
-			include.then_some(r.to_owned())
-		})
-		.collect()
-		.await
-}

src/api/client/sync/v5.rs

@@ -0,0 +1,886 @@
+use std::{
+	cmp::{self, Ordering},
+	collections::{BTreeMap, BTreeSet, HashMap, HashSet},
+	time::Duration,
+};
+
+use axum::extract::State;
+use conduwuit::{
+	debug, error, extract_variant, trace,
+	utils::{
+		math::{ruma_from_usize, usize_from_ruma},
+		BoolExt, IterStream, ReadyExt, TryFutureExtExt,
+	},
+	warn, Error, Result,
+};
+use futures::{FutureExt, StreamExt, TryFutureExt};
+use ruma::{
+	api::client::{
+		error::ErrorKind,
+		sync::sync_events::{self, DeviceLists, UnreadNotificationsCount},
+	},
+	events::{
+		room::member::{MembershipState, RoomMemberEventContent},
+		AnyRawAccountDataEvent, AnySyncEphemeralRoomEvent, StateEventType, TimelineEventType,
+	},
+	serde::Raw,
+	state_res::TypeStateKey,
+	uint, DeviceId, OwnedEventId, OwnedRoomId, OwnedUserId, RoomId, UInt, UserId,
+};
+use service::{rooms::read_receipt::pack_receipts, PduCount};
+
+use super::{filter_rooms, share_encrypted_room};
+use crate::{
+	client::{ignored_filter, sync::load_timeline, DEFAULT_BUMP_TYPES},
+	Ruma,
+};
+
+type SyncInfo<'a> = (&'a UserId, &'a DeviceId, u64, &'a sync_events::v5::Request);
+
+/// `POST /_matrix/client/unstable/org.matrix.simplified_msc3575/sync`
+/// ([MSC4186])
+///
+/// A simplified version of sliding sync ([MSC3575]).
+///
+/// Get all new events in a sliding window of rooms since the last sync or a
+/// given point in time.
+///
+/// [MSC3575]: https://github.com/matrix-org/matrix-spec-proposals/pull/3575
+/// [MSC4186]: https://github.com/matrix-org/matrix-spec-proposals/pull/4186
+pub(crate) async fn sync_events_v5_route(
+	State(services): State<crate::State>,
+	body: Ruma<sync_events::v5::Request>,
+) -> Result<sync_events::v5::Response> {
+	debug_assert!(DEFAULT_BUMP_TYPES.is_sorted(), "DEFAULT_BUMP_TYPES is not sorted");
+	let sender_user = body.sender_user.as_ref().expect("user is authenticated");
+	let sender_device = body.sender_device.as_ref().expect("user is authenticated");
+	let mut body = body.body;
+
+	// Setup watchers, so if there's no response, we can wait for them
+	let watcher = services.sync.watch(sender_user, sender_device);
+
+	let next_batch = services.globals.next_count()?;
+
+	let conn_id = body.conn_id.clone();
+
+	let globalsince = body
+		.pos
+		.as_ref()
+		.and_then(|string| string.parse().ok())
+		.unwrap_or(0);
+
+	if globalsince != 0
+		&& !services.sync.snake_connection_cached(
+			sender_user.clone(),
+			sender_device.clone(),
+			conn_id.clone(),
+		) {
+		debug!("Restarting sync stream because it was gone from the database");
+		return Err(Error::Request(
+			ErrorKind::UnknownPos,
+			"Connection data lost since last time".into(),
+			http::StatusCode::BAD_REQUEST,
+		));
+	}
+
+	// Client / User requested an initial sync
+	if globalsince == 0 {
+		services.sync.forget_snake_sync_connection(
+			sender_user.clone(),
+			sender_device.clone(),
+			conn_id.clone(),
+		);
+	}
+
+	// Get sticky parameters from cache
+	let known_rooms = services.sync.update_snake_sync_request_with_cache(
+		sender_user.clone(),
+		sender_device.clone(),
+		&mut body,
+	);
+
+	let all_joined_rooms: Vec<_> = services
+		.rooms
+		.state_cache
+		.rooms_joined(sender_user)
+		.map(ToOwned::to_owned)
+		.collect()
+		.await;
+
+	let all_invited_rooms: Vec<_> = services
+		.rooms
+		.state_cache
+		.rooms_invited(sender_user)
+		.map(|r| r.0)
+		.collect()
+		.await;
+
+	let all_knocked_rooms: Vec<_> = services
+		.rooms
+		.state_cache
+		.rooms_knocked(sender_user)
+		.map(|r| r.0)
+		.collect()
+		.await;
+
+	let all_rooms: Vec<&RoomId> = all_joined_rooms
+		.iter()
+		.map(AsRef::as_ref)
+		.chain(all_invited_rooms.iter().map(AsRef::as_ref))
+		.chain(all_knocked_rooms.iter().map(AsRef::as_ref))
+		.collect();
+
+	let all_joined_rooms = all_joined_rooms.iter().map(AsRef::as_ref).collect();
+	let all_invited_rooms = all_invited_rooms.iter().map(AsRef::as_ref).collect();
+
+	let pos = next_batch.clone().to_string();
+
+	let mut todo_rooms: TodoRooms = BTreeMap::new();
+
+	let sync_info: SyncInfo<'_> = (sender_user, sender_device, globalsince, &body);
+	let mut response = sync_events::v5::Response {
+		txn_id: body.txn_id.clone(),
+		pos,
+		lists: BTreeMap::new(),
+		rooms: BTreeMap::new(),
+		extensions: sync_events::v5::response::Extensions {
+			account_data: collect_account_data(services, sync_info).await,
+			e2ee: collect_e2ee(services, sync_info, &all_joined_rooms).await?,
+			to_device: collect_to_device(services, sync_info, next_batch).await,
+			receipts: collect_receipts(services).await,
+			typing: sync_events::v5::response::Typing::default(),
+		},
+	};
+
+	handle_lists(
+		services,
+		sync_info,
+		&all_invited_rooms,
+		&all_joined_rooms,
+		&all_rooms,
+		&mut todo_rooms,
+		&known_rooms,
+		&mut response,
+	)
+	.await;
+
+	fetch_subscriptions(services, sync_info, &known_rooms, &mut todo_rooms).await;
+
+	response.rooms = process_rooms(
+		services,
+		sender_user,
+		next_batch,
+		&all_invited_rooms,
+		&todo_rooms,
+		&mut response,
+		&body,
+	)
+	.await?;
+
+	if response.rooms.iter().all(|(id, r)| {
+		r.timeline.is_empty()
+			&& r.required_state.is_empty()
+			&& !response.extensions.receipts.rooms.contains_key(id)
+	}) && response
+		.extensions
+		.to_device
+		.clone()
+		.is_none_or(|to| to.events.is_empty())
+	{
+		// Hang a few seconds so requests are not spammed
+		// Stop hanging if new info arrives
+		let default = Duration::from_secs(30);
+		let duration = cmp::min(body.timeout.unwrap_or(default), default);
+		_ = tokio::time::timeout(duration, watcher).await;
+	}
+
+	trace!(
+		rooms=?response.rooms.len(),
+		account_data=?response.extensions.account_data.rooms.len(),
+		receipts=?response.extensions.receipts.rooms.len(),
+		"responding to request with"
+	);
+	Ok(response)
+}
+
+type KnownRooms = BTreeMap<String, BTreeMap<OwnedRoomId, u64>>;
+pub(crate) type TodoRooms = BTreeMap<OwnedRoomId, (BTreeSet<TypeStateKey>, usize, u64)>;
+
+async fn fetch_subscriptions(
+	services: crate::State,
+	(sender_user, sender_device, globalsince, body): SyncInfo<'_>,
+	known_rooms: &KnownRooms,
+	todo_rooms: &mut TodoRooms,
+) {
+	let mut known_subscription_rooms = BTreeSet::new();
+	for (room_id, room) in &body.room_subscriptions {
+		if !services.rooms.metadata.exists(room_id).await {
+			continue;
+		}
+		let todo_room =
+			todo_rooms
+				.entry(room_id.clone())
+				.or_insert((BTreeSet::new(), 0_usize, u64::MAX));
+
+		let limit: UInt = room.timeline_limit;
+
+		todo_room.0.extend(room.required_state.iter().cloned());
+		todo_room.1 = todo_room.1.max(usize_from_ruma(limit));
+		// 0 means unknown because it got out of date
+		todo_room.2 = todo_room.2.min(
+			known_rooms
+				.get("subscriptions")
+				.and_then(|k| k.get(room_id))
+				.copied()
+				.unwrap_or(0),
+		);
+		known_subscription_rooms.insert(room_id.clone());
+	}
+	// where this went (protomsc says it was removed)
+	//for r in body.unsubscribe_rooms {
+	//	known_subscription_rooms.remove(&r);
+	//	body.room_subscriptions.remove(&r);
+	//}
+
+	if let Some(conn_id) = &body.conn_id {
+		services.sync.update_snake_sync_known_rooms(
+			sender_user,
+			sender_device,
+			conn_id.clone(),
+			"subscriptions".to_owned(),
+			known_subscription_rooms,
+			globalsince,
+		);
+	}
+}
+
+#[allow(clippy::too_many_arguments)]
+async fn handle_lists<'a>(
+	services: crate::State,
+	(sender_user, sender_device, globalsince, body): SyncInfo<'_>,
+	all_invited_rooms: &Vec<&'a RoomId>,
+	all_joined_rooms: &Vec<&'a RoomId>,
+	all_rooms: &Vec<&'a RoomId>,
+	todo_rooms: &'a mut TodoRooms,
+	known_rooms: &'a KnownRooms,
+	response: &'_ mut sync_events::v5::Response,
+) -> KnownRooms {
+	for (list_id, list) in &body.lists {
+		let active_rooms = match list.filters.clone().and_then(|f| f.is_invite) {
+			| Some(true) => all_invited_rooms,
+			| Some(false) => all_joined_rooms,
+			| None => all_rooms,
+		};
+
+		let active_rooms = match list.filters.clone().map(|f| f.not_room_types) {
+			| Some(filter) if filter.is_empty() => active_rooms,
+			| Some(value) => &filter_rooms(&services, active_rooms, &value, true).await,
+			| None => active_rooms,
+		};
+
+		let mut new_known_rooms: BTreeSet<OwnedRoomId> = BTreeSet::new();
+
+		let ranges = list.ranges.clone();
+
+		for mut range in ranges {
+			range.0 = uint!(0);
+			range.1 = range
+				.1
+				.clamp(range.0, UInt::try_from(active_rooms.len()).unwrap_or(UInt::MAX));
+
+			let room_ids =
+				active_rooms[usize_from_ruma(range.0)..usize_from_ruma(range.1)].to_vec();
+
+			let new_rooms: BTreeSet<OwnedRoomId> =
+				room_ids.clone().into_iter().map(From::from).collect();
+			new_known_rooms.extend(new_rooms);
+			//new_known_rooms.extend(room_ids..cloned());
+			for room_id in room_ids {
+				let todo_room = todo_rooms.entry(room_id.to_owned()).or_insert((
+					BTreeSet::new(),
+					0_usize,
+					u64::MAX,
+				));
+
+				let limit: usize = usize_from_ruma(list.room_details.timeline_limit).min(100);
+
+				todo_room
+					.0
+					.extend(list.room_details.required_state.iter().cloned());
+
+				todo_room.1 = todo_room.1.max(limit);
+				// 0 means unknown because it got out of date
+				todo_room.2 = todo_room.2.min(
+					known_rooms
+						.get(list_id.as_str())
+						.and_then(|k| k.get(room_id))
+						.copied()
+						.unwrap_or(0),
+				);
+			}
+		}
+		response
+			.lists
+			.insert(list_id.clone(), sync_events::v5::response::List {
+				count: ruma_from_usize(active_rooms.len()),
+			});
+
+		if let Some(conn_id) = &body.conn_id {
+			services.sync.update_snake_sync_known_rooms(
+				sender_user,
+				sender_device,
+				conn_id.clone(),
+				list_id.clone(),
+				new_known_rooms,
+				globalsince,
+			);
+		}
+	}
+	BTreeMap::default()
+}
+
+async fn process_rooms(
+	services: crate::State,
+	sender_user: &UserId,
+	next_batch: u64,
+	all_invited_rooms: &[&RoomId],
+	todo_rooms: &TodoRooms,
+	response: &mut sync_events::v5::Response,
+	body: &sync_events::v5::Request,
+) -> Result<BTreeMap<OwnedRoomId, sync_events::v5::response::Room>> {
+	let mut rooms = BTreeMap::new();
+	for (room_id, (required_state_request, timeline_limit, roomsince)) in todo_rooms {
+		let roomsincecount = PduCount::Normal(*roomsince);
+
+		let mut timestamp: Option<_> = None;
+		let mut invite_state = None;
+		let (timeline_pdus, limited);
+		let new_room_id: &RoomId = (*room_id).as_ref();
+		if all_invited_rooms.contains(&new_room_id) {
+			// TODO: figure out a timestamp we can use for remote invites
+			invite_state = services
+				.rooms
+				.state_cache
+				.invite_state(sender_user, room_id)
+				.await
+				.ok();
+
+			(timeline_pdus, limited) = (Vec::new(), true);
+		} else {
+			(timeline_pdus, limited) = match load_timeline(
+				&services,
+				sender_user,
+				room_id,
+				roomsincecount,
+				Some(PduCount::from(next_batch)),
+				*timeline_limit,
+			)
+			.await
+			{
+				| Ok(value) => value,
+				| Err(err) => {
+					warn!("Encountered missing timeline in {}, error {}", room_id, err);
+					continue;
+				},
+			};
+		}
+
+		if body.extensions.account_data.enabled == Some(true) {
+			response.extensions.account_data.rooms.insert(
+				room_id.to_owned(),
+				services
+					.account_data
+					.changes_since(Some(room_id), sender_user, *roomsince)
+					.ready_filter_map(|e| extract_variant!(e, AnyRawAccountDataEvent::Room))
+					.collect()
+					.await,
+			);
+		}
+
+		let last_privateread_update = services
+			.rooms
+			.read_receipt
+			.last_privateread_update(sender_user, room_id)
+			.await > *roomsince;
+
+		let private_read_event = if last_privateread_update {
+			services
+				.rooms
+				.read_receipt
+				.private_read_get(room_id, sender_user)
+				.await
+				.ok()
+		} else {
+			None
+		};
+
+		let mut receipts: Vec<Raw<AnySyncEphemeralRoomEvent>> = services
+			.rooms
+			.read_receipt
+			.readreceipts_since(room_id, *roomsince)
+			.filter_map(|(read_user, _ts, v)| async move {
+				services
+					.users
+					.user_is_ignored(read_user, sender_user)
+					.await
+					.or_some(v)
+			})
+			.collect()
+			.await;
+
+		if let Some(private_read_event) = private_read_event {
+			receipts.push(private_read_event);
+		}
+
+		let receipt_size = receipts.len();
+
+		if receipt_size > 0 {
+			response
+				.extensions
+				.receipts
+				.rooms
+				.insert(room_id.clone(), pack_receipts(Box::new(receipts.into_iter())));
+		}
+
+		if roomsince != &0
+			&& timeline_pdus.is_empty()
+			&& response
+				.extensions
+				.account_data
+				.rooms
+				.get(room_id)
+				.is_none_or(Vec::is_empty)
+			&& receipt_size == 0
+		{
+			continue;
+		}
+
+		let prev_batch = timeline_pdus
+			.first()
+			.map_or(Ok::<_, Error>(None), |(pdu_count, _)| {
+				Ok(Some(match pdu_count {
+					| PduCount::Backfilled(_) => {
+						error!("timeline in backfill state?!");
+						"0".to_owned()
+					},
+					| PduCount::Normal(c) => c.to_string(),
+				}))
+			})?
+			.or_else(|| {
+				if roomsince != &0 {
+					Some(roomsince.to_string())
+				} else {
+					None
+				}
+			});
+
+		let room_events: Vec<_> = timeline_pdus
+			.iter()
+			.stream()
+			.filter_map(|item| ignored_filter(&services, item.clone(), sender_user))
+			.map(|(_, pdu)| pdu.to_sync_room_event())
+			.collect()
+			.await;
+
+		for (_, pdu) in timeline_pdus {
+			let ts = pdu.origin_server_ts;
+			if DEFAULT_BUMP_TYPES.binary_search(&pdu.kind).is_ok()
+				&& timestamp.is_none_or(|time| time <= ts)
+			{
+				timestamp = Some(ts);
+			}
+		}
+
+		let required_state = required_state_request
+			.iter()
+			.stream()
+			.filter_map(|state| async move {
+				services
+					.rooms
+					.state_accessor
+					.room_state_get(room_id, &state.0, &state.1)
+					.await
+					.map(|s| s.to_sync_state_event())
+					.ok()
+			})
+			.collect()
+			.await;
+
+		// Heroes
+		let heroes: Vec<_> = services
+			.rooms
+			.state_cache
+			.room_members(room_id)
+			.ready_filter(|member| *member != sender_user)
+			.filter_map(|user_id| {
+				services
+					.rooms
+					.state_accessor
+					.get_member(room_id, user_id)
+					.map_ok(|memberevent| sync_events::v5::response::Hero {
+						user_id: user_id.into(),
+						name: memberevent.displayname,
+						avatar: memberevent.avatar_url,
+					})
+					.ok()
+			})
+			.take(5)
+			.collect()
+			.await;
+
+		let name = match heroes.len().cmp(&(1_usize)) {
+			| Ordering::Greater => {
+				let firsts = heroes[1..]
+					.iter()
+					.map(|h| h.name.clone().unwrap_or_else(|| h.user_id.to_string()))
+					.collect::<Vec<_>>()
+					.join(", ");
+
+				let last = heroes[0]
+					.name
+					.clone()
+					.unwrap_or_else(|| heroes[0].user_id.to_string());
+
+				Some(format!("{firsts} and {last}"))
+			},
+			| Ordering::Equal => Some(
+				heroes[0]
+					.name
+					.clone()
+					.unwrap_or_else(|| heroes[0].user_id.to_string()),
+			),
+			| Ordering::Less => None,
+		};
+
+		let heroes_avatar = if heroes.len() == 1 {
+			heroes[0].avatar.clone()
+		} else {
+			None
+		};
+
+		rooms.insert(room_id.clone(), sync_events::v5::response::Room {
+			name: services
+				.rooms
+				.state_accessor
+				.get_name(room_id)
+				.await
+				.ok()
+				.or(name),
+			avatar: if let Some(heroes_avatar) = heroes_avatar {
+				ruma::JsOption::Some(heroes_avatar)
+			} else {
+				match services.rooms.state_accessor.get_avatar(room_id).await {
+					| ruma::JsOption::Some(avatar) => ruma::JsOption::from_option(avatar.url),
+					| ruma::JsOption::Null => ruma::JsOption::Null,
+					| ruma::JsOption::Undefined => ruma::JsOption::Undefined,
+				}
+			},
+			initial: Some(roomsince == &0),
+			is_dm: None,
+			invite_state,
+			unread_notifications: UnreadNotificationsCount {
+				highlight_count: Some(
+					services
+						.rooms
+						.user
+						.highlight_count(sender_user, room_id)
+						.await
+						.try_into()
+						.expect("notification count can't go that high"),
+				),
+				notification_count: Some(
+					services
+						.rooms
+						.user
+						.notification_count(sender_user, room_id)
+						.await
+						.try_into()
+						.expect("notification count can't go that high"),
+				),
+			},
+			timeline: room_events,
+			required_state,
+			prev_batch,
+			limited,
+			joined_count: Some(
+				services
+					.rooms
+					.state_cache
+					.room_joined_count(room_id)
+					.await
+					.unwrap_or(0)
+					.try_into()
+					.unwrap_or_else(|_| uint!(0)),
+			),
+			invited_count: Some(
+				services
+					.rooms
+					.state_cache
+					.room_invited_count(room_id)
+					.await
+					.unwrap_or(0)
+					.try_into()
+					.unwrap_or_else(|_| uint!(0)),
+			),
+			num_live: None, // Count events in timeline greater than global sync counter
+			bump_stamp: timestamp,
+			heroes: Some(heroes),
+		});
+	}
+	Ok(rooms)
+}
+async fn collect_account_data(
+	services: crate::State,
+	(sender_user, _, globalsince, body): (&UserId, &DeviceId, u64, &sync_events::v5::Request),
+) -> sync_events::v5::response::AccountData {
+	let mut account_data = sync_events::v5::response::AccountData {
+		global: Vec::new(),
+		rooms: BTreeMap::new(),
+	};
+
+	if !body.extensions.account_data.enabled.unwrap_or(false) {
+		return sync_events::v5::response::AccountData::default();
+	}
+
+	account_data.global = services
+		.account_data
+		.changes_since(None, sender_user, globalsince)
+		.ready_filter_map(|e| extract_variant!(e, AnyRawAccountDataEvent::Global))
+		.collect()
+		.await;
+
+	if let Some(rooms) = &body.extensions.account_data.rooms {
+		for room in rooms {
+			account_data.rooms.insert(
+				room.clone(),
+				services
+					.account_data
+					.changes_since(Some(room), sender_user, globalsince)
+					.ready_filter_map(|e| extract_variant!(e, AnyRawAccountDataEvent::Room))
+					.collect()
+					.await,
+			);
+		}
+	}
+
+	account_data
+}
+
+async fn collect_e2ee<'a>(
+	services: crate::State,
+	(sender_user, sender_device, globalsince, body): (
+		&UserId,
+		&DeviceId,
+		u64,
+		&sync_events::v5::Request,
+	),
+	all_joined_rooms: &'a Vec<&'a RoomId>,
+) -> Result<sync_events::v5::response::E2EE> {
+	if !body.extensions.e2ee.enabled.unwrap_or(false) {
+		return Ok(sync_events::v5::response::E2EE::default());
+	}
+	let mut left_encrypted_users = HashSet::new(); // Users that have left any encrypted rooms the sender was in
+	let mut device_list_changes = HashSet::new();
+	let mut device_list_left = HashSet::new();
+	// Look for device list updates of this account
+	device_list_changes.extend(
+		services
+			.users
+			.keys_changed(sender_user, globalsince, None)
+			.map(ToOwned::to_owned)
+			.collect::<Vec<_>>()
+			.await,
+	);
+
+	for room_id in all_joined_rooms {
+		let Ok(current_shortstatehash) =
+			services.rooms.state.get_room_shortstatehash(room_id).await
+		else {
+			error!("Room {room_id} has no state");
+			continue;
+		};
+
+		let since_shortstatehash = services
+			.rooms
+			.user
+			.get_token_shortstatehash(room_id, globalsince)
+			.await
+			.ok();
+
+		let encrypted_room = services
+			.rooms
+			.state_accessor
+			.state_get(current_shortstatehash, &StateEventType::RoomEncryption, "")
+			.await
+			.is_ok();
+
+		if let Some(since_shortstatehash) = since_shortstatehash {
+			// Skip if there are only timeline changes
+			if since_shortstatehash == current_shortstatehash {
+				continue;
+			}
+
+			let since_encryption = services
+				.rooms
+				.state_accessor
+				.state_get(since_shortstatehash, &StateEventType::RoomEncryption, "")
+				.await;
+
+			let since_sender_member: Option<RoomMemberEventContent> = services
+				.rooms
+				.state_accessor
+				.state_get_content(
+					since_shortstatehash,
+					&StateEventType::RoomMember,
+					sender_user.as_str(),
+				)
+				.ok()
+				.await;
+
+			let joined_since_last_sync = since_sender_member
+				.as_ref()
+				.is_none_or(|member| member.membership != MembershipState::Join);
+
+			let new_encrypted_room = encrypted_room && since_encryption.is_err();
+
+			if encrypted_room {
+				let current_state_ids: HashMap<_, OwnedEventId> = services
+					.rooms
+					.state_accessor
+					.state_full_ids(current_shortstatehash)
+					.await?;
+
+				let since_state_ids = services
+					.rooms
+					.state_accessor
+					.state_full_ids(since_shortstatehash)
+					.await?;
+
+				for (key, id) in current_state_ids {
+					if since_state_ids.get(&key) != Some(&id) {
+						let Ok(pdu) = services.rooms.timeline.get_pdu(&id).await else {
+							error!("Pdu in state not found: {id}");
+							continue;
+						};
+						if pdu.kind == TimelineEventType::RoomMember {
+							if let Some(state_key) = &pdu.state_key {
+								let user_id =
+									OwnedUserId::parse(state_key.clone()).map_err(|_| {
+										Error::bad_database("Invalid UserId in member PDU.")
+									})?;
+
+								if user_id == *sender_user {
+									continue;
+								}
+
+								let content: RoomMemberEventContent = pdu.get_content()?;
+								match content.membership {
+									| MembershipState::Join => {
+										// A new user joined an encrypted room
+										if !share_encrypted_room(
+											&services,
+											sender_user,
+											&user_id,
+											Some(room_id),
+										)
+										.await
+										{
+											device_list_changes.insert(user_id);
+										}
+									},
+									| MembershipState::Leave => {
+										// Write down users that have left encrypted rooms we
+										// are in
+										left_encrypted_users.insert(user_id);
+									},
+									| _ => {},
+								}
+							}
+						}
+					}
+				}
+				if joined_since_last_sync || new_encrypted_room {
+					// If the user is in a new encrypted room, give them all joined users
+					device_list_changes.extend(
+						services
+						.rooms
+						.state_cache
+						.room_members(room_id)
+						// Don't send key updates from the sender to the sender
+						.ready_filter(|user_id| sender_user != *user_id)
+						// Only send keys if the sender doesn't share an encrypted room with the target
+						// already
+						.filter_map(|user_id| {
+							share_encrypted_room(&services, sender_user, user_id, Some(room_id))
+								.map(|res| res.or_some(user_id.to_owned()))
+						})
+						.collect::<Vec<_>>()
+						.await,
+					);
+				}
+			}
+		}
+		// Look for device list updates in this room
+		device_list_changes.extend(
+			services
+				.users
+				.room_keys_changed(room_id, globalsince, None)
+				.map(|(user_id, _)| user_id)
+				.map(ToOwned::to_owned)
+				.collect::<Vec<_>>()
+				.await,
+		);
+	}
+
+	for user_id in left_encrypted_users {
+		let dont_share_encrypted_room =
+			!share_encrypted_room(&services, sender_user, &user_id, None).await;
+
+		// If the user doesn't share an encrypted room with the target anymore, we need
+		// to tell them
+		if dont_share_encrypted_room {
+			device_list_left.insert(user_id);
+		}
+	}
+
+	Ok(sync_events::v5::response::E2EE {
+		device_lists: DeviceLists {
+			changed: device_list_changes.into_iter().collect(),
+			left: device_list_left.into_iter().collect(),
+		},
+		device_one_time_keys_count: services
+			.users
+			.count_one_time_keys(sender_user, sender_device)
+			.await,
+		device_unused_fallback_key_types: None,
+	})
+}
+
+async fn collect_to_device(
+	services: crate::State,
+	(sender_user, sender_device, globalsince, body): SyncInfo<'_>,
+	next_batch: u64,
+) -> Option<sync_events::v5::response::ToDevice> {
+	if !body.extensions.to_device.enabled.unwrap_or(false) {
+		return None;
+	}
+
+	services
+		.users
+		.remove_to_device_events(sender_user, sender_device, globalsince)
+		.await;
+
+	Some(sync_events::v5::response::ToDevice {
+		next_batch: next_batch.to_string(),
+		events: services
+			.users
+			.get_to_device_events(sender_user, sender_device)
+			.collect()
+			.await,
+	})
+}
+
+async fn collect_receipts(_services: crate::State) -> sync_events::v5::response::Receipts {
+	sync_events::v5::response::Receipts { rooms: BTreeMap::new() }
+	// TODO: get explicitly requested read receipts
+}

src/api/client/typing.rs

@@ -1,5 +1,5 @@
 use axum::extract::State;
-use conduwuit::Err;
+use conduwuit::{utils::math::Tried, Err};
 use ruma::api::client::typing::create_typing_event;
 
 use crate::{utils, Result, Ruma};
@@ -31,17 +31,15 @@ pub(crate) async fn create_typing_event_route(
 		let duration = utils::clamp(
 			duration.as_millis().try_into().unwrap_or(u64::MAX),
 			services
-				.globals
+				.server
 				.config
 				.typing_client_timeout_min_s
-				.checked_mul(1000)
-				.unwrap(),
+				.try_mul(1000)?,
 			services
-				.globals
+				.server
 				.config
 				.typing_client_timeout_max_s
-				.checked_mul(1000)
-				.unwrap(),
+				.try_mul(1000)?,
 		);
 		services
 			.rooms

src/api/client/unversioned.rs

@@ -52,6 +52,7 @@ pub(crate) async fn get_supported_versions_route(
 			("org.matrix.msc4180".to_owned(), true), /* stable flag for 3916 (https://github.com/matrix-org/matrix-spec-proposals/pull/4180) */
 			("uk.tcpip.msc4133".to_owned(), true), /* Extending User Profile API with Key:Value Pairs (https://github.com/matrix-org/matrix-spec-proposals/pull/4133) */
 			("us.cloke.msc4175".to_owned(), true), /* Profile field for user time zone (https://github.com/matrix-org/matrix-spec-proposals/pull/4175) */
+			("org.matrix.simplified_msc3575".to_owned(), true), /* Simplified Sliding sync (https://github.com/matrix-org/matrix-spec-proposals/pull/4186) */
 		]),
 	};
 

src/api/client/voip.rs

@@ -38,7 +38,7 @@ pub(crate) async fn turn_server_route(
 		let user = body.sender_user.unwrap_or_else(|| {
 			UserId::parse_with_server_name(
 				utils::random_string(RANDOM_USER_ID_LENGTH).to_lowercase(),
-				&services.globals.config.server_name,
+				&services.server.config.server_name,
 			)
 			.unwrap()
 		});

src/api/router.rs

@@ -34,6 +34,7 @@ pub fn build(router: Router<State>, server: &Server) -> Router<State> {
 		.ruma_route(&client::register_route)
 		.ruma_route(&client::get_login_types_route)
 		.ruma_route(&client::login_route)
+		.ruma_route(&client::login_token_route)
 		.ruma_route(&client::whoami_route)
 		.ruma_route(&client::logout_route)
 		.ruma_route(&client::logout_all_route)
@@ -99,6 +100,7 @@ pub fn build(router: Router<State>, server: &Server) -> Router<State> {
 		.ruma_route(&client::join_room_by_id_route)
 		.ruma_route(&client::join_room_by_id_or_alias_route)
 		.ruma_route(&client::joined_members_route)
+		.ruma_route(&client::knock_room_route)
 		.ruma_route(&client::leave_room_route)
 		.ruma_route(&client::forget_room_route)
 		.ruma_route(&client::joined_rooms_route)
@@ -144,6 +146,7 @@ pub fn build(router: Router<State>, server: &Server) -> Router<State> {
 		)
 		.ruma_route(&client::sync_events_route)
 		.ruma_route(&client::sync_events_v4_route)
+		.ruma_route(&client::sync_events_v5_route)
 		.ruma_route(&client::get_context_route)
 		.ruma_route(&client::get_message_events_route)
 		.ruma_route(&client::search_events_route)
@@ -204,8 +207,10 @@ pub fn build(router: Router<State>, server: &Server) -> Router<State> {
 			.ruma_route(&server::get_room_state_route)
 			.ruma_route(&server::get_room_state_ids_route)
 			.ruma_route(&server::create_leave_event_template_route)
+			.ruma_route(&server::create_knock_event_template_route)
 			.ruma_route(&server::create_leave_event_v1_route)
 			.ruma_route(&server::create_leave_event_v2_route)
+			.ruma_route(&server::create_knock_event_v1_route)
 			.ruma_route(&server::create_join_event_template_route)
 			.ruma_route(&server::create_join_event_v1_route)
 			.ruma_route(&server::create_join_event_v2_route)

src/api/router/auth.rs

@@ -71,7 +71,7 @@ pub(super) async fn auth(
 		match metadata {
 			| &get_public_rooms::v3::Request::METADATA => {
 				if !services
-					.globals
+					.server
 					.config
 					.allow_public_room_directory_without_auth
 				{
@@ -94,7 +94,7 @@ pub(super) async fn auth(
 			| &get_display_name::v3::Request::METADATA
 			| &get_avatar_url::v3::Request::METADATA
 			| &get_timezone_key::unstable::Request::METADATA => {
-				if services.globals.config.require_auth_for_profile_requests {
+				if services.server.config.require_auth_for_profile_requests {
 					match token {
 						| Token::Appservice(_) | Token::User(_) => {
 							// we should have validated the token above
@@ -127,7 +127,7 @@ pub(super) async fn auth(
 		}),
 		| (AuthScheme::AccessToken, Token::None) => match metadata {
 			| &get_turn_server_info::v3::Request::METADATA => {
-				if services.globals.config.turn_allow_guests {
+				if services.server.config.turn_allow_guests {
 					Ok(Auth {
 						origin: None,
 						sender_user: None,

src/api/router/request.rs

@@ -32,7 +32,7 @@ pub(super) async fn from(
 	let query = serde_html_form::from_str(query)
 		.map_err(|e| err!(Request(Unknown("Failed to read query parameters: {e}"))))?;
 
-	let max_body_size = services.globals.config.max_request_size;
+	let max_body_size = services.server.config.max_request_size;
 
 	let body = axum::body::to_bytes(body, max_body_size)
 		.await

src/api/server/backfill.rs

@@ -2,10 +2,10 @@ use std::cmp;
 
 use axum::extract::State;
 use conduwuit::{
-	utils::{IterStream, ReadyExt},
+	utils::{stream::TryTools, IterStream, ReadyExt},
 	PduCount, Result,
 };
-use futures::{FutureExt, StreamExt};
+use futures::{FutureExt, StreamExt, TryStreamExt};
 use ruma::{api::federation::backfill::get_backfill, uint, MilliSecondsSinceUnixEpoch};
 
 use super::AccessCheck;
@@ -57,26 +57,30 @@ pub(crate) async fn get_backfill_route(
 			.rooms
 			.timeline
 			.pdus_rev(None, &body.room_id, Some(from.saturating_add(1)))
-			.await?
-			.take(limit)
-			.filter_map(|(_, pdu)| async move {
-				services
+			.try_take(limit)
+			.try_filter_map(|(_, pdu)| async move {
+				Ok(services
 					.rooms
 					.state_accessor
 					.server_can_see_event(body.origin(), &pdu.room_id, &pdu.event_id)
 					.await
-					.then_some(pdu)
+					.then_some(pdu))
 			})
-			.filter_map(|pdu| async move {
-				services
+			.try_filter_map(|pdu| async move {
+				Ok(services
 					.rooms
 					.timeline
 					.get_pdu_json(&pdu.event_id)
 					.await
-					.ok()
+					.ok())
 			})
-			.then(|pdu| services.sending.convert_to_outgoing_federation_event(pdu))
-			.collect()
-			.await,
+			.and_then(|pdu| {
+				services
+					.sending
+					.convert_to_outgoing_federation_event(pdu)
+					.map(Ok)
+			})
+			.try_collect()
+			.await?,
 	})
 }

src/api/server/invite.rs

@@ -6,8 +6,9 @@ use ruma::{
 	api::{client::error::ErrorKind, federation::membership::create_invite},
 	events::room::member::{MembershipState, RoomMemberEventContent},
 	serde::JsonObject,
-	CanonicalJsonValue, OwnedEventId, OwnedUserId, UserId,
+	CanonicalJsonValue, OwnedUserId, UserId,
 };
+use service::pdu::gen_event_id;
 
 use crate::Ruma;
 
@@ -36,7 +37,7 @@ pub(crate) async fn create_invite_route(
 
 	if let Some(server) = body.room_id.server_name() {
 		if services
-			.globals
+			.server
 			.config
 			.forbidden_remote_server_names
 			.contains(&server.to_owned())
@@ -46,7 +47,7 @@ pub(crate) async fn create_invite_route(
 	}
 
 	if services
-		.globals
+		.server
 		.config
 		.forbidden_remote_server_names
 		.contains(body.origin())
@@ -86,12 +87,7 @@ pub(crate) async fn create_invite_route(
 		.map_err(|e| err!(Request(InvalidParam("Failed to sign event: {e}"))))?;
 
 	// Generate event id
-	let event_id = OwnedEventId::parse(format!(
-		"${}",
-		ruma::signatures::reference_hash(&signed_event, &body.room_version)
-			.expect("ruma can calculate reference hashes")
-	))
-	.expect("ruma's reference hashes are valid event ids");
+	let event_id = gen_event_id(&signed_event, &body.room_version)?;
 
 	// Add event_id back
 	signed_event.insert("event_id".to_owned(), CanonicalJsonValue::String(event_id.to_string()));
@@ -115,12 +111,12 @@ pub(crate) async fn create_invite_route(
 	let mut invite_state = body.invite_room_state.clone();
 
 	let mut event: JsonObject = serde_json::from_str(body.event.get())
-		.map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "Invalid invite event bytes."))?;
+		.map_err(|e| err!(Request(BadJson("Invalid invite event PDU: {e}"))))?;
 
 	event.insert("event_id".to_owned(), "$placeholder".into());
 
 	let pdu: PduEvent = serde_json::from_value(event.into())
-		.map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "Invalid invite event."))?;
+		.map_err(|e| err!(Request(BadJson("Invalid invite event PDU: {e}"))))?;
 
 	invite_state.push(pdu.to_stripped_state_event());
 

src/api/server/make_join.rs

@@ -42,7 +42,7 @@ pub(crate) async fn create_join_event_template_route(
 		.await?;
 
 	if services
-		.globals
+		.server
 		.config
 		.forbidden_remote_server_names
 		.contains(body.origin())
@@ -59,7 +59,7 @@ pub(crate) async fn create_join_event_template_route(
 
 	if let Some(server) = body.room_id.server_name() {
 		if services
-			.globals
+			.server
 			.config
 			.forbidden_remote_server_names
 			.contains(&server.to_owned())

src/api/server/make_knock.rs

@@ -1,5 +1,5 @@
 use axum::extract::State;
-use conduwuit::Err;
+use conduwuit::{debug_warn, Err};
 use ruma::{
 	api::{client::error::ErrorKind, federation::knock::create_knock_event_template},
 	events::room::member::{MembershipState, RoomMemberEventContent},
@@ -15,7 +15,8 @@ use crate::{service::pdu::PduBuilder, Error, Result, Ruma};
 ///
 /// Creates a knock template.
 pub(crate) async fn create_knock_event_template_route(
-	State(services): State<crate::State>, body: Ruma<create_knock_event_template::v1::Request>,
+	State(services): State<crate::State>,
+	body: Ruma<create_knock_event_template::v1::Request>,
 ) -> Result<create_knock_event_template::v1::Response> {
 	if !services.rooms.metadata.exists(&body.room_id).await {
 		return Err!(Request(NotFound("Room is unknown to this server.")));
@@ -33,14 +34,14 @@ pub(crate) async fn create_knock_event_template_route(
 		.await?;
 
 	if services
-		.globals
+		.server
 		.config
 		.forbidden_remote_server_names
 		.contains(body.origin())
 	{
 		warn!(
-			"Server {} for remote user {} tried knocking room ID {} which has a server name that is globally \
-			 forbidden. Rejecting.",
+			"Server {} for remote user {} tried knocking room ID {} which has a server name \
+			 that is globally forbidden. Rejecting.",
 			body.origin(),
 			&body.user_id,
 			&body.room_id,
@@ -50,7 +51,7 @@ pub(crate) async fn create_knock_event_template_route(
 
 	if let Some(server) = body.room_id.server_name() {
 		if services
-			.globals
+			.server
 			.config
 			.forbidden_remote_server_names
 			.contains(&server.to_owned())
@@ -63,29 +64,44 @@ pub(crate) async fn create_knock_event_template_route(
 
 	if matches!(room_version_id, V1 | V2 | V3 | V4 | V5 | V6) {
 		return Err(Error::BadRequest(
-			ErrorKind::IncompatibleRoomVersion {
-				room_version: room_version_id,
-			},
+			ErrorKind::IncompatibleRoomVersion { room_version: room_version_id },
 			"Room version does not support knocking.",
 		));
 	}
 
 	if !body.ver.contains(&room_version_id) {
 		return Err(Error::BadRequest(
-			ErrorKind::IncompatibleRoomVersion {
-				room_version: room_version_id,
-			},
+			ErrorKind::IncompatibleRoomVersion { room_version: room_version_id },
 			"Your homeserver does not support the features required to knock on this room.",
 		));
 	}
 
 	let state_lock = services.rooms.state.mutex.lock(&body.room_id).await;
 
+	if let Ok(membership) = services
+		.rooms
+		.state_accessor
+		.get_member(&body.room_id, &body.user_id)
+		.await
+	{
+		if membership.membership == MembershipState::Ban {
+			debug_warn!(
+				"Remote user {} is banned from {} but attempted to knock",
+				&body.user_id,
+				&body.room_id
+			);
+			return Err!(Request(Forbidden("You cannot knock on a room you are banned from.")));
+		}
+	}
+
 	let (_pdu, mut pdu_json) = services
 		.rooms
 		.timeline
 		.create_hash_and_sign_event(
-			PduBuilder::state(body.user_id.to_string(), &RoomMemberEventContent::new(MembershipState::Knock)),
+			PduBuilder::state(
+				body.user_id.to_string(),
+				&RoomMemberEventContent::new(MembershipState::Knock),
+			),
 			&body.user_id,
 			&body.room_id,
 			&state_lock,

src/api/server/make_leave.rs

@@ -9,7 +9,7 @@ use serde_json::value::to_raw_value;
 use super::make_join::maybe_strip_event_id;
 use crate::{service::pdu::PduBuilder, Ruma};
 
-/// # `PUT /_matrix/federation/v1/make_leave/{roomId}/{eventId}`
+/// # `GET /_matrix/federation/v1/make_leave/{roomId}/{eventId}`
 ///
 /// Creates a leave template.
 pub(crate) async fn create_leave_event_template_route(
@@ -21,7 +21,9 @@ pub(crate) async fn create_leave_event_template_route(
 	}
 
 	if body.user_id.server_name() != body.origin() {
-		return Err!(Request(BadJson("Not allowed to leave on behalf of another server/user.")));
+		return Err!(Request(Forbidden(
+			"Not allowed to leave on behalf of another server/user."
+		)));
 	}
 
 	// ACL check origin

src/api/server/media.rs

@@ -14,7 +14,12 @@ use crate::Ruma;
 /// # `GET /_matrix/federation/v1/media/download/{mediaId}`
 ///
 /// Load media from our server.
-#[tracing::instrument(skip_all, fields(%client), name = "media_get")]
+#[tracing::instrument(
+	name = "media_get",
+	level = "debug",
+	skip_all,
+	fields(%client)
+)]
 pub(crate) async fn get_content_route(
 	State(services): State<crate::State>,
 	InsecureClientIp(client): InsecureClientIp,
@@ -51,7 +56,12 @@ pub(crate) async fn get_content_route(
 /// # `GET /_matrix/federation/v1/media/thumbnail/{mediaId}`
 ///
 /// Load media thumbnail from our server.
-#[tracing::instrument(skip_all, fields(%client), name = "media_thumbnail_get")]
+#[tracing::instrument(
+	name = "media_thumbnail_get",
+	level = "debug",
+	skip_all,
+	fields(%client)
+)]
 pub(crate) async fn get_content_thumbnail_route(
 	State(services): State<crate::State>,
 	InsecureClientIp(client): InsecureClientIp,

src/api/server/mod.rs

@@ -6,6 +6,7 @@ pub(super) mod hierarchy;
 pub(super) mod invite;
 pub(super) mod key;
 pub(super) mod make_join;
+pub(super) mod make_knock;
 pub(super) mod make_leave;
 pub(super) mod media;
 pub(super) mod openid;
@@ -13,6 +14,7 @@ pub(super) mod publicrooms;
 pub(super) mod query;
 pub(super) mod send;
 pub(super) mod send_join;
+pub(super) mod send_knock;
 pub(super) mod send_leave;
 pub(super) mod state;
 pub(super) mod state_ids;
@@ -28,6 +30,7 @@ pub(super) use hierarchy::*;
 pub(super) use invite::*;
 pub(super) use key::*;
 pub(super) use make_join::*;
+pub(super) use make_knock::*;
 pub(super) use make_leave::*;
 pub(super) use media::*;
 pub(super) use openid::*;
@@ -35,6 +38,7 @@ pub(super) use publicrooms::*;
 pub(super) use query::*;
 pub(super) use send::*;
 pub(super) use send_join::*;
+pub(super) use send_knock::*;
 pub(super) use send_leave::*;
 pub(super) use state::*;
 pub(super) use state_ids::*;

src/api/server/publicrooms.rs

@@ -20,7 +20,7 @@ pub(crate) async fn get_public_rooms_filtered_route(
 	body: Ruma<get_public_rooms_filtered::v1::Request>,
 ) -> Result<get_public_rooms_filtered::v1::Response> {
 	if !services
-		.globals
+		.server
 		.config
 		.allow_public_room_directory_over_federation
 	{

src/api/server/query.rs

@@ -63,7 +63,7 @@ pub(crate) async fn get_profile_information_route(
 	body: Ruma<get_profile_information::v1::Request>,
 ) -> Result<get_profile_information::v1::Response> {
 	if !services
-		.globals
+		.server
 		.config
 		.allow_inbound_profile_lookup_federation_requests
 	{

src/api/server/send.rs

@@ -39,7 +39,15 @@ type ResolvedMap = BTreeMap<OwnedEventId, Result<()>>;
 /// # `PUT /_matrix/federation/v1/send/{txnId}`
 ///
 /// Push EDUs and PDUs to this server.
-#[tracing::instrument(skip_all, fields(%client, origin = body.origin().as_str()), name = "send")]
+#[tracing::instrument(
+	name = "send",
+	level = "debug",
+	skip_all,
+	fields(
+		%client,
+		origin = body.origin().as_str()
+	),
+)]
 pub(crate) async fn send_transaction_message_route(
 	State(services): State<crate::State>,
 	InsecureClientIp(client): InsecureClientIp,
@@ -135,6 +143,7 @@ async fn handle_pdus(
 			.rooms
 			.event_handler
 			.handle_incoming_pdu(origin, &room_id, &event_id, value, true)
+			.boxed()
 			.await
 			.map(|_| ());
 
@@ -300,7 +309,7 @@ async fn handle_edu_typing(
 	origin: &ServerName,
 	typing: TypingContent,
 ) {
-	if !services.globals.config.allow_incoming_typing {
+	if !services.server.config.allow_incoming_typing {
 		return;
 	}
 
@@ -335,7 +344,7 @@ async fn handle_edu_typing(
 		if typing.typing {
 			let timeout = utils::millis_since_unix_epoch().saturating_add(
 				services
-					.globals
+					.server
 					.config
 					.typing_federation_timeout_s
 					.saturating_mul(1000),

src/api/server/send_join.rs

@@ -186,14 +186,13 @@ async fn create_join_event(
 		.map_err(|e| err!(Request(InvalidParam(warn!("Failed to sign send_join event: {e}")))))?;
 
 	let origin: OwnedServerName = serde_json::from_value(
-		serde_json::to_value(
-			value
-				.get("origin")
-				.ok_or_else(|| err!(Request(BadJson("Event missing origin property."))))?,
-		)
-		.expect("CanonicalJson is valid json value"),
+		value
+			.get("origin")
+			.ok_or_else(|| err!(Request(BadJson("Event does not have an origin server name."))))?
+			.clone()
+			.into(),
 	)
-	.map_err(|e| err!(Request(BadJson(warn!("origin field is not a valid server name: {e}")))))?;
+	.map_err(|e| err!(Request(BadJson("Event has an invalid origin server name: {e}"))))?;
 
 	let mutex_lock = services
 		.rooms
@@ -269,7 +268,7 @@ pub(crate) async fn create_join_event_v1_route(
 	body: Ruma<create_join_event::v1::Request>,
 ) -> Result<create_join_event::v1::Response> {
 	if services
-		.globals
+		.server
 		.config
 		.forbidden_remote_server_names
 		.contains(body.origin())
@@ -285,7 +284,7 @@ pub(crate) async fn create_join_event_v1_route(
 
 	if let Some(server) = body.room_id.server_name() {
 		if services
-			.globals
+			.server
 			.config
 			.forbidden_remote_server_names
 			.contains(&server.to_owned())
@@ -317,7 +316,7 @@ pub(crate) async fn create_join_event_v2_route(
 	body: Ruma<create_join_event::v2::Request>,
 ) -> Result<create_join_event::v2::Response> {
 	if services
-		.globals
+		.server
 		.config
 		.forbidden_remote_server_names
 		.contains(body.origin())
@@ -327,7 +326,7 @@ pub(crate) async fn create_join_event_v2_route(
 
 	if let Some(server) = body.room_id.server_name() {
 		if services
-			.globals
+			.server
 			.config
 			.forbidden_remote_server_names
 			.contains(&server.to_owned())

src/api/server/send_knock.rs

@@ -1,7 +1,8 @@
 use axum::extract::State;
-use conduwuit::{err, pdu::gen_event_id_canonical_json, warn, Err, Error, PduEvent, Result};
+use conduwuit::{err, pdu::gen_event_id_canonical_json, warn, Err, PduEvent, Result};
+use futures::FutureExt;
 use ruma::{
-	api::{client::error::ErrorKind, federation::knock::send_knock},
+	api::federation::knock::send_knock,
 	events::{
 		room::member::{MembershipState, RoomMemberEventContent},
 		StateEventType,
@@ -17,16 +18,18 @@ use crate::Ruma;
 ///
 /// Submits a signed knock event.
 pub(crate) async fn create_knock_event_v1_route(
-	State(services): State<crate::State>, body: Ruma<send_knock::v1::Request>,
+	State(services): State<crate::State>,
+	body: Ruma<send_knock::v1::Request>,
 ) -> Result<send_knock::v1::Response> {
 	if services
-		.globals
+		.server
 		.config
 		.forbidden_remote_server_names
 		.contains(body.origin())
 	{
 		warn!(
-			"Server {} tried knocking room ID {} who has a server name that is globally forbidden. Rejecting.",
+			"Server {} tried knocking room ID {} who has a server name that is globally \
+			 forbidden. Rejecting.",
 			body.origin(),
 			&body.room_id,
 		);
@@ -35,13 +38,14 @@ pub(crate) async fn create_knock_event_v1_route(
 
 	if let Some(server) = body.room_id.server_name() {
 		if services
-			.globals
+			.server
 			.config
 			.forbidden_remote_server_names
 			.contains(&server.to_owned())
 		{
 			warn!(
-				"Server {} tried knocking room ID {} which has a server name that is globally forbidden. Rejecting.",
+				"Server {} tried knocking room ID {} which has a server name that is globally \
+				 forbidden. Rejecting.",
 				body.origin(),
 				&body.room_id,
 			);
@@ -50,7 +54,7 @@ pub(crate) async fn create_knock_event_v1_route(
 	}
 
 	if !services.rooms.metadata.exists(&body.room_id).await {
-		return Err(Error::BadRequest(ErrorKind::NotFound, "Room is unknown to this server."));
+		return Err!(Request(NotFound("Room is unknown to this server.")));
 	}
 
 	// ACL check origin server
@@ -74,44 +78,42 @@ pub(crate) async fn create_knock_event_v1_route(
 	let event_type: StateEventType = serde_json::from_value(
 		value
 			.get("type")
-			.ok_or_else(|| Error::BadRequest(ErrorKind::InvalidParam, "Event missing type property."))?
+			.ok_or_else(|| err!(Request(InvalidParam("Event has no event type."))))?
 			.clone()
 			.into(),
 	)
-	.map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "Event has invalid event type."))?;
+	.map_err(|e| err!(Request(InvalidParam("Event has invalid event type: {e}"))))?;
 
 	if event_type != StateEventType::RoomMember {
-		return Err(Error::BadRequest(
-			ErrorKind::InvalidParam,
+		return Err!(Request(InvalidParam(
 			"Not allowed to send non-membership state event to knock endpoint.",
-		));
+		)));
 	}
 
 	let content: RoomMemberEventContent = serde_json::from_value(
 		value
 			.get("content")
-			.ok_or_else(|| Error::BadRequest(ErrorKind::InvalidParam, "Event missing content property"))?
+			.ok_or_else(|| err!(Request(InvalidParam("Membership event has no content"))))?
 			.clone()
 			.into(),
 	)
-	.map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "Event content is empty or invalid"))?;
+	.map_err(|e| err!(Request(InvalidParam("Event has invalid membership content: {e}"))))?;
 
 	if content.membership != MembershipState::Knock {
-		return Err(Error::BadRequest(
-			ErrorKind::InvalidParam,
-			"Not allowed to send a non-knock membership event to knock endpoint.",
-		));
+		return Err!(Request(InvalidParam(
+			"Not allowed to send a non-knock membership event to knock endpoint."
+		)));
 	}
 
 	// ACL check sender server name
 	let sender: OwnedUserId = serde_json::from_value(
 		value
 			.get("sender")
-			.ok_or_else(|| Error::BadRequest(ErrorKind::InvalidParam, "Event missing sender property."))?
+			.ok_or_else(|| err!(Request(InvalidParam("Event has no sender user ID."))))?
 			.clone()
 			.into(),
 	)
-	.map_err(|_| Error::BadRequest(ErrorKind::BadJson, "sender is not a valid user ID."))?;
+	.map_err(|e| err!(Request(BadJson("Event sender is not a valid user ID: {e}"))))?;
 
 	services
 		.rooms
@@ -127,36 +129,32 @@ pub(crate) async fn create_knock_event_v1_route(
 	let state_key: OwnedUserId = serde_json::from_value(
 		value
 			.get("state_key")
-			.ok_or_else(|| Error::BadRequest(ErrorKind::InvalidParam, "Event missing state_key property."))?
+			.ok_or_else(|| err!(Request(InvalidParam("Event does not have a state_key"))))?
 			.clone()
 			.into(),
 	)
-	.map_err(|_| Error::BadRequest(ErrorKind::BadJson, "state_key is invalid or not a user ID."))?;
+	.map_err(|e| err!(Request(BadJson("Event does not have a valid state_key: {e}"))))?;
 
 	if state_key != sender {
-		return Err(Error::BadRequest(
-			ErrorKind::InvalidParam,
-			"State key does not match sender user",
-		));
+		return Err!(Request(InvalidParam("state_key does not match sender user of event.")));
 	};
 
 	let origin: OwnedServerName = serde_json::from_value(
-		serde_json::to_value(
-			value
-				.get("origin")
-				.ok_or_else(|| Error::BadRequest(ErrorKind::InvalidParam, "Event missing origin property."))?,
-		)
-		.expect("CanonicalJson is valid json value"),
+		value
+			.get("origin")
+			.ok_or_else(|| err!(Request(BadJson("Event does not have an origin server name."))))?
+			.clone()
+			.into(),
 	)
-	.map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "origin is not a server name."))?;
+	.map_err(|e| err!(Request(BadJson("Event has an invalid origin server name: {e}"))))?;
 
 	let mut event: JsonObject = serde_json::from_str(body.pdu.get())
-		.map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "Invalid knock event PDU."))?;
+		.map_err(|e| err!(Request(InvalidParam("Invalid knock event PDU: {e}"))))?;
 
 	event.insert("event_id".to_owned(), "$placeholder".into());
 
 	let pdu: PduEvent = serde_json::from_value(event.into())
-		.map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "Invalid knock event PDU."))?;
+		.map_err(|e| err!(Request(InvalidParam("Invalid knock event PDU: {e}"))))?;
 
 	let mutex_lock = services
 		.rooms
@@ -169,19 +167,18 @@ pub(crate) async fn create_knock_event_v1_route(
 		.rooms
 		.event_handler
 		.handle_incoming_pdu(&origin, &body.room_id, &event_id, value.clone(), true)
+		.boxed()
 		.await?
 		.ok_or_else(|| err!(Request(InvalidParam("Could not accept as timeline event."))))?;
 
 	drop(mutex_lock);
 
-	let knock_room_state = services.rooms.state.summary_stripped(&pdu).await;
-
 	services
 		.sending
 		.send_pdu_room(&body.room_id, &pdu_id)
 		.await?;
 
-	Ok(send_knock::v1::Response {
-		knock_room_state,
-	})
+	let knock_room_state = services.rooms.state.summary_stripped(&pdu).await;
+
+	Ok(send_knock::v1::Response { knock_room_state })
 }

src/api/server/utils.rs

@@ -1,6 +1,6 @@
 use conduwuit::{implement, is_false, Err, Result};
 use conduwuit_service::Services;
-use futures::{future::OptionFuture, join, FutureExt};
+use futures::{future::OptionFuture, join, FutureExt, StreamExt};
 use ruma::{EventId, RoomId, ServerName};
 
 pub(super) struct AccessCheck<'a> {
@@ -31,6 +31,15 @@ pub(super) async fn check(&self) -> Result {
 		.state_cache
 		.server_in_room(self.origin, self.room_id);
 
+	// if any user on our homeserver is trying to knock this room, we'll need to
+	// acknowledge bans or leaves
+	let user_is_knocking = self
+		.services
+		.rooms
+		.state_cache
+		.room_members_knocked(self.room_id)
+		.count();
+
 	let server_can_see: OptionFuture<_> = self
 		.event_id
 		.map(|event_id| {
@@ -42,14 +51,14 @@ pub(super) async fn check(&self) -> Result {
 		})
 		.into();
 
-	let (world_readable, server_in_room, server_can_see, acl_check) =
-		join!(world_readable, server_in_room, server_can_see, acl_check);
+	let (world_readable, server_in_room, server_can_see, acl_check, user_is_knocking) =
+		join!(world_readable, server_in_room, server_can_see, acl_check, user_is_knocking);
 
 	if !acl_check {
 		return Err!(Request(Forbidden("Server access denied.")));
 	}
 
-	if !world_readable && !server_in_room {
+	if !world_readable && !server_in_room && user_is_knocking == 0 {
 		return Err!(Request(Forbidden("Server is not in room.")));
 	}
 

src/core/Cargo.toml

@@ -36,6 +36,7 @@ jemalloc_stats = [
     "tikv-jemalloc-ctl/stats",
     "tikv-jemallocator/stats",
 ]
+jemalloc_conf = []
 hardened_malloc = [
 	"dep:hardened_malloc-rs"
 ]
@@ -50,6 +51,9 @@ zstd_compression = [
 ]
 perf_measurements = []
 sentry_telemetry = []
+conduwuit_mods = [
+    "dep:libloading"
+]
 
 [dependencies]
 argon2.workspace = true
@@ -71,11 +75,11 @@ figment.workspace = true
 futures.workspace = true
 http-body-util.workspace = true
 http.workspace = true
-image.workspace = true
 ipaddress.workspace = true
 itertools.workspace = true
 libc.workspace = true
 libloading.workspace = true
+libloading.optional = true
 log.workspace = true
 num-traits.workspace = true
 rand.workspace = true

src/core/alloc/default.rs

@@ -1,8 +1,11 @@
 //! Default allocator with no special features
 
+/// Always returns Ok
+pub fn trim<I: Into<Option<usize>>>(_: I) -> crate::Result { Ok(()) }
+
 /// Always returns None
 #[must_use]
-pub fn memory_stats() -> Option<String> { None }
+pub fn memory_stats(_opts: &str) -> Option<String> { None }
 
 /// Always returns None
 #[must_use]

src/core/alloc/hardened.rs

@@ -3,11 +3,13 @@
 #[global_allocator]
 static HMALLOC: hardened_malloc_rs::HardenedMalloc = hardened_malloc_rs::HardenedMalloc;
 
-#[must_use]
-//TODO: get usage
-pub fn memory_usage() -> Option<string> { None }
+pub fn trim<I: Into<Option<usize>>>(_: I) -> crate::Result { Ok(()) }
 
 #[must_use]
-pub fn memory_stats() -> Option<String> {
+//TODO: get usage
+pub fn memory_usage() -> Option<String> { None }
+
+#[must_use]
+pub fn memory_stats(_opts: &str) -> Option<String> {
 	Some("Extended statistics are not available from hardened_malloc.".to_owned())
 }

src/core/alloc/je.rs

@@ -1,18 +1,58 @@
 //! jemalloc allocator
 
-use std::ffi::{c_char, c_void};
+use std::{
+	cell::OnceCell,
+	ffi::{c_char, c_void, CStr},
+	fmt::Debug,
+	sync::RwLock,
+};
 
+use arrayvec::ArrayVec;
+use tikv_jemalloc_ctl as mallctl;
 use tikv_jemalloc_sys as ffi;
 use tikv_jemallocator as jemalloc;
 
+use crate::{
+	err, is_equal_to, is_nonzero,
+	utils::{math, math::Tried},
+	Result,
+};
+
+#[cfg(feature = "jemalloc_conf")]
+#[unsafe(no_mangle)]
+pub static malloc_conf: &[u8] = b"\
+metadata_thp:always\
+,percpu_arena:percpu\
+,background_thread:true\
+,max_background_threads:-1\
+,lg_extent_max_active_fit:4\
+,oversize_threshold:16777216\
+,tcache_max:2097152\
+,dirty_decay_ms:16000\
+,muzzy_decay_ms:144000\
+,prof_active:false\
+\0";
+
 #[global_allocator]
 static JEMALLOC: jemalloc::Jemalloc = jemalloc::Jemalloc;
+static CONTROL: RwLock<()> = RwLock::new(());
+
+type Name = ArrayVec<u8, NAME_MAX>;
+type Key = ArrayVec<usize, KEY_SEGS>;
+
+const NAME_MAX: usize = 128;
+const KEY_SEGS: usize = 8;
+
+#[crate::ctor]
+fn _static_initialization() {
+	acq_epoch().expect("pre-initialization of jemalloc failed");
+	acq_epoch().expect("pre-initialization of jemalloc failed");
+}
 
 #[must_use]
 #[cfg(feature = "jemalloc_stats")]
 pub fn memory_usage() -> Option<String> {
 	use mallctl::stats;
-	use tikv_jemalloc_ctl as mallctl;
 
 	let mibs = |input: Result<usize, mallctl::Error>| {
 		let input = input.unwrap_or_default();
@@ -22,6 +62,9 @@ pub fn memory_usage() -> Option<String> {
 		kibs / 1024.0
 	};
 
+	// Acquire the epoch; ensure latest stats are pulled in
+	acq_epoch().ok()?;
+
 	let allocated = mibs(stats::allocated::read());
 	let active = mibs(stats::active::read());
 	let mapped = mibs(stats::mapped::read());
@@ -39,34 +82,293 @@ pub fn memory_usage() -> Option<String> {
 #[cfg(not(feature = "jemalloc_stats"))]
 pub fn memory_usage() -> Option<String> { None }
 
-#[must_use]
-pub fn memory_stats() -> Option<String> {
-	const MAX_LENGTH: usize = 65536 - 4096;
+pub fn memory_stats(opts: &str) -> Option<String> {
+	const MAX_LENGTH: usize = 1_048_576;
 
-	let opts_s = "d";
 	let mut str = String::new();
-
 	let opaque = std::ptr::from_mut(&mut str).cast::<c_void>();
-	let opts_p: *const c_char = std::ffi::CString::new(opts_s)
+	let opts_p: *const c_char = std::ffi::CString::new(opts)
 		.expect("cstring")
 		.into_raw()
 		.cast_const();
 
+	// Acquire the epoch; ensure latest stats are pulled in
+	acq_epoch().ok()?;
+
 	// SAFETY: calls malloc_stats_print() with our string instance which must remain
 	// in this frame. https://docs.rs/tikv-jemalloc-sys/latest/tikv_jemalloc_sys/fn.malloc_stats_print.html
 	unsafe { ffi::malloc_stats_print(Some(malloc_stats_cb), opaque, opts_p) };
 
 	str.truncate(MAX_LENGTH);
-	Some(format!("<pre><code>{str}</code></pre>"))
+
+	Some(str)
 }
 
 unsafe extern "C" fn malloc_stats_cb(opaque: *mut c_void, msg: *const c_char) {
 	// SAFETY: we have to trust the opaque points to our String
-	let res: &mut String = unsafe { opaque.cast::<String>().as_mut().unwrap() };
+	let res: &mut String = unsafe {
+		opaque
+			.cast::<String>()
+			.as_mut()
+			.expect("failed to cast void* to &mut String")
+	};
 
 	// SAFETY: we have to trust the string is null terminated.
-	let msg = unsafe { std::ffi::CStr::from_ptr(msg) };
+	let msg = unsafe { CStr::from_ptr(msg) };
 
 	let msg = String::from_utf8_lossy(msg.to_bytes());
 	res.push_str(msg.as_ref());
 }
+
+macro_rules! mallctl {
+	($name:expr) => {{
+		thread_local! {
+			static KEY: OnceCell<Key> = OnceCell::default();
+		};
+
+		KEY.with(|once| {
+			once.get_or_init(move || key($name).expect("failed to translate name into mib key"))
+				.clone()
+		})
+	}};
+}
+
+pub mod this_thread {
+	use super::{is_nonzero, key, math, Debug, Key, OnceCell, Result};
+
+	thread_local! {
+		static ALLOCATED_BYTES: OnceCell<&'static u64> = const { OnceCell::new() };
+		static DEALLOCATED_BYTES: OnceCell<&'static u64> = const { OnceCell::new() };
+	}
+
+	pub fn trim() -> Result { decay().and_then(|()| purge()) }
+
+	pub fn purge() -> Result { notify(mallctl!("arena.0.purge")) }
+
+	pub fn decay() -> Result { notify(mallctl!("arena.0.decay")) }
+
+	pub fn idle() -> Result { super::notify(&mallctl!("thread.idle")) }
+
+	pub fn flush() -> Result { super::notify(&mallctl!("thread.tcache.flush")) }
+
+	pub fn set_muzzy_decay(decay_ms: isize) -> Result<isize> {
+		set(mallctl!("arena.0.muzzy_decay_ms"), decay_ms)
+	}
+
+	pub fn get_muzzy_decay() -> Result<isize> { get(mallctl!("arena.0.muzzy_decay_ms")) }
+
+	pub fn set_dirty_decay(decay_ms: isize) -> Result<isize> {
+		set(mallctl!("arena.0.dirty_decay_ms"), decay_ms)
+	}
+
+	pub fn get_dirty_decay() -> Result<isize> { get(mallctl!("arena.0.dirty_decay_ms")) }
+
+	pub fn cache_enable(enable: bool) -> Result<bool> {
+		super::set::<u8>(&mallctl!("thread.tcache.enabled"), enable.into()).map(is_nonzero!())
+	}
+
+	pub fn is_cache_enabled() -> Result<bool> {
+		super::get::<u8>(&mallctl!("thread.tcache.enabled")).map(is_nonzero!())
+	}
+
+	pub fn set_arena(id: usize) -> Result<usize> {
+		super::set::<u32>(&mallctl!("thread.arena"), id.try_into()?).and_then(math::try_into)
+	}
+
+	pub fn arena_id() -> Result<usize> {
+		super::get::<u32>(&mallctl!("thread.arena")).and_then(math::try_into)
+	}
+
+	pub fn prof_enable(enable: bool) -> Result<bool> {
+		super::set::<u8>(&mallctl!("thread.prof.active"), enable.into()).map(is_nonzero!())
+	}
+
+	pub fn is_prof_enabled() -> Result<bool> {
+		super::get::<u8>(&mallctl!("thread.prof.active")).map(is_nonzero!())
+	}
+
+	pub fn reset_peak() -> Result { super::notify(&mallctl!("thread.peak.reset")) }
+
+	pub fn peak() -> Result<u64> { super::get(&mallctl!("thread.peak.read")) }
+
+	#[inline]
+	#[must_use]
+	pub fn allocated() -> u64 {
+		*ALLOCATED_BYTES.with(|once| init_tls_cell(once, "thread.allocatedp"))
+	}
+
+	#[inline]
+	#[must_use]
+	pub fn deallocated() -> u64 {
+		*DEALLOCATED_BYTES.with(|once| init_tls_cell(once, "thread.deallocatedp"))
+	}
+
+	fn notify(key: Key) -> Result { super::notify_by_arena(Some(arena_id()?), key) }
+
+	fn set<T>(key: Key, val: T) -> Result<T>
+	where
+		T: Copy + Debug,
+	{
+		super::set_by_arena(Some(arena_id()?), key, val)
+	}
+
+	fn get<T>(key: Key) -> Result<T>
+	where
+		T: Copy + Debug,
+	{
+		super::get_by_arena(Some(arena_id()?), key)
+	}
+
+	fn init_tls_cell(cell: &OnceCell<&'static u64>, name: &str) -> &'static u64 {
+		cell.get_or_init(|| {
+			let ptr: *const u64 = super::get(&mallctl!(name)).expect("failed to obtain pointer");
+
+			// SAFETY: ptr points directly to the internal state of jemalloc for this thread
+			unsafe { ptr.as_ref() }.expect("pointer must not be null")
+		})
+	}
+}
+
+pub fn stats_reset() -> Result { notify(&mallctl!("stats.mutexes.reset")) }
+
+pub fn prof_reset() -> Result { notify(&mallctl!("prof.reset")) }
+
+pub fn prof_enable(enable: bool) -> Result<bool> {
+	set::<u8>(&mallctl!("prof.active"), enable.into()).map(is_nonzero!())
+}
+
+pub fn is_prof_enabled() -> Result<bool> {
+	get::<u8>(&mallctl!("prof.active")).map(is_nonzero!())
+}
+
+pub fn trim<I: Into<Option<usize>> + Copy>(arena: I) -> Result {
+	decay(arena).and_then(|()| purge(arena))
+}
+
+pub fn purge<I: Into<Option<usize>>>(arena: I) -> Result {
+	notify_by_arena(arena.into(), mallctl!("arena.4096.purge"))
+}
+
+pub fn decay<I: Into<Option<usize>>>(arena: I) -> Result {
+	notify_by_arena(arena.into(), mallctl!("arena.4096.decay"))
+}
+
+pub fn set_muzzy_decay<I: Into<Option<usize>>>(arena: I, decay_ms: isize) -> Result<isize> {
+	if let Some(arena) = arena.into() {
+		set_by_arena(Some(arena), mallctl!("arena.4096.muzzy_decay_ms"), decay_ms)
+	} else {
+		set(&mallctl!("arenas.muzzy_decay_ms"), decay_ms)
+	}
+}
+
+pub fn set_dirty_decay<I: Into<Option<usize>>>(arena: I, decay_ms: isize) -> Result<isize> {
+	if let Some(arena) = arena.into() {
+		set_by_arena(Some(arena), mallctl!("arena.4096.dirty_decay_ms"), decay_ms)
+	} else {
+		set(&mallctl!("arenas.dirty_decay_ms"), decay_ms)
+	}
+}
+
+#[inline]
+#[must_use]
+pub fn is_affine_arena() -> bool { is_percpu_arena() || is_phycpu_arena() }
+
+#[inline]
+#[must_use]
+pub fn is_percpu_arena() -> bool { percpu_arenas().is_ok_and(is_equal_to!("percpu")) }
+
+#[inline]
+#[must_use]
+pub fn is_phycpu_arena() -> bool { percpu_arenas().is_ok_and(is_equal_to!("phycpu")) }
+
+pub fn percpu_arenas() -> Result<&'static str> {
+	let ptr = get::<*const c_char>(&mallctl!("opt.percpu_arena"))?;
+	//SAFETY: ptr points to a null-terminated string returned for opt.percpu_arena.
+	let cstr = unsafe { CStr::from_ptr(ptr) };
+	cstr.to_str().map_err(Into::into)
+}
+
+pub fn arenas() -> Result<usize> {
+	get::<u32>(&mallctl!("arenas.narenas")).and_then(math::try_into)
+}
+
+pub fn inc_epoch() -> Result<u64> { xchg(&mallctl!("epoch"), 1_u64) }
+
+pub fn acq_epoch() -> Result<u64> { xchg(&mallctl!("epoch"), 0_u64) }
+
+fn notify_by_arena(id: Option<usize>, mut key: Key) -> Result {
+	key[1] = id.unwrap_or(4096);
+	notify(&key)
+}
+
+fn set_by_arena<T>(id: Option<usize>, mut key: Key, val: T) -> Result<T>
+where
+	T: Copy + Debug,
+{
+	key[1] = id.unwrap_or(4096);
+	set(&key, val)
+}
+
+fn get_by_arena<T>(id: Option<usize>, mut key: Key) -> Result<T>
+where
+	T: Copy + Debug,
+{
+	key[1] = id.unwrap_or(4096);
+	get(&key)
+}
+
+fn notify(key: &Key) -> Result { xchg(key, ()) }
+
+fn set<T>(key: &Key, val: T) -> Result<T>
+where
+	T: Copy + Debug,
+{
+	let _lock = CONTROL.write()?;
+	let res = xchg(key, val)?;
+	inc_epoch()?;
+
+	Ok(res)
+}
+
+fn get<T>(key: &Key) -> Result<T>
+where
+	T: Copy + Debug,
+{
+	acq_epoch()?;
+	acq_epoch()?;
+
+	// SAFETY: T must be perfectly valid to receive value.
+	unsafe { mallctl::raw::read_mib(key.as_slice()) }.map_err(map_err)
+}
+
+fn xchg<T>(key: &Key, val: T) -> Result<T>
+where
+	T: Copy + Debug,
+{
+	// SAFETY: T must be the exact expected type.
+	unsafe { mallctl::raw::update_mib(key.as_slice(), val) }.map_err(map_err)
+}
+
+fn key(name: &str) -> Result<Key> {
+	// tikv asserts the output buffer length is tight to the number of required mibs
+	// so we slice that down here.
+	let segs = name.chars().filter(is_equal_to!(&'.')).count().try_add(1)?;
+
+	let name = self::name(name)?;
+	let mut buf = [0_usize; KEY_SEGS];
+	mallctl::raw::name_to_mib(name.as_slice(), &mut buf[0..segs])
+		.map_err(map_err)
+		.map(move |()| buf.into_iter().take(segs).collect())
+}
+
+fn name(name: &str) -> Result<Name> {
+	let mut buf = Name::new();
+	buf.try_extend_from_slice(name.as_bytes())?;
+	buf.try_extend_from_slice(b"\0")?;
+
+	Ok(buf)
+}
+
+fn map_err(error: tikv_jemalloc_ctl::Error) -> crate::Error {
+	err!("mallctl: {}", error.to_string())
+}

src/core/alloc/mod.rs

@@ -4,7 +4,7 @@
 #[cfg(all(not(target_env = "msvc"), feature = "jemalloc"))]
 pub mod je;
 #[cfg(all(not(target_env = "msvc"), feature = "jemalloc"))]
-pub use je::{memory_stats, memory_usage};
+pub use je::{memory_stats, memory_usage, trim};
 
 #[cfg(all(not(target_env = "msvc"), feature = "hardened_malloc", not(feature = "jemalloc")))]
 pub mod hardened;
@@ -13,7 +13,7 @@ pub mod hardened;
 	feature = "hardened_malloc",
 	not(feature = "jemalloc")
 ))]
-pub use hardened::{memory_stats, memory_usage};
+pub use hardened::{memory_stats, memory_usage, trim};
 
 #[cfg(any(
 	target_env = "msvc",
@@ -24,4 +24,4 @@ pub mod default;
 	target_env = "msvc",
 	all(not(feature = "hardened_malloc"), not(feature = "jemalloc"))
 ))]
-pub use default::{memory_stats, memory_usage};
+pub use default::{memory_stats, memory_usage, trim};

src/core/config/check.rs

@@ -4,7 +4,7 @@ use either::Either;
 use figment::Figment;
 
 use super::DEPRECATED_KEYS;
-use crate::{debug, debug_info, debug_warn, error, warn, Config, Err, Result};
+use crate::{debug, debug_info, debug_warn, error, warn, Config, Err, Result, Server};
 
 #[allow(clippy::cognitive_complexity)]
 pub fn check(config: &Config) -> Result<()> {
@@ -233,6 +233,16 @@ pub fn check(config: &Config) -> Result<()> {
 		}
 	}
 
+	if !Server::available_room_versions()
+		.any(|(version, _)| version == config.default_room_version)
+	{
+		return Err!(Config(
+			"default_room_version",
+			"Room version {:?} is not available",
+			config.default_room_version
+		));
+	}
+
 	Ok(())
 }
 

src/core/config/manager.rs

@@ -0,0 +1,128 @@
+use std::{
+	cell::{Cell, RefCell},
+	ops::Deref,
+	ptr,
+	ptr::null_mut,
+	sync::{
+		atomic::{AtomicPtr, Ordering},
+		Arc,
+	},
+};
+
+use super::Config;
+use crate::{implement, Result};
+
+/// The configuration manager is an indirection to reload the configuration for
+/// the server while it is running. In order to not burden or clutter the many
+/// callsites which query for configuration items, this object implements Deref
+/// for the actively loaded configuration.
+pub struct Manager {
+	active: AtomicPtr<Config>,
+}
+
+thread_local! {
+	static INDEX: Cell<usize> = 0.into();
+	static HANDLE: RefCell<Handles> = const {
+		RefCell::new([const { None }; HISTORY])
+	};
+}
+
+type Handle = Option<Arc<Config>>;
+type Handles = [Handle; HISTORY];
+
+const HISTORY: usize = 8;
+
+impl Manager {
+	pub(crate) fn new(config: Config) -> Self {
+		let config = Arc::new(config);
+		Self {
+			active: AtomicPtr::new(Arc::into_raw(config).cast_mut()),
+		}
+	}
+}
+
+impl Drop for Manager {
+	fn drop(&mut self) {
+		let config = self.active.swap(null_mut(), Ordering::AcqRel);
+
+		// SAFETY: The active pointer was set using an Arc::into_raw(). We're obliged to
+		// reconstitute that into Arc otherwise it will leak.
+		unsafe { Arc::from_raw(config) };
+	}
+}
+
+impl Deref for Manager {
+	type Target = Arc<Config>;
+
+	fn deref(&self) -> &Self::Target { HANDLE.with_borrow_mut(|handle| self.load(handle)) }
+}
+
+/// Update the active configuration, returning prior configuration.
+#[implement(Manager)]
+#[tracing::instrument(skip_all)]
+pub fn update(&self, config: Config) -> Result<Arc<Config>> {
+	let config = Arc::new(config);
+	let new = Arc::into_raw(config);
+	let old = self.active.swap(new.cast_mut(), Ordering::AcqRel);
+
+	// SAFETY: The old active pointer was set using an Arc::into_raw(). We're
+	// obliged to reconstitute that into Arc otherwise it will leak.
+	Ok(unsafe { Arc::from_raw(old) })
+}
+
+#[implement(Manager)]
+fn load(&self, handle: &mut [Option<Arc<Config>>]) -> &'static Arc<Config> {
+	let config = self.active.load(Ordering::Acquire);
+
+	// Branch taken after config reload or first access by this thread.
+	if handle[INDEX.get()]
+		.as_ref()
+		.is_none_or(|handle| !ptr::eq(config, Arc::as_ptr(handle)))
+	{
+		INDEX.set(INDEX.get().wrapping_add(1).wrapping_rem(HISTORY));
+		return load_miss(handle, INDEX.get(), config);
+	}
+
+	let config: &Arc<Config> = handle[INDEX.get()]
+		.as_ref()
+		.expect("handle was already cached for this thread");
+
+	// SAFETY: The caller should not hold multiple references at a time directly
+	// into Config, as a subsequent reference might invalidate the thread's cache
+	// causing another reference to dangle.
+	//
+	// This is a highly unusual pattern as most config values are copied by value or
+	// used immediately without running overlap with another value. Even if it does
+	// actually occur somewhere, the window of danger is limited to the config being
+	// reloaded while the reference is held and another access is made by the same
+	// thread into a different config value. This is mitigated by creating a buffer
+	// of old configs rather than discarding at the earliest opportunity; the odds
+	// of this scenario are thus astronomical.
+	unsafe { std::mem::transmute(config) }
+}
+
+#[tracing::instrument(
+	name = "miss",
+	level = "trace",
+	skip_all,
+	fields(%index, ?config)
+)]
+#[allow(clippy::transmute_ptr_to_ptr)]
+fn load_miss(
+	handle: &mut [Option<Arc<Config>>],
+	index: usize,
+	config: *const Config,
+) -> &'static Arc<Config> {
+	// SAFETY: The active pointer was set prior and always remains valid. We're
+	// reconstituting the Arc here but as a new reference, so the count is
+	// incremented. This instance will be cached in the thread-local.
+	let config = unsafe {
+		Arc::increment_strong_count(config);
+		Arc::from_raw(config)
+	};
+
+	// SAFETY: See the note on the transmute above. The caller should not hold more
+	// than one reference at a time directly into Config, as the second access
+	// might invalidate the thread's cache, dangling the reference to the first.
+	unsafe { std::mem::transmute(handle[index].insert(config)) }
+}

src/core/config/mod.rs

@@ -1,11 +1,11 @@
 pub mod check;
+pub mod manager;
 pub mod proxy;
 
 use std::{
 	collections::{BTreeMap, BTreeSet, HashSet},
-	fmt,
 	net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr},
-	path::PathBuf,
+	path::{Path, PathBuf},
 };
 
 use conduwuit_macros::config_example_generator;
@@ -15,7 +15,6 @@ use either::{
 };
 use figment::providers::{Env, Format, Toml};
 pub use figment::{value::Value as FigmentValue, Figment};
-use itertools::Itertools;
 use regex::RegexSet;
 use ruma::{
 	api::client::discovery::discover_support::ContactRole, OwnedRoomOrAliasId, OwnedServerName,
@@ -24,8 +23,8 @@ use ruma::{
 use serde::{de::IgnoredAny, Deserialize};
 use url::Url;
 
-pub use self::check::check;
 use self::proxy::ProxyConfig;
+pub use self::{check::check, manager::Manager};
 use crate::{err, error::Error, utils::sys, Result};
 
 /// All the config options for conduwuit.
@@ -147,22 +146,6 @@ pub struct Config {
 	#[serde(default = "default_database_backups_to_keep")]
 	pub database_backups_to_keep: i16,
 
-	/// Set this to any float value in megabytes for conduwuit to tell the
-	/// database engine that this much memory is available for database-related
-	/// caches.
-	///
-	/// May be useful if you have significant memory to spare to increase
-	/// performance.
-	///
-	/// Similar to the individual LRU caches, this is scaled up with your CPU
-	/// core count.
-	///
-	/// This defaults to 128.0 + (64.0 * CPU core count).
-	///
-	/// default: varies by system
-	#[serde(default = "default_db_cache_capacity_mb")]
-	pub db_cache_capacity_mb: f64,
-
 	/// Text which will be added to the end of the user's displayname upon
 	/// registration with a space before the text. In Conduit, this was the
 	/// lightning bolt emoji.
@@ -205,6 +188,38 @@ pub struct Config {
 	)]
 	pub cache_capacity_modifier: f64,
 
+	/// Set this to any float value in megabytes for conduwuit to tell the
+	/// database engine that this much memory is available for database read
+	/// caches.
+	///
+	/// May be useful if you have significant memory to spare to increase
+	/// performance.
+	///
+	/// Similar to the individual LRU caches, this is scaled up with your CPU
+	/// core count.
+	///
+	/// This defaults to 128.0 + (64.0 * CPU core count).
+	///
+	/// default: varies by system
+	#[serde(default = "default_db_cache_capacity_mb")]
+	pub db_cache_capacity_mb: f64,
+
+	/// Set this to any float value in megabytes for conduwuit to tell the
+	/// database engine that this much memory is available for database write
+	/// caches.
+	///
+	/// May be useful if you have significant memory to spare to increase
+	/// performance.
+	///
+	/// Similar to the individual LRU caches, this is scaled up with your CPU
+	/// core count.
+	///
+	/// This defaults to 48.0 + (4.0 * CPU core count).
+	///
+	/// default: varies by system
+	#[serde(default = "default_db_write_buffer_capacity_mb")]
+	pub db_write_buffer_capacity_mb: f64,
+
 	/// default: varies by system
 	#[serde(default = "default_pdu_cache_capacity")]
 	pub pdu_cache_capacity: u32,
@@ -477,16 +492,22 @@ pub struct Config {
 	#[serde(default)]
 	pub allow_registration: bool,
 
+	/// Enabling this setting opens registration to anyone without restrictions.
+	/// This makes your server vulnerable to abuse
 	#[serde(default)]
 	pub yes_i_am_very_very_sure_i_want_an_open_registration_server_prone_to_abuse: bool,
 
 	/// A static registration token that new users will have to provide when
 	/// creating an account. If unset and `allow_registration` is true,
-	/// registration is open without any condition.
+	/// you must set
+	/// `yes_i_am_very_very_sure_i_want_an_open_registration_server_prone_to_abuse`
+	/// to true to allow open registration without any conditions.
 	///
 	/// YOU NEED TO EDIT THIS OR USE registration_token_file.
 	///
 	/// example: "o&^uCtes4HPf0Vu@F20jQeeWE7"
+	///
+	/// display: sensitive
 	pub registration_token: Option<String>,
 
 	/// Path to a file on the system that gets read for the registration token.
@@ -621,6 +642,7 @@ pub struct Config {
 	#[serde(default = "default_tracing_flame_output_path")]
 	pub tracing_flame_output_path: String,
 
+	#[cfg(not(doctest))]
 	/// Examples:
 	///
 	/// - No proxy (default):
@@ -654,8 +676,6 @@ pub struct Config {
 	#[serde(default)]
 	pub proxy: ProxyConfig,
 
-	pub jwt_secret: Option<String>,
-
 	/// Servers listed here will be used to gather public keys of other servers
 	/// (notary trusted key servers).
 	///
@@ -752,6 +772,24 @@ pub struct Config {
 	#[serde(default = "default_openid_token_ttl")]
 	pub openid_token_ttl: u64,
 
+	/// Allow an existing session to mint a login token for another client.
+	/// This requires interactive authentication, but has security ramifications
+	/// as a malicious client could use the mechanism to spawn more than one
+	/// session.
+	/// Enabled by default.
+	#[serde(default = "true_fn")]
+	pub login_via_existing_session: bool,
+
+	/// Login token expiration/TTL in milliseconds.
+	///
+	/// These are short-lived tokens for the m.login.token endpoint.
+	/// This is used to allow existing sessions to create new sessions.
+	/// see login_via_existing_session.
+	///
+	/// default: 120000
+	#[serde(default = "default_login_token_ttl")]
+	pub login_token_ttl: u64,
+
 	/// Static TURN username to provide the client if not using a shared secret
 	/// ("turn_secret"), It is recommended to use a shared secret over static
 	/// credentials.
@@ -761,6 +799,8 @@ pub struct Config {
 	/// Static TURN password to provide the client if not using a shared secret
 	/// ("turn_secret"). It is recommended to use a shared secret over static
 	/// credentials.
+	///
+	/// display: sensitive
 	#[serde(default)]
 	pub turn_password: String,
 
@@ -782,6 +822,8 @@ pub struct Config {
 	///
 	/// This is more secure, but if needed you can use traditional static
 	/// username/password credentials.
+	///
+	/// display: sensitive
 	#[serde(default)]
 	pub turn_secret: String,
 
@@ -925,6 +967,9 @@ pub struct Config {
 	/// magic number and translated to the library's default compression level
 	/// as they all differ. See their `kDefaultCompressionLevel`.
 	///
+	/// Note when using the default value we may override it with a setting
+	/// tailored specifically conduwuit.
+	///
 	/// default: 32767
 	#[serde(default = "default_rocksdb_compression_level")]
 	pub rocksdb_compression_level: i32,
@@ -940,6 +985,9 @@ pub struct Config {
 	/// less likely for this data to be used. Research your chosen compression
 	/// algorithm.
 	///
+	/// Note when using the default value we may override it with a setting
+	/// tailored specifically conduwuit.
+	///
 	/// default: 32767
 	#[serde(default = "default_rocksdb_bottommost_compression_level")]
 	pub rocksdb_bottommost_compression_level: i32,
@@ -952,7 +1000,7 @@ pub struct Config {
 	/// if you're trying to reduce storage usage from the database.
 	///
 	/// See https://github.com/facebook/rocksdb/wiki/Compression for more details.
-	#[serde(default)]
+	#[serde(default = "true_fn")]
 	pub rocksdb_bottommost_compression: bool,
 
 	/// Database recovery mode (for RocksDB WAL corruption).
@@ -1073,6 +1121,8 @@ pub struct Config {
 	/// security purposes.
 	///
 	/// example: "F670$2CP@Hw8mG7RY1$%!#Ic7YA"
+	///
+	/// display: sensitive
 	pub emergency_password: Option<String>,
 
 	/// default: "/_matrix/push/v1/notify"
@@ -1522,6 +1572,7 @@ pub struct Config {
 
 	/// Sentry reporting URL, if a custom one is desired.
 	///
+	/// display: sensitive
 	/// default: "https://fe2eb4536aa04949e28eff3128d64757@o4506996327251968.ingest.us.sentry.io/4506996334657536"
 	#[serde(default = "default_sentry_endpoint")]
 	pub sentry_endpoint: Option<Url>,
@@ -1646,7 +1697,7 @@ pub struct Config {
 	/// responsiveness for many users at the cost of throughput for each.
 	///
 	/// Setting this value to 0.0 causes the stream width to be fixed at the
-	/// value of stream_width_default. The default is 1.0 to match the
+	/// value of stream_width_default. The default scale is 1.0 to match the
 	/// capabilities detected for the system.
 	///
 	/// default: 1.0
@@ -1677,6 +1728,11 @@ pub struct Config {
 	#[serde(default)]
 	pub sender_workers: usize,
 
+	/// Enables listener sockets; can be set to false to disable listening. This
+	/// option is intended for developer/diagnostic purposes only.
+	#[serde(default = "true_fn")]
+	pub listening: bool,
+
 	#[serde(flatten)]
 	#[allow(clippy::zero_sized_map_values)]
 	// this is a catchall, the map shouldn't be zero at runtime
@@ -1755,14 +1811,17 @@ const DEPRECATED_KEYS: &[&str; 9] = &[
 
 impl Config {
 	/// Pre-initialize config
-	pub fn load(paths: Option<&[PathBuf]>) -> Result<Figment> {
-		let paths_files = paths.into_iter().flatten().map(Toml::file);
-
+	pub fn load<'a, I>(paths: I) -> Result<Figment>
+	where
+		I: Iterator<Item = &'a Path>,
+	{
 		let envs = [Env::var("CONDUIT_CONFIG"), Env::var("CONDUWUIT_CONFIG")];
-		let envs_files = envs.into_iter().flatten().map(Toml::file);
 
-		let config = envs_files
-			.chain(paths_files)
+		let config = envs
+			.into_iter()
+			.flatten()
+			.map(Toml::file)
+			.chain(paths.map(Toml::file))
 			.fold(Figment::new(), |config, file| config.merge(file.nested()))
 			.merge(Env::prefixed("CONDUIT_").global().split("__"))
 			.merge(Env::prefixed("CONDUWUIT_").global().split("__"));
@@ -1815,409 +1874,6 @@ impl Config {
 	pub fn check(&self) -> Result<(), Error> { check(self) }
 }
 
-impl fmt::Display for Config {
-	fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-		writeln!(f, "Active config values:\n").expect("wrote line to formatter stream");
-		let mut line = |key: &str, val: &str| {
-			writeln!(f, "{key}: {val}").expect("wrote line to formatter stream");
-		};
-
-		line("Server name", self.server_name.host());
-		line("Database path", &self.database_path.to_string_lossy());
-		line(
-			"Database backup path",
-			self.database_backup_path
-				.as_ref()
-				.map_or("", |path| path.to_str().unwrap_or("")),
-		);
-		line("Database backups to keep", &self.database_backups_to_keep.to_string());
-		line("Database cache capacity (MB)", &self.db_cache_capacity_mb.to_string());
-		line("Cache capacity modifier", &self.cache_capacity_modifier.to_string());
-		line("PDU cache capacity", &self.pdu_cache_capacity.to_string());
-		line("Auth chain cache capacity", &self.auth_chain_cache_capacity.to_string());
-		line("Short eventid cache capacity", &self.shorteventid_cache_capacity.to_string());
-		line("Eventid short cache capacity", &self.eventidshort_cache_capacity.to_string());
-		line("Short statekey cache capacity", &self.shortstatekey_cache_capacity.to_string());
-		line("Statekey short cache capacity", &self.statekeyshort_cache_capacity.to_string());
-		line(
-			"Server visibility cache capacity",
-			&self.server_visibility_cache_capacity.to_string(),
-		);
-		line(
-			"User visibility cache capacity",
-			&self.user_visibility_cache_capacity.to_string(),
-		);
-		line("Stateinfo cache capacity", &self.stateinfo_cache_capacity.to_string());
-		line(
-			"Roomid space hierarchy cache capacity",
-			&self.roomid_spacehierarchy_cache_capacity.to_string(),
-		);
-		line("DNS cache entry limit", &self.dns_cache_entries.to_string());
-		line("DNS minimum TTL", &self.dns_min_ttl.to_string());
-		line("DNS minimum NXDOMAIN TTL", &self.dns_min_ttl_nxdomain.to_string());
-		line("DNS attempts", &self.dns_attempts.to_string());
-		line("DNS timeout", &self.dns_timeout.to_string());
-		line("DNS fallback to TCP", &self.dns_tcp_fallback.to_string());
-		line("DNS query over TCP only", &self.query_over_tcp_only.to_string());
-		line("Query all nameservers", &self.query_all_nameservers.to_string());
-		line("Maximum request size (bytes)", &self.max_request_size.to_string());
-		line("Sender retry backoff limit", &self.sender_retry_backoff_limit.to_string());
-		line("Request connect timeout", &self.request_conn_timeout.to_string());
-		line("Request timeout", &self.request_timeout.to_string());
-		line("Request total timeout", &self.request_total_timeout.to_string());
-		line("Idle connections per host", &self.request_idle_per_host.to_string());
-		line("Request pool idle timeout", &self.request_idle_timeout.to_string());
-		line("Well_known connect timeout", &self.well_known_conn_timeout.to_string());
-		line("Well_known timeout", &self.well_known_timeout.to_string());
-		line("Federation timeout", &self.federation_timeout.to_string());
-		line("Federation pool idle per host", &self.federation_idle_per_host.to_string());
-		line("Federation pool idle timeout", &self.federation_idle_timeout.to_string());
-		line("Sender timeout", &self.sender_timeout.to_string());
-		line("Sender pool idle timeout", &self.sender_idle_timeout.to_string());
-		line("Appservice timeout", &self.appservice_timeout.to_string());
-		line("Appservice pool idle timeout", &self.appservice_idle_timeout.to_string());
-		line("Pusher pool idle timeout", &self.pusher_idle_timeout.to_string());
-		line("Allow registration", &self.allow_registration.to_string());
-		line(
-			"Registration token",
-			if self.registration_token.is_none()
-				&& self.registration_token_file.is_none()
-				&& self.allow_registration
-			{
-				"not set (⚠️ open registration!)"
-			} else if self.registration_token.is_none() && self.registration_token_file.is_none()
-			{
-				"not set"
-			} else {
-				"set"
-			},
-		);
-		line(
-			"Registration token file path",
-			self.registration_token_file
-				.as_ref()
-				.map_or("", |path| path.to_str().unwrap_or_default()),
-		);
-		line(
-			"Allow guest registration (inherently false if allow registration is false)",
-			&self.allow_guest_registration.to_string(),
-		);
-		line(
-			"Log guest registrations in admin room",
-			&self.log_guest_registrations.to_string(),
-		);
-		line(
-			"Allow guests to auto join rooms",
-			&self.allow_guests_auto_join_rooms.to_string(),
-		);
-		line("New user display name suffix", &self.new_user_displayname_suffix);
-		line("Allow encryption", &self.allow_encryption.to_string());
-		line("Allow federation", &self.allow_federation.to_string());
-		line("Federation loopback", &self.federation_loopback.to_string());
-		line(
-			"Require authentication for profile requests",
-			&self.require_auth_for_profile_requests.to_string(),
-		);
-		line(
-			"Allow incoming federated presence requests (updates)",
-			&self.allow_incoming_presence.to_string(),
-		);
-		line(
-			"Allow outgoing federated presence requests (updates)",
-			&self.allow_outgoing_presence.to_string(),
-		);
-		line(
-			"Allow local presence requests (updates)",
-			&self.allow_local_presence.to_string(),
-		);
-		line(
-			"Allow incoming remote read receipts",
-			&self.allow_incoming_read_receipts.to_string(),
-		);
-		line(
-			"Allow outgoing remote read receipts",
-			&self.allow_outgoing_read_receipts.to_string(),
-		);
-		line(
-			"Block non-admin room invites (local and remote, admins can still send and receive \
-			 invites)",
-			&self.block_non_admin_invites.to_string(),
-		);
-		line("Enable admin escape commands", &self.admin_escape_commands.to_string());
-		line(
-			"Activate admin console after startup",
-			&self.admin_console_automatic.to_string(),
-		);
-		line("Execute admin commands after startup", &self.admin_execute.join(", "));
-		line(
-			"Continue startup even if some commands fail",
-			&self.admin_execute_errors_ignore.to_string(),
-		);
-		line("Filter for admin command log capture", &self.admin_log_capture);
-		line("Admin room tag", &self.admin_room_tag);
-		line("Allow outgoing federated typing", &self.allow_outgoing_typing.to_string());
-		line("Allow incoming federated typing", &self.allow_incoming_typing.to_string());
-		line(
-			"Incoming federated typing timeout",
-			&self.typing_federation_timeout_s.to_string(),
-		);
-		line("Client typing timeout minimum", &self.typing_client_timeout_min_s.to_string());
-		line("Client typing timeout maxmimum", &self.typing_client_timeout_max_s.to_string());
-		line("Allow device name federation", &self.allow_device_name_federation.to_string());
-		line(
-			"Allow incoming profile lookup federation requests",
-			&self
-				.allow_inbound_profile_lookup_federation_requests
-				.to_string(),
-		);
-		line(
-			"Auto deactivate banned room join attempts",
-			&self.auto_deactivate_banned_room_attempts.to_string(),
-		);
-		line("Notification push path", &self.notification_push_path);
-		line("Allow room creation", &self.allow_room_creation.to_string());
-		line(
-			"Allow public room directory over federation",
-			&self.allow_public_room_directory_over_federation.to_string(),
-		);
-		line(
-			"Allow public room directory without authentication",
-			&self.allow_public_room_directory_without_auth.to_string(),
-		);
-		line(
-			"Lockdown public room directory (only allow admins to publish)",
-			&self.lockdown_public_room_directory.to_string(),
-		);
-		line("JWT secret", match self.jwt_secret {
-			| Some(_) => "set",
-			| None => "not set",
-		});
-		line(
-			"Trusted key servers",
-			&self
-				.trusted_servers
-				.iter()
-				.map(|server| server.host())
-				.join(", "),
-		);
-		line("OpenID Token TTL", &self.openid_token_ttl.to_string());
-		line(
-			"TURN username",
-			if self.turn_username.is_empty() {
-				"not set"
-			} else {
-				&self.turn_username
-			},
-		);
-		line("TURN password", {
-			if self.turn_password.is_empty() {
-				"not set"
-			} else {
-				"set"
-			}
-		});
-		line("TURN secret", {
-			if self.turn_secret.is_empty() && self.turn_secret_file.is_none() {
-				"not set"
-			} else {
-				"set"
-			}
-		});
-		line("TURN secret file path", {
-			self.turn_secret_file
-				.as_ref()
-				.map_or("", |path| path.to_str().unwrap_or_default())
-		});
-		line("Turn TTL", &self.turn_ttl.to_string());
-		line("Turn URIs", {
-			let mut lst = Vec::with_capacity(self.turn_uris.len());
-			for item in self.turn_uris.iter().cloned().enumerate() {
-				let (_, uri): (usize, String) = item;
-				lst.push(uri);
-			}
-			&lst.join(", ")
-		});
-		line("Auto Join Rooms", {
-			let mut lst = Vec::with_capacity(self.auto_join_rooms.len());
-			for room in &self.auto_join_rooms {
-				lst.push(room);
-			}
-			&lst.into_iter().join(", ")
-		});
-		line("Zstd HTTP Compression", &self.zstd_compression.to_string());
-		line("Gzip HTTP Compression", &self.gzip_compression.to_string());
-		line("Brotli HTTP Compression", &self.brotli_compression.to_string());
-		line("RocksDB database LOG level", &self.rocksdb_log_level);
-		line("RocksDB database LOG to stderr", &self.rocksdb_log_stderr.to_string());
-		line("RocksDB database LOG time-to-roll", &self.rocksdb_log_time_to_roll.to_string());
-		line("RocksDB Max LOG Files", &self.rocksdb_max_log_files.to_string());
-		line(
-			"RocksDB database max LOG file size",
-			&self.rocksdb_max_log_file_size.to_string(),
-		);
-		line(
-			"RocksDB database optimize for spinning disks",
-			&self.rocksdb_optimize_for_spinning_disks.to_string(),
-		);
-		line("RocksDB Direct-IO", &self.rocksdb_direct_io.to_string());
-		line("RocksDB Parallelism Threads", &self.rocksdb_parallelism_threads.to_string());
-		line("RocksDB Compression Algorithm", &self.rocksdb_compression_algo);
-		line("RocksDB Compression Level", &self.rocksdb_compression_level.to_string());
-		line(
-			"RocksDB Bottommost Compression Level",
-			&self.rocksdb_bottommost_compression_level.to_string(),
-		);
-		line(
-			"RocksDB Bottommost Level Compression",
-			&self.rocksdb_bottommost_compression.to_string(),
-		);
-		line("RocksDB Recovery Mode", &self.rocksdb_recovery_mode.to_string());
-		line("RocksDB Repair Mode", &self.rocksdb_repair.to_string());
-		line("RocksDB Read-only Mode", &self.rocksdb_read_only.to_string());
-		line("RocksDB Secondary Mode", &self.rocksdb_secondary.to_string());
-		line(
-			"RocksDB Compaction Idle Priority",
-			&self.rocksdb_compaction_prio_idle.to_string(),
-		);
-		line(
-			"RocksDB Compaction Idle IOPriority",
-			&self.rocksdb_compaction_ioprio_idle.to_string(),
-		);
-		line("RocksDB Compaction enabled", &self.rocksdb_compaction.to_string());
-		line("RocksDB Statistics level", &self.rocksdb_stats_level.to_string());
-		line("Media integrity checks on startup", &self.media_startup_check.to_string());
-		line("Media compatibility filesystem links", &self.media_compat_file_link.to_string());
-		line("Prune missing media from database", &self.prune_missing_media.to_string());
-		line("Allow legacy (unauthenticated) media", &self.allow_legacy_media.to_string());
-		line("Freeze legacy (unauthenticated) media", &self.freeze_legacy_media.to_string());
-		line("Prevent Media Downloads From", {
-			let mut lst = Vec::with_capacity(self.prevent_media_downloads_from.len());
-			for domain in &self.prevent_media_downloads_from {
-				lst.push(domain.host());
-			}
-			&lst.join(", ")
-		});
-		line("Forbidden Remote Server Names (\"Global\" ACLs)", {
-			let mut lst = Vec::with_capacity(self.forbidden_remote_server_names.len());
-			for domain in &self.forbidden_remote_server_names {
-				lst.push(domain.host());
-			}
-			&lst.join(", ")
-		});
-		line("Forbidden Remote Room Directory Server Names", {
-			let mut lst =
-				Vec::with_capacity(self.forbidden_remote_room_directory_server_names.len());
-			for domain in &self.forbidden_remote_room_directory_server_names {
-				lst.push(domain.host());
-			}
-			&lst.join(", ")
-		});
-		line("Outbound Request IP Range (CIDR) Denylist", {
-			let mut lst = Vec::with_capacity(self.ip_range_denylist.len());
-			for item in self.ip_range_denylist.iter().cloned().enumerate() {
-				let (_, ip): (usize, String) = item;
-				lst.push(ip);
-			}
-			&lst.join(", ")
-		});
-		line("Forbidden usernames", {
-			&self.forbidden_usernames.patterns().iter().join(", ")
-		});
-		line("Forbidden room aliases", {
-			&self.forbidden_alias_names.patterns().iter().join(", ")
-		});
-		line(
-			"URL preview bound interface",
-			self.url_preview_bound_interface
-				.as_ref()
-				.map(Either::as_ref)
-				.map(|either| either.map_left(ToString::to_string))
-				.map(Either::either_into::<String>)
-				.unwrap_or_default()
-				.as_str(),
-		);
-		line(
-			"URL preview domain contains allowlist",
-			&self.url_preview_domain_contains_allowlist.join(", "),
-		);
-		line(
-			"URL preview domain explicit allowlist",
-			&self.url_preview_domain_explicit_allowlist.join(", "),
-		);
-		line(
-			"URL preview domain explicit denylist",
-			&self.url_preview_domain_explicit_denylist.join(", "),
-		);
-		line(
-			"URL preview URL contains allowlist",
-			&self.url_preview_url_contains_allowlist.join(", "),
-		);
-		line("URL preview maximum spider size", &self.url_preview_max_spider_size.to_string());
-		line("URL preview check root domain", &self.url_preview_check_root_domain.to_string());
-		line(
-			"Allow check for updates / announcements check",
-			&self.allow_check_for_updates.to_string(),
-		);
-		line("Enable netburst on startup", &self.startup_netburst.to_string());
-		#[cfg(feature = "sentry_telemetry")]
-		line("Sentry.io reporting and tracing", &self.sentry.to_string());
-		#[cfg(feature = "sentry_telemetry")]
-		line("Sentry.io send server_name in logs", &self.sentry_send_server_name.to_string());
-		#[cfg(feature = "sentry_telemetry")]
-		line("Sentry.io tracing sample rate", &self.sentry_traces_sample_rate.to_string());
-		line("Sentry.io attach stacktrace", &self.sentry_attach_stacktrace.to_string());
-		line("Sentry.io send panics", &self.sentry_send_panic.to_string());
-		line("Sentry.io send errors", &self.sentry_send_error.to_string());
-		line("Sentry.io tracing filter", &self.sentry_filter);
-		line(
-			"Well-known server name",
-			self.well_known
-				.server
-				.as_ref()
-				.map_or("", |server| server.as_str()),
-		);
-		line(
-			"Well-known client URL",
-			self.well_known
-				.client
-				.as_ref()
-				.map_or("", |url| url.as_str()),
-		);
-		line(
-			"Well-known support email",
-			self.well_known
-				.support_email
-				.as_ref()
-				.map_or("", |str| str.as_ref()),
-		);
-		line(
-			"Well-known support Matrix ID",
-			self.well_known
-				.support_mxid
-				.as_ref()
-				.map_or("", |mxid| mxid.as_str()),
-		);
-		line(
-			"Well-known support role",
-			self.well_known
-				.support_role
-				.as_ref()
-				.map_or("", |role| role.as_str()),
-		);
-		line(
-			"Well-known support page/URL",
-			self.well_known
-				.support_page
-				.as_ref()
-				.map_or("", |url| url.as_str()),
-		);
-		line("Enable the tokio-console", &self.tokio_console.to_string());
-		line("Admin room notices", &self.admin_room_notices.to_string());
-
-		Ok(())
-	}
-}
-
 fn true_fn() -> bool { true }
 
 fn default_address() -> ListeningAddr {
@@ -2232,6 +1888,8 @@ fn default_unix_socket_perms() -> u32 { 660 }
 
 fn default_database_backups_to_keep() -> i16 { 1 }
 
+fn default_db_write_buffer_capacity_mb() -> f64 { 48.0 + parallelism_scaled_f64(4.0) }
+
 fn default_db_cache_capacity_mb() -> f64 { 128.0 + parallelism_scaled_f64(64.0) }
 
 fn default_pdu_cache_capacity() -> u32 { parallelism_scaled_u32(10_000).saturating_add(100_000) }
@@ -2360,6 +2018,8 @@ fn default_notification_push_path() -> String { "/_matrix/push/v1/notify".to_own
 
 fn default_openid_token_ttl() -> u64 { 60 * 60 }
 
+fn default_login_token_ttl() -> u64 { 2 * 60 * 1000 }
+
 fn default_turn_ttl() -> u64 { 60 * 60 * 24 }
 
 fn default_presence_idle_timeout_s() -> u64 { 5 * 60 }

src/core/error/mod.rs

@@ -4,7 +4,7 @@ mod panic;
 mod response;
 mod serde;
 
-use std::{any::Any, borrow::Cow, convert::Infallible, fmt, sync::PoisonError};
+use std::{any::Any, borrow::Cow, convert::Infallible, sync::PoisonError};
 
 pub use self::{err::visit, log::*};
 
@@ -17,7 +17,7 @@ pub enum Error {
 
 	// std
 	#[error(transparent)]
-	Fmt(#[from] fmt::Error),
+	Fmt(#[from] std::fmt::Error),
 	#[error(transparent)]
 	FromUtf8(#[from] std::string::FromUtf8Error),
 	#[error("I/O error: {0}")]
@@ -27,6 +27,10 @@ pub enum Error {
 	#[error(transparent)]
 	ParseInt(#[from] std::num::ParseIntError),
 	#[error(transparent)]
+	Std(#[from] Box<dyn std::error::Error + Send>),
+	#[error(transparent)]
+	ThreadAccessError(#[from] std::thread::AccessError),
+	#[error(transparent)]
 	TryFromInt(#[from] std::num::TryFromIntError),
 	#[error(transparent)]
 	TryFromSlice(#[from] std::array::TryFromSliceError),
@@ -48,8 +52,6 @@ pub enum Error {
 	Http(#[from] http::Error),
 	#[error(transparent)]
 	HttpHeader(#[from] http::header::InvalidHeaderValue),
-	#[error("Image error: {0}")]
-	Image(#[from] image::error::ImageError),
 	#[error("Join error: {0}")]
 	JoinError(#[from] tokio::task::JoinError),
 	#[error(transparent)]
@@ -129,6 +131,10 @@ pub enum Error {
 }
 
 impl Error {
+	#[inline]
+	#[must_use]
+	pub fn from_errno() -> Self { Self::Io(std::io::Error::last_os_error()) }
+
 	//#[deprecated]
 	pub fn bad_database(message: &'static str) -> Self {
 		crate::err!(Database(error!("{message}")))
@@ -191,8 +197,10 @@ impl Error {
 	pub fn is_not_found(&self) -> bool { self.status_code() == http::StatusCode::NOT_FOUND }
 }
 
-impl fmt::Debug for Error {
-	fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "{}", self.message()) }
+impl std::fmt::Debug for Error {
+	fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+		write!(f, "{}", self.message())
+	}
 }
 
 impl<T> From<PoisonError<T>> for Error {

src/core/info/room_version.rs

@@ -20,6 +20,8 @@ pub const STABLE_ROOM_VERSIONS: &[RoomVersionId] = &[
 pub const UNSTABLE_ROOM_VERSIONS: &[RoomVersionId] =
 	&[RoomVersionId::V2, RoomVersionId::V3, RoomVersionId::V4, RoomVersionId::V5];
 
+type RoomVersion = (RoomVersionId, RoomVersionStability);
+
 impl crate::Server {
 	#[inline]
 	pub fn supported_room_version(&self, version: &RoomVersionId) -> bool {
@@ -28,15 +30,13 @@ impl crate::Server {
 
 	#[inline]
 	pub fn supported_room_versions(&self) -> impl Iterator<Item = RoomVersionId> + '_ {
-		self.available_room_versions()
+		Self::available_room_versions()
 			.filter(|(_, stability)| self.supported_stability(stability))
 			.map(at!(0))
 	}
 
 	#[inline]
-	pub fn available_room_versions(
-		&self,
-	) -> impl Iterator<Item = (RoomVersionId, RoomVersionStability)> {
+	pub fn available_room_versions() -> impl Iterator<Item = RoomVersion> {
 		available_room_versions()
 	}
 
@@ -46,7 +46,7 @@ impl crate::Server {
 	}
 }
 
-pub fn available_room_versions() -> impl Iterator<Item = (RoomVersionId, RoomVersionStability)> {
+pub fn available_room_versions() -> impl Iterator<Item = RoomVersion> {
 	let unstable_room_versions = UNSTABLE_ROOM_VERSIONS
 		.iter()
 		.cloned()