remove ProcSubset=pid from systemd units for now

they appear to cause strange rust malloc issues on Debian systems

Signed-off-by: strawberry <strawberry@puppygock.gay>

.github/workflows/ci.yml

@@ -16,7 +16,6 @@ on:
           - 'docker/**'
         branches:
             - main
-            - change-ci-cache
         tags:
           - '*'
     # Allows you to run this workflow manually from the Actions tab
@@ -24,7 +23,7 @@ on:
 
 concurrency:
     group: ${{ github.head_ref || github.ref_name }}
-    cancel-in-progress: true
+    cancel-in-progress: false
 
 env:
     # sccache only on main repo
@@ -51,8 +50,11 @@ env:
     # Get error output from nix that we can actually use, and use our binary caches for the earlier CI steps
     NIX_CONFIG: |
       show-trace = true
-      extra-substituters = https://attic.kennel.juneis.dog/conduit https://attic.kennel.juneis.dog/conduwuit https://cache.lix.systems https://conduwuit.cachix.org
+      extra-substituters = https://attic.kennel.juneis.dog/conduwuit https://attic.kennel.juneis.dog/conduit https://cache.lix.systems https://conduwuit.cachix.org https://aseipp-nix-cache.freetls.fastly.net
       extra-trusted-public-keys = conduit:eEKoUwlQGDdYmAI/Q/0slVlegqh/QmAvQd7HBSm21Wk= conduwuit:BbycGUgTISsltcmH0qNjFR9dbrQNYgdIAcmViSGoVTE= cache.lix.systems:aBnZUw8zA7H35Cz2RyKFVs3H4PlGTLawyY5KRbvJR8o= conduwuit.cachix.org-1:MFRm6jcnfTf0jSAbmvLfhO3KBMt4px+1xaereWXp8Xg=
+      experimental-features = nix-command flakes
+      extra-experimental-features = nix-command flakes
+      accept-flake-config = true
     # complement uses libolm
     NIXPKGS_ALLOW_INSECURE: 1
 
@@ -64,12 +66,23 @@ jobs:
     tests:
         name: Test
         runs-on: ubuntu-latest
-        env:
-          CARGO_PROFILE: "test"
         steps:
             - name: Free Disk Space (Ubuntu)
               uses: jlumbroso/free-disk-space@main
 
+            - name: Free up more runner space
+              run: |
+                set +o pipefail
+                # large docker images
+                sudo docker image prune --all --force || true
+                # large packages
+                sudo apt-get purge -y '^llvm-.*' 'php.*' '^mongodb-.*' '^mysql-.*' azure-cli google-cloud-cli google-chrome-stable firefox powershell microsoft-edge-stable || true
+                sudo apt-get autoremove -y
+                sudo apt-get clean
+                # large folders
+                sudo rm -rf /var/lib/apt/lists/* /usr/local/games /usr/local/sqlpackage /usr/local/.ghcup /usr/local/share/powershell /usr/local/share/edge_driver /usr/local/share/gecko_driver /usr/local/share/chromium /usr/local/share/chromedriver-linux64 /usr/local/share/vcpkg /usr/local/lib/python* /usr/local/lib/node_modules /usr/local/julia* /opt/mssql-tools /etc/skel /usr/share/vim /usr/share/postgresql /usr/share/man /usr/share/apache-maven-* /usr/share/R /usr/share/alsa /usr/share/miniconda /usr/share/grub /usr/share/gradle-* /usr/share/locale /usr/share/texinfo /usr/share/kotlinc /usr/share/swift /usr/share/doc /usr/share/az_9.3.0 /usr/share/sbt /usr/share/ri /usr/share/icons /usr/share/java /usr/share/fonts /usr/lib/google-cloud-sdk /usr/lib/jvm /usr/lib/mono /usr/lib/R /usr/lib/postgresql /usr/lib/heroku /usr/lib/gcc
+                set -o pipefail
+
             - name: Sync repository
               uses: actions/checkout@v4
 
@@ -85,13 +98,7 @@ jobs:
                     exit 1
                   fi
 
-            - uses: nixbuild/nix-quick-install-action@v28
+            - uses: nixbuild/nix-quick-install-action@master
-
-            - name: Enable Cachix binary cache
-              run: |
-                  nix profile install nixpkgs#cachix
-                  cachix use crane
-                  cachix use nix-community
 
             - name: Restore and cache Nix store
               uses: nix-community/cache-nix-action@v5.1.0
@@ -114,11 +121,20 @@ jobs:
                 # always save the cache
                 save-always: true
 
+            - name: Enable Cachix binary cache
+              run: |
+                  nix profile install nixpkgs#cachix
+                  cachix use crane
+                  cachix use nix-community
+
             - name: Apply Nix binary cache configuration
               run: |
                   sudo tee -a "${XDG_CONFIG_HOME:-$HOME/.config}/nix/nix.conf" > /dev/null <<EOF
-                  extra-substituters = https://attic.kennel.juneis.dog/conduit https://attic.kennel.juneis.dog/conduwuit https://cache.lix.systems https://conduwuit.cachix.org
+                  extra-substituters = https://attic.kennel.juneis.dog/conduwuit https://attic.kennel.juneis.dog/conduit https://cache.lix.systems https://conduwuit.cachix.org https://aseipp-nix-cache.freetls.fastly.net
                   extra-trusted-public-keys = conduit:eEKoUwlQGDdYmAI/Q/0slVlegqh/QmAvQd7HBSm21Wk= conduwuit:BbycGUgTISsltcmH0qNjFR9dbrQNYgdIAcmViSGoVTE= cache.lix.systems:aBnZUw8zA7H35Cz2RyKFVs3H4PlGTLawyY5KRbvJR8o= conduwuit.cachix.org-1:MFRm6jcnfTf0jSAbmvLfhO3KBMt4px+1xaereWXp8Xg=
+                  experimental-features = nix-command flakes
+                  extra-experimental-features = nix-command flakes
+                  accept-flake-config = true
                   EOF
 
             - name: Use alternative Nix binary caches if specified
@@ -132,29 +148,16 @@ jobs:
             - name: Prepare build environment
               run: |
                   echo 'source $HOME/.nix-profile/share/nix-direnv/direnvrc' > "$HOME/.direnvrc"
-                  nix profile install --impure --inputs-from . nixpkgs#direnv nixpkgs#nix-direnv
+                  nix profile install --inputs-from . nixpkgs#direnv nixpkgs#nix-direnv
                   direnv allow
-                  nix develop .#all-features --command true --impure
+                  nix develop .#all-features --command true
 
             - name: Cache CI dependencies
               run: |
-                  # attic nix binary cache server is very, very terribly flakey. nothing i can do to fix it other than retry multiple times here
+                  bin/nix-build-and-cache ci
-                  ATTEMPTS=3
+                  bin/nix-build-and-cache just '.#devShells.x86_64-linux.default'
-                  SUCCESS=false
+                  bin/nix-build-and-cache just '.#devShells.x86_64-linux.all-features'
-                  while (( ATTEMPTS-- > 0 ))
+                  bin/nix-build-and-cache just '.#devShells.x86_64-linux.dynamic'
-                  do
-                    bin/nix-build-and-cache ci
-                    if [[ $? == 0 ]]; then
-                      SUCCESS=true
-                      break
-                    else
-                      sleep 3
-                    fi
-                  done
-
-                  if [[ $SUCCESS == "false" ]]; then
-                    exit 1
-                  fi
 
             # use sccache for Rust
             - name: Run sccache-cache
@@ -167,10 +170,14 @@ jobs:
                 cache-all-crates: "true"
 
             - name: Run CI tests
+              env:
+                CARGO_PROFILE: "test"
               run: |
                   direnv exec . engage > >(tee -a test_output.log)
 
             - name: Run Complement tests
+              env:
+                CARGO_PROFILE: "test"
               run: |
                   # the nix devshell sets $COMPLEMENT_SRC, so "/dev/null" is no-op
                   direnv exec . bin/complement "/dev/null" complement_test_logs.jsonl complement_test_results.jsonl > >(tee -a test_output.log)
@@ -218,7 +225,7 @@ jobs:
                       echo '```' >> $GITHUB_STEP_SUMMARY
                   fi
 
-            - name: Run cargo clean test artifacts
+            - name: Run cargo clean test artifacts to free up space
               run: |
                   cargo clean --profile test
 
@@ -229,8 +236,8 @@ jobs:
         strategy:
             matrix:
                 include:
-                    - target: aarch64-unknown-linux-musl
+                    - target: aarch64-linux-musl
-                    - target: x86_64-unknown-linux-musl
+                    - target: x86_64-linux-musl
         steps:
             - name: Free Disk Space (Ubuntu)
               uses: jlumbroso/free-disk-space@main
@@ -240,12 +247,6 @@ jobs:
 
             - uses: nixbuild/nix-quick-install-action@v28
 
-            - name: Enable Cachix binary cache
-              run: |
-                  nix profile install nixpkgs#cachix
-                  cachix use crane
-                  cachix use nix-community
-
             - name: Restore and cache Nix store
               uses: nix-community/cache-nix-action@v5.1.0
               with:
@@ -267,11 +268,20 @@ jobs:
                 # always save the cache
                 save-always: true
 
+            - name: Enable Cachix binary cache
+              run: |
+                  nix profile install nixpkgs#cachix
+                  cachix use crane
+                  cachix use nix-community
+
             - name: Apply Nix binary cache configuration
               run: |
                   sudo tee -a "${XDG_CONFIG_HOME:-$HOME/.config}/nix/nix.conf" > /dev/null <<EOF
-                  extra-substituters = https://attic.kennel.juneis.dog/conduit https://attic.kennel.juneis.dog/conduwuit https://cache.lix.systems https://conduwuit.cachix.org
+                  extra-substituters = https://attic.kennel.juneis.dog/conduwuit https://attic.kennel.juneis.dog/conduit https://cache.lix.systems https://conduwuit.cachix.org https://aseipp-nix-cache.freetls.fastly.net
                   extra-trusted-public-keys = conduit:eEKoUwlQGDdYmAI/Q/0slVlegqh/QmAvQd7HBSm21Wk= conduwuit:BbycGUgTISsltcmH0qNjFR9dbrQNYgdIAcmViSGoVTE= cache.lix.systems:aBnZUw8zA7H35Cz2RyKFVs3H4PlGTLawyY5KRbvJR8o= conduwuit.cachix.org-1:MFRm6jcnfTf0jSAbmvLfhO3KBMt4px+1xaereWXp8Xg=
+                  experimental-features = nix-command flakes
+                  extra-experimental-features = nix-command flakes
+                  accept-flake-config = true
                   EOF
 
             - name: Use alternative Nix binary caches if specified
@@ -301,26 +311,17 @@ jobs:
 
             - name: Build static ${{ matrix.target }}
               run: |
-                  CARGO_DEB_TARGET_TUPLE=$(echo ${{ matrix.target }} | grep -o -E '^([^-]*-){3}[^-]*')
+                  if [[ ${{ matrix.target }} == "x86_64-linux-musl" ]]
+                  then
+                    CARGO_DEB_TARGET_TUPLE="x86_64-unknown-linux-musl"
+                  elif [[ ${{ matrix.target }} == "aarch64-linux-musl" ]]
+                  then
+                    CARGO_DEB_TARGET_TUPLE="aarch64-unknown-linux-musl"
+                  fi
+
                   SOURCE_DATE_EPOCH=$(git log -1 --pretty=%ct)
 
-                  # attic nix binary cache server is very, very terribly flakey. nothing i can do to fix it other than retry multiple times here
+                  bin/nix-build-and-cache just .#static-${{ matrix.target }}-all-features
-                  ATTEMPTS=3
-                  SUCCESS=false
-                  while (( ATTEMPTS-- > 0 ))
-                  do
-                    bin/nix-build-and-cache just .#static-${{ matrix.target }}-all-features
-                    if [[ $? == 0 ]]; then
-                      SUCCESS=true
-                      break
-                    else
-                      sleep 3
-                    fi
-                  done
-
-                  if [[ $SUCCESS == "false" ]]; then
-                    exit 1
-                  fi
 
                   mkdir -v -p target/release/
                   mkdir -v -p target/$CARGO_DEB_TARGET_TUPLE/release/
@@ -341,26 +342,17 @@ jobs:
 
             - name: Build static debug ${{ matrix.target }}
               run: |
-                  CARGO_DEB_TARGET_TUPLE=$(echo ${{ matrix.target }} | grep -o -E '^([^-]*-){3}[^-]*')
+                  if [[ ${{ matrix.target }} == "x86_64-linux-musl" ]]
+                  then
+                    CARGO_DEB_TARGET_TUPLE="x86_64-unknown-linux-musl"
+                  elif [[ ${{ matrix.target }} == "aarch64-linux-musl" ]]
+                  then
+                    CARGO_DEB_TARGET_TUPLE="aarch64-unknown-linux-musl"
+                  fi
+
                   SOURCE_DATE_EPOCH=$(git log -1 --pretty=%ct)
 
-                  # attic nix binary cache server is very, very terribly flakey. nothing i can do to fix it other than retry multiple times here
+                  bin/nix-build-and-cache just .#static-${{ matrix.target }}-all-features-debug
-                  ATTEMPTS=3
-                  SUCCESS=false
-                  while (( ATTEMPTS-- > 0 ))
-                  do
-                    bin/nix-build-and-cache just .#static-${{ matrix.target }}-all-features-debug
-                    if [[ $? == 0 ]]; then
-                      SUCCESS=true
-                      break
-                    else
-                      sleep 3
-                    fi
-                  done
-
-                  if [[ $SUCCESS == "false" ]]; then
-                    exit 1
-                  fi
 
                   # > warning: dev profile is not supported and will be a hard error in the future. cargo-deb is for making releases, and it doesn't make sense to use it with dev profiles.
                   # so we need to coerce cargo-deb into thinking this is a release binary
@@ -423,45 +415,13 @@ jobs:
 
             - name: Build OCI image ${{ matrix.target }}
               run: |
-                  # attic nix binary cache server is very, very terribly flakey. nothing i can do to fix it other than retry multiple times here
+                  bin/nix-build-and-cache just .#oci-image-${{ matrix.target }}-all-features
-                  ATTEMPTS=3
-                  SUCCESS=false
-                  while (( ATTEMPTS-- > 0 ))
-                  do
-                    bin/nix-build-and-cache just .#oci-image-${{ matrix.target }}-all-features
-                    if [[ $? == 0 ]]; then
-                      SUCCESS=true
-                      break
-                    else
-                      sleep 3
-                    fi
-                  done
-
-                  if [[ $SUCCESS == "false" ]]; then
-                    exit 1
-                  fi
 
                   cp -v -f result oci-image-${{ matrix.target }}.tar.gz
 
             - name: Build debug OCI image ${{ matrix.target }}
               run: |
-                  # attic nix binary cache server is very, very terribly flakey. nothing i can do to fix it other than retry multiple times here
+                  bin/nix-build-and-cache just .#oci-image-${{ matrix.target }}-all-features-debug
-                  ATTEMPTS=3
-                  SUCCESS=false
-                  while (( ATTEMPTS-- > 0 ))
-                  do
-                    bin/nix-build-and-cache just .#oci-image-${{ matrix.target }}-all-features-debug
-                    if [[ $? == 0 ]]; then
-                      SUCCESS=true
-                      break
-                    else
-                      sleep 3
-                    fi
-                  done
-
-                  if [[ $SUCCESS == "false" ]]; then
-                    exit 1
-                  fi
 
                   cp -v -f result oci-image-${{ matrix.target }}-debug.tar.gz
 
@@ -481,6 +441,71 @@ jobs:
                   if-no-files-found: error
                   compression-level: 0
 
+    build_mac_binaries:
+        name: Build MacOS Binaries
+        strategy:
+          matrix:
+            os: [macos-latest, macos-13]
+        runs-on: ${{ matrix.os }}
+        steps:
+            - name: Sync repository
+              uses: actions/checkout@v4
+            - name: Tag comparison check
+              if: ${{ startsWith(github.ref, 'refs/tags/v') && !endsWith(github.ref, '-rc') }}
+              run: |
+                  # Tag mismatch with latest repo tag check to prevent potential downgrades
+                  LATEST_TAG=$(git describe --tags `git rev-list --tags --max-count=1`)
+                  if [ $LATEST_TAG != ${{ github.ref_name }} ]; then
+                    echo '# WARNING: Attempting to run this workflow for a tag that is not the latest repo tag. Aborting.'
+                    echo '# WARNING: Attempting to run this workflow for a tag that is not the latest repo tag. Aborting.' >> $GITHUB_STEP_SUMMARY
+                    exit 1
+                  fi
+            # use sccache for Rust
+            - name: Run sccache-cache
+              if: (github.event.pull_request.draft != true) && (vars.DOCKER_USERNAME != '') && (vars.GITLAB_USERNAME != '') && (vars.SCCACHE_ENDPOINT != '') && (github.event.pull_request.user.login != 'renovate[bot]')
+              uses: mozilla-actions/sccache-action@main
+            # use rust-cache
+            - uses: Swatinem/rust-cache@v2
+              with:
+                cache-all-crates: "true"
+            # Nix can't do portable macOS builds yet
+            - name: Build macOS x86_64 binary
+              if: ${{ matrix.os == 'macos-13' }}
+              run: |
+                  CONDUWUIT_VERSION_EXTRA="$(git rev-parse --short HEAD)" cargo build --release
+                  cp -v -f target/release/conduit conduwuit-macos-x86_64
+                  otool -L conduwuit-macos-x86_64
+            # quick smoke test of the x86_64 macOS binary
+            - name: Run x86_64 macOS release binary
+              if: ${{ matrix.os == 'macos-13' }}
+              run: |
+                  ./conduwuit-macos-x86_64 --version
+            - name: Build macOS arm64 binary
+              if: ${{ matrix.os == 'macos-latest' }}
+              run: |
+                  CONDUWUIT_VERSION_EXTRA="$(git rev-parse --short HEAD)" cargo build --release
+                  cp -v -f target/release/conduit conduwuit-macos-arm64
+                  otool -L conduwuit-macos-arm64
+            # quick smoke test of the arm64 macOS binary
+            - name: Run arm64 macOS release binary
+              if: ${{ matrix.os == 'macos-latest' }}
+              run: |
+                  ./conduwuit-macos-arm64 --version
+            - name: Upload macOS x86_64 binary
+              if: ${{ matrix.os == 'macos-13' }}
+              uses: actions/upload-artifact@v4
+              with:
+                  name: conduwuit-macos-x86_64
+                  path: conduwuit-macos-x86_64
+                  if-no-files-found: error
+            - name: Upload macOS arm64 binary
+              if: ${{ matrix.os == 'macos-latest' }}
+              uses: actions/upload-artifact@v4
+              with:
+                  name: conduwuit-macos-arm64
+                  path: conduwuit-macos-arm64
+                  if-no-files-found: error
+
     docker:
         name: Docker publish
         runs-on: ubuntu-latest
@@ -531,10 +556,10 @@ jobs:
 
             - name: Move OCI images into position
               run: |
-                  mv -v oci-image-x86_64-unknown-linux-musl/*.tar.gz oci-image-amd64.tar.gz
+                  mv -v oci-image-x86_64-linux-musl/*.tar.gz oci-image-amd64.tar.gz
-                  mv -v oci-image-aarch64-unknown-linux-musl/*.tar.gz oci-image-arm64v8.tar.gz
+                  mv -v oci-image-aarch64-linux-musl/*.tar.gz oci-image-arm64v8.tar.gz
-                  mv -v oci-image-x86_64-unknown-linux-musl-debug/*.tar.gz oci-image-amd64-debug.tar.gz
+                  mv -v oci-image-x86_64-linux-musl-debug/*.tar.gz oci-image-amd64-debug.tar.gz
-                  mv -v oci-image-aarch64-unknown-linux-musl-debug/*.tar.gz oci-image-arm64v8-debug.tar.gz
+                  mv -v oci-image-aarch64-linux-musl-debug/*.tar.gz oci-image-arm64v8-debug.tar.gz
 
             - name: Load and push amd64 image
               if: ${{ (vars.DOCKER_USERNAME != '') && (env.DOCKERHUB_TOKEN != '') }}

.github/workflows/documentation.yml

@@ -24,8 +24,11 @@ env:
   # Get error output from nix that we can actually use, and use our binary caches for the earlier CI steps
   NIX_CONFIG: |
     show-trace = true
-    extra-substituters = https://attic.kennel.juneis.dog/conduit https://attic.kennel.juneis.dog/conduwuit https://cache.lix.systems https://conduwuit.cachix.org
+    extra-substituters = extra-substituters = https://attic.kennel.juneis.dog/conduwuit https://attic.kennel.juneis.dog/conduit https://cache.lix.systems https://conduwuit.cachix.org https://aseipp-nix-cache.freetls.fastly.net
     extra-trusted-public-keys = conduit:eEKoUwlQGDdYmAI/Q/0slVlegqh/QmAvQd7HBSm21Wk= conduwuit:BbycGUgTISsltcmH0qNjFR9dbrQNYgdIAcmViSGoVTE= cache.lix.systems:aBnZUw8zA7H35Cz2RyKFVs3H4PlGTLawyY5KRbvJR8o= conduwuit.cachix.org-1:MFRm6jcnfTf0jSAbmvLfhO3KBMt4px+1xaereWXp8Xg=
+    experimental-features = nix-command flakes
+    extra-experimental-features = nix-command flakes
+    accept-flake-config = true
 
 # Allow only one concurrent deployment, skipping runs queued between the run in-progress and latest queued.
 # However, do NOT cancel in-progress runs as we want to allow these production deployments to complete.
@@ -57,13 +60,7 @@ jobs:
         if: github.event_name != 'pull_request'
         uses: actions/configure-pages@v5
 
-      - uses: nixbuild/nix-quick-install-action@v28
+      - uses: nixbuild/nix-quick-install-action@master
-
-      - name: Enable Cachix binary cache
-        run: |
-            nix profile install nixpkgs#cachix
-            cachix use crane
-            cachix use nix-community
 
       - name: Restore and cache Nix store
         uses: nix-community/cache-nix-action@v5.1.0
@@ -86,11 +83,20 @@ jobs:
           # always save the cache
           save-always: true
 
+      - name: Enable Cachix binary cache
+        run: |
+            nix profile install nixpkgs#cachix
+            cachix use crane
+            cachix use nix-community
+
       - name: Apply Nix binary cache configuration
         run: |
             sudo tee -a "${XDG_CONFIG_HOME:-$HOME/.config}/nix/nix.conf" > /dev/null <<EOF
-            extra-substituters = https://attic.kennel.juneis.dog/conduit https://attic.kennel.juneis.dog/conduwuit https://cache.lix.systems https://conduwuit.cachix.org
+            extra-substituters = https://attic.kennel.juneis.dog/conduwuit https://attic.kennel.juneis.dog/conduit https://cache.lix.systems https://conduwuit.cachix.org https://aseipp-nix-cache.freetls.fastly.net
             extra-trusted-public-keys = conduit:eEKoUwlQGDdYmAI/Q/0slVlegqh/QmAvQd7HBSm21Wk= conduwuit:BbycGUgTISsltcmH0qNjFR9dbrQNYgdIAcmViSGoVTE= cache.lix.systems:aBnZUw8zA7H35Cz2RyKFVs3H4PlGTLawyY5KRbvJR8o= conduwuit.cachix.org-1:MFRm6jcnfTf0jSAbmvLfhO3KBMt4px+1xaereWXp8Xg=
+            experimental-features = nix-command flakes
+            extra-experimental-features = nix-command flakes
+            accept-flake-config = true
             EOF
 
       - name: Use alternative Nix binary caches if specified
@@ -110,23 +116,7 @@ jobs:
 
       - name: Cache CI dependencies
         run: |
-            # attic nix binary cache server is very, very terribly flakey. nothing i can do to fix it other than retry multiple times here
+            bin/nix-build-and-cache ci
-            ATTEMPTS=3
-            SUCCESS=false
-            while (( ATTEMPTS-- > 0 ))
-            do
-              bin/nix-build-and-cache ci
-              if [[ $? == 0 ]]; then
-                SUCCESS=true
-                break
-              else
-                sleep 3
-              fi
-            done
-
-            if [[ $SUCCESS == "false" ]]; then
-              exit 1
-            fi
 
       - name: Run lychee and markdownlint
         run: |
@@ -135,23 +125,7 @@ jobs:
 
       - name: Build documentation (book)
         run: |
-            # attic nix binary cache server is very, very terribly flakey. nothing i can do to fix it other than retry multiple times here
+            bin/nix-build-and-cache just .#book
-            ATTEMPTS=3
-            SUCCESS=false
-            while (( ATTEMPTS-- > 0 ))
-            do
-              bin/nix-build-and-cache just .#book
-              if [[ $? == 0 ]]; then
-                SUCCESS=true
-                break
-              else
-                sleep 3
-              fi
-            done
-
-            if [[ $SUCCESS == "false" ]]; then
-              exit 1
-            fi
 
             cp -r --dereference result public
 

.github/workflows/trivy.yml

@@ -26,7 +26,7 @@ jobs:
         uses: actions/checkout@v4
 
       - name: Run Trivy code and vulnerability scanner on repo
-        uses: aquasecurity/trivy-action@0.24.0
+        uses: aquasecurity/trivy-action@0.28.0
         with:
           scan-type: repo
           format: sarif
@@ -34,7 +34,7 @@ jobs:
           severity: CRITICAL,HIGH,MEDIUM,LOW
 
       - name: Run Trivy code and vulnerability scanner on filesystem
-        uses: aquasecurity/trivy-action@0.24.0
+        uses: aquasecurity/trivy-action@0.28.0
         with:
           scan-type: fs
           format: sarif

.gitlab-ci.yml

@@ -10,6 +10,13 @@ variables:
   FF_USE_FASTZIP: true
   # Print progress reports for cache and artifact transfers
   TRANSFER_METER_FREQUENCY: 5s
+  NIX_CONFIG: |
+    show-trace = true
+    extra-substituters = https://attic.kennel.juneis.dog/conduit https://attic.kennel.juneis.dog/conduwuit https://cache.lix.systems https://conduwuit.cachix.org
+    extra-trusted-public-keys = conduit:eEKoUwlQGDdYmAI/Q/0slVlegqh/QmAvQd7HBSm21Wk= conduwuit:BbycGUgTISsltcmH0qNjFR9dbrQNYgdIAcmViSGoVTE= cache.lix.systems:aBnZUw8zA7H35Cz2RyKFVs3H4PlGTLawyY5KRbvJR8o= conduwuit.cachix.org-1:MFRm6jcnfTf0jSAbmvLfhO3KBMt4px+1xaereWXp8Xg=
+    experimental-features = nix-command flakes
+    extra-experimental-features = nix-command flakes
+    accept-flake-config = true
 
 # Avoid duplicate pipelines
 # See: https://docs.gitlab.com/ee/ci/yaml/workflow.html#switch-between-branch-pipelines-and-merge-request-pipelines
@@ -23,6 +30,9 @@ workflow:
 before_script:
   # Enable nix-command and flakes
   - if command -v nix > /dev/null; then echo "experimental-features = nix-command flakes" >> /etc/nix/nix.conf; fi
+  - if command -v nix > /dev/null; then echo "extra-experimental-features = nix-command flakes" >> /etc/nix/nix.conf; fi
+  # Accept flake config from "untrusted" users
+  - if command -v nix > /dev/null; then echo "accept-flake-config = true" >> /etc/nix/nix.conf; fi
 
   # Add conduwuit binary cache
   - if command -v nix > /dev/null; then echo "extra-substituters = https://attic.kennel.juneis.dog/conduwuit" >> /etc/nix/nix.conf; fi
@@ -47,6 +57,8 @@ before_script:
   - if command -v nix > /dev/null; then echo "extra-substituters = https://nix-community.cachix.org" >> /etc/nix/nix.conf; fi
   - if command -v nix > /dev/null; then echo "extra-trusted-public-keys = nix-community.cachix.org-1:mB9FSh9qf2dCimDSUo8Zy7bkq5CX+/rkCWyvRCYg3Fs=" >> /etc/nix/nix.conf; fi
 
+  - if command -v nix > /dev/null; then echo "extra-substituters = https://aseipp-nix-cache.freetls.fastly.net" >> /etc/nix/nix.conf; fi
+
   # Install direnv and nix-direnv
   - if command -v nix > /dev/null; then nix-env -iA nixpkgs.direnv nixpkgs.nix-direnv; fi
 
@@ -58,7 +70,7 @@ before_script:
 
 ci:
   stage: ci
-  image: nixos/nix:2.24.4
+  image: nixos/nix:2.24.9
   script:
     # Cache CI dependencies
     - ./bin/nix-build-and-cache ci
@@ -83,31 +95,31 @@ ci:
 
 artifacts:
   stage: artifacts
-  image: nixos/nix:2.24.4
+  image: nixos/nix:2.24.9
   script:
-    - ./bin/nix-build-and-cache just .#static-x86_64-unknown-linux-musl
+    - ./bin/nix-build-and-cache just .#static-x86_64-linux-musl
-    - cp result/bin/conduit x86_64-unknown-linux-musl
+    - cp result/bin/conduit x86_64-linux-musl
 
     - mkdir -p target/release
     - cp result/bin/conduit target/release
     - direnv exec . cargo deb --no-build --no-strip
-    - mv target/debian/*.deb x86_64-unknown-linux-musl.deb
+    - mv target/debian/*.deb x86_64-linux-musl.deb
 
     # Since the OCI image package is based on the binary package, this has the
     # fun side effect of uploading the normal binary too. Conduit users who are
     # deploying with Nix can leverage this fact by adding our binary cache to
     # their systems.
     #
-    # Note that although we have an `oci-image-x86_64-unknown-linux-musl`
+    # Note that although we have an `oci-image-x86_64-linux-musl`
     # output, we don't build it because it would be largely redundant to this
     # one since it's all containerized anyway.
     - ./bin/nix-build-and-cache just .#oci-image
     - cp result oci-image-amd64.tar.gz
 
-    - ./bin/nix-build-and-cache just .#static-aarch64-unknown-linux-musl
+    - ./bin/nix-build-and-cache just .#static-aarch64-linux-musl
-    - cp result/bin/conduit aarch64-unknown-linux-musl
+    - cp result/bin/conduit aarch64-linux-musl
 
-    - ./bin/nix-build-and-cache just .#oci-image-aarch64-unknown-linux-musl
+    - ./bin/nix-build-and-cache just .#oci-image-aarch64-linux-musl
     - cp result oci-image-arm64v8.tar.gz
 
     - ./bin/nix-build-and-cache just .#book
@@ -115,9 +127,9 @@ artifacts:
     - cp -r --dereference result public
   artifacts:
     paths:
-      - x86_64-unknown-linux-musl
+      - x86_64-linux-musl
-      - aarch64-unknown-linux-musl
+      - aarch64-linux-musl
-      - x86_64-unknown-linux-musl.deb
+      - x86_64-linux-musl.deb
       - oci-image-amd64.tar.gz
       - oci-image-arm64v8.tar.gz
       - public

CONTRIBUTING.md

@@ -1,7 +1,7 @@
 # Contributing guide
 
 This page is for about contributing to conduwuit. The
-[development](development.md) page may be of interest for you as well.
+[development](./development.md) page may be of interest for you as well.
 
 If you would like to work on an [issue][issues] that is not assigned, preferably
 ask in the Matrix room first at [#conduwuit:puppygock.gay][conduwuit-matrix],
@@ -67,7 +67,7 @@ failing from your changes, please review the logs (they are uploaded as
 artifacts) and determine if they're intended or not.
 
 If you'd like to run Complement locally using Nix, see the
-[testing](docs/development/testing.md) page.
+[testing](development/testing.md) page.
 
 [Sytest][sytest] support will come soon.
 
@@ -128,7 +128,10 @@ Direct all PRs/MRs to the `main` branch.
 
 By sending a pull request or patch, you are agreeing that your changes are
 allowed to be licenced under the Apache-2.0 licence and all of your conduct is
-in line with the Contributor's Covenant.
+in line with the Contributor's Covenant, and conduwuit's Code of Conduct.
+
+Contribution by users who violate either of these code of conducts will not have
+their contributions accepted.
 
 [issues]: https://github.com/girlbossceo/conduwuit/issues
 [conduwuit-matrix]: https://matrix.to/#/#conduwuit:puppygock.gay

Cargo.toml

@@ -19,8 +19,8 @@ license = "Apache-2.0"
 # See also `rust-toolchain.toml`
 readme = "README.md"
 repository = "https://github.com/girlbossceo/conduwuit"
-rust-version = "1.80.1"
+rust-version = "1.82.0"
-version = "0.4.6"
+version = "0.4.7"
 
 [workspace.metadata.crane]
 name = "conduit"
@@ -69,7 +69,7 @@ version = "0.8.5"
 
 # Used for the http request / response body type for Ruma endpoints used with reqwest
 [workspace.dependencies.bytes]
-version = "1.7.1"
+version = "1.7.2"
 
 [workspace.dependencies.http-body-util]
 version = "0.1.1"
@@ -101,22 +101,21 @@ features = ["typed-header", "tracing"]
 [workspace.dependencies.axum-server]
 version = "0.7.1"
 default-features = false
-features = ["tls-rustls"]
 
 # to listen on both HTTP and HTTPS if listening on TLS dierctly from conduwuit for complement or sytest
 [workspace.dependencies.axum-server-dual-protocol]
 version = "0.7"
 
 [workspace.dependencies.axum-client-ip]
-version = "0.6.0"
+version = "0.6.1"
 
 [workspace.dependencies.tower]
-version = "0.5.0"
+version = "0.5.1"
 default-features = false
 features = ["util"]
 
 [workspace.dependencies.tower-http]
-version = "0.5.2"
+version = "0.6.0"
 default-features = false
 features = [
     "add-extension",
@@ -129,10 +128,10 @@ features = [
 ]
 
 [workspace.dependencies.rustls]
-version = "0.23.12"
+version = "0.23.13"
 
 [workspace.dependencies.reqwest]
-version = "0.12.7"
+version = "0.12.8"
 default-features = false
 features = [
 	"rustls-tls-native-roots",
@@ -200,7 +199,7 @@ default-features = false
 
 # used for conduit's CLI and admin room command parsing
 [workspace.dependencies.clap]
-version = "4.5.15"
+version = "4.5.20"
 default-features = false
 features = [
 	"std",
@@ -248,7 +247,7 @@ features = ["alloc", "std"]
 default-features = false
 
 [workspace.dependencies.hyper]
-version = "1.4.1"
+version = "1.5.0"
 default-features = false
 features = [
 	"server",
@@ -257,7 +256,8 @@ features = [
 ]
 
 [workspace.dependencies.hyper-util]
-version = "0.1.6"
+# 0.1.9 causes DNS issues
+version = "=0.1.8"
 default-features = false
 features = [
 	"client",
@@ -314,7 +314,7 @@ version = "0.1.2"
 [workspace.dependencies.ruma]
 git = "https://github.com/girlbossceo/ruwuma"
 #branch = "conduwuit-changes"
-rev = "d7ddcd036f81edb257ab9371f9cadd46444e8a90"
+rev = "9900d0676564883cfade556d6e8da2a2c9061efd"
 features = [
     "compat",
     "rand",
@@ -329,18 +329,22 @@ features = [
     "ring-compat",
     "identifiers-validation",
     "unstable-unspecified",
+    "unstable-msc2409",
     "unstable-msc2448",
     "unstable-msc2666",
     "unstable-msc2867",
     "unstable-msc2870",
     "unstable-msc3026",
     "unstable-msc3061",
+    "unstable-msc3245",
     "unstable-msc3266",
     "unstable-msc3381", # polls
     "unstable-msc3489", # beacon / live location
     "unstable-msc3575",
+    "unstable-msc4075",
     "unstable-msc4121",
     "unstable-msc4125",
+    "unstable-msc4186",
     "unstable-extensible-events",
 ]
 
@@ -403,17 +407,17 @@ version = "0.34.0"
 # jemalloc usage
 [workspace.dependencies.tikv-jemalloc-sys]
 git = "https://github.com/girlbossceo/jemallocator"
-rev = "c32af15f3b440ae5e46c3404f78b19093bbd5294"
+rev = "d87938bfddc26377dd7fdf14bbcd345f3ab19442"
 default-features = false
 features = ["unprefixed_malloc_on_supported_platforms"]
 [workspace.dependencies.tikv-jemallocator]
 git = "https://github.com/girlbossceo/jemallocator"
-rev = "c32af15f3b440ae5e46c3404f78b19093bbd5294"
+rev = "d87938bfddc26377dd7fdf14bbcd345f3ab19442"
 default-features = false
 features = ["unprefixed_malloc_on_supported_platforms"]
 [workspace.dependencies.tikv-jemalloc-ctl]
 git = "https://github.com/girlbossceo/jemallocator"
-rev = "c32af15f3b440ae5e46c3404f78b19093bbd5294"
+rev = "d87938bfddc26377dd7fdf14bbcd345f3ab19442"
 default-features = false
 features = ["use_std"]
 
@@ -442,7 +446,7 @@ version = "0.4.3"
 default-features = false
 
 [workspace.dependencies.termimad]
-version = "0.30.0"
+version = "0.30.1"
 default-features = false
 
 [workspace.dependencies.checked_ops]
@@ -457,7 +461,7 @@ features = ["full", "extra-traits"]
 version = "1.0.36"
 
 [workspace.dependencies.proc-macro2]
-version = "1.0.86"
+version = "1.0.89"
 
 
 #
@@ -608,7 +612,7 @@ inherits = "release"
 # and can be raised if build times are tolerable.
 
 [profile.dev]
-debug = 1
+debug = "full"
 opt-level = 0
 panic = "unwind"
 debug-assertions = true
@@ -715,12 +719,16 @@ opt-level = 'z'
 # primarily used for CI
 [profile.test]
 inherits = "dev"
+strip = false
+opt-level = 0
 codegen-units = 16
 incremental = false
 
 [profile.test.package.'*']
 inherits = "dev"
 debug = 0
+strip = false
+opt-level = 0
 codegen-units = 16
 incremental = false
 
@@ -808,6 +816,7 @@ significant_drop_tightening = { level = "allow", priority = 1 }      # TODO
 pedantic = { level = "warn", priority = -1 }
 
 ## some sadness
+too_long_first_doc_paragraph = { level = "allow", priority = 1 }
 doc_markdown = { level = "allow", priority = 1 }
 enum_glob_use = { level = "allow", priority = 1 }
 if_not_else = { level = "allow", priority = 1 }

README.md

@@ -9,7 +9,7 @@ Artifacts](https://github.com/girlbossceo/conduwuit/actions/workflows/ci.yml/bad
 
 <!-- ANCHOR_END: catchphrase -->
 
-Visit the [Conduwuit documentation](https://conduwuit.puppyirl.gay/) for more
+Visit the [conduwuit documentation](https://conduwuit.puppyirl.gay/) for more
 information.
 
 <!-- ANCHOR: body -->
@@ -23,9 +23,9 @@ to communicate with users outside of Matrix, like a community on Discord.
 
 #### What is the goal?
 
-An efficient Matrix homeserver that's easy to set up and just works. You can
+A high-performance and efficient Matrix homeserver that's easy to set up and
-install it on a mini-computer like the Raspberry Pi to host Matrix for your
+just works. You can install it on a mini-computer like the Raspberry Pi to
-family, friends or company.
+host Matrix for your family, friends or company.
 
 #### Can I try it out?
 
@@ -42,9 +42,28 @@ transfem.dev is also listed at
 
 #### What is the current status?
 
-conduwuit is a hard fork of Conduit which is in beta, meaning you can join and
+conduwuit is technically a hard fork of Conduit, which is in Beta. The Beta status
-participate in most Matrix rooms, but not all features are supported and you
+initially was inherited from Conduit, however overtime this Beta status is rapidly
-might run into bugs from time to time.
+becoming less and less relevant as our codebase significantly diverges more and more.
+
+conduwuit is quite stable and very usable as a daily driver and for a low-medium
+sized homeserver. There is still a lot of more work to be done, but it is in a far
+better place than the project was in early 2024.
+
+#### How is conduwuit funded? Is conduwuit sustainable?
+
+conduwuit has no external funding. This is made possible purely in my freetime with
+contributors, also in their free time, and only by user-curated donations.
+
+conduwuit has existed since around November 2023, but [only became more publicly known
+in March/April 2024](https://matrix.org/blog/2024/04/26/this-week-in-matrix-2024-04-26/#conduwuit-website)
+and we have no plans in stopping or slowing down any time soon!
+
+#### Can I migrate or switch from Conduit?
+
+conduwuit is a complete drop-in replacement for Conduit. As long as you are using RocksDB,
+the only "migration" you need to do is replace the binary or container image. There
+is no harm or additional steps required for using conduwuit.
 
 <!-- ANCHOR_END: body -->
 
@@ -59,8 +78,12 @@ If you run into any question, feel free to
 
 #### Donate
 
+conduwuit development is purely made possible by myself and contributors. I do
+not get paid to work on this, and I work on it in my free time. Donations are
+heavily appreciated! 💜🥺
+
 - Liberapay: <https://liberapay.com/girlbossceo>
-- Ko-fi: <https://ko-fi.com/puppygock>
+- Ko-fi (note they take a fee): <https://ko-fi.com/puppygock>
 - GitHub Sponsors: <https://github.com/sponsors/girlbossceo>
 
 #### Logo

arch/conduwuit.service

@@ -15,7 +15,7 @@ DevicePolicy=closed
 LockPersonality=yes
 MemoryDenyWriteExecute=yes
 NoNewPrivileges=yes
-ProcSubset=pid
+#ProcSubset=pid
 ProtectClock=yes
 ProtectControlGroups=yes
 ProtectHome=yes

bin/complement

@@ -15,7 +15,7 @@ LOG_FILE="$2"
 # A `.jsonl` file to write test results to
 RESULTS_FILE="$3"
 
-OCI_IMAGE="complement-conduit:main"
+OCI_IMAGE="complement-conduwuit:main"
 
 # Complement tests that are skipped due to flakiness/reliability issues
 SKIPPED_COMPLEMENT_TESTS='-skip=TestClientSpacesSummary.*|TestJoinFederatedRoomFromApplicationServiceBridgeUser.*|TestJumpToDateEndpoint.*'
@@ -34,7 +34,7 @@ toplevel="$(git rev-parse --show-toplevel)"
 
 pushd "$toplevel" > /dev/null
 
-bin/nix-build-and-cache just .#static-complement
+bin/nix-build-and-cache just .#linux-complement
 
 docker load < result
 popd > /dev/null

bin/nix-build-and-cache

@@ -26,7 +26,12 @@ just() {
         "$ATTIC_TOKEN"
 
     # Find all output paths of the installables and their build dependencies
-    readarray -t derivations < <(nix path-info --derivation "$@")
+    #readarray -t derivations < <(nix path-info --derivation "$@")
+    derivations=()
+    while IFS=$'\n' read derivation; do
+        derivations+=("$derivation")
+    done < <(nix path-info --derivation "$@")
+
     cache=()
     for derivation in "${derivations[@]}"; do
         cache+=(
@@ -34,6 +39,9 @@ just() {
         )
     done
 
+    withattic() {
+        nix shell --inputs-from "$toplevel" attic --command xargs attic push "$@" <<< "${cache[*]}"
+    }
     # Upload them to Attic (conduit store)
     #
     # Use `xargs` and a here-string because something would probably explode if
@@ -41,8 +49,7 @@ just() {
     # store paths include a newline in them.
     (
         IFS=$'\n'
-        nix shell --inputs-from "$toplevel" attic -c xargs \
+        withattic conduit || withattic conduit || withattic conduit || true
-            attic push conduit <<< "${cache[*]}"
     )
 
     # main "conduwuit" store
@@ -59,8 +66,7 @@ just() {
     # store paths include a newline in them.
     (
         IFS=$'\n'
-        nix shell --inputs-from "$toplevel" attic -c xargs \
+        withattic conduwuit || withattic conduwuit || withattic conduwuit || true
-            attic push conduwuit <<< "${cache[*]}"
 
         # push to cachix if available
         if [ "$CACHIX_AUTH_TOKEN" ]; then
@@ -76,8 +82,8 @@ ci() {
         --inputs-from "$toplevel"
 
         # Keep sorted
-        "$toplevel#devShells.x86_64-linux.default"
+        #"$toplevel#devShells.x86_64-linux.default"
-        "$toplevel#devShells.x86_64-linux.all-features"
+        #"$toplevel#devShells.x86_64-linux.all-features"
         attic#default
         cachix#default
         nixpkgs#direnv

conduwuit-example.toml

@@ -351,6 +351,14 @@ allow_profile_lookup_federation_requests = true
 # defaults to true
 #admin_escape_commands = true
 
+# Controls whether admin room notices like account registrations, password changes, account deactivations,
+# room directory publications, etc will be sent to the admin room.
+#
+# Update notices and normal admin command responses will still be sent.
+#
+# defaults to true
+#admin_room_notices = true
+
 
 ### Misc
 
@@ -363,6 +371,11 @@ allow_profile_lookup_federation_requests = true
 # Defaults to "info"
 #log = "info"
 
+# controls whether logs will be outputted with ANSI colours
+#
+# defaults to true
+#log_colors = true
+
 # controls whether encrypted rooms and events are allowed (default true)
 #allow_encryption = false
 
@@ -481,6 +494,7 @@ allow_profile_lookup_federation_requests = true
 ### Generic database options
 
 # Set this to any float value to multiply conduwuit's in-memory LRU caches with.
+# By default, the caches scale automatically with cpu-core-count.
 # May be useful if you have significant memory to spare to increase performance.
 #
 # This was previously called `conduit_cache_capacity_modifier`
@@ -490,7 +504,7 @@ allow_profile_lookup_federation_requests = true
 
 # Set this to any float value in megabytes for conduwuit to tell the database engine that this much memory is available for database-related caches.
 # May be useful if you have significant memory to spare to increase performance.
-# Defaults to 256.0
+# Defaults to 128.0 + (64.0 * CPU core count).
 #db_cache_capacity_mb = 256.0
 
 
@@ -844,9 +858,20 @@ allow_profile_lookup_federation_requests = true
 
 # vector list of TURN URIs/servers to use
 #
+# replace "example.turn.uri" with your TURN domain, such as the coturn "realm".
+# if using TURN over TLS, replace "turn:" with "turns:"
+#
 # No default
 #turn_uris = ["turn:example.turn.uri?transport=udp", "turn:example.turn.uri?transport=tcp"]
 
+# TURN secret to use that's read from the file path specified
+#
+# this takes priority over "turn_secret" first, and falls back to "turn_secret" if invalid or
+# failed to open.
+#
+# no default
+#turn_secret_file = "/path/to/secret.txt"
+
 # TURN secret to use for generating the HMAC-SHA1 hash apart of username and password generation
 #
 # this is more secure, but if needed you can use traditional username/password below.

debian/conduwuit.service

@@ -22,7 +22,7 @@ DevicePolicy=closed
 LockPersonality=yes
 MemoryDenyWriteExecute=yes
 NoNewPrivileges=yes
-ProcSubset=pid
+#ProcSubset=pid
 ProtectClock=yes
 ProtectControlGroups=yes
 ProtectHome=yes

deps/rust-rocksdb/Cargo.toml

@@ -27,7 +27,7 @@ malloc-usable-size = ["rust-rocksdb/malloc-usable-size"]
 
 [dependencies.rust-rocksdb]
 git = "https://github.com/girlbossceo/rust-rocksdb-zaidoon1"
-rev = "5383ca8173299066b516406e3a2cf945ead891cb"
+rev = "c1e5523eae095a893deaf9056128c7dbc2d5fd73"
 #branch = "master"
 default-features = false
 

development.md

@@ -0,0 +1 @@
+docs/development.md

docs/contributing.md

@@ -1 +0,0 @@
-{{#include ../CONTRIBUTING.md}}

docs/contributing.md

@@ -0,0 +1 @@
+../CONTRIBUTING.md

docs/deploying/generic.md

@@ -13,18 +13,38 @@ what you need.
 
 Prebuilt fully static musl binaries can be downloaded from the latest tagged
 release [here](https://github.com/girlbossceo/conduwuit/releases/latest) or
-`main` CI branch workflow artifact output. These also include Debian packages.
+`main` CI branch workflow artifact output. These also include Debian/Ubuntu packages.
+
 These binaries have jemalloc and io_uring statically linked and included with
-them.
+them, so no additional dynamic dependencies need to be installed.
 
 Alternatively, you may compile the binary yourself. We recommend using
-[Lix](https://lix.systems) to build conduwuit as this has the most guaranteed
+Nix (or [Lix](https://lix.systems)) to build conduwuit as this has the most guaranteed
-reproducibiltiy and easiest to get a build environment and output going.
+reproducibiltiy and easiest to get a build environment and output going. This also
+allows easy cross-compilation.
+
+You can run the `nix build -L .#static-x86_64-linux-musl-all-features` or
+`nix build -L .#static-aarch64-linux-musl-all-features` commands based
+on architecture to cross-compile the necessary static binary located at
+`result/bin/conduit`. This is reproducible with the static binaries produced in our CI.
 
 Otherwise, follow standard Rust project build guides (installing git and cloning
 the repo, getting the Rust toolchain via rustup, installing LLVM toolchain +
 libclang for RocksDB, installing liburing for io_uring and RocksDB, etc).
 
+## Migrating from Conduit
+
+As mentioned in the README, there is little to no steps needed to migrate
+from Conduit. As long as you are using the RocksDB database backend, just
+replace the binary / container image / etc.
+
+**Note**: If you are relying on Conduit's "automatic delegation" feature,
+this will **NOT** work on conduwuit and you must configure delegation manually.
+This is not a mistake and no support for this feature will be added.
+
+See the `[global.well_known]` config section, or configure your web server
+appropriately to send the delegation responses.
+
 ## Adding a conduwuit user
 
 While conduwuit can run as any user it is better to use dedicated users for
@@ -58,6 +78,8 @@ The systemd unit for conduwuit can be found
 [here](../configuration/examples.md#example-systemd-unit-file). You may need to
 change the `ExecStart=` path to where you placed the conduwuit binary.
 
+On systems where rsyslog is used alongside journald (i.e. Red Hat-based distros and OpenSUSE), put `$EscapeControlCharactersOnReceive off` inside `/etc/rsyslog.conf` to allow color in logs.
+
 ## Creating the conduwuit configuration file
 
 Now we need to create the conduwuit's config file in
@@ -74,26 +96,47 @@ If you are using a dedicated user for conduwuit, you will need to allow it to
 read the config. To do that you can run this:
 
 ```bash
-sudo chown -R root:root /etc/conduwuit sudo chmod -R 755 /etc/conduwuit
+sudo chown -R root:root /etc/conduwuit
+sudo chmod -R 755 /etc/conduwuit
 ```
 
 If you use the default database path you also need to run this:
 
 ```bash
-sudo mkdir -p /var/lib/conduwuit/ sudo chown -R conduwuit:conduwuit /var/lib/conduwuit/
+sudo mkdir -p /var/lib/conduwuit/
+sudo chown -R conduwuit:conduwuit /var/lib/conduwuit/
 sudo chmod 700 /var/lib/conduwuit/
 ```
 
 ## Setting up the Reverse Proxy
 
 Refer to the documentation or various guides online of your chosen reverse proxy
-software. A [Caddy](https://caddyserver.com/) example will be provided as this
+software. There are many examples of basic Apache/Nginx reverse proxy setups
+out there.
+
+A [Caddy](https://caddyserver.com/) example will be provided as this
 is the recommended reverse proxy for new users and is very trivial to use
 (handles TLS, reverse proxy headers, etc transparently with proper defaults).
 
 Lighttpd is not supported as it seems to mess with the `X-Matrix` Authorization
 header, making federation non-functional. If using Apache, you need to use
-`nocanon` to prevent this.
+`nocanon` in your `ProxyPass` directive to prevent this (note that Apache
+isn't very good as a general reverse proxy).
+
+Nginx users may need to set `proxy_buffering off;` if there are issues with
+uploading media like images.
+
+You will need to reverse proxy everything under following routes:
+- `/_matrix/` - core Matrix C-S and S-S APIs
+- `/_conduwuit/` - ad-hoc conduwuit routes such as `/local_user_count` and
+`/server_version`
+
+You can optionally reverse proxy the following individual routes:
+- `/.well-known/matrix/client` and `/.well-known/matrix/server` if using
+conduwuit to perform delegation
+- `/.well-known/matrix/support` if using conduwuit to send the homeserver admin
+contact and support page (formerly known as MSC1929)
+- `/` if you would like to see `hewwo from conduwuit woof!` at the root
 
 ### Caddy
 
@@ -142,6 +185,9 @@ curl https://your.server.name/_conduwuit/server_version
 
 # If using port 8448
 curl https://your.server.name:8448/_conduwuit/server_version
+
+# If federation is enabled
+curl https://your.server.name:8448/_matrix/federation/v1/version
 ```
 
 - To check if your server can talk with other homeservers, you can use the

docs/deploying/nixos.md

@@ -1,11 +1,15 @@
 # conduwuit for NixOS
 
-conduwuit can be acquired by [Lix][lix] from various places:
+conduwuit can be acquired by Nix (or [Lix][lix]) from various places:
 
 * The `flake.nix` at the root of the repo
 * The `default.nix` at the root of the repo
 * From conduwuit's binary cache
 
+A community maintained NixOS package is available at [`conduwuit`](https://search.nixos.org/packages?channel=unstable&show=conduwuit&from=0&size=50&sort=relevance&type=packages&query=conduwuit)
+
+### Binary cache
+
 A binary cache for conduwuit that the CI/CD publishes to is available at the
 following places (both are the same just different names):
 
@@ -17,25 +21,57 @@ https://attic.kennel.juneis.dog/conduwuit
 conduwuit:BbycGUgTISsltcmH0qNjFR9dbrQNYgdIAcmViSGoVTE=
 ```
 
-The binary caches have been recreated recently due to attic issues. The old
+The binary caches were recreated some months ago due to attic issues. The old public
-public keys were:
+keys were:
 
 ```
 conduit:Isq8FGyEC6FOXH6nD+BOeAA+bKp6X6UIbupSlGEPuOg=
 conduwuit:lYPVh7o1hLu1idH4Xt2QHaRa49WRGSAqzcfFd94aOTw=
 ```
 
-If specifying a URL in your flake, please use the GitHub remote:
-`github:girlbossceo/conduwuit`
 
-The `flake.nix` and `default.nix` do not (currently) provide a NixOS module, so
+If specifying a Git remote URL in your flake, you can use any remotes that
-(for now) [`services.matrix-conduit`][module] from Nixpkgs should be used to
+are specified on the README (the mirrors), such as the GitHub: `github:girlbossceo/conduwuit`
-configure conduwuit.
 
-If you want to run the latest code, you should get conduwuit from the
+### NixOS module
-`flake.nix` or `default.nix` and set
+
-[`services.matrix-conduit.package`][package] appropriately.
+The `flake.nix` and `default.nix` do not currently provide a NixOS module (contributions
+welcome!), so [`services.matrix-conduit`][module] from Nixpkgs can be used to configure
+conduwuit.
+
+If you want to run the latest code, you should get conduwuit from the `flake.nix`
+or `default.nix` and set [`services.matrix-conduit.package`][package]
+appropriately to use conduwuit instead of Conduit.
+
+### UNIX sockets
+
+Due to the lack of a conduwuit NixOS module, when using the `services.matrix-conduit` module
+it is not possible to use UNIX sockets. This is because the UNIX socket option does not exist
+in Conduit, and their module forces listening on `[::1]:6167` by default if unspecified.
+
+Additionally, the [`matrix-conduit` systemd unit][systemd-unit] in the module does not allow
+the `AF_UNIX` socket address family in their systemd unit's `RestrictAddressFamilies=` which
+disallows the namespace from accessing or creating UNIX sockets.
+
+There is no known workaround these. A conduwuit NixOS configuration module must be developed and
+published by the community.
+
+### jemalloc and hardened profile
+
+conduwuit uses jemalloc by default. This may interfere with the [`hardened.nix` profile][hardened.nix]
+due to them using `scudo` by default. You must either disable/hide `scudo` from conduwuit, or
+disable jemalloc like so:
+
+```nix
+let
+    conduwuit = pkgs.unstable.conduwuit.override {
+      enableJemalloc = false;
+    };
+in
+```
 
 [lix]: https://lix.systems/
 [module]: https://search.nixos.org/options?channel=unstable&query=services.matrix-conduit
 [package]: https://search.nixos.org/options?channel=unstable&query=services.matrix-conduit.package
+[hardened.nix]: https://github.com/NixOS/nixpkgs/blob/master/nixos/modules/profiles/hardened.nix#L22
+[systemd-unit]: https://github.com/NixOS/nixpkgs/blob/master/nixos/modules/services/matrix/conduit.nix#L132

docs/development.md

@@ -1,10 +1,74 @@
 # Development
 
 Information about developing the project. If you are only interested in using
-it, you can safely ignore this section. If you plan on contributing, see the
+it, you can safely ignore this page. If you plan on contributing, see the
-[contributor's guide](contributing.md).
+[contributor's guide](./contributing.md).
 
-## List of forked dependencies During conduwuit development, we have had to fork
+## conduwuit project layout
+
+conduwuit uses a collection of sub-crates, packages, or workspace members
+that indicate what each general area of code is for. All of the workspace
+members are under `src/`. The workspace definition is at the top level / root
+`Cargo.toml`.
+
+The crate names are generally self-explanatory:
+- `admin` is the admin room
+- `api` is the HTTP API, Matrix C-S and S-S endpoints, etc
+- `core` is core conduwuit functionality like config loading, error definitions,
+global utilities, logging infrastructure, etc
+- `database` is RocksDB methods, helpers, RocksDB config, and general database definitions,
+utilities, or functions
+- `macros` are conduwuit Rust [macros][macros] like general helper macros, logging
+and error handling macros, and [syn][syn] and [procedural macros][proc-macro]
+used for admin room commands and others
+- `main` is the "primary" sub-crate. This is where the `main()` function lives,
+tokio worker and async initialisation, Sentry initialisation, [clap][clap] init,
+and signal handling. If you are adding new [Rust features][features], they *must*
+go here.
+- `router` is the webserver and request handling bits, using axum, tower, tower-http,
+hyper, etc, and the [global server state][state] to access `services`.
+- `service` is the high-level database definitions and functions for data,
+outbound/sending code, and other business logic such as media fetching.
+
+It is highly unlikely you will ever need to add a new workspace member, but
+if you truly find yourself needing to, we recommend reaching out to us in
+the Matrix room for discussions about it beforehand.
+
+The primary inspiration for this design was apart of hot reloadable development,
+to support "conduwuit as a library" where specific parts can simply be swapped out.
+There is evidence Conduit wanted to go this route too as `axum` is technically an
+optional feature in Conduit, and can be compiled without the binary or axum library
+for handling inbound web requests; but it was never completed or worked.
+
+See the Rust documentation on [Workspaces][workspaces] for general questions
+and information on Cargo workspaces.
+
+## Adding compile-time [features][features]
+
+If you'd like to add a compile-time feature, you must first define it in
+the `main` workspace crate located in `src/main/Cargo.toml`. The feature must
+enable a feature in the other workspace crate(s) you intend to use it in. Then
+the said workspace crate(s) must define the feature there in its `Cargo.toml`.
+
+So, if this is adding a feature to the API such as `woof`, you define the feature
+in the `api` crate's `Cargo.toml` as `woof = []`. The feature definition in `main`'s
+`Cargo.toml` will be `woof = ["conduit-api/woof"]`.
+
+The rationale for this is due to Rust / Cargo not supporting
+["workspace level features"][9], we must make a choice of; either scattering
+features all over the workspace crates, making it difficult for anyone to add
+or remove default features; or define all the features in one central workspace
+crate that propagate down/up to the other workspace crates. It is a Cargo pitfall,
+and we'd like to see better developer UX in Rust's Workspaces.
+
+Additionally, the definition of one single place makes "feature collection" in our
+Nix flake a million times easier instead of collecting and deduping them all from
+searching in all the workspace crates' `Cargo.toml`s. Though we wouldn't need to
+do this if Rust supported workspace-level features to begin with.
+
+## List of forked dependencies
+
+During conduwuit development, we have had to fork
 some dependencies to support our use-cases in some areas. This ranges from
 things said upstream project won't accept for any reason, faster-paced
 development (unresponsive or slow upstream), conduwuit-specific usecases, or
@@ -53,3 +117,11 @@ RUSTFLAGS="--cfg tokio_unstable" cargo build \
 [6]: https://github.com/tokio-rs/tracing/
 [7]: https://docs.rs/tokio-console/latest/tokio_console/
 [8]: https://github.com/zaidoon1/
+[9]: https://github.com/rust-lang/cargo/issues/12162
+[workspaces]: https://doc.rust-lang.org/cargo/reference/workspaces.html
+[macros]: https://doc.rust-lang.org/book/ch19-06-macros.html
+[syn]: https://docs.rs/syn/latest/syn/
+[proc-macro]: https://doc.rust-lang.org/reference/procedural-macros.html
+[clap]: https://docs.rs/clap/latest/clap/
+[features]: https://doc.rust-lang.org/cargo/reference/features.html
+[state]: https://docs.rs/axum/latest/axum/extract/struct.State.html

docs/development/testing.md

@@ -5,8 +5,8 @@
 Have a look at [Complement's repository][complement] for an explanation of what
 it is.
 
-To test against Complement, with [Lix][lix] and direnv installed and set up, you
+To test against Complement, with Nix (or [Lix](https://lix.systems) and direnv installed
-can:
+and set up, you can:
 
 * Run `./bin/complement "$COMPLEMENT_SRC" ./path/to/logs.jsonl
 ./path/to/results.jsonl` to build a Complement image, run the tests, and output
@@ -18,6 +18,5 @@ Complement OCI image outputted to `result` (it's a `.tar.gz` file)
 output from the commit/revision you want to test (e.g. from main)
 [here][ci-workflows]
 
-[lix]: https://lix.systems/
 [ci-workflows]: https://github.com/girlbossceo/conduwuit/actions/workflows/ci.yml?query=event%3Apush+is%3Asuccess+actor%3Agirlbossceo
 [complement]: https://github.com/matrix-org/complement

docs/turn.md

@@ -21,6 +21,22 @@ These same values need to be set in conduwuit. See the [example
 config](configuration/examples.md) in the TURN section for configuring these and
 restart conduwuit after.
 
+`turn_secret` or a path to `turn_secret_file` must have a value of your
+coturn `static-auth-secret`, or use `turn_username` and `turn_password`
+if using legacy username:password TURN authentication (not preferred).
+
+`turn_uris` must be the list of TURN URIs you would like to send to the client.
+Typically you will just replace the example domain `example.turn.uri` with the
+`realm` you set from the example config.
+
+If you are using TURN over TLS, you can replace `turn:` with `turns:` in the
+`turn_uris` config option to instruct clients to attempt to connect to
+TURN over TLS. This is highly recommended.
+
+If you need unauthenticated access to the TURN URIs, or some clients may be
+having trouble, you can enable `turn_guest_access` in conduwuit which disables
+authentication for the TURN URI endpoint `/_matrix/client/v3/voip/turnServer`
+
 ### Run
 
 Run the [Coturn](https://hub.docker.com/r/coturn/coturn) image using

engage.toml

@@ -152,7 +152,7 @@ cargo clippy \
 [[task]]
 name = "lychee"
 group = "lints"
-script = "lychee --verbose --offline docs *.md --exclude development.md"
+script = "lychee --verbose --offline docs *.md --exclude development.md --exclude contributing.md --exclude testing.md"
 
 [[task]]
 name = "markdownlint"

flake.lock

@@ -4,16 +4,16 @@
       "inputs": {
         "crane": "crane",
         "flake-compat": "flake-compat",
-        "flake-utils": "flake-utils",
+        "flake-parts": "flake-parts",
         "nixpkgs": "nixpkgs",
         "nixpkgs-stable": "nixpkgs-stable"
       },
       "locked": {
-        "lastModified": 1724226964,
+        "lastModified": 1729116596,
-        "narHash": "sha256-cltFh4su2vcFidxKp7LuEgX3ZGLfPy0DCdrQZ/QTe68=",
+        "narHash": "sha256-NnLMLIXGZtAscUF4dCShksuQ1nOGF6Y2dEeyj0rBbUg=",
         "owner": "zhaofengli",
         "repo": "attic",
-        "rev": "6d9aeaef0a067d664cb11bb7704f7ec373d47fb2",
+        "rev": "2b05b7d986cf6009b1c1ef7daa4961cd1a658782",
         "type": "github"
       },
       "original": {
@@ -28,14 +28,14 @@
         "devenv": "devenv",
         "flake-compat": "flake-compat_3",
         "git-hooks": "git-hooks",
-        "nixpkgs": "nixpkgs_3"
+        "nixpkgs": "nixpkgs_4"
       },
       "locked": {
-        "lastModified": 1724232775,
+        "lastModified": 1728672398,
-        "narHash": "sha256-6u2DycIEgrgNYlLxyGqdFVmBNiKIitnQKJ1pbRP5oko=",
+        "narHash": "sha256-KxuGSoVUFnQLB2ZcYODW7AVPAh9JqRlD5BrfsC/Q4qs=",
         "owner": "cachix",
         "repo": "cachix",
-        "rev": "03b6cb3f953097bff378fb8b9ea094bd091a4ec7",
+        "rev": "aac51f698309fd0f381149214b7eee213c66ef0a",
         "type": "github"
       },
       "original": {
@@ -53,12 +53,51 @@
           "devenv",
           "flake-compat"
         ],
+        "git-hooks": [
+          "cachix",
+          "devenv",
+          "pre-commit-hooks"
+        ],
         "nixpkgs": [
           "cachix",
           "devenv",
           "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1726520618,
+        "narHash": "sha256-jOsaBmJ/EtX5t/vbylCdS7pWYcKGmWOKg4QKUzKr6dA=",
+        "owner": "cachix",
+        "repo": "cachix",
+        "rev": "695525f9086542dfb09fde0871dbf4174abbf634",
+        "type": "github"
+      },
+      "original": {
+        "owner": "cachix",
+        "repo": "cachix",
+        "type": "github"
+      }
+    },
+    "cachix_3": {
+      "inputs": {
+        "devenv": "devenv_3",
+        "flake-compat": [
+          "cachix",
+          "devenv",
+          "cachix",
+          "devenv",
+          "flake-compat"
+        ],
+        "nixpkgs": [
+          "cachix",
+          "devenv",
+          "cachix",
+          "devenv",
+          "nixpkgs"
         ],
         "pre-commit-hooks": [
+          "cachix",
+          "devenv",
           "cachix",
           "devenv",
           "pre-commit-hooks"
@@ -81,11 +120,11 @@
     "complement": {
       "flake": false,
       "locked": {
-        "lastModified": 1722323564,
+        "lastModified": 1724347376,
-        "narHash": "sha256-6w6/N8walz4Ayc9zu7iySqJRmGFukhkaICLn4dweAcA=",
+        "narHash": "sha256-y0e/ULDJ92IhNQZsS/06g0s+AYZ82aJfrIO9qEse94c=",
         "owner": "matrix-org",
         "repo": "complement",
-        "rev": "6e4426a9e63233f9821a4d2382bfed145244183f",
+        "rev": "39733c1b2f8314800776748cc7164f9a34650686",
         "type": "github"
       },
       "original": {
@@ -117,17 +156,12 @@
       }
     },
     "crane_2": {
-      "inputs": {
-        "nixpkgs": [
-          "nixpkgs"
-        ]
-      },
       "locked": {
-        "lastModified": 1724006180,
+        "lastModified": 1729741221,
-        "narHash": "sha256-PVxPj0Ga2fMYMtcT9ARCthF+4U71YkOT7ZjgD/vf1Aw=",
+        "narHash": "sha256-8AHZZXs1lFkERfBY0C8cZGElSo33D/et7NKEpLRmvzo=",
         "owner": "ipetkov",
         "repo": "crane",
-        "rev": "7ce92819802bc583b7e82ebc08013a530f22209f",
+        "rev": "f235b656ee5b2bfd6d94c3bfd67896a575d4a6ed",
         "type": "github"
       },
       "original": {
@@ -144,7 +178,7 @@
           "cachix",
           "flake-compat"
         ],
-        "nix": "nix_2",
+        "nix": "nix_3",
         "nixpkgs": [
           "cachix",
           "nixpkgs"
@@ -154,6 +188,43 @@
           "git-hooks"
         ]
       },
+      "locked": {
+        "lastModified": 1727963652,
+        "narHash": "sha256-os0EDjn7QVXL6RtHNb9TrZLXVm2Tc5/nZKk3KpbTzd8=",
+        "owner": "cachix",
+        "repo": "devenv",
+        "rev": "cb0052e25dbcc8267b3026160dc73cddaac7d5fd",
+        "type": "github"
+      },
+      "original": {
+        "owner": "cachix",
+        "repo": "devenv",
+        "type": "github"
+      }
+    },
+    "devenv_2": {
+      "inputs": {
+        "cachix": "cachix_3",
+        "flake-compat": [
+          "cachix",
+          "devenv",
+          "cachix",
+          "flake-compat"
+        ],
+        "nix": "nix_2",
+        "nixpkgs": [
+          "cachix",
+          "devenv",
+          "cachix",
+          "nixpkgs"
+        ],
+        "pre-commit-hooks": [
+          "cachix",
+          "devenv",
+          "cachix",
+          "git-hooks"
+        ]
+      },
       "locked": {
         "lastModified": 1723156315,
         "narHash": "sha256-0JrfahRMJ37Rf1i0iOOn+8Z4CLvbcGNwa2ChOAVrp/8=",
@@ -168,9 +239,11 @@
         "type": "github"
       }
     },
-    "devenv_2": {
+    "devenv_3": {
       "inputs": {
         "flake-compat": [
+          "cachix",
+          "devenv",
           "cachix",
           "devenv",
           "cachix",
@@ -180,6 +253,8 @@
         "nixpkgs": "nixpkgs_2",
         "poetry2nix": "poetry2nix",
         "pre-commit-hooks": [
+          "cachix",
+          "devenv",
           "cachix",
           "devenv",
           "cachix",
@@ -209,11 +284,11 @@
         "rust-analyzer-src": "rust-analyzer-src"
       },
       "locked": {
-        "lastModified": 1724221791,
+        "lastModified": 1729751566,
-        "narHash": "sha256-mKX67QPnUybOopVph/LhOV1G/H4EvPxDIfSmbufrVdA=",
+        "narHash": "sha256-99u/hrgBdi8bxSXZc9ZbNkR5EL1htrkbd3lsbKzS60g=",
         "owner": "nix-community",
         "repo": "fenix",
-        "rev": "e88b38a5a3834e039d413a88f8150a75ef6453ef",
+        "rev": "f32a2d484091a6dc98220b1f4a2c2d60b7c97c64",
         "type": "github"
       },
       "original": {
@@ -288,27 +363,53 @@
         "type": "github"
       }
     },
-    "flake-utils": {
+    "flake-parts": {
       "inputs": {
-        "systems": "systems"
+        "nixpkgs-lib": [
+          "attic",
+          "nixpkgs"
+        ]
       },
       "locked": {
-        "lastModified": 1710146030,
+        "lastModified": 1722555600,
-        "narHash": "sha256-SZ5L6eA7HJ/nmkzGG7/ISclqe6oZdOZTNoesiInkXPQ=",
+        "narHash": "sha256-XOQkdLafnb/p9ij77byFQjDf5m5QYl9b2REiVClC+x4=",
-        "owner": "numtide",
+        "owner": "hercules-ci",
-        "repo": "flake-utils",
+        "repo": "flake-parts",
-        "rev": "b1d9ab70662946ef0850d488da1c9019f3a9752a",
+        "rev": "8471fe90ad337a8074e957b69ca4d0089218391d",
         "type": "github"
       },
       "original": {
-        "owner": "numtide",
+        "owner": "hercules-ci",
-        "repo": "flake-utils",
+        "repo": "flake-parts",
         "type": "github"
       }
     },
-    "flake-utils_2": {
+    "flake-parts_2": {
       "inputs": {
-        "systems": "systems_2"
+        "nixpkgs-lib": [
+          "cachix",
+          "devenv",
+          "nix",
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1712014858,
+        "narHash": "sha256-sB4SWl2lX95bExY2gMFG5HIzvva5AVMJd4Igm+GpZNw=",
+        "owner": "hercules-ci",
+        "repo": "flake-parts",
+        "rev": "9126214d0a59633752a136528f5f3b9aa8565b7d",
+        "type": "github"
+      },
+      "original": {
+        "owner": "hercules-ci",
+        "repo": "flake-parts",
+        "type": "github"
+      }
+    },
+    "flake-utils": {
+      "inputs": {
+        "systems": "systems"
       },
       "locked": {
         "lastModified": 1689068808,
@@ -324,9 +425,24 @@
         "type": "github"
       }
     },
+    "flake-utils_2": {
+      "locked": {
+        "lastModified": 1667395993,
+        "narHash": "sha256-nuEHfE/LcWyuSWnS8t12N1wc105Qtau+/OdUAjtQ0rA=",
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "rev": "5aed5285a952e0b949eb3ba02c12fa4fcfef535f",
+        "type": "github"
+      },
+      "original": {
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "type": "github"
+      }
+    },
     "flake-utils_3": {
       "inputs": {
-        "systems": "systems_3"
+        "systems": "systems_2"
       },
       "locked": {
         "lastModified": 1710146030,
@@ -357,11 +473,11 @@
         "nixpkgs-stable": "nixpkgs-stable_2"
       },
       "locked": {
-        "lastModified": 1723202784,
+        "lastModified": 1727854478,
-        "narHash": "sha256-qbhjc/NEGaDbyy0ucycubq4N3//gDFFH3DOmp1D3u1Q=",
+        "narHash": "sha256-/odH2nUMAwkMgOS2nG2z0exLQNJS4S2LfMW0teqU7co=",
         "owner": "cachix",
         "repo": "git-hooks.nix",
-        "rev": "c7012d0c18567c889b948781bc74a501e92275d1",
+        "rev": "5f58871c9657b5fc0a7f65670fe2ba99c26c1d79",
         "type": "github"
       },
       "original": {
@@ -392,14 +508,30 @@
         "type": "github"
       }
     },
+    "libgit2": {
+      "flake": false,
+      "locked": {
+        "lastModified": 1697646580,
+        "narHash": "sha256-oX4Z3S9WtJlwvj0uH9HlYcWv+x1hqp8mhXl7HsLu2f0=",
+        "owner": "libgit2",
+        "repo": "libgit2",
+        "rev": "45fd9ed7ae1a9b74b957ef4f337bc3c8b3df01b5",
+        "type": "github"
+      },
+      "original": {
+        "owner": "libgit2",
+        "repo": "libgit2",
+        "type": "github"
+      }
+    },
     "liburing": {
       "flake": false,
       "locked": {
-        "lastModified": 1724199144,
+        "lastModified": 1725659644,
-        "narHash": "sha256-MVjnwO6EbKzzSrU51dSseLarZ1fRp+6SagAf/nE/XZU=",
+        "narHash": "sha256-WjnpmopfvFoUbubIu9bki+Y6P4YXDfvnW4+72hniq3g=",
         "owner": "axboe",
         "repo": "liburing",
-        "rev": "2d4e799017d64cd2f8304503eef9064931bb3fbd",
+        "rev": "0fe5c09195c0918f89582dd6ff098a58a0bdf62a",
         "type": "github"
       },
       "original": {
@@ -417,6 +549,8 @@
           "devenv",
           "cachix",
           "devenv",
+          "cachix",
+          "devenv",
           "nixpkgs"
         ],
         "nixpkgs-regression": "nixpkgs-regression"
@@ -459,6 +593,8 @@
           "devenv",
           "cachix",
           "devenv",
+          "cachix",
+          "devenv",
           "poetry2nix",
           "nixpkgs"
         ]
@@ -480,11 +616,15 @@
     "nix_2": {
       "inputs": {
         "flake-compat": [
+          "cachix",
+          "devenv",
           "cachix",
           "devenv",
           "flake-compat"
         ],
         "nixpkgs": [
+          "cachix",
+          "devenv",
           "cachix",
           "devenv",
           "nixpkgs"
@@ -506,13 +646,42 @@
         "type": "github"
       }
     },
+    "nix_3": {
+      "inputs": {
+        "flake-compat": [
+          "cachix",
+          "devenv",
+          "flake-compat"
+        ],
+        "flake-parts": "flake-parts_2",
+        "libgit2": "libgit2",
+        "nixpkgs": "nixpkgs_3",
+        "nixpkgs-23-11": "nixpkgs-23-11",
+        "nixpkgs-regression": "nixpkgs-regression_3",
+        "pre-commit-hooks": "pre-commit-hooks"
+      },
+      "locked": {
+        "lastModified": 1727438425,
+        "narHash": "sha256-X8ES7I1cfNhR9oKp06F6ir4Np70WGZU5sfCOuNBEwMg=",
+        "owner": "domenkozar",
+        "repo": "nix",
+        "rev": "f6c5ae4c1b2e411e6b1e6a8181cc84363d6a7546",
+        "type": "github"
+      },
+      "original": {
+        "owner": "domenkozar",
+        "ref": "devenv-2.24",
+        "repo": "nix",
+        "type": "github"
+      }
+    },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1723827930,
+        "lastModified": 1726042813,
-        "narHash": "sha256-EU+W5F6y2CVNxGrGIMpY7nSVYq72WRChYxF4zpjx0y4=",
+        "narHash": "sha256-LnNKCCxnwgF+575y0pxUdlGZBO/ru1CtGHIqQVfvjlA=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "d4a7a4d0e066278bfb0d77bd2a7adde1c0ec9e3d",
+        "rev": "159be5db480d1df880a0135ca0bfed84c2f88353",
         "type": "github"
       },
       "original": {
@@ -522,6 +691,22 @@
         "type": "github"
       }
     },
+    "nixpkgs-23-11": {
+      "locked": {
+        "lastModified": 1717159533,
+        "narHash": "sha256-oamiKNfr2MS6yH64rUn99mIZjc45nGJlj9eGth/3Xuw=",
+        "owner": "NixOS",
+        "repo": "nixpkgs",
+        "rev": "a62e6edd6d5e1fa0329b8653c801147986f8d446",
+        "type": "github"
+      },
+      "original": {
+        "owner": "NixOS",
+        "repo": "nixpkgs",
+        "rev": "a62e6edd6d5e1fa0329b8653c801147986f8d446",
+        "type": "github"
+      }
+    },
     "nixpkgs-regression": {
       "locked": {
         "lastModified": 1643052045,
@@ -554,18 +739,34 @@
         "type": "github"
       }
     },
-    "nixpkgs-stable": {
+    "nixpkgs-regression_3": {
       "locked": {
-        "lastModified": 1720535198,
+        "lastModified": 1643052045,
-        "narHash": "sha256-zwVvxrdIzralnSbcpghA92tWu2DV2lwv89xZc8MTrbg=",
+        "narHash": "sha256-uGJ0VXIhWKGXxkeNnq4TvV3CIOkUJ3PAoLZ3HMzNVMw=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "205fd4226592cc83fd4c0885a3e4c9c400efabb5",
+        "rev": "215d4d0fd80ca5163643b03a33fde804a29cc1e2",
         "type": "github"
       },
       "original": {
         "owner": "NixOS",
-        "ref": "nixos-23.11",
+        "repo": "nixpkgs",
+        "rev": "215d4d0fd80ca5163643b03a33fde804a29cc1e2",
+        "type": "github"
+      }
+    },
+    "nixpkgs-stable": {
+      "locked": {
+        "lastModified": 1724316499,
+        "narHash": "sha256-Qb9MhKBUTCfWg/wqqaxt89Xfi6qTD3XpTzQ9eXi3JmE=",
+        "owner": "NixOS",
+        "repo": "nixpkgs",
+        "rev": "797f7dc49e0bc7fab4b57c021cdf68f595e47841",
+        "type": "github"
+      },
+      "original": {
+        "owner": "NixOS",
+        "ref": "nixos-24.05",
         "repo": "nixpkgs",
         "type": "github"
       }
@@ -604,27 +805,27 @@
     },
     "nixpkgs_3": {
       "locked": {
-        "lastModified": 1722813957,
+        "lastModified": 1717432640,
-        "narHash": "sha256-IAoYyYnED7P8zrBFMnmp7ydaJfwTnwcnqxUElC1I26Y=",
+        "narHash": "sha256-+f9c4/ZX5MWDOuB1rKoWj+lBNm0z0rs4CK47HBLxy1o=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "cb9a96f23c491c081b38eab96d22fa958043c9fa",
+        "rev": "88269ab3044128b7c2f4c7d68448b2fb50456870",
         "type": "github"
       },
       "original": {
         "owner": "NixOS",
-        "ref": "nixos-unstable",
+        "ref": "release-24.05",
         "repo": "nixpkgs",
         "type": "github"
       }
     },
     "nixpkgs_4": {
       "locked": {
-        "lastModified": 1724271409,
+        "lastModified": 1727802920,
-        "narHash": "sha256-z4nw9HxkaXEn+5OT8ljLVL2oataHvAzUQ1LEi8Fp+SY=",
+        "narHash": "sha256-HP89HZOT0ReIbI7IJZJQoJgxvB2Tn28V6XS3MNKnfLs=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "36a9aeaaa17a2d4348498275f9fe530cd4f9e519",
+        "rev": "27e30d177e57d912d614c88c622dcfdb2e6e6515",
         "type": "github"
       },
       "original": {
@@ -634,15 +835,33 @@
         "type": "github"
       }
     },
+    "nixpkgs_5": {
+      "locked": {
+        "lastModified": 1725534445,
+        "narHash": "sha256-Yd0FK9SkWy+ZPuNqUgmVPXokxDgMJoGuNpMEtkfcf84=",
+        "owner": "NixOS",
+        "repo": "nixpkgs",
+        "rev": "9bb1e7571aadf31ddb4af77fc64b2d59580f9a39",
+        "type": "github"
+      },
+      "original": {
+        "owner": "NixOS",
+        "ref": "nixpkgs-unstable",
+        "repo": "nixpkgs",
+        "type": "github"
+      }
+    },
     "poetry2nix": {
       "inputs": {
-        "flake-utils": "flake-utils_2",
+        "flake-utils": "flake-utils",
         "nix-github-actions": "nix-github-actions",
         "nixpkgs": [
           "cachix",
           "devenv",
           "cachix",
           "devenv",
+          "cachix",
+          "devenv",
           "nixpkgs"
         ]
       },
@@ -660,19 +879,59 @@
         "type": "github"
       }
     },
+    "pre-commit-hooks": {
+      "inputs": {
+        "flake-compat": [
+          "cachix",
+          "devenv",
+          "nix"
+        ],
+        "flake-utils": "flake-utils_2",
+        "gitignore": [
+          "cachix",
+          "devenv",
+          "nix"
+        ],
+        "nixpkgs": [
+          "cachix",
+          "devenv",
+          "nix",
+          "nixpkgs"
+        ],
+        "nixpkgs-stable": [
+          "cachix",
+          "devenv",
+          "nix",
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1712897695,
+        "narHash": "sha256-nMirxrGteNAl9sWiOhoN5tIHyjBbVi5e2tgZUgZlK3Y=",
+        "owner": "cachix",
+        "repo": "pre-commit-hooks.nix",
+        "rev": "40e6053ecb65fcbf12863338a6dcefb3f55f1bf8",
+        "type": "github"
+      },
+      "original": {
+        "owner": "cachix",
+        "repo": "pre-commit-hooks.nix",
+        "type": "github"
+      }
+    },
     "rocksdb": {
       "flake": false,
       "locked": {
-        "lastModified": 1724285323,
+        "lastModified": 1729712930,
-        "narHash": "sha256-k60kreKQ0v+bQ16yBd2SfLYpuNjMw2qoRmZL/S3k6CU=",
+        "narHash": "sha256-jlp4kPkRTpoJaUdobEoHd8rCGAQNBy4ZHZ6y5zL/ibw=",
         "owner": "girlbossceo",
         "repo": "rocksdb",
-        "rev": "5a67ad7ce46328578ee5587fb0c23faa03d14e67",
+        "rev": "871eda6953c3f399aae39808dcfccdd014885beb",
         "type": "github"
       },
       "original": {
         "owner": "girlbossceo",
-        "ref": "v9.5.2",
+        "ref": "v9.7.3",
         "repo": "rocksdb",
         "type": "github"
       }
@@ -688,18 +947,18 @@
         "flake-utils": "flake-utils_3",
         "liburing": "liburing",
         "nix-filter": "nix-filter",
-        "nixpkgs": "nixpkgs_4",
+        "nixpkgs": "nixpkgs_5",
         "rocksdb": "rocksdb"
       }
     },
     "rust-analyzer-src": {
       "flake": false,
       "locked": {
-        "lastModified": 1724153119,
+        "lastModified": 1729715509,
-        "narHash": "sha256-WxpvDJDttkINkXmUA/W5o11lwLPYhATAgu0QUAacZ2g=",
+        "narHash": "sha256-jUDN4e1kObbksb4sc+57NEeujBEDRdLCOu9wiE3RZdM=",
         "owner": "rust-lang",
         "repo": "rust-analyzer",
-        "rev": "3723e5910c14f0ffbd13de474b8a8fcc74db04ce",
+        "rev": "40492e15d49b89cf409e2c5536444131fac49429",
         "type": "github"
       },
       "original": {
@@ -738,21 +997,6 @@
         "repo": "default",
         "type": "github"
       }
-    },
-    "systems_3": {
-      "locked": {
-        "lastModified": 1681028828,
-        "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
-        "owner": "nix-systems",
-        "repo": "default",
-        "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
-        "type": "github"
-      },
-      "original": {
-        "owner": "nix-systems",
-        "repo": "default",
-        "type": "github"
-      }
     }
   },
   "root": "root",

flake.nix

@@ -3,13 +3,13 @@
     attic.url = "github:zhaofengli/attic?ref=main";
     cachix.url = "github:cachix/cachix?ref=master";
     complement = { url = "github:matrix-org/complement?ref=main"; flake = false; };
-    crane = { url = "github:ipetkov/crane?ref=master"; inputs.nixpkgs.follows = "nixpkgs"; };
+    crane = { url = "github:ipetkov/crane?ref=master"; };
     fenix = { url = "github:nix-community/fenix?ref=main"; inputs.nixpkgs.follows = "nixpkgs"; };
     flake-compat = { url = "github:edolstra/flake-compat?ref=master"; flake = false; };
     flake-utils.url = "github:numtide/flake-utils?ref=main";
     nix-filter.url = "github:numtide/nix-filter?ref=main";
-    nixpkgs.url = "github:NixOS/nixpkgs?ref=nixos-unstable";
+    nixpkgs.url = "github:NixOS/nixpkgs?ref=nixpkgs-unstable";
-    rocksdb = { url = "github:girlbossceo/rocksdb?ref=v9.5.2"; flake = false; };
+    rocksdb = { url = "github:girlbossceo/rocksdb?ref=v9.7.3"; flake = false; };
     liburing = { url = "github:axboe/liburing?ref=master"; flake = false; };
   };
 
@@ -27,7 +27,7 @@
         file = ./rust-toolchain.toml;
 
         # See also `rust-toolchain.toml`
-        sha256 = "sha256-3jVIIf5XPnUU1CRaTyAiO0XHVbJl12MSx3eucTXCjtE=";
+        sha256 = "sha256-yMuSb5eQPO/bHv+Bcf/US8LVMbf/G/0MSfiPwBhiPpk=";
       };
 
       mkScope = pkgs: pkgs.lib.makeScope pkgs.newScope (self: {
@@ -38,7 +38,23 @@
         inherit inputs;
         main = self.callPackage ./nix/pkgs/main {};
         oci-image = self.callPackage ./nix/pkgs/oci-image {};
-        rocksdb = pkgs.rocksdb.overrideAttrs (old: {
+        tini = pkgs.tini.overrideAttrs {
+            # newer clang/gcc is unhappy with tini-static: <https://3.dog/~strawberry/pb/c8y4>
+            patches = [ (pkgs.fetchpatch {
+                url = "https://patch-diff.githubusercontent.com/raw/krallin/tini/pull/224.patch";
+                hash = "sha256-4bTfAhRyIT71VALhHY13hUgbjLEUyvgkIJMt3w9ag3k=";
+              })
+            ];
+        };
+        liburing = pkgs.liburing.overrideAttrs {
+          # Tests weren't building
+          outputs = [ "out" "dev" "man" ];
+          buildFlags = [ "library" ];
+          src = inputs.liburing;
+        };
+        rocksdb = (pkgs.rocksdb.override {
+          liburing = self.liburing;
+        }).overrideAttrs (old: {
           src = inputs.rocksdb;
           version = pkgs.lib.removePrefix
             "v"
@@ -76,18 +92,20 @@
           # preInstall hooks has stuff for messing with ldb/sst_dump which we dont need or use
           preInstall = "";
         });
-        # TODO: remove once https://github.com/NixOS/nixpkgs/pull/314945 is available
-        liburing = pkgs.liburing.overrideAttrs (old: {
-          # the configure script doesn't support these, and unconditionally
-          # builds both static and dynamic libraries.
-          configureFlags = pkgs.lib.subtractLists
-            [ "--enable-static" "--disable-shared" ]
-            old.configureFlags;
-        });
       });
 
       scopeHost = mkScope pkgsHost;
       scopeHostStatic = mkScope pkgsHostStatic;
+      scopeCrossLinux = mkScope pkgsHost.pkgsLinux.pkgsStatic;
+      mkCrossScope = crossSystem:
+        let pkgsCrossStatic = (import inputs.nixpkgs {
+          inherit system;
+          crossSystem = {
+           config = crossSystem;
+          };
+        }).pkgsStatic;
+         in
+        mkScope pkgsCrossStatic;
 
       mkDevShell = scope: scope.pkgs.mkShell {
         env = scope.main.env // {
@@ -119,6 +137,9 @@
           engage
           cargo-audit
 
+          # Required by hardened-malloc.rs dep
+          binutils
+
           # Needed for producing Debian packages
           cargo-deb
 
@@ -145,12 +166,21 @@
 
           # needed so we can get rid of gcc and other unused deps that bloat OCI images
           removeReferencesTo
-        ])
+        ]
+        # liburing is Linux-exclusive
+        ++ lib.optional stdenv.hostPlatform.isLinux liburing
+        # needed to build Rust applications on macOS
+        ++ lib.optionals stdenv.hostPlatform.isDarwin [
+            # https://github.com/NixOS/nixpkgs/issues/206242
+            # ld: library not found for -liconv
+            libiconv
+            # https://stackoverflow.com/questions/69869574/properly-adding-darwin-apple-sdk-to-a-nix-shell
+            # https://discourse.nixos.org/t/compile-a-rust-binary-on-macos-dbcrossbar/8612
+            pkgsBuildHost.darwin.apple_sdk.frameworks.Security
+            ])
         ++ scope.main.buildInputs
         ++ scope.main.propagatedBuildInputs
         ++ scope.main.nativeBuildInputs;
-
-        meta.broken = scope.main.meta.broken;
       };
     in
     {
@@ -224,6 +254,8 @@
 
         complement = scopeHost.complement;
         static-complement = scopeHostStatic.complement;
+        # macOS containers don't exist, so the complement images must be forced to linux
+        linux-complement = (mkCrossScope "${pkgsHost.hostPlatform.qemuArch}-linux-musl").complement;
       }
       //
       builtins.listToAttrs
@@ -232,14 +264,7 @@
             (crossSystem:
               let
                 binaryName = "static-${crossSystem}";
-                pkgsCrossStatic =
+                scopeCrossStatic = mkCrossScope crossSystem;
-                  (import inputs.nixpkgs {
-                    inherit system;
-                    crossSystem = {
-                      config = crossSystem;
-                    };
-                  }).pkgsStatic;
-                scopeCrossStatic = mkScope pkgsCrossStatic;
               in
               [
                 # An output for a statically-linked binary
@@ -369,11 +394,20 @@
                     };
                   };
                 }
+
+                # An output for a complement OCI image for the specified platform
+                {
+                  name = "complement-${crossSystem}";
+                  value = scopeCrossStatic.complement;
+                }
               ]
             )
             [
-              "x86_64-unknown-linux-musl"
+              #"x86_64-apple-darwin"
-              "aarch64-unknown-linux-musl"
+              #"aarch64-apple-darwin"
+              "x86_64-linux-gnu"
+              "x86_64-linux-musl"
+              "aarch64-linux-musl"
             ]
           )
         );

nix/pkgs/book/default.nix

@@ -14,8 +14,10 @@ stdenv.mkDerivation {
     include = [
       "book.toml"
       "conduwuit-example.toml"
+      "CODE_OF_CONDUCT.md"
       "CONTRIBUTING.md"
       "README.md"
+      "development.md"
       "debian/conduwuit.service"
       "debian/README.md"
       "arch/conduwuit.service"

nix/pkgs/complement/config.toml

@@ -16,6 +16,7 @@ url_preview_domain_contains_allowlist = ["*"]
 media_compat_file_link = false
 media_startup_check = false
 rocksdb_direct_io = false
+log_colors = false
 
 [global.tls]
 certs = "/certificate.crt"

nix/pkgs/complement/default.nix

@@ -18,6 +18,15 @@ let
     all_features = true;
     disable_release_max_log_level = true;
     disable_features = [
+        # no reason to use jemalloc for complement, just has compatibility/build issues
+        "jemalloc"
+        # console/CLI stuff isn't used or relevant for complement
+        "console"
+        "tokio_console"
+        # sentry telemetry isn't useful for complement, disabled by default anyways
+        "sentry_telemetry"
+        # the containers don't use or need systemd signal support
+        "systemd"
         # this is non-functional on nix for some reason
         "hardened_malloc"
         # dont include experimental features
@@ -57,7 +66,7 @@ let
 in
 
 dockerTools.buildImage {
-  name = "complement-${main.pname}";
+  name = "complement-conduwuit";
   tag = "main";
 
   copyToRoot = buildEnv {
@@ -78,7 +87,7 @@ dockerTools.buildImage {
       "${lib.getExe start}"
     ];
 
-    Entrypoint = if !stdenv.isDarwin
+    Entrypoint = if !stdenv.hostPlatform.isDarwin
       # Use the `tini` init system so that signals (e.g. ctrl+c/SIGINT)
       # are handled as expected
       then [ "${lib.getExe' tini "tini"}" "--" ]

nix/pkgs/main/cross-compilation-env.nix

@@ -1,5 +1,6 @@
 { lib
 , pkgsBuildHost
+, pkgsBuildTarget
 , rust
 , stdenv
 }:
@@ -13,12 +14,6 @@ lib.optionalAttrs stdenv.hostPlatform.isStatic {
     lib.concatStringsSep
       " "
       ([]
-        ++ lib.optionals
-          stdenv.targetPlatform.isx86_64
-          [ "-C" "target-cpu=x86-64-v2" ]
-        ++ lib.optionals
-          stdenv.targetPlatform.isAarch64
-          [ "-C" "target-cpu=cortex-a55" ] # cortex-a55 == ARMv8.2-a
         # This disables PIE for static builds, which isn't great in terms
         # of security. Unfortunately, my hand is forced because nixpkgs'
         # `libstdc++.a` is built without `-fPIE`, which precludes us from
@@ -41,7 +36,7 @@ lib.optionalAttrs stdenv.hostPlatform.isStatic {
             # including it here. Linkers are weird.
             (stdenv.hostPlatform.isAarch64 || stdenv.hostPlatform.isx86_64)
               && stdenv.hostPlatform.isStatic
-              && !stdenv.isDarwin
+              && !stdenv.hostPlatform.isDarwin
               && !stdenv.cc.bintools.isLLVM
           )
           [
@@ -58,11 +53,12 @@ lib.optionalAttrs stdenv.hostPlatform.isStatic {
 # even covers the case of build scripts that need native code compiled and
 # run on the build platform (I think).
 #
-# [0]: https://github.com/NixOS/nixpkgs/blob/5cdb38bb16c6d0a38779db14fcc766bc1b2394d6/pkgs/build-support/rust/lib/default.nix#L57-L80
+# [0]: https://github.com/NixOS/nixpkgs/blob/nixpkgs-unstable/pkgs/build-support/rust/lib/default.nix#L48-L68
 //
 (
   let
     inherit (rust.lib) envVars;
+    shouldUseLLD = platform: platform.isAarch64 && platform.isStatic && !stdenv.hostPlatform.isDarwin;
   in
   lib.optionalAttrs
     (stdenv.targetPlatform.rust.rustcTarget
@@ -70,23 +66,30 @@ lib.optionalAttrs stdenv.hostPlatform.isStatic {
     (
       let
         inherit (stdenv.targetPlatform.rust) cargoEnvVarTarget;
+        linkerForTarget = if shouldUseLLD stdenv.targetPlatform
+          && !stdenv.cc.bintools.isLLVM # whether stdenv's linker is lld already
+          then "${pkgsBuildTarget.llvmPackages.bintools}/bin/${stdenv.cc.targetPrefix}ld.lld"
+          else envVars.ccForTarget;
       in
       {
         "CC_${cargoEnvVarTarget}" = envVars.ccForTarget;
         "CXX_${cargoEnvVarTarget}" = envVars.cxxForTarget;
-        "CARGO_TARGET_${cargoEnvVarTarget}_LINKER" =
+        "CARGO_TARGET_${cargoEnvVarTarget}_LINKER" = linkerForTarget;
-          envVars.linkerForTarget;
       }
     )
   //
   (
     let
       inherit (stdenv.hostPlatform.rust) cargoEnvVarTarget rustcTarget;
+      linkerForHost = if shouldUseLLD stdenv.targetPlatform
+        && !stdenv.cc.bintools.isLLVM
+        then "${pkgsBuildHost.llvmPackages.bintools}/bin/${stdenv.cc.targetPrefix}ld.lld"
+        else envVars.ccForHost;
     in
     {
       "CC_${cargoEnvVarTarget}" = envVars.ccForHost;
       "CXX_${cargoEnvVarTarget}" = envVars.cxxForHost;
-      "CARGO_TARGET_${cargoEnvVarTarget}_LINKER" = envVars.linkerForHost;
+      "CARGO_TARGET_${cargoEnvVarTarget}_LINKER" = linkerForHost;
       CARGO_BUILD_TARGET = rustcTarget;
     }
   )
@@ -98,7 +101,7 @@ lib.optionalAttrs stdenv.hostPlatform.isStatic {
     {
       "CC_${cargoEnvVarTarget}" = envVars.ccForBuild;
       "CXX_${cargoEnvVarTarget}" = envVars.cxxForBuild;
-      "CARGO_TARGET_${cargoEnvVarTarget}_LINKER" = envVars.linkerForBuild;
+      "CARGO_TARGET_${cargoEnvVarTarget}_LINKER" = envVars.ccForBuild;
       HOST_CC = "${pkgsBuildHost.stdenv.cc}/bin/cc";
       HOST_CXX = "${pkgsBuildHost.stdenv.cc}/bin/c++";
     }

nix/pkgs/main/default.nix

@@ -6,6 +6,7 @@
 , libiconv
 , liburing
 , pkgsBuildHost
+, pkgsBuildTarget
 , rocksdb
 , removeReferencesTo
 , rust
@@ -40,7 +41,7 @@ features'' = lib.subtractLists disable_features' features';
 
 featureEnabled = feature : builtins.elem feature features'';
 
-enableLiburing = featureEnabled "io_uring" && !stdenv.isDarwin;
+enableLiburing = featureEnabled "io_uring" && !stdenv.hostPlatform.isDarwin;
 
 # This derivation will set the JEMALLOC_OVERRIDE variable, causing the
 # tikv-jemalloc-sys crate to use the nixpkgs jemalloc instead of building it's
@@ -72,35 +73,13 @@ buildDepsOnlyEnv =
       # jemalloc symbols are prefixed.
       #
       # [1]: https://github.com/tikv/jemallocator/blob/ab0676d77e81268cd09b059260c75b38dbef2d51/jemalloc-sys/src/env.rs#L17
-      enableJemalloc = featureEnabled "jemalloc" && !stdenv.isDarwin;
+      enableJemalloc = featureEnabled "jemalloc" && !stdenv.hostPlatform.isDarwin;
 
       # for some reason enableLiburing in nixpkgs rocksdb is default true
       # which breaks Darwin entirely
       enableLiburing = enableLiburing;
     }).overrideAttrs (old: {
-      # TODO: static rocksdb fails to build on darwin, also see <https://github.com/NixOS/nixpkgs/issues/320448>
-      # build log at <https://girlboss.ceo/~strawberry/pb/JjGH>
-      meta.broken = stdenv.hostPlatform.isStatic && stdenv.isDarwin;
-
       enableLiburing = enableLiburing;
-
-      sse42Support = stdenv.targetPlatform.isx86_64;
-
-      cmakeFlags = if stdenv.targetPlatform.isx86_64
-        then lib.subtractLists [ "-DPORTABLE=1" ] old.cmakeFlags
-        ++ lib.optionals stdenv.targetPlatform.isx86_64 [
-          "-DPORTABLE=x86-64-v2"
-          "-DUSE_SSE=1"
-          "-DHAVE_SSE=1"
-          "-DHAVE_SSE42=1"
-        ]
-        else if stdenv.targetPlatform.isAarch64
-        then lib.subtractLists [ "-DPORTABLE=1" ] old.cmakeFlags
-        ++ lib.optionals stdenv.targetPlatform.isAarch64 [
-          # cortex-a73 == ARMv8-A
-          "-DPORTABLE=armv8-a"
-        ]
-        else old.cmakeFlags;
     });
   in
   {
@@ -117,6 +96,7 @@ buildDepsOnlyEnv =
     inherit
       lib
       pkgsBuildHost
+      pkgsBuildTarget
       rust
       stdenv;
   });
@@ -127,11 +107,7 @@ buildPackageEnv = {
   # Only needed in static stdenv because these are transitive dependencies of rocksdb
   CARGO_BUILD_RUSTFLAGS = buildDepsOnlyEnv.CARGO_BUILD_RUSTFLAGS
     + lib.optionalString (enableLiburing && stdenv.hostPlatform.isStatic)
-      " -L${lib.getLib liburing}/lib -luring"
+      " -L${lib.getLib liburing}/lib -luring";
-    + lib.optionalString stdenv.targetPlatform.isx86_64
-      " -Ctarget-cpu=x86-64-v2"
-    + lib.optionalString stdenv.targetPlatform.isAarch64
-      " -Ctarget-cpu=cortex-a73"; # cortex-a73 == ARMv8-A
 };
 
 
@@ -159,7 +135,16 @@ commonAttrs = {
     dontStrip = profile == "dev" || profile == "test";
     dontPatchELF = profile == "dev" || profile == "test";
 
-    buildInputs = lib.optional (featureEnabled "jemalloc") rust-jemalloc-sys';
+    buildInputs = lib.optional (featureEnabled "jemalloc") rust-jemalloc-sys'
+    # needed to build Rust applications on macOS
+    ++ lib.optionals stdenv.hostPlatform.isDarwin [
+        # https://github.com/NixOS/nixpkgs/issues/206242
+        # ld: library not found for -liconv
+        libiconv
+        # https://stackoverflow.com/questions/69869574/properly-adding-darwin-apple-sdk-to-a-nix-shell
+        # https://discourse.nixos.org/t/compile-a-rust-binary-on-macos-dbcrossbar/8612
+        pkgsBuildHost.darwin.apple_sdk.frameworks.Security
+      ];
 
     nativeBuildInputs = [
       # bindgen needs the build platform's libclang. Apparently due to "splicing
@@ -176,8 +161,10 @@ commonAttrs = {
       # needed so we can get rid of gcc and other unused deps that bloat OCI images
       removeReferencesTo
   ]
-  ++ lib.optionals stdenv.isDarwin [
+  # needed to build Rust applications on macOS
+  ++ lib.optionals stdenv.hostPlatform.isDarwin [
       # https://github.com/NixOS/nixpkgs/issues/206242
+      # ld: library not found for -liconv
       libiconv
 
       # https://stackoverflow.com/questions/69869574/properly-adding-darwin-apple-sdk-to-a-nix-shell
@@ -189,7 +176,7 @@ commonAttrs = {
     #
     # <https://github.com/input-output-hk/haskell.nix/issues/829>
     postInstall = with pkgsBuildHost; ''
-        find "$out" -type f -exec remove-references-to -t ${stdenv.cc} -t ${gcc} -t ${libgcc} -t ${linuxHeaders} -t ${libidn2} -t ${libunistring} '{}' +
+        find "$out" -type f -exec remove-references-to -t ${stdenv.cc} -t ${gcc} -t ${rustc.unwrapped} -t ${rustc} -t ${libidn2} -t ${libunistring} '{}' +
     '';
  };
 in

nix/pkgs/oci-image/default.nix

@@ -16,7 +16,7 @@ dockerTools.buildLayeredImage {
     dockerTools.caCertificates
   ];
   config = {
-    Entrypoint = if !stdenv.isDarwin
+    Entrypoint = if !stdenv.hostPlatform.isDarwin
       # Use the `tini` init system so that signals (e.g. ctrl+c/SIGINT)
       # are handled as expected
       then [ "${lib.getExe' tini "tini"}" "--" ]

rust-toolchain.toml

@@ -2,8 +2,6 @@
 #
 # Other files that need upkeep when this changes:
 #
-# * `.gitlab-ci.yml`
-# * `.github/workflows/ci.yml`
 # * `Cargo.toml`
 # * `flake.nix`
 #
@@ -11,13 +9,20 @@
 # If you're having trouble making the relevant changes, bug a maintainer.
 
 [toolchain]
-channel = "1.80.1"
+channel = "1.82.0"
+profile = "minimal"
 components = [
     # For rust-analyzer
     "rust-src",
+    "rust-analyzer",
+    # For CI and editors
+    "rustfmt",
+    "clippy",
 ]
 targets = [
+    #"x86_64-apple-darwin",
     "x86_64-unknown-linux-gnu",
     "x86_64-unknown-linux-musl",
     "aarch64-unknown-linux-musl",
+    #"aarch64-apple-darwin",
 ]

src/admin/media/commands.rs

@@ -1,6 +1,6 @@
 use std::time::Duration;
 
-use conduit::{debug, info, trace, utils::time::parse_timepoint_ago, warn, Result};
+use conduit::{debug, debug_info, debug_warn, error, info, trace, utils::time::parse_timepoint_ago, Result};
 use conduit_service::media::Dim;
 use ruma::{
 	events::room::message::RoomMessageEventContent, EventId, Mxc, MxcUri, OwnedMxcUri, OwnedServerName, ServerName,
@@ -19,7 +19,7 @@ pub(super) async fn delete(
 	}
 
 	if let Some(mxc) = mxc {
-		debug!("Got MXC URL: {mxc}");
+		trace!("Got MXC URL: {mxc}");
 		self.services
 			.media
 			.delete(&mxc.as_str().try_into()?)
@@ -28,11 +28,12 @@ pub(super) async fn delete(
 		return Ok(RoomMessageEventContent::text_plain(
 			"Deleted the MXC from our database and on our filesystem.",
 		));
-	} else if let Some(event_id) = event_id {
+	}
-		debug!("Got event ID to delete media from: {event_id}");
 
-		let mut mxc_urls = vec![];
+	if let Some(event_id) = event_id {
-		let mut mxc_deletion_count: usize = 0;
+		trace!("Got event ID to delete media from: {event_id}");
+
+		let mut mxc_urls = Vec::with_capacity(4);
 
 		// parsing the PDU for any MXC URLs begins here
 		if let Some(event_json) = self.services.rooms.timeline.get_pdu_json(&event_id)? {
@@ -124,18 +125,28 @@ pub(super) async fn delete(
 		}
 
 		if mxc_urls.is_empty() {
-			// we shouldn't get here (should have errored earlier) but just in case for
-			// whatever reason we do...
 			info!("Parsed event ID {event_id} but did not contain any MXC URLs.");
 			return Ok(RoomMessageEventContent::text_plain("Parsed event ID but found no MXC URLs."));
 		}
 
+		let mut mxc_deletion_count: usize = 0;
+
 		for mxc_url in mxc_urls {
-			self.services
+			match self
+				.services
 				.media
 				.delete(&mxc_url.as_str().try_into()?)
-				.await?;
+				.await
-			mxc_deletion_count = mxc_deletion_count.saturating_add(1);
+			{
+				Ok(()) => {
+					debug_info!("Successfully deleted {mxc_url} from filesystem and database");
+					mxc_deletion_count = mxc_deletion_count.saturating_add(1);
+				},
+				Err(e) => {
+					debug_warn!("Failed to delete {mxc_url}, ignoring error and skipping: {e}");
+					continue;
+				},
+			}
 		}
 
 		return Ok(RoomMessageEventContent::text_plain(format!(
@@ -158,34 +169,62 @@ pub(super) async fn delete_list(&self) -> Result<RoomMessageEventContent> {
 		));
 	}
 
+	let mut failed_parsed_mxcs: usize = 0;
+
 	let mxc_list = self
 		.body
 		.to_vec()
 		.drain(1..self.body.len().checked_sub(1).unwrap())
-		.collect::<Vec<_>>();
+		.filter_map(|mxc_s| {
+			mxc_s
+				.try_into()
+				.inspect_err(|e| {
+					debug_warn!("Failed to parse user-provided MXC URI: {e}");
+
+					failed_parsed_mxcs = failed_parsed_mxcs.saturating_add(1);
+				})
+				.ok()
+		})
+		.collect::<Vec<Mxc<'_>>>();
 
 	let mut mxc_deletion_count: usize = 0;
 
-	for mxc in mxc_list {
+	for mxc in &mxc_list {
-		debug!("Deleting MXC {mxc} in bulk");
+		trace!(%failed_parsed_mxcs, %mxc_deletion_count, "Deleting MXC {mxc} in bulk");
-		self.services.media.delete(&mxc.try_into()?).await?;
+		match self.services.media.delete(mxc).await {
-		mxc_deletion_count = mxc_deletion_count
+			Ok(()) => {
-			.checked_add(1)
+				debug_info!("Successfully deleted {mxc} from filesystem and database");
-			.expect("mxc_deletion_count should not get this high");
+				mxc_deletion_count = mxc_deletion_count.saturating_add(1);
+			},
+			Err(e) => {
+				debug_warn!("Failed to delete {mxc}, ignoring error and skipping: {e}");
+				continue;
+			},
+		}
 	}
 
 	Ok(RoomMessageEventContent::text_plain(format!(
-		"Finished bulk MXC deletion, deleted {mxc_deletion_count} total MXCs from our database and the filesystem.",
+		"Finished bulk MXC deletion, deleted {mxc_deletion_count} total MXCs from our database and the filesystem. \
+		 {failed_parsed_mxcs} MXCs failed to be parsed from the database.",
 	)))
 }
 
 #[admin_command]
-pub(super) async fn delete_past_remote_media(&self, duration: String, force: bool) -> Result<RoomMessageEventContent> {
+pub(super) async fn delete_past_remote_media(
+	&self, duration: String, before: bool, after: bool, yes_i_want_to_delete_local_media: bool,
+) -> Result<RoomMessageEventContent> {
+	if before && after {
+		return Ok(RoomMessageEventContent::text_plain(
+			"Please only pick one argument, --before or --after.",
+		));
+	}
+	assert!(!(before && after), "--before and --after should not be specified together");
+
 	let duration = parse_timepoint_ago(&duration)?;
 	let deleted_count = self
 		.services
 		.media
-		.delete_all_remote_media_at_after_time(duration, force)
+		.delete_all_remote_media_at_after_time(duration, before, after, yes_i_want_to_delete_local_media)
 		.await?;
 
 	Ok(RoomMessageEventContent::text_plain(format!(
@@ -194,14 +233,10 @@ pub(super) async fn delete_past_remote_media(&self, duration: String, force: boo
 }
 
 #[admin_command]
-pub(super) async fn delete_all_from_user(&self, username: String, force: bool) -> Result<RoomMessageEventContent> {
+pub(super) async fn delete_all_from_user(&self, username: String) -> Result<RoomMessageEventContent> {
 	let user_id = parse_local_user_id(self.services, &username)?;
 
-	let deleted_count = self
+	let deleted_count = self.services.media.delete_from_user(&user_id).await?;
-		.services
-		.media
-		.delete_from_user(&user_id, force)
-		.await?;
 
 	Ok(RoomMessageEventContent::text_plain(format!(
 		"Deleted {deleted_count} total files.",
@@ -210,34 +245,36 @@ pub(super) async fn delete_all_from_user(&self, username: String, force: bool) -
 
 #[admin_command]
 pub(super) async fn delete_all_from_server(
-	&self, server_name: Box<ServerName>, force: bool,
+	&self, server_name: Box<ServerName>, yes_i_want_to_delete_local_media: bool,
 ) -> Result<RoomMessageEventContent> {
-	if server_name == self.services.globals.server_name() {
+	if server_name == self.services.globals.server_name() && !yes_i_want_to_delete_local_media {
-		return Ok(RoomMessageEventContent::text_plain("This command only works for remote media."));
+		return Ok(RoomMessageEventContent::text_plain(
+			"This command only works for remote media by default.",
+		));
 	}
 
-	let Ok(all_mxcs) = self.services.media.get_all_mxcs().await else {
+	let Ok(all_mxcs) = self
+		.services
+		.media
+		.get_all_mxcs()
+		.await
+		.inspect_err(|e| error!("Failed to get MXC URIs from our database: {e}"))
+	else {
 		return Ok(RoomMessageEventContent::text_plain("Failed to get MXC URIs from our database"));
 	};
 
 	let mut deleted_count: usize = 0;
 
 	for mxc in all_mxcs {
-		let mxc_server_name = match mxc.server_name() {
+		let Ok(mxc_server_name) = mxc.server_name().inspect_err(|e| {
-			Ok(server_name) => server_name,
+			debug_warn!("Failed to parse MXC {mxc} server name from database, ignoring error and skipping: {e}");
-			Err(e) => {
+		}) else {
-				if force {
+			continue;
-					warn!("Failed to parse MXC {mxc} server name from database, ignoring error and skipping: {e}");
-					continue;
-				}
-
-				return Ok(RoomMessageEventContent::text_plain(format!(
-					"Failed to parse MXC {mxc} server name from database: {e}",
-				)));
-			},
 		};
 
-		if mxc_server_name != server_name || self.services.globals.server_is_ours(mxc_server_name) {
+		if mxc_server_name != server_name
+			|| (self.services.globals.server_is_ours(mxc_server_name) && !yes_i_want_to_delete_local_media)
+		{
 			trace!("skipping MXC URI {mxc}");
 			continue;
 		}
@@ -249,12 +286,8 @@ pub(super) async fn delete_all_from_server(
 				deleted_count = deleted_count.saturating_add(1);
 			},
 			Err(e) => {
-				if force {
+				debug_warn!("Failed to delete {mxc}, ignoring error and skipping: {e}");
-					warn!("Failed to delete {mxc}, ignoring error and skipping: {e}");
+				continue;
-					continue;
-				}
-
-				return Ok(RoomMessageEventContent::text_plain(format!("Failed to delete MXC {mxc}: {e}")));
 			},
 		}
 	}

src/admin/media/mod.rs

@@ -10,7 +10,7 @@ use crate::admin_command_dispatch;
 #[derive(Debug, Subcommand)]
 pub(super) enum MediaCommand {
 	/// - Deletes a single media file from our database and on the filesystem
-	///   via a single MXC URL
+	///   via a single MXC URL or event ID (not redacted)
 	Delete {
 		/// The MXC URL to delete
 		#[arg(long)]
@@ -23,37 +23,44 @@ pub(super) enum MediaCommand {
 	},
 
 	/// - Deletes a codeblock list of MXC URLs from our database and on the
-	///   filesystem
+	///   filesystem. This will always ignore errors.
 	DeleteList,
 
-	/// - Deletes all remote media in the last X amount of time using filesystem
+	/// - Deletes all remote media in the last/after "X" time using filesystem
-	///   metadata first created at date.
+	///   metadata first created at date, or fallback to last modified date.
+	///   This will always ignore errors by default.
+	///
+	/// Synapse
 	DeletePastRemoteMedia {
-		/// - The duration (at or after), e.g. "5m" to delete all media in the
+		/// - The duration (at or after/before), e.g. "5m" to delete all media
-		///   past 5 minutes
+		///   in the past or up to 5 minutes
 		duration: String,
 
-		/// Continues deleting remote media if an undeletable object is found
+		#[arg(long, short)]
-		#[arg(short, long)]
+		before: bool,
-		force: bool,
+
+		#[arg(long, short)]
+		after: bool,
+
+		/// Long argument to delete local media
+		#[arg(long)]
+		yes_i_want_to_delete_local_media: bool,
 	},
 
-	/// - Deletes all the local media from a local user on our server
+	/// - Deletes all the local media from a local user on our server. This will
+	///   always ignore errors by default.
 	DeleteAllFromUser {
 		username: String,
-
-		/// Continues deleting media if an undeletable object is found
-		#[arg(short, long)]
-		force: bool,
 	},
 
-	/// - Deletes all remote media from the specified remote server
+	/// - Deletes all remote media from the specified remote server. This will
+	///   always ignore errors by default.
 	DeleteAllFromServer {
 		server_name: Box<ServerName>,
 
-		/// Continues deleting media if an undeletable object is found
+		/// Long argument to delete local media
-		#[arg(short, long)]
+		#[arg(long)]
-		force: bool,
+		yes_i_want_to_delete_local_media: bool,
 	},
 
 	GetFileInfo {
@@ -82,10 +89,10 @@ pub(super) enum MediaCommand {
 		#[arg(short, long, default_value("10000"))]
 		timeout: u32,
 
-		#[arg(short, long)]
+		#[arg(short, long, default_value("800"))]
 		width: u32,
 
-		#[arg(short, long)]
+		#[arg(short, long, default_value("800"))]
 		height: u32,
 	},
 }

src/admin/query/mod.rs

@@ -2,6 +2,7 @@ mod account_data;
 mod appservice;
 mod globals;
 mod presence;
+mod pusher;
 mod resolver;
 mod room_alias;
 mod room_state_cache;
@@ -13,7 +14,7 @@ use conduit::Result;
 
 use self::{
 	account_data::AccountDataCommand, appservice::AppserviceCommand, globals::GlobalsCommand,
-	presence::PresenceCommand, resolver::ResolverCommand, room_alias::RoomAliasCommand,
+	presence::PresenceCommand, pusher::PusherCommand, resolver::ResolverCommand, room_alias::RoomAliasCommand,
 	room_state_cache::RoomStateCacheCommand, sending::SendingCommand, users::UsersCommand,
 };
 use crate::admin_command_dispatch;
@@ -57,4 +58,8 @@ pub(super) enum QueryCommand {
 	/// - resolver service
 	#[command(subcommand)]
 	Resolver(ResolverCommand),
+
+	/// - pusher service
+	#[command(subcommand)]
+	Pusher(PusherCommand),
 }

src/admin/query/pusher.rs

@@ -0,0 +1,32 @@
+use clap::Subcommand;
+use conduit::Result;
+use ruma::{events::room::message::RoomMessageEventContent, UserId};
+
+use crate::Command;
+
+#[derive(Debug, Subcommand)]
+pub(crate) enum PusherCommand {
+	/// - Returns all the pushers for the user.
+	GetPushers {
+		/// Full user ID
+		user_id: Box<UserId>,
+	},
+}
+
+pub(super) async fn process(subcommand: PusherCommand, context: &Command<'_>) -> Result<RoomMessageEventContent> {
+	let services = context.services;
+
+	match subcommand {
+		PusherCommand::GetPushers {
+			user_id,
+		} => {
+			let timer = tokio::time::Instant::now();
+			let results = services.pusher.get_pushers(&user_id)?;
+			let query_time = timer.elapsed();
+
+			Ok(RoomMessageEventContent::notice_markdown(format!(
+				"Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```"
+			)))
+		},
+	}
+}

src/admin/room/info.rs

@@ -10,6 +10,10 @@ pub(crate) enum RoomInfoCommand {
 	/// - List joined members in a room
 	ListJoinedMembers {
 		room_id: Box<RoomId>,
+
+		/// Lists only our local users in the specified room
+		#[arg(long)]
+		local_only: bool,
 	},
 
 	/// - Displays room topic
@@ -22,7 +26,7 @@ pub(crate) enum RoomInfoCommand {
 }
 
 #[admin_command]
-async fn list_joined_members(&self, room_id: Box<RoomId>) -> Result<RoomMessageEventContent> {
+async fn list_joined_members(&self, room_id: Box<RoomId>, local_only: bool) -> Result<RoomMessageEventContent> {
 	let room_name = self
 		.services
 		.rooms
@@ -37,7 +41,15 @@ async fn list_joined_members(&self, room_id: Box<RoomId>) -> Result<RoomMessageE
 		.rooms
 		.state_cache
 		.room_members(&room_id)
-		.filter_map(Result::ok);
+		.filter_map(|member| {
+			if local_only {
+				member
+					.ok()
+					.filter(|user| self.services.globals.user_is_local(user))
+			} else {
+				member.ok()
+			}
+		});
 
 	let member_info = members
 		.into_iter()

src/admin/user/commands.rs

@@ -1,12 +1,16 @@
 use std::{collections::BTreeMap, fmt::Write as _};
 
-use api::client::{join_room_by_id_helper, leave_all_rooms, leave_room, update_avatar_url, update_displayname};
+use api::client::{full_user_deactivate, join_room_by_id_helper, leave_room};
 use conduit::{error, info, utils, warn, PduBuilder, Result};
 use ruma::{
 	events::{
-		room::{message::RoomMessageEventContent, redaction::RoomRedactionEventContent},
+		room::{
+			message::RoomMessageEventContent,
+			power_levels::{RoomPowerLevels, RoomPowerLevelsEventContent},
+			redaction::RoomRedactionEventContent,
+		},
 		tag::{TagEvent, TagEventContent, TagInfo},
-		RoomAccountDataEventType, TimelineEventType,
+		RoomAccountDataEventType, StateEventType, TimelineEventType,
 	},
 	EventId, OwnedRoomId, OwnedRoomOrAliasId, OwnedUserId, RoomId,
 };
@@ -111,6 +115,7 @@ pub(super) async fn create_user(&self, username: String, password: Option<String
 					Some("Automatically joining this room upon registration".to_owned()),
 					&[room_id_server_name.to_owned(), self.services.globals.server_name().to_owned()],
 					None,
+					&None,
 				)
 				.await
 				{
@@ -128,6 +133,22 @@ pub(super) async fn create_user(&self, username: String, password: Option<String
 
 	// we dont add a device since we're not the user, just the creator
 
+	// if this account creation is from the CLI / --execute, invite the first user
+	// to admin room
+	if let Some(admin_room) = self.services.admin.get_admin_room()? {
+		if self
+			.services
+			.rooms
+			.state_cache
+			.room_joined_count(&admin_room)?
+			== Some(1)
+		{
+			self.services.admin.make_user_admin(&user_id).await?;
+
+			warn!("Granting {user_id} admin privileges as the first user");
+		}
+	}
+
 	// Inhibit login does not work for guests
 	Ok(RoomMessageEventContent::text_plain(format!(
 		"Created user with user_id: {user_id} and password: `{password}`"
@@ -163,9 +184,8 @@ pub(super) async fn deactivate(&self, no_leave_rooms: bool, user_id: String) ->
 			.rooms_joined(&user_id)
 			.filter_map(Result::ok)
 			.collect();
-		update_displayname(self.services, user_id.clone(), None, all_joined_rooms.clone()).await?;
+
-		update_avatar_url(self.services, user_id.clone(), None, None, all_joined_rooms).await?;
+		full_user_deactivate(self.services, &user_id, all_joined_rooms).await?;
-		leave_all_rooms(self.services, &user_id).await;
 	}
 
 	Ok(RoomMessageEventContent::text_plain(format!(
@@ -271,9 +291,7 @@ pub(super) async fn deactivate_all(&self, no_leave_rooms: bool, force: bool) ->
 						.rooms_joined(&user_id)
 						.filter_map(Result::ok)
 						.collect();
-					update_displayname(self.services, user_id.clone(), None, all_joined_rooms.clone()).await?;
+					full_user_deactivate(self.services, &user_id, all_joined_rooms).await?;
-					update_avatar_url(self.services, user_id.clone(), None, None, all_joined_rooms).await?;
-					leave_all_rooms(self.services, &user_id).await;
 				}
 			},
 			Err(e) => {
@@ -343,7 +361,7 @@ pub(super) async fn force_join_room(
 		self.services.globals.user_is_local(&user_id),
 		"Parsed user_id must be a local user"
 	);
-	join_room_by_id_helper(self.services, &user_id, &room_id, None, &[], None).await?;
+	join_room_by_id_helper(self.services, &user_id, &room_id, None, &[], None, &None).await?;
 
 	Ok(RoomMessageEventContent::notice_markdown(format!(
 		"{user_id} has been joined to {room_id}.",
@@ -369,23 +387,83 @@ pub(super) async fn force_leave_room(
 }
 
 #[admin_command]
-pub(super) async fn make_user_admin(&self, user_id: String) -> Result<RoomMessageEventContent> {
+pub(super) async fn force_demote(
+	&self, user_id: String, room_id: OwnedRoomOrAliasId,
+) -> Result<RoomMessageEventContent> {
 	let user_id = parse_local_user_id(self.services, &user_id)?;
-	let displayname = self
+	let room_id = self.services.rooms.alias.resolve(&room_id).await?;
-		.services
-		.users
-		.displayname(&user_id)?
-		.unwrap_or_else(|| user_id.to_string());
 
 	assert!(
 		self.services.globals.user_is_local(&user_id),
 		"Parsed user_id must be a local user"
 	);
-	self.services
+
-		.admin
+	let state_lock = self.services.rooms.state.mutex.lock(&room_id).await;
-		.make_user_admin(&user_id, displayname)
+
+	let room_power_levels = self
+		.services
+		.rooms
+		.state_accessor
+		.room_state_get(&room_id, &StateEventType::RoomPowerLevels, "")?
+		.as_ref()
+		.and_then(|event| serde_json::from_str(event.content.get()).ok()?)
+		.and_then(|content: RoomPowerLevelsEventContent| content.into());
+
+	let user_can_demote_self = room_power_levels
+		.as_ref()
+		.is_some_and(|power_levels_content| {
+			RoomPowerLevels::from(power_levels_content.clone()).user_can_change_user_power_level(&user_id, &user_id)
+		}) || self
+		.services
+		.rooms
+		.state_accessor
+		.room_state_get(&room_id, &StateEventType::RoomCreate, "")?
+		.as_ref()
+		.is_some_and(|event| event.sender == user_id);
+
+	if !user_can_demote_self {
+		return Ok(RoomMessageEventContent::notice_markdown(
+			"User is not allowed to modify their own power levels in the room.",
+		));
+	}
+
+	let mut power_levels_content = room_power_levels.unwrap_or_default();
+	power_levels_content.users.remove(&user_id);
+
+	let event_id = self
+		.services
+		.rooms
+		.timeline
+		.build_and_append_pdu(
+			PduBuilder {
+				event_type: TimelineEventType::RoomPowerLevels,
+				content: to_raw_value(&power_levels_content).expect("event is valid, we just created it"),
+				unsigned: None,
+				state_key: Some(String::new()),
+				redacts: None,
+				timestamp: None,
+			},
+			&user_id,
+			&room_id,
+			&state_lock,
+		)
 		.await?;
 
+	Ok(RoomMessageEventContent::notice_markdown(format!(
+		"User {user_id} demoted themselves to the room default power level in {room_id} - {event_id}"
+	)))
+}
+
+#[admin_command]
+pub(super) async fn make_user_admin(&self, user_id: String) -> Result<RoomMessageEventContent> {
+	let user_id = parse_local_user_id(self.services, &user_id)?;
+
+	assert!(
+		self.services.globals.user_is_local(&user_id),
+		"Parsed user_id must be a local user"
+	);
+	self.services.admin.make_user_admin(&user_id).await?;
+
 	Ok(RoomMessageEventContent::notice_markdown(format!(
 		"{user_id} has been granted admin privileges.",
 	)))

src/admin/user/mod.rs

@@ -79,6 +79,13 @@ pub(super) enum UserCommand {
 		room_id: OwnedRoomOrAliasId,
 	},
 
+	/// - Forces the specified user to drop their power levels to the room
+	///   default, if their permissions allow and the auth check permits
+	ForceDemote {
+		user_id: String,
+		room_id: OwnedRoomOrAliasId,
+	},
+
 	/// - Grant server-admin privileges to a user.
 	MakeUserAdmin {
 		user_id: String,

src/api/client/account.rs

@@ -2,7 +2,7 @@ use std::fmt::Write;
 
 use axum::extract::State;
 use axum_client_ip::InsecureClientIp;
-use conduit::{debug_info, error, info, utils, warn, Error, Result};
+use conduit::{debug_info, error, info, utils, warn, Error, PduBuilder, Result};
 use register::RegistrationKind;
 use ruma::{
 	api::client::{
@@ -15,9 +15,17 @@ use ruma::{
 		error::ErrorKind,
 		uiaa::{AuthFlow, AuthType, UiaaInfo},
 	},
-	events::{room::message::RoomMessageEventContent, GlobalAccountDataEventType},
+	events::{
+		room::{
+			message::RoomMessageEventContent,
+			power_levels::{RoomPowerLevels, RoomPowerLevelsEventContent},
+		},
+		GlobalAccountDataEventType, StateEventType, TimelineEventType,
+	},
 	push, OwnedRoomId, UserId,
 };
+use serde_json::value::to_raw_value;
+use service::Services;
 
 use super::{join_room_by_id_helper, DEVICE_ID_LENGTH, SESSION_ID_LENGTH, TOKEN_LENGTH};
 use crate::Ruma;
@@ -296,50 +304,61 @@ pub(crate) async fn register_route(
 
 	debug_info!(%user_id, %device_id, "User account was created");
 
+	let device_display_name = body.initial_device_display_name.clone().unwrap_or_default();
+
 	// log in conduit admin channel if a non-guest user registered
 	if body.appservice_info.is_none() && !is_guest {
-		info!("New user \"{user_id}\" registered on this server.");
+		if !device_display_name.is_empty() {
-		services
+			info!("New user \"{user_id}\" registered on this server with device display name: {device_display_name}");
-			.admin
+
-			.send_message(RoomMessageEventContent::notice_plain(format!(
+			if services.globals.config.admin_room_notices {
-				"New user \"{user_id}\" registered on this server from IP {client}."
+				services
-			)))
+					.admin
-			.await;
+					.send_message(RoomMessageEventContent::notice_plain(format!(
+						"New user \"{user_id}\" registered on this server from IP {client} and device display name \
+						 \"{device_display_name}\""
+					)))
+					.await;
+			}
+		} else {
+			info!("New user \"{user_id}\" registered on this server.");
+
+			if services.globals.config.admin_room_notices {
+				services
+					.admin
+					.send_message(RoomMessageEventContent::notice_plain(format!(
+						"New user \"{user_id}\" registered on this server from IP {client}"
+					)))
+					.await;
+			}
+		}
 	}
 
 	// log in conduit admin channel if a guest registered
 	if body.appservice_info.is_none() && is_guest && services.globals.log_guest_registrations() {
 		info!("New guest user \"{user_id}\" registered on this server.");
 
-		if let Some(device_display_name) = &body.initial_device_display_name {
+		if !device_display_name.is_empty() {
-			if body
+			if services.globals.config.admin_room_notices {
-				.initial_device_display_name
-				.as_ref()
-				.is_some_and(|device_display_name| !device_display_name.is_empty())
-			{
 				services
 					.admin
 					.send_message(RoomMessageEventContent::notice_plain(format!(
-						"Guest user \"{user_id}\" with device display name `{device_display_name}` registered on this \
+						"Guest user \"{user_id}\" with device display name \"{device_display_name}\" registered on \
-						 server from IP {client}."
+						 this server from IP {client}"
-					)))
-					.await;
-			} else {
-				services
-					.admin
-					.send_message(RoomMessageEventContent::notice_plain(format!(
-						"Guest user \"{user_id}\" with no device display name registered on this server from IP \
-						 {client}.",
 					)))
 					.await;
 			}
 		} else {
-			services
+			#[allow(clippy::collapsible_else_if)]
-				.admin
+			if services.globals.config.admin_room_notices {
-				.send_message(RoomMessageEventContent::notice_plain(format!(
+				services
-					"Guest user \"{user_id}\" with no device display name registered on this server from IP {client}.",
+					.admin
-				)))
+					.send_message(RoomMessageEventContent::notice_plain(format!(
-				.await;
+						"Guest user \"{user_id}\" with no device display name registered on this server from IP \
+						 {client}",
+					)))
+					.await;
+			}
 		}
 	}
 
@@ -348,10 +367,7 @@ pub(crate) async fn register_route(
 	if !is_guest {
 		if let Some(admin_room) = services.admin.get_admin_room()? {
 			if services.rooms.state_cache.room_joined_count(&admin_room)? == Some(1) {
-				services
+				services.admin.make_user_admin(&user_id).await?;
-					.admin
-					.make_user_admin(&user_id, displayname)
-					.await?;
 
 				warn!("Granting {user_id} admin privileges as the first user");
 			}
@@ -380,6 +396,7 @@ pub(crate) async fn register_route(
 					Some("Automatically joining this room upon registration".to_owned()),
 					&[room_id_server_name.to_owned(), services.globals.server_name().to_owned()],
 					None,
+					&body.appservice_info,
 				)
 				.await
 				{
@@ -476,12 +493,15 @@ pub(crate) async fn change_password_route(
 	}
 
 	info!("User {sender_user} changed their password.");
-	services
+
-		.admin
+	if services.globals.config.admin_room_notices {
-		.send_message(RoomMessageEventContent::notice_plain(format!(
+		services
-			"User {sender_user} changed their password."
+			.admin
-		)))
+			.send_message(RoomMessageEventContent::notice_plain(format!(
-		.await;
+				"User {sender_user} changed their password."
+			)))
+			.await;
+	}
 
 	Ok(change_password::v3::Response {})
 }
@@ -556,9 +576,6 @@ pub(crate) async fn deactivate_route(
 		return Err(Error::BadRequest(ErrorKind::NotJson, "Not json."));
 	}
 
-	// Remove devices and mark account as deactivated
-	services.users.deactivate_account(sender_user)?;
-
 	// Remove profile pictures and display name
 	let all_joined_rooms: Vec<OwnedRoomId> = services
 		.rooms
@@ -566,19 +583,19 @@ pub(crate) async fn deactivate_route(
 		.rooms_joined(sender_user)
 		.filter_map(Result::ok)
 		.collect();
-	super::update_displayname(&services, sender_user.clone(), None, all_joined_rooms.clone()).await?;
-	super::update_avatar_url(&services, sender_user.clone(), None, None, all_joined_rooms).await?;
 
-	// Make the user leave all rooms before deactivation
+	full_user_deactivate(&services, sender_user, all_joined_rooms).await?;
-	super::leave_all_rooms(&services, sender_user).await;
 
 	info!("User {sender_user} deactivated their account.");
-	services
+
-		.admin
+	if services.globals.config.admin_room_notices {
-		.send_message(RoomMessageEventContent::notice_plain(format!(
+		services
-			"User {sender_user} deactivated their account."
+			.admin
-		)))
+			.send_message(RoomMessageEventContent::notice_plain(format!(
-		.await;
+				"User {sender_user} deactivated their account."
+			)))
+			.await;
+	}
 
 	Ok(deactivate::v3::Response {
 		id_server_unbind_result: ThirdPartyIdRemovalStatus::NoSupport,
@@ -648,3 +665,86 @@ pub(crate) async fn check_registration_token_validity(
 		valid: reg_token == body.token,
 	})
 }
+
+/// Runs through all the deactivation steps:
+///
+/// - Mark as deactivated
+/// - Removing display name
+/// - Removing avatar URL and blurhash
+/// - Removing all profile data
+/// - Leaving all rooms (and forgets all of them)
+pub async fn full_user_deactivate(
+	services: &Services, user_id: &UserId, all_joined_rooms: Vec<OwnedRoomId>,
+) -> Result<()> {
+	services.users.deactivate_account(user_id)?;
+
+	super::update_displayname(services, user_id, None, all_joined_rooms.clone()).await?;
+	super::update_avatar_url(services, user_id, None, None, all_joined_rooms.clone()).await?;
+
+	let all_profile_keys = services
+		.users
+		.all_profile_keys(user_id)
+		.filter_map(Result::ok);
+
+	for (profile_key, _profile_value) in all_profile_keys {
+		if let Err(e) = services.users.set_profile_key(user_id, &profile_key, None) {
+			warn!("Failed removing {user_id} profile key {profile_key}: {e}");
+		}
+	}
+
+	for room_id in all_joined_rooms {
+		let state_lock = services.rooms.state.mutex.lock(&room_id).await;
+
+		let room_power_levels = services
+			.rooms
+			.state_accessor
+			.room_state_get(&room_id, &StateEventType::RoomPowerLevels, "")?
+			.as_ref()
+			.and_then(|event| serde_json::from_str(event.content.get()).ok()?)
+			.and_then(|content: RoomPowerLevelsEventContent| content.into());
+
+		let user_can_demote_self = room_power_levels
+			.as_ref()
+			.is_some_and(|power_levels_content| {
+				RoomPowerLevels::from(power_levels_content.clone()).user_can_change_user_power_level(user_id, user_id)
+			}) || services
+			.rooms
+			.state_accessor
+			.room_state_get(&room_id, &StateEventType::RoomCreate, "")?
+			.as_ref()
+			.is_some_and(|event| event.sender == user_id);
+
+		if user_can_demote_self {
+			let mut power_levels_content = room_power_levels.unwrap_or_default();
+			power_levels_content.users.remove(user_id);
+
+			// ignore errors so deactivation doesn't fail
+			if let Err(e) = services
+				.rooms
+				.timeline
+				.build_and_append_pdu(
+					PduBuilder {
+						event_type: TimelineEventType::RoomPowerLevels,
+						content: to_raw_value(&power_levels_content).expect("event is valid, we just created it"),
+						unsigned: None,
+						state_key: Some(String::new()),
+						redacts: None,
+						timestamp: None,
+					},
+					user_id,
+					&room_id,
+					&state_lock,
+				)
+				.await
+			{
+				warn!(%room_id, %user_id, "Failed to demote user's own power level: {e}");
+			} else {
+				info!("Demoted {user_id} in {room_id} as part of account deactivation");
+			}
+		}
+	}
+
+	super::leave_all_rooms(services, user_id).await;
+
+	Ok(())
+}

src/api/client/capabilities.rs

@@ -7,6 +7,7 @@ use ruma::{
 	},
 	RoomVersionId,
 };
+use serde_json::json;
 
 use crate::{Result, Ruma};
 
@@ -42,6 +43,11 @@ pub(crate) async fn get_capabilities_route(
 		enabled: false,
 	};
 
+	// MSC4133 capability
+	capabilities
+		.set("uk.tcpip.msc4133.profile_fields", json!({"enabled": true}))
+		.expect("this is valid JSON we created");
+
 	Ok(get_capabilities::v3::Response {
 		capabilities,
 	})

src/api/client/directory.rs

@@ -1,6 +1,6 @@
 use axum::extract::State;
 use axum_client_ip::InsecureClientIp;
-use conduit::{err, info, warn, Error, Result};
+use conduit::{err, info, warn, Err, Error, Result};
 use ruma::{
 	api::{
 		client::{
@@ -124,6 +124,10 @@ pub(crate) async fn set_room_visibility_route(
 		return Err(Error::BadRequest(ErrorKind::NotFound, "Room not found"));
 	}
 
+	if services.users.is_deactivated(sender_user).unwrap_or(false) && body.appservice_info.is_none() {
+		return Err!(Request(Forbidden("Guests cannot publish to room directories")));
+	}
+
 	if !user_can_publish_room(&services, sender_user, &body.room_id)? {
 		return Err(Error::BadRequest(
 			ErrorKind::forbidden(),
@@ -133,20 +137,26 @@ pub(crate) async fn set_room_visibility_route(
 
 	match &body.visibility {
 		room::Visibility::Public => {
-			if services.globals.config.lockdown_public_room_directory && !services.users.is_admin(sender_user)? {
+			if services.globals.config.lockdown_public_room_directory
+				&& !services.users.is_admin(sender_user)?
+				&& body.appservice_info.is_none()
+			{
 				info!(
 					"Non-admin user {sender_user} tried to publish {0} to the room directory while \
 					 \"lockdown_public_room_directory\" is enabled",
 					body.room_id
 				);
-				services
+
-					.admin
+				if services.globals.config.admin_room_notices {
-					.send_text(&format!(
+					services
-						"Non-admin user {sender_user} tried to publish {0} to the room directory while \
+						.admin
-						 \"lockdown_public_room_directory\" is enabled",
+						.send_text(&format!(
-						body.room_id
+							"Non-admin user {sender_user} tried to publish {0} to the room directory while \
-					))
+							 \"lockdown_public_room_directory\" is enabled",
-					.await;
+							body.room_id
+						))
+						.await;
+				}
 
 				return Err(Error::BadRequest(
 					ErrorKind::forbidden(),
@@ -155,10 +165,13 @@ pub(crate) async fn set_room_visibility_route(
 			}
 
 			services.rooms.directory.set_public(&body.room_id)?;
-			services
+
-				.admin
+			if services.globals.config.admin_room_notices {
-				.send_text(&format!("{sender_user} made {} public to the room directory", body.room_id))
+				services
-				.await;
+					.admin
+					.send_text(&format!("{sender_user} made {} public to the room directory", body.room_id))
+					.await;
+			}
 			info!("{sender_user} made {0} public to the room directory", body.room_id);
 		},
 		room::Visibility::Private => services.rooms.directory.set_not_public(&body.room_id)?,

src/api/client/membership.rs

@@ -40,13 +40,10 @@ use ruma::{
 	OwnedUserId, RoomId, RoomVersionId, ServerName, UserId,
 };
 use serde_json::value::{to_raw_value, RawValue as RawJsonValue};
-use service::{rooms::state::RoomMutexGuard, Services};
+use service::{appservice::RegistrationInfo, rooms::state::RoomMutexGuard, Services};
 use tokio::sync::RwLock;
 
-use crate::{
+use crate::{client::full_user_deactivate, Ruma};
-	client::{update_avatar_url, update_displayname},
-	Ruma,
-};
 
 /// Checks if the room is banned in any way possible and the sender user is not
 /// an admin.
@@ -74,16 +71,15 @@ async fn banned_room_check(
 
 				if services.globals.config.auto_deactivate_banned_room_attempts {
 					warn!("Automatically deactivating user {user_id} due to attempted banned room join");
-					services
-						.admin
-						.send_message(RoomMessageEventContent::text_plain(format!(
-							"Automatically deactivating user {user_id} due to attempted banned room join from IP \
-							 {client_ip}"
-						)))
-						.await;
 
-					if let Err(e) = services.users.deactivate_account(user_id) {
+					if services.globals.config.admin_room_notices {
-						warn!(%user_id, %e, "Failed to deactivate account");
+						services
+							.admin
+							.send_message(RoomMessageEventContent::text_plain(format!(
+								"Automatically deactivating user {user_id} due to attempted banned room join from IP \
+								 {client_ip}"
+							)))
+							.await;
 					}
 
 					let all_joined_rooms: Vec<OwnedRoomId> = services
@@ -93,9 +89,7 @@ async fn banned_room_check(
 						.filter_map(Result::ok)
 						.collect();
 
-					update_displayname(services, user_id.into(), None, all_joined_rooms.clone()).await?;
+					full_user_deactivate(services, user_id, all_joined_rooms).await?;
-					update_avatar_url(services, user_id.into(), None, None, all_joined_rooms).await?;
-					leave_all_rooms(services, user_id).await;
 				}
 
 				return Err(Error::BadRequest(
@@ -117,16 +111,15 @@ async fn banned_room_check(
 
 				if services.globals.config.auto_deactivate_banned_room_attempts {
 					warn!("Automatically deactivating user {user_id} due to attempted banned room join");
-					services
-						.admin
-						.send_message(RoomMessageEventContent::text_plain(format!(
-							"Automatically deactivating user {user_id} due to attempted banned room join from IP \
-							 {client_ip}"
-						)))
-						.await;
 
-					if let Err(e) = services.users.deactivate_account(user_id) {
+					if services.globals.config.admin_room_notices {
-						warn!(%user_id, %e, "Failed to deactivate account");
+						services
+							.admin
+							.send_message(RoomMessageEventContent::text_plain(format!(
+								"Automatically deactivating user {user_id} due to attempted banned room join from IP \
+								 {client_ip}"
+							)))
+							.await;
 					}
 
 					let all_joined_rooms: Vec<OwnedRoomId> = services
@@ -136,9 +129,7 @@ async fn banned_room_check(
 						.filter_map(Result::ok)
 						.collect();
 
-					update_displayname(services, user_id.into(), None, all_joined_rooms.clone()).await?;
+					full_user_deactivate(services, user_id, all_joined_rooms).await?;
-					update_avatar_url(services, user_id.into(), None, None, all_joined_rooms).await?;
-					leave_all_rooms(services, user_id).await;
 				}
 
 				return Err(Error::BadRequest(
@@ -209,6 +200,7 @@ pub(crate) async fn join_room_by_id_route(
 		body.reason.clone(),
 		&servers,
 		body.third_party_signed.as_ref(),
+		&body.appservice_info,
 	)
 	.await
 }
@@ -228,13 +220,14 @@ pub(crate) async fn join_room_by_id_or_alias_route(
 	body: Ruma<join_room_by_id_or_alias::v3::Request>,
 ) -> Result<join_room_by_id_or_alias::v3::Response> {
 	let sender_user = body.sender_user.as_deref().expect("user is authenticated");
+	let appservice_info = &body.appservice_info;
 	let body = body.body;
 
 	let (servers, room_id) = match OwnedRoomId::try_from(body.room_id_or_alias) {
 		Ok(room_id) => {
 			banned_room_check(&services, sender_user, Some(&room_id), room_id.server_name(), client).await?;
 
-			let mut servers = body.server_name.clone();
+			let mut servers = body.via.clone();
 			servers.extend(
 				services
 					.rooms
@@ -267,13 +260,13 @@ pub(crate) async fn join_room_by_id_or_alias_route(
 			let response = services
 				.rooms
 				.alias
-				.resolve_alias(&room_alias, Some(&body.server_name.clone()))
+				.resolve_alias(&room_alias, Some(&body.via.clone()))
 				.await?;
 			let (room_id, mut pre_servers) = response;
 
 			banned_room_check(&services, sender_user, Some(&room_id), Some(room_alias.server_name()), client).await?;
 
-			let mut servers = body.server_name;
+			let mut servers = body.via;
 			if let Some(pre_servers) = &mut pre_servers {
 				servers.append(pre_servers);
 			}
@@ -310,6 +303,7 @@ pub(crate) async fn join_room_by_id_or_alias_route(
 		body.reason.clone(),
 		&servers,
 		body.third_party_signed.as_ref(),
+		appservice_info,
 	)
 	.await?;
 
@@ -660,11 +654,11 @@ pub(crate) async fn joined_members_route(
 
 pub async fn join_room_by_id_helper(
 	services: &Services, sender_user: &UserId, room_id: &RoomId, reason: Option<String>, servers: &[OwnedServerName],
-	third_party_signed: Option<&ThirdPartySigned>,
+	third_party_signed: Option<&ThirdPartySigned>, appservice_info: &Option<RegistrationInfo>,
 ) -> Result<join_room_by_id::v3::Response> {
 	let state_lock = services.rooms.state.mutex.lock(room_id).await;
 
-	let user_is_guest = services.users.is_deactivated(sender_user).unwrap_or(false);
+	let user_is_guest = services.users.is_deactivated(sender_user).unwrap_or(false) && appservice_info.is_none();
 
 	if matches!(services.rooms.state_accessor.guest_can_join(room_id), Ok(false)) && user_is_guest {
 		return Err!(Request(Forbidden("Guests are not allowed to join this room")));

src/api/client/mod.rs

@@ -37,6 +37,7 @@ pub(super) mod unversioned;
 pub(super) mod user_directory;
 pub(super) mod voip;
 
+pub use account::full_user_deactivate;
 pub(super) use account::*;
 pub(super) use alias::*;
 pub(super) use appservice::*;

src/api/client/profile.rs

@@ -1,5 +1,5 @@
 use axum::extract::State;
-use conduit::{pdu::PduBuilder, warn, Error, Result};
+use conduit::{pdu::PduBuilder, warn, Err, Error, Result};
 use ruma::{
 	api::{
 		client::{
@@ -10,7 +10,7 @@ use ruma::{
 	},
 	events::{room::member::RoomMemberEventContent, StateEventType, TimelineEventType},
 	presence::PresenceState,
-	OwnedMxcUri, OwnedRoomId, OwnedUserId,
+	OwnedMxcUri, OwnedRoomId, UserId,
 };
 use serde_json::value::to_raw_value;
 use service::Services;
@@ -26,20 +26,25 @@ pub(crate) async fn set_displayname_route(
 	State(services): State<crate::State>, body: Ruma<set_display_name::v3::Request>,
 ) -> Result<set_display_name::v3::Response> {
 	let sender_user = body.sender_user.as_ref().expect("user is authenticated");
+
+	if *sender_user != body.user_id && body.appservice_info.is_none() {
+		return Err!(Request(Forbidden("You cannot update the profile of another user")));
+	}
+
 	let all_joined_rooms: Vec<OwnedRoomId> = services
 		.rooms
 		.state_cache
-		.rooms_joined(sender_user)
+		.rooms_joined(&body.user_id)
 		.filter_map(Result::ok)
 		.collect();
 
-	update_displayname(&services, sender_user.clone(), body.displayname.clone(), all_joined_rooms).await?;
+	update_displayname(&services, &body.user_id, body.displayname.clone(), all_joined_rooms).await?;
 
 	if services.globals.allow_local_presence() {
 		// Presence update
 		services
 			.presence
-			.ping_presence(sender_user, &PresenceState::Online)?;
+			.ping_presence(&body.user_id, &PresenceState::Online)?;
 	}
 
 	Ok(set_display_name::v3::Response {})
@@ -110,16 +115,21 @@ pub(crate) async fn set_avatar_url_route(
 	State(services): State<crate::State>, body: Ruma<set_avatar_url::v3::Request>,
 ) -> Result<set_avatar_url::v3::Response> {
 	let sender_user = body.sender_user.as_ref().expect("user is authenticated");
+
+	if *sender_user != body.user_id && body.appservice_info.is_none() {
+		return Err!(Request(Forbidden("You cannot update the profile of another user")));
+	}
+
 	let all_joined_rooms: Vec<OwnedRoomId> = services
 		.rooms
 		.state_cache
-		.rooms_joined(sender_user)
+		.rooms_joined(&body.user_id)
 		.filter_map(Result::ok)
 		.collect();
 
 	update_avatar_url(
 		&services,
-		sender_user.clone(),
+		&body.user_id,
 		body.avatar_url.clone(),
 		body.blurhash.clone(),
 		all_joined_rooms,
@@ -130,7 +140,7 @@ pub(crate) async fn set_avatar_url_route(
 		// Presence update
 		services
 			.presence
-			.ping_presence(sender_user, &PresenceState::Online)?;
+			.ping_presence(&body.user_id, &PresenceState::Online)?;
 	}
 
 	Ok(set_avatar_url::v3::Response {})
@@ -196,7 +206,7 @@ pub(crate) async fn get_avatar_url_route(
 
 /// # `GET /_matrix/client/v3/profile/{userId}`
 ///
-/// Returns the displayname, avatar_url and blurhash of the user.
+/// Returns the displayname, avatar_url, blurhash, and tz of the user.
 ///
 /// - If user is on another server and we do not have a local copy already,
 ///   fetch profile over federation.
@@ -232,11 +242,23 @@ pub(crate) async fn get_profile_route(
 				.users
 				.set_blurhash(&body.user_id, response.blurhash.clone())
 				.await?;
+			services
+				.users
+				.set_timezone(&body.user_id, response.tz.clone())
+				.await?;
+
+			for (profile_key, profile_key_value) in &response.custom_profile_fields {
+				services
+					.users
+					.set_profile_key(&body.user_id, profile_key, Some(profile_key_value.clone()))?;
+			}
 
 			return Ok(get_profile::v3::Response {
 				displayname: response.displayname,
 				avatar_url: response.avatar_url,
 				blurhash: response.blurhash,
+				tz: response.tz,
+				custom_profile_fields: response.custom_profile_fields,
 			});
 		}
 	}
@@ -251,13 +273,19 @@ pub(crate) async fn get_profile_route(
 		avatar_url: services.users.avatar_url(&body.user_id)?,
 		blurhash: services.users.blurhash(&body.user_id)?,
 		displayname: services.users.displayname(&body.user_id)?,
+		tz: services.users.timezone(&body.user_id)?,
+		custom_profile_fields: services
+			.users
+			.all_profile_keys(&body.user_id)
+			.filter_map(Result::ok)
+			.collect(),
 	})
 }
 
 pub async fn update_displayname(
-	services: &Services, user_id: OwnedUserId, displayname: Option<String>, all_joined_rooms: Vec<OwnedRoomId>,
+	services: &Services, user_id: &UserId, displayname: Option<String>, all_joined_rooms: Vec<OwnedRoomId>,
 ) -> Result<()> {
-	let current_display_name = services.users.displayname(&user_id).unwrap_or_default();
+	let current_display_name = services.users.displayname(user_id).unwrap_or_default();
 
 	if displayname == current_display_name {
 		return Ok(());
@@ -265,7 +293,7 @@ pub async fn update_displayname(
 
 	services
 		.users
-		.set_displayname(&user_id, displayname.clone())
+		.set_displayname(user_id, displayname.clone())
 		.await?;
 
 	// Send a new join membership event into all joined rooms
@@ -309,11 +337,11 @@ pub async fn update_displayname(
 }
 
 pub async fn update_avatar_url(
-	services: &Services, user_id: OwnedUserId, avatar_url: Option<OwnedMxcUri>, blurhash: Option<String>,
+	services: &Services, user_id: &UserId, avatar_url: Option<OwnedMxcUri>, blurhash: Option<String>,
 	all_joined_rooms: Vec<OwnedRoomId>,
 ) -> Result<()> {
-	let current_avatar_url = services.users.avatar_url(&user_id).unwrap_or_default();
+	let current_avatar_url = services.users.avatar_url(user_id).unwrap_or_default();
-	let current_blurhash = services.users.blurhash(&user_id).unwrap_or_default();
+	let current_blurhash = services.users.blurhash(user_id).unwrap_or_default();
 
 	if current_avatar_url == avatar_url && current_blurhash == blurhash {
 		return Ok(());
@@ -321,11 +349,11 @@ pub async fn update_avatar_url(
 
 	services
 		.users
-		.set_avatar_url(&user_id, avatar_url.clone())
+		.set_avatar_url(user_id, avatar_url.clone())
 		.await?;
 	services
 		.users
-		.set_blurhash(&user_id, blurhash.clone())
+		.set_blurhash(user_id, blurhash.clone())
 		.await?;
 
 	// Send a new join membership event into all joined rooms
@@ -370,14 +398,14 @@ pub async fn update_avatar_url(
 }
 
 pub async fn update_all_rooms(
-	services: &Services, all_joined_rooms: Vec<(PduBuilder, &OwnedRoomId)>, user_id: OwnedUserId,
+	services: &Services, all_joined_rooms: Vec<(PduBuilder, &OwnedRoomId)>, user_id: &UserId,
 ) {
 	for (pdu_builder, room_id) in all_joined_rooms {
 		let state_lock = services.rooms.state.mutex.lock(room_id).await;
 		if let Err(e) = services
 			.rooms
 			.timeline
-			.build_and_append_pdu(pdu_builder, &user_id, room_id, &state_lock)
+			.build_and_append_pdu(pdu_builder, user_id, room_id, &state_lock)
 			.await
 		{
 			warn!(%user_id, %room_id, %e, "Failed to update/send new profile join membership update in room");

src/api/client/room.rs

@@ -1,7 +1,7 @@
 use std::{cmp::max, collections::BTreeMap};
 
 use axum::extract::State;
-use conduit::{debug_info, debug_warn, err};
+use conduit::{debug_info, debug_warn, err, Err};
 use ruma::{
 	api::client::{
 		error::ErrorKind,
@@ -64,6 +64,7 @@ const TRANSFERABLE_STATE_EVENTS: &[StateEventType; 9] = &[
 /// - Send events listed in initial state
 /// - Send events implied by `name` and `topic`
 /// - Send invite events
+#[allow(clippy::large_stack_frames)]
 pub(crate) async fn create_room_route(
 	State(services): State<crate::State>, body: Ruma<create_room::v3::Request>,
 ) -> Result<create_room::v3::Response> {
@@ -92,6 +93,31 @@ pub(crate) async fn create_room_route(
 		));
 	}
 
+	if body.visibility == room::Visibility::Public
+		&& services.globals.config.lockdown_public_room_directory
+		&& !services.users.is_admin(sender_user)?
+		&& body.appservice_info.is_none()
+	{
+		info!(
+			"Non-admin user {sender_user} tried to publish {0} to the room directory while \
+			 \"lockdown_public_room_directory\" is enabled",
+			&room_id
+		);
+
+		if services.globals.config.admin_room_notices {
+			services
+				.admin
+				.send_text(&format!(
+					"Non-admin user {sender_user} tried to publish {0} to the room directory while \
+					 \"lockdown_public_room_directory\" is enabled",
+					&room_id
+				))
+				.await;
+		}
+
+		return Err!(Request(Forbidden("Publishing rooms to the room directory is not allowed")));
+	}
+
 	let _short_id = services.rooms.short.get_or_create_shortroomid(&room_id)?;
 	let state_lock = services.rooms.state.mutex.lock(&room_id).await;
 
@@ -119,6 +145,7 @@ pub(crate) async fn create_room_route(
 		None => services.globals.default_room_version(),
 	};
 
+	#[allow(clippy::single_match_else)]
 	let content = match &body.creation_content {
 		Some(content) => {
 			use RoomVersionId::*;
@@ -230,8 +257,7 @@ pub(crate) async fn create_room_route(
 		_ => RoomPreset::PrivateChat, // Room visibility should not be custom
 	});
 
-	let mut users = BTreeMap::new();
+	let mut users = BTreeMap::from_iter([(sender_user.clone(), int!(100))]);
-	users.insert(sender_user.clone(), int!(100));
 
 	if preset == RoomPreset::TrustedPrivateChat {
 		for invite_ in &body.invite {
@@ -450,6 +476,14 @@ pub(crate) async fn create_room_route(
 
 	if body.visibility == room::Visibility::Public {
 		services.rooms.directory.set_public(&room_id)?;
+
+		if services.globals.config.admin_room_notices {
+			services
+				.admin
+				.send_text(&format!("{sender_user} made {} public to the room directory", &room_id))
+				.await;
+		}
+		info!("{sender_user} made {0} public to the room directory", &room_id);
 	}
 
 	info!("{sender_user} created a room with room ID {room_id}");
@@ -821,10 +855,18 @@ fn default_power_levels_content(
 	power_levels_content["events"]["m.room.history_visibility"] =
 		serde_json::to_value(100).expect("100 is valid Value");
 
+	// always allow users to respond (not post new) to polls. this is primarily
+	// useful in read-only announcement rooms that post a public poll.
+	power_levels_content["events"]["org.matrix.msc3381.poll.response"] =
+		serde_json::to_value(0).expect("0 is valid Value");
+	power_levels_content["events"]["m.poll.response"] = serde_json::to_value(0).expect("0 is valid Value");
+
 	// synapse does this too. clients do not expose these permissions. it prevents
 	// default users from calling public rooms, for obvious reasons.
 	if *visibility == room::Visibility::Public {
 		power_levels_content["events"]["m.call.invite"] = serde_json::to_value(50).expect("50 is valid Value");
+		power_levels_content["events"]["m.call"] = serde_json::to_value(50).expect("50 is valid Value");
+		power_levels_content["events"]["m.call.member"] = serde_json::to_value(50).expect("50 is valid Value");
 		power_levels_content["events"]["org.matrix.msc3401.call"] =
 			serde_json::to_value(50).expect("50 is valid Value");
 		power_levels_content["events"]["org.matrix.msc3401.call.member"] =

src/api/client/sync.rs

@@ -25,12 +25,12 @@ use ruma::{
 		},
 		uiaa::UiaaResponse,
 	},
+	directory::RoomTypeFilter,
 	events::{
 		presence::PresenceEvent,
 		room::member::{MembershipState, RoomMemberEventContent},
 		AnyRawAccountDataEvent, StateEventType, TimelineEventType,
 	},
-	room::RoomType,
 	serde::Raw,
 	state_res::Event,
 	uint, DeviceId, EventId, MilliSecondsSinceUnixEpoch, OwnedRoomId, OwnedUserId, RoomId, UInt, UserId,
@@ -1760,32 +1760,23 @@ pub(crate) async fn sync_events_v4_route(
 }
 
 fn filter_rooms(
-	rooms: &[OwnedRoomId], State(services): State<crate::State>, filter: &[Option<RoomType>], negate: bool,
+	rooms: &[OwnedRoomId], State(services): State<crate::State>, filter: &[RoomTypeFilter], negate: bool,
 ) -> Vec<OwnedRoomId> {
 	return rooms
 		.iter()
-		.filter(|r| {
+		.filter(|r| match services.rooms.state_accessor.get_room_type(r) {
-			match services.rooms.state_accessor.get_room_type(r) {
+			Err(e) => {
-				Err(e) => {
+				warn!("Requested room type for {}, but could not retrieve with error {}", r, e);
-					warn!("Requested room type for {}, but could not retrieve with error {}", r, e);
+				false
-					false
+			},
-				},
+			Ok(result) => {
-				Ok(None) => {
+				let result = RoomTypeFilter::from(result);
-					// For rooms which do not have a room type, use 'null' to include them
+				if negate {
-					if negate {
+					!filter.contains(&result)
-						!filter.contains(&None)
+				} else {
-					} else {
+					filter.is_empty() || filter.contains(&result)
-						filter.contains(&None)
+				}
-					}
+			},
-				},
-				Ok(Some(room_type)) => {
-					if negate {
-						!filter.contains(&Some(room_type))
-					} else {
-						filter.is_empty() || filter.contains(&Some(room_type))
-					}
-				},
-			}
 		})
 		.cloned()
 		.collect();

src/api/client/unstable.rs

@@ -1,12 +1,27 @@
+use std::collections::BTreeMap;
+
 use axum::extract::State;
 use axum_client_ip::InsecureClientIp;
-use conduit::warn;
+use conduit::{warn, Err};
 use ruma::{
-	api::client::{error::ErrorKind, membership::mutual_rooms, room::get_summary},
+	api::{
+		client::{
+			error::ErrorKind,
+			membership::mutual_rooms,
+			profile::{
+				delete_profile_key, delete_timezone_key, get_profile_key, get_timezone_key, set_profile_key,
+				set_timezone_key,
+			},
+			room::get_summary,
+		},
+		federation,
+	},
 	events::room::member::MembershipState,
+	presence::PresenceState,
 	OwnedRoomId,
 };
 
+use super::{update_avatar_url, update_displayname};
 use crate::{Error, Result, Ruma, RumaResponse};
 
 /// # `GET /_matrix/client/unstable/uk.half-shot.msc2666/user/mutual_rooms`
@@ -161,3 +176,327 @@ pub(crate) async fn get_room_summary(
 			.unwrap_or_else(|_e| None),
 	})
 }
+
+/// # `DELETE /_matrix/client/unstable/uk.tcpip.msc4133/profile/:user_id/us.cloke.msc4175.tz`
+///
+/// Deletes the `tz` (timezone) of a user, as per MSC4133 and MSC4175.
+///
+/// - Also makes sure other users receive the update using presence EDUs
+pub(crate) async fn delete_timezone_key_route(
+	State(services): State<crate::State>, body: Ruma<delete_timezone_key::unstable::Request>,
+) -> Result<delete_timezone_key::unstable::Response> {
+	let sender_user = body.sender_user.as_ref().expect("user is authenticated");
+
+	if *sender_user != body.user_id && body.appservice_info.is_none() {
+		return Err!(Request(Forbidden("You cannot update the profile of another user")));
+	}
+
+	services.users.set_timezone(&body.user_id, None).await?;
+
+	if services.globals.allow_local_presence() {
+		// Presence update
+		services
+			.presence
+			.ping_presence(&body.user_id, &PresenceState::Online)?;
+	}
+
+	Ok(delete_timezone_key::unstable::Response {})
+}
+
+/// # `PUT /_matrix/client/unstable/uk.tcpip.msc4133/profile/:user_id/us.cloke.msc4175.tz`
+///
+/// Updates the `tz` (timezone) of a user, as per MSC4133 and MSC4175.
+///
+/// - Also makes sure other users receive the update using presence EDUs
+pub(crate) async fn set_timezone_key_route(
+	State(services): State<crate::State>, body: Ruma<set_timezone_key::unstable::Request>,
+) -> Result<set_timezone_key::unstable::Response> {
+	let sender_user = body.sender_user.as_ref().expect("user is authenticated");
+
+	if *sender_user != body.user_id && body.appservice_info.is_none() {
+		return Err!(Request(Forbidden("You cannot update the profile of another user")));
+	}
+
+	services
+		.users
+		.set_timezone(&body.user_id, body.tz.clone())
+		.await?;
+
+	if services.globals.allow_local_presence() {
+		// Presence update
+		services
+			.presence
+			.ping_presence(&body.user_id, &PresenceState::Online)?;
+	}
+
+	Ok(set_timezone_key::unstable::Response {})
+}
+
+/// # `PUT /_matrix/client/unstable/uk.tcpip.msc4133/profile/{user_id}/{field}`
+///
+/// Updates the profile key-value field of a user, as per MSC4133.
+///
+/// This also handles the avatar_url and displayname being updated.
+pub(crate) async fn set_profile_key_route(
+	State(services): State<crate::State>, body: Ruma<set_profile_key::unstable::Request>,
+) -> Result<set_profile_key::unstable::Response> {
+	let sender_user = body.sender_user.as_ref().expect("user is authenticated");
+
+	if *sender_user != body.user_id && body.appservice_info.is_none() {
+		return Err!(Request(Forbidden("You cannot update the profile of another user")));
+	}
+
+	if body.kv_pair.is_empty() {
+		return Err!(Request(BadJson(
+			"The key-value pair JSON body is empty. Use DELETE to delete a key"
+		)));
+	}
+
+	if body.kv_pair.len() > 1 {
+		// TODO: support PATCH or "recursively" adding keys in some sort
+		return Err!(Request(BadJson("This endpoint can only take one key-value pair at a time")));
+	}
+
+	let Some(profile_key_value) = body.kv_pair.get(&body.key) else {
+		return Err!(Request(BadJson(
+			"The key does not match the URL field key, or JSON body is empty (use DELETE)"
+		)));
+	};
+
+	if body
+		.kv_pair
+		.keys()
+		.any(|key| key.starts_with("u.") && !profile_key_value.is_string())
+	{
+		return Err!(Request(BadJson("u.* profile key fields must be strings")));
+	}
+
+	if body.kv_pair.keys().any(|key| key.len() > 128) {
+		return Err!(Request(BadJson("Key names cannot be longer than 128 bytes")));
+	}
+
+	if body.key == "displayname" {
+		let all_joined_rooms: Vec<OwnedRoomId> = services
+			.rooms
+			.state_cache
+			.rooms_joined(&body.user_id)
+			.filter_map(Result::ok)
+			.collect();
+
+		update_displayname(&services, &body.user_id, Some(profile_key_value.to_string()), all_joined_rooms).await?;
+	} else if body.key == "avatar_url" {
+		let mxc = ruma::OwnedMxcUri::from(profile_key_value.to_string());
+
+		let all_joined_rooms: Vec<OwnedRoomId> = services
+			.rooms
+			.state_cache
+			.rooms_joined(&body.user_id)
+			.filter_map(Result::ok)
+			.collect();
+
+		update_avatar_url(&services, &body.user_id, Some(mxc), None, all_joined_rooms).await?;
+	} else {
+		services
+			.users
+			.set_profile_key(&body.user_id, &body.key, Some(profile_key_value.clone()))?;
+	}
+
+	if services.globals.allow_local_presence() {
+		// Presence update
+		services
+			.presence
+			.ping_presence(&body.user_id, &PresenceState::Online)?;
+	}
+
+	Ok(set_profile_key::unstable::Response {})
+}
+
+/// # `DELETE /_matrix/client/unstable/uk.tcpip.msc4133/profile/{user_id}/{field}`
+///
+/// Deletes the profile key-value field of a user, as per MSC4133.
+///
+/// This also handles the avatar_url and displayname being updated.
+pub(crate) async fn delete_profile_key_route(
+	State(services): State<crate::State>, body: Ruma<delete_profile_key::unstable::Request>,
+) -> Result<delete_profile_key::unstable::Response> {
+	let sender_user = body.sender_user.as_ref().expect("user is authenticated");
+
+	if *sender_user != body.user_id && body.appservice_info.is_none() {
+		return Err!(Request(Forbidden("You cannot update the profile of another user")));
+	}
+
+	if body.kv_pair.len() > 1 {
+		// TODO: support PATCH or "recursively" adding keys in some sort
+		return Err!(Request(BadJson("This endpoint can only take one key-value pair at a time")));
+	}
+
+	if body.key == "displayname" {
+		let all_joined_rooms: Vec<OwnedRoomId> = services
+			.rooms
+			.state_cache
+			.rooms_joined(&body.user_id)
+			.filter_map(Result::ok)
+			.collect();
+
+		update_displayname(&services, &body.user_id, None, all_joined_rooms).await?;
+	} else if body.key == "avatar_url" {
+		let all_joined_rooms: Vec<OwnedRoomId> = services
+			.rooms
+			.state_cache
+			.rooms_joined(&body.user_id)
+			.filter_map(Result::ok)
+			.collect();
+
+		update_avatar_url(&services, &body.user_id, None, None, all_joined_rooms).await?;
+	} else {
+		services
+			.users
+			.set_profile_key(&body.user_id, &body.key, None)?;
+	}
+
+	if services.globals.allow_local_presence() {
+		// Presence update
+		services
+			.presence
+			.ping_presence(&body.user_id, &PresenceState::Online)?;
+	}
+
+	Ok(delete_profile_key::unstable::Response {})
+}
+
+/// # `GET /_matrix/client/unstable/uk.tcpip.msc4133/profile/:user_id/us.cloke.msc4175.tz`
+///
+/// Returns the `timezone` of the user as per MSC4133 and MSC4175.
+///
+/// - If user is on another server and we do not have a local copy already fetch
+///   `timezone` over federation
+pub(crate) async fn get_timezone_key_route(
+	State(services): State<crate::State>, body: Ruma<get_timezone_key::unstable::Request>,
+) -> Result<get_timezone_key::unstable::Response> {
+	if !services.globals.user_is_local(&body.user_id) {
+		// Create and update our local copy of the user
+		if let Ok(response) = services
+			.sending
+			.send_federation_request(
+				body.user_id.server_name(),
+				federation::query::get_profile_information::v1::Request {
+					user_id: body.user_id.clone(),
+					field: None, // we want the full user's profile to update locally as well
+				},
+			)
+			.await
+		{
+			if !services.users.exists(&body.user_id)? {
+				services.users.create(&body.user_id, None)?;
+			}
+
+			services
+				.users
+				.set_displayname(&body.user_id, response.displayname.clone())
+				.await?;
+			services
+				.users
+				.set_avatar_url(&body.user_id, response.avatar_url.clone())
+				.await?;
+			services
+				.users
+				.set_blurhash(&body.user_id, response.blurhash.clone())
+				.await?;
+			services
+				.users
+				.set_timezone(&body.user_id, response.tz.clone())
+				.await?;
+
+			return Ok(get_timezone_key::unstable::Response {
+				tz: response.tz,
+			});
+		}
+	}
+
+	if !services.users.exists(&body.user_id)? {
+		// Return 404 if this user doesn't exist and we couldn't fetch it over
+		// federation
+		return Err(Error::BadRequest(ErrorKind::NotFound, "Profile was not found."));
+	}
+
+	Ok(get_timezone_key::unstable::Response {
+		tz: services.users.timezone(&body.user_id)?,
+	})
+}
+
+/// # `GET /_matrix/client/unstable/uk.tcpip.msc4133/profile/{userId}/{field}}`
+///
+/// Gets the profile key-value field of a user, as per MSC4133.
+///
+/// - If user is on another server and we do not have a local copy already fetch
+///   `timezone` over federation
+pub(crate) async fn get_profile_key_route(
+	State(services): State<crate::State>, body: Ruma<get_profile_key::unstable::Request>,
+) -> Result<get_profile_key::unstable::Response> {
+	let mut profile_key_value: BTreeMap<String, serde_json::Value> = BTreeMap::new();
+
+	if !services.globals.user_is_local(&body.user_id) {
+		// Create and update our local copy of the user
+		if let Ok(response) = services
+			.sending
+			.send_federation_request(
+				body.user_id.server_name(),
+				federation::query::get_profile_information::v1::Request {
+					user_id: body.user_id.clone(),
+					field: None, // we want the full user's profile to update locally as well
+				},
+			)
+			.await
+		{
+			if !services.users.exists(&body.user_id)? {
+				services.users.create(&body.user_id, None)?;
+			}
+
+			services
+				.users
+				.set_displayname(&body.user_id, response.displayname.clone())
+				.await?;
+			services
+				.users
+				.set_avatar_url(&body.user_id, response.avatar_url.clone())
+				.await?;
+			services
+				.users
+				.set_blurhash(&body.user_id, response.blurhash.clone())
+				.await?;
+			services
+				.users
+				.set_timezone(&body.user_id, response.tz.clone())
+				.await?;
+
+			if let Some(value) = response.custom_profile_fields.get(&body.key) {
+				profile_key_value.insert(body.key.clone(), value.clone());
+				services
+					.users
+					.set_profile_key(&body.user_id, &body.key, Some(value.clone()))?;
+			} else {
+				return Err!(Request(NotFound("The requested profile key does not exist.")));
+			}
+
+			return Ok(get_profile_key::unstable::Response {
+				value: profile_key_value,
+			});
+		}
+	}
+
+	if !services.users.exists(&body.user_id)? {
+		// Return 404 if this user doesn't exist and we couldn't fetch it over
+		// federation
+		return Err(Error::BadRequest(ErrorKind::NotFound, "Profile was not found."));
+	}
+
+	if let Some(value) = services.users.profile_key(&body.user_id, &body.key)? {
+		profile_key_value.insert(body.key.clone(), value);
+	} else {
+		return Err!(Request(NotFound("The requested profile key does not exist.")));
+	}
+
+	Ok(get_profile_key::unstable::Response {
+		value: profile_key_value,
+	})
+}

src/api/client/unversioned.rs

@@ -55,6 +55,8 @@ pub(crate) async fn get_supported_versions_route(
 			("org.matrix.msc3575".to_owned(), true), /* sliding sync (https://github.com/matrix-org/matrix-spec-proposals/pull/3575/files#r1588877046) */
 			("org.matrix.msc3916.stable".to_owned(), true), /* authenticated media (https://github.com/matrix-org/matrix-spec-proposals/pull/3916) */
 			("org.matrix.msc4180".to_owned(), true), /* stable flag for 3916 (https://github.com/matrix-org/matrix-spec-proposals/pull/4180) */
+			("uk.tcpip.msc4133".to_owned(), true), /* Extending User Profile API with Key:Value Pairs (https://github.com/matrix-org/matrix-spec-proposals/pull/4133) */
+			("us.cloke.msc4175".to_owned(), true), /* Profile field for user time zone (https://github.com/matrix-org/matrix-spec-proposals/pull/4175) */
 		]),
 	};
 

src/api/client/voip.rs

@@ -24,7 +24,7 @@ pub(crate) async fn turn_server_route(
 		return Err!(Request(NotFound("Not Found")));
 	}
 
-	let turn_secret = services.globals.turn_secret().clone();
+	let turn_secret = services.globals.turn_secret.clone();
 
 	let (username, password) = if !turn_secret.is_empty() {
 		let expiry = SecondsSinceUnixEpoch::from_system_time(

src/api/router.rs

@@ -22,6 +22,12 @@ use crate::{client, server};
 pub fn build(router: Router<State>, server: &Server) -> Router<State> {
 	let config = &server.config;
 	let mut router = router
+        .ruma_route(client::get_timezone_key_route)
+        .ruma_route(client::get_profile_key_route)
+        .ruma_route(client::set_profile_key_route)
+        .ruma_route(client::delete_profile_key_route)
+        .ruma_route(client::set_timezone_key_route)
+        .ruma_route(client::delete_timezone_key_route)
         .ruma_route(client::appservice_ping)
 		.ruma_route(client::get_supported_versions_route)
 		.ruma_route(client::get_register_available_route)

src/api/router/auth.rs

@@ -236,6 +236,7 @@ async fn auth_server(
 		}
 	}
 
+	#[allow(clippy::or_fun_call)]
 	let signature_uri = CanonicalJsonValue::String(
 		request
 			.parts

src/api/server/query.rs

@@ -1,3 +1,5 @@
+use std::collections::BTreeMap;
+
 use axum::extract::State;
 use conduit::{Error, Result};
 use get_profile_information::v1::ProfileField;
@@ -75,6 +77,8 @@ pub(crate) async fn get_profile_information_route(
 	let mut displayname = None;
 	let mut avatar_url = None;
 	let mut blurhash = None;
+	let mut tz = None;
+	let mut custom_profile_fields = BTreeMap::new();
 
 	match &body.field {
 		Some(ProfileField::DisplayName) => {
@@ -84,12 +88,24 @@ pub(crate) async fn get_profile_information_route(
 			avatar_url = services.users.avatar_url(&body.user_id)?;
 			blurhash = services.users.blurhash(&body.user_id)?;
 		},
-		// TODO: what to do with custom
+		Some(custom_field) => {
-		Some(_) => {},
+			if let Some(value) = services
+				.users
+				.profile_key(&body.user_id, custom_field.as_str())?
+			{
+				custom_profile_fields.insert(custom_field.to_string(), value);
+			}
+		},
 		None => {
 			displayname = services.users.displayname(&body.user_id)?;
 			avatar_url = services.users.avatar_url(&body.user_id)?;
 			blurhash = services.users.blurhash(&body.user_id)?;
+			tz = services.users.timezone(&body.user_id)?;
+			custom_profile_fields = services
+				.users
+				.all_profile_keys(&body.user_id)
+				.filter_map(Result::ok)
+				.collect();
 		},
 	}
 
@@ -97,5 +113,7 @@ pub(crate) async fn get_profile_information_route(
 		displayname,
 		avatar_url,
 		blurhash,
+		tz,
+		custom_profile_fields,
 	})
 }

src/core/Cargo.toml

@@ -52,10 +52,6 @@ zstd_compression = [
 perf_measurements = []
 sentry_telemetry = []
 
-# these do nothing, these are purely for informing users to update their build scripts if they use one
-rocksdb = []
-sha256_media = []
-
 [dependencies]
 argon2.workspace = true
 arrayvec.workspace = true
@@ -83,7 +79,6 @@ regex.workspace = true
 reqwest.workspace = true
 ring.workspace = true
 ruma.workspace = true
-rustls.workspace = true
 sanitize-filename.workspace = true
 serde_json.workspace = true
 serde_regex.workspace = true

src/core/config/check.rs

@@ -9,22 +9,6 @@ pub fn check(config: &Config) -> Result<()> {
 		info!("Note: conduwuit was built without optimisations (i.e. debug build)");
 	}
 
-	// prevents catching this in `--all-features`
-	if cfg!(all(feature = "rocksdb", not(feature = "sha256_media"))) {
-		warn!(
-			"Note the rocksdb feature was deleted from conduwuit. SQLite support was removed and RocksDB is the only \
-			 supported backend now. Please update your build script to remove this feature."
-		);
-	}
-
-	// prevents catching this in `--all-features`
-	if cfg!(all(feature = "sha256_media", not(feature = "rocksdb"))) {
-		warn!(
-			"Note the sha256_media feature was deleted from conduwuit, it is now fully integrated in a \
-			 forwards-compatible way. Please update your build script to remove this feature."
-		);
-	}
-
 	warn_deprecated(config);
 	warn_unknown_key(config);
 

src/core/config/mod.rs

@@ -21,7 +21,7 @@ use url::Url;
 
 pub use self::check::check;
 use self::proxy::ProxyConfig;
-use crate::{error::Error, Err, Result};
+use crate::{error::Error, utils::sys, Err, Result};
 
 pub mod check;
 pub mod proxy;
@@ -184,6 +184,8 @@ pub struct Config {
 	pub query_trusted_key_servers_first: bool,
 	#[serde(default = "default_log")]
 	pub log: String,
+	#[serde(default = "true_fn", alias = "log_colours")]
+	pub log_colors: bool,
 	#[serde(default = "default_openid_token_ttl")]
 	pub openid_token_ttl: u64,
 	#[serde(default)]
@@ -194,6 +196,7 @@ pub struct Config {
 	pub turn_uris: Vec<String>,
 	#[serde(default)]
 	pub turn_secret: String,
+	pub turn_secret_file: Option<PathBuf>,
 	#[serde(default = "default_turn_ttl")]
 	pub turn_ttl: u64,
 
@@ -374,6 +377,13 @@ pub struct Config {
 	#[serde(default)]
 	pub test: BTreeSet<String>,
 
+	/// Controls whether admin room notices like account registrations, password
+	/// changes, account deactivations, room directory publications, etc will
+	/// be sent to the admin room. Update notices and normal admin command
+	/// responses will still be sent.
+	#[serde(default = "true_fn")]
+	pub admin_room_notices: bool,
+
 	#[serde(flatten)]
 	#[allow(clippy::zero_sized_map_values)] // this is a catchall, the map shouldn't be zero at runtime
 	catchall: BTreeMap<String, IgnoredAny>,
@@ -426,29 +436,26 @@ const DEPRECATED_KEYS: &[&str; 9] = &[
 
 impl Config {
 	/// Pre-initialize config
-	pub fn load(path: &Option<PathBuf>) -> Result<Figment> {
+	pub fn load(paths: &Option<Vec<PathBuf>>) -> Result<Figment> {
 		let raw_config = if let Some(config_file_env) = Env::var("CONDUIT_CONFIG") {
-			Figment::new()
+			Figment::new().merge(Toml::file(config_file_env).nested())
-				.merge(Toml::file(config_file_env).nested())
-				.merge(Env::prefixed("CONDUIT_").global().split("__"))
-				.merge(Env::prefixed("CONDUWUIT_").global().split("__"))
 		} else if let Some(config_file_arg) = Env::var("CONDUWUIT_CONFIG") {
-			Figment::new()
+			Figment::new().merge(Toml::file(config_file_arg).nested())
-				.merge(Toml::file(config_file_arg).nested())
+		} else if let Some(config_file_args) = paths {
-				.merge(Env::prefixed("CONDUIT_").global().split("__"))
+			let mut figment = Figment::new();
-				.merge(Env::prefixed("CONDUWUIT_").global().split("__"))
+
-		} else if let Some(config_file_arg) = path {
+			for config in config_file_args {
-			Figment::new()
+				figment = figment.merge(Toml::file(config).nested());
-				.merge(Toml::file(config_file_arg).nested())
+			}
-				.merge(Env::prefixed("CONDUIT_").global().split("__"))
+
-				.merge(Env::prefixed("CONDUWUIT_").global().split("__"))
+			figment
 		} else {
 			Figment::new()
-				.merge(Env::prefixed("CONDUIT_").global().split("__"))
-				.merge(Env::prefixed("CONDUWUIT_").global().split("__"))
 		};
 
-		Ok(raw_config)
+		Ok(raw_config
+			.merge(Env::prefixed("CONDUIT_").global().split("__"))
+			.merge(Env::prefixed("CONDUWUIT_").global().split("__")))
 	}
 
 	/// Finalize config
@@ -466,7 +473,11 @@ impl Config {
 
 	#[must_use]
 	pub fn get_bind_addrs(&self) -> Vec<SocketAddr> {
-		let mut addrs = Vec::new();
+		let mut addrs = Vec::with_capacity(
+			self.get_bind_hosts()
+				.len()
+				.saturating_add(self.get_bind_ports().len()),
+		);
 		for host in &self.get_bind_hosts() {
 			for port in &self.get_bind_ports() {
 				addrs.push(SocketAddr::new(*host, *port));
@@ -684,15 +695,20 @@ impl fmt::Display for Config {
 			}
 		});
 		line("TURN secret", {
-			if self.turn_secret.is_empty() {
+			if self.turn_secret.is_empty() && self.turn_secret_file.is_none() {
 				"not set"
 			} else {
 				"set"
 			}
 		});
+		line("TURN secret file path", {
+			self.turn_secret_file
+				.as_ref()
+				.map_or("", |path| path.to_str().unwrap_or_default())
+		});
 		line("Turn TTL", &self.turn_ttl.to_string());
 		line("Turn URIs", {
-			let mut lst = vec![];
+			let mut lst = Vec::with_capacity(self.turn_uris.len());
 			for item in self.turn_uris.iter().cloned().enumerate() {
 				let (_, uri): (usize, String) = item;
 				lst.push(uri);
@@ -700,7 +716,7 @@ impl fmt::Display for Config {
 			&lst.join(", ")
 		});
 		line("Auto Join Rooms", {
-			let mut lst = vec![];
+			let mut lst = Vec::with_capacity(self.auto_join_rooms.len());
 			for room in &self.auto_join_rooms {
 				lst.push(room);
 			}
@@ -752,28 +768,28 @@ impl fmt::Display for Config {
 		line("Allow legacy (unauthenticated) media", &self.allow_legacy_media.to_string());
 		line("Freeze legacy (unauthenticated) media", &self.freeze_legacy_media.to_string());
 		line("Prevent Media Downloads From", {
-			let mut lst = vec![];
+			let mut lst = Vec::with_capacity(self.prevent_media_downloads_from.len());
 			for domain in &self.prevent_media_downloads_from {
 				lst.push(domain.host());
 			}
 			&lst.join(", ")
 		});
 		line("Forbidden Remote Server Names (\"Global\" ACLs)", {
-			let mut lst = vec![];
+			let mut lst = Vec::with_capacity(self.forbidden_remote_server_names.len());
 			for domain in &self.forbidden_remote_server_names {
 				lst.push(domain.host());
 			}
 			&lst.join(", ")
 		});
 		line("Forbidden Remote Room Directory Server Names", {
-			let mut lst = vec![];
+			let mut lst = Vec::with_capacity(self.forbidden_remote_room_directory_server_names.len());
 			for domain in &self.forbidden_remote_room_directory_server_names {
 				lst.push(domain.host());
 			}
 			&lst.join(", ")
 		});
-		line("Outbound Request IP Range Denylist", {
+		line("Outbound Request IP Range (CIDR) Denylist", {
-			let mut lst = vec![];
+			let mut lst = Vec::with_capacity(self.ip_range_denylist.len());
 			for item in self.ip_range_denylist.iter().cloned().enumerate() {
 				let (_, ip): (usize, String) = item;
 				lst.push(ip);
@@ -862,6 +878,7 @@ impl fmt::Display for Config {
 				.map_or("", |url| url.as_str()),
 		);
 		line("Enable the tokio-console", &self.tokio_console.to_string());
+		line("Admin room notices", &self.admin_room_notices.to_string());
 
 		Ok(())
 	}
@@ -887,29 +904,29 @@ fn default_database_backups_to_keep() -> i16 { 1 }
 
 fn default_database_backend() -> String { "rocksdb".to_owned() }
 
-fn default_db_cache_capacity_mb() -> f64 { 256.0 }
+fn default_db_cache_capacity_mb() -> f64 { 128.0 + parallelism_scaled_f64(64.0) }
 
-fn default_pdu_cache_capacity() -> u32 { 150_000 }
+fn default_pdu_cache_capacity() -> u32 { parallelism_scaled_u32(10_000).saturating_add(100_000) }
 
 fn default_cache_capacity_modifier() -> f64 { 1.0 }
 
-fn default_auth_chain_cache_capacity() -> u32 { 100_000 }
+fn default_auth_chain_cache_capacity() -> u32 { parallelism_scaled_u32(10_000).saturating_add(100_000) }
 
-fn default_shorteventid_cache_capacity() -> u32 { 500_000 }
+fn default_shorteventid_cache_capacity() -> u32 { parallelism_scaled_u32(50_000).saturating_add(100_000) }
 
-fn default_eventidshort_cache_capacity() -> u32 { 100_000 }
+fn default_eventidshort_cache_capacity() -> u32 { parallelism_scaled_u32(25_000).saturating_add(100_000) }
 
-fn default_shortstatekey_cache_capacity() -> u32 { 100_000 }
+fn default_shortstatekey_cache_capacity() -> u32 { parallelism_scaled_u32(10_000).saturating_add(100_000) }
 
-fn default_statekeyshort_cache_capacity() -> u32 { 100_000 }
+fn default_statekeyshort_cache_capacity() -> u32 { parallelism_scaled_u32(10_000).saturating_add(100_000) }
 
-fn default_server_visibility_cache_capacity() -> u32 { 100 }
+fn default_server_visibility_cache_capacity() -> u32 { parallelism_scaled_u32(500) }
 
-fn default_user_visibility_cache_capacity() -> u32 { 100 }
+fn default_user_visibility_cache_capacity() -> u32 { parallelism_scaled_u32(1000) }
 
-fn default_stateinfo_cache_capacity() -> u32 { 100 }
+fn default_stateinfo_cache_capacity() -> u32 { parallelism_scaled_u32(1000) }
 
-fn default_roomid_spacehierarchy_cache_capacity() -> u32 { 100 }
+fn default_roomid_spacehierarchy_cache_capacity() -> u32 { parallelism_scaled_u32(1000) }
 
 fn default_dns_cache_entries() -> u32 { 32768 }
 
@@ -1087,3 +1104,13 @@ fn default_admin_log_capture() -> String {
 }
 
 fn default_admin_room_tag() -> String { "m.server_notice".to_owned() }
+
+#[allow(clippy::as_conversions, clippy::cast_precision_loss)]
+fn parallelism_scaled_f64(val: f64) -> f64 { val * (sys::available_parallelism() as f64) }
+
+fn parallelism_scaled_u32(val: u32) -> u32 {
+	let val = val.try_into().expect("failed to cast u32 to usize");
+	parallelism_scaled(val).try_into().unwrap_or(u32::MAX)
+}
+
+fn parallelism_scaled(val: usize) -> usize { val.saturating_mul(sys::available_parallelism()) }

src/core/debug.rs

@@ -61,7 +61,7 @@ pub fn set_panic_trap() {
 
 #[inline(always)]
 #[allow(deprecated_in_future)]
-fn panic_handler(info: &panic::PanicInfo<'_>, next: &dyn Fn(&panic::PanicInfo<'_>)) {
+fn panic_handler(info: &panic::PanicHookInfo<'_>, next: &dyn Fn(&panic::PanicHookInfo<'_>)) {
 	trap();
 	next(info);
 }

src/database/maps.rs

@@ -94,6 +94,7 @@ pub const MAPS: &[&str] = &[
 	"userid_presenceid",
 	"userid_selfsigningkeyid",
 	"userid_usersigningkeyid",
+	"useridprofilekey_value",
 	"openidtoken_expiresatuserid",
 	"userroomid_highlightcount",
 	"userroomid_invitestate",

src/main/Cargo.toml

@@ -42,6 +42,7 @@ default = [
 	"gzip_compression",
 	"io_uring",
 	"jemalloc",
+	"jemalloc_stats",
 	"release_max_log_level",
 	"sentry_telemetry",
 	"systemd",
@@ -65,6 +66,9 @@ console = [
 #	"conduit-router/dev_release_log_level",
 #	"conduit-service/dev_release_log_level",
 #]
+direct_tls = [
+    "conduit-router/direct_tls"
+]
 element_hacks = [
 	"conduit-api/element_hacks",
 	"conduit-service/element_hacks",

src/main/clap.rs

@@ -14,7 +14,7 @@ use conduit::{
 pub(crate) struct Args {
 	#[arg(short, long)]
 	/// Path to the config TOML file (optional)
-	pub(crate) config: Option<PathBuf>,
+	pub(crate) config: Option<Vec<PathBuf>>,
 
 	/// Override a configuration variable using TOML 'key=value' syntax
 	#[arg(long, short('O'))]

src/main/tracing.rs

@@ -18,7 +18,7 @@ pub(crate) fn init(config: &Config) -> Result<(LogLevelReloadHandles, TracingFla
 	let reload_handles = LogLevelReloadHandles::default();
 
 	let console_filter = EnvFilter::try_new(&config.log).map_err(|e| err!(Config("log", "{e}.")))?;
-	let console_layer = tracing_subscriber::fmt::Layer::new();
+	let console_layer = tracing_subscriber::fmt::Layer::new().with_ansi(config.log_colors);
 	let (console_reload_filter, console_reload_handle) = reload::Layer::new(console_filter.clone());
 	reload_handles.add("console", Box::new(console_reload_handle));
 

src/router/Cargo.toml

@@ -42,9 +42,16 @@ systemd = [
 	"dep:sd-notify",
 ]
 
+direct_tls = [
+    "axum-server/tls-rustls",
+    "dep:rustls",
+    "dep:axum-server-dual-protocol",
+]
+
 [dependencies]
 axum-client-ip.workspace = true
 axum-server-dual-protocol.workspace = true
+axum-server-dual-protocol.optional = true
 axum-server.workspace = true
 axum.workspace = true
 conduit-admin.workspace = true
@@ -63,6 +70,7 @@ hyper.workspace = true
 hyper-util.workspace = true
 ruma.workspace = true
 rustls.workspace = true
+rustls.optional = true
 sentry.optional = true
 sentry-tower.optional = true
 sentry-tower.workspace = true

src/router/serve/mod.rs

@@ -1,4 +1,5 @@
 mod plain;
+#[cfg(feature = "direct_tls")]
 mod tls;
 mod unix;
 
@@ -23,7 +24,14 @@ pub(super) async fn serve(
 	if cfg!(unix) && config.unix_socket_path.is_some() {
 		unix::serve(server, app, shutdown).await
 	} else if config.tls.is_some() {
-		tls::serve(server, app, handle, addrs).await
+		#[cfg(feature = "direct_tls")]
+		return tls::serve(server, app, handle, addrs).await;
+
+		#[cfg(not(feature = "direct_tls"))]
+		return conduit::Err!(Config(
+			"tls",
+			"conduwuit was not built with direct TLS support (\"direct_tls\")"
+		));
 	} else {
 		plain::serve(server, app, handle, addrs).await
 	}

src/router/serve/tls.rs

@@ -20,7 +20,9 @@ pub(super) async fn serve(
 
 	// we use ring for ruma and hashing state, but aws-lc-rs is the new default.
 	// without this, TLS mode will panic.
-	_ = rustls::crypto::aws_lc_rs::default_provider().install_default();
+	rustls::crypto::aws_lc_rs::default_provider()
+		.install_default()
+		.expect("failed to initialise aws-lc-rs rustls crypto provider");
 
 	debug!("Using direct TLS. Certificate path {certs} and certificate private key path {key}",);
 	info!(

src/service/admin/create.rs

@@ -101,8 +101,7 @@ pub async fn create_admin_room(services: &Services) -> Result<()> {
 		.await?;
 
 	// 3. Power levels
-	let mut users = BTreeMap::new();
+	let users = BTreeMap::from_iter([(server_user.clone(), 100.into())]);
-	users.insert(server_user.clone(), 100.into());
 
 	services
 		.rooms

src/service/admin/grant.rs

@@ -21,7 +21,7 @@ impl super::Service {
 	/// Invite the user to the conduit admin room.
 	///
 	/// In conduit, this is equivalent to granting admin privileges.
-	pub async fn make_user_admin(&self, user_id: &UserId, displayname: String) -> Result<()> {
+	pub async fn make_user_admin(&self, user_id: &UserId) -> Result<()> {
 		let Some(room_id) = self.get_admin_room()? else {
 			return Ok(());
 		};
@@ -65,7 +65,7 @@ impl super::Service {
 					event_type: TimelineEventType::RoomMember,
 					content: to_raw_value(&RoomMemberEventContent {
 						membership: MembershipState::Join,
-						displayname: Some(displayname),
+						displayname: None,
 						avatar_url: None,
 						is_direct: None,
 						third_party_invite: None,
@@ -86,9 +86,7 @@ impl super::Service {
 			.await?;
 
 		// Set power level
-		let mut users = BTreeMap::new();
+		let users = BTreeMap::from_iter([(server_user.clone(), 100.into()), (user_id.to_owned(), 100.into())]);
-		users.insert(server_user.clone(), 100.into());
-		users.insert(user_id.to_owned(), 100.into());
 
 		self.services
 			.timeline

src/service/globals/migrations.rs

@@ -24,6 +24,14 @@ use crate::{media, Services};
 ///   equal or lesser version. These are expected to be backward-compatible.
 pub(crate) const DATABASE_VERSION: u64 = 13;
 
+/// Conduit's database version.
+///
+/// Conduit bumped the database version to 16, but did not introduce any
+/// breaking changes. Their database migrations are extremely fragile and risky,
+/// and also do not really apply to us, so just to retain Conduit -> conduwuit
+/// compatibility we'll check for both versions.
+pub(crate) const CONDUIT_DATABASE_VERSION: u64 = 16;
+
 pub(crate) async fn migrations(services: &Services) -> Result<()> {
 	// Matrix resource ownership is based on the server name; changing it
 	// requires recreating the database from scratch.
@@ -54,6 +62,7 @@ async fn fresh(services: &Services) -> Result<()> {
 		.db
 		.bump_database_version(DATABASE_VERSION)?;
 
+	db["global"].insert(b"feat_sha256_media", &[])?;
 	db["global"].insert(b"fix_bad_double_separator_in_state_cache", &[])?;
 	db["global"].insert(b"retroactively_fix_bad_data_from_roomuserid_joined", &[])?;
 
@@ -147,9 +156,11 @@ async fn migrate(services: &Services) -> Result<()> {
 		retroactively_fix_bad_data_from_roomuserid_joined(services).await?;
 	}
 
-	assert_eq!(
+	let version_match = services.globals.db.database_version().unwrap() == DATABASE_VERSION
-		services.globals.db.database_version().unwrap(),
+		|| services.globals.db.database_version().unwrap() == CONDUIT_DATABASE_VERSION;
-		DATABASE_VERSION,
+
+	assert!(
+		version_match,
 		"Failed asserting local database version {} is equal to known latest conduwuit database version {}",
 		services.globals.db.database_version().unwrap(),
 		DATABASE_VERSION,

src/service/globals/mod.rs

@@ -40,6 +40,7 @@ pub struct Service {
 	pub stateres_mutex: Arc<Mutex<()>>,
 	pub server_user: OwnedUserId,
 	pub admin_alias: OwnedRoomAliasId,
+	pub turn_secret: String,
 }
 
 type RateLimitState = (Instant, u32); // Time if last failed try, number of failed tries
@@ -84,6 +85,17 @@ impl crate::Service for Service {
 			.collect::<Result<_, String>>()
 			.map_err(|e| err!(Config("ip_range_denylist", e)))?;
 
+		let turn_secret = config
+			.turn_secret_file
+			.as_ref()
+			.map_or(config.turn_secret.clone(), |path| {
+				std::fs::read_to_string(path).unwrap_or_else(|e| {
+					error!("Failed to read the TURN secret file: {e}");
+
+					config.turn_secret.clone()
+				})
+			});
+
 		let mut s = Self {
 			db,
 			config: config.clone(),
@@ -99,6 +111,7 @@ impl crate::Service for Service {
 				.expect("#admins:server_name is valid alias name"),
 			server_user: UserId::parse_with_server_name(String::from("conduit"), &config.server_name)
 				.expect("@conduit:server_name is valid"),
+			turn_secret,
 		};
 
 		if !s
@@ -207,8 +220,6 @@ impl Service {
 
 	pub fn turn_username(&self) -> &String { &self.config.turn_username }
 
-	pub fn turn_secret(&self) -> &String { &self.config.turn_secret }
-
 	pub fn allow_profile_lookup_federation_requests(&self) -> bool {
 		self.config.allow_profile_lookup_federation_requests
 	}
@@ -264,12 +275,15 @@ impl Service {
 	pub fn block_non_admin_invites(&self) -> bool { self.config.block_non_admin_invites }
 
 	pub fn supported_room_versions(&self) -> Vec<RoomVersionId> {
-		let mut room_versions: Vec<RoomVersionId> = Vec::with_capacity(self.stable_room_versions.len());
+		if self.config.allow_unstable_room_versions {
-		room_versions.extend(self.stable_room_versions.clone());
+			self.stable_room_versions
-		if self.allow_unstable_room_versions() {
+				.clone()
-			room_versions.extend(self.unstable_room_versions.clone());
+				.into_iter()
-		};
+				.chain(self.unstable_room_versions.clone())
-		room_versions
+				.collect()
+		} else {
+			self.stable_room_versions.clone()
+		}
 	}
 
 	/// This returns an empty `Ok(BTreeMap<..>)` when there are no keys found

src/service/media/mod.rs

@@ -10,7 +10,7 @@ use std::{path::PathBuf, sync::Arc, time::SystemTime};
 use async_trait::async_trait;
 use base64::{engine::general_purpose, Engine as _};
 use conduit::{
-	debug, debug_error, debug_info, err, error, trace,
+	debug, debug_error, debug_info, debug_warn, err, error, trace,
 	utils::{self, MutexMap},
 	warn, Err, Result, Server,
 };
@@ -99,45 +99,46 @@ impl Service {
 	pub async fn delete(&self, mxc: &Mxc<'_>) -> Result<()> {
 		if let Ok(keys) = self.db.search_mxc_metadata_prefix(mxc) {
 			for key in keys {
-				trace!(?mxc, ?key, "Deleting from filesystem");
+				trace!(?mxc, "MXC Key: {key:?}");
+				debug_info!(?mxc, "Deleting from filesystem");
+
 				if let Err(e) = self.remove_media_file(&key).await {
-					error!(?mxc, ?key, "Failed to remove media file: {e}");
+					debug_error!(?mxc, "Failed to remove media file: {e}");
 				}
 
-				trace!(?mxc, ?key, "Deleting from database");
+				debug_info!(?mxc, "Deleting from database");
-				if let Err(e) = self.db.delete_file_mxc(mxc) {
+				_ = self.db.delete_file_mxc(mxc);
-					error!(?mxc, ?key, "Failed to remove media from database: {e}");
-				}
 			}
 
 			Ok(())
 		} else {
-			Err!(Database(error!(
+			Err!(Database(error!("Failed to find any media keys for MXC {mxc} in our database.")))
-				"Failed to find any media keys for MXC {mxc:?} in our database."
-			)))
 		}
 	}
 
 	/// Deletes all media by the specified user
 	///
 	/// currently, this is only practical for local users
-	pub async fn delete_from_user(&self, user: &UserId, force: bool) -> Result<usize> {
+	pub async fn delete_from_user(&self, user: &UserId) -> Result<usize> {
 		let mxcs = self.db.get_all_user_mxcs(user);
 		let mut deletion_count: usize = 0;
 
 		for mxc in mxcs {
-			let mxc: Mxc<'_> = mxc.as_str().try_into()?;
+			let Ok(mxc) = mxc.as_str().try_into().inspect_err(|e| {
-			debug_info!("Deleting MXC {mxc} by user {user} from database and filesystem");
+				debug_error!(?mxc, "Failed to parse MXC URI from database: {e}");
-			if force {
+			}) else {
-				_ = self
+				continue;
-					.delete(&mxc)
+			};
-					.await
-					.inspect_err(|e| warn!("Failed to delete {mxc} from user {user}, ignoring error: {e}"));
-			} else {
-				self.delete(&mxc).await?;
-			}
 
-			deletion_count = deletion_count.saturating_add(1);
+			debug_info!(%deletion_count, "Deleting MXC {mxc} by user {user} from database and filesystem");
+			match self.delete(&mxc).await {
+				Ok(()) => {
+					deletion_count = deletion_count.saturating_add(1);
+				},
+				Err(e) => {
+					debug_error!(%deletion_count, "Failed to delete {mxc} from user {user}, ignoring error: {e}");
+				},
+			}
 		}
 
 		Ok(deletion_count)
@@ -176,9 +177,6 @@ impl Service {
 		for key in all_keys {
 			trace!("Full MXC key from database: {key:?}");
 
-			// we need to get the MXC URL from the first part of the key (the first 0xff /
-			// 255 push). this is all necessary because of conduit using magic keys for
-			// media
 			let mut parts = key.split(|&b| b == 0xFF);
 			let mxc = parts
 				.next()
@@ -189,31 +187,33 @@ impl Service {
 				.transpose()?;
 
 			let Some(mxc_s) = mxc else {
-				return Err!(Database("Parsed MXC URL unicode bytes from database but still is None"));
+				debug_warn!(?mxc, "Parsed MXC URL unicode bytes from database but is still invalid");
+				continue;
 			};
 
 			trace!("Parsed MXC key to URL: {mxc_s}");
 			let mxc = OwnedMxcUri::from(mxc_s);
 
-			mxcs.push(mxc);
+			if mxc.is_valid() {
+				mxcs.push(mxc);
+			} else {
+				debug_warn!("{mxc:?} from database was found to not be valid");
+			}
 		}
 
 		Ok(mxcs)
 	}
 
 	/// Deletes all remote only media files in the given at or after
-	/// time/duration. Returns a u32 with the amount of media files deleted.
+	/// time/duration. Returns a usize with the amount of media files deleted.
-	pub async fn delete_all_remote_media_at_after_time(&self, time: SystemTime, force: bool) -> Result<usize> {
+	pub async fn delete_all_remote_media_at_after_time(
+		&self, time: SystemTime, before: bool, after: bool, yes_i_want_to_delete_local_media: bool,
+	) -> Result<usize> {
 		let all_keys = self.db.get_all_media_keys();
-
 		let mut remote_mxcs = Vec::with_capacity(all_keys.len());
 
 		for key in all_keys {
 			trace!("Full MXC key from database: {key:?}");
-
-			// we need to get the MXC URL from the first part of the key (the first 0xff /
-			// 255 push). this is all necessary because of conduit using magic keys for
-			// media
 			let mut parts = key.split(|&b| b == 0xFF);
 			let mxc = parts
 				.next()
@@ -224,35 +224,30 @@ impl Service {
 				.transpose()?;
 
 			let Some(mxc_s) = mxc else {
-				return Err!(Database("Parsed MXC URL unicode bytes from database but still is None"));
+				debug_warn!(?mxc, "Parsed MXC URL unicode bytes from database but is still invalid");
+				continue;
 			};
 
 			trace!("Parsed MXC key to URL: {mxc_s}");
 			let mxc = OwnedMxcUri::from(mxc_s);
-			if mxc.server_name() == Ok(self.services.globals.server_name()) {
+			if (mxc.server_name() == Ok(self.services.globals.server_name()) && !yes_i_want_to_delete_local_media)
-				debug!("Ignoring local media MXC: {mxc}");
+				|| !mxc.is_valid()
-				// ignore our own MXC URLs as this would be local media.
+			{
+				debug!("Ignoring local or broken media MXC: {mxc}");
 				continue;
 			}
 
 			let path = self.get_media_file(&key);
-			debug!("MXC path: {path:?}");
 
 			let file_metadata = match fs::metadata(path.clone()).await {
 				Ok(file_metadata) => file_metadata,
 				Err(e) => {
-					if force {
+					error!("Failed to obtain file metadata for MXC {mxc} at file path \"{path:?}\", skipping: {e}");
-						error!("Failed to obtain file metadata for MXC {mxc} at file path \"{path:?}\", skipping: {e}");
+					continue;
-						continue;
-					}
-
-					return Err!(Database(
-						"Failed to obtain file metadata for MXC {mxc} at file path \"{path:?}\": {e}"
-					));
 				},
 			};
 
-			debug!("File metadata: {file_metadata:?}");
+			trace!(%mxc, ?path, "File metadata: {file_metadata:?}");
 
 			let file_created_at = match file_metadata.created() {
 				Ok(value) => value,
@@ -261,33 +256,36 @@ impl Service {
 					file_metadata.modified()?
 				},
 				Err(err) => {
-					if force {
+					error!("Could not delete MXC {mxc} at path {path:?}: {err:?}. Skipping...");
-						error!("Could not delete MXC {mxc} at path {path:?}: {err:?}. Skipping...");
+					continue;
-						continue;
-					}
-
-					return Err(err.into());
 				},
 			};
 
 			debug!("File created at: {file_created_at:?}");
-			if file_created_at <= time {
+
-				debug!("File is within user duration, pushing to list of file paths and keys to delete.");
+			if file_created_at >= time && before {
+				debug!("File is within (before) user duration, pushing to list of file paths and keys to delete.");
+				remote_mxcs.push(mxc.to_string());
+			} else if file_created_at <= time && after {
+				debug!("File is not within (after) user duration, pushing to list of file paths and keys to delete.");
 				remote_mxcs.push(mxc.to_string());
 			}
 		}
 
-		debug!(
-			"Finished going through all our media in database for eligible keys to delete, checking if these are empty"
-		);
 		if remote_mxcs.is_empty() {
 			return Err!(Database("Did not found any eligible MXCs to delete."));
 		}
 
-		debug_info!("Deleting media now in the past {time:?}.");
+		debug_info!("Deleting media now in the past {time:?}");
+
 		let mut deletion_count: usize = 0;
+
 		for mxc in remote_mxcs {
-			let mxc: Mxc<'_> = mxc.as_str().try_into()?;
+			let Ok(mxc) = mxc.as_str().try_into() else {
+				debug_warn!("Invalid MXC in database, skipping");
+				continue;
+			};
+
 			debug_info!("Deleting MXC {mxc} from database and filesystem");
 
 			match self.delete(&mxc).await {
@@ -295,12 +293,8 @@ impl Service {
 					deletion_count = deletion_count.saturating_add(1);
 				},
 				Err(e) => {
-					if force {
+					warn!("Failed to delete {mxc}, ignoring error and skipping: {e}");
-						warn!("Failed to delete {mxc}, ignoring error and skipping: {e}");
+					continue;
-						continue;
-					}
-
-					return Err!(Database(warn!("Failed to delete MXC {mxc}: {e}")));
 				},
 			}
 		}

src/service/rooms/short/data.rs

@@ -63,6 +63,7 @@ impl Data {
 			.iter()
 			.enumerate()
 		{
+			#[allow(clippy::single_match_else)]
 			match short {
 				Some(short) => ret.push(
 					utils::u64_from_bytes(short).map_err(|_| Error::bad_database("Invalid shorteventid in db."))?,

src/service/sending/sender.rs

@@ -479,6 +479,8 @@ impl Service {
 						.collect::<Vec<_>>(),
 				)))
 					.into(),
+				ephemeral: Vec::new(),
+				to_device: Vec::new(),
 			},
 		)
 		.await
@@ -636,7 +638,9 @@ impl Service {
 					.pdus
 					.iter()
 					.filter(|(_, res)| res.is_err())
-					.for_each(|(pdu_id, res)| warn!("error for {pdu_id} from remote: {res:?}"));
+					.for_each(
+						|(pdu_id, res)| warn!(%transaction_id, %server, "error sending PDU {pdu_id} to remote server: {res:?}"),
+					);
 			})
 			.map(|_| dest.clone())
 			.map_err(|e| (dest.clone(), e))

src/service/updates/mod.rs

@@ -1,13 +1,13 @@
 use std::{sync::Arc, time::Duration};
 
 use async_trait::async_trait;
-use conduit::{err, info, utils, warn, Error, Result};
+use conduit::{debug, err, info, utils, warn, Error, Result};
 use database::Map;
 use ruma::events::room::message::RoomMessageEventContent;
 use serde::Deserialize;
 use tokio::{sync::Notify, time::interval};
 
-use crate::{admin, client, Dep};
+use crate::{admin, client, globals, Dep};
 
 pub struct Service {
 	services: Services,
@@ -19,6 +19,7 @@ pub struct Service {
 struct Services {
 	admin: Dep<admin::Service>,
 	client: Dep<client::Service>,
+	globals: Dep<globals::Service>,
 }
 
 #[derive(Deserialize)]
@@ -42,6 +43,7 @@ impl crate::Service for Service {
 	fn build(args: crate::Args<'_>) -> Result<Arc<Self>> {
 		Ok(Arc::new(Self {
 			services: Services {
+				globals: args.depend::<globals::Service>("globals"),
 				admin: args.depend::<admin::Service>("admin"),
 				client: args.depend::<client::Service>("client"),
 			},
@@ -52,6 +54,10 @@ impl crate::Service for Service {
 	}
 
 	async fn worker(self: Arc<Self>) -> Result<()> {
+		if !self.services.globals.allow_check_for_updates() {
+			debug!("Disabling update check");
+			return Ok(());
+		}
 		let mut i = interval(self.interval);
 		loop {
 			tokio::select! {

src/service/users/data.rs

@@ -32,6 +32,7 @@ pub struct Data {
 	userid_password: Arc<Map>,
 	userid_selfsigningkeyid: Arc<Map>,
 	userid_usersigningkeyid: Arc<Map>,
+	useridprofilekey_value: Arc<Map>,
 	services: Services,
 }
 
@@ -64,6 +65,7 @@ impl Data {
 			userid_password: db["userid_password"].clone(),
 			userid_selfsigningkeyid: db["userid_selfsigningkeyid"].clone(),
 			userid_usersigningkeyid: db["userid_usersigningkeyid"].clone(),
+			useridprofilekey_value: db["useridprofilekey_value"].clone(),
 			services: Services {
 				server: args.server.clone(),
 				globals: args.depend::<globals::Service>("globals"),
@@ -231,6 +233,111 @@ impl Data {
 			.transpose()
 	}
 
+	/// Gets a specific user profile key
+	pub(super) fn profile_key(&self, user_id: &UserId, profile_key: &str) -> Result<Option<serde_json::Value>> {
+		let mut key = user_id.as_bytes().to_vec();
+		key.push(0xFF);
+		key.extend_from_slice(profile_key.as_bytes());
+
+		self.useridprofilekey_value
+			.get(&key)?
+			.map_or(Ok(None), |bytes| Ok(Some(serde_json::from_slice(&bytes).unwrap())))
+	}
+
+	/// Gets all the user's profile keys and values in an iterator
+	pub(super) fn all_profile_keys<'a>(
+		&'a self, user_id: &UserId,
+	) -> Box<dyn Iterator<Item = Result<(String, serde_json::Value)>> + 'a + Send> {
+		let prefix = user_id.as_bytes().to_vec();
+
+		Box::new(
+			self.useridprofilekey_value
+				.scan_prefix(prefix)
+				.map(|(key, value)| {
+					let profile_key_name = utils::string_from_bytes(
+						key.rsplit(|&b| b == 0xFF)
+							.next()
+							.ok_or_else(|| err!(Database("Profile key in db is invalid")))?,
+					)
+					.map_err(|e| err!(Database("Profile key in db is invalid. {e}")))?;
+
+					let profile_key_value = serde_json::from_slice(&value)
+						.map_err(|e| err!(Database("Profile key in db is invalid. {e}")))?;
+
+					Ok((profile_key_name, profile_key_value))
+				}),
+		)
+	}
+
+	/// Sets a new profile key value, removes the key if value is None
+	pub(super) fn set_profile_key(
+		&self, user_id: &UserId, profile_key: &str, profile_key_value: Option<serde_json::Value>,
+	) -> Result<()> {
+		let mut key = user_id.as_bytes().to_vec();
+		key.push(0xFF);
+		key.extend_from_slice(profile_key.as_bytes());
+
+		// TODO: insert to the stable MSC4175 key when it's stable
+		if let Some(value) = profile_key_value {
+			let value = serde_json::to_vec(&value).unwrap();
+
+			self.useridprofilekey_value.insert(&key, &value)
+		} else {
+			self.useridprofilekey_value.remove(&key)
+		}
+	}
+
+	/// Get the timezone of a user.
+	pub(super) fn timezone(&self, user_id: &UserId) -> Result<Option<String>> {
+		// first check the unstable prefix
+		let mut key = user_id.as_bytes().to_vec();
+		key.push(0xFF);
+		key.extend_from_slice(b"us.cloke.msc4175.tz");
+
+		let value = self
+			.useridprofilekey_value
+			.get(&key)?
+			.map(|bytes| utils::string_from_bytes(&bytes).map_err(|e| err!(Database("Timezone in db is invalid. {e}"))))
+			.transpose()
+			.unwrap();
+
+		// TODO: transparently migrate unstable key usage to the stable key once MSC4133
+		// and MSC4175 are stable, likely a remove/insert in this block
+		if value.is_none() || value.as_ref().is_some_and(String::is_empty) {
+			// check the stable prefix
+			let mut key = user_id.as_bytes().to_vec();
+			key.push(0xFF);
+			key.extend_from_slice(b"m.tz");
+
+			return self
+				.useridprofilekey_value
+				.get(&key)?
+				.map(|bytes| {
+					utils::string_from_bytes(&bytes).map_err(|e| err!(Database("Timezone in db is invalid. {e}")))
+				})
+				.transpose();
+		}
+
+		Ok(value)
+	}
+
+	/// Sets a new timezone or removes it if timezone is None.
+	pub(super) fn set_timezone(&self, user_id: &UserId, timezone: Option<String>) -> Result<()> {
+		let mut key = user_id.as_bytes().to_vec();
+		key.push(0xFF);
+		key.extend_from_slice(b"us.cloke.msc4175.tz");
+
+		// TODO: insert to the stable MSC4175 key when it's stable
+		if let Some(timezone) = timezone {
+			self.useridprofilekey_value
+				.insert(&key, timezone.as_bytes())?;
+		} else {
+			self.useridprofilekey_value.remove(&key)?;
+		}
+
+		Ok(())
+	}
+
 	/// Sets a new avatar_url or removes it if avatar_url is None.
 	pub(super) fn set_blurhash(&self, user_id: &UserId, blurhash: Option<String>) -> Result<()> {
 		if let Some(blurhash) = blurhash {

src/service/users/mod.rs

@@ -327,6 +327,33 @@ impl Service {
 	/// Get the blurhash of a user.
 	pub fn blurhash(&self, user_id: &UserId) -> Result<Option<String>> { self.db.blurhash(user_id) }
 
+	pub fn timezone(&self, user_id: &UserId) -> Result<Option<String>> { self.db.timezone(user_id) }
+
+	/// Gets a specific user profile key
+	pub fn profile_key(&self, user_id: &UserId, profile_key: &str) -> Result<Option<serde_json::Value>> {
+		self.db.profile_key(user_id, profile_key)
+	}
+
+	/// Gets all the user's profile keys and values in an iterator
+	pub fn all_profile_keys<'a>(
+		&'a self, user_id: &UserId,
+	) -> Box<dyn Iterator<Item = Result<(String, serde_json::Value)>> + 'a + Send> {
+		self.db.all_profile_keys(user_id)
+	}
+
+	/// Sets a new profile key value, removes the key if value is None
+	pub fn set_profile_key(
+		&self, user_id: &UserId, profile_key: &str, profile_key_value: Option<serde_json::Value>,
+	) -> Result<()> {
+		self.db
+			.set_profile_key(user_id, profile_key, profile_key_value)
+	}
+
+	/// Sets a new tz or removes it if tz is None.
+	pub async fn set_timezone(&self, user_id: &UserId, tz: Option<String>) -> Result<()> {
+		self.db.set_timezone(user_id, tz)
+	}
+
 	/// Sets a new blurhash or removes it if blurhash is None.
 	pub async fn set_blurhash(&self, user_id: &UserId, blurhash: Option<String>) -> Result<()> {
 		self.db.set_blurhash(user_id, blurhash)

tests/test_results/complement/test_results.jsonl

@@ -64,6 +64,7 @@
 {"Action":"pass","Test":"TestFederationRoomsInvite/Parallel/Invited_user_can_reject_invite_over_federation_several_times"}
 {"Action":"pass","Test":"TestFederationRoomsInvite/Parallel/Invited_user_has_'is_direct'_flag_in_prev_content_after_joining"}
 {"Action":"pass","Test":"TestFederationRoomsInvite/Parallel/Remote_invited_user_can_see_room_metadata"}
+{"Action":"pass","Test":"TestFederationThumbnail"}
 {"Action":"fail","Test":"TestGetMissingEventsGapFilling"}
 {"Action":"fail","Test":"TestInboundCanReturnMissingEvents"}
 {"Action":"fail","Test":"TestInboundCanReturnMissingEvents/Inbound_federation_can_return_missing_events_for_invited_visibility"}