Compare commits

..
Author SHA1 Message Date
timedout beb78fa5d4 chore: Add news fragment 2026-11-21 16:48:46 +00:00
timedout de9e9b28e2 fix: Modernise knocking helpers 2026-11-21 16:36:18 +00:00
49 changed files with 1273 additions and 774 deletions

No files matched your search

+6 -14
View File
@@ -49,8 +49,6 @@ jobs:
run: npm ci
- name: Build Rspress documentation
env:
SITE_PATH: ${{ github.ref != 'refs/heads/main' && format('/pr/{0}/', '${{ forgejo.event.number }}') || '/' }}
run: npm run docs:build
- name: Install Wrangler
@@ -64,16 +62,10 @@ jobs:
apiToken: ${{ secrets.CLOUDFLARE_API_TOKEN }}
command: pages deploy ./doc_build --branch="main" --commit-dirty=true --project-name="${{ vars.CLOUDFLARE_PROJECT_NAME }}"
- name: Deploy to Grebedoc (Preview)
if: github.ref != 'refs/heads/main'
uses: actions/git-pages@v2
- name: Deploy to Cloudflare Pages (Preview)
if: github.ref != 'refs/heads/main' && vars.CLOUDFLARE_PROJECT_NAME != ''
uses: https://github.com/cloudflare/wrangler-action@953926a2e2182532811c01a25e53647d93bf07c0 # v4
with:
site: https://preview.continuwuity.org
path: "pr/${{ forgejo.event.number }}"
server: grebedoc.dev
token: ${{ forgejo.token }}
source: doc_build/
strict: true
pretty-urls: true
expires: 14
comment: ${{ forgejo.sha }}
accountId: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }}
apiToken: ${{ secrets.CLOUDFLARE_API_TOKEN }}
command: pages deploy ./doc_build --branch="${{ github.head_ref || github.ref_name }}" --commit-dirty=true --project-name="${{ vars.CLOUDFLARE_PROJECT_NAME }}"
Generated
+29 -18
View File
@@ -497,7 +497,7 @@ dependencies = [
"regex",
"rustc-hash",
"shlex",
"syn 2.0.119",
"syn 3.0.10",
]
[[package]]
@@ -671,7 +671,7 @@ checksum = "82f4b26e751e711a5302649417f2da046dce6391b2ea30a4820f37462314f0b9"
dependencies = [
"semver",
"serde",
"toml 1.2.2+spec-1.1.0",
"toml 1.2.1+spec-1.1.0",
]
[[package]]
@@ -759,14 +759,14 @@ checksum = "157a8ba7b480713b56f4c09fd13fc3e0a22a5dfab8097ba61cbc5feef950788a"
dependencies = [
"glob",
"libc",
"libloading",
"libloading 0.8.9",
]
[[package]]
name = "clap"
version = "4.6.9"
version = "4.6.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "844c5c0b65be192cb68d570cca1b21a6444431d24e39c084dc0336177015337c"
checksum = "ac230478587d0269d8d6998b82ae69ea51509d55df66d9e92edd1f986904cd4a"
dependencies = [
"clap_builder",
"clap_derive",
@@ -784,9 +784,9 @@ dependencies = [
[[package]]
name = "clap_derive"
version = "4.6.9"
version = "4.6.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b4c174f11f949b9ca8c60580925e282b9cbd5454750e506f08648b2038080cd7"
checksum = "895c9d8d2ac2dd8b8a29f3724a14e72aaaa6debcf8c38c82f97305c6a172767a"
dependencies = [
"heck",
"proc-macro2",
@@ -997,6 +997,7 @@ dependencies = [
"ipaddress",
"lettre",
"libc",
"libloading 0.9.1",
"libmimalloc-sys",
"lock_api 0.5.0",
"maplit",
@@ -1029,7 +1030,7 @@ dependencies = [
"tikv-jemallocator",
"tokio",
"tokio-metrics",
"toml 1.2.2+spec-1.1.0",
"toml 1.2.1+spec-1.1.0",
"tracing",
"tracing-core",
"tracing-subscriber",
@@ -2104,9 +2105,9 @@ checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb"
[[package]]
name = "form_urlencoded"
version = "1.2.3"
version = "1.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "afa489199eaffb1211795df1c6236ce1ad0d41a6fa45250add73e3b02420f06e"
checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf"
dependencies = [
"percent-encoding",
]
@@ -3240,6 +3241,16 @@ dependencies = [
"windows-link",
]
[[package]]
name = "libloading"
version = "0.9.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a1832accd5a4ca2795ace4d05d15c4039be15b62d3b950eff99746a990e3cc10"
dependencies = [
"cfg-if",
"windows-link",
]
[[package]]
name = "libm"
version = "0.2.16"
@@ -5135,7 +5146,7 @@ dependencies = [
"ruma-identifiers-validation",
"serde",
"syn 3.0.10",
"toml 1.2.2+spec-1.1.0",
"toml 1.2.1+spec-1.1.0",
]
[[package]]
@@ -5641,7 +5652,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "969cde0349344dbbf58b5f9badbfb8f806db13969364eb8007c057ef687007d9"
dependencies = [
"annotate-snippets",
"base64 0.23.1",
"base64 0.22.1",
"encoding_rs_io",
"granit-parser",
"nohash-hasher",
@@ -6395,9 +6406,9 @@ dependencies = [
[[package]]
name = "toml"
version = "1.2.2+spec-1.1.0"
version = "1.2.1+spec-1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5426cb1b964690d913f6c5c1a1ae0b42f428ec506bb4d452876d7a209c9bca3b"
checksum = "d5755c51863cdac711f1956baf8eb9588e060755f7d3c4c979e70ea1e1f32cd7"
dependencies = [
"indexmap 2.14.2",
"serde_core",
@@ -6454,9 +6465,9 @@ dependencies = [
[[package]]
name = "toml_parser"
version = "1.1.6+spec-1.1.0"
version = "1.1.5+spec-1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e6769c452d71ba30696c6b37770c439a9cc9110f47f7d5cdbeda79840ef5ea31"
checksum = "baa693a8032d7e1cada7d0041e96126df243179ff061456783ac7f12bda4744c"
dependencies = [
"winnow 1.0.5",
]
@@ -6853,9 +6864,9 @@ checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1"
[[package]]
name = "url"
version = "2.5.9"
version = "2.5.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f00cd2407bb42c0ef17be88d4cf020f5cceffd121665bac9b8b9493a369b3b1a"
checksum = "ff67a8a4397373c3ef660812acab3268222035010ab8680ec4215f38ba3d0eed"
dependencies = [
"form_urlencoded",
"idna",
+54
View File
@@ -269,6 +269,9 @@ features = [
[workspace.dependencies.tokio-metrics]
version = "0.5.2"
[workspace.dependencies.libloading]
version = "0.9.0"
[workspace.dependencies.wildmatch]
version = "2.7.0"
@@ -750,23 +753,74 @@ inherits = "release"
# Developer profile
#
# To enable hot-reloading:
# 1. Uncomment all of the rustflags here.
# 2. Uncomment crate-type=dylib in src/*/Cargo.toml
#
# opt-level, mir-opt-level, validate-mir are not known to interfere with reloading
# and can be raised if build times are tolerable.
[profile.dev]
debug = "full"
opt-level = 0
panic = "unwind"
debug-assertions = true
incremental = true
#rustflags = [
# '--cfg', 'conduwuit_mods',
# '-Ztime-passes',
# '-Zmir-opt-level=0',
# '-Zvalidate-mir=false',
# '-Ztls-model=global-dynamic',
# '-Cprefer-dynamic=true',
# '-Zstaticlib-prefer-dynamic=true',
# '-Zstaticlib-allow-rdylib-deps=true',
# '-Zpacked-bundled-libs=false',
# '-Zplt=true',
# '-Crpath=true',
# '-Clink-arg=-Wl,--as-needed',
# '-Clink-arg=-Wl,--allow-shlib-undefined',
# '-Clink-arg=-Wl,-z,keep-text-section-prefix',
# '-Clink-arg=-Wl,-z,lazy',
#]
[profile.dev.package.conduwuit_core]
inherits = "dev"
[profile.dev.package.conduwuit]
inherits = "dev"
#rustflags = [
# '--cfg', 'conduwuit_mods',
# '-Ztime-passes',
# '-Zmir-opt-level=0',
# '-Zvalidate-mir=false',
# '-Ztls-model=global-dynamic',
# '-Cprefer-dynamic=true',
# '-Zexport-executable-symbols=true',
# '-Zplt=true',
# '-Crpath=true',
# '-Clink-arg=-Wl,--as-needed',
# '-Clink-arg=-Wl,--allow-shlib-undefined',
# '-Clink-arg=-Wl,--export-dynamic',
# '-Clink-arg=-Wl,-z,lazy',
#]
[profile.dev.package.'*']
inherits = "dev"
debug = 'limited'
codegen-units = 1
opt-level = 'z'
#rustflags = [
# '--cfg', 'conduwuit_mods',
# '-Ztls-model=global-dynamic',
# '-Cprefer-dynamic=true',
# '-Zstaticlib-prefer-dynamic=true',
# '-Zstaticlib-allow-rdylib-deps=true',
# '-Zpacked-bundled-libs=true',
# '-Zplt=true',
# '-Clink-arg=-Wl,--as-needed',
# '-Clink-arg=-Wl,-z,lazy',
# '-Clink-arg=-Wl,-z,nodelete',
#]
# primarily used for CI
[profile.test]
+1
View File
@@ -0,0 +1 @@
Modernised the room knocking helper functions to make them match the behaviour of our room join helpers, which are safer and more efficient. This also fixes a bug that caused an error when knocking on rooms not local to the server. Contributed by @nex.
+5
View File
@@ -18,5 +18,10 @@
"type": "file",
"name": "testing",
"label": "Testing"
},
{
"type": "file",
"name": "hot_reload",
"label": "Hot Reloading"
}
]
+200
View File
@@ -0,0 +1,200 @@
# Hot Reloading ("Live" Development)
Note that hot reloading has not been refactored in quite a while and is not
guaranteed to work at this time.
### Summary
When developing in debug-builds with the nightly toolchain, Continuwuity is modular
using dynamic libraries and various parts of the application are hot-reloadable
while the server is running: http api handlers, admin commands, services,
database, etc. These are all split up into individual workspace crates as seen
in the `src/` directory. Changes to sourcecode in a crate rebuild that crate and
subsequent crates depending on it. Reloading then occurs for the changed crates.
Release builds still produce static binaries which are unaffected. Rust's
soundness guarantees are in full force. Thus you cannot hot-reload release
binaries.
### Requirements
Currently, this development setup only works on x86_64 and aarch64 Linux glibc.
[musl explicitly does not support hot reloadable libraries, and does not
implement `dlclose`][2]. macOS does not fully support our usage of `RTLD_GLOBAL`
possibly due to some thread-local issues. [This Rust issue][3] may be of
relevance, specifically [this comment][4]. It may be possible to get it working
on only very modern macOS versions such as at least Sonoma, as currently loading
dylibs is supported, but not unloading them in our setup, and the cited comment
mentions an Apple WWDC confirming there have been TLS changes to somewhat make
this possible.
As mentioned above this requires the nightly toolchain. This is due to reliance
on various Cargo.toml features that are only available on nightly, most
specifically `RUSTFLAGS` in Cargo.toml. Some of the implementation could also be
simpler based on other various nightly features. We hope lots of nightly
features start making it out of nightly sooner as there have been dozens of very
helpful features that have been stuck in nightly ("unstable") for at least 5+
years that would make this simpler. We encourage greater community consensus to
move these features into stability.
This currently only works on x86_64/aarch64 Linux with a glibc C library. musl C
library, macOS, and likely other host architectures are not supported (if other
architectures work, feel free to let us know and/or make a PR updating this).
This should work on GNU ld and lld (rust-lld) and gcc/clang, however if you
happen to have linker issues it's recommended to try using `mold` or `gold`
linkers, and please let us know in the [Continuwuity Matrix room][7] the linker
error and what linker solved this issue so we can figure out a solution. Ideally
there should be minimal friction to using this, and in the future a build script
(`build.rs`) may be suitable to making this easier to use if the capabilities
allow us.
### Usage
As of 19 May 2024, the instructions for using this are:
0. Have patience. Don't hesitate to join the [Continuwuity Matrix room][7] to
receive help using this. As indicated by the various rustflags used and some
of the interesting issues linked at the bottom, this is definitely not something
the Rust ecosystem or toolchain is used to doing.
1. Install the nightly toolchain using rustup. You may need to use `rustup
override set nightly` in your local Continuwuity directory, or use `cargo
+nightly` for all actions.
2. Uncomment `cargo-features` at the top level / root Cargo.toml
3. Scroll down to the `# Developer profile` section and uncomment ALL the
rustflags for each dev profile and their respective packages.
4. In each workspace crate's Cargo.toml (everything under `src/*` AND
`deps/rust-rocksdb/Cargo.toml`), uncomment the `dylib` crate type under
`[lib]`.
5. Due to [this rpath issue][5], you must export the `LD_LIBRARY_PATH`
environment variable to your nightly Rust toolchain library directory. If
using rustup (hopefully), use this: `export
LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$HOME/.rustup/toolchains/nightly-x86_64-unknown-linux-gnu/lib/`
6. Start the server. You can use `cargo +nightly run` for this along with the
standard.
7. Make some changes where you need to.
8. In a separate terminal window in the same directory (or using a terminal
multiplexer like tmux), run the *build* Cargo command `cargo +nightly build`.
Cargo should only rebuild what was changed / what's necessary, so it should
not be rebuilding all the crates.
9. In your Continuwuity server terminal, hit/send `CTRL+C` signal. This will tell
Continuwuity to find which libraries need to be reloaded, and reloads them as
necessary.
10. If there were no errors, it will tell you it successfully reloaded `#`
modules, and your changes should now be visible. Repeat 7 - 9 as needed.
To shutdown Continuwuity in this setup, hit/send `CTRL+\`. Normal builds still
shutdown with `CTRL+C` as usual.
Steps 1 - 5 are the initial first-time steps for using this. To remove the hot
reload setup, revert/comment all the Cargo.toml changes.
As mentioned in the requirements section, if you happen to have some linker
issues, try using the `-fuse-ld=` rustflag and specify mold or gold in all the
`rustflags` definitions in the top level Cargo.toml, and please let us know in
the [Continuwuity Matrix room][7] the problem. mold can be installed typically
through your distro, and gold is provided by the binutils package.
It's possible a helper script can be made to do all of this, or most preferably
a specially made build script (build.rs). `cargo watch` support will be
implemented soon which will eliminate the need to manually run `cargo build` all
together.
### Addendum
Conduit was inherited as a single crate without modularity or reloading in its
design. Reasonable partitioning and abstraction allowed a split into several
crates, though many circular dependencies had to be corrected. The resulting
crates now form a directed graph as depicted in figures below. The interfacing
between these crates is still extremely broad which is not mitigable.
Initially [hot_lib_reload][6] was investigated but found appropriate for a
project designed with modularity through limited interfaces, not a large and
complex existing codebase. Instead a bespoke solution built directly on
[libloading][8] satisfied our constraints. This required relatively minimal
modifications and zero maintenance burden compared to what would be required
otherwise. The technical difference lies with relocation processing: we leverage
global bindings (`RTLD_GLOBAL`) in a very intentional way. Most libraries and
off-the-shelf module systems (such as [hot_lib_reload][6]) restrict themselves
to local bindings (`RTLD_LOCAL`). This allows them to release software to
multiple platforms with much greater consistency, but at the cost of burdening
applications to explicitly manage these bindings. In our case with an optional
feature for developers, we shrug any such requirement to enjoy the cost/benefit
on platforms where global relocations are properly cooperative.
To make use of `RTLD_GLOBAL` the application has to be oriented as a directed
acyclic graph. The primary rule is simple and illustrated in the figure below:
**no crate is allowed to call a function or use a variable from a crate below
it.**
![Continuwuity's dynamic library setup diagram - created by Jason
Volk](./assets/libraries.png)
When a symbol is referenced between crates they become bound: **crates cannot be
unloaded until their calling crates are first unloaded.** Thus we start the
reloading process from the crate which has no callers. There is a small problem
though: the first crate is called by the base executable itself! This is solved
by using an `RTLD_LOCAL` binding for just one link between the main executable
and the first crate, freeing the executable from all modules as no global
binding ever occurs between them.
![Continuwuity's reload and load order diagram - created by Jason
Volk](./assets/reload_order.png)
Proper resource management is essential for reliable reloading to occur. This is
a very basic ask in RAII-idiomatic Rust and the exposure to reloading hazards is
remarkably low, generally stemming from poor patterns and practices.
Unfortunately static analysis doesn't enforce reload-safety programmatically
(though it could one day), for now hazards can be avoided by knowing a few basic
do's and dont's:
1. Understand that code is memory. Just like one is forbidden from referencing
free'd memory, one must not transfer control to free'd code. Exposure to this
is primarily from two things:
- Callbacks, which this project makes very little use of.
- Async tasks, which are addressed below.
2. Tie all resources to a scope or object lifetime with greatest possible
symmetry (locality). For our purposes this applies to code resources, which
means async blocks and tokio tasks.
- **Never spawn a task without receiving and storing its JoinHandle**.
- **Always wait on join handles** before leaving a scope or in another cleanup
function called by an owning scope.
3. Know any minor specific quirks documented in code or here:
- Don't use `tokio::spawn`, instead use our `Handle` in `core/server.rs`, which
is reachable in most of the codebase via `services()` or other state. This is
due to some bugs or assumptions made in tokio, as it happens in `unsafe {}`
blocks, which are mitigated by circumventing some thread-local variables. Using
runtime handles is good practice in any case.
The initial implementation PR is available [here][1].
### Interesting related issues/bugs
- [DT_RUNPATH produced in binary with rpath = true is wrong (cargo)][5]
- [Disabling MIR Optimization in Rust Compilation
(cargo)](https://internals.rust-lang.org/t/disabling-mir-optimization-in-rust-compilation/19066/5)
- [Workspace-level metadata
(cargo-deb)](https://github.com/kornelski/cargo-deb/issues/68)
[1]: https://forgejo.ellis.link/continuwuation/continuwuity/pulls/387
[2]: https://wiki.musl-libc.org/functional-differences-from-glibc.html#Unloading-libraries
[3]: https://github.com/rust-lang/rust/issues/28794
[4]: https://github.com/rust-lang/rust/issues/28794#issuecomment-368693049
[5]: https://github.com/rust-lang/cargo/issues/12746
[6]: https://crates.io/crates/hot-lib-reloader/
[7]: https://matrix.to/#/#continuwuity:continuwuity.org?via=continuwuity.org&via=ellis.link&via=explodie.org&via=matrix.org
[8]: https://crates.io/crates/libloading
+7
View File
@@ -37,6 +37,13 @@ It is highly unlikely you will ever need to add a new workspace member, but if
you truly find yourself needing to, we recommend reaching out to us in the
Matrix room for discussions about it beforehand.
The primary inspiration for this design was apart of hot reloadable development,
to support "Continuwuity as a library" where specific parts can simply be
swapped out. There is evidence Conduit wanted to go this route too as `axum` is
technically an optional feature in Conduit, and can be compiled without the
binary or axum library for handling inbound web requests; but it was never
completed or worked.
See the Rust documentation on [Workspaces][workspaces] for general questions and
information on Cargo workspaces.
+4
View File
@@ -36,6 +36,10 @@ List database backups
Send a message to the admin room
## `!admin server reload`
Hot-reload the server
## `!admin server restart`
Restart the server
+38 -18
View File
@@ -11,7 +11,7 @@
"devDependencies": {
"@rspress/core": "^2.0.0",
"@rspress/plugin-client-redirects": "^2.0.0",
"@types/node": "^26.6.6",
"@rspress/plugin-sitemap": "^2.0.0",
"typescript": "^7.0.0"
}
},
@@ -230,6 +230,9 @@
"arm64"
],
"dev": true,
"libc": [
"glibc"
],
"license": "MIT",
"optional": true,
"os": [
@@ -244,6 +247,9 @@
"arm64"
],
"dev": true,
"libc": [
"musl"
],
"license": "MIT",
"optional": true,
"os": [
@@ -258,6 +264,9 @@
"ppc64"
],
"dev": true,
"libc": [
"glibc"
],
"license": "MIT",
"optional": true,
"os": [
@@ -272,6 +281,9 @@
"riscv64"
],
"dev": true,
"libc": [
"glibc"
],
"license": "MIT",
"optional": true,
"os": [
@@ -286,6 +298,9 @@
"riscv64"
],
"dev": true,
"libc": [
"musl"
],
"license": "MIT",
"optional": true,
"os": [
@@ -300,6 +315,9 @@
"s390x"
],
"dev": true,
"libc": [
"glibc"
],
"license": "MIT",
"optional": true,
"os": [
@@ -314,6 +332,9 @@
"x64"
],
"dev": true,
"libc": [
"glibc"
],
"license": "MIT",
"optional": true,
"os": [
@@ -328,6 +349,9 @@
"x64"
],
"dev": true,
"libc": [
"musl"
],
"license": "MIT",
"optional": true,
"os": [
@@ -500,6 +524,19 @@
"@rspress/core": "^2.1.0-beta.2"
}
},
"node_modules/@rspress/plugin-sitemap": {
"version": "2.1.4",
"resolved": "https://registry.npmjs.org/@rspress/plugin-sitemap/-/plugin-sitemap-2.1.4.tgz",
"integrity": "sha512-TObDTXgBKFAHEWEth47aehQSYtFfpIkqM4w14a0hUbPLkbu2AZ2cKnMzFTKK+W6I/3LN54scvfGYTL5CPQsDXw==",
"dev": true,
"license": "MIT",
"engines": {
"node": "^20.19.0 || >=22.12.0"
},
"peerDependencies": {
"@rspress/core": "^2.1.0-beta.2"
}
},
"node_modules/@rspress/shared": {
"version": "2.1.4",
"resolved": "https://registry.npmjs.org/@rspress/shared/-/shared-2.1.4.tgz",
@@ -733,16 +770,6 @@
"dev": true,
"license": "MIT"
},
"node_modules/@types/node": {
"version": "26.6.6",
"resolved": "https://registry.npmjs.org/@types/node/-/node-26.6.6.tgz",
"integrity": "sha512-MzQj+8Kx3XI40OmUAE6Qoh58ZMNstNNMwn2nfd3qWsfe7nDNSqJNzes4njhz6AHwrSHOffSLwXU6yb474Q4aog==",
"dev": true,
"license": "MIT",
"dependencies": {
"undici-types": "~8.9.0"
}
},
"node_modules/@types/react": {
"version": "19.3.0",
"resolved": "https://registry.npmjs.org/@types/react/-/react-19.3.0.tgz",
@@ -3815,13 +3842,6 @@
"@typescript/typescript-win32-x64": "7.0.2"
}
},
"node_modules/undici-types": {
"version": "8.9.0",
"resolved": "https://registry.npmjs.org/undici-types/-/undici-types-8.9.0.tgz",
"integrity": "sha512-KTDyRTYX8sWmKXAikPHHSyc63CRPETMctyjKFupcC6OBLXT3xsN0e9aF7m+mIXutFWpUXuedtowG7iLOzp0kQg==",
"dev": true,
"license": "MIT"
},
"node_modules/unhead": {
"version": "2.1.17",
"resolved": "https://registry.npmjs.org/unhead/-/unhead-2.1.17.tgz",
+2 -2
View File
@@ -20,11 +20,11 @@
"keywords": [],
"author": "",
"license": "ISC",
"type": "module",
"type": "commonjs",
"devDependencies": {
"@rspress/core": "^2.0.0",
"@rspress/plugin-client-redirects": "^2.0.0",
"@types/node": "^26.6.6",
"@rspress/plugin-sitemap": "^2.0.0",
"typescript": "^7.0.0"
}
}
+9 -8
View File
@@ -1,13 +1,10 @@
import { defineConfig } from '@rspress/core';
import { pluginSitemap } from '@rspress/plugin-sitemap';
import { pluginClientRedirects } from '@rspress/plugin-client-redirects';
const sitePath: string = process.env.SITE_PATH ?? '/';
export default defineConfig({
root: 'docs',
title: 'Continuwuity',
siteOrigin: sitePath != "/" ? "https://preview.continuwuity.org" : "https://continuwuity.org",
base: sitePath,
description: 'A community-driven Matrix homeserver',
icon: '/assets/logo.svg',
logo: {
@@ -78,10 +75,14 @@ export default defineConfig({
lastUpdated: true,
enableContentAnimation: true,
enableAppearanceAnimation: false,
footer: {},
footer: {
},
},
plugins: [pluginClientRedirects({
plugins: [pluginSitemap({
siteUrl: 'https://continuwuity.org', // TODO: Set automatically in build pipeline
}),
pluginClientRedirects({
redirects: [{
from: '^/config_reference$',
to: '/reference/config'
@@ -104,8 +105,8 @@ export default defineConfig({
from: "^/source(|_code)$",
to: "https://forgejo.ellis.link/continuwuation/continuwuity",
}, {
from: "^/(CONTRIBUTING.md|contributing)$",
to: "/development/contributing"
from: "^/(CONTRIBUTING.md|contributing)$",
to: "/development/contributing"
}
]
})],
+3
View File
@@ -34,6 +34,9 @@ pub(crate) use crate::{context::Context, utils::get_room_info};
pub(crate) const PAGE_SIZE: usize = 100;
conduwuit::mod_ctor! {}
conduwuit::mod_dtor! {}
pub use crate::admin::AdminCommand;
/// Install the admin command processor
+8
View File
@@ -110,6 +110,14 @@ impl crate::Context<'_> {
self.write_str("Notice was sent to #admins").await
}
pub(super) async fn reload_mods(&self) -> Result {
self.bail_restricted()?;
self.services.server.reload()?;
self.write_str("Reloading server...").await
}
#[cfg(unix)]
pub(super) async fn restart(&self, force: bool) -> Result {
use conduwuit::utils::sys::current_exe_deleted;
+4
View File
@@ -39,6 +39,10 @@ pub enum ServerCommand {
message: Vec<String>,
},
/// Hot-reload the server
#[command(name = "reload", alias = "reload-mods")]
ReloadMods,
#[cfg(unix)]
/// Restart the server
Restart {
+221 -416
View File
@@ -1,9 +1,10 @@
use std::{borrow::Borrow, collections::HashMap, sync::Arc};
use std::collections::{HashMap, hash_map::Entry};
use axum::extract::State;
use conduwuit::{
Err, Result, debug, debug_info, debug_warn, err, info,
Err, EventTypeExt, Result, debug, debug_warn, err, info,
matrix::{
StateKey,
event::gen_event_id,
pdu::{PartialPdu, PduEvent},
},
@@ -15,24 +16,23 @@ use conduwuit::{
use futures::{FutureExt, StreamExt};
use ruma::{
CanonicalJsonObject, CanonicalJsonValue, OwnedEventId, OwnedRoomId, OwnedServerName,
OwnedUserId, RoomId, UserId,
api::{client::knock::knock_room, federation},
OwnedUserId, RoomId, RoomVersionId, UserId,
api::{
client::knock::knock_room,
error::{ErrorKind, IncompatibleRoomVersionErrorData},
federation,
federation::membership::RawStrippedState,
},
canonical_json::to_canonical_value,
events::{
StateEventType,
room::{
join_rules::{AllowRule, JoinRule},
member::{MembershipState, RoomMemberEventContent},
},
room::member::{MembershipState, RoomMemberEventContent},
},
room_version_rules::RoomVersionRules,
};
use service::{
Services,
rooms::{
membership::validate_remote_member_event_stub,
state::RoomMutexGuard,
state_compressor::{CompressedState, HashSetCompressStateEvent},
},
rooms::{membership::validate_remote_member_event_stub, state::RoomMutexGuard},
};
use super::banned_room_check;
@@ -187,121 +187,24 @@ async fn knock_room_by_id_helper(
}
}
// For knock_restricted rooms, check if the user meets the restricted
// conditions If they do, attempt to join instead of knock
// This is not mentioned in the spec, but should be allowable (we're allowed
// to auto-join invites to knocked rooms)
let join_rule = services.rooms.state_accessor.get_join_rules(room_id).await;
if let JoinRule::KnockRestricted(restricted) = &join_rule {
let restriction_rooms: Vec<_> = restricted
.allow
.iter()
.filter_map(|a| match a {
| AllowRule::RoomMembership(r) => Some(&r.room_id),
| _ => None,
})
.collect();
// Check if the user is in any of the allowed rooms
let mut user_meets_restrictions = false;
for restriction_room_id in &restriction_rooms {
if services
.rooms
.state_cache
.is_joined(sender_user, restriction_room_id)
.await
{
user_meets_restrictions = true;
break;
}
}
// If the user meets the restrictions, try joining instead
if user_meets_restrictions {
debug_info!(
"{sender_user} meets the restricted criteria in knock_restricted room \
{room_id}, attempting to join instead of knock"
);
// For this case, we need to drop the state lock and get a new one
// in join_room_by_id_helper We need to release the lock here and
// let join_room_by_id_helper acquire it again
drop(state_lock);
match services
.rooms
.membership
.join_room(sender_user, room_id, reason.clone(), servers)
.await
{
| Ok(_) => return Ok(knock_room::v3::Response::new(room_id.to_owned())),
| Err(e) => {
debug_warn!(
"Failed to convert knock to join for {sender_user} in {room_id}: {e:?}"
);
// Get a new state lock for the remaining knock logic
let new_state_lock = services.rooms.state.mutex.lock(room_id).await;
let server_in_room = services
.rooms
.state_cache
.server_in_room(services.globals.server_name(), room_id)
.await;
let local_knock = server_in_room
|| servers.is_empty()
|| (servers.len() == 1 && services.globals.server_is_ours(&servers[0]));
if local_knock {
knock_room_helper_local(
services,
sender_user,
room_id,
reason,
servers,
new_state_lock,
)
.boxed()
.await?;
} else {
knock_room_helper_remote(
services,
sender_user,
room_id,
reason,
servers,
new_state_lock,
)
.boxed()
.await?;
}
return Ok(knock_room::v3::Response::new(room_id.to_owned()));
},
}
}
} else if !matches!(join_rule, JoinRule::Knock | JoinRule::KnockRestricted(_)) {
debug_warn!(
"{sender_user} attempted to knock on room {room_id} but its join rule is \
{join_rule:?}, not knock or knock_restricted"
);
}
let server_in_room = services
.rooms
.state_cache
.server_in_room(services.globals.server_name(), room_id)
.await;
let local_knock = server_in_room
|| servers.is_empty()
|| (servers.len() == 1 && services.globals.server_is_ours(&servers[0]));
if local_knock {
knock_room_helper_local(services, sender_user, room_id, reason, servers, state_lock)
if !server_in_room && !servers.iter().any(|sn| services.globals.server_is_ours(sn)) {
return Err!(Request(NotFound(
"Room {room_id} not found and no remote servers were provided to discover it"
)));
} else if server_in_room {
// Local knocks can't fail unless the room is corrupted, so we don't
// fall back to remote knocks like we do with remote joins.
knock_room_helper_local(services, sender_user, room_id, reason, state_lock)
.boxed()
.await?;
} else {
knock_room_helper_remote(services, sender_user, room_id, reason, servers, state_lock)
knock_room_helper_remote(services, sender_user, room_id, reason, servers)
.boxed()
.await?;
}
@@ -314,11 +217,8 @@ async fn knock_room_helper_local(
sender_user: &UserId,
room_id: &RoomId,
reason: Option<String>,
servers: &[OwnedServerName],
state_lock: RoomMutexGuard,
) -> Result {
debug_info!("We can knock locally");
let room_version = services.rooms.state.get_room_version(room_id).await?;
let room_version_rules = room_version
.rules()
@@ -333,8 +233,7 @@ async fn knock_room_helper_local(
content.avatar_url = services.users.avatar_url(sender_user).await.ok();
content.reason.clone_from(&reason.clone());
// Try normal knock first
let Err(error) = services
services
.rooms
.timeline
.build_and_append_pdu(
@@ -344,129 +243,7 @@ async fn knock_room_helper_local(
&state_lock,
)
.await
else {
return Ok(());
};
if servers.is_empty() || (servers.len() == 1 && services.globals.server_is_ours(&servers[0]))
{
return Err(error);
}
let (make_knock_response, remote_server) =
make_knock_request(services, sender_user, room_id, servers).await?;
info!("make_knock finished");
let room_version = make_knock_response.room_version;
let room_version_rules = room_version
.rules()
.expect("room version should have defined rules");
if !services.server.supported_room_version(&room_version) {
return Err!(BadServerResponse("Remote room version {room_version} is not supported"));
}
let mut knock_event_stub = serde_json::from_str::<CanonicalJsonObject>(
make_knock_response.event.get(),
)
.map_err(|e| {
err!(BadServerResponse("Invalid make_knock event json received from server: {e:?}"))
})?;
validate_remote_member_event_stub(
&MembershipState::Knock,
sender_user,
room_id,
&knock_event_stub,
)?;
knock_event_stub.insert(
"origin".to_owned(),
CanonicalJsonValue::String(services.globals.server_name().as_str().to_owned()),
);
knock_event_stub.insert(
"origin_server_ts".to_owned(),
CanonicalJsonValue::Integer(
utils::millis_since_unix_epoch()
.try_into()
.expect("Timestamp is valid js_int value"),
),
);
knock_event_stub.insert(
"content".to_owned(),
to_canonical_value(content).expect("event is valid, we just created it"),
);
// In order to create a compatible ref hash (EventID) the `hashes` field
// needs to be present
services
.server_keys
.hash_and_sign_event(&mut knock_event_stub, &room_version_rules)?;
// Generate event id
let event_id = gen_event_id(&knock_event_stub, &room_version_rules)?;
// Add event_id
knock_event_stub
.insert("event_id".to_owned(), CanonicalJsonValue::String(event_id.clone().into()));
// It has enough fields to be called a proper event now
let knock_event = knock_event_stub;
info!("Asking {remote_server} for send_knock in room {room_id}");
let send_knock_request = federation::membership::create_knock_event::v1::Request::new(
room_id.to_owned(),
event_id.clone(),
services
.sending
.convert_to_outgoing_federation_event(knock_event.clone())
.await,
);
services
.sending
.send_federation_request(&remote_server, send_knock_request)
.await?;
info!("send_knock finished");
services
.rooms
.short
.get_or_create_shortroomid(room_id)
.await;
info!("Parsing knock event");
let parsed_knock_pdu = PduEvent::from_id_val(&event_id, knock_event.clone())
.map_err(|e| err!(BadServerResponse("Invalid knock event PDU: {e:?}")))?;
info!("Updating membership locally to knock state with provided stripped state events");
// TODO: this call does not appear to do anything because
// `update_membership` doesn't call `mark_as_knock`. investigate further,
// ideally with the aim of removing this call entirely -- Ginger thinks
// `update_membership` should only be called from `force_state` and
// `append_pdu`.
services
.rooms
.state_cache
.update_membership(room_id, sender_user, &parsed_knock_pdu, false)
.await?;
info!("Appending room knock event locally");
services
.rooms
.timeline
.progress_extremities(&parsed_knock_pdu)
.await?;
services
.rooms
.timeline
.append_pdu(&parsed_knock_pdu, knock_event, room_id)
.await;
Ok(())
.map(|_| ())
}
async fn knock_room_helper_remote(
@@ -475,16 +252,14 @@ async fn knock_room_helper_remote(
room_id: &RoomId,
reason: Option<String>,
servers: &[OwnedServerName],
state_lock: RoomMutexGuard,
) -> Result {
info!("Knocking {room_id} over federation.");
info!(via=?servers, "Knocking on {room_id} over federation.");
let (make_knock_response, remote_server) =
let (mut stub, room_version, remote_server) =
make_knock_request(services, sender_user, room_id, servers).await?;
info!("make_knock finished");
let room_version = make_knock_response.room_version;
let room_version_rules = room_version
.rules()
.expect("room version should have defined rules");
@@ -493,16 +268,7 @@ async fn knock_room_helper_remote(
return Err!(BadServerResponse("Remote room version {room_version} is not supported"));
}
let mut knock_event_stub: CanonicalJsonObject =
serde_json::from_str(make_knock_response.event.get()).map_err(|e| {
err!(BadServerResponse("Invalid make_knock event json received from server: {e:?}"))
})?;
knock_event_stub.insert(
"origin".to_owned(),
CanonicalJsonValue::String(services.globals.server_name().as_str().to_owned()),
);
knock_event_stub.insert(
stub.insert(
"origin_server_ts".to_owned(),
CanonicalJsonValue::Integer(
utils::millis_since_unix_epoch()
@@ -516,7 +282,7 @@ async fn knock_room_helper_remote(
knock_content.avatar_url = services.users.avatar_url(sender_user).await.ok();
knock_content.reason = reason;
knock_event_stub.insert(
stub.insert(
"content".to_owned(),
to_canonical_value(knock_content).expect("event is valid, we just created it"),
);
@@ -525,17 +291,15 @@ async fn knock_room_helper_remote(
// needs to be present
services
.server_keys
.hash_and_sign_event(&mut knock_event_stub, &room_version_rules)?;
.hash_and_sign_event(&mut stub, &room_version_rules)?;
// Generate event id
let event_id = gen_event_id(&knock_event_stub, &room_version_rules)?;
let event_id = gen_event_id(&stub, &room_version_rules)?;
// Add event_id
knock_event_stub
.insert("event_id".to_owned(), CanonicalJsonValue::String(event_id.clone().into()));
// It has enough fields to be called a proper event now
let knock_event = knock_event_stub;
// Just verify that this event isn't hilariously malformed before we sent it
// off.
info!("Parsing knock event");
PduEvent::from_id_val(&event_id, stub.clone())
.map_err(|e| err!(BadServerResponse("Invalid knock event PDU: {e:?}")))?;
info!("Asking {remote_server} for send_knock in room {room_id}");
let request = federation::membership::create_knock_event::v1::Request::new(
@@ -543,7 +307,7 @@ async fn knock_room_helper_remote(
event_id.clone(),
services
.sending
.convert_to_outgoing_federation_event(knock_event.clone())
.convert_to_outgoing_federation_event(stub.clone())
.await,
);
@@ -554,147 +318,173 @@ async fn knock_room_helper_remote(
info!("send_knock finished");
info!("Validating send_knock response");
services
.rooms
.short
.get_or_create_shortroomid(room_id)
.await;
let (create_event_id, mut state) = validate_knock_state(
services,
&send_knock_response.knock_room_state,
&room_version_rules,
room_id.to_owned(),
)
.await?;
state.insert(StateEventType::RoomMember.with_state_key(sender_user.as_str()), stub);
let create_event_json = state
.get(&StateEventType::RoomCreate.with_state_key(""))
.expect("must have create event in knock state by this point");
info!("Parsing knock event");
let parsed_knock_pdu = PduEvent::from_id_val(&event_id, knock_event.clone())
.map_err(|e| err!(BadServerResponse("Invalid knock event PDU: {e:?}")))?;
info!("Going through send_knock response knock state events");
let state = send_knock_response
.knock_room_state
.iter()
.map(|event| {
#[allow(deprecated)]
let raw_value = match event {
| federation::membership::RawStrippedState::Stripped(raw_state) =>
&raw_state.clone().into_json(),
| federation::membership::RawStrippedState::Pdu(raw_value) => raw_value,
| _ => panic!("unknown raw stripped state type"),
};
serde_json::from_str::<CanonicalJsonObject>(raw_value.get())
})
.filter_map(Result::ok);
let mut state_map: HashMap<u64, OwnedEventId> = HashMap::new();
for mut event in state {
let Some(state_key) = event.get("state_key") else {
debug_warn!("send_knock stripped state event missing state_key: {event:?}");
continue;
};
let Some(event_type) = event.get("type") else {
debug_warn!("send_knock stripped state event missing event type: {event:?}");
continue;
};
let Ok(state_key) = serde_json::from_value::<String>(state_key.clone().into()) else {
debug_warn!("send_knock stripped state event has invalid state_key: {event:?}");
continue;
};
let Ok(event_type) = serde_json::from_value::<StateEventType>(event_type.clone().into())
else {
debug_warn!("send_knock stripped state event has invalid event type: {event:?}");
continue;
};
let event_id = gen_event_id(&event, &room_version_rules)?;
let shortstatekey = services
.rooms
.short
.get_or_create_shortstatekey(&event_type, &state_key)
.await;
event.insert("event_id".into(), CanonicalJsonValue::String(event_id.to_string()));
services.rooms.outlier.add_pdu_outlier(&event_id, &event);
state_map.insert(shortstatekey, event_id.clone());
}
info!("Compressing state from send_knock");
let compressed: CompressedState = services
.rooms
.state_compressor
.compress_state_events(state_map.iter().map(|(ssk, eid)| (ssk, eid.borrow())))
.collect()
.await;
debug!("Saving compressed state");
let HashSetCompressStateEvent {
shortstatehash: statehash_before_knock,
added,
removed,
} = services
.rooms
.state_compressor
.save_state(room_id, Arc::new(compressed))
.await?;
debug!("Forcing state for new room");
// Much like when receiving an invite over federation, if we're in this room
// we'll receive the knock PDU over /send. Otherwise, we need to add it as
// an outlier. Since this function isn't a fallback when local knocks fail,
// we can do this unconditionally.
services
.rooms
.state
.force_state(room_id, statehash_before_knock, added, removed, &state_lock)
.await?;
let statehash_after_knock = services
.rooms
.state
.append_to_state(&parsed_knock_pdu, room_id)
.await?;
info!("Updating membership locally to knock state with provided stripped state events");
// TODO: see TODO on the other call to `update_membership`
.outlier
.add_pdu_outlier(&create_event_id, create_event_json);
services.rooms.state_cache.mark_as_knocked(
sender_user,
room_id,
Some(state.values().collect()),
);
services
.rooms
.state_cache
.update_membership(room_id, sender_user, &parsed_knock_pdu, false)
.await?;
info!("Appending room knock event locally");
services
.rooms
.timeline
.progress_extremities(&parsed_knock_pdu)
.await?;
services
.rooms
.timeline
.append_pdu(&parsed_knock_pdu, knock_event, room_id)
.update_joined_count(room_id)
.await;
info!("Setting final room state for new room");
// We set the room state after inserting the pdu, so that we never have a
// moment in time where events in the current room state do not exist
services
.rooms
.state
.set_room_state(room_id, statehash_after_knock, &state_lock);
services.sync.wake(sender_user).await;
Ok(())
}
/// Validates and builds a state map based on the returned knock state.
///
/// This is very similar to
/// `conduwuit_api::server::invite::validate_invite_state`, but with fixed up
/// error messages.
///
/// Returns the create event's event ID, and the partial state map.
// TODO(nex): this function is LITERALLY the same as validate_invite state,
// except the errors and logs say "knock" instead of "invite". There's probably
// a more elegant way to do this without duplicating the ENTIRE function.
async fn validate_knock_state(
services: &Services,
knock_state: &[RawStrippedState],
room_version_rules: &RoomVersionRules,
room_id: OwnedRoomId,
) -> Result<(OwnedEventId, HashMap<(StateEventType, StateKey), CanonicalJsonObject>)> {
trace!(?knock_state, "Raw knock state");
let mut knock_state_map: HashMap<(StateEventType, StateKey), _> =
HashMap::with_capacity(knock_state.len());
let mut create_event_id: Option<OwnedEventId> = None;
for (idx, knock_state_event) in knock_state.iter().cloned().enumerate() {
trace!(%idx, ?knock_state_event, "Knock state event");
// Stripped state hasn't been sent over federation since v1.16.
let RawStrippedState::Pdu(raw_pdu) = knock_state_event else {
debug!(%idx, "Knock state event is not a PDU");
return Err!(Request(InvalidParam(
"PDU in knock state (index {idx}) violates the room event format"
)));
};
let (state_event_room_id, state_event_id, state_event_json) = services
.rooms
.event_handler
.parse_incoming_pdu(&raw_pdu, Some(room_version_rules))
.await
.map_err(|e| {
err!(Request(InvalidParam(debug_warn!("Invalid PDU in knock state: {e}"))))
})?;
if state_event_room_id != room_id {
return Err!(Request(InvalidParam(debug_warn!(
%state_event_room_id,
%room_id,
"PDU in knock state ({state_event_id}) belongs to the wrong room"
))));
}
services
.server_keys
.verify_event_json(&state_event_json, room_version_rules)
.await
.map_err(|e| {
err!(Request(InvalidParam("Signature verification failed on knock event: {e}")))
})?;
let Some(state_key) = state_event_json.get("state_key").and_then(|k| k.as_str()) else {
return Err!(Request(InvalidParam(debug_info!(
"PDU in knock state ({state_event_id}) is not a state event"
))));
};
let Some(event_type) = state_event_json.get("type").and_then(|k| k.as_str()) else {
return Err!(Request(InvalidParam(debug_warn!(
"PDU in knock state ({state_event_id}) is not an event?"
))));
};
let key = StateEventType::from(event_type).with_state_key(state_key);
match knock_state_map.entry(key) {
| Entry::Occupied(entry) =>
return Err!(Request(InvalidParam(
"Duplicate state events in knock state for state key: {:?}",
entry.key(),
))),
| Entry::Vacant(entry) => {
if entry.key().0 == StateEventType::RoomCreate {
// Ensure this is a legal create event.
let pdu_event =
PduEvent::from_id_val(&state_event_id, state_event_json.clone())
.expect("must be able to create pdu event from event json");
debug!("Validating discovered create event in knock room state");
crate::server::invite::validate_invite_create_event(
&pdu_event,
room_version_rules,
)
.await?;
create_event_id = Some(state_event_id);
}
entry.insert(state_event_json);
},
}
}
let Some(create_event_id) = create_event_id else {
return Err!(Request(InvalidParam(debug_warn!(
parsed_state=?knock_state_map,
"Knock state does not contain the m.room.create event"
))));
};
knock_state_map.iter().try_for_each(|(key, event_json)| {
service::rooms::event_handler::Service::pdu_format_check_1(
event_json,
room_version_rules,
&create_event_id,
)
.map_err(|e| {
err!(Request(InvalidParam(
"PDU in knock state for {key:?} violates the room event format: {e}"
)))
})
})?;
Ok((create_event_id, knock_state_map))
}
async fn make_knock_request(
services: &Services,
sender_user: &UserId,
room_id: &RoomId,
servers: &[OwnedServerName],
) -> Result<(federation::membership::prepare_knock_event::v1::Response, OwnedServerName)> {
let mut make_knock_response_and_server =
Err!(BadServerResponse("No server available to assist in knocking."));
let mut make_knock_counter: usize = 0;
for remote_server in servers {
) -> Result<(CanonicalJsonObject, RoomVersionId, OwnedServerName)> {
for (idx, remote_server) in servers.iter().enumerate() {
if services.globals.server_is_ours(remote_server) {
continue;
}
info!("Asking {remote_server} for make_knock ({make_knock_counter})");
info!("Asking {remote_server} for make_knock ({})", idx.saturating_add(1));
let mut request = federation::membership::prepare_knock_event::v1::Request::new(
room_id.to_owned(),
@@ -708,36 +498,51 @@ async fn make_knock_request(
.await;
trace!("make_knock response: {make_knock_response:?}");
make_knock_counter = make_knock_counter.saturating_add(1);
if let Ok(r) = &make_knock_response {
if let Err(e) = validate_remote_member_event_stub(
&MembershipState::Knock,
sender_user,
room_id,
&to_canonical_object(&r.event)?,
) {
warn!("make_knock response from {remote_server} failed validation: {e}");
continue;
}
}
make_knock_response_and_server = make_knock_response.map(|r| (r, remote_server.clone()));
if make_knock_response_and_server.is_ok() {
break;
}
if make_knock_counter > 40 {
warn!(
"50 servers failed to provide valid make_knock response, assuming no server can \
assist in knocking."
);
make_knock_response_and_server =
Err!(BadServerResponse("No server available to assist in knocking."));
return make_knock_response_and_server;
match make_knock_response
.and_then(|r| {
to_canonical_object(&r.event)
.map(|o| (o, r.room_version))
.map_err(Into::into)
})
.and_then(|(stub, room_version)| {
validate_remote_member_event_stub(
&MembershipState::Knock,
sender_user,
room_id,
&stub,
)
.map(|()| (stub, room_version))
}) {
| Ok((evt, room_version)) =>
return Ok((evt, room_version, remote_server.to_owned())),
| Err(e) => match e.kind() {
| ErrorKind::IncompatibleRoomVersion(IncompatibleRoomVersionErrorData {
room_version,
..
}) => {
warn!(
"{remote_server} reports the room we are trying to knock on is \
v{room_version}, which we do not support."
);
return Err(e);
},
| ErrorKind::Forbidden => {
warn!("{remote_server} refuses to let us knock: {e}.");
return Err(e);
},
| ErrorKind::NotFound => {
info!(
"{remote_server} does not know about {room_id}: {e}. Will continue \
trying."
);
},
| _ => {
info!("{remote_server} failed to make_knock: {e}. Will continue trying.");
},
},
}
}
make_knock_response_and_server
info!("All {} servers were unable to assist in knocking {room_id} :(", servers.len());
Err!(BadServerResponse("No server available to assist in knocking."))
}
+3
View File
@@ -16,3 +16,6 @@ pub mod client_ip;
pub mod admin;
pub(crate) use self::router::{Ruma, RumaResponse, State};
conduwuit::mod_ctor! {}
conduwuit::mod_dtor! {}
+1 -1
View File
@@ -414,7 +414,7 @@ struct MFederate {
///
/// 1. It passes auth checks (aka is valid)
/// 2. The room is federated (there's no point persisting unfederated rooms)
async fn validate_invite_create_event(
pub(crate) async fn validate_invite_create_event(
pdu: &PduEvent,
room_version_rules: &RoomVersionRules,
) -> Result {
+5
View File
@@ -20,6 +20,9 @@ crate-type = [
brotli_compression = [
"reqwest/brotli",
]
conduwuit_mods = [
"dep:libloading"
]
gzip_compression = [
"reqwest/gzip",
]
@@ -82,6 +85,8 @@ http-body-util.workspace = true
http.workspace = true
ipaddress.workspace = true
libc.workspace = true
libloading.workspace = true
libloading.optional = true
lettre.workspace = true
num-traits.workspace = true
rand.workspace = true
+85 -142
View File
@@ -1,6 +1,8 @@
use std::{
cell::{Cell, RefCell},
ops::Deref,
ptr::NonNull,
ptr,
ptr::null_mut,
sync::{
Arc,
atomic::{AtomicPtr, Ordering},
@@ -8,178 +10,119 @@ use std::{
};
use super::Config;
use crate::{Result, SyncMutex};
use crate::Result;
/// The configuration manager is an indirection to reload the configuration for
/// the server while it is running. In order to not burden or clutter the many
/// callsites which query for configuration items, this object implements Deref
/// for the actively loaded configuration.
///
/// Every installed `Arc<Config>` is kept at a stable heap address in `retired`
/// for the entire lifetime of `Manager` (intentionally retaining prior configs
/// across the rare, admin-triggered reloads until shutdown). Because
/// `Deref::deref(&self)` ties the returned `&Arc<Config>` lifetime to `&self`,
/// readers can perform a single `Acquire` load with no locking, no TLS, and no
/// risk of dangling or aliasing UB across reloads.
pub struct Manager {
active: AtomicPtr<Arc<Config>>,
/// Append-only owner of every `Box<Arc<Config>>` ever published to
/// `active` (including the currently active config). Entries are only
/// freed when `Manager` itself is dropped (`&mut self`), guaranteeing that
/// any `&Arc<Config>` returned by `Deref::deref` remains valid and
/// immutable for the lifetime of `self`.
retired: SyncMutex<Vec<PinnedConfig>>,
active: AtomicPtr<Config>,
}
/// Heap-pinned `Arc<Config>` stored as a raw `NonNull` rather than `Box` so
/// moving it (e.g., when `Vec` reallocates) never retags or invalidates raw
/// pointers previously published to `Manager::active`.
struct PinnedConfig(NonNull<Arc<Config>>);
// SAFETY: `Arc<Config>` is `Send + Sync`, and `PinnedConfig` uniquely owns the
// `Box<Arc<Config>>` allocation until it is dropped in `Manager::drop`.
unsafe impl Send for PinnedConfig {}
impl PinnedConfig {
fn new(config: Config) -> Self {
let ptr = Box::into_raw(Box::new(Arc::new(config)));
Self(NonNull::new(ptr).expect("Box::into_raw returned null"))
}
#[inline]
const fn as_ptr(&self) -> *mut Arc<Config> { self.0.as_ptr() }
thread_local! {
static INDEX: Cell<usize> = 0.into();
static HANDLE: RefCell<Handles> = const {
RefCell::new([const { None }; HISTORY])
};
}
impl Drop for PinnedConfig {
fn drop(&mut self) {
// SAFETY: `self.0` was created via `Box::into_raw` in
// `PinnedConfig::new` and is only dropped when `Manager` itself is
// dropped (`&mut self`), after all borrows derived from `Deref::deref`
// have ended.
unsafe {
drop(Box::from_raw(self.0.as_ptr()));
}
}
}
type Handle = Option<Arc<Config>>;
type Handles = [Handle; HISTORY];
const HISTORY: usize = 8;
impl Manager {
pub(crate) fn new(config: Config) -> Self {
let pinned = PinnedConfig::new(config);
let ptr = pinned.as_ptr();
let config = Arc::new(config);
Self {
active: AtomicPtr::new(ptr),
retired: SyncMutex::new(vec![pinned]),
active: AtomicPtr::new(Arc::into_raw(config).cast_mut()),
}
}
/// Update the active configuration, returning prior configuration.
#[tracing::instrument(skip_all, level = "info")]
pub fn update(&self, config: Config) -> Result<Arc<Config>> {
let pinned = PinnedConfig::new(config);
let new_ptr = pinned.as_ptr();
let config = Arc::new(config);
let new = Arc::into_raw(config);
let old = self.active.swap(new.cast_mut(), Ordering::AcqRel);
let mut retired = self.retired.lock();
// Push before publishing to `active` so ownership is recorded even if
// `Vec::push` were to unwind.
retired.push(pinned);
// SAFETY: The old active pointer was set using an Arc::into_raw().
// We're obliged to reconstitute that into Arc otherwise it will leak.
Ok(unsafe { Arc::from_raw(old) })
}
// `Release` publishes the initialized `Box<Arc<Config>>` to readers
// performing `Acquire` loads in `Deref::deref`.
let old_ptr = self.active.swap(new_ptr, Ordering::AcqRel);
fn load(&self, handle: &mut [Option<Arc<Config>>]) -> &'static Arc<Config> {
let config = self.active.load(Ordering::Acquire);
// SAFETY: `old_ptr` points to a `PinnedConfig` still owned by `retired`
// and cannot be freed until `Manager` is dropped.
let old = unsafe { Arc::clone(&*old_ptr) };
// Branch taken after config reload or first access by this thread.
if handle[INDEX.get()]
.as_ref()
.is_none_or(|handle| !ptr::eq(config, Arc::as_ptr(handle)))
{
INDEX.set(INDEX.get().wrapping_add(1).wrapping_rem(HISTORY));
return load_miss(handle, INDEX.get(), config);
}
Ok(old)
let config: &Arc<Config> = handle[INDEX.get()]
.as_ref()
.expect("handle was already cached for this thread");
// SAFETY: The caller should not hold multiple references at a time
// directly into Config, as a subsequent reference might invalidate
// the thread's cache causing another reference to dangle.
//
// This is a highly unusual pattern as most config values are copied by
// value or used immediately without running overlap with another
// value. Even if it does actually occur somewhere, the window of
// danger is limited to the config being reloaded while the reference
// is held and another access is made by the same thread into a
// different config value. This is mitigated by creating a buffer
// of old configs rather than discarding at the earliest opportunity;
// the odds of this scenario are thus astronomical.
unsafe { std::mem::transmute(config) }
}
}
impl Drop for Manager {
fn drop(&mut self) {
let config = self.active.swap(null_mut(), Ordering::AcqRel);
// SAFETY: The active pointer was set using an Arc::into_raw(). We're
// obliged to reconstitute that into Arc otherwise it will leak.
unsafe { Arc::from_raw(config) };
}
}
impl Deref for Manager {
type Target = Arc<Config>;
#[inline]
fn deref(&self) -> &Self::Target {
let ptr = self.active.load(Ordering::Acquire);
// SAFETY: `ptr` was initialized in `Manager::new` or published via
// `Ordering::AcqRel` in `Manager::update` and points to a
// `PinnedConfig` in `self.retired`. Entries in `self.retired` are never
// removed or mutated until `Manager` is dropped (`&mut self`), so the
// returned `&'self Arc<Config>` remains valid and immutable for the
// entire borrow of `self`.
unsafe { &*ptr }
}
fn deref(&self) -> &Self::Target { HANDLE.with_borrow_mut(|handle| self.load(handle)) }
}
#[cfg(test)]
mod tests {
use std::thread;
#[tracing::instrument(
name = "miss",
level = "trace",
skip_all,
fields(%index, ?config)
)]
#[allow(clippy::transmute_ptr_to_ptr)]
fn load_miss(
handle: &mut [Option<Arc<Config>>],
index: usize,
config: *const Config,
) -> &'static Arc<Config> {
// SAFETY: The active pointer was set prior and always remains valid. We're
// reconstituting the Arc here but as a new reference, so the count is
// incremented. This instance will be cached in the thread-local.
let config = unsafe {
Arc::increment_strong_count(config);
Arc::from_raw(config)
};
use super::*;
fn test_config(server_name: &str) -> Config {
use figment::{
Figment,
providers::{Format, Toml},
};
let raw = format!("server_name = \"{server_name}\"\ndatabase_path = \"/tmp\"\n");
Figment::new()
.merge(Toml::string(&raw))
.extract()
.expect("valid test config")
}
#[test]
fn deref_survives_many_reloads() {
let manager = Manager::new(test_config("initial.example"));
let first: &Arc<Config> = &manager;
let first_name: &str = first.server_name.as_str();
for i in 0..32 {
let prev = manager
.update(test_config(&format!("reload-{i}.example")))
.expect("update succeeds");
if i == 0 {
assert_eq!(prev.server_name.as_str(), "initial.example");
}
}
// Original reference remains valid and unchanged after > HISTORY
// reloads.
assert_eq!(first_name, "initial.example");
assert_eq!(manager.server_name.as_str(), "reload-31.example");
}
#[test]
fn concurrent_reads_and_updates() {
let manager = Manager::new(test_config("start.example"));
thread::scope(|s| {
for _ in 0..4 {
s.spawn(|| {
for _ in 0..200 {
let cfg: &Arc<Config> = &manager;
assert!(cfg.server_name.as_str().ends_with(".example"));
}
});
}
s.spawn(|| {
for i in 0..32 {
let _ = manager
.update(test_config(&format!("iter-{i}.example")))
.expect("update succeeds");
}
});
});
}
#[test]
fn reference_outlives_scoped_thread() {
let manager = Manager::new(test_config("scoped.example"));
let cfg_ref: &Arc<Config> = thread::scope(|s| s.spawn(|| &*manager).join().unwrap());
assert_eq!(cfg_ref.server_name.as_str(), "scoped.example");
}
// SAFETY: See the note on the transmute above. The caller should not hold
// more than one reference at a time directly into Config, as the second
// access might invalidate the thread's cache, dangling the reference to
// the first.
unsafe { std::mem::transmute(handle[index].insert(config)) }
}
+13
View File
@@ -8,6 +8,19 @@ pub mod info;
pub mod log;
pub mod matrix;
pub mod metrics;
pub mod mods;
#[cfg(any(not(conduwuit_mods), not(feature = "conduwuit_mods")))]
pub mod mods {
#[macro_export]
macro_rules! mod_ctor {
() => {};
}
#[macro_export]
macro_rules! mod_dtor {
() => {};
}
}
pub mod server;
pub mod utils;
+28
View File
@@ -0,0 +1,28 @@
use std::sync::atomic::{AtomicI32, Ordering};
const ORDERING: Ordering = Ordering::Relaxed;
static STATIC_DTORS: AtomicI32 = AtomicI32::new(0);
/// Called by Module::unload() to indicate module is about to be unloaded and
/// static destruction is intended. This will allow verifying it actually took
/// place.
pub(crate) fn prepare() {
let count = STATIC_DTORS.fetch_sub(1, ORDERING);
debug_assert!(count <= 0, "STATIC_DTORS should not be greater than zero.");
}
/// Called by static destructor of a module. This call should only be found
/// inside a mod_fini! macro. Do not call from anywhere else.
#[inline(always)]
pub fn report() { let _count = STATIC_DTORS.fetch_add(1, ORDERING); }
/// Called by Module::unload() (see check()) with action in case a check()
/// failed. This can allow a stuck module to be noted while allowing for other
/// independent modules to be diagnosed.
pub(crate) fn check_and_reset() -> bool { STATIC_DTORS.swap(0, ORDERING) == 0 }
/// Called by Module::unload() after unload to verify static destruction took
/// place. A call to prepare() must be made prior to Module::unload() and making
/// this call.
#[allow(dead_code)]
pub(crate) fn check() -> bool { STATIC_DTORS.load(ORDERING) == 0 }
+44
View File
@@ -0,0 +1,44 @@
#[macro_export]
macro_rules! mod_ctor {
( $($body:block)? ) => {
$crate::mod_init! {{
$crate::debug_info!("Module loaded");
$($body)?
}}
}
}
#[macro_export]
macro_rules! mod_dtor {
( $($body:block)? ) => {
$crate::mod_fini! {{
$crate::debug_info!("Module unloading");
$($body)?
$crate::mods::canary::report();
}}
}
}
#[macro_export]
macro_rules! mod_init {
($body:block) => {
#[used]
#[cfg_attr(target_family = "unix", unsafe(link_section = ".init_array"))]
static MOD_INIT: unsafe extern "C" fn() = { _mod_init };
#[cfg_attr(target_family = "unix", unsafe(link_section = ".text.startup"))]
unsafe extern "C" fn _mod_init() -> () $body
};
}
#[macro_export]
macro_rules! mod_fini {
($body:block) => {
#[used]
#[cfg_attr(target_family = "unix", unsafe(link_section = ".fini_array"))]
static MOD_FINI: unsafe extern "C" fn() = { _mod_fini };
#[cfg_attr(target_family = "unix", unsafe(link_section = ".text.startup"))]
unsafe extern "C" fn _mod_fini() -> () $body
};
}
+11
View File
@@ -0,0 +1,11 @@
#![cfg(all(conduwuit_mods, feature = "conduwuit_mods"))]
pub(crate) use libloading::os::unix::{Library, Symbol};
pub mod canary;
pub mod macros;
pub mod module;
pub mod new;
pub mod path;
pub use module::Module;
+75
View File
@@ -0,0 +1,75 @@
use std::{
ffi::{CString, OsString},
time::SystemTime,
};
use super::{Library, Symbol, canary, new, path};
use crate::{Result, error};
pub struct Module {
handle: Option<Library>,
loaded: SystemTime,
path: OsString,
}
impl Module {
pub fn from_name(name: &str) -> Result<Self> { Self::from_path(path::from_name(name)?) }
pub fn from_path(path: OsString) -> Result<Self> {
Ok(Self {
handle: Some(new::from_path(&path)?),
loaded: SystemTime::now(),
path,
})
}
pub fn unload(&mut self) {
canary::prepare();
self.close();
if !canary::check_and_reset() {
let name = self.name().expect("Module is named");
error!("Module {name:?} is stuck and failed to unload.");
}
}
pub(crate) fn close(&mut self) {
if let Some(handle) = self.handle.take() {
handle.close().expect("Module handle closed");
}
}
pub fn get<Prototype>(&self, name: &str) -> Result<Symbol<Prototype>> {
let cname = CString::new(name.to_owned()).expect("terminated string from provided name");
let handle = self
.handle
.as_ref()
.expect("backing library loaded by this instance");
// SAFETY: Calls dlsym(3) on unix platforms. This might not have to be
// unsafe if wrapped in libloading with_dlerror().
let sym = unsafe { handle.get::<Prototype>(cname.as_bytes()) };
let sym = sym.expect("symbol found; binding successful");
Ok(sym)
}
pub fn deleted(&self) -> Result<bool> {
let mtime = path::mtime(self.path())?;
let res = mtime > self.loaded;
Ok(res)
}
pub fn name(&self) -> Result<String> { path::to_name(self.path()) }
#[must_use]
pub fn path(&self) -> &OsString { &self.path }
}
impl Drop for Module {
fn drop(&mut self) {
if self.handle.is_some() {
self.unload();
}
}
}
+23
View File
@@ -0,0 +1,23 @@
use std::ffi::OsStr;
use super::{Library, path};
use crate::{Err, Result};
const OPEN_FLAGS: i32 = libloading::os::unix::RTLD_LAZY | libloading::os::unix::RTLD_GLOBAL;
pub fn from_name(name: &str) -> Result<Library> {
let path = path::from_name(name)?;
from_path(&path)
}
pub fn from_path(path: &OsStr) -> Result<Library> {
//SAFETY: Calls dlopen(3) on unix platforms. This might not have to be
// unsafe if wrapped in with_dlerror.
let lib = unsafe { Library::open(Some(path), OPEN_FLAGS) };
if let Err(e) = lib {
let name = path::to_name(path)?;
return Err!("Loading module {name:?} failed: {e}");
}
Ok(lib.expect("module loaded"))
}
+41
View File
@@ -0,0 +1,41 @@
use std::{
env::current_exe,
ffi::{OsStr, OsString},
path::{Path, PathBuf},
time::SystemTime,
};
use libloading::library_filename;
use crate::Result;
pub fn from_name(name: &str) -> Result<OsString> {
let root = PathBuf::new();
let exe_path = current_exe()?;
let exe_dir = exe_path.parent().unwrap_or(&root);
let mut mod_path = exe_dir.to_path_buf();
let mod_file = library_filename(name);
mod_path.push(mod_file);
Ok(mod_path.into_os_string())
}
pub fn to_name(path: &OsStr) -> Result<String> {
let path = Path::new(path);
let name = path
.file_stem()
.expect("path file stem")
.to_str()
.expect("name string");
let name = name.strip_prefix("lib").unwrap_or(name).to_owned();
Ok(name)
}
pub fn mtime(path: &OsStr) -> Result<SystemTime> {
let meta = std::fs::metadata(path)?;
let mtime = meta.modified()?;
Ok(mtime)
}
+27
View File
@@ -27,6 +27,10 @@ pub struct Server {
/// observable used on shutdown and should not be modified.
pub stopping: AtomicBool,
/// Reload/shutdown desired indicator; when false, shutdown is desired. This
/// is an observable used on shutdown and modifying is not recommended.
pub reloading: AtomicBool,
/// Restart desired; when true, restart it desired after shutdown.
pub restarting: AtomicBool,
@@ -56,6 +60,7 @@ impl Server {
config: config::Manager::new(config),
started: SystemTime::now(),
stopping: AtomicBool::new(false),
reloading: AtomicBool::new(false),
restarting: AtomicBool::new(false),
runtime,
signal: broadcast::channel::<&'static str>(1).0,
@@ -64,6 +69,25 @@ impl Server {
}
}
pub fn reload(&self) -> Result<()> {
if cfg!(any(not(conduwuit_mods), not(feature = "conduwuit_mods"))) {
return Err!("Reloading not enabled");
}
if self.reloading.swap(true, Ordering::AcqRel) {
return Err!("Reloading already in progress");
}
if self.stopping.swap(true, Ordering::AcqRel) {
return Err!("Shutdown already in progress");
}
self.signal("SIGINT").inspect_err(|_| {
self.stopping.store(false, Ordering::Release);
self.reloading.store(false, Ordering::Release);
})
}
pub fn restart(&self) -> Result {
if self.restarting.swap(true, Ordering::AcqRel) {
return Err!("Restart already in progress");
@@ -122,6 +146,9 @@ impl Server {
#[inline]
pub fn is_stopping(&self) -> bool { self.stopping.load(Ordering::Relaxed) }
#[inline]
pub fn is_reloading(&self) -> bool { self.reloading.load(Ordering::Relaxed) }
#[inline]
pub fn is_restarting(&self) -> bool { self.restarting.load(Ordering::Relaxed) }
+3 -47
View File
@@ -62,28 +62,14 @@ macro_rules! validated {
}
#[inline]
#[allow(clippy::as_conversions, clippy::cast_possible_truncation, clippy::cast_sign_loss)]
#[allow(clippy::as_conversions)]
pub fn usize_from_f64(val: f64) -> Result<usize, Error> {
// Exclusive upper bound for values whose truncation fits in `usize`.
// Note that `usize::MAX as f64` rounds up to 2^64 on 64-bit targets, so a
// strict `<` check against 2^(usize::BITS) rejects all out-of-range values
// on both 32-bit and 64-bit targets.
#[allow(clippy::cast_precision_loss)]
const MAX_USIZE_EXCLUSIVE: f64 = (1_u128 << usize::BITS) as f64;
if !val.is_finite() {
return Err!(Arithmetic("Converting non-finite float to unsigned integer"));
}
if val < 0.0 {
return Err!(Arithmetic("Converting negative float to unsigned integer"));
}
if val >= MAX_USIZE_EXCLUSIVE {
return Err!(Arithmetic("Converting out-of-range float to unsigned integer"));
}
Ok(val as usize)
//SAFETY: <https://doc.rust-lang.org/std/primitive.f64.html#method.to_int_unchecked>
Ok(unsafe { val.to_int_unchecked::<usize>() })
}
#[inline]
@@ -125,33 +111,3 @@ fn try_into_err<Dst: TryFrom<Src>, Src>(e: <Dst as TryFrom<Src>>::Error) -> Erro
#[inline]
pub fn clamp<T: Ord>(val: T, min: T, max: T) -> T { cmp::min(cmp::max(val, min), max) }
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn usize_from_f64_valid() {
assert_eq!(usize_from_f64(0.0).unwrap(), 0);
assert_eq!(usize_from_f64(-0.0).unwrap(), 0);
assert_eq!(usize_from_f64(42.9).unwrap(), 42);
assert_eq!(usize_from_f64(1_048_576.0).unwrap(), 1_048_576);
}
#[test]
fn usize_from_f64_rejects_invalid() {
usize_from_f64(-1.0).unwrap_err();
usize_from_f64(-0.5).unwrap_err();
usize_from_f64(f64::NAN).unwrap_err();
usize_from_f64(f64::INFINITY).unwrap_err();
usize_from_f64(f64::NEG_INFINITY).unwrap_err();
usize_from_f64(1e30).unwrap_err();
#[cfg(target_pointer_width = "64")]
{
// `usize::MAX as f64` rounds up to 2^64, which is out of range.
#[allow(clippy::as_conversions, clippy::cast_precision_loss)]
let rounded_max = usize::MAX as f64;
usize_from_f64(rounded_max).unwrap_err();
}
}
}
+5 -3
View File
@@ -1,6 +1,7 @@
use std::convert::Infallible;
use super::Result;
use super::{DebugInspect, Result};
use crate::error;
pub trait UnwrapInfallible<T> {
fn unwrap_infallible(self) -> T;
@@ -9,7 +10,8 @@ pub trait UnwrapInfallible<T> {
impl<T> UnwrapInfallible<T> for Result<T, Infallible> {
#[inline]
fn unwrap_infallible(self) -> T {
let Ok(val) = self;
val
// SAFETY: Branchless unwrap for errors that can never happen. In debug
// mode this is asserted.
unsafe { self.debug_inspect_err(error::infallible).unwrap_unchecked() }
}
}
+4 -5
View File
@@ -32,12 +32,11 @@ impl<'a> From<&'a str> for &'a Unquoted {
fn from(s: &'a str) -> &'a Unquoted {
let s: &'a str = s.unquote_infallible();
// SAFETY: `Unquoted` is `#[repr(transparent)]` over `str`, so
// `*const str` and `*const Unquoted` have identical layout and
// metadata, and the returned reference preserves lifetime `'a`.
#[allow(clippy::as_conversions)]
//SAFETY: This is a pattern I lifted from ruma-identifiers for
// strong-type strs by wrapping in a tuple-struct.
#[allow(clippy::transmute_ptr_to_ptr)]
unsafe {
&*(std::ptr::from_ref::<str>(s) as *const Unquoted)
std::mem::transmute(s)
}
}
}
+13 -5
View File
@@ -120,13 +120,21 @@ pub fn cores_available() -> impl Iterator<Item = Id> { from_mask(*CORES_AVAILABL
pub fn getcpu() -> Result<usize> {
use crate::{Error, utils::math};
// SAFETY: `sched_getcpu(2)` takes no pointers and has no preconditions.
// Note that it may still return -1 with errno set (e.g. under seccomp
// filters or kernels without getcpu support), so the error branch below
// must remain live.
// SAFETY: This is part of an interface with many low-level calls taking
// many raw params, but it's unclear why this specific call is unsafe.
// Nevertheless the value obtained here is semantically unsafe because it
// can change on the instruction boundary trailing its own acquisition and
// also any other time.
let ret: i32 = unsafe { libc::sched_getcpu() };
if ret < 0 {
#[cfg(target_os = "linux")]
// SAFETY: On modern linux systems with a vdso if we can optimize away the branch checking
// for error (see getcpu(2)) then this system call becomes a memory access.
unsafe {
std::hint::assert_unchecked(ret >= 0);
};
if ret == -1 {
return Err(Error::from_errno());
}
+12
View File
@@ -472,5 +472,17 @@ impl<'a, 'de: 'a> de::MapAccess<'de> for &'a mut Deserializer<'de> {
}
}
// activate when stable; too soon now
//#[cfg(debug_assertions)]
#[inline]
fn deserialize_str(input: &[u8]) -> Result<&str> { string::str_from_bytes(input) }
//#[cfg(not(debug_assertions))]
#[cfg(disable)]
#[inline]
fn deserialize_str(input: &[u8]) -> Result<&str> {
// SAFETY: Strings were written by the serializer to the database. Assuming
// no database corruption, the string will be valid. Database corruption is
// detected via rocksdb checksums.
unsafe { std::str::from_utf8_unchecked(input) }
}
-4
View File
@@ -46,10 +46,6 @@ use crate::{Engine, watchers::Watchers};
pub struct Map {
name: &'static str,
watchers: Watchers,
/// SAFETY / DROP ORDER: `cf` holds a lifetime-erased RocksDB column family
/// handle (see `open::open`) and MUST be declared before `db` so that the
/// handle is destroyed before `Engine` (and the underlying RocksDB `DB`)
/// can be dropped.
cf: Arc<ColumnFamily>,
db: Arc<Engine>,
read_options: ReadOptions,
+1 -15
View File
@@ -1,24 +1,10 @@
use std::sync::Arc;
use rocksdb::{BoundColumnFamily, ColumnFamily};
use rocksdb::ColumnFamily;
use crate::Engine;
pub(super) fn open(db: &Arc<Engine>, name: &str) -> Arc<ColumnFamily> {
// Compile-time guard: `BoundColumnFamily` and `ColumnFamily` are both
// `#[repr(Rust)]` in `rust-rocksdb`, so fail compilation if their size or
// alignment ever diverge.
const {
assert!(
size_of::<BoundColumnFamily<'_>>() == size_of::<ColumnFamily>(),
"BoundColumnFamily and ColumnFamily must have identical size"
);
assert!(
align_of::<BoundColumnFamily<'_>>() == align_of::<ColumnFamily>(),
"BoundColumnFamily and ColumnFamily must have identical alignment"
);
}
let bounded_arc = db.cf(name);
let bounded_ptr = Arc::into_raw(bounded_arc);
let cf_ptr = bounded_ptr.cast::<ColumnFamily>();
-11
View File
@@ -12,17 +12,6 @@ impl super::Map {
/// Iterate key-value entries in the map from the beginning.
///
/// - Result is deserialized
///
/// # Unsoundness
/// Because Rust's `Stream` trait does not support lending items tied to
/// `&mut self`, `stream::slice_longevity` unsafely extends the lifetime of
/// raw RocksDB iterator slices to `'a` (the borrow of `Arc<Map>`). As a
/// consequence, borrowed deserialized types (e.g. `&str`, `&UserId`,
/// `&[u8]`) reference the underlying RocksDB cursor buffer and are
/// invalidated on the next poll of the stream. Do not hold borrowed items
/// across polls (e.g. via `collect()`, `ready_chunks()`, `peekable()`,
/// `buffered()`, `buffer_unordered()`, `broadband`, or `wideband`) without
/// converting them to owned values first.
pub fn stream<'a, K, V>(
self: &'a Arc<Self>,
) -> impl Stream<Item = Result<KeyVal<'a, K, V>>> + Send
+3
View File
@@ -5,6 +5,9 @@ extern crate rust_rocksdb as rocksdb;
conduwuit_macros::introspect_crate! {}
conduwuit::mod_ctor! {}
conduwuit::mod_dtor! {}
#[cfg(test)]
mod benches;
mod cork;
+11 -23
View File
@@ -54,14 +54,8 @@ pub(crate) struct Get {
/// Note: only initial seek is supported at this time on the assumption rocksdb
/// prefetching prevents mid-iteration polls from blocking on I/O.
pub(crate) struct Seek {
/// SAFETY / DROP ORDER: `state` wraps a RocksDB
/// `DBRawIteratorWithThreadMode` referencing `map.db.db` whose lifetime has
/// been erased to `'static`. `state` MUST be declared before `map` so the
/// iterator is destroyed before `map` (and potentially the last
/// `Arc<Engine>`) can be dropped if `Seek` is dropped early (e.g. on
/// cancellation or queue closure).
pub(crate) state: stream::State<'static>,
pub(crate) map: Arc<Map>,
pub(crate) state: stream::State<'static>,
pub(crate) dir: Direction,
pub(crate) key: Option<KeyBuf>,
pub(crate) res: Option<ResultSender<stream::State<'static>>>,
@@ -422,30 +416,24 @@ impl Drop for Pool {
}
fn into_send_get(result: BatchResult<'_>) -> BatchResult<'static> {
// SAFETY: Necessary to send `Handle` (which wraps
// `rocksdb::DBPinnableSlice`) through the oneshot channel. The lifetime on
// `Handle` ties it to the RocksDB `DB` instance; callers of `execute_get`
// borrow `&self.db.pool` (which is owned by `Engine`), ensuring the
// received `BatchResult<'_>` cannot outlive the database.
// SAFETY: Necessary to send the Handle (rust_rocksdb::PinnableSlice)
// through the channel. The lifetime on the handle is a device by
// rust-rocksdb to associate a database lifetime with its assets. The
// Handle must be dropped before the database is dropped.
unsafe { std::mem::transmute(result) }
}
fn into_recv_get<'a>(result: BatchResult<'static>) -> BatchResult<'a> {
// SAFETY: Shrinking the erased `'static` lifetime back to `'a` (borrowed
// from `&self.db.pool`). `SmallVec<A>` is invariant over `A`, so this
// cannot be done via implicit covariance.
// SAFETY: This is to receive the Handle from the channel.
unsafe { std::mem::transmute(result) }
}
pub(crate) fn into_send_seek(result: stream::State<'_>) -> stream::State<'static> {
// SAFETY: Necessary to send `stream::State` through the pool channel.
// While in flight on the queue, `Seek` owns an `Arc<Map>` (declared after
// `state` so `state` drops first on early cancellation); once received back
// from `execute_iter`, the lifetime is shrunk via `into_recv_seek` to the
// caller's borrow of `&self.db.pool` (and `&'a Arc<Map>`), ensuring the
// returned `stream::State<'_>` cannot outlive the database.
// SAFETY: Necessary to send the State through the channel; see above.
unsafe { std::mem::transmute(result) }
}
#[inline]
const fn into_recv_seek<'a>(result: stream::State<'static>) -> stream::State<'a> { result }
fn into_recv_seek(result: stream::State<'static>) -> stream::State<'static> {
// SAFETY: This is to receive the State from the channel; see above.
unsafe { std::mem::transmute(result) }
}
+16 -14
View File
@@ -133,20 +133,22 @@ fn keyval_longevity<'a, 'b: 'a>(item: KeyVal<'a>) -> KeyVal<'b> {
}
fn slice_longevity<'a, 'b: 'a>(item: &'a Slice) -> &'b Slice {
// SAFETY: the lifetime of the data returned by
// the RocksDB cursor is only valid until the next movement of the cursor,
// but is unsafely extended here to match the lifetime `'b` of the `Map`
// borrow because `futures::Stream` is not a lending stream.
// SAFETY: The lifetime of the data returned by the rocksdb cursor is only
// valid between each movement of the cursor. It is hereby unsafely
// extended to match the lifetime of the cursor itself. This is due to the
// limitation of the Stream trait where the Item is incapable of conveying
// a lifetime; this is due to GAT's being unstable during its development.
// This unsafety can be removed as soon as this limitation is addressed by
// an upcoming version.
//
// Callers MUST NOT hold borrowed items (or borrowed deserialized views such
// as `&str`, `&UserId`, or `&[u8]`) across subsequent polls of the stream —
// in particular, never call `collect()`, `ready_chunks()`, `peekable()`,
// `buffered()`, `buffer_unordered()`, `broadband`, or `wideband` on a
// stream yielding borrowed references without first converting items to
// owned values (e.g. `map(ToOwned::to_owned)`). The compiler does NOT
// reject such escapes when items borrow from `'b`.
//
// TODO: Fix this, it is old code and language changes may have made it
// unnecessary
// We have done our best to mitigate the implications of this in conjunction
// with the deserialization API such that borrows being held across
// movements of the cursor do not happen accidentally. The compiler will
// still error when values herein produced try to leave a closure passed to
// a StreamExt API. But escapes can happen if you explicitly and
// intentionally attempt it, and there will be no compiler error or
// warning. This is primarily the case with calling collect() without a
// preceding map(ToOwned::to_owned). A collection of references here is
// illegal, but this will not be enforced by the compiler.
unsafe { std::mem::transmute(item) }
}
+16
View File
@@ -1,12 +1,28 @@
use conduwuit::Result;
use rocksdb::{Direction, ErrorKind, IteratorMode};
//#[cfg(debug_assertions)]
macro_rules! unhandled {
($msg:literal) => {
unimplemented!($msg)
};
}
// activate when stable; we're not ready for this yet
#[cfg(disable)] // #[cfg(not(debug_assertions))]
macro_rules! unhandled {
($msg:literal) => {
// SAFETY: Eliminates branches for serializing and deserializing types never
// encountered in the codebase. This can promote optimization and reduce
// codegen. The developer must verify for every invoking callsite that the
// unhandled type is in no way involved and could not possibly be
// encountered.
unsafe {
std::hint::unreachable_unchecked();
}
};
}
pub(crate) use unhandled;
#[inline]
+3
View File
@@ -175,6 +175,9 @@ zstd_compression = [
"conduwuit-router/zstd_compression",
"conduwuit-service/zstd_compression",
]
conduwuit_mods = [
"conduwuit-core/conduwuit_mods",
]
bindgen-static = [
# "bindgen/static"
# "clang-sys/static"
+38 -2
View File
@@ -10,6 +10,7 @@ mod clap;
mod deadlock;
mod logging;
mod metrics;
mod mods;
mod panic;
mod restart;
mod runtime;
@@ -62,8 +63,9 @@ pub fn run_with_args(args: &Args) -> Result<()> {
Ok(())
}
/// Operate the server. This will start, run and stop the server within the
/// asynchronous runtime.
/// Operate the server normally in release-mode static builds. This will start,
/// run and stop the server within the asynchronous runtime.
#[cfg(any(not(conduwuit_mods), not(feature = "conduwuit_mods")))]
#[tracing::instrument(
name = "main",
parent = None,
@@ -112,3 +114,37 @@ async fn async_main(server: &Arc<Server>) -> Result<(), Error> {
debug_info!("Exit runtime");
Ok(())
}
/// Operate the server in developer-mode dynamic builds. This will start, run,
/// and hot-reload portions of the server as-needed before returning for an
/// actual shutdown. This is not available in release-mode or static builds.
#[cfg(all(conduwuit_mods, feature = "conduwuit_mods"))]
async fn async_main(server: &Arc<Server>) -> Result<(), Error> {
let mut starts = true;
let mut reloads = true;
while reloads {
if let Err(error) = mods::open(server).await {
error!("Loading router: {error}");
return Err(error);
}
let result = mods::run(server, starts).await;
if let Ok(result) = result {
(starts, reloads) = result;
}
let force = !reloads || result.is_err();
if let Err(error) = mods::close(server, force).await {
error!("Unloading router: {error}");
return Err(error);
}
if let Err(error) = result {
error!("{error}");
return Err(error);
}
}
debug_info!("Exit runtime");
Ok(())
}
+163
View File
@@ -0,0 +1,163 @@
#![cfg(all(conduwuit_mods, feature = "conduwuit_mods"))]
#[unsafe(no_link)]
extern crate conduwuit_service;
use std::{
future::Future,
pin::Pin,
sync::{Arc, atomic::Ordering},
};
use conduwuit_core::{Error, Result, debug, error, mods};
use conduwuit_service::Services;
use crate::Server;
type StartFuncResult = Pin<Box<dyn Future<Output = Result<Arc<Services>>> + Send>>;
type StartFuncProto = fn(&Arc<conduwuit_core::Server>) -> StartFuncResult;
type RunFuncResult = Pin<Box<dyn Future<Output = Result<()>> + Send>>;
type RunFuncProto = fn(&Arc<Services>) -> RunFuncResult;
type StopFuncResult = Pin<Box<dyn Future<Output = Result<()>> + Send>>;
type StopFuncProto = fn(Arc<Services>) -> StopFuncResult;
const RESTART_THRESH: &str = "conduwuit_service";
const MODULE_NAMES: &[&str] = &[
//"conduwuit_core",
"conduwuit_database",
"conduwuit_service",
"conduwuit_api",
"conduwuit_admin",
"conduwuit_router",
];
#[cfg(panic_trap)]
conduwuit_core::mod_init! {{
conduwuit_core::debug::set_panic_trap();
}}
pub(crate) async fn run(server: &Arc<Server>, starts: bool) -> Result<(bool, bool), Error> {
let main_lock = server.mods.read().await;
let main_mod = (*main_lock).last().expect("main module loaded");
if starts {
let start = main_mod.get::<StartFuncProto>("start")?;
match start(&server.server).await {
| Ok(services) => server.services.lock().await.insert(services),
| Err(error) => {
error!("Starting server: {error}");
return Err(error);
},
};
}
server.server.stopping.store(false, Ordering::Release);
let run = main_mod.get::<RunFuncProto>("run")?;
if let Err(error) = run(server
.services
.lock()
.await
.as_ref()
.expect("services initialized"))
.await
{
error!("Running server: {error}");
return Err(error);
}
let reloads = server.server.reloading.swap(false, Ordering::AcqRel);
let stops = !reloads || stale_from_mods(&main_lock)? <= restart_thresh();
let starts = reloads && stops;
if stops {
let stop = main_mod.get::<StopFuncProto>("stop")?;
if let Err(error) = stop(
server
.services
.lock()
.await
.take()
.expect("services initialized"),
)
.await
{
error!("Stopping server: {error}");
return Err(error);
}
}
Ok((starts, reloads))
}
pub(crate) async fn open(server: &Arc<Server>) -> Result<usize, Error> {
let mut mods_lock = server.mods.write().await;
let mods: &mut Vec<mods::Module> = &mut mods_lock;
debug!(
available = %available(),
loaded = %mods.len(),
"Loading modules",
);
for (i, name) in MODULE_NAMES.iter().enumerate() {
if mods.get(i).is_none() {
mods.push(mods::Module::from_name(name)?);
}
}
Ok(mods.len())
}
pub(crate) async fn close(server: &Arc<Server>, force: bool) -> Result<usize, Error> {
let stale = stale_count(server).await;
let mut mods_lock = server.mods.write().await;
let mods: &mut Vec<mods::Module> = &mut mods_lock;
debug!(
available = %available(),
loaded = %mods.len(),
stale = %stale,
force,
"Unloading modules",
);
while mods.last().is_some() {
let module = &mods.last().expect("module");
if force || module.deleted()? {
mods.pop();
} else {
break;
}
}
Ok(mods.len())
}
async fn stale_count(server: &Arc<Server>) -> usize {
let watermark = stale(server).await.unwrap_or(available());
available().saturating_sub(watermark)
}
async fn stale(server: &Arc<Server>) -> Result<usize, Error> {
let mods_lock = server.mods.read().await;
stale_from_mods(&mods_lock)
}
fn stale_from_mods(mods: &[mods::Module]) -> Result<usize, Error> {
for (i, module) in mods.iter().enumerate() {
if module.deleted()? {
return Ok(i);
}
}
Ok(mods.len())
}
fn restart_thresh() -> usize {
MODULE_NAMES
.iter()
.position(|&name| name.ends_with(RESTART_THRESH))
.unwrap_or(MODULE_NAMES.len())
}
const fn available() -> usize { MODULE_NAMES.len() }
+7
View File
@@ -27,6 +27,10 @@ pub(crate) struct Server {
#[cfg(feature = "sentry_telemetry")]
_sentry_guard: Option<::sentry::ClientInitGuard>,
#[cfg(all(conduwuit_mods, feature = "conduwuit_mods"))]
// Module instances; TODO: move to mods::loaded mgmt vector
pub(crate) mods: tokio::sync::RwLock<Vec<conduwuit_core::mods::Module>>,
}
impl Server {
@@ -83,6 +87,9 @@ impl Server {
#[cfg(feature = "sentry_telemetry")]
_sentry_guard: sentry_guard,
#[cfg(all(conduwuit_mods, feature = "conduwuit_mods"))]
mods: tokio::sync::RwLock::new(Vec::new()),
}))
}
+4 -1
View File
@@ -12,6 +12,7 @@ pub(super) async fn signal(server: Arc<Server>) {
use unix::SignalKind;
const CONSOLE: bool = cfg!(feature = "console");
const RELOADING: bool = cfg!(all(conduwuit_mods, feature = "conduwuit_mods", not(CONSOLE)));
let mut quit = unix::signal(SignalKind::quit()).expect("SIGQUIT handler");
let mut term = unix::signal(SignalKind::terminate()).expect("SIGTERM handler");
@@ -29,7 +30,9 @@ pub(super) async fn signal(server: Arc<Server>) {
}
warn!("Received {sig}");
let result = if matches!(sig, "SIGQUIT" | "SIGTERM") || (!CONSOLE && sig == "SIGINT") {
let result = if RELOADING && sig == "SIGINT" {
server.server.reload()
} else if matches!(sig, "SIGQUIT" | "SIGTERM") || (!CONSOLE && sig == "SIGINT") {
server.server.shutdown()
} else {
server.server.signal(sig)
+13 -3
View File
@@ -16,7 +16,11 @@ use conduwuit::{Error, Result, Server};
use conduwuit_service::Services;
use futures::{Future, FutureExt, TryFutureExt};
pub fn start(
conduwuit::mod_ctor! {}
conduwuit::mod_dtor! {}
#[unsafe(no_mangle)]
pub extern "Rust" fn start(
server: &Arc<Server>,
) -> Pin<Box<dyn Future<Output = Result<Arc<Services>>> + Send>> {
AssertUnwindSafe(run::start(server.clone()))
@@ -26,7 +30,10 @@ pub fn start(
.boxed()
}
pub fn stop(services: Arc<Services>) -> Pin<Box<dyn Future<Output = Result<()>> + Send>> {
#[unsafe(no_mangle)]
pub extern "Rust" fn stop(
services: Arc<Services>,
) -> Pin<Box<dyn Future<Output = Result<()>> + Send>> {
AssertUnwindSafe(run::stop(services))
.catch_unwind()
.map_err(Error::from_panic)
@@ -34,7 +41,10 @@ pub fn stop(services: Arc<Services>) -> Pin<Box<dyn Future<Output = Result<()>>
.boxed()
}
pub fn run(services: &Arc<Services>) -> Pin<Box<dyn Future<Output = Result<()>> + Send>> {
#[unsafe(no_mangle)]
pub extern "Rust" fn run(
services: &Arc<Services>,
) -> Pin<Box<dyn Future<Output = Result<()>> + Send>> {
AssertUnwindSafe(run::run(services.clone()))
.catch_unwind()
.map_err(Error::from_panic)
+1 -5
View File
@@ -8,11 +8,7 @@ use ruma::api::error::ErrorKind;
pub(crate) fn build(services: &Arc<Services>) -> (Router, Guard) {
let router = Router::<state::State>::new();
// SAFETY: `guard` is returned alongside `router` and held in `serve()`
// until the HTTP server has finished serving all requests and dropped
// `router`, ensuring no `State` copy can be dereferenced after `Services`
// is dropped.
let (state, guard) = unsafe { state::create(services.clone()) };
let (state, guard) = state::create(services.clone());
let router = conduwuit_api::router::build(router, state)
.merge(conduwuit_web::build(services))
.fallback(not_found)
+9 -3
View File
@@ -44,13 +44,19 @@ pub mod transactions;
pub mod uiaa;
pub mod users;
use std::{collections::BTreeMap, sync::LazyLock};
use std::collections::BTreeMap;
use conduwuit::matrix::versions::versions;
use ruma::api::SupportedVersions;
pub(crate) use service::{Args, Dep, Service};
pub use crate::services::Services;
conduwuit::mod_ctor! {}
conduwuit::mod_dtor! {}
use std::sync::LazyLock;
use conduwuit::matrix::versions::versions;
use ruma::api::SupportedVersions;
pub static SUPPORTED_VERSIONS: LazyLock<SupportedVersions> =
LazyLock::new(|| SupportedVersions::from_parts(&versions(), &BTreeMap::new()));
+3 -5
View File
@@ -4,18 +4,16 @@ use conduwuit::{Err, Event, Pdu, Result, is_not_empty, utils::ReadyExt};
use database::{Json, serialize_key};
use futures::StreamExt;
use ruma::{
OwnedServerName, RoomId, UserId,
CanonicalJsonObject, OwnedServerName, RoomId, UserId,
api::federation::membership::RawStrippedState,
events::{
AnyStrippedStateEvent, GlobalAccountDataEventType, RoomAccountDataEventType,
StateEventType,
GlobalAccountDataEventType, RoomAccountDataEventType, StateEventType,
direct::DirectEvent,
room::{
create::RoomCreateEventContent,
member::{MembershipState, RoomMemberEventContent},
},
},
serde::Raw,
};
use ruminuwuity::invite_permission_config::FilterLevel;
@@ -274,7 +272,7 @@ impl super::Service {
&self,
user_id: &UserId,
room_id: &RoomId,
knocked_state: Option<Vec<Raw<AnyStrippedStateEvent>>>,
knocked_state: Option<Vec<&CanonicalJsonObject>>,
) {
let userroom_id = (user_id, room_id);
let userroom_id = serialize_key(userroom_id).expect("failed to serialize userroom_id");
+1 -9
View File
@@ -11,15 +11,7 @@ pub struct Guard {
services: Arc<Services>,
}
/// Create a raw-pointer `State` handle paired with a `Guard` that keeps
/// `Services` alive.
///
/// # Safety
/// `State` is `Copy` and dereferences its raw pointer without reference
/// counting. The caller must ensure that no `State` (nor any copy of it) is
/// dereferenced after `Guard` and all other `Arc<Services>` clones have been
/// dropped.
pub unsafe fn create(services: Arc<Services>) -> (State, Guard) {
pub fn create(services: Arc<Services>) -> (State, Guard) {
let state = State {
services: Arc::into_raw(services.clone()),
};