docs: Generated database docs

.typos.toml

@@ -1,5 +1,5 @@
 [files]
-extend-exclude = ["*.csr", "*.lock"]
+extend-exclude = ["*.csr"]
 
 [default.extend-words]
 "allocatedp" = "allocatedp"

Cargo.lock

@@ -771,7 +771,7 @@ dependencies = [
 
 [[package]]
 name = "conduwuit"
-version = "0.5.0-rc.5"
+version = "0.5.0-rc.6"
 dependencies = [
  "clap",
  "conduwuit_admin",
@@ -800,7 +800,7 @@ dependencies = [
 
 [[package]]
 name = "conduwuit_admin"
-version = "0.5.0-rc.5"
+version = "0.5.0-rc.6"
 dependencies = [
  "clap",
  "conduwuit_api",
@@ -821,7 +821,7 @@ dependencies = [
 
 [[package]]
 name = "conduwuit_api"
-version = "0.5.0-rc.5"
+version = "0.5.0-rc.6"
 dependencies = [
  "async-trait",
  "axum",
@@ -853,14 +853,14 @@ dependencies = [
 
 [[package]]
 name = "conduwuit_build_metadata"
-version = "0.5.0-rc.5"
+version = "0.5.0-rc.6"
 dependencies = [
  "built 0.8.0",
 ]
 
 [[package]]
 name = "conduwuit_core"
-version = "0.5.0-rc.5"
+version = "0.5.0-rc.6"
 dependencies = [
  "argon2",
  "arrayvec",
@@ -919,7 +919,7 @@ dependencies = [
 
 [[package]]
 name = "conduwuit_database"
-version = "0.5.0-rc.5"
+version = "0.5.0-rc.6"
 dependencies = [
  "async-channel",
  "conduwuit_core",
@@ -937,7 +937,7 @@ dependencies = [
 
 [[package]]
 name = "conduwuit_macros"
-version = "0.5.0-rc.5"
+version = "0.5.0-rc.6"
 dependencies = [
  "itertools 0.14.0",
  "proc-macro2",
@@ -947,7 +947,7 @@ dependencies = [
 
 [[package]]
 name = "conduwuit_router"
-version = "0.5.0-rc.5"
+version = "0.5.0-rc.6"
 dependencies = [
  "axum",
  "axum-client-ip",
@@ -981,7 +981,7 @@ dependencies = [
 
 [[package]]
 name = "conduwuit_service"
-version = "0.5.0-rc.5"
+version = "0.5.0-rc.6"
 dependencies = [
  "async-trait",
  "base64 0.22.1",
@@ -1018,7 +1018,7 @@ dependencies = [
 
 [[package]]
 name = "conduwuit_web"
-version = "0.5.0-rc.5"
+version = "0.5.0-rc.6"
 dependencies = [
  "askama",
  "axum",
@@ -3695,7 +3695,7 @@ dependencies = [
 [[package]]
 name = "ruma"
 version = "0.10.1"
-source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=a48665b682be1016cea53ea5e7787442dfe7c1de#a48665b682be1016cea53ea5e7787442dfe7c1de"
+source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=d6870a7fb7f6cccff63f7fd0ff6c581bad80e983#d6870a7fb7f6cccff63f7fd0ff6c581bad80e983"
 dependencies = [
  "assign",
  "js_int",
@@ -3715,7 +3715,7 @@ dependencies = [
 [[package]]
 name = "ruma-appservice-api"
 version = "0.10.0"
-source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=a48665b682be1016cea53ea5e7787442dfe7c1de#a48665b682be1016cea53ea5e7787442dfe7c1de"
+source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=d6870a7fb7f6cccff63f7fd0ff6c581bad80e983#d6870a7fb7f6cccff63f7fd0ff6c581bad80e983"
 dependencies = [
  "js_int",
  "ruma-common",
@@ -3727,7 +3727,7 @@ dependencies = [
 [[package]]
 name = "ruma-client-api"
 version = "0.18.0"
-source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=a48665b682be1016cea53ea5e7787442dfe7c1de#a48665b682be1016cea53ea5e7787442dfe7c1de"
+source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=d6870a7fb7f6cccff63f7fd0ff6c581bad80e983#d6870a7fb7f6cccff63f7fd0ff6c581bad80e983"
 dependencies = [
  "as_variant",
  "assign",
@@ -3750,7 +3750,7 @@ dependencies = [
 [[package]]
 name = "ruma-common"
 version = "0.13.0"
-source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=a48665b682be1016cea53ea5e7787442dfe7c1de#a48665b682be1016cea53ea5e7787442dfe7c1de"
+source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=d6870a7fb7f6cccff63f7fd0ff6c581bad80e983#d6870a7fb7f6cccff63f7fd0ff6c581bad80e983"
 dependencies = [
  "as_variant",
  "base64 0.22.1",
@@ -3782,7 +3782,7 @@ dependencies = [
 [[package]]
 name = "ruma-events"
 version = "0.28.1"
-source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=a48665b682be1016cea53ea5e7787442dfe7c1de#a48665b682be1016cea53ea5e7787442dfe7c1de"
+source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=d6870a7fb7f6cccff63f7fd0ff6c581bad80e983#d6870a7fb7f6cccff63f7fd0ff6c581bad80e983"
 dependencies = [
  "as_variant",
  "indexmap 2.9.0",
@@ -3807,7 +3807,7 @@ dependencies = [
 [[package]]
 name = "ruma-federation-api"
 version = "0.9.0"
-source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=a48665b682be1016cea53ea5e7787442dfe7c1de#a48665b682be1016cea53ea5e7787442dfe7c1de"
+source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=d6870a7fb7f6cccff63f7fd0ff6c581bad80e983#d6870a7fb7f6cccff63f7fd0ff6c581bad80e983"
 dependencies = [
  "bytes",
  "headers",
@@ -3829,7 +3829,7 @@ dependencies = [
 [[package]]
 name = "ruma-identifiers-validation"
 version = "0.9.5"
-source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=a48665b682be1016cea53ea5e7787442dfe7c1de#a48665b682be1016cea53ea5e7787442dfe7c1de"
+source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=d6870a7fb7f6cccff63f7fd0ff6c581bad80e983#d6870a7fb7f6cccff63f7fd0ff6c581bad80e983"
 dependencies = [
  "js_int",
  "thiserror 2.0.12",
@@ -3838,7 +3838,7 @@ dependencies = [
 [[package]]
 name = "ruma-identity-service-api"
 version = "0.9.0"
-source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=a48665b682be1016cea53ea5e7787442dfe7c1de#a48665b682be1016cea53ea5e7787442dfe7c1de"
+source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=d6870a7fb7f6cccff63f7fd0ff6c581bad80e983#d6870a7fb7f6cccff63f7fd0ff6c581bad80e983"
 dependencies = [
  "js_int",
  "ruma-common",
@@ -3848,7 +3848,7 @@ dependencies = [
 [[package]]
 name = "ruma-macros"
 version = "0.13.0"
-source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=a48665b682be1016cea53ea5e7787442dfe7c1de#a48665b682be1016cea53ea5e7787442dfe7c1de"
+source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=d6870a7fb7f6cccff63f7fd0ff6c581bad80e983#d6870a7fb7f6cccff63f7fd0ff6c581bad80e983"
 dependencies = [
  "cfg-if",
  "proc-macro-crate",
@@ -3863,7 +3863,7 @@ dependencies = [
 [[package]]
 name = "ruma-push-gateway-api"
 version = "0.9.0"
-source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=a48665b682be1016cea53ea5e7787442dfe7c1de#a48665b682be1016cea53ea5e7787442dfe7c1de"
+source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=d6870a7fb7f6cccff63f7fd0ff6c581bad80e983#d6870a7fb7f6cccff63f7fd0ff6c581bad80e983"
 dependencies = [
  "js_int",
  "ruma-common",
@@ -3875,7 +3875,7 @@ dependencies = [
 [[package]]
 name = "ruma-signatures"
 version = "0.15.0"
-source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=a48665b682be1016cea53ea5e7787442dfe7c1de#a48665b682be1016cea53ea5e7787442dfe7c1de"
+source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=d6870a7fb7f6cccff63f7fd0ff6c581bad80e983#d6870a7fb7f6cccff63f7fd0ff6c581bad80e983"
 dependencies = [
  "base64 0.22.1",
  "ed25519-dalek",

Cargo.toml

@@ -21,7 +21,7 @@ license = "Apache-2.0"
 readme = "README.md"
 repository = "https://forgejo.ellis.link/continuwuation/continuwuity"
 rust-version = "1.86.0"
-version = "0.5.0-rc.5"
+version = "0.5.0-rc.6"
 
 [workspace.metadata.crane]
 name = "conduwuit"
@@ -350,7 +350,7 @@ version = "0.1.2"
 [workspace.dependencies.ruma]
 git = "https://forgejo.ellis.link/continuwuation/ruwuma"
 #branch = "conduwuit-changes"
-rev = "a48665b682be1016cea53ea5e7787442dfe7c1de"
+rev = "d6870a7fb7f6cccff63f7fd0ff6c581bad80e983"
 features = [
     "compat",
     "rand",
@@ -381,7 +381,7 @@ features = [
     "unstable-msc4121",
     "unstable-msc4125",
     "unstable-msc4186",
-    "unstable-msc4203", # sending to-device events to appservices
+    "unstable-msc4203", # sending to-device events to appservices 
     "unstable-msc4210", # remove legacy mentions
     "unstable-extensible-events",
     "unstable-pdu",

docs/DATABASE_MERMAID_DIAGRAMS.md

@@ -0,0 +1,350 @@
+# Continuwuity Database Mermaid Diagrams
+
+This document contains visual representations of the Continuwuity database schema using Mermaid diagrams.
+
+## 1. Core Event Storage Architecture
+
+```mermaid
+graph TD
+    A[Matrix Event ID<br/>48 bytes] --> B[eventid_shorteventid]
+    B --> C[Short Event ID<br/>8 bytes]
+    C --> D[shorteventid_eventid]
+    D --> A
+    
+    A --> E[eventid_pduid]
+    E --> F[PDU ID<br/>16 bytes]
+    F --> G[pduid_pdu<br/>Main Event Storage]
+    
+    A --> H[eventid_outlierpdu<br/>Outlier Events]
+    
+    C --> I[shorteventid_authchain<br/>Authorization Chains]
+    C --> J[shorteventid_shortstatehash<br/>Event → State Mapping]
+    
+    G -.->|Shared Cache| H
+    
+    style G fill:#e1f5fe
+    style H fill:#e1f5fe
+    style A fill:#fff3e0
+    style C fill:#f3e5f5
+```
+
+## 2. Room State Management System
+
+```mermaid
+graph TD
+    A[Room State Key] --> B[statekey_shortstatekey]
+    B --> C[Short State Key<br/>8 bytes]
+    C --> D[shortstatekey_statekey]
+    D --> A
+    
+    E[Full State Hash] --> F[statehash_shortstatehash]
+    F --> G[Short State Hash<br/>8 bytes]
+    
+    G --> H[shortstatehash_statediff<br/>State Differences]
+    G --> I[roomid_shortstatehash<br/>Current Room State]
+    G --> J[roomsynctoken_shortstatehash<br/>Sync Token Mapping]
+    
+    K[Room ID] --> I
+    L[Sync Token] --> J
+    
+    style G fill:#e8f5e8
+    style I fill:#fff3e0
+    style J fill:#f0f4ff
+```
+
+## 3. User Authentication and Identity Flow
+
+```mermaid
+graph TD
+    A[User ID] --> B[userid_password<br/>Password Hashes]
+    
+    A --> C[userid_displayname]
+    A --> D[userid_avatarurl]
+    D --> E[userid_blurhash]
+    A --> F[useridprofilekey_value<br/>Custom Profile]
+    
+    G[Access Token] --> H[token_userdeviceid]
+    H --> I[User + Device ID]
+    I --> J[userdeviceid_token]
+    J --> G
+    
+    I --> K[userdeviceid_metadata<br/>Device Info]
+    I --> L[userdevicesessionid_uiaainfo<br/>Auth Sessions]
+    I --> M[userdevicetxnid_response<br/>Transaction Cache]
+    
+    N[OpenID Token] --> O[openidtoken_expiresatuserid]
+    P[Login Token] --> Q[logintoken_expiresatuserid]
+    
+    style H fill:#e1f5fe
+    style J fill:#e1f5fe
+    style B fill:#ffebee
+```
+
+## 4. Room Membership Bidirectional System
+
+```mermaid
+graph TD
+    A[Room ID + User ID] --> B[roomuserid_joined<br/>Room → User View]
+    C[User ID + Room ID] --> D[userroomid_joined<br/>User → Room View]
+    
+    B -.->|Bidirectional| D
+    
+    A --> E[roomuserid_invitecount]
+    C --> F[userroomid_invitestate]
+    E -.->|Related| F
+    
+    A --> G[roomuserid_leftcount]
+    C --> H[userroomid_leftstate]
+    G -.->|Related| H
+    
+    A --> I[roomuserid_knockedcount]
+    C --> J[userroomid_knockedstate]
+    I -.->|Related| J
+    
+    K[Room ID] --> L[roomid_joinedcount<br/>Total Joined]
+    K --> M[roomid_invitedcount<br/>Total Invited]
+    
+    N[Historical] --> O[roomuseroncejoinedids<br/>Ever Joined Tracking]
+    
+    style B fill:#e8f5e8
+    style D fill:#e8f5e8
+    style L fill:#fff3e0
+    style M fill:#fff3e0
+```
+
+## 5. Cryptography and Key Management Chain
+
+```mermaid
+graph TD
+    A[User ID] --> B[userid_devicelistversion<br/>Device List Changes]
+    
+    A --> C[userid_masterkeyid<br/>Master Signing Key]
+    A --> D[userid_selfsigningkeyid<br/>Self Signing Key]
+    A --> E[userid_usersigningkeyid<br/>User Signing Key]
+    
+    F[Key ID] --> G[keyid_key<br/>Actual Keys]
+    
+    C --> G
+    D --> G
+    E --> G
+    
+    H[Key Change ID] --> I[keychangeid_userid<br/>Change Notifications]
+    
+    J[One-Time Key ID] --> K[onetimekeyid_onetimekeys<br/>OTK Storage]
+    A --> L[userid_lastonetimekeyupdate<br/>Last OTK Update]
+    
+    M[Backup ID] --> N[backupid_algorithm<br/>Backup Algorithm]
+    M --> O[backupid_etag<br/>Backup Versioning]
+    P[Backup Key ID] --> Q[backupkeyid_backup<br/>Backed Up Keys]
+    
+    style G fill:#e1f5fe
+    style I fill:#fff3e0
+    style K fill:#f3e5f5
+    style Q fill:#e8f5e8
+```
+
+## 6. Federation and Server Communication
+
+```mermaid
+graph TD
+    A[Server Name] --> B[servername_destination<br/>Cached Destinations]
+    A --> C[servername_override<br/>Cached Overrides]
+    A --> D[server_signingkeys<br/>Federation Keys]
+    A --> E[servername_educount<br/>EDU Counters]
+    
+    F[Server + Event] --> G[servernameevent_data<br/>Server Events]
+    H[Server Current] --> I[servercurrentevent_data<br/>Current State]
+    
+    J[Room ID] --> K[roomserverids<br/>Room → Servers]
+    L[Server Name] --> M[serverroomids<br/>Server → Rooms]
+    
+    K -.->|Bidirectional| M
+    
+    N[Room ID] --> O[roomid_inviteviaservers<br/>Invitation Routing]
+    
+    style B fill:#e1f5fe
+    style C fill:#e1f5fe
+    style K fill:#e8f5e8
+    style M fill:#e8f5e8
+```
+
+## 7. Push Notifications and Read Tracking
+
+```mermaid
+graph TD
+    A[Sender Key] --> B[senderkey_pusher<br/>Push Endpoints]
+    C[Push Key] --> D[pushkey_deviceid<br/>Device Mapping]
+    
+    B -.->|Related| D
+    
+    E[Read Receipt ID] --> F[readreceiptid_readreceipt<br/>Public Receipts]
+    
+    G[Room + User] --> H[roomuserid_privateread<br/>Private Read Markers]
+    G --> I[roomuserid_lastprivatereadupdate<br/>Update Timestamps]
+    
+    J[User + Room] --> K[userroomid_highlightcount<br/>Mention Count]
+    J --> L[userroomid_notificationcount<br/>Notification Count]
+    
+    style F fill:#e8f5e8
+    style H fill:#f3e5f5
+    style K fill:#fff3e0
+    style L fill:#fff3e0
+```
+
+## 8. Media and Content Management
+
+```mermaid
+graph TD
+    A[Media ID] --> B[mediaid_file<br/>File Metadata]
+    A --> C[mediaid_user<br/>Uploader Tracking]
+    
+    B -.->|Related| C
+    
+    D[URL] --> E[url_previews<br/>Preview Cache]
+    
+    F[User ID] --> G[userfilterid_filter<br/>Sync Filters]
+    H[Lazy Load] --> I[lazyloadedids<br/>Member Event Tracking]
+    
+    style B fill:#e1f5fe
+    style C fill:#e1f5fe
+    style E fill:#f0f4ff
+```
+
+## 9. Account Data and Presence System
+
+```mermaid
+graph TD
+    A[Room + User + Type] --> B[roomusertype_roomuserdataid<br/>Account Data Index]
+    B --> C[Room User Data ID]
+    C --> D[roomuserdataid_accountdata<br/>Actual Account Data]
+    
+    E[User ID] --> F[userid_presenceid<br/>Presence Mapping]
+    F --> G[Presence ID]
+    G --> H[presenceid_presence<br/>Presence Data]
+    
+    I[To-Device ID] --> J[todeviceid_events<br/>Device Messages]
+    
+    style D fill:#e8f5e8
+    style H fill:#f3e5f5
+    style J fill:#fff3e0
+```
+
+## 10. Global Configuration and Access Control
+
+```mermaid
+graph TD
+    A[Global Config] --> B[global<br/>Server Settings]
+    
+    C[Room Categories] --> D[publicroomids<br/>Public Rooms]
+    C --> E[bannedroomids<br/>Banned Rooms]
+    C --> F[disabledroomids<br/>Disabled Rooms]
+    
+    G[App Service ID] --> H[id_appserviceregistrations<br/>Application Services]
+    
+    I[Token Management] --> J[tokenids<br/>Token Allocation]
+    
+    K[Relations] --> L[tofrom_relation<br/>Event Relations]
+    K --> M[threadid_userids<br/>Thread Participants]
+    K --> N[referencedevents<br/>Referenced Events]
+    K --> O[softfailedeventids<br/>Failed Events]
+    
+    style B fill:#e1f5fe
+    style D fill:#e8f5e8
+    style E fill:#ffebee
+    style F fill:#ffebee
+```
+
+## 11. Complete System Overview
+
+```mermaid
+graph TB
+    subgraph "Identity Management"
+        UI[User Identity]
+        UA[User Auth]
+        UD[User Devices]
+        UP[User Profile]
+    end
+    
+    subgraph "Event Storage"
+        ES[Event Storage]
+        EID[Event ID Mapping]
+        EO[Outlier Events]
+    end
+    
+    subgraph "Room Management"
+        RS[Room State]
+        RM[Room Membership]
+        RMeta[Room Metadata]
+    end
+    
+    subgraph "Cryptography"
+        DK[Device Keys]
+        CS[Cross Signing]
+        KB[Key Backups]
+    end
+    
+    subgraph "Federation"
+        FS[Federation Servers]
+        FK[Federation Keys]
+        FE[Federation Events]
+    end
+    
+    subgraph "Communication"
+        PUSH[Push Notifications]
+        RT[Read Tracking]
+        DM[Device Messages]
+    end
+    
+    subgraph "Content"
+        MC[Media Content]
+        UP2[URL Previews]
+        AD[Account Data]
+    end
+    
+    UI --> UA
+    UA --> UD
+    UI --> UP
+    
+    ES --> EID
+    ES --> EO
+    EID --> RS
+    
+    RS --> RM
+    RM --> RMeta
+    
+    UD --> DK
+    DK --> CS
+    CS --> KB
+    
+    RM --> FS
+    FS --> FK
+    FK --> FE
+    
+    UD --> PUSH
+    RM --> RT
+    UD --> DM
+    
+    UI --> MC
+    MC --> UP2
+    UI --> AD
+    
+    style UI fill:#e8f5e8
+    style ES fill:#e1f5fe
+    style RS fill:#f3e5f5
+    style DK fill:#fff3e0
+    style FS fill:#f0f4ff
+```
+
+## Diagram Legend
+
+- **Blue boxes** (`#e1f5fe`): Core storage tables
+- **Green boxes** (`#e8f5e8`): Membership and relationship tables  
+- **Purple boxes** (`#f3e5f5`): ID mapping and compression tables
+- **Orange boxes** (`#fff3e0`): Count and metadata tables
+- **Light blue boxes** (`#f0f4ff`): Sync and federation tables
+- **Red boxes** (`#ffebee`): Access control and security tables
+- **Solid arrows**: Direct relationships
+- **Dotted arrows**: Bidirectional or related tables
+- **Shared Cache notation**: Tables that share memory pools
+
+These diagrams show how Continuwuity's 89 database tables interconnect to provide a complete Matrix homeserver implementation with optimized storage patterns and efficient relationship management.

docs/DATABASE_RELATIONSHIPS.md

@@ -0,0 +1,354 @@
+# Continuwuity Database Column Relationships
+
+This document analyzes how the 89 database columns in Continuwuity relate to each other, showing the data flow and dependencies between tables.
+
+## Core Identity Mapping System
+
+### Event ID Management
+
+The system uses a sophisticated event ID mapping system to optimize storage:
+
+```
+eventid_shorteventid ←→ shorteventid_eventid
+         ↓                      ↓
+   eventid_pduid         shorteventid_authchain
+         ↓                      ↓
+     pduid_pdu         shorteventid_shortstatehash
+         ↓
+eventid_outlierpdu
+```
+
+**Relationships:**
+
+- `eventid_shorteventid` + `shorteventid_eventid`: Bidirectional mapping between full Matrix event IDs (48 bytes) and compact short IDs (8 bytes)
+- `eventid_pduid`: Maps event IDs to PDU IDs (16-byte internal identifiers)
+- `pduid_pdu`: Main event storage using PDU IDs as keys
+- `eventid_outlierpdu`: Stores events not yet part of the timeline (outliers)
+- `shorteventid_authchain`: Authorization chains using short event IDs
+- `shorteventid_shortstatehash`: Links events to room state using short IDs
+
+### Room ID Management
+
+Similar optimization for room identifiers:
+
+```
+roomid_shortroomid
+       ↓
+   (used in keys for room-related tables)
+```
+
+### State Management
+
+Complex state tracking with compression:
+
+```
+statekey_shortstatekey ←→ shortstatekey_statekey
+         ↓
+statehash_shortstatehash
+         ↓
+shortstatehash_statediff
+         ↓
+roomid_shortstatehash
+```
+
+**Relationships:**
+
+- `statekey_shortstatekey` + `shortstatekey_statekey`: Bidirectional mapping for state keys
+- `statehash_shortstatehash`: Maps full state hashes to 8-byte compressed versions
+- `shortstatehash_statediff`: Stores state differences between versions
+- `roomid_shortstatehash`: Current state hash for each room
+
+## User and Authentication Flow
+
+### User Authentication Chain
+
+```
+userid_password → token_userdeviceid ←→ userdeviceid_token
+                         ↓
+                 userdeviceid_metadata
+                         ↓
+              userdevicesessionid_uiaainfo
+```
+
+**Relationships:**
+
+- `userid_password`: Stores user password hashes
+- `token_userdeviceid` + `userdeviceid_token`: Bidirectional mapping between access tokens and devices
+- `userdeviceid_metadata`: Device information (name, type, etc.)
+- `userdevicesessionid_uiaainfo`: User-Interactive Authentication session data
+
+### User Profile Data
+
+```
+userid_displayname
+userid_avatarurl → userid_blurhash
+useridprofilekey_value
+```
+
+**Relationships:**
+
+- Profile data is stored separately per attribute
+- `userid_blurhash` complements `userid_avatarurl` for progressive loading
+
+### Token Management
+
+```
+openidtoken_expiresatuserid
+logintoken_expiresatuserid  
+tokenids
+```
+
+**Relationships:**
+
+- Separate token types have separate expiration tracking
+- `tokenids` manages token ID allocation
+
+## Room Membership System
+
+### Membership State Tracking
+
+```
+roomuserid_joined ←→ userroomid_joined
+roomuserid_invitecount ←→ userroomid_invitestate
+roomuserid_leftcount ←→ userroomid_leftstate
+roomuserid_knockedcount ←→ userroomid_knockedstate
+```
+
+**Relationships:**
+
+- Bidirectional indexes: room→user and user→room perspectives
+- Count tables track membership transitions
+- State tables store membership event data
+
+### Room Counts and Metadata
+
+```
+roomid_joinedcount ← roomuserid_joined
+roomid_invitedcount ← roomuserid_invitecount
+roomuseroncejoinedids (historical tracking)
+```
+
+**Relationships:**
+
+- Count tables are derived from individual membership records
+- Historical tracking for users who ever joined
+
+### Federation Integration
+
+```
+roomserverids ←→ serverroomids
+roomid_inviteviaservers
+```
+
+**Relationships:**
+
+- Bidirectional tracking of which servers participate in which rooms
+- Via servers for invitation routing
+
+## Cryptography and Security
+
+### Device Key Management
+
+```
+userid_devicelistversion
+         ↓
+keyid_key ← userid_masterkeyid
+         ↓     userid_selfsigningkeyid
+keychangeid_userid ← userid_usersigningkeyid
+         ↓
+onetimekeyid_onetimekeys
+         ↓
+userid_lastonetimekeyupdate
+```
+
+**Relationships:**
+
+- Device list versions track changes requiring key updates
+- Different key types stored separately with references from user records
+- Key changes trigger notifications
+- One-time keys managed with update timestamps
+
+### Key Backup System
+
+```
+backupid_algorithm
+backupid_etag → backupkeyid_backup
+```
+
+**Relationships:**
+
+- Backup metadata (algorithm, versioning) linked to actual backed-up keys
+
+## Push Notifications and Read Tracking
+
+### Push Infrastructure
+
+```
+senderkey_pusher ←→ pushkey_deviceid
+```
+
+**Relationships:**
+
+- Bidirectional mapping between push keys and devices
+
+### Read Receipt System
+
+```
+readreceiptid_readreceipt
+roomuserid_privateread
+roomuserid_lastprivatereadupdate
+userroomid_highlightcount
+userroomid_notificationcount
+```
+
+**Relationships:**
+
+- Public read receipts vs private read markers
+- Highlight/notification counts per user-room pair
+- Update tracking for private reads
+
+## Media and Content
+
+### Media Storage
+
+```
+mediaid_file ←→ mediaid_user
+url_previews
+```
+
+**Relationships:**
+
+- File metadata linked to uploader tracking
+- URL previews cached separately
+
+## Sync and Timeline
+
+### Sync Token Management
+
+```
+roomsynctoken_shortstatehash
+lazyloadedids
+```
+
+**Relationships:**
+
+- Sync tokens map to room state for efficient delta computation
+- Lazy loading tracking for member events
+
+### Event Relations
+
+```
+tofrom_relation
+threadid_userids
+referencedevents
+softfailedeventids
+```
+
+**Relationships:**
+
+- Event relations track replies, edits, reactions
+- Thread participant tracking
+- Referenced events and soft failures
+
+## Federation and Server Management
+
+### Server Discovery and Communication
+
+```
+servername_destination (cached)
+servername_override (cached)
+server_signingkeys
+servername_educount
+servercurrentevent_data
+servernameevent_data
+```
+
+**Relationships:**
+
+- Destination resolution with caching
+- Server signing keys for federation
+- EDU (Ephemeral Data Unit) counting
+- Current and historical server events
+
+## Account Data and Presence
+
+### Account Data Storage
+
+```
+roomusertype_roomuserdataid → roomuserdataid_accountdata
+userid_presenceid → presenceid_presence
+```
+
+**Relationships:**
+
+- Account data indexed by room+user+type, pointing to actual data
+- Presence data separated from user records with ID mapping
+
+## Global Configuration
+
+### Application Services
+
+```
+id_appserviceregistrations
+```
+
+### Global Settings
+
+```
+global
+publicroomids
+bannedroomids
+disabledroomids
+```
+
+**Relationships:**
+
+- Global server configuration
+- Room access control lists
+
+## Performance Optimizations
+
+### Shared Cache Relationships
+
+- `eventid_outlierpdu` and `pduid_pdu` share cache because they both store PDU data
+- Related tables are grouped for memory efficiency
+
+### Transaction Management
+
+```
+userdevicetxnid_response
+todeviceid_events
+```
+
+**Relationships:**
+
+- Transaction ID response caching
+- To-device message queuing
+
+## Data Flow Examples
+
+### Sending a Message
+
+1. `pduid_pdu` ← stores the PDU
+2. `eventid_pduid` ← maps event ID to PDU ID
+3. `eventid_shorteventid` ← creates short ID mapping
+4. `shorteventid_shortstatehash` ← links to room state
+5. `userroomid_notificationcount` ← updates notification counts
+6. `readreceiptid_readreceipt` ← processes read receipts
+
+### User Login
+
+1. `userid_password` ← validates credentials
+2. `userdeviceid_token` ← creates device token
+3. `token_userdeviceid` ← creates reverse mapping
+4. `userdeviceid_metadata` ← stores device info
+
+### Room Join
+
+1. `roomuserid_joined` ← records membership
+2. `userroomid_joined` ← creates reverse index
+3. `roomid_joinedcount` ← updates room count
+4. `roomuseroncejoinedids` ← historical tracking
+5. `roomserverids` ← federation tracking
+
+This relational structure allows Continuwuity to efficiently handle Matrix protocol operations while maintaining data consistency and enabling fast lookups from multiple perspectives.

docs/DATABASE_SCHEMA.md

@@ -0,0 +1,485 @@
+# Continuwuity Database Schema Documentation
+
+Continuwuity is a Matrix protocol implementation using RocksDB as its storage backend. The database is organized into column families (called "Maps" in the codebase), each serving specific purposes in the Matrix homeserver functionality.
+
+| Table Name | Access Pattern | Key Size | Value Size | Description |
+|------------|---------------|----------|------------|-------------|
+| `alias_roomid` | RANDOM_SMALL | - | - | Maps room alias to room ID |
+| `alias_userid` | RANDOM_SMALL | - | - | Maps room alias to user ID |
+| `aliasid_alias` | RANDOM_SMALL | - | - | Maps alias ID to alias string |
+| `backupid_algorithm` | RANDOM_SMALL | - | - | Key backup algorithms |
+| `backupid_etag` | RANDOM_SMALL | - | - | Key backup ETags |
+| `backupkeyid_backup` | RANDOM_SMALL | - | - | Backed up keys |
+| `bannedroomids` | RANDOM_SMALL | - | - | Set of banned room IDs |
+| `disabledroomids` | RANDOM_SMALL | - | - | Set of disabled room IDs |
+| `eventid_outlierpdu` | RANDOM | 48 bytes | 1488 bytes | Outlier PDUs (shared cache with pduid_pdu) |
+| `eventid_pduid` | RANDOM | 48 bytes | 16 bytes | Event ID to PDU ID mapping |
+| `eventid_shorteventid` | RANDOM | 48 bytes | 8 bytes | Event ID to short event ID |
+| `global` | RANDOM_SMALL | - | - | Global server configuration |
+| `id_appserviceregistrations` | RANDOM_SMALL | - | - | Application service registrations |
+| `keychangeid_userid` | RANDOM | - | - | Key change notifications |
+| `keyid_key` | RANDOM_SMALL | - | - | Cryptographic keys |
+| `lazyloadedids` | RANDOM_SMALL | - | - | Lazy-loaded member events |
+| `mediaid_file` | RANDOM_SMALL | - | - | Media file metadata |
+| `mediaid_user` | RANDOM_SMALL | - | - | Media uploader tracking |
+| `onetimekeyid_onetimekeys` | RANDOM_SMALL | - | - | One-time keys |
+| `pduid_pdu` | SEQUENTIAL | 16 bytes | 1520 bytes | Main PDU storage (shared cache with eventid_outlierpdu) |
+| `publicroomids` | RANDOM_SMALL | - | - | Public room IDs |
+| `pushkey_deviceid` | RANDOM_SMALL | - | - | Push key to device mapping |
+| `presenceid_presence` | SEQUENTIAL_SMALL | - | - | User presence data |
+| `readreceiptid_readreceipt` | RANDOM | - | - | Read receipts |
+| `referencedevents` | RANDOM | - | - | Referenced events |
+| `roomid_invitedcount` | RANDOM_SMALL | - | - | Room invited user count |
+| `roomid_inviteviaservers` | RANDOM_SMALL | - | - | Room invite via servers |
+| `roomid_joinedcount` | RANDOM_SMALL | - | - | Room joined user count |
+| `roomid_pduleaves` | RANDOM_SMALL | - | - | PDU leaves per room |
+| `roomid_shortroomid` | RANDOM_SMALL | - | 8 bytes | Room ID to short room ID |
+| `roomid_shortstatehash` | RANDOM_SMALL | - | 8 bytes | Room ID to state hash |
+| `roomserverids` | RANDOM_SMALL | - | - | Server IDs per room |
+| `roomsynctoken_shortstatehash` | SEQUENTIAL | - | 8 bytes | Sync token to state hash (special compression) |
+| `roomuserdataid_accountdata` | RANDOM_SMALL | - | - | Room account data |
+| `roomuserid_invitecount` | RANDOM_SMALL | - | 8 bytes | Room-user invite count |
+| `roomuserid_joined` | RANDOM_SMALL | - | - | Room-user joined status |
+| `roomuserid_lastprivatereadupdate` | RANDOM_SMALL | - | - | Last private read update |
+| `roomuserid_leftcount` | RANDOM | - | 8 bytes | Room-user leave count |
+| `roomuserid_knockedcount` | RANDOM_SMALL | - | 8 bytes | Room-user knock count |
+| `roomuserid_privateread` | RANDOM_SMALL | - | - | Private read markers |
+| `roomuseroncejoinedids` | RANDOM | - | - | Users who ever joined |
+| `roomusertype_roomuserdataid` | RANDOM_SMALL | - | - | Account data type mapping |
+| `senderkey_pusher` | RANDOM_SMALL | - | - | Push notification senders |
+| `server_signingkeys` | RANDOM | - | - | Server signing keys |
+| `servercurrentevent_data` | RANDOM_SMALL | - | - | Current server events |
+| `servername_destination` | RANDOM_SMALL_CACHE | - | - | Server destinations (cached) |
+| `servername_educount` | RANDOM_SMALL | - | - | EDU counters |
+| `servername_override` | RANDOM_SMALL_CACHE | - | - | Server name overrides (cached) |
+| `servernameevent_data` | RANDOM | - | 128 bytes | Server event data |
+| `serverroomids` | RANDOM_SMALL | - | - | Rooms per server |
+| `shorteventid_authchain` | SEQUENTIAL | 8 bytes | - | Event authorization chains |
+| `shorteventid_eventid` | SEQUENTIAL_SMALL | 8 bytes | 48 bytes | Short event ID to event ID |
+| `shorteventid_shortstatehash` | SEQUENTIAL | 8 bytes | 8 bytes | Event to state hash mapping |
+| `shortstatehash_statediff` | SEQUENTIAL_SMALL | 8 bytes | - | State differences |
+| `shortstatekey_statekey` | RANDOM_SMALL | 8 bytes | 1016 bytes | Short state key to state key |
+| `softfailedeventids` | RANDOM_SMALL | 48 bytes | - | Soft-failed events |
+| `statehash_shortstatehash` | RANDOM | - | 8 bytes | State hash to short hash |
+| `statekey_shortstatekey` | RANDOM | 1016 bytes | 8 bytes | State key to short key |
+| `threadid_userids` | SEQUENTIAL_SMALL | - | - | Thread participants |
+| `todeviceid_events` | RANDOM | - | - | To-device messages |
+| `tofrom_relation` | RANDOM_SMALL | 8 bytes | 8 bytes | Event relations |
+| `token_userdeviceid` | RANDOM_SMALL | - | - | Token to device mapping |
+| `tokenids` | RANDOM | - | - | Token ID management |
+| `url_previews` | RANDOM | - | - | URL preview cache |
+| `userdeviceid_metadata` | RANDOM_SMALL | - | - | Device metadata |
+| `userdeviceid_token` | RANDOM_SMALL | - | - | Device tokens |
+| `userdevicesessionid_uiaainfo` | RANDOM_SMALL | - | - | UIAA session info |
+| `userdevicetxnid_response` | RANDOM_SMALL | - | - | Transaction responses |
+| `userfilterid_filter` | RANDOM_SMALL | - | - | User sync filters |
+| `userid_avatarurl` | RANDOM_SMALL | - | - | User avatar URLs |
+| `userid_blurhash` | RANDOM_SMALL | - | - | Avatar blurhashes |
+| `userid_devicelistversion` | RANDOM_SMALL | - | - | Device list versions |
+| `userid_displayname` | RANDOM_SMALL | - | - | User display names |
+| `userid_lastonetimekeyupdate` | RANDOM_SMALL | - | - | Last OTK update time |
+| `userid_masterkeyid` | RANDOM_SMALL | - | - | Master signing keys |
+| `userid_password` | RANDOM | - | - | Password hashes |
+| `userid_presenceid` | RANDOM_SMALL | - | - | User presence mapping |
+| `userid_selfsigningkeyid` | RANDOM_SMALL | - | - | Self-signing keys |
+| `userid_usersigningkeyid` | RANDOM_SMALL | - | - | User-signing keys |
+| `useridprofilekey_value` | RANDOM_SMALL | - | - | Custom profile fields |
+| `openidtoken_expiresatuserid` | RANDOM_SMALL | - | - | OpenID tokens |
+| `logintoken_expiresatuserid` | RANDOM_SMALL | - | - | Login tokens |
+| `userroomid_highlightcount` | RANDOM | - | - | Highlight counts |
+| `userroomid_invitestate` | RANDOM_SMALL | - | - | User invite states |
+| `userroomid_joined` | RANDOM | - | - | User joined rooms |
+| `userroomid_leftstate` | RANDOM | - | - | User leave states |
+| `userroomid_knockedstate` | RANDOM_SMALL | - | - | User knock states |
+| `userroomid_notificationcount` | RANDOM | - | - | Notification counts |
+
+## Access Pattern Definitions
+
+### RANDOM
+
+- Large datasets with random updates across keyspace
+- Compaction priority: OldestSmallestSeqFirst
+- Write buffer: 32MB
+- Cache shards: 128
+- Compression: Zstd level -3
+- Bottommost compression: level 2
+
+### SEQUENTIAL  
+
+- Large datasets with append-heavy updates
+- Compaction priority: OldestLargestSeqFirst
+- Write buffer: 64MB
+- Level size: 32MB
+- File size: 2MB
+- Compression: Zstd level -2
+
+### RANDOM_SMALL
+
+- Small datasets with random updates
+- Compaction style: Universal
+- Write buffer: 16MB
+- Level size: 512KB
+- File size: 128KB
+- Block size: 512 bytes
+- Compression: Zstd level -4
+
+### SEQUENTIAL_SMALL
+
+- Small datasets with sequential updates
+- Compaction style: Universal
+- Write buffer: 16MB
+- Level size: 1MB
+- File size: 512KB
+- Compression: Zstd level -4
+
+### RANDOM_SMALL_CACHE
+
+- Small persistent caches with TTL
+- Compaction style: FIFO
+- Size limit: 64MB
+- TTL: 14 days
+- Unique cache allocation
+
+## Special Configurations
+
+### Shared Cache Tables
+
+- `eventid_outlierpdu` and `pduid_pdu` share cache pool
+- Optimizes memory usage for related event data
+
+### High-Performance Tables
+
+- `roomsynctoken_shortstatehash`: Special compression settings for sync performance
+- `pduid_pdu`: Large block size (2KB) for efficient event storage
+- `eventid_outlierpdu`: Optimized for outlier PDU handling
+
+### Cache-Only Tables
+
+- `servername_destination`: FIFO cache for server resolution
+- `servername_override`: FIFO cache for server overrides
+
+## Data Types and Sizes
+
+### Event IDs
+
+- Full event IDs: 48 bytes (Matrix event ID format)
+- Short event IDs: 8 bytes (internal optimization)
+
+### Room IDs  
+
+- Full room IDs: Variable length Matrix room ID
+- Short room IDs: 8 bytes (internal optimization)
+
+### PDU Data
+
+- PDU ID: 16 bytes
+- PDU content: ~1520 bytes average
+- Outlier PDUs: ~1488 bytes average
+
+### State Data
+
+- State keys: Up to 1016 bytes
+- Short state keys: 8 bytes
+- State hashes: 8 bytes (shortened)
+
+This technical reference shows how Continuwuity optimizes storage for different types of Matrix data, using appropriate RocksDB configurations for each access pattern.
+
+## Database Architecture
+
+## Column Families (Maps)
+
+### Room Management
+
+#### Room Aliases
+
+- **`alias_roomid`**: Maps room alias to room ID
+- **`alias_userid`**: Maps room alias to user ID (for alias management)
+- **`aliasid_alias`**: Maps alias ID to actual alias string
+
+#### Room Metadata
+
+- **`roomid_shortroomid`**: Maps room ID to short room ID (8-byte identifier)
+- **`roomid_shortstatehash`**: Maps room ID to current state hash
+- **`roomid_pduleaves`**: Tracks PDU leaves for each room
+- **`roomid_invitedcount`**: Count of invited users per room
+- **`roomid_joinedcount`**: Count of joined users per room
+- **`roomid_inviteviaservers`**: Via servers for room invites
+- **`publicroomids`**: Set of public room IDs
+- **`bannedroomids`**: Set of banned room IDs
+- **`disabledroomids`**: Set of disabled room IDs
+
+#### Room State
+
+- **`shortstatehash_statediff`**: State differences between state hashes
+- **`statehash_shortstatehash`**: Maps full state hash to short state hash (8-byte)
+- **`statekey_shortstatekey`**: Maps state key to short state key (8-byte)
+- **`shortstatekey_statekey`**: Reverse mapping from short state key to full state key
+- **`roomsynctoken_shortstatehash`**: Maps room sync tokens to state hashes
+
+### Events and Timeline
+
+#### Event Storage
+
+- **`eventid_pduid`**: Maps event ID to PDU ID (16-byte identifier)
+- **`eventid_shorteventid`**: Maps event ID to short event ID (8-byte)
+- **`eventid_outlierpdu`**: Stores outlier PDUs (events not yet in timeline)
+- **`pduid_pdu`**: Main PDU storage (PDU ID to PDU data)
+- **`shorteventid_eventid`**: Reverse mapping from short event ID to full event ID
+- **`shorteventid_authchain`**: Authorization chains for events
+- **`shorteventid_shortstatehash`**: Maps events to their state hashes
+
+#### Event Relationships
+
+- **`tofrom_relation`**: Event relations (replies, edits, reactions)
+- **`threadid_userids`**: Thread participants tracking
+- **`referencedevents`**: Referenced events tracking
+- **`softfailedeventids`**: Events that soft-failed state resolution
+
+### User Management
+
+#### User Identity
+
+- **`userid_displayname`**: User display names
+- **`userid_avatarurl`**: User avatar URLs
+- **`userid_blurhash`**: Avatar blurhash values
+- **`userid_password`**: Password hashes
+- **`useridprofilekey_value`**: Custom profile fields
+
+#### User Devices and Sessions
+
+- **`userdeviceid_metadata`**: Device metadata (name, type, etc.)
+- **`userdeviceid_token`**: Device access tokens
+- **`token_userdeviceid`**: Reverse token to device mapping
+- **`userdevicesessionid_uiaainfo`**: User-Interactive Auth session data
+- **`userdevicetxnid_response`**: Transaction ID to response caching
+
+#### User Preferences
+
+- **`userfilterid_filter`**: Sync filter definitions
+- **`lazyloadedids`**: Lazy-loaded member event tracking
+
+### Cryptography and Security
+
+#### Device Keys
+
+- **`keyid_key`**: Cryptographic keys storage
+- **`userid_devicelistversion`**: Device list versions for users
+- **`userid_lastonetimekeyupdate`**: Last one-time key update timestamps
+- **`onetimekeyid_onetimekeys`**: One-time keys storage
+
+#### Cross-Signing
+
+- **`userid_masterkeyid`**: Master signing keys
+- **`userid_selfsigningkeyid`**: Self-signing keys
+- **`userid_usersigningkeyid`**: User-signing keys
+- **`keychangeid_userid`**: Key change notifications
+
+#### Key Backups
+
+- **`backupid_algorithm`**: Backup algorithm information
+- **`backupid_etag`**: Backup ETags for versioning
+- **`backupkeyid_backup`**: Backed up keys
+
+### Room Membership
+
+#### Membership States
+
+- **`roomuserid_joined`**: Current joined room members
+- **`roomuserid_invitecount`**: Invite counts per room-user
+- **`roomuserid_leftcount`**: Leave counts per room-user
+- **`roomuserid_knockedcount`**: Knock counts per room-user
+- **`roomuseroncejoinedids`**: Users who have ever joined rooms
+
+#### Membership Events
+
+- **`userroomid_joined`**: User's joined rooms
+- **`userroomid_invitestate`**: Invite state events
+- **`userroomid_leftstate`**: Leave state events
+- **`userroomid_knockedstate`**: Knock state events
+
+### Push Notifications and Read Receipts
+
+#### Push Infrastructure
+
+- **`senderkey_pusher`**: Push notification endpoints
+- **`pushkey_deviceid`**: Push key to device mappings
+
+#### Read Tracking
+
+- **`readreceiptid_readreceipt`**: Read receipt storage
+- **`roomuserid_privateread`**: Private read markers
+- **`roomuserid_lastprivatereadupdate`**: Last private read updates
+- **`userroomid_highlightcount`**: Highlight/mention counts
+- **`userroomid_notificationcount`**: Notification counts per room
+
+### Media and Content
+
+#### Media Storage
+
+- **`mediaid_file`**: Media file metadata
+- **`mediaid_user`**: Media uploader tracking
+- **`url_previews`**: URL preview cache
+
+### Federation and Server-to-Server
+
+#### Server Management
+
+- **`server_signingkeys`**: Server signing keys
+- **`servername_destination`**: Server destination resolution
+- **`servername_educount`**: Ephemeral Data Unit counters
+- **`servername_override`**: Server name overrides for federation
+- **`servernameevent_data`**: Server event data
+- **`roomserverids`**: Servers participating in rooms
+- **`serverroomids`**: Rooms per server
+- **`servercurrentevent_data`**: Current server event state
+
+### Application Services
+
+- **`id_appserviceregistrations`**: Application service registrations
+
+### Account Data and Presence
+
+#### Account Data
+
+- **`roomuserdataid_accountdata`**: Room-specific account data
+- **`roomusertype_roomuserdataid`**: Account data type mappings
+
+#### Presence
+
+- **`presenceid_presence`**: User presence information
+- **`userid_presenceid`**: User to presence ID mapping
+
+### To-Device Messages
+
+- **`todeviceid_events`**: Direct device-to-device messages
+
+### Authentication Tokens
+
+- **`openidtoken_expiresatuserid`**: OpenID Connect tokens
+- **`logintoken_expiresatuserid`**: Login tokens
+- **`tokenids`**: Token ID management
+
+### Global Configuration
+
+- **`global`**: Global server settings and state
+
+## Key Design Patterns
+
+### Short Identifiers
+
+Many tables use "short" versions of identifiers (8-byte integers) to reduce storage overhead:
+
+- `shortroomid` for room IDs
+- `shorteventid` for event IDs
+- `shortstatekey` for state keys
+- `shortstatehash` for state hashes
+
+### Composite Keys
+
+Key naming follows a pattern of `{primary}_{secondary}` to create efficient lookups:
+
+- `roomuserid_*` for room-user relationships
+- `userroomid_*` for user-room relationships
+- `eventid_*` for event-related data
+
+### Performance Optimizations
+
+- **Cache sharing**: Related tables share cache pools (e.g., `eventid_outlierpdu` and `pduid_pdu`)
+- **Access patterns**: Tables are optimized for their specific usage (RANDOM vs SEQUENTIAL)
+- **Compression**: Different compression levels based on data characteristics
+- **Block sizes**: Tuned based on expected key/value sizes
+
+## Storage Efficiency
+
+The schema is designed for efficiency in a Matrix homeserver context:
+
+- Large event data uses sequential storage patterns
+- Lookup tables use random access patterns
+- Small metadata uses compressed storage
+- Caching is strategically shared between related data
+
+This design allows Continuwuity to efficiently handle the complex relationships and high-volume data typical in Matrix federation while maintaining good performance characteristics for both reads and writes.
+
+## Column Relationships and Data Flow
+
+### Core Event Storage Chain
+
+The heart of the Matrix homeserver is event storage, which uses several interconnected tables:
+
+- `eventid_shorteventid` ↔ `shorteventid_eventid`: Bidirectional mapping for event ID compression (48 bytes → 8 bytes)
+- `eventid_pduid`: Maps Matrix event IDs to internal PDU IDs (16 bytes)
+- `pduid_pdu`: Main event storage using PDU IDs as keys (shares cache with `eventid_outlierpdu`)
+- `eventid_outlierpdu`: Stores events not yet integrated into the timeline
+- `shorteventid_authchain`: Authorization chains using compressed event IDs
+- `shorteventid_shortstatehash`: Links events to room state snapshots
+
+### Room State Management
+
+Room state is tracked through multiple interconnected tables:
+
+- `statekey_shortstatekey` ↔ `shortstatekey_statekey`: Bidirectional state key compression
+- `statehash_shortstatehash`: Compresses state hashes from full size to 8 bytes
+- `shortstatehash_statediff`: Stores incremental state changes
+- `roomid_shortstatehash`: Current state hash for each room
+- `roomsynctoken_shortstatehash`: Maps sync tokens to state for efficient delta sync
+
+### User Identity and Authentication
+
+User management involves several related tables:
+
+- `userid_password` → authentication base
+- `token_userdeviceid` ↔ `userdeviceid_token`: Bidirectional token↔device mapping
+- `userdeviceid_metadata`: Device information storage
+- `userid_displayname`, `userid_avatarurl`, `userid_blurhash`: Profile data
+- `openidtoken_expiresatuserid`, `logintoken_expiresatuserid`: Token management
+
+### Room Membership Tracking
+
+Membership uses bidirectional indexes for efficient queries:
+
+- `roomuserid_joined` ↔ `userroomid_joined`: Current membership from both perspectives
+- `roomuserid_invitecount` ↔ `userroomid_invitestate`: Invitation tracking
+- `roomuserid_leftcount` ↔ `userroomid_leftstate`: Leave event tracking
+- `roomid_joinedcount`, `roomid_invitedcount`: Aggregate room statistics
+- `roomuseroncejoinedids`: Historical membership tracking
+
+### Cryptography and Security Chain
+
+End-to-end encryption involves coordinated key management:
+
+- `userid_devicelistversion`: Tracks when device lists change
+- `keyid_key`: Stores actual cryptographic keys
+- `userid_masterkeyid`, `userid_selfsigningkeyid`, `userid_usersigningkeyid`: Cross-signing keys
+- `onetimekeyid_onetimekeys` → `userid_lastonetimekeyupdate`: One-time key lifecycle
+- `keychangeid_userid`: Key change notifications
+- `backupid_algorithm`, `backupid_etag` → `backupkeyid_backup`: Key backup system
+
+### Federation and Server Communication
+
+Server-to-server communication requires coordinated tracking:
+
+- `roomserverids` ↔ `serverroomids`: Bidirectional room↔server participation
+- `servername_destination`, `servername_override`: Server resolution (both cached)
+- `server_signingkeys`: Federation authentication
+- `servername_educount`: Ephemeral data unit tracking
+- `servernameevent_data`, `servercurrentevent_data`: Server event state
+
+### Read Tracking and Notifications
+
+Message read tracking involves multiple coordinated updates:
+
+- `readreceiptid_readreceipt`: Public read receipts
+- `roomuserid_privateread`, `roomuserid_lastprivatereadupdate`: Private read markers
+- `userroomid_highlightcount`, `userroomid_notificationcount`: Per-room notification counts
+- `senderkey_pusher` ↔ `pushkey_deviceid`: Push notification routing
+
+### Account Data and Preferences
+
+User preferences and account data use a two-level structure:
+
+- `roomusertype_roomuserdataid` → `roomuserdataid_accountdata`: Type index points to actual data
+- `userid_presenceid` → `presenceid_presence`: Presence data separation
+- `userfilterid_filter`: Sync filter definitions
+- `lazyloadedids`: Lazy loading state tracking
+
+This interconnected design allows Continuwuity to efficiently handle Matrix protocol operations while maintaining data consistency and enabling fast lookups from multiple perspectives.

docs/static/announcements.json

@@ -4,6 +4,10 @@
         {
             "id": 1,
             "message": "Welcome to Continuwuity! Important announcements about the project will appear here."
+        },
+        {
+            "id": 2,
+            "message": "🎉 Continuwuity v0.5.0-rc.6 is now available! This release includes improved knock-restricted room handling, automatic support contact configuration, and a new HTML landing page. Check [the release notes for full details](https://forgejo.ellis.link/continuwuation/continuwuity/releases/tag/v0.5.0-rc.6) and upgrade instructions."
         }
     ]
-}
+}

docs/static/announcements.schema.json

@@ -3,7 +3,7 @@
     "$id": "https://continwuity.org/schema/announcements.schema.json",
     "type": "object",
     "properties": {
-      "updates": {
+      "announcements": {
         "type": "array",
         "items": {
           "type": "object",
@@ -16,6 +16,10 @@
             },
             "date": {
               "type": "string"
+            },
+            "mention_room": {
+              "type": "boolean",
+              "description": "Whether to mention the room (@room) when posting this announcement"
             }
           },
           "required": [
@@ -26,6 +30,6 @@
       }
     },
     "required": [
-      "updates"
+      "announcements"
     ]
   }

src/api/client/sync/v3.rs

@@ -808,7 +808,7 @@ async fn load_joined_room(
 			let typings = services
 				.rooms
 				.typing
-				.typings_event_for_user(room_id, sender_user)
+				.typings_all(room_id, sender_user)
 				.await?;
 
 			Ok(vec![serde_json::from_str(&serde_json::to_string(&typings)?)?])

src/api/client/sync/v5.rs

@@ -33,7 +33,6 @@ use ruma::{
 	events::{
 		AnyRawAccountDataEvent, AnySyncEphemeralRoomEvent, StateEventType, TimelineEventType,
 		room::member::{MembershipState, RoomMemberEventContent},
-		typing::TypingEventContent,
 	},
 	serde::Raw,
 	uint,
@@ -206,9 +205,6 @@ pub(crate) async fn sync_events_v5_route(
 		_ = tokio::time::timeout(duration, watcher).await;
 	}
 
-	let typing = collect_typing_events(services, sender_user, &body, &todo_rooms).await?;
-	response.extensions.typing = typing;
-
 	trace!(
 		rooms = ?response.rooms.len(),
 		account_data = ?response.extensions.account_data.rooms.len(),
@@ -292,8 +288,6 @@ where
 	Rooms: Iterator<Item = &'a RoomId> + Clone + Send + 'a,
 	AllRooms: Iterator<Item = &'a RoomId> + Clone + Send + 'a,
 {
-	// TODO MSC4186: Implement remaining list filters: is_dm, is_encrypted,
-	// room_types.
 	for (list_id, list) in &body.lists {
 		let active_rooms: Vec<_> = match list.filters.as_ref().and_then(|f| f.is_invite) {
 			| None => all_rooms.clone().collect(),
@@ -671,62 +665,6 @@ where
 	}
 	Ok(rooms)
 }
-
-async fn collect_typing_events(
-	services: &Services,
-	sender_user: &UserId,
-	body: &sync_events::v5::Request,
-	todo_rooms: &TodoRooms,
-) -> Result<sync_events::v5::response::Typing> {
-	if !body.extensions.typing.enabled.unwrap_or(false) {
-		return Ok(sync_events::v5::response::Typing::default());
-	}
-	let rooms: Vec<_> = body.extensions.typing.rooms.clone().unwrap_or_else(|| {
-		body.room_subscriptions
-			.keys()
-			.map(ToOwned::to_owned)
-			.collect()
-	});
-	let lists: Vec<_> = body
-		.extensions
-		.typing
-		.lists
-		.clone()
-		.unwrap_or_else(|| body.lists.keys().map(ToOwned::to_owned).collect::<Vec<_>>());
-
-	if rooms.is_empty() && lists.is_empty() {
-		return Ok(sync_events::v5::response::Typing::default());
-	}
-
-	let mut typing_response = sync_events::v5::response::Typing::default();
-	for (room_id, (required_state_request, timeline_limit, roomsince)) in todo_rooms {
-		if services.rooms.typing.last_typing_update(room_id).await? <= *roomsince {
-			continue;
-		}
-
-		match services
-			.rooms
-			.typing
-			.typing_users_for_user(room_id, sender_user)
-			.await
-		{
-			| Ok(typing_users) => {
-				typing_response.rooms.insert(
-					room_id.to_owned(), // Already OwnedRoomId
-					Raw::new(&sync_events::v5::response::SyncTypingEvent {
-						content: TypingEventContent::new(typing_users),
-					})?,
-				);
-			},
-			| Err(e) => {
-				warn!(%room_id, "Failed to get typing events for room: {}", e);
-			},
-		}
-	}
-
-	Ok(typing_response)
-}
-
 async fn collect_account_data(
 	services: &Services,
 	(sender_user, _, globalsince, body): (&UserId, &DeviceId, u64, &sync_events::v5::Request),

src/service/announcements/mod.rs

@@ -20,7 +20,7 @@ use std::{sync::Arc, time::Duration};
 use async_trait::async_trait;
 use conduwuit::{Result, Server, debug, info, warn};
 use database::{Deserialized, Map};
-use ruma::events::room::message::RoomMessageEventContent;
+use ruma::events::{Mentions, room::message::RoomMessageEventContent};
 use serde::Deserialize;
 use tokio::{
 	sync::Notify,
@@ -53,6 +53,8 @@ struct CheckForAnnouncementsResponseEntry {
 	id: u64,
 	date: Option<String>,
 	message: String,
+	#[serde(default, skip_serializing_if = "bool::not")]
+	mention_room: bool,
 }
 
 const CHECK_FOR_ANNOUNCEMENTS_URL: &str =
@@ -139,19 +141,20 @@ impl Service {
 		} else {
 			info!("[announcements] {:#}", announcement.message);
 		}
+		let mut message = RoomMessageEventContent::text_markdown(format!(
+			"### New announcement{}\n\n{}",
+			announcement
+				.date
+				.as_ref()
+				.map_or_else(String::new, |date| format!(" - `{date}`")),
+			announcement.message
+		));
 
-		self.services
-			.admin
-			.send_message(RoomMessageEventContent::text_markdown(format!(
-				"### New announcement{}\n\n{}",
-				announcement
-					.date
-					.as_ref()
-					.map_or_else(String::new, |date| format!(" - `{date}`")),
-				announcement.message
-			)))
-			.await
-			.ok();
+		if announcement.mention_room {
+			message = message.add_mentions(Mentions::with_room_mention());
+		}
+
+		self.services.admin.send_message(message).await.ok();
 	}
 
 	#[inline]

src/service/rooms/typing/mod.rs

@@ -179,15 +179,18 @@ impl Service {
 			.unwrap_or(0))
 	}
 
-	pub async fn typing_users_for_user(
+	/// Returns a new typing EDU.
+	pub async fn typings_all(
 		&self,
 		room_id: &RoomId,
 		sender_user: &UserId,
-	) -> Result<Vec<OwnedUserId>> {
+	) -> Result<SyncEphemeralRoomEvent<ruma::events::typing::TypingEventContent>> {
 		let room_typing_indicators = self.typing.read().await.get(room_id).cloned();
 
 		let Some(typing_indicators) = room_typing_indicators else {
-			return Ok(Vec::new());
+			return Ok(SyncEphemeralRoomEvent {
+				content: ruma::events::typing::TypingEventContent { user_ids: Vec::new() },
+			});
 		};
 
 		let user_ids: Vec<_> = typing_indicators
@@ -204,19 +207,8 @@ impl Service {
 			.collect()
 			.await;
 
-		Ok(user_ids)
-	}
-
-	/// Returns a new typing EDU.
-	pub async fn typings_event_for_user(
-		&self,
-		room_id: &RoomId,
-		sender_user: &UserId,
-	) -> Result<SyncEphemeralRoomEvent<ruma::events::typing::TypingEventContent>> {
 		Ok(SyncEphemeralRoomEvent {
-			content: ruma::events::typing::TypingEventContent {
-				user_ids: self.typing_users_for_user(room_id, sender_user).await?,
-			},
+			content: ruma::events::typing::TypingEventContent { user_ids },
 		})
 	}