feat(complement): Simplify local test runs

.forgejo/workflows/complement.yml

@@ -0,0 +1,87 @@
+name: Complement
+
+on:
+  pull_request:
+    types: [labeled]
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+concurrency:
+  group: complement-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  complement:
+    name: Matrix compliance tests
+    if: github.event_name == 'workflow_dispatch' || github.event.label.name == 'Testing/Complement'
+    runs-on: ubuntu-latest
+    services:
+      dind:
+        image: docker:dind
+        options: --privileged
+        env:
+          DOCKER_TLS_CERTDIR: ""
+    env:
+      DOCKER_HOST: tcp://dind:2375
+      DOCKER_TLS_CERTDIR: ""
+
+    steps:
+      - name: Checkout Continuwuity
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
+        with:
+          persist-credentials: false
+
+      - name: Checkout Complement
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
+        with:
+          repository: matrix-org/complement
+          ref: 6d2fdc286c2b44faaddd1037205869b2242a4005
+          path: complement-source
+          persist-credentials: false
+
+      - name: Setup LLVM
+        uses: ./.forgejo/actions/setup-llvm-with-apt
+        with:
+          extra-packages: golang-go jq liburing-dev liburing2
+
+      - name: Setup Rust with caching
+        uses: ./.forgejo/actions/setup-rust
+        with:
+          github-token: ${{ secrets.GH_PUBLIC_RO }}
+
+      - name: Build Complement image
+        run: |
+          cargo build --locked -p conduwuit
+          docker build --tag continuwuity:complement --file docker/complement.Dockerfile .
+
+      - name: Run Complement tests
+        id: complement
+        shell: bash
+        run: |
+          set +e
+          COMPLEMENT_HS_PORT_BINDING_IP="$(getent ahostsv4 dind | awk 'NR == 1 { print $1 }')" \
+          COMPLEMENT_HOSTNAME_RUNNING_COMPLEMENT=dind \
+          ./bin/complement complement-source complement_test_logs.jsonl complement_test_results.jsonl | tee complement_output.log
+          status=${PIPESTATUS[0]}
+          set -e
+
+          if [ ! -s complement_test_results.jsonl ]; then
+            exit "$status"
+          fi
+
+          diff --unified --color=always \
+            tests/test_results/complement/test_results.jsonl \
+            complement_test_results.jsonl
+
+      - name: Upload Complement artifacts
+        if: always()
+        uses: forgejo/upload-artifact@v4
+        with:
+          name: complement-${{ github.run_id }}
+          path: |
+            complement_output.log
+            complement_test_logs.jsonl
+            complement_test_results.jsonl
+          if-no-files-found: warn

CONTRIBUTING.md

@@ -75,7 +75,7 @@ cargo +nightly fmt
 
 ### Matrix tests
 
-Continuwuity uses [Complement][complement] for Matrix protocol compliance testing. Complement tests are run manually by developers, and documentation on how to run these tests locally is currently being developed.
+Continuwuity uses [Complement][complement] for Matrix protocol compliance testing. Complement tests are run manually by developers; see the [local testing instructions][complement-testing].
 
 If your changes are done to fix Matrix tests, please note that in your pull request. If more Complement tests start failing from your changes, please review the logs and determine if they're intended or not.
 
@@ -174,6 +174,7 @@ continuwuity Matrix rooms for Code of Conduct violations.
 [issues]: https://forgejo.ellis.link/continuwuation/continuwuity/issues
 [continuwuity-matrix]: https://matrix.to/#/#continuwuity:continuwuity.org?via=continuwuity.org&via=ellis.link&via=explodie.org&via=matrix.org
 [complement]: https://github.com/matrix-org/complement/
+[complement-testing]: docs/development/testing.mdx#complement
 [sytest]: https://github.com/matrix-org/sytest/
 [nodejs-download]: https://nodejs.org/en/download
 [rspress]: https://rspress.rs/

bin/complement

@@ -2,14 +2,34 @@
 
 set -euo pipefail
 
-# The root path where complement is available.
-COMPLEMENT_SRC="${COMPLEMENT_SRC:-$1}"
+toplevel="$(git rev-parse --show-toplevel)"
+DEFAULT_COMPLEMENT_SRC="$toplevel/target/complement"
+DEFAULT_COMPLEMENT_REVISION="6d2fdc286c2b44faaddd1037205869b2242a4005"
 
-# A `.jsonl` file to write test logs to
-LOG_FILE="${2:-tests/test_results/complement/test_logs.jsonl}"
+MSC="${COMPLEMENT_MSC:-}"
+if [ "${1:-}" = "--msc" ]; then
+    MSC="${2:-}"
+    shift 2
+fi
 
-# A `.jsonl` file to write test results to
-RESULTS_FILE="${3:-tests/test_results/complement/test_results.jsonl}"
+if [ -n "$MSC" ]; then
+    MSC="${MSC#MSC}"
+    MSC="${MSC#msc}"
+    if ! [[ "$MSC" =~ ^[0-9]+$ ]]; then
+        echo "--msc must be an MSC number, for example: --msc 3391"
+        exit 2
+    fi
+fi
+
+# The root path where Complement is available.
+if [ -n "${COMPLEMENT_SRC:-}" ]; then
+    LOG_FILE="${1:-tests/test_results/complement/test_logs.jsonl}"
+    RESULTS_FILE="${2:-tests/test_results/complement/test_results.jsonl}"
+else
+    COMPLEMENT_SRC="${1:-$DEFAULT_COMPLEMENT_SRC}"
+    LOG_FILE="${2:-tests/test_results/complement/test_logs.jsonl}"
+    RESULTS_FILE="${3:-tests/test_results/complement/test_results.jsonl}"
+fi
 
 # The base docker image to use for complement tests
 # You can build the default with `docker build -t continuwuity:complement -f ./docker/complement.Dockerfile .`
@@ -19,45 +39,93 @@ COMPLEMENT_BASE_IMAGE="${COMPLEMENT_BASE_IMAGE:-continuwuity:complement}"
 # Complement tests that are skipped due to flakiness/reliability issues or we don't implement such features and won't for a long time
 SKIPPED_COMPLEMENT_TESTS='TestPartialStateJoin.*|TestRoomDeleteAlias/Parallel/Regular_users_can_add_and_delete_aliases_when_m.*|TestRoomDeleteAlias/Parallel/Can_delete_canonical_alias|TestUnbanViaInvite.*|TestRoomState/Parallel/GET_/publicRooms_lists.*"|TestRoomDeleteAlias/Parallel/Users_with_sufficient_power-level_can_delete_other.*'
 
-# $COMPLEMENT_SRC needs to be a directory to Complement source code
-if [ -f "$COMPLEMENT_SRC" ]; then
-    echo "\$COMPLEMENT_SRC must be a directory/path to Complement source code"
+# Download the pinned upstream Complement checkout on first use. A local
+# checkout can be supplied as the first argument or with COMPLEMENT_SRC.
+if [ ! -e "$COMPLEMENT_SRC" ]; then
+    mkdir -p "$(dirname "$COMPLEMENT_SRC")"
+    git init --quiet "$COMPLEMENT_SRC"
+    git -C "$COMPLEMENT_SRC" remote add origin https://github.com/matrix-org/complement.git
+    git -C "$COMPLEMENT_SRC" fetch --depth 1 origin "$DEFAULT_COMPLEMENT_REVISION"
+    git -C "$COMPLEMENT_SRC" checkout --quiet --detach FETCH_HEAD
+fi
+
+if [ ! -d "$COMPLEMENT_SRC" ] || [ ! -f "$COMPLEMENT_SRC/go.mod" ]; then
+    echo "\$COMPLEMENT_SRC must be a directory containing Complement source code"
     exit 1
 fi
 
-# quick test to make sure we can actually write to $LOG_FILE and $RESULTS_FILE
-touch $LOG_FILE && rm -v $LOG_FILE
-touch $RESULTS_FILE && rm -v $RESULTS_FILE
-
-toplevel="$(git rev-parse --show-toplevel)"
-
 pushd "$toplevel" > /dev/null
 
+for command in docker go jq; do
+    if ! command -v "$command" > /dev/null; then
+        echo "missing required command: $command"
+        exit 1
+    fi
+done
+
+if ! docker image inspect "$COMPLEMENT_BASE_IMAGE" > /dev/null 2>&1; then
+    echo "Complement base image not found: $COMPLEMENT_BASE_IMAGE"
+    echo "Build it with: cargo build -p conduwuit && docker build -t $COMPLEMENT_BASE_IMAGE -f docker/complement.Dockerfile ."
+    exit 1
+fi
+
+mkdir -p "$(dirname "$LOG_FILE")" "$(dirname "$RESULTS_FILE")"
+LOG_TMP="$(mktemp "${LOG_FILE}.XXXXXX")"
+RESULTS_TMP="$(mktemp "${RESULTS_FILE}.XXXXXX")"
+trap 'rm -f "$LOG_TMP" "$RESULTS_TMP"' EXIT
 
 echo ""
 echo "running go test with:"
 echo "\$COMPLEMENT_SRC: $COMPLEMENT_SRC"
+COMPLEMENT_REVISION="$(git -C "$COMPLEMENT_SRC" rev-parse HEAD 2>/dev/null || echo unknown)"
+echo "\$COMPLEMENT_REVISION: $COMPLEMENT_REVISION"
+echo "\$COMPLEMENT_MSC: ${MSC:-all}"
 echo "\$COMPLEMENT_BASE_IMAGE: $COMPLEMENT_BASE_IMAGE"
 echo "\$RESULTS_FILE: $RESULTS_FILE"
 echo "\$LOG_FILE: $LOG_FILE"
 echo ""
 
-# It's okay (likely, even) that `go test` exits nonzero
+TEST_PACKAGES=(./tests/...)
+if [ -n "$MSC" ]; then
+    test_files="$({
+        grep -rilE --include='*.go' "msc$MSC([^0-9]|$)" "$COMPLEMENT_SRC/tests" || true
+        find "$COMPLEMENT_SRC/tests" -type f -name '*.go' -print | while IFS= read -r file; do
+            if [[ "$(basename "$file")" =~ [Mm][Ss][Cc]$MSC([^0-9]|$) ]]; then
+                printf '%s\n' "$file"
+            fi
+        done
+    } | sort -u)"
+    if [ -z "$test_files" ]; then
+        echo "no Complement tests mention MSC$MSC"
+        exit 1
+    fi
+    TEST_PACKAGES=()
+    while IFS= read -r test_file; do
+        test_dir="$(dirname "$test_file")"
+        TEST_PACKAGES+=(".${test_dir#"$COMPLEMENT_SRC"}")
+    done <<< "$test_files"
+fi
+
 # `COMPLEMENT_ENABLE_DIRTY_RUNS=1`  reuses the same complement container for faster complement, at the possible expense of test environment pollution
-set +o pipefail
-env \
-    -C "$COMPLEMENT_SRC" \
+set +e
+(cd "$COMPLEMENT_SRC" && \
     COMPLEMENT_BASE_IMAGE="$COMPLEMENT_BASE_IMAGE" \
-    go test -tags="conduwuit_blacklist" -skip="$SKIPPED_COMPLEMENT_TESTS" -v -timeout 1h -json ./tests/... | tee "$LOG_FILE"
-set -o pipefail
+    go test -tags="conduwuit_blacklist" -skip="$SKIPPED_COMPLEMENT_TESTS" -v -timeout 1h -json "${TEST_PACKAGES[@]}"
+) | tee "$LOG_TMP"
+test_status=${PIPESTATUS[0]}
+set -e
 
 # Post-process the results into an easy-to-compare format, sorted by Test name for reproducible results
-jq -s -c 'sort_by(.Test)[]' < "$LOG_FILE" | jq -c '
+jq -s -c 'sort_by(.Test)[]' < "$LOG_TMP" | jq -c '
     select(
         (.Action == "pass" or .Action == "fail" or .Action == "skip")
         and .Test != null
     ) | {Action: .Action, Test: .Test}
-    ' > "$RESULTS_FILE"
+    ' > "$RESULTS_TMP"
+
+mv "$LOG_TMP" "$LOG_FILE"
+mv "$RESULTS_TMP" "$RESULTS_FILE"
+trap - EXIT
 
 echo ""
 echo ""
@@ -65,3 +133,5 @@ echo "complement logs saved at $LOG_FILE"
 echo "complement results saved at $RESULTS_FILE"
 echo ""
 echo ""
+
+exit "$test_status"

docs/development/testing.mdx

@@ -3,30 +3,65 @@
 ## Complement
 
 Have a look at [Complement's repository][complement] for an explanation of what
-it is.
+it is. Continuwuity runs Complement manually; it is not part of the normal test
+suite or default CI gate.
 
-To test against Complement, with Nix (or [Lix](https://lix.systems) and
-[direnv installed and set up][direnv] (run `direnv allow` after setting up the hook), you can:
+### Run it locally
 
-* Run `./bin/complement "$COMPLEMENT_SRC"` to build a Complement image, run
-the tests, and output the logs and results to the specified paths. This will also output the OCI image
-at `result`
-* Run `nix build .#complement` from the root of the repository to just build a
-Complement OCI image outputted to `result` (it's a `.tar.gz` file)
-* Or download the latest Complement OCI image from the CI workflow artifacts
-output from the commit/revision you want to test (e.g. from main)
-[here][ci-workflows]
+Install Git, Rust, Docker, Go, and `jq`. From this repository's root, build the
+server image and run the suite:
 
-If you want to use your own prebuilt OCI image (such as from our CI) without needing
-Nix installed, put the image at `complement_oci_image.tar.gz` in the root of the repo
-and run the script.
+```bash
+cargo build -p conduwuit
+docker build -t continuwuity:complement -f docker/complement.Dockerfile .
+./bin/complement
+```
 
-If you're on macOS and need to build an image, run `nix build .#linux-complement`.
+The run can take up to an hour. The runner writes the full Go test stream to
+`tests/test_results/complement/test_logs.jsonl` and a sorted pass/fail/skip
+summary to `tests/test_results/complement/test_results.jsonl`. It exits nonzero
+when Complement reports failing tests, after writing both files.
 
-We have a Complement fork as some tests have needed to be fixed. This can be found
-at [continuwuation/complement](https://forgejo.ellis.link/continuwuation/complement)
+On first use, the runner downloads the pinned upstream Complement revision to
+`target/complement`. Pass a checkout path or set `COMPLEMENT_SRC` to run a
+different revision or the [Continuwuity fork][complement-fork].
+
+To run test packages that mention an MSC, pass its number:
+
+```bash
+./bin/complement --msc 3391
+```
+
+This selects packages from Complement's source comments and file names, so it
+can include tests shared with that MSC but does not claim a complete mapping.
+
+To use an image built elsewhere, load and name it before running the script:
+
+```bash
+docker load < complement_oci_image.tar.gz
+COMPLEMENT_BASE_IMAGE=your-image:tag ./bin/complement
+```
+
+`COMPLEMENT_ENABLE_DIRTY_RUNS=1` reuses Complement containers for faster local
+runs, but can let state leak between tests. Do not use it when validating a
+failure or updating the committed result summary.
+
+The checked-in result summary is a baseline, not an assertion that every test
+passes. Compare a manual run with it when working on a compliance fix:
+
+```bash
+git diff -- tests/test_results/complement/test_results.jsonl
+```
+
+### Run it in CI
+
+Add the `Testing/Complement` label to a pull request to run the suite in
+Forgejo Actions. The workflow builds the image from the pull request, checks
+out upstream Complement, and uploads the full logs and result summary as an
+artifact. It passes only when the summary matches the checked-in baseline, so
+expected existing failures do not hide regressions.
+
+You can also start the workflow manually from the Actions page.
 
-[ci-workflows]:
-https://forgejo.ellis.link/continuwuation/continuwuity/actions/?workflow=ci.yml&actor=0&status=1
 [complement]: https://github.com/matrix-org/complement
-[direnv]: https://direnv.net/docs/hook.html
+[complement-fork]: https://forgejo.ellis.link/continuwuation/complement