diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
index 7dbd83908e8f965b9d4f9892497cb92261218146..fcbd40b74625a26680c7a2a7b7784865aadc059a 100644
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/tests.yml
@@ -56,6 +56,7 @@ jobs:
             - 'pyproject.toml'
             - 'poetry.lock'
             - 'docker/**'
+            - 'scripts-dev/complement.sh'
 
           linting:
             - 'synapse/**'
diff --git a/changelog.d/16466.misc b/changelog.d/16466.misc
new file mode 100644
index 0000000000000000000000000000000000000000..471056bb0fbd046531f0eeb1f8bc90c7c21707c9
--- /dev/null
+++ b/changelog.d/16466.misc
@@ -0,0 +1 @@
+Update complement.sh to match new public API shape.
diff --git a/scripts-dev/complement.sh b/scripts-dev/complement.sh
index 8416b556743724b43074c34806663239f7ba1af9..3e0cddb527da286a7b9cd8712550ecd0ebfd9018 100755
--- a/scripts-dev/complement.sh
+++ b/scripts-dev/complement.sh
@@ -214,7 +214,7 @@ fi
 
 extra_test_args=()
 
-test_tags="synapse_blacklist,msc3874,msc3890,msc3391,msc3930,faster_joins"
+test_packages="./tests/csapi ./tests ./tests/msc3874 ./tests/msc3890 ./tests/msc3391 ./tests/msc3930 ./tests/msc3902"
 
 # All environment variables starting with PASS_ will be shared.
 # (The prefix is stripped off before reaching the container.)
@@ -277,4 +277,4 @@ export PASS_SYNAPSE_LOG_TESTING=1
 echo "Images built; running complement"
 cd "$COMPLEMENT_DIR"
 
-go test -v -tags $test_tags -count=1 "${extra_test_args[@]}" "$@" ./tests/...
+go test -v -tags "synapse_blacklist" -count=1 "${extra_test_args[@]}" "$@" $test_packages