diff --git a/.github/workflows/complement_tests.yml b/.github/workflows/complement_tests.yml new file mode 100644 index 0000000000..41b7be6192 --- /dev/null +++ b/.github/workflows/complement_tests.yml @@ -0,0 +1,176 @@ +# Re-usable workflow (https://docs.github.com/en/actions/how-tos/reuse-automations/reuse-workflows) +name: Reusable Complement testing + +on: + workflow_call: + inputs: + use_latest_deps: + type: boolean + default: false + use_twisted_trunk: + type: boolean + default: false + +# Control the permissions granted to `GITHUB_TOKEN`. +permissions: + # `actions/checkout` reads the repository (also see + # https://github.com/actions/checkout/tree/de0fac2e4500dabe0009e67214ff5f5447ce83dd/#recommended-permissions) + contents: read + +env: + RUST_VERSION: 1.87.0 + +jobs: + complement: + runs-on: ubuntu-latest + + strategy: + fail-fast: false + matrix: + include: + - arrangement: monolith + database: SQLite + + - arrangement: monolith + database: Postgres + + - arrangement: workers + database: Postgres + + steps: + - name: Checkout synapse codebase + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + path: synapse + + # Log Docker system info for debugging (compare with your local environment) and + # tracking GitHub runner changes over time (can easily compare a run from last + # week with the current one in question). + - run: docker system info + shell: bash + + - name: Install Rust + uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master + with: + toolchain: ${{ env.RUST_VERSION }} + - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 + + # We use `poetry` in `complement.sh` + - uses: matrix-org/setup-python-poetry@5bbf6603c5c930615ec8a29f1b5d7d258d905aa4 # v2.0.0 + with: + poetry-version: "2.2.1" + # Matches the `path` where we checkout Synapse above + working-directory: "synapse" + + - name: Prepare Complement's Prerequisites + run: synapse/.ci/scripts/setup_complement_prerequisites.sh + + - uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0 + with: + cache-dependency-path: complement/go.sum + go-version-file: complement/go.mod + + # This step is specific to the 'Twisted trunk' test run: + - name: Patch dependencies + if: ${{ inputs.use_twisted_trunk }} + run: | + set -x + DEBIAN_FRONTEND=noninteractive sudo apt-get install -yqq python3 pipx + pipx install poetry==2.2.1 + + poetry remove -n twisted + poetry add -n --extras tls git+https://github.com/twisted/twisted.git#trunk + poetry lock + working-directory: synapse + + # Run the image sanity check test first as this is the first thing we want to know + # about (are we actually testing what we expect?) and we don't want to debug + # downstream failures (wild goose chase). + - name: Sanity check Complement image + id: run_sanity_check_complement_image_test + # -p=1: We're using `-p 1` to force the test packages to run serially as GHA boxes + # are underpowered and don't like running tons of Synapse instances at once. + # -json: Output JSON format so that gotestfmt can parse it. + # + # tee /tmp/gotest-complement.log: We tee the output to a file so that we can re-process it + # later on for better formatting with gotestfmt. But we still want the command + # to output to the terminal as it runs so we can see what's happening in + # real-time. + run: | + set -o pipefail + COMPLEMENT_DIR=`pwd`/complement synapse/scripts-dev/complement.sh --in-repo -p 1 -json -run 'TestSynapseVersion/Synapse_version_matches_current_git_checkout' 2>&1 | tee /tmp/gotest-sanity-check-complement.log + shell: bash + env: + POSTGRES: ${{ (matrix.database == 'Postgres') && 1 || '' }} + WORKERS: ${{ (matrix.arrangement == 'workers') && 1 || '' }} + + - name: Formatted sanity check Complement test logs + # Always run this step if we attempted to run the Complement tests. + if: always() && steps.run_sanity_check_complement_image_test.outcome != 'skipped' + # We do not hide successful tests in `gotestfmt` here as the list of sanity + # check tests is so short. Feel free to change this when we get more tests. + # + # Note that the `-hide` argument is interpreted by `gotestfmt`. From it, + # it derives several values under `$settings` and passes them to our + # custom `.ci/complement_package.gotpl` template to render the output. + run: cat /tmp/gotest-sanity-check-complement.log | gotestfmt -hide "successful-downloads,empty-packages" + + - name: Run Complement Tests + id: run_complement_tests + # -p=1: We're using `-p 1` to force the test packages to run serially as GHA boxes + # are underpowered and don't like running tons of Synapse instances at once. + # -json: Output JSON format so that gotestfmt can parse it. + # + # tee /tmp/gotest-complement.log: We tee the output to a file so that we can re-process it + # later on for better formatting with gotestfmt. But we still want the command + # to output to the terminal as it runs so we can see what's happening in + # real-time. + run: | + set -o pipefail + COMPLEMENT_DIR=`pwd`/complement synapse/scripts-dev/complement.sh -p 1 -json 2>&1 | tee /tmp/gotest-complement.log + shell: bash + env: + POSTGRES: ${{ (matrix.database == 'Postgres') && 1 || '' }} + WORKERS: ${{ (matrix.arrangement == 'workers') && 1 || '' }} + TEST_ONLY_IGNORE_POETRY_LOCKFILE: ${{ inputs.use_latest_deps && 1 || '' }} + TEST_ONLY_SKIP_DEP_HASH_VERIFICATION: ${{ inputs.use_twisted_trunk && 1 || '' }} + + - name: Formatted Complement test logs (only failing are shown) + # Always run this step if we attempted to run the Complement tests. + if: always() && steps.run_complement_tests.outcome != 'skipped' + # Hide successful tests in order to reduce the verbosity of the otherwise very large output. + # + # Note that the `-hide` argument is interpreted by `gotestfmt`. From it, + # it derives several values under `$settings` and passes them to our + # custom `.ci/complement_package.gotpl` template to render the output. + run: cat /tmp/gotest-complement.log | gotestfmt -hide "successful-downloads,successful-tests,empty-packages" + + - name: Run in-repo Complement Tests + id: run_in_repo_complement_tests + # -p=1: We're using `-p 1` to force the test packages to run serially as GHA boxes + # are underpowered and don't like running tons of Synapse instances at once. + # -json: Output JSON format so that gotestfmt can parse it. + # + # tee /tmp/gotest-in-repo-complement.log: We tee the output to a file so that we can re-process it + # later on for better formatting with gotestfmt. But we still want the command + # to output to the terminal as it runs so we can see what's happening in + # real-time. + run: | + set -o pipefail + COMPLEMENT_DIR=`pwd`/complement synapse/scripts-dev/complement.sh --in-repo -p 1 -json 2>&1 | tee /tmp/gotest-in-repo-complement.log + shell: bash + env: + POSTGRES: ${{ (matrix.database == 'Postgres') && 1 || '' }} + WORKERS: ${{ (matrix.arrangement == 'workers') && 1 || '' }} + TEST_ONLY_IGNORE_POETRY_LOCKFILE: ${{ inputs.use_latest_deps && 1 || '' }} + TEST_ONLY_SKIP_DEP_HASH_VERIFICATION: ${{ inputs.use_twisted_trunk && 1 || '' }} + + - name: Formatted in-repo Complement test logs (only failing are shown) + # Always run this step if we attempted to run the Complement tests. + if: always() && steps.run_in_repo_complement_tests.outcome != 'skipped' + # Hide successful tests in order to reduce the verbosity of the otherwise very large output. + # + # Note that the `-hide` argument is interpreted by `gotestfmt`. From it, + # it derives several values under `$settings` and passes them to our + # custom `.ci/complement_package.gotpl` template to render the output. + run: cat /tmp/gotest-in-repo-complement.log | gotestfmt -hide "successful-downloads,successful-tests,empty-packages" diff --git a/.github/workflows/famedly-tests.yml b/.github/workflows/famedly-tests.yml index b2ce55fe0d..052a48f452 100644 --- a/.github/workflows/famedly-tests.yml +++ b/.github/workflows/famedly-tests.yml @@ -442,7 +442,7 @@ jobs: - name: Run invite-checker tests working-directory: synapse-invite-checker - run: hatch run pip install -r synapse-requirements.txt && hatch run cov + run: hatch run pip install -r synapse-requirements.txt && hatch test -p -c - name: Display Hatch Environment Info if: always() @@ -576,7 +576,7 @@ jobs: - name: Run famedly-control tests working-directory: famedly-control-synapse - run: hatch run pip install -r synapse-requirements.txt && hatch run cov + run: hatch run pip install -r synapse-requirements.txt && hatch test -p -c - name: Display Hatch Environment Info if: always() diff --git a/.github/workflows/latest_deps.yml b/.github/workflows/latest_deps.yml index 746223ef4a..d03a929507 100644 --- a/.github/workflows/latest_deps.yml +++ b/.github/workflows/latest_deps.yml @@ -181,84 +181,11 @@ jobs: /logs/**/*.log* complement: + uses: ./.github/workflows/complement_tests.yml needs: check_repo if: "!failure() && !cancelled() && needs.check_repo.outputs.should_run_workflow == 'true'" - runs-on: ubuntu-latest - - strategy: - fail-fast: false - matrix: - include: - - arrangement: monolith - database: SQLite - - - arrangement: monolith - database: Postgres - - - arrangement: workers - database: Postgres - - steps: - - name: Check out synapse codebase - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - with: - path: synapse - - - name: Prepare Complement's Prerequisites - run: synapse/.ci/scripts/setup_complement_prerequisites.sh - - - uses: actions/setup-go@4b73464bb391d4059bd26b0524d20df3927bd417 # v6.3.0 - with: - cache-dependency-path: complement/go.sum - go-version-file: complement/go.mod - - - name: Run Complement Tests - id: run_complement_tests - # -p=1: We're using `-p 1` to force the test packages to run serially as GHA boxes - # are underpowered and don't like running tons of Synapse instances at once. - # -json: Output JSON format so that gotestfmt can parse it. - # - # tee /tmp/gotest-complement.log: We tee the output to a file so that we can re-process it - # later on for better formatting with gotestfmt. But we still want the command - # to output to the terminal as it runs so we can see what's happening in - # real-time. - run: | - set -o pipefail - COMPLEMENT_DIR=`pwd`/complement synapse/scripts-dev/complement.sh -p 1 -json 2>&1 | tee /tmp/gotest-complement.log - shell: bash - env: - POSTGRES: ${{ (matrix.database == 'Postgres') && 1 || '' }} - WORKERS: ${{ (matrix.arrangement == 'workers') && 1 || '' }} - TEST_ONLY_IGNORE_POETRY_LOCKFILE: 1 - - - name: Formatted Complement test logs - # Always run this step if we attempted to run the Complement tests. - if: always() && steps.run_complement_tests.outcome != 'skipped' - run: cat /tmp/gotest-complement.log | gotestfmt -hide "successful-downloads,empty-packages" - - - name: Run in-repo Complement Tests - id: run_in_repo_complement_tests - # -p=1: We're using `-p 1` to force the test packages to run serially as GHA boxes - # are underpowered and don't like running tons of Synapse instances at once. - # -json: Output JSON format so that gotestfmt can parse it. - # - # tee /tmp/gotest-in-repo-complement.log: We tee the output to a file so that we can re-process it - # later on for better formatting with gotestfmt. But we still want the command - # to output to the terminal as it runs so we can see what's happening in - # real-time. - run: | - set -o pipefail - COMPLEMENT_DIR=`pwd`/complement synapse/scripts-dev/complement.sh --in-repo -p 1 -json 2>&1 | tee /tmp/gotest-in-repo-complement.log - shell: bash - env: - POSTGRES: ${{ (matrix.database == 'Postgres') && 1 || '' }} - WORKERS: ${{ (matrix.arrangement == 'workers') && 1 || '' }} - TEST_ONLY_IGNORE_POETRY_LOCKFILE: 1 - - - name: Formatted in-repo Complement test logs - # Always run this step if we attempted to run the Complement tests. - if: always() && steps.run_in_repo_complement_tests.outcome != 'skipped' - run: cat /tmp/gotest-in-repo-complement.log | gotestfmt -hide "successful-downloads,empty-packages" + with: + use_latest_deps: true # Open an issue if the build fails, so we know about it. # Only do this if we're not experimenting with this action in a PR. diff --git a/.github/workflows/release-artifacts.yml b/.github/workflows/release-artifacts.yml index c313a64250..8c625be96e 100644 --- a/.github/workflows/release-artifacts.yml +++ b/.github/workflows/release-artifacts.yml @@ -64,7 +64,7 @@ jobs: uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0 - name: Set up docker layer caching - uses: actions/cache@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.3 + uses: actions/cache@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4 with: path: /tmp/.buildx-cache key: ${{ runner.os }}-buildx-${{ github.sha }} diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 488208291f..2d548a3883 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -174,7 +174,7 @@ jobs: # Cribbed from # https://github.com/AustinScola/mypy-cache-github-action/blob/85ea4f2972abed39b33bd02c36e341b28ca59213/src/restore.ts#L10-L17 - name: Restore/persist mypy's cache - uses: actions/cache@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.3 + uses: actions/cache@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4 with: path: | .mypy_cache @@ -668,145 +668,11 @@ jobs: schema_diff complement: + uses: ./.github/workflows/complement_tests.yml if: "${{ !failure() && !cancelled() && needs.changes.outputs.integration == 'true' }}" needs: - linting-done - changes - runs-on: ubuntu-latest - - strategy: - fail-fast: false - matrix: - include: - - arrangement: monolith - database: SQLite - - - arrangement: monolith - database: Postgres - - - arrangement: workers - database: Postgres - - steps: - - name: Checkout synapse codebase - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - with: - path: synapse - - # Log Docker system info for debugging (compare with your local environment) and - # tracking GitHub runner changes over time (can easily compare a run from last - # week with the current one in question). - - run: docker system info - shell: bash - - - name: Install Rust - uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master - with: - toolchain: ${{ env.RUST_VERSION }} - - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 - - # We use `poetry` in `complement.sh` - - uses: matrix-org/setup-python-poetry@5bbf6603c5c930615ec8a29f1b5d7d258d905aa4 # v2.0.0 - with: - poetry-version: "2.2.1" - # Matches the `path` where we checkout Synapse above - working-directory: "synapse" - - - name: Prepare Complement's Prerequisites - run: synapse/.ci/scripts/setup_complement_prerequisites.sh - - - uses: actions/setup-go@4b73464bb391d4059bd26b0524d20df3927bd417 # v6.3.0 - with: - cache-dependency-path: complement/go.sum - go-version-file: complement/go.mod - - # Run the image sanity check test first as this is the first thing we want to know - # about (are we actually testing what we expect?) and we don't want to debug - # downstream failures (wild goose chase). - - name: Sanity check Complement image - id: run_sanity_check_complement_image_test - # -p=1: We're using `-p 1` to force the test packages to run serially as GHA boxes - # are underpowered and don't like running tons of Synapse instances at once. - # -json: Output JSON format so that gotestfmt can parse it. - # - # tee /tmp/gotest-complement.log: We tee the output to a file so that we can re-process it - # later on for better formatting with gotestfmt. But we still want the command - # to output to the terminal as it runs so we can see what's happening in - # real-time. - run: | - set -o pipefail - COMPLEMENT_DIR=`pwd`/complement synapse/scripts-dev/complement.sh --in-repo -p 1 -json -run 'TestSynapseVersion/Synapse_version_matches_current_git_checkout' 2>&1 | tee /tmp/gotest-sanity-check-complement.log - shell: bash - env: - POSTGRES: ${{ (matrix.database == 'Postgres') && 1 || '' }} - WORKERS: ${{ (matrix.arrangement == 'workers') && 1 || '' }} - - - name: Formatted sanity check Complement test logs - # Always run this step if we attempted to run the Complement tests. - if: always() && steps.run_sanity_check_complement_image_test.outcome != 'skipped' - # We do not hide successful tests in `gotestfmt` here as the list of sanity - # check tests is so short. Feel free to change this when we get more tests. - # - # Note that the `-hide` argument is interpreted by `gotestfmt`. From it, - # it derives several values under `$settings` and passes them to our - # custom `.ci/complement_package.gotpl` template to render the output. - run: cat /tmp/gotest-sanity-check-complement.log | gotestfmt -hide "successful-downloads,empty-packages" - - - name: Run Complement Tests - id: run_complement_tests - # -p=1: We're using `-p 1` to force the test packages to run serially as GHA boxes - # are underpowered and don't like running tons of Synapse instances at once. - # -json: Output JSON format so that gotestfmt can parse it. - # - # tee /tmp/gotest-complement.log: We tee the output to a file so that we can re-process it - # later on for better formatting with gotestfmt. But we still want the command - # to output to the terminal as it runs so we can see what's happening in - # real-time. - run: | - set -o pipefail - COMPLEMENT_DIR=`pwd`/complement synapse/scripts-dev/complement.sh -p 1 -json 2>&1 | tee /tmp/gotest-complement.log - shell: bash - env: - POSTGRES: ${{ (matrix.database == 'Postgres') && 1 || '' }} - WORKERS: ${{ (matrix.arrangement == 'workers') && 1 || '' }} - - - name: Formatted Complement test logs (only failing are shown) - # Always run this step if we attempted to run the Complement tests. - if: always() && steps.run_complement_tests.outcome != 'skipped' - # Hide successful tests in order to reduce the verbosity of the otherwise very large output. - # - # Note that the `-hide` argument is interpreted by `gotestfmt`. From it, - # it derives several values under `$settings` and passes them to our - # custom `.ci/complement_package.gotpl` template to render the output. - run: cat /tmp/gotest-complement.log | gotestfmt -hide "successful-downloads,successful-tests,empty-packages" - - - name: Run in-repo Complement Tests - id: run_in_repo_complement_tests - # -p=1: We're using `-p 1` to force the test packages to run serially as GHA boxes - # are underpowered and don't like running tons of Synapse instances at once. - # -json: Output JSON format so that gotestfmt can parse it. - # - # tee /tmp/gotest-in-repo-complement.log: We tee the output to a file so that we can re-process it - # later on for better formatting with gotestfmt. But we still want the command - # to output to the terminal as it runs so we can see what's happening in - # real-time. - run: | - set -o pipefail - COMPLEMENT_DIR=`pwd`/complement synapse/scripts-dev/complement.sh --in-repo -p 1 -json 2>&1 | tee /tmp/gotest-in-repo-complement.log - shell: bash - env: - POSTGRES: ${{ (matrix.database == 'Postgres') && 1 || '' }} - WORKERS: ${{ (matrix.arrangement == 'workers') && 1 || '' }} - - - name: Formatted in-repo Complement test logs (only failing are shown) - # Always run this step if we attempted to run the Complement tests. - if: always() && steps.run_in_repo_complement_tests.outcome != 'skipped' - # Hide successful tests in order to reduce the verbosity of the otherwise very large output. - # - # Note that the `-hide` argument is interpreted by `gotestfmt`. From it, - # it derives several values under `$settings` and passes them to our - # custom `.ci/complement_package.gotpl` template to render the output. - run: cat /tmp/gotest-in-repo-complement.log | gotestfmt -hide "successful-downloads,successful-tests,empty-packages" cargo-test: if: ${{ needs.changes.outputs.rust == 'true' }} diff --git a/.github/workflows/twisted_trunk.yml b/.github/workflows/twisted_trunk.yml index be73e5283b..d9d61152fb 100644 --- a/.github/workflows/twisted_trunk.yml +++ b/.github/workflows/twisted_trunk.yml @@ -154,96 +154,11 @@ jobs: /logs/**/*.log* complement: + uses: ./.github/workflows/complement_tests.yml needs: check_repo if: "!failure() && !cancelled() && needs.check_repo.outputs.should_run_workflow == 'true'" - runs-on: ubuntu-latest - - strategy: - fail-fast: false - matrix: - include: - - arrangement: monolith - database: SQLite - - - arrangement: monolith - database: Postgres - - - arrangement: workers - database: Postgres - - steps: - - name: Run actions/checkout@v4 for synapse - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - with: - path: synapse - - - name: Prepare Complement's Prerequisites - run: synapse/.ci/scripts/setup_complement_prerequisites.sh - - - uses: actions/setup-go@4b73464bb391d4059bd26b0524d20df3927bd417 # v6.3.0 - with: - cache-dependency-path: complement/go.sum - go-version-file: complement/go.mod - - # This step is specific to the 'Twisted trunk' test run: - - name: Patch dependencies - run: | - set -x - DEBIAN_FRONTEND=noninteractive sudo apt-get install -yqq python3 pipx - pipx install poetry==2.1.1 - - poetry remove -n twisted - poetry add -n --extras tls git+https://github.com/twisted/twisted.git#trunk - poetry lock - working-directory: synapse - - - name: Run Complement Tests - id: run_complement_tests - # -p=1: We're using `-p 1` to force the test packages to run serially as GHA boxes - # are underpowered and don't like running tons of Synapse instances at once. - # -json: Output JSON format so that gotestfmt can parse it. - # - # tee /tmp/gotest-complement.log: We tee the output to a file so that we can re-process it - # later on for better formatting with gotestfmt. But we still want the command - # to output to the terminal as it runs so we can see what's happening in - # real-time. - run: | - set -o pipefail - COMPLEMENT_DIR=`pwd`/complement synapse/scripts-dev/complement.sh -p 1 -json 2>&1 | tee /tmp/gotest-complement.log - shell: bash - env: - POSTGRES: ${{ (matrix.database == 'Postgres') && 1 || '' }} - WORKERS: ${{ (matrix.arrangement == 'workers') && 1 || '' }} - TEST_ONLY_SKIP_DEP_HASH_VERIFICATION: 1 - - - name: Formatted Complement test logs - # Always run this step if we attempted to run the Complement tests. - if: always() && steps.run_complement_tests.outcome != 'skipped' - run: cat /tmp/gotest-complement.log | gotestfmt -hide "successful-downloads,empty-packages" - - - name: Run in-repo Complement Tests - id: run_in_repo_complement_tests - # -p=1: We're using `-p 1` to force the test packages to run serially as GHA boxes - # are underpowered and don't like running tons of Synapse instances at once. - # -json: Output JSON format so that gotestfmt can parse it. - # - # tee /tmp/gotest-in-repo-complement.log: We tee the output to a file so that we can re-process it - # later on for better formatting with gotestfmt. But we still want the command - # to output to the terminal as it runs so we can see what's happening in - # real-time. - run: | - set -o pipefail - COMPLEMENT_DIR=`pwd`/complement synapse/scripts-dev/complement.sh --in-repo -p 1 -json 2>&1 | tee /tmp/gotest-in-repo-complement.log - shell: bash - env: - POSTGRES: ${{ (matrix.database == 'Postgres') && 1 || '' }} - WORKERS: ${{ (matrix.arrangement == 'workers') && 1 || '' }} - TEST_ONLY_SKIP_DEP_HASH_VERIFICATION: 1 - - - name: Formatted in-repo Complement test logs - # Always run this step if we attempted to run the Complement tests. - if: always() && steps.run_in_repo_complement_tests.outcome != 'skipped' - run: cat /tmp/gotest-in-repo-complement.log | gotestfmt -hide "successful-downloads,empty-packages" + with: + use_twisted_trunk: true # open an issue if the build fails, so we know about it. open-issue: diff --git a/CHANGES.md b/CHANGES.md index 51122e5097..81c30219bf 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,56 @@ +# Synapse 1.152.0 (2026-04-28) + +No significant changes since 1.152.0rc1. + +## Configuration changes needed for deployments using workers + +For deployments using workers, please note that this version introduces a new `quarantined_media_changes` stream writer, which may require configuration changes. +Please see the [the relevant section in the upgrade notes](https://github.com/element-hq/synapse/blob/develop/docs/upgrade.md#upgrading-to-v11520) for details. + +Without configuring this new stream writer, only the main process will be able to handle the `/media/quarantine` admin API endpoints for quarantining media. + +## No Famedly additions for v1.152.0_1 + +# Synapse 1.152.0rc1 (2026-04-22) + +## Features + +- Add a ["Listing quarantined media changes" Admin API](https://element-hq.github.io/synapse/latest/admin_api/media_admin_api.html#listing-quarantined-media-changes) for retrieving a paginated record of when media became (un)quarantined. ([\#19558](https://github.com/element-hq/synapse/issues/19558), [\#19677](https://github.com/element-hq/synapse/issues/19677), [\#19694](https://github.com/element-hq/synapse/issues/19694)) +- Advertise [MSC4445](https://github.com/matrix-org/matrix-spec-proposals/pull/4445) sync timeline order in `unstable_features`. ([\#19642](https://github.com/element-hq/synapse/issues/19642)) +- Report the Rust compiler version used in the Prometheus metrics. Contributed by Noah Markert. ([\#19643](https://github.com/element-hq/synapse/issues/19643)) +- Passthrough 'article' and 'profile' OpenGraph metadata on URL preview requests. ([\#19659](https://github.com/element-hq/synapse/issues/19659)) +- Add a way to re-sign local events with a new signing key. ([\#19668](https://github.com/element-hq/synapse/issues/19668)) +- Support [MSC4450: Identity Provider selection for User-Interactive Authentication with Legacy Single Sign-On](https://github.com/matrix-org/matrix-spec-proposals/pull/4450). ([\#19693](https://github.com/element-hq/synapse/issues/19693)) +- Add experimental support for [MSC4242](https://github.com/matrix-org/matrix-spec-proposals/pull/4242): State DAGs. Excludes federation support. ([\#19424](https://github.com/element-hq/synapse/issues/19424)) +- Adds [Admin API](https://element-hq.github.io/synapse/latest/usage/administration/admin_api/index.html) endpoints to + list, fetch and delete user reports. ([\#19657](https://github.com/element-hq/synapse/issues/19657)) +- Reduce database disk space usage by pruning old rows from `device_lists_changes_in_room`. ([\#19473](https://github.com/element-hq/synapse/issues/19473), [\#19709](https://github.com/element-hq/synapse/issues/19709)) + +## Bugfixes + +- Reject `device_keys: null` in the request to [`POST /_matrix/client/v3/keys/upload`](https://spec.matrix.org/v1.16/client-server-api/#post_matrixclientv3keysupload), as per the spec. This was temporarily allowed as a workaround for misbehaving clients. ([\#19637](https://github.com/element-hq/synapse/issues/19637)) +- Fix database migrations failing on platforms where SQLite is configured with `SQLITE_DBCONFIG_DEFENSIVE` by default, such as macOS. ([\#19690](https://github.com/element-hq/synapse/issues/19690)) +- Fix a bug introduced in v1.145 where a non-admin could bypass admin checks for downloading remote quarantined media. This relied on the media already being previously present on the homeserver. ([\#19639](https://github.com/element-hq/synapse/issues/19639)) + +## Improved Documentation + +- Include a workaround for running the unit tests with SQLite under recent versions of MacOS. ([\#19615](https://github.com/element-hq/synapse/issues/19615)) +- Fix Docker image link typo in worker docs. ([\#19645](https://github.com/element-hq/synapse/issues/19645)) +- Update the developer stream docs for creating a new stream to point out `_setup_sequence(...)` in `portdb`. ([\#19675](https://github.com/element-hq/synapse/issues/19675)) +- Update the developer stream docs for creating a new stream to highlight places that require documentation updates. ([\#19696](https://github.com/element-hq/synapse/issues/19696)) + +## Internal Changes + +- Update CI to use re-usable Complement GitHub CI workflow. ([\#19533](https://github.com/element-hq/synapse/issues/19533)) +- Fix docstring for `limit` argument in `_maybe_backfill_inner(...)`. ([\#19630](https://github.com/element-hq/synapse/issues/19630)) +- Document context for why increase timeout for policy server requests. ([\#19633](https://github.com/element-hq/synapse/issues/19633)) +- Run lint script to format Complement tests introduced in [#19509](https://github.com/element-hq/synapse/pull/19509). ([\#19636](https://github.com/element-hq/synapse/issues/19636)) +- Small simplifications to the events class. ([\#19680](https://github.com/element-hq/synapse/issues/19680), [\#19712](https://github.com/element-hq/synapse/issues/19712)) +- Introduce `spam_checker_spammy` internal event metadata. ([\#19453](https://github.com/element-hq/synapse/issues/19453)) +- Add a `FilteredEvent` class that saves us copying events. ([\#19640](https://github.com/element-hq/synapse/issues/19640)) +- Convert `EventInternalMetadata` to use `Arc>`. ([\#19669](https://github.com/element-hq/synapse/issues/19669)) + + # Synapse 1.151.0 (2026-04-07) ## Bugfixes diff --git a/Cargo.lock b/Cargo.lock index 5cb2ab58af..e1747182e3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1078,6 +1078,15 @@ version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "357703d41365b4b27c590e3ed91eabb1b663f07c4c084095e60cbed4362dff0d" +[[package]] +name = "rustc_version" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" +dependencies = [ + "semver", +] + [[package]] name = "rustls" version = "0.23.31" @@ -1169,6 +1178,12 @@ dependencies = [ "libc", ] +[[package]] +name = "semver" +version = "1.0.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d767eb0aabc880b29956c35734170f26ed551a859dbd361d140cdbeca61ab1e2" + [[package]] name = "serde" version = "1.0.228" @@ -1330,6 +1345,7 @@ dependencies = [ "pythonize", "regex", "reqwest", + "rustc_version", "serde", "serde_json", "sha2", diff --git a/contrib/grafana/synapse.json b/contrib/grafana/synapse.json index 21be6ebb86..917b7f2ae2 100644 --- a/contrib/grafana/synapse.json +++ b/contrib/grafana/synapse.json @@ -6803,6 +6803,155 @@ ], "title": "Stale extremity dropping", "type": "timeseries" + }, + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "description": "For a given percentage P, the number X where P% of events were persisted to rooms with X state DAG forward extremities or fewer.", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 50 + }, + "id": 181, + "options": { + "alertThreshold": true + }, + "pluginVersion": "9.2.2", + "targets": [ + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "histogram_quantile(0.5, rate(synapse_storage_msc4242_state_dag_forward_extremities_persisted_bucket{server_name=\"$server_name\"}[$bucket_size]) and on (instance, job, index) (synapse_storage_events_persisted_events_total > 0))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "50%", + "refId": "A" + }, + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "histogram_quantile(0.75, rate(synapse_storage_msc4242_state_dag_forward_extremities_persisted_bucket{server_name=\"$server_name\"}[$bucket_size]) and on (instance, job, index) (synapse_storage_events_persisted_events_total > 0))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "75%", + "refId": "B" + }, + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "histogram_quantile(0.90, rate(synapse_storage_msc4242_state_dag_forward_extremities_persisted_bucket{server_name=\"$server_name\"}[$bucket_size]) and on (instance, job, index) (synapse_storage_events_persisted_events_total > 0))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "90%", + "refId": "C" + }, + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "histogram_quantile(0.99, rate(synapse_storage_msc4242_state_dag_forward_extremities_persisted_bucket{server_name=\"$server_name\"}[$bucket_size]) and on (instance, job, index) (synapse_storage_events_persisted_events_total > 0))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "99%", + "refId": "D" + } + ], + "title": "Events persisted, by number of state DAG forward extremities in room (quantiles)", + "type": "timeseries" + }, + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "description": "Colour reflects the number of events persisted to rooms with the given number of state DAG forward extremities, or fewer.", + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 50 + }, + "id": 127, + "options": { + "calculate": false, + "calculation": {}, + "cellGap": 1, + "cellValues": {}, + "color": { + "exponent": 0.5, + "fill": "#5794F2", + "min": 0, + "mode": "opacity", + "reverse": false, + "scale": "exponential", + "scheme": "Oranges", + "steps": 128 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "showValue": "never", + "tooltip": { + "show": true, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "decimals": 0, + "reverse": false, + "unit": "short" + } + }, + "pluginVersion": "9.2.2", + "targets": [ + { + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "expr": "rate(synapse_storage_msc4242_state_dag_forward_extremities_persisted_bucket{server_name=\"$server_name\"}[$bucket_size]) and on (instance, job, index) (synapse_storage_events_persisted_events_total > 0)", + "format": "heatmap", + "intervalFactor": 1, + "legendFormat": "{{le}}", + "refId": "A" + } + ], + "title": "Events persisted, by number of state DAG forward extremities in room (heatmap)", + "type": "heatmap" } ], "title": "Extremities", @@ -7705,4 +7854,4 @@ "uid": "000000012", "version": 1, "weekStart": "" -} +} \ No newline at end of file diff --git a/debian/changelog b/debian/changelog index f3dc1eb0c2..ff9dfe3e13 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,15 @@ +matrix-synapse-py3 (1.152.0) stable; urgency=medium + + * New Synapse release 1.152.0. + + -- Synapse Packaging team Tue, 28 Apr 2026 11:45:01 +0100 + +matrix-synapse-py3 (1.152.0~rc1) stable; urgency=medium + + * New Synapse release 1.152.0rc1. + + -- Synapse Packaging team Wed, 22 Apr 2026 12:03:58 +0100 + matrix-synapse-py3 (1.151.0) stable; urgency=medium * New synapse release 1.151.0. diff --git a/docker/configure_workers_and_start.py b/docker/configure_workers_and_start.py index 1b8d4f9989..26c8556eff 100755 --- a/docker/configure_workers_and_start.py +++ b/docker/configure_workers_and_start.py @@ -119,7 +119,7 @@ }, "media_repository": { "app": "synapse.app.generic_worker", - "listener_resources": ["media", "client"], + "listener_resources": ["media", "client", "replication"], "endpoint_patterns": [ "^/_matrix/media/", "^/_synapse/admin/v1/purge_media_cache$", diff --git a/docs/admin_api/media_admin_api.md b/docs/admin_api/media_admin_api.md index 6b96eb3356..750be85bbe 100644 --- a/docs/admin_api/media_admin_api.md +++ b/docs/admin_api/media_admin_api.md @@ -247,6 +247,42 @@ Response: {} ``` +## Listing quarantined media changes + +When media is quarantined or unquarantined, a change record is created in the +database. This API returns those change records in the order they were created. + +**Note**: This API should be considered *best-effort* and expected to have missing or +duplicate records. Currently, this only captures any media explicitly (un)quarantined by +the media quarantine admin API, and the other cases are tracked by +https://github.com/element-hq/synapse/issues/19672. Historical media uploaded before +Synapse 1.152.0 is backfilled in a background update on a best-effort basis. + +Each page has a maximum of 100 records. The first page has the oldest records, +paginating forwards with each `next_batch` value. + +Request: + +``` +GET /_synapse/admin/v1/media/quarantine_changes?from=2 +``` + +Where `from` is the `next_batch` value from a previous request. It is optional +and defaults to the first page (the value `0`). + +Response: + +```json +{ + "next_batch": 4, + "changes": [ + { "origin": "example.org", "media_id": "abcdefg12345...", "quarantined": true }, + { "origin": "example.org", "media_id": "abcdefg12345...", "quarantined": false }, + { "origin": "another.example.org", "media_id": "abcdefg12345...", "quarantined": true } + ] +} +``` + # Delete local media This API deletes the *local* media from the disk of your own server. This includes any local thumbnails and copies of media downloaded from diff --git a/docs/development/synapse_architecture/streams.md b/docs/development/synapse_architecture/streams.md index 1c081ab9d6..49b67d1186 100644 --- a/docs/development/synapse_architecture/streams.md +++ b/docs/development/synapse_architecture/streams.md @@ -158,6 +158,8 @@ These rough notes and links may help you to create a new stream and add all the necessary registration and event handling. **Create your stream:** +- Create a Postgres-specific database delta file to [add a new `SEQUENCE`](https://github.com/element-hq/synapse/blob/35b55e962aa0bed3b2da5a3c12e3783ddf7604ca/synapse/storage/schema/main/delta/93/01_sticky_events_seq.sql.postgres#L14-L18) (this will be referenced by the `MultiWriterIdGenerator` below). +- Update `synapse/_scripts/synapse_port_db.py` so it knows about your new `SEQUENCE`: [add a new `_setup_sequence(...)`](https://github.com/element-hq/synapse/blob/35b55e962aa0bed3b2da5a3c12e3783ddf7604ca/synapse/_scripts/synapse_port_db.py#L883C24-L888) - [create a stream class and stream row class](https://github.com/element-hq/synapse/blob/4367fb2d078c52959aeca0fe6874539c53e8360d/synapse/replication/tcp/streams/_base.py#L728) - will need an [ID generator](https://github.com/element-hq/synapse/blob/4367fb2d078c52959aeca0fe6874539c53e8360d/synapse/storage/databases/main/thread_subscriptions.py#L75) - may need [writer configuration](https://github.com/element-hq/synapse/blob/4367fb2d078c52959aeca0fe6874539c53e8360d/synapse/config/workers.py#L177), if there isn't already an obvious source of configuration for which workers should be designated as writers to your new stream. @@ -177,6 +179,13 @@ necessary registration and event handling. - don't forget the super call - add local-only [invalidations to your writer transactions](https://github.com/element-hq/synapse/blob/4367fb2d078c52959aeca0fe6874539c53e8360d/synapse/storage/databases/main/thread_subscriptions.py#L201) +**Update docs:** +- Update the [*Stream + writers*](https://github.com/element-hq/synapse/blob/develop/docs/workers.md#stream-writers) + section in the worker docs with a new section for the stream +- If this stream can only be handled by specific workers, add a new section to the + [upgrade notes](https://github.com/element-hq/synapse/blob/develop/docs/upgrade.md). + **For streams to be used in sync:** - add a new field to [`StreamToken`](https://github.com/element-hq/synapse/blob/4367fb2d078c52959aeca0fe6874539c53e8360d/synapse/types/__init__.py#L1003) - add a new [`StreamKeyType`](https://github.com/element-hq/synapse/blob/4367fb2d078c52959aeca0fe6874539c53e8360d/synapse/types/__init__.py#L999) diff --git a/docs/upgrade.md b/docs/upgrade.md index 0833ae5a3c..5c69446c91 100644 --- a/docs/upgrade.md +++ b/docs/upgrade.md @@ -117,6 +117,19 @@ each upgrade are complete before moving on to the next upgrade, to avoid stacking them up. You can monitor the currently running background updates with [the Admin API](usage/administration/admin_api/background_updates.html#status). +# Upgrading to v1.152.0 + +## Workers which quarantine media must be stream writers + +A new [`quarantined_media_changes` stream writer](./workers.md#the-quarantined_media_changes-stream) is +introduced. Existing deployments which route the `/quarantine_media` endpoints to a +worker (instead of the main process) *must* also add those workers to the +`quarantined_media_changes` stream writer list. Quarantining media will not work without +this. + +If your deployment does not use workers, or instead uses the main process for +quarantining media, you do not need to make any changes to your configuration. + # Upgrading to v1.150.0 ## Removal of the `systemd` pip extra diff --git a/docs/usage/administration/admin_api/background_updates.md b/docs/usage/administration/admin_api/background_updates.md index 7b75ee5587..7a78b8964b 100644 --- a/docs/usage/administration/admin_api/background_updates.md +++ b/docs/usage/administration/admin_api/background_updates.md @@ -107,3 +107,6 @@ The following JSON body parameters are available: - `job_name` - A string which job to run. Valid values are: - `populate_stats_process_rooms` - Recalculate the stats for all rooms. - `regenerate_directory` - Recalculate the [user directory](../../../user_directory.md) if it is stale or out of sync. + - `event_resign` - Re-sign all locally-sent events with the current signing key. This is useful after rotating the server's signing key to ensure all historical events are signed with the new key. Optional additional parameters: + - `old_key` - Only re-sign events whose signature verifies against this key. Format: `"ed25519:key_id base64_public_key"` (e.g. `"ed25519:my_old_key XGX0JRS2Af3be3knz2fBiRbApjm2Dh61gXDJA8kcJNI"`). + - `before_ts` - Only re-sign events with a `received_ts` less than this value (milliseconds since the epoch). diff --git a/docs/workers.md b/docs/workers.md index c2aef33e16..8d3aad19c6 100644 --- a/docs/workers.md +++ b/docs/workers.md @@ -60,10 +60,10 @@ virtualenv, these can be installed with: pip install "matrix-synapse[redis]" ``` -Note that these dependencies are included when synapse is installed with `pip -install matrix-synapse[all]`. They are also included in the debian packages from -`packages.matrix.org` and in the docker images at -https://hub.docker.com/r/ectorim/synapse/. +Note that these dependencies are included when Synapse is installed with `pip install +matrix-synapse[all]`. They are also included in the [Debian +packages](setup/installation.md#debianubuntu) and in the [Docker +images](setup/installation.md#docker-images-and-ansible-playbooks). To make effective use of the workers, you will need to configure an HTTP reverse-proxy such as nginx or haproxy, which will direct incoming requests to @@ -576,6 +576,14 @@ configured as stream writer for the `device_lists` stream: ^/_matrix/client/(api/v1|r0|v3|unstable)/keys/device_signing/upload$ ^/_matrix/client/(api/v1|r0|v3|unstable)/keys/signatures/upload$ +##### The `quarantined_media_changes` stream + +The `quarantined_media_changes` stream supports multiple writers. The following endpoints +can be handled by any worker, but should be routed directly to one of the workers +configured as stream writer for the `quarantined_media_changes` stream: + + ^/_synapse/admin/v1/quarantine_media/.*$ + #### Restrict outbound federation traffic to a specific set of workers The diff --git a/poetry.lock b/poetry.lock index 4ed2b96bf9..b6578ec161 100644 --- a/poetry.lock +++ b/poetry.lock @@ -26,15 +26,15 @@ files = [ [[package]] name = "authlib" -version = "1.6.9" +version = "1.6.11" description = "The ultimate Python library in building OAuth and OpenID Connect servers and clients." optional = true python-versions = ">=3.9" groups = ["main"] markers = "extra == \"oidc\" or extra == \"jwt\" or extra == \"all\"" files = [ - {file = "authlib-1.6.9-py2.py3-none-any.whl", hash = "sha256:f08b4c14e08f0861dc18a32357b33fbcfd2ea86cfe3fe149484b4d764c4a0ac3"}, - {file = "authlib-1.6.9.tar.gz", hash = "sha256:d8f2421e7e5980cc1ddb4e32d3f5fa659cfaf60d8eaf3281ebed192e4ab74f04"}, + {file = "authlib-1.6.11-py2.py3-none-any.whl", hash = "sha256:c8687a9a26451c51a34a06fa17bb97cb15bba46a6a626755e2d7f50da8bff3e3"}, + {file = "authlib-1.6.11.tar.gz", hash = "sha256:64db35b9b01aeccb4715a6c9a6613a06f2bd7be2ab9d2eb89edd1dfc7580a38f"}, ] [package.dependencies] @@ -453,61 +453,61 @@ files = [ [[package]] name = "cryptography" -version = "46.0.6" +version = "46.0.7" description = "cryptography is a package which provides cryptographic recipes and primitives to Python developers." optional = false python-versions = "!=3.9.0,!=3.9.1,>=3.8" groups = ["main", "dev"] files = [ - {file = "cryptography-46.0.6-cp311-abi3-macosx_10_9_universal2.whl", hash = "sha256:64235194bad039a10bb6d2d930ab3323baaec67e2ce36215fd0952fad0930ca8"}, - {file = "cryptography-46.0.6-cp311-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:26031f1e5ca62fcb9d1fcb34b2b60b390d1aacaa15dc8b895a9ed00968b97b30"}, - {file = "cryptography-46.0.6-cp311-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:9a693028b9cbe51b5a1136232ee8f2bc242e4e19d456ded3fa7c86e43c713b4a"}, - {file = "cryptography-46.0.6-cp311-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:67177e8a9f421aa2d3a170c3e56eca4e0128883cf52a071a7cbf53297f18b175"}, - {file = "cryptography-46.0.6-cp311-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:d9528b535a6c4f8ff37847144b8986a9a143585f0540fbcb1a98115b543aa463"}, - {file = "cryptography-46.0.6-cp311-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:22259338084d6ae497a19bae5d4c66b7ca1387d3264d1c2c0e72d9e9b6a77b97"}, - {file = "cryptography-46.0.6-cp311-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:760997a4b950ff00d418398ad73fbc91aa2894b5c1db7ccb45b4f68b42a63b3c"}, - {file = "cryptography-46.0.6-cp311-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:3dfa6567f2e9e4c5dceb8ccb5a708158a2a871052fa75c8b78cb0977063f1507"}, - {file = "cryptography-46.0.6-cp311-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:cdcd3edcbc5d55757e5f5f3d330dd00007ae463a7e7aa5bf132d1f22a4b62b19"}, - {file = "cryptography-46.0.6-cp311-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:d4e4aadb7fc1f88687f47ca20bb7227981b03afaae69287029da08096853b738"}, - {file = "cryptography-46.0.6-cp311-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:2b417edbe8877cda9022dde3a008e2deb50be9c407eef034aeeb3a8b11d9db3c"}, - {file = "cryptography-46.0.6-cp311-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:380343e0653b1c9d7e1f55b52aaa2dbb2fdf2730088d48c43ca1c7c0abb7cc2f"}, - {file = "cryptography-46.0.6-cp311-abi3-win32.whl", hash = "sha256:bcb87663e1f7b075e48c3be3ecb5f0b46c8fc50b50a97cf264e7f60242dca3f2"}, - {file = "cryptography-46.0.6-cp311-abi3-win_amd64.whl", hash = "sha256:6739d56300662c468fddb0e5e291f9b4d084bead381667b9e654c7dd81705124"}, - {file = "cryptography-46.0.6-cp314-cp314t-macosx_10_9_universal2.whl", hash = "sha256:2ef9e69886cbb137c2aef9772c2e7138dc581fad4fcbcf13cc181eb5a3ab6275"}, - {file = "cryptography-46.0.6-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:7f417f034f91dcec1cb6c5c35b07cdbb2ef262557f701b4ecd803ee8cefed4f4"}, - {file = "cryptography-46.0.6-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:d24c13369e856b94892a89ddf70b332e0b70ad4a5c43cf3e9cb71d6d7ffa1f7b"}, - {file = "cryptography-46.0.6-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:aad75154a7ac9039936d50cf431719a2f8d4ed3d3c277ac03f3339ded1a5e707"}, - {file = "cryptography-46.0.6-cp314-cp314t-manylinux_2_28_ppc64le.whl", hash = "sha256:3c21d92ed15e9cfc6eb64c1f5a0326db22ca9c2566ca46d845119b45b4400361"}, - {file = "cryptography-46.0.6-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:4668298aef7cddeaf5c6ecc244c2302a2b8e40f384255505c22875eebb47888b"}, - {file = "cryptography-46.0.6-cp314-cp314t-manylinux_2_31_armv7l.whl", hash = "sha256:8ce35b77aaf02f3b59c90b2c8a05c73bac12cea5b4e8f3fbece1f5fddea5f0ca"}, - {file = "cryptography-46.0.6-cp314-cp314t-manylinux_2_34_aarch64.whl", hash = "sha256:c89eb37fae9216985d8734c1afd172ba4927f5a05cfd9bf0e4863c6d5465b013"}, - {file = "cryptography-46.0.6-cp314-cp314t-manylinux_2_34_ppc64le.whl", hash = "sha256:ed418c37d095aeddf5336898a132fba01091f0ac5844e3e8018506f014b6d2c4"}, - {file = "cryptography-46.0.6-cp314-cp314t-manylinux_2_34_x86_64.whl", hash = "sha256:69cf0056d6947edc6e6760e5f17afe4bea06b56a9ac8a06de9d2bd6b532d4f3a"}, - {file = "cryptography-46.0.6-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:8e7304c4f4e9490e11efe56af6713983460ee0780f16c63f219984dab3af9d2d"}, - {file = "cryptography-46.0.6-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:b928a3ca837c77a10e81a814a693f2295200adb3352395fad024559b7be7a736"}, - {file = "cryptography-46.0.6-cp314-cp314t-win32.whl", hash = "sha256:97c8115b27e19e592a05c45d0dd89c57f81f841cc9880e353e0d3bf25b2139ed"}, - {file = "cryptography-46.0.6-cp314-cp314t-win_amd64.whl", hash = "sha256:c797e2517cb7880f8297e2c0f43bb910e91381339336f75d2c1c2cbf811b70b4"}, - {file = "cryptography-46.0.6-cp38-abi3-macosx_10_9_universal2.whl", hash = "sha256:12cae594e9473bca1a7aceb90536060643128bb274fcea0fc459ab90f7d1ae7a"}, - {file = "cryptography-46.0.6-cp38-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:639301950939d844a9e1c4464d7e07f902fe9a7f6b215bb0d4f28584729935d8"}, - {file = "cryptography-46.0.6-cp38-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:ed3775295fb91f70b4027aeba878d79b3e55c0b3e97eaa4de71f8f23a9f2eb77"}, - {file = "cryptography-46.0.6-cp38-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:8927ccfbe967c7df312ade694f987e7e9e22b2425976ddbf28271d7e58845290"}, - {file = "cryptography-46.0.6-cp38-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:b12c6b1e1651e42ab5de8b1e00dc3b6354fdfd778e7fa60541ddacc27cd21410"}, - {file = "cryptography-46.0.6-cp38-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:063b67749f338ca9c5a0b7fe438a52c25f9526b851e24e6c9310e7195aad3b4d"}, - {file = "cryptography-46.0.6-cp38-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:02fad249cb0e090b574e30b276a3da6a149e04ee2f049725b1f69e7b8351ec70"}, - {file = "cryptography-46.0.6-cp38-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:7e6142674f2a9291463e5e150090b95a8519b2fb6e6aaec8917dd8d094ce750d"}, - {file = "cryptography-46.0.6-cp38-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:456b3215172aeefb9284550b162801d62f5f264a081049a3e94307fe20792cfa"}, - {file = "cryptography-46.0.6-cp38-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:341359d6c9e68834e204ceaf25936dffeafea3829ab80e9503860dcc4f4dac58"}, - {file = "cryptography-46.0.6-cp38-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:9a9c42a2723999a710445bc0d974e345c32adfd8d2fac6d8a251fa829ad31cfb"}, - {file = "cryptography-46.0.6-cp38-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:6617f67b1606dfd9fe4dbfa354a9508d4a6d37afe30306fe6c101b7ce3274b72"}, - {file = "cryptography-46.0.6-cp38-abi3-win32.whl", hash = "sha256:7f6690b6c55e9c5332c0b59b9c8a3fb232ebf059094c17f9019a51e9827df91c"}, - {file = "cryptography-46.0.6-cp38-abi3-win_amd64.whl", hash = "sha256:79e865c642cfc5c0b3eb12af83c35c5aeff4fa5c672dc28c43721c2c9fdd2f0f"}, - {file = "cryptography-46.0.6-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:2ea0f37e9a9cf0df2952893ad145fd9627d326a59daec9b0802480fa3bcd2ead"}, - {file = "cryptography-46.0.6-pp311-pypy311_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:a3e84d5ec9ba01f8fd03802b2147ba77f0c8f2617b2aff254cedd551844209c8"}, - {file = "cryptography-46.0.6-pp311-pypy311_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:12f0fa16cc247b13c43d56d7b35287ff1569b5b1f4c5e87e92cc4fcc00cd10c0"}, - {file = "cryptography-46.0.6-pp311-pypy311_pp73-manylinux_2_34_aarch64.whl", hash = "sha256:50575a76e2951fe7dbd1f56d181f8c5ceeeb075e9ff88e7ad997d2f42af06e7b"}, - {file = "cryptography-46.0.6-pp311-pypy311_pp73-manylinux_2_34_x86_64.whl", hash = "sha256:90e5f0a7b3be5f40c3a0a0eafb32c681d8d2c181fc2a1bdabe9b3f611d9f6b1a"}, - {file = "cryptography-46.0.6-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:6728c49e3b2c180ef26f8e9f0a883a2c585638db64cf265b49c9ba10652d430e"}, - {file = "cryptography-46.0.6.tar.gz", hash = "sha256:27550628a518c5c6c903d84f637fbecf287f6cb9ced3804838a1295dc1fd0759"}, + {file = "cryptography-46.0.7-cp311-abi3-macosx_10_9_universal2.whl", hash = "sha256:ea42cbe97209df307fdc3b155f1b6fa2577c0defa8f1f7d3be7d31d189108ad4"}, + {file = "cryptography-46.0.7-cp311-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:b36a4695e29fe69215d75960b22577197aca3f7a25b9cf9d165dcfe9d80bc325"}, + {file = "cryptography-46.0.7-cp311-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:5ad9ef796328c5e3c4ceed237a183f5d41d21150f972455a9d926593a1dcb308"}, + {file = "cryptography-46.0.7-cp311-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:73510b83623e080a2c35c62c15298096e2a5dc8d51c3b4e1740211839d0dea77"}, + {file = "cryptography-46.0.7-cp311-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:cbd5fb06b62bd0721e1170273d3f4d5a277044c47ca27ee257025146c34cbdd1"}, + {file = "cryptography-46.0.7-cp311-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:420b1e4109cc95f0e5700eed79908cef9268265c773d3a66f7af1eef53d409ef"}, + {file = "cryptography-46.0.7-cp311-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:24402210aa54baae71d99441d15bb5a1919c195398a87b563df84468160a65de"}, + {file = "cryptography-46.0.7-cp311-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:8a469028a86f12eb7d2fe97162d0634026d92a21f3ae0ac87ed1c4a447886c83"}, + {file = "cryptography-46.0.7-cp311-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:9694078c5d44c157ef3162e3bf3946510b857df5a3955458381d1c7cfc143ddb"}, + {file = "cryptography-46.0.7-cp311-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:42a1e5f98abb6391717978baf9f90dc28a743b7d9be7f0751a6f56a75d14065b"}, + {file = "cryptography-46.0.7-cp311-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:91bbcb08347344f810cbe49065914fe048949648f6bd5c2519f34619142bbe85"}, + {file = "cryptography-46.0.7-cp311-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:5d1c02a14ceb9148cc7816249f64f623fbfee39e8c03b3650d842ad3f34d637e"}, + {file = "cryptography-46.0.7-cp311-abi3-win32.whl", hash = "sha256:d23c8ca48e44ee015cd0a54aeccdf9f09004eba9fc96f38c911011d9ff1bd457"}, + {file = "cryptography-46.0.7-cp311-abi3-win_amd64.whl", hash = "sha256:397655da831414d165029da9bc483bed2fe0e75dde6a1523ec2fe63f3c46046b"}, + {file = "cryptography-46.0.7-cp314-cp314t-macosx_10_9_universal2.whl", hash = "sha256:d151173275e1728cf7839aaa80c34fe550c04ddb27b34f48c232193df8db5842"}, + {file = "cryptography-46.0.7-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:db0f493b9181c7820c8134437eb8b0b4792085d37dbb24da050476ccb664e59c"}, + {file = "cryptography-46.0.7-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:ebd6daf519b9f189f85c479427bbd6e9c9037862cf8fe89ee35503bd209ed902"}, + {file = "cryptography-46.0.7-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:b7b412817be92117ec5ed95f880defe9cf18a832e8cafacf0a22337dc1981b4d"}, + {file = "cryptography-46.0.7-cp314-cp314t-manylinux_2_28_ppc64le.whl", hash = "sha256:fbfd0e5f273877695cb93baf14b185f4878128b250cc9f8e617ea0c025dfb022"}, + {file = "cryptography-46.0.7-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:ffca7aa1d00cf7d6469b988c581598f2259e46215e0140af408966a24cf086ce"}, + {file = "cryptography-46.0.7-cp314-cp314t-manylinux_2_31_armv7l.whl", hash = "sha256:60627cf07e0d9274338521205899337c5d18249db56865f943cbe753aa96f40f"}, + {file = "cryptography-46.0.7-cp314-cp314t-manylinux_2_34_aarch64.whl", hash = "sha256:80406c3065e2c55d7f49a9550fe0c49b3f12e5bfff5dedb727e319e1afb9bf99"}, + {file = "cryptography-46.0.7-cp314-cp314t-manylinux_2_34_ppc64le.whl", hash = "sha256:c5b1ccd1239f48b7151a65bc6dd54bcfcc15e028c8ac126d3fada09db0e07ef1"}, + {file = "cryptography-46.0.7-cp314-cp314t-manylinux_2_34_x86_64.whl", hash = "sha256:d5f7520159cd9c2154eb61eb67548ca05c5774d39e9c2c4339fd793fe7d097b2"}, + {file = "cryptography-46.0.7-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:fcd8eac50d9138c1d7fc53a653ba60a2bee81a505f9f8850b6b2888555a45d0e"}, + {file = "cryptography-46.0.7-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:65814c60f8cc400c63131584e3e1fad01235edba2614b61fbfbfa954082db0ee"}, + {file = "cryptography-46.0.7-cp314-cp314t-win32.whl", hash = "sha256:fdd1736fed309b4300346f88f74cd120c27c56852c3838cab416e7a166f67298"}, + {file = "cryptography-46.0.7-cp314-cp314t-win_amd64.whl", hash = "sha256:e06acf3c99be55aa3b516397fe42f5855597f430add9c17fa46bf2e0fb34c9bb"}, + {file = "cryptography-46.0.7-cp38-abi3-macosx_10_9_universal2.whl", hash = "sha256:462ad5cb1c148a22b2e3bcc5ad52504dff325d17daf5df8d88c17dda1f75f2a4"}, + {file = "cryptography-46.0.7-cp38-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:84d4cced91f0f159a7ddacad249cc077e63195c36aac40b4150e7a57e84fffe7"}, + {file = "cryptography-46.0.7-cp38-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:128c5edfe5e5938b86b03941e94fac9ee793a94452ad1365c9fc3f4f62216832"}, + {file = "cryptography-46.0.7-cp38-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:5e51be372b26ef4ba3de3c167cd3d1022934bc838ae9eaad7e644986d2a3d163"}, + {file = "cryptography-46.0.7-cp38-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:cdf1a610ef82abb396451862739e3fc93b071c844399e15b90726ef7470eeaf2"}, + {file = "cryptography-46.0.7-cp38-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:1d25aee46d0c6f1a501adcddb2d2fee4b979381346a78558ed13e50aa8a59067"}, + {file = "cryptography-46.0.7-cp38-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:cdfbe22376065ffcf8be74dc9a909f032df19bc58a699456a21712d6e5eabfd0"}, + {file = "cryptography-46.0.7-cp38-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:abad9dac36cbf55de6eb49badd4016806b3165d396f64925bf2999bcb67837ba"}, + {file = "cryptography-46.0.7-cp38-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:935ce7e3cfdb53e3536119a542b839bb94ec1ad081013e9ab9b7cfd478b05006"}, + {file = "cryptography-46.0.7-cp38-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:35719dc79d4730d30f1c2b6474bd6acda36ae2dfae1e3c16f2051f215df33ce0"}, + {file = "cryptography-46.0.7-cp38-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:7bbc6ccf49d05ac8f7d7b5e2e2c33830d4fe2061def88210a126d130d7f71a85"}, + {file = "cryptography-46.0.7-cp38-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:a1529d614f44b863a7b480c6d000fe93b59acee9c82ffa027cfadc77521a9f5e"}, + {file = "cryptography-46.0.7-cp38-abi3-win32.whl", hash = "sha256:f247c8c1a1fb45e12586afbb436ef21ff1e80670b2861a90353d9b025583d246"}, + {file = "cryptography-46.0.7-cp38-abi3-win_amd64.whl", hash = "sha256:506c4ff91eff4f82bdac7633318a526b1d1309fc07ca76a3ad182cb5b686d6d3"}, + {file = "cryptography-46.0.7-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:fc9ab8856ae6cf7c9358430e49b368f3108f050031442eaeb6b9d87e4dcf4e4f"}, + {file = "cryptography-46.0.7-pp311-pypy311_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:d3b99c535a9de0adced13d159c5a9cf65c325601aa30f4be08afd680643e9c15"}, + {file = "cryptography-46.0.7-pp311-pypy311_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:d02c738dacda7dc2a74d1b2b3177042009d5cab7c7079db74afc19e56ca1b455"}, + {file = "cryptography-46.0.7-pp311-pypy311_pp73-manylinux_2_34_aarch64.whl", hash = "sha256:04959522f938493042d595a736e7dbdff6eb6cc2339c11465b3ff89343b65f65"}, + {file = "cryptography-46.0.7-pp311-pypy311_pp73-manylinux_2_34_x86_64.whl", hash = "sha256:3986ac1dee6def53797289999eabe84798ad7817f3e97779b5061a95b0ee4968"}, + {file = "cryptography-46.0.7-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:258514877e15963bd43b558917bc9f54cf7cf866c38aa576ebf47a77ddbc43a4"}, + {file = "cryptography-46.0.7.tar.gz", hash = "sha256:e4cfd68c5f3e0bfdad0d38e023239b96a2fe84146481852dffbcca442c245aa5"}, ] [package.dependencies] @@ -521,7 +521,7 @@ nox = ["nox[uv] (>=2024.4.15)"] pep8test = ["check-sdist", "click (>=8.0.1)", "mypy (>=1.14)", "ruff (>=0.11.11)"] sdist = ["build (>=1.0.0)"] ssh = ["bcrypt (>=3.1.5)"] -test = ["certifi (>=2024)", "cryptography-vectors (==46.0.6)", "pretend (>=0.7)", "pytest (>=7.4.0)", "pytest-benchmark (>=4.0)", "pytest-cov (>=2.10.1)", "pytest-xdist (>=3.5.0)"] +test = ["certifi (>=2024)", "cryptography-vectors (==46.0.7)", "pretend (>=0.7)", "pytest (>=7.4.0)", "pytest-benchmark (>=4.0)", "pytest-cov (>=2.10.1)", "pytest-xdist (>=3.5.0)"] test-randomorder = ["pytest-randomly"] [[package]] @@ -698,118 +698,118 @@ protobuf = ["grpcio-tools (>=1.78.0)"] [[package]] name = "hiredis" -version = "3.3.0" +version = "3.3.1" description = "Python wrapper for hiredis" optional = true python-versions = ">=3.8" groups = ["main"] markers = "extra == \"redis\" or extra == \"all\"" files = [ - {file = "hiredis-3.3.0-cp310-cp310-macosx_10_15_universal2.whl", hash = "sha256:9937d9b69321b393fbace69f55423480f098120bc55a3316e1ca3508c4dbbd6f"}, - {file = "hiredis-3.3.0-cp310-cp310-macosx_10_15_x86_64.whl", hash = "sha256:50351b77f89ba6a22aff430b993653847f36b71d444509036baa0f2d79d1ebf4"}, - {file = "hiredis-3.3.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:1d00bce25c813eec45a2f524249f58daf51d38c9d3347f6f643ae53826fc735a"}, - {file = "hiredis-3.3.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0ef840d9f142556ed384180ed8cdf14ff875fcae55c980cbe5cec7adca2ef4d8"}, - {file = "hiredis-3.3.0-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:88bc79d7e9b94d17ed1bd8b7f2815ed0eada376ed5f48751044e5e4d179aa2f2"}, - {file = "hiredis-3.3.0-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:7165c7363e59b258e1875c51f35c0b2b9901e6c691037b487d8a0ace2c137ed2"}, - {file = "hiredis-3.3.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8c3be446f0c38fbe6863a7cf4522c9a463df6e64bee87c4402e9f6d7d2e7f869"}, - {file = "hiredis-3.3.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:96f9a27643279853b91a1fb94a88b559e55fdecec86f1fcd5f2561492be52e47"}, - {file = "hiredis-3.3.0-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:0a5eebb170de1b415c78ae5ca3aee17cff8b885df93c2055d54320e789d838f4"}, - {file = "hiredis-3.3.0-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:200678547ac3966bac3e38df188211fdc13d5f21509c23267e7def411710e112"}, - {file = "hiredis-3.3.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:dd9d78c5363a858f9dc5e698e5e1e402b83c00226cba294f977a92c53092b549"}, - {file = "hiredis-3.3.0-cp310-cp310-win32.whl", hash = "sha256:a0d31ff178b913137a7a08c7377e93805914755a15c3585e203d0d74496456c0"}, - {file = "hiredis-3.3.0-cp310-cp310-win_amd64.whl", hash = "sha256:7b41833c8f0d4c7fbfaa867c8ed9a4e4aaa71d7c54e4806ed62da2d5cd27b40d"}, - {file = "hiredis-3.3.0-cp311-cp311-macosx_10_15_universal2.whl", hash = "sha256:63ee6c1ae6a2462a2439eb93c38ab0315cd5f4b6d769c6a34903058ba538b5d6"}, - {file = "hiredis-3.3.0-cp311-cp311-macosx_10_15_x86_64.whl", hash = "sha256:31eda3526e2065268a8f97fbe3d0e9a64ad26f1d89309e953c80885c511ea2ae"}, - {file = "hiredis-3.3.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:a26bae1b61b7bcafe3d0d0c7d012fb66ab3c95f2121dbea336df67e344e39089"}, - {file = "hiredis-3.3.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b9546079f7fd5c50fbff9c791710049b32eebe7f9b94debec1e8b9f4c048cba2"}, - {file = "hiredis-3.3.0-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:ae327fc13b1157b694d53f92d50920c0051e30b0c245f980a7036e299d039ab4"}, - {file = "hiredis-3.3.0-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:4016e50a8be5740a59c5af5252e5ad16c395021a999ad24c6604f0d9faf4d346"}, - {file = "hiredis-3.3.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c17b473f273465a3d2168a57a5b43846165105ac217d5652a005e14068589ddc"}, - {file = "hiredis-3.3.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:9ecd9b09b11bd0b8af87d29c3f5da628d2bdc2a6c23d2dd264d2da082bd4bf32"}, - {file = "hiredis-3.3.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:00fb04eac208cd575d14f246e74a468561081ce235937ab17d77cde73aefc66c"}, - {file = "hiredis-3.3.0-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:60814a7d0b718adf3bfe2c32c6878b0e00d6ae290ad8e47f60d7bba3941234a6"}, - {file = "hiredis-3.3.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:fcbd1a15e935aa323b5b2534b38419511b7909b4b8ee548e42b59090a1b37bb1"}, - {file = "hiredis-3.3.0-cp311-cp311-win32.whl", hash = "sha256:73679607c5a19f4bcfc9cf6eb54480bcd26617b68708ac8b1079da9721be5449"}, - {file = "hiredis-3.3.0-cp311-cp311-win_amd64.whl", hash = "sha256:30a4df3d48f32538de50648d44146231dde5ad7f84f8f08818820f426840ae97"}, - {file = "hiredis-3.3.0-cp312-cp312-macosx_10_15_universal2.whl", hash = "sha256:5b8e1d6a2277ec5b82af5dce11534d3ed5dffeb131fd9b210bc1940643b39b5f"}, - {file = "hiredis-3.3.0-cp312-cp312-macosx_10_15_x86_64.whl", hash = "sha256:c4981de4d335f996822419e8a8b3b87367fcef67dc5fb74d3bff4df9f6f17783"}, - {file = "hiredis-3.3.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:1706480a683e328ae9ba5d704629dee2298e75016aa0207e7067b9c40cecc271"}, - {file = "hiredis-3.3.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0a95cef9989736ac313639f8f545b76b60b797e44e65834aabbb54e4fad8d6c8"}, - {file = "hiredis-3.3.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:ca2802934557ccc28a954414c245ba7ad904718e9712cb67c05152cf6b9dd0a3"}, - {file = "hiredis-3.3.0-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:fe730716775f61e76d75810a38ee4c349d3af3896450f1525f5a4034cf8f2ed7"}, - {file = "hiredis-3.3.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:749faa69b1ce1f741f5eaf743435ac261a9262e2d2d66089192477e7708a9abc"}, - {file = "hiredis-3.3.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:95c9427f2ac3f1dd016a3da4e1161fa9d82f221346c8f3fdd6f3f77d4e28946c"}, - {file = "hiredis-3.3.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:c863ee44fe7bff25e41f3a5105c936a63938b76299b802d758f40994ab340071"}, - {file = "hiredis-3.3.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:2213c7eb8ad5267434891f3241c7776e3bafd92b5933fc57d53d4456247dc542"}, - {file = "hiredis-3.3.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:a172bae3e2837d74530cd60b06b141005075db1b814d966755977c69bd882ce8"}, - {file = "hiredis-3.3.0-cp312-cp312-win32.whl", hash = "sha256:cb91363b9fd6d41c80df9795e12fffbaf5c399819e6ae8120f414dedce6de068"}, - {file = "hiredis-3.3.0-cp312-cp312-win_amd64.whl", hash = "sha256:04ec150e95eea3de9ff8bac754978aa17b8bf30a86d4ab2689862020945396b0"}, - {file = "hiredis-3.3.0-cp313-cp313-macosx_10_15_universal2.whl", hash = "sha256:b7048b4ec0d5dddc8ddd03da603de0c4b43ef2540bf6e4c54f47d23e3480a4fa"}, - {file = "hiredis-3.3.0-cp313-cp313-macosx_10_15_x86_64.whl", hash = "sha256:e5f86ce5a779319c15567b79e0be806e8e92c18bb2ea9153e136312fafa4b7d6"}, - {file = "hiredis-3.3.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:fbdb97a942e66016fff034df48a7a184e2b7dc69f14c4acd20772e156f20d04b"}, - {file = "hiredis-3.3.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b0fb4bea72fe45ff13e93ddd1352b43ff0749f9866263b5cca759a4c960c776f"}, - {file = "hiredis-3.3.0-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:85b9baf98050e8f43c2826ab46aaf775090d608217baf7af7882596aef74e7f9"}, - {file = "hiredis-3.3.0-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:69079fb0f0ebb61ba63340b9c4bce9388ad016092ca157e5772eb2818209d930"}, - {file = "hiredis-3.3.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c17f77b79031ea4b0967d30255d2ae6e7df0603ee2426ad3274067f406938236"}, - {file = "hiredis-3.3.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:45d14f745fc177bc05fc24bdf20e2b515e9a068d3d4cce90a0fb78d04c9c9d9a"}, - {file = "hiredis-3.3.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:ba063fdf1eff6377a0c409609cbe890389aefddfec109c2d20fcc19cfdafe9da"}, - {file = "hiredis-3.3.0-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:1799cc66353ad066bfdd410135c951959da9f16bcb757c845aab2f21fc4ef099"}, - {file = "hiredis-3.3.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:2cbf71a121996ffac82436b6153290815b746afb010cac19b3290a1644381b07"}, - {file = "hiredis-3.3.0-cp313-cp313-win32.whl", hash = "sha256:a7cbbc6026bf03659f0b25e94bbf6e64f6c8c22f7b4bc52fe569d041de274194"}, - {file = "hiredis-3.3.0-cp313-cp313-win_amd64.whl", hash = "sha256:a8def89dd19d4e2e4482b7412d453dec4a5898954d9a210d7d05f60576cedef6"}, - {file = "hiredis-3.3.0-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:c135bda87211f7af9e2fd4e046ab433c576cd17b69e639a0f5bb2eed5e0e71a9"}, - {file = "hiredis-3.3.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:2f855c678230aed6fc29b962ce1cc67e5858a785ef3a3fd6b15dece0487a2e60"}, - {file = "hiredis-3.3.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:4059c78a930cbb33c391452ccce75b137d6f89e2eebf6273d75dafc5c2143c03"}, - {file = "hiredis-3.3.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:334a3f1d14c253bb092e187736c3384203bd486b244e726319bbb3f7dffa4a20"}, - {file = "hiredis-3.3.0-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:fd137b147235447b3d067ec952c5b9b95ca54b71837e1b38dbb2ec03b89f24fc"}, - {file = "hiredis-3.3.0-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:8f88f4f2aceb73329ece86a1cb0794fdbc8e6d614cb5ca2d1023c9b7eb432db8"}, - {file = "hiredis-3.3.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:550f4d1538822fc75ebf8cf63adc396b23d4958bdbbad424521f2c0e3dfcb169"}, - {file = "hiredis-3.3.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:54b14211fbd5930fc696f6fcd1f1f364c660970d61af065a80e48a1fa5464dd6"}, - {file = "hiredis-3.3.0-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:c9e96f63dbc489fc86f69951e9f83dadb9582271f64f6822c47dcffa6fac7e4a"}, - {file = "hiredis-3.3.0-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:106e99885d46684d62ab3ec1d6b01573cc0e0083ac295b11aaa56870b536c7ec"}, - {file = "hiredis-3.3.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:087e2ef3206361281b1a658b5b4263572b6ba99465253e827796964208680459"}, - {file = "hiredis-3.3.0-cp314-cp314-win32.whl", hash = "sha256:80638ebeab1cefda9420e9fedc7920e1ec7b4f0513a6b23d58c9d13c882f8065"}, - {file = "hiredis-3.3.0-cp314-cp314-win_amd64.whl", hash = "sha256:a68aaf9ba024f4e28cf23df9196ff4e897bd7085872f3a30644dca07fa787816"}, - {file = "hiredis-3.3.0-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:f7f80442a32ce51ee5d89aeb5a84ee56189a0e0e875f1a57bbf8d462555ae48f"}, - {file = "hiredis-3.3.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:a1a67530da714954ed50579f4fe1ab0ddbac9c43643b1721c2cb226a50dde263"}, - {file = "hiredis-3.3.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:616868352e47ab355559adca30f4f3859f9db895b4e7bc71e2323409a2add751"}, - {file = "hiredis-3.3.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e799b79f3150083e9702fc37e6243c0bd47a443d6eae3f3077b0b3f510d6a145"}, - {file = "hiredis-3.3.0-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:9ef1dfb0d2c92c3701655e2927e6bbe10c499aba632c7ea57b6392516df3864b"}, - {file = "hiredis-3.3.0-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:c290da6bc2a57e854c7da9956cd65013483ede935677e84560da3b848f253596"}, - {file = "hiredis-3.3.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:fd8c438d9e1728f0085bf9b3c9484d19ec31f41002311464e75b69550c32ffa8"}, - {file = "hiredis-3.3.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:1bbc6b8a88bbe331e3ebf6685452cebca6dfe6d38a6d4efc5651d7e363ba28bd"}, - {file = "hiredis-3.3.0-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:55d8c18fe9a05496c5c04e6eccc695169d89bf358dff964bcad95696958ec05f"}, - {file = "hiredis-3.3.0-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:4ddc79afa76b805d364e202a754666cb3c4d9c85153cbfed522871ff55827838"}, - {file = "hiredis-3.3.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:8e8a4b8540581dcd1b2b25827a54cfd538e0afeaa1a0e3ca87ad7126965981cc"}, - {file = "hiredis-3.3.0-cp314-cp314t-win32.whl", hash = "sha256:298593bb08487753b3afe6dc38bac2532e9bac8dcee8d992ef9977d539cc6776"}, - {file = "hiredis-3.3.0-cp314-cp314t-win_amd64.whl", hash = "sha256:b442b6ab038a6f3b5109874d2514c4edf389d8d8b553f10f12654548808683bc"}, - {file = "hiredis-3.3.0-cp38-cp38-macosx_10_15_universal2.whl", hash = "sha256:114c0b9f1b5fad99edae38e747018aead358a4f4e9720cc1876495d78cdb8276"}, - {file = "hiredis-3.3.0-cp38-cp38-macosx_10_15_x86_64.whl", hash = "sha256:c6d91a5e6904ed7eca21d74b041e03f2ad598dd08a6065b06a776974fe5d003c"}, - {file = "hiredis-3.3.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:76374faa075e996c895cbe106ba923852a9f8146f2aa59eba22111c5e5ec6316"}, - {file = "hiredis-3.3.0-cp38-cp38-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:50a54397bd104c2e2f5b7696bbdab8ba2973d3075e4deb932adb025b8863de91"}, - {file = "hiredis-3.3.0-cp38-cp38-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:15edee02cc9cc06e07e2bcfae07e283e640cc1aeedd08b4c6934bf1a0113c607"}, - {file = "hiredis-3.3.0-cp38-cp38-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:ff3179a57745d0f8d71fa8bf3ea3944d3f557dcfa4431304497987fecad381dd"}, - {file = "hiredis-3.3.0-cp38-cp38-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bdb7cd9e1e73db78f145a09bb837732790d0912eb963dee5768631faf2ece162"}, - {file = "hiredis-3.3.0-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:4d3b4e0d4445faf9041c52a98cb5d2b65c4fcaebb2aa02efa7c6517c4917f7e8"}, - {file = "hiredis-3.3.0-cp38-cp38-musllinux_1_2_ppc64le.whl", hash = "sha256:ffea6c407cff532c7599d3ec9e8502c2c865753cebab044f3dfce9afbf71a8df"}, - {file = "hiredis-3.3.0-cp38-cp38-musllinux_1_2_s390x.whl", hash = "sha256:bcd745a28e1b3216e42680d91e142a42569dfad68a6f40535080c47b0356c796"}, - {file = "hiredis-3.3.0-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:4c18a97ea55d1a58f5c3adfe236b3e7cccedc6735cbd36ab1c786c52fd823667"}, - {file = "hiredis-3.3.0-cp38-cp38-win32.whl", hash = "sha256:77eacd969e3c6ff50c2b078c27d2a773c652248a5d81af5765a8663478d0bc02"}, - {file = "hiredis-3.3.0-cp38-cp38-win_amd64.whl", hash = "sha256:161a4a595a53475587aef8dc549d0527962879b0c5d62f7947b44ba7e5084b76"}, - {file = "hiredis-3.3.0-cp39-cp39-macosx_10_15_universal2.whl", hash = "sha256:1203697a7ebadc7cf873acc189df9e44fcb377b636e6660471707ac8d5bcba68"}, - {file = "hiredis-3.3.0-cp39-cp39-macosx_10_15_x86_64.whl", hash = "sha256:9a7ea2344d277317160da4911f885bcf7dfd8381b830d76b442f7775b41544b3"}, - {file = "hiredis-3.3.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:9bd7c9a089cf4e4f4b5a61f412c76293449bac6b0bf92bb49a3892850bd5c899"}, - {file = "hiredis-3.3.0-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:294de11e3995128c784534e327d1f9382b88dc5407356465df7934c710e8392d"}, - {file = "hiredis-3.3.0-cp39-cp39-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:4a3aab895358368f81f9546a7cd192b6fb427f785cb1a8853cf9db38df01e9ca"}, - {file = "hiredis-3.3.0-cp39-cp39-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:eaf8418e33e23d6d7ef0128eff4c06ab3040d40b9bbc8a24d6265d751a472596"}, - {file = "hiredis-3.3.0-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:41aea51949142bad4e40badb0396392d7f4394791e4097a0951ab75bcc58ff84"}, - {file = "hiredis-3.3.0-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:1f9a5f84a8bd29ac5b9953b27e8ba5508396afeabf1d165611a1e31fbd90a0e1"}, - {file = "hiredis-3.3.0-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:a5f9fde56550ebbe962f437a4c982b0856d03aea7fab09e30fa6c0f9be992b40"}, - {file = "hiredis-3.3.0-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:c567aab02612d91f3e747fc492100ae894515194f85d6fb6bb68958c0e718721"}, - {file = "hiredis-3.3.0-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:ca97c5e6f9e9b9f0aed61b70fed2d594ce2f7472905077d2d10b307c50a41008"}, - {file = "hiredis-3.3.0-cp39-cp39-win32.whl", hash = "sha256:776dc5769d5eb05e969216de095377ff61c802414a74bd3c24a4ca8526c897ab"}, - {file = "hiredis-3.3.0-cp39-cp39-win_amd64.whl", hash = "sha256:538a9f5fbb3a8a4ef0c3abd309cccb90cd2ba9976fcc2b44193af9507d005b48"}, - {file = "hiredis-3.3.0.tar.gz", hash = "sha256:105596aad9249634361815c574351f1bd50455dc23b537c2940066c4a9dea685"}, + {file = "hiredis-3.3.1-cp310-cp310-macosx_10_15_universal2.whl", hash = "sha256:f525734382a47f9828c9d6a1501522c78d5935466d8e2be1a41ba40ca5bb922b"}, + {file = "hiredis-3.3.1-cp310-cp310-macosx_10_15_x86_64.whl", hash = "sha256:6e2e1024f0a021777740cb7c633a0efb2c4a4bc570f508223a8dcbcf79f99ef9"}, + {file = "hiredis-3.3.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:c1d68c6980d4690a4550bd3db6c03146f7be68ef5d08d38bb1fb68b3e9c32fe3"}, + {file = "hiredis-3.3.1-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0caf3fc8af0767794b335753781c3fa35f2a3e975c098edbc8f733d35d6a95e4"}, + {file = "hiredis-3.3.1-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:81a1669b6631976b1dc9d3d58ed1ab3333e9f52feb91a2a1fb8241101ac3b665"}, + {file = "hiredis-3.3.1-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:c8139e9011117822391c5bcfd674c5948fb1e4b8cb9adf6f13d9890859ee3a1a"}, + {file = "hiredis-3.3.1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:042e57de8a2cae91e3e7c0af32960ea2c5107b2f27f68a740295861e68780a8a"}, + {file = "hiredis-3.3.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:65f6ac06a9f0c32c254660ec6a9329d81d589e8f5d0a9837a941d5424a6be1ef"}, + {file = "hiredis-3.3.1-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:002fc0201b9af1cc8960e27cdc501ad1f8cdd6dbadb2091c6ddbd4e5ace6cb77"}, + {file = "hiredis-3.3.1-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:9ebae74ce2b977c2fcb22d6a10aa0acb730022406977b2bcb6ddd6788f5c414a"}, + {file = "hiredis-3.3.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:8a52b24cd710690c4a7e191c7e300136ad2ecb3c68ffe7e95b598e76de166e5e"}, + {file = "hiredis-3.3.1-cp310-cp310-win32.whl", hash = "sha256:1ebc307a87b099d0877dbd2bdc0bae427258e7ec67f60a951e89027f8dc2568f"}, + {file = "hiredis-3.3.1-cp310-cp310-win_amd64.whl", hash = "sha256:62cc62284541bb2a86c898c7d5e8388661cade91c184cb862095ed547e80588f"}, + {file = "hiredis-3.3.1-cp311-cp311-macosx_10_15_universal2.whl", hash = "sha256:26f899cde0279e4b7d370716ff80320601c2bd93cdf3e774a42bdd44f65b41f8"}, + {file = "hiredis-3.3.1-cp311-cp311-macosx_10_15_x86_64.whl", hash = "sha256:a2f049c3f3c83e886cd1f53958e2a1ebb369be626bef9e50d8b24d79864f1df6"}, + {file = "hiredis-3.3.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:5f316cf2d0558f5027aab19dde7d7e4901c26c21fa95367bc37784e8f547bbf2"}, + {file = "hiredis-3.3.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:03baa381964b8df356d19ec4e3a6ae656044249a87b0def257fe1e08dbaf6094"}, + {file = "hiredis-3.3.1-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:304481241e081bc26f0778b2c2b99f9c43917e4e724a016dcc9439b7ab12c726"}, + {file = "hiredis-3.3.1-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:8597c35c9e82f65fd5897c4a2188c65d7daf10607b102960137b23d261cd957b"}, + {file = "hiredis-3.3.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ad940dc2db545dc978cb41cb9a683e2ff328f3ef581230b9ca40ff6c3d01d542"}, + {file = "hiredis-3.3.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:156be6a0c736ee145cfe0fb155d0e96cec8d4872cf8b4f76ad6a2ee6ab391d0a"}, + {file = "hiredis-3.3.1-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:583de2f16528e66081cbdfe510d8488c2de73039dc00aada7d22bd49d73a4a94"}, + {file = "hiredis-3.3.1-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:c24c1460486b6b36083252c2db21a814becf8495ccd0e76b7286623e37239b63"}, + {file = "hiredis-3.3.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:a58a58cef0d911b1717154179a9ff47852249c536ea5966bde4370b6b20638ff"}, + {file = "hiredis-3.3.1-cp311-cp311-win32.whl", hash = "sha256:e0db44cf81e4d7b94f3776b9f89111f74ed6bbdbfd42a22bc4a5ce0644d3e060"}, + {file = "hiredis-3.3.1-cp311-cp311-win_amd64.whl", hash = "sha256:1f7bceb03a1b934872ffe3942eaeed7c7e09096e67b53f095b81f39c7a819113"}, + {file = "hiredis-3.3.1-cp312-cp312-macosx_10_15_universal2.whl", hash = "sha256:60543f3b068b16a86e99ed96b7fdae71cdc1d8abdfe9b3f82032a555e52ece7e"}, + {file = "hiredis-3.3.1-cp312-cp312-macosx_10_15_x86_64.whl", hash = "sha256:2611bfaaadc5e8d43fb7967f9bbf1110c8beaa83aee2f2d812c76f11cfb56c6a"}, + {file = "hiredis-3.3.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:8e3754ce60e1b11b0afad9a053481ff184d2ee24bea47099107156d1b84a84aa"}, + {file = "hiredis-3.3.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e89dabf436ee79b358fd970dcbed6333a36d91db73f27069ca24a02fb138a404"}, + {file = "hiredis-3.3.1-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:4f7e242eab698ad0be5a4b2ec616fa856569c57455cc67c625fd567726290e5f"}, + {file = "hiredis-3.3.1-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:53148a4e21057541b6d8e493b2ea1b500037ddf34433c391970036f3cbce00e3"}, + {file = "hiredis-3.3.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c25132902d3eff38781e0d54f27a0942ec849e3c07dbdce83c4d92b7e43c8dce"}, + {file = "hiredis-3.3.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3fb6573efa15a29c12c0c0f7170b14e7c1347fe4bb39b6a15b779f46015cc929"}, + {file = "hiredis-3.3.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:487658e1db83c1ee9fbbac6a43039ea76957767a5987ffb16b590613f9e68297"}, + {file = "hiredis-3.3.1-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:a1d190790ee39b8b7adeeb10fc4090dc4859eb4e75ed27bd8108710eef18f358"}, + {file = "hiredis-3.3.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:a42c7becd4c9ec4ab5769c754eb61112777bdc6e1c1525e2077389e193b5f5aa"}, + {file = "hiredis-3.3.1-cp312-cp312-win32.whl", hash = "sha256:17ec8b524055a88b80d76c177dbbbe475a25c17c5bf4b67bdbdbd0629bcae838"}, + {file = "hiredis-3.3.1-cp312-cp312-win_amd64.whl", hash = "sha256:0fac4af8515e6cca74fc701169ae4dc9a71a90e9319c9d21006ec9454b43aa2f"}, + {file = "hiredis-3.3.1-cp313-cp313-macosx_10_15_universal2.whl", hash = "sha256:afe3c3863f16704fb5d7c2c6ff56aaf9e054f6d269f7b4c9074c5476178d1aba"}, + {file = "hiredis-3.3.1-cp313-cp313-macosx_10_15_x86_64.whl", hash = "sha256:f19ee7dc1ef8a6497570d91fa4057ba910ad98297a50b8c44ff37589f7c89d17"}, + {file = "hiredis-3.3.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:09f5e510f637f2c72d2a79fb3ad05f7b6211e057e367ca5c4f97bb3d8c9d71f4"}, + {file = "hiredis-3.3.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1b46e96b50dad03495447860510daebd2c96fd44ed25ba8ccb03e9f89eaa9d34"}, + {file = "hiredis-3.3.1-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:b4fe7f38aa8956fcc1cea270e62601e0e11066aff78e384be70fd283d30293b6"}, + {file = "hiredis-3.3.1-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:2b96da7e365d6488d2a75266a662cbe3cc14b28c23dd9b0c9aa04b5bc5c20192"}, + {file = "hiredis-3.3.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:52d5641027d6731bc7b5e7d126a5158a99784a9f8c6de3d97ca89aca4969e9f8"}, + {file = "hiredis-3.3.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:eddeb9a153795cf6e615f9f3cef66a1d573ff3b6ee16df2b10d1d1c2f2baeaa8"}, + {file = "hiredis-3.3.1-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:011a9071c3df4885cac7f58a2623feac6c8e2ad30e6ba93c55195af05ce61ff5"}, + {file = "hiredis-3.3.1-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:264ee7e9cb6c30dc78da4ecf71d74cf14ca122817c665d838eda8b4384bce1b0"}, + {file = "hiredis-3.3.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:6d1434d0bcc1b3ef048bae53f26456405c08aeed9827e65b24094f5f3a6793f1"}, + {file = "hiredis-3.3.1-cp313-cp313-win32.whl", hash = "sha256:f915a34fb742e23d0d61573349aa45d6f74037fde9d58a9f340435eff8d62736"}, + {file = "hiredis-3.3.1-cp313-cp313-win_amd64.whl", hash = "sha256:d8e56e0d1fe607bfff422633f313aec9191c3859ab99d11ff097e3e6e068000c"}, + {file = "hiredis-3.3.1-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:439f9a5cc8f9519ce208a24cdebfa0440fef26aa682a40ba2c92acb10a53f5e0"}, + {file = "hiredis-3.3.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:3724f0e58c6ff76fd683429945491de71324ab1bc0ad943a8d68cb0932d24075"}, + {file = "hiredis-3.3.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:29fe35e3c6fe03204e75c86514f452591957a1e06b05d86e10d795455b71c355"}, + {file = "hiredis-3.3.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d42f3a13290f89191568fc113d95a3d2c8759cdd8c3672f021d8b7436f909e75"}, + {file = "hiredis-3.3.1-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2afc675b831f7552da41116fffffca4340f387dc03f56d6ec0c7895ab0b59a10"}, + {file = "hiredis-3.3.1-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:4106201cd052d9eabe3cb7b5a24b0fe37307792bda4fcb3cf6ddd72f697828e8"}, + {file = "hiredis-3.3.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8887bf0f31e4b550bd988c8863b527b6587d200653e9375cd91eea2b944b7424"}, + {file = "hiredis-3.3.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:1ac7697365dbe45109273b34227fee6826b276ead9a4a007e0877e1d3f0fcf21"}, + {file = "hiredis-3.3.1-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:2b6da6e07359107c653a809b3cff2d9ccaeedbafe33c6f16434aef6f53ce4a2b"}, + {file = "hiredis-3.3.1-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:ce334915f5d31048f76a42c607bf26687cf045eb1bc852b7340f09729c6a64fc"}, + {file = "hiredis-3.3.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:ee11fd431f83d8a5b29d370b9d79a814d3218d30113bdcd44657e9bdf715fc92"}, + {file = "hiredis-3.3.1-cp314-cp314-win32.whl", hash = "sha256:e0356561b4a97c83b9ee3de657a41b8d1a1781226853adaf47b550bb988fda6f"}, + {file = "hiredis-3.3.1-cp314-cp314-win_amd64.whl", hash = "sha256:80aba5f85d6227faee628ae28d1c3b69c661806a0636548ac56c68782606454f"}, + {file = "hiredis-3.3.1-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:907f7b5501a534030738f0f27459a612d2266fd0507b007bb8f3e6de08167920"}, + {file = "hiredis-3.3.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:de94b409f49eb6a588ebdd5872e826caec417cd77c17af0fb94f2128427f1a2a"}, + {file = "hiredis-3.3.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:79cd03e7ff550c17758a7520bf437c156d3d4c8bb74214deeafa69cda49c85a4"}, + {file = "hiredis-3.3.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6ffa7ba2e2da1f806f3181b9730b3e87ba9dbfec884806725d4584055ba3faa6"}, + {file = "hiredis-3.3.1-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:ee37fe8cf081b72dea72f96a0ee604f492ec02252eb77dc26ff6eec3f997b580"}, + {file = "hiredis-3.3.1-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:9bfdeff778d3f7ff449ca5922ab773899e7d31e26a576028b06a5e9cf0ed8c34"}, + {file = "hiredis-3.3.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:027ce4fabfeff5af5b9869d5524770877f9061d118bc36b85703ae3faf5aad8e"}, + {file = "hiredis-3.3.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:dcea8c3f53674ae68e44b12e853b844a1d315250ca6677b11ec0c06aff85e86c"}, + {file = "hiredis-3.3.1-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:0b5ff2f643f4b452b0597b7fe6aa35d398cb31d8806801acfafb1558610ea2aa"}, + {file = "hiredis-3.3.1-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:3586c8a5f56d34b9dddaaa9e76905f31933cac267251006adf86ec0eef7d0400"}, + {file = "hiredis-3.3.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:a110d19881ca78a88583d3b07231e7c6864864f5f1f3491b638863ea45fa8708"}, + {file = "hiredis-3.3.1-cp314-cp314t-win32.whl", hash = "sha256:98fd5b39410e9d69e10e90d0330e35650becaa5dd2548f509b9598f1f3c6124d"}, + {file = "hiredis-3.3.1-cp314-cp314t-win_amd64.whl", hash = "sha256:ab1f646ff531d70bfd25f01e60708dfa3d105eb458b7dedd9fe9a443039fd809"}, + {file = "hiredis-3.3.1-cp38-cp38-macosx_10_15_universal2.whl", hash = "sha256:a3af4e9f277d6b8acd369dc44a723a055752fca9d045094383af39f90a3e3729"}, + {file = "hiredis-3.3.1-cp38-cp38-macosx_10_15_x86_64.whl", hash = "sha256:526db52e5234a9463520e960a509d6c1bd5128d1ab1b569cbf459fe39189e8ab"}, + {file = "hiredis-3.3.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:90d6b9f2652303aefd2c5a26a5e14cb74a3a63d10faa642c08d790e99442a088"}, + {file = "hiredis-3.3.1-cp38-cp38-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4479e36d263251dba8ab8ea81adf07e7f1163603c7102c5de1e130b83b4fad3b"}, + {file = "hiredis-3.3.1-cp38-cp38-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:b2390ad81c03d93ef1d5afd18ffcf5935de827f1a2b96b2c829437968bdabccb"}, + {file = "hiredis-3.3.1-cp38-cp38-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:65c05b79cb8366c123357b354a16f9fc3f7187159422f143638d1c26b7240ed4"}, + {file = "hiredis-3.3.1-cp38-cp38-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:09d41a3a965f7c261223d516ebda607aee4d8440dd7637f01af9a4c05872f0c4"}, + {file = "hiredis-3.3.1-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:113e098e4a6b3cc5500e05e7cb1548ba9e83de5fe755941b11f6020a76e6c03a"}, + {file = "hiredis-3.3.1-cp38-cp38-musllinux_1_2_ppc64le.whl", hash = "sha256:e31e92b61d56244047ad600812e16f7587a6172f74810fd919ff993af12b9149"}, + {file = "hiredis-3.3.1-cp38-cp38-musllinux_1_2_s390x.whl", hash = "sha256:77c5d2bebbc9d06691abb512a31d0f54e1562af0b872891463a67a949b5278ef"}, + {file = "hiredis-3.3.1-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:137c14905ea6f2933967200bc7b2a0c8ec9387888b273fd0004f25b994fd0343"}, + {file = "hiredis-3.3.1-cp38-cp38-win32.whl", hash = "sha256:f2f94355affd51088f57f8674b0e294704c3c7c3d7d3b1545310f5b135d4843b"}, + {file = "hiredis-3.3.1-cp38-cp38-win_amd64.whl", hash = "sha256:b1e3b9f4bf9a4120510ba77a77b2fb674893cd6795653545152bb11a79eecfcb"}, + {file = "hiredis-3.3.1-cp39-cp39-macosx_10_15_universal2.whl", hash = "sha256:743b85bd6902856cac457ddd8cd7dd48c89c47d641b6016ff5e4d015bfbd4799"}, + {file = "hiredis-3.3.1-cp39-cp39-macosx_10_15_x86_64.whl", hash = "sha256:b37df4b10cb15dedfc203f69312d8eedd617b941c21df58c13af59496c53ad0f"}, + {file = "hiredis-3.3.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:8650158217b469d8b6087f490929211b0493a9121154c4efaafd1dec9e19319e"}, + {file = "hiredis-3.3.1-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c74bd9926954e7e575f9cd9890f63defd90cd8f812dfbf8e1efb72acc9355456"}, + {file = "hiredis-3.3.1-cp39-cp39-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2f1c1b2e8f00b71e6214234d313f655a3a27cd4384b054126ce04073c1d47045"}, + {file = "hiredis-3.3.1-cp39-cp39-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:01cf82a514bc4fd145b99333c28523e61b7a9ad051a245804323ebf4e7b1c6a6"}, + {file = "hiredis-3.3.1-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:db46baf157feefd88724e6a7f145fe996a5990a8604ed9292b45d563360e513b"}, + {file = "hiredis-3.3.1-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:5e55d90b431b0c6b64ae5a624208d4aea318566d31872e595ee723c0f5b9a79f"}, + {file = "hiredis-3.3.1-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:40ae8a7041fcb328a6bc7202d8c4e6e0d38d434b2e3880b1ee8ed754f17cd836"}, + {file = "hiredis-3.3.1-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:d14229beaa76e66c3a25f9477d973336441ca820df853679a98796256813316f"}, + {file = "hiredis-3.3.1-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:b3df9447f9209f9aa0434ca74050e9509670c1ad99398fe5807abb90e5f3a014"}, + {file = "hiredis-3.3.1-cp39-cp39-win32.whl", hash = "sha256:48ff424f8aa36aacd9fdaa68efeb27d2e8771f293af4305bdb15d92194ca6631"}, + {file = "hiredis-3.3.1-cp39-cp39-win_amd64.whl", hash = "sha256:318f772dd321404075d406825266e574ee0f4751be1831424c2ebd5722609398"}, + {file = "hiredis-3.3.1.tar.gz", hash = "sha256:da6f0302360e99d32bc2869772692797ebadd536e1b826d0103c72ba49d38698"}, ] [[package]] @@ -2564,14 +2564,14 @@ typing-extensions = ">=4.14.1" [[package]] name = "pygithub" -version = "2.8.1" +version = "2.9.0" description = "Use the full Github API v3" optional = false -python-versions = ">=3.8" +python-versions = ">=3.9" groups = ["dev"] files = [ - {file = "pygithub-2.8.1-py3-none-any.whl", hash = "sha256:23a0a5bca93baef082e03411bf0ce27204c32be8bfa7abc92fe4a3e132936df0"}, - {file = "pygithub-2.8.1.tar.gz", hash = "sha256:341b7c78521cb07324ff670afd1baa2bf5c286f8d9fd302c1798ba594a5400c9"}, + {file = "pygithub-2.9.0-py3-none-any.whl", hash = "sha256:5e2b260ce327bffce9b00f447b65953ef7078ffe93e5a5425624a3075483927c"}, + {file = "pygithub-2.9.0.tar.gz", hash = "sha256:a26abda1222febba31238682634cad11d8b966137ed6cc3c5e445b29a11cb0a4"}, ] [package.dependencies] @@ -2583,14 +2583,14 @@ urllib3 = ">=1.26.0" [[package]] name = "pygments" -version = "2.19.2" +version = "2.20.0" description = "Pygments is a syntax highlighting package written in Python." optional = false -python-versions = ">=3.8" +python-versions = ">=3.9" groups = ["dev"] files = [ - {file = "pygments-2.19.2-py3-none-any.whl", hash = "sha256:86540386c03d588bb81d44bc3928634ff26449851e99741617ecb9037ee5ec0b"}, - {file = "pygments-2.19.2.tar.gz", hash = "sha256:636cb2477cec7f8952536970bc533bc43743542f70392ae026374600add5b887"}, + {file = "pygments-2.20.0-py3-none-any.whl", hash = "sha256:81a9e26dd42fd28a23a2d169d86d7ac03b46e2f8b59ed4698fb4785f946d0176"}, + {file = "pygments-2.20.0.tar.gz", hash = "sha256:6757cd03768053ff99f3039c1a36d6c0aa0b263438fcab17520b30a303a82b5f"}, ] [package.extras] @@ -3225,15 +3225,15 @@ doc = ["Sphinx", "sphinx-rtd-theme"] [[package]] name = "sentry-sdk" -version = "2.54.0" +version = "2.57.0" description = "Python client for Sentry (https://sentry.io)" optional = true python-versions = ">=3.6" groups = ["main"] markers = "extra == \"sentry\" or extra == \"all\"" files = [ - {file = "sentry_sdk-2.54.0-py2.py3-none-any.whl", hash = "sha256:fd74e0e281dcda63afff095d23ebcd6e97006102cdc8e78a29f19ecdf796a0de"}, - {file = "sentry_sdk-2.54.0.tar.gz", hash = "sha256:2620c2575128d009b11b20f7feb81e4e4e8ae08ec1d36cbc845705060b45cc1b"}, + {file = "sentry_sdk-2.57.0-py2.py3-none-any.whl", hash = "sha256:812c8bf5ff3d2f0e89c82f5ce80ab3a6423e102729c4706af7413fd1eb480585"}, + {file = "sentry_sdk-2.57.0.tar.gz", hash = "sha256:4be8d1e71c32fb27f79c577a337ac8912137bba4bcbc64a4ec1da4d6d8dc5199"}, ] [package.dependencies] @@ -3244,6 +3244,7 @@ urllib3 = ">=1.26.11" aiohttp = ["aiohttp (>=3.5)"] anthropic = ["anthropic (>=0.16)"] arq = ["arq (>=0.23)"] +asyncio = ["httpcore[asyncio] (==1.*)"] asyncpg = ["asyncpg (>=0.23)"] beam = ["apache-beam (>=2.12)"] bottle = ["bottle (>=0.12.13)"] @@ -3264,7 +3265,7 @@ huggingface-hub = ["huggingface_hub (>=0.22)"] langchain = ["langchain (>=0.0.210)"] langgraph = ["langgraph (>=0.6.6)"] launchdarkly = ["launchdarkly-server-sdk (>=9.8.0)"] -litellm = ["litellm (>=1.77.5)"] +litellm = ["litellm (>=1.77.5,!=1.82.7,!=1.82.8)"] litestar = ["litestar (>=2.0.0)"] loguru = ["loguru (>=0.5)"] mcp = ["mcp (>=1.15.0)"] @@ -3407,20 +3408,20 @@ files = [ [[package]] name = "sqlglot" -version = "29.0.1" +version = "30.2.1" description = "An easily customizable SQL parser and transpiler" optional = false python-versions = ">=3.9" groups = ["dev"] files = [ - {file = "sqlglot-29.0.1-py3-none-any.whl", hash = "sha256:06a473ea6c2b3632ac67bd38e687a6860265bf4156e66b54adeda15d07f00c65"}, - {file = "sqlglot-29.0.1.tar.gz", hash = "sha256:0010b4f77fb996c8d25dd4b16f3654e6da163ff1866ceabc70b24e791c203048"}, + {file = "sqlglot-30.2.1-py3-none-any.whl", hash = "sha256:f23d9ee9427ef9d20df15f9b0ffa57d9eb45e52b012219a349d1e6b50ed926d1"}, + {file = "sqlglot-30.2.1.tar.gz", hash = "sha256:ef4a67cc6f66a8043085eb8ea95fa9541c1625dffa9145ad4e9815a7ba60a199"}, ] [package.extras] -c = ["sqlglotc"] -dev = ["duckdb (>=0.6)", "mypy", "pandas", "pandas-stubs", "pdoc", "pre-commit", "pyperf", "python-dateutil", "pytz", "ruff (==0.7.2)", "types-python-dateutil", "types-pytz", "typing_extensions"] -rs = ["sqlglotrs (==0.13.0)"] +c = ["sqlglotc (==30.2.1)"] +dev = ["duckdb (>=0.6)", "pandas", "pandas-stubs", "pdoc", "pre-commit", "pyperf", "python-dateutil", "pytz", "ruff (==0.15.6)", "setuptools_scm", "sqlglot-mypy (>=1.19.1.post1)", "types-python-dateutil", "types-pytz", "typing_extensions"] +rs = ["sqlglotc (==30.2.1)", "sqlglotrs (==0.13.0)"] [[package]] name = "threadloop" @@ -4049,4 +4050,4 @@ url-preview = ["lxml"] [metadata] lock-version = "2.1" python-versions = ">=3.10.0,<4.0.0" -content-hash = "98f0a1e0ebea0391137935787fd30ecca61062cdfb308d1f1e8b77c6700cbe17" +content-hash = "6b14e814692dc21ef76e136cceb6b778a18ba9e54c2766a38da71c1cbb87a779" diff --git a/pyproject.toml b/pyproject.toml index ab44627039..4d6566ab36 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "matrix-synapse" -version = "1.151.0" +version = "1.152.0" description = "Homeserver for the Matrix decentralised comms protocol" readme = "README.rst" authors = [ @@ -29,7 +29,10 @@ dependencies = [ # We require 2.0.0 for immutabledict support. "canonicaljson>=2.0.0,<3.0.0", # we use the type definitions added in signedjson 1.1. - "signedjson>=1.1.0,<2.0.0", + # 1.1.0 erroneously removed decode_verify_key_base64 (reintroduced in 1.1.1). + # 1.1.1 is mispackaged (importlib-metadata dependency without minimum version bound) + # 1.1.2, 1.1.3 and 1.1.4 were all released on the same day, so no good reason to use the older version. + "signedjson>=1.1.4,<2.0.0", # validating SSL certs for IP addresses requires service_identity 18.1. "service-identity>=18.1.0", # Twisted 18.9 introduces some logger improvements that the structured @@ -134,7 +137,7 @@ saml2 = [ "defusedxml>=0.7.1", # via pysaml2 "pytz>=2018.3", # via pysaml2 ] -oidc = ["authlib>=0.15.1"] +oidc = ["authlib>=1.6.11"] url-preview = ["lxml>=4.6.3"] sentry = ["sentry-sdk>=0.7.2"] opentracing-jaeger = [ @@ -189,7 +192,7 @@ all = [ # saml2 "pysaml2>=4.5.0", # oidc and jwt - "authlib>=0.15.1", + "authlib>=1.6.11", # url-preview "lxml>=4.6.3", # sentry diff --git a/rust/Cargo.toml b/rust/Cargo.toml index 8199a4e02b..bca2f6ed70 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -64,3 +64,4 @@ default = ["extension-module"] [build-dependencies] blake2 = "0.10.4" hex = "0.4.3" +rustc_version = "0.4.1" diff --git a/rust/build.rs b/rust/build.rs index 8755f3bfa3..2a99a9b95d 100644 --- a/rust/build.rs +++ b/rust/build.rs @@ -8,6 +8,7 @@ use std::path::PathBuf; use blake2::{Blake2b512, Digest}; +use rustc_version::version_meta; fn main() -> Result<(), std::io::Error> { let mut dirs = vec![PathBuf::from("src")]; @@ -48,6 +49,11 @@ fn main() -> Result<(), std::io::Error> { let hex_digest = hex::encode(hasher.finalize()); println!("cargo:rustc-env=SYNAPSE_RUST_DIGEST={hex_digest}"); + let rustc_version = version_meta() + .map(|v| v.short_version_string) + .unwrap_or_else(|_| "unknown".to_string()); + println!("cargo:rustc-env=SYNAPSE_RUSTC_VERSION={}", rustc_version,); + // The default rules don't pick up trivial changes to the workspace config // files, but we need to rebuild if those change to pick up the changed // hashes. diff --git a/rust/src/events/internal_metadata.rs b/rust/src/events/internal_metadata.rs index be71eef126..6fd3d06b00 100644 --- a/rust/src/events/internal_metadata.rs +++ b/rust/src/events/internal_metadata.rs @@ -32,12 +32,16 @@ //! attributes, but for small number of keys is actually faster than using a //! hash or btree map. -use std::{num::NonZeroI64, ops::Deref}; +use std::{ + num::NonZeroI64, + ops::Deref, + sync::{Arc, RwLock, RwLockReadGuard, RwLockWriteGuard}, +}; use anyhow::Context; use log::warn; use pyo3::{ - exceptions::PyAttributeError, + exceptions::{PyAttributeError, PyRuntimeError}, pybacked::PyBackedStr, pyclass, pymethods, types::{PyAnyMethods, PyDict, PyDictMethods, PyString}, @@ -55,11 +59,13 @@ enum EventInternalMetadataData { SoftFailed(bool), ProactivelySend(bool), PolicyServerSpammy(bool), + SpamCheckerSpammy(bool), Redacted(bool), TxnId(Box), DelayId(Box), TokenId(i64), DeviceId(Box), + CalculatedAuthEventIDs(Vec), // MSC4242: State DAGs } impl EventInternalMetadataData { @@ -105,6 +111,13 @@ impl EventInternalMetadataData { .to_owned() .into_any(), ), + EventInternalMetadataData::SpamCheckerSpammy(o) => ( + pyo3::intern!(py, "spam_checker_spammy"), + o.into_pyobject(py) + .unwrap_infallible() + .to_owned() + .into_any(), + ), EventInternalMetadataData::Redacted(o) => ( pyo3::intern!(py, "redacted"), o.into_pyobject(py) @@ -128,6 +141,10 @@ impl EventInternalMetadataData { pyo3::intern!(py, "device_id"), o.into_pyobject(py).unwrap_infallible().into_any(), ), + EventInternalMetadataData::CalculatedAuthEventIDs(o) => ( + pyo3::intern!(py, "calculated_auth_event_ids"), + o.into_pyobject(py).unwrap().into_any(), + ), } } @@ -173,6 +190,11 @@ impl EventInternalMetadataData { .extract() .with_context(|| format!("'{key_str}' has invalid type"))?, ), + "spam_checker_spammy" => EventInternalMetadataData::SpamCheckerSpammy( + value + .extract() + .with_context(|| format!("'{key_str}' has invalid type"))?, + ), "redacted" => EventInternalMetadataData::Redacted( value .extract() @@ -201,6 +223,11 @@ impl EventInternalMetadataData { .map(String::into_boxed_str) .with_context(|| format!("'{key_str}' has invalid type"))?, ), + "calculated_auth_event_ids" => EventInternalMetadataData::CalculatedAuthEventIDs( + value + .extract() + .with_context(|| format!("'{key_str}' has invalid type"))?, + ), _ => return Ok(None), }; @@ -222,19 +249,6 @@ macro_rules! get_property_opt { }; } -/// Helper macro to find the given field in internal metadata, raising an -/// attribute error if not found. -macro_rules! get_property { - ($self:expr, $name:ident) => { - get_property_opt!($self, $name).ok_or_else(|| { - PyAttributeError::new_err(format!( - "'EventInternalMetadata' has no attribute '{}'", - stringify!($name), - )) - }) - }; -} - /// Helper macro to set the give field. macro_rules! set_property { ($self:expr, $name:ident, $obj:expr) => { @@ -249,75 +263,28 @@ macro_rules! set_property { }; } -#[pyclass] +/// The inner data for `EventInternalMetadata`, containing all fields and +/// business logic with pure Rust types. #[derive(Clone)] -pub struct EventInternalMetadata { +struct EventInternalMetadataInner { /// The fields of internal metadata. This functions as a mapping. data: Vec, /// The stream ordering of this event. None, until it has been persisted. - #[pyo3(get, set)] - stream_ordering: Option, - #[pyo3(get, set)] - instance_name: Option, + pub stream_ordering: Option, + pub instance_name: Option, /// The event ID of the redaction event, if this event has been redacted. /// This is set dynamically at load time and is not persisted to the database. - #[pyo3(get, set)] - redacted_by: Option, + pub redacted_by: Option, /// whether this event is an outlier (ie, whether we have the state at that /// point in the DAG) - #[pyo3(get, set)] - outlier: bool, + pub outlier: bool, } -#[pymethods] -impl EventInternalMetadata { - #[new] - fn new(dict: &Bound<'_, PyDict>) -> PyResult { - let mut data = Vec::with_capacity(dict.len()); - - for (key, value) in dict.iter() { - match EventInternalMetadataData::from_python_pair(&key, &value) { - Ok(Some(entry)) => data.push(entry), - Ok(None) => {} - Err(err) => { - warn!("Ignoring internal metadata field '{key}', as failed to convert to Rust due to {err}") - } - } - } - - data.shrink_to_fit(); - - Ok(EventInternalMetadata { - data, - stream_ordering: None, - instance_name: None, - redacted_by: None, - outlier: false, - }) - } - - fn copy(&self) -> Self { - self.clone() - } - - /// Get a dict holding the data stored in the `internal_metadata` column in the database. - /// - /// Note that `outlier` and `stream_ordering` are stored in separate columns so are not returned here. - fn get_dict(&self, py: Python<'_>) -> PyResult> { - let dict = PyDict::new(py); - - for entry in &self.data { - let (key, value) = entry.to_python_pair(py); - dict.set_item(key, value)?; - } - - Ok(dict.into()) - } - - fn is_outlier(&self) -> bool { +impl EventInternalMetadataInner { + pub fn is_outlier(&self) -> bool { self.outlier } @@ -334,11 +301,11 @@ impl EventInternalMetadata { /// outliers. /// /// See also - /// https://famedly.github.io/synapse/latest/development/room-dag-concepts.html#out-of-band-membership-events. + /// https://element-hq.github.io/synapse/develop/development/room-dag-concepts.html#out-of-band-membership-events. /// /// (Added in synapse 0.99.0, so may be unreliable for events received /// before that) - fn is_out_of_band_membership(&self) -> bool { + pub fn is_out_of_band_membership(&self) -> bool { get_property_opt!(self, OutOfBandMembership) .copied() .unwrap_or(false) @@ -350,9 +317,8 @@ impl EventInternalMetadata { /// /// returns a str with the name of the server this event is sent on behalf /// of. - fn get_send_on_behalf_of(&self) -> Option<&str> { - let s = get_property_opt!(self, SendOnBehalfOf); - s.map(|a| a.deref()) + pub fn get_send_on_behalf_of(&self) -> Option<&str> { + get_property_opt!(self, SendOnBehalfOf).map(|a| a.deref()) } /// Whether the redaction event needs to be rechecked when fetching @@ -363,7 +329,7 @@ impl EventInternalMetadata { /// /// If the sender of the redaction event is allowed to redact any event /// due to auth rules, then this will always return false. - fn need_to_check_redaction(&self) -> bool { + pub fn need_to_check_redaction(&self) -> bool { get_property_opt!(self, RecheckRedaction) .copied() .unwrap_or(false) @@ -376,7 +342,7 @@ impl EventInternalMetadata { /// clients. /// 2. They should not be added to the forward extremities (and therefore /// not to current state). - fn is_soft_failed(&self) -> bool { + pub fn is_soft_failed(&self) -> bool { get_property_opt!(self, SoftFailed) .copied() .unwrap_or(false) @@ -386,7 +352,7 @@ impl EventInternalMetadata { /// /// This is used for sending dummy events internally. Servers and clients /// can still explicitly fetch the event. - fn should_proactively_send(&self) -> bool { + pub fn should_proactively_send(&self) -> bool { get_property_opt!(self, ProactivelySend) .copied() .unwrap_or(true) @@ -396,129 +362,431 @@ impl EventInternalMetadata { /// /// This is used for efficiently checking whether an event has been marked /// as redacted without needing to make another database call. - fn is_redacted(&self) -> bool { + pub fn is_redacted(&self) -> bool { get_property_opt!(self, Redacted).copied().unwrap_or(false) } /// Whether this event can trigger a push notification - fn is_notifiable(&self) -> bool { + pub fn is_notifiable(&self) -> bool { !self.outlier || self.is_out_of_band_membership() } - // ** The following are the getters and setters of the various properties ** + pub fn get_out_of_band_membership(&self) -> Option { + get_property_opt!(self, OutOfBandMembership).copied() + } + + pub fn get_recheck_redaction(&self) -> Option { + get_property_opt!(self, RecheckRedaction).copied() + } + + pub fn get_soft_failed(&self) -> Option { + get_property_opt!(self, SoftFailed).copied() + } + + pub fn get_proactively_send(&self) -> Option { + get_property_opt!(self, ProactivelySend).copied() + } + + pub fn get_policy_server_spammy(&self) -> bool { + get_property_opt!(self, PolicyServerSpammy) + .copied() + .unwrap_or(false) + } + + pub fn get_redacted(&self) -> Option { + get_property_opt!(self, Redacted).copied() + } + + pub fn get_txn_id(&self) -> Option<&str> { + get_property_opt!(self, TxnId).map(|s| s.deref()) + } + + pub fn get_delay_id(&self) -> Option<&str> { + get_property_opt!(self, DelayId).map(|s| s.deref()) + } + + pub fn get_calculated_auth_event_ids(&self) -> Option<&Vec> { + get_property_opt!(self, CalculatedAuthEventIDs) + } + + pub fn get_token_id(&self) -> Option { + get_property_opt!(self, TokenId).copied() + } + + pub fn get_device_id(&self) -> Option<&str> { + get_property_opt!(self, DeviceId).map(|s| s.deref()) + } + + pub fn set_out_of_band_membership(&mut self, obj: bool) { + set_property!(self, OutOfBandMembership, obj); + } + + pub fn set_send_on_behalf_of(&mut self, obj: String) { + set_property!(self, SendOnBehalfOf, obj.into_boxed_str()); + } + + pub fn set_recheck_redaction(&mut self, obj: bool) { + set_property!(self, RecheckRedaction, obj); + } + + pub fn set_soft_failed(&mut self, obj: bool) { + set_property!(self, SoftFailed, obj); + } + + pub fn set_proactively_send(&mut self, obj: bool) { + set_property!(self, ProactivelySend, obj); + } + + pub fn set_policy_server_spammy(&mut self, obj: bool) { + set_property!(self, PolicyServerSpammy, obj); + } + + fn get_spam_checker_spammy(&self) -> bool { + get_property_opt!(self, SpamCheckerSpammy) + .copied() + .unwrap_or(false) + } + + fn set_spam_checker_spammy(&mut self, obj: bool) { + set_property!(self, SpamCheckerSpammy, obj); + } + + pub fn set_redacted(&mut self, obj: bool) { + set_property!(self, Redacted, obj); + } + + pub fn set_txn_id(&mut self, obj: String) { + set_property!(self, TxnId, obj.into_boxed_str()); + } + + pub fn set_delay_id(&mut self, obj: String) { + set_property!(self, DelayId, obj.into_boxed_str()); + } + + pub fn set_token_id(&mut self, obj: i64) { + set_property!(self, TokenId, obj); + } + + pub fn set_device_id(&mut self, obj: String) { + set_property!(self, DeviceId, obj.into_boxed_str()); + } + + pub fn set_calculated_auth_event_ids(&mut self, obj: Vec) { + set_property!(self, CalculatedAuthEventIDs, obj); + } +} + +#[pyclass(frozen)] +#[derive(Clone)] +pub struct EventInternalMetadata { + inner: Arc>, +} + +impl EventInternalMetadata { + fn read_inner(&self) -> PyResult> { + self.inner + .read() + .map_err(|_| PyRuntimeError::new_err("EventInternalMetadata lock poisoned")) + } + + /// Get a write lock on the inner data. + /// + /// Note that callers should be careful not to panic while holding the write + /// lock, as this will poison the lock.q + fn write_inner(&self) -> PyResult> { + self.inner + .write() + .map_err(|_| PyRuntimeError::new_err("EventInternalMetadata lock poisoned")) + } +} + +/// Helper to convert `None` to an `AttributeError` for a property getter. +fn attr_err(val: Option, name: &str) -> PyResult { + val.ok_or_else(|| { + PyAttributeError::new_err(format!("'EventInternalMetadata' has no attribute '{name}'",)) + }) +} + +#[pymethods] +impl EventInternalMetadata { + #[new] + fn new(dict: &Bound<'_, PyDict>) -> PyResult { + let mut data = Vec::with_capacity(dict.len()); + + for (key, value) in dict.iter() { + match EventInternalMetadataData::from_python_pair(&key, &value) { + Ok(Some(entry)) => data.push(entry), + Ok(None) => {} + Err(err) => { + warn!("Ignoring internal metadata field '{key}', as failed to convert to Rust due to {err}") + } + } + } + + data.shrink_to_fit(); + + Ok(EventInternalMetadata { + inner: Arc::new(RwLock::new(EventInternalMetadataInner { + data, + stream_ordering: None, + instance_name: None, + redacted_by: None, + outlier: false, + })), + }) + } + + fn copy(&self) -> PyResult { + let guard = self.read_inner()?; + Ok(EventInternalMetadata { + inner: Arc::new(RwLock::new(guard.clone())), + }) + } + + /// Get a dict holding the data stored in the `internal_metadata` column in + /// the database. + /// + /// Note that `outlier` and `stream_ordering` are stored in separate columns + /// so are not returned here. + fn get_dict(&self, py: Python<'_>) -> PyResult> { + let guard = self.read_inner()?; + let dict = PyDict::new(py); + + for entry in &guard.data { + let (key, value) = entry.to_python_pair(py); + dict.set_item(key, value)?; + } + + Ok(dict.into()) + } + + fn is_outlier(&self) -> PyResult { + Ok(self.read_inner()?.is_outlier()) + } + + fn is_out_of_band_membership(&self) -> PyResult { + Ok(self.read_inner()?.is_out_of_band_membership()) + } + + fn get_send_on_behalf_of(&self) -> PyResult> { + Ok(self + .read_inner()? + .get_send_on_behalf_of() + .map(|s| s.to_owned())) + } + + fn need_to_check_redaction(&self) -> PyResult { + Ok(self.read_inner()?.need_to_check_redaction()) + } + + fn is_soft_failed(&self) -> PyResult { + Ok(self.read_inner()?.is_soft_failed()) + } + + fn should_proactively_send(&self) -> PyResult { + Ok(self.read_inner()?.should_proactively_send()) + } + + fn is_redacted(&self) -> PyResult { + Ok(self.read_inner()?.is_redacted()) + } + + fn is_notifiable(&self) -> PyResult { + Ok(self.read_inner()?.is_notifiable()) + } + + #[getter] + fn get_stream_ordering(&self) -> PyResult> { + Ok(self.read_inner()?.stream_ordering) + } + #[setter] + fn set_stream_ordering(&self, val: Option) -> PyResult<()> { + self.write_inner()?.stream_ordering = val; + Ok(()) + } + + #[getter] + fn get_instance_name(&self) -> PyResult> { + Ok(self.read_inner()?.instance_name.clone()) + } + #[setter] + fn set_instance_name(&self, val: Option) -> PyResult<()> { + self.write_inner()?.instance_name = val; + Ok(()) + } + + #[getter] + fn get_redacted_by(&self) -> PyResult> { + Ok(self.read_inner()?.redacted_by.clone()) + } + #[setter] + fn set_redacted_by(&self, val: Option) -> PyResult<()> { + self.write_inner()?.redacted_by = val; + Ok(()) + } + + #[getter] + fn get_outlier(&self) -> PyResult { + Ok(self.read_inner()?.outlier) + } + #[setter] + fn set_outlier(&self, val: bool) -> PyResult<()> { + self.write_inner()?.outlier = val; + Ok(()) + } #[getter] fn get_out_of_band_membership(&self) -> PyResult { - let bool = get_property!(self, OutOfBandMembership)?; - Ok(*bool) + attr_err( + self.read_inner()?.get_out_of_band_membership(), + "out_of_band_membership", + ) } #[setter] - fn set_out_of_band_membership(&mut self, obj: bool) { - set_property!(self, OutOfBandMembership, obj); + fn set_out_of_band_membership(&self, obj: bool) -> PyResult<()> { + self.write_inner()?.set_out_of_band_membership(obj); + Ok(()) } #[getter(send_on_behalf_of)] - fn getter_send_on_behalf_of(&self) -> PyResult<&str> { - let s = get_property!(self, SendOnBehalfOf)?; - Ok(s) + fn getter_send_on_behalf_of(&self) -> PyResult { + let guard = self.read_inner()?; + attr_err( + guard.get_send_on_behalf_of().map(|s| s.to_owned()), + "send_on_behalf_of", + ) } #[setter] - fn set_send_on_behalf_of(&mut self, obj: String) { - set_property!(self, SendOnBehalfOf, obj.into_boxed_str()); + fn set_send_on_behalf_of(&self, obj: String) -> PyResult<()> { + self.write_inner()?.set_send_on_behalf_of(obj); + Ok(()) } #[getter] fn get_recheck_redaction(&self) -> PyResult { - let bool = get_property!(self, RecheckRedaction)?; - Ok(*bool) + attr_err( + self.read_inner()?.get_recheck_redaction(), + "recheck_redaction", + ) } #[setter] - fn set_recheck_redaction(&mut self, obj: bool) { - set_property!(self, RecheckRedaction, obj); + fn set_recheck_redaction(&self, obj: bool) -> PyResult<()> { + self.write_inner()?.set_recheck_redaction(obj); + Ok(()) } #[getter] fn get_soft_failed(&self) -> PyResult { - let bool = get_property!(self, SoftFailed)?; - Ok(*bool) + attr_err(self.read_inner()?.get_soft_failed(), "soft_failed") } #[setter] - fn set_soft_failed(&mut self, obj: bool) { - set_property!(self, SoftFailed, obj); + fn set_soft_failed(&self, obj: bool) -> PyResult<()> { + self.write_inner()?.set_soft_failed(obj); + Ok(()) } #[getter] fn get_proactively_send(&self) -> PyResult { - let bool = get_property!(self, ProactivelySend)?; - Ok(*bool) + attr_err( + self.read_inner()?.get_proactively_send(), + "proactively_send", + ) } #[setter] - fn set_proactively_send(&mut self, obj: bool) { - set_property!(self, ProactivelySend, obj); + fn set_proactively_send(&self, obj: bool) -> PyResult<()> { + self.write_inner()?.set_proactively_send(obj); + Ok(()) } #[getter] fn get_policy_server_spammy(&self) -> PyResult { - Ok(get_property_opt!(self, PolicyServerSpammy) - .copied() - .unwrap_or(false)) + Ok(self.read_inner()?.get_policy_server_spammy()) } #[setter] - fn set_policy_server_spammy(&mut self, obj: bool) { - set_property!(self, PolicyServerSpammy, obj); + fn set_policy_server_spammy(&self, obj: bool) -> PyResult<()> { + self.write_inner()?.set_policy_server_spammy(obj); + Ok(()) + } + + #[getter] + fn get_spam_checker_spammy(&self) -> PyResult { + Ok(self.read_inner()?.get_spam_checker_spammy()) + } + #[setter] + fn set_spam_checker_spammy(&self, obj: bool) -> PyResult<()> { + self.write_inner()?.set_spam_checker_spammy(obj); + Ok(()) } #[getter] fn get_redacted(&self) -> PyResult { - let bool = get_property!(self, Redacted)?; - Ok(*bool) + attr_err(self.read_inner()?.get_redacted(), "redacted") } #[setter] - fn set_redacted(&mut self, obj: bool) { - set_property!(self, Redacted, obj); + fn set_redacted(&self, obj: bool) -> PyResult<()> { + self.write_inner()?.set_redacted(obj); + Ok(()) } /// The transaction ID, if it was set when the event was created. #[getter] - fn get_txn_id(&self) -> PyResult<&str> { - let s = get_property!(self, TxnId)?; - Ok(s) + fn get_txn_id(&self) -> PyResult { + let guard = self.read_inner()?; + attr_err(guard.get_txn_id().map(|s| s.to_owned()), "txn_id") } #[setter] - fn set_txn_id(&mut self, obj: String) { - set_property!(self, TxnId, obj.into_boxed_str()); + fn set_txn_id(&self, obj: String) -> PyResult<()> { + self.write_inner()?.set_txn_id(obj); + Ok(()) + } + + /// The calculated auth event IDs, if it was set when the event was created. + #[getter] + fn get_calculated_auth_event_ids(&self) -> PyResult> { + let guard = self.read_inner()?; + attr_err( + guard.get_calculated_auth_event_ids().cloned(), + "calculated_auth_event_ids", + ) + } + #[setter] + fn set_calculated_auth_event_ids(&self, obj: Vec) -> PyResult<()> { + self.write_inner()?.set_calculated_auth_event_ids(obj); + Ok(()) } /// The delay ID, set only if the event was a delayed event. #[getter] - fn get_delay_id(&self) -> PyResult<&str> { - let s = get_property!(self, DelayId)?; - Ok(s) + fn get_delay_id(&self) -> PyResult { + let guard = self.read_inner()?; + attr_err(guard.get_delay_id().map(|s| s.to_owned()), "delay_id") } #[setter] - fn set_delay_id(&mut self, obj: String) { - set_property!(self, DelayId, obj.into_boxed_str()); + fn set_delay_id(&self, obj: String) -> PyResult<()> { + self.write_inner()?.set_delay_id(obj); + Ok(()) } /// The access token ID of the user who sent this event, if any. #[getter] fn get_token_id(&self) -> PyResult { - let r = get_property!(self, TokenId)?; - Ok(*r) + attr_err(self.read_inner()?.get_token_id(), "token_id") } #[setter] - fn set_token_id(&mut self, obj: i64) { - set_property!(self, TokenId, obj); + fn set_token_id(&self, obj: i64) -> PyResult<()> { + self.write_inner()?.set_token_id(obj); + Ok(()) } /// The device ID of the user who sent this event, if any. #[getter] - fn get_device_id(&self) -> PyResult<&str> { - let s = get_property!(self, DeviceId)?; - Ok(s) + fn get_device_id(&self) -> PyResult { + let guard = self.read_inner()?; + attr_err(guard.get_device_id().map(|s| s.to_owned()), "device_id") } #[setter] - fn set_device_id(&mut self, obj: String) { - set_property!(self, DeviceId, obj.into_boxed_str()); + fn set_device_id(&self, obj: String) -> PyResult<()> { + self.write_inner()?.set_device_id(obj); + Ok(()) } } diff --git a/rust/src/lib.rs b/rust/src/lib.rs index a6e01fce64..3b049a51b7 100644 --- a/rust/src/lib.rs +++ b/rust/src/lib.rs @@ -30,6 +30,14 @@ fn get_rust_file_digest() -> &'static str { env!("SYNAPSE_RUST_DIGEST") } +/// Returns the `rustc` version used when this native module was built. +/// +/// This value is embedded at build time, so it can be exported as a prometheus metrics. +#[pyfunction] +pub fn get_rustc_version() -> &'static str { + env!("SYNAPSE_RUSTC_VERSION") +} + /// Formats the sum of two numbers as string. #[pyfunction] #[pyo3(text_signature = "(a, b, /)")] @@ -50,6 +58,7 @@ fn reset_logging_config() { fn synapse_rust(py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_function(wrap_pyfunction!(sum_as_string, m)?)?; m.add_function(wrap_pyfunction!(get_rust_file_digest, m)?)?; + m.add_function(wrap_pyfunction!(get_rustc_version, m)?)?; m.add_function(wrap_pyfunction!(reset_logging_config, m)?)?; acl::register_module(py, m)?; diff --git a/rust/src/room_versions.rs b/rust/src/room_versions.rs index fbcc32516a..dbc962174d 100644 --- a/rust/src/room_versions.rs +++ b/rust/src/room_versions.rs @@ -47,6 +47,9 @@ impl EventFormatVersions { /// MSC4291 room IDs as hashes: introduced for room HydraV11 #[classattr] const ROOM_V11_HYDRA_PLUS: i32 = 4; + /// MSC4242 state DAGs: adds prev_state_events, removes auth_events + #[classattr] + const ROOM_VMSC4242: i32 = 5; } /// Enum to identify the state resolution algorithms. @@ -146,6 +149,14 @@ pub struct RoomVersion { /// /// In these room versions, we are stricter with event size validation. pub strict_event_byte_limits_room_versions: bool, + /// MSC4242: State DAGs. Creates events with prev_state_events instead of auth_events and derives + /// state from it. Events are always processed in causal order without any gaps in the DAG + /// (prev_state_events are always known), guaranteeing that processed events have a path to the + /// create event. This is an emergent property of state DAGs as asserting that there is a path + /// to the create event every time we insert an event would be prohibitively expensive. + /// This is similar to how doubly-linked lists can potentially not refer to previous items correctly + /// without verifying the list's integrity, but doing it on every insert is too expensive. + pub msc4242_state_dags: bool, } const ROOM_VERSION_V1: RoomVersion = RoomVersion { @@ -170,6 +181,7 @@ const ROOM_VERSION_V1: RoomVersion = RoomVersion { msc4289_creator_power_enabled: false, msc4291_room_ids_as_hashes: false, strict_event_byte_limits_room_versions: false, + msc4242_state_dags: false, }; const ROOM_VERSION_V2: RoomVersion = RoomVersion { @@ -194,6 +206,7 @@ const ROOM_VERSION_V2: RoomVersion = RoomVersion { msc4289_creator_power_enabled: false, msc4291_room_ids_as_hashes: false, strict_event_byte_limits_room_versions: false, + msc4242_state_dags: false, }; const ROOM_VERSION_V3: RoomVersion = RoomVersion { @@ -218,6 +231,7 @@ const ROOM_VERSION_V3: RoomVersion = RoomVersion { msc4289_creator_power_enabled: false, msc4291_room_ids_as_hashes: false, strict_event_byte_limits_room_versions: false, + msc4242_state_dags: false, }; const ROOM_VERSION_V4: RoomVersion = RoomVersion { @@ -242,6 +256,7 @@ const ROOM_VERSION_V4: RoomVersion = RoomVersion { msc4289_creator_power_enabled: false, msc4291_room_ids_as_hashes: false, strict_event_byte_limits_room_versions: false, + msc4242_state_dags: false, }; const ROOM_VERSION_V5: RoomVersion = RoomVersion { @@ -266,6 +281,7 @@ const ROOM_VERSION_V5: RoomVersion = RoomVersion { msc4289_creator_power_enabled: false, msc4291_room_ids_as_hashes: false, strict_event_byte_limits_room_versions: false, + msc4242_state_dags: false, }; const ROOM_VERSION_V6: RoomVersion = RoomVersion { @@ -290,6 +306,7 @@ const ROOM_VERSION_V6: RoomVersion = RoomVersion { msc4289_creator_power_enabled: false, msc4291_room_ids_as_hashes: false, strict_event_byte_limits_room_versions: false, + msc4242_state_dags: false, }; const ROOM_VERSION_V7: RoomVersion = RoomVersion { @@ -314,6 +331,7 @@ const ROOM_VERSION_V7: RoomVersion = RoomVersion { msc4289_creator_power_enabled: false, msc4291_room_ids_as_hashes: false, strict_event_byte_limits_room_versions: false, + msc4242_state_dags: false, }; const ROOM_VERSION_V8: RoomVersion = RoomVersion { @@ -338,6 +356,7 @@ const ROOM_VERSION_V8: RoomVersion = RoomVersion { msc4289_creator_power_enabled: false, msc4291_room_ids_as_hashes: false, strict_event_byte_limits_room_versions: false, + msc4242_state_dags: false, }; const ROOM_VERSION_V9: RoomVersion = RoomVersion { @@ -362,6 +381,7 @@ const ROOM_VERSION_V9: RoomVersion = RoomVersion { msc4289_creator_power_enabled: false, msc4291_room_ids_as_hashes: false, strict_event_byte_limits_room_versions: false, + msc4242_state_dags: false, }; const ROOM_VERSION_V10: RoomVersion = RoomVersion { @@ -386,6 +406,7 @@ const ROOM_VERSION_V10: RoomVersion = RoomVersion { msc4289_creator_power_enabled: false, msc4291_room_ids_as_hashes: false, strict_event_byte_limits_room_versions: false, + msc4242_state_dags: false, }; /// MSC3389 (Redaction changes for events with a relation) based on room version "10". @@ -411,6 +432,7 @@ const ROOM_VERSION_MSC3389V10: RoomVersion = RoomVersion { msc4289_creator_power_enabled: false, msc4291_room_ids_as_hashes: false, strict_event_byte_limits_room_versions: true, + msc4242_state_dags: false, }; /// MSC1767 (Extensible Events) based on room version "10". @@ -436,6 +458,7 @@ const ROOM_VERSION_MSC1767V10: RoomVersion = RoomVersion { msc4289_creator_power_enabled: false, msc4291_room_ids_as_hashes: false, strict_event_byte_limits_room_versions: false, + msc4242_state_dags: false, }; /// MSC3757 (Restricting who can overwrite a state event) based on room version "10". @@ -461,6 +484,7 @@ const ROOM_VERSION_MSC3757V10: RoomVersion = RoomVersion { msc4289_creator_power_enabled: false, msc4291_room_ids_as_hashes: false, strict_event_byte_limits_room_versions: false, + msc4242_state_dags: false, }; const ROOM_VERSION_V11: RoomVersion = RoomVersion { @@ -485,6 +509,7 @@ const ROOM_VERSION_V11: RoomVersion = RoomVersion { msc4289_creator_power_enabled: false, msc4291_room_ids_as_hashes: false, strict_event_byte_limits_room_versions: true, // Changed from v10 + msc4242_state_dags: false, }; /// MSC3757 (Restricting who can overwrite a state event) based on room version "11". @@ -510,6 +535,7 @@ const ROOM_VERSION_MSC3757V11: RoomVersion = RoomVersion { msc4289_creator_power_enabled: false, msc4291_room_ids_as_hashes: false, strict_event_byte_limits_room_versions: true, + msc4242_state_dags: false, }; const ROOM_VERSION_HYDRA_V11: RoomVersion = RoomVersion { @@ -534,6 +560,7 @@ const ROOM_VERSION_HYDRA_V11: RoomVersion = RoomVersion { msc4289_creator_power_enabled: true, // Changed from v11 msc4291_room_ids_as_hashes: true, // Changed from v11 strict_event_byte_limits_room_versions: true, + msc4242_state_dags: false, }; const ROOM_VERSION_V12: RoomVersion = RoomVersion { @@ -558,6 +585,32 @@ const ROOM_VERSION_V12: RoomVersion = RoomVersion { msc4289_creator_power_enabled: true, // Changed from v11 msc4291_room_ids_as_hashes: true, // Changed from v11 strict_event_byte_limits_room_versions: true, + msc4242_state_dags: false, +}; + +const ROOM_VERSION_MSC4242V12: RoomVersion = RoomVersion { + identifier: "org.matrix.msc4242.12", + disposition: RoomDisposition::UNSTABLE, + event_format: EventFormatVersions::ROOM_VMSC4242, + state_res: StateResolutionVersions::V2_1, + enforce_key_validity: true, + special_case_aliases_auth: false, + strict_canonicaljson: true, + limit_notifications_power_levels: true, + implicit_room_creator: true, + updated_redaction_rules: true, + restricted_join_rule: true, + restricted_join_rule_fix: true, + knock_join_rule: true, + msc3389_relation_redactions: false, + knock_restricted_join_rule: true, + enforce_int_power_levels: true, + msc3931_push_features: &[], + msc3757_enabled: false, + msc4289_creator_power_enabled: true, + msc4291_room_ids_as_hashes: true, + strict_event_byte_limits_room_versions: true, + msc4242_state_dags: true, }; /// Helper class for managing the known room versions, and providing dict-like @@ -800,6 +853,10 @@ impl RoomVersions { fn V12(py: Python<'_>) -> PyResult> { ROOM_VERSION_V12.into_py_any(py) } + #[classattr] + fn MSC4242v12(py: Python<'_>) -> PyResult> { + ROOM_VERSION_MSC4242V12.into_py_any(py) + } } /// Called when registering modules with python. @@ -814,11 +871,12 @@ pub fn register_module(py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> child_module.add_class::()?; // Build KNOWN_EVENT_FORMAT_VERSIONS as a frozenset - let known_ef: [i32; 4] = [ + let known_ef: [i32; 5] = [ EventFormatVersions::ROOM_V1_V2, EventFormatVersions::ROOM_V3, EventFormatVersions::ROOM_V4_PLUS, EventFormatVersions::ROOM_V11_HYDRA_PLUS, + EventFormatVersions::ROOM_VMSC4242, ]; let known_event_format_versions = PyFrozenSet::new(py, known_ef)?; child_module.add("KNOWN_EVENT_FORMAT_VERSIONS", known_event_format_versions)?; diff --git a/schema/synapse-config.schema.yaml b/schema/synapse-config.schema.yaml index be765ac5ed..9385feff29 100644 --- a/schema/synapse-config.schema.yaml +++ b/schema/synapse-config.schema.yaml @@ -1,5 +1,5 @@ $schema: https://famedly.github.io/synapse/latest/schema/v1/meta.schema.json -$id: https://famedly.github.io/synapse/schema/synapse/v1.151/synapse-config.schema.json +$id: https://famedly.github.io/synapse/schema/synapse/v1.152/synapse-config.schema.json type: object properties: famedly_maximum_refresh_token_lifetime: diff --git a/synapse/_scripts/synapse_port_db.py b/synapse/_scripts/synapse_port_db.py index eedceb170e..0b8a289d92 100755 --- a/synapse/_scripts/synapse_port_db.py +++ b/synapse/_scripts/synapse_port_db.py @@ -136,6 +136,7 @@ "users": ["shadow_banned", "approved", "locked", "suspended"], "un_partial_stated_event_stream": ["rejection_status_changed"], "users_who_share_rooms": ["share_private"], + "quarantined_media_changes": ["quarantined"], } @@ -912,6 +913,10 @@ def alter_table(txn: LoggingTransaction) -> None: await self._setup_autoincrement_sequence( "state_groups_pending_deletion", "sequence_number" ) + await self._setup_sequence( + "quarantined_media_id_seq", + [("quarantined_media_changes", "stream_id")], + ) # Step 3. Get tables. self.progress.set_state("Fetching tables") diff --git a/synapse/app/admin_cmd.py b/synapse/app/admin_cmd.py index 0614c805da..67189e91e7 100644 --- a/synapse/app/admin_cmd.py +++ b/synapse/app/admin_cmd.py @@ -34,6 +34,7 @@ from synapse.config.homeserver import HomeServerConfig from synapse.config.logger import setup_logging from synapse.events import EventBase +from synapse.events.utils import FilteredEvent from synapse.handlers.admin import ExfiltrationWriter from synapse.server import HomeServer from synapse.storage.database import DatabasePool, LoggingDatabaseConnection @@ -150,14 +151,14 @@ def __init__(self, user_id: str, directory: str | None = None): if list(os.listdir(self.base_directory)): raise Exception("Directory must be empty") - def write_events(self, room_id: str, events: list[EventBase]) -> None: + def write_events(self, room_id: str, filtered_events: list[FilteredEvent]) -> None: room_directory = os.path.join(self.base_directory, "rooms", room_id) os.makedirs(room_directory, exist_ok=True) events_file = os.path.join(room_directory, "events") with open(events_file, "a") as f: - for event in events: - json.dump(event.get_pdu_json(), fp=f) + for filtered_event in filtered_events: + json.dump(filtered_event.event.get_pdu_json(), fp=f) def write_state( self, room_id: str, event_id: str, state: StateMap[EventBase] @@ -175,7 +176,7 @@ def write_state( def write_invite( self, room_id: str, event: EventBase, state: StateMap[EventBase] ) -> None: - self.write_events(room_id, [event]) + self.write_events(room_id, [FilteredEvent.state(event)]) # We write the invite state somewhere else as they aren't full events # and are only a subset of the state at the event. @@ -191,7 +192,7 @@ def write_invite( def write_knock( self, room_id: str, event: EventBase, state: StateMap[EventBase] ) -> None: - self.write_events(room_id, [event]) + self.write_events(room_id, [FilteredEvent.state(event)]) # We write the knock state somewhere else as they aren't full events # and are only a subset of the state at the event. diff --git a/synapse/appservice/api.py b/synapse/appservice/api.py index d4e9d50b96..66c962e17d 100644 --- a/synapse/appservice/api.py +++ b/synapse/appservice/api.py @@ -40,7 +40,7 @@ TransactionUnusedFallbackKeys, ) from synapse.events import EventBase -from synapse.events.utils import SerializeEventConfig +from synapse.events.utils import FilteredEvent, SerializeEventConfig from synapse.http.client import SimpleHttpClient, is_unknown_endpoint from synapse.logging import opentracing from synapse.metrics import SERVER_NAME_LABEL @@ -545,7 +545,7 @@ async def _serialize( ) -> list[JsonDict]: time_now = self.clock.time_msec() return await self._event_serializer.serialize_events( - list(events), + [FilteredEvent(event=e, membership=None) for e in events], time_now, config=SerializeEventConfig( as_client_event=True, diff --git a/synapse/config/experimental.py b/synapse/config/experimental.py index 91bef558ed..630a08f868 100644 --- a/synapse/config/experimental.py +++ b/synapse/config/experimental.py @@ -482,6 +482,12 @@ def read_config( # Enable room version (and thus applicable push rules from MSC3931/3932) KNOWN_ROOM_VERSIONS.add_room_version(RoomVersions.MSC1767v10) + # MSC4242: State DAGs + self.msc4242_enabled: bool = experimental.get("msc4242_enabled", False) + if self.msc4242_enabled: + # Enable the room version + KNOWN_ROOM_VERSIONS.add_room_version(RoomVersions.MSC4242v12) + # MSC3391: Removing account data. self.msc3391_enabled = experimental.get("msc3391_enabled", False) @@ -607,3 +613,9 @@ def read_config( # Note that sticky events persisted before this feature is enabled will not be # considered sticky by the local homeserver. self.msc4354_enabled: bool = experimental.get("msc4354_enabled", False) + + # MSC4450: Identity Provider selection for User-Interactive Authentication + # with Legacy Single Sign-On (`m.login.sso`) + # Tracked in: https://github.com/element-hq/synapse/issues/19691 + # Note that this is only applicable to legacy auth, not MAS integration (OAuth 2.0). + self.msc4450_enabled: bool = experimental.get("msc4450_enabled", False) diff --git a/synapse/config/workers.py b/synapse/config/workers.py index da321d8a7a..b4c4b8835f 100644 --- a/synapse/config/workers.py +++ b/synapse/config/workers.py @@ -142,6 +142,8 @@ class WriterLocations: push_rules: The instances that write to the push stream. Currently can only be a single instance. device_lists: The instances that write to the device list stream. + quarantined_media_changes: The instances that write to the quarantined media + changes stream. """ events: list[str] = attr.ib( @@ -180,6 +182,10 @@ class WriterLocations: default=["master"], converter=_instance_to_list_converter, ) + quarantined_media_changes: list[str] = attr.ib( + default=[MAIN_PROCESS_INSTANCE_NAME], + converter=_instance_to_list_converter, + ) @attr.s(auto_attribs=True) diff --git a/synapse/crypto/event_signing.py b/synapse/crypto/event_signing.py index d13d5d04c3..d789c06a9c 100644 --- a/synapse/crypto/event_signing.py +++ b/synapse/crypto/event_signing.py @@ -27,7 +27,7 @@ from canonicaljson import encode_canonical_json from signedjson.sign import sign_json -from signedjson.types import SigningKey +from signedjson.types import SigningKey, VerifyKey from unpaddedbase64 import decode_base64, encode_base64 from synapse.api.errors import Codes, SynapseError @@ -35,7 +35,7 @@ from synapse.events import EventBase from synapse.events.utils import prune_event, prune_event_dict from synapse.logging.opentracing import trace -from synapse.types import JsonDict +from synapse.types import JsonDict, UserID logger = logging.getLogger(__name__) @@ -192,3 +192,54 @@ def add_hashes_and_signatures( event_dict["signatures"] = compute_event_signature( room_version, event_dict, signature_name=signature_name, signing_key=signing_key ) + + +def resign_event( + ev: EventBase, + server_name: str, + signing_key: SigningKey, + time_now: int | None = None, +) -> JsonDict: + """Re-sign the provided event with the given signing key. Any existing signatures on the event + for this server_name are removed. + + If there has been no signature for this event by this server_name, the event is still re-signed. + If there have been signatures on this event by this server_name, the event is not re-checked for + validity. As such, only events that have valid signatures should be passed into this function + e.g. from the event_json table in the database. + """ + event_dict = ev.get_pdu_json(time_now=time_now) + event_dict["signatures"].pop( + server_name, None + ) # remove existing signatures for this server_name + event_dict["signatures"].update( + compute_event_signature( + ev.room_version, + event_dict, + server_name, + signing_key, + ) + ) + return event_dict + + +def event_needs_resigning( + ev: EventBase, server_name: str, verify_key: VerifyKey +) -> bool: + """Check if this event needs re-signing. + + This returns True if all of the following are True: + - the event `sender` domain matches the `server_name` provided. + - the event has not been already signed with this `verify_key`. + """ + sender = UserID.from_string(ev.sender) + if sender.domain != server_name: + return False + want_key_id = verify_key.alg + ":" + verify_key.version + signed_with_current_key_id = ev.signatures.get(server_name, {}).get( + want_key_id, None + ) + if signed_with_current_key_id: + return False + + return True diff --git a/synapse/event_auth.py b/synapse/event_auth.py index f39f55b472..ca528ae235 100644 --- a/synapse/event_auth.py +++ b/synapse/event_auth.py @@ -61,7 +61,7 @@ EventFormatVersions, RoomVersion, ) -from synapse.events import is_creator +from synapse.events import FrozenEventVMSC4242, is_creator from synapse.state import CREATE_KEY from synapse.storage.databases.main.events_worker import EventRedactBehaviour from synapse.types import ( @@ -186,6 +186,70 @@ async def check_state_independent_auth_rules( # 1.5 Otherwise, allow return + # State DAGs 2. Considering the event's prev_state_events: + if event.room_version.msc4242_state_dags: + prev_state_events_ids = set(cast(FrozenEventVMSC4242, event).prev_state_events) + # Fetch all of the `prev_state_events` + prev_state_events = {} + # Try to load the `prev_state_events` from `batched_auth_events` initially as + # that can save us a database hit. + if batched_auth_events is not None: + prev_state_events = { + event_id: value + for event_id in prev_state_events_ids + if (value := batched_auth_events.get(event_id)) is not None + } + # Fetch the rest of the `prev_state_events` + missing_prev_state_events_ids = prev_state_events_ids - set( + prev_state_events.keys() + ) + fetched_prev_state_events = await store.get_events( + missing_prev_state_events_ids, + redact_behaviour=EventRedactBehaviour.as_is, + allow_rejected=True, + ) + prev_state_events.update(fetched_prev_state_events) + if len(prev_state_events) != len(prev_state_events_ids): + # we should have all the `prev_state_events` by now, so if we do not, that suggests + # a Synapse programming error + known_prev_state_event_ids = set(prev_state_events) + raise AssertionError( + f"Event {event.event_id} has unknown prev_state_events " + + f"({len(prev_state_events)}/{len(prev_state_events_ids)} known)" + + f"{prev_state_events_ids - known_prev_state_event_ids} missing " + + f"out of {prev_state_events_ids}" + ) + for prev_state_event in prev_state_events.values(): + # 2.1 If there are entries which do not belong in the same room, reject. + if prev_state_event.room_id != event.room_id: + raise AuthError( + 403, + "During auth for event %s in room %s, found event %s in prev_state_events " + "which belongs to a different room %s" + % ( + event.event_id, + event.room_id, + prev_state_event.event_id, + prev_state_event.room_id, + ), + ) + # 2.2 If there are entries which do not have a state_key, reject. + if not prev_state_event.is_state(): + raise AuthError( + 403, + f"During auth for event {event.event_id} in room {event.room_id}, event has a " + + f"prev_state_event which is not state: {prev_state_event.event_id}", + ) + # 2.3 If there are entries which were themselves rejected under the checks performed on + # receipt of a PDU, reject. + if prev_state_event.rejected_reason is not None: + raise AuthError( + 403, + f"During auth for event {event.event_id} in room {event.room_id}, event has a " + + f"prev_state_event which is rejected ({prev_state_event.rejected_reason}): " + + f"{prev_state_event.event_id}", + ) + # 2. Reject if event has auth_events that: ... auth_events: ChainMap[str, EventBase] = ChainMap() if batched_auth_events: @@ -358,7 +422,7 @@ def check_state_dependent_auth_rules( # a user is allowed to issue invites. Fixes # https://github.com/vector-im/vector-web/issues/1208 hopefully if event.type == EventTypes.ThirdPartyInvite: - user_level = get_user_power_level(event.user_id, auth_dict) + user_level = get_user_power_level(event.sender, auth_dict) invite_level = get_named_level(auth_dict, "invite", 0) if user_level < invite_level: @@ -409,8 +473,8 @@ def _check_size_limits(event: "EventBase") -> None: # Codepoint size check: Synapse always enforced these limits, so apply # them strictly. - if len(event.user_id) > 255: - raise EventSizeError("'user_id' too large", unpersistable=True) + if len(event.sender) > 255: + raise EventSizeError("'sender' too large", unpersistable=True) if len(event.room_id) > 255: raise EventSizeError("'room_id' too large", unpersistable=True) if event.is_state() and len(event.state_key) > 255: @@ -423,8 +487,8 @@ def _check_size_limits(event: "EventBase") -> None: strict_byte_limits = event.room_version.strict_event_byte_limits_room_versions # Byte size check: if these fail, then be lenient to avoid breaking rooms. - if len(event.user_id.encode("utf-8")) > 255: - raise EventSizeError("'user_id' too large", unpersistable=strict_byte_limits) + if len(event.sender.encode("utf-8")) > 255: + raise EventSizeError("'sender' too large", unpersistable=strict_byte_limits) if len(event.room_id.encode("utf-8")) > 255: raise EventSizeError("'room_id' too large", unpersistable=strict_byte_limits) if event.is_state() and len(event.state_key.encode("utf-8")) > 255: @@ -450,6 +514,12 @@ def _check_create(event: "EventBase") -> None: if event.prev_event_ids(): raise AuthError(403, "Create event has prev events") + # State DAGs 1.2 If it has any prev_state_events, reject. + if event.room_version.msc4242_state_dags: + assert isinstance(event, FrozenEventVMSC4242) + if len(event.prev_state_events) > 0: + raise AuthError(403, "Create event has prev state events") + if event.room_version.msc4291_room_ids_as_hashes: # 1.2 If the create event has a room_id, reject if "room_id" in event: @@ -544,7 +614,7 @@ def _is_membership_change_allowed( raise AuthError(403, "This room has been marked as unfederatable.") # get info about the caller - key = (EventTypes.Member, event.user_id) + key = (EventTypes.Member, event.sender) caller = auth_events.get(key) caller_in_room = caller and caller.membership == Membership.JOIN @@ -569,7 +639,7 @@ def _is_membership_change_allowed( else: join_rule = JoinRules.INVITE - user_level = get_user_power_level(event.user_id, auth_events) + user_level = get_user_power_level(event.sender, auth_events) target_level = get_user_power_level(target_user_id, auth_events) invite_level = get_named_level(auth_events, "invite", 0) @@ -587,7 +657,7 @@ def _is_membership_change_allowed( "membership": membership, "join_rule": join_rule, "target_user_id": target_user_id, - "event.user_id": event.user_id, + "event.sender": event.sender, }, ) @@ -607,14 +677,14 @@ def _is_membership_change_allowed( if ( (caller_invited or caller_knocked) and Membership.LEAVE == membership - and target_user_id == event.user_id + and target_user_id == event.sender ): return if not caller_in_room: # caller isn't joined raise UnstableSpecAuthError( 403, - "%s not in room %s." % (event.user_id, event.room_id), + "%s not in room %s." % (event.sender, event.room_id), errcode=Codes.NOT_JOINED, ) @@ -645,7 +715,7 @@ def _is_membership_change_allowed( # * They are already joined (it's a NOOP). # * The room is public. # * The room is restricted and the user meets the allows rules. - if event.user_id != target_user_id: + if event.sender != target_user_id: raise AuthError(403, "Cannot force another user to join.") elif target_banned: raise AuthError(403, "You are banned from this room") @@ -705,7 +775,7 @@ def _is_membership_change_allowed( "You cannot unban user %s." % (target_user_id,), errcode=Codes.INSUFFICIENT_POWER, ) - elif target_user_id != event.user_id: + elif target_user_id != event.sender: kick_level = get_named_level(auth_events, "kick", 50) if user_level < kick_level or user_level <= target_level: @@ -733,7 +803,7 @@ def _is_membership_change_allowed( or join_rule != JoinRules.KNOCK_RESTRICTED ): raise AuthError(403, "You don't have permission to knock") - elif target_user_id != event.user_id: + elif target_user_id != event.sender: raise AuthError(403, "You cannot knock for other users") elif target_in_room: raise UnstableSpecAuthError( @@ -752,10 +822,10 @@ def _is_membership_change_allowed( def _check_event_sender_in_room( event: "EventBase", auth_events: StateMap["EventBase"] ) -> None: - key = (EventTypes.Member, event.user_id) + key = (EventTypes.Member, event.sender) member_event = auth_events.get(key) - _check_joined_room(member_event, event.user_id, event.room_id) + _check_joined_room(member_event, event.sender, event.room_id) def _check_joined_room( @@ -809,7 +879,7 @@ def _can_send_event(event: "EventBase", auth_events: StateMap["EventBase"]) -> b power_levels_event = get_power_level_event(auth_events) send_level = get_send_level(event.type, state_key, power_levels_event) - user_level = get_user_power_level(event.user_id, auth_events) + user_level = get_user_power_level(event.sender, auth_events) if user_level < send_level: raise UnstableSpecAuthError( @@ -822,7 +892,7 @@ def _can_send_event(event: "EventBase", auth_events: StateMap["EventBase"]) -> b if ( state_key is not None and state_key.startswith("@") - and state_key != event.user_id + and state_key != event.sender ): if event.room_version.msc3757_enabled: try: @@ -841,7 +911,7 @@ def _can_send_event(event: "EventBase", auth_events: StateMap["EventBase"]) -> b ) if ( # sender is owner of the state key - state_key_user_id == event.user_id + state_key_user_id == event.sender # sender has higher PL than the owner of the state key or user_level > get_user_power_level(state_key_user_id, auth_events) ): @@ -868,7 +938,7 @@ def check_redaction( AuthError if the event sender is definitely not allowed to redact the target event. """ - user_level = get_user_power_level(event.user_id, auth_events) + user_level = get_user_power_level(event.sender, auth_events) redact_level = get_named_level(auth_events, "redact", 50) @@ -966,7 +1036,7 @@ def _check_power_levels( if not current_state: return - user_level = get_user_power_level(event.user_id, auth_events) + user_level = get_user_power_level(event.sender, auth_events) # Check other levels: levels_to_check: list[tuple[str, str | None]] = [ @@ -1020,7 +1090,7 @@ def _check_power_levels( if new_level == old_level: continue - if dir == "users" and level_to_check != event.user_id: + if dir == "users" and level_to_check != event.sender: if old_level == user_level: raise AuthError( 403, @@ -1137,7 +1207,7 @@ def _verify_third_party_invite( if invite_event.sender != event.sender: return False - if event.user_id != invite_event.user_id: + if event.sender != invite_event.sender: return False if signed["mxid"] != event.state_key: diff --git a/synapse/events/__init__.py b/synapse/events/__init__.py index 35b0506f66..51e4ca0c45 100644 --- a/synapse/events/__init__.py +++ b/synapse/events/__init__.py @@ -34,6 +34,7 @@ ) import attr +from typing_extensions import deprecated from unpaddedbase64 import encode_base64 from synapse.api.constants import ( @@ -44,10 +45,7 @@ ) from synapse.api.room_versions import EventFormatVersions, RoomVersion, RoomVersions from synapse.synapse_rust.events import EventInternalMetadata -from synapse.types import ( - JsonDict, - StrCollection, -) +from synapse.types import JsonDict, StateKey, StrCollection from synapse.util.caches import intern_dict from synapse.util.duration import Duration from synapse.util.frozenutils import freeze, unfreeze @@ -221,6 +219,8 @@ def __init__( # get_state_key() (and a check for None). state_key: DictProperty[str] = DictProperty("state_key") type: DictProperty[str] = DictProperty("type") + + # This is a deprecated property, use `sender` instead. Only used by modules. user_id: DictProperty[str] = DictProperty("sender") @property @@ -288,9 +288,6 @@ def get_templated_pdu_json(self) -> JsonDict: return template_json - def __getitem__(self, field: str) -> Any | None: - return self._dict[field] - def __contains__(self, field: str) -> bool: return field in self._dict @@ -373,6 +370,11 @@ def __repr__(self) -> str: ">" ) + # Using `__getitem__` is deprecated. Only used by modules. + @deprecated("Use attribute access instead") + def __getitem__(self, field: str) -> Any | None: + return self._dict[field] + class FrozenEvent(EventBase): format_version = EventFormatVersions.ROOM_V1_V2 # All events of this type are V1 @@ -585,9 +587,60 @@ def auth_event_ids(self) -> StrCollection: return [*self._dict["auth_events"], create_event_id] +class FrozenEventVMSC4242(FrozenEventV4): + """FrozenEventVMSC4242, which differs from FrozenEventV4 only in the addition of prev_state_events""" + + format_version = EventFormatVersions.ROOM_VMSC4242 + prev_state_events: DictProperty[list[str]] = DictProperty("prev_state_events") + + def __init__( + self, + event_dict: JsonDict, + room_version: RoomVersion, + internal_metadata_dict: JsonDict | None = None, + rejected_reason: str | None = None, + ): + # Similar to how we assert event_id isn't in V2+ events, we do the same with auth_events. + # We don't expect `auth_events` in the wire format because we calculate it from prev_state_events. + assert "auth_events" not in event_dict + super().__init__( + event_dict=event_dict, + room_version=room_version, + internal_metadata_dict=internal_metadata_dict, + rejected_reason=rejected_reason, + ) + + def auth_event_ids(self) -> StrCollection: + """Returns the list of _calculated_ auth event IDs. + + Returns: + The list of event IDs of this event's auth events + """ + # Catches cases where we accidentally call auth_event_ids() prior to calculating what they + # actually are. The exception being the m.room.create event which has no auth events. + if self.type != EventTypes.Create: + assert len(self.internal_metadata.calculated_auth_event_ids) > 0 + return self.internal_metadata.calculated_auth_event_ids + + def __repr__(self) -> str: + rejection = f"REJECTED={self.rejected_reason}, " if self.rejected_reason else "" + + return ( + f"<{self.__class__.__name__} " + f"{rejection}" + f"event_id={self.event_id}, " + f"type={self.get('type')}, " + f"state_key={self.get('state_key')}, " + f"prev_events={self.get('prev_events')}, " + f"prev_state_events={self.get('prev_state_events')}, " + f"outlier={self.internal_metadata.is_outlier()}" + ">" + ) + + def _event_type_from_format_version( format_version: int, -) -> type[FrozenEvent | FrozenEventV2 | FrozenEventV3]: +) -> type[FrozenEvent | FrozenEventV2 | FrozenEventV3 | FrozenEventVMSC4242]: """Returns the python type to use to construct an Event object for the given event format version. @@ -604,6 +657,8 @@ def _event_type_from_format_version( return FrozenEventV2 elif format_version == EventFormatVersions.ROOM_V4_PLUS: return FrozenEventV3 + elif format_version == EventFormatVersions.ROOM_VMSC4242: + return FrozenEventVMSC4242 elif format_version == EventFormatVersions.ROOM_V11_HYDRA_PLUS: return FrozenEventV4 else: @@ -665,6 +720,24 @@ def relation_from_event(event: EventBase) -> _EventRelation | None: return _EventRelation(parent_id, rel_type, aggregation_key) +def event_exists_in_state_dag( + event: Union["EventBase", "EventBuilder", "EventMetadata", "StateKey"], +) -> bool: + """Given an event, returns true if this event should form part of the state DAG. + Only valid for room versions which use a state DAG (MSC4242).""" + state_key = None + if isinstance(event, EventMetadata): + state_key = event.state_key + elif isinstance(event, tuple): # StateKey + # can't use StateKey else you get: + # "Subscripted generics cannot be used with class and instance checks" + state_key = event[1] + else: + state_key = event.state_key if event.is_state() else None + + return state_key is not None + + def is_creator(create: EventBase, user_id: str) -> bool: """ Return true if the provided user ID is the room creator. @@ -699,3 +772,13 @@ class StrippedStateEvent: state_key: str sender: str content: dict[str, Any] + + +@attr.s(slots=True, frozen=True, auto_attribs=True) +class EventMetadata: + """Returned by `get_metadata_for_events`""" + + room_id: str + event_type: str + state_key: str | None + rejection_reason: str | None diff --git a/synapse/events/builder.py b/synapse/events/builder.py index 2cd1bf6106..78eb98e1e5 100644 --- a/synapse/events/builder.py +++ b/synapse/events/builder.py @@ -132,6 +132,7 @@ async def build( prev_event_ids: list[str], auth_event_ids: list[str] | None, depth: int | None = None, + prev_state_events: list[str] | None = None, ) -> EventBase: """Transform into a fully signed and hashed event @@ -143,10 +144,51 @@ async def build( depth: Override the depth used to order the event in the DAG. Should normally be set to None, which will cause the depth to be calculated based on the prev_events. - + prev_state_events: The event IDs to use as prev_state_events. + Only applicable on MSC4242 state DAG rooms. If this is supplied, auth_event_ids + must not be specified unless this event is part of a batch such that the builder + will be unable to compute the auth_event_ids due to the events not being persisted + yet. Returns: The signed and hashed event. """ + # If the caller specifies this, make sure the room version supports it. + if prev_state_events: + assert self.room_version.msc4242_state_dags + if self.room_version.msc4242_state_dags: + assert prev_state_events is not None + if self.room_id: + state_ids = await self._state.compute_state_after_events( + self.room_id, + prev_state_events, + state_filter=StateFilter.from_types( + auth_types_for_event(self.room_version, self) + ), + await_full_state=False, + ) + # When we create rooms we only insert the create+member events, and batch the rest. + # Therefore, we may not have state_ids from compute_state_after_events as the + # prev_state_events are unknown. If this happens, the caller provides the auth events + # to use instead. + calculated_auth_event_ids: list[ + str + ] = [] # assume it's the create event which has [] + if len(state_ids) == 0 and len(prev_state_events) > 0: + # it's a batched event, so we should have been provided the auth_events + assert auth_event_ids and len(auth_event_ids) > 0 + calculated_auth_event_ids = auth_event_ids + else: + calculated_auth_event_ids = ( + self._event_auth_handler.compute_auth_events(self, state_ids) + ) + else: + # event is a state DAG event and is the create event (room_id is not provided), + # therefore there are no auth_events. + calculated_auth_event_ids = [] + assert self.type == EventTypes.Create and self.state_key == "" + self.internal_metadata.calculated_auth_event_ids = calculated_auth_event_ids + auth_event_ids = calculated_auth_event_ids + # Create events always have empty auth_events. if self.type == EventTypes.Create and self.is_state() and self.state_key == "": auth_event_ids = [] @@ -155,6 +197,8 @@ async def build( if auth_event_ids is None: # Every non-create event must have a room ID assert self.room_id is not None + # this block must not be hit for MSC4242 rooms as it resolves state with prev_events + assert not self.room_version.msc4242_state_dags state_ids = await self._state.compute_state_after_events( self.room_id, prev_event_ids, @@ -231,7 +275,6 @@ async def build( # rejected by other servers (and so that they can be persisted in # the db) depth = min(depth, MAX_DEPTH) - event_dict: dict[str, Any] = { "auth_events": auth_events, "prev_events": prev_events, @@ -241,8 +284,6 @@ async def build( "unsigned": self.unsigned, "depth": depth, } - if self.room_id is not None: - event_dict["room_id"] = self.room_id if self.room_version.msc4291_room_ids_as_hashes: # In MSC4291: the create event has no room ID as the create event ID /is/ the room ID. @@ -262,6 +303,14 @@ async def build( auth_event_ids.remove(create_event_id) event_dict["auth_events"] = auth_event_ids + if self.room_version.msc4242_state_dags: + # Auth events are removed entirely on state DAG rooms + event_dict.pop("auth_events") + assert prev_state_events is not None + event_dict["prev_state_events"] = prev_state_events + if self.room_id is not None: + event_dict["room_id"] = self.room_id + if self.is_state(): event_dict["state_key"] = self._state_key diff --git a/synapse/events/utils.py b/synapse/events/utils.py index 76ebac8b17..f038fb5578 100644 --- a/synapse/events/utils.py +++ b/synapse/events/utils.py @@ -41,6 +41,7 @@ MAX_PDU_SIZE, EventContentFields, EventTypes, + EventUnsignedContentFields, RelationTypes, ) from synapse.api.errors import Codes, SynapseError @@ -155,6 +156,10 @@ def prune_event_dict(room_version: RoomVersion, event_dict: JsonDict) -> JsonDic # Earlier room versions from had additional allowed keys. if not room_version.updated_redaction_rules: allowed_keys.extend(["prev_state", "membership", "origin"]) + # Custom room versions add new allowed keys and remove others + if room_version.msc4242_state_dags: + allowed_keys.extend(["prev_state_events"]) + allowed_keys.remove("auth_events") event_type = event_dict["type"] @@ -416,6 +421,50 @@ def format_event_for_client_v2_without_room_id(d: JsonDict) -> JsonDict: return d +@attr.s(slots=True, frozen=True, auto_attribs=True) +class FilteredEvent: + """An event annotated with per-user data for client serialization. + + Produced by filter_and_transform_events_for_client. Carries the user's + membership at the time of the event so serialization can inject it into + unsigned.membership (MSC4115) without cloning the underlying event. + """ + + event: "EventBase" + """The event to be serialized.""" + + membership: str | None + """The user whose requesting the event's membership at the time of the + event was sent. + + This is None if we didn't compute the membership. In Synapse this happens a) + when returning state events to state endpoints, or b) when the event is + returned to an admin. + + According to the spec we don't have to include the membership for any events + if we don't want to, especially if its expensive to compute. In practice + clients really only care about events in the room timeline so that in + encrypted room they can determine if they should be able to decrypt the + event or not. + """ + + @classmethod + def state(cls, event: "EventBase") -> "FilteredEvent": + """Wrap a state event with no per-user membership annotation. + + The event must be a state event (i.e. have a state_key). + """ + assert event.is_state(), ( + f"FilteredEvent.state() called with non-state event {event.event_id}" + ) + return cls(event=event, membership=None) + + @classmethod + def admin_override(cls, event: "EventBase") -> "FilteredEvent": + """Wrap an event that bypasses visibility filtering due to admin privileges.""" + return cls(event=event, membership=None) + + @attr.s(slots=True, frozen=True, auto_attribs=True) class SerializeEventConfig: as_client_event: bool = True @@ -435,6 +484,9 @@ class SerializeEventConfig: # only server admins can see through other configuration. For example, # whether an event was soft failed by the server. include_admin_metadata: bool = False + # Whether MSC4354 (sticky events) is enabled. When True, the sticky TTL + # will be computed and included in the unsigned section of sticky events. + msc4354_enabled: bool = False @only_event_fields.validator def _validate_only_event_fields( @@ -461,6 +513,7 @@ def _serialize_event( time_now_ms: int, *, config: SerializeEventConfig = _DEFAULT_SERIALIZE_EVENT_CONFIG, + membership: str | None = None, ) -> JsonDict: """Serialize event for clients @@ -468,6 +521,8 @@ def _serialize_event( e time_now_ms config: Event serialization config + membership: The requesting user's membership at the time of the event, + to be injected into unsigned.membership (MSC4115). Returns: The serialized event dictionary. @@ -564,6 +619,23 @@ def _serialize_event( if e.internal_metadata.policy_server_spammy: d["unsigned"]["io.element.synapse.policy_server_spammy"] = True + if config.msc4354_enabled: + sticky_duration = e.sticky_duration() + if sticky_duration: + expires_at = ( + # min() ensures that the origin server can't lie about the time and + # send the event 'in the future', as that would allow them to exceed + # the 1 hour limit on stickiness duration. + min(e.origin_server_ts, time_now_ms) + sticky_duration.as_millis() + ) + if expires_at > time_now_ms: + d["unsigned"][EventUnsignedContentFields.STICKY_TTL] = ( + expires_at - time_now_ms + ) + + if membership is not None: + d["unsigned"][EventUnsignedContentFields.MEMBERSHIP] = membership + return d @@ -577,13 +649,15 @@ class EventClientSerializer: def __init__(self, hs: "HomeServer") -> None: self._store = hs.get_datastores().main self._auth = hs.get_auth() + self._config = hs.config + self._clock = hs.get_clock() self._add_extra_fields_to_unsigned_client_event_callbacks: list[ ADD_EXTRA_FIELDS_TO_UNSIGNED_CLIENT_EVENT_CALLBACK ] = [] async def serialize_event( self, - event: JsonDict | EventBase, + event: JsonDict | FilteredEvent, time_now: int, *, config: SerializeEventConfig = _DEFAULT_SERIALIZE_EVENT_CONFIG, @@ -605,7 +679,7 @@ async def serialize_event( The serialized event """ # To handle the case of presence events and the like - if not isinstance(event, EventBase): + if not isinstance(event, FilteredEvent): return event # Force-enable server admin metadata because the only time an event with @@ -617,11 +691,16 @@ async def serialize_event( ): config = make_config_for_admin(config) - serialized_event = _serialize_event(event, time_now, config=config) + if self._config.experimental.msc4354_enabled: + config = attr.evolve(config, msc4354_enabled=True) + + serialized_event = _serialize_event( + event.event, time_now, config=config, membership=event.membership + ) # If the event was redacted, fetch the redaction event from the database # and include it in the serialized event's unsigned section. - redacted_by: str | None = event.internal_metadata.redacted_by + redacted_by: str | None = event.event.internal_metadata.redacted_by if redacted_by is not None: serialized_event.setdefault("unsigned", {})["redacted_by"] = redacted_by if redaction_map is not None: @@ -648,7 +727,7 @@ async def serialize_event( new_unsigned = {} for callback in self._add_extra_fields_to_unsigned_client_event_callbacks: - u = await callback(event) + u = await callback(event.event) new_unsigned.update(u) if new_unsigned: @@ -666,9 +745,9 @@ async def serialize_event( # Check if there are any bundled aggregations to include with the event. if bundle_aggregations: - if event.event_id in bundle_aggregations: + if event.event.event_id in bundle_aggregations: await self._inject_bundled_aggregations( - event, + event.event, time_now, config, bundle_aggregations, @@ -720,7 +799,7 @@ async def _inject_bundled_aggregations( # `sender` of the edit; however MSC3925 proposes extending it to the whole # of the edit, which is what we do here. serialized_aggregations[RelationTypes.REPLACE] = await self.serialize_event( - event_aggregations.replace, + FilteredEvent(event=event_aggregations.replace, membership=None), time_now, config=config, ) @@ -730,7 +809,7 @@ async def _inject_bundled_aggregations( thread = event_aggregations.thread serialized_latest_event = await self.serialize_event( - thread.latest_event, + FilteredEvent(event=thread.latest_event, membership=None), time_now, config=config, bundle_aggregations=bundled_aggregations, @@ -755,7 +834,7 @@ async def _inject_bundled_aggregations( @trace async def serialize_events( self, - events: Collection[JsonDict | EventBase], + events: Collection[JsonDict | FilteredEvent], time_now: int, *, config: SerializeEventConfig = _DEFAULT_SERIALIZE_EVENT_CONFIG, @@ -780,11 +859,13 @@ async def serialize_events( ) # Batch-fetch all redaction events in one go rather than one per event. - redaction_ids = { - e.internal_metadata.redacted_by - for e in events - if isinstance(e, EventBase) and e.internal_metadata.redacted_by is not None - } + redaction_ids: set[str] = set() + for e in events: + base = e.event if isinstance(e, FilteredEvent) else e + if isinstance(base, EventBase): + redacted_by = base.internal_metadata.redacted_by + if redacted_by is not None: + redaction_ids.add(redacted_by) redaction_map = ( await self._store.get_events(redaction_ids) if redaction_ids else {} ) diff --git a/synapse/events/validator.py b/synapse/events/validator.py index b27f8a942a..ff22b2287f 100644 --- a/synapse/events/validator.py +++ b/synapse/events/validator.py @@ -63,14 +63,17 @@ def validate_new(self, event: EventBase, config: HomeServerConfig) -> None: if event.format_version == EventFormatVersions.ROOM_V1_V2: EventID.from_string(event.event_id) - required = [ + required = { "auth_events", "content", "hashes", "prev_events", "sender", "type", - ] + } + if event.room_version.msc4242_state_dags: + required.remove("auth_events") + required.add("prev_state_events") for k in required: if k not in event: diff --git a/synapse/federation/federation_base.py b/synapse/federation/federation_base.py index 04ba5b86db..fe0710a0bf 100644 --- a/synapse/federation/federation_base.py +++ b/synapse/federation/federation_base.py @@ -177,7 +177,9 @@ async def _check_sigs_and_hash( # Note: we don't redact the event so admins can inspect the event after the # fact. Other processes may redact the event, but that won't be applied to # the database copy of the event until the server's config requires it. - return pdu + # + # We also *don't* return early here as we would still like to evaluate + # `spam_checker_spammy`, for completeness. spam_check = await self._spam_checker_module_callbacks.check_event_for_spam(pdu) @@ -194,6 +196,8 @@ async def _check_sigs_and_hash( # using the event in prev_events). redacted_event = prune_event(pdu) redacted_event.internal_metadata.soft_failed = True + # Mark this as spam so we don't re-evaluate soft-failure status. + redacted_event.internal_metadata.spam_checker_spammy = True return redacted_event return pdu diff --git a/synapse/federation/federation_client.py b/synapse/federation/federation_client.py index 55151ca549..78a1900c73 100644 --- a/synapse/federation/federation_client.py +++ b/synapse/federation/federation_client.py @@ -1108,6 +1108,11 @@ async def send_join( SynapseError: if the chosen remote server returns a 300/400 code, or no servers successfully handle the request. """ + # See related restriction in /createRoom requests in handlers/room.py + if room_version.msc4242_state_dags: + raise UnsupportedRoomVersionError( + "Homeserver does not support this room version over federation" + ) async def send_request(destination: str) -> SendJoinResult: response = await self._do_send_join( diff --git a/synapse/handlers/admin.py b/synapse/handlers/admin.py index 2fb0e5814f..51a752472f 100644 --- a/synapse/handlers/admin.py +++ b/synapse/handlers/admin.py @@ -32,7 +32,8 @@ from synapse.api.constants import Direction, EventTypes, Membership from synapse.api.errors import SynapseError -from synapse.events import EventBase +from synapse.events import EventBase, FrozenEventVMSC4242 +from synapse.events.utils import FilteredEvent from synapse.types import ( JsonMapping, Requester, @@ -251,32 +252,40 @@ async def export_user_data(self, user_id: str, writer: "ExfiltrationWriter") -> topological=last_event.depth, ) - events = await filter_and_transform_events_for_client( + filtered_events = await filter_and_transform_events_for_client( self._storage_controllers, user_id, events, ) - writer.write_events(room_id, events) + writer.write_events(room_id, filtered_events) # Update the extremity tracking dicts - for event in events: + for filtered_event in filtered_events: # Check if we have any prev events that haven't been # processed yet, and add those to the appropriate dicts. - unseen_events = set(event.prev_event_ids()) - written_events + unseen_events = ( + set(filtered_event.event.prev_event_ids()) - written_events + ) if unseen_events: - event_to_unseen_prevs[event.event_id] = unseen_events + event_to_unseen_prevs[filtered_event.event.event_id] = ( + unseen_events + ) for unseen in unseen_events: unseen_to_child_events.setdefault(unseen, set()).add( - event.event_id + filtered_event.event.event_id ) # Now check if this event is an unseen prev event, if so # then we remove this event from the appropriate dicts. - for child_id in unseen_to_child_events.pop(event.event_id, []): - event_to_unseen_prevs[child_id].discard(event.event_id) + for child_id in unseen_to_child_events.pop( + filtered_event.event.event_id, [] + ): + event_to_unseen_prevs[child_id].discard( + filtered_event.event.event_id + ) - written_events.add(event.event_id) + written_events.add(filtered_event.event.event_id) logger.info( "Written %d events in room %s", len(written_events), room_id @@ -485,9 +494,16 @@ async def _redact_all_events( event_dict["redacts"] = event.event_id try: + prev_state_events = None + if room_version.msc4242_state_dags: + assert isinstance(event, FrozenEventVMSC4242) + prev_state_events = event.prev_state_events + assert prev_state_events is not None, ( + "Parent event of redaction has no `prev_state_events` which should be impossible as `prev_state_events` is a required field in MSC4242 rooms" + ) # set the prev event to the offending message to allow for redactions # to be processed in the case where the user has been kicked/banned before - # redactions are requested + # redactions are requested. ( redaction, _, @@ -496,6 +512,7 @@ async def _redact_all_events( event_dict, prev_event_ids=[event.event_id], ratelimit=False, + prev_state_events=prev_state_events, ) except Exception as ex: logger.info( @@ -511,7 +528,7 @@ class ExfiltrationWriter(metaclass=abc.ABCMeta): """Interface used to specify how to write exported data.""" @abc.abstractmethod - def write_events(self, room_id: str, events: list[EventBase]) -> None: + def write_events(self, room_id: str, events: list[FilteredEvent]) -> None: """Write a batch of events for a room.""" raise NotImplementedError() diff --git a/synapse/handlers/auth.py b/synapse/handlers/auth.py index d439380197..e2c41fc168 100644 --- a/synapse/handlers/auth.py +++ b/synapse/handlers/auth.py @@ -1774,13 +1774,17 @@ def _do_validate_hash(checked_hash: bytes) -> bool: else: return False - async def start_sso_ui_auth(self, request: SynapseRequest, session_id: str) -> str: + async def start_sso_ui_auth( + self, request: SynapseRequest, session_id: str, preferred_idp_id: str | None + ) -> str: """ Get the HTML for the SSO redirect confirmation page. Args: request: The incoming HTTP request session_id: The user interactive authentication session ID. + preferred_idp_id: The ID of the identity provider to use. If `None` one will + be picked unpredictably from those the user has already signed in with. Returns: The HTML to render. @@ -1804,15 +1808,26 @@ async def start_sso_ui_auth(self, request: SynapseRequest, session_id: str) -> s # it not being offered. raise SynapseError(400, "User has no SSO identities") - # for now, just pick one - idp_id, sso_auth_provider = next(iter(idps.items())) - if len(idps) > 0: - logger.warning( - "User %r has previously logged in with multiple SSO IdPs; arbitrarily " - "picking %r", - user_id_to_verify, - idp_id, - ) + if preferred_idp_id is not None: + # Use the idp specified by the client. + sso_auth_provider = idps.get(preferred_idp_id) + if sso_auth_provider is None: + raise SynapseError( + 400, + f"Unknown preferred Identity Provider: '{preferred_idp_id}'", + errcode=Codes.INVALID_PARAM, + ) + else: + idp_id, sso_auth_provider = next(iter(idps.items())) + if len(idps) > 0: + # We arbitrarily picked an IdP from multiple potential + # candidates. This may be undesirable for the user. + logger.warning( + "User %r has previously logged in with multiple SSO IdPs; arbitrarily " + "picking %r", + user_id_to_verify, + idp_id, + ) redirect_url = await sso_auth_provider.handle_redirect_request( request, None, session_id diff --git a/synapse/handlers/device.py b/synapse/handlers/device.py index 9a371651fb..2225466648 100644 --- a/synapse/handlers/device.py +++ b/synapse/handlers/device.py @@ -58,6 +58,7 @@ DeviceListUpdates, JsonDict, JsonMapping, + MultiWriterStreamToken, ScheduledTask, StrCollection, StreamKeyType, @@ -1193,7 +1194,16 @@ async def handle_room_un_partial_stated(self, room_id: str) -> None: changes = await self.store.get_device_list_changes_in_room( room_id, device_lists_stream_id ) - local_changes = {(u, d) for u, d in changes if self.hs.is_mine_id(u)} + if changes is not None: + local_changes = {(u, d) for u, d in changes if self.hs.is_mine_id(u)} + else: + # The `device_lists_stream_id` is too old, so we need to fall back + # to looking for changes for all local users. + local_users = await self.store.get_local_users_in_room(room_id) + local_changes = await self.store.get_device_changes_for_users( + MultiWriterStreamToken(stream=device_lists_stream_id), local_users + ) + if not local_changes: return diff --git a/synapse/handlers/events.py b/synapse/handlers/events.py index f6517def9c..2518716bc7 100644 --- a/synapse/handlers/events.py +++ b/synapse/handlers/events.py @@ -25,8 +25,7 @@ from synapse.api.constants import EduTypes, EventTypes, Membership, PresenceState from synapse.api.errors import AuthError, SynapseError -from synapse.events import EventBase -from synapse.events.utils import SerializeEventConfig +from synapse.events.utils import FilteredEvent, SerializeEventConfig from synapse.handlers.presence import format_user_presence_state from synapse.storage.databases.main.events_worker import EventRedactBehaviour from synapse.streams.config import PaginationConfig @@ -102,19 +101,19 @@ async def get_stream( # joined room, we need to send down presence for those users. to_add: list[JsonDict] = [] for event in events: - if not isinstance(event, EventBase): + if not isinstance(event, FilteredEvent): continue - if event.type == EventTypes.Member: - if event.membership != Membership.JOIN: + if event.event.type == EventTypes.Member: + if event.event.membership != Membership.JOIN: continue # Send down presence. - if event.state_key == requester.user.to_string(): + if event.event.state_key == requester.user.to_string(): # Send down presence for everyone in the room. users: Iterable[str] = await self.store.get_users_in_room( - event.room_id + event.event.room_id ) else: - users = [event.state_key] + users = [event.event.state_key] states = await presence_handler.get_states(users) to_add.extend( @@ -155,7 +154,7 @@ async def get_event( room_id: str | None, event_id: str, show_redacted: bool = False, - ) -> EventBase | None: + ) -> FilteredEvent | None: """Retrieve a single specified event on behalf of a user. The event will be transformed in a user-specific and time-specific way, e.g. having unsigned metadata added or being erased depending on who is accessing. diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 9050c2f934..f18cfce9e7 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -240,7 +240,9 @@ async def _maybe_backfill_inner( Args: room_id: The room to backfill in. current_depth: The depth to check at for any upcoming backfill points. - limit: The max number of events to request from the remote federated server. + limit: The number of events that the pagination request will + return. This is used as part of the heuristic to decide if we + should back paginate. processing_start_time: The time when `maybe_backfill` started processing. Only used for timing. If `None`, no timing observation will be made. @@ -1182,7 +1184,7 @@ async def _make_and_verify_event( # We should assert some things. # FIXME: Do this in a nicer way assert event.type == EventTypes.Member - assert event.user_id == user_id + assert event.sender == user_id assert event.state_key == user_id assert event.room_id == room_id return origin, event, room_version diff --git a/synapse/handlers/initial_sync.py b/synapse/handlers/initial_sync.py index 1e5e98a59b..591a0aefd3 100644 --- a/synapse/handlers/initial_sync.py +++ b/synapse/handlers/initial_sync.py @@ -30,7 +30,7 @@ Membership, ) from synapse.api.errors import SynapseError -from synapse.events.utils import SerializeEventConfig +from synapse.events.utils import FilteredEvent, SerializeEventConfig from synapse.events.validator import EventValidator from synapse.handlers.presence import format_user_presence_state from synapse.handlers.receipts import ReceiptEventSource @@ -186,7 +186,7 @@ async def handle_room(event: RoomsForUser) -> None: invite_event = await self.store.get_event(event.event_id) d["invite"] = await self._event_serializer.serialize_event( - invite_event, + FilteredEvent.state(event=invite_event), time_now, config=serializer_options, ) @@ -225,7 +225,7 @@ async def handle_room(event: RoomsForUser) -> None: ) ).addErrback(unwrapFirstError) - messages = await filter_and_transform_events_for_client( + filtered_messages = await filter_and_transform_events_for_client( self._storage_controllers, user_id, messages, @@ -240,7 +240,7 @@ async def handle_room(event: RoomsForUser) -> None: d["messages"] = { "chunk": ( await self._event_serializer.serialize_events( - messages, + filtered_messages, time_now=time_now, config=serializer_options, ) @@ -250,7 +250,7 @@ async def handle_room(event: RoomsForUser) -> None: } d["state"] = await self._event_serializer.serialize_events( - current_state.values(), + [FilteredEvent.state(e) for e in current_state.values()], time_now=time_now, config=serializer_options, ) @@ -382,7 +382,9 @@ async def _room_initial_sync_parted( room_id, limit=pagin_config.limit, end_token=stream_token ) - messages = await filter_and_transform_events_for_client( + filtered_messages: list[ + FilteredEvent + ] = await filter_and_transform_events_for_client( self._storage_controllers, requester.user.to_string(), messages, @@ -402,7 +404,7 @@ async def _room_initial_sync_parted( "chunk": ( # Don't bundle aggregations as this is a deprecated API. await self._event_serializer.serialize_events( - messages, time_now, config=serialize_options + filtered_messages, time_now, config=serialize_options ) ), "start": await start_token.to_string(self.store), @@ -411,7 +413,9 @@ async def _room_initial_sync_parted( "state": ( # Don't bundle aggregations as this is a deprecated API. await self._event_serializer.serialize_events( - room_state.values(), time_now, config=serialize_options + [FilteredEvent.state(e) for e in room_state.values()], + time_now, + config=serialize_options, ) ), "presence": [], @@ -435,7 +439,7 @@ async def _room_initial_sync_joined( serialize_options = SerializeEventConfig(requester=requester) # Don't bundle aggregations as this is a deprecated API. state = await self._event_serializer.serialize_events( - current_state.values(), + [FilteredEvent.state(e) for e in current_state.values()], time_now, config=serialize_options, ) @@ -456,9 +460,7 @@ async def get_presence() -> list[JsonDict]: if not self.hs.config.server.presence_enabled: return [] - states = await presence_handler.get_states( - [m.user_id for m in room_members] - ) + states = await presence_handler.get_states([m.sender for m in room_members]) return [ { @@ -496,7 +498,9 @@ async def get_receipts() -> list[JsonMapping]: ).addErrback(unwrapFirstError) ) - messages = await filter_and_transform_events_for_client( + filtered_messages: list[ + FilteredEvent + ] = await filter_and_transform_events_for_client( self._storage_controllers, requester.user.to_string(), messages, @@ -512,7 +516,7 @@ async def get_receipts() -> list[JsonMapping]: "chunk": ( # Don't bundle aggregations as this is a deprecated API. await self._event_serializer.serialize_events( - messages, time_now, config=serialize_options + filtered_messages, time_now, config=serialize_options ) ), "start": await start_token.to_string(self.store), diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index fc5dceb80b..bfade58063 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -53,7 +53,7 @@ from synapse.api.room_versions import KNOWN_ROOM_VERSIONS from synapse.api.urls import ConsentURIBuilder from synapse.event_auth import validate_event_for_room_version -from synapse.events import EventBase, relation_from_event +from synapse.events import EventBase, FrozenEventVMSC4242, relation_from_event from synapse.events.builder import EventBuilder from synapse.events.snapshot import ( EventContext, @@ -61,7 +61,11 @@ UnpersistedEventContext, UnpersistedEventContextBase, ) -from synapse.events.utils import SerializeEventConfig, maybe_upsert_event_field +from synapse.events.utils import ( + FilteredEvent, + SerializeEventConfig, + maybe_upsert_event_field, +) from synapse.events.validator import EventValidator from synapse.handlers.directory import DirectoryHandler from synapse.handlers.worker_lock import NEW_EVENT_DURING_PURGE_LOCK_NAME @@ -261,7 +265,7 @@ async def get_state_events( room_state = room_state_events[membership_event_id] events = await self._event_serializer.serialize_events( - room_state.values(), + [FilteredEvent.state(e) for e in room_state.values()], self.clock.time_msec(), config=SerializeEventConfig(requester=requester), ) @@ -585,6 +589,7 @@ async def create_event( state_map: StateMap[str] | None = None, for_batch: bool = False, current_state_group: int | None = None, + prev_state_events: list[str] | None = None, delay_id: str | None = None, ) -> tuple[EventBase, UnpersistedEventContextBase]: """ @@ -640,6 +645,10 @@ async def create_event( current_state_group: the current state group, used only for creating events for batch persisting + prev_state_events: + The state event IDs which represent the current forward extremities of the state DAG. + Only applicable on room versions which use a state DAG (MSC4242). + delay_id: The delay ID of this event, if it was a delayed event. Raises: @@ -744,6 +753,7 @@ async def create_event( state_map=state_map, for_batch=for_batch, current_state_group=current_state_group, + prev_state_events=prev_state_events, ) # In an ideal world we wouldn't need the second part of this condition. However, @@ -895,7 +905,7 @@ async def deduplicate_state_event( if not prev_event: return None - if prev_event and event.user_id == prev_event.user_id: + if prev_event and event.sender == prev_event.sender: prev_content = encode_canonical_json(prev_event.content) next_content = encode_canonical_json(event.content) if prev_content == next_content: @@ -972,6 +982,7 @@ async def create_and_send_nonmember_event( ignore_shadow_ban: bool = False, outlier: bool = False, depth: int | None = None, + prev_state_events: list[str] | None = None, delay_id: str | None = None, ) -> tuple[EventBase, int]: """ @@ -1001,6 +1012,9 @@ async def create_and_send_nonmember_event( depth: Override the depth used to order the event in the DAG. Should normally be set to None, which will cause the depth to be calculated based on the prev_events. + prev_state_events: + The state event IDs which represent the current forward extremities of the state DAG. + Only applicable on room versions which use a state DAG (MSC4242). delay_id: The delay ID of this event, if it was a delayed event. Returns: @@ -1098,6 +1112,7 @@ async def create_and_send_nonmember_event( ignore_shadow_ban=ignore_shadow_ban, outlier=outlier, depth=depth, + prev_state_events=prev_state_events, delay_id=delay_id, ) @@ -1112,6 +1127,7 @@ async def _create_and_send_nonmember_event_locked( ignore_shadow_ban: bool = False, outlier: bool = False, depth: int | None = None, + prev_state_events: list[str] | None = None, delay_id: str | None = None, ) -> tuple[EventBase, int]: room_id = event_dict["room_id"] @@ -1141,6 +1157,7 @@ async def _create_and_send_nonmember_event_locked( state_event_ids=state_event_ids, outlier=outlier, depth=depth, + prev_state_events=prev_state_events, delay_id=delay_id, ) context = await unpersisted_context.persist(event) @@ -1236,6 +1253,7 @@ async def create_new_client_event( state_map: StateMap[str] | None = None, for_batch: bool = False, current_state_group: int | None = None, + prev_state_events: list[str] | None = None, ) -> tuple[EventBase, UnpersistedEventContextBase]: """Create a new event for a local client. If bool for_batch is true, will create an event using the prev_event_ids, and will create an event context for @@ -1277,9 +1295,30 @@ async def create_new_client_event( current_state_group: the current state group, used only for creating events for batch persisting + prev_state_events: + The state event IDs which represent the current forward extremities of the state DAG. + Only applicable on room versions which use a state DAG (MSC4242). + If unset, populates them from the current state dag forward extremities. + Returns: Tuple of created event, UnpersistedEventContext """ + if builder.room_version.msc4242_state_dags: + assert auth_event_ids is None + # (kegan) I can't find any call-site which uses this. We can't risk letting in + # untrusted input, so for now assert that we aren't told about any state. + assert state_event_ids is None + + if builder.room_id: + if prev_state_events is None: + prev_state_events = list( + await self.store.get_state_dag_extremities(builder.room_id) + ) + else: + # create event doesn't need prev_state_events to be fetched, but it must be non-None. + assert builder.type == EventTypes.Create and builder.state_key == "" + prev_state_events = [] + # Strip down the state_event_ids to only what we need to auth the event. # For example, we don't need extra m.room.member that don't match event.sender if state_event_ids is not None: @@ -1353,7 +1392,10 @@ async def create_new_client_event( assert state_map is not None auth_ids = self._event_auth_handler.compute_auth_events(builder, state_map) event = await builder.build( - prev_event_ids=prev_event_ids, auth_event_ids=auth_ids, depth=depth + prev_event_ids=prev_event_ids, + auth_event_ids=auth_ids, + depth=depth, + prev_state_events=prev_state_events, ) context: UnpersistedEventContextBase = ( @@ -1370,6 +1412,7 @@ async def create_new_client_event( prev_event_ids=prev_event_ids, auth_event_ids=auth_event_ids, depth=depth, + prev_state_events=prev_state_events, ) # Pass on the outlier property from the builder to the event @@ -1541,7 +1584,7 @@ async def handle_new_client_event( EventTypes.Message, EventTypes.Encrypted, ]: - await self.store.set_room_participation(event.user_id, event.room_id) + await self.store.set_room_participation(event.sender, event.room_id) if event.internal_metadata.is_out_of_band_membership(): # the only sort of out-of-band-membership events we expect to see here are @@ -1559,6 +1602,20 @@ async def handle_new_client_event( auth_event = event_id_to_event.get(event_id) if auth_event: batched_auth_events[event_id] = auth_event + if event.room_version.msc4242_state_dags: + assert isinstance(event, FrozenEventVMSC4242) + # State DAG rooms will check that the prev_state_events are not rejected. + # To do that, we need to make sure we pass in the prev_state_events as + # batched_auth_events, else we will fail the event due to the + # prev_state_events not existing in the database. + for prev_state_event_id in event.prev_state_events: + prev_state_event = event_id_to_event.get( + prev_state_event_id + ) + if prev_state_event: + batched_auth_events[prev_state_event_id] = ( + prev_state_event + ) await self._event_auth_handler.check_auth_rules_from_context( event, batched_auth_events ) @@ -1813,7 +1870,10 @@ async def cache_joined_hosts_for_events( # set for a while, so that the expiry time is reset. state_entry = await self.state.resolve_state_groups_for_events( - event.room_id, event_ids=event.prev_event_ids() + event.room_id, + event_ids=event.prev_state_events + if isinstance(event, FrozenEventVMSC4242) + else event.prev_event_ids(), ) if state_entry.state_group: @@ -2098,7 +2158,7 @@ async def persist_and_notify_client_events( "Could not find event %s" % (event.redacts,) ) - if event.user_id != original_event.user_id: + if event.sender != original_event.sender: raise AuthError( 403, "You don't have permission to redact events" ) @@ -2384,9 +2444,16 @@ async def _rebuild_event_after_third_party_rules( # case. prev_event_ids = await self.store.get_prev_events_for_room(builder.room_id) + prev_state_events = None + if original_event.room_version.msc4242_state_dags: + prev_state_events = list( + await self.store.get_state_dag_extremities(builder.room_id) + ) + event = await builder.build( prev_event_ids=prev_event_ids, auth_event_ids=None, + prev_state_events=prev_state_events, ) # we rebuild the event context, to be on the safe side. If nothing else, diff --git a/synapse/handlers/pagination.py b/synapse/handlers/pagination.py index 368fa3e007..ab02e3acb8 100644 --- a/synapse/handlers/pagination.py +++ b/synapse/handlers/pagination.py @@ -29,6 +29,7 @@ from synapse.api.errors import SynapseError from synapse.api.filtering import Filter from synapse.events import EventBase +from synapse.events.utils import FilteredEvent from synapse.handlers.relations import BundledAggregations from synapse.handlers.worker_lock import NEW_EVENT_DURING_PURGE_LOCK_NAME from synapse.logging.opentracing import trace @@ -79,7 +80,7 @@ class GetMessagesResult: Everything needed to serialize a `/messages` response. """ - messages_chunk: list[EventBase] + messages_chunk: list[FilteredEvent] """ A list of room events. @@ -688,16 +689,18 @@ async def get_messages( events = await event_filter.filter(events) if not use_admin_priviledge: - events = await filter_and_transform_events_for_client( + filtered_events = await filter_and_transform_events_for_client( self._storage_controllers, user_id, events, is_peeking=(member_event_id is None), ) + else: + filtered_events = [FilteredEvent.admin_override(e) for e in events] # if after the filter applied there are no more events # return immediately - but there might be more in next_token batch - if not events: + if not filtered_events: return GetMessagesResult( messages_chunk=[], bundled_aggregations={}, @@ -707,16 +710,16 @@ async def get_messages( ) state = None - if event_filter and event_filter.lazy_load_members and len(events) > 0: + if event_filter and event_filter.lazy_load_members and len(filtered_events) > 0: # TODO: remove redundant members # FIXME: we also care about invite targets etc. state_filter = StateFilter.from_types( - (EventTypes.Member, event.sender) for event in events + (EventTypes.Member, event.event.sender) for event in filtered_events ) state_ids = await self._state_storage_controller.get_state_ids_for_event( - events[0].event_id, state_filter=state_filter + filtered_events[0].event.event_id, state_filter=state_filter ) if state_ids: @@ -724,11 +727,11 @@ async def get_messages( state = list(state_dict.values()) aggregations = await self._relations_handler.get_bundled_aggregations( - events, user_id + filtered_events, user_id ) return GetMessagesResult( - messages_chunk=events, + messages_chunk=filtered_events, bundled_aggregations=aggregations, state=state, start_token=from_token, diff --git a/synapse/handlers/relations.py b/synapse/handlers/relations.py index d7d3002fbe..ee4f8d672e 100644 --- a/synapse/handlers/relations.py +++ b/synapse/handlers/relations.py @@ -33,7 +33,7 @@ from synapse.api.constants import Direction, EventTypes, RelationTypes from synapse.api.errors import SynapseError from synapse.events import EventBase, relation_from_event -from synapse.events.utils import SerializeEventConfig +from synapse.events.utils import FilteredEvent, SerializeEventConfig from synapse.logging.context import make_deferred_yieldable, run_in_background from synapse.logging.opentracing import trace from synapse.storage.databases.main.relations import ThreadsNextBatch, _RelatedEvent @@ -139,7 +139,7 @@ async def get_relations( # not passing them in here we should get a better cache hit rate). related_events, next_token = await self._main_store.get_relations_for_event( event_id=event_id, - event=event, + event=event.event, room_id=room_id, relation_type=relation_type, event_type=event_type, @@ -154,7 +154,9 @@ async def get_relations( [e.event_id for e in related_events] ) - events = await filter_and_transform_events_for_client( + filtered_events: list[ + FilteredEvent + ] = await filter_and_transform_events_for_client( self._storage_controllers, user_id, events, @@ -164,14 +166,14 @@ async def get_relations( # The relations returned for the requested event do include their # bundled aggregations. aggregations = await self.get_bundled_aggregations( - events, requester.user.to_string() + filtered_events, requester.user.to_string() ) now = self._clock.time_msec() serialize_options = SerializeEventConfig(requester=requester) return_value: JsonDict = { "chunk": await self._event_serializer.serialize_events( - events, + filtered_events, now, bundle_aggregations=aggregations, config=serialize_options, @@ -389,7 +391,7 @@ async def _get_threads_for_events( potential_events, _ = await self._main_store.get_relations_for_event( room_id, event_id, - event, + event.event, RelationTypes.THREAD, direction=Direction.FORWARDS, ) @@ -417,7 +419,7 @@ async def _get_threads_for_events( potential_events[-1].event_id, ) continue - latest_thread_event = event + latest_thread_event = event.event results[event_id] = _ThreadAggregation( latest_event=latest_thread_event, @@ -432,12 +434,12 @@ async def _get_threads_for_events( @trace async def get_bundled_aggregations( - self, events: Iterable[EventBase], user_id: str + self, filtered_events: Iterable[FilteredEvent], user_id: str ) -> dict[str, BundledAggregations]: """Generate bundled aggregations for events. Args: - events: The iterable of events to calculate bundled aggregations for. + filtered_events: The iterable of filtered events to calculate bundled aggregations for. user_id: The user requesting the bundled aggregations. Returns: @@ -453,7 +455,9 @@ async def get_bundled_aggregations( events_by_id = {} # A map of event ID to the relation in that event, if there is one. relations_by_id: dict[str, str] = {} - for event in events: + for filtered_event in filtered_events: + event = filtered_event.event + # State events do not get bundled aggregations. if event.is_state(): continue @@ -599,7 +603,9 @@ async def get_threads( # Limit the returned threads to those the user has participated in. events = [event for event in events if participated[event.event_id]] - events = await filter_and_transform_events_for_client( + filtered_events: list[ + FilteredEvent + ] = await filter_and_transform_events_for_client( self._storage_controllers, user_id, events, @@ -607,12 +613,12 @@ async def get_threads( ) aggregations = await self.get_bundled_aggregations( - events, requester.user.to_string() + filtered_events, requester.user.to_string() ) now = self._clock.time_msec() serialized_events = await self._event_serializer.serialize_events( - events, now, bundle_aggregations=aggregations + filtered_events, now, bundle_aggregations=aggregations ) return_value: JsonDict = {"chunk": serialized_events} diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index 024e98ccf4..d4a8e327f2 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -65,9 +65,9 @@ from synapse.api.ratelimiting import Ratelimiter from synapse.api.room_versions import KNOWN_ROOM_VERSIONS, RoomVersion from synapse.event_auth import validate_event_for_room_version -from synapse.events import EventBase +from synapse.events import EventBase, event_exists_in_state_dag from synapse.events.snapshot import UnpersistedEventContext -from synapse.events.utils import copy_and_fixup_power_levels_contents +from synapse.events.utils import FilteredEvent, copy_and_fixup_power_levels_contents from synapse.handlers.relations import BundledAggregations from synapse.rest.admin._base import assert_user_is_admin from synapse.streams import EventSource @@ -109,9 +109,9 @@ @attr.s(slots=True, frozen=True, auto_attribs=True) class EventContext: - events_before: list[EventBase] - event: EventBase - events_after: list[EventBase] + events_before: list[FilteredEvent] + event: FilteredEvent + events_after: list[FilteredEvent] state: list[EventBase] aggregations: dict[str, BundledAggregations] start: str @@ -1250,6 +1250,10 @@ async def create_room( creation_content = config.get("creation_content", {}) # override any attempt to set room versions via the creation_content creation_content["room_version"] = room_version.identifier + # We do not currently support federating state DAG rooms. + # See related restriction in /send_join requests in federation_client.py. + if room_version.msc4242_state_dags: + creation_content[EventContentFields.FEDERATE] = False # trusted private chats have the invited users marked as additional creators if ( @@ -1499,6 +1503,11 @@ async def _send_events_for_new_room( # the most recently created event prev_event: list[str] = [] + # This should be the most recently created state event as we create each event + prev_state_events: list[str] | None = ( + [] if room_version.msc4242_state_dags else None + ) + # a map of event types, state keys -> event_ids. We collect these mappings this as events are # created (but not persisted to the db) to determine state for future created events # (as this info can't be pulled from the db) @@ -1525,6 +1534,7 @@ async def create_event( """ nonlocal depth nonlocal prev_event + nonlocal prev_state_events # Create the event dictionary. event_dict = {"type": etype, "content": content} @@ -1538,6 +1548,7 @@ async def create_event( creator, event_dict, prev_event_ids=prev_event, + prev_state_events=prev_state_events, depth=depth, # Take a copy to ensure each event gets a unique copy of # state_map since it is modified below. @@ -1548,7 +1559,8 @@ async def create_event( depth += 1 prev_event = [new_event.event_id] state_map[(new_event.type, new_event.state_key)] = new_event.event_id - + if room_version.msc4242_state_dags and event_exists_in_state_dag(new_event): + prev_state_events = [new_event.event_id] return new_event, new_unpersisted_context preset_config, config = self._room_preset_config(room_config) @@ -1581,6 +1593,8 @@ async def create_event( ignore_shadow_ban=True, ) last_sent_event_id = ev.event_id + if room_version.msc4242_state_dags: + prev_state_events = [ev.event_id] member_event_id, _ = await self.room_member_handler.update_membership( creator, @@ -1592,8 +1606,11 @@ async def create_event( new_room=True, prev_event_ids=[last_sent_event_id], depth=depth, + prev_state_events=prev_state_events, ) prev_event = [member_event_id] + if room_version.msc4242_state_dags: + prev_state_events = [member_event_id] # update the depth and state map here as the membership event has been created # through a different code path @@ -1929,9 +1946,9 @@ async def get_event_context( # The user is peeking if they aren't in the room already is_peeking = not is_user_in_room - async def filter_evts(events: list[EventBase]) -> list[EventBase]: + async def filter_evts(events: list[EventBase]) -> list[FilteredEvent]: if use_admin_priviledge: - return events + return [FilteredEvent.admin_override(e) for e in events] return await filter_and_transform_events_for_client( self._storage_controllers, user.to_string(), @@ -1959,31 +1976,33 @@ async def filter_evts(events: list[EventBase]) -> list[EventBase]: events_before = await event_filter.filter(events_before) events_after = await event_filter.filter(events_after) - events_before = await filter_evts(events_before) - events_after = await filter_evts(events_after) + filtered_events_before = await filter_evts(events_before) + filtered_events_after = await filter_evts(events_after) # filter_evts can return a pruned event in case the user is allowed to see that # there's something there but not see the content, so use the event that's in # `filtered` rather than the event we retrieved from the datastore. - event = filtered[0] + filtered_event = filtered[0] # Fetch the aggregations. aggregations = await self._relations_handler.get_bundled_aggregations( - itertools.chain(events_before, (event,), events_after), + itertools.chain( + filtered_events_before, (filtered_event,), filtered_events_after + ), user.to_string(), ) - if events_after: - last_event_id = events_after[-1].event_id + if filtered_events_after: + last_event_id = filtered_events_after[-1].event.event_id else: last_event_id = event_id if event_filter and event_filter.lazy_load_members: state_filter = StateFilter.from_lazy_load_member_list( - ev.sender + ev.event.sender for ev in itertools.chain( - events_before, - (event,), - events_after, + filtered_events_before, + (filtered_event,), + filtered_events_after, ) ) else: @@ -2006,9 +2025,9 @@ async def filter_evts(events: list[EventBase]) -> list[EventBase]: token = StreamToken.START return EventContext( - events_before=events_before, - event=event, - events_after=events_after, + events_before=filtered_events_before, + event=filtered_event, + events_after=filtered_events_after, state=state_events, aggregations=aggregations, start=await token.copy_and_replace( diff --git a/synapse/handlers/room_member.py b/synapse/handlers/room_member.py index b2e678e90e..236c8ca03c 100644 --- a/synapse/handlers/room_member.py +++ b/synapse/handlers/room_member.py @@ -36,9 +36,11 @@ from synapse.api.errors import ( AuthError, Codes, + NotFoundError, PartialStateConflictError, ShadowBanError, SynapseError, + UnsupportedRoomVersionError, ) from synapse.api.ratelimiting import Ratelimiter from synapse.event_auth import get_named_level, get_power_level_event @@ -408,6 +410,7 @@ async def _local_membership_update( require_consent: bool = True, outlier: bool = False, origin_server_ts: int | None = None, + prev_state_events: list[str] | None = None, delay_id: str | None = None, ) -> tuple[str, int]: """ @@ -494,6 +497,7 @@ async def _local_membership_update( depth=depth, require_consent=require_consent, outlier=outlier, + prev_state_events=prev_state_events, delay_id=delay_id, ) context = await unpersisted_context.persist(event) @@ -590,6 +594,7 @@ async def update_membership( state_event_ids: list[str] | None = None, depth: int | None = None, origin_server_ts: int | None = None, + prev_state_events: list[str] | None = None, delay_id: str | None = None, ) -> tuple[str, int]: """Update a user's membership in a room. @@ -684,6 +689,7 @@ async def update_membership( state_event_ids=state_event_ids, depth=depth, origin_server_ts=origin_server_ts, + prev_state_events=prev_state_events, delay_id=delay_id, ) @@ -707,6 +713,7 @@ async def update_membership_locked( state_event_ids: list[str] | None = None, depth: int | None = None, origin_server_ts: int | None = None, + prev_state_events: list[str] | None = None, delay_id: str | None = None, ) -> tuple[str, int]: """Helper for update_membership. @@ -951,10 +958,21 @@ async def update_membership_locked( require_consent=require_consent, outlier=outlier, origin_server_ts=origin_server_ts, + prev_state_events=prev_state_events, delay_id=delay_id, ) - latest_event_ids = await self.store.get_prev_events_for_room(room_id) + is_state_dags = False + try: + room_version = await self.store.get_room_version(room_id) + is_state_dags = room_version.msc4242_state_dags + except (NotFoundError, UnsupportedRoomVersionError): + pass + + if is_state_dags: + latest_event_ids = list(await self.store.get_state_dag_extremities(room_id)) + else: + latest_event_ids = await self.store.get_prev_events_for_room(room_id) is_partial_state_room = await self.store.is_partial_state_room(room_id) partial_state_before_join = await self.state_handler.compute_state_after_events( @@ -1165,6 +1183,8 @@ async def update_membership_locked( # see: https://github.com/matrix-org/synapse/issues/7139 if len(latest_event_ids) == 0: latest_event_ids = [invite.event_id] + if invite.room_version.msc4242_state_dags: + prev_state_events = [invite.event_id] # or perhaps this is a remote room that a local user has knocked on elif current_membership_type == Membership.KNOCK: @@ -1210,6 +1230,7 @@ async def update_membership_locked( require_consent=require_consent, outlier=outlier, origin_server_ts=origin_server_ts, + prev_state_events=prev_state_events, delay_id=delay_id, ) @@ -2108,10 +2129,21 @@ async def _generate_local_out_of_band_leave( # # the prev_events consist solely of the previous membership event. prev_event_ids = [previous_membership_event.event_id] - auth_event_ids = ( - list(previous_membership_event.auth_event_ids()) + prev_event_ids - ) + auth_event_ids = None + # Authorise the leave by referencing the previous membership + prev_state_event_ids = None + if previous_membership_event.room_version.msc4242_state_dags: + prev_state_event_ids = [ + previous_membership_event.event_id, + ] + else: + auth_event_ids = ( + list(previous_membership_event.auth_event_ids()) + prev_event_ids + ) + # State DAG rooms should not have auth events specified + # Normal rooms should not have prev state event IDs specified + assert not (prev_state_event_ids is not None and auth_event_ids is not None) # Try several times, it could fail with PartialStateConflictError # in handle_new_client_event, cf comment in except block. max_retries = 5 @@ -2127,6 +2159,7 @@ async def _generate_local_out_of_band_leave( prev_event_ids=prev_event_ids, auth_event_ids=auth_event_ids, outlier=True, + prev_state_events=prev_state_event_ids, ) context = await unpersisted_context.persist(event) event.internal_metadata.out_of_band_membership = True diff --git a/synapse/handlers/room_policy.py b/synapse/handlers/room_policy.py index fee3c6cfaf..01943e1991 100644 --- a/synapse/handlers/room_policy.py +++ b/synapse/handlers/room_policy.py @@ -223,11 +223,16 @@ async def ask_policy_server_to_sign_event( return # Ask the policy server to sign this event. - # We set a smallish timeout here as we don't want to block event sending too long. try: signature = await self._federation_client.ask_policy_server_to_sign_event( policy_server.server_name, event, + # We set a smallish timeout here as we don't want to block event sending + # too long. + # + # We were previously seeing regular timeouts with media + # scanning/checking when the timeout was set to 3s. 30s was chosen based + # on vibes and light real world testing. timeout=30000, ) # TODO: We can *probably* remove this when we remove unstable MSC4284 support. diff --git a/synapse/handlers/search.py b/synapse/handlers/search.py index 56c047b0e8..30e072d011 100644 --- a/synapse/handlers/search.py +++ b/synapse/handlers/search.py @@ -29,8 +29,7 @@ from synapse.api.constants import EventTypes, Membership from synapse.api.errors import NotFoundError, SynapseError from synapse.api.filtering import Filter -from synapse.events import EventBase -from synapse.events.utils import SerializeEventConfig +from synapse.events.utils import FilteredEvent, SerializeEventConfig from synapse.types import JsonDict, Requester, StrCollection, StreamKeyType, UserID from synapse.types.state import StateFilter from synapse.visibility import filter_and_transform_events_for_client @@ -48,7 +47,7 @@ class _SearchResult: # A mapping of event ID to the rank of that event. rank_map: dict[str, int] # A list of the resulting events. - allowed_events: list[EventBase] + allowed_events: list[FilteredEvent] # A map of room ID to results. room_groups: dict[str, JsonDict] # A set of event IDs to highlight. @@ -355,12 +354,12 @@ async def _search( state_results = {} if include_state: - for room_id in {e.room_id for e in search_result.allowed_events}: + for room_id in {e.event.room_id for e in search_result.allowed_events}: state = await self._storage_controllers.state.get_current_state(room_id) state_results[room_id] = list(state.values()) aggregations = await self._relations_handler.get_bundled_aggregations( - # Generate an iterable of EventBase for all the events that will be + # Generate an iterable of FilteredEvent for all the events that will be # returned, including contextual events. itertools.chain( # The events_before and events_after for each context. @@ -396,14 +395,14 @@ async def _search( results = [ { - "rank": search_result.rank_map[e.event_id], + "rank": search_result.rank_map[e.event.event_id], "result": await self._event_serializer.serialize_event( e, time_now, bundle_aggregations=aggregations, config=serialize_options, ), - "context": contexts.get(e.event_id, {}), + "context": contexts.get(e.event.event_id, {}), } for e in search_result.allowed_events ] @@ -417,7 +416,9 @@ async def _search( if state_results: rooms_cat_res["state"] = { room_id: await self._event_serializer.serialize_events( - state_events, time_now, config=serialize_options + [FilteredEvent.state(e) for e in state_events], + time_now, + config=serialize_options, ) for room_id, state_events in state_results.items() } @@ -485,19 +486,19 @@ async def _search_by_rank( filtered_events, ) - events.sort(key=lambda e: -rank_map[e.event_id]) + events.sort(key=lambda e: -rank_map[e.event.event_id]) allowed_events = events[: search_filter.limit] for e in allowed_events: rm = room_groups.setdefault( - e.room_id, {"results": [], "order": rank_map[e.event_id]} + e.event.room_id, {"results": [], "order": rank_map[e.event.event_id]} ) - rm["results"].append(e.event_id) + rm["results"].append(e.event.event_id) s = sender_group.setdefault( - e.sender, {"results": [], "order": rank_map[e.event_id]} + e.event.sender, {"results": [], "order": rank_map[e.event.event_id]} ) - s["results"].append(e.event_id) + s["results"].append(e.event.event_id) return ( _SearchResult( @@ -549,7 +550,7 @@ async def _search_by_recent( highlights = set() - room_events: list[EventBase] = [] + room_events: list[FilteredEvent] = [] i = 0 pagination_token = batch_token @@ -595,11 +596,11 @@ async def _search_by_recent( pagination_token = results[-1]["pagination_token"] for event in room_events: - group = room_groups.setdefault(event.room_id, {"results": []}) - group["results"].append(event.event_id) + group = room_groups.setdefault(event.event.room_id, {"results": []}) + group["results"].append(event.event.event_id) if room_events and len(room_events) >= search_filter.limit: - last_event_id = room_events[-1].event_id + last_event_id = room_events[-1].event.event_id pagination_token = results_map[last_event_id]["pagination_token"] # We want to respect the given batch group and group keys so @@ -632,7 +633,7 @@ async def _search_by_recent( async def _calculate_event_contexts( self, user: UserID, - allowed_events: list[EventBase], + allowed_events: list[FilteredEvent], before_limit: int, after_limit: int, include_profile: bool, @@ -658,7 +659,7 @@ async def _calculate_event_contexts( contexts = {} for event in allowed_events: res = await self.store.get_events_around( - event.room_id, event.event_id, before_limit, after_limit + event.event.room_id, event.event.event_id, before_limit, after_limit ) logger.info( @@ -692,14 +693,14 @@ async def _calculate_event_contexts( if include_profile: senders = { - ev.sender + ev.event.sender for ev in itertools.chain(events_before, [event], events_after) } if events_after: - last_event_id = events_after[-1].event_id + last_event_id = events_after[-1].event.event_id else: - last_event_id = event.event_id + last_event_id = event.event.event_id state_filter = StateFilter.from_types( [(EventTypes.Member, sender) for sender in senders] @@ -718,6 +719,6 @@ async def _calculate_event_contexts( if s.type == EventTypes.Member and s.state_key in senders } - contexts[event.event_id] = context + contexts[event.event.event_id] = context return contexts diff --git a/synapse/handlers/sliding_sync/__init__.py b/synapse/handlers/sliding_sync/__init__.py index 6feb6c292e..1cc587d4a7 100644 --- a/synapse/handlers/sliding_sync/__init__.py +++ b/synapse/handlers/sliding_sync/__init__.py @@ -23,7 +23,7 @@ from synapse.api.constants import Direction, EventTypes, Membership from synapse.events import EventBase -from synapse.events.utils import strip_event +from synapse.events.utils import FilteredEvent, strip_event from synapse.handlers.relations import BundledAggregations from synapse.handlers.sliding_sync.extensions import SlidingSyncExtensionHandler from synapse.handlers.sliding_sync.room_lists import ( @@ -679,7 +679,7 @@ async def get_room_sync_data( # membership. Currently, we have to make all of these optional because # `invite`/`knock` rooms only have `stripped_state`. See # https://github.com/matrix-org/matrix-spec-proposals/pull/3575#discussion_r1653045932 - timeline_events: list[EventBase] = [] + timeline_events: list[FilteredEvent] = [] bundled_aggregations: dict[str, BundledAggregations] | None = None limited: bool | None = None prev_batch_token: StreamToken | None = None @@ -739,7 +739,7 @@ async def get_room_sync_data( # Use `stream_ordering` for updates else paginate_room_events_by_stream_ordering ) - timeline_events, new_room_key, limited = await pagination_method( + raw_timeline_events, new_room_key, limited = await pagination_method( room_id=room_id, # The bounds are reversed so we can paginate backwards # (from newer to older events) starting at to_bound. @@ -752,13 +752,13 @@ async def get_room_sync_data( # We want to return the events in ascending order (the last event is the # most recent). - timeline_events.reverse() + raw_timeline_events.reverse() # Make sure we don't expose any events that the client shouldn't see timeline_events = await filter_and_transform_events_for_client( self.storage_controllers, user.to_string(), - timeline_events, + raw_timeline_events, is_peeking=room_membership_for_user_at_to_token.membership != Membership.JOIN, filter_send_to_client=True, @@ -778,12 +778,17 @@ async def get_room_sync_data( if from_token is not None: for timeline_event in reversed(timeline_events): # This fields should be present for all persisted events - assert timeline_event.internal_metadata.stream_ordering is not None - assert timeline_event.internal_metadata.instance_name is not None + assert ( + timeline_event.event.internal_metadata.stream_ordering + is not None + ) + assert ( + timeline_event.event.internal_metadata.instance_name is not None + ) persisted_position = PersistedEventPosition( - instance_name=timeline_event.internal_metadata.instance_name, - stream=timeline_event.internal_metadata.stream_ordering, + instance_name=timeline_event.event.internal_metadata.instance_name, + stream=timeline_event.event.internal_metadata.stream_ordering, ) if persisted_position.persisted_after( from_token.stream_token.room_key @@ -1061,13 +1066,13 @@ async def get_room_sync_data( if timeline_events is not None: for timeline_event in timeline_events: # Anyone who sent a message is relevant - timeline_membership.add(timeline_event.sender) + timeline_membership.add(timeline_event.event.sender) # We also care about invite, ban, kick, targets, # etc. - if timeline_event.type == EventTypes.Member: + if timeline_event.event.type == EventTypes.Member: timeline_membership.add( - timeline_event.state_key + timeline_event.event.state_key ) # The client needs to know the membership of everyone in @@ -1480,7 +1485,7 @@ async def _get_bump_stamp( self, room_id: str, to_token: StreamToken, - timeline: list[EventBase], + timeline: list[FilteredEvent], check_outside_timeline: bool, ) -> int | None: """Get a bump stamp for the room, if we have a bump event and it has @@ -1500,8 +1505,8 @@ async def _get_bump_stamp( # those matches. We iterate backwards and take the stream ordering # of the first event that matches the bump event types. for timeline_event in reversed(timeline): - if timeline_event.type in SLIDING_SYNC_DEFAULT_BUMP_EVENT_TYPES: - new_bump_stamp = timeline_event.internal_metadata.stream_ordering + if timeline_event.event.type in SLIDING_SYNC_DEFAULT_BUMP_EVENT_TYPES: + new_bump_stamp = timeline_event.event.internal_metadata.stream_ordering # All persisted events have a stream ordering assert new_bump_stamp is not None diff --git a/synapse/handlers/sliding_sync/extensions.py b/synapse/handlers/sliding_sync/extensions.py index 9b7a01df14..4a324e9661 100644 --- a/synapse/handlers/sliding_sync/extensions.py +++ b/synapse/handlers/sliding_sync/extensions.py @@ -761,7 +761,7 @@ async def handle_previously_room(room_id: str) -> None: # in the timeline to avoid bloating and blowing up the sync response # as the number of users in the room increases. (this behavior is part of the spec) initial_rooms_and_event_ids = [ - (room_id, event.event_id) + (room_id, event.event.event_id) for room_id in initial_rooms if room_id in actual_room_response_map for event in actual_room_response_map[room_id].timeline_events diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index c8ef5e2aa6..c88f703ae9 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -43,6 +43,7 @@ from synapse.api.presence import UserPresenceState from synapse.api.room_versions import KNOWN_ROOM_VERSIONS from synapse.events import EventBase +from synapse.events.utils import FilteredEvent from synapse.handlers.relations import BundledAggregations from synapse.logging import issue9533_logger from synapse.logging.context import current_context @@ -123,7 +124,7 @@ class SyncConfig: @attr.s(slots=True, frozen=True, auto_attribs=True) class TimelineBatch: prev_batch: StreamToken - events: Sequence[EventBase] + events: Sequence[FilteredEvent] limited: bool # A mapping of event ID to the bundled aggregations for the above events. # This is only calculated if limited is true. @@ -148,7 +149,7 @@ class JoinedSyncResult: state: StateMap[EventBase] ephemeral: list[JsonDict] account_data: list[JsonDict] - sticky: list[EventBase] + sticky: list[FilteredEvent] unread_notifications: JsonDict unread_thread_notifications: JsonDict summary: JsonDict | None @@ -699,6 +700,7 @@ async def _load_filtered_recents( log_kv({"limited": limited}) + filtered_recents: list[FilteredEvent] if potential_recents: recents = await sync_config.filter_collection.filter_room_timeline( potential_recents @@ -725,29 +727,32 @@ async def _load_filtered_recents( ) ) - recents = await filter_and_transform_events_for_client( + filtered_recents = await filter_and_transform_events_for_client( self._storage_controllers, sync_config.user.to_string(), recents, always_include_ids=current_state_ids, ) - log_kv({"recents_after_visibility_filtering": len(recents)}) + log_kv({"recents_after_visibility_filtering": len(filtered_recents)}) else: - recents = [] + filtered_recents = [] if not limited or block_all_timeline: prev_batch_token = upto_token - if recents: - assert recents[0].internal_metadata.stream_ordering + if filtered_recents: + assert filtered_recents[0].event.internal_metadata.stream_ordering room_key = RoomStreamToken( - stream=recents[0].internal_metadata.stream_ordering - 1 + stream=filtered_recents[ + 0 + ].event.internal_metadata.stream_ordering + - 1 ) prev_batch_token = upto_token.copy_and_replace( StreamKeyType.ROOM, room_key ) return TimelineBatch( - events=recents, prev_batch=prev_batch_token, limited=False + events=filtered_recents, prev_batch=prev_batch_token, limited=False ) filtering_factor = 2 @@ -764,7 +769,7 @@ async def _load_filtered_recents( elif since_token and not newly_joined_room: since_key = since_token.room_key - while limited and len(recents) < timeline_limit and max_repeat: + while limited and len(filtered_recents) < timeline_limit and max_repeat: # For initial `/sync`, we want to view a historical section of the # timeline; to fetch events by `topological_ordering` (best # representation of the room DAG as others were seeing it at the time). @@ -835,26 +840,35 @@ async def _load_filtered_recents( ) ) - loaded_recents = await filter_and_transform_events_for_client( + loaded_filtered_recents: list[ + FilteredEvent + ] = await filter_and_transform_events_for_client( self._storage_controllers, sync_config.user.to_string(), loaded_recents, always_include_ids=current_state_ids, ) - log_kv({"loaded_recents_after_client_filtering": len(loaded_recents)}) + log_kv( + { + "loaded_recents_after_client_filtering": len( + loaded_filtered_recents + ) + } + ) - loaded_recents.extend(recents) - recents = loaded_recents + loaded_filtered_recents.extend(filtered_recents) + filtered_recents = loaded_filtered_recents max_repeat -= 1 - if len(recents) > timeline_limit: + if len(filtered_recents) > timeline_limit: limited = True - recents = recents[-timeline_limit:] - assert recents[0].internal_metadata.stream_ordering + filtered_recents = filtered_recents[-timeline_limit:] + assert filtered_recents[0].event.internal_metadata.stream_ordering room_key = RoomStreamToken( - stream=recents[0].internal_metadata.stream_ordering - 1 + stream=filtered_recents[0].event.internal_metadata.stream_ordering + - 1 ) prev_batch_token = upto_token.copy_and_replace(StreamKeyType.ROOM, room_key) @@ -865,12 +879,12 @@ async def _load_filtered_recents( if limited or newly_joined_room: bundled_aggregations = ( await self._relations_handler.get_bundled_aggregations( - recents, sync_config.user.to_string() + filtered_recents, sync_config.user.to_string() ) ) return TimelineBatch( - events=recents, + events=filtered_recents, prev_batch=prev_batch_token, # Also mark as limited if this is a new room or there has been a gap # (to force client to paginate the gap). @@ -976,8 +990,8 @@ async def compute_summary( # ...or ones which are in the timeline... for ev in batch.events: - if ev.type == EventTypes.Member: - existing_members.add(ev.state_key) + if ev.event.type == EventTypes.Member: + existing_members.add(ev.event.state_key) # ...and then ensure any missing ones get included in state. missing_hero_event_ids = [ @@ -1084,32 +1098,34 @@ async def compute_state_delta( first_event_by_sender_map = {} for event in batch.events: # Build the map from user IDs to the first timeline event they sent. - if event.sender not in first_event_by_sender_map: - first_event_by_sender_map[event.sender] = event + if event.event.sender not in first_event_by_sender_map: + first_event_by_sender_map[event.event.sender] = event.event # When using `state_after`, there is no special treatment with # regards to state also being in the `timeline`. Always fetch # relevant membership regardless of whether the state event is in # the `timeline`. if sync_config.use_state_after: - members_to_fetch.add(event.sender) + members_to_fetch.add(event.event.sender) # For `state`, the client is supposed to do a flawed re-construction # of state over time by starting with the given `state` and layering # on state from the `timeline` as you go (flawed because state # resolution). In this case, we only need their membership in # `state` when their membership isn't already in the `timeline`. - elif (EventTypes.Member, event.sender) not in timeline_state: - members_to_fetch.add(event.sender) + elif (EventTypes.Member, event.event.sender) not in timeline_state: + members_to_fetch.add(event.event.sender) # FIXME: we also care about invite targets etc. - if event.is_state(): - timeline_state[(event.type, event.state_key)] = event.event_id + if event.event.is_state(): + timeline_state[(event.event.type, event.event.state_key)] = ( + event.event.event_id + ) else: timeline_state = { - (event.type, event.state_key): event.event_id + (event.event.type, event.event.state_key): event.event.event_id for event in batch.events - if event.is_state() + if event.event.is_state() } # Now calculate the state to return in the sync response for the room. @@ -1340,7 +1356,7 @@ async def _compute_state_delta_for_full_sync( # timeline, but that is good enough here. state_at_timeline_start = ( await self._state_storage_controller.get_state_ids_for_event( - batch.events[0].event_id, + batch.events[0].event.event_id, state_filter=state_filter, await_full_state=await_full_state, ) @@ -1470,10 +1486,10 @@ async def _compute_state_delta_for_incremental_sync( prev_event_id = last_event_id_prev_batch for e in batch.events: - if e.prev_event_ids() != [prev_event_id]: + if e.event.prev_event_ids() != [prev_event_id]: is_linear_timeline = False break - prev_event_id = e.event_id + prev_event_id = e.event.event_id if is_linear_timeline and not batch.limited: state_ids: StateMap[str] = {} @@ -1487,7 +1503,7 @@ async def _compute_state_delta_for_incremental_sync( state_ids = ( await self._state_storage_controller.get_state_ids_for_event( - batch.events[0].event_id, + batch.events[0].event.event_id, # we only want members! state_filter=StateFilter.from_types( (EventTypes.Member, member) @@ -1501,7 +1517,7 @@ async def _compute_state_delta_for_incremental_sync( if batch: state_at_timeline_start = ( await self._state_storage_controller.get_state_ids_for_event( - batch.events[0].event_id, + batch.events[0].event.event_id, state_filter=state_filter, await_full_state=await_full_state, ) @@ -2854,7 +2870,7 @@ async def _generate_room_entry( # if there are membership changes in the timeline, or # if membership has changed during a gappy sync, or # if this is an initial sync. - any(ev.type == EventTypes.Member for ev in batch.events) + any(ev.event.type == EventTypes.Member for ev in batch.events) or ( # XXX: this may include false positives in the form of LL # members which have snuck into state @@ -2870,7 +2886,7 @@ async def _generate_room_entry( if room_builder.rtype == "joined": unread_notifications: dict[str, int] = {} - sticky_events: list[EventBase] = [] + sticky_events: list[FilteredEvent] = [] if sticky_event_ids: # As per MSC4354: # Remove sticky events that are already in the timeline, else we will needlessly duplicate @@ -2880,7 +2896,7 @@ async def _generate_room_entry( # This is particularly important given the risk of sticky events spam since # anyone can send sticky events, so halving the bandwidth on average for each sticky # event is helpful. - timeline_event_id_set = {ev.event_id for ev in batch.events} + timeline_event_id_set = {ev.event.event_id for ev in batch.events} # Must preserve sticky event stream order sticky_event_ids = [ e for e in sticky_event_ids if e not in timeline_event_id_set @@ -3144,7 +3160,8 @@ def calculate_user_changes(self) -> tuple[AbstractSet[str], AbstractSet[str]]: if self.since_token: for joined_sync in self.joined: it = itertools.chain( - joined_sync.state.values(), joined_sync.timeline.events + joined_sync.state.values(), + (e.event for e in joined_sync.timeline.events), ) for event in it: if event.type == EventTypes.Member: diff --git a/synapse/handlers/thread_subscriptions.py b/synapse/handlers/thread_subscriptions.py index 539672c7fe..29cb045d00 100644 --- a/synapse/handlers/thread_subscriptions.py +++ b/synapse/handlers/thread_subscriptions.py @@ -53,7 +53,7 @@ async def get_thread_subscription_settings( raise NotFoundError("No such thread root") return await self.store.get_subscription_for_thread( - user_id.to_string(), event.room_id, thread_root_event_id + user_id.to_string(), event.event.room_id, thread_root_event_id ) async def subscribe_user_to_thread( @@ -103,7 +103,7 @@ async def subscribe_user_to_thread( ) if autosub_cause_event is None: raise NotFoundError("Automatic subscription event not found") - relation = relation_from_event(autosub_cause_event) + relation = relation_from_event(autosub_cause_event.event) if ( relation is None or relation.rel_type != RelationTypes.THREAD @@ -115,7 +115,9 @@ async def subscribe_user_to_thread( errcode=Codes.MSC4306_NOT_IN_THREAD, ) - automatic_event_orderings = EventOrderings.from_event(autosub_cause_event) + automatic_event_orderings = EventOrderings.from_event( + autosub_cause_event.event + ) else: automatic_event_orderings = None @@ -174,7 +176,7 @@ async def unsubscribe_user_from_thread( outcome = await self.store.unsubscribe_user_from_thread( user_id.to_string(), - event.room_id, + event.event.room_id, thread_root_event_id, ) diff --git a/synapse/media/preview_html.py b/synapse/media/preview_html.py index 22ad581f82..4f9315f4c9 100644 --- a/synapse/media/preview_html.py +++ b/synapse/media/preview_html.py @@ -278,17 +278,15 @@ def parse_html_to_open_graph(tree: "etree._Element") -> dict[str, str | None]: # "og:video:height" : "720", # "og:video:secure_url": "https://www.youtube.com/v/LXDBoHyjmtw?version=3", - og = _get_meta_tags(tree, "property", "og") + ogRoot = _get_meta_tags(tree, "property", "og") - # TODO: Search for properties specific to the different Open Graph types, - # such as article: meta tags, e.g.: - # - # "article:publisher" : "https://www.facebook.com/thethudonline" /> - # "article:author" content="https://www.facebook.com/thethudonline" /> - # "article:tag" content="baby" /> - # "article:section" content="Breaking News" /> - # "article:published_time" content="2016-03-31T19:58:24+00:00" /> - # "article:modified_time" content="2016-04-01T18:31:53+00:00" /> + # https://ogp.me/#type_article + ogArticle = _get_meta_tags(tree, "property", "article") + # https://ogp.me/#type_profile + ogProfile = _get_meta_tags(tree, "property", "profile") + + # Merge as-is + og = ogRoot | ogArticle | ogProfile # Search for Twitter Card (twitter:) meta tags, e.g.: # diff --git a/synapse/metrics/__init__.py b/synapse/metrics/__init__.py index 83abd91fad..a86debf9f2 100644 --- a/synapse/metrics/__init__.py +++ b/synapse/metrics/__init__.py @@ -62,6 +62,7 @@ import synapse.metrics._reactor_metrics # noqa: F401 from synapse.metrics._gc import MIN_TIME_BETWEEN_GCS, install_gc_manager from synapse.metrics._types import Collector +from synapse.synapse_rust import get_rustc_version from synapse.types import StrSequence from synapse.util import SYNAPSE_VERSION @@ -69,6 +70,9 @@ METRICS_PREFIX = "/_synapse/metrics" +# Rust version used for compilation +RUSTC_VERSION = get_rustc_version() + HAVE_PROC_SELF_STAT = os.path.exists("/proc/self/stat") SERVER_NAME_LABEL = "server_name" @@ -672,12 +676,15 @@ def collect(self) -> Iterable[Metric]: # consider this process-level because all Synapse homeservers running in the process # will use the same Synapse version. build_info = Gauge( # type: ignore[missing-server-name-label] - "synapse_build_info", "Build information", ["pythonversion", "version", "osversion"] + "synapse_build_info", + "Build information", + ["pythonversion", "version", "osversion", "rustcversion"], ) build_info.labels( " ".join([platform.python_implementation(), platform.python_version()]), SYNAPSE_VERSION, " ".join([platform.system(), platform.release()]), + RUSTC_VERSION, ).set(1) # Loaded modules info diff --git a/synapse/notifier.py b/synapse/notifier.py index 93d438def7..f1cec74462 100644 --- a/synapse/notifier.py +++ b/synapse/notifier.py @@ -41,6 +41,7 @@ from synapse.api.constants import EduTypes, EventTypes, HistoryVisibility, Membership from synapse.api.errors import AuthError from synapse.events import EventBase +from synapse.events.utils import FilteredEvent from synapse.handlers.presence import format_user_presence_state from synapse.logging import issue9533_logger from synapse.logging.context import PreserveLoggingContext @@ -210,7 +211,7 @@ def new_listener(self, token: StreamToken) -> "Deferred[StreamToken]": @attr.s(slots=True, frozen=True, auto_attribs=True) class EventStreamResult: - events: list[JsonDict | EventBase] + events: list[JsonDict | FilteredEvent] start_token: StreamToken end_token: StreamToken @@ -765,7 +766,7 @@ async def check_for_updates( # The events fetched from each source are a JsonDict, EventBase, or # UserPresenceState, but see below for UserPresenceState being # converted to JsonDict. - events: list[JsonDict | EventBase] = [] + events: list[JsonDict | FilteredEvent] = [] end_token = from_token for keyname, source in self.event_sources.sources.get_sources(): diff --git a/synapse/push/httppusher.py b/synapse/push/httppusher.py index fdfae234be..ca63a99e3e 100644 --- a/synapse/push/httppusher.py +++ b/synapse/push/httppusher.py @@ -501,7 +501,7 @@ async def dispatch_push_event( "event_id": event.event_id, "room_id": event.room_id, "type": event.type, - "sender": event.user_id, + "sender": event.sender, "prio": priority, } if not self.disable_badge_count: diff --git a/synapse/push/mailer.py b/synapse/push/mailer.py index d18630e80b..1ebbc6d4f3 100644 --- a/synapse/push/mailer.py +++ b/synapse/push/mailer.py @@ -543,8 +543,10 @@ async def _get_notif_vars( results.events_before + [notif_event], ) - for event in the_events: - messagevars = await self._get_message_vars(notif, event, room_state_ids) + for filtered_event in the_events: + messagevars = await self._get_message_vars( + notif, filtered_event.event, room_state_ids + ) if messagevars is not None: ret["messages"].append(messagevars) diff --git a/synapse/replication/tcp/streams/__init__.py b/synapse/replication/tcp/streams/__init__.py index 067847617f..e41573cf68 100644 --- a/synapse/replication/tcp/streams/__init__.py +++ b/synapse/replication/tcp/streams/__init__.py @@ -39,6 +39,7 @@ PresenceStream, PushersStream, PushRulesStream, + QuarantinedMediaStream, ReceiptsStream, StickyEventsStream, Stream, @@ -73,6 +74,7 @@ ThreadSubscriptionsStream, UnPartialStatedRoomStream, UnPartialStatedEventStream, + QuarantinedMediaStream, ) } @@ -96,4 +98,5 @@ "ThreadSubscriptionsStream", "UnPartialStatedRoomStream", "UnPartialStatedEventStream", + "QuarantinedMediaStream", ] diff --git a/synapse/replication/tcp/streams/_base.py b/synapse/replication/tcp/streams/_base.py index 1ea6b4fa85..a73f767add 100644 --- a/synapse/replication/tcp/streams/_base.py +++ b/synapse/replication/tcp/streams/_base.py @@ -808,3 +808,50 @@ async def _update_function( return [], to_token, False return rows, rows[-1][0], len(updates) == limit + + +@attr.s(slots=True, auto_attribs=True) +class QuarantinedMediaStreamRow: + """Row for QuarantinedMediaStream""" + + # We store the origin and media_id as media is scoped to the origin and are uniquely + # identified by (origin, media_id). + + origin: str + media_id: str + quarantined: bool + + +class QuarantinedMediaStream(_StreamFromIdGen): + """Stream to track changes to (un)quarantined media.""" + + NAME = "quarantined_media" + ROW_TYPE = QuarantinedMediaStreamRow + + def __init__(self, hs: "HomeServer"): + self.store = hs.get_datastores().main + super().__init__( + hs.get_instance_name(), + self._update_function, + self.store._quarantined_media_changes_id_gen, + ) + + async def _update_function( + self, instance_name: str, from_token: Token, to_token: Token, limit: int + ) -> StreamUpdateResult: + updates = await self.store.get_quarantined_media_changes( + from_id=from_token, to_id=to_token, limit=limit + ) + rows = [ + ( + update.stream_id, + # Args to `QuarantinedMediaStreamRow` + (update.origin, update.media_id, update.quarantined), + ) + for update in updates + ] + + if not rows: + return [], to_token, False + + return rows, rows[-1][0], len(updates) == limit diff --git a/synapse/rest/admin/__init__.py b/synapse/rest/admin/__init__.py index 88de8b3f4a..8702fe056f 100644 --- a/synapse/rest/admin/__init__.py +++ b/synapse/rest/admin/__init__.py @@ -97,6 +97,10 @@ LargestRoomsStatistics, UserMediaStatisticsRestServlet, ) +from synapse.rest.admin.user_reports import ( + UserReportDetailRestServlet, + UserReportsRestServlet, +) from synapse.rest.admin.username_available import UsernameAvailableRestServlet from synapse.rest.admin.users import ( AccountDataRestServlet, @@ -313,6 +317,8 @@ def register_servlets(hs: "HomeServer", http_server: HttpServer) -> None: LargestRoomsStatistics(hs).register(http_server) EventReportDetailRestServlet(hs).register(http_server) EventReportsRestServlet(hs).register(http_server) + UserReportsRestServlet(hs).register(http_server) + UserReportDetailRestServlet(hs).register(http_server) AccountDataRestServlet(hs).register(http_server) PushersRestServlet(hs).register(http_server) MakeRoomAdminRestServlet(hs).register(http_server) diff --git a/synapse/rest/admin/background_updates.py b/synapse/rest/admin/background_updates.py index 96190c416d..e693a0afd3 100644 --- a/synapse/rest/admin/background_updates.py +++ b/synapse/rest/admin/background_updates.py @@ -18,6 +18,7 @@ # [This file includes modifications made by New Vector Limited] # # +import json import logging from http import HTTPStatus from typing import TYPE_CHECKING @@ -150,6 +151,24 @@ async def on_POST(self, request: SynapseRequest) -> tuple[int, JsonDict]: "populate_user_directory_process_users", ), ] + elif job_name == "event_resign": + old_key = body.get("old_key") + if old_key is not None and not isinstance(old_key, str): + raise SynapseError( + HTTPStatus.BAD_REQUEST, + "'old_key' must be a string", + ) + before_ts = body.get("before_ts") + if before_ts is not None and not isinstance(before_ts, int): + raise SynapseError( + HTTPStatus.BAD_REQUEST, + "'before_ts' must be an integer", + ) + progress = { + "old_key": old_key, + "before_ts": before_ts, + } + jobs = [("event_resign", json.dumps(progress), "")] else: raise SynapseError(HTTPStatus.BAD_REQUEST, "Invalid job_name") diff --git a/synapse/rest/admin/events.py b/synapse/rest/admin/events.py index 8da7a67820..1c311b0471 100644 --- a/synapse/rest/admin/events.py +++ b/synapse/rest/admin/events.py @@ -3,6 +3,7 @@ from synapse.api.errors import NotFoundError from synapse.events.utils import ( + FilteredEvent, SerializeEventConfig, format_event_raw, ) @@ -66,7 +67,9 @@ async def on_GET( ) res = { "event": await self._event_serializer.serialize_event( - event, self._clock.time_msec(), config=config + FilteredEvent.admin_override(event), + self._clock.time_msec(), + config=config, ) } diff --git a/synapse/rest/admin/media.py b/synapse/rest/admin/media.py index d5346fe0d5..1633cca884 100644 --- a/synapse/rest/admin/media.py +++ b/synapse/rest/admin/media.py @@ -230,6 +230,80 @@ async def on_POST( return HTTPStatus.OK, {} +class ListQuarantineChanges(RestServlet): + """Lists the quarantine changes to media. + + Uses the pagination format described by https://spec.matrix.org/v1.18/appendices/#pagination + """ + + PATTERNS = admin_patterns("/media/quarantine_changes$") + + def __init__(self, hs: "HomeServer"): + self.store = hs.get_datastores().main + self.auth = hs.get_auth() + self.server_name = hs.hostname + self.replication = hs.get_replication_data_handler() + + async def on_GET(self, request: SynapseRequest) -> tuple[int, JsonDict]: + await assert_requester_is_admin(self.auth, request) + + from_id = parse_integer(request, "from", default=0) + limit = 100 # arbitrary; not enough to cause problems (hopefully) + + # Validate the `from` token + max_id = await self.store.get_max_allocated_quarantined_media_stream_id() + if from_id > max_id: + # The caller is trying to get future data, which we don't allow because + # we know it's an invalid state that should never happen. We could + # wait until we reach the token but we might as well not waste our + # resources on that which is why `wait_for_quarantined_media_stream_id(...)` + # has assertions around this. + raise SynapseError( + HTTPStatus.BAD_REQUEST, + "The `from` token is considered invalid because it includes stream positions " + "greater than the furthest persisted position across all of the workers." + "This indicates either a Synapse programming error (as we should never hand out " + "invalid future tokens) or a fabricated `from` token. If you've modified the token, " + "you can try paginating from the beginning again.", + errcode=Codes.INVALID_PARAM, + ) + + # We need to wait to ensure that our current worker is actually caught up with + # the stream position, otherwise we might not return what we think we're returning. + if not await self.store.wait_for_quarantined_media_stream_id(from_id): + raise SynapseError( + HTTPStatus.INTERNAL_SERVER_ERROR, + "Timed out while waiting for the worker serving this request to catch up to the given " + "`from` stream position. Assuming this is a valid `from` token, this indicates an issue " + "with Synapse or the worker deployment lagging behind the replication stream. Please try " + "the request again later.", + errcode=Codes.UNKNOWN, + ) + + to_id = await self.store.get_current_quarantined_media_stream_id() + changes = await self.store.get_quarantined_media_changes( + from_id=from_id, + to_id=to_id, + limit=limit, + ) + + serialized_changes = [ + { + "origin": c.origin if c.origin is not None else self.server_name, + "media_id": c.media_id, + "quarantined": c.quarantined, + } + for c in changes + ] + + # We know the last record will have the highest stream ID, so use that one. If + # there aren't any records, just return the `to_id` value because it'll be the + # furthest stream position possible. + next_batch = changes[-1].stream_id if len(changes) > 0 else to_id + + return HTTPStatus.OK, {"next_batch": next_batch, "changes": serialized_changes} + + class ProtectMediaByID(RestServlet): """Protect local media from being quarantined.""" @@ -529,6 +603,7 @@ def register_servlets_for_media_repo(hs: "HomeServer", http_server: HttpServer) QuarantineMediaByID(hs).register(http_server) UnquarantineMediaByID(hs).register(http_server) QuarantineMediaByUser(hs).register(http_server) + ListQuarantineChanges(hs).register(http_server) ProtectMediaByID(hs).register(http_server) UnprotectMediaByID(hs).register(http_server) ListMediaInRoom(hs).register(http_server) diff --git a/synapse/rest/admin/rooms.py b/synapse/rest/admin/rooms.py index a886859ffa..61511b9360 100644 --- a/synapse/rest/admin/rooms.py +++ b/synapse/rest/admin/rooms.py @@ -29,6 +29,7 @@ from synapse.api.errors import AuthError, Codes, NotFoundError, SynapseError from synapse.api.filtering import Filter from synapse.events.utils import ( + FilteredEvent, SerializeEventConfig, ) from synapse.handlers.pagination import ( @@ -529,7 +530,9 @@ async def on_GET( ) events = await self.store.get_events(event_ids.values()) now = self.clock.time_msec() - room_state = await self._event_serializer.serialize_events(events.values(), now) + room_state = await self._event_serializer.serialize_events( + [FilteredEvent.state(e) for e in events.values()], now + ) ret = {"state": room_state} return HTTPStatus.OK, ret @@ -897,7 +900,8 @@ async def on_GET( bundle_aggregations=event_context.aggregations, ), "state": await self._event_serializer.serialize_events( - event_context.state, time_now + [FilteredEvent.state(e) for e in event_context.state], + time_now, ), "start": event_context.start, "end": event_context.end, diff --git a/synapse/rest/admin/user_reports.py b/synapse/rest/admin/user_reports.py new file mode 100644 index 0000000000..119dc86517 --- /dev/null +++ b/synapse/rest/admin/user_reports.py @@ -0,0 +1,173 @@ +# +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright (C) 2026 Element Creations, Ltd +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . +# + + +import logging +from http import HTTPStatus +from typing import TYPE_CHECKING + +from synapse.api.constants import Direction +from synapse.api.errors import Codes, NotFoundError, SynapseError +from synapse.http.servlet import RestServlet, parse_enum, parse_integer, parse_string +from synapse.http.site import SynapseRequest +from synapse.rest.admin._base import admin_patterns, assert_requester_is_admin +from synapse.types import JsonDict + +if TYPE_CHECKING: + from synapse.server import HomeServer + +logger = logging.getLogger(__name__) + + +class UserReportsRestServlet(RestServlet): + """ + List all reported users that are known to the homeserver. Results are returned + in a dictionary containing report information. Supports pagination. + The requester must have administrator access in Synapse. + + GET /_synapse/admin/v1/user_reports + returns: + 200 OK with list of reports if success otherwise an error. + + Args: + The parameters `from` and `limit` are required only for pagination. + By default, a `limit` of 100 is used. + The parameter `dir` can be used to define the order of results. + The `user_id` query parameter filters by the user ID of the reporter of the target user. + The `target_user_id` query parameter filters by user id of the target user. + Returns: + A list of user reprots and an integer representing the total number of user + reports that exist given this query + """ + + PATTERNS = admin_patterns("/user_reports$") + + def __init__(self, hs: "HomeServer"): + self._auth = hs.get_auth() + self._store = hs.get_datastores().main + + async def on_GET(self, request: SynapseRequest) -> tuple[int, JsonDict]: + await assert_requester_is_admin(self._auth, request) + + start = parse_integer(request, "from", default=0) + limit = parse_integer(request, "limit", default=100) + direction = parse_enum(request, "dir", Direction, Direction.BACKWARDS) + user_id = parse_string(request, "user_id") + target_user_id = parse_string(request, "target_user_id") + + if start < 0: + raise SynapseError( + HTTPStatus.BAD_REQUEST, + "The start parameter must be a positive integer.", + errcode=Codes.INVALID_PARAM, + ) + + if limit < 0: + raise SynapseError( + HTTPStatus.BAD_REQUEST, + "The limit parameter must be a positive integer.", + errcode=Codes.INVALID_PARAM, + ) + + user_reports, total = await self._store.get_user_reports_paginate( + start, limit, direction, user_id, target_user_id + ) + ret = {"user_reports": user_reports, "total": total} + if (start + limit) < total: + ret["next_token"] = start + len(user_reports) + + return HTTPStatus.OK, ret + + +class UserReportDetailRestServlet(RestServlet): + """ + Get a specific user report that is known to the homeserver. Results are returned + in a dictionary containing report information. + The requester must have administrator access in Synapse. + + GET /_synapse/admin/v1/user_reports/ + returns: + 200 OK with details report if success otherwise an error. + + Args: + The parameter `report_id` is the ID of the user report in the database. + Returns: + JSON blob of information about the user report + """ + + PATTERNS = admin_patterns("/user_reports/(?P[^/]*)$") + + def __init__(self, hs: "HomeServer"): + self._auth = hs.get_auth() + self._store = hs.get_datastores().main + + async def on_GET( + self, request: SynapseRequest, report_id: str + ) -> tuple[int, JsonDict]: + await assert_requester_is_admin(self._auth, request) + + message = ( + "The report_id parameter must be a string representing a positive integer." + ) + try: + resolved_report_id = int(report_id) + except ValueError: + raise SynapseError( + HTTPStatus.BAD_REQUEST, message, errcode=Codes.INVALID_PARAM + ) + + if resolved_report_id < 0: + raise SynapseError( + HTTPStatus.BAD_REQUEST, message, errcode=Codes.INVALID_PARAM + ) + + ret = await self._store.get_user_report(resolved_report_id) + if not ret: + raise NotFoundError("User report not found") + + id, received_ts, target_user_id, user_id, reason = ret + response = { + "id": id, + "received_ts": received_ts, + "target_user_id": target_user_id, + "user_id": user_id, + "reason": reason, + } + + return HTTPStatus.OK, response + + async def on_DELETE( + self, request: SynapseRequest, report_id: str + ) -> tuple[int, JsonDict]: + await assert_requester_is_admin(self._auth, request) + + message = ( + "The report_id parameter must be a string representing a positive integer." + ) + try: + resolved_report_id = int(report_id) + except ValueError: + raise SynapseError( + HTTPStatus.BAD_REQUEST, message, errcode=Codes.INVALID_PARAM + ) + + if resolved_report_id < 0: + raise SynapseError( + HTTPStatus.BAD_REQUEST, message, errcode=Codes.INVALID_PARAM + ) + + if await self._store.delete_user_report(resolved_report_id): + return HTTPStatus.OK, {} + + raise NotFoundError("User report not found") diff --git a/synapse/rest/client/auth.py b/synapse/rest/client/auth.py index f325499044..566c9c98c5 100644 --- a/synapse/rest/client/auth.py +++ b/synapse/rest/client/auth.py @@ -20,13 +20,13 @@ # import logging -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Optional from twisted.web.server import Request from synapse.api.auth.mas import MasDelegatedAuth from synapse.api.constants import LoginType -from synapse.api.errors import LoginError, SynapseError +from synapse.api.errors import Codes, LoginError, SynapseError from synapse.api.urls import CLIENT_API_PREFIX from synapse.http.server import HttpServer, respond_with_html, respond_with_redirect from synapse.http.servlet import RestServlet, parse_string @@ -61,12 +61,27 @@ def __init__(self, hs: "HomeServer"): hs.config.registration.registration_token_template ) self.success_template = hs.config.registration.fallback_success_template + self._msc4450_enabled = hs.config.experimental.msc4450_enabled async def on_GET(self, request: SynapseRequest, stagetype: str) -> None: session = parse_string(request, "session") if not session: raise SynapseError(400, "No session supplied") + # MSC4450 query parameter which allows clients to specify the Identity Provider + # they wish to use for legacy SSO during User-Interactive Authentication. + idp_id: Optional[str] = None + + if self._msc4450_enabled: + idp_id = parse_string(request, "io.element.idp_id") + + if idp_id is not None and stagetype != LoginType.SSO: + raise SynapseError( + 400, + Codes.INVALID_PARAM, + "idp_id can only be specified for the `m.login.sso` auth type", + ) + # We support the unstable (`org.matrix.cross_signing_reset`) name from MSC4312 until # enough clients have adopted the stable name (`m.oauth`). # Note: `org.matrix.cross_signing_reset` *is* the stable name of the *action* in the @@ -118,7 +133,7 @@ async def on_GET(self, request: SynapseRequest, stagetype: str) -> None: elif stagetype == LoginType.SSO: # Display a confirmation page which prompts the user to # re-authenticate with their SSO provider. - html = await self.auth_handler.start_sso_ui_auth(request, session) + html = await self.auth_handler.start_sso_ui_auth(request, session, idp_id) elif stagetype == LoginType.REGISTRATION_TOKEN: html = self.registration_token_template.render( diff --git a/synapse/rest/client/keys.py b/synapse/rest/client/keys.py index 89b68331f2..2c65a55ea1 100644 --- a/synapse/rest/client/keys.py +++ b/synapse/rest/client/keys.py @@ -159,6 +159,23 @@ class KeyObject(RequestBodyModel): device_keys: DeviceKeys | None = None """Identity keys for the device. May be absent if no new identity keys are required.""" + @field_validator("device_keys", mode="before") + @classmethod + def validate_device_keys_not_null(cls, v: Any) -> Any: + """Reject explicit `null` for `device_keys` while still allowing + the field to be omitted (in which case the default `None` is used). + + The spec says `device_keys` may be omitted, but when present it + must be a `DeviceKeys` object — not `null`. + + Pydantic's experimental `Missing` sentinel would be a cleaner way + to express this, but it's not stable yet: + https://docs.pydantic.dev/latest/concepts/experimental/#missing-sentinel + """ + if v is None: + raise ValueError("device_keys must not be null") + return v + fallback_keys: Mapping[StrictStr, StrictStr | KeyObject] | None = None """ The public key which should be used if the device's one-time keys are @@ -241,7 +258,7 @@ async def on_POST(self, request: SynapseRequest) -> tuple[int, JsonDict]: 400, "To upload keys, you must pass device_id when authenticating" ) - if "device_keys" in body and isinstance(body["device_keys"], dict): + if "device_keys" in body: # Validate the provided `user_id` and `device_id` fields in # `device_keys` match that of the requesting user. We can't do # this directly in the pydantic model as we don't have access @@ -249,13 +266,13 @@ async def on_POST(self, request: SynapseRequest) -> tuple[int, JsonDict]: # # TODO: We could use ValidationInfo when we switch to Pydantic v2. # https://docs.pydantic.dev/latest/concepts/validators/#validation-info - if body["device_keys"].get("user_id") != user_id: + if body["device_keys"]["user_id"] != user_id: raise SynapseError( code=HTTPStatus.BAD_REQUEST, errcode=Codes.BAD_JSON, msg="Provided `user_id` in `device_keys` does not match that of the authenticated user", ) - if body["device_keys"].get("device_id") != device_id: + if body["device_keys"]["device_id"] != device_id: raise SynapseError( code=HTTPStatus.BAD_REQUEST, errcode=Codes.BAD_JSON, diff --git a/synapse/rest/client/media.py b/synapse/rest/client/media.py index 4db3b01576..15f58acb95 100644 --- a/synapse/rest/client/media.py +++ b/synapse/rest/client/media.py @@ -253,6 +253,7 @@ async def on_GET( ), send_cors=True, ) + return set_cors_headers(request) set_corp_headers(request) diff --git a/synapse/rest/client/notifications.py b/synapse/rest/client/notifications.py index 2420e9fffb..f80a43b297 100644 --- a/synapse/rest/client/notifications.py +++ b/synapse/rest/client/notifications.py @@ -24,6 +24,7 @@ from synapse.api.constants import ReceiptTypes from synapse.events.utils import ( + FilteredEvent, SerializeEventConfig, format_event_for_client_v2_without_room_id, ) @@ -111,7 +112,7 @@ async def on_GET(self, request: SynapseRequest) -> tuple[int, JsonDict]: "ts": pa.received_ts, "event": ( await self._event_serializer.serialize_event( - notif_events[pa.event_id], + FilteredEvent(event=notif_events[pa.event_id], membership=None), now, config=serialize_options, ) diff --git a/synapse/rest/client/room.py b/synapse/rest/client/room.py index 65d9c130ef..83664814a6 100644 --- a/synapse/rest/client/room.py +++ b/synapse/rest/client/room.py @@ -53,6 +53,7 @@ from synapse.api.filtering import Filter from synapse.events.utils import ( EventClientSerializer, + FilteredEvent, SerializeEventConfig, format_event_for_client_v2, ) @@ -286,7 +287,7 @@ async def on_GET( if format == "event": event = await self._event_serializer.serialize_event( - data, + FilteredEvent.state(data), self.clock.time_msec(), config=SerializeEventConfig( event_format=format_event_for_client_v2, @@ -866,7 +867,9 @@ async def encode_messages_response( serialized_result[ "state" ] = await serialize_deps.event_serializer.serialize_events( - get_messages_result.state, time_now, config=serialize_options + [FilteredEvent.state(e) for e in get_messages_result.state], + time_now, + config=serialize_options, ) return serialized_result @@ -1172,7 +1175,7 @@ async def on_GET( config=serializer_options, ), "state": await self._event_serializer.serialize_events( - event_context.state, + [FilteredEvent.state(e) for e in event_context.state], time_now, config=serializer_options, ), diff --git a/synapse/rest/client/sync.py b/synapse/rest/client/sync.py index 710d097eab..c3cf0dc3c4 100644 --- a/synapse/rest/client/sync.py +++ b/synapse/rest/client/sync.py @@ -18,7 +18,6 @@ # [This file includes modifications made by New Vector Limited] # # -import itertools import logging from collections import defaultdict from typing import TYPE_CHECKING, Any, Mapping @@ -31,6 +30,7 @@ from synapse.api.presence import UserPresenceState from synapse.api.ratelimiting import Ratelimiter from synapse.events.utils import ( + FilteredEvent, SerializeEventConfig, format_event_for_client_v2_without_room_id, format_event_raw, @@ -448,7 +448,9 @@ async def encode_invited( invited = {} for room in rooms: invite = await self._event_serializer.serialize_event( - room.invite, time_now, config=serialize_options + FilteredEvent.state(event=room.invite), + time_now, + config=serialize_options, ) unsigned = dict(invite.get("unsigned", {})) invite["unsigned"] = unsigned @@ -484,7 +486,9 @@ async def encode_knocked( knocked = {} for room in rooms: knock = await self._event_serializer.serialize_event( - room.knock, time_now, config=serialize_options + FilteredEvent.state(event=room.knock), + time_now, + config=serialize_options, ) # Extract the `unsigned` key from the knock event. @@ -574,7 +578,7 @@ async def encode_room( state_events = state_dict.values() - for event in itertools.chain(state_events, timeline_events): + for event in state_events: # We've had bug reports that events were coming down under the # wrong room. if event.room_id != room.room_id: @@ -584,9 +588,21 @@ async def encode_room( room.room_id, event.room_id, ) + for filtered_event in timeline_events: + # We've had bug reports that events were coming down under the + # wrong room. + if filtered_event.event.room_id != room.room_id: + logger.warning( + "Event %r is under room %r instead of %r", + filtered_event.event.event_id, + room.room_id, + filtered_event.event.room_id, + ) serialized_state = await self._event_serializer.serialize_events( - state_events, time_now, config=serialize_options + [FilteredEvent.state(e) for e in state_events], + time_now, + config=serialize_options, ) serialized_timeline = await self._event_serializer.serialize_events( timeline_events, @@ -974,7 +990,7 @@ async def encode_rooms( ): serialized_required_state = ( await self.event_serializer.serialize_events( - room_result.required_state, + [FilteredEvent.state(e) for e in room_result.required_state], time_now, config=serialize_options, ) diff --git a/synapse/rest/client/versions.py b/synapse/rest/client/versions.py index 7bf4b12e8b..bb1711f2cf 100644 --- a/synapse/rest/client/versions.py +++ b/synapse/rest/client/versions.py @@ -206,6 +206,8 @@ async def on_GET(self, request: SynapseRequest) -> tuple[int, JsonDict]: "org.matrix.msc4354": self.config.experimental.msc4354_enabled, # MSC4380: Invite blocking "org.matrix.msc4380.stable": True, + # MSC4445: Sync timeline order + "org.matrix.msc4445.initial_sync_timeline_topological_ordering": True, }, }, ) diff --git a/synapse/state/__init__.py b/synapse/state/__init__.py index a92233c863..2f0e3f2c3e 100644 --- a/synapse/state/__init__.py +++ b/synapse/state/__init__.py @@ -37,7 +37,7 @@ from synapse.api.constants import EventTypes from synapse.api.room_versions import KNOWN_ROOM_VERSIONS, StateResolutionVersions -from synapse.events import EventBase +from synapse.events import EventBase, FrozenEventVMSC4242 from synapse.events.snapshot import ( EventContext, UnpersistedEventContext, @@ -239,31 +239,6 @@ async def compute_state_after_events( ) return await ret.get_state(self._state_storage_controller, state_filter) - async def get_current_user_ids_in_room( - self, room_id: str, latest_event_ids: StrCollection - ) -> set[str]: - """ - Get the users IDs who are currently in a room. - - Note: This is much slower than using the equivalent method - `DataStore.get_users_in_room` or `DataStore.get_users_in_room_with_profiles`, - so this should only be used when wanting the users at a particular point - in the room. - - Args: - room_id: The ID of the room. - latest_event_ids: Precomputed list of latest event IDs. Will be computed if None. - Returns: - Set of user IDs in the room. - """ - - assert latest_event_ids is not None - - logger.debug("calling resolve_state_groups from get_current_user_ids_in_room") - entry = await self.resolve_state_groups_for_events(room_id, latest_event_ids) - state = await entry.get_state(self._state_storage_controller, StateFilter.all()) - return await self.store.get_joined_user_ids_from_state(room_id, state) - async def get_hosts_in_room_at_events( self, room_id: str, event_ids: StrCollection ) -> frozenset[str]: @@ -303,7 +278,8 @@ async def calculate_context_info( membership events. `False` if `state_ids_before_event` is the full state. `None` when `state_ids_before_event` is not provided. In this case, the - flag will be calculated based on `event`'s prev events. + flag will be calculated based on `event`'s `prev_events` or `prev_state_events` + for state DAG rooms. state_group_before_event: the current state group at the time of event, if known Returns: @@ -337,7 +313,11 @@ async def calculate_context_info( # (This is slightly racy - the prev-events might get fixed up before we use # their states - but I don't think that really matters; it just means we # might redundantly recalculate the state for this event later.) - prev_event_ids = event.prev_event_ids() + prev_event_ids = frozenset( + event.prev_state_events + if isinstance(event, FrozenEventVMSC4242) + else event.prev_event_ids() + ) incomplete_prev_events = await self.store.get_partial_state_events( prev_event_ids ) @@ -355,7 +335,7 @@ async def calculate_context_info( entry = await self.resolve_state_groups_for_events( event.room_id, - event.prev_event_ids(), + prev_event_ids, await_full_state=False, ) diff --git a/synapse/storage/controllers/persist_events.py b/synapse/storage/controllers/persist_events.py index 2948227807..7cc6a39639 100644 --- a/synapse/storage/controllers/persist_events.py +++ b/synapse/storage/controllers/persist_events.py @@ -35,6 +35,7 @@ Generic, Iterable, TypeVar, + cast, ) import attr @@ -43,7 +44,9 @@ from twisted.internet import defer from synapse.api.constants import EventTypes, Membership -from synapse.events import EventBase +from synapse.api.errors import SynapseError +from synapse.api.room_versions import KNOWN_ROOM_VERSIONS +from synapse.events import EventBase, FrozenEventVMSC4242, event_exists_in_state_dag from synapse.events.snapshot import EventContext, EventPersistencePair from synapse.handlers.worker_lock import NEW_EVENT_DURING_PURGE_LOCK_NAME from synapse.logging.context import PreserveLoggingContext, make_deferred_yieldable @@ -68,6 +71,7 @@ from synapse.types.state import StateFilter from synapse.util.async_helpers import ObservableDeferred, yieldable_gather_results from synapse.util.metrics import Measure +from synapse.util.stringutils import shortstr if TYPE_CHECKING: from synapse.server import HomeServer @@ -111,6 +115,14 @@ buckets=(0, 1, 2, 3, 5, 7, 10, 15, 20, 50, 100, 200, 500, "+Inf"), ) +# The number of forward extremities for each new event. +msc4242_state_dag_forward_extremities_counter = Histogram( + "synapse_storage_msc4242_state_dag_forward_extremities_persisted", + "Number of forward extremities for each new event in the state DAG", + labelnames=[SERVER_NAME_LABEL], + buckets=(1, 2, 3, 5, 7, 10, 15, 20, 50, 100, 200, 500, "+Inf"), +) + state_resolutions_during_persistence = Counter( "synapse_storage_events_state_resolutions_during_persistence", "Number of times we had to do state res to calculate new current state", @@ -529,7 +541,15 @@ async def _calculate_current_state(self, room_id: str) -> StateMap[str]: Returns: map from (type, state_key) to event id for the current state in the room """ - latest_event_ids = await self.main_store.get_latest_event_ids_in_room(room_id) + room_version = await self.main_store.get_room_version_id(room_id) + room_version_obj = KNOWN_ROOM_VERSIONS[room_version] + if room_version_obj.msc4242_state_dags: + latest_event_ids = await self.main_store.get_state_dag_extremities(room_id) + else: + latest_event_ids = await self.main_store.get_latest_event_ids_in_room( + room_id + ) + state_groups = set( ( await self.main_store._get_state_group_for_events(latest_event_ids) @@ -551,7 +571,6 @@ async def _calculate_current_state(self, room_id: str) -> StateMap[str]: # Avoid a circular import. from synapse.state import StateResolutionStore - room_version = await self.main_store.get_room_version_id(room_id) res = await self._state_resolution_handler.resolve_state_groups( room_id, room_version, @@ -615,28 +634,52 @@ async def _persist_event_batch( for x in range(0, len(events_and_contexts), 100) ] + # Get the room version for the first event. This room version is the same for all events + # as events_and_contexts is all for one room. + assert len(events_and_contexts) > 0 + room_version = events_and_contexts[0][0].room_version + for chunk in chunks: # We can't easily parallelize these since different chunks # might contain the same event. :( new_forward_extremities = None state_delta_for_room = None + new_state_dag_extrems = None if not backfilled: - with Measure( - self._clock, - name="_calculate_state_and_extrem", - server_name=self.server_name, - ): - # Work out the new "current state" for the room. - # We do this by working out what the new extremities are and then - # calculating the state from that. - ( - new_forward_extremities, - state_delta_for_room, - ) = await self._calculate_new_forward_extremities_and_state_delta( - room_id, chunk - ) + if room_version.msc4242_state_dags: + with Measure( + self._clock, + name="_process_state_dag_forward_extremities_and_state_delta", + server_name=self.server_name, + ): + assert all( + isinstance(ev, FrozenEventVMSC4242) for ev, _ in chunk + ) + ( + new_forward_extremities, # for prev_events + state_delta_for_room, # for state groups + new_state_dag_extrems, # for prev_state_events + ) = await self._process_state_dag_forward_extremities_and_state_delta( + room_id, + cast(list[tuple[FrozenEventVMSC4242, EventContext]], chunk), + ) + else: + with Measure( + self._clock, + name="_calculate_state_and_extrem", + server_name=self.server_name, + ): + # Work out the new "current state" for the room. + # We do this by working out what the new extremities are and then + # calculating the state from that. + ( + new_forward_extremities, + state_delta_for_room, + ) = await self._calculate_new_forward_extremities_and_state_delta( + room_id, chunk + ) with Measure( self._clock, @@ -666,6 +709,7 @@ async def _persist_event_batch( use_negative_stream_ordering=backfilled, inhibit_local_membership_updates=backfilled, new_event_links=new_event_links, + new_state_dag_forward_extremities=new_state_dag_extrems, ) return replaced_events @@ -793,6 +837,216 @@ async def _calculate_new_forward_extremities_and_state_delta( return (new_forward_extremities, delta) + async def _process_state_dag_forward_extremities_and_state_delta( + self, + room_id: str, + event_contexts: list[tuple[FrozenEventVMSC4242, EventContext]], + ) -> tuple[set[str] | None, DeltaState | None, set[str] | None]: + """Process the forwards extremities for state DAG rooms. + Returns: + - the new room dag extremities which should be written when these events are persisted. + - the state delta for the room, if applicable. + - the new state dag extremities which should be written when these events are persisted. + + NB: this does not write them because if it did, new events may see them _before_ the events + get persisted, causing failures in retrieving state groups. + """ + # Update forward extremities + # ...for the state DAG + existing_state_dag_fwd_extrems = ( + await self.main_store.get_state_dag_extremities(room_id) + ) + new_state_dag_fwd_extrems = await self._calculate_new_state_dag_extremities( + room_id, + existing_state_dag_fwd_extrems, + event_contexts, + ) + # ...and the room DAG + existing_room_dag_fwd_extrems = ( + await self.main_store.get_latest_event_ids_in_room(room_id) + ) + new_room_dag_fwd_extrems = await self._calculate_new_extremities( + room_id, + cast(list[EventPersistencePair], event_contexts), + existing_room_dag_fwd_extrems, + ) + assert new_room_dag_fwd_extrems, ( + f"No room dag forward extremities left in room {room_id}!" + ) + + # See if we need to calculate a state delta + if new_state_dag_fwd_extrems == existing_state_dag_fwd_extrems: + # No change in state extremities, so no new state to calculate + return new_room_dag_fwd_extrems, None, new_state_dag_fwd_extrems + + with Measure( + self._clock, + name="persist_events.state_dag.get_new_state_after_events", + server_name=self.server_name, + ): + (current_state, delta_ids, _) = await self._get_new_state_after_events( + room_id, + cast(list[EventPersistencePair], event_contexts), + existing_state_dag_fwd_extrems, + new_state_dag_fwd_extrems, + # do not prune forward extremities in the state DAG + # else we lose eventual delivery + should_prune=False, + ) + + # Following logic cargoculted from _calculate_new_forward_extremities_and_state_delta + # If either are not None then there has been a change, + # and we need to work out the delta (or use that + # given) + delta = None + if delta_ids is not None: + # If there is a delta we know that we've + # only added or replaced state, never + # removed keys entirely. + delta = DeltaState([], delta_ids) + elif current_state is not None: + with Measure( + self._clock, + name="persist_events.calculate_state_delta", + server_name=self.server_name, + ): + delta = await self._calculate_state_delta(room_id, current_state) + + if delta: + # If we have a change of state then lets check + # whether we're actually still a member of the room, + # or if our last user left. If we're no longer in + # the room then we delete the current state and + # extremities. + is_still_joined = await self._is_server_still_joined( + room_id, + cast(list[EventPersistencePair], event_contexts), + delta, + ) + if not is_still_joined: + logger.info("Server no longer in room %s", room_id) + delta.no_longer_in_room = True + + return new_room_dag_fwd_extrems, delta, new_state_dag_fwd_extrems + + async def _calculate_new_state_dag_extremities( + self, + room_id: str, + existing_fwd_extrems: frozenset[str], + event_contexts: list[tuple[FrozenEventVMSC4242, EventContext]], + ) -> set[str]: + """Calculate the new state dag forward extremities. Modifies existing_fwd_extrems. + + Assumes that event_contexts are only state events which should be in the state DAG. + + Raises: + SynapseError: if the new events include unknown prev_state_events + AssertionError: if there are no state DAG forward extremities remaining in the room + """ + # Events are always processed in causal order without any gaps in the DAG + # (prev_state_events are always known), guaranteeing that processed events have a path to the + # create event. This is an emergent property of state DAGs as asserting that there is a path + # to the create event every time we insert an event would be prohibitively expensive. + # This is similar to how doubly-linked lists can potentially not refer to previous items correctly + # without verifying the list's integrity, but doing it on every insert is too expensive. + + # filter out events which don't belong in the state dag. + new_state_events_contexts = [ + (e, ctx) for e, ctx in event_contexts if event_exists_in_state_dag(e) + ] + if len(new_state_events_contexts) == 0: + # if there are no state events being persisted, then the fwd extremities of the state dag + # do not change. + return set(existing_fwd_extrems) + + # This logic is very similar to _calculate_new_extremities with a few key differences: + # - We do not "Remove any events which are prev_events of any existing events." because the + # state DAG mandates that events are processed in causal order, so there MUST NOT be any + # existing, processed events which have the to-be-persisted events as prev_state_events. + # - We don't care if they are an "outlier" in the main room dag, so long as they AREN'T + # an outlier on the state dag, which this function checks, so we don't check outlier-ness. + # - We allow *soft-failed* events to become forward extremities, as per the MSC. We do not + # allow *rejected* events to become forward extremities though. + + rejected_events = [ev for ev, ctx in new_state_events_contexts if ctx.rejected] + new_state_events = [ + ev for ev, ctx in new_state_events_contexts if not ctx.rejected + ] + # We want to check that we are not missing any prev_state_events. + # To do this, we include rejected events in this check because other events may point to them. + # If we didn't include them, we might incorrectly say we are missing events when we are not. + all_new_state_events = set(rejected_events + new_state_events) + + # First, verify that we know all prev_state_events. If we fail this check then we don't have + # a complete DAG and that is bad, so bail out. + + # Start with them all missing. + missing_prev_state_events = { + e_id for event in all_new_state_events for e_id in event.prev_state_events + } + + # remove prev events which appear in all_events + missing_prev_state_events.difference_update( + event.event_id for event in all_new_state_events + ) + # the rest of these events should be present in the DB. Some of them may be forward extremities, + # some may not be, that's ok. + seen_events = await self.main_store.have_seen_events( + room_id, + missing_prev_state_events, + ) + missing_prev_state_events.difference_update(seen_events) + + if len(missing_prev_state_events) > 0: + logger.error( + "_calculate_new_state_dag_extremities: missing the following prev_state_events in room %s : %s", + room_id, + missing_prev_state_events, + ) + logger.error( + "_calculate_new_state_dag_extremities: was handling %s", + shortstr([ev.event_id for ev in all_new_state_events]), + ) + raise SynapseError( + code=500, + msg=f"missing {len(missing_prev_state_events)} prev_state_events in room {room_id}", + ) + + # Now calculate the forward extremities. + + # start with the existing forward extremities + result = set(existing_fwd_extrems) + + # add all the new events to the list + result.update(event.event_id for event in new_state_events) + + # Now remove all events which are prev_state_events of any of the new events + result.difference_update( + e_id for event in new_state_events for e_id in event.prev_state_events + ) + + # Finally handle the case where the new events have rejected/soft-failed `prev_state_events`. + # If they do we need to remove them and their `prev_state_events`, + # otherwise we end up with dangling extremities. + # Specifically, this handles the case where (F=fwd extrem, SF=soft-failed, N=new event) + # F <-- SF <-- SF <-- N + # where we want to remove F as a forward extremity and replace with N. + existing_prevs = await self.persist_events_store._get_prevs_before_rejected( + (e_id for event in new_state_events for e_id in event.prev_state_events), + include_soft_failed=False, + ) + result.difference_update(existing_prevs) + + # We only update metrics for events that change forward extremities + if result != existing_fwd_extrems: + msc4242_state_dag_forward_extremities_counter.labels( + **{SERVER_NAME_LABEL: self.server_name} + ).observe(len(result)) + + # There should always be at least one forward extremity. + assert result, f"No state dag forward extremities left in room {room_id}!" + return result + async def _calculate_new_extremities( self, room_id: str, @@ -859,6 +1113,7 @@ async def _get_new_state_after_events( events_context: list[EventPersistencePair], old_latest_event_ids: AbstractSet[str], new_latest_event_ids: set[str], + should_prune: bool = True, ) -> tuple[StateMap[str] | None, StateMap[str] | None, set[str]]: """Calculate the current state dict after adding some new events to a room @@ -873,9 +1128,15 @@ async def _get_new_state_after_events( old_latest_event_ids: the old forward extremities for the room. - new_latest_event_ids : + new_latest_event_ids: the new forward extremities for the room. + should_prune: + if true, attempt to prune the forward extremities. + Pruning means we will not communicate some new events to other servers, + which can compromise eventual delivery, so graphs which are fully synchronised + e.g. state DAGs should not prune. + Returns: Returns a tuple of two state maps and a set of new forward extremities. @@ -1015,7 +1276,7 @@ async def _get_new_state_after_events( # If the returned state matches the state group of one of the new # forward extremities then we check if we are able to prune some state # extremities. - if res.state_group and res.state_group in new_state_groups: + if should_prune and res.state_group and res.state_group in new_state_groups: new_latest_event_ids = await self._prune_extremities( room_id, new_latest_event_ids, diff --git a/synapse/storage/databases/main/devices.py b/synapse/storage/databases/main/devices.py index e9ecf46411..8670d68f38 100644 --- a/synapse/storage/databases/main/devices.py +++ b/synapse/storage/databases/main/devices.py @@ -79,6 +79,19 @@ BG_UPDATE_REMOVE_DUP_OUTBOUND_POKES = "remove_dup_outbound_pokes" +# Background update name for adding an index on +# `device_lists_changes_in_room.inserted_ts`. +BG_UPDATE_ADD_INSERTED_TS_INDEX = "device_lists_changes_in_room_inserted_ts_idx" + + +# Prunes entries out of the `device_lists_changes_in_room` table that are more +# than this old. +PRUNE_DEVICE_LISTS_CHANGES_IN_ROOM_AGE = Duration(days=30) + +# The number of rows to delete at once when pruning old entries out of the +# `device_lists_changes_in_room` table. +PRUNE_DEVICE_LISTS_BATCH_SIZE = 1000 + class DeviceWorkerStore(RoomMemberWorkerStore, EndToEndKeyWorkerStore): _device_list_id_gen: MultiWriterIdGenerator @@ -194,6 +207,10 @@ def __init__( self.clock.looping_call( self._prune_old_outbound_device_pokes, Duration(hours=1) ) + self.clock.looping_call( + self._prune_device_lists_changes_in_room, + Duration(hours=1), + ) def process_replication_rows( self, stream_name: str, instance_name: str, token: int, rows: Iterable[Any] @@ -1143,6 +1160,35 @@ async def get_users_whose_devices_changed( The set of user_ids whose devices have changed since `from_key` (exclusive) until `to_key` (inclusive). """ + return { + user_id + for user_id, _ in await self.get_device_changes_for_users( + from_key, user_ids, to_key + ) + } + + @cancellable + async def get_device_changes_for_users( + self, + from_key: MultiWriterStreamToken, + user_ids: Collection[str], + to_key: MultiWriterStreamToken | None = None, + ) -> set[tuple[str, str]]: + """Get set of user/device ID tuple whose devices have changed since `from_key` that + are in the given list of user_ids. + + Args: + from_key: The minimum device lists stream token to query device list changes for, + exclusive. + user_ids: If provided, only check if these users have changed their device lists. + Otherwise changes from all users are returned. + to_key: The maximum device lists stream token to query device list changes for, + inclusive. If None then no upper limit is applied. + + Returns: + The set of user/device ID tuples whose devices have changed since `from_key` + (exclusive) until `to_key` (inclusive). + """ # Get set of users who *may* have changed. Users not in the returned # list have definitely not changed. user_ids_to_check = self._device_list_stream_cache.get_entities_changed( @@ -1156,18 +1202,18 @@ async def get_users_whose_devices_changed( if to_key is None: to_key = self.get_device_stream_token() - def _get_users_whose_devices_changed_txn( + def get_device_changes_for_users_txn( txn: LoggingTransaction, from_key: MultiWriterStreamToken, to_key: MultiWriterStreamToken, - ) -> set[str]: + ) -> set[tuple[str, str]]: sql = """ - SELECT user_id, stream_id, instance_name + SELECT user_id, device_id, stream_id, instance_name FROM device_lists_stream WHERE ? < stream_id AND stream_id <= ? AND %s """ - changes: set[str] = set() + changes: set[tuple[str, str]] = set() # Query device changes with a batch of users at a time for chunk in batch_iter(user_ids_to_check, 100): @@ -1179,8 +1225,8 @@ def _get_users_whose_devices_changed_txn( [from_key.stream, to_key.get_max_stream_pos()] + args, ) changes.update( - user_id - for (user_id, stream_id, instance_name) in txn + (user_id, device_id) + for (user_id, device_id, stream_id, instance_name) in txn if MultiWriterStreamToken.is_stream_position_in_range( low=from_key, high=to_key, @@ -1192,8 +1238,8 @@ def _get_users_whose_devices_changed_txn( return changes return await self.db_pool.runInteraction( - "get_users_whose_devices_changed", - _get_users_whose_devices_changed_txn, + "get_device_changes_for_users", + get_device_changes_for_users_txn, from_key, to_key, ) @@ -1699,17 +1745,22 @@ def get_devices_not_accessed_since_txn( return devices - @cached() - async def _get_min_device_lists_changes_in_room(self) -> int: - """Returns the minimum stream ID that we have entries for - `device_lists_changes_in_room` + def _get_max_pruned_device_lists_changes_in_room_txn( + self, txn: LoggingTransaction + ) -> int: + """Returns the maximum stream ID that has been pruned from + `device_lists_changes_in_room`. + + Any queries for stream IDs less than this value cannot be answered + completely, as the data has been deleted. """ - return await self.db_pool.simple_select_one_onecol( - table="device_lists_changes_in_room", + return self.db_pool.simple_select_one_onecol_txn( + txn, + table="device_lists_changes_in_room_max_pruned_stream_id", keyvalues={}, - retcol="COALESCE(MIN(stream_id), 0)", - desc="get_min_device_lists_changes_in_room", + retcol="stream_id", + allow_none=False, ) @cancellable @@ -1728,55 +1779,54 @@ async def get_device_list_changes_in_rooms( if not room_ids: return set() - min_stream_id = await self._get_min_device_lists_changes_in_room() - - # Return early if there are no rows to process in device_lists_changes_in_room - if min_stream_id > from_token.stream: - return None - changed_room_ids = self._device_list_room_stream_cache.get_entities_changed( room_ids, from_token.stream ) if not changed_room_ids: return set() - sql = """ - SELECT user_id, stream_id, instance_name - FROM device_lists_changes_in_room - WHERE {clause} AND stream_id > ? AND stream_id <= ? - """ - def _get_device_list_changes_in_rooms_txn( txn: LoggingTransaction, - chunk: list[str], - ) -> set[str]: - clause, args = make_in_list_sql_clause( - self.database_engine, "room_id", chunk + ) -> set[str] | None: + # Check if the from_token is too old (i.e. data has been pruned). + max_pruned_stream_id = ( + self._get_max_pruned_device_lists_changes_in_room_txn(txn) ) - args.append(from_token.stream) - args.append(to_token.get_max_stream_pos()) - - txn.execute(sql.format(clause=clause), args) - return { - user_id - for (user_id, stream_id, instance_name) in txn - if MultiWriterStreamToken.is_stream_position_in_range( - low=from_token, - high=to_token, - instance_name=instance_name, - pos=stream_id, + if max_pruned_stream_id > from_token.stream: + return None + + changes: set[str] = set() + + for chunk in batch_iter(changed_room_ids, 1000): + clause, args = make_in_list_sql_clause( + self.database_engine, "room_id", chunk ) - } - - changes = set() - for chunk in batch_iter(changed_room_ids, 1000): - changes |= await self.db_pool.runInteraction( - "get_device_list_changes_in_rooms", - _get_device_list_changes_in_rooms_txn, - chunk, - ) + args.append(from_token.stream) + args.append(to_token.get_max_stream_pos()) - return changes + sql = f""" + SELECT user_id, stream_id, instance_name + FROM device_lists_changes_in_room + WHERE {clause} AND stream_id > ? AND stream_id <= ? + """ + txn.execute(sql, args) + changes.update( + user_id + for (user_id, stream_id, instance_name) in txn + if MultiWriterStreamToken.is_stream_position_in_range( + low=from_token, + high=to_token, + instance_name=instance_name, + pos=stream_id, + ) + ) + + return changes + + return await self.db_pool.runInteraction( + "get_device_list_changes_in_rooms", + _get_device_list_changes_in_rooms_txn, + ) async def get_all_device_list_changes(self, from_id: int, to_id: int) -> set[str]: """Return the set of rooms where devices have changed since the given @@ -1785,46 +1835,66 @@ async def get_all_device_list_changes(self, from_id: int, to_id: int) -> set[str Will raise an exception if the given stream ID is too old. """ - min_stream_id = await self._get_min_device_lists_changes_in_room() - - if min_stream_id > from_id: - raise Exception("stream ID is too old") - - sql = """ - SELECT DISTINCT room_id FROM device_lists_changes_in_room - WHERE stream_id > ? AND stream_id <= ? - """ - def _get_all_device_list_changes_txn( txn: LoggingTransaction, - ) -> set[str]: + ) -> set[str] | None: + # Check if the from_token is too old (i.e. data has been pruned). + max_pruned_stream_id = ( + self._get_max_pruned_device_lists_changes_in_room_txn(txn) + ) + if max_pruned_stream_id > from_id: + logger.warning( + "Given stream ID is too old %d < %d", + from_id, + max_pruned_stream_id, + ) + return None + + sql = """ + SELECT DISTINCT room_id FROM device_lists_changes_in_room + WHERE stream_id > ? AND stream_id <= ? + """ + txn.execute(sql, (from_id, to_id)) return {room_id for (room_id,) in txn} - return await self.db_pool.runInteraction( + room_ids = await self.db_pool.runInteraction( "get_all_device_list_changes", _get_all_device_list_changes_txn, ) + if room_ids is None: + raise Exception(f"Given stream ID is too old {from_id}") + + return room_ids + async def get_device_list_changes_in_room( self, room_id: str, min_stream_id: int - ) -> Collection[tuple[str, str]]: + ) -> Collection[tuple[str, str]] | None: """Get all device list changes that happened in the room since the given stream ID. Returns: Collection of user ID/device ID tuples of all devices that have - changed - """ - - sql = """ - SELECT DISTINCT user_id, device_id FROM device_lists_changes_in_room - WHERE room_id = ? AND stream_id > ? + changed, or None if the given stream ID is too old and so a complete + list cannot be calculated. """ def get_device_list_changes_in_room_txn( txn: LoggingTransaction, - ) -> Collection[tuple[str, str]]: + ) -> Collection[tuple[str, str]] | None: + # Check if the from_token is too old (i.e. data has been pruned). + max_pruned_stream_id = ( + self._get_max_pruned_device_lists_changes_in_room_txn(txn) + ) + if max_pruned_stream_id > min_stream_id: + return None + + sql = """ + SELECT DISTINCT user_id, device_id FROM device_lists_changes_in_room + WHERE room_id = ? AND stream_id > ? + """ + txn.execute(sql, (room_id, min_stream_id)) return cast(Collection[tuple[str, str]], txn.fetchall()) @@ -2160,6 +2230,8 @@ def _add_device_outbound_room_poke_txn( encoded_context = json_encoder.encode(context) + now = self.clock.time_msec() + # The `device_lists_changes_in_room.stream_id` column matches the # corresponding `stream_id` of the update in the `device_lists_stream` # table, i.e. all rows persisted for the same device update will have @@ -2175,6 +2247,7 @@ def _add_device_outbound_room_poke_txn( "instance_name", "converted_to_destinations", "opentracing_context", + "inserted_ts", ), values=[ ( @@ -2186,6 +2259,7 @@ def _add_device_outbound_room_poke_txn( # We only need to calculate outbound pokes for local users not self.hs.is_mine_id(user_id), encoded_context, + now, ) for room_id in room_ids for device_id, stream_id in zip(device_ids, stream_ids) @@ -2401,6 +2475,154 @@ async def set_device_change_last_converted_pos( desc="set_device_change_last_converted_pos", ) + @wrap_as_background_process("prune_device_lists_changes_in_room") + async def _prune_device_lists_changes_in_room(self) -> None: + """Delete old entries out of the `device_lists_changes_in_room`, so that + the table doesn't grow indefinitely. + """ + + # Let's only do this pruning if the index on inserted_ts has been + # created, otherwise this query will be very inefficient. + has_index_been_created = ( + await self.db_pool.updates.has_completed_background_update( + BG_UPDATE_ADD_INSERTED_TS_INDEX + ) + ) + if not has_index_been_created: + return + + prune_before_ts = ( + self.clock.time_msec() - PRUNE_DEVICE_LISTS_CHANGES_IN_ROOM_AGE.as_millis() + ) + + # Get stream ID corresponding to the prune_before_ts timestamp. We can + # delete all rows with a stream ID less than or equal to this, as they + # will be older than the cutoff. + # + # Some rows will have a NULL inserted_ts (due to being inserted before + # the column was added), but we can assume that the timestamp will + # monotonically increase with stream ID, so we can safely ignore those + # rows when calculating the cutoff stream ID. This means that we may end + # up keeping some rows with a non-NULL inserted_ts that are older than + # the cutoff, but that's better than accidentally deleting rows that are + # newer than the cutoff. + cutoff_sql = """ + SELECT stream_id FROM device_lists_changes_in_room + WHERE inserted_ts <= ? AND inserted_ts IS NOT NULL + ORDER BY inserted_ts DESC + LIMIT 1 + """ + + def get_prune_before_stream_id_txn(txn: LoggingTransaction) -> int | None: + txn.execute(cutoff_sql, (prune_before_ts,)) + row = txn.fetchone() + return row[0] if row else None + + prune_before_stream_id = await self.db_pool.runInteraction( + "prune_device_lists_changes_in_room_get_stream_id", + get_prune_before_stream_id_txn, + ) + + if prune_before_stream_id is None: + return + + # Get the max stream ID in the table so we avoid deleting it. We need + # to keep the latest row so that we can calculate the maximum stream ID + # used. + max_stream_id = await self.db_pool.simple_select_one_onecol( + table="device_lists_changes_in_room", + keyvalues={}, + retcol="MAX(stream_id)", + desc="prune_device_lists_changes_in_room_get_max_stream_id", + ) + if prune_before_stream_id >= max_stream_id: + prune_before_stream_id = max_stream_id - 1 + + logger.debug( + "Pruning device_lists_changes_in_room before stream ID %d (timestamp %d)", + prune_before_stream_id, + prune_before_ts, + ) + + # Now delete all rows with stream_id less than the + # prune_before_stream_id. + # + # We also delete in batches to avoid massive churn when initially + # clearing out all the old entries. + # + # We set a minimum stream ID so that when we delete in batches the + # database doesn't have to scan through all the (dead) tuples that were just + # deleted to find the next batch to delete. + + # The minimum stream ID to delete in the next batch, c.f. comment above. + # We default to 0 here as that is less than all possible stream IDs. + min_stream_id = 0 + + def prune_device_lists_changes_in_room_txn(txn: LoggingTransaction) -> int: + nonlocal min_stream_id + + delete_sql = """ + DELETE FROM device_lists_changes_in_room + WHERE stream_id IN ( + SELECT stream_id FROM device_lists_changes_in_room + WHERE ? < stream_id AND stream_id <= ? + ORDER BY stream_id ASC + LIMIT ? + ) + RETURNING stream_id + """ + txn.execute( + delete_sql, + (min_stream_id, prune_before_stream_id, PRUNE_DEVICE_LISTS_BATCH_SIZE), + ) + + # We can't use rowcount as that is incorrect on SQLite when using + # RETURNING. + num_deleted = 0 + for row in txn: + num_deleted += 1 + min_stream_id = max(min_stream_id, row[0]) + + if num_deleted: + # Update the max pruned stream ID tracking table so that the + # safety check knows data up to this point has been deleted. + self.db_pool.simple_update_one_txn( + txn, + table="device_lists_changes_in_room_max_pruned_stream_id", + keyvalues={}, + updatevalues={"stream_id": min_stream_id}, + ) + + return num_deleted + + progress_num_rows_deleted = 0 + while True: + batch_deleted = await self.db_pool.runInteraction( + "prune_device_lists_changes_in_room", + prune_device_lists_changes_in_room_txn, + ) + + finished = batch_deleted < PRUNE_DEVICE_LISTS_BATCH_SIZE + + progress_num_rows_deleted += batch_deleted + + # Periodically report progress in the logs. We do this either when + # we've deleted a significant number of rows or when we've finished + # deleting all rows in this round. + if finished or progress_num_rows_deleted > 10000: + logger.info( + "Pruned %d rows from device_lists_changes_in_room", + progress_num_rows_deleted, + ) + progress_num_rows_deleted = 0 + + if finished: + break + + # Sleep for a short time to avoid hammering the database too much if + # there are a lot of rows to delete. + await self.clock.sleep(Duration(milliseconds=100)) + class DeviceBackgroundUpdateStore(SQLBaseStore): _instance_name: str @@ -2459,6 +2681,15 @@ def __init__( columns=["room_id", "stream_id"], ) + # Add indexes to speed up pruning of device_lists_changes_in_room + self.db_pool.updates.register_background_index_update( + BG_UPDATE_ADD_INSERTED_TS_INDEX, + index_name="device_lists_changes_in_room_inserted_ts_idx", + table="device_lists_changes_in_room", + columns=["inserted_ts"], + where_clause="inserted_ts IS NOT NULL", + ) + async def _drop_device_list_streams_non_unique_indexes( self, progress: JsonDict, batch_size: int ) -> int: diff --git a/synapse/storage/databases/main/event_federation.py b/synapse/storage/databases/main/event_federation.py index cc7083b605..415926eb0a 100644 --- a/synapse/storage/databases/main/event_federation.py +++ b/synapse/storage/databases/main/event_federation.py @@ -1493,6 +1493,15 @@ async def get_latest_event_ids_in_room(self, room_id: str) -> frozenset[str]: ) return frozenset(event_ids) + async def get_state_dag_extremities(self, room_id: str) -> frozenset[str]: + event_ids = await self.db_pool.simple_select_onecol( + table="msc4242_state_dag_forward_extremities", + keyvalues={"room_id": room_id}, + retcol="event_id", + desc="get_state_dag_extremities", + ) + return frozenset(event_ids) + async def get_min_depth(self, room_id: str) -> int | None: """For the given room, get the minimum depth we have seen for it.""" return await self.db_pool.runInteraction( diff --git a/synapse/storage/databases/main/events.py b/synapse/storage/databases/main/events.py index 941a5f9f3a..12c918eca6 100644 --- a/synapse/storage/databases/main/events.py +++ b/synapse/storage/databases/main/events.py @@ -48,7 +48,9 @@ from synapse.api.room_versions import RoomVersions from synapse.events import ( EventBase, + FrozenEventVMSC4242, StrippedStateEvent, + event_exists_in_state_dag, is_creator, relation_from_event, ) @@ -295,6 +297,7 @@ async def _persist_events_and_state_updates( new_event_links: dict[str, NewEventChainLinks], use_negative_stream_ordering: bool = False, inhibit_local_membership_updates: bool = False, + new_state_dag_forward_extremities: set[str] | None = None, ) -> None: """Persist a set of events alongside updates to the current state and forward extremities tables. @@ -315,6 +318,8 @@ async def _persist_events_and_state_updates( from being updated by these events. This should be set to True for backfilled events because backfilled events in the past do not affect the current local state. + new_state_dag_forward_extremities: A set of event IDs that are the new forward + extremities for the state DAG for this room. MSC4242 only. Returns: Resolves when the events have been persisted @@ -379,6 +384,7 @@ async def _persist_events_and_state_updates( new_forward_extremities=new_forward_extremities, new_event_links=new_event_links, sliding_sync_table_changes=sliding_sync_table_changes, + new_state_dag_forward_extremities=new_state_dag_forward_extremities, ) persist_event_counter.labels(**{SERVER_NAME_LABEL: self.server_name}).inc( len(events_and_contexts) @@ -962,8 +968,10 @@ def _get_events_which_are_prevs_txn( return results - async def _get_prevs_before_rejected(self, event_ids: Iterable[str]) -> set[str]: - """Get soft-failed ancestors to remove from the extremities. + async def _get_prevs_before_rejected( + self, event_ids: Iterable[str], include_soft_failed: bool = True + ) -> set[str]: + """Get soft-failed/rejected ancestors to remove from the extremities. Given a set of events, find all those that have been soft-failed or rejected. Returns those soft failed/rejected events and their prev @@ -976,7 +984,8 @@ async def _get_prevs_before_rejected(self, event_ids: Iterable[str]) -> set[str] Args: event_ids: Events to find prev events for. Note that these must have already been persisted. - + include_soft_failed: Soft-failed events are included in the search. If false, only + rejected events are included. Returns: The previous events. """ @@ -1016,7 +1025,7 @@ def _get_prevs_before_rejected_txn( continue soft_failed = db_to_json(metadata).get("soft_failed") - if soft_failed or rejected: + if (include_soft_failed and soft_failed) or rejected: to_recursively_check.append(prev_event_id) existing_prevs.add(prev_event_id) @@ -1038,6 +1047,7 @@ def _persist_events_txn( new_forward_extremities: set[str] | None, new_event_links: dict[str, NewEventChainLinks], sliding_sync_table_changes: SlidingSyncTableChanges | None, + new_state_dag_forward_extremities: set[str] | None = None, ) -> None: """Insert some number of room events into the necessary database tables. @@ -1146,6 +1156,11 @@ def _persist_events_txn( max_stream_order=max_stream_order, ) + if new_state_dag_forward_extremities: + self._set_state_dag_extremities_txn( + txn, room_id, new_state_dag_forward_extremities + ) + self._persist_transaction_ids_txn(txn, events_and_contexts) # Insert into event_to_state_groups. @@ -2475,6 +2490,29 @@ def _update_forward_extremities_txn( ], ) + def _set_state_dag_extremities_txn( + self, txn: LoggingTransaction, room_id: str, new_extrems: Collection[str] + ) -> None: + self.db_pool.simple_delete_txn( + txn, + table="msc4242_state_dag_forward_extremities", + keyvalues={ + "room_id": room_id, + }, + ) + self.db_pool.simple_insert_many_txn( + txn, + table="msc4242_state_dag_forward_extremities", + keys=("room_id", "event_id"), + values=[ + ( + room_id, + event_id, + ) + for event_id in new_extrems + ], + ) + @classmethod def _filter_events_and_contexts_for_duplicates( cls, events_and_contexts: list[EventPersistencePair] @@ -2859,6 +2897,12 @@ def _update_metadata_tables_txn( self._handle_event_relations(txn, event) + if event.room_version.msc4242_state_dags and event_exists_in_state_dag( + event + ): + assert isinstance(event, FrozenEventVMSC4242) + self._store_state_dag_edges(txn, event) + # Store the labels for this event. labels = event.content.get(EventContentFields.LABELS) if labels: @@ -2935,6 +2979,36 @@ def local_prefill() -> None: txn.async_call_after(external_prefill) txn.call_after(local_prefill) + def _store_state_dag_edges( + self, txn: LoggingTransaction, event: FrozenEventVMSC4242 + ) -> None: + # the create event has no edge but we still need to persist it as get_state_dag just + # yanks all rows in this table. It's a bit gross to store NULL as the prev_state_event_id + # though. + if len(event.prev_state_events) == 0 and event.type == EventTypes.Create: + self.db_pool.simple_insert_txn( + txn, + table="msc4242_state_dag_edges", + values={ + "room_id": event.room_id, + "event_id": event.event_id, + "prev_state_event_id": None, + }, + ) + return + assert len(event.prev_state_events) > 0 + self.db_pool.simple_upsert_many_txn( + txn, + table="msc4242_state_dag_edges", + key_names=["room_id", "event_id", "prev_state_event_id"], + key_values=[ + (event.room_id, event.event_id, prev_state_event) + for prev_state_event in event.prev_state_events + ], + value_names=(), + value_values=(), + ) + def _store_redaction(self, txn: LoggingTransaction, event: EventBase) -> None: assert event.redacts is not None self.db_pool.simple_upsert_txn( @@ -3028,7 +3102,7 @@ def _store_room_members_txn( event.event_id, event.internal_metadata.stream_ordering, event.state_key, - event.user_id, + event.sender, event.room_id, event.membership, non_null_str_or_none(event.content.get("displayname")), @@ -3456,7 +3530,13 @@ def _store_event_state_mappings_txn( """ state_groups = {} for event, context in events_and_contexts: - if event.internal_metadata.is_outlier(): + # state dag rooms allow outliers to have state, as `/get_missing_events` state dag events are nominally + # outliers (not present in the timeline) but do need state persisted so we can calculate + # what the auth_events are for the event. + if ( + not event.room_version.msc4242_state_dags + and event.internal_metadata.is_outlier() + ): # double-check that we don't have any events that claim to be outliers # *and* have partial state (which is meaningless: we should have no # state at all for an outlier) diff --git a/synapse/storage/databases/main/events_bg_updates.py b/synapse/storage/databases/main/events_bg_updates.py index 934cd157ca..d2623f0760 100644 --- a/synapse/storage/databases/main/events_bg_updates.py +++ b/synapse/storage/databases/main/events_bg_updates.py @@ -23,6 +23,8 @@ from typing import TYPE_CHECKING, cast import attr +from signedjson.key import decode_verify_key_base64, get_verify_key +from signedjson.sign import SignatureVerifyException, verify_signed_json from synapse.api.constants import ( MAX_DEPTH, @@ -31,7 +33,12 @@ RelationTypes, ) from synapse.api.room_versions import KNOWN_ROOM_VERSIONS +from synapse.crypto.event_signing import ( + event_needs_resigning, + resign_event, +) from synapse.events import EventBase, make_event_from_dict +from synapse.events.utils import prune_event_dict from synapse.storage._base import SQLBaseStore, db_to_json, make_in_list_sql_clause from synapse.storage.database import ( DatabasePool, @@ -39,6 +46,7 @@ LoggingTransaction, make_tuple_comparison_clause, ) +from synapse.storage.databases.main.cache import CacheInvalidationWorkerStore from synapse.storage.databases.main.events import ( SLIDING_SYNC_RELEVANT_STATE_SET, PersistEventsStore, @@ -48,6 +56,7 @@ ) from synapse.storage.databases.main.events_worker import ( DatabaseCorruptionError, + EventRedactBehaviour, InvalidEventError, ) from synapse.storage.databases.main.state_deltas import StateDeltasStore @@ -112,7 +121,9 @@ class _JoinedRoomStreamOrderingUpdate: most_recent_bump_stamp: int | None -class EventsBackgroundUpdatesStore(StreamWorkerStore, StateDeltasStore, SQLBaseStore): +class EventsBackgroundUpdatesStore( + StreamWorkerStore, StateDeltasStore, CacheInvalidationWorkerStore, SQLBaseStore +): def __init__( self, database: DatabasePool, @@ -346,6 +357,11 @@ def __init__( _BackgroundUpdates.FIXUP_MAX_DEPTH_CAP, self.fixup_max_depth_cap_bg_update ) + self.db_pool.updates.register_background_update_handler( + _BackgroundUpdates.EVENT_RESIGN, + self._resign_events, + ) + # We want this to run on the main database at startup before we start processing # events. # @@ -1370,7 +1386,7 @@ def _event_arbitrary_relations_txn(txn: LoggingTransaction) -> int: ) # Iterate the parent IDs and invalidate caches. - self._invalidate_cache_and_stream_bulk( # type: ignore[attr-defined] + self._invalidate_cache_and_stream_bulk( txn, self.get_relations_for_event, # type: ignore[attr-defined] { @@ -1381,7 +1397,7 @@ def _event_arbitrary_relations_txn(txn: LoggingTransaction) -> int: for r in relations_to_insert }, ) - self._invalidate_cache_and_stream_bulk( # type: ignore[attr-defined] + self._invalidate_cache_and_stream_bulk( txn, self.get_thread_summary, # type: ignore[attr-defined] {(r[1],) for r in relations_to_insert}, @@ -2713,6 +2729,177 @@ def redo_max_depth_bg_update_txn(txn: LoggingTransaction) -> tuple[bool, int]: return num_rooms + async def _resign_events(self, progress: dict, batch_size: int) -> int: + """Retroactively re-sign events signed with a different key than the + current signing key. + + Optional progress parameters: + old_key: If set, only re-sign events whose signature can be + verified with this key. Format: "algorithm:key_id base64key" + (e.g. "ed25519:my_old_key XGX0JRS2Af3be3k..."). + before_ts: If set, only re-sign events with a received_ts less + than this value (milliseconds since epoch). + """ + + # Read optional filter parameters from progress. These are set once + # when the job is created and preserved across batches. + old_key_str: str | None = progress.get("old_key") + before_ts: int | None = progress.get("before_ts") + + # Parse the old verify key if provided. + old_verify_key = None + if old_key_str is not None: + parts = old_key_str.split(" ", 1) + if len(parts) == 2: + key_id, key_base64 = parts + alg, _, version = key_id.partition(":") + old_verify_key = decode_verify_key_base64(alg, version, key_base64) + else: + raise ValueError( + f"Invalid old_key format: expected 'algorithm:version base64key', got {old_key_str!r}" + ) + + # Load the next set of candidate events to re-sign. + # Returns the event IDs and the highest stream position for those events. + # If no event IDs are returned, this signals the background update is complete. + def _fetch_next_events_txn( + txn: LoggingTransaction, + ) -> tuple[list[str], int]: + # Start from the minimum 32-bit integer to ensure we cover events + # with negative stream orderings (e.g. from backfill). + last_stream_pos: int = progress.get("last_stream_pos", -(1 << 31)) + + sql = """ + SELECT event_id, stream_ordering FROM events + WHERE stream_ordering > ? AND sender LIKE ? + """ + args: list[object] = [ + last_stream_pos, + f"%:{self.hs.hostname}", + ] + + if before_ts is not None: + sql += " AND received_ts < ?" + args.append(before_ts) + + sql += " ORDER BY stream_ordering ASC LIMIT ?" + args.append(batch_size) + + txn.execute(sql, args) + event_rows: list[tuple[str, int]] = txn.fetchall() + if not event_rows: + return [], last_stream_pos + + last_stream_pos = event_rows[-1][1] + return [row[0] for row in event_rows], last_stream_pos + + next_event_ids, max_stream_pos = await self.db_pool.runInteraction( + "_resign_events._fetch_next_events", + _fetch_next_events_txn, + ) + logger.debug( + "Resign[num_checking=%d,sp=%d]", len(next_event_ids), max_stream_pos + ) + + if not next_event_ids: + await self.db_pool.updates._end_background_update( + _BackgroundUpdates.EVENT_RESIGN + ) + return 0 + + next_events = await self.get_events_as_list( + next_event_ids, + redact_behaviour=EventRedactBehaviour.as_is, + ) + current_verify_key = get_verify_key(self.hs.signing_key) + + # Re-sign any events that need it. + # A list of event IDs and their newly signed event dicts. + resigned_events: list[tuple[str, JsonDict]] = [] + for event in next_events: + if not event_needs_resigning(event, self.hs.hostname, current_verify_key): + continue + + # If old_key is set, only re-sign events whose signature verifies + # with the provided old key. + if old_verify_key is not None: + old_key_id = f"{old_verify_key.alg}:{old_verify_key.version}" + server_sigs = event.signatures.get(self.hs.hostname, {}) + if old_key_id not in server_sigs: + # Event wasn't signed with this key ID at all, skip. + continue + + # Verify the signature is genuinely from this key. We prune + # first since signatures are computed over the redacted form. + pruned = prune_event_dict(event.room_version, event.get_pdu_json()) + try: + verify_signed_json(pruned, self.hs.hostname, old_verify_key) + except SignatureVerifyException: + # In this case, the key ID was right but the signature doesn't match + # the public key we had. We definitely need to log about this. + logger.warning( + "Event %s has a signature for key %s that does not " + "verify — skipping", + event.event_id, + old_key_id, + ) + continue + + event_dict = resign_event(event, self.hs.hostname, self.hs.signing_key) + resigned_events.append((event.event_id, event_dict)) + + # Atomically write the new stream pos progress with the new signatures, + # else we may update the pos and crash before writing the new + # signatures, thus not re-signing at all! + def _write_events_txn( + txn: LoggingTransaction, + events_to_write: list[tuple[str, JsonDict]], + max_stream_pos: int, + ) -> None: + if events_to_write: + self.db_pool.simple_update_many_txn( + txn, + "event_json", + key_names=["event_id"], + key_values=[[event_id] for event_id, _ in events_to_write], + value_names=["json"], + value_values=[ + [json_encoder.encode(event_dict)] + for _, event_dict in events_to_write + ], + ) + # Always update the progress even if we re-sign nothing. + self.db_pool.updates._background_update_progress_txn( + txn, + _BackgroundUpdates.EVENT_RESIGN, + progress={ + "last_stream_pos": max_stream_pos, + "old_key": old_key_str, + "before_ts": before_ts, + }, + ) + + # Invalidate the event cache for re-signed events so that other + # workers also pick up the new signatures. + for event_id, _ in events_to_write: + self.invalidate_get_event_cache_after_txn(txn, event_id) + self._send_invalidation_to_replication( + txn, "_get_event_cache", (event_id,) + ) + + await self.db_pool.runInteraction( + "_resign_events._write_events_txn", + _write_events_txn, + resigned_events, + max_stream_pos, + ) + + logger.info("Re-signed %d events", len(resigned_events)) + + # Even if we don't re-sign them, we need to let the background updater + # know we're still churning through the events. + return len(next_event_ids) + def _resolve_stale_data_in_sliding_sync_tables( txn: LoggingTransaction, diff --git a/synapse/storage/databases/main/purge_events.py b/synapse/storage/databases/main/purge_events.py index d55ea5cf7d..fe8079c201 100644 --- a/synapse/storage/databases/main/purge_events.py +++ b/synapse/storage/databases/main/purge_events.py @@ -71,6 +71,10 @@ # so must be deleted first. "sliding_sync_joined_rooms", "sliding_sync_membership_snapshots", + # Note: msc4242_state_dag_forward_extremities/edges have a foreign key to the `events` table + # so must be deleted first. + "msc4242_state_dag_forward_extremities", + "msc4242_state_dag_edges", "events", "federation_inbound_events_staging", "receipts_graph", diff --git a/synapse/storage/databases/main/room.py b/synapse/storage/databases/main/room.py index 7ac88e4c2a..a0c42082f0 100644 --- a/synapse/storage/databases/main/room.py +++ b/synapse/storage/databases/main/room.py @@ -21,6 +21,7 @@ # import logging +from dataclasses import dataclass from enum import Enum from typing import ( TYPE_CHECKING, @@ -44,6 +45,7 @@ from synapse.api.room_versions import RoomVersion, RoomVersions from synapse.config.homeserver import HomeServerConfig from synapse.events import EventBase +from synapse.replication.tcp.streams._base import QuarantinedMediaStream from synapse.replication.tcp.streams.partial_state import UnPartialStatedRoomStream from synapse.storage._base import ( db_to_json, @@ -58,8 +60,14 @@ from synapse.storage.databases.main.cache import CacheInvalidationWorkerStore from synapse.storage.types import Cursor from synapse.storage.util.id_generators import IdGenerator, MultiWriterIdGenerator -from synapse.types import JsonDict, RetentionPolicy, StrCollection, ThirdPartyInstanceID +from synapse.types import ( + JsonDict, + RetentionPolicy, + StrCollection, + ThirdPartyInstanceID, +) from synapse.util.caches.descriptors import cached, cachedList +from synapse.util.duration import Duration from synapse.util.json import json_encoder from synapse.util.stringutils import MXC_REGEX @@ -102,6 +110,14 @@ class RoomStats(LargestRoomStats): public: bool +@dataclass(frozen=True) +class QuarantinedMediaUpdate: + stream_id: int # for the quarantined_media_changes stream + origin: str + media_id: str + quarantined: bool + + class RoomSortOrder(Enum): """ Enum to define the sorting method used when returning rooms with get_rooms_paginate @@ -162,11 +178,169 @@ def __init__( writers=["master"], ) + self._can_write_quarantined_media_changes = ( + self._instance_name in hs.config.worker.writers.quarantined_media_changes + ) + + self._quarantined_media_changes_id_gen: MultiWriterIdGenerator = ( + MultiWriterIdGenerator( + db_conn=db_conn, + db=database, + notifier=hs.get_replication_notifier(), + stream_name=QuarantinedMediaStream.NAME, + server_name=self.server_name, + instance_name=self._instance_name, + tables=[("quarantined_media_changes", "instance_name", "stream_id")], + sequence_name="quarantined_media_id_seq", + writers=hs.config.worker.writers.quarantined_media_changes, + ) + ) + + # Register a background update to flag already-quarantined media in the quarantine + # media changes table. This is to populate the API endpoint which consumes the + # table with initial data that callers expect (namely, a list of currently + # quarantined media). + self.db_pool.updates.register_background_update_handler( + _BackgroundUpdates.FLAG_EXISTING_QUARANTINED_MEDIA, + self._flag_existing_quarantined_media, + ) + + async def _flag_existing_quarantined_media( + self, progress: JsonDict, batch_size: int + ) -> int: + """Background update function to flag existing already-quarantined media in + the new `quarantine_media_changes` table. + + This only flags quarantined media as the API which reads the table is only + concerned with *changes* to the quarantined state - media does not start as + quarantined, so if it's already quarantined then it has changed state at + some point. Media which isn't quarantined has not changed state (as far as + this function can tell). + + We don't know when the media was originally quarantined, so this inserts as if + the media was quarantined now and are processed in an arbitrary order. + + Further, due to lack of timestamp or history, media which was quarantined then + unquarantined will not be picked up by this background task. + + Function signature is as per `register_background_update_handler` requirements. + + Args: + progress: The progress dictionary from the background update. + batch_size: The number of rows to process in each batch. + + Returns: + The number of rows inserted. + """ + # Note: we actually process 2x the batch_size per batch because we use it twice: + # once for the local_media_repository table, and once for the remote_media_cache + # table. This is fine though - we're still doing the work in batches. + + # We track the progress for the local and remote tables separately just in case + # there are media ID collisions that would make a mess of `ORDER BY media_id + # LIMIT ?`. If there are collisions towards the end of the returned set, the LIMIT + # might cut off some rows and make a future call with `WHERE media_id > ?` miss + # them too. By tracking each table separately, we avoid this kind of issue. + last_local_media_id = progress.get("last_local_media_id", "") + last_remote_media_id = progress.get("last_remote_media_id", "") + last_remote_origin = progress.get("last_remote_origin", "") + + # The `ORDER BY` here would normally miss records if the admin (un)quarantined a + # record, but that doesn't affect the background update because we also insert + # into the stream table upon quarantine status changing. Worst case is the admin + # newly quarantines some media, adding a row to the stream table, then we run + # over it again in the background update, adding a second row. Duplicate rows are + # non-issues for us. + # + # Another similar issue is if Synapse is downgraded partway through the background + # update then has more media quarantined. If Synapse is later upgraded again, the + # media that was quarantined while downgraded will only be imported if the media + # IDs are ordered higher than the last processed media ID. Media that has a lower + # ID will be skipped by the background update. + # + # Note: Already-quarantined media is indicated by the `quarantined_by` field being + # non-null. We only want quarantined media, per docstring above. + # + # This background update is considered *best effort* for the reasons above. Data + # might be missing or duplicated, and that's just going to have to be okay. This + # is further reinforced by not all changes being captured by the table anyway. + # See https://github.com/element-hq/synapse/issues/19672 for more details. + def flag_quarantined(txn: LoggingTransaction) -> int: + # It doesn't matter which order we do these in, as long as we do both of them. + txn.execute( + """ + SELECT NULL AS media_origin, media_id + FROM local_media_repository + WHERE quarantined_by IS NOT NULL + AND media_id > ? + ORDER BY media_id + LIMIT ? + """, + (last_local_media_id, batch_size), + ) + local_media_result = cast(list[tuple[str | None, str]], txn.fetchall()) + if len(local_media_result) > 0: + self._insert_quarantine_changes_txn(txn, local_media_result, True) + + # We use a >= ? on the media origin to avoid missing records when media IDs + # collide between origins (the table's unique constraint is on `(media_origin, media_id)`). + # Filtering by `(media_origin, media_id)` also makes sure we're using an index. + txn.execute( + """ + SELECT media_origin, media_id + FROM remote_media_cache + WHERE quarantined_by IS NOT NULL + AND media_origin >= ? AND media_id > ? + ORDER BY media_origin, media_id + LIMIT ? + """, + (last_remote_origin, last_remote_media_id, batch_size), + ) + remote_media_result = cast(list[tuple[str | None, str]], txn.fetchall()) + if len(remote_media_result) > 0: + self._insert_quarantine_changes_txn(txn, remote_media_result, True) + + self.db_pool.updates._background_update_progress_txn( + txn, + _BackgroundUpdates.FLAG_EXISTING_QUARANTINED_MEDIA, + { + "last_local_media_id": local_media_result[-1][1] + if len(local_media_result) > 0 + else last_local_media_id, + "last_remote_media_id": remote_media_result[-1][1] + if len(remote_media_result) > 0 + else last_remote_media_id, + "last_remote_origin": remote_media_result[-1][0] + if len(remote_media_result) > 0 + else last_remote_origin, + }, + ) + + return len(local_media_result) + len(remote_media_result) + + logger.info( + "Flagging existing quarantined media with local offset %s and remote offset %s/%s", + last_local_media_id, + last_remote_origin, + last_remote_media_id, + ) + num_flagged = await self.db_pool.runInteraction( + "_flag_existing_quarantined_media.flag_quarantined", + flag_quarantined, + ) + if num_flagged <= 0: # probably never negative, but why trust computers? + await self.db_pool.updates._end_background_update( + _BackgroundUpdates.FLAG_EXISTING_QUARANTINED_MEDIA + ) + return num_flagged + def process_replication_position( self, stream_name: str, instance_name: str, token: int ) -> None: if stream_name == UnPartialStatedRoomStream.NAME: self._un_partial_stated_rooms_stream_id_gen.advance(instance_name, token) + elif stream_name == QuarantinedMediaStream.NAME: + self._quarantined_media_changes_id_gen.advance(instance_name, token) return super().process_replication_position(stream_name, instance_name, token) async def store_room( @@ -1128,6 +1302,188 @@ def _get_media_ids_by_user_txn( return local_media_ids + async def get_current_quarantined_media_stream_id(self) -> int: + """Gets the position of the quarantined media changes stream. + + Returns: + int - the current stream ID + """ + return self._quarantined_media_changes_id_gen.get_current_token() + + async def get_max_allocated_quarantined_media_stream_id(self) -> int: + """Gets the maximum allocated position of the quarantined media changes stream. + + Returns: + int - the maximum stream ID + """ + return await self._quarantined_media_changes_id_gen.get_max_allocated_token() + + async def wait_for_quarantined_media_stream_id(self, target_id: int) -> bool: + """Waits until the quarantined media changes stream reaches the given stream ID. + + See https://github.com/element-hq/synapse/pull/19644 for more details. + + TODO: Replace function and call sites with https://github.com/element-hq/synapse/pull/19644 + + Args: + target_id: The stream ID to wait for. + + Returns: + True when caught up to the target stream ID. + False when timing out while waiting. + """ + # We ideally would use something like `wait_for_stream_position` in the meantime, + # but that short circuits if the instance name matches the current instance name. + # Doing so means that if *another* writer is actually leading the to_id, then we'll + # assume that we're caught up when we aren't. + # + # NOTE: Because this is implemented to wait for stream positions by integer ID, + # we're technically waiting for *all* workers to catch up rather than just waiting + # for *our* worker to catch up. This is okay for now because the quarantined media + # stream should be pretty fast to update, and if it's not then the only thing we're + # affecting is an admin API that probably has a tool automatically retrying requests + # anyway. https://github.com/element-hq/synapse/pull/19644 does the waiting properly + # so this should be replaced by that (or similar). + + # Get the minimum shared position/ID across all workers + current_id = self._quarantined_media_changes_id_gen.get_current_token() + if current_id >= target_id: + return True # nothing to wait for: we're already caught up. + + # "This should never happen". Tokens we hand out via the API should exist. If they + # don't, then we're in a bad state and need to explode. + max_persisted_position = ( + await self._quarantined_media_changes_id_gen.get_max_allocated_token() + ) + assert max_persisted_position >= target_id, ( + f"Unable to wait for invalid future token (token={target_id} has positions " + f"ahead of our max persisted position={max_persisted_position})" + ) + + # Start waiting until we've caught up to the `stream_token` + start = self.clock.time_msec() + logged = False + while True: + # Like above, get the minimum shared ID across all workers + current_id = self._quarantined_media_changes_id_gen.get_current_token() + if current_id >= target_id: + return True + + now = self.clock.time_msec() + + # Timed out + if now - start > 10_000: + return False + + if not logged: + logger.info( + "Waiting for current token to reach %s; currently at %s", + target_id, + current_id, + ) + logged = True + + # TODO: be better + await self.clock.sleep(Duration(milliseconds=500)) + + async def get_quarantined_media_changes( + self, *, from_id: int, to_id: int, limit: int + ) -> list[QuarantinedMediaUpdate]: + """ + Get updates to quarantined media in stream ordering since `from_id`. + + Paginating forwards: from_id < x <= to_id, (ascending order) + + Args: + from_id: The starting stream ID (exclusive) + to_id: The ending stream ID (inclusive) + limit: The maximum number of rows to return + + Returns: + List of `QuarantinedMediaUpdate` update rows in stream ordering (ascending order). + + Raises: + SynapseError: If waiting for `to_id` took too long. + """ + if to_id < from_id: + # the to_id is behind the from_id, which means no results + return [] + + return await self.db_pool.runInteraction( + "get_quarantined_media_changes", + self._get_quarantined_media_changes_txn, + from_id, + to_id, + limit, + ) + + def _get_quarantined_media_changes_txn( + self, txn: LoggingTransaction, from_id: int, to_id: int, limit: int + ) -> list[QuarantinedMediaUpdate]: + txn.execute( + """ + SELECT stream_id, origin, media_id, quarantined + FROM quarantined_media_changes + WHERE ? < stream_id AND stream_id <= ? + ORDER BY stream_id ASC + LIMIT ? + """, + (from_id, to_id, limit), + ) + return [ + QuarantinedMediaUpdate( + stream_id=stream_id, + origin=origin, + media_id=media_id, + quarantined=quarantined, + ) + for stream_id, origin, media_id, quarantined in txn + ] + + def _insert_quarantine_changes_txn( + self, + txn: LoggingTransaction, + origins_and_media_ids: list[tuple[str | None, str]], + quarantined: bool, + ) -> None: + """Records media being (un)quarantined in the stream. + + Args: + txn (cursor) + origins_and_media_ids: The [origin, media_id] tuples to record. The origin + may be None if the media is local. + quarantined: Whether the media is being quarantined or unquarantined. + """ + assert self._can_write_quarantined_media_changes + medias_with_stream_ids = zip( + origins_and_media_ids, + self._quarantined_media_changes_id_gen.get_next_mult_txn( + txn, len(origins_and_media_ids) + ), + strict=True, + ) + self.db_pool.simple_insert_many_txn( + txn, + "quarantined_media_changes", + keys=( + "instance_name", + "stream_id", + "origin", + "media_id", + "quarantined", + ), + values=[ + ( + self._instance_name, + stream_id, + origin, + media_id, + quarantined, + ) + for (origin, media_id), stream_id in medias_with_stream_ids + ], + ) + def _quarantine_local_media_txn( self, txn: LoggingTransaction, @@ -1161,9 +1517,23 @@ def _quarantine_local_media_txn( if quarantined_by is not None: sql += " AND safe_from_quarantine = FALSE" + sql += " RETURNING media_id" + txn.execute(sql, [quarantined_by] + sql_many_clause_args) - # Note that a rowcount of -1 can be used to indicate no rows were affected. - total_media_quarantined += txn.rowcount if txn.rowcount > 0 else 0 + media_ids_affected = txn.fetchall() + # We use `len(media_ids_affected)` here and below because both queries may + # affect fewer or more rows than the `media_ids` input. For example, if the + # media_ids point to already-quarantined media, then nothing was updated. + # Similarly, the below query might find more media than the `media_ids` + # because it's searching for hashes instead. + total_media_quarantined += len(media_ids_affected) + if len(media_ids_affected) > 0: + # Flag media that was newly (un)quarantined in the changes table. + self._insert_quarantine_changes_txn( + txn, + [(None, media_id) for (media_id,) in media_ids_affected], + quarantined_by is not None, + ) # Update any media that was identified via hash. if hashes: @@ -1178,8 +1548,17 @@ def _quarantine_local_media_txn( if quarantined_by is not None: sql += " AND safe_from_quarantine = FALSE" + sql += " RETURNING media_id" + txn.execute(sql, [quarantined_by] + sql_many_clause_args) - total_media_quarantined += txn.rowcount if txn.rowcount > 0 else 0 + media_ids_affected = txn.fetchall() + total_media_quarantined += len(media_ids_affected) + if len(media_ids_affected) > 0: + self._insert_quarantine_changes_txn( + txn, + [(None, media_id) for (media_id,) in media_ids_affected], + quarantined_by is not None, + ) return total_media_quarantined @@ -1212,10 +1591,18 @@ def _quarantine_remote_media_txn( sql = f""" UPDATE remote_media_cache SET quarantined_by = ? - WHERE {sql_in_list_clause}""" + WHERE {sql_in_list_clause} + RETURNING media_origin, media_id""" txn.execute(sql, [quarantined_by] + sql_args) - total_media_quarantined += txn.rowcount if txn.rowcount > 0 else 0 + media_ids_affected = cast(list[tuple[str | None, str]], txn.fetchall()) + total_media_quarantined += len(media_ids_affected) + if len(media_ids_affected) > 0: + self._insert_quarantine_changes_txn( + txn, + media_ids_affected, + quarantined_by is not None, + ) if hashes: sql_many_clause_sql, sql_many_clause_args = make_in_list_sql_clause( @@ -1224,9 +1611,17 @@ def _quarantine_remote_media_txn( sql = f""" UPDATE remote_media_cache SET quarantined_by = ? - WHERE {sql_many_clause_sql}""" + WHERE {sql_many_clause_sql} + RETURNING media_origin, media_id""" txn.execute(sql, [quarantined_by] + sql_many_clause_args) - total_media_quarantined += txn.rowcount if txn.rowcount > 0 else 0 + media_ids_affected = cast(list[tuple[str | None, str]], txn.fetchall()) + total_media_quarantined += len(media_ids_affected) + if len(media_ids_affected) > 0: + self._insert_quarantine_changes_txn( + txn, + media_ids_affected, + quarantined_by is not None, + ) return total_media_quarantined @@ -1882,6 +2277,132 @@ async def delete_event_report(self, report_id: int) -> bool: return True + async def get_user_report( + self, report_id: int + ) -> tuple[int, int, str, str, str] | None: + """Retrieve a user report + + Args: + report_id: ID of user report in database + Returns: + JSON dict of information from a user report or None if the + report does not exist. + """ + + return await self.db_pool.simple_select_one( + table="user_reports", + keyvalues={"id": report_id}, + retcols=("id", "received_ts", "target_user_id", "user_id", "reason"), + allow_none=True, + desc="get_user_report", + ) + + async def get_user_reports_paginate( + self, + start: int, + limit: int, + direction: Direction = Direction.BACKWARDS, + user_id: str | None = None, + target_user_id: str | None = None, + ) -> tuple[list[JsonDict], int]: + """Retrieve a paginated list of user reports + + Args: + start: event offset to begin the query from + limit: number of rows to retrieve + direction: Whether to fetch the most recent first (backwards) or the + oldest first (forwards) + user_id: search for user_id of the reporter. Ignored if user_id is None + target_user_id: search for user_id of the target. Ignored if target_user_id is None + Returns: + Tuple of: + json list of user reports + total number of user reports matching the filter criteria + """ + + def _get_user_reports_paginate_txn( + txn: LoggingTransaction, + ) -> tuple[list[dict[str, Any]], int]: + filters = [] + args: list[object] = [] + + if user_id: + filters.append("user_id LIKE ?") + args.extend(["%" + user_id + "%"]) + if target_user_id: + filters.append("target_user_id LIKE ?") + args.extend(["%" + target_user_id + "%"]) + + if direction == Direction.BACKWARDS: + order = "DESC" + else: + order = "ASC" + + where_clause = "WHERE " + " AND ".join(filters) if len(filters) > 0 else "" + + sql = f""" + SELECT COUNT(*) as total_user_reports + FROM user_reports {where_clause} + """ + txn.execute(sql, args) + count = cast(tuple[int], txn.fetchone())[0] + + sql = f""" + SELECT + id, + received_ts, + target_user_id, + user_id, + reason + FROM user_reports + {where_clause} + ORDER BY received_ts {order} + LIMIT ? + OFFSET ? + """ + + args += [limit, start] + txn.execute(sql, args) + + user_reports = [] + for row in txn: + user_reports.append( + { + "id": row[0], + "received_ts": row[1], + "target_user_id": row[2], + "user_id": row[3], + "reason": row[4], + } + ) + + return user_reports, count + + return await self.db_pool.runInteraction( + "get_user_reports_paginate", _get_user_reports_paginate_txn + ) + + async def delete_user_report(self, report_id: int) -> bool: + """Remove a user report from database. + + Args: + report_id: Report to delete + + Returns: + Whether the report was successfully deleted or not. + """ + try: + await self.db_pool.simple_delete_one( + table="user_reports", + keyvalues={"id": report_id}, + desc="delete_user_report", + ) + except StoreError: + # Deletion failed because report does not exist + return False + + return True + async def set_room_is_public(self, room_id: str, is_public: bool) -> None: await self.db_pool.simple_update_one( table="rooms", @@ -2001,6 +2522,7 @@ class _BackgroundUpdates: REPLACE_ROOM_DEPTH_MIN_DEPTH = "replace_room_depth_min_depth" POPULATE_ROOMS_CREATOR_COLUMN = "populate_rooms_creator_column" ADD_ROOM_TYPE_COLUMN = "add_room_type_column" + FLAG_EXISTING_QUARANTINED_MEDIA = "flag_existing_quarantined_media" _REPLACE_ROOM_DEPTH_SQL_COMMANDS = ( diff --git a/synapse/storage/databases/main/state.py b/synapse/storage/databases/main/state.py index cfde107b48..87523e6f18 100644 --- a/synapse/storage/databases/main/state.py +++ b/synapse/storage/databases/main/state.py @@ -38,7 +38,7 @@ from synapse.api.constants import EventContentFields, EventTypes, Membership from synapse.api.errors import NotFoundError, UnsupportedRoomVersionError from synapse.api.room_versions import KNOWN_ROOM_VERSIONS, RoomVersion -from synapse.events import EventBase +from synapse.events import EventBase, EventMetadata from synapse.events.snapshot import EventContext from synapse.logging.opentracing import trace from synapse.replication.tcp.streams import UnPartialStatedEventStream @@ -78,16 +78,6 @@ class Sentinel: ROOM_UNKNOWN_SENTINEL = Sentinel() -@attr.s(slots=True, frozen=True, auto_attribs=True) -class EventMetadata: - """Returned by `get_metadata_for_events`""" - - room_id: str - event_type: str - state_key: str | None - rejection_reason: str | None - - def _retrieve_and_check_room_version(room_id: str, room_version_id: str) -> RoomVersion: v = KNOWN_ROOM_VERSIONS.get(room_version_id) if not v: diff --git a/synapse/storage/databases/main/sticky_events.py b/synapse/storage/databases/main/sticky_events.py index 38b84443df..eee6b92415 100644 --- a/synapse/storage/databases/main/sticky_events.py +++ b/synapse/storage/databases/main/sticky_events.py @@ -302,10 +302,14 @@ def insert_sticky_events_txn( Skips inserting events: - if they are considered spammy by the policy server; (unsure if correct, track: https://github.com/matrix-org/matrix-spec-proposals/pull/4354#discussion_r2727593350) + - if they are considered spammy by a Synapse spam checker module; - if they are rejected; - if they are outliers (they should be reconsidered for insertion when de-outliered); or - if they are not sticky (e.g. if the stickiness expired). + Note: Soft-failed sticky events ARE inserted, as their soft-failed status + could be re-evaluated later. + Skipping the insertion of these types of 'invalid' events is useful for performance reasons because they would fill up the table yet we wouldn't show them to clients anyway. @@ -321,7 +325,12 @@ def insert_sticky_events_txn( sticky_events: list[tuple[EventBase, int]] = [] for ev in events: # MSC: Note: policy servers and other similar antispam techniques still apply to these events. - if ev.internal_metadata.policy_server_spammy: + # We don't filter out soft-failed events altogether (in case they get re-evaluated later), + # so filter out `spam_checker_spammy` events specifically as we don't want to re-evaluate _those_ later. + if ( + ev.internal_metadata.policy_server_spammy + or ev.internal_metadata.spam_checker_spammy + ): continue # We shouldn't be passed rejected events, but if we do, we filter them out too. if ev.rejected_reason is not None: @@ -332,7 +341,7 @@ def insert_sticky_events_txn( sticky_duration = ev.sticky_duration() if sticky_duration is None: continue - # Calculate the end time as start_time + effecitve sticky duration + # Calculate the end time as start_time + effective sticky duration expires_at = min(ev.origin_server_ts, now_ms) + sticky_duration.as_millis() # Filter out already expired sticky events if expires_at <= now_ms: diff --git a/synapse/storage/engines/sqlite.py b/synapse/storage/engines/sqlite.py index 3b1b19c00e..316e188dc9 100644 --- a/synapse/storage/engines/sqlite.py +++ b/synapse/storage/engines/sqlite.py @@ -21,6 +21,7 @@ import platform import sqlite3 import struct +import sys import threading from typing import TYPE_CHECKING, Any, Mapping @@ -90,6 +91,20 @@ def on_new_connection(self, db_conn: "LoggingDatabaseConnection") -> None: # We need to import here to avoid an import loop. from synapse.storage.prepare_database import prepare_database + if sys.version_info >= (3, 12): + # Opportunistically disable the SQLITE_DBCONFIG_DEFENSIVE + # flag on the database, as some of our database migrations + # alter the schema and this is forbidden in defensive mode. + # + # This is only known to be necessary on macOS, though SQLite can + # in theory be configured to be defensive by default on any + # platform. + # + # The constant for this is only exposed in the sqlite3 module + # on Python >= 3.12. + assert isinstance(db_conn.conn, sqlite3.Connection) + db_conn.conn.setconfig(sqlite3.SQLITE_DBCONFIG_DEFENSIVE, False) + if self._is_in_memory: # In memory databases need to be rebuilt each time. Ideally we'd # reuse the same connection as we do when starting up, but that diff --git a/synapse/storage/schema/__init__.py b/synapse/storage/schema/__init__.py index c16e492cc7..b5fd2ddfa4 100644 --- a/synapse/storage/schema/__init__.py +++ b/synapse/storage/schema/__init__.py @@ -174,6 +174,7 @@ Changes in SCHEMA_VERSION = 94 - Add `recheck` column (boolean, default true) to the `redactions` table. + - MSC4242: Add state DAG tables. """ diff --git a/synapse/storage/schema/main/delta/94/03_device_lists_room_timestamp.sql b/synapse/storage/schema/main/delta/94/03_device_lists_room_timestamp.sql new file mode 100644 index 0000000000..b0ae1eaaf6 --- /dev/null +++ b/synapse/storage/schema/main/delta/94/03_device_lists_room_timestamp.sql @@ -0,0 +1,18 @@ +-- +-- This file is licensed under the Affero General Public License (AGPL) version 3. +-- +-- Copyright (C) 2025 Element Creations, Ltd +-- +-- This program is free software: you can redistribute it and/or modify +-- it under the terms of the GNU Affero General Public License as +-- published by the Free Software Foundation, either version 3 of the +-- License, or (at your option) any later version. +-- +-- See the GNU Affero General Public License for more details: +-- . + +ALTER TABLE device_lists_changes_in_room ADD COLUMN inserted_ts BIGINT; + +-- Add a background update to add index +INSERT INTO background_updates (ordering, update_name, progress_json) VALUES + (9403, 'device_lists_changes_in_room_inserted_ts_idx', '{}'); diff --git a/synapse/storage/schema/main/delta/94/03_quarantined_media_tracking.sql b/synapse/storage/schema/main/delta/94/03_quarantined_media_tracking.sql new file mode 100644 index 0000000000..0eb74d85d4 --- /dev/null +++ b/synapse/storage/schema/main/delta/94/03_quarantined_media_tracking.sql @@ -0,0 +1,46 @@ +-- +-- This file is licensed under the Affero General Public License (AGPL) version 3. +-- +-- Copyright (C) 2026 Element Creations Ltd +-- +-- This program is free software: you can redistribute it and/or modify +-- it under the terms of the GNU Affero General Public License as +-- published by the Free Software Foundation, either version 3 of the +-- License, or (at your option) any later version. +-- +-- See the GNU Affero General Public License for more details: +-- . + +-- Represents a stream of when media is quarantined and unquarantined. Note that it's possible for duplicate rows to +-- exist in this table. When the background update which backfills this table is running, it orders quarantined media +-- by `media_id`. This table is also populated when an admin newly quarantines a piece of media. If the admin happens +-- to newly quarantine media that hasn't yet been processed by the background update, both the admin's API call and the +-- background update will add a row to this table. If the background update has already progressed past the media's ID, +-- then only the admin's API call will add a row to this table. +-- +-- Note also that this table might not be inserted with all possible cases of media being quarantined. For example, if +-- media is quarantined by hash upon upload or URL preview, it might not show up here. See https://github.com/element-hq/synapse/issues/19672 +-- for more details. +-- +-- Overall, this table is very much intended to be *best effort* for media that was quarantined before the table existed. +CREATE TABLE quarantined_media_changes ( + -- Position in the quarantined media stream + stream_id INTEGER NOT NULL PRIMARY KEY, + + -- Name of the worker sending this (makes us compatible with multiple writers) + instance_name TEXT NOT NULL, + + -- Media origin. NULL if local media. + -- We store the origin and media_id as media is scoped to the origin and are uniquely identified by (origin, media_id). + origin TEXT NULL, + + -- Media ID at the origin. + media_id TEXT NOT NULL, + + -- True if quarantined at this position, false otherwise. + quarantined BOOLEAN NOT NULL +); + +-- Start the background update to populate existing quarantined media in the table. See update handler for more details. +INSERT INTO background_updates (ordering, update_name, progress_json) VALUES + (9305, 'flag_existing_quarantined_media', '{}'); diff --git a/synapse/storage/schema/main/delta/94/03_quarantined_media_tracking_seq.sql.postgres b/synapse/storage/schema/main/delta/94/03_quarantined_media_tracking_seq.sql.postgres new file mode 100644 index 0000000000..85f50ba8e7 --- /dev/null +++ b/synapse/storage/schema/main/delta/94/03_quarantined_media_tracking_seq.sql.postgres @@ -0,0 +1,18 @@ +-- +-- This file is licensed under the Affero General Public License (AGPL) version 3. +-- +-- Copyright (C) 2026 Element Creations Ltd +-- +-- This program is free software: you can redistribute it and/or modify +-- it under the terms of the GNU Affero General Public License as +-- published by the Free Software Foundation, either version 3 of the +-- License, or (at your option) any later version. +-- +-- See the GNU Affero General Public License for more details: +-- . + +CREATE SEQUENCE quarantined_media_id_seq; +-- Synapse streams start at 2, because the default position is 1 +-- so any item inserted at position 1 is ignored. +-- We have to use nextval not START WITH 2, see https://github.com/element-hq/synapse/issues/18712 +SELECT nextval('quarantined_media_id_seq'); diff --git a/synapse/storage/schema/main/delta/94/03_state_dag_fwd_extrems.sql b/synapse/storage/schema/main/delta/94/03_state_dag_fwd_extrems.sql new file mode 100644 index 0000000000..bc5c738ba5 --- /dev/null +++ b/synapse/storage/schema/main/delta/94/03_state_dag_fwd_extrems.sql @@ -0,0 +1,38 @@ +-- +-- This file is licensed under the Affero General Public License (AGPL) version 3. +-- +-- Copyright (C) 2026 Element Creations, Ltd +-- +-- This program is free software: you can redistribute it and/or modify +-- it under the terms of the GNU Affero General Public License as +-- published by the Free Software Foundation, either version 3 of the +-- License, or (at your option) any later version. +-- +-- See the GNU Affero General Public License for more details: +-- . + +CREATE TABLE IF NOT EXISTS msc4242_state_dag_forward_extremities( + -- we always expect the room to exist. If it gets removed, delete fwd extremities. + room_id TEXT NOT NULL REFERENCES rooms(room_id) ON DELETE CASCADE, + event_id TEXT NOT NULL REFERENCES events(event_id) ON DELETE CASCADE, + -- it doesn't make sense to reference the same event multiple times, and this uniqueness + -- index is also used to delete events once they are no longer forward extremities. + UNIQUE (event_id) +); +-- When creating events, we want to select all forward extremities for a room which this index helps with. +CREATE INDEX msc4242_state_dag_room ON msc4242_state_dag_forward_extremities(room_id); + + +CREATE TABLE IF NOT EXISTS msc4242_state_dag_edges( + -- Deleting the room deletes the state DAG. + room_id TEXT NOT NULL REFERENCES rooms(room_id) ON DELETE CASCADE, + -- the event IDs being referenced must exist (hence REFERENCES) and we do not want to accidentally delete + -- the event and create a hole in the state DAG. It is not possible for a state + -- DAG room to function with an holey DAG, so these events _cannot_ be purged. To purge them, the + -- entire room would need to be deleted. + event_id TEXT NOT NULL REFERENCES events(event_id), + -- one of the `prev_state_events` for this event ID. We must have it since we must have the entire state DAG. + -- can be NULL for the create event. + prev_state_event_id TEXT REFERENCES events(event_id) +); +CREATE UNIQUE INDEX msc4242_state_dag_edges_key ON msc4242_state_dag_edges(room_id, event_id, prev_state_event_id); diff --git a/synapse/storage/schema/main/delta/94/04_device_lists_changes_max_pruned.sql b/synapse/storage/schema/main/delta/94/04_device_lists_changes_max_pruned.sql new file mode 100644 index 0000000000..73836841da --- /dev/null +++ b/synapse/storage/schema/main/delta/94/04_device_lists_changes_max_pruned.sql @@ -0,0 +1,34 @@ +-- +-- This file is licensed under the Affero General Public License (AGPL) version 3. +-- +-- Copyright (C) 2026 Element Creations Ltd +-- +-- This program is free software: you can redistribute it and/or modify +-- it under the terms of the GNU Affero General Public License as +-- published by the Free Software Foundation, either version 3 of the +-- License, or (at your option) any later version. +-- +-- See the GNU Affero General Public License for more details: +-- . + + +-- Tracks the maximum stream_id that has been deleted (pruned) from the +-- device_lists_changes_in_room table. This is used to determine whether it's +-- safe to read from that table for a given stream_id — if the requested +-- stream_id is < the value here, the data has been pruned and the table cannot +-- provide a complete answer. +-- +-- We need a separate table, rather than looking at the minimum stream_id in the +-- device_lists_changes_in_room table, because not all valid stream IDs will +-- have entries in the table. This could lead to situations where the minimum +-- stream ID was potentially much more recent than when we actually pruned. This +-- would cause us to incorrectly think that the table was not safe to read from, +-- when in fact it was. +CREATE TABLE IF NOT EXISTS device_lists_changes_in_room_max_pruned_stream_id ( + Lock CHAR(1) NOT NULL DEFAULT 'X' UNIQUE, + stream_id BIGINT NOT NULL +); + +-- We assume that nothing has been deleted from the device_lists_changes_in_room +-- table, so we can set the initial value to 0. +INSERT INTO device_lists_changes_in_room_max_pruned_stream_id (stream_id) VALUES (0); diff --git a/synapse/synapse_rust/__init__.pyi b/synapse/synapse_rust/__init__.pyi index d25c609106..cb3eb7df07 100644 --- a/synapse/synapse_rust/__init__.pyi +++ b/synapse/synapse_rust/__init__.pyi @@ -1,3 +1,4 @@ def sum_as_string(a: int, b: int) -> str: ... def get_rust_file_digest() -> str: ... def reset_logging_config() -> None: ... +def get_rustc_version() -> str: ... diff --git a/synapse/synapse_rust/events.pyi b/synapse/synapse_rust/events.pyi index 73a9b75ca0..b3b4c3d14b 100644 --- a/synapse/synapse_rust/events.pyi +++ b/synapse/synapse_rust/events.pyi @@ -38,6 +38,19 @@ class EventInternalMetadata: policy_server_spammy: bool """whether the policy server indicated that this event is spammy""" + spam_checker_spammy: bool + """Whether a spam checker module indicated that this event is spammy + + Note that spam checkers also cause the event to be marked as soft-failed. + + This flags exists for two reasons: + 1. as debugging information + 2. to prevent the soft-failed re-evaluation of spammy events + (the re-evaluation behaviour originates from MSC4354 Sticky Events) + + Note that historical spammy events won't have this flag. + """ + txn_id: str """The transaction ID, if it was set when the event was created.""" delay_id: str @@ -47,6 +60,9 @@ class EventInternalMetadata: device_id: str """The device ID of the user who sent this event, if any.""" + # MSC4242 state dags + calculated_auth_event_ids: list[str] + def get_dict(self) -> JsonDict: ... def is_outlier(self) -> bool: ... def copy(self) -> "EventInternalMetadata": ... diff --git a/synapse/synapse_rust/room_versions.pyi b/synapse/synapse_rust/room_versions.pyi index 909e3a1c26..9bbb538f18 100644 --- a/synapse/synapse_rust/room_versions.pyi +++ b/synapse/synapse_rust/room_versions.pyi @@ -31,6 +31,8 @@ class EventFormatVersions: """MSC1884-style format: introduced for room v4""" ROOM_V11_HYDRA_PLUS: int """MSC4291 room IDs as hashes: introduced for room HydraV11""" + ROOM_VMSC4242: int + """MSC4242 state DAGs: adds prev_state_events, removes auth_events""" KNOWN_EVENT_FORMAT_VERSIONS: frozenset[int] @@ -113,6 +115,14 @@ class RoomVersion: rather than in codepoints. If true, this room version uses stricter event size validation.""" + msc4242_state_dags: bool + """MSC4242: State DAGs. Creates events with prev_state_events instead of auth_events and derives + state from it. Events are always processed in causal order without any gaps in the DAG + (prev_state_events are always known), guaranteeing that processed events have a path to the + create event. This is an emergent property of state DAGs as asserting that there is a path + to the create event every time we insert an event would be prohibitively expensive. + This is similar to how doubly-linked lists can potentially not refer to previous items correctly + without verifying the list's integrity, but doing it on every insert is too expensive.""" class RoomVersions: V1: RoomVersion @@ -132,6 +142,7 @@ class RoomVersions: MSC3757v11: RoomVersion HydraV11: RoomVersion V12: RoomVersion + MSC4242v12: RoomVersion class KnownRoomVersionsMapping(Mapping[str, RoomVersion]): def add_room_version(self, room_version: RoomVersion) -> None: ... diff --git a/synapse/types/handlers/sliding_sync.py b/synapse/types/handlers/sliding_sync.py index 694b3e1645..1a84bf1ff8 100644 --- a/synapse/types/handlers/sliding_sync.py +++ b/synapse/types/handlers/sliding_sync.py @@ -34,6 +34,7 @@ from synapse.api.constants import EventTypes from synapse.events import EventBase +from synapse.events.utils import FilteredEvent from synapse.types import ( DeviceListUpdates, JsonDict, @@ -185,7 +186,7 @@ class StrippedHero: # Should be empty for invite/knock rooms with `stripped_state` required_state: list[EventBase] # Should be empty for invite/knock rooms with `stripped_state` - timeline_events: list[EventBase] + timeline_events: list[FilteredEvent] bundled_aggregations: dict[str, "BundledAggregations"] | None # Optional because it's only relevant to invite/knock rooms stripped_state: list[JsonDict] diff --git a/synapse/types/storage/__init__.py b/synapse/types/storage/__init__.py index 992c36caba..6b857aeb0e 100644 --- a/synapse/types/storage/__init__.py +++ b/synapse/types/storage/__init__.py @@ -66,3 +66,5 @@ class _BackgroundUpdates: FIXUP_MAX_DEPTH_CAP = "fixup_max_depth_cap" REDACTIONS_RECHECK_BG_UPDATE = "redactions_recheck" + + EVENT_RESIGN = "event_resign" diff --git a/synapse/visibility.py b/synapse/visibility.py index 5ba2a14a24..fc3e9dfa49 100644 --- a/synapse/visibility.py +++ b/synapse/visibility.py @@ -31,14 +31,13 @@ from synapse.api.constants import ( EventTypes, - EventUnsignedContentFields, HistoryVisibility, JoinRules, Membership, ) from synapse.events import EventBase from synapse.events.snapshot import EventContext -from synapse.events.utils import clone_event, prune_event +from synapse.events.utils import FilteredEvent, prune_event from synapse.logging.opentracing import trace from synapse.storage.controllers import StorageControllers from synapse.storage.databases.main import DataStore @@ -82,7 +81,7 @@ async def filter_and_transform_events_for_client( is_peeking: bool = False, always_include_ids: frozenset[str] = frozenset(), filter_send_to_client: bool = True, -) -> list[EventBase]: +) -> list[FilteredEvent]: """ Check which events a user is allowed to see. If the user can see the event but its sender asked for their data to be erased, prune the content of the event. @@ -102,8 +101,8 @@ async def filter_and_transform_events_for_client( also be called to check whether a user can see the state at a given point. Returns: - The filtered events. The `unsigned` data is annotated with the membership state - of `user_id` at each event. + The filtered events, wrapped in FilteredEvent with the requesting user's + membership at each event annotated for use during serialization (MSC4115). """ # Filter out events that have been soft failed so that we don't relay them # to clients, unless they're a server admin and want that to happen. @@ -176,7 +175,7 @@ async def filter_and_transform_events_for_client( room_id ] = await storage.main.get_retention_policy_for_room(room_id) - def allowed(event: EventBase) -> EventBase | None: + def allowed(event: EventBase) -> FilteredEvent | None: state_after_event = event_id_to_state.get(event.event_id) filtered = _check_client_allowed_to_see_event( user_id=user_id, @@ -233,28 +232,9 @@ def allowed(event: EventBase) -> EventBase | None: else Membership.LEAVE ) - # Copy the event before updating the unsigned data: this shouldn't be persisted - # to the cache! - cloned = clone_event(filtered) - cloned.unsigned[EventUnsignedContentFields.MEMBERSHIP] = user_membership - if storage.main.config.experimental.msc4354_enabled: - sticky_duration = cloned.sticky_duration() - if sticky_duration: - now_ms = storage.main.clock.time_msec() - expires_at = ( - # min() ensures that the origin server can't lie about the time and - # send the event 'in the future', as that would allow them to exceed - # the 1 hour limit on stickiness duration. - min(cloned.origin_server_ts, now_ms) + sticky_duration.as_millis() - ) - if expires_at > now_ms: - cloned.unsigned[EventUnsignedContentFields.STICKY_TTL] = ( - expires_at - now_ms - ) - - return cloned + return FilteredEvent(event=filtered, membership=user_membership) - # Check each event: gives an iterable of None or (a modified) EventBase. + # Check each event: gives an iterable of None or a FilteredEvent. filtered_events = map(allowed, events) # Turn it into a list and remove None entries before returning. diff --git a/tests/crypto/test_event_signing.py b/tests/crypto/test_event_signing.py index 9cdc1604da..334ff64bc2 100644 --- a/tests/crypto/test_event_signing.py +++ b/tests/crypto/test_event_signing.py @@ -19,12 +19,23 @@ # # -from signedjson.key import decode_signing_key_base64 +from typing import TypedDict + +from signedjson.key import ( + decode_signing_key_base64, + generate_signing_key, + get_verify_key, +) from signedjson.types import SigningKey from synapse.api.room_versions import RoomVersions -from synapse.crypto.event_signing import add_hashes_and_signatures -from synapse.events import make_event_from_dict +from synapse.crypto.event_signing import ( + add_hashes_and_signatures, + event_needs_resigning, + resign_event, +) +from synapse.events import EventBase, make_event_from_dict +from synapse.types import JsonDict from tests import unittest @@ -107,3 +118,121 @@ def test_sign_message(self) -> None: "Ay4aj2b5oJ1k8INYZ9n3KnszCflM0emwcmQQ7vxpbdc" "Sv9bkJxIZdWX1IJllcZLq89+D3sSabE+vqPtZs9akDw", ) + + +class EventResigningTestCase(unittest.TestCase): + def setUp(self) -> None: + self.signing_key: SigningKey = decode_signing_key_base64( + KEY_ALG, KEY_VER, SIGNING_KEY_SEED + ) + + def test_resign(self) -> None: + event_dict: JsonDict = { + "content": {"body": "Here is the message content"}, + "event_id": "$fffff:" + HOSTNAME, + "origin_server_ts": 1000000, + "type": "m.room.message", + "room_id": "!r:" + HOSTNAME, + "sender": "@u:" + HOSTNAME, + "signatures": {}, + "unsigned": {"age_ts": 1000000}, + } + add_hashes_and_signatures( + RoomVersions.V1, event_dict, HOSTNAME, self.signing_key + ) + event = make_event_from_dict(event_dict) + self.assertIn(HOSTNAME, event.signatures) + self.assertIn(KEY_NAME, event.signatures[HOSTNAME]) + signature = event.signatures[HOSTNAME][KEY_NAME] + + # Re-sign with a different key + signing_key_2: SigningKey = generate_signing_key("2") + key_name_2 = "ed25519:2" + + resigned_event = resign_event(event, HOSTNAME, signing_key_2) + self.assertIn(HOSTNAME, resigned_event["signatures"]) + self.assertIn(key_name_2, resigned_event["signatures"][HOSTNAME]) + self.assertEqual( + len(resigned_event["signatures"][HOSTNAME]), 1 + ) # the previous signature was removed. + self.assertNotEqual( + signature, resigned_event["signatures"][HOSTNAME][key_name_2] + ) # different signatures + + # Repeat but with an event without any signatures. + event_dict = { + "content": {"body": "Here is the message content"}, + "event_id": "$fffff:" + HOSTNAME, + "origin_server_ts": 1000000, + "type": "m.room.message", + "room_id": "!r:" + HOSTNAME, + "sender": "@u:" + HOSTNAME, + "signatures": {}, + "unsigned": {"age_ts": 1000000}, + } + event = make_event_from_dict(event_dict) + resigned_event = resign_event(event, HOSTNAME, signing_key_2) + self.assertIn(HOSTNAME, resigned_event["signatures"]) + self.assertIn(key_name_2, resigned_event["signatures"][HOSTNAME]) + self.assertEqual(len(resigned_event["signatures"][HOSTNAME]), 1) + + def test_event_needs_resigning(self) -> None: + event_that_needs_resigning_dict: JsonDict = { + "content": {"body": "Here is the message content"}, + "event_id": "$fffff:" + HOSTNAME, + "origin_server_ts": 1000000, + "type": "m.room.message", + "room_id": "!r:" + HOSTNAME, + "sender": "@u:" + HOSTNAME, + "unsigned": {"age_ts": 1000000}, + } + internal_metadata: JsonDict = {} + event_that_needs_resigning = make_event_from_dict( + event_that_needs_resigning_dict, + RoomVersions.V1, + internal_metadata, + ) + self.assertEqual( + event_needs_resigning( + event_that_needs_resigning, HOSTNAME, get_verify_key(self.signing_key) + ), + True, + ) + + class TestCase(TypedDict): + name: str + event: EventBase + + events_that_dont_need_resigning: list[TestCase] = [ + { + "name": "sender domain isn't ours", + "event": make_event_from_dict( + {**event_that_needs_resigning_dict, "sender": "@u:somewhereelse"}, + RoomVersions.V1, + internal_metadata, + ), + }, + { + "name": "already signed with this key", + "event": make_event_from_dict( + { + **event_that_needs_resigning_dict, + "signatures": { + HOSTNAME: { + KEY_NAME: "thisisntchecked", + }, + }, + }, + RoomVersions.V1, + internal_metadata, + ), + }, + ] + for test_case in events_that_dont_need_resigning: + self.assertEqual( + event_needs_resigning( + test_case["event"], HOSTNAME, get_verify_key(self.signing_key) + ), + False, + test_case["name"], + ) diff --git a/tests/events/test_auto_accept_invites.py b/tests/events/test_auto_accept_invites.py index 72ade45758..e0ebdf0bca 100644 --- a/tests/events/test_auto_accept_invites.py +++ b/tests/events/test_auto_accept_invites.py @@ -380,7 +380,7 @@ async def test_ignore_invite_for_missing_user(self) -> None: join_updates, _ = sync_join(self, inviting_user_id) # Assert that the last event in the room was not a member event for the target user. self.assertEqual( - join_updates[0].timeline.events[-1].content["membership"], "invite" + join_updates[0].timeline.events[-1].event.content["membership"], "invite" ) @override_config( @@ -423,7 +423,7 @@ async def test_ignore_invite_for_deactivated_user(self) -> None: join_updates, b = sync_join(self, inviting_user_id) # Assert that the last event in the room was not a member event for the target user. self.assertEqual( - join_updates[0].timeline.events[-1].content["membership"], "invite" + join_updates[0].timeline.events[-1].event.content["membership"], "invite" ) @override_config( @@ -466,7 +466,7 @@ async def test_ignore_invite_for_suspended_user(self) -> None: join_updates, b = sync_join(self, inviting_user_id) # Assert that the last event in the room was not a member event for the target user. self.assertEqual( - join_updates[0].timeline.events[-1].content["membership"], "invite" + join_updates[0].timeline.events[-1].event.content["membership"], "invite" ) @override_config( @@ -509,7 +509,7 @@ async def test_ignore_invite_for_locked_user(self) -> None: join_updates, b = sync_join(self, inviting_user_id) # Assert that the last event in the room was not a member event for the target user. self.assertEqual( - join_updates[0].timeline.events[-1].content["membership"], "invite" + join_updates[0].timeline.events[-1].event.content["membership"], "invite" ) diff --git a/tests/events/test_utils.py b/tests/events/test_utils.py index af44b5dec1..12ef42866d 100644 --- a/tests/events/test_utils.py +++ b/tests/events/test_utils.py @@ -28,6 +28,7 @@ from synapse.api.room_versions import RoomVersions from synapse.events import EventBase, make_event_from_dict from synapse.events.utils import ( + FilteredEvent, PowerLevelsContent, SerializeEventConfig, _split_field, @@ -655,7 +656,7 @@ def serialize( ) -> JsonDict: return self.get_success( self._event_serializer.serialize_event( - ev, + FilteredEvent(event=ev, membership=None), 1479807801915, config=SerializeEventConfig( only_event_fields=fields, diff --git a/tests/federation/test_federation_server.py b/tests/federation/test_federation_server.py index cb24566f39..6ed9e96037 100644 --- a/tests/federation/test_federation_server.py +++ b/tests/federation/test_federation_server.py @@ -432,7 +432,14 @@ def _test_get_extremities_common(self, room_version: str) -> None: self.assertEqual(channel.json_body["error"], "Server is banned from room") self.assertEqual(channel.json_body["errcode"], "M_FORBIDDEN") - @parameterized.expand([(k,) for k in KNOWN_ROOM_VERSIONS.keys()]) + # FIXME: Exclude MSC4242 room versions whilst it lacks federation support + @parameterized.expand( + [ + (k,) + for k in KNOWN_ROOM_VERSIONS.keys() + if k != RoomVersions.MSC4242v12.identifier + ] + ) @override_config( {"use_frozen_dicts": True, "experimental_features": {"msc4370_enabled": True}} ) @@ -440,7 +447,14 @@ def test_get_extremities_with_frozen_dicts(self, room_version: str) -> None: """Test GET /extremities with USE_FROZEN_DICTS=True""" self._test_get_extremities_common(room_version) - @parameterized.expand([(k,) for k in KNOWN_ROOM_VERSIONS.keys()]) + # FIXME: Exclude MSC4242 room versions whilst it lacks federation support + @parameterized.expand( + [ + (k,) + for k in KNOWN_ROOM_VERSIONS.keys() + if k != RoomVersions.MSC4242v12.identifier + ] + ) @override_config( {"use_frozen_dicts": False, "experimental_features": {"msc4370_enabled": True}} ) @@ -573,12 +587,18 @@ def _test_send_join_common(self, room_version: str) -> None: @override_config({"use_frozen_dicts": True}) def test_send_join_with_frozen_dicts(self, room_version: str) -> None: """Test send_join with USE_FROZEN_DICTS=True""" + if room_version == RoomVersions.MSC4242v12.identifier: + # TODO: This room version doesn't work over federation in this PR. + return self._test_send_join_common(room_version) @parameterized.expand([(k,) for k in KNOWN_ROOM_VERSIONS.keys()]) @override_config({"use_frozen_dicts": False}) def test_send_join_without_frozen_dicts(self, room_version: str) -> None: """Test send_join with USE_FROZEN_DICTS=False""" + if room_version == RoomVersions.MSC4242v12.identifier: + # TODO: This room version doesn't work over federation in this PR. + return self._test_send_join_common(room_version) def test_send_join_partial_state(self) -> None: diff --git a/tests/handlers/test_admin.py b/tests/handlers/test_admin.py index 49bd3ba3f4..a368363d7e 100644 --- a/tests/handlers/test_admin.py +++ b/tests/handlers/test_admin.py @@ -81,7 +81,8 @@ def test_single_public_joined_room(self) -> None: # Check that the right number of events were written counter = Counter( - (event.type, getattr(event, "state_key", None)) for event in written_events + (event.event.type, getattr(event.event, "state_key", None)) + for event in written_events ) self.assertEqual(counter[(EventTypes.Message, None)], 2) self.assertEqual(counter[(EventTypes.Member, self.user1)], 1) @@ -119,7 +120,8 @@ def test_single_private_joined_room(self) -> None: # Check that the right number of events were written counter = Counter( - (event.type, getattr(event, "state_key", None)) for event in written_events + (event.event.type, getattr(event.event, "state_key", None)) + for event in written_events ) self.assertEqual(counter[(EventTypes.Message, None)], 1) self.assertEqual(counter[(EventTypes.Member, self.user1)], 1) @@ -151,7 +153,8 @@ def test_single_left_room(self) -> None: # Check that the right number of events were written counter = Counter( - (event.type, getattr(event, "state_key", None)) for event in written_events + (event.event.type, getattr(event.event, "state_key", None)) + for event in written_events ) self.assertEqual(counter[(EventTypes.Message, None)], 2) self.assertEqual(counter[(EventTypes.Member, self.user1)], 1) @@ -192,7 +195,8 @@ def test_single_left_rejoined_private_room(self) -> None: # Check that the right number of events were written counter = Counter( - (event.type, getattr(event, "state_key", None)) for event in written_events + (event.event.type, getattr(event.event, "state_key", None)) + for event in written_events ) self.assertEqual(counter[(EventTypes.Message, None)], 2) self.assertEqual(counter[(EventTypes.Member, self.user1)], 1) diff --git a/tests/handlers/test_appservice.py b/tests/handlers/test_appservice.py index 6336edb108..7f67d0ca5e 100644 --- a/tests/handlers/test_appservice.py +++ b/tests/handlers/test_appservice.py @@ -568,8 +568,8 @@ def test_match_interesting_room_members( # notify us because the interesting user is joined to the room where the # message was sent. self.assertEqual(service, interested_appservice) - self.assertEqual(events[0]["type"], "m.room.message") - self.assertEqual(events[0]["sender"], alice) + self.assertEqual(events[0].type, "m.room.message") + self.assertEqual(events[0].sender, alice) else: self.send_mock.assert_not_called() @@ -628,8 +628,8 @@ def test_application_services_receive_events_sent_by_interesting_local_user( # Events sent from an interesting local user should also be picked up as # interesting to the appservice. self.assertEqual(service, interested_appservice) - self.assertEqual(events[0]["type"], "m.room.message") - self.assertEqual(events[0]["sender"], alice) + self.assertEqual(events[0].type, "m.room.message") + self.assertEqual(events[0].sender, alice) def test_sending_read_receipt_batches_to_application_services(self) -> None: """Tests that a large batch of read receipts are sent correctly to diff --git a/tests/handlers/test_device.py b/tests/handlers/test_device.py index f99e3cd4a2..9e44b1dc1e 100644 --- a/tests/handlers/test_device.py +++ b/tests/handlers/test_device.py @@ -21,20 +21,42 @@ # from unittest import mock +from unittest.mock import AsyncMock, Mock, patch +import signedjson.key +from parameterized import parameterized +from signedjson.types import SigningKey + +from twisted.internet import defer from twisted.internet.defer import ensureDeferred from twisted.internet.testing import MemoryReactor -from synapse.api.constants import RoomEncryptionAlgorithms +from synapse.api.constants import EventTypes, JoinRules, RoomEncryptionAlgorithms from synapse.api.errors import NotFoundError, SynapseError +from synapse.api.room_versions import RoomVersions from synapse.appservice import ApplicationService +from synapse.crypto.event_signing import add_hashes_and_signatures +from synapse.events import EventBase, FrozenEventV3 +from synapse.federation.federation_client import SendJoinResult +from synapse.federation.transport.client import ( + StateRequestResponse, + TransportLayerClient, +) +from synapse.federation.units import Transaction from synapse.handlers.device import MAX_DEVICE_DISPLAY_NAME_LEN, DeviceWriterHandler from synapse.rest import admin from synapse.rest.client import devices, login, register from synapse.server import HomeServer from synapse.storage.databases.main.appservice import _make_exclusive_regex -from synapse.types import JsonDict, UserID, create_requester +from synapse.types import ( + JsonDict, + StateMap, + UserID, + create_requester, + get_domain_from_id, +) from synapse.util.clock import Clock +from synapse.util.duration import Duration from synapse.util.task_scheduler import TaskScheduler from tests import unittest @@ -581,3 +603,334 @@ def test_dehydrate_v2_and_fetch_events(self) -> None: self.assertTrue(len(res["next_batch"]) > 1) self.assertEqual(len(res["events"]), 1) self.assertEqual(res["events"][0]["content"]["body"], "foo") + + +@patch("synapse.crypto.keyring.Keyring.process_request", AsyncMock(return_value=None)) +class DeviceUnPartialStateTestCase(unittest.HomeserverTestCase): + """Tests that local device list changes during partial state are sent to + remote servers when the room un-partials.""" + + servlets = [ + admin.register_servlets, + login.register_servlets, + ] + + # The two remote servers to fake + REMOTE1_SERVER_NAME = "remote1" + REMOTE1_SERVER_SIGNATURE_KEY = signedjson.key.generate_signing_key("test") + REMOTE1_USER = f"@user:{REMOTE1_SERVER_NAME}" + + REMOTE2_SERVER_NAME = "remote2" + REMOTE2_SERVER_SIGNATURE_KEY = signedjson.key.generate_signing_key("test") + REMOTE2_USER = f"@user:{REMOTE2_SERVER_NAME}" + + def default_config(self) -> JsonDict: + config = super().default_config() + # Enable federation so that get_device_updates_by_remote works. + config["federation_sender_instances"] = ["master"] + return config + + def make_homeserver(self, reactor: MemoryReactor, clock: Clock) -> HomeServer: + # Mock the federation transport client to prevent actual network calls. + self.federation_transport_client = AsyncMock(TransportLayerClient) + + self.federation_transport_client.send_transaction.return_value = {} + + hs = self.setup_test_homeserver( + federation_transport_client=self.federation_transport_client, + ) + handler = hs.get_device_handler() + assert isinstance(handler, DeviceWriterHandler) + self.device_handler = handler + self.store = hs.get_datastores().main + + return hs + + def _build_public_room(self) -> StateMap[EventBase]: + """Build a public room DAG that has REMOTE1 in it""" + + room_id = f"!room:{self.REMOTE1_SERVER_NAME}" + room_version = RoomVersions.V10 + + events: list[EventBase] = [] + + # First we make the create event + create_event_dict: JsonDict = { + "auth_events": [], + "content": { + "creator": self.REMOTE1_USER, + "room_version": room_version.identifier, + }, + "depth": 0, + "origin_server_ts": 0, + "prev_events": [], + "room_id": room_id, + "sender": self.REMOTE1_USER, + "state_key": "", + "type": EventTypes.Create, + } + + add_hashes_and_signatures( + room_version, + create_event_dict, + self.REMOTE1_SERVER_NAME, + self.REMOTE1_SERVER_SIGNATURE_KEY, + ) + + create_event = FrozenEventV3(create_event_dict, room_version, {}, None) + events.append(create_event) + + room_version = self.hs.config.server.default_room_version + join_event_dict: JsonDict = { + "auth_events": [ + create_event.event_id, + ], + "content": {"membership": "join"}, + "depth": 1, + "origin_server_ts": 100, + "prev_events": [create_event.event_id], + "sender": self.REMOTE1_USER, + "state_key": self.REMOTE1_USER, + "room_id": room_id, + "type": EventTypes.Member, + } + add_hashes_and_signatures( + room_version, + join_event_dict, + self.hs.hostname, + self.hs.signing_key, + ) + join_event = FrozenEventV3(join_event_dict, room_version, {}, None) + events.append(join_event) + + # Then set the join rules to public + join_rules_event_dict: JsonDict = { + "auth_events": [create_event.event_id, join_event.event_id], + "content": {"join_rule": JoinRules.PUBLIC}, + "depth": 2, + "origin_server_ts": 200, + "prev_events": [join_event.event_id], + "room_id": room_id, + "sender": self.REMOTE1_USER, + "state_key": "", + "type": EventTypes.JoinRules, + } + + add_hashes_and_signatures( + room_version, + join_rules_event_dict, + self.REMOTE1_SERVER_NAME, + self.REMOTE1_SERVER_SIGNATURE_KEY, + ) + join_rules_event = FrozenEventV3(join_rules_event_dict, room_version, {}, None) + events.append(join_rules_event) + + return {(event.type, event.state_key): event for event in events} + + def _build_signed_join_event( + self, + room_id: str, + user: str, + signing_key: SigningKey, + state: StateMap[EventBase], + ) -> FrozenEventV3: + """Build a join event for the local user, signed by the local server.""" + + latest_event = max(state.values(), key=lambda e: e.depth) + + room_version = self.hs.config.server.default_room_version + join_event_dict: JsonDict = { + "auth_events": [ + state[(EventTypes.Create, "")].event_id, + state[(EventTypes.JoinRules, "")].event_id, + ], + "content": {"membership": "join"}, + "depth": latest_event.depth + 1, + "origin_server_ts": latest_event.origin_server_ts + 100, + "prev_events": [latest_event.event_id], + "sender": user, + "state_key": user, + "room_id": room_id, + "type": EventTypes.Member, + } + add_hashes_and_signatures( + room_version, + join_event_dict, + get_domain_from_id(user), + signing_key, + ) + return FrozenEventV3(join_event_dict, room_version, {}, None) + + @parameterized.expand([("not_pruned", False), ("pruned", True)]) + @patch( + "synapse.storage.databases.main.devices.PRUNE_DEVICE_LISTS_CHANGES_IN_ROOM_AGE", + Duration(minutes=1), + ) + def test_local_device_changes_sent_to_new_servers_on_un_partial_state( + self, _test_suffix: str, prune_device_lists_change_in_room: bool + ) -> None: + """When a room un-partials, local device list changes made during the + partial state period should be sent to remote servers that were NOT + known at the time of the partial join. + + We do this by creating a room with one remote server, partialling + joining it, then receiving a join event from a second remote server. The + second remote server should receive a device list update EDU for any + local device changes that happened during the partial state period. + + We parameterize this test over whether during the unpartial process we + prune the `device_list_changes_in_room` table, to check that the + unpartial process correctly handles the case. + """ + + local_user = self.register_user("alice", "password") + self.login("alice", "password") + + # Build the remote room's state events. + room_state = self._build_public_room() + + # Before joining, we mock out the federation endpoints that are used + # during the unpartial process, so that we can control when the + # unpartial process completes. + get_room_state_ids_deferred: defer.Deferred[JsonDict] = defer.Deferred() + get_room_state_deferred: defer.Deferred[StateRequestResponse] = defer.Deferred() + self.federation_transport_client.get_room_state_ids = Mock( + side_effect=[get_room_state_ids_deferred] + ) + self.federation_transport_client.get_room_state = Mock( + side_effect=[get_room_state_deferred] + ) + + # Now make the local server partially join the room. + room_id = room_state[(EventTypes.Create, "")].room_id + room_version = room_state[(EventTypes.Create, "")].room_version + + local_join_event = self._build_signed_join_event( + room_id, local_user, self.hs.signing_key, room_state + ) + + # Mock the federation client endpoints for the partial join. + mock_make_membership_event = AsyncMock( + return_value=(self.REMOTE1_SERVER_NAME, local_join_event, room_version) + ) + mock_send_join = AsyncMock( + return_value=SendJoinResult( + local_join_event, + self.REMOTE1_SERVER_NAME, + state=list(room_state.values()), + auth_chain=list(room_state.values()), + partial_state=True, + # Only REMOTE1_SERVER_NAME is known at join time. + servers_in_room={self.REMOTE1_SERVER_NAME}, + ) + ) + + fed_handler = self.hs.get_federation_handler() + fed_client = self.hs.get_federation_client() + with ( + patch.object( + fed_client, "make_membership_event", mock_make_membership_event + ), + patch.object(fed_client, "send_join", mock_send_join), + ): + self.get_success( + fed_handler.do_invite_join( + [self.REMOTE1_SERVER_NAME], room_id, local_user, {} + ) + ) + + # The room should now be in partial state. + self.assertTrue(self.get_success(self.store.is_partial_state_room(room_id))) + + # A local device change happens while the room is in partial state. + self.get_success( + self.store.add_device_change_to_streams( + local_user, ["NEW_DEVICE"], [room_id] + ) + ) + + if prune_device_lists_change_in_room: + # Add a device change for another room, as we won't prune the most + # recent change. + self.get_success( + self.store.add_device_change_to_streams( + "@other:user", ["device1"], ["!some:room"] + ) + ) + + # Now prune the device list changes for the room. This simulates the + # case where the unpartial process prunes the + # `device_list_changes_in_room` table before processing the device + # list changes. + self.reactor.advance(120) # Advance past the pruning threshold + self.get_success(self.store._prune_device_lists_changes_in_room()) + + # Assert we actually pruned the device list changes for the room. + room_ids = self.get_success( + self.store.db_pool.simple_select_onecol( + table="device_lists_changes_in_room", + keyvalues={}, + retcol="room_id", + ) + ) + self.assertCountEqual(room_ids, ["!some:room"]) + + # Join the second server + new_state = dict(room_state) + new_state[(EventTypes.Member, local_user)] = local_join_event + join_event_2 = self._build_signed_join_event( + room_id, + self.REMOTE2_USER, + self.REMOTE2_SERVER_SIGNATURE_KEY, + new_state, + ) + + self.get_success( + self.hs.get_federation_event_handler().on_receive_pdu( + self.REMOTE2_SERVER_NAME, join_event_2 + ) + ) + + # Some EDUs may get sent out immediately, such as presence updates. + # However, we only care about the device list update EDU sent by the + # unpartialling process. Let's wait a few seconds and reset the mock. + self.reactor.advance(5) + self.federation_transport_client.send_transaction.reset_mock() + + # We now unblock the unpartial processs by returning the room state and + # state ids. This should trigger the device list update to be sent to + # REMOTE2_SERVER_NAME. + self.federation_transport_client.get_room_state_ids.assert_called_once_with( + self.REMOTE1_SERVER_NAME, + room_id, + event_id=local_join_event.prev_event_ids()[0], + ) + + get_room_state_ids_deferred.callback( + { + "pdu_ids": [event.event_id for event in room_state.values()], + "auth_event_ids": [], + } + ) + get_room_state_deferred.callback( + StateRequestResponse( + state=list(room_state.values()), + auth_events=[], + ) + ) + + # The device list EDU isn't necessarily sent out immediately + self.reactor.advance(30) + + # Check that only one transaction was sent, and that it contains the + # device list update EDU for the new device to REMOTE2_SERVER_NAME. + self.federation_transport_client.send_transaction.assert_called_once() + args, _ = self.federation_transport_client.send_transaction.call_args + transaction: Transaction = args[0] + + self.assertEqual(transaction.destination, self.REMOTE2_SERVER_NAME) + self.assertEqual(len(transaction.edus), 1) + + edu = transaction.edus[0] + self.assertEqual(edu["edu_type"], "m.device_list_update") + self.assertEqual(edu["content"]["device_id"], "NEW_DEVICE") diff --git a/tests/handlers/test_federation.py b/tests/handlers/test_federation.py index 7085531548..e4a41cf1ae 100644 --- a/tests/handlers/test_federation.py +++ b/tests/handlers/test_federation.py @@ -19,7 +19,7 @@ # # import logging -from typing import Collection, cast +from typing import Collection from unittest import TestCase from unittest.mock import AsyncMock, Mock, patch @@ -140,7 +140,7 @@ def test_rejected_message_event_state(self) -> None: "content": {}, "room_id": room_id, "sender": "@yetanotheruser:" + OTHER_SERVER, - "depth": cast(int, join_event["depth"]) + 1, + "depth": join_event.depth + 1, "prev_events": [join_event.event_id], "auth_events": [], "origin_server_ts": self.clock.time_msec(), @@ -192,7 +192,7 @@ def test_rejected_state_event_state(self) -> None: "content": {}, "room_id": room_id, "sender": "@yetanotheruser:" + OTHER_SERVER, - "depth": cast(int, join_event["depth"]) + 1, + "depth": join_event.depth + 1, "prev_events": [join_event.event_id], "auth_events": [], "origin_server_ts": self.clock.time_msec(), diff --git a/tests/handlers/test_federation_event.py b/tests/handlers/test_federation_event.py index 3d856b9346..1aaa86e2e8 100644 --- a/tests/handlers/test_federation_event.py +++ b/tests/handlers/test_federation_event.py @@ -1121,7 +1121,9 @@ async def get_event( return {"pdus": [missing_event.get_pdu_json()]} async def get_room_state_ids( - destination: str, room_id: str, event_id: str + destination: str, + room_id: str, + event_id: str, ) -> JsonDict: self.assertEqual(destination, self.OTHER_SERVER_NAME) self.assertEqual(event_id, missing_event.event_id) @@ -1131,7 +1133,10 @@ async def get_room_state_ids( } async def get_room_state( - room_version: RoomVersion, destination: str, room_id: str, event_id: str + room_version: RoomVersion, + destination: str, + room_id: str, + event_id: str, ) -> StateRequestResponse: self.assertEqual(destination, self.OTHER_SERVER_NAME) self.assertEqual(event_id, missing_event.event_id) diff --git a/tests/handlers/test_sync.py b/tests/handlers/test_sync.py index 18ec2ca6b6..b9dee1c954 100644 --- a/tests/handlers/test_sync.py +++ b/tests/handlers/test_sync.py @@ -307,7 +307,7 @@ def test_ban_wins_race_with_join(self) -> None: self.assertEqual(len(alice_sync_result.joined), 1) self.assertEqual(alice_sync_result.joined[0].room_id, room_id) last_room_creation_event_id = ( - alice_sync_result.joined[0].timeline.events[-1].event_id + alice_sync_result.joined[0].timeline.events[-1].event.event_id ) # Eve, a ne'er-do-well, registers. @@ -402,7 +402,7 @@ def test_state_includes_changes_on_forks(self) -> None: ) ) last_room_creation_event_id = ( - initial_sync_result.joined[0].timeline.events[-1].event_id + initial_sync_result.joined[0].timeline.events[-1].event.event_id ) # Send a state event, and a regular event, both using the same prev ID @@ -437,7 +437,7 @@ def test_state_includes_changes_on_forks(self) -> None: self.assertEqual(room_sync.room_id, room_id) self.assertTrue(room_sync.timeline.limited) self.assertEqual( - [e.event_id for e in room_sync.timeline.events], + [e.event.event_id for e in room_sync.timeline.events], [e3_event, e4_event], ) self.assertEqual( @@ -476,7 +476,7 @@ def test_state_includes_changes_on_forks_when_events_excluded(self) -> None: ) ) last_room_creation_event_id = ( - initial_sync_result.joined[0].timeline.events[-1].event_id + initial_sync_result.joined[0].timeline.events[-1].event.event_id ) # Send a state event, and a regular event, both using the same prev ID @@ -521,7 +521,7 @@ def test_state_includes_changes_on_forks_when_events_excluded(self) -> None: self.assertEqual(room_sync.room_id, room_id) self.assertTrue(room_sync.timeline.limited) self.assertEqual( - [e.event_id for e in room_sync.timeline.events], + [e.event.event_id for e in room_sync.timeline.events], [e3_event], ) self.assertEqual( @@ -563,7 +563,7 @@ def test_state_includes_changes_on_long_lived_forks(self) -> None: ) ) last_room_creation_event_id = ( - initial_sync_result.joined[0].timeline.events[-1].event_id + initial_sync_result.joined[0].timeline.events[-1].event.event_id ) # Send a state event, and a regular event, both using the same prev ID @@ -593,7 +593,7 @@ def test_state_includes_changes_on_long_lived_forks(self) -> None: self.assertEqual(room_sync.room_id, room_id) self.assertTrue(room_sync.timeline.limited) self.assertEqual( - [e.event_id for e in room_sync.timeline.events], + [e.event.event_id for e in room_sync.timeline.events], [e3_event], ) @@ -632,7 +632,7 @@ def test_state_includes_changes_on_long_lived_forks(self) -> None: self.assertEqual(room_sync.room_id, room_id) self.assertFalse(room_sync.timeline.limited) self.assertEqual( - [e.event_id for e in room_sync.timeline.events], + [e.event.event_id for e in room_sync.timeline.events], [e4_event], ) @@ -701,7 +701,7 @@ def test_state_includes_changes_on_ungappy_syncs(self) -> None: ) ) last_room_creation_event_id = ( - initial_sync_result.joined[0].timeline.events[-1].event_id + initial_sync_result.joined[0].timeline.events[-1].event.event_id ) # Send a state event, and a regular event, both using the same prev ID @@ -728,7 +728,7 @@ def test_state_includes_changes_on_ungappy_syncs(self) -> None: room_sync = initial_sync_result.joined[0] self.assertEqual(room_sync.room_id, room_id) self.assertEqual( - [e.event_id for e in room_sync.timeline.events], + [e.event.event_id for e in room_sync.timeline.events], [e3_event], ) if self.use_state_after: @@ -757,7 +757,7 @@ def test_state_includes_changes_on_ungappy_syncs(self) -> None: self.assertEqual(room_sync.room_id, room_id) self.assertFalse(room_sync.timeline.limited) self.assertEqual( - [e.event_id for e in room_sync.timeline.events], + [e.event.event_id for e in room_sync.timeline.events], [e4_event, e5_event], ) @@ -855,7 +855,7 @@ def test_archived_rooms_do_not_include_state_after_leave( # The last three events in the timeline should be those leading up to the # leave self.assertEqual( - [e.event_id for e in sync_room_result.timeline.events[-3:]], + [e.event.event_id for e in sync_room_result.timeline.events[-3:]], [before_message_event, before_state_event, leave_event], ) @@ -947,7 +947,7 @@ async def _check_sigs_and_hash_for_pulled_events_and_fetch( ) event_ids = [] for event in sync_result.joined[0].timeline.events: - event_ids.append(event.event_id) + event_ids.append(event.event.event_id) self.assertNotIn(call_event.event_id, event_ids) # it will come down in a private room, though @@ -995,7 +995,7 @@ async def _check_sigs_and_hash_for_pulled_events_and_fetch( ) priv_event_ids = [] for event in private_sync_result.joined[0].timeline.events: - priv_event_ids.append(event.event_id) + priv_event_ids.append(event.event.event_id) self.assertIn(private_call_event.event_id, priv_event_ids) diff --git a/tests/media/test_html_preview.py b/tests/media/test_html_preview.py index d3f1e8833a..ddcbbee897 100644 --- a/tests/media/test_html_preview.py +++ b/tests/media/test_html_preview.py @@ -433,6 +433,28 @@ def test_twitter_tag(self) -> None: }, ) + def test_extended_opengraph(self) -> None: + """Ensure we pull in profile and article data from opengraph.""" + html = b""" + + + + + + """ + tree = decode_body(html, "http://example.com/test.html") + assert tree is not None + og = parse_html_to_open_graph(tree) + self.assertEqual( + og, + { + "og:title": None, + "og:description": "My description", + "profile:username": "myname", + "article:published_time": "2026-04-07T10:07:37Z", + }, + ) + def test_nested_nodes(self) -> None: """A body with some nested nodes. Tests that we iterate over children in the right order (and don't reverse the order of the text).""" diff --git a/tests/module_api/test_api.py b/tests/module_api/test_api.py index 12c8942bc8..f1b20a12ec 100644 --- a/tests/module_api/test_api.py +++ b/tests/module_api/test_api.py @@ -828,6 +828,24 @@ async def _on_logged_out_mock( # Ensure the pushers were deleted after the callback. self.assertEqual(len(self.hs.get_pusherpool().pushers[user_id].values()), 0) + def test_event_deprecated_methods(self) -> None: + """Test that deprecated methods on events are still functional.""" + user_id = self.register_user("user", "password") + tok = self.login("user", "password") + + room_id = self.helper.create_room_as(tok=tok) + + state = self.get_success( + self.hs.get_storage_controllers().state.get_current_state(room_id) + ) + create_event = state[(EventTypes.Create, "")] + + # `.user_id` is a deprecated alias for `.sender`. + self.assertEqual(create_event.user_id, user_id) + + # The event supports looking up keys via `__getitem__` although deprecated + self.assertEqual(create_event["room_id"], room_id) + class ModuleApiWorkerTestCase(BaseModuleApiTestCase, BaseMultiWorkerStreamTestCase): """For testing ModuleApi functionality in a multi-worker setup""" diff --git a/tests/module_api/test_spamchecker.py b/tests/module_api/test_spamchecker.py index 42ef969ce0..572f7bdbcf 100644 --- a/tests/module_api/test_spamchecker.py +++ b/tests/module_api/test_spamchecker.py @@ -12,22 +12,33 @@ # . # # +from http import HTTPStatus from typing import Literal from twisted.internet.testing import MemoryReactor -from synapse.api.constants import EventContentFields, EventTypes +from synapse.api.constants import ( + EventContentFields, + EventTypes, + Membership, +) +from synapse.api.room_versions import RoomVersions from synapse.config.server import DEFAULT_ROOM_VERSION +from synapse.events import make_event_from_dict +from synapse.module_api import EventBase from synapse.rest import admin, login, room, room_upgrade_rest_servlet from synapse.server import HomeServer from synapse.types import Codes, JsonDict from synapse.util.clock import Clock +from tests import unittest from tests.server import FakeChannel from tests.unittest import HomeserverTestCase class SpamCheckerTestCase(HomeserverTestCase): + """Tests for the spam checker module API.""" + servlets = [ room.register_servlets, admin.register_servlets, @@ -284,3 +295,178 @@ async def user_may_send_state_event( self.assertEqual(channel.code, 403) self.assertEqual(channel.json_body["errcode"], Codes.FORBIDDEN) + + +class FederatedEventSpamCheckMetadataTestCase(unittest.FederatingHomeserverTestCase): + servlets = [ + admin.register_servlets, + login.register_servlets, + room.register_servlets, + ] + + def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: + super().prepare(reactor, clock, hs) + self._module_api = hs.get_module_api() + self._store = hs.get_datastores().main + self._storage_controllers = hs.get_storage_controllers() + self._federation_event_handler = hs.get_federation_event_handler() + self._federation_server = hs.get_federation_server() + self._state_handler = hs.get_state_handler() + self._persistence_controller = hs.get_storage_controllers().persistence + + # Create a room + user1_id = self.register_user("user1", "pass") + user1_tok = self.login(user1_id, "pass") + self.room_id = self.helper.create_room_as( + user1_id, + tok=user1_tok, + is_public=True, + room_version=RoomVersions.V10.identifier, + ) + + # Prepare a join for the 'remote' user + state_map = self.get_success( + self._storage_controllers.state.get_current_state(self.room_id) + ) + forward_extremity_event_ids = self.get_success( + self.hs.get_datastores().main.get_latest_event_ids_in_room(self.room_id) + ) + self.remote_user_id = f"@remoteuser:{self.OTHER_SERVER_NAME}" + self.remote_user_join_event = make_event_from_dict( + self.add_hashes_and_signatures_from_other_server( + { + "room_id": self.room_id, + "sender": self.remote_user_id, + "state_key": self.remote_user_id, + "depth": 1000, + "origin_server_ts": 1, + "type": EventTypes.Member, + "content": {"membership": Membership.JOIN}, + "auth_events": [ + state_map[(EventTypes.Create, "")].event_id, + state_map[(EventTypes.JoinRules, "")].event_id, + ], + "prev_events": list(forward_extremity_event_ids), + } + ), + room_version=RoomVersions.V10, + ) + + # Send the join + self.get_success( + self._federation_event_handler.on_receive_pdu( + self.OTHER_SERVER_NAME, self.remote_user_join_event + ) + ) + + # Check the join made it to the 'local' view of the room + self.helper.get_event( + room_id=self.room_id, + event_id=self.remote_user_join_event.event_id, + tok=user1_tok, + expect_code=HTTPStatus.OK, + ) + + def test_federated_events_with_spam_checker_metadata(self) -> None: + """ + Simulates receiving spammy and non-spammy events over federation, + then checks their `spam_checker_spammy` flag is set properly. + """ + + async def check_event_for_spam(event: EventBase) -> Literal["NOT_SPAM"] | Codes: + if event.type == EventTypes.Message: + if "ham" not in event.content["body"]: + return Codes.FORBIDDEN + return "NOT_SPAM" + + # Register a spam checker callback that only allows messages with 'ham' + self._module_api.register_spam_checker_callbacks( + check_event_for_spam=check_event_for_spam + ) + + # Prepare a spammy and a non-spammy event. + forward_extremity_event_ids = self.get_success( + self._store.get_latest_event_ids_in_room(self.room_id) + ) + state_map = self.get_success( + self._storage_controllers.state.get_current_state(self.room_id) + ) + spammy_event = make_event_from_dict( + self.add_hashes_and_signatures_from_other_server( + { + "room_id": self.room_id, + "sender": self.remote_user_id, + "depth": 2000, + "origin_server_ts": 2, + "type": EventTypes.Message, + "content": {"body": "this is spam", "msgtype": "m.text"}, + "auth_events": [ + state_map[(EventTypes.Create, "")].event_id, + state_map[(EventTypes.JoinRules, "")].event_id, + state_map[(EventTypes.Member, self.remote_user_id)].event_id, + ], + "prev_events": list(forward_extremity_event_ids), + } + ), + room_version=RoomVersions.V10, + ) + non_spammy_event = make_event_from_dict( + self.add_hashes_and_signatures_from_other_server( + { + "room_id": self.room_id, + "sender": self.remote_user_id, + "depth": 2000, + "origin_server_ts": 2, + "type": EventTypes.Message, + "content": {"body": "delicious ham", "msgtype": "m.text"}, + "auth_events": [ + state_map[(EventTypes.Create, "")].event_id, + state_map[(EventTypes.JoinRules, "")].event_id, + state_map[(EventTypes.Member, self.remote_user_id)].event_id, + ], + "prev_events": list(forward_extremity_event_ids), + } + ), + room_version=RoomVersions.V10, + ) + + # Receive these events over federation + # We need to let the federation server have them because it will + # invoke `_check_sigs_and_hash` which invokes the spam checker. + self.get_success( + self._federation_server._handle_received_pdu( + self.OTHER_SERVER_NAME, spammy_event + ) + ) + self.get_success( + self._federation_server._handle_received_pdu( + self.OTHER_SERVER_NAME, non_spammy_event + ) + ) + + # Retrieve the events from the database + retrieved_spammy_event = self.get_success( + self._store.get_event(spammy_event.event_id, allow_rejected=True) + ) + retrieved_non_spammy_event = self.get_success( + self._store.get_event(non_spammy_event.event_id, allow_rejected=True) + ) + + # Assert the spammy flags (and soft-failed flags, for good measure) are set properly + self.assertTrue( + retrieved_spammy_event.internal_metadata.spam_checker_spammy, + "Spammy inbound event should be marked as spam_checker_spammy!", + ) + self.assertTrue( + retrieved_spammy_event.internal_metadata.is_soft_failed(), + "Spammy inbound event should be soft-failed.", + ) + + self.assertFalse( + retrieved_non_spammy_event.internal_metadata.spam_checker_spammy, + "Non-spammy inbound event should not be marked as spam_checker_spammy!", + ) + self.assertFalse( + retrieved_non_spammy_event.internal_metadata.is_soft_failed(), + "Non-spammy inbound event should not be soft-failed.", + ) diff --git a/tests/rest/admin/test_admin.py b/tests/rest/admin/test_admin.py index 77d824dcd8..b77a72ec4a 100644 --- a/tests/rest/admin/test_admin.py +++ b/tests/rest/admin/test_admin.py @@ -18,12 +18,15 @@ # [This file includes modifications made by New Vector Limited] # # +from __future__ import annotations import urllib.parse -from typing import cast +from typing import Any, cast +from unittest.mock import Mock from parameterized import parameterized +from twisted.internet.defer import Deferred from twisted.internet.testing import MemoryReactor from twisted.web.resource import Resource @@ -70,6 +73,24 @@ def create_resource_dict(self) -> dict[str, Resource]: resources["/_matrix/media"] = self.hs.get_media_repository_resource() return resources + def make_homeserver(self, reactor: MemoryReactor, clock: Clock) -> HomeServer: + self.fetches: list[tuple[tuple[Any, ...], dict[str, Any]]] = [] + + # A remote fetch of media that was not intentional. + # Used to check that remote media fetches do NOT happen. + def unexpected_remote_fetch(*args: Any, **kwargs: Any) -> Deferred[Any]: + self.fetches.append((args, kwargs)) + return Deferred() + + client = Mock() + client.federation_get_file = unexpected_remote_fetch + client.get_file = unexpected_remote_fetch + + return self.setup_test_homeserver( + clock=clock, + federation_http_client=client, + ) + def _ensure_quarantined( self, user_tok: str, @@ -176,6 +197,28 @@ def test_admin_can_bypass_quarantine(self) -> None: ), ) + def test_non_admin_bypass_does_not_fetch_remote_media(self) -> None: + self.register_user("nonadmin", "pass", admin=False) + non_admin_user_tok = self.login("nonadmin", "pass") + + channel = self.make_request( + "GET", + "/_matrix/client/v1/media/download/example.com/remote_media" + "?admin_unsafely_bypass_quarantine=true", + shorthand=False, + access_token=non_admin_user_tok, + await_result=False, + ) + self.pump() + + self.assertEqual(400, channel.code, msg=channel.json_body) + self.assertEqual( + channel.json_body["error"], + "Must be a server admin to bypass quarantine", + ) + # Check that a remote fetch attempt did not occur. + self.assertEqual(self.fetches, []) + @parameterized.expand( [ # Attempt quarantine media APIs as non-admin diff --git a/tests/rest/admin/test_media.py b/tests/rest/admin/test_media.py index 8cc54cc80c..6469d305b2 100644 --- a/tests/rest/admin/test_media.py +++ b/tests/rest/admin/test_media.py @@ -756,6 +756,133 @@ def _access_media( self.assertFalse(os.path.exists(local_path)) +class ListQuarantinedMediaChangesTestCase(_AdminMediaTests): + def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: + self.store = hs.get_datastores().main + self.server_name = hs.hostname + self.admin_user = self.register_user("admin", "pass", admin=True) + self.admin_user_tok = self.login("admin", "pass") + + def test_no_auth(self) -> None: + """ + Try to list quarantined media changes without authentication. + """ + + channel = self.make_request( + "GET", + "/_synapse/admin/v1/media/quarantine_changes", + ) + + self.assertEqual(401, channel.code, msg=channel.json_body) + self.assertEqual(Codes.MISSING_TOKEN, channel.json_body["errcode"]) + + def test_requester_is_not_admin(self) -> None: + """ + If the user is not a server admin, an error is returned. + """ + self.other_user = self.register_user("user", "pass") + self.other_user_token = self.login("user", "pass") + + channel = self.make_request( + "GET", + "/_synapse/admin/v1/media/quarantine_changes", + access_token=self.other_user_token, + ) + + self.assertEqual(403, channel.code, msg=channel.json_body) + self.assertEqual(Codes.FORBIDDEN, channel.json_body["errcode"]) + + def _quarantine_local_media(self, media_id: str, admin_user_tok: str) -> None: + channel = self.make_request( + "POST", + "/_synapse/admin/v1/media/quarantine/%s/%s" + % ( + self.server_name, + media_id, + ), + access_token=admin_user_tok, + ) + self.assertEqual(200, channel.code, msg=channel.json_body) + + def _local_upload(self, admin_user_tok: str) -> str: + response = self.helper.upload_media( + SMALL_PNG, tok=admin_user_tok, expect_code=200 + ) + origin_and_media_id = response["content_uri"][6:] # Cut off 'mxc://' + _origin, media_id = origin_and_media_id.split("/") + return media_id + + def test_list_quarantined_media(self) -> None: + """ + Ensure we actually get results for each page and that pagination is seamless. + """ + # Upload 105 media objects to test multiple pages + self.media_ids = [self._local_upload(self.admin_user_tok) for _ in range(105)] + + # No changes before quarantine + channel = self.make_request( + "GET", + "/_synapse/admin/v1/media/quarantine_changes", + access_token=self.admin_user_tok, + ) + self.assertEqual(200, channel.code, msg=channel.json_body) + self.assertEqual(0, len(channel.json_body["changes"])) + + # We expect to continue from the current stream position because we have no changes + self.assertEqual(1, channel.json_body["next_batch"]) + + # Quarantine by hash should kick in to get the other 104 media objects + self._quarantine_local_media(self.media_ids[0], self.admin_user_tok) + + # Page 1 (implied ?from=0) + channel = self.make_request( + "GET", + "/_synapse/admin/v1/media/quarantine_changes", + access_token=self.admin_user_tok, + ) + self.assertEqual(200, channel.code, msg=channel.json_body) + self.assertEqual(100, len(channel.json_body["changes"])) + self.assertEqual(101, channel.json_body["next_batch"]) + for row in channel.json_body["changes"]: + self.assertIn( + row["media_id"], + self.media_ids[0:100], + ) + self.assertEqual(row["origin"], self.server_name) + self.assertEqual(row["quarantined"], True) + + # Page 2 (explicit ?from, using next_batch) + channel = self.make_request( + "GET", + f"/_synapse/admin/v1/media/quarantine_changes?from={channel.json_body['next_batch']}", + access_token=self.admin_user_tok, + ) + self.assertEqual(200, channel.code, msg=channel.json_body) + self.assertEqual(5, len(channel.json_body["changes"])) + self.assertEqual(106, channel.json_body["next_batch"]) + for row in channel.json_body["changes"]: + self.assertIn( + row["media_id"], + self.media_ids[100:], + ) + self.assertEqual(row["origin"], self.server_name) + self.assertEqual(row["quarantined"], True) + + def test_list_quarantined_media_bounds_high(self) -> None: + """ + Ensure out of bounds (token stream position greater than our furthest persisted + position) requests with high `from` values are met with an appropriate error. + """ + # Page that's very much out of range + channel = self.make_request( + "GET", + "/_synapse/admin/v1/media/quarantine_changes?from=900000", + access_token=self.admin_user_tok, + ) + self.assertEqual(400, channel.code, msg=channel.json_body) + self.assertEqual(Codes.INVALID_PARAM, channel.json_body["errcode"]) + + class QuarantineMediaByIDTestCase(_AdminMediaTests): def upload_media_and_return_media_id(self, data: bytes) -> str: # Upload some media into the room diff --git a/tests/rest/admin/test_user_reports.py b/tests/rest/admin/test_user_reports.py new file mode 100644 index 0000000000..94ae242d86 --- /dev/null +++ b/tests/rest/admin/test_user_reports.py @@ -0,0 +1,644 @@ +# +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright (C) 2026 Element Creations Ltd +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . +# + + +from twisted.internet.testing import MemoryReactor + +import synapse.rest.admin +from synapse.api.errors import Codes +from synapse.rest.client import login, reporting, room +from synapse.server import HomeServer +from synapse.types import JsonDict +from synapse.util.clock import Clock + +from tests import unittest + + +class UserReportsTestCase(unittest.HomeserverTestCase): + servlets = [ + synapse.rest.admin.register_servlets, + login.register_servlets, + room.register_servlets, + reporting.register_servlets, + ] + + def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: + self.admin_user = self.register_user("admin", "pass", admin=True) + self.admin_user_tok = self.login("admin", "pass") + + self.users = {} + for i in range(10): + self.users[i] = self.register_user(f"user{i}", "pass") + + # users 1 and 2 report all other users + reporter_1_tok = self.login(self.users[0], "pass") + reporter_2_tok = self.login(self.users[1], "pass") + for num, user in self.users.items(): + if num <= 1: + continue + if num % 2 == 0: + self._report_user(user, reporter_1_tok) + else: + self._report_user(user, reporter_2_tok) + + self.url = "/_synapse/admin/v1/user_reports" + + def test_no_auth(self) -> None: + """ + Try to get a user report without authentication. + """ + channel = self.make_request("GET", self.url, {}) + + self.assertEqual(401, channel.code, msg=channel.json_body) + self.assertEqual(Codes.MISSING_TOKEN, channel.json_body["errcode"]) + + def test_requester_is_no_admin(self) -> None: + """ + If the user is not a server admin, an error 403 is returned. + """ + rando_tok = self.login(self.users[4], "pass") + channel = self.make_request( + "GET", + self.url, + access_token=rando_tok, + ) + + self.assertEqual(403, channel.code, msg=channel.json_body) + self.assertEqual(Codes.FORBIDDEN, channel.json_body["errcode"]) + + def test_default_success(self) -> None: + """ + Testing list of reported users + """ + + channel = self.make_request( + "GET", + self.url, + access_token=self.admin_user_tok, + ) + + self.assertEqual(200, channel.code, msg=channel.json_body) + self.assertEqual(channel.json_body["total"], 8) + self.assertEqual(len(channel.json_body["user_reports"]), 8) + self.assertNotIn("next_token", channel.json_body) + self._check_fields(channel.json_body["user_reports"]) + + def test_limit(self) -> None: + """ + Testing list of reported users with limit + """ + + channel = self.make_request( + "GET", + self.url + "?limit=5", + access_token=self.admin_user_tok, + ) + + self.assertEqual(200, channel.code, msg=channel.json_body) + self.assertEqual(channel.json_body["total"], 8) + self.assertEqual(len(channel.json_body["user_reports"]), 5) + self.assertEqual(channel.json_body["next_token"], 5) + self._check_fields(channel.json_body["user_reports"]) + + def test_from(self) -> None: + """ + Testing list of reported users with a defined starting point (from) + """ + + channel = self.make_request( + "GET", + self.url + "?from=5", + access_token=self.admin_user_tok, + ) + + self.assertEqual(200, channel.code, msg=channel.json_body) + self.assertEqual(channel.json_body["total"], 8) + self.assertEqual(len(channel.json_body["user_reports"]), 3) + self.assertNotIn("next_token", channel.json_body) + self._check_fields(channel.json_body["user_reports"]) + + def test_limit_and_from(self) -> None: + """ + Testing list of reported users with a defined starting point and limit + """ + + channel = self.make_request( + "GET", + self.url + "?from=2&limit=5", + access_token=self.admin_user_tok, + ) + + self.assertEqual(200, channel.code, msg=channel.json_body) + self.assertEqual(channel.json_body["total"], 8) + self.assertEqual(channel.json_body["next_token"], 7) + self.assertEqual(len(channel.json_body["user_reports"]), 5) + self._check_fields(channel.json_body["user_reports"]) + + def test_filter_by_target_user_id(self) -> None: + """ + Testing list of reported users with a filter of target_user_id + """ + + channel = self.make_request( + "GET", + self.url + "?target_user_id=%s" % self.users[3], + access_token=self.admin_user_tok, + ) + + self.assertEqual(200, channel.code, msg=channel.json_body) + self.assertEqual(channel.json_body["total"], 1) + self.assertEqual(len(channel.json_body["user_reports"]), 1) + self.assertNotIn("next_token", channel.json_body) + self._check_fields(channel.json_body["user_reports"]) + + for report in channel.json_body["user_reports"]: + self.assertEqual(report["target_user_id"], self.users[3]) + + def test_filter_user(self) -> None: + """ + Testing list of reported users with a filter of reporting user + """ + + channel = self.make_request( + "GET", + self.url + "?user_id=%s" % self.users[0], + access_token=self.admin_user_tok, + ) + + self.assertEqual(200, channel.code, msg=channel.json_body) + self.assertEqual(channel.json_body["total"], 4) + self.assertEqual(len(channel.json_body["user_reports"]), 4) + self.assertNotIn("next_token", channel.json_body) + self._check_fields(channel.json_body["user_reports"]) + + for report in channel.json_body["user_reports"]: + self.assertEqual(report["user_id"], self.users[0]) + + def test_filter_user_and_target_user(self) -> None: + """ + Testing list of reported users with a filter of reporting user and target_user + """ + + channel = self.make_request( + "GET", + self.url + "?user_id=%s&target_user_id=%s" % (self.users[1], self.users[7]), + access_token=self.admin_user_tok, + ) + + self.assertEqual(200, channel.code, msg=channel.json_body) + self.assertEqual(channel.json_body["total"], 1) + self.assertEqual(len(channel.json_body["user_reports"]), 1) + self.assertNotIn("next_token", channel.json_body) + self._check_fields(channel.json_body["user_reports"]) + + for report in channel.json_body["user_reports"]: + self.assertEqual(report["user_id"], self.users[1]) + self.assertEqual(report["target_user_id"], self.users[7]) + + def test_valid_search_order(self) -> None: + """ + Testing search order. Order by timestamps. + """ + + # fetch the most recent first, largest timestamp + channel = self.make_request( + "GET", + self.url + "?dir=b", + access_token=self.admin_user_tok, + ) + + self.assertEqual(200, channel.code, msg=channel.json_body) + self.assertEqual(channel.json_body["total"], 8) + self.assertEqual(len(channel.json_body["user_reports"]), 8) + report = 1 + while report < len(channel.json_body["user_reports"]): + self.assertGreaterEqual( + channel.json_body["user_reports"][report - 1]["received_ts"], + channel.json_body["user_reports"][report]["received_ts"], + ) + report += 1 + + # fetch the oldest first, smallest timestamp + channel = self.make_request( + "GET", + self.url + "?dir=f", + access_token=self.admin_user_tok, + ) + + self.assertEqual(200, channel.code, msg=channel.json_body) + self.assertEqual(channel.json_body["total"], 8) + self.assertEqual(len(channel.json_body["user_reports"]), 8) + report = 1 + while report < len(channel.json_body["user_reports"]): + self.assertLessEqual( + channel.json_body["user_reports"][report - 1]["received_ts"], + channel.json_body["user_reports"][report]["received_ts"], + ) + report += 1 + + def test_invalid_search_order(self) -> None: + """ + Testing that a invalid search order returns a 400 + """ + + channel = self.make_request( + "GET", + self.url + "?dir=bar", + access_token=self.admin_user_tok, + ) + + self.assertEqual(400, channel.code, msg=channel.json_body) + self.assertEqual(Codes.INVALID_PARAM, channel.json_body["errcode"]) + self.assertEqual( + "Query parameter 'dir' must be one of ['b', 'f']", + channel.json_body["error"], + ) + + def test_limit_is_negative(self) -> None: + """ + Testing that a negative limit parameter returns a 400 + """ + + channel = self.make_request( + "GET", + self.url + "?limit=-5", + access_token=self.admin_user_tok, + ) + + self.assertEqual(400, channel.code, msg=channel.json_body) + self.assertEqual(Codes.INVALID_PARAM, channel.json_body["errcode"]) + + def test_from_is_negative(self) -> None: + """ + Testing that a negative from parameter returns a 400 + """ + + channel = self.make_request( + "GET", + self.url + "?from=-5", + access_token=self.admin_user_tok, + ) + + self.assertEqual(400, channel.code, msg=channel.json_body) + self.assertEqual(Codes.INVALID_PARAM, channel.json_body["errcode"]) + + def test_next_token(self) -> None: + """ + Testing that `next_token` appears at the right place + """ + + # `next_token` does not appear + # Number of results is the number of entries + channel = self.make_request( + "GET", + self.url + "?limit=8", + access_token=self.admin_user_tok, + ) + + self.assertEqual(200, channel.code, msg=channel.json_body) + self.assertEqual(channel.json_body["total"], 8) + self.assertEqual(len(channel.json_body["user_reports"]), 8) + self.assertNotIn("next_token", channel.json_body) + + # `next_token` does not appear + # Number of max results is larger than the number of entries + channel = self.make_request( + "GET", + self.url + "?limit=10", + access_token=self.admin_user_tok, + ) + + self.assertEqual(200, channel.code, msg=channel.json_body) + self.assertEqual(channel.json_body["total"], 8) + self.assertEqual(len(channel.json_body["user_reports"]), 8) + self.assertNotIn("next_token", channel.json_body) + + # `next_token` does appear + # Number of max results is smaller than the number of entries + channel = self.make_request( + "GET", + self.url + "?limit=6", + access_token=self.admin_user_tok, + ) + + self.assertEqual(200, channel.code, msg=channel.json_body) + self.assertEqual(channel.json_body["total"], 8) + self.assertEqual(len(channel.json_body["user_reports"]), 6) + self.assertEqual(channel.json_body["next_token"], 6) + + # Check + # Set `from` to value of `next_token` for request remaining entries + # `next_token` does not appear + channel = self.make_request( + "GET", + self.url + "?from=6", + access_token=self.admin_user_tok, + ) + + self.assertEqual(200, channel.code, msg=channel.json_body) + self.assertEqual(channel.json_body["total"], 8) + self.assertEqual(len(channel.json_body["user_reports"]), 2) + self.assertNotIn("next_token", channel.json_body) + + def _report_user(self, target_user: str, reporter_tok: str) -> None: + """Report a user""" + channel = self.make_request( + "POST", + "_matrix/client/v3/users/%s/report" % (target_user), + {"reason": "stone-cold bummer"}, + access_token=reporter_tok, + shorthand=False, + ) + self.assertEqual(200, channel.code, msg=channel.json_body) + + def _check_fields(self, content: list[JsonDict]) -> None: + """Checks that all attributes are present in a user report""" + for c in content: + self.assertIn("id", c) + self.assertIn("received_ts", c) + self.assertIn("user_id", c) + self.assertIn("target_user_id", c) + self.assertIn("reason", c) + + +class UserReportDetailTestCase(unittest.HomeserverTestCase): + servlets = [ + synapse.rest.admin.register_servlets, + login.register_servlets, + room.register_servlets, + reporting.register_servlets, + ] + + def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: + self.admin_user = self.register_user("admin", "pass", admin=True) + self.admin_user_tok = self.login("admin", "pass") + + self.bad_user = self.register_user("user", "pass") + self.bad_user_tok = self.login("user", "pass") + + channel = self.make_request( + "POST", + "_matrix/client/v3/users/%s/report" % (self.bad_user), + {"reason": "stone-cold bummer"}, + access_token=self.admin_user_tok, + shorthand=False, + ) + self.assertEqual(200, channel.code, msg=channel.json_body) + + # first created user report gets `id`=2 + self.url = "/_synapse/admin/v1/user_reports/2" + + def test_no_auth(self) -> None: + """ + Try to get user report without authentication. + """ + channel = self.make_request("GET", self.url, {}) + + self.assertEqual(401, channel.code, msg=channel.json_body) + self.assertEqual(Codes.MISSING_TOKEN, channel.json_body["errcode"]) + + def test_requester_is_no_admin(self) -> None: + """ + If the user is not a server admin, an error 403 is returned. + """ + + channel = self.make_request( + "GET", + self.url, + access_token=self.bad_user_tok, + ) + + self.assertEqual(403, channel.code, msg=channel.json_body) + self.assertEqual(Codes.FORBIDDEN, channel.json_body["errcode"]) + + def test_default_success(self) -> None: + """ + Testing get a reported user + """ + + channel = self.make_request( + "GET", + self.url, + access_token=self.admin_user_tok, + ) + + self.assertEqual(200, channel.code, msg=channel.json_body) + self._check_fields(channel.json_body) + + def test_invalid_report_id(self) -> None: + """ + Testing that an invalid `report_id` returns a 400. + """ + + # `report_id` is negative + channel = self.make_request( + "GET", + "/_synapse/admin/v1/user_reports/-123", + access_token=self.admin_user_tok, + ) + + self.assertEqual(400, channel.code, msg=channel.json_body) + self.assertEqual(Codes.INVALID_PARAM, channel.json_body["errcode"]) + self.assertEqual( + "The report_id parameter must be a string representing a positive integer.", + channel.json_body["error"], + ) + + # `report_id` is a non-numerical string + channel = self.make_request( + "GET", + "/_synapse/admin/v1/user_reports/abcdef", + access_token=self.admin_user_tok, + ) + + self.assertEqual(400, channel.code, msg=channel.json_body) + self.assertEqual(Codes.INVALID_PARAM, channel.json_body["errcode"]) + self.assertEqual( + "The report_id parameter must be a string representing a positive integer.", + channel.json_body["error"], + ) + + # `report_id` is undefined + channel = self.make_request( + "GET", + "/_synapse/admin/v1/user_reports/", + access_token=self.admin_user_tok, + ) + + self.assertEqual(400, channel.code, msg=channel.json_body) + self.assertEqual(Codes.INVALID_PARAM, channel.json_body["errcode"]) + self.assertEqual( + "The report_id parameter must be a string representing a positive integer.", + channel.json_body["error"], + ) + + def test_report_id_not_found(self) -> None: + """ + Testing that a not existing `report_id` returns a 404. + """ + + channel = self.make_request( + "GET", + "/_synapse/admin/v1/user_reports/123", + access_token=self.admin_user_tok, + ) + + self.assertEqual(404, channel.code, msg=channel.json_body) + self.assertEqual(Codes.NOT_FOUND, channel.json_body["errcode"]) + self.assertEqual("User report not found", channel.json_body["error"]) + + def _check_fields(self, content: JsonDict) -> None: + """Checks that all attributes are present in a user report""" + self.assertIn("id", content) + self.assertIn("received_ts", content) + self.assertIn("target_user_id", content) + self.assertIn("user_id", content) + self.assertIn("reason", content) + + +class DeleteUserReportTestCase(unittest.HomeserverTestCase): + servlets = [ + synapse.rest.admin.register_servlets, + login.register_servlets, + ] + + def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: + self._store = hs.get_datastores().main + + self.admin_user = self.register_user("admin", "pass", admin=True) + self.admin_user_tok = self.login("admin", "pass") + + self.bad_user = self.register_user("user", "pass") + self.other_user_tok = self.login("user", "pass") + + # create report + self.get_success( + self._store.add_user_report( + self.bad_user, + self.admin_user, + "super bummer", + self.clock.time_msec(), + ) + ) + + self.url = "/_synapse/admin/v1/user_reports/2" + + def test_no_auth(self) -> None: + """ + Try to delete user report without authentication. + """ + channel = self.make_request("DELETE", self.url) + + self.assertEqual(401, channel.code, msg=channel.json_body) + self.assertEqual(Codes.MISSING_TOKEN, channel.json_body["errcode"]) + + def test_requester_is_no_admin(self) -> None: + """ + If the user is not a server admin, an error 403 is returned. + """ + + channel = self.make_request( + "DELETE", + self.url, + access_token=self.other_user_tok, + ) + + self.assertEqual(403, channel.code, msg=channel.json_body) + self.assertEqual(Codes.FORBIDDEN, channel.json_body["errcode"]) + + def test_delete_success(self) -> None: + """ + Testing delete a report. + """ + + channel = self.make_request( + "DELETE", + self.url, + access_token=self.admin_user_tok, + ) + + self.assertEqual(200, channel.code, msg=channel.json_body) + self.assertEqual({}, channel.json_body) + + channel = self.make_request( + "GET", + self.url, + access_token=self.admin_user_tok, + ) + + # check that report was deleted + self.assertEqual(404, channel.code, msg=channel.json_body) + self.assertEqual(Codes.NOT_FOUND, channel.json_body["errcode"]) + + def test_invalid_report_id(self) -> None: + """ + Testing that an invalid `report_id` returns a 400. + """ + + # `report_id` is negative + channel = self.make_request( + "DELETE", + "/_synapse/admin/v1/user_reports/-123", + access_token=self.admin_user_tok, + ) + + self.assertEqual(400, channel.code, msg=channel.json_body) + self.assertEqual(Codes.INVALID_PARAM, channel.json_body["errcode"]) + self.assertEqual( + "The report_id parameter must be a string representing a positive integer.", + channel.json_body["error"], + ) + + # `report_id` is a non-numerical string + channel = self.make_request( + "DELETE", + "/_synapse/admin/v1/user_reports/abcdef", + access_token=self.admin_user_tok, + ) + + self.assertEqual(400, channel.code, msg=channel.json_body) + self.assertEqual(Codes.INVALID_PARAM, channel.json_body["errcode"]) + self.assertEqual( + "The report_id parameter must be a string representing a positive integer.", + channel.json_body["error"], + ) + + # `report_id` is undefined + channel = self.make_request( + "DELETE", + "/_synapse/admin/v1/user_reports/", + access_token=self.admin_user_tok, + ) + + self.assertEqual(400, channel.code, msg=channel.json_body) + self.assertEqual(Codes.INVALID_PARAM, channel.json_body["errcode"]) + self.assertEqual( + "The report_id parameter must be a string representing a positive integer.", + channel.json_body["error"], + ) + + def test_report_id_not_found(self) -> None: + """ + Testing that a not existing `report_id` returns a 404. + """ + + channel = self.make_request( + "DELETE", + "/_synapse/admin/v1/user_reports/123", + access_token=self.admin_user_tok, + ) + + self.assertEqual(404, channel.code, msg=channel.json_body) + self.assertEqual(Codes.NOT_FOUND, channel.json_body["errcode"]) + self.assertEqual("User report not found", channel.json_body["error"]) diff --git a/tests/rest/client/sliding_sync/test_rooms_meta.py b/tests/rest/client/sliding_sync/test_rooms_meta.py index 9e3f8aaf94..4ff07b4d26 100644 --- a/tests/rest/client/sliding_sync/test_rooms_meta.py +++ b/tests/rest/client/sliding_sync/test_rooms_meta.py @@ -12,6 +12,7 @@ # . # import logging +from typing import Any from parameterized import parameterized, parameterized_class @@ -966,7 +967,7 @@ def test_rooms_bump_stamp_backfill(self) -> None: creator = "@user:other" room_id = "!foo:other" room_version = RoomVersions.V10 - shared_kwargs = { + shared_kwargs: dict[str, Any] = { "room_id": room_id, "room_version": room_version.identifier, } diff --git a/tests/rest/client/test_auth.py b/tests/rest/client/test_auth.py index 3a9ab82aa2..e7cfae4693 100644 --- a/tests/rest/client/test_auth.py +++ b/tests/rest/client/test_auth.py @@ -52,6 +52,62 @@ from tests.server import FakeChannel from tests.unittest import override_config, skip_unless +TEST_MULTIPLE_OIDC_IDP_ID1 = "friendface" +TEST_MULTIPLE_OIDC_IDP_ID2 = "potatocloud" +TEST_MULTIPLE_OIDC_IDP_ID3 = "svnswitch" + +TEST_MULTIPLE_OIDC_ISSUER1 = "https://friendface.invalid/" +TEST_MULTIPLE_OIDC_ISSUER2 = "https://potato.invalid/" +TEST_MULTIPLE_OIDC_ISSUER3 = "https://subversionswitch.invalid/" + +TEST_MULTIPLE_OIDC_PROVIDERS = [ + { + "idp_id": TEST_MULTIPLE_OIDC_IDP_ID1, + "idp_name": "FriendFace", + "idp_icon": "mxc://example.invalid/friendface", + "idp_brand": "friendface", + "issuer": TEST_MULTIPLE_OIDC_ISSUER1, + "client_id": "id-for-friendface", + "client_secret": "secret-for-friendface", + "scopes": ["openid"], + "discover": False, + "authorization_endpoint": f"{TEST_MULTIPLE_OIDC_ISSUER1}oidc/auth", + "token_endpoint": f"{TEST_MULTIPLE_OIDC_ISSUER1}oidc/token", + "jwks_uri": f"{TEST_MULTIPLE_OIDC_ISSUER1}oidc/jwks", + }, + { + "idp_id": TEST_MULTIPLE_OIDC_IDP_ID2, + "idp_name": "Potato Cloud", + "idp_icon": "mxc://example.invalid/potatocloud", + "idp_brand": "potato", + "issuer": TEST_MULTIPLE_OIDC_ISSUER2, + "client_id": "id-for-potato", + "client_secret": "secret-for-potato", + "scopes": ["openid"], + "discover": False, + "authorization_endpoint": f"{TEST_MULTIPLE_OIDC_ISSUER2}oidc/auth", + "token_endpoint": f"{TEST_MULTIPLE_OIDC_ISSUER2}oidc/token", + "jwks_uri": f"{TEST_MULTIPLE_OIDC_ISSUER2}oidc/jwks", + }, + { + "idp_id": TEST_MULTIPLE_OIDC_IDP_ID3, + "idp_name": "SubversionSwitch", + "idp_icon": "mxc://example.invalid/svnswitch", + "idp_brand": "svnswitch", + "issuer": TEST_MULTIPLE_OIDC_ISSUER3, + "client_id": "id-for-svnswitch", + "client_secret": "secret-for-svnswitch", + "scopes": ["openid"], + "discover": False, + "authorization_endpoint": f"{TEST_MULTIPLE_OIDC_ISSUER3}oidc/auth", + "token_endpoint": f"{TEST_MULTIPLE_OIDC_ISSUER3}oidc/token", + "jwks_uri": f"{TEST_MULTIPLE_OIDC_ISSUER3}oidc/jwks", + }, +] +""" +`oidc_providers` config example for multiple OIDC providers. +""" + class DummyRecaptchaChecker(UserInteractiveAuthChecker): def __init__(self, hs: HomeServer) -> None: @@ -515,6 +571,90 @@ def test_ui_auth_via_sso(self) -> None: body={"auth": {"session": session_id}}, ) + @skip_unless(HAS_OIDC, "requires OIDC") + @override_config( + { + "oidc_providers": TEST_MULTIPLE_OIDC_PROVIDERS, + "experimental_features": {"msc4450_enabled": True}, + } + ) + def test_msc4450_select_idp_id(self) -> None: + """ + Test for MSC4450: Identity Provider selection for + User-Interactive Authentication with Legacy Single Sign-On. + + We configure 3 OIDC providers and then check that we can select + which one to redirect to for User-Interactive Authentication. + """ + + # Attach the user to the OIDC providers manually + for idp_id in ( + TEST_MULTIPLE_OIDC_IDP_ID1, + TEST_MULTIPLE_OIDC_IDP_ID2, + ): + self.get_success( + self.hs.get_datastores().main.record_user_external_id( + # `oidc-` is a magic prefix needed for OIDC providers + f"oidc-{idp_id}", + # arbitrary/opaque provider-specific external ID, doesn't really matter + "some.ext.user.id", + self.user, + ) + ) + + # Start a user-interactive authentication session by + # calling the device deletion API + channel = self.delete_device( + self.user_tok, self.device_id, HTTPStatus.UNAUTHORIZED + ) + auth_session = channel.json_body["session"] + flows = channel.json_body["flows"] + self.assertCountEqual( + flows, [{"stages": ["m.login.sso"]}, {"stages": ["m.login.password"]}] + ) + + # Try to start the User-Interactive Auth against both identity providers. + # Make sure we get the identity provider that we ask for. + # + # We need to try against both to be sure that Synapse doesn't just conveniently happen to + # arbitrarily select the identity provider we test. + for idp_id, provider_config in ( + ( + TEST_MULTIPLE_OIDC_IDP_ID1, + TEST_MULTIPLE_OIDC_PROVIDERS[0], + ), + ( + TEST_MULTIPLE_OIDC_IDP_ID2, + TEST_MULTIPLE_OIDC_PROVIDERS[1], + ), + ): + endpoint = f"/_matrix/client/v3/auth/m.login.sso/fallback/web?session={auth_session}&io.element.idp_id=oidc-{idp_id}" + channel = self.make_request("GET", endpoint) + self.assertEqual( + channel.code, + HTTPStatus.OK, + f"Failed to use the {endpoint} endpoint as part of the UIA flow for idp_id={idp_id} : response_body={channel.text_body}", + ) + + # This 'Continue with ...' text is templated by `synapse/res/templates/sso_auth_confirm.html` + self.assertIn( + f"Continue with {provider_config['idp_name']}", + channel.text_body, + ) + + self.assertIn(provider_config["authorization_endpoint"], channel.text_body) + + # Test that we can't use the 3rd OIDC provider as we're not + # registered with it + channel = self.make_request( + "GET", + f"/_matrix/client/v3/auth/m.login.sso/fallback/web?session={auth_session}&io.element.idp_id=oidc-{TEST_MULTIPLE_OIDC_IDP_ID3}", + ) + self.assertEqual(channel.code, HTTPStatus.BAD_REQUEST, channel.json_body) + self.assertEqual( + channel.json_body["errcode"], Codes.INVALID_PARAM, channel.json_body + ) + @skip_unless(HAS_OIDC, "requires OIDC") @override_config({"oidc_config": TEST_OIDC_CONFIG}) def test_does_not_offer_password_for_sso_user(self) -> None: diff --git a/tests/rest/client/test_keys.py b/tests/rest/client/test_keys.py index 9c83a284d7..fc21d83ef2 100644 --- a/tests/rest/client/test_keys.py +++ b/tests/rest/client/test_keys.py @@ -160,9 +160,12 @@ def test_upload_keys_fails_on_invalid_user_id_or_device_id(self) -> None: channel.result, ) - def test_upload_keys_succeeds_when_fields_are_explicitly_set_to_null(self) -> None: + def test_upload_keys_rejects_device_keys_set_to_null(self) -> None: """ - This is a regression test for https://github.com/element-hq/synapse/pull/19023. + Test that sending `device_keys: null` is rejected per spec. + The spec says `device_keys` may be omitted, but not set to `null`. + + See https://github.com/element-hq/synapse/issues/19030. """ device_id = "DEVICE_ID" self.register_user("alice", "wonderland") @@ -173,12 +176,10 @@ def test_upload_keys_succeeds_when_fields_are_explicitly_set_to_null(self) -> No "/_matrix/client/v3/keys/upload", { "device_keys": None, - "one_time_keys": None, - "fallback_keys": None, }, alice_token, ) - self.assertEqual(channel.code, HTTPStatus.OK, channel.result) + self.assertEqual(channel.code, HTTPStatus.BAD_REQUEST, channel.result) class KeyQueryTestCase(unittest.HomeserverTestCase): diff --git a/tests/rest/client/test_retention.py b/tests/rest/client/test_retention.py index 82a3b5b337..1f8d9154ca 100644 --- a/tests/rest/client/test_retention.py +++ b/tests/rest/client/test_retention.py @@ -23,6 +23,7 @@ from twisted.internet.testing import MemoryReactor from synapse.api.constants import EventTypes +from synapse.events.utils import FilteredEvent from synapse.rest import admin from synapse.rest.client import login, room from synapse.server import HomeServer @@ -173,7 +174,9 @@ def test_visibility(self) -> None: # We should only get one event back. self.assertEqual(len(filtered_events), 1, filtered_events) # That event should be the second, not outdated event. - self.assertEqual(filtered_events[0].event_id, valid_event_id, filtered_events) + self.assertEqual( + filtered_events[0].event.event_id, valid_event_id, filtered_events + ) def _test_retention_event_purged(self, room_id: str, increment: float) -> None: """Run the following test scenario to test the message retention policy support: @@ -253,7 +256,11 @@ def get_event(self, event_id: str, expect_none: bool = False) -> JsonDict: assert event is not None time_now = self.clock.time_msec() - serialized = self.get_success(self.serializer.serialize_event(event, time_now)) + serialized = self.get_success( + self.serializer.serialize_event( + FilteredEvent(event=event, membership=None), time_now + ) + ) return serialized diff --git a/tests/storage/test_devices.py b/tests/storage/test_devices.py index 1d1979e19f..b153c74980 100644 --- a/tests/storage/test_devices.py +++ b/tests/storage/test_devices.py @@ -19,15 +19,21 @@ # # +import itertools from typing import Collection +from unittest.mock import patch from twisted.internet.testing import MemoryReactor import synapse.api.errors from synapse.api.constants import EduTypes from synapse.server import HomeServer +from synapse.storage.databases.main.devices import ( + PRUNE_DEVICE_LISTS_CHANGES_IN_ROOM_AGE, +) from synapse.types import JsonDict from synapse.util.clock import Clock +from synapse.util.duration import Duration from tests.unittest import HomeserverTestCase @@ -351,3 +357,101 @@ def test_update_unknown_device(self) -> None: synapse.api.errors.StoreError, ) self.assertEqual(404, exc.value.code) + + @patch("synapse.storage.databases.main.devices.PRUNE_DEVICE_LISTS_BATCH_SIZE", 5) + def test_prune_old_device_lists_changes_in_room(self) -> None: + """Test that old entries in the `device_lists_changes_in_room` table are pruned properly.""" + + # Pretend the user is in a few rooms. + room_ids = [f"!room{i}:test" for i in range(20)] + + # Create a generator for device IDs so we can easily create many unique + # device IDs without having to keep track of the count ourselves. + device_id_gen = (f"device_id{i}" for i in itertools.count()) + + def get_devices_in_room_status() -> tuple[int, str]: + """Helper function to get the count of entries in + `device_lists_changes_in_room` and the minimum device_id.""" + return self.get_success( + self.store.db_pool.simple_select_one( + table="device_lists_changes_in_room", + keyvalues={}, + retcols=("COUNT(*)", "MIN(device_id)"), + ) + ) + + # First we add some initial entries to the `device_lists_changes_in_room`. + self.get_success( + self.store.add_device_change_to_streams( + user_id="@user_id:test", + device_ids=[next(device_id_gen) for _ in range(10)], + room_ids=room_ids, + ) + ) + + # Advance the reactor a while, but not long enough to trigger pruning. + self.reactor.advance(Duration(hours=1).as_secs()) + + # The `device_lists_changes_in_room` table should now have 10 * + # len(room_ids) entries, and the minimum device_id should be + # `device_id0`. + count, min_device_id = get_devices_in_room_status() + self.assertEqual(count, 10 * len(room_ids)) + self.assertEqual(min_device_id, "device_id0") + + # Record the max pruned stream ID before pruning, so we can check + # that this correctly updates after pruning. + starting_max_pruned_id = self.get_success( + self.store.db_pool.runInteraction( + "get_max_pruned_device_lists_changes_in_room", + self.store._get_max_pruned_device_lists_changes_in_room_txn, + ) + ) + + # Now we add some more entries. + self.get_success( + self.store.add_device_change_to_streams( + user_id="@user_id:test", + device_ids=[next(device_id_gen) for _ in range(10)], + room_ids=room_ids, + ) + ) + + # Advance the reactor a while more, so that the first batch of entries is + # now old enough to be pruned. + self.reactor.advance( + (PRUNE_DEVICE_LISTS_CHANGES_IN_ROOM_AGE - Duration(minutes=30)).as_secs() + ) + + # Advance repeatedly a bit so that the pruning process can run to completion. + for _ in range(10): + self.reactor.advance(Duration(milliseconds=110).as_secs()) + + # Check that the old entries have been pruned, and the new entries are still there. + count, min_device_id = get_devices_in_room_status() + self.assertEqual(count, 10 * len(room_ids)) + self.assertEqual(min_device_id, "device_id10") + + # We should always keep the most recent entries, even if they are old enough to be pruned. + self.reactor.advance( + (PRUNE_DEVICE_LISTS_CHANGES_IN_ROOM_AGE + Duration(minutes=30)).as_secs() + ) + + # Advance repeatedly a bit so that the pruning process can run to completion. + for _ in range(10): + self.reactor.advance(Duration(milliseconds=110).as_secs()) + + count, min_device_id = get_devices_in_room_status() + # We should always keep the most recent entries so that we can + # calculate the maximum stream ID used. + self.assertEqual(count, len(room_ids)) + self.assertEqual(min_device_id, "device_id19") + + # Check that the max pruned stream ID has been advanced after pruning. + max_pruned_id = self.get_success( + self.store.db_pool.runInteraction( + "get_max_pruned_device_lists_changes_in_room", + self.store._get_max_pruned_device_lists_changes_in_room_txn, + ) + ) + self.assertGreater(max_pruned_id, starting_max_pruned_id) diff --git a/tests/storage/test_events_bg_updates.py b/tests/storage/test_events_bg_updates.py index a5b53de77f..aceacec98e 100644 --- a/tests/storage/test_events_bg_updates.py +++ b/tests/storage/test_events_bg_updates.py @@ -14,17 +14,23 @@ # +import json + +import signedjson.key from canonicaljson import encode_canonical_json from twisted.internet.testing import MemoryReactor from synapse.api.constants import MAX_DEPTH from synapse.api.room_versions import RoomVersion, RoomVersions +from synapse.rest import admin +from synapse.rest.client import login, room from synapse.server import HomeServer +from synapse.storage.background_updates import BackgroundUpdater from synapse.types.storage import _BackgroundUpdates from synapse.util.clock import Clock -from tests.unittest import HomeserverTestCase +from tests.unittest import HomeserverTestCase, override_config class TestFixupMaxDepthCapBgUpdate(HomeserverTestCase): @@ -287,3 +293,163 @@ def test_batching(self) -> None: self.assertTrue(self._get_recheck("$redact5:test")) self.assertFalse(self._get_recheck("$redact6:test")) self.assertTrue(self._get_recheck("$redact7:test")) + + +class TestResignEventsBgUpdate(HomeserverTestCase): + """Test the background update that re-signs events.""" + + servlets = [ + admin.register_servlets, + room.register_servlets, + login.register_servlets, + ] + + def prepare( + self, reactor: MemoryReactor, clock: Clock, homeserver: HomeServer + ) -> None: + self.updates: BackgroundUpdater = self.hs.get_datastores().main.db_pool.updates + self.store = self.hs.get_datastores().main + self.db_pool = self.store.db_pool + + self.room_id = "!testroom:example.com" + + @override_config({"caches": {"global_factor": 1}, "event_cache_size": "999"}) + def test_events_are_resigned_after_bg_update_runs(self) -> None: + """Test that the background update correctly re-signs existing events with the + new key""" + + # Ensure all background updates have finished running + self.wait_for_background_updates() + + # Set up a room with a local and remote user in it. + self.register_user("user", "pass") + token = self.login("user", "pass") + + # Create new room + room_id = self.helper.create_room_as( + "user", room_version=RoomVersions.V12.identifier, tok=token + ) + + # Send a message + body = self.helper.send(room_id, body="Test", tok=token) + + old_event = self.get_success(self.store.get_event(body["event_id"])) + old_key_id = f"{self.hs.signing_key.alg}:{self.hs.signing_key.version}" + + # Ensure the message event is in the cache so that we test the cache is + # invalidated properly + res = self.store._get_event_cache.get_local((old_event.event_id,)) + self.assertEqual(res.event, old_event, "Event not cached as expected.") # type: ignore + + # Ensure message event is signed with original signing key + self.assertIn( + old_key_id, old_event.signatures[self.hs.config.server.server_name] + ) + + # Generate a new signing key + self.hs.signing_key = signedjson.key.generate_signing_key("new-test-key") + + # Reinsert the background update as it was already run at the start of + # the test. + self.get_success( + self.db_pool.simple_insert( + table="background_updates", + values={ + "update_name": "event_resign", + "progress_json": "{}", + }, + ) + ) + self.updates.start_doing_background_updates() + # Ensure the background updates have finished running + self.wait_for_background_updates() + + # Get the event from the database again + new_event = self.get_success(self.store.get_event(body["event_id"])) + new_key_id = f"{self.hs.signing_key.alg}:{self.hs.signing_key.version}" + + # Ensure message event is signed with new signing key, and not with the original + # signing key + self.assertNotIn( + old_key_id, new_event.signatures[self.hs.config.server.server_name] + ) + self.assertIn( + new_key_id, new_event.signatures[self.hs.config.server.server_name] + ) + + @override_config({"caches": {"global_factor": 1}, "event_cache_size": "999"}) + def test_old_key_filter(self) -> None: + """Test that old_key parameter causes only events whose signature + verifies against the provided key to be re-signed.""" + + self.wait_for_background_updates() + + self.register_user("user2", "pass") + token = self.login("user2", "pass") + + room_id = self.helper.create_room_as( + "user2", room_version=RoomVersions.V12.identifier, tok=token + ) + body = self.helper.send(room_id, body="Test old_key", tok=token) + + old_signing_key = self.hs.signing_key + old_key_id = f"{old_signing_key.alg}:{old_signing_key.version}" + old_verify_key = signedjson.key.get_verify_key(old_signing_key) + old_key_param = ( + f"{old_verify_key.alg}:{old_verify_key.version} " + f"{signedjson.key.encode_verify_key_base64(old_verify_key)}" + ) + + # Generate a new signing key + self.hs.signing_key = signedjson.key.generate_signing_key("new-test-key-2") + + # Generate a different key but reuse the same key ID/version, to + # ensure we're filtering on the actual public key, not just the ID. + wrong_key = signedjson.key.generate_signing_key(old_signing_key.version) + wrong_verify_key = signedjson.key.get_verify_key(wrong_key) + wrong_key_param = ( + f"{old_verify_key.alg}:{old_verify_key.version} " + f"{signedjson.key.encode_verify_key_base64(wrong_verify_key)}" + ) + + # Insert BG update with old_key filter pointing to a WRONG key + self.get_success( + self.db_pool.simple_insert( + table="background_updates", + values={ + "update_name": "event_resign", + "progress_json": json.dumps({"old_key": wrong_key_param}), + }, + ) + ) + self.updates.start_doing_background_updates() + self.wait_for_background_updates() + + # Event should NOT have been re-signed (wrong key) + event_after = self.get_success(self.store.get_event(body["event_id"])) + self.assertIn( + old_key_id, event_after.signatures[self.hs.config.server.server_name] + ) + + # Now insert BG update with the CORRECT old key + self.get_success( + self.db_pool.simple_insert( + table="background_updates", + values={ + "update_name": "event_resign", + "progress_json": json.dumps({"old_key": old_key_param}), + }, + ) + ) + self.updates.start_doing_background_updates() + self.wait_for_background_updates() + + # Event should now be re-signed + new_event = self.get_success(self.store.get_event(body["event_id"])) + new_key_id = f"{self.hs.signing_key.alg}:{self.hs.signing_key.version}" + self.assertNotIn( + old_key_id, new_event.signatures[self.hs.config.server.server_name] + ) + self.assertIn( + new_key_id, new_event.signatures[self.hs.config.server.server_name] + ) diff --git a/tests/storage/test_msc4242_state_dag.py b/tests/storage/test_msc4242_state_dag.py new file mode 100644 index 0000000000..8775e5c8eb --- /dev/null +++ b/tests/storage/test_msc4242_state_dag.py @@ -0,0 +1,371 @@ +# +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright (C) 2026 Element Creations, Ltd +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . + +from typing import Iterable +from unittest.mock import Mock + +from twisted.test.proto_helpers import MemoryReactor + +from synapse.api.constants import EventTypes +from synapse.api.errors import SynapseError +from synapse.api.room_versions import RoomVersions +from synapse.events import FrozenEventVMSC4242, make_event_from_dict +from synapse.events.snapshot import EventContext +from synapse.rest.client import room +from synapse.server import HomeServer +from synapse.util.clock import Clock + +from tests.unittest import HomeserverTestCase, override_config + + +class MSC4242StateDagsTests(HomeserverTestCase): + user_id = "@user1:server" + servlets = [room.register_servlets] + + def make_homeserver(self, reactor: MemoryReactor, clock: Clock) -> HomeServer: + hs = self.setup_test_homeserver("server") + return hs + + def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: + self.room_id = self.helper.create_room_as( + self.user_id, + room_version=RoomVersions.MSC4242v12.identifier, + ) + + self.store = hs.get_datastores().main + self._storage_controllers = self.hs.get_storage_controllers() + + def _get_prev_state_events(self, event_id: str) -> list[str]: + ev = self.helper.get_event(self.room_id, event_id) + prev_state_events: list[str] | None = ev.get("prev_state_events", None) + assert prev_state_events is not None + return prev_state_events + + @override_config({"experimental_features": {"msc4242_enabled": True}}) + def test_forward_extremities_are_calculated(self) -> None: + """ + Check that forward extremities are set as prev_state_events and that they don't change + for non-state events. + """ + # they don't change for messages + first_event_id = self.helper.send(self.room_id, body="test1")["event_id"] + first_prev_state_events = self._get_prev_state_events(first_event_id) + assert len(first_prev_state_events) == 1 + second_id = self.helper.send(self.room_id, body="test2")["event_id"] + second_prev_state_events = self._get_prev_state_events(second_id) + assert len(second_prev_state_events) == 1 + self.assertIncludes( + set(first_prev_state_events), set(second_prev_state_events), exact=True + ) + + # send an auth event, which should change the prev_state_events on *subsequent* events + join_rule_state_event_id = self.helper.send_state( + self.room_id, + EventTypes.JoinRules, + { + "join_rule": "knock", + }, + tok="nope", + )["event_id"] + join_rule_prev_state_event_ids = self._get_prev_state_events( + join_rule_state_event_id + ) + self.assertIncludes( + set(second_prev_state_events), + set(join_rule_prev_state_event_ids), + exact=True, + ) + + # prev_state_events should always point to the join rule now + third_event_id = self.helper.send(self.room_id, body="test3")["event_id"] + third_prev_state_events = self._get_prev_state_events(third_event_id) + self.assertIncludes( + set(third_prev_state_events), {join_rule_state_event_id}, exact=True + ) + # and non-auth state should also update prev_state_events + name_state_event_id = self.helper.send_state( + self.room_id, + EventTypes.Name, + { + "name": "State DAGs!", + }, + tok="nope", + )["event_id"] + name_prev_state_event_ids = self._get_prev_state_events(name_state_event_id) + self.assertIncludes( + set(name_prev_state_event_ids), {join_rule_state_event_id}, exact=True + ) + fourth_event_id = self.helper.send(self.room_id, body="test4")["event_id"] + fourth_prev_state_events = self._get_prev_state_events(fourth_event_id) + self.assertIncludes( + set(fourth_prev_state_events), {name_state_event_id}, exact=True + ) + + +class MSC4242EventPersistenceStateDagsStoreTestCase(HomeserverTestCase): + servlets = [ + room.register_servlets, + ] + + def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: + self.store = hs.get_datastores().main + persistence = hs.get_storage_controllers().persistence + assert persistence is not None + self.persistence = persistence + self.room_id = "!foo:bar" + self.seen_event_ids: set[str] = set() + self.persistence.main_store = Mock(spec=["have_seen_events"]) + self.persistence.main_store.have_seen_events.side_effect = ( + self._have_seen_events + ) + self.rejected_event_ids_and_their_prevs: set[str] = set() + self.persistence.persist_events_store = Mock( + spec=["_get_prevs_before_rejected"] + ) + self.persistence.persist_events_store._get_prevs_before_rejected.side_effect = ( + self._get_prevs_before_rejected + ) + + async def _have_seen_events( + self, room_id: str, event_ids: Iterable[str] + ) -> set[str]: + unknown_events = set(event_ids) + return self.seen_event_ids.intersection(unknown_events) + + async def _get_prevs_before_rejected( + self, event_ids: Iterable[str], include_soft_failed: bool = True + ) -> set[str]: + return self.rejected_event_ids_and_their_prevs + + def _make_event( + self, + id: str, + prev_state_events: list[str], + rejected: bool = False, + ) -> tuple[FrozenEventVMSC4242, EventContext]: + ev = make_event_from_dict( + { + "prev_state_events": prev_state_events, + "content": { + "membership": "join", + }, + "sender": "@unimportant:info", + "state_key": "@unimportant:info", + "type": "m.room.member", + "room_id": self.room_id, + }, + room_version=RoomVersions.MSC4242v12, + ) + assert isinstance(ev, FrozenEventVMSC4242) + ev._event_id = id + ctx = Mock() + ctx.rejected = rejected + return ev, ctx + + def _test( + self, + current_fwds: list[str], + new_events: list[tuple[FrozenEventVMSC4242, EventContext]], + want_new_extrems: set[str], + want_raises: bool = False, + ) -> None: + """ + Tests the logic of _calculate_new_state_dag_extremities. + + Tests that the new extremities calculated as a result of processing current_fwds and new_events + matches want_new_extrems or raises if want_raises is True. + """ + coroutine = self.persistence._calculate_new_state_dag_extremities( + self.room_id, + frozenset(current_fwds), + new_events, + ) + if want_raises: + f = self.get_failure(coroutine, SynapseError) + assert f is not None + return + + new_extrems = set(self.get_success(coroutine)) + self.assertIncludes( + new_extrems, + set(want_new_extrems), + exact=True, + message=f"want_new_extrems={want_new_extrems} got={new_extrems}", + ) + + @override_config({"experimental_features": {"msc4242_enabled": True}}) + def test_calculate_new_state_dag_extremities_simple(self) -> None: + # Simple linear chain + self._test( + current_fwds=[], + new_events=[ + self._make_event("$1", []), + self._make_event("$2", ["$1"]), + self._make_event("$3", ["$2"]), + self._make_event("$4", ["$3"]), + ], + want_new_extrems={"$4"}, + ) + + @override_config({"experimental_features": {"msc4242_enabled": True}}) + def test_calculate_new_state_dag_extremities_fork(self) -> None: + # Simple fork so we end up with two forward extrems + self._test( + current_fwds=[], + new_events=[ + self._make_event("$1", []), + self._make_event("$2", ["$1"]), + self._make_event("$3", ["$2"]), + self._make_event("$4", ["$2"]), + ], + want_new_extrems={"$3", "$4"}, + ) + + @override_config({"experimental_features": {"msc4242_enabled": True}}) + def test_calculate_new_state_dag_extremities_merge(self) -> None: + # Simple fork so we end up with two forward extrems + self._test( + current_fwds=[], + new_events=[ + self._make_event("$1", []), + self._make_event("$2", ["$1"]), + self._make_event("$3", ["$1"]), + self._make_event("$4", ["$2", "$3"]), + ], + want_new_extrems={"$4"}, + ) + + @override_config({"experimental_features": {"msc4242_enabled": True}}) + def test_calculate_new_state_dag_extremities_fork_on_existing(self) -> None: + # Fork where we are adding to older events + self.seen_event_ids = {"$1", "$2", "$3"} + self._test( + current_fwds=["$3"], + new_events=[ + self._make_event("$4", ["$3"]), # append to the forward extrem + self._make_event("$5", ["$1"]), # append to the root + ], + want_new_extrems={"$4", "$5"}, + ) + + @override_config({"experimental_features": {"msc4242_enabled": True}}) + def test_calculate_new_state_dag_extremities_merge_on_existing(self) -> None: + # Merge where we are merging to older events + self.seen_event_ids = {"$1", "$2", "$3"} + self._test( + current_fwds=["$3"], + new_events=[ + self._make_event("$4", ["$3", "$2"]), + ], + want_new_extrems={"$4"}, + ) + + @override_config({"experimental_features": {"msc4242_enabled": True}}) + def test_calculate_new_state_dag_extremities_merge_on_not_current(self) -> None: + # Merge where we are merging to older events + self.seen_event_ids = {"$1", "$2", "$3"} + self._test( + current_fwds=["$3"], + new_events=[ + self._make_event("$4", ["$1", "$2"]), + ], + want_new_extrems={"$3", "$4"}, + ) + + @override_config({"experimental_features": {"msc4242_enabled": True}}) + def test_calculate_new_state_dag_extremities_append_with_rejected(self) -> None: + # rejected events cannot be forward extremities + self.seen_event_ids = {"$1", "$2", "$3"} + self._test( + current_fwds=["$3"], + new_events=[ + self._make_event("$4", ["$3"], rejected=True), + ], + want_new_extrems={"$3"}, + ) + + self._test( + current_fwds=["$3"], + new_events=[ + self._make_event("$4", ["$3"], rejected=True), + self._make_event("$5", ["$4"], rejected=True), + ], + want_new_extrems={"$3"}, + ) + + @override_config({"experimental_features": {"msc4242_enabled": True}}) + def test_calculate_new_state_dag_extremities_append_with_rejected_in_chain( + self, + ) -> None: + # rejected events cannot be forward extremities, but events that come after them can. + # this shouldn't cause multiple forward extremities. + self.seen_event_ids = {"$1", "$2", "$3"} + self.rejected_event_ids_and_their_prevs = {"$4", "$3"} + self._test( + current_fwds=["$3"], + new_events=[ + self._make_event("$4", ["$3"], rejected=True), + self._make_event("$5", ["$4"]), + ], + want_new_extrems={"$5"}, + ) + + @override_config({"experimental_features": {"msc4242_enabled": True}}) + def test_calculate_new_state_dag_extremities_missing_prevs_raises(self) -> None: + self._test( + current_fwds=[], + new_events=[ + self._make_event("$1", []), + self._make_event("$2", ["$1"]), + self._make_event("$3", ["$unknown"]), + self._make_event("$4", ["$3"]), + ], + want_new_extrems={"$4"}, + want_raises=True, + ) + + @override_config({"experimental_features": {"msc4242_enabled": True}}) + def test_calculate_new_state_dag_extremities_complex(self) -> None: + """ + 1 + | \ + 2 4 + | + 3 + + Exists already, then becomes... + + 1______ + | \\ | + 2 4 5R + | | | + 3--7 6R + | \\ / \ + 10R 8 9 + + """ + # Merge where we are merging to older events + self.seen_event_ids = {"$1", "$2", "$3", "$4"} + self.rejected_event_ids_and_their_prevs = {"$1", "$5", "$6", "$3", "$10"} + self._test( + current_fwds=["$3", "$4"], + new_events=[ + self._make_event("$5", ["$1"], rejected=True), + self._make_event("$6", ["$5"], rejected=True), + self._make_event("$7", ["$4", "$3"]), + self._make_event("$8", ["$6", "$7"]), + self._make_event("$9", ["$6"]), + self._make_event("$10", ["$3"], rejected=True), + ], + want_new_extrems={"$8", "$9"}, + ) diff --git a/tests/storage/test_redaction.py b/tests/storage/test_redaction.py index c82ccf1600..c346245706 100644 --- a/tests/storage/test_redaction.py +++ b/tests/storage/test_redaction.py @@ -152,7 +152,7 @@ def test_redact(self) -> None: self.assertObjectHasAttributes( { "type": EventTypes.Message, - "user_id": self.u_alice.to_string(), + "sender": self.u_alice.to_string(), "content": {"body": "t", "msgtype": "message"}, }, event, @@ -173,7 +173,7 @@ def test_redact(self) -> None: self.assertObjectHasAttributes( { "type": EventTypes.Message, - "user_id": self.u_alice.to_string(), + "sender": self.u_alice.to_string(), "content": {}, }, event, @@ -191,7 +191,7 @@ def test_redact_join(self) -> None: self.assertObjectHasAttributes( { "type": EventTypes.Member, - "user_id": self.u_bob.to_string(), + "sender": self.u_bob.to_string(), "content": {"membership": Membership.JOIN, "blue": "red"}, }, event, @@ -212,7 +212,7 @@ def test_redact_join(self) -> None: self.assertObjectHasAttributes( { "type": EventTypes.Member, - "user_id": self.u_bob.to_string(), + "sender": self.u_bob.to_string(), "content": {"membership": Membership.JOIN}, }, event, @@ -232,6 +232,7 @@ async def build( prev_event_ids: list[str], auth_event_ids: list[str] | None, depth: int | None = None, + prev_state_events: list[str] | None = None, ) -> EventBase: built_event = await self._base_builder.build( prev_event_ids=prev_event_ids, auth_event_ids=auth_event_ids @@ -328,7 +329,7 @@ def test_redact_censor(self) -> None: self.assertObjectHasAttributes( { "type": EventTypes.Message, - "user_id": self.u_alice.to_string(), + "sender": self.u_alice.to_string(), "content": {"body": "t", "msgtype": "message"}, }, event, @@ -347,7 +348,7 @@ def test_redact_censor(self) -> None: self.assertObjectHasAttributes( { "type": EventTypes.Message, - "user_id": self.u_alice.to_string(), + "sender": self.u_alice.to_string(), "content": {}, }, event, diff --git a/tests/storage/test_room.py b/tests/storage/test_room.py index f8c5260fa2..b4fb5d5628 100644 --- a/tests/storage/test_room.py +++ b/tests/storage/test_room.py @@ -23,9 +23,12 @@ from synapse.api.room_versions import RoomVersions from synapse.server import HomeServer +from synapse.storage.databases.main.room import _BackgroundUpdates +from synapse.storage.types import Cursor from synapse.types import RoomAlias, RoomID, UserID from synapse.util.clock import Clock +from tests.rest.admin.test_media import _AdminMediaTests from tests.unittest import HomeserverTestCase @@ -67,3 +70,89 @@ def test_get_room_with_stats_unknown_room(self) -> None: self.assertIsNone( self.get_success(self.store.get_room_with_stats("!uknown:test")) ) + + +class FlagExistingQuarantinedMediaBackgroundUpdatesTestCase(_AdminMediaTests): + """ + Test the `flag_existing_quarantined_media` background update. + """ + + def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: + self.store = hs.get_datastores().main + + def test_populates_quarantined_only(self) -> None: + self.register_user("admin", "pass", admin=True) + admin_user_tok = self.login("admin", "pass") + + # Upload two distinct media items so we can quarantine one. If they shared content, + # then the quarantine-by-hash code would hit both. + _unaffected_media_upload_response = self.helper.upload_media( + b"first content", tok=admin_user_tok, expect_code=200 + ) + # Upload the media we're going to quarantine + media_upload_response = self.helper.upload_media( + b"second content", tok=admin_user_tok, expect_code=200 + ) + # Extract media ID from the response + quarantined_media_origin_and_media_id = media_upload_response["content_uri"][ + 6: + ] # cut off 'mxc://' + quarantined_media_origin, quarantined_media_id = ( + quarantined_media_origin_and_media_id.split("/") + ) + + # Ideally we'd also upload remote media to ensure that `remote_media_cache` gets picked up, but that's + # a little tricky to set up in the test here. We hope that local and remote media + # are treated similarly during the background update. + + # Quarantine the media like an admin would. Because the quarantine API also inserts + # a record into the database for us, we'll clear out the `quarantined_media_changes` + # table before running the background update. This will simulate already-quarantined + # media being in the database prior to the background update. + channel = self.make_request( + "POST", + "/_synapse/admin/v1/media/quarantine/%s/%s" + % ( + quarantined_media_origin, + quarantined_media_id, + ), + access_token=admin_user_tok, + ) + self.assertEqual(200, channel.code, msg=channel.json_body) + + # Do that table clear we mentioned above + def _wipe_table(txn: Cursor) -> None: + txn.execute("DELETE FROM quarantined_media_changes") + + self.get_success( + self.store.db_pool.runInteraction( + "test_populates_quarantined_only._wipe_table", _wipe_table + ) + ) + + # Insert and run the background update + self.get_success( + self.store.db_pool.simple_insert( + table="background_updates", + values={ + "update_name": _BackgroundUpdates.FLAG_EXISTING_QUARANTINED_MEDIA, + "progress_json": "{}", + }, + ) + ) + self.store.db_pool.updates._all_done = False + self.wait_for_background_updates() + + # Check that the changes table is now populated, and has exactly 1 quarantined + # media object in it (the one we quarantined). + changes: list[tuple[str | None, str, bool]] = self.get_success( + self.store.db_pool.simple_select_list( + "quarantined_media_changes", + None, + retcols=("origin", "media_id", "quarantined"), + ) + ) + self.assertEqual(len(changes), 1) + self.assertEqual(changes[0][0], None) # origin (local media) + self.assertEqual(changes[0][1], quarantined_media_id) # media_id + self.assertEqual(changes[0][2], True) # quarantined flag diff --git a/tests/storage/test_roommember.py b/tests/storage/test_roommember.py index f8d64e8ce6..711c9ba4ac 100644 --- a/tests/storage/test_roommember.py +++ b/tests/storage/test_roommember.py @@ -20,7 +20,7 @@ # # import logging -from typing import cast +from typing import Any, cast from twisted.internet.testing import MemoryReactor @@ -238,7 +238,7 @@ def test_join_locally_forgotten_room(self) -> None: creator = "@user:other" room_id = "!foo:other" room_version = RoomVersions.V10 - shared_kwargs = { + shared_kwargs: dict[str, Any] = { "room_id": room_id, "room_version": room_version.identifier, } diff --git a/tests/storage/test_sliding_sync_tables.py b/tests/storage/test_sliding_sync_tables.py index f5bbd49663..39226ff9be 100644 --- a/tests/storage/test_sliding_sync_tables.py +++ b/tests/storage/test_sliding_sync_tables.py @@ -18,7 +18,7 @@ # # import logging -from typing import cast +from typing import Any, cast import attr from parameterized import parameterized @@ -873,7 +873,7 @@ def test_joined_room_bump_stamp_backfill(self) -> None: creator = "@user:other" room_id = "!foo:other" room_version = RoomVersions.V10 - shared_kwargs = { + shared_kwargs: dict[str, Any] = { "room_id": room_id, "room_version": room_version.identifier, } diff --git a/tests/storage/test_sticky_events.py b/tests/storage/test_sticky_events.py index 60243cb2f4..e77b362f52 100644 --- a/tests/storage/test_sticky_events.py +++ b/tests/storage/test_sticky_events.py @@ -30,6 +30,7 @@ from synapse.util.duration import Duration from tests import unittest +from tests.test_utils.event_injection import inject_event from tests.utils import USE_POSTGRES_FOR_TESTS @@ -276,3 +277,154 @@ def test_outlier_events_not_in_table(self) -> None: ) self.assertEqual(len(updates), 1) self.assertEqual(updates[0].event_id, event_non_outlier.event_id) + + def test_soft_failed_events_are_tracked(self) -> None: + """ + Tests that sticky events marked as soft_failed ARE inserted + into the sticky_events table, as their soft-failed status can be re-evaluated later, + as per MSC4354. + """ + user_id = self.register_user("testuser", "pass") + token = self.login(user_id, "pass") + room_id = self.helper.create_room_as(user_id, tok=token) + + start_id = self.store.get_max_sticky_events_stream_id() + + # Create and persist a sticky event that is soft-failed + soft_failed_sticky_event = self.get_success( + inject_event( + self.hs, + room_id=room_id, + sender=user_id, + type=EventTypes.Message, + content={"body": "spam checker spammy message", "msgtype": "m.text"}, + internal_metadata={"soft_failed": True}, + # Corresponds to StickyEvent.EVENT_FIELD_NAME + msc4354_sticky=StickyEventField( + duration_ms=Duration(minutes=1).as_millis() + ), + ) + ) + + end_id = self.store.get_max_sticky_events_stream_id() + + updates = self.get_success( + self.store.get_updated_sticky_events( + from_id=start_id, to_id=end_id, limit=10 + ) + ) + + self.assertEqual(len(updates), 1) + self.assertEqual(updates[0].event_id, soft_failed_sticky_event.event_id) + + def test_policy_server_spammy_events_are_not_tracked(self) -> None: + """ + Tests that sticky events marked as policy_server_spammy are NOT inserted + into the sticky_events table, as they are exempt from the soft-failed + re-evaluation logic. + """ + user_id = self.register_user("testuser", "pass") + token = self.login(user_id, "pass") + room_id = self.helper.create_room_as(user_id, tok=token) + + start_id = self.store.get_max_sticky_events_stream_id() + + # Create and persist a sticky event that is marked policy_server_spammy + # N.B. policy_server_spammy events are always soft-failed too + _spammy_sticky_event = self.get_success( + inject_event( + self.hs, + room_id=room_id, + sender=user_id, + type=EventTypes.Message, + content={"body": "spam checker spammy message", "msgtype": "m.text"}, + internal_metadata={"soft_failed": True, "policy_server_spammy": True}, + # Corresponds to StickyEvent.EVENT_FIELD_NAME + msc4354_sticky=StickyEventField( + duration_ms=Duration(minutes=1).as_millis() + ), + ) + ) + + # Also insert a valid sticky event as a canary for the test setup + valid_sticky_event = self.get_success( + inject_event( + self.hs, + room_id=room_id, + sender=user_id, + type=EventTypes.Message, + content={"body": "normal sticky", "msgtype": "m.text"}, + # Corresponds to StickyEvent.EVENT_FIELD_NAME + msc4354_sticky=StickyEventField( + duration_ms=Duration(minutes=1).as_millis() + ), + ) + ) + + end_id = self.store.get_max_sticky_events_stream_id() + + # Verify only the regular event was inserted + updates = self.get_success( + self.store.get_updated_sticky_events( + from_id=start_id, to_id=end_id, limit=10 + ) + ) + + self.assertEqual(len(updates), 1) + self.assertEqual(updates[0].event_id, valid_sticky_event.event_id) + + def test_spam_checker_spammy_events_are_not_tracked(self) -> None: + """ + Tests that sticky events marked as spam_checker_spammy are NOT inserted + into the sticky_events table, as they are exempt from the soft-failed + re-evaluation logic. + """ + user_id = self.register_user("testuser", "pass") + token = self.login(user_id, "pass") + room_id = self.helper.create_room_as(user_id, tok=token) + + start_id = self.store.get_max_sticky_events_stream_id() + + # Create and persist a sticky event that is marked spam_checker_spammy + # N.B. spam_checker_spammy events are always soft-failed too + _spammy_sticky_event = self.get_success( + inject_event( + self.hs, + room_id=room_id, + sender=user_id, + type=EventTypes.Message, + content={"body": "spam checker spammy message", "msgtype": "m.text"}, + internal_metadata={"soft_failed": True, "spam_checker_spammy": True}, + # Corresponds to StickyEvent.EVENT_FIELD_NAME + msc4354_sticky=StickyEventField( + duration_ms=Duration(minutes=1).as_millis() + ), + ) + ) + + # Also insert a valid sticky event as a canary for the test setup + valid_sticky_event = self.get_success( + inject_event( + self.hs, + room_id=room_id, + sender=user_id, + type=EventTypes.Message, + content={"body": "normal sticky", "msgtype": "m.text"}, + # Corresponds to StickyEvent.EVENT_FIELD_NAME + msc4354_sticky=StickyEventField( + duration_ms=Duration(minutes=1).as_millis() + ), + ) + ) + + end_id = self.store.get_max_sticky_events_stream_id() + + # Verify only the valid sticky event was inserted + updates = self.get_success( + self.store.get_updated_sticky_events( + from_id=start_id, to_id=end_id, limit=10 + ) + ) + + self.assertEqual(len(updates), 1) + self.assertEqual(updates[0].event_id, valid_sticky_event.event_id) diff --git a/tests/test_event_auth.py b/tests/test_event_auth.py index 934a2fd307..9258f0d4dc 100644 --- a/tests/test_event_auth.py +++ b/tests/test_event_auth.py @@ -20,15 +20,16 @@ # import unittest +from collections import namedtuple from typing import Any, Collection, Iterable from parameterized import parameterized from synapse import event_auth -from synapse.api.constants import EventContentFields +from synapse.api.constants import EventContentFields, RejectedReason from synapse.api.errors import AuthError, SynapseError from synapse.api.room_versions import EventFormatVersions, RoomVersion, RoomVersions -from synapse.events import EventBase, make_event_from_dict +from synapse.events import EventBase, event_exists_in_state_dag, make_event_from_dict from synapse.storage.databases.main.events_worker import EventRedactBehaviour from synapse.types import JsonDict, get_domain_from_id @@ -374,6 +375,195 @@ def test_msc2432_alias_event(self) -> None: auth_events, ) + def test_msc4242_state_dag_rules(self) -> None: + """Tests additional rules in place for state DAG rooms. + + 1. m.room.create => if it has any prev_state_events, reject. + 2. Considering the event's prev_state_events: + i. If there are entries which do not belong in the same room, reject. + ii. If there are entries which do not have a state_key, reject. + iii. If there are entries which were themselves rejected under the checks performed on receipt of a PDU, reject. + """ + creator = "@creator:example.com" + room_version = RoomVersions.MSC4242v12 + + create_event = make_event_from_dict( + { + "type": "m.room.create", + "sender": creator, + "state_key": "", + "content": {"creator": creator}, + "prev_events": [], + "prev_state_events": [], + }, + room_version, + ) + create_event_2 = make_event_from_dict( + { + "type": "m.room.create", + "sender": creator, + "state_key": "", + "content": {"creator": creator, "another": "room"}, + "prev_events": [], + "prev_state_events": [], + }, + room_version, + ) + room_id = create_event.room_id + another_room_id = create_event_2.room_id + join_event = make_event_from_dict( + { + "room_id": room_id, + "type": "m.room.member", + "sender": creator, + "state_key": creator, + "content": {"membership": "join"}, + "prev_events": [create_event.event_id], + "prev_state_events": [create_event.event_id], + }, + room_version, + {"calculated_auth_event_ids": [create_event.event_id]}, + ) + event_in_another_room = make_event_from_dict( + { + "room_id": another_room_id, + "type": "m.room.join_rules", + "sender": creator, + "state_key": "", + "content": {"join_rule": "public"}, + "prev_events": [join_event.event_id], + "prev_state_events": [join_event.event_id], + }, + room_version, + {"calculated_auth_event_ids": [create_event.event_id, join_event.event_id]}, + ) + msg_event = make_event_from_dict( + { + "room_id": room_id, + "type": "m.room.message", + "sender": creator, + "content": {"msgtype": "m.text", "body": "I am a message"}, + "prev_events": [join_event.event_id], + "prev_state_events": [join_event.event_id], + }, + room_version, + {"calculated_auth_event_ids": [create_event.event_id, join_event.event_id]}, + ) + rejected_event = make_event_from_dict( + { + "room_id": room_id, + "type": "m.room.name", + "sender": creator, + "state_key": "", + "content": {"name": "REJECTED"}, + "prev_events": [join_event.event_id], + "prev_state_events": [join_event.event_id], + }, + room_version, + {"calculated_auth_event_ids": [create_event.event_id, join_event.event_id]}, + rejected_reason=RejectedReason.AUTH_ERROR, + ) + RejectingTestCase = namedtuple( + "RejectingTestCase", "name events_in_store test_event" + ) + rejecting_test_cases = [ + RejectingTestCase( + name="create event has prev_state_events", + events_in_store=[], + test_event=make_event_from_dict( + { + "type": "m.room.create", + "sender": creator, + "state_key": "", + "content": {"creator": creator}, + "prev_events": [], + "prev_state_events": [create_event.event_id], + }, + room_version, + {}, + ), + ), + RejectingTestCase( + name="prev_state_event belongs in a different room", + events_in_store=[create_event, join_event, event_in_another_room], + test_event=make_event_from_dict( + { + "room_id": room_id, + "type": "m.room.name", + "sender": creator, + "state_key": "", + "content": {"name": "prev_state_event is in another room"}, + "prev_events": [join_event.event_id], + "prev_state_events": [event_in_another_room.event_id], + }, + room_version, + { + "calculated_auth_event_ids": [ + create_event.event_id, + join_event.event_id, + ] + }, + ), + ), + RejectingTestCase( + name="prev_state_event is a message event", + events_in_store=[create_event, join_event, msg_event], + test_event=make_event_from_dict( + { + "room_id": room_id, + "type": "m.room.name", + "sender": creator, + "state_key": "", + "content": {"name": "prev state event is a message"}, + "prev_events": [msg_event.event_id], + "prev_state_events": [msg_event.event_id], + }, + room_version, + { + "calculated_auth_event_ids": [ + create_event.event_id, + join_event.event_id, + ] + }, + ), + ), + RejectingTestCase( + name="prev_state_event was rejected", + events_in_store=[create_event, join_event, rejected_event], + test_event=make_event_from_dict( + { + "room_id": room_id, + "type": "m.room.name", + "sender": creator, + "state_key": "", + "content": {"name": "prev state event was rejected"}, + "prev_events": [join_event.event_id], + "prev_state_events": [rejected_event.event_id], + }, + room_version, + { + "calculated_auth_event_ids": [ + create_event.event_id, + join_event.event_id, + ] + }, + ), + ), + ] + + for test_case in rejecting_test_cases: + event_store = _StubEventSourceStore() + event_store.add_events(test_case.events_in_store) + + with self.assertRaises( + AuthError, msg=f"test case {test_case.name} was not rejected" + ): + get_awaitable_result( + event_auth.check_state_independent_auth_rules( + event_store, test_case.test_event + ) + ) + @parameterized.expand([(RoomVersions.V1, True), (RoomVersions.V6, False)]) def test_notifications( self, room_version: RoomVersion, allow_modification: bool @@ -769,6 +959,105 @@ def create_event(pl_event_content: dict[str, Any]) -> EventBase: with self.assertRaises(SynapseError): event_auth._check_power_levels(event.room_version, event, {}) + def test_event_exists_in_state_dag(self) -> None: + events_that_exist_in_state_dag = [ + { + "type": "m.room.create", + "state_key": "", + "content": {}, + }, + { + "type": "m.room.join_rules", + "state_key": "", + "content": {}, + }, + { + "type": "m.room.power_levels", + "state_key": "", + "content": {}, + }, + { + "type": "m.room.server_acl", + "state_key": "", + "content": {}, + }, + { + "type": "m.room.member", + "state_key": "@alice:somewhere", + "content": {}, + }, + { + "type": "m.room.third_party_invite", + "state_key": "flibble", + "content": {}, + }, + { + "type": "m.room.create", + "state_key": " ", + "content": {}, + }, + { + "type": "m.room.join_rules", + "state_key": " ", + "content": {}, + }, + { + "type": "m.room.power_levels", + "state_key": " ", + "content": {}, + }, + { + "type": "m.room.name", + "state_key": "", + "content": {}, + }, + { + "type": "m.room.member", + "state_key": "", + "content": {}, + }, + { + "type": "m.room.member", + "state_key": "hello_world", + "content": {}, + }, + ] + events_that_dont_exist_in_state_dag = [ + { + "type": "m.room.message", + "content": {}, + }, + { + "type": "m.room.create", + "content": {}, + }, + { + "type": "m.room.join_rules", + "content": {}, + }, + { + "type": "m.room.power_levels", + "content": {}, + }, + ] + + def check_events(events: list[dict], should_exist: bool) -> None: + for ev in events: + base = { + "room_id": TEST_ROOM_ID, + "sender": "@test:test.com", + "signatures": {"test.com": {"ed25519:0": "some9signature"}}, + } + base.update(ev) + event = make_event_from_dict(base, RoomVersions.V10) + got = event_exists_in_state_dag(event) + self.assertEqual( + got, should_exist, f"{ev} should_exist={should_exist} but got {got}" + ) + + check_events(events_that_exist_in_state_dag, should_exist=True) + check_events(events_that_dont_exist_in_state_dag, should_exist=False) + # helpers for making events TEST_DOMAIN = "example.com" diff --git a/tests/test_utils/event_injection.py b/tests/test_utils/event_injection.py index a90fc5884d..36f6baa123 100644 --- a/tests/test_utils/event_injection.py +++ b/tests/test_utils/event_injection.py @@ -18,7 +18,7 @@ # [This file includes modifications made by New Vector Limited] # # -from typing import Any +from typing import Any, Mapping import synapse.server from synapse.api.constants import EventTypes @@ -63,6 +63,8 @@ async def inject_event( hs: synapse.server.HomeServer, room_version: str | None = None, prev_event_ids: list[str] | None = None, + *, + internal_metadata: Mapping[str, Any] | None = None, **kwargs: Any, ) -> EventBase: """Inject a generic event into a room @@ -72,9 +74,12 @@ async def inject_event( room_version: the version of the room we're inserting into. if not specified, will be looked up prev_event_ids: prev_events for the event. If not specified, will be looked up + internal_metadata: Dict representing the event's internal metadata; see `EventBase.internal_metadata` kwargs: fields for the event to be created """ - event, context = await create_event(hs, room_version, prev_event_ids, **kwargs) + event, context = await create_event( + hs, room_version, prev_event_ids, internal_metadata=internal_metadata, **kwargs + ) persistence = hs.get_storage_controllers().persistence assert persistence is not None @@ -88,8 +93,11 @@ async def create_event( hs: synapse.server.HomeServer, room_version: str | None = None, prev_event_ids: list[str] | None = None, + *, + internal_metadata: Mapping[str, Any] | None = None, **kwargs: Any, ) -> tuple[EventBase, EventContext]: + internal_metadata = internal_metadata or {} if room_version is None: room_version = await hs.get_datastores().main.get_room_version_id( kwargs["room_id"] @@ -106,11 +114,13 @@ async def create_event( ) # Copy over writable internal_metadata, if set - # Dev note: This isn't everything that's writable. `for k,v` doesn't work here :( - if kwargs.get("internal_metadata", {}).get("soft_failed", False): - event.internal_metadata.soft_failed = True - if kwargs.get("internal_metadata", {}).get("policy_server_spammy", False): - event.internal_metadata.policy_server_spammy = True + if internal_metadata: + for key, value in internal_metadata.items(): + # Note: this calls the relevant `#[setter]` function in the (Rust) event class' + # internal metadata struct. + # Will reject unknown keys with exceptions. + # This is desirable for our test suite anyway. + setattr(event.internal_metadata, key, value) context = await unpersisted_context.persist(event) diff --git a/tests/test_visibility.py b/tests/test_visibility.py index b50faa2a49..9a5efbdd39 100644 --- a/tests/test_visibility.py +++ b/tests/test_visibility.py @@ -22,7 +22,7 @@ from twisted.test.proto_helpers import MemoryReactor -from synapse.api.constants import AccountDataTypes, EventUnsignedContentFields +from synapse.api.constants import AccountDataTypes from synapse.api.room_versions import RoomVersions from synapse.events import EventBase, make_event_from_dict from synapse.events.snapshot import EventContext @@ -341,7 +341,7 @@ def test_normal_operation_as_admin(self) -> None: ) self.assertEqual( [e.event_id for e in [self.regular_event]], - [e.event_id for e in filtered_events], + [e.event.event_id for e in filtered_events], ) def test_see_soft_failed_events(self) -> None: @@ -380,7 +380,7 @@ def test_see_soft_failed_events(self) -> None: ) self.assertEqual( [e.event_id for e in [self.regular_event, self.soft_failed_event]], - [e.event_id for e in filtered_events], + [e.event.event_id for e in filtered_events], ) def test_see_policy_server_spammy_events(self) -> None: @@ -427,7 +427,7 @@ def test_see_policy_server_spammy_events(self) -> None: ) self.assertEqual( [e.event_id for e in [self.regular_event, self.spammy_event]], - [e.event_id for e in filtered_events], + [e.event.event_id for e in filtered_events], ) def test_see_soft_failed_and_policy_server_spammy_events(self) -> None: @@ -477,7 +477,7 @@ def test_see_soft_failed_and_policy_server_spammy_events(self) -> None: e.event_id for e in [self.regular_event, self.soft_failed_event, self.spammy_event] ], - [e.event_id for e in filtered_events], + [e.event.event_id for e in filtered_events], ) @@ -559,14 +559,11 @@ def test_joined_history_visibility(self) -> None: # and messages sent between the two, but not before or after. self.assertEqual( [e.event_id for e in [join_event, during_event, leave_event]], - [e.event_id for e in joiner_filtered_events], + [e.event.event_id for e in joiner_filtered_events], ) self.assertEqual( ["join", "join", "leave"], - [ - e.unsigned[EventUnsignedContentFields.MEMBERSHIP] - for e in joiner_filtered_events - ], + [e.membership for e in joiner_filtered_events], ) # The resident user should see all the events. @@ -581,14 +578,11 @@ def test_joined_history_visibility(self) -> None: after_event, ] ], - [e.event_id for e in resident_filtered_events], + [e.event.event_id for e in resident_filtered_events], ) self.assertEqual( ["join", "join", "join", "join", "join"], - [ - e.unsigned[EventUnsignedContentFields.MEMBERSHIP] - for e in resident_filtered_events - ], + [e.membership for e in resident_filtered_events], ) @@ -651,15 +645,12 @@ def test_out_of_band_invite_rejection(self) -> None: ) ) self.assertEqual( - [e.event_id for e in filtered_events], + [e.event.event_id for e in filtered_events], [e.event_id for e in [invite_event, reject_event]], ) self.assertEqual( ["invite", "leave"], - [ - e.unsigned[EventUnsignedContentFields.MEMBERSHIP] - for e in filtered_events - ], + [e.membership for e in filtered_events], ) # other users should see neither