From 3651bf52cec47a16297c676b4b6b8eff280189e5 Mon Sep 17 00:00:00 2001 From: Sumner Evans Date: Fri, 25 Mar 2022 01:01:09 -0600 Subject: [PATCH 01/29] Remove re-assignment of self.clock in UploadResource Signed-off-by: Sumner Evans --- synapse/rest/media/upload_resource.py | 1 - 1 file changed, 1 deletion(-) diff --git a/synapse/rest/media/upload_resource.py b/synapse/rest/media/upload_resource.py index 949326d85dac..066d2e534f18 100644 --- a/synapse/rest/media/upload_resource.py +++ b/synapse/rest/media/upload_resource.py @@ -42,7 +42,6 @@ def __init__(self, hs: "HomeServer", media_repo: "MediaRepository"): self.clock = hs.get_clock() self.auth = hs.get_auth() self.max_upload_size = hs.config.media.max_upload_size - self.clock = hs.get_clock() async def on_POST(self, request: SynapseRequest) -> None: requester = await self.auth.get_user_by_req(request) From 1bcb58777ca67c1234381fa0fb57df59baa2e523 Mon Sep 17 00:00:00 2001 From: Sumner Evans Date: Fri, 15 Apr 2022 09:01:08 -0600 Subject: [PATCH 02/29] config: add option to rate limit media creation Signed-off-by: Sumner Evans --- docs/usage/configuration/config_documentation.md | 13 +++++++++++++ synapse/config/ratelimiting.py | 7 +++++++ 2 files changed, 20 insertions(+) diff --git a/docs/usage/configuration/config_documentation.md b/docs/usage/configuration/config_documentation.md index a673975e0411..1608dfca4476 100644 --- a/docs/usage/configuration/config_documentation.md +++ b/docs/usage/configuration/config_documentation.md @@ -1753,6 +1753,19 @@ rc_third_party_invite: burst_count: 10 ``` --- +### `rc_media_create` + +This option ratelimits creation of MXC URIs via the `/_matrix/media/v1/create` +endpoint based on the account that's creating the media. Defaults to +`per_second: 10`, `burst_count: 50`. + +Example configuration: +```yaml +rc_media_create: + per_second: 10 + burst_count: 50 +``` +--- ### `rc_federation` Defines limits on federation requests. diff --git a/synapse/config/ratelimiting.py b/synapse/config/ratelimiting.py index 4efbaeac0d7f..b1fcaf71a3d1 100644 --- a/synapse/config/ratelimiting.py +++ b/synapse/config/ratelimiting.py @@ -204,3 +204,10 @@ def read_config(self, config: JsonDict, **kwargs: Any) -> None: "rc_third_party_invite", defaults={"per_second": 0.0025, "burst_count": 5}, ) + + # Ratelimit create media requests: + self.rc_media_create = RatelimitSettings.parse( + config, + "rc_media_create", + defaults={"per_second": 10, "burst_count": 50}, + ) From f9e0504fd2c67ee4964a80a92e05de2bd06342d6 Mon Sep 17 00:00:00 2001 From: Sumner Evans Date: Tue, 31 May 2022 09:19:57 -0600 Subject: [PATCH 03/29] config: add option for how long to wait until media ID expires --- docs/usage/configuration/config_documentation.md | 10 ++++++++++ synapse/config/repository.py | 4 ++++ 2 files changed, 14 insertions(+) diff --git a/docs/usage/configuration/config_documentation.md b/docs/usage/configuration/config_documentation.md index 1608dfca4476..e77d84372cb6 100644 --- a/docs/usage/configuration/config_documentation.md +++ b/docs/usage/configuration/config_documentation.md @@ -1827,6 +1827,16 @@ Example configuration: media_store_path: "DATADIR/media_store" ``` --- +### `unused_expiration_time` + +How long to wait in milliseconds before expiring created media IDs. Defaults to +"24h" + +Example configuration: +```yaml +unused_expiration_time: "1h" +``` +--- ### `media_storage_providers` Media storage providers allow media to be stored in different diff --git a/synapse/config/repository.py b/synapse/config/repository.py index f6cfdd3e048a..dba9926779ba 100644 --- a/synapse/config/repository.py +++ b/synapse/config/repository.py @@ -141,6 +141,10 @@ def read_config(self, config: JsonDict, **kwargs: Any) -> None: "prevent_media_downloads_from", [] ) + self.unused_expiration_time = self.parse_duration( + config.get("unused_expiration_time", "24h") + ) + self.media_store_path = self.ensure_directory( config.get("media_store_path", "media_store") ) From f0d53e873f8e1d328314e7bf7c5a6bd98472425a Mon Sep 17 00:00:00 2001 From: Sumner Evans Date: Thu, 20 Apr 2023 01:45:32 -0600 Subject: [PATCH 04/29] media/create: add /create endpoint See matrix-org/matrix-spec-proposals#2246 for details Signed-off-by: Sumner Evans --- synapse/media/media_repository.py | 24 +++++ synapse/rest/media/create_resource.py | 87 +++++++++++++++++++ .../rest/media/media_repository_resource.py | 3 +- .../databases/main/media_repository.py | 19 ++++ .../80/02_add_unused_expires_at_for_media.sql | 22 +++++ 5 files changed, 154 insertions(+), 1 deletion(-) create mode 100644 synapse/rest/media/create_resource.py create mode 100644 synapse/storage/schema/main/delta/80/02_add_unused_expires_at_for_media.sql diff --git a/synapse/media/media_repository.py b/synapse/media/media_repository.py index 1957426c6ae7..8ec91fdf8883 100644 --- a/synapse/media/media_repository.py +++ b/synapse/media/media_repository.py @@ -80,6 +80,7 @@ def __init__(self, hs: "HomeServer"): self.store = hs.get_datastores().main self.max_upload_size = hs.config.media.max_upload_size self.max_image_pixels = hs.config.media.max_image_pixels + self.unused_expiration_time = hs.config.media.unused_expiration_time Thumbnailer.set_limits(self.max_image_pixels) @@ -185,6 +186,29 @@ def mark_recently_accessed(self, server_name: Optional[str], media_id: str) -> N else: self.recently_accessed_locals.add(media_id) + @trace + async def create_media_id(self, auth_user: UserID) -> Tuple[str, int]: + """Create and store a media ID for a local user and return the MXC URI and its + expiration. + + Args: + auth_user: The user_id of the uploader + + Returns: + A tuple containing the MXC URI of the stored content and the timestamp at + which the MXC URI expires. + """ + media_id = random_string(24) + now = self.clock.time_msec() + unused_expires_at = now + self.unused_expiration_time + await self.store.store_local_media_id( + media_id=media_id, + time_now_ms=now, + user_id=auth_user, + unused_expires_at=unused_expires_at, + ) + return f"mxc://{self.server_name}/{media_id}", unused_expires_at + @trace async def create_content( self, diff --git a/synapse/rest/media/create_resource.py b/synapse/rest/media/create_resource.py new file mode 100644 index 000000000000..df79d528cb4d --- /dev/null +++ b/synapse/rest/media/create_resource.py @@ -0,0 +1,87 @@ +# Copyright 2023 Beeper Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import re +from typing import TYPE_CHECKING + +from synapse.api.errors import LimitExceededError +from synapse.api.ratelimiting import Ratelimiter +from synapse.http.server import respond_with_json +from synapse.http.servlet import RestServlet +from synapse.http.site import SynapseRequest + +if TYPE_CHECKING: + from synapse.media.media_repository import MediaRepository + from synapse.server import HomeServer + +logger = logging.getLogger(__name__) + + +class CreateResource(RestServlet): + PATTERNS = [re.compile("/_matrix/media/v1/create")] + + def __init__(self, hs: "HomeServer", media_repo: "MediaRepository"): + super().__init__() + + self.media_repo = media_repo + self.clock = hs.get_clock() + self.auth = hs.get_auth() + + # A rate limiter for creating new media IDs. + self._create_media_rate_limiter = Ratelimiter( + store=hs.get_datastores().main, + clock=self.clock, + cfg=hs.config.ratelimiting.rc_media_create, + ) + + async def _async_render_OPTIONS(self, request: SynapseRequest) -> None: + respond_with_json(request, 200, {}, send_cors=True) + + async def _async_render_POST(self, request: SynapseRequest) -> None: + requester = await self.auth.get_user_by_req(request) + + # If the create media requests for the user are over the limit, drop them. + await self._create_media_rate_limiter.ratelimit(requester) + + ( + reached_pending_limit, + first_expiration_ts, + ) = await self.media_repo.reached_pending_media_limit( + requester.user, self.max_pending_media_uploads + ) + if reached_pending_limit: + raise LimitExceededError( + limiter_name="max_pending_media_uploads", + retry_after_ms=first_expiration_ts - self.clock.time_msec(), + ) + + content_uri, unused_expires_at = await self.media_repo.create_media_id( + requester.user + ) + + logger.info( + "Created Media URI %r that if unused will expire at %d", + content_uri, + unused_expires_at, + ) + respond_with_json( + request, + 200, + { + "content_uri": content_uri, + "unused_expires_at": unused_expires_at, + }, + send_cors=True, + ) diff --git a/synapse/rest/media/media_repository_resource.py b/synapse/rest/media/media_repository_resource.py index 2089bb10296c..e92b0716ca1a 100644 --- a/synapse/rest/media/media_repository_resource.py +++ b/synapse/rest/media/media_repository_resource.py @@ -18,6 +18,7 @@ from synapse.http.server import HttpServer, JsonResource from .config_resource import MediaConfigResource +from .create_resource import CreateResource from .download_resource import DownloadResource from .preview_url_resource import PreviewUrlResource from .thumbnail_resource import ThumbnailResource @@ -91,7 +92,7 @@ def register_servlets(http_server: HttpServer, hs: "HomeServer") -> None: # Note that many of these should not exist as v1 endpoints, but empirically # a lot of traffic still goes to them. - + CreateResource(hs, media_repo).register(http_server) UploadResource(hs, media_repo).register(http_server) DownloadResource(hs, media_repo).register(http_server) ThumbnailResource(hs, media_repo, media_repo.media_storage).register( diff --git a/synapse/storage/databases/main/media_repository.py b/synapse/storage/databases/main/media_repository.py index 3f80a64dc5d0..d07967c883b2 100644 --- a/synapse/storage/databases/main/media_repository.py +++ b/synapse/storage/databases/main/media_repository.py @@ -391,6 +391,25 @@ def _get_local_media_ids_txn(txn: LoggingTransaction) -> List[str]: "get_local_media_ids", _get_local_media_ids_txn ) + @trace + async def store_local_media_id( + self, + media_id: str, + time_now_ms: int, + user_id: UserID, + unused_expires_at: int, + ) -> None: + await self.db_pool.simple_insert( + "local_media_repository", + { + "media_id": media_id, + "created_ts": time_now_ms, + "user_id": user_id.to_string(), + "unused_expires_at": unused_expires_at, + }, + desc="store_local_media_id", + ) + @trace async def store_local_media( self, diff --git a/synapse/storage/schema/main/delta/80/02_add_unused_expires_at_for_media.sql b/synapse/storage/schema/main/delta/80/02_add_unused_expires_at_for_media.sql new file mode 100644 index 000000000000..ed464820cece --- /dev/null +++ b/synapse/storage/schema/main/delta/80/02_add_unused_expires_at_for_media.sql @@ -0,0 +1,22 @@ +/* Copyright 2023 Beeper Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +-- Add new colums to the `local_media_repository` to keep track of when the +-- media ID must be used by. This is to support async uploads (see MSC2246). + +ALTER TABLE local_media_repository + ADD COLUMN unused_expires_at BIGINT DEFAULT NULL; + +CREATE INDEX CONCURRENTLY ON local_media_repository (unused_expires_at); From d29b9d3768fec61c381dcf76bd42d4e662af96c6 Mon Sep 17 00:00:00 2001 From: Sumner Evans Date: Fri, 15 Apr 2022 19:24:54 -0600 Subject: [PATCH 05/29] changelog: add entry for async uploads Signed-off-by: Sumner Evans --- changelog.d/15503.feature | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/15503.feature diff --git a/changelog.d/15503.feature b/changelog.d/15503.feature new file mode 100644 index 000000000000..b6ca97a2cf50 --- /dev/null +++ b/changelog.d/15503.feature @@ -0,0 +1 @@ +Add support for asynchronous uploads as defined by [MSC2246](https://github.com/matrix-org/matrix-spec-proposals/pull/2246). Contributed by @sumnerevans at @beeper. From e2007fae0cd74cc706e73fc82e9865f131f277d8 Mon Sep 17 00:00:00 2001 From: Sumner Evans Date: Thu, 20 Apr 2023 02:14:54 -0600 Subject: [PATCH 06/29] media/upload: add support for async uploads Signed-off-by: Sumner Evans --- synapse/api/errors.py | 2 + synapse/media/media_repository.py | 70 +++++++++++++++++++ synapse/rest/media/upload_resource.py | 63 +++++++++++++++-- .../databases/main/media_repository.py | 26 +++++++ 4 files changed, 157 insertions(+), 4 deletions(-) diff --git a/synapse/api/errors.py b/synapse/api/errors.py index fdb2955be82b..fbd8b16ec39b 100644 --- a/synapse/api/errors.py +++ b/synapse/api/errors.py @@ -83,6 +83,8 @@ class Codes(str, Enum): USER_DEACTIVATED = "M_USER_DEACTIVATED" # USER_LOCKED = "M_USER_LOCKED" USER_LOCKED = "ORG_MATRIX_MSC3939_USER_LOCKED" + NOT_YET_UPLOADED = "M_NOT_YET_UPLOADED" + CANNOT_OVERWRITE_MEDIA = "M_CANNOT_OVERWRITE_MEDIA" # Part of MSC3848 # https://github.com/matrix-org/matrix-spec-proposals/pull/3848 diff --git a/synapse/media/media_repository.py b/synapse/media/media_repository.py index 8ec91fdf8883..be2942c0afd9 100644 --- a/synapse/media/media_repository.py +++ b/synapse/media/media_repository.py @@ -27,6 +27,7 @@ from twisted.internet.defer import Deferred from synapse.api.errors import ( + Codes, FederationDeniedError, HttpResponseException, NotFoundError, @@ -209,6 +210,75 @@ async def create_media_id(self, auth_user: UserID) -> Tuple[str, int]: ) return f"mxc://{self.server_name}/{media_id}", unused_expires_at + @trace + async def verify_can_upload(self, media_id: str, auth_user: UserID) -> None: + """Verify that the media ID can be uploaded to by the given user. This + function checks that: + + * the media ID exists + * the media ID does not already have content + * the user uploading is the same as the one who created the media ID + * the media ID has not expired + + Args: + media_id: The media ID to verify + auth_user: The user_id of the uploader + """ + media = await self.store.get_local_media(media_id) + if media is None: + raise SynapseError(404, "Unknow media ID", errcode=Codes.NOT_FOUND) + + if media["user_id"] != auth_user.to_string(): + raise SynapseError( + 403, + "Only the creator of the media ID can upload to it", + errcode=Codes.FORBIDDEN, + ) + + if media.get("media_length") is not None: + raise SynapseError( + 409, + "Media ID already has content", + errcode=Codes.CANNOT_OVERWRITE_MEDIA, + ) + + if media.get("unused_expires_at", 0) < self.clock.time_msec(): + raise NotFoundError("Media ID has expired") + + @trace + async def update_content( + self, + media_id: str, + media_type: str, + upload_name: Optional[str], + content: IO, + content_length: int, + auth_user: UserID, + ) -> None: + """Update the content of the given media ID. + + Args: + media_id: The media ID to replace. + media_type: The content type of the file. + upload_name: The name of the file, if provided. + content: A file like object that is the content to store + content_length: The length of the content + auth_user: The user_id of the uploader + """ + file_info = FileInfo(server_name=None, file_id=media_id) + fname = await self.media_storage.store_file(content, file_info) + logger.info("Stored local media in file %r", fname) + + await self.store.update_local_media( + media_id=media_id, + media_type=media_type, + upload_name=upload_name, + media_length=content_length, + user_id=auth_user, + ) + + await self._generate_thumbnails(None, media_id, media_id, media_type) + @trace async def create_content( self, diff --git a/synapse/rest/media/upload_resource.py b/synapse/rest/media/upload_resource.py index 066d2e534f18..b6bf0a0a8f05 100644 --- a/synapse/rest/media/upload_resource.py +++ b/synapse/rest/media/upload_resource.py @@ -15,12 +15,13 @@ import logging import re -from typing import IO, TYPE_CHECKING, Dict, List, Optional +from typing import IO, TYPE_CHECKING, Dict, List, Optional, Tuple from synapse.api.errors import Codes, SynapseError from synapse.http.server import respond_with_json from synapse.http.servlet import RestServlet, parse_bytes_from_args from synapse.http.site import SynapseRequest +from synapse.media._base import parse_media_id from synapse.media.media_storage import SpamMediaException if TYPE_CHECKING: @@ -29,6 +30,9 @@ logger = logging.getLogger(__name__) +# The name of the lock to use when uploading media. +_UPLOAD_MEDIA_LOCK_NAME = "upload_media" + class UploadResource(RestServlet): PATTERNS = [re.compile("/_matrix/media/(r0|v3|v1)/upload")] @@ -39,12 +43,13 @@ def __init__(self, hs: "HomeServer", media_repo: "MediaRepository"): self.media_repo = media_repo self.filepaths = media_repo.filepaths self.store = hs.get_datastores().main - self.clock = hs.get_clock() + self.server_name = hs.hostname self.auth = hs.get_auth() self.max_upload_size = hs.config.media.max_upload_size - async def on_POST(self, request: SynapseRequest) -> None: - requester = await self.auth.get_user_by_req(request) + def _get_file_metadata( + self, request: SynapseRequest + ) -> Tuple[int, Optional[str], str]: raw_content_length = request.getHeader("Content-Length") if raw_content_length is None: raise SynapseError(msg="Request must specify a Content-Length", code=400) @@ -87,6 +92,15 @@ async def on_POST(self, request: SynapseRequest) -> None: # disposition = headers.getRawHeaders(b"Content-Disposition")[0] # TODO(markjh): parse content-dispostion + return content_length, upload_name, media_type + + async def _async_render_OPTIONS(self, request: SynapseRequest) -> None: + respond_with_json(request, 200, {}, send_cors=True) + + async def on_POST(self, request: SynapseRequest) -> None: + requester = await self.auth.get_user_by_req(request) + content_length, upload_name, media_type = self._get_file_metadata(request) + try: content: IO = request.content # type: ignore content_uri = await self.media_repo.create_content( @@ -102,3 +116,44 @@ async def on_POST(self, request: SynapseRequest) -> None: respond_with_json( request, 200, {"content_uri": str(content_uri)}, send_cors=True ) + + async def _async_render_PUT(self, request: SynapseRequest) -> None: + requester = await self.auth.get_user_by_req(request) + server_name, media_id, _ = parse_media_id(request) + + if server_name != self.server_name: + raise SynapseError( + 404, + "Non-local server name specified", + errcode=Codes.NOT_FOUND, + ) + + lock = await self.store.try_acquire_lock(_UPLOAD_MEDIA_LOCK_NAME, media_id) + if not lock: + raise SynapseError( + 409, + "Media ID cannot be overwritten", + errcode=Codes.CANNOT_OVERWRITE_MEDIA, + ) + + async with lock: + await self.media_repo.verify_can_upload(media_id, requester.user) + content_length, upload_name, media_type = self._get_file_metadata(request) + + try: + content: IO = request.content # type: ignore + await self.media_repo.update_content( + media_id, + media_type, + upload_name, + content, + content_length, + requester.user, + ) + except SpamMediaException: + # For uploading of media we want to respond with a 400, instead of + # the default 404, as that would just be confusing. + raise SynapseError(400, "Bad content") + + logger.info("Uploaded content for media ID %r", media_id) + respond_with_json(request, 200, {}, send_cors=True) diff --git a/synapse/storage/databases/main/media_repository.py b/synapse/storage/databases/main/media_repository.py index d07967c883b2..a902414aaf98 100644 --- a/synapse/storage/databases/main/media_repository.py +++ b/synapse/storage/databases/main/media_repository.py @@ -202,6 +202,8 @@ async def get_local_media(self, media_id: str) -> Optional[LocalMedia]: "url_cache", "last_access_ts", "safe_from_quarantine", + "user_id", + "unused_expires_at", ), allow_none=True, desc="get_local_media", @@ -435,6 +437,30 @@ async def store_local_media( desc="store_local_media", ) + async def update_local_media( + self, + media_id: str, + media_type: str, + upload_name: Optional[str], + media_length: int, + user_id: UserID, + url_cache: Optional[str] = None, + ) -> None: + await self.db_pool.simple_update_one( + "local_media_repository", + keyvalues={ + "user_id": user_id.to_string(), + "media_id": media_id, + }, + updatevalues={ + "media_type": media_type, + "upload_name": upload_name, + "media_length": media_length, + "url_cache": url_cache, + }, + desc="update_local_media", + ) + async def mark_local_media_as_safe(self, media_id: str, safe: bool = True) -> None: """Mark a local media as safe or unsafe from quarantining.""" await self.db_pool.simple_update_one( From baa96f6af4899f8eea792685348cb6b91b285ae9 Mon Sep 17 00:00:00 2001 From: Sumner Evans Date: Fri, 28 Apr 2023 08:07:30 -0600 Subject: [PATCH 07/29] media/{download,thumbnail}: support timeout_ms parameter Signed-off-by: Sumner Evans Co-authored-by: Patrick Cloke --- synapse/media/_base.py | 6 ++ synapse/media/media_repository.py | 105 ++++++++++++++++++++--- synapse/rest/media/download_resource.py | 22 +++-- synapse/rest/media/thumbnail_resource.py | 59 +++++++++---- 4 files changed, 158 insertions(+), 34 deletions(-) diff --git a/synapse/media/_base.py b/synapse/media/_base.py index 860e5ddca2e3..9d88a711cf5c 100644 --- a/synapse/media/_base.py +++ b/synapse/media/_base.py @@ -83,6 +83,12 @@ "audio/x-flac", ] +# Default timeout_ms for download and thumbnail requests +DEFAULT_MAX_TIMEOUT_MS = 20_000 + +# Maximum allowed timeout_ms for download and thumbnail requests +MAXIMUM_ALLOWED_MAX_TIMEOUT_MS = 60_000 + def respond_404(request: SynapseRequest) -> None: assert request.path is not None diff --git a/synapse/media/media_repository.py b/synapse/media/media_repository.py index be2942c0afd9..133003a8d529 100644 --- a/synapse/media/media_repository.py +++ b/synapse/media/media_repository.py @@ -17,7 +17,7 @@ import os import shutil from io import BytesIO -from typing import IO, TYPE_CHECKING, Dict, List, Optional, Set, Tuple +from typing import IO, TYPE_CHECKING, Any, Dict, List, Optional, Set, Tuple import attr from matrix_common.types.mxc_uri import MXCUri @@ -33,8 +33,10 @@ NotFoundError, RequestSendFailed, SynapseError, + cs_error, ) from synapse.config.repository import ThumbnailRequirement +from synapse.http.server import respond_with_json from synapse.http.site import SynapseRequest from synapse.logging.context import defer_to_thread from synapse.logging.opentracing import trace @@ -325,8 +327,70 @@ async def create_content( return MXCUri(self.server_name, media_id) + def respond_not_yet_uploaded(self, request: SynapseRequest) -> None: + respond_with_json( + request, + 504, + cs_error("Media has not been uploaded yet", code=Codes.NOT_YET_UPLOADED), + send_cors=True, + ) + + async def get_local_media_info( + self, request: SynapseRequest, media_id: str, max_timeout_ms: int + ) -> Optional[Dict[str, Any]]: + """Gets the info dictionary for given local media ID. If the media has + not been uploaded yet, this function will wait up to ``max_timeout_ms`` + milliseconds for the media to be uploaded. + + Args: + request: The incoming request. + media_id: The media ID of the content. (This is the same as + the file_id for local content.) + max_timeout_ms: the maximum number of milliseconds to wait for the + media to be uploaded. + + Returns: + Either the info dictionary for the given local media ID or + ``None``. If ``None``, then no further processing is necessary as + this function will send the necessary JSON response. + """ + wait_until = self.clock.time_msec() + max_timeout_ms + while True: + # Get the info for the media + media_info = await self.store.get_local_media(media_id) + if not media_info: + respond_404(request) + return None + + if media_info["quarantined_by"]: + logger.info("Media is quarantined") + respond_404(request) + return None + + # The file has been uploaded, so stop looping + if media_info.get("media_length") is not None: + return media_info + + # Check if the media ID has expired and still hasn't been uploaded to. + unused_expires_at = media_info.get("unused_expires_at", 0) + if 0 < unused_expires_at < self.clock.time_msec(): + respond_404(request) + return None + + if self.clock.time_msec() >= wait_until: + break + + await self.clock.sleep(0.5) + + self.respond_not_yet_uploaded(request) + return None + async def get_local_media( - self, request: SynapseRequest, media_id: str, name: Optional[str] + self, + request: SynapseRequest, + media_id: str, + name: Optional[str], + max_timeout_ms: int, ) -> None: """Responds to requests for local media, if exists, or returns 404. @@ -336,11 +400,13 @@ async def get_local_media( the file_id for local content.) name: Optional name that, if specified, will be used as the filename in the Content-Disposition header of the response. + max_timeout_ms: the maximum number of milliseconds to wait for the + media to be uploaded. Returns: Resolves once a response has successfully been written to request """ - media_info = await self.store.get_local_media(media_id) + media_info = await self.get_local_media_info(request, media_id, max_timeout_ms) if not media_info or media_info.quarantined_by: respond_404(request) return @@ -367,6 +433,7 @@ async def get_remote_media( server_name: str, media_id: str, name: Optional[str], + max_timeout_ms: int, ) -> None: """Respond to requests for remote media. @@ -376,6 +443,8 @@ async def get_remote_media( media_id: The media ID of the content (as defined by the remote server). name: Optional name that, if specified, will be used as the filename in the Content-Disposition header of the response. + max_timeout_ms: the maximum number of milliseconds to wait for the + media to be uploaded. Returns: Resolves once a response has successfully been written to request @@ -401,11 +470,13 @@ async def get_remote_media( key = (server_name, media_id) async with self.remote_media_linearizer.queue(key): responder, media_info = await self._get_remote_media_impl( - server_name, media_id + server_name, media_id, max_timeout_ms ) # We deliberately stream the file outside the lock - if responder: + if responder and media_info: + media_type = media_info.media_type + media_length = media_info.media_length upload_name = name if name else media_info.upload_name await respond_with_responder( request, @@ -414,11 +485,15 @@ async def get_remote_media( media_info.media_length, upload_name, ) + elif media_info: + # The media exists (but the responder doesn't) meaning that this file hasn't + # been uploaded yet. + self.respond_not_yet_uploaded(request) else: respond_404(request) async def get_remote_media_info( - self, server_name: str, media_id: str + self, server_name: str, media_id: str, max_timeout_ms: int ) -> RemoteMedia: """Gets the media info associated with the remote file, downloading if necessary. @@ -426,6 +501,8 @@ async def get_remote_media_info( Args: server_name: Remote server_name where the media originated. media_id: The media ID of the content (as defined by the remote server). + max_timeout_ms: the maximum number of milliseconds to wait for the + media to be uploaded. Returns: The media info of the file @@ -441,7 +518,7 @@ async def get_remote_media_info( key = (server_name, media_id) async with self.remote_media_linearizer.queue(key): responder, media_info = await self._get_remote_media_impl( - server_name, media_id + server_name, media_id, max_timeout_ms ) # Ensure we actually use the responder so that it releases resources @@ -452,7 +529,7 @@ async def get_remote_media_info( return media_info async def _get_remote_media_impl( - self, server_name: str, media_id: str + self, server_name: str, media_id: str, max_timeout_ms: int ) -> Tuple[Optional[Responder], RemoteMedia]: """Looks for media in local cache, if not there then attempt to download from remote server. @@ -461,6 +538,8 @@ async def _get_remote_media_impl( server_name: Remote server_name where the media originated. media_id: The media ID of the content (as defined by the remote server). + max_timeout_ms: the maximum number of milliseconds to wait for the + media to be uploaded. Returns: A tuple of responder and the media info of the file. @@ -493,8 +572,7 @@ async def _get_remote_media_impl( try: media_info = await self._download_remote_file( - server_name, - media_id, + server_name, media_id, max_timeout_ms ) except SynapseError: raise @@ -527,6 +605,7 @@ async def _download_remote_file( self, server_name: str, media_id: str, + max_timeout_ms: int, ) -> RemoteMedia: """Attempt to download the remote file from the given server name, using the given file_id as the local id. @@ -536,7 +615,8 @@ async def _download_remote_file( media_id: The media ID of the content (as defined by the remote server). This is different than the file_id, which is locally generated. - file_id: Local file ID + max_timeout_ms: the maximum number of milliseconds to wait for the + media to be uploaded. Returns: The media info of the file. @@ -560,7 +640,8 @@ async def _download_remote_file( # tell the remote server to 404 if it doesn't # recognise the server_name, to make sure we don't # end up with a routing loop. - "allow_remote": "false" + "allow_remote": "false", + "timeout_ms": str(max_timeout_ms), }, ) except RequestSendFailed as e: diff --git a/synapse/rest/media/download_resource.py b/synapse/rest/media/download_resource.py index 65b9ff52faaf..60cd87548c13 100644 --- a/synapse/rest/media/download_resource.py +++ b/synapse/rest/media/download_resource.py @@ -17,9 +17,13 @@ from typing import TYPE_CHECKING, Optional from synapse.http.server import set_corp_headers, set_cors_headers -from synapse.http.servlet import RestServlet, parse_boolean +from synapse.http.servlet import RestServlet, parse_boolean, parse_integer from synapse.http.site import SynapseRequest -from synapse.media._base import respond_404 +from synapse.media._base import ( + DEFAULT_MAX_TIMEOUT_MS, + MAXIMUM_ALLOWED_MAX_TIMEOUT_MS, + respond_404, +) from synapse.util.stringutils import parse_and_validate_server_name if TYPE_CHECKING: @@ -65,12 +69,16 @@ async def on_GET( ) # Limited non-standard form of CSP for IE11 request.setHeader(b"X-Content-Security-Policy", b"sandbox;") - request.setHeader( - b"Referrer-Policy", - b"no-referrer", + request.setHeader(b"Referrer-Policy", b"no-referrer") + max_timeout_ms = parse_integer( + request, "timeout_ms", default=DEFAULT_MAX_TIMEOUT_MS ) + max_timeout_ms = min(max_timeout_ms, MAXIMUM_ALLOWED_MAX_TIMEOUT_MS) + if self._is_mine_server_name(server_name): - await self.media_repo.get_local_media(request, media_id, file_name) + await self.media_repo.get_local_media( + request, media_id, file_name, max_timeout_ms + ) else: allow_remote = parse_boolean(request, "allow_remote", default=True) if not allow_remote: @@ -83,5 +91,5 @@ async def on_GET( return await self.media_repo.get_remote_media( - request, server_name, media_id, file_name + request, server_name, media_id, file_name, max_timeout_ms ) diff --git a/synapse/rest/media/thumbnail_resource.py b/synapse/rest/media/thumbnail_resource.py index efda8b4ab4a9..f01f2dd342de 100644 --- a/synapse/rest/media/thumbnail_resource.py +++ b/synapse/rest/media/thumbnail_resource.py @@ -23,6 +23,8 @@ from synapse.http.servlet import RestServlet, parse_integer, parse_string from synapse.http.site import SynapseRequest from synapse.media._base import ( + DEFAULT_MAX_TIMEOUT_MS, + MAXIMUM_ALLOWED_MAX_TIMEOUT_MS, FileInfo, ThumbnailInfo, respond_404, @@ -75,15 +77,19 @@ async def on_GET( method = parse_string(request, "method", "scale") # TODO Parse the Accept header to get an prioritised list of thumbnail types. m_type = "image/png" + max_timeout_ms = parse_integer( + request, "timeout_ms", default=DEFAULT_MAX_TIMEOUT_MS + ) + max_timeout_ms = min(max_timeout_ms, MAXIMUM_ALLOWED_MAX_TIMEOUT_MS) if self._is_mine_server_name(server_name): if self.dynamic_thumbnails: await self._select_or_generate_local_thumbnail( - request, media_id, width, height, method, m_type + request, media_id, width, height, method, m_type, max_timeout_ms ) else: await self._respond_local_thumbnail( - request, media_id, width, height, method, m_type + request, media_id, width, height, method, m_type, max_timeout_ms ) self.media_repo.mark_recently_accessed(None, media_id) else: @@ -95,14 +101,21 @@ async def on_GET( respond_404(request) return - if self.dynamic_thumbnails: - await self._select_or_generate_remote_thumbnail( - request, server_name, media_id, width, height, method, m_type - ) - else: - await self._respond_remote_thumbnail( - request, server_name, media_id, width, height, method, m_type - ) + remote_resp_function = ( + self._select_or_generate_remote_thumbnail + if self.dynamic_thumbnails + else self._respond_remote_thumbnail + ) + await remote_resp_function( + request, + server_name, + media_id, + width, + height, + method, + m_type, + max_timeout_ms, + ) self.media_repo.mark_recently_accessed(server_name, media_id) async def _respond_local_thumbnail( @@ -113,9 +126,11 @@ async def _respond_local_thumbnail( height: int, method: str, m_type: str, + max_timeout_ms: int, ) -> None: - media_info = await self.store.get_local_media(media_id) - + media_info = await self.media_repo.get_local_media_info( + request, media_id, max_timeout_ms + ) if not media_info: respond_404(request) return @@ -146,8 +161,11 @@ async def _select_or_generate_local_thumbnail( desired_height: int, desired_method: str, desired_type: str, + max_timeout_ms: int, ) -> None: - media_info = await self.store.get_local_media(media_id) + media_info = await self.media_repo.get_local_media_info( + request, media_id, max_timeout_ms + ) if not media_info: respond_404(request) @@ -206,8 +224,14 @@ async def _select_or_generate_remote_thumbnail( desired_height: int, desired_method: str, desired_type: str, + max_timeout_ms: int, ) -> None: - media_info = await self.media_repo.get_remote_media_info(server_name, media_id) + media_info = await self.media_repo.get_remote_media_info( + server_name, media_id, max_timeout_ms + ) + if not media_info: + respond_404(request) + return thumbnail_infos = await self.store.get_remote_media_thumbnails( server_name, media_id @@ -263,11 +287,16 @@ async def _respond_remote_thumbnail( height: int, method: str, m_type: str, + max_timeout_ms: int, ) -> None: # TODO: Don't download the whole remote file # We should proxy the thumbnail from the remote server instead of # downloading the remote file and generating our own thumbnails. - media_info = await self.media_repo.get_remote_media_info(server_name, media_id) + media_info = await self.media_repo.get_remote_media_info( + server_name, media_id, max_timeout_ms + ) + if not media_info: + return thumbnail_infos = await self.store.get_remote_media_thumbnails( server_name, media_id From e4c4f36871a8fbea0ae1241ae329944963b51709 Mon Sep 17 00:00:00 2001 From: Sumner Evans Date: Fri, 9 Jun 2023 11:04:22 -0600 Subject: [PATCH 08/29] tests/media/test_media_storage: add timeout_ms Signed-off-by: Sumner Evans --- tests/media/test_media_storage.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/media/test_media_storage.py b/tests/media/test_media_storage.py index a8e7a76b298e..f262304c3daa 100644 --- a/tests/media/test_media_storage.py +++ b/tests/media/test_media_storage.py @@ -318,7 +318,9 @@ def _req( self.assertEqual( self.fetches[0][2], "/_matrix/media/r0/download/" + self.media_id ) - self.assertEqual(self.fetches[0][3], {"allow_remote": "false"}) + self.assertEqual( + self.fetches[0][3], {"allow_remote": "false", "timeout_ms": "20000"} + ) headers = { b"Content-Length": [b"%d" % (len(self.test_image.data))], From a6b2192b51a2f4ef3ef5c26528aea444b2b40c8b Mon Sep 17 00:00:00 2001 From: Sumner Evans Date: Fri, 9 Jun 2023 13:51:20 -0600 Subject: [PATCH 09/29] config: add option for max pending media uploads Signed-off-by: Sumner Evans --- docs/usage/configuration/config_documentation.md | 11 +++++++++++ synapse/config/repository.py | 2 ++ 2 files changed, 13 insertions(+) diff --git a/docs/usage/configuration/config_documentation.md b/docs/usage/configuration/config_documentation.md index e77d84372cb6..f2bb0e8200e5 100644 --- a/docs/usage/configuration/config_documentation.md +++ b/docs/usage/configuration/config_documentation.md @@ -1827,6 +1827,17 @@ Example configuration: media_store_path: "DATADIR/media_store" ``` --- +### `max_pending_media_uploads` + +How many *pending media uploads* can a given user have? A pending media upload +is a created MXC URI that (a) is not expired (the `unused_expires_at` timestamp +has not passed) and (b) the media has not yet been uploaded for. Defaults to 5. + +Example configuration: +```yaml +max_pending_media_uploads: 5 +``` +--- ### `unused_expiration_time` How long to wait in milliseconds before expiring created media IDs. Defaults to diff --git a/synapse/config/repository.py b/synapse/config/repository.py index dba9926779ba..839c026d70d7 100644 --- a/synapse/config/repository.py +++ b/synapse/config/repository.py @@ -145,6 +145,8 @@ def read_config(self, config: JsonDict, **kwargs: Any) -> None: config.get("unused_expiration_time", "24h") ) + self.max_pending_media_uploads = config.get("max_pending_media_uploads", 5) + self.media_store_path = self.ensure_directory( config.get("media_store_path", "media_store") ) From e2e4ad57e5cb25a94bd220ed7a9367900cde314a Mon Sep 17 00:00:00 2001 From: Sumner Evans Date: Fri, 9 Jun 2023 17:20:25 -0600 Subject: [PATCH 10/29] media/create: enforce limit on number of pending uploads Signed-off-by: Sumner Evans --- synapse/media/media_repository.py | 20 ++++++++++++ synapse/rest/media/create_resource.py | 1 + .../databases/main/media_repository.py | 31 +++++++++++++++++++ 3 files changed, 52 insertions(+) diff --git a/synapse/media/media_repository.py b/synapse/media/media_repository.py index 133003a8d529..d735c36ed7d4 100644 --- a/synapse/media/media_repository.py +++ b/synapse/media/media_repository.py @@ -212,6 +212,26 @@ async def create_media_id(self, auth_user: UserID) -> Tuple[str, int]: ) return f"mxc://{self.server_name}/{media_id}", unused_expires_at + @trace + async def reached_pending_media_limit( + self, auth_user: UserID, limit: int + ) -> Tuple[bool, int]: + """Check if the user is over the limit for pending media uploads. + + Args: + auth_user: The user_id of the uploader + limit: The maximum number of pending media uploads a user is allowed to have + + Returns: + A tuple with a boolean and an integer indicating whether the user has too + many pending media uploads and the timestamp at which the first pending + media will expire, respectively. + """ + pending, first_expiration_ts = await self.store.count_pending_media( + user_id=auth_user + ) + return pending >= limit, first_expiration_ts + @trace async def verify_can_upload(self, media_id: str, auth_user: UserID) -> None: """Verify that the media ID can be uploaded to by the given user. This diff --git a/synapse/rest/media/create_resource.py b/synapse/rest/media/create_resource.py index df79d528cb4d..09d1d3caf003 100644 --- a/synapse/rest/media/create_resource.py +++ b/synapse/rest/media/create_resource.py @@ -38,6 +38,7 @@ def __init__(self, hs: "HomeServer", media_repo: "MediaRepository"): self.media_repo = media_repo self.clock = hs.get_clock() self.auth = hs.get_auth() + self.max_pending_media_uploads = hs.config.media.max_pending_media_uploads # A rate limiter for creating new media IDs. self._create_media_rate_limiter = Ratelimiter( diff --git a/synapse/storage/databases/main/media_repository.py b/synapse/storage/databases/main/media_repository.py index a902414aaf98..959ee7978c13 100644 --- a/synapse/storage/databases/main/media_repository.py +++ b/synapse/storage/databases/main/media_repository.py @@ -27,6 +27,7 @@ import attr from synapse.api.constants import Direction +from synapse.api.errors import StoreError from synapse.logging.opentracing import trace from synapse.media._base import ThumbnailInfo from synapse.storage._base import SQLBaseStore @@ -470,6 +471,36 @@ async def mark_local_media_as_safe(self, media_id: str, safe: bool = True) -> No desc="mark_local_media_as_safe", ) + async def count_pending_media(self, user_id: UserID) -> Tuple[int, int]: + """Count the number of pending media for a user. + + Returns: + A tuple of two integers: the total pending media requests and the earliest + expiration timestamp. + """ + + def get_pending_media_txn(txn: LoggingTransaction) -> Tuple[int, int]: + sql = """ + SELECT COUNT(*), MIN(created_at) + FROM local_media_repository + WHERE user_id = ? + AND created_at > ? + AND media_length IS NULL + """ + txn.execute( + sql, + ( + user_id.to_string(), + self._clock.time_msec() - self.unused_expiration_time, + ), + ) + row = txn.fetchone() + if not row: + raise StoreError(404, "Failed to count pending media for user") + return row[0], row[1] or 0 + + return await self.db_pool.runInteraction("get_url_cache", get_pending_media_txn) + async def get_url_cache(self, url: str, ts: int) -> Optional[UrlCache]: """Get the media_id and ts for a cached URL as of the given timestamp Returns: From 97c792c8ee9f6f1d8a90e469bde7785f3f4ea0d7 Mon Sep 17 00:00:00 2001 From: Sumner Evans Date: Mon, 4 Sep 2023 18:34:02 -0600 Subject: [PATCH 11/29] media/create: don't store unused expiration time in database Signed-off-by: Sumner Evans Co-authored-by: Patrick Cloke --- synapse/media/media_repository.py | 13 +++++------ .../databases/main/media_repository.py | 7 +++--- .../80/02_add_unused_expires_at_for_media.sql | 22 ------------------- 3 files changed, 9 insertions(+), 33 deletions(-) delete mode 100644 synapse/storage/schema/main/delta/80/02_add_unused_expires_at_for_media.sql diff --git a/synapse/media/media_repository.py b/synapse/media/media_repository.py index d735c36ed7d4..de2ccca8aed7 100644 --- a/synapse/media/media_repository.py +++ b/synapse/media/media_repository.py @@ -203,14 +203,12 @@ async def create_media_id(self, auth_user: UserID) -> Tuple[str, int]: """ media_id = random_string(24) now = self.clock.time_msec() - unused_expires_at = now + self.unused_expiration_time await self.store.store_local_media_id( media_id=media_id, time_now_ms=now, user_id=auth_user, - unused_expires_at=unused_expires_at, ) - return f"mxc://{self.server_name}/{media_id}", unused_expires_at + return f"mxc://{self.server_name}/{media_id}", now + self.unused_expiration_time @trace async def reached_pending_media_limit( @@ -264,7 +262,8 @@ async def verify_can_upload(self, media_id: str, auth_user: UserID) -> None: errcode=Codes.CANNOT_OVERWRITE_MEDIA, ) - if media.get("unused_expires_at", 0) < self.clock.time_msec(): + expired_time_ms = self.clock.time_msec() - self.unused_expiration_time + if media.created_ts < expired_time_ms: raise NotFoundError("Media ID has expired") @trace @@ -392,12 +391,12 @@ async def get_local_media_info( return media_info # Check if the media ID has expired and still hasn't been uploaded to. - unused_expires_at = media_info.get("unused_expires_at", 0) - if 0 < unused_expires_at < self.clock.time_msec(): + now = self.clock.time_msec() + if media_info.get("created_ts", now) + self.unused_expiration_time < now: respond_404(request) return None - if self.clock.time_msec() >= wait_until: + if now >= wait_until: break await self.clock.sleep(0.5) diff --git a/synapse/storage/databases/main/media_repository.py b/synapse/storage/databases/main/media_repository.py index 959ee7978c13..4e65f01074e2 100644 --- a/synapse/storage/databases/main/media_repository.py +++ b/synapse/storage/databases/main/media_repository.py @@ -150,6 +150,8 @@ def __init__( self._drop_media_index_without_method, ) + self.unused_expiration_time = hs.config.media.unused_expiration_time + async def _drop_media_index_without_method( self, progress: JsonDict, batch_size: int ) -> int: @@ -204,7 +206,6 @@ async def get_local_media(self, media_id: str) -> Optional[LocalMedia]: "last_access_ts", "safe_from_quarantine", "user_id", - "unused_expires_at", ), allow_none=True, desc="get_local_media", @@ -400,7 +401,6 @@ async def store_local_media_id( media_id: str, time_now_ms: int, user_id: UserID, - unused_expires_at: int, ) -> None: await self.db_pool.simple_insert( "local_media_repository", @@ -408,7 +408,6 @@ async def store_local_media_id( "media_id": media_id, "created_ts": time_now_ms, "user_id": user_id.to_string(), - "unused_expires_at": unused_expires_at, }, desc="store_local_media_id", ) @@ -497,7 +496,7 @@ def get_pending_media_txn(txn: LoggingTransaction) -> Tuple[int, int]: row = txn.fetchone() if not row: raise StoreError(404, "Failed to count pending media for user") - return row[0], row[1] or 0 + return row[0], (row[1] + self.unused_expiration_time if row[1] else 0) return await self.db_pool.runInteraction("get_url_cache", get_pending_media_txn) diff --git a/synapse/storage/schema/main/delta/80/02_add_unused_expires_at_for_media.sql b/synapse/storage/schema/main/delta/80/02_add_unused_expires_at_for_media.sql deleted file mode 100644 index ed464820cece..000000000000 --- a/synapse/storage/schema/main/delta/80/02_add_unused_expires_at_for_media.sql +++ /dev/null @@ -1,22 +0,0 @@ -/* Copyright 2023 Beeper Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - --- Add new colums to the `local_media_repository` to keep track of when the --- media ID must be used by. This is to support async uploads (see MSC2246). - -ALTER TABLE local_media_repository - ADD COLUMN unused_expires_at BIGINT DEFAULT NULL; - -CREATE INDEX CONCURRENTLY ON local_media_repository (unused_expires_at); From 52e89156e4769c806b6e521fa34bd8618805251a Mon Sep 17 00:00:00 2001 From: Patrick Cloke Date: Thu, 26 Oct 2023 08:30:55 -0400 Subject: [PATCH 12/29] media/upload: split async servlet into separate class Signed-off-by: Sumner Evans Co-authored-by: Patrick Cloke --- .../rest/media/media_repository_resource.py | 5 ++-- synapse/rest/media/upload_resource.py | 23 ++++++++++++------- 2 files changed, 18 insertions(+), 10 deletions(-) diff --git a/synapse/rest/media/media_repository_resource.py b/synapse/rest/media/media_repository_resource.py index e92b0716ca1a..ca65116b8487 100644 --- a/synapse/rest/media/media_repository_resource.py +++ b/synapse/rest/media/media_repository_resource.py @@ -22,7 +22,7 @@ from .download_resource import DownloadResource from .preview_url_resource import PreviewUrlResource from .thumbnail_resource import ThumbnailResource -from .upload_resource import UploadResource +from .upload_resource import AsyncUploadServlet, UploadServlet if TYPE_CHECKING: from synapse.server import HomeServer @@ -93,7 +93,8 @@ def register_servlets(http_server: HttpServer, hs: "HomeServer") -> None: # Note that many of these should not exist as v1 endpoints, but empirically # a lot of traffic still goes to them. CreateResource(hs, media_repo).register(http_server) - UploadResource(hs, media_repo).register(http_server) + UploadServlet(hs, media_repo).register(http_server) + AsyncUploadServlet(hs, media_repo).register(http_server) DownloadResource(hs, media_repo).register(http_server) ThumbnailResource(hs, media_repo, media_repo.media_storage).register( http_server diff --git a/synapse/rest/media/upload_resource.py b/synapse/rest/media/upload_resource.py index b6bf0a0a8f05..d059ffee9908 100644 --- a/synapse/rest/media/upload_resource.py +++ b/synapse/rest/media/upload_resource.py @@ -21,7 +21,6 @@ from synapse.http.server import respond_with_json from synapse.http.servlet import RestServlet, parse_bytes_from_args from synapse.http.site import SynapseRequest -from synapse.media._base import parse_media_id from synapse.media.media_storage import SpamMediaException if TYPE_CHECKING: @@ -34,9 +33,7 @@ _UPLOAD_MEDIA_LOCK_NAME = "upload_media" -class UploadResource(RestServlet): - PATTERNS = [re.compile("/_matrix/media/(r0|v3|v1)/upload")] - +class BaseUploadServlet(RestServlet): def __init__(self, hs: "HomeServer", media_repo: "MediaRepository"): super().__init__() @@ -94,8 +91,9 @@ def _get_file_metadata( return content_length, upload_name, media_type - async def _async_render_OPTIONS(self, request: SynapseRequest) -> None: - respond_with_json(request, 200, {}, send_cors=True) + +class UploadServlet(BaseUploadServlet): + PATTERNS = [re.compile("/_matrix/media/(r0|v3|v1)/upload$")] async def on_POST(self, request: SynapseRequest) -> None: requester = await self.auth.get_user_by_req(request) @@ -117,9 +115,18 @@ async def on_POST(self, request: SynapseRequest) -> None: request, 200, {"content_uri": str(content_uri)}, send_cors=True ) - async def _async_render_PUT(self, request: SynapseRequest) -> None: + +class AsyncUploadServlet(BaseUploadServlet): + PATTERNS = [ + re.compile( + "/_matrix/media/v1/upload/(?P[^/]*)/(?P[^/]*)$" + ) + ] + + async def on_PUT( + self, request: SynapseRequest, server_name: str, media_id: str + ) -> None: requester = await self.auth.get_user_by_req(request) - server_name, media_id, _ = parse_media_id(request) if server_name != self.server_name: raise SynapseError( From 961bfa0fa61e46c044028d23217f8c6b7e89500b Mon Sep 17 00:00:00 2001 From: Patrick Cloke Date: Wed, 8 Nov 2023 09:16:22 -0500 Subject: [PATCH 13/29] Do not fail upload if thumbnailing fails. --- synapse/media/media_repository.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/synapse/media/media_repository.py b/synapse/media/media_repository.py index de2ccca8aed7..edf8e82b5046 100644 --- a/synapse/media/media_repository.py +++ b/synapse/media/media_repository.py @@ -298,7 +298,10 @@ async def update_content( user_id=auth_user, ) - await self._generate_thumbnails(None, media_id, media_id, media_type) + try: + await self._generate_thumbnails(None, media_id, media_id, media_type) + except Exception as e: + logger.info("Failed to generate thumbnails: %s", e) @trace async def create_content( From b49a50b641d955332e1c499e373d8e7247e1caf1 Mon Sep 17 00:00:00 2001 From: Sumner Evans Date: Thu, 9 Nov 2023 16:17:11 -0700 Subject: [PATCH 14/29] fixup! media/upload: add support for async uploads Signed-off-by: Sumner Evans --- synapse/media/media_repository.py | 4 ++-- synapse/storage/databases/main/media_repository.py | 8 ++++++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/synapse/media/media_repository.py b/synapse/media/media_repository.py index edf8e82b5046..67a4ec79e41d 100644 --- a/synapse/media/media_repository.py +++ b/synapse/media/media_repository.py @@ -248,14 +248,14 @@ async def verify_can_upload(self, media_id: str, auth_user: UserID) -> None: if media is None: raise SynapseError(404, "Unknow media ID", errcode=Codes.NOT_FOUND) - if media["user_id"] != auth_user.to_string(): + if media.user_id != auth_user.to_string(): raise SynapseError( 403, "Only the creator of the media ID can upload to it", errcode=Codes.FORBIDDEN, ) - if media.get("media_length") is not None: + if media.media_length is not None: raise SynapseError( 409, "Media ID already has content", diff --git a/synapse/storage/databases/main/media_repository.py b/synapse/storage/databases/main/media_repository.py index 4e65f01074e2..334fc15a4365 100644 --- a/synapse/storage/databases/main/media_repository.py +++ b/synapse/storage/databases/main/media_repository.py @@ -50,13 +50,14 @@ class LocalMedia: media_id: str media_type: str - media_length: int + media_length: Optional[int] upload_name: str created_ts: int url_cache: Optional[str] last_access_ts: int quarantined_by: Optional[str] safe_from_quarantine: bool + user_id: Optional[str] @attr.s(slots=True, frozen=True, auto_attribs=True) @@ -222,6 +223,7 @@ async def get_local_media(self, media_id: str) -> Optional[LocalMedia]: url_cache=row[5], last_access_ts=row[6], safe_from_quarantine=row[7], + user_id=row[8], ) async def get_local_media_by_user_paginate( @@ -276,7 +278,8 @@ def get_local_media_by_user_paginate_txn( url_cache, last_access_ts, quarantined_by, - safe_from_quarantine + safe_from_quarantine, + user_id FROM local_media_repository WHERE user_id = ? ORDER BY {order_by_column} {order}, media_id ASC @@ -299,6 +302,7 @@ def get_local_media_by_user_paginate_txn( last_access_ts=row[6], quarantined_by=row[7], safe_from_quarantine=bool(row[8]), + user_id=row[9], ) for row in txn ] From 4e4e8d01f3f575a0a29d51adfa667f0864553a2c Mon Sep 17 00:00:00 2001 From: Sumner Evans Date: Thu, 9 Nov 2023 16:19:40 -0700 Subject: [PATCH 15/29] fixup! media/{download,thumbnail}: support timeout_ms parameter Signed-off-by: Sumner Evans --- synapse/media/media_repository.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/synapse/media/media_repository.py b/synapse/media/media_repository.py index 67a4ec79e41d..fdefd392f825 100644 --- a/synapse/media/media_repository.py +++ b/synapse/media/media_repository.py @@ -497,8 +497,6 @@ async def get_remote_media( # We deliberately stream the file outside the lock if responder and media_info: - media_type = media_info.media_type - media_length = media_info.media_length upload_name = name if name else media_info.upload_name await respond_with_responder( request, From 37c79704d0d98729f56212fb1eb03f605e7bb305 Mon Sep 17 00:00:00 2001 From: Sumner Evans Date: Thu, 9 Nov 2023 16:22:06 -0700 Subject: [PATCH 16/29] fixup! media/{download,thumbnail}: support timeout_ms parameter Signed-off-by: Sumner Evans --- synapse/media/media_repository.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/synapse/media/media_repository.py b/synapse/media/media_repository.py index fdefd392f825..4e4e8843790c 100644 --- a/synapse/media/media_repository.py +++ b/synapse/media/media_repository.py @@ -505,10 +505,6 @@ async def get_remote_media( media_info.media_length, upload_name, ) - elif media_info: - # The media exists (but the responder doesn't) meaning that this file hasn't - # been uploaded yet. - self.respond_not_yet_uploaded(request) else: respond_404(request) From acadec01455702236a3b59fd94328e091c6d3a4e Mon Sep 17 00:00:00 2001 From: Sumner Evans Date: Thu, 9 Nov 2023 16:28:14 -0700 Subject: [PATCH 17/29] fixup! media/create: add /create endpoint Signed-off-by: Sumner Evans --- synapse/media/media_repository.py | 8 +++----- synapse/rest/media/create_resource.py | 4 +--- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/synapse/media/media_repository.py b/synapse/media/media_repository.py index 4e4e8843790c..7d8788ffcc5f 100644 --- a/synapse/media/media_repository.py +++ b/synapse/media/media_repository.py @@ -84,6 +84,7 @@ def __init__(self, hs: "HomeServer"): self.max_upload_size = hs.config.media.max_upload_size self.max_image_pixels = hs.config.media.max_image_pixels self.unused_expiration_time = hs.config.media.unused_expiration_time + self.max_pending_media_uploads = hs.config.media.max_pending_media_uploads Thumbnailer.set_limits(self.max_image_pixels) @@ -211,14 +212,11 @@ async def create_media_id(self, auth_user: UserID) -> Tuple[str, int]: return f"mxc://{self.server_name}/{media_id}", now + self.unused_expiration_time @trace - async def reached_pending_media_limit( - self, auth_user: UserID, limit: int - ) -> Tuple[bool, int]: + async def reached_pending_media_limit(self, auth_user: UserID) -> Tuple[bool, int]: """Check if the user is over the limit for pending media uploads. Args: auth_user: The user_id of the uploader - limit: The maximum number of pending media uploads a user is allowed to have Returns: A tuple with a boolean and an integer indicating whether the user has too @@ -228,7 +226,7 @@ async def reached_pending_media_limit( pending, first_expiration_ts = await self.store.count_pending_media( user_id=auth_user ) - return pending >= limit, first_expiration_ts + return pending >= self.max_pending_media_uploads, first_expiration_ts @trace async def verify_can_upload(self, media_id: str, auth_user: UserID) -> None: diff --git a/synapse/rest/media/create_resource.py b/synapse/rest/media/create_resource.py index 09d1d3caf003..7717435a1e2e 100644 --- a/synapse/rest/media/create_resource.py +++ b/synapse/rest/media/create_resource.py @@ -59,9 +59,7 @@ async def _async_render_POST(self, request: SynapseRequest) -> None: ( reached_pending_limit, first_expiration_ts, - ) = await self.media_repo.reached_pending_media_limit( - requester.user, self.max_pending_media_uploads - ) + ) = await self.media_repo.reached_pending_media_limit(requester.user) if reached_pending_limit: raise LimitExceededError( limiter_name="max_pending_media_uploads", From 58364ad02fbdd8f6c2e587512cf1c7093eff6f4b Mon Sep 17 00:00:00 2001 From: Sumner Evans Date: Thu, 9 Nov 2023 16:35:08 -0700 Subject: [PATCH 18/29] fixup! media/create: don't store unused expiration time in database Signed-off-by: Sumner Evans --- synapse/media/media_repository.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/synapse/media/media_repository.py b/synapse/media/media_repository.py index 7d8788ffcc5f..5214fc73a809 100644 --- a/synapse/media/media_repository.py +++ b/synapse/media/media_repository.py @@ -17,7 +17,7 @@ import os import shutil from io import BytesIO -from typing import IO, TYPE_CHECKING, Any, Dict, List, Optional, Set, Tuple +from typing import IO, TYPE_CHECKING, Dict, List, Optional, Set, Tuple import attr from matrix_common.types.mxc_uri import MXCUri @@ -54,7 +54,7 @@ from synapse.media.thumbnailer import Thumbnailer, ThumbnailError from synapse.media.url_previewer import UrlPreviewer from synapse.metrics.background_process_metrics import run_as_background_process -from synapse.storage.databases.main.media_repository import RemoteMedia +from synapse.storage.databases.main.media_repository import LocalMedia, RemoteMedia from synapse.types import UserID from synapse.util.async_helpers import Linearizer from synapse.util.retryutils import NotRetryingDestination @@ -357,7 +357,7 @@ def respond_not_yet_uploaded(self, request: SynapseRequest) -> None: async def get_local_media_info( self, request: SynapseRequest, media_id: str, max_timeout_ms: int - ) -> Optional[Dict[str, Any]]: + ) -> Optional[LocalMedia]: """Gets the info dictionary for given local media ID. If the media has not been uploaded yet, this function will wait up to ``max_timeout_ms`` milliseconds for the media to be uploaded. @@ -382,18 +382,19 @@ async def get_local_media_info( respond_404(request) return None - if media_info["quarantined_by"]: + if media_info.quarantined_by: logger.info("Media is quarantined") respond_404(request) return None # The file has been uploaded, so stop looping - if media_info.get("media_length") is not None: + if media_info.media_length is not None: return media_info # Check if the media ID has expired and still hasn't been uploaded to. now = self.clock.time_msec() - if media_info.get("created_ts", now) + self.unused_expiration_time < now: + expired_time_ms = now - self.unused_expiration_time + if media_info.created_ts < expired_time_ms: respond_404(request) return None From a514c84d026cb263b52d4f7d0931b3420323336b Mon Sep 17 00:00:00 2001 From: Sumner Evans Date: Thu, 9 Nov 2023 16:39:08 -0700 Subject: [PATCH 19/29] fixup! media/create: enforce limit on number of pending uploads Co-authored-by: Patrick Cloke --- synapse/storage/databases/main/media_repository.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/storage/databases/main/media_repository.py b/synapse/storage/databases/main/media_repository.py index 334fc15a4365..2db954064bcc 100644 --- a/synapse/storage/databases/main/media_repository.py +++ b/synapse/storage/databases/main/media_repository.py @@ -499,7 +499,7 @@ def get_pending_media_txn(txn: LoggingTransaction) -> Tuple[int, int]: ) row = txn.fetchone() if not row: - raise StoreError(404, "Failed to count pending media for user") + return 0, 0 return row[0], (row[1] + self.unused_expiration_time if row[1] else 0) return await self.db_pool.runInteraction("get_url_cache", get_pending_media_txn) From b298258e1d8f242668fde4a45b06ae5755998510 Mon Sep 17 00:00:00 2001 From: Sumner Evans Date: Thu, 9 Nov 2023 16:42:51 -0700 Subject: [PATCH 20/29] fixup! media/create: enforce limit on number of pending uploads Signed-off-by: Sumner Evans --- synapse/storage/databases/main/media_repository.py | 1 - 1 file changed, 1 deletion(-) diff --git a/synapse/storage/databases/main/media_repository.py b/synapse/storage/databases/main/media_repository.py index 2db954064bcc..480a249b684a 100644 --- a/synapse/storage/databases/main/media_repository.py +++ b/synapse/storage/databases/main/media_repository.py @@ -27,7 +27,6 @@ import attr from synapse.api.constants import Direction -from synapse.api.errors import StoreError from synapse.logging.opentracing import trace from synapse.media._base import ThumbnailInfo from synapse.storage._base import SQLBaseStore From 1cff556c7506dee2057122f524c372ac1ac5b2c2 Mon Sep 17 00:00:00 2001 From: Sumner Evans Date: Mon, 13 Nov 2023 12:31:49 -0700 Subject: [PATCH 21/29] fixup! media/create: add /create endpoint Co-authored-by: Patrick Cloke --- synapse/rest/media/create_resource.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/synapse/rest/media/create_resource.py b/synapse/rest/media/create_resource.py index 7717435a1e2e..994afdf13ca4 100644 --- a/synapse/rest/media/create_resource.py +++ b/synapse/rest/media/create_resource.py @@ -47,10 +47,7 @@ def __init__(self, hs: "HomeServer", media_repo: "MediaRepository"): cfg=hs.config.ratelimiting.rc_media_create, ) - async def _async_render_OPTIONS(self, request: SynapseRequest) -> None: - respond_with_json(request, 200, {}, send_cors=True) - - async def _async_render_POST(self, request: SynapseRequest) -> None: + async def on_POST(self, request: SynapseRequest) -> None: requester = await self.auth.get_user_by_req(request) # If the create media requests for the user are over the limit, drop them. From 07afcfb25050b0d5689cc0b8cb6b05ec7064d832 Mon Sep 17 00:00:00 2001 From: Sumner Evans Date: Mon, 13 Nov 2023 12:33:19 -0700 Subject: [PATCH 22/29] fixup! media/create: enforce limit on number of pending uploads Co-authored-by: Patrick Cloke --- synapse/storage/databases/main/media_repository.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/synapse/storage/databases/main/media_repository.py b/synapse/storage/databases/main/media_repository.py index 480a249b684a..626e892a0fbc 100644 --- a/synapse/storage/databases/main/media_repository.py +++ b/synapse/storage/databases/main/media_repository.py @@ -483,10 +483,10 @@ async def count_pending_media(self, user_id: UserID) -> Tuple[int, int]: def get_pending_media_txn(txn: LoggingTransaction) -> Tuple[int, int]: sql = """ - SELECT COUNT(*), MIN(created_at) + SELECT COUNT(*), MIN(created_ts) FROM local_media_repository WHERE user_id = ? - AND created_at > ? + AND created_ts > ? AND media_length IS NULL """ txn.execute( @@ -501,7 +501,7 @@ def get_pending_media_txn(txn: LoggingTransaction) -> Tuple[int, int]: return 0, 0 return row[0], (row[1] + self.unused_expiration_time if row[1] else 0) - return await self.db_pool.runInteraction("get_url_cache", get_pending_media_txn) + return await self.db_pool.runInteraction("get_pending_media", get_pending_media_txn) async def get_url_cache(self, url: str, ts: int) -> Optional[UrlCache]: """Get the media_id and ts for a cached URL as of the given timestamp From c377b5d38d8a8a6221bcc87df7d9d6f07085a5e6 Mon Sep 17 00:00:00 2001 From: Sumner Evans Date: Mon, 13 Nov 2023 12:39:02 -0700 Subject: [PATCH 23/29] fixup! media/upload: split async servlet into separate class Co-authored-by: Patrick Cloke --- synapse/rest/media/upload_resource.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/rest/media/upload_resource.py b/synapse/rest/media/upload_resource.py index d059ffee9908..62d3e228a874 100644 --- a/synapse/rest/media/upload_resource.py +++ b/synapse/rest/media/upload_resource.py @@ -119,7 +119,7 @@ async def on_POST(self, request: SynapseRequest) -> None: class AsyncUploadServlet(BaseUploadServlet): PATTERNS = [ re.compile( - "/_matrix/media/v1/upload/(?P[^/]*)/(?P[^/]*)$" + "/_matrix/media/v3/upload/(?P[^/]*)/(?P[^/]*)$" ) ] From 697438039c4887e8e7814c5c73662739ae0cdd16 Mon Sep 17 00:00:00 2001 From: Sumner Evans Date: Tue, 14 Nov 2023 10:01:01 -0700 Subject: [PATCH 24/29] fixup! media/{download,thumbnail}: support timeout_ms parameter Signed-off-by: Sumner Evans --- synapse/media/media_repository.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/synapse/media/media_repository.py b/synapse/media/media_repository.py index 5214fc73a809..91b01bfce0ce 100644 --- a/synapse/media/media_repository.py +++ b/synapse/media/media_repository.py @@ -379,11 +379,12 @@ async def get_local_media_info( # Get the info for the media media_info = await self.store.get_local_media(media_id) if not media_info: + logger.info("Media %s is unknown", media_id) respond_404(request) return None if media_info.quarantined_by: - logger.info("Media is quarantined") + logger.info("Media %s is quarantined", media_id) respond_404(request) return None @@ -395,6 +396,7 @@ async def get_local_media_info( now = self.clock.time_msec() expired_time_ms = now - self.unused_expiration_time if media_info.created_ts < expired_time_ms: + logger.info("Media %s has expired without being uploaded", media_id) respond_404(request) return None @@ -403,6 +405,7 @@ async def get_local_media_info( await self.clock.sleep(0.5) + logger.info("Media %s has not yet been uploaded", media_id) self.respond_not_yet_uploaded(request) return None From 8b225ee464f6688f87141997414f3f80e3f4de4e Mon Sep 17 00:00:00 2001 From: Sumner Evans Date: Tue, 14 Nov 2023 10:01:54 -0700 Subject: [PATCH 25/29] fixup! media/{download,thumbnail}: support timeout_ms parameter Signed-off-by: Sumner Evans --- synapse/rest/media/thumbnail_resource.py | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/synapse/rest/media/thumbnail_resource.py b/synapse/rest/media/thumbnail_resource.py index f01f2dd342de..681f2a5a273f 100644 --- a/synapse/rest/media/thumbnail_resource.py +++ b/synapse/rest/media/thumbnail_resource.py @@ -132,11 +132,6 @@ async def _respond_local_thumbnail( request, media_id, max_timeout_ms ) if not media_info: - respond_404(request) - return - if media_info.quarantined_by: - logger.info("Media is quarantined") - respond_404(request) return thumbnail_infos = await self.store.get_local_media_thumbnails(media_id) @@ -168,11 +163,6 @@ async def _select_or_generate_local_thumbnail( ) if not media_info: - respond_404(request) - return - if media_info.quarantined_by: - logger.info("Media is quarantined") - respond_404(request) return thumbnail_infos = await self.store.get_local_media_thumbnails(media_id) From c5e4f041c397edd784fe168f28a5e231a9c451ab Mon Sep 17 00:00:00 2001 From: Sumner Evans Date: Tue, 14 Nov 2023 10:02:25 -0700 Subject: [PATCH 26/29] fixup! media/{download,thumbnail}: support timeout_ms parameter Signed-off-by: Sumner Evans --- synapse/media/media_repository.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/synapse/media/media_repository.py b/synapse/media/media_repository.py index 91b01bfce0ce..bf976b9e7c2c 100644 --- a/synapse/media/media_repository.py +++ b/synapse/media/media_repository.py @@ -431,8 +431,7 @@ async def get_local_media( Resolves once a response has successfully been written to request """ media_info = await self.get_local_media_info(request, media_id, max_timeout_ms) - if not media_info or media_info.quarantined_by: - respond_404(request) + if not media_info: return self.mark_recently_accessed(None, media_id) From c3d3afe4c0a09f5ad5681ce212c01915f642d7b7 Mon Sep 17 00:00:00 2001 From: Sumner Evans Date: Tue, 14 Nov 2023 10:05:36 -0700 Subject: [PATCH 27/29] fixup! media/create: enforce limit on number of pending uploads Signed-off-by: Sumner Evans --- synapse/storage/databases/main/media_repository.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/synapse/storage/databases/main/media_repository.py b/synapse/storage/databases/main/media_repository.py index 626e892a0fbc..c5121e323862 100644 --- a/synapse/storage/databases/main/media_repository.py +++ b/synapse/storage/databases/main/media_repository.py @@ -501,7 +501,9 @@ def get_pending_media_txn(txn: LoggingTransaction) -> Tuple[int, int]: return 0, 0 return row[0], (row[1] + self.unused_expiration_time if row[1] else 0) - return await self.db_pool.runInteraction("get_pending_media", get_pending_media_txn) + return await self.db_pool.runInteraction( + "get_pending_media", get_pending_media_txn + ) async def get_url_cache(self, url: str, ts: int) -> Optional[UrlCache]: """Get the media_id and ts for a cached URL as of the given timestamp From 8debbdf5c158c7d593c2ed104518e563cef40350 Mon Sep 17 00:00:00 2001 From: Patrick Cloke Date: Wed, 15 Nov 2023 08:23:15 -0500 Subject: [PATCH 28/29] Fix reference to potentially undefined config. --- synapse/storage/databases/main/media_repository.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/synapse/storage/databases/main/media_repository.py b/synapse/storage/databases/main/media_repository.py index c5121e323862..c55d58bdbd49 100644 --- a/synapse/storage/databases/main/media_repository.py +++ b/synapse/storage/databases/main/media_repository.py @@ -150,7 +150,10 @@ def __init__( self._drop_media_index_without_method, ) - self.unused_expiration_time = hs.config.media.unused_expiration_time + if hs.config.media.can_load_media_repo: + self.unused_expiration_time = hs.config.media.unused_expiration_time + else: + self.unused_expiration_time = None async def _drop_media_index_without_method( self, progress: JsonDict, batch_size: int @@ -489,6 +492,7 @@ def get_pending_media_txn(txn: LoggingTransaction) -> Tuple[int, int]: AND created_ts > ? AND media_length IS NULL """ + assert self.unused_expiration_time is not None txn.execute( sql, ( From 38badff6aa7c73bf77aa3df73d794aa96c23c23c Mon Sep 17 00:00:00 2001 From: Patrick Cloke Date: Wed, 15 Nov 2023 08:26:10 -0500 Subject: [PATCH 29/29] Appease mypy --- synapse/storage/databases/main/media_repository.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/synapse/storage/databases/main/media_repository.py b/synapse/storage/databases/main/media_repository.py index c55d58bdbd49..149135b8b5fe 100644 --- a/synapse/storage/databases/main/media_repository.py +++ b/synapse/storage/databases/main/media_repository.py @@ -151,7 +151,9 @@ def __init__( ) if hs.config.media.can_load_media_repo: - self.unused_expiration_time = hs.config.media.unused_expiration_time + self.unused_expiration_time: Optional[ + int + ] = hs.config.media.unused_expiration_time else: self.unused_expiration_time = None