Allow downloading of torrents with invalid SSL certs (#8433)

src/tribler/core/libtorrent/download_manager/download_manager.py

@@ -632,7 +632,7 @@ async def start_download_from_uri(self, uri: str, config: DownloadConfig | None
         """
         logger.info("Start download from URI: %s", uri)
 
-        uri = await unshorten(uri)
+        uri, _ = await unshorten(uri)
         scheme = URL(uri).scheme
 
         if scheme in ("http", "https"):

src/tribler/core/libtorrent/restapi/torrentinfo_endpoint.py

@@ -158,9 +158,8 @@ async def get_torrent_info(self, request: Request) -> RESTResponse:  # noqa: C90
                                     "message": "uri parameter missing"
                                 }}, status=HTTP_BAD_REQUEST)
 
-        uri = await unshorten(p_uri)
+        uri, valid_cert = await unshorten(p_uri)
         scheme = URL(uri).scheme
-        valid_cert = True
 
         if scheme == "file":
             file_path = url_to_path(uri)

src/tribler/core/libtorrent/torrents.py

@@ -5,12 +5,13 @@
 from contextlib import suppress
 from hashlib import sha1
 from os.path import getsize
-from typing import TYPE_CHECKING, Callable, Iterable, TypedDict, TypeVar
+from typing import TYPE_CHECKING, Callable, TypedDict, TypeVar
 
 import libtorrent as lt
 from typing_extensions import ParamSpec
 
 if TYPE_CHECKING:
+    from collections.abc import Iterable
     from pathlib import Path
 
     from tribler.core.libtorrent.download_manager.download import Download
@@ -20,7 +21,7 @@
 logger = logging.getLogger(__name__)
 WrappedParams = ParamSpec("WrappedParams")
 WrappedReturn = TypeVar("WrappedReturn")
-Wrapped = Callable[WrappedParams, WrappedReturn]
+Wrapped = Callable[WrappedParams, WrappedReturn]  # We are forced to type ignore in Python 3.9, revisit in 3.10!
 
 
 def check_handle(default: WrappedReturn) -> Wrapped:
@@ -31,7 +32,9 @@ def check_handle(default: WrappedReturn) -> Wrapped:
     """
 
     def wrap(f: Wrapped) -> Wrapped:
-        def invoke_func(self: Download, *args: WrappedParams.args, **kwargs: WrappedParams.kwargs) -> WrappedReturn:
+        def invoke_func(self: Download,
+                        *args: WrappedParams.args, **kwargs: WrappedParams.kwargs  # type: ignore[valid-type]
+                        ) -> WrappedReturn:
             if self.handle and self.handle.is_valid():
                 return f(self, *args, **kwargs)
             return default
@@ -48,8 +51,9 @@ def require_handle(func: Wrapped) -> Wrapped:
     Author(s): Egbert Bouman
     """
 
-    def invoke_func(self: Download, *args: WrappedParams.args,
-                    **kwargs: WrappedParams.kwargs) -> Future[WrappedReturn | None]:
+    def invoke_func(self: Download,
+                    *args: WrappedParams.args, **kwargs: WrappedParams.kwargs  # type: ignore[valid-type]
+                    ) -> Future[WrappedReturn | None]:
         result_future: Future[WrappedReturn | None] = Future()
 
         def done_cb(fut: Future[lt.torrent_handle]) -> None:
@@ -86,7 +90,9 @@ def check_vod(default: WrappedReturn) -> Wrapped:
     """
 
     def wrap(f: Wrapped) -> Wrapped:
-        def invoke_func(self: Stream, *args: WrappedParams.args, **kwargs: WrappedParams.kwargs) -> WrappedReturn:
+        def invoke_func(self: Stream,
+                        *args: WrappedParams.args, **kwargs: WrappedParams.kwargs  # type: ignore[valid-type]
+                        ) -> WrappedReturn:
             if self.enabled:
                 return f(self, *args, **kwargs)
             return default

src/tribler/core/libtorrent/uris.py

@@ -2,7 +2,7 @@
 import os
 from pathlib import Path
 
-from aiohttp import ClientSession
+from aiohttp import ClientConnectorCertificateError, ClientSession
 from aiohttp.hdrs import LOCATION
 from yarl import URL
 
@@ -37,26 +37,33 @@ def url_to_path(file_url: str) -> str:
     return str(Path(*path))
 
 
-async def unshorten(uri: str) -> str:
+async def unshorten(uri: str) -> tuple[str, bool]:
     """
     Unshorten a URI if it is a short URI. Return the original URI if it is not a short URI.
 
     :param uri: A string representing the shortened URL that needs to be unshortened.
-    :return: The unshortened URL. If the original URL does not redirect to another URL, the original URL is returned.
+    :return: The unshortened URL, or the URL if not redirected, and whether the certificate is valid.
     """
     scheme = URL(uri).scheme
+    cert_valid = True
     if scheme not in ("http", "https"):
-        return uri
+        return uri, cert_valid
 
     logger.info("Unshortening URI: %s", uri)
 
     async with ClientSession() as session:
         try:
-            async with await session.get(uri, allow_redirects=False) as response:
-                if response.status in (301, 302, 303, 307, 308):
-                    uri = response.headers.get(LOCATION, uri)
+            try:
+                async with await session.get(uri, allow_redirects=False) as response:
+                    if response.status in (301, 302, 303, 307, 308):
+                        uri = response.headers.get(LOCATION, uri)
+            except ClientConnectorCertificateError:
+                cert_valid = False
+                async with await session.get(uri, allow_redirects=False, ssl=False) as response:
+                    if response.status in (301, 302, 303, 307, 308):
+                        uri = response.headers.get(LOCATION, uri)
         except Exception as e:
             logger.warning("Error while unshortening a URI: %s: %s", e.__class__.__name__, str(e), exc_info=e)
 
     logger.info("Unshorted URI: %s", uri)
-    return uri
+    return uri, cert_valid

src/tribler/core/rss/rss.py

@@ -73,7 +73,7 @@ async def resolve(self, urls: set[str]) -> None:
         """
         for url in urls:
             try:
-                uri = await unshorten(url)
+                uri, _ = await unshorten(url)
                 response = await query_uri(uri, valid_cert=False)
             except (ServerConnectionError, ClientResponseError, SSLError, ClientConnectorError,
                     AsyncTimeoutError, ValueError) as e:

src/tribler/test_unit/core/libtorrent/restapi/test_torrentinfo_endpoint.py

@@ -18,11 +18,11 @@
 from tribler.test_unit.core.libtorrent.mocks import TORRENT_WITH_DIRS_CONTENT
 
 
-async def mock_unshorten(uri: str) -> str:
+async def mock_unshorten(uri: str) -> tuple[str, bool]:
     """
     Don't following links.
     """
-    return uri
+    return uri, True
 
 
 class TestTorrentInfoEndpoint(TestBase):

src/tribler/test_unit/core/libtorrent/test_uris.py

@@ -109,7 +109,7 @@ async def test_unshorten_non_http_https(self) -> None:
         """
         url = "udp://tracker.example.com/"
 
-        unshortened = await unshorten(url)
+        unshortened, _ = await unshorten(url)
 
         self.assertEqual(url, unshortened)
 
@@ -126,7 +126,7 @@ async def test_unshorten_non_redirect(self) -> None:
                     __aenter__=AsyncMock(return_value=Mock(status=200, headers={istr("Location"): "test"}))
                 ))))
         ))}):
-            unshortened = await unshorten(url)
+            unshortened, _ = await unshorten(url)
 
         self.assertEqual(url, unshortened)
 
@@ -143,7 +143,7 @@ async def test_unshorten_redirect_no_location(self) -> None:
                     __aenter__=AsyncMock(return_value=Mock(status=301, headers={}))
                 ))))
         ))}):
-            unshortened = await unshorten(url)
+            unshortened, _ = await unshorten(url)
 
         self.assertEqual(url, unshortened)
 
@@ -160,6 +160,6 @@ async def test_unshorten_redirect(self) -> None:
                     __aenter__=AsyncMock(return_value=Mock(status=301, headers={istr("Location"): "test"}))
                 ))))
         ))}):
-            unshortened = await unshorten(url)
+            unshortened, _ = await unshorten(url)
 
         self.assertEqual("test", unshortened)