Slow cyberstorm api endpoints (TS-2516)

django/thunderstore/api/cyberstorm/tests/test_markdown.py

@@ -5,6 +5,7 @@
 from rest_framework.test import APIClient
 
 from thunderstore.api.cyberstorm.views.markdown import get_package_version
+from thunderstore.cache.utils import get_cache
 from thunderstore.repository.factories import PackageVersionFactory
 from thunderstore.repository.models import Package
 
@@ -61,21 +62,29 @@ def test_get_package_version__raises_for_inactive_package_version(
 
 @pytest.mark.django_db
 def test_readme_api_view__prerenders_markup(api_client: APIClient) -> None:
+    cache = get_cache("markdown_render")
     v = PackageVersionFactory(readme="# Very **strong** header")
 
+    assert cache.get(f"rendered_html:readme:{v.id}") is None
+
     response = api_client.get(
         f"/api/cyberstorm/package/{v.package.namespace}/{v.package.name}/latest/readme/",
     )
     actual = response.json()
+    expected_html = "<h1>Very <strong>strong</strong> header</h1>\n"
 
-    assert actual["html"] == "<h1>Very <strong>strong</strong> header</h1>\n"
+    assert cache.get(f"rendered_html:readme:{v.id}") == expected_html
+    assert cache.get(f"rendering_status:readme:{v.id}") is None
+    assert cache.get(f"lock.rendered_html:readme:{v.id}") is None
+    assert actual["html"] == expected_html
 
 
 @pytest.mark.django_db
 @pytest.mark.parametrize(
     ("markdown", "markup"),
     (
         ("", ""),
+        ("   ", ""),
         ("Oh hai!", "<p>Oh hai!</p>\n"),
     ),
 )
@@ -84,13 +93,23 @@ def test_changelog_api_view__prerenders_markup(
     markdown: Optional[str],
     markup: str,
 ) -> None:
+    cache = get_cache("markdown_render")
     v = PackageVersionFactory(changelog=markdown)
 
+    assert cache.get(f"rendered_html:changelog:{v.id}") is None
+
     response = api_client.get(
         f"/api/cyberstorm/package/{v.package.namespace}/{v.package.name}/latest/changelog/",
     )
     actual = response.json()
 
+    if markup == "":
+        # We ignore empty strings and dont render them so no need for cache
+        assert cache.get(f"rendered_html:changelog:{v.id}") == None
+    else:
+        assert cache.get(f"rendered_html:changelog:{v.id}") == markup
+    assert cache.get(f"rendering_status:changelog:{v.id}") is None
+    assert cache.get(f"lock.rendered_html:changelog:{v.id}") is None
     assert actual["html"] == markup
 
 

django/thunderstore/api/cyberstorm/views/markdown.py

@@ -5,8 +5,8 @@
 from rest_framework.generics import RetrieveAPIView, get_object_or_404
 
 from thunderstore.api.utils import CyberstormAutoSchemaMixin
-from thunderstore.markdown.templatetags.markdownify import render_markdown
 from thunderstore.repository.models import Package, PackageVersion
+from thunderstore.repository.services.markdown import render_markdown_service
 
 
 class CyberstormMarkdownResponseSerializer(serializers.Serializer):
@@ -29,7 +29,15 @@
             version_number=self.kwargs.get("version_number"),
         )
 
-        return {"html": render_markdown(package_version.readme)}
+        readme = package_version.readme
+        if readme is None:
+            raise Http404("README not found for this package version.")
+
+        return render_markdown_service(
+            markdown=readme,
+            key="readme",
+            object_id=package_version.id,
+        )
 
 
 class PackageVersionChangelogAPIView(CyberstormAutoSchemaMixin, RetrieveAPIView):
@@ -48,10 +56,15 @@
             version_number=self.kwargs.get("version_number"),
         )
 
-        if package_version.changelog is None:
-            raise Http404
+        changelog = package_version.changelog
+        if changelog is None:
+            raise Http404("CHANGELOG not found for this package version.")
 
-        return {"html": render_markdown(package_version.changelog)}
+        return render_markdown_service(
+            markdown=changelog,
+            key="changelog",
+            object_id=package_version.id,
+        )
 
 
 def get_package_version(

django/thunderstore/core/settings.py

@@ -109,6 +109,7 @@
     REDIS_URL_LEGACY=(str, None),
     REDIS_URL_PROFILES=(str, None),
     REDIS_URL_DOWNLOADS=(str, None),
+    REDIS_MARKDOWN_RENDER=(str, None),
     DB_CERT_DIR=(str, ""),
     DB_CLIENT_CERT=(str, ""),
     DB_CLIENT_KEY=(str, ""),
@@ -390,6 +391,7 @@ class CeleryQueues:
     BackgroundCache = "background.cache"
     BackgroundTask = "background.task"
     BackgroundLongRunning = "background.long_running"
+    BackgroundMarkdownRender = "background.markdown_render"
 
 
 CELERY_BROKER_URL = env.str("CELERY_BROKER_URL")
@@ -509,6 +511,10 @@ def get_redis_cache(env_key: str, fallback_key: Optional[str] = None):
         **get_redis_cache("REDIS_URL_DOWNLOADS", "REDIS_URL"),
         "TIMEOUT": None,
     },
+    "markdown_render": {
+        **get_redis_cache("REDIS_MARKDOWN_RENDER", "REDIS_URL"),
+        "TIMEOUT": None,
+    },
 }
 
 

django/thunderstore/core/tests/test_celery.py

@@ -40,6 +40,7 @@ def test_task():
     "thunderstore.repository.tasks.process_package_submission",
     "thunderstore.repository.tasks.cleanup_package_submissions",
     "thunderstore.repository.tasks.log_version_download",
+    "thunderstore.repository.tasks.render_markdown",
     "thunderstore.webhooks.tasks.process_audit_event",
 )
 

django/thunderstore/repository/services/markdown.py

@@ -0,0 +1,34 @@
+import time
+
+from thunderstore.cache.utils import get_cache
+from thunderstore.repository.tasks.markdown import render_markdown_to_html
+
+cache = get_cache("markdown_render")
+
+
+def render_markdown_service(markdown: str, key: str, object_id: int) -> dict:
+    if markdown.strip() == "":
+        return {"html": ""}
+
+    cache_key = f"rendered_html:{key}:{object_id}"
+    status_key = f"rendering_status:{key}:{object_id}"
+
+    html = cache.get(cache_key)
+    if html is not None:
+        return {"html": html}
+
+    if cache.get(status_key) is None:
+        cache.set(status_key, "in_progress", timeout=300)
+        render_markdown_to_html.delay(
+            markdown=markdown,
+            cache_key=cache_key,
+            status_key=status_key,
+        )
+
+    for _ in range(5):  # wait up to 5s total
+        html = cache.get(cache_key)
+        if html is not None:
+            return {"html": html}
+        time.sleep(1)
+
+    return {"html": "<em>Loading...</em>"}

django/thunderstore/repository/tasks/markdown.py

@@ -0,0 +1,32 @@
+from celery import shared_task
+
+from thunderstore.cache.utils import get_cache
+from thunderstore.core.settings import CeleryQueues
+from thunderstore.markdown.templatetags.markdownify import render_markdown
+
+cache = get_cache("markdown_render")
+
+
+@shared_task(
+    queue=CeleryQueues.BackgroundMarkdownRender,
+    name="thunderstore.repository.tasks.render_markdown",
+)
+def render_markdown_to_html(
+    markdown: str,
+    cache_key: str,
+    status_key: str,
+) -> str:
+    cached_html = cache.get(cache_key)
+    if cached_html is not None:
+        return cached_html
+
+    try:
+        html = render_markdown(markdown)
+    except Exception as error:
+        cache.delete(status_key)
+        raise error
+
+    cache.set(cache_key, html)
+    cache.delete(status_key)
+
+    return html

django/thunderstore/repository/tasks/tests/test_markdown.py

@@ -0,0 +1,40 @@
+from unittest.mock import patch
+
+import pytest
+
+from thunderstore.cache.utils import get_cache
+from thunderstore.repository.tasks.markdown import render_markdown_to_html
+
+RENDER_MARKDOWN_PATH = "thunderstore.markdown.templatetags.markdownify.render_markdown"
+
+
+def test_markdown_cache_hit_returns_html():
+    cache = get_cache("markdown_render")
+    cache_key = "rendered_html:test_key:1"
+    cache.set(cache_key, "<p>Cached HTML</p>")
+
+    result = render_markdown_to_html("", cache_key, "status_key")
+    assert result == "<p>Cached HTML</p>"
+
+
+def test_markdown_cache_miss_triggers_rendering_task():
+    cache = get_cache("markdown_render")
+    cache_key = "rendered_html:test_key:1"
+    status_key = "rendering_status:test_key:1"
+
+    result = render_markdown_to_html("test markdown", cache_key, status_key)
+    assert result == "<p>test markdown</p>\n"
+    assert cache.get(cache_key) == "<p>test markdown</p>\n"
+    assert cache.get(status_key) is None
+
+
+def test_markdown_rendering_exception():
+    cache = get_cache("markdown_render")
+    cache_key = "rendered_html:test_key:1"
+    status_key = "rendering_status:test_key:1"
+
+    with pytest.raises(Exception):
+        render_markdown_to_html(None, cache_key, status_key)
+
+    assert cache.get(cache_key) is None
+    assert cache.get(status_key) is None

django/thunderstore/repository/tests/test_markdown_service.py

@@ -0,0 +1,87 @@
+from unittest.mock import patch
+
+import pytest
+
+from thunderstore.cache.utils import get_cache
+from thunderstore.repository.services.markdown import render_markdown_service
+
+
+@pytest.mark.django_db
+def test_render_markdown_empty_input():
+    result = render_markdown_service("", "changelog", 1)
+    assert result == {"html": ""}
+
+    result = render_markdown_service("    ", "changelog", 1)
+    assert result == {"html": ""}
+
+
+@pytest.mark.django_db
+def test_render_markdown_cached_html(package_version):
+    cache = get_cache("markdown_render")
+    cache_key = f"rendered_html:changelog:{package_version.id}"
+    cache.set(cache_key, "<p>Cached HTML</p>")
+
+    result = render_markdown_service(
+        package_version.changelog, "changelog", package_version.id
+    )
+    assert result == {"html": "<p>Cached HTML</p>"}
+
+
+@pytest.mark.django_db
+def test_render_markdown_no_cached_html(package_version):
+    cache = get_cache("markdown_render")
+    cache_key = f"rendered_html:changelog:{package_version.id}"
+    cache.delete(cache_key)
+
+    result = render_markdown_service(
+        package_version.changelog, "changelog", package_version.id
+    )
+    assert result == {"html": "<h1>This is an example changelog</h1>\n"}
+
+
+@pytest.mark.django_db
+def test_render_markdown_in_progress(package_version):
+    cache = get_cache("markdown_render")
+    status_key = f"rendering_status:changelog:{package_version.id}"
+    cache.set(status_key, "in_progress")
+
+    with patch("time.sleep") as sleep_mock:
+        result = render_markdown_service(
+            package_version.changelog, "changelog", package_version.id
+        )
+        assert result == {"html": "<em>Loading...</em>"}
+        sleep_mock.assert_called()
+
+
+@pytest.mark.django_db
+def test_render_markdown_html_available_after_wait(package_version):
+    cache = get_cache("markdown_render")
+    status_key = f"rendering_status:changelog:{package_version.id}"
+    cache_key = f"rendered_html:changelog:{package_version.id}"
+
+    cache.set(status_key, "in_progress")
+
+    def mock_cache_get(key):
+        if key == cache_key:
+            return "<p>Rendered HTML</p>"
+        return None
+
+    with patch("time.sleep"), patch.object(cache, "get", side_effect=mock_cache_get):
+        result = render_markdown_service(
+            package_version.changelog, "changelog", package_version.id
+        )
+        assert result == {"html": "<p>Rendered HTML</p>"}
+
+
+@pytest.mark.django_db
+def test_render_markdown_html_unavailable_after_wait(package_version):
+    cache = get_cache("markdown_render")
+    status_key = f"rendering_status:changelog:{package_version.id}"
+
+    cache.set(status_key, "in_progress")
+
+    with patch("time.sleep"), patch.object(cache, "get", return_value=None):
+        result = render_markdown_service(
+            package_version.changelog, "changelog", package_version.id
+        )
+        assert result == {"html": "<em>Loading...</em>"}

docker-compose.yml

@@ -8,12 +8,13 @@ x-django-service: &django-service
       BUILD_INSTALL_EXTRAS: ${BUILD_INSTALL_EXTRAS}
   environment:
     CELERY_BROKER_URL: "pyamqp://django:django@rabbitmq/django"
-    CELERY_QUEUES: "celery,background.cache,background.task,background.long_running,log.downloads"
+    CELERY_QUEUES: "celery,background.cache,background.task,background.long_running,log.downloads,background.markdown_render"
     DATABASE_URL: "psql://django:django@dbpool/django"
     REDIS_URL: "redis://redis:6379/0"
     REDIS_URL_LEGACY: "redis://redis:6379/1"
     REDIS_URL_PROFILES: "redis://redis:6379/2"
     REDIS_URL_DOWNLOADS: "redis://redis:6379/3"
+    REDIS_MARKDOWN_RENDER: "redis://redis:6379/4"
     PROTOCOL: "http://"
 
     AWS_ACCESS_KEY_ID: "thunderstore"