ref(grouping): Allow rate-limiting of grouping config upgrades (#102441)

This adds a sample rate to grouping config upgrades, in case we decide we want to roll out the new config bit by bit. It can also serve as a killswitch should we ever decide we need one.

Author: lobsterkatie

src/sentry/grouping/ingest/config.py

@@ -3,6 +3,7 @@
 import logging
 import time
 from collections.abc import MutableMapping
+from random import random
 from typing import Any
 
 from django.conf import settings
@@ -59,6 +60,17 @@ def update_or_set_grouping_config_if_needed(project: Project, source: str) -> st
     if cache.get(cache_key) is not None:
         return "skipped - race condition"
 
+    # Check if we're rate-limiting upgrades. Note that we only allow skips here if the project will
+    # be left with a valid config record even if it's skipped.
+    upgrade_sample_rate = options.get("grouping.config_transition.config_upgrade_sample_rate")
+    if (
+        upgrade_sample_rate < 1
+        and current_config_is_valid
+        and project_option_exists
+        and random() > upgrade_sample_rate
+    ):
+        return "skipped - sample rate"
+
     try:
         with locks.get(lock_key, duration=60, name="grouping-update-lock").acquire():
             if cache.get(cache_key) is not None:

src/sentry/options/defaults.py

@@ -2821,6 +2821,15 @@
     flags=FLAG_AUTOMATOR_MODIFIABLE,
 )
 
+# This is here in case we want to roll out a new config gradually. Can also be used as a killswitch
+# for config upgrades if one is ever needed.
+register(
+    "grouping.config_transition.config_upgrade_sample_rate",
+    type=Float,
+    default=1.0,
+    flags=FLAG_AUTOMATOR_MODIFIABLE,
+)
+
 
 # Sample rate for double writing to experimental dsn
 register(

tests/sentry/event_manager/test_event_manager_grouping.py

@@ -19,6 +19,7 @@
 from sentry.models.project import Project
 from sentry.testutils.cases import TestCase
 from sentry.testutils.helpers.eventprocessing import save_new_event
+from sentry.testutils.helpers.options import override_options
 from sentry.testutils.pytest.fixtures import django_db_all
 from sentry.testutils.silo import assume_test_silo_mode_of
 from sentry.testutils.skips import requires_snuba
@@ -239,6 +240,80 @@ def test_no_ops_if_grouping_config_project_option_exists_and_is_current(
         assert self.project.get_option("sentry:secondary_grouping_config") is None
         assert self.project.get_option("sentry:secondary_grouping_expiry") == 0
 
+    @patch(
+        "sentry.event_manager.update_or_set_grouping_config_if_needed",
+        wraps=update_or_set_grouping_config_if_needed,
+    )
+    def test_no_ops_if_sample_rate_test_fails(self, update_config_spy: MagicMock):
+        with (
+            # Ensure our die roll will fall outside the sample rate
+            patch("sentry.grouping.ingest.config.random", return_value=0.1121),
+            override_options({"grouping.config_transition.config_upgrade_sample_rate": 0.0908}),
+        ):
+            self.project.update_option("sentry:grouping_config", NO_MSG_PARAM_CONFIG)
+            assert self.project.get_option("sentry:grouping_config") == NO_MSG_PARAM_CONFIG
+
+            save_new_event({"message": "Dogs are great!"}, self.project)
+
+            update_config_spy.assert_called_with(self.project, "ingest")
+
+            # After the function has been called, the config hasn't changed and no transition has
+            # started
+            assert self.project.get_option("sentry:grouping_config") == NO_MSG_PARAM_CONFIG
+            assert self.project.get_option("sentry:secondary_grouping_config") is None
+            assert self.project.get_option("sentry:secondary_grouping_expiry") == 0
+
+    @patch(
+        "sentry.event_manager.update_or_set_grouping_config_if_needed",
+        wraps=update_or_set_grouping_config_if_needed,
+    )
+    def test_ignores_sample_rate_if_current_config_is_invalid(self, update_config_spy: MagicMock):
+        with (
+            # Ensure our die roll will fall outside the sample rate
+            patch("sentry.grouping.ingest.config.random", return_value=0.1121),
+            override_options({"grouping.config_transition.config_upgrade_sample_rate": 0.0908}),
+        ):
+            self.project.update_option("sentry:grouping_config", "not_a_real_config")
+            assert self.project.get_option("sentry:grouping_config") == "not_a_real_config"
+
+            save_new_event({"message": "Dogs are great!"}, self.project)
+
+            update_config_spy.assert_called_with(self.project, "ingest")
+
+            # The config has been updated, but no transition has started because we can't calculate
+            # a secondary hash using a config that doesn't exist
+            assert self.project.get_option("sentry:grouping_config") == DEFAULT_GROUPING_CONFIG
+            assert self.project.get_option("sentry:secondary_grouping_config") is None
+            assert self.project.get_option("sentry:secondary_grouping_expiry") == 0
+
+    @patch(
+        "sentry.event_manager.update_or_set_grouping_config_if_needed",
+        wraps=update_or_set_grouping_config_if_needed,
+    )
+    def test_ignores_sample_rate_if_no_record_exists(self, update_config_spy: MagicMock):
+        with (
+            # Ensure our die roll will fall outside the sample rate
+            patch("sentry.grouping.ingest.config.random", return_value=0.1121),
+            override_options({"grouping.config_transition.config_upgrade_sample_rate": 0.0908}),
+        ):
+            assert self.project.get_option("sentry:grouping_config") == DEFAULT_GROUPING_CONFIG
+            assert not ProjectOption.objects.filter(
+                project_id=self.project.id, key="sentry:grouping_config"
+            ).exists()
+
+            save_new_event({"message": "Dogs are great!"}, self.project)
+
+            update_config_spy.assert_called_with(self.project, "ingest")
+
+            # The config hasn't been updated, but now the project has its own record. No transition
+            # has started because the config was already up to date.
+            assert self.project.get_option("sentry:grouping_config") == DEFAULT_GROUPING_CONFIG
+            assert ProjectOption.objects.filter(
+                project_id=self.project.id, key="sentry:grouping_config"
+            ).exists()
+            assert self.project.get_option("sentry:secondary_grouping_config") is None
+            assert self.project.get_option("sentry:secondary_grouping_expiry") == 0
+
 
 class PlaceholderTitleTest(TestCase):
     """