Eng 1740 fallback to database identities if identities have expired from redis

CHANGELOG.md

@@ -35,6 +35,7 @@ Changes can also be flagged with a GitHub label for tracking purposes. The URL o
 - Limit action center tree expansion to 4 levels (Database/Schema/Table/Field) by treating fields as leaf nodes [#6907](https://github.com/ethyca/fides/pull/6907)
 - Allowing pending (new) requests to be resubmitted [#6916](https://github.com/ethyca/fides/pull/6916)
 - Allowing dynamic Celery environment variables to be loaded if they match the `FIDES__CELERY__` prefix [#6916](https://github.com/ethyca/fides/pull/6916)
+- PrivacyRequest cache now falls back to the db for identity and custom fields [#6896](https://github.com/ethyca/fides/pull/6896)
 
 ### Developer Experience
 - Added keyboard navigation to CustomList component [#6903](https://github.com/ethyca/fides/pull/6903)

src/fides/api/api/v1/endpoints/privacy_request_endpoints.py

@@ -1359,8 +1359,9 @@ def restart_privacy_request_from_failure(
     )
 
     # Automatically resubmit the request if the cache has expired
+
     if (
-        not privacy_request.get_cached_identity_data()
+        not privacy_request.verify_cache_for_identity_data()
         and privacy_request.status != PrivacyRequestStatus.complete
     ):
         logger.info(

src/fides/api/models/privacy_request/privacy_request.py

@@ -4,7 +4,7 @@
 
 import json
 from datetime import datetime, timedelta
-from typing import TYPE_CHECKING, Any, Dict, List, Optional, Set, Union
+from typing import TYPE_CHECKING, Any, Dict, List, Optional, Set, Tuple, Union
 
 from celery.result import AsyncResult
 from loguru import logger
@@ -692,12 +692,29 @@ def persist_masking_secrets(
                 },
             )
 
-    def get_cached_identity_data(self) -> Dict[str, Any]:
-        """Retrieves any identity data pertaining to this request from the cache"""
+    def identity_prefix_cache_and_keys(self) -> Tuple[str, FidesopsRedis, List[str]]:
+        """Returns the prefix and cache keys for the identity data for this request"""
         prefix = f"id-{self.id}-identity-*"
         cache: FidesopsRedis = get_cache()
         keys = cache.keys(prefix)
-        result = {}
+        return prefix, cache, keys
+
+    def verify_cache_for_identity_data(self) -> bool:
+        """Verifies if the identity data is cached for this request"""
+        _, _, keys = self.identity_prefix_cache_and_keys()
+        return len(keys) > 0
+
+    def get_cached_identity_data(self) -> Dict[str, Any]:
+        """Retrieves any identity data pertaining to this request from the cache"""
+        result: Dict[str, Any] = {}
+        prefix, cache, keys = self.identity_prefix_cache_and_keys()
+
+        if not keys:
+            logger.debug(f"Cache miss for request {self.id}, falling back to DB")
+            identity = self.get_persisted_identity()
+            self.cache_identity(identity)
+            keys = cache.keys(prefix)
+
         for key in keys:
             value = cache.get(key)
             if value:
@@ -715,10 +732,25 @@ def get_cached_identity_data(self) -> Dict[str, Any]:
 
     def get_cached_custom_privacy_request_fields(self) -> Dict[str, Any]:
         """Retrieves any custom fields pertaining to this request from the cache"""
+        result: Dict[str, Any] = {}
         prefix = f"id-{self.id}-custom-privacy-request-field-*"
+
         cache: FidesopsRedis = get_cache()
         keys = cache.keys(prefix)
-        result = {}
+
+        if not keys:
+            logger.debug(f"Cache miss for request {self.id}, falling back to DB")
+            custom_privacy_request_fields = (
+                self.get_persisted_custom_privacy_request_fields()
+            )
+            self.cache_custom_privacy_request_fields(
+                {
+                    key: CustomPrivacyRequestFieldSchema(**value)
+                    for key, value in custom_privacy_request_fields.items()
+                }
+            )
+            keys = cache.keys(prefix)
+
         for key in keys:
             value = cache.get(key)
             if value:

tests/ops/models/privacy_request/test_privacy_request.py

@@ -43,7 +43,12 @@
     PrivacyRequestStatus,
 )
 from fides.api.schemas.redis_cache import Identity, LabeledIdentity
-from fides.api.util.cache import FidesopsRedis, get_cache, get_identity_cache_key
+from fides.api.util.cache import (
+    FidesopsRedis,
+    get_cache,
+    get_custom_privacy_request_field_cache_key,
+    get_identity_cache_key,
+)
 from fides.api.util.constants import API_DATE_FORMAT
 from fides.config import CONFIG
 
@@ -254,6 +259,99 @@ def test_delete_privacy_request_removes_cached_data(
     assert cache.get(key) is None
 
 
+def test_cache_identity_fallback_to_db(
+    db: Session,
+    privacy_request_with_email_identity: PrivacyRequest,
+    cache: FidesopsRedis,
+    loguru_caplog,
+) -> None:
+    identity = privacy_request_with_email_identity.get_persisted_identity()
+    privacy_request_with_email_identity.cache_identity(identity)
+    key = get_identity_cache_key(
+        privacy_request_id=privacy_request_with_email_identity.id,
+        identity_attribute="email",
+    )
+    cached_identity_data = (
+        privacy_request_with_email_identity.get_cached_identity_data()
+    )
+    assert cached_identity_data != {}
+    cache.delete(key)
+    assert cache.get(key) is None
+    assert (
+        privacy_request_with_email_identity.get_cached_identity_data()
+        == cached_identity_data
+    )
+    assert (
+        f"Cache miss for request {privacy_request_with_email_identity.id}, falling back to DB"
+        in loguru_caplog.messages[-1]
+    )
+
+
+def test_cache_identity_fallback_to_db_no_persisted_identity(
+    db: Session,
+    cache: FidesopsRedis,
+    loguru_caplog,
+    policy: Policy,
+) -> None:
+    privacy_request = PrivacyRequest.create(
+        db=db,
+        data={
+            "policy_id": policy.id,
+            "status": "pending",
+        },
+    )
+    key = get_identity_cache_key(
+        privacy_request_id=privacy_request.id,
+        identity_attribute="email",
+    )
+    cached_identity_data = privacy_request.get_cached_identity_data()
+    assert cached_identity_data == {}
+    cache.delete(key)
+    assert cache.get(key) is None
+    assert privacy_request.get_cached_identity_data() == {}
+    assert (
+        f"Cache miss for request {privacy_request.id}, falling back to DB"
+        in loguru_caplog.messages[-1]
+    )
+
+
+def test_custom_privacy_request_fields_fallback_to_db(
+    db: Session,
+    privacy_request: PrivacyRequest,
+    cache: FidesopsRedis,
+    loguru_caplog,
+) -> None:
+    custom_privacy_request_field = CustomPrivacyRequestField(
+        label="Test",
+        value="test",
+    )
+    privacy_request.persist_custom_privacy_request_fields(
+        db=db,
+        custom_privacy_request_fields=[custom_privacy_request_field],
+    )
+    privacy_request.cache_custom_privacy_request_fields(
+        custom_privacy_request_fields=[custom_privacy_request_field],
+    )
+    key = get_custom_privacy_request_field_cache_key(
+        privacy_request_id=privacy_request.id,
+        custom_privacy_request_field=custom_privacy_request_field.label,
+    )
+    cached_custom_privacy_request_fields = (
+        privacy_request.get_cached_custom_privacy_request_fields()
+    )
+    assert cached_custom_privacy_request_fields is not None
+    cache.delete(key)
+    assert cache.get(key) is None
+    assert (
+        privacy_request.get_cached_custom_privacy_request_fields()
+        == cached_custom_privacy_request_fields
+    )
+    assert (
+        f"Cache miss for request {privacy_request.id}, falling back to DB"
+        in loguru_caplog.messages[-1]
+    )
+
+
 @pytest.mark.parametrize(
     "privacy_request,expected_status",
     [

tests/ops/service/privacy_request/test_email_batch_send.py

@@ -78,21 +78,6 @@ def second_privacy_request_awaiting_erasure_email_send(
     privacy_request.delete(db)
 
 
-@pytest.fixture(scope="function")
-def third_privacy_request_awaiting_erasure_email_send(
-    db: Session, erasure_policy: Policy
-) -> PrivacyRequest:
-    """Add a third erasure privacy request w/ no identity in this state for these tests"""
-    privacy_request = _create_privacy_request_for_policy(
-        db,
-        erasure_policy,
-    )
-    privacy_request.status = PrivacyRequestStatus.awaiting_email_send
-    privacy_request.save(db)
-    yield privacy_request
-    privacy_request.delete(db)
-
-
 class TestConsentEmailBatchSend:
     @mock.patch(
         "fides.api.service.connectors.consent_email_connector.send_single_consent_email",
@@ -982,22 +967,23 @@ def test_send_erasure_email(
         requeue_privacy_requests,
         send_single_erasure_email,
         db,
+        erasure_policy: Policy,
         privacy_request_awaiting_erasure_email_send,
         second_privacy_request_awaiting_consent_email_send,
-        third_privacy_request_awaiting_erasure_email_send,
         attentive_email_connection_config,
     ) -> None:
         """
         Test for batch erasure email, also verifies that a privacy request
         queued for a consent email doesn't trigger an erasure email.
         """
         # third_privacy_request_awaiting_erasure_email_send has no identities
-        cache = get_cache()
-        all_keys = get_all_cache_keys_for_privacy_request(
-            privacy_request_id=third_privacy_request_awaiting_erasure_email_send.id
+        third_privacy_request_awaiting_erasure_email_send = PrivacyRequest.create(
+            db=db,
+            data={
+                "policy_id": erasure_policy.id,
+                "status": PrivacyRequestStatus.awaiting_email_send,
+            },
         )
-        for key in all_keys:
-            cache.delete(key)
 
         exit_state = send_email_batch.delay().get()
         assert exit_state == EmailExitState.complete
@@ -1073,22 +1059,23 @@ def test_send_generic_erasure_email(
         requeue_privacy_requests,
         send_single_erasure_email,
         db,
+        erasure_policy: Policy,
         privacy_request_awaiting_erasure_email_send,
         second_privacy_request_awaiting_consent_email_send,
-        third_privacy_request_awaiting_erasure_email_send,
         generic_erasure_email_connection_config,
     ) -> None:
         """
         Test for batch erasure email, also verifies that a privacy request
         queued for a consent email doesn't trigger an erasure email.
         """
         # third_privacy_request_awaiting_erasure_email_send has no identities
-        cache = get_cache()
-        all_keys = get_all_cache_keys_for_privacy_request(
-            privacy_request_id=third_privacy_request_awaiting_erasure_email_send.id
+        third_privacy_request_awaiting_erasure_email_send = PrivacyRequest.create(
+            db=db,
+            data={
+                "policy_id": erasure_policy.id,
+                "status": PrivacyRequestStatus.awaiting_email_send,
+            },
         )
-        for key in all_keys:
-            cache.delete(key)
 
         exit_state = send_email_batch.delay().get()
         assert exit_state == EmailExitState.complete