Sprint 223 Staging Deploy

.github/workflows/code-style-checks.yaml

@@ -37,6 +37,8 @@ jobs:
         with:
           files: |
             **.py
+          files_ignore: |
+            **/migrations/**
           separator: " "
 
       - name: Check If `changed-python-files` Is An Empty String

pyproject.toml

@@ -32,9 +32,9 @@ dependencies = [
     "django-debug-toolbar==4.3.*",
     "django-extensions==3.2.*",
     "django-spaghetti-and-meatballs==0.4.*",
-    "Django==4.2.*",
+    "Django==5.2.*",
     "django_cte==1.3.*",
-    "djangorestframework==3.15.*",
+    "djangorestframework==3.17.*",
     "docutils>=0.10,<0.17",
     "drf-api-tracking==1.8.4",
     "drf-extensions==0.7.*",
@@ -233,3 +233,6 @@ pycodestyle.max-line-length = 120
     "PLR0913", # too-many-arguments; avoid conflict with too many fixtures
     "PLR0915", # too-many-statements; avoid conflict with long fixtures
 ]
+"**/disaster/management/sql/spark/**.py" = [
+    "E501", # Line too long
+]

usaspending_api/awards/migrations/0115_rename_financialaccountsbyawards_disaster_emergency_fund_submission_award_piid_fain_uri_parent_award.py

@@ -0,0 +1,18 @@
+# Generated by Django 5.2.14 on 2026-05-07 12:57
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('awards', '0114_alter_ctodlinkageupdates_award_id'),
+    ]
+
+    operations = [
+        migrations.RenameIndex(
+            model_name='financialaccountsbyawards',
+            new_name='financial_a_disaste_396e45_idx',
+            old_fields=('disaster_emergency_fund', 'submission', 'award', 'piid', 'fain', 'uri', 'parent_award_id', 'transaction_obligated_amount', 'gross_outlay_amount_by_award_cpe'),
+        ),
+    ]

usaspending_api/awards/models/financial_accounts_by_awards.py

@@ -1,6 +1,6 @@
 from django.db import models
-from django_cte import CTEManager
 from django.db.models import Q
+from django_cte import CTEManager
 
 from usaspending_api.common.models import DataSourceTrackedModel
 
@@ -154,21 +154,6 @@ class FinancialAccountsByAwards(AbstractFinancialAccountsByAwards):
     class Meta:
         managed = True
         db_table = "financial_accounts_by_awards"
-        index_together = [
-            # This index dramatically sped up disaster endpoint queries.  VERY IMPORTANT!  It needs
-            # to cover all of the fields being queried in order to eek out maximum performance.
-            [
-                "disaster_emergency_fund",
-                "submission",
-                "award",
-                "piid",
-                "fain",
-                "uri",
-                "parent_award_id",
-                "transaction_obligated_amount",
-                "gross_outlay_amount_by_award_cpe",
-            ]
-        ]
         indexes = [
             models.Index(
                 fields=["submission_id", "treasury_account_id"],
@@ -193,4 +178,19 @@ class Meta:
                 name="faba_subid_awardkey_sums_idx",
                 condition=Q(disaster_emergency_fund__in=["L", "M", "N", "O", "P", "U", "V"]),
             ),
+            # This index dramatically sped up disaster endpoint queries.  VERY IMPORTANT!  It needs
+            # to cover all of the fields being queried in order to eek out maximum performance.
+            models.Index(
+                fields=[
+                    "disaster_emergency_fund",
+                    "submission",
+                    "award",
+                    "piid",
+                    "fain",
+                    "uri",
+                    "parent_award_id",
+                    "transaction_obligated_amount",
+                    "gross_outlay_amount_by_award_cpe",
+                ]
+            ),
         ]

usaspending_api/awards/models/transaction_normalized.py

@@ -152,7 +152,9 @@ def __str__(self):
     class Meta:
         managed = False
         db_table = "vw_transaction_normalized"
-        index_together = ["award", "action_date"]
+        indexes = [
+            models.Index(fields=["award", "action_date"])
+        ]
 
 
 NORM_ALT_COL_NAMES_IN_TRANSACTION_SEARCH = {

usaspending_api/common/cache.py

@@ -1,6 +1,9 @@
 import hashlib
 import json
+from typing import Any, Callable
 
+from rest_framework.request import Request
+from rest_framework.views import APIView
 from rest_framework_extensions.key_constructor import bits
 from rest_framework_extensions.key_constructor.constructors import DefaultKeyConstructor
 
@@ -12,7 +15,9 @@ class PathKeyBit(bits.QueryParamsKeyBit):
     Adds query path as a key bit
     """
 
-    def get_source_dict(self, params, view_instance, view_method, request, args, kwargs):
+    def get_source_dict(
+        self, params: dict, view_instance: APIView, view_method: Callable, request: Request, args: Any, kwargs: Any
+    ) -> dict[str, str]:
         return {"path": request.path}
 
 
@@ -22,14 +27,17 @@ class GetPostQueryParamsKeyBit(bits.QueryParamsKeyBit):
     directives in a POST request (i.e., request.data) as well as GET parameters
     """
 
-    def get_source_dict(self, params, view_instance, view_method, request, args, kwargs):
+    def get_source_dict(
+        self, params: dict, view_instance: APIView, view_method: Callable, request: Request, args: Any, kwargs: Any
+    ) -> dict[str, str]:
 
         if hasattr(view_instance, "cache_key_whitelist"):
             whitelist = view_instance.cache_key_whitelist
             params = {}
             for param in whitelist:
-                if param in request.query_params:
-                    params[param] = request.query_params[param]
+                # Currently, the only view with a whitelist attr is RecipientCount
+                # RecipientCount only reads the request body, not query params.
+                # Take this into consideration when implementing whitelist attr
                 if param in request.data:
                     params[param] = request.data[param]
         else:
@@ -43,14 +51,14 @@ def get_source_dict(self, params, view_instance, view_method, request, args, kwa
 
 class USAspendingKeyConstructor(DefaultKeyConstructor):
     """
-    Handle cache key construction for API requests. If we never need to create more nuanced keys, see the
+    Handle cache key construction for API requests. If we ever need to create more nuanced keys, see the
     drf-extensions documentation: http://chibisov.github.io/drf-extensions/docs/#default-key-constructor
     """
 
     path_bit = PathKeyBit()
     request_params = GetPostQueryParamsKeyBit()
 
-    def prepare_key(self, key_dict):
+    def prepare_key(self, key_dict: dict) -> str:
         # Order the key_dict using the order_nested_object function to make sure cache keys are always exactly the same
         ordered_key_dict = json.dumps(order_nested_object(key_dict))
         key_hex = hashlib.md5(ordered_key_dict.encode("utf-8"), usedforsecurity=False).hexdigest()

usaspending_api/common/helpers/orm_helpers.py

@@ -1,23 +1,20 @@
-from datetime import date
 from functools import reduce
 from operator import add
 from typing import List, Union
 
 from django.contrib.postgres.aggregates import StringAgg
-from django.db import DEFAULT_DB_ALIAS
+from django.db import DEFAULT_DB_ALIAS, connections
+from django.db.backends.postgresql.psycopg_any import mogrify
 from django.db.models import Aggregate, Case, CharField, F, Func, IntegerField, QuerySet, TextField, Value, When
 from django.db.models.functions import Cast, Coalesce, Concat, LPad
 
 from usaspending_api.awards.v2.lookups.lookups import (
+    all_award_types_mappings,
     assistance_type_mapping,
     procurement_type_mapping,
-    all_award_types_mappings,
 )
 
 
-TYPES_TO_QUOTE_IN_SQL = (str, date)
-
-
 class AwardGroupsException(Exception):
     """Custom Exception for a specific event"""
 
@@ -103,7 +100,7 @@ class StringAggWithDefault(StringAgg):
     output_field = TextField()
 
 
-def get_fyp_notation(relation_name=None):
+def get_fyp_notation(relation_name: str = None) -> Concat:
     """
     Generates FYyyyyPpp syntax from submission table.  relation_name is the Django ORM
     relation name from the foreign key table to the submission table.
@@ -117,7 +114,7 @@ def get_fyp_notation(relation_name=None):
     )
 
 
-def get_fyq_notation(relation_name=None):
+def get_fyq_notation(relation_name: str = None) -> Concat:
     """
     Generates FYyyyyPpp syntax from submission table.  relation_name is the Django ORM
     relation name from the foreign key table to the submission table.
@@ -131,7 +128,7 @@ def get_fyq_notation(relation_name=None):
     )
 
 
-def get_fyp_or_q_notation(relation_name=None):
+def get_fyp_or_q_notation(relation_name: str = None) -> Case:
     """
     Generates FYyyyyQq or FYyyyyPpp syntax from submission table.  relation_name is the Django ORM
     relation name from the foreign key table to the submission table.
@@ -144,7 +141,7 @@ def get_fyp_or_q_notation(relation_name=None):
     )
 
 
-def get_gtas_fyp_notation():
+def get_gtas_fyp_notation() -> Concat:
     """
     Generates FYyyyyPpp syntax from gtas_sf133_balances table.
     """
@@ -157,31 +154,21 @@ def get_gtas_fyp_notation():
 
 
 def generate_raw_quoted_query(queryset: QuerySet) -> str:
-    """Generates the raw sql from a queryset with quotable types quoted.
+    """Generates the raw sql from a queryset parameters safely quoted by the DB driver.
 
-    This function provided benefit since the Django queryset.query doesn't quote
-        some types such as dates and strings. If Django is updated to fix this,
-        please use that instead.
-
-    Note: To add new python data types that should be quoted in queryset.query output,
-        add them to TYPES_TO_QUOTE_IN_SQL global
+    Compiles the queryset to parameterized SQL (%s placeholders) and uses psycopg
+    client-side binding to quote values.
     """
     sql, params = queryset.query.get_compiler(DEFAULT_DB_ALIAS).as_sql()
-    str_fix_params = []
-    for param in params:
-        if isinstance(param, TYPES_TO_QUOTE_IN_SQL):
-            # single quotes are escaped with two '' for strings in sql
-            param = param.replace("'", "''") if isinstance(param, str) else param
-            str_fix_param = "'{}'".format(param)
-        elif isinstance(param, list):
-            str_fix_param = "ARRAY{}".format(param)
-        else:
-            str_fix_param = param
-        str_fix_params.append(str_fix_param)
-    return sql % tuple(str_fix_params)
-
-
-def obtain_category_from_award_group(type_list):
+    if not params:
+        return sql
+    quoted_sql = mogrify(sql, params, connections[DEFAULT_DB_ALIAS])
+    if isinstance(quoted_sql, bytes):
+        quoted_sql = quoted_sql.decode()
+    return quoted_sql
+
+
+def obtain_category_from_award_group(type_list: list) -> str:
     if not type_list:
         raise AwardGroupsException("Invalid award type list: No types provided.")
 
@@ -210,7 +197,7 @@ def award_types_are_valid_groups(type_list: list) -> bool:
     return is_valid
 
 
-def subaward_types_are_valid_groups(type_list):
+def subaward_types_are_valid_groups(type_list: list) -> bool:
     """Check to ensure the award type list is a subset of one and only one award group.
 
     Groups: are "Procurement" and "Assistance"

usaspending_api/common/helpers/pydantic_error_formatter.py

@@ -1,26 +1,52 @@
+from typing import Any
+
 from pydantic import ValidationError
 
+# Pydantic V2 error types mapped to the type names our API has always used
+API_TYPE_BY_ERROR_TYPE = {
+    "bool_parsing": "boolean",
+    "bool_type": "boolean",
+    "date_from_datetime_parsing": "date",
+    "date_parsing": "date",
+    "dict_type": "object",
+    "float_parsing": "float",
+    "float_type": "float",
+    "int_parsing": "integer",
+    "int_type": "integer",
+    "list_type": "array",
+    "model_attributes_type": "object",
+    "model_type": "object",
+    "string_type": "text",
+}
+
+
+def _key_name(loc: tuple[str | int, ...]) -> str:
+    """Convert a Pydantic error location into the pipe delimited key used in our API messages.
+
+    List indexes and union member tags are dropped, so ("filters", "recipient_locations", 0, "state") becomes
+    "filters|recipient_locations|state" and ("filters", "naics_codes", "list[str]") becomes "filters|"naics_codes".
+    """
+
+    field_names = [part for part in loc if isinstance(part, str) and "[" not in part and part.islower()]
+    return "|".join(field_names)
+
 
-def pydantic_error_formatter(error: ValidationError) -> str:
-    errors: list[dict] = error.errors()
+def pydantic_error_formatter(error: ValidationError) -> str:  # noqa: PLR0911
+    errors: list[dict[str, Any]] = error.errors()
+    key_name = _key_name(errors[0]["loc"])
 
-    for error in errors:
-        # Missing required fields
-        if error.get('msg') == 'Field required':
-            key_name = error['loc'][0]
+    key_errors = [err for err in errors if _key_name(err["loc"]) == key_name]
+    rule_error = next((err for err in key_errors if err["type"] not in API_TYPE_BY_ERROR_TYPE), None)
 
-            return f"Missing value: '{key_name}' is a required field"
+    if rule_error is None:
+        expected_types = ", ".join(dict.fromkeys(API_TYPE_BY_ERROR_TYPE[err["type"]] for err in key_errors))
+        return f"Invalid value in '{key_name}'. '{errors[0]['input']}' is not a valid type ({expected_types})."
 
-        # Incorrect type for a filter
-        else:
-            key_name = error["loc"][0] + "|" + error["loc"][1] if len(error['loc']) > 1 else error['loc'][0]
-            bad_value = error['input']
+    if rule_error["type"] == "missing":
+        return f"Missing value: '{key_name}' is a required field"
 
-            if error['type'].endswith('_type'):
-                expected_type = error['type'].strip("_type")
-            elif error.get('ctx'):
-                expected_type = error['ctx']['expected_type']
-            else:
-                expected_type = "dictionary"
+    if rule_error["type"] == "literal_error":
+        return f"Field '{key_name}' is outside valid values {rule_error['ctx']['expected']}"
 
-            return f"Invalid value in '{key_name}'. '{bad_value}' is not a valid type ({expected_type})."
+    message: str = rule_error["msg"].removeprefix("Value error, ")
+    return f"Invalid value in '{key_name}': {message}"

usaspending_api/common/management/sql/disaster_spending_contracts.sql

@@ -182,7 +182,7 @@ SELECT
     "transaction_fpds"."cost_accounting_standards" AS "cost_accounting_standards_clause_code",
     "transaction_fpds"."cost_accounting_stand_desc" AS "cost_accounting_standards_clause",
     "transaction_fpds"."government_furnished_prope" AS "government_furnished_property_code",
-    "transaction_fpds"."government_furnished_prope" AS "government_furnished_property",
+    "transaction_fpds"."government_furnished_desc" AS "government_furnished_property",
     "transaction_fpds"."sea_transportation" AS "sea_transportation_code",
     "transaction_fpds"."sea_transportation_desc" AS "sea_transportation",
     "transaction_fpds"."consolidated_contract" AS "consolidated_contract_code",

usaspending_api/common/pydantic_base_models/codes.py

@@ -12,8 +12,7 @@ def check_at_least_one_key(self) -> Self:
         if self.require is None and self.exclude is None:
             raise PydanticCustomError(
                 "missing_required_field",
-                'At least one of "require" or "exclude" must be provided',
-                {"expected_type": "array, object"},
+                'At least one of "require" or "exclude" must be provided'
             )
         return self
 
@@ -27,8 +26,7 @@ def check_at_least_one_key(self) -> Self:
         if self.require is None and self.exclude is None:
             raise PydanticCustomError(
                 "missing_required_field",
-                'At least one of "require" or "exclude" must be provided',
-                {"expected_type": "array, object"},
+                'At least one of "require" or "exclude" must be provided'
             )
         return self
 
@@ -42,8 +40,7 @@ def check_at_least_one_key(self) -> Self:
         if self.require is None and self.exclude is None:
             raise PydanticCustomError(
                 "missing_required_field",
-                'At least one of "require" or "exclude" must be provided',
-                {"expected_type": "array, object"}
+                'At least one of "require" or "exclude" must be provided'
             )
         return self