Add core__procedure table

cumulus_library/.sqlfluff

@@ -250,6 +250,18 @@ schema =
             'gender': True, 
             'address': True, 
             'birthdate': True
+        },
+        'procedure': {
+            'encounter': {
+                'reference': True,
+            },
+            'id': True,
+            'performedPeriod': {
+              'start': True, 'end': True,
+            },
+            'subject': {
+                'reference': True,
+            },
         }
     }
 source_table = source_table

cumulus_library/builders/counts.py

@@ -299,6 +299,32 @@ def count_patient(
             fhir_resource="patient",
         )
 
+    def count_procedure(
+        self,
+        table_name: str,
+        source_table: str,
+        table_cols: list,
+        where_clauses: list | None = None,
+        min_subject: int | None = None,
+    ) -> str:
+        """wrapper method for constructing procedure counts tables
+
+        :param table_name: The name of the table to create. Must start with study prefix
+        :param source_table: The table to create counts data from
+        :param table_cols: The columns from the source table to add to the count table
+        :param where_clauses: An array of where clauses to use for filtering the data
+        :param min_subject: An integer setting the minimum bin size for inclusion
+            (default: 10)
+        """
+        return self.get_count_query(
+            table_name,
+            source_table,
+            table_cols,
+            where_clauses=where_clauses,
+            min_subject=min_subject,
+            fhir_resource="procedure",
+        )
+
     # End of wrapper section
     # ----------------------------------------------------------------------
 

cumulus_library/builders/statistics_templates/counts_templates.py

@@ -20,11 +20,12 @@ class CountableFhirResource(Enum):
     DIAGNOSTICREPORT = "diagnosticreport"
     DOCUMENTREFERENCE = "documentreference"
     ENCOUNTER = "encounter"
+    MEDICATION = "medication"
+    MEDICATIONREQUEST = "medicationrequest"
     NONE = None
     OBSERVATION = "observation"
     PATIENT = "patient"
-    MEDICATION = "medication"
-    MEDICATIONREQUEST = "medicationrequest"
+    PROCEDURE = "procedure"
 
 
 @dataclass

cumulus_library/studies/core/builder_procedure.py

@@ -0,0 +1,30 @@
+import cumulus_library
+from cumulus_library.studies.core.core_templates import core_templates
+from cumulus_library.template_sql import sql_utils
+
+expected_table_cols = {
+    "procedure": {
+        "id": [],
+        "status": [],
+        "subject": sql_utils.REFERENCE,
+        "encounter": sql_utils.REFERENCE,
+        "performedDateTime": [],
+        "performedPeriod": ["start", "end"],
+    }
+}
+
+
+class CoreProcedureBuilder(cumulus_library.BaseTableBuilder):
+    display_text = "Creating Procedure tables..."
+
+    def prepare_queries(self, *args, config: cumulus_library.StudyConfig, **kwargs):
+        code_sources = [
+            sql_utils.CodeableConceptConfig(
+                source_table="procedure",
+                column_hierarchy=[("code", dict)],
+                target_table="core__procedure_dn_code",
+            ),
+        ]
+        self.queries += sql_utils.denormalize_complex_objects(config.db, code_sources)
+        validated_schema = sql_utils.validate_schema(config.db, expected_table_cols)
+        self.queries.append(core_templates.get_core_template("procedure", validated_schema))

cumulus_library/studies/core/core_templates/completion_utils.jinja

@@ -45,6 +45,7 @@ in this table.
                 AND BOOL_OR(ec.table_name = 'documentreference')
                 AND BOOL_OR(ec.table_name = 'medicationrequest')
                 AND BOOL_OR(ec.table_name = 'observation')
+                AND BOOL_OR(ec.table_name = 'procedure')
             ) AS is_complete
         FROM etl__completion_encounters AS ece
         INNER JOIN temp_completion_times AS tct ON tct.encounter_id = ece.encounter_id

cumulus_library/studies/core/core_templates/procedure.sql.jinja

@@ -0,0 +1,88 @@
+{% import 'core_utils.jinja' as utils %}
+
+-- This table includes all fields of interest to the US Core Procedure profile.
+-- EXCEPT FOR:
+-- * the 'performedAge' and 'performedRange' fields, simply because they are annoying to
+--   represent and not frequently used. They aren't even marked as Must Support by the profile,
+--   only performedDateTime actually is, but we do support performedPeriod as well, since EHRs
+--   often like to use periods.
+--
+-- AND ADDING:
+-- * the `encounter` field, because come on, why isn't it in the profile
+--
+-- There are lots of interesting possible fields to support from the base FHIR spec that aren't
+-- in the US Core profile, like reasonCode, bodySite, and outcome. But EHR support seems low since
+-- they aren't in the profile, so they have been left out so far.
+--
+-- US Core profile for reference:
+-- * http://hl7.org/fhir/us/core/STU4/StructureDefinition-us-core-procedure.html
+
+CREATE TABLE core__procedure AS
+WITH temp_procedure AS (
+    SELECT
+        {{- utils.basic_cols('procedure', 'src', ['id']) }},
+        {{-
+            utils.nullable_cols(
+                'procedure',
+                'src',
+                [
+                    'status',
+                    ('subject', 'reference', 'subject_ref'),
+                    ('encounter', 'reference', 'encounter_ref'),
+                ],
+                schema
+            )
+        }},
+        {{-
+            utils.truncate_date_cols(
+                'procedure',
+                'src',
+                [
+                    ('performedDateTime', 'day'),
+                    ('performedDateTime', 'week'),
+                    ('performedDateTime', 'month'),
+                    ('performedDateTime', 'year'),
+                    ('performedPeriod', 'start', 'performedPeriod_start_day', 'day'),
+                    ('performedPeriod', 'start', 'performedPeriod_start_week', 'week'),
+                    ('performedPeriod', 'start', 'performedPeriod_start_month', 'month'),
+                    ('performedPeriod', 'start', 'performedPeriod_start_year', 'year'),
+                    ('performedPeriod', 'end', 'performedPeriod_end_day', 'day'),
+                    ('performedPeriod', 'end', 'performedPeriod_end_week', 'week'),
+                    ('performedPeriod', 'end', 'performedPeriod_end_month', 'month'),
+                    ('performedPeriod', 'end', 'performedPeriod_end_year', 'year'),
+                ],
+                schema
+            )
+        }}
+    FROM "procedure" AS src
+)
+
+SELECT
+    tp.id,
+    tp.status,
+
+    dn_code.code AS code_code,
+    dn_code.system AS code_system,
+    dn_code.display AS code_display,
+
+    tp.performedDateTime_day,
+    tp.performedDateTime_week,
+    tp.performedDateTime_month,
+    tp.performedDateTime_year,
+
+    tp.performedPeriod_start_day,
+    tp.performedPeriod_start_week,
+    tp.performedPeriod_start_month,
+    tp.performedPeriod_start_year,
+
+    tp.performedPeriod_end_day,
+    tp.performedPeriod_end_week,
+    tp.performedPeriod_end_month,
+    tp.performedPeriod_end_year,
+
+    concat('Procedure/', tp.id) AS procedure_ref,
+    tp.subject_ref,
+    tp.encounter_ref
+
+FROM temp_procedure AS tp
+LEFT JOIN core__procedure_dn_code AS dn_code ON tp.id = dn_code.id;

cumulus_library/studies/core/count_core.py

@@ -129,6 +129,20 @@ def count_core_patient(self):
         cols = ["gender", "race_display", "ethnicity_display"]
         return self.count_patient(table_name, from_table, cols)
 
+    def count_core_procedure(self, duration: str = "month"):
+        table_name = self.get_table_name("count_procedure", duration=duration)
+        from_table = self.get_table_name("procedure")
+        cols = [
+            ["code_display", "varchar", None],
+            # The performed date is annoyingly spread across three fields: performedDateTime,
+            # performedPeriod.start, and performedPeriod.end.
+            # Rather than do some fancy collation, we just use performedDateTime.
+            # It's the only "must support" performed field, and period seems less common.
+            # These core counts are just a rough idea of the data, not a polished final product.
+            [f"performedDateTime_{duration}", "date", None],
+        ]
+        return self.count_procedure(table_name, from_table, cols)
+
     def prepare_queries(self, *args, **kwargs):
         super().prepare_queries(*args, **kwargs)
         self.queries = [
@@ -144,6 +158,7 @@ def prepare_queries(self, *args, **kwargs):
             self.count_core_encounter_priority(duration="month"),
             self.count_core_medicationrequest(duration="month"),
             self.count_core_observation_lab(duration="month"),
+            self.count_core_procedure(duration="month"),
             self.count_core_patient(),
         ]
 

cumulus_library/studies/core/manifest.toml

@@ -11,6 +11,7 @@ file_names = [
     "builder_documentreference.py",
     "builder_medicationrequest.py",
     "builder_observation.py",
+    "builder_procedure.py",
     "observation_type.sql",
     "meta_date.sql",
     "count_core.py"
@@ -31,6 +32,7 @@ count_list = [
     "core__count_medicationrequest_month",
     "core__count_observation_lab_month",
     "core__count_patient",
+    "core__count_procedure_month",
 ]
 meta_list = [
     "core__meta_date",

cumulus_library/studies/core/reference_sql/builder_encounter.sql

@@ -588,6 +588,7 @@ temp_encounter_completion AS (
                 AND BOOL_OR(ec.table_name = 'documentreference')
                 AND BOOL_OR(ec.table_name = 'medicationrequest')
                 AND BOOL_OR(ec.table_name = 'observation')
+                AND BOOL_OR(ec.table_name = 'procedure')
             ) AS is_complete
         FROM etl__completion_encounters AS ece
         INNER JOIN temp_completion_times AS tct ON tct.encounter_id = ece.encounter_id
@@ -740,6 +741,7 @@ temp_encounter_completion AS (
                 AND BOOL_OR(ec.table_name = 'documentreference')
                 AND BOOL_OR(ec.table_name = 'medicationrequest')
                 AND BOOL_OR(ec.table_name = 'observation')
+                AND BOOL_OR(ec.table_name = 'procedure')
             ) AS is_complete
         FROM etl__completion_encounters AS ece
         INNER JOIN temp_completion_times AS tct ON tct.encounter_id = ece.encounter_id

cumulus_library/studies/core/reference_sql/count_core.sql

@@ -1108,6 +1108,61 @@ CREATE TABLE core__count_observation_lab_month AS (
 
 -- ###########################################################
 
+CREATE TABLE core__count_procedure_month AS (
+    WITH
+    filtered_table AS (
+        SELECT
+            s.subject_ref,
+            --noqa: disable=RF03, AL02
+            s."code_display",
+            s."performedDateTime_month"
+            --noqa: enable=RF03, AL02
+        FROM core__procedure AS s
+    ),
+
+    null_replacement AS (
+        SELECT
+            subject_ref,
+            coalesce(
+                cast(code_display AS varchar),
+                'cumulus__none'
+            ) AS code_display,
+            coalesce(
+                cast(performedDateTime_month AS varchar),
+                'cumulus__none'
+            ) AS performedDateTime_month
+        FROM filtered_table
+    ),
+
+    powerset AS (
+        SELECT
+            count(DISTINCT subject_ref) AS cnt_subject_ref,
+            "code_display",
+            "performedDateTime_month",
+            concat_ws(
+                '-',
+                COALESCE("code_display",''),
+                COALESCE("performedDateTime_month",'')
+            ) AS id
+        FROM null_replacement
+        GROUP BY
+            cube(
+            "code_display",
+            "performedDateTime_month"
+            )
+    )
+
+    SELECT
+        p.cnt_subject_ref AS cnt,
+        p."code_display",
+        p."performedDateTime_month"
+    FROM powerset AS p
+    WHERE 
+        cnt_subject_ref >= 10
+);
+
+-- ###########################################################
+
 CREATE TABLE core__count_patient AS (
     WITH
     filtered_table AS (

docs/core-study-details.md

@@ -95,6 +95,8 @@ Per resource, the optional fields are as follows:
   - encounter
   - valueCodeableConcept
   - interpretation
+- Procedure
+  - encounter
 
 ## Deprecation Notice
 

tests/core/test_core.py

@@ -13,21 +13,25 @@
     [
         ("core__allergyintolerance"),
         ("core__condition"),
+        ("core__diagnosticreport"),
         ("core__documentreference"),
         ("core__encounter"),
         ("core__medicationrequest"),
         ("core__observation"),
         ("core__observation_lab"),
         ("core__observation_vital_signs"),
         ("core__patient"),
+        ("core__procedure"),
         ("core__count_allergyintolerance_month"),
         ("core__count_condition_month"),
+        ("core__count_diagnosticreport_month"),
         ("core__count_documentreference_month"),
         ("core__count_encounter_month"),
         ("core__count_encounter_all_types_month"),
         ("core__count_observation_lab_month"),
         ("core__count_medicationrequest_month"),
         ("core__count_patient"),
+        ("core__count_procedure_month"),
     ],
 )
 def test_core_tables(mock_db_core, table):