feat: dynamically check if the missing_* columsn exist

Author: sgratzl

src/server/_db.py

@@ -1,14 +1,26 @@
-from sqlalchemy import MetaData, create_engine
+from typing import Dict, List
+from sqlalchemy import MetaData, create_engine, inspect
 from sqlalchemy.engine import Engine
+from sqlalchemy.engine.reflection import Inspector
 
 from ._config import SQLALCHEMY_DATABASE_URI, SQLALCHEMY_ENGINE_OPTIONS
 
 engine: Engine = create_engine(SQLALCHEMY_DATABASE_URI, **SQLALCHEMY_ENGINE_OPTIONS)
 metadata = MetaData(bind=engine)
+inspector: Inspector = inspect(engine)
 
 TABLE_OPTIONS = dict(
     mysql_engine="InnoDB",
     # mariadb_engine="InnoDB",
     mysql_charset="utf8mb4",
     # mariadb_charset="utf8",
 )
+
+
+def sql_table_has_columns(table: str, columns: List[str]) -> bool:
+    """
+    checks whether the given table has all the given columns defined
+    """
+    table_columns: List[Dict] = inspector.get_columns(table)
+    table_column_names = set(str(d.get("name", "")).lower() for d in table_columns)
+    return all(c.lower() in table_column_names for c in columns)

src/server/endpoints/covidcast.py

@@ -31,6 +31,7 @@
     require_all,
     require_any,
 )
+from .._db import sql_table_has_columns
 from .._pandas import as_pandas
 from .covidcast_utils import compute_trend, compute_trends, compute_correlations, compute_trend_value, CovidcastMetaEntry, AllSignalsMap
 from ..utils import shift_time_value, date_to_time_value, time_value_to_iso
@@ -134,7 +135,7 @@ def handle():
     q = QueryBuilder("covidcast", "t")
 
     fields_string = ["geo_value", "signal"]
-    fields_int = ["time_value", "direction", "issue", "lag", "missing_value", "missing_stderr", "missing_sample_size"]
+    fields_int = ["time_value", "direction", "issue", "lag"]
     fields_float = ["value", "stderr", "sample_size"]
     if is_compatibility_mode():
         q.set_order("signal", "time_value", "geo_value", "issue")
@@ -144,6 +145,15 @@ def handle():
         q.set_order("source", "signal", "time_type", "time_value", "geo_type", "geo_value", "issue")
     q.set_fields(fields_string, fields_int, fields_float)
 
+    missing_fields = ["missing_value", "missing_stderr", "missing_sample_size"]
+    fields_int.extend(missing_fields)
+    if sql_table_has_columns("covidcast", missing_fields):
+        # real fields
+        q.fields.extend([f"{q.alias}.{field}" for field in missing_fields])
+    else:
+        # fake fields
+        q.fields.extend([f"0 as {field}" for field in missing_fields])
+
     # basic query info
     # data type of each field
     # build the source, signal, time, and location (type and id) filters