remove dependence on and references to db docker definition in delphi/operations repo

.github/workflows/ci.yaml

@@ -51,7 +51,6 @@ jobs:
 
       - name: Build docker images
         run: |
-          docker build -t delphi_database -f repos/delphi/operations/dev/docker/database/Dockerfile .
           docker build -t delphi_python -f repos/delphi/operations/dev/docker/python/Dockerfile .
           docker build -t delphi_database_epidata -f ./repos/delphi/delphi-epidata/dev/docker/database/epidata/Dockerfile .
           docker build -t delphi_web_python -f repos/delphi/delphi-epidata/dev/docker/python/Dockerfile .

dev/docker/database/epidata/Dockerfile

@@ -13,7 +13,7 @@ ENV MYSQL_ROOT_PASSWORD pass
 # create the `epidata` database
 ENV MYSQL_DATABASE epidata
 
-# create the `epi` user account with a development-only password
+# create the `user` account with a development-only password
 ENV MYSQL_USER user
 ENV MYSQL_PASSWORD pass
 

dev/docker/database/epidata/README.md

@@ -1,10 +1,10 @@
 # `delphi_database_epidata`
 
-This image extends Delphi's database by:
+This image extends a Percona database by:
 
-- adding the `epi` user account
-- adding the `epidata` database
-- creating empty tables in `epidata`
+- adding the `user` account
+- adding the `epidata` &  other appropriate databases
+- creating empty tables in those databases
 
 To start a container from this image, run:
 

dev/docker/python/Dockerfile

@@ -1,4 +1,4 @@
-# start with the `delphi_python` image
+# start with the `delphi_python` image, which is built from repos/delphi/operations/dev/docker/python/Dockerfile
 FROM delphi_python
 
 RUN pip install --no-cache-dir -r repos/delphi/delphi-epidata/requirements.txt -r repos/delphi/delphi-epidata/requirements.dev.txt

dev/local/Makefile

@@ -78,7 +78,7 @@ web:
 	@# Build the web_epidata image
 	@cd repos/delphi/delphi-epidata;\
 		docker build -t delphi_web_epidata -f ./devops/Dockerfile .;\
-		cd ../../../
+		cd -
 
 	@# Run the web server
 	@docker run --rm -p 127.0.0.1:10080:80 \
@@ -97,10 +97,6 @@ db:
 	@# Setup virtual network if it doesn't exist
 	@docker network ls | grep delphi-net || docker network create --driver bridge delphi-net
 
-	@# Only build prereqs if we need them
-	@docker images delphi_database | grep delphi || \
-		docker build -t delphi_database -f repos/delphi/operations/dev/docker/database/Dockerfile .
-
 	@# Build the database_epidata image
 	@docker build -t delphi_database_epidata \
 		-f repos/delphi/delphi-epidata/dev/docker/database/epidata/Dockerfile .

docs/epidata_development.md

@@ -119,8 +119,8 @@ above. The base images are built first, followed by the derived
   the Epidata API to the `delphi_web` image.
 - The
   [`delphi_database_epidata` image](https://github.com/cmu-delphi/delphi-epidata/blob/main/dev/docker/database/epidata/README.md)
-  adds the `epi` user account, `epidata` database, and relevant tables
-  (initially empty) to the `delphi_database` image.
+  adds user accounts, `epidata` & other appropriate databases, and relevant tables
+  (initially empty) to a Percona database image.
 
 From the root of your workspace, all of the images can be built as follows:
 

docs/new_endpoint_tutorial.md

@@ -277,8 +277,6 @@ docker build -t delphi_web \
   -f repos/delphi/operations/dev/docker/web/Dockerfile .
 docker build -t delphi_web_epidata \
   -f repos/delphi/delphi-epidata/dev/docker/web/epidata/Dockerfile .
-docker build -t delphi_database \
-  -f repos/delphi/operations/dev/docker/database/Dockerfile .
 docker build -t delphi_database_epidata \
   -f repos/delphi/delphi-epidata/dev/docker/database/epidata/Dockerfile .
 

src/server/_query.py

@@ -22,7 +22,7 @@
 from ._exceptions import DatabaseErrorException
 from ._validate import DateRange, extract_strings
 from ._params import GeoPair, SourceSignalPair, TimePair
-from .utils import dates_to_ranges
+from .utils import time_values_to_ranges, days_to_ranges, weeks_to_ranges
 
 
 def date_string(value: int) -> str:
@@ -90,7 +90,7 @@ def filter_dates(
     param_key: str,
     params: Dict[str, Any],
 ):
-    ranges = dates_to_ranges(values)
+    ranges = time_values_to_ranges(values)
     return filter_values(field, ranges, param_key, params, date_string)
 
 
@@ -187,7 +187,7 @@ def filter_pair(pair: TimePair, i) -> str:
         params[type_param] = pair.time_type
         if isinstance(pair.time_values, bool) and pair.time_values:
             return f"{type_field} = :{type_param}"
-        ranges = dates_to_ranges(pair.time_values)
+        ranges = weeks_to_ranges(pair.time_values) if pair.is_week else days_to_ranges(pair.time_values)
         return f"({type_field} = :{type_param} AND {filter_integers(time_field, cast(Sequence[Union[int, Tuple[int,int]]], ranges), type_param, params)})"
 
     parts = [filter_pair(p, i) for i, p in enumerate(values)]

src/server/utils/__init__.py

@@ -1 +1 @@
-from .dates import shift_time_value, date_to_time_value, time_value_to_iso, time_value_to_date, days_in_range, weeks_in_range, shift_week_value, week_to_time_value, week_value_to_week, guess_time_value_is_day, dates_to_ranges
+from .dates import shift_time_value, date_to_time_value, time_value_to_iso, time_value_to_date, days_in_range, weeks_in_range, shift_week_value, week_to_time_value, week_value_to_week, guess_time_value_is_day, time_values_to_ranges, days_to_ranges, weeks_to_ranges

src/server/utils/dates.py

@@ -1,12 +1,13 @@
 from typing import (
+    Callable,
     Optional,
     Sequence,
     Tuple,
     Union
 )
 from datetime import date, timedelta
 from epiweeks import Week, Year
-
+import logging
 
 def time_value_to_date(value: int) -> date:
     year, month, day = value // 10000, (value % 10000) // 100, value % 100
@@ -26,7 +27,7 @@ def week_value_to_week(value: int) -> Week:
 
 def guess_time_value_is_day(value: int) -> bool:
     # YYYYMMDD type and not YYYYMM
-    return len(str(value)) > 6
+    return len(str(value)) == 8
 
 def guess_time_value_is_week(value: int) -> bool:
     # YYYYWW type and not YYYYMMDD
@@ -77,7 +78,7 @@ def weeks_in_range(week_range: Tuple[int, int]) -> int:
         acc += year.totalweeks()
     return acc + 1  # same week should lead to 1 week that will be queried
 
-def dates_to_ranges(values: Optional[Sequence[Union[Tuple[int, int], int]]]) -> Optional[Sequence[Union[Tuple[int, int], int]]]:
+def time_values_to_ranges(values: Optional[Sequence[Union[Tuple[int, int], int]]]) -> Optional[Sequence[Union[Tuple[int, int], int]]]:
     """
     Converts a mixed list of dates and date ranges to an optimized list where dates are merged into ranges where possible.
     e.g. [20200101, 20200102, (20200101, 20200104), 20200106] -> [(20200101, 20200104), 20200106]
@@ -87,84 +88,55 @@ def dates_to_ranges(values: Optional[Sequence[Union[Tuple[int, int], int]]]) ->
         return values
 
     # determine whether the list is of days (YYYYMMDD) or weeks (YYYYWW) based on first element
-    try:
-        if (isinstance(values[0], tuple) and guess_time_value_is_day(values[0][0]))\
-            or (isinstance(values[0], int) and guess_time_value_is_day(values[0])):
-            return days_to_ranges(values)
-        elif (isinstance(values[0], tuple) and guess_time_value_is_week(values[0][0]))\
-            or (isinstance(values[0], int) and guess_time_value_is_week(values[0])):
-            return weeks_to_ranges(values)
-        else:
-            return values
-    except:
+    first_element = values[0][0] if isinstance(values[0], tuple) else values[0]
+    if guess_time_value_is_day(first_element):
+        return days_to_ranges(values)
+    elif guess_time_value_is_week(first_element):
+        return weeks_to_ranges(values)
+    else:
         return values
 
 def days_to_ranges(values: Sequence[Union[Tuple[int, int], int]]) -> Sequence[Union[Tuple[int, int], int]]:
-    intervals = []
-
-    # populate list of intervals based on original values
-    for v in values:
-        if isinstance(v, int):
-            # 20200101 -> [20200101, 20200101]
-            intervals.append([time_value_to_date(v), time_value_to_date(v)])
-        else: # tuple
-            # (20200101, 20200102) -> [20200101, 20200102]
-            intervals.append([time_value_to_date(v[0]), time_value_to_date(v[1])])
-
-    intervals.sort(key=lambda x: x[0])
-
-    # merge overlapping intervals https://leetcode.com/problems/merge-intervals/
-    merged = []
-    for interval in intervals:
-        # no overlap, append the interval
-        # caveat: we subtract 1 from interval[0] so that contiguous intervals are considered overlapping. i.e. [1, 1], [2, 2] -> [1, 2]
-        if not merged or merged[-1][1] < interval[0] - timedelta(days=1):
-            merged.append(interval)
-        # overlap, merge the current and previous intervals
-        else:
-            merged[-1][1] = max(merged[-1][1], interval[1])
-
-    # convert intervals from dates back to integers
-    ranges = []
-    for m in merged:
-        if m[0] == m[1]:
-            ranges.append(date_to_time_value(m[0]))
-        else:
-            ranges.append((date_to_time_value(m[0]), date_to_time_value(m[1])))
-
-    return ranges
+    return _to_ranges(values, time_value_to_date, date_to_time_value, timedelta(days=1))
 
 def weeks_to_ranges(values: Sequence[Union[Tuple[int, int], int]]) -> Sequence[Union[Tuple[int, int], int]]:
-    intervals = []
-
-    # populate list of intervals based on original values
-    for v in values:
-        if isinstance(v, int):
-            # 202001 -> [202001, 202001]
-            intervals.append([week_value_to_week(v), week_value_to_week(v)])
-        else: # tuple
-            # (202001, 202002) -> [202001, 202002]
-            intervals.append([week_value_to_week(v[0]), week_value_to_week(v[1])])
-
-    intervals.sort(key=lambda x: x[0])
-
-    # merge overlapping intervals https://leetcode.com/problems/merge-intervals/
-    merged = []
-    for interval in intervals:
-        # no overlap, append the interval
-        # caveat: we subtract 1 from interval[0] so that contiguous intervals are considered overlapping. i.e. [1, 1], [2, 2] -> [1, 2]
-        if not merged or merged[-1][1] < interval[0] - 1:
-            merged.append(interval)
-        # overlap, merge the current and previous intervals
-        else:
-            merged[-1][1] = max(merged[-1][1], interval[1])
-
-    # convert intervals from weeks back to integers
-    ranges = []
-    for m in merged:
-        if m[0] == m[1]:
-            ranges.append(week_to_time_value(m[0]))
-        else:
-            ranges.append((week_to_time_value(m[0]), week_to_time_value(m[1])))
-
-    return ranges
+    return _to_ranges(values, week_value_to_week, week_to_time_value, 1)
+
+def _to_ranges(values: Sequence[Union[Tuple[int, int], int]], value_to_date: Callable, date_to_value: Callable, time_unit: Union[int, timedelta]) -> Sequence[Union[Tuple[int, int], int]]:
+    try:
+        intervals = []
+
+        # populate list of intervals based on original date/week values
+        for v in values:
+            if isinstance(v, int):
+                # 20200101 -> [20200101, 20200101]
+                intervals.append([value_to_date(v), value_to_date(v)])
+            else: # tuple
+                # (20200101, 20200102) -> [20200101, 20200102]
+                intervals.append([value_to_date(v[0]), value_to_date(v[1])])
+
+        intervals.sort()
+
+        # merge overlapping intervals https://leetcode.com/problems/merge-intervals/
+        merged = []
+        for interval in intervals:
+            # no overlap, append the interval
+            # caveat: we subtract 1 from interval[0] so that contiguous intervals are considered overlapping. i.e. [1, 1], [2, 2] -> [1, 2]
+            if not merged or merged[-1][1] < interval[0] - time_unit:
+                merged.append(interval)
+            # overlap, merge the current and previous intervals
+            else:
+                merged[-1][1] = max(merged[-1][1], interval[1])
+
+        # convert intervals from dates/weeks back to integers
+        ranges = []
+        for m in merged:
+            if m[0] == m[1]:
+                ranges.append(date_to_value(m[0]))
+            else:
+                ranges.append((date_to_value(m[0]), date_to_value(m[1])))
+
+        return ranges
+    except Exception as e:
+        logging.info('bad input to date ranges', input=values, exception=e)
+        return values

tests/server/test_query.py

@@ -278,3 +278,24 @@ def test_filter_time_pairs(self):
                 "((t = :p_0t AND (v BETWEEN :p_0t_0 AND :p_0t_0_2)))",
             )
             self.assertEqual(params, {"p_0t": "day", "p_0t_0": 20201201, "p_0t_0_2": 20201203})
+        with self.subTest("dedupe"):
+            params = {}
+            self.assertEqual(
+                filter_time_pairs("t", "v", [TimePair("day", [20200101, 20200101, (20200101, 20200101), 20200101])], "p", params),
+                "((t = :p_0t AND (v = :p_0t_0)))",
+            )
+            self.assertEqual(params, {"p_0t": "day", "p_0t_0": 20200101})
+        with self.subTest("merge single range"):
+            params = {}
+            self.assertEqual(
+                filter_time_pairs("t", "v", [TimePair("day", [20200101, 20200102, (20200101, 20200104)])], "p", params),
+                "((t = :p_0t AND (v BETWEEN :p_0t_0 AND :p_0t_0_2)))",
+            )
+            self.assertEqual(params, {"p_0t": "day", "p_0t_0": 20200101, "p_0t_0_2": 20200104})
+        with self.subTest("merge ranges and singles"):
+            params = {}
+            self.assertEqual(
+                filter_time_pairs("t", "v", [TimePair("day", [20200101, 20200103, (20200105, 20200107)])], "p", params),
+                "((t = :p_0t AND (v = :p_0t_0 OR v = :p_0t_1 OR v BETWEEN :p_0t_2 AND :p_0t_2_2)))",
+            )
+            self.assertEqual(params, {"p_0t": "day", "p_0t_0": 20200101, "p_0t_1": 20200103, 'p_0t_2': 20200105, 'p_0t_2_2': 20200107})           

tests/server/utils/test_dates.py

@@ -2,7 +2,7 @@
 from datetime import date
 from epiweeks import Week
 
-from delphi.epidata.server.utils.dates import time_value_to_date, date_to_time_value, shift_time_value, time_value_to_iso, days_in_range, weeks_in_range, week_to_time_value, week_value_to_week, dates_to_ranges
+from delphi.epidata.server.utils.dates import time_value_to_date, date_to_time_value, shift_time_value, time_value_to_iso, days_in_range, weeks_in_range, week_to_time_value, week_value_to_week, time_values_to_ranges
 
 
 class UnitTests(unittest.TestCase):
@@ -41,16 +41,21 @@ def test_week_to_time_value(self):
         self.assertEqual(week_to_time_value(Week(2021, 1)), 202101)
         self.assertEqual(week_to_time_value(Week(2020, 42)), 202042)
 
-    def test_dates_to_ranges(self):
-        self.assertEqual(dates_to_ranges(None), None)
-        self.assertEqual(dates_to_ranges([]), [])
+    def test_time_values_to_ranges(self):
+        self.assertEqual(time_values_to_ranges(None), None)
+        self.assertEqual(time_values_to_ranges([]), [])
         # days
-        self.assertEqual(dates_to_ranges([20200101]), [20200101])
-        self.assertEqual(dates_to_ranges([(20200101, 20200105)]), [(20200101, 20200105)])
-        self.assertEqual(dates_to_ranges([20211231, (20211230, 20220102), 20220102]), [(20211230, 20220102)])
-        self.assertEqual(dates_to_ranges([20200101, 20200102, (20200101, 20200104), 20200106]), [(20200101, 20200104), 20200106])
+        self.assertEqual(time_values_to_ranges([20200101]), [20200101])
+        self.assertEqual(time_values_to_ranges([(20200101, 20200105)]), [(20200101, 20200105)])
+        self.assertEqual(time_values_to_ranges([20211231, (20211230, 20220102), 20220102]), [(20211230, 20220102)])
+        self.assertEqual(time_values_to_ranges([20200101, 20200102, (20200101, 20200104), 20200106]), [(20200101, 20200104), 20200106])
         # weeks
-        self.assertEqual(dates_to_ranges([202001]), [202001])
-        self.assertEqual(dates_to_ranges([(202001, 202005)]), [(202001, 202005)])
-        self.assertEqual(dates_to_ranges([202051, (202050, 202102), 202101]), [(202050, 202102)])
-        self.assertEqual(dates_to_ranges([202050, 202051, (202050, 202101), 202103]), [(202050, 202101), 202103])
+        self.assertEqual(time_values_to_ranges([202001]), [202001])
+        self.assertEqual(time_values_to_ranges([(202001, 202005)]), [(202001, 202005)])
+        self.assertEqual(time_values_to_ranges([202051, (202050, 202102), 202101]), [(202050, 202102)])
+        self.assertEqual(time_values_to_ranges([202050, 202051, (202050, 202101), 202103]), [(202050, 202101), 202103])
+        # non-contiguous integers that represent actually contiguous time objects should join to become a range:
+        self.assertEqual(time_values_to_ranges([20200228, 20200301]), [20200228, 20200301]) # this is NOT a range because 2020 was a leap year
+        self.assertEqual(time_values_to_ranges([20210228, 20210301]), [(20210228, 20210301)]) # this becomes a range because these dates are indeed consecutive
+        # individual weeks become a range (2020 is a rare year with 53 weeks)
+        self.assertEqual(time_values_to_ranges([202051, 202052, 202053, 202101, 202102]), [(202051, 202102)])