Merge pull request #1024 from cmu-delphi/release/delphi-epidata-0.4.1

Release Delphi Epidata 0.4.1

Author: krivard

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.4.0
+current_version = 0.4.1
 commit = False
 tag = False
 

README.md

@@ -26,6 +26,8 @@ You should now have the following directory structure:
 │   ├── .dockerignore -> repos/delphi/delphi-epidata/dev/local/.dockerignore
 │   ├── Makefile -> repos/delphi/delphi-epidata/dev/local/Makefile
 │   ├── repos
+│   │   ├── pyproject.toml -> delphi/delphi-epidata/dev/local/pyproject.toml
+│   │   ├── setup.cfg -> delphi/delphi-epidata/dev/local/setup.cfg
 │   │   └── delphi
 │   │       ├── delphi-epidata
 │   │       ├── flu-contest
@@ -52,6 +54,16 @@ $ [sudo] make test pdb=1
 $ [sudo] make test test=repos/delphi/delphi-epidata/integrations/acquisition
 ```
 
+Enabling features like code autocompletion and linting in your editor
+requires one extra step (prerequisites: up-to-date pip and setuptools v64+):
+
+```sh
+$ cd repos
+
+# Installs the working directory as an "editable package"
+$ pip install -e . --config-settings editable_mode=strict
+```
+
 # COVIDcast
 
 At the present, our primary focus is developing and expanding the

dev/local/install.sh

@@ -2,14 +2,17 @@
 # Bootstrap delphi-epidata development
 #
 # Downloads the repos needed for local delphi-epidata development into current dir 
-# and provides a Makefile with Docker control commands.
+# and provides a Makefile with Docker control commands
+# as well as pyproject/setup.cfg files for IDE mappings.
 #
 # Creates the directory structure:
 #
 #   driver/
 #     .dockerignore
 #     Makefile
 #     repos/
+#       pyproject.toml
+#       setup.cfg
 #       delphi/
 #         operations/
 #         delphi-epidata/
@@ -43,3 +46,7 @@ cd ../../
 
 ln -s repos/delphi/delphi-epidata/dev/local/Makefile
 ln -s repos/delphi/delphi-epidata/dev/local/.dockerignore
+cd repos
+ln -s delphi/delphi-epidata/dev/local/pyproject.toml
+ln -s delphi/delphi-epidata/dev/local/setup.cfg
+cd -

dev/local/pyproject.toml

@@ -0,0 +1,3 @@
+[build-system]
+requires = ["setuptools>=65", "wheel"]
+build-backend = "setuptools.build_meta"

dev/local/setup.cfg

@@ -0,0 +1,71 @@
+[metadata]
+name = Delphi Development
+version = 0.4.0
+
+[options]
+packages =
+    delphi.epidata
+    delphi.epidata.acquisition
+    delphi.epidata.acquisition.afhsb
+    delphi.epidata.acquisition.cdcp
+    delphi.epidata.acquisition.covid_hosp
+    delphi.epidata.acquisition.covid_hosp.common
+    delphi.epidata.acquisition.covid_hosp.facility
+    delphi.epidata.acquisition.covid_hosp.state_daily
+    delphi.epidata.acquisition.covid_hosp.state_timeseries
+    delphi.epidata.acquisition.covidcast
+    delphi.epidata.acquisition.covidcast_nowcast
+    delphi.epidata.acquisition.ecdc
+    delphi.epidata.acquisition.flusurv
+    delphi.epidata.acquisition.fluview
+    delphi.epidata.acquisition.ght
+    delphi.epidata.acquisition.kcdc
+    delphi.epidata.acquisition.nidss
+    delphi.epidata.acquisition.norostat
+    delphi.epidata.acquisition.paho
+    delphi.epidata.acquisition.quidel
+    delphi.epidata.acquisition.twtr
+    delphi.epidata.acquisition.wiki
+    delphi.epidata.client
+    delphi.epidata.server
+    delphi.epidata.server.covidcast_issues_migration
+    delphi.epidata.server.endpoints
+    delphi.epidata.server.endpoints.covidcast_utils
+    delphi.epidata.server.utils
+    delphi.flu_contest
+    delphi.flu_contest
+    delphi.flu_contest.archefilter
+    delphi.flu_contest.covid
+    delphi.flu_contest.epicast
+    delphi.flu_contest.forecasters
+    delphi.flu_contest.hosp
+    delphi.flu_contest.main
+    delphi.flu_contest.uploads
+    delphi.flu_contest.utils
+    delphi.github_deploy_repo
+    delphi.github_deploy_repo.actions
+    delphi.nowcast
+    delphi.nowcast.experiments
+    delphi.nowcast.fusion
+    delphi.nowcast.obsolete
+    delphi.nowcast.sensors
+    delphi.nowcast.util
+    delphi.operations
+    delphi.operations.database_metrics
+    delphi.operations.screenshots
+    delphi.operations.screenshots.covidcast
+    delphi.utils
+    delphi.utils.geo
+    delphi.utils.obsolete
+    undefx.py3tester
+    undefx.undef_analysis
+
+package_dir =
+    delphi.epidata = delphi/delphi-epidata/src
+    delphi.flu_contest = delphi/flu-contest/src
+    delphi.github_deploy_repo = delphi/github-deploy-repo/src
+    delphi.nowcast = delphi/nowcast/src
+    delphi.operations = delphi/operations/src
+    delphi.utils = delphi/utils/src
+    undefx.py3tester = undefx/py3tester/src
+    undefx.undef_analysis = undefx/undef-analysis

docs/api/covid_hosp.md

@@ -11,11 +11,22 @@ Hospital Capacity by State" datasets provided by the US Department of
 Health & Human Services via healthdata.gov. The latter provides more frequent updates,
 so it is combined with the former to create a single dataset which is as recent as possible.
 
+HHS performs up to four days of forward-fill for missing values in the
+[facility-level data](covid_hosp_facility.md) which are aggregated to make this
+state-level dataset. This sometimes results in repeated values in the state-level data.
+A sequence of two repeated values is extremely common, and longer sequences are rare.
+Repeated values added in this way are sometimes updated if the underlying missing data can
+be completed at a later date.
+
+Starting October 1, 2022, some facilities are only required to report annually.
+
 For more information, see the
 [official description and data dictionary at healthdata.gov](https://healthdata.gov/Hospital/COVID-19-Reported-Patient-Impact-and-Hospital-Capa/g62h-syeh)
 for "COVID-19 Reported Patient Impact and Hospital Capacity by State Timeseries,"
 as well as the [official description](https://healthdata.gov/dataset/COVID-19-Reported-Patient-Impact-and-Hospital-Capa/6xf2-c3ie)
-for "COVID-19 Reported Patient Impact and Hospital Capacity by State."
+for "COVID-19 Reported Patient Impact and Hospital Capacity by State." The data elements,
+cadence, and how the data are being used in the federal response are documented in
+[a FAQ published by Health & Human Services](https://www.hhs.gov/sites/default/files/covid-19-faqs-hospitals-hospital-laboratory-acute-care-facility-data-reporting.pdf).
 
 General topics not specific to any particular data source are discussed in the
 [API overview](README.md). Such topics include:
@@ -65,7 +76,7 @@ If `issues` is not specified, then the most recent issue is used by default.
 | `epidata[].state` | state pertaining to this row | string |
 | `epidata[].date` | date pertaining to this row | integer |
 | `epidata[].issue` | the date on which the dataset containing this row was published | integer |
-| `epidata[].*` | see the [data dictionary](https://healthdata.gov/covid-19-reported-patient-impact-and-hospital-capacity-state-data-dictionary) |  |
+| `epidata[].*` | see the [data dictionary](https://healthdata.gov/Hospital/COVID-19-Reported-Patient-Impact-and-Hospital-Capa/g62h-syeh). Last synced: 2021-10-21 |  |
 | `message` | `success` or error message | string |
 
 # Example URLs
@@ -149,25 +160,31 @@ The following sample shows how to import the library and fetch MA on 2020-05-10
 ### Python
 
 Optionally install the package using pip(env):
-````bash
+```bash
 pip install delphi-epidata
-````
+```
 
 Otherwise, place `delphi_epidata.py` from this repo next to your python script.
 
-````python
+```python
 # Import
 from delphi_epidata import Epidata
 # Fetch data
 res = Epidata.covid_hosp('MA', 20200510)
 print(res['result'], res['message'], len(res['epidata']))
-````
+```
 
 # Repair Log
 
 If we ever need to repair the data record due to a bug in our code (not at the
 source), we will update the list below.
 
+## October 21, 2021
+
+All issues between 20210430 and 20211021 were re-uploaded to include new columns added by
+HHS. If you pulled these issues before October 21, the data you received was correct, but
+was missing the added columns.
+
 ## January 22, 2021
 
 The following issues were repaired:

docs/api/covid_hosp_facility.md

@@ -9,9 +9,16 @@ This data source is a mirror of the "COVID-19 Reported Patient Impact and
 Hospital Capacity by Facility" dataset provided by the US Department of Health
 & Human Services via healthdata.gov.
 
+HHS performs up to four days of forward-fill for missing values.
+
+Starting October 1, 2022, some facilities are only required to report annually.
+
 See the
 [official description and data dictionary at healthdata.gov](https://healthdata.gov/Hospital/COVID-19-Reported-Patient-Impact-and-Hospital-Capa/anag-cw7u)
-for more information.
+for more information. The data elements, cadence, and how the data are being used in the
+federal response are documented in
+[a FAQ published by Health & Human Services](https://www.hhs.gov/sites/default/files/covid-19-faqs-hospitals-hospital-laboratory-acute-care-facility-data-reporting.pdf).
+
 
 General topics not specific to any particular data source are discussed in the
 [API overview](README.md). Such topics include:
@@ -74,7 +81,7 @@ has been renamed here for clarity.
 | `epidata[].hospital_pk` | facility identified by this row | string |
 | `epidata[].collection_week` | Friday's date in the week pertaining to this row | integer |
 | `epidata[].publication_date` | the date on which the dataset containing this row was published | integer |
-| `epidata[].*` | see the [data dictionary](https://healthdata.gov/covid-19-reported-patient-impact-and-hospital-capacity-facility-data-dictionary) |  |
+| `epidata[].*` | see the [data dictionary](https://healthdata.gov/Hospital/COVID-19-Reported-Patient-Impact-and-Hospital-Capa/anag-cw7u) |  |
 | `message` | `success` or error message | string |
 
 # Example URLs

docs/api/covid_hosp_facility_lookup.md

@@ -54,10 +54,21 @@ supported.
 | --- | --- | --- |
 | `result` | result code: 1 = success, 2 = too many results, -2 = no results | integer |
 | `epidata` | list of results | array of objects |
-| `epidata[].hospital_pk` | facility identified by this row | string |
-| `epidata[].*` | see the [data dictionary](https://healthdata.gov/covid-19-reported-patient-impact-and-hospital-capacity-facility-data-dictionary) |  |
+| `epidata[].hospital_pk` | unique identifier for this facility (will match CCN if CCN exists) | string |
+| `epidata[].state` | two-letter state code | string |
+| `epidata[].ccn` | CMS Certification Number for this facility | string |
+| `epidata[].hospital_name` | facility name | string |
+| `epidata[].address` | facility address | string |
+| `epidata[].city` | facility city | string |
+| `epidata[].zip` | 5-digit ZIP code | string |
+| `epidata[].hospital_subtype` | one of: Childrens Hospitals, Critical Access Hospitals, Long Term, Psychiatric, Rehabilitation, Short Term  | string |
+| `epidata[].fips_code` | 5-digit FIPS county code | string |
+| `epidata[].is_metro_micro` | 1 if this facility serves a metropolitan or micropolitan area, 0 otherwise | integer |
 | `message` | `success` or error message | string |
 
+Use the `hospital_pk` value when querying
+[the COVID-19 Reported Patient Impact and Hospital Capacity by Facility endpoint](covid_hosp_facility.md).
+
 # Example URLs
 
 ### Lookup facilities in the city of Southlake (TX)

src/client/delphi_epidata.R

@@ -15,7 +15,7 @@ Epidata <- (function() {
   # API base url
   BASE_URL <- 'https://delphi.cmu.edu/epidata/api.php'
 
-  client_version <- '0.4.0'
+  client_version <- '0.4.1'
 
   # Helper function to cast values and/or ranges to strings
   .listitem <- function(value) {

src/client/delphi_epidata.js

@@ -22,7 +22,7 @@
   }
 })(this, function (exports, fetchImpl, jQuery) {
   const BASE_URL = "https://delphi.cmu.edu/epidata/";
-  const client_version = "0.4.0";
+  const client_version = "0.4.1";
 
   // Helper function to cast values and/or ranges to strings
   function _listitem(value) {

src/client/packaging/npm/package.json

@@ -2,7 +2,7 @@
   "name": "delphi_epidata",
   "description": "Delphi Epidata API Client",
   "authors": "Delphi Group",
-  "version": "0.4.0",
+  "version": "0.4.1",
   "license": "MIT",
   "homepage": "https://github.com/cmu-delphi/delphi-epidata",
   "bugs": {

src/client/packaging/pypi/delphi_epidata/__init__.py

@@ -1,4 +1,4 @@
 from .delphi_epidata import Epidata
 
 name = 'delphi_epidata'
-__version__ = '0.4.0'
+__version__ = '0.4.1'

src/client/packaging/pypi/setup.py

@@ -5,7 +5,7 @@
 
 setuptools.setup(
     name="delphi_epidata",
-    version="0.4.0",
+    version="0.4.1",
     author="David Farrow",
     author_email="[email protected]",
     description="A programmatic interface to Delphi's Epidata API.",

src/server/_config.py

@@ -5,7 +5,7 @@
 
 load_dotenv()
 
-VERSION = "0.4.0"
+VERSION = "0.4.1"
 
 MAX_RESULTS = int(10e6)
 MAX_COMPATIBILITY_RESULTS = int(3650)

src/server/_query.py

@@ -22,7 +22,7 @@
 from ._exceptions import DatabaseErrorException
 from ._validate import DateRange, extract_strings
 from ._params import GeoPair, SourceSignalPair, TimePair
-from .utils import dates_to_ranges
+from .utils import time_values_to_ranges, days_to_ranges, weeks_to_ranges
 
 
 def date_string(value: int) -> str:
@@ -90,7 +90,7 @@ def filter_dates(
     param_key: str,
     params: Dict[str, Any],
 ):
-    ranges = dates_to_ranges(values)
+    ranges = time_values_to_ranges(values)
     return filter_values(field, ranges, param_key, params, date_string)
 
 
@@ -187,7 +187,7 @@ def filter_pair(pair: TimePair, i) -> str:
         params[type_param] = pair.time_type
         if isinstance(pair.time_values, bool) and pair.time_values:
             return f"{type_field} = :{type_param}"
-        ranges = dates_to_ranges(pair.time_values)
+        ranges = weeks_to_ranges(pair.time_values) if pair.is_week else days_to_ranges(pair.time_values)
         return f"({type_field} = :{type_param} AND {filter_integers(time_field, cast(Sequence[Union[int, Tuple[int,int]]], ranges), type_param, params)})"
 
     parts = [filter_pair(p, i) for i, p in enumerate(values)]

src/server/utils/__init__.py

@@ -1 +1 @@
-from .dates import shift_time_value, date_to_time_value, time_value_to_iso, time_value_to_date, days_in_range, weeks_in_range, shift_week_value, week_to_time_value, week_value_to_week, guess_time_value_is_day, dates_to_ranges
+from .dates import shift_time_value, date_to_time_value, time_value_to_iso, time_value_to_date, days_in_range, weeks_in_range, shift_week_value, week_to_time_value, week_value_to_week, guess_time_value_is_day, time_values_to_ranges, days_to_ranges, weeks_to_ranges