Skip to content

Commit e32fd2f

Browse files
melange396melange396
authored
Merge branch 'dev' into rm_operations_database_docker
2 parents f8654e9 + c6f2e80 commit e32fd2f

File tree

87 files changed

+1478
-874
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

87 files changed

+1478
-874
lines changed

‎.bumpversion.cfg

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
[bumpversion]
2-
current_version = 0.4.0
2+
current_version = 0.4.3
33
commit = False
44
tag = False
55

‎README.md

+12
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,8 @@ You should now have the following directory structure:
2626
│ ├── .dockerignore -> repos/delphi/delphi-epidata/dev/local/.dockerignore
2727
│ ├── Makefile -> repos/delphi/delphi-epidata/dev/local/Makefile
2828
│ ├── repos
29+
│ │ ├── pyproject.toml -> delphi/delphi-epidata/dev/local/pyproject.toml
30+
│ │ ├── setup.cfg -> delphi/delphi-epidata/dev/local/setup.cfg
2931
│ │ └── delphi
3032
│ │ ├── delphi-epidata
3133
│ │ ├── flu-contest
@@ -52,6 +54,16 @@ $ [sudo] make test pdb=1
5254
$ [sudo] make test test=repos/delphi/delphi-epidata/integrations/acquisition
5355
```
5456

57+
Enabling features like code autocompletion and linting in your editor
58+
requires one extra step (prerequisites: up-to-date pip and setuptools v64+):
59+
60+
```sh
61+
$ cd repos
62+
63+
# Installs the working directory as an "editable package"
64+
$ pip install -e . --config-settings editable_mode=strict
65+
```
66+
5567
# COVIDcast
5668

5769
At the present, our primary focus is developing and expanding the

‎dev/local/Makefile

+15-1
Original file line numberDiff line numberDiff line change
@@ -83,7 +83,7 @@ web:
8383
@# Run the web server
8484
@docker run --rm -p 127.0.0.1:10080:80 \
8585
--env "SQLALCHEMY_DATABASE_URI=mysql+mysqldb://user:pass@delphi_database_epidata:3306/epidata" \
86-
--env "FLASK_SECRET=abc" --env "FLASK_PREFIX=/epidata" \
86+
--env "FLASK_SECRET=abc" --env "FLASK_PREFIX=/epidata" --env "LOG_DEBUG" \
8787
--network delphi-net --name delphi_web_epidata \
8888
delphi_web_epidata >$(LOG_WEB) 2>&1 &
8989

@@ -135,6 +135,20 @@ test:
135135
--env "FLASK_SECRET=abc" \
136136
delphi_web_python python -m pytest --import-mode importlib $(pdb) $(test) | tee test_output_$(NOW).log
137137

138+
.PHONY=bash
139+
bash:
140+
@docker run -it --rm --network delphi-net \
141+
--mount type=bind,source=$(CWD)repos/delphi/delphi-epidata,target=/usr/src/app/repos/delphi/delphi-epidata,readonly \
142+
--mount type=bind,source=$(CWD)repos/delphi/delphi-epidata/src,target=/usr/src/app/delphi/epidata,readonly \
143+
--env "SQLALCHEMY_DATABASE_URI=mysql+mysqldb://user:pass@delphi_database_epidata:3306/epidata" \
144+
--env "FLASK_SECRET=abc" \
145+
delphi_web_python bash
146+
147+
.PHONY=sql
148+
sql:
149+
@docker run --rm -it --network delphi-net --cap-add=sys_nice \
150+
percona mysql --user=user --password=pass --port 3306 --host delphi_database_epidata epidata
151+
138152
.PHONY=clean
139153
clean:
140154
@docker images -f "dangling=true" -q | xargs docker rmi >/dev/null 2>&1

‎dev/local/install.sh

+8-1
Original file line numberDiff line numberDiff line change
@@ -2,14 +2,17 @@
22
# Bootstrap delphi-epidata development
33
#
44
# Downloads the repos needed for local delphi-epidata development into current dir
5-
# and provides a Makefile with Docker control commands.
5+
# and provides a Makefile with Docker control commands
6+
# as well as pyproject/setup.cfg files for IDE mappings.
67
#
78
# Creates the directory structure:
89
#
910
# driver/
1011
# .dockerignore
1112
# Makefile
1213
# repos/
14+
# pyproject.toml
15+
# setup.cfg
1316
# delphi/
1417
# operations/
1518
# delphi-epidata/
@@ -43,3 +46,7 @@ cd ../../
4346

4447
ln -s repos/delphi/delphi-epidata/dev/local/Makefile
4548
ln -s repos/delphi/delphi-epidata/dev/local/.dockerignore
49+
cd repos
50+
ln -s delphi/delphi-epidata/dev/local/pyproject.toml
51+
ln -s delphi/delphi-epidata/dev/local/setup.cfg
52+
cd -

‎dev/local/pyproject.toml

+3
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
[build-system]
2+
requires = ["setuptools>=65", "wheel"]
3+
build-backend = "setuptools.build_meta"

‎dev/local/setup.cfg

+71
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,71 @@
1+
[metadata]
2+
name = Delphi Development
3+
version = 0.4.0
4+
5+
[options]
6+
packages =
7+
delphi.epidata
8+
delphi.epidata.acquisition
9+
delphi.epidata.acquisition.afhsb
10+
delphi.epidata.acquisition.cdcp
11+
delphi.epidata.acquisition.covid_hosp
12+
delphi.epidata.acquisition.covid_hosp.common
13+
delphi.epidata.acquisition.covid_hosp.facility
14+
delphi.epidata.acquisition.covid_hosp.state_daily
15+
delphi.epidata.acquisition.covid_hosp.state_timeseries
16+
delphi.epidata.acquisition.covidcast
17+
delphi.epidata.acquisition.covidcast_nowcast
18+
delphi.epidata.acquisition.ecdc
19+
delphi.epidata.acquisition.flusurv
20+
delphi.epidata.acquisition.fluview
21+
delphi.epidata.acquisition.ght
22+
delphi.epidata.acquisition.kcdc
23+
delphi.epidata.acquisition.nidss
24+
delphi.epidata.acquisition.norostat
25+
delphi.epidata.acquisition.paho
26+
delphi.epidata.acquisition.quidel
27+
delphi.epidata.acquisition.twtr
28+
delphi.epidata.acquisition.wiki
29+
delphi.epidata.client
30+
delphi.epidata.server
31+
delphi.epidata.server.covidcast_issues_migration
32+
delphi.epidata.server.endpoints
33+
delphi.epidata.server.endpoints.covidcast_utils
34+
delphi.epidata.server.utils
35+
delphi.flu_contest
36+
delphi.flu_contest
37+
delphi.flu_contest.archefilter
38+
delphi.flu_contest.covid
39+
delphi.flu_contest.epicast
40+
delphi.flu_contest.forecasters
41+
delphi.flu_contest.hosp
42+
delphi.flu_contest.main
43+
delphi.flu_contest.uploads
44+
delphi.flu_contest.utils
45+
delphi.github_deploy_repo
46+
delphi.github_deploy_repo.actions
47+
delphi.nowcast
48+
delphi.nowcast.experiments
49+
delphi.nowcast.fusion
50+
delphi.nowcast.obsolete
51+
delphi.nowcast.sensors
52+
delphi.nowcast.util
53+
delphi.operations
54+
delphi.operations.database_metrics
55+
delphi.operations.screenshots
56+
delphi.operations.screenshots.covidcast
57+
delphi.utils
58+
delphi.utils.geo
59+
delphi.utils.obsolete
60+
undefx.py3tester
61+
undefx.undef_analysis
62+
63+
package_dir =
64+
delphi.epidata = delphi/delphi-epidata/src
65+
delphi.flu_contest = delphi/flu-contest/src
66+
delphi.github_deploy_repo = delphi/github-deploy-repo/src
67+
delphi.nowcast = delphi/nowcast/src
68+
delphi.operations = delphi/operations/src
69+
delphi.utils = delphi/utils/src
70+
undefx.py3tester = undefx/py3tester/src
71+
undefx.undef_analysis = undefx/undef-analysis

‎docs/Gemfile.lock

+1-1
Original file line numberDiff line numberDiff line change
@@ -211,7 +211,7 @@ GEM
211211
jekyll-seo-tag (~> 2.1)
212212
minitest (5.14.4)
213213
multipart-post (2.1.1)
214-
nokogiri (1.13.6)
214+
nokogiri (1.13.9)
215215
mini_portile2 (~> 2.8.0)
216216
racc (~> 1.4)
217217
octokit (4.20.0)

‎docs/api/covid_hosp.md

+23-6
Original file line numberDiff line numberDiff line change
@@ -11,11 +11,22 @@ Hospital Capacity by State" datasets provided by the US Department of
1111
Health & Human Services via healthdata.gov. The latter provides more frequent updates,
1212
so it is combined with the former to create a single dataset which is as recent as possible.
1313

14+
HHS performs up to four days of forward-fill for missing values in the
15+
[facility-level data](covid_hosp_facility.md) which are aggregated to make this
16+
state-level dataset. This sometimes results in repeated values in the state-level data.
17+
A sequence of two repeated values is extremely common, and longer sequences are rare.
18+
Repeated values added in this way are sometimes updated if the underlying missing data can
19+
be completed at a later date.
20+
21+
Starting October 1, 2022, some facilities are only required to report annually.
22+
1423
For more information, see the
1524
[official description and data dictionary at healthdata.gov](https://healthdata.gov/Hospital/COVID-19-Reported-Patient-Impact-and-Hospital-Capa/g62h-syeh)
1625
for "COVID-19 Reported Patient Impact and Hospital Capacity by State Timeseries,"
1726
as well as the [official description](https://healthdata.gov/dataset/COVID-19-Reported-Patient-Impact-and-Hospital-Capa/6xf2-c3ie)
18-
for "COVID-19 Reported Patient Impact and Hospital Capacity by State."
27+
for "COVID-19 Reported Patient Impact and Hospital Capacity by State." The data elements,
28+
cadence, and how the data are being used in the federal response are documented in
29+
[a FAQ published by Health & Human Services](https://www.hhs.gov/sites/default/files/covid-19-faqs-hospitals-hospital-laboratory-acute-care-facility-data-reporting.pdf).
1930

2031
General topics not specific to any particular data source are discussed in the
2132
[API overview](README.md). Such topics include:
@@ -65,7 +76,7 @@ If `issues` is not specified, then the most recent issue is used by default.
6576
| `epidata[].state` | state pertaining to this row | string |
6677
| `epidata[].date` | date pertaining to this row | integer |
6778
| `epidata[].issue` | the date on which the dataset containing this row was published | integer |
68-
| `epidata[].*` | see the [data dictionary](https://healthdata.gov/covid-19-reported-patient-impact-and-hospital-capacity-state-data-dictionary) | |
79+
| `epidata[].*` | see the [data dictionary](https://healthdata.gov/Hospital/COVID-19-Reported-Patient-Impact-and-Hospital-Capa/g62h-syeh). Last synced: 2021-10-21 | |
6980
| `message` | `success` or error message | string |
7081

7182
# Example URLs
@@ -149,25 +160,31 @@ The following sample shows how to import the library and fetch MA on 2020-05-10
149160
### Python
150161

151162
Optionally install the package using pip(env):
152-
````bash
163+
```bash
153164
pip install delphi-epidata
154-
````
165+
```
155166

156167
Otherwise, place `delphi_epidata.py` from this repo next to your python script.
157168

158-
````python
169+
```python
159170
# Import
160171
from delphi_epidata import Epidata
161172
# Fetch data
162173
res = Epidata.covid_hosp('MA', 20200510)
163174
print(res['result'], res['message'], len(res['epidata']))
164-
````
175+
```
165176

166177
# Repair Log
167178

168179
If we ever need to repair the data record due to a bug in our code (not at the
169180
source), we will update the list below.
170181

182+
## October 21, 2021
183+
184+
All issues between 20210430 and 20211021 were re-uploaded to include new columns added by
185+
HHS. If you pulled these issues before October 21, the data you received was correct, but
186+
was missing the added columns.
187+
171188
## January 22, 2021
172189

173190
The following issues were repaired:

‎docs/api/covid_hosp_facility.md

+9-2
Original file line numberDiff line numberDiff line change
@@ -9,9 +9,16 @@ This data source is a mirror of the "COVID-19 Reported Patient Impact and
99
Hospital Capacity by Facility" dataset provided by the US Department of Health
1010
& Human Services via healthdata.gov.
1111

12+
HHS performs up to four days of forward-fill for missing values.
13+
14+
Starting October 1, 2022, some facilities are only required to report annually.
15+
1216
See the
1317
[official description and data dictionary at healthdata.gov](https://healthdata.gov/Hospital/COVID-19-Reported-Patient-Impact-and-Hospital-Capa/anag-cw7u)
14-
for more information.
18+
for more information. The data elements, cadence, and how the data are being used in the
19+
federal response are documented in
20+
[a FAQ published by Health & Human Services](https://www.hhs.gov/sites/default/files/covid-19-faqs-hospitals-hospital-laboratory-acute-care-facility-data-reporting.pdf).
21+
1522

1623
General topics not specific to any particular data source are discussed in the
1724
[API overview](README.md). Such topics include:
@@ -74,7 +81,7 @@ has been renamed here for clarity.
7481
| `epidata[].hospital_pk` | facility identified by this row | string |
7582
| `epidata[].collection_week` | Friday's date in the week pertaining to this row | integer |
7683
| `epidata[].publication_date` | the date on which the dataset containing this row was published | integer |
77-
| `epidata[].*` | see the [data dictionary](https://healthdata.gov/covid-19-reported-patient-impact-and-hospital-capacity-facility-data-dictionary) | |
84+
| `epidata[].*` | see the [data dictionary](https://healthdata.gov/Hospital/COVID-19-Reported-Patient-Impact-and-Hospital-Capa/anag-cw7u) | |
7885
| `message` | `success` or error message | string |
7986

8087
# Example URLs

‎docs/api/covid_hosp_facility_lookup.md

+13-2
Original file line numberDiff line numberDiff line change
@@ -54,10 +54,21 @@ supported.
5454
| --- | --- | --- |
5555
| `result` | result code: 1 = success, 2 = too many results, -2 = no results | integer |
5656
| `epidata` | list of results | array of objects |
57-
| `epidata[].hospital_pk` | facility identified by this row | string |
58-
| `epidata[].*` | see the [data dictionary](https://healthdata.gov/covid-19-reported-patient-impact-and-hospital-capacity-facility-data-dictionary) | |
57+
| `epidata[].hospital_pk` | unique identifier for this facility (will match CCN if CCN exists) | string |
58+
| `epidata[].state` | two-letter state code | string |
59+
| `epidata[].ccn` | CMS Certification Number for this facility | string |
60+
| `epidata[].hospital_name` | facility name | string |
61+
| `epidata[].address` | facility address | string |
62+
| `epidata[].city` | facility city | string |
63+
| `epidata[].zip` | 5-digit ZIP code | string |
64+
| `epidata[].hospital_subtype` | one of: Childrens Hospitals, Critical Access Hospitals, Long Term, Psychiatric, Rehabilitation, Short Term | string |
65+
| `epidata[].fips_code` | 5-digit FIPS county code | string |
66+
| `epidata[].is_metro_micro` | 1 if this facility serves a metropolitan or micropolitan area, 0 otherwise | integer |
5967
| `message` | `success` or error message | string |
6068

69+
Use the `hospital_pk` value when querying
70+
[the COVID-19 Reported Patient Impact and Hospital Capacity by Facility endpoint](covid_hosp_facility.md).
71+
6172
# Example URLs
6273

6374
### Lookup facilities in the city of Southlake (TX)

‎integrations/acquisition/covid_hosp/facility/test_scenarios.py

+27-10
Original file line numberDiff line numberDiff line change
@@ -76,7 +76,7 @@ def test_acquire_dataset(self):
7676
response = Epidata.covid_hosp_facility(
7777
'450822', Epidata.range(20200101, 20210101))
7878
self.assertEqual(response['result'], 1)
79-
self.assertEqual(len(response['epidata']), 1)
79+
self.assertEqual(len(response['epidata']), 2)
8080
row = response['epidata'][0]
8181
for k,v in expected_spotchecks.items():
8282
self.assertTrue(
@@ -101,9 +101,9 @@ def test_acquire_dataset(self):
101101
response = Epidata.covid_hosp_facility(
102102
'450822', Epidata.range(20200101, 20210101))
103103
self.assertEqual(response['result'], 1)
104-
self.assertEqual(len(response['epidata']), 1)
104+
self.assertEqual(len(response['epidata']), 2)
105105

106-
@freeze_time("2021-03-16")
106+
@freeze_time("2021-03-17")
107107
def test_facility_lookup(self):
108108
"""Lookup facilities using various filters."""
109109

@@ -120,7 +120,7 @@ def test_facility_lookup(self):
120120
self.assertTrue(acquired)
121121

122122
# texas ground truth, sorted by `hospital_pk`
123-
# see sample data at testdata/acquisition/covid_hosp/facility/dataset_old.csv
123+
# see sample data at testdata/acquisition/covid_hosp/facility/dataset.csv
124124
texas_hospitals = [{
125125
'hospital_pk': '450771',
126126
'state': 'TX',
@@ -139,7 +139,7 @@ def test_facility_lookup(self):
139139
'hospital_name': 'MEDICAL CITY LAS COLINAS',
140140
'address': '6800 N MACARTHUR BLVD',
141141
'city': 'IRVING',
142-
'zip': '75039',
142+
'zip': '77777', # most-recent collection week should take precedence
143143
'hospital_subtype': 'Short Term',
144144
'fips_code': '48113',
145145
'is_metro_micro': 1,
@@ -150,7 +150,7 @@ def test_facility_lookup(self):
150150
'hospital_name': 'RANKIN HOSPITAL MEDICAL CLINIC',
151151
'address': '1611 SPUR 576',
152152
'city': 'RANKIN',
153-
'zip': '79778',
153+
'zip': '99999', # most-recent collection week should take precedence
154154
'hospital_subtype': 'Critical Access Hospitals',
155155
'fips_code': '48461',
156156
'is_metro_micro': 0,
@@ -160,16 +160,16 @@ def test_facility_lookup(self):
160160
response = Epidata.covid_hosp_facility_lookup(state='tx')
161161
self.assertEqual(response['epidata'], texas_hospitals)
162162

163-
with self.subTest(name='by ccn'):
164-
response = Epidata.covid_hosp_facility_lookup(ccn='450771')
163+
with self.subTest(name='by zip'):
164+
response = Epidata.covid_hosp_facility_lookup(zip='75093')
165165
self.assertEqual(response['epidata'], texas_hospitals[0:1])
166166

167167
with self.subTest(name='by city'):
168168
response = Epidata.covid_hosp_facility_lookup(city='irving')
169169
self.assertEqual(response['epidata'], texas_hospitals[1:2])
170170

171-
with self.subTest(name='by zip'):
172-
response = Epidata.covid_hosp_facility_lookup(zip='79778')
171+
with self.subTest(name='by ccn'):
172+
response = Epidata.covid_hosp_facility_lookup(ccn='451329')
173173
self.assertEqual(response['epidata'], texas_hospitals[2:3])
174174

175175
with self.subTest(name='by fips_code'):
@@ -179,3 +179,20 @@ def test_facility_lookup(self):
179179
with self.subTest(name='no results'):
180180
response = Epidata.covid_hosp_facility_lookup(state='not a state')
181181
self.assertEqual(response['result'], -2)
182+
183+
# update facility info
184+
mock_network = MagicMock()
185+
mock_network.fetch_metadata.return_value = \
186+
self.test_utils.load_sample_metadata('metadata_update_facility.csv')
187+
mock_network.fetch_dataset.return_value = \
188+
self.test_utils.load_sample_dataset('dataset_update_facility.csv')
189+
190+
# acquire sample data into local database
191+
with self.subTest(name='second acquisition'):
192+
acquired = Update.run(network=mock_network)
193+
self.assertTrue(acquired)
194+
195+
texas_hospitals[1]['zip'] = '88888'
196+
with self.subTest(name='by city after update'):
197+
response = Epidata.covid_hosp_facility_lookup(city='irving')
198+
self.assertEqual(response['epidata'], texas_hospitals[1:2])

0 commit comments

Comments
 (0)