cmu-delphi · krivard · Feb 9, 2023 · Aug 2, 2022 · Jan 20, 2023 · Jan 20, 2023
diff --git a/.bumpversion.cfg b/.bumpversion.cfg
@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.4.5
+current_version = 0.4.6
 commit = False
 tag = False
 

diff --git a/dev/local/Makefile b/dev/local/Makefile
@@ -36,6 +36,7 @@
 #   pdb=1         Drops you into debug mode upon test failure, if running tests.
 #   test=         Only runs tests in the directories provided here, e.g.
 #                 repos/delphi/delphi-epidata/tests/acquisition/covidcast
+#   sql=          Overrides the default SQL connection string.
 
 
 # Set optional argument defaults
@@ -49,6 +50,12 @@ ifndef test
 	test=repos/delphi/delphi-epidata/tests repos/delphi/delphi-epidata/integrations
 endif
 
+ifdef sql
+	sqlalchemy_uri:=$(sql)
+else
+	sqlalchemy_uri:=mysql+mysqldb://user:pass@delphi_database_epidata:3306/epidata
+endif
+
 SHELL:=/bin/sh
 
 # Get the Makefile's absolute path: https://stackoverflow.com/a/324782/4784655
@@ -78,7 +85,7 @@ web:
 
 	@# Run the web server
 	@docker run --rm -p 127.0.0.1:10080:80 \
-		--env "SQLALCHEMY_DATABASE_URI=mysql+mysqldb://user:pass@delphi_database_epidata:3306/epidata" \
+		--env "SQLALCHEMY_DATABASE_URI=$(sqlalchemy_uri)" \
 		--env "FLASK_SECRET=abc" --env "FLASK_PREFIX=/epidata" --env "LOG_DEBUG" \
 		--network delphi-net --name delphi_web_epidata \
 		delphi_web_epidata >$(LOG_WEB) 2>&1 &
@@ -123,7 +130,7 @@ test:
 	@docker run -i --rm --network delphi-net \
 		--mount type=bind,source=$(CWD)repos/delphi/delphi-epidata,target=/usr/src/app/repos/delphi/delphi-epidata,readonly \
 		--mount type=bind,source=$(CWD)repos/delphi/delphi-epidata/src,target=/usr/src/app/delphi/epidata,readonly \
-		--env "SQLALCHEMY_DATABASE_URI=mysql+mysqldb://user:pass@delphi_database_epidata:3306/epidata" \
+		--env "SQLALCHEMY_DATABASE_URI=$(sqlalchemy_uri)" \
 		--env "FLASK_SECRET=abc" \
 		delphi_web_python python -m pytest --import-mode importlib $(pdb) $(test) | tee test_output_$(NOW).log
 
@@ -132,7 +139,7 @@ bash:
 	@docker run -it --rm --network delphi-net \
 		--mount type=bind,source=$(CWD)repos/delphi/delphi-epidata,target=/usr/src/app/repos/delphi/delphi-epidata,readonly \
 		--mount type=bind,source=$(CWD)repos/delphi/delphi-epidata/src,target=/usr/src/app/delphi/epidata,readonly \
-		--env "SQLALCHEMY_DATABASE_URI=mysql+mysqldb://user:pass@delphi_database_epidata:3306/epidata" \
+		--env "SQLALCHEMY_DATABASE_URI=$(sqlalchemy_uri)" \
 		--env "FLASK_SECRET=abc" \
 		delphi_web_python bash
 

diff --git a/dev/local/setup.cfg b/dev/local/setup.cfg
@@ -1,6 +1,6 @@
 [metadata]
 name = Delphi Development
-version = 0.4.5
+version = 0.4.6
 
 [options]
 packages =

diff --git a/docs/api/covidcast-signals/_source-template.md b/docs/api/covidcast-signals/_source-template.md
@@ -62,6 +62,16 @@ geographic coverage, limits in its interpretation (symptoms in a survey aren't
 always caused by COVID, our healthcare partner only is part of the market, there
 may be a demographic bias in respondents, etc.), known inaccuracies, etc.
 
+## Missingness
+
+Describe *all* situations under which a value may not be reported, and what that
+means. If the signal ever reports NA, describe what that means and how it is
+different from missingness. For example:
+
+When fewer than 100 survey responses are received in a geographic area on a
+specific day, no data is reported for that area on that day; an API query for
+all reported geographic areas on that day will not include it.
+
 ## Lag and Backfill
 
 If this signal is reported with a consistent lag, describe it here.

diff --git a/docs/symptom-survey/end-of-survey.md b/docs/symptom-survey/end-of-survey.md
@@ -1,7 +1,7 @@
 ---
 title: End of CTIS Data Collection
 parent: COVID-19 Trends and Impact Survey
-nav_order: 10
+nav_order: 11
 ---
 
 # End of CTIS Data Collection

diff --git a/docs/symptom-survey/index.md b/docs/symptom-survey/index.md
@@ -16,12 +16,6 @@ international version is [conducted by the University of
 Maryland](https://covidmap.umd.edu/). Data collection [ceased on June 25,
 2022](end-of-survey.md).
 
-The [survey results dashboard](https://delphi.cmu.edu/covidcast/survey-results/)
-provides a high-level summary of survey results. Geographically aggregated data
-from this survey is publicly available through the [COVIDcast API](../api/covidcast.md)
-as the [`fb-survey` data source](../api/covidcast-signals/fb-survey.md). Demographic breakdowns of survey
-data are publicly available as [downloadable contingency tables](contingency-tables.md).
-
 The [CTIS Methodology
 Report](https://dataforgood.facebook.com/dfg/resources/CTIS-methodology-report)
 describes the survey design, data collection process, weighting, and aggregation
@@ -34,6 +28,16 @@ access to the data, see our page on getting [data access](data-access.md).
 If you have questions about the survey or getting access to data, contact us at
 <[email protected]>.
 
+## Results
+
+The [survey results dashboard](https://delphi.cmu.edu/covidcast/survey-results/)
+provides a high-level summary of survey results. Geographically aggregated data
+from this survey is publicly available through the [COVIDcast API](../api/covidcast.md)
+as the [`fb-survey` data source](../api/covidcast-signals/fb-survey.md). Demographic breakdowns of survey
+data are publicly available as [downloadable contingency tables](contingency-tables.md).
+
+CTIS data has been used in [numerous peer-reviewed publications](publications.md).
+
 ## Credits
 
 The US COVID-19 Trends and Impact Survey (CTIS) is a project of the [Delphi

diff --git a/docs/symptom-survey/publications.md b/docs/symptom-survey/publications.md
@@ -0,0 +1,224 @@
+---
+title: Publications
+parent: COVID-19 Trends and Impact Survey
+nav_order: 10
+---
+
+# Publications Using CTIS
+
+Since 2020, academic and nonprofit researchers have used data from the US
+COVID-19 Trends and Impact Survey in peer-reviewed research publications.
+
+Our primary description of the survey and its results over the first year of
+operation was published as part of the [special feature "Beyond Cases and
+Deaths: The Benefits of Auxiliary Data Streams in Tracking the COVID-19
+Pandemic"](https://www.pnas.org/topic/548) in *PNAS*:
+
+- J. Salomon, A. Reinhart, A. Bilinski, E. J. Chua, W. La Motte-Kerr, M. M.
+  Rönn, M. B. Reitsma, K. A. Morris, S. LaRocca, T. H. Farag, F. Kreuter, R.
+  Rosenfeld, and R. J. Tibshirani (2021). [The US COVID-19 Trends and Impact
+  Survey: Continuous real-time measurement of COVID-19 symptoms, risks,
+  protective behaviors, testing, and
+  vaccination](https://doi.org/10.1073/pnas.2111454118). *Proceedings of the
+  National Academy of Sciences* 118 (51) e2111454118.
+
+## Peer Reviewed Papers
+
+Research publications using the survey data include:
+
+- Rebecca L. Weintraub et al (2023). [Identifying COVID-19 Vaccine Deserts and
+  Ways to Reduce Them: A Digital Tool to Support Public Health
+  Decision-Making](https://doi.org/10.2105/AJPH.2022.307198). *American Journal
+  of Public Health*.
+- Anzalone AJ, Sun J, Vinson AJ, Beasley WH, Hillegass WB, Murray K, et al.
+  (2023). [Community risks for SARS-CoV-2 infection among fully vaccinated US
+  adults by rurality: A retrospective cohort study from the National COVID
+  Cohort Collaborative](https://doi.org/10.1371/journal.pone.0279968). *PLoS
+  ONE* 18(1): e0279968.
+- Rufino, J., Baquero, C., Frey, D. et al (2023). [Using survey data to estimate
+  the impact of the omicron variant on vaccine efficacy against COVID-19
+  infection](https://doi.org/10.1038/s41598-023-27951-3). *Scientific Reports*
+  13, 900 (2023).
+- Rader, B., Astley, C.M., Sewalk, K. et al (2022). [Spatial modeling of vaccine
+  deserts as barriers to controlling
+  SARS-CoV-2](https://doi.org/10.1038/s43856-022-00183-8). *Communications
+  Medicine* 2, 141.
+- C. Lupton-Smith, E. Badillo Goicoechea, M. Collins, J. Lessler, M. K.
+  Grabowski & E. A. Stuart (2022). [Consistency between Household and County
+  Measures of Onsite Schooling during the COVID-19
+  Pandemic](https://doi.org/10.1080/19345747.2022.2131660). *Journal of Research
+  on Educational Effectiveness*.
+- Nguyen, Q.C., Yardi, I., Gutierrez, F.X.M. et al. (2022). [Leveraging 13
+  million responses to the U.S. COVID-19 Trends and Impact Survey to examine
+  vaccine hesitancy, vaccination, and mask wearing, January 2021-February
+  2022](https://doi.org/10.1186/s12889-022-14286-3). *BMC Public Health* 22,
+  1911.
+- J. G. Lu (2022). [Two large-scale global studies on COVID-19 vaccine hesitancy
+  over time: Culture, uncertainty avoidance, and vaccine side-effect
+  concerns](https://doi.org/10.1037/pspa0000320). *Journal of Personality and
+  Social Psychology*.
+- J. M. Cox-Ganser, P. K. Henneberger, D. N. Weissman, G. Guthrie, and C. P.
+  Groth (2022). [COVID-19 test positivity by occupation using the Delphi US
+  COVID-19 Trends and Impact Survey, September–November
+  2020](https://doi.org/10.1002/ajim.23410). *American Journal of Industrial
+  Medicine* 65 (9), 721-730.
+- M. Jahja, A. Chin, and R.J. Tibshirani (2022). [Real-Time Estimation of
+  COVID-19 Infections: Deconvolution and Sensor
+  Fusion](https://doi.org/10.1214/22-STS856). *Statistical Science* 37 (2),
+  207-228.
+- Henneberger, PK, Cox-Ganser, JM, Guthrie, GM, Groth, CP (2022). [Estimates of
+  COVID-19 vaccine uptake in major occupational groups and detailed occupational
+  categories in the United States, April–May
+  2021](https://doi.org/10.1002/ajim.23370). *American Journal of Industrial
+  Medicine* 65 (7), 525-536.
+- K. E. Wiens, C. P. Smith, E. Badillo-Goicoechea, K. H. Grantz, M. K.
+  Grabowski, A. S. Azman, E. A. Stuart, and J. Lessler (2022). [In-person
+  schooling and associated COVID-19 risk in the United States over spring
+  semester 2021](https://doi.org/10.1126/sciadv.abm9128). *Science Advances* 8,
+  eabm9128.
+- F. Petersen, A. Errore, and P. Karaca-Mandic (2022). [Lifting statewide mask
+  mandates and COVID-19 cases: A synthetic control
+  study](https://doi.org/10.1097/MLR.0000000000001725). *Medical Care* 60 (7),
+  538-544.
+- F. Pierri, B. L. Perry, M. R. DeVerna, et al. (2022). [Online misinformation
+  is linked to early COVID-19 vaccination hesitancy and
+  refusal](https://doi.org/10.1038/s41598-022-10070-w). *Scientific Reports* 12,
+  5966.
+- A. Pilehvari, J. Ton, M. R. Mohan, A. Marathe, and A. Vullikanti (2022).
+  [Drivers and Predictors of COVID-19 Vaccine Hesitancy in
+  Virginia](https://doi.org/10.1007/978-3-030-96188-6_8). In: Yang, Z., von
+  Briesen, E. (eds), *Proceedings of the 2021 Conference of The Computational
+  Social Science Society of the Americas*. CSSSA 2021.
+- L. S. Flor, J. Friedman, C. N. Spencer, et al. (2022). [Quantifying the
+  effects of the COVID-19 pandemic on gender equality on health, social, and
+  economic indicators: a comprehensive review of data from March, 2020, to
+  September, 2021](https://doi.org/10.1016/S0140-6736(22)00008-3). *The Lancet*
+  399 (10344), 2381-2397.
+- D. P. Do and R. Frank (2022). [Prior COVID-19 infection: an underappreciated
+  factor in vaccine hesitancy in the
+  USA](https://doi.org/10.1093/pubmed/fdab404). *Journal of Public Health* 44
+  (2), 471-474.
+- W. C. King, M. Rubinstein, A. Reinhart, and R. J. Mejia (2021). [Time trends,
+  factors associated with, and reasons for COVID-19 vaccine hesitancy: A massive
+  online survey of US adults from January-May
+  2021](https://doi.org/10.1371/journal.pone.0260731). *PLoS ONE* 16 (12),
+  e0260731.
+- C. Lupton-Smith, E. Badillo-Goicochea, T.-H. Chang, H. Maniates, K. E. Riehm,
+  I. Schmid, and E. A. Stuart (2021). [Factors associated with county-level
+  mental health during the COVID-19
+  pandemic](https://doi.org/10.1002/jcop.22785). *Journal of Community
+  Psychology* 50 (5), 2431-2442.
+- D. P. Do and R. Frank (2021). [U.S. frontline workers and COVID-19
+  inequities](https://doi.org/10.1016/j.ypmed.2021.106833). *Preventive
+  Medicine* 153, 106833.
+- W. C. King, M. Rubinstein, A. Reinhart, and R. J. Mejia (2021). [COVID-19
+  vaccine hesitancy January-May 2021 among 18–64 year old US adults by
+  employment and occupation](https://doi.org/10.1016/j.pmedr.2021.101569).
+  *Preventive Medicine Reports* 24, 101569.
+- C. H. Sudre, A. Keshet, M. S. Graham, A. D. Joshi, S. Shilo, H. Rossman, B.
+  Murray, E. Molteni, K. Klaser, L. D. Canas, M. Antonelli, L. H. Nguyen, D. A.
+  Drew, M. Modat, J. Capdevila Pujol, S. Ganesh, J. Wolf, T. Meir, A. T. Chan,
+  C. J. Steves, T. D. Spector, J. S. Brownstein, E. Segal, S. Ourselin, and C.
+  M. Astley (2021). [Anosmia, ageusia, and other COVID-19-like symptoms in
+  association with a positive SARS-CoV-2 test, across six national digital
+  surveillance platforms: an observational
+  study](https://doi.org/10.1016/S2589-7500(21)00115-1). *The Lancet Digital
+  Health* 3 (9), e577-e586.
+- R. Sukumaran, P. Patwa, S. T V, S. Shankar, R. Kanaparti, J. Bae, Y. Mathur,
+  A. Singh, A. Chopra, M. Kang, P. Ramaswamy, and R. Raskar (2021). [COVID-19
+  outbreak prediction and analysis using self reported
+  symptoms](https://doi.org/10.35566/jbds/v1n1/p8). *Journal of Behavioral Data
+  Science* 1 (1), 154–169.
+- D. Adjodah, K. Dinakar, M. Chinazzi, S. P. Fraiberger, A. Pentland, S. Bates,
+  K. Staller, A. Vespignani, and D. L. Bhatt (2021). [Association between
+  COVID-19 outcomes and mask mandates, adherence, and
+  attitudes](https://doi.org/10.1371/journal.pone.0252315). *PLoS ONE* 16 (6),
+  e0252315.
+- C. M. Zipfel, V. Colizza, and S. Bansal (2021). [The missing season: The
+  impacts of the COVID-19 pandemic on
+  influenza](https://doi.org/10.1016/j.vaccine.2021.05.049). *Vaccine* 39 (28),
+  3645-3648.
+- J. Lessler, M. K. Grabowski, K. H. Grantz, E. Badillo-Goicoechea, C. J. E.
+  Metcalf, C. Lupton-Smith, A. S. Azman, and E. A. Stuart (2021). [Household
+  COVID-19 risk and in-person
+  schooling](https://doi.org/10.1126/science.abh2939). *Science* 372 (6545),
+  1092-1097.
+- Doerr, A.J. (2021). [Locked (Down) and Loaded (Language): Effect of Policy and
+  Speech on COVID-19 Outcomes](https://doi.org/10.1177/15480518211012404).
+  *Journal of Leadership & Organizational Studies* 28 (3), 340-348.
+- Fischer CB, Adrien N, Silguero JJ, Hopper JJ, Chowdhury AI, Werler MM (2021).
+  [Mask adherence and rate of COVID-19 across the United
+  States](https://doi.org/10.1371/journal.pone.0249891). *PLoS ONE* 16 (4),
+  e0249891.
+- Bilinski, A., Emanuel, E., Salomon, J.A. and Venkataramani, A. (2021). [Better
+  Late Than Never: Trends in COVID-19 Infection Rates, Risk Perceptions, and
+  Behavioral Responses in the USA](https://doi.org/10.1007/s11606-021-06633-8).
+  *Journal of General Internal Medicine* 36, 1825-1828.
+- Molteni, E., Astley, C.M., Ma, W. et al. (2021). [Symptoms and syndromes
+  associated with SARS-CoV-2 infection and severity in pregnant women from two
+  community cohorts](https://doi.org/10.1038/s41598-021-86452-3). *Scientific
+  Reports* 11, 6928.
+- Rebeiro, P., Aronoff, D, and Smith, M.K. (2021). [The Impact of State
+  Mask-Wearing Requirements on the Growth of COVID-19 Cases, Hospitalizations,
+  and Deaths in the United States](https://doi.org/10.1093/cid/ciab101).
+  *Clinical Infectious Diseases* 73 (9), 1703–1706.
+- Rader, White, Burns, Chen, Brilliant, Cohen, Shaman, Brilliant, Kraemer,
+  Moritz, Hawkins, Scarpino, Astley, and Brownstein (2021). [Mask-wearing and
+  control of SARS-CoV-2 transmission in the USA: a cross-sectional
+  study](https://doi.org/10.1016/S2589-7500(20)30293-4). *Lancet Digital Health*
+  3 (3), e148-e157.
+- Flaxman AD, Henning DJ and Duber HC (2020). [The relative incidence of
+  COVID-19 in healthcare workers versus non-healthcare workers: evidence from a
+  web-based survey of Facebook users in the United
+  States](https://doi.org/10.12688/gatesopenres.13202.2). *Gates Open Research*,
+  4:174.
+- Kreuter, F., Barkay, N., Bilinski, A., Bradford, A., Chiu, S., Eliat, R., Fan,
+  J., Galili, T., Haimovich, D., Kim, B., LaRocca, S., Li, Y., Morris, K.,
+  Presser, S., Sarig, T., Salomon, J. A., Stewart, K., Stuart, E. A., &
+  Tibshirani, R. J. (2020). [Partnering with a global platform to inform
+  research and public policy
+  making](https://doi.org/10.18148/srm/2020.v14i2.7761). *Survey Research
+  Methods*, 14 (2), 159-163.
+
+## Blog Posts and Reports
+
+* Surgo Ventures, Resolve to Save Lives (2021). [COVID-19 Vaccine Precision
+  Response Toolkit: An End-to-End Vaccination Improvement Framework to Improve
+  COVID-19 Vaccine
+  Uptake](https://surgoventures.org/resource-library/increasing-covid-19vaccine-uptake-a-four-step-framework-to-promote-access-acceptance-and-equity).
+* Alejandra Arrieta, Emmanuela Gakidou, Heidi Larson, Erin Mullany, and
+  Christopher Troeger, April 1, 2021. [Through Understanding and Empathy, We Can
+  Convince Women to Get the COVID-19
+  Vaccine](https://www.thinkglobalhealth.org/article/through-understanding-and-empathy-we-can-convince-women-get-covid-19-vaccine).
+  Think Global Health.
+* Joseph Friedman, Silvia Montoya and Emmanuela Gakidou, March 8, 2021. [Gender
+  Equality in the Global Return to
+  School](https://www.thinkglobalhealth.org/article/gender-equality-global-return-school).
+  Think Global Health.
+* Leanna Morinishi, Kris Barkume, Esther Kim, and Alex Reinhart, February
+  2, 2021. [Home for the Holidays? The Impact of US Holidays on Social Behaviors
+  and Preventative
+  Measures](https://delphi.cmu.edu/blog/2021/02/02/home-for-the-holidays-the-impact-of-us-holidays-on-social-behaviors-and-preventative-measures/).
+  Delphi blog.
+* Alex Reinhart, Esther Kim, Andy Garcia, and Sarah LaRocca, January 28, 2021.
+  [Using the COVID-19 Symptom Survey to Track Vaccination Uptake and Sentiment
+  in the United
+  States](https://delphi.cmu.edu/blog/2021/01/28/using-the-covid-19-symptom-survey-to-track-vaccination-uptake-and-sentiment-in-the-united-states/).
+  Delphi blog.
+* Alex Reinhart, October 12, 2020. [New and Improved COVID Symptom Survey Tracks
+  Testing and
+  Mask-Wearing](https://delphi.cmu.edu/blog/2020/10/12/new-and-improved-covid-symptom-survey-tracks-testing-and-mask-wearing/).
+  Delphi blog.
+* Ryan Tibshirani, September 21, 2020. [Can Symptom Surveys Improve COVID-19
+  Forecasts?](https://delphi.cmu.edu/blog/2020/09/21/can-symptoms-surveys-improve-covid-19-forecasts/)
+  Delphi blog.
+* Alex Reinhart and Ryan Tibshirani, August 26, 2020. [COVID-19 Symptom Surveys
+  through
+  Facebook](https://delphi.cmu.edu/blog/2020/08/26/covid-19-symptom-surveys-through-facebook/).
+  Delphi blog.
+
+## Send Your Publications
+
+If you have used the survey data, or the aggregate data available in the
+COVIDcast API, to publish research results, please contact us at
+<[email protected]> so we can include your work here.
diff --git a/integrations/acquisition/covidcast/delete_batch.csv b/integrations/acquisition/covidcast/delete_batch.csv
@@ -1,4 +1,4 @@
 geo_id,value,stderr,sample_size,issue,time_value,geo_type,signal,source
-d_nonlatest,0,0,0,1,0,geo,sig,src
-d_latest,   0,0,0,3,0,geo,sig,src
-d_justone,  0,0,0,1,0,geo,sig,src
+d_nonlatest,0,0,0,1,0,county,sig,src
+d_latest,   0,0,0,3,0,county,sig,src
+d_justone,  0,0,0,1,0,county,sig,src
diff --git a/integrations/acquisition/covidcast/test_csv_uploading.py b/integrations/acquisition/covidcast/test_csv_uploading.py
@@ -213,8 +213,8 @@ def test_uploading(self):
         "time_value": [20200419],
         "signal": [signal_name],
         "direction": [None]})], axis=1).rename(columns=uploader_column_rename)
-      expected_values_df["missing_value"].iloc[0] = Nans.OTHER
-      expected_values_df["missing_sample_size"].iloc[0] = Nans.NOT_MISSING
+      expected_values_df.loc[0, "missing_value"] = Nans.OTHER
+      expected_values_df.loc[0, "missing_sample_size"] = Nans.NOT_MISSING
       expected_values = expected_values_df.to_dict(orient="records")
       expected_response = {'result': 1, 'epidata': self.apply_lag(expected_values), 'message': 'success'}