From c2890b3576c4bc9583a1ff0881bf615329e285c8 Mon Sep 17 00:00:00 2001 From: dsweber2 Date: Wed, 22 May 2024 17:15:06 -0500 Subject: [PATCH 1/4] add warnings for unreasonable weekday effect --- _delphi_utils_python/delphi_utils/weekday.py | 14 +++++++++----- .../download_claims_ftp_files.py | 14 +++++++++----- 2 files changed, 18 insertions(+), 10 deletions(-) diff --git a/_delphi_utils_python/delphi_utils/weekday.py b/_delphi_utils_python/delphi_utils/weekday.py index ba5d75815..427cf58b1 100644 --- a/_delphi_utils_python/delphi_utils/weekday.py +++ b/_delphi_utils_python/delphi_utils/weekday.py @@ -19,9 +19,10 @@ def get_params(data, denominator_col, numerator_cols, date_col, scales, logger): series column in the data. """ tmp = data.reset_index() - denoms = tmp.groupby(date_col).sum()[denominator_col] - nums = tmp.groupby(date_col).sum()[numerator_cols] - + denoms = tmp.groupby(date_col).sum(numeric_only = True)[denominator_col] + nums = tmp.groupby(date_col).sum(numeric_only = True)[numerator_cols] + if nums.shape[0] < 7: + logger.warning("Trying to handle weekday effects with fewer than 7 days worth of data. This will probably not work.") # Construct design matrix to have weekday indicator columns and then day # indicators. X = np.zeros((nums.shape[0], 6 + nums.shape[0])) @@ -40,7 +41,9 @@ def get_params(data, denominator_col, numerator_cols, date_col, scales, logger): logger.error("Unable to calculate weekday correction") else: params[i,:] = result - + if np.exp(-params).max() == np.inf: + logger.warning("largest weekday correction is infinite. Defaulting to no correction") + params = np.zeros((nums.shape[1], X.shape[1])) return params @staticmethod @@ -93,11 +96,12 @@ def _fit(X, scales, npnums, npdenoms): for scale in scales: try: prob = cp.Problem(cp.Minimize((-ll + lmbda * penalty) / scale)) - _ = prob.solve() + _ = prob.solve(solver = cp.ECOS) return b.value except SolverError: # If the magnitude of the objective function is too large, an error is # thrown; Rescale the objective function by going through loop + print(f"Solver didn't work at {scale}x") continue return None diff --git a/doctor_visits/delphi_doctor_visits/download_claims_ftp_files.py b/doctor_visits/delphi_doctor_visits/download_claims_ftp_files.py index 002d4a7c9..ee11f80a4 100644 --- a/doctor_visits/delphi_doctor_visits/download_claims_ftp_files.py +++ b/doctor_visits/delphi_doctor_visits/download_claims_ftp_files.py @@ -61,12 +61,16 @@ def download(ftp_credentials, out_path, logger): client = paramiko.SSHClient() client.set_missing_host_key_policy(AllowAnythingPolicy()) - client.connect(ftp_credentials["host"], - username=ftp_credentials["user"], - password=ftp_credentials["pass"], - port=ftp_credentials["port"]) + client.connect( + ftp_credentials["host"], + username=ftp_credentials["user"], + password=ftp_credentials["pass"], + port=ftp_credentials["port"], + allow_agent=False, + look_for_keys=False, + ) sftp = client.open_sftp() - sftp.chdir('/optum/receiving') + sftp.chdir("/optum/receiving") # go through files in recieving dir files_to_download = [] From 7b5a4ea9ff21d353abe0c189548629113e261f81 Mon Sep 17 00:00:00 2001 From: dsweber2 Date: Wed, 22 May 2024 17:36:27 -0500 Subject: [PATCH 2/4] include Sunday and warn if big but finite --- _delphi_utils_python/delphi_utils/weekday.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/_delphi_utils_python/delphi_utils/weekday.py b/_delphi_utils_python/delphi_utils/weekday.py index 427cf58b1..18c2a75a0 100644 --- a/_delphi_utils_python/delphi_utils/weekday.py +++ b/_delphi_utils_python/delphi_utils/weekday.py @@ -41,9 +41,15 @@ def get_params(data, denominator_col, numerator_cols, date_col, scales, logger): logger.error("Unable to calculate weekday correction") else: params[i,:] = result - if np.exp(-params).max() == np.inf: + # the values multiplied by in calc_adjustment + # Sunday, the last day here, is a sum of the other days + effective_multipliers = np.exp(np.concatenate([-params[:,:6], params[:,:6].sum(axis=0, keepdims = True)])) + + if effective_multipliers.max() == np.inf or effective_multipliers.min() == -np.inf: logger.warning("largest weekday correction is infinite. Defaulting to no correction") params = np.zeros((nums.shape[1], X.shape[1])) + else if effective_multipliers.max() > 1000: + logger.warning("largest weekday correction is over a factor of 1000. This is probably an error, but may be associated with a zero value.") return params @staticmethod From b36be28469d2b95deb26734681fa77ccd7c39db0 Mon Sep 17 00:00:00 2001 From: dsweber2 Date: Wed, 22 May 2024 17:37:43 -0500 Subject: [PATCH 3/4] delphi-utils version bump --- _delphi_utils_python/delphi_utils/weekday.py | 2 +- _delphi_utils_python/setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/_delphi_utils_python/delphi_utils/weekday.py b/_delphi_utils_python/delphi_utils/weekday.py index 18c2a75a0..c30923812 100644 --- a/_delphi_utils_python/delphi_utils/weekday.py +++ b/_delphi_utils_python/delphi_utils/weekday.py @@ -48,7 +48,7 @@ def get_params(data, denominator_col, numerator_cols, date_col, scales, logger): if effective_multipliers.max() == np.inf or effective_multipliers.min() == -np.inf: logger.warning("largest weekday correction is infinite. Defaulting to no correction") params = np.zeros((nums.shape[1], X.shape[1])) - else if effective_multipliers.max() > 1000: + elif effective_multipliers.max() > 1000: logger.warning("largest weekday correction is over a factor of 1000. This is probably an error, but may be associated with a zero value.") return params diff --git a/_delphi_utils_python/setup.py b/_delphi_utils_python/setup.py index 046dc5d3a..e9b6236b7 100644 --- a/_delphi_utils_python/setup.py +++ b/_delphi_utils_python/setup.py @@ -27,7 +27,7 @@ setup( name="delphi_utils", - version="0.3.23", + version="0.3.24", description="Shared Utility Functions for Indicators", long_description=long_description, long_description_content_type="text/markdown", From b39a540de7fda46829e5cb10b9773ba85818428c Mon Sep 17 00:00:00 2001 From: dsweber2 Date: Thu, 23 May 2024 13:14:57 -0500 Subject: [PATCH 4/4] version unbump --- _delphi_utils_python/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/_delphi_utils_python/setup.py b/_delphi_utils_python/setup.py index e9b6236b7..046dc5d3a 100644 --- a/_delphi_utils_python/setup.py +++ b/_delphi_utils_python/setup.py @@ -27,7 +27,7 @@ setup( name="delphi_utils", - version="0.3.24", + version="0.3.23", description="Shared Utility Functions for Indicators", long_description=long_description, long_description_content_type="text/markdown",