add warnings for unreasonable weekday effect

Author: dsweber2

_delphi_utils_python/delphi_utils/weekday.py

@@ -19,9 +19,10 @@ def get_params(data, denominator_col, numerator_cols, date_col, scales, logger):
         series column in the data.
         """
         tmp = data.reset_index()
-        denoms = tmp.groupby(date_col).sum()[denominator_col]
-        nums = tmp.groupby(date_col).sum()[numerator_cols]
-
+        denoms = tmp.groupby(date_col).sum(numeric_only = True)[denominator_col]
+        nums = tmp.groupby(date_col).sum(numeric_only = True)[numerator_cols]
+        if nums.shape[0] < 7:
+            logger.warning("Trying to handle weekday effects with fewer than 7 days worth of data. This will probably not work.")
         # Construct design matrix to have weekday indicator columns and then day
         # indicators.
         X = np.zeros((nums.shape[0], 6 + nums.shape[0]))
@@ -40,7 +41,9 @@ def get_params(data, denominator_col, numerator_cols, date_col, scales, logger):
                 logger.error("Unable to calculate weekday correction")
             else:
                 params[i,:] = result
-
+        if np.exp(-params).max() == np.inf:
+            logger.warning("largest weekday correction is infinite. Defaulting to no correction")
+            params = np.zeros((nums.shape[1], X.shape[1]))
         return params
 
     @staticmethod
@@ -93,11 +96,12 @@ def _fit(X, scales, npnums, npdenoms):
         for scale in scales:
             try:
                 prob = cp.Problem(cp.Minimize((-ll + lmbda * penalty) / scale))
-                _ = prob.solve()
+                _ = prob.solve(solver = cp.ECOS)
                 return b.value
             except SolverError:
                 # If the magnitude of the objective function is too large, an error is
                 # thrown; Rescale the objective function by going through loop
+                print(f"Solver didn't work at {scale}x")
                 continue
         return None
 

doctor_visits/delphi_doctor_visits/download_claims_ftp_files.py

@@ -61,12 +61,16 @@ def download(ftp_credentials, out_path, logger):
     client = paramiko.SSHClient()
     client.set_missing_host_key_policy(AllowAnythingPolicy())
 
-    client.connect(ftp_credentials["host"],
-                   username=ftp_credentials["user"],
-                   password=ftp_credentials["pass"],
-                   port=ftp_credentials["port"])
+    client.connect(
+        ftp_credentials["host"],
+        username=ftp_credentials["user"],
+        password=ftp_credentials["pass"],
+        port=ftp_credentials["port"],
+        allow_agent=False,
+        look_for_keys=False,
+    )
     sftp = client.open_sftp()
-    sftp.chdir('/optum/receiving')
+    sftp.chdir("/optum/receiving")
 
     # go through files in recieving dir
     files_to_download = []