Make filetype errors more actionable

Author: jlchang

ingest/ingest_files.py

@@ -82,10 +82,20 @@ class IngestFiles:
         "ignore", "Your application has authenticated using end user credentials"
     )
 
+    ALLOWED_FILE_EXTENSIONS = {
+        "text/csv": [".csv"],
+        "text/plain": [".txt"],
+        "text/tab-separated-values": [".tsv"],
+        "dataframe": [".tsv"],
+        "application/x-hdf5": [".h5ad", ".h5", ".hdf5"],
+    }
+
     def __init__(self, file_path, allowed_file_types):
         self.file_path = file_path
-        # define filetype for h5ad file extension
+        # valid suffixes for Anndata (h5ad file extensions)
         mimetypes.add_type('application/x-hdf5', '.h5ad')
+        mimetypes.add_type('application/x-hdf5', '.h5')
+        mimetypes.add_type('application/x-hdf5', '.hdf5')
         # File is remote (in GCS bucket) when running via PAPI,
         # and typically local when developing
         self.is_remote_file = IngestFiles.is_remote_file(file_path)
@@ -237,10 +247,19 @@ def open_file(self, file_path, open_as=None, start_point: int = 0, **kwargs):
                     open_file,
                 )
         else:
-            msg = (
-                f"Unsupported file format. Allowed file MIME types are: "
-                f"{' '.join(self.allowed_file_types)}"
-            )
+            expected_suffixes = []
+            for t in self.allowed_file_types:
+                expected_suffixes.extend(self.ALLOWED_FILE_EXTENSIONS[t])
+            if file_type == None:
+                msg = (
+                    f"File type not detected for {file_path}, expected file endings are: "
+                    f"{' '.join(expected_suffixes)}"
+                )
+            else:
+                msg = (
+                    f"Unsupported file format {file_path}. Expected file suffix are: "
+                    f"{' '.join(expected_suffixes)}"
+                )
             log_exception(IngestFiles.dev_logger, IngestFiles.user_logger, msg)
             raise ValueError(msg)
 

ingest/monitor.py

@@ -44,9 +44,25 @@ def setup_logger(logger_name, log_file, level=logging.DEBUG, format="default"):
     return logger
 
 
+def bypass_mongo_writes():
+    """Check if developer has set environment variable to bypass writing data to MongoDB
+        BYPASS_MONGO_WRITES='yes'
+    """
+    if os.environ.get("BYPASS_MONGO_WRITES") is not None:
+        skip = os.environ["BYPASS_MONGO_WRITES"]
+        if skip == "yes":
+            return True
+        else:
+            return False
+    else:
+        return False
+
+
 def log_exception(dev_logger, user_logger, exception):
     user_logger.critical(str(exception))
     dev_logger.exception(exception)
+    if bypass_mongo_writes():
+        print(str(exception))
 
 
 # Modified from https://jdkandersson.com/2019/05/19/testing-decorated-python-functions/

tests/test_h5ad.py

@@ -27,9 +27,9 @@ def test_minimal_valid_h5ad(self):
             good_input.validate(), "expect known good file to open with scanpy"
         )
 
-    def test_truncated_h5ad(self):
+    def test_truncated_anndata(self):
         truncated_input = H5adIngestor(
-            "../tests/data/h5ad/bad.h5ad",
+            "../tests/data/h5ad/bad.h5",
             "addedfeed000000000000000",
             "dec0dedfeed0000000000000",
         )
@@ -38,14 +38,14 @@ def test_truncated_h5ad(self):
         # an exception before assertRaises gets called
         self.assertRaisesRegex(
             ValueError,
-            "Scanpy cannot read file, \"../tests/data/h5ad/bad.h5ad\".",
+            "Scanpy cannot read file, \"../tests/data/h5ad/bad.h5\".",
             lambda: truncated_input.obtain_adata(),
         )
         self.assertFalse(truncated_input.validate())
 
-    def test_input_not_h5ad(self):
+    def test_input_bad_suffix(self):
         bad_input = H5adIngestor(
-            "../tests/data/h5ad/bad.h5",
+            "../tests/data/h5ad/bad.foo",
             "addedfeed000000000000000",
             "dec0dedfeed0000000000000",
         )
@@ -54,7 +54,7 @@ def test_input_not_h5ad(self):
         # an exception before assertRaises gets called
         self.assertRaisesRegex(
             ValueError,
-            "Unsupported file format. Allowed file MIME types are: application/x-hdf5",
+            "File type not detected for ../tests/data/h5ad/bad.foo, expected file endings are: .h5ad .h5 .hdf5",
             lambda: bad_input.obtain_adata(),
         )
         self.assertFalse(bad_input.validate())