Merge pull request #266 from broadinstitute/jlc_add_h5_extensions

Accept additional file extensions for AnnData ingest (SCP-4557)

Author: jlchang

ingest/h5ad.py renamed to ingest/anndata_.py

@@ -7,7 +7,7 @@
     from .monitor import log_exception
 
 
-class H5adIngestor(IngestFiles):
+class AnnDataIngestor(IngestFiles):
     ALLOWED_FILE_TYPES = ['application/x-hdf5']
 
     def __init__(self, file_path, study_file_id, study_id, **kwargs):

ingest/cli_parser.py

@@ -331,19 +331,21 @@ def create_parser():
         "--gene-file", help="Path to .genes.tsv file"
     )
 
-    # h5ad subparsers
-    parser_h5ad = subparsers.add_parser(
-        "ingest_h5ad", help="Indicates that h5ad file is being ingested"
+    # AnnData subparsers
+    parser_anndata = subparsers.add_parser(
+        "ingest_anndata", help="Indicates that AnnData file is being ingested"
     )
 
-    parser_h5ad.add_argument(
-        "--ingest-h5ad",
+    parser_anndata.add_argument(
+        "--ingest-anndata",
         required=True,
         action="store_true",
-        help="Indicates that ingest of h5ad file should be invoked",
+        help="Indicates that ingest of AnnData file should be invoked",
     )
 
-    parser_h5ad.add_argument("--h5ad-file", required=True, help="Path to h5ad file")
+    parser_anndata.add_argument(
+        "--anndata-file", required=True, help="Path to AnnData file"
+    )
 
     return parser
 

ingest/ingest_files.py

@@ -82,10 +82,21 @@ class IngestFiles:
         "ignore", "Your application has authenticated using end user credentials"
     )
 
+    ALLOWED_FILE_EXTENSIONS = {
+        "text/csv": [".csv"],
+        "text/plain": [".txt"],
+        "text/tab-separated-values": [".tsv"],
+        "dataframe": [".tsv"],
+        "application/x-hdf5": [".h5ad", ".h5", ".hdf5"],
+    }
+
     def __init__(self, file_path, allowed_file_types):
         self.file_path = file_path
-        # define filetype for h5ad file extension
+        # valid suffixes for AnnData ingest (expecting .h5ad)
+        # including hdf5 file extensions - AnnData files should be valid hdf5
         mimetypes.add_type('application/x-hdf5', '.h5ad')
+        mimetypes.add_type('application/x-hdf5', '.h5')
+        mimetypes.add_type('application/x-hdf5', '.hdf5')
         # File is remote (in GCS bucket) when running via PAPI,
         # and typically local when developing
         self.is_remote_file = IngestFiles.is_remote_file(file_path)
@@ -199,7 +210,7 @@ def open_file(self, file_path, open_as=None, start_point: int = 0, **kwargs):
             "text/plain": self.open_txt,
             "text/tab-separated-values": self.open_tsv,
             "dataframe": self.open_pandas,
-            "application/x-hdf5": self.open_h5ad,
+            "application/x-hdf5": self.open_anndata,
         }
 
         if start_point != 0:
@@ -237,10 +248,19 @@ def open_file(self, file_path, open_as=None, start_point: int = 0, **kwargs):
                     open_file,
                 )
         else:
-            msg = (
-                f"Unsupported file format. Allowed file MIME types are: "
-                f"{' '.join(self.allowed_file_types)}"
-            )
+            expected_suffixes = []
+            for t in self.allowed_file_types:
+                expected_suffixes.extend(self.ALLOWED_FILE_EXTENSIONS[t])
+            if file_type == None:
+                msg = (
+                    f"File type not detected for {file_path}, expected file endings are: "
+                    f"{' '.join(expected_suffixes)}"
+                )
+            else:
+                msg = (
+                    f"Unsupported file format {file_path}. Expected file suffix are: "
+                    f"{' '.join(expected_suffixes)}"
+                )
             log_exception(IngestFiles.dev_logger, IngestFiles.user_logger, msg)
             raise ValueError(msg)
 
@@ -311,7 +331,7 @@ def open_pandas(self, file_path, file_type, **kwargs):
         else:
             raise ValueError("File must be tab or comma delimited")
 
-    def open_h5ad(self, file_path, **kwargs):
+    def open_anndata(self, file_path, **kwargs):
         """Opens file as AnnData object """
         try:
             return sc.read_h5ad(file_path, backed='r')

ingest/ingest_pipeline.py

@@ -26,8 +26,8 @@
 # Ingest dense file
 python ingest_pipeline.py --study-id 5d276a50421aa9117c982845 --study-file-id 5dd5ae25421aa910a723a337 ingest_expression --taxon-name 'Homo sapiens' --taxon-common-name human --ncbi-taxid 9606 --matrix-file ../tests/data/dense_matrix_19_genes_1000_cells.txt --matrix-file-type dense
 
-# Ingest h5ad file
-python ingest_pipeline.py  --study-id 5d276a50421aa9117c982845 --study-file-id 5dd5ae25421aa910a723a337 ingest_h5ad --h5ad-file ../tests/data/test.h5ad
+# Ingest AnnData file
+python ingest_pipeline.py  --study-id 5d276a50421aa9117c982845 --study-file-id 5dd5ae25421aa910a723a337 ingest_anndata --anndata-file ../tests/data/anndata/test.h5ad
 
 # Subsample cluster and metadata file
 python ingest_pipeline.py --study-id 5d276a50421aa9117c982845 --study-file-id 5dd5ae25421aa910a723a337 ingest_subsample --cluster-file ../tests/data/test_1k_cluster_Data.csv --name custer1 --cell-metadata-file ../tests/data/test_1k_metadata_Data.csv --subsample
@@ -51,12 +51,8 @@
 from contextlib import nullcontext
 from typing import Dict, Generator, List, Tuple, Union
 from wsgiref.simple_server import WSGIRequestHandler  # noqa: F401
-
-
 from bson.objectid import ObjectId
 
-
-# from google.cloud.logging.resource import Resource
 try:
     # Used when importing internally and in tests
     from ingest_files import IngestFiles
@@ -82,10 +78,13 @@
     from clusters import Clusters
     from expression_files.mtx import MTXIngestor
     from expression_files.dense_ingestor import DenseIngestor
-    from h5ad import H5adIngestor
     from monitor import setup_logger, log_exception
     from de import DifferentialExpression
 
+    # scanpy uses anndata python package, disamibguate local anndata
+    # using underscore https://peps.python.org/pep-0008/#naming-conventions
+    from anndata_ import AnnDataIngestor
+
 except ImportError:
     # Used when importing as external package, e.g. imports in single_cell_portal code
     from .ingest_files import IngestFiles
@@ -103,7 +102,7 @@
     from .clusters import Clusters
     from .expression_files.dense_ingestor import DenseIngestor
     from .expression_files.mtx import MTXIngestor
-    from .h5ad import H5adIngestor
+    from .anndata import AnnDataIngestor
     from .cli_parser import create_parser, validate_arguments
     from .de import DifferentialExpression
 
@@ -127,7 +126,7 @@ def __init__(
         matrix_file_type: str = None,
         cell_metadata_file: str = None,
         cluster_file: str = None,
-        h5ad_file: str = None,
+        anndata_file: str = None,
         subsample=False,
         ingest_cell_metadata=False,
         ingest_cluster=False,
@@ -147,7 +146,7 @@ def __init__(
         else:
             self.db = None
         self.cluster_file = cluster_file
-        self.h5ad_file = h5ad_file
+        self.anndata_file = anndata_file
         self.kwargs = kwargs
         self.cell_metadata_file = cell_metadata_file
         self.props = {}
@@ -479,15 +478,15 @@ def subsample(self):
         return 0
 
     @custom_metric(config.get_metric_properties)
-    def ingest_h5ad(self):
-        """Ingests h5ad files."""
-        self.h5ad = H5adIngestor(
-            self.h5ad_file, self.study_id, self.study_file_id, **self.kwargs
+    def ingest_anndata(self):
+        """Ingests anndata files."""
+        self.anndata = AnnDataIngestor(
+            self.anndata_file, self.study_id, self.study_file_id, **self.kwargs
         )
-        if self.h5ad.validate():
+        if self.anndata.validate():
             self.report_validation("success")
             return 0
-        # scanpy unable to open h5ad file
+        # scanpy unable to open AnnData file
         else:
             self.report_validation("failure")
             return 1
@@ -541,11 +540,11 @@ def run_ingest(ingest, arguments, parsed_args):
             config.set_parent_event_name("ingest-pipeline:subsample:ingest")
             status_subsample = ingest.subsample()
             status.append(status_subsample)
-    elif "ingest_h5ad" in arguments:
-        if arguments["ingest_h5ad"]:
-            config.set_parent_event_name("ingest-pipeline:h5ad:ingest")
-            status_h5ad = ingest.ingest_h5ad()
-            status.append(status_h5ad)
+    elif "ingest_anndata" in arguments:
+        if arguments["ingest_anndata"]:
+            config.set_parent_event_name("ingest-pipeline:anndata:ingest")
+            status_anndata = ingest.ingest_anndata()
+            status.append(status_anndata)
     elif "differential_expression" in arguments:
         config.set_parent_event_name("ingest-pipeline:differential-expression")
         status_de = ingest.calculate_de()

ingest/monitor.py

@@ -44,9 +44,22 @@ def setup_logger(logger_name, log_file, level=logging.DEBUG, format="default"):
     return logger
 
 
+def bypass_mongo_writes():
+    """Check if developer has set environment variable to bypass writing data to MongoDB
+        BYPASS_MONGO_WRITES='yes'
+    """
+    if os.environ.get("BYPASS_MONGO_WRITES") is not None:
+        skip = os.environ["BYPASS_MONGO_WRITES"]
+        return skip == "yes"
+    else:
+        return False
+
+
 def log_exception(dev_logger, user_logger, exception):
     user_logger.critical(str(exception))
     dev_logger.exception(exception)
+    if bypass_mongo_writes():
+        print(str(exception))
 
 
 # Modified from https://jdkandersson.com/2019/05/19/testing-decorated-python-functions/

tests/data/h5ad/bad.h5ad renamed to tests/data/anndata/bad.foo

@@ -1,35 +1,28 @@
-""" test_de.py
-    integration test to verify that de process generates expected output
+""" test_anndata.py
+    verify basic AnnData validation works as expected
 """
 
 import unittest
 import sys
-import hashlib
-import os
-import glob
-from unittest.mock import patch
-import scanpy as sc
-
 
 sys.path.append("../ingest")
-from ingest_files import IngestFiles
-from h5ad import H5adIngestor
+from anndata_ import AnnDataIngestor
 
 
-class TestH5adIngestor(unittest.TestCase):
-    def test_minimal_valid_h5ad(self):
-        good_input = H5adIngestor(
-            "../tests/data/h5ad/test.h5ad",
+class TestAnnDataIngestor(unittest.TestCase):
+    def test_minimal_valid_anndata(self):
+        good_input = AnnDataIngestor(
+            "../tests/data/anndata/test.h5ad",
             "addedfeed000000000000000",
             "dec0dedfeed0000000000000",
         )
         self.assertTrue(
             good_input.validate(), "expect known good file to open with scanpy"
         )
 
-    def test_truncated_h5ad(self):
-        truncated_input = H5adIngestor(
-            "../tests/data/h5ad/bad.h5ad",
+    def test_truncated_anndata(self):
+        truncated_input = AnnDataIngestor(
+            "../tests/data/anndata/bad.h5",
             "addedfeed000000000000000",
             "dec0dedfeed0000000000000",
         )
@@ -38,14 +31,14 @@ def test_truncated_h5ad(self):
         # an exception before assertRaises gets called
         self.assertRaisesRegex(
             ValueError,
-            "Scanpy cannot read file, \"../tests/data/h5ad/bad.h5ad\".",
+            "Scanpy cannot read file, \"../tests/data/anndata/bad.h5\".",
             lambda: truncated_input.obtain_adata(),
         )
         self.assertFalse(truncated_input.validate())
 
-    def test_input_not_h5ad(self):
-        bad_input = H5adIngestor(
-            "../tests/data/h5ad/bad.h5",
+    def test_input_bad_suffix(self):
+        bad_input = AnnDataIngestor(
+            "../tests/data/anndata/bad.foo",
             "addedfeed000000000000000",
             "dec0dedfeed0000000000000",
         )
@@ -54,7 +47,7 @@ def test_input_not_h5ad(self):
         # an exception before assertRaises gets called
         self.assertRaisesRegex(
             ValueError,
-            "Unsupported file format. Allowed file MIME types are: application/x-hdf5",
+            "File type not detected for ../tests/data/anndata/bad.foo, expected file endings are: .h5ad .h5 .hdf5",
             lambda: bad_input.obtain_adata(),
         )
         self.assertFalse(bad_input.validate())