restore rank info to DE output files

address PR comments

Author: jlchang

ingest/cli_parser.py

@@ -255,7 +255,7 @@ def create_parser():
     )
 
     parser_differential_expression.add_argument(
-        "--differential_expression",
+        "--differential-expression",
         required=True,
         action="store_true",
         help="Indicates that differential expression analysis should be invoked",

ingest/de.py

@@ -72,6 +72,8 @@ def __init__(
     @staticmethod
     def get_cluster_cells(cluster_cells):
         """ ID cells in cluster file """
+        # cluster_cells.tolist() yields a list of lists that needs to be flattened
+        # using extend converts a single-value list to a plain value
         cluster_cell_values = cluster_cells.tolist()
         cluster_cell_list = []
         for value in cluster_cell_values:
@@ -156,9 +158,9 @@ def execute_de(self):
                 self.genes,
                 self.barcodes,
             )
-            DifferentialExpression.de_logger.info(f"preparing DE on sparse matrix")
+            DifferentialExpression.de_logger.info("preparing DE on sparse matrix")
         else:
-            self.prepare_h5ad(
+            self.run_h5ad(
                 self.cluster,
                 self.metadata,
                 self.matrix_file_path,
@@ -168,10 +170,10 @@ def execute_de(self):
                 self.cluster_name,
                 self.method,
             )
-            DifferentialExpression.de_logger.info(f"preparing DE on dense matrix")
+            DifferentialExpression.de_logger.info("preparing DE on dense matrix")
 
     @staticmethod
-    def prepare_h5ad(
+    def run_h5ad(
         cluster,
         metadata,
         matrix_file_path,
@@ -183,16 +185,15 @@ def prepare_h5ad(
         genes=None,
         barcodes=None,
     ):
-        """
-        """
+
         de_cells = DifferentialExpression.get_cluster_cells(cluster.file['NAME'].values)
         de_annots = DifferentialExpression.subset_annots(metadata, de_cells)
 
         if matrix_file_type == "dense":
-            # will need try/except
+            # will need try/except (SCP-4205)
             adata = sc.read(matrix_file_path)
         else:
-            # MTX DE UNTESTED
+            # MTX DE UNTESTED (SCP-4203)
             # will want try/except here to catch failed data object composition
             adata = sc.read_mtx(matrix_file_path)
             # For AnnData, obs are cells and vars are genes
@@ -206,7 +207,7 @@ def prepare_h5ad(
 
         adata = DifferentialExpression.subset_adata(adata, de_cells)
 
-        # will need try/except
+        # will need try/except (SCP-4205)
         adata.obs = DifferentialExpression.order_annots(de_annots, adata.obs_names)
 
         sc.pp.normalize_total(adata, target_sum=1e4)
@@ -238,8 +239,9 @@ def prepare_h5ad(
             out_file = (
                 f'{cluster_name}--{annotation}--{str(group_filename)}--{method}.tsv'
             )
-
-            rank.to_csv(out_file, sep='\t', float_format='%.4g', index=False)
+            # Round numbers to 4 significant digits while respecting fixed point
+            # and scientific notation (note: trailing zeros are removed)
+            rank.to_csv(out_file, sep='\t', float_format='%.4g')
 
         # Provide h5ad of DE analysis as reference computable object
         # DifferentialExpression.de_logger.info(f"Writing DE h5ad file")

ingest/ingest_pipeline.py

@@ -95,7 +95,7 @@
     from .expression_files.dense_ingestor import DenseIngestor
     from .expression_files.mtx import MTXIngestor
     from .cli_parser import create_parser, validate_arguments
-    from .de import DifferentialExpression, prepare_h5ad
+    from .de import DifferentialExpression
 
 
 class IngestPipeline:
@@ -476,7 +476,7 @@ def calculate_de(self):
             **self.kwargs,
         )
         de.execute_de()
-        # ToDo: surface failed DE for analytics
+        # ToDo: surface failed DE for analytics (SCP-4206)
         return 0
 
     def report_validation(self, status):
@@ -512,12 +512,9 @@ def run_ingest(ingest, arguments, parsed_args):
             status_subsample = ingest.subsample()
             status.append(status_subsample)
     elif "differential_expression" in arguments:
-        if arguments["differential_expression"]:
-            config.set_parent_event_name(
-                "ingest-pipeline:differential_expression:ingest"
-            )
-            status_de = ingest.calculate_de()
-            status.append(status_de)
+        config.set_parent_event_name("ingest-pipeline:differential-expression")
+        status_de = ingest.calculate_de()
+        status.append(status_de)
 
     return status, status_cell_metadata