Merge pull request #146 from broadinstitute/development

Release 1.6.2

Author: eweitz

ingest/expression_files/dense_ingestor.py

@@ -249,27 +249,19 @@ def transform(self):
                 raise ValueError(f"Duplicate gene: {gene}")
             self.gene_names[gene] = True
 
-            if len(exp_scores) > 0:
-                data_arrays, gene_models, num_processed = self.create_models(
-                    exp_cells,
-                    exp_scores,
-                    gene,
-                    None,
-                    gene_models,
-                    data_arrays,
-                    num_processed,
-                    False,
-                )
-        # load any remaining models (this is necessary here since there isn't
-        # an easy way to detect the last line of the file in the iteration above
-        if len(gene_models) > 0:
-            self.create_models(
+            data_arrays, gene_models, num_processed = self.create_models(
                 exp_cells,
                 exp_scores,
                 gene,
                 None,
                 gene_models,
                 data_arrays,
                 num_processed,
-                True,
+                False,
+            )
+        # Load any remaining models. This is necessary because the amount of
+        # models maybe less than the batch size.
+        if len(gene_models) > 0:
+            self.create_models(
+                [], [], None, None, gene_models, data_arrays, num_processed, True
             )

ingest/expression_files/expression_files.py

@@ -203,19 +203,22 @@ def create_models(
             """
         current_data_arrays = []
         start_time = datetime.datetime.now()
-
         model_id = ObjectId()
-        gene_models.append(
-            GeneExpression.create_gene_model(
-                name=gene,
-                study_file_id=self.study_file_id,
-                study_id=self.study_id,
-                gene_id=gene_id,
-                _id=model_id,
+
+        GeneExpression.dev_logger.debug(f"Creating models for {gene}")
+        if gene:
+            gene_models.append(
+                GeneExpression.create_gene_model(
+                    name=gene,
+                    study_file_id=self.study_file_id,
+                    study_id=self.study_id,
+                    gene_id=gene_id,
+                    _id=model_id,
+                )
             )
-        )
-        if len(data_arrays) > 0:
-            # Data arrays for cells
+        # Make data array models for genes with expression data
+        if len(exp_scores) > 0:
+            # Data array model for cells
             for cell_data_array in GeneExpression.create_data_arrays(
                 name=f"{gene} Cells",
                 array_type="cells",
@@ -225,7 +228,7 @@ def create_models(
                 **self.data_array_kwargs,
             ):
                 current_data_arrays.append(cell_data_array)
-            # Data arrays for expression values
+            # Data array model for expression values
             for exp_value_data_array in GeneExpression.create_data_arrays(
                 name=f"{gene} Expression",
                 array_type="expression",
@@ -236,12 +239,18 @@ def create_models(
             ):
                 current_data_arrays.append(exp_value_data_array)
         this_batch_size = len(data_arrays) + len(current_data_arrays)
-        # Determine if models should be batched
+        # Determine if models should be batched/loaded
         if this_batch_size >= GeneExpression.DATA_ARRAY_BATCH_SIZE or force:
-            self.load(gene_models, GeneExpression.COLLECTION_NAME)
-            self.load(data_arrays, DataArray.COLLECTION_NAME)
+            if force:
+                # Add new data arrays
+                data_arrays += current_data_arrays
+                current_data_arrays.clear()
+            if len(data_arrays) > 0:
+                self.load(data_arrays, DataArray.COLLECTION_NAME)
+            if len(gene_models) > 0:
+                self.load(gene_models, GeneExpression.COLLECTION_NAME)
             num_processed += len(gene_models)
-            print(
+            GeneExpression.dev_logger.info(
                 f"Processed {num_processed} genes. "
                 f"{str(datetime.datetime.now() - start_time)} "
                 f"elapsed"

ingest/expression_files/mtx.py

@@ -143,6 +143,19 @@ def get_mtx_dimensions(file_handler) -> List:
                     raise e
         raise ValueError("MTX file did not contain data")
 
+    @staticmethod
+    def get_features(feature_row: str):
+        """Determines gene id and gene name from a given row:str in a feature
+            file
+        """
+        feature_data = feature_row.split("\t")
+        gene_id = feature_data[0]
+        gene_name = feature_data[0]
+        if len(feature_data) >= 2:
+            # gene_name field is present
+            gene_name = feature_data[1]
+        return gene_id, gene_name
+
     def execute_ingest(self):
         """Parses MTX files"""
         self.extract_feature_barcode_matrices()
@@ -190,11 +203,16 @@ def transform(self):
             if current_idx != prev_idx:
                 if not MTXIngestor.is_sorted(current_idx, visited_expression_idx):
                     raise ValueError("MTX file must be sorted")
+                GeneExpression.dev_logger.debug(
+                    f"Processing {self.genes[prev_idx - 1]}"
+                )
                 visited_expression_idx.append(current_idx)
                 if prev_idx != 0:
                     # Expressed cells and scores are associated with prior gene
-                    prev_gene_id, prev_gene = self.genes[prev_idx - 1].split("\t")
-                    # Ff the previous gene exists, load its models
+                    prev_gene_id, prev_gene = MTXIngestor.get_features(
+                        self.genes[prev_idx - 1]
+                    )
+                    # If the previous gene exists, load its models
                     data_arrays, gene_models, num_processed = self.create_models(
                         exp_cells,
                         exp_scores,
@@ -213,7 +231,9 @@ def transform(self):
             exp_cells.append(exp_cell)
             exp_scores.append(exp_score)
         # Create data array for last row
-        current_gene_id, current_gene = self.genes[prev_idx - 1].split("\t")
+        current_gene_id, current_gene = MTXIngestor.get_features(
+            self.genes[prev_idx - 1]
+        )
         self.create_models(
             exp_cells,
             exp_scores,

tests/data/dense_matrix_no_exp_data.txt

@@ -0,0 +1,5 @@
+GENE	CELL_0001	CELL_0002	CELL_0003	CELL_0004
+Itm2a	0	0	0	0
+Sergef	0	0	0	0
+Chil5	0	0	0	0
+Fam109a	0	0	0	0