Merge pull request #222 from broadinstitute/development

Release 1.12.3

Author: jlchang

ingest/cell_metadata.py

@@ -14,6 +14,7 @@
 from typing import Dict, Generator, List, Tuple, Union  # noqa: F401
 import copy
 import json
+import logging
 
 from bson.objectid import ObjectId
 from mypy_extensions import TypedDict
@@ -27,10 +28,18 @@
         validate_input_metadata,
         write_metadata_to_bq,
     )
+    from monitor import setup_logger
 except ImportError:
     # Used when importing as external package, e.g. imports in single_cell_portal code
     from .annotations import Annotations
     from .ingest_files import DataArray, IngestFiles
+    from ..monitor import setup_logger
+
+dev_logger = setup_logger(__name__, "log.txt", format="support_configs")
+
+user_logger = setup_logger(
+    __name__ + ".user_logger", "user_log.txt", level=logging.ERROR
+)
 
 
 class CellMetadata(Annotations):
@@ -141,16 +150,22 @@ def transform(self):
                 if not self.numeric_array_columns.get(annot_name)
                 else "group"
             )
+
+            group = True if annot_type == "group" else False
+            # should not store annotations with >200 unique values for viz
+            # annot_header is the column of data, which includes name and type
+            # large is any annotation with more than 200 + 2 unique values
+            unique_values = list(self.file[annot_header].unique())
+            large = True if len(unique_values) > 202 else False
+
             yield AnnotationModel(
                 annot_header,
                 self.Model(
                     {
                         "name": annot_name,
                         "annotation_type": stored_mongo_annot_type,
                         # unique values from "group" type annotations else []
-                        "values": list(self.file[annot_header].unique())
-                        if annot_type == "group"
-                        else [],
+                        "values": unique_values if (group and not large) else [],
                         "study_file_id": self.study_file_id,
                         "study_id": self.study_id,
                     }

ingest/expression_files/dense_ingestor.py

@@ -187,6 +187,8 @@ def filter_expression_scores(scores: List, cells: List):
                     ):
                         valid_expression_scores.append(expression_score)
                         associated_cells.append(cells[idx])
+            except ValueError as err:
+                raise ValueError(f"Expected numeric expression score - {err}")
             except Exception:
                 raise ValueError("Score '{expression_score}' is not valid")
         return valid_expression_scores, associated_cells

ingest/validation/validate_metadata.py

@@ -1031,7 +1031,10 @@ def report_issues(metadata):
                             is_valid = True
                         except re.error:
                             pass
-                        issue_msg = f"{issue_text} [ Error count: {len(cells)} ]"
+                        if issue_type == "error":
+                            issue_msg = f"{issue_text} [ Error count: {len(cells)} ]"
+                        elif issue_type == "warn":
+                            issue_msg = f"{issue_text} [ Warn count: {len(cells)} ]"
                         record_issue(issue_type, issue_msg)
                     else:
                         record_issue(issue_type, issue_text)