Close file

Author: knapii-developments

.circleci/config.yml

@@ -4,6 +4,9 @@
 #
 version: 2.1
 
+orbs:
+  codecov: codecov/[email protected]
+
 jobs:
   build:
     docker:
@@ -61,9 +64,7 @@ jobs:
             export FIRESTORE_EMULATOR_HOST=localhost:8081
             . venv/bin/activate
             cd tests
-            coverage run -m pytest
-            coverage report --include *scp-ingest-pipeline/ingest*
-            coverage html --include *scp-ingest-pipeline/ingest*
+            pytest --cov-report=xml --cov=../ingest/
 
-      - store_artifacts:
-          path: tests/htmlcov
+      - codecov/upload:
+          file: tests/coverage.xml

README.md

@@ -2,6 +2,7 @@
 File Ingest Pipeline for Single Cell Portal
 
 [![Build status](https://img.shields.io/circleci/build/github/broadinstitute/scp-ingest-pipeline.svg)](https://circleci.com/gh/broadinstitute/scp-ingest-pipeline)
+[![Code coverage](https://codecov.io/gh/broadinstitute/scp-ingest-pipeline/branch/master/graph/badge.svg)](https://codecov.io/gh/broadinstitute/scp-ingest-pipeline)
 
 The SCP Ingest Pipeline is an ETL pipeline for single-cell RNA-seq data.  
 

ingest/cell_metadata.py

@@ -61,7 +61,6 @@ def __init__(self, file_path, file_id: str, study_accession: str, *args, **kwarg
         # lambda below initializes new key with nested dictionary as value and avoids KeyError
         self.issues = defaultdict(lambda: defaultdict(lambda: defaultdict(list)))
         self.ontology = defaultdict(lambda: defaultdict(list))
-        self.type = defaultdict(list)
         self.cells = []
         self.is_valid_file = self.validate_format()
 
@@ -270,6 +269,13 @@ def validate_type_annotations(self):
                 # string for error reporting
                 if 'Unnamed' in t:
                     invalid_types.append('<empty value>')
+                # Duplicated metadata header name causes type annotation issue.
+                # Side effect of Pandas adding a suffix to uniquefy the header.
+                # These invalid annotations should not be included in invalid
+                # type annotation count. This exception may cause miscount of
+                # type annot errors if user-supplied annotation has period.
+                elif '.' in t:
+                    pass
                 else:
                     invalid_types.append(t)
         if invalid_types:

ingest/ingest_pipeline.py

@@ -51,19 +51,15 @@
 from ingest_files import IngestFiles
 from subsample import SubSample
 from loom import Loom
-from validation.validate_metadata import (
-    collect_jsonschema_errors,
-    validate_collected_ontology_data,
-    report_issues,
-)
+from validation.validate_metadata import validate_input_metadata, report_issues
 
 # Ingest file types
 EXPRESSION_FILE_TYPES = ["dense", "mtx", "loom"]
 
 
 class IngestPipeline(object):
     # File location for metadata json convention
-    JSON_CONVENTION = 'gs://fc-bcc55e6c-bec3-4b2e-9fb2-5e1526ddfcd2/metadata_conventions/AMC_v1.1.1.json'
+    JSON_CONVENTION = 'gs://fc-bcc55e6c-bec3-4b2e-9fb2-5e1526ddfcd2/metadata_conventions/AMC_v1.1.3/AMC_v1.1.3.json'
 
     def __init__(
         self,
@@ -234,11 +230,12 @@ def load_subsample(self, doc):
 
     def has_valid_metadata_convention(self):
         """ Determines if cell metadata file follows metadata convention"""
-        json_file = IngestFiles(self.JSON_CONVENTION, ['application/json'])
-        convention = json.load(json_file.file)
+        with open(self.JSON_CONVENTION, 'r') as f:
+            json_file = IngestFiles(self.JSON_CONVENTION, ['application/json'])
+            convention = json.load(json_file.file)
+            validate_input_metadata(self.cell_metadata, convention)
 
-        collect_jsonschema_errors(self.cell_metadata, convention)
-        validate_collected_ontology_data(self.cell_metadata, convention)
+        f.close()
         return not report_issues(self.cell_metadata)
 
     def ingest_expression(self) -> None: