Merge pull request #41 from broadinstitute/ea-subsample-cluster

Ea subsample cluster

Author: knapii-developments

ingest/ingest_files.py

@@ -88,6 +88,7 @@ def open_file(self, file_path, open_as=None, start_point: int = 0):
         file_connections = {
             "text/csv": self.open_csv(open_file),
             "text/plain": open_file,
+            "application/json": open_file,
             "text/tab-separated-values": self.open_tsv(open_file),
             "dataframe": self.open_pandas,
         }

ingest/ingest_pipeline.py

@@ -19,7 +19,8 @@
 python ingest_pipeline.py --study-accession SCP1 --file-id 123abc ingest_cell_metadata --cell-metadata-file ../tests/data/metadata_valid.tsv --ingest-cell-metadata
 
 # Ingest Cell Metadata file against convention
-python ingest_pipeline.py --study-accession SCP1 --file-id 123abc ingest_cell_metadata --cell-metadata-file ../tests/data/metadata_valid.tsv --ingest-cell-metadata --validate-convention
+!! Please note that you must have permission to the SCP bucket
+python ingest_pipeline.py --study-accession SCP1 --file-id 123abc ingest_cell_metadata --cell-metadata-file ../tests/data/valid_array_v1.1.3.tsv --ingest-cell-metadata --validate-convention
 
 # Ingest dense file
 python ingest_pipeline.py --study-accession SCP1 --file-id 123abc ingest_expression --taxon-name 'Homo sapiens' --taxon-common-name human --ncbi-taxid 9606 --matrix-file ../tests/data/dense_matrix_19_genes_100k_cells.txt --matrix-file-type dense
@@ -47,6 +48,7 @@
 from google.api_core import exceptions
 from google.cloud import firestore
 from mtx import Mtx
+from ingest_files import IngestFiles
 from subsample import SubSample
 from loom import Loom
 from validation.validate_metadata import validate_input_metadata, report_issues
@@ -57,7 +59,7 @@
 
 class IngestPipeline(object):
     # File location for metadata json convention
-    JSON_CONVENTION = 'DoNotTouch/AMC_v0.8.json'
+    JSON_CONVENTION = 'gs://fc-bcc55e6c-bec3-4b2e-9fb2-5e1526ddfcd2/metadata_conventions/AMC_v1.1.3/AMC_v1.1.3.json'
 
     def __init__(
         self,
@@ -229,8 +231,11 @@ def load_subsample(self, doc):
     def has_valid_metadata_convention(self):
         """ Determines if cell metadata file follows metadata convention"""
         with open(self.JSON_CONVENTION, 'r') as f:
-            convention = json.load(f)
+            json_file = IngestFiles(self.JSON_CONVENTION, ['application/json'])
+            convention = json.load(json_file.file)
             validate_input_metadata(self.cell_metadata, convention)
+
+        f.close()
         return not report_issues(self.cell_metadata)
 
     def ingest_expression(self) -> None:
@@ -272,6 +277,7 @@ def ingest_cell_metadata(self):
             if self.kwargs['validate_convention'] is not None:
                 if self.kwargs['validate_convention']:
                     if self.has_valid_metadata_convention():
+                        print("it works!")
                         pass
                     else:
                         return 1