Validate against metadata convention in google bucket

Author: knapii-developments

ingest/ingest_files.py

@@ -88,6 +88,7 @@ def open_file(self, file_path, open_as=None, start_point: int = 0):
         file_connections = {
             "text/csv": self.open_csv(open_file),
             "text/plain": open_file,
+            "application/json": open_file,
             "text/tab-separated-values": self.open_tsv(open_file),
             "dataframe": self.open_pandas,
         }

ingest/ingest_pipeline.py

@@ -19,7 +19,8 @@
 python ingest_pipeline.py --study-accession SCP1 --file-id 123abc ingest_cell_metadata --cell-metadata-file ../tests/data/metadata_valid.tsv --ingest-cell-metadata
 
 # Ingest Cell Metadata file against convention
-python ingest_pipeline.py --study-accession SCP1 --file-id 123abc ingest_cell_metadata --cell-metadata-file ../tests/data/metadata_valid.tsv --ingest-cell-metadata --validate-convention
+!! Please note that you must have permission to the SCP bucket
+python ingest_pipeline.py --study-accession SCP1 --file-id 123abc ingest_cell_metadata --cell-metadata-file ../tests/data/valid_array_v1.1.3.tsv --ingest-cell-metadata --validate-convention
 
 # Ingest dense file
 python ingest_pipeline.py --study-accession SCP1 --file-id 123abc ingest_expression --taxon-name 'Homo sapiens' --taxon-common-name human --ncbi-taxid 9606 --matrix-file ../tests/data/dense_matrix_19_genes_100k_cells.txt --matrix-file-type dense
@@ -47,6 +48,7 @@
 from google.api_core import exceptions
 from google.cloud import firestore
 from mtx import Mtx
+from ingest_files import IngestFiles
 from subsample import SubSample
 from loom import Loom
 from validation.validate_metadata import (
@@ -61,7 +63,7 @@
 
 class IngestPipeline(object):
     # File location for metadata json convention
-    JSON_CONVENTION = 'DoNotTouch/AMC_v0.8.json'
+    JSON_CONVENTION = 'gs://fc-bcc55e6c-bec3-4b2e-9fb2-5e1526ddfcd2/metadata_conventions/AMC_v1.1.1.json'
 
     def __init__(
         self,
@@ -232,8 +234,8 @@ def load_subsample(self, doc):
 
     def has_valid_metadata_convention(self):
         """ Determines if cell metadata file follows metadata convention"""
-        with open(self.JSON_CONVENTION, 'r') as f:
-            convention = json.load(f)
+        json_file = IngestFiles(self.JSON_CONVENTION, ['application/json'])
+        convention = json.load(json_file.file)
 
         collect_jsonschema_errors(self.cell_metadata, convention)
         validate_collected_ontology_data(self.cell_metadata, convention)
@@ -278,6 +280,7 @@ def ingest_cell_metadata(self):
             if self.kwargs['validate_convention'] is not None:
                 if self.kwargs['validate_convention']:
                     if self.has_valid_metadata_convention():
+                        print("it works!")
                         pass
                     else:
                         return 1