Merge pull request #45 from broadinstitute/jlc_gzip_ingest

Add support for gzip files [SCP-1884]

Author: jlchang

ingest/ingest_files.py

@@ -8,6 +8,7 @@
 import mimetypes
 import os
 import re
+import gzip
 
 import pandas as pd
 from google.cloud import storage
@@ -19,6 +20,7 @@ def __init__(self, file_path, allowed_file_types, *, open_as=None):
         # File is remote (in GCS bucket) when running via PAPI,
         # and typically local when developing
         self.is_remote_file = file_path[:5] == "gs://"
+        self.is_gzip_file = self.get_file_type(file_path)[1] == 'gzip'
 
         self.verify_file_exists(file_path)
 
@@ -67,9 +69,13 @@ def resolve_path(self, file_path):
         """
         if self.is_remote_file:
             file_path = self.download_from_bucket(file_path)
-
         # Remove BOM with encoding ='utf - 8 - sig'
-        return open(file_path, encoding="utf-8-sig")
+        if self.is_gzip_file:
+            open_file = gzip.open(file_path, 'rt', encoding='utf-8-sig')
+        else:
+            open_file = open(file_path, encoding="utf-8-sig")
+
+        return open_file
 
     def reset_file(self, start_point, open_as=None):
         """Restart file reader at point that's equal to start_point.

ingest/ingest_pipeline.py

@@ -16,7 +16,7 @@
 python ingest_pipeline.py --study-accession SCP1 --file-id 123abc ingest_cluster --cluster-file ../tests/data/test_1k_cluster_Data.csv --ingest-cluster --name cluster1 --domain-ranges "{'x':[-1, 1], 'y':[-1, 1], 'z':[-1, 1]}"
 
 # Ingest Cell Metadata file
-python ingest_pipeline.py --study-accession SCP1 --file-id 123abc ingest_cell_metadata --cell-metadata-file ../tests/data/metadata_valid.tsv --ingest-cell-metadata
+python ingest_pipeline.py --study-accession SCP1 --file-id 123abc ingest_cell_metadata --cell-metadata-file ../tests/data/valid_v1.1.1.tsv --ingest-cell-metadata
 
 # Ingest Cell Metadata file against convention
 !! Please note that you must have permission to the SCP bucket

tests/data/dense_matrix_19_genes_100k_cells.txt.gz

@@ -218,6 +218,47 @@ def test_ingest_local_dense_matrix(self):
 
         self.assertEqual(model, expected_model)
 
+    def test_ingest_local_compressed_dense_matrix(self):
+        """Ingest Pipeline should extract and transform local dense matrices
+            from compressed file in the same manner as uncompressed file
+        """
+
+        args = [
+            '--study-accession',
+            'SCP1',
+            '--file-id',
+            '1234abc',
+            'ingest_expression',
+            '--taxon-name',
+            'Homo sapiens',
+            '--taxon-common-name',
+            'human',
+            '--ncbi-taxid',
+            '9606',
+            '--genome-assembly-accession',
+            'GCA_000001405.15',
+            '--genome-annotation',
+            'Ensembl 94',
+            '--matrix-file',
+            '../tests/data/dense_matrix_19_genes_100k_cells.txt.gz',
+            '--matrix-file-type',
+            'dense',
+        ]
+        ingest = self.setup_ingest(args)
+
+        models = ingest.load_expression_data_args[0]
+
+        # Verify that 19 gene models were passed into load method
+        num_models = len(models)
+        expected_num_models = 19
+        self.assertEqual(num_models, expected_num_models)
+
+        # Verify that the first gene model looks as expected
+        mock_dir = 'dense_matrix_19_genes_100k_cells_txt'
+        model, expected_model = get_nth_gene_models(0, models, mock_dir)
+
+        self.assertEqual(model, expected_model)
+
     def test_ingest_missing_local_file(self):
         """Ingest Pipeline should throw error for missing local file
         """