Merge pull request #44 from broadinstitute/ew-remove-firestore-emulator

Remove Firestore emulator [SCP-1968]

Author: eweitz

.circleci/config.yml

@@ -20,28 +20,9 @@ jobs:
       # Download and cache dependencies
       - restore_cache:
           keys:
-            - v2-dependencies-{{ checksum "requirements.txt" }}
+            - v3-dependencies-{{ checksum "requirements.txt" }}
             # fallback to using the latest cache if no exact match is found
-            - v2-dependencies-
-
-      - run:
-          name: Install Firestore emulator
-          command: |
-            # Add the Cloud SDK distribution URI as a package source
-            echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] http://packages.cloud.google.com/apt cloud-sdk main" | sudo tee -a /etc/apt/sources.list.d/google-cloud-sdk.list
-
-            # Import the Google Cloud Platform public key
-            curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo apt-key --keyring /usr/share/keyrings/cloud.google.gpg add -
-            
-            # Enable `add-apt-repository` command
-            sudo apt-get --yes install software-properties-common
-
-            # Enable installation of OpenJDK, needed by gcloud
-            sudo add-apt-repository --yes ppa:openjdk-r/ppa
-
-            # Update the package list and install the Cloud SDK
-            sudo apt-get --yes update && sudo apt-get --yes install google-cloud-sdk
-            sudo apt-get --yes install google-cloud-sdk google-cloud-sdk-firestore-emulator
+            - v3-dependencies-
 
       - run:
           name: Install Python dependencies
@@ -54,14 +35,11 @@ jobs:
       - save_cache:
           paths:
             - ./venv
-          key: v2-dependencies-{{ checksum "requirements.txt" }}
+          key: v3-dependencies-{{ checksum "requirements.txt" }}
 
       - run:
           name: Run tests
           command: |
-            env
-            gcloud beta emulators firestore start --host-port localhost:8081 &
-            export FIRESTORE_EMULATOR_HOST=localhost:8081
             . venv/bin/activate
             cd tests
             pytest --cov-report=xml --cov=../ingest/

.gitignore

@@ -13,4 +13,6 @@ __pycache__/
 *.log
 .DS_Store
 .coverage
+coverage.xml
+issues.json
 htmlcov/

ingest/ingest_pipeline.py

@@ -73,18 +73,14 @@ def __init__(
         subsample=False,
         ingest_cell_metadata=False,
         ingest_cluster=False,
-        db=None,
         **kwargs,
     ):
         """Initializes variables in ingest service."""
         self.file_id = file_id
         self.study_accession = study_accession
         self.matrix_file = matrix_file
         self.matrix_file_type = matrix_file_type
-        if db is not None:
-            self.db = db
-        else:
-            self.db = firestore.Client()
+        self.db = firestore.Client()
         self.cluster_file = cluster_file
         self.kwargs = kwargs
         self.cell_metadata_file = cell_metadata_file

tests/gcp_mocks.py

@@ -53,3 +53,10 @@ def download_to_filename(self, filename):
             copyfile(self.name, filename)
 
     return MockStorageBlob(*args, **kwargs)
+
+
+def mock_firestore_client():
+    """Mocks firestore.Client() by returning nothing upon initializing client
+    See notes in mock_load_expression_data for context.
+    """
+    return

tests/gene_model_0.txt

@@ -0,0 +1,2 @@
+{'name': 'RPP25L', 'gene_id': 'ENSG00000164967', 'source_file_type': ('Mtx',), 'cell_names': ['AACGTTGGTTAAAGTG-1'], 'expression_scores': [1.0], 'subdocument': {'cell_names': ['AACGTTGGTTAAAGTG-1'], 'expression_scores': [1.0], 'source_file_type': 'Mtx'}, 'top_level_doc': {'file_id': '1234abc', 'searchable_name': 'rpp25l', 'name': 'RPP25L',
+'gene_id': 'ENSG00000164967', 'study_accession': 'SCP1', 'taxon_name': 'Homo sapiens', 'taxon_common_name': 'human', 'ncbi_taxid': '9606', 'genome_assembly_accession': 'GCA_000001405.15', 'genome_annotation': 'Ensembl 94'}}

tests/mock_data/dense_matrix_19_genes_100k_cells_txt/gene_doc_0.txt

@@ -0,0 +1 @@
+{'name': 'A1BG', 'gene_id': None, 'source_file_type': ('Dense',), 'cell_names': ['"Granuloma10_270"', '"Granuloma10_902"', '"Granuloma5_1043"', '"Granuloma5_316"'], 'expression_scores': [0.602, 1.934, 1.161, 0.82], 'subdocument': {'cell_names': ['"Granuloma10_270"', '"Granuloma10_902"', '"Granuloma5_1043"', '"Granuloma5_316"'], 'expression_scores': [0.602, 1.934, 1.161, 0.82], 'source_file_type': 'Dense'}, 'top_level_doc': {'file_id': '1234abc', 'searchable_name': 'a1bg', 'name': 'A1BG', 'gene_id': None, 'study_accession': 'SCP1', 'taxon_name': 'Homo sapiens', 'taxon_common_name': 'human', 'ncbi_taxid': '9606', 'genome_assembly_accession': 'GCA_000001405.15', 'genome_annotation': 'Ensembl 94'}}

tests/mock_data/dense_matrix_19_genes_100k_cells_txt/gene_model_0.txt

@@ -0,0 +1 @@
+{'name': 'A2M', 'gene_id': None, 'source_file_type': ('loom',), 'cell_names': ['BM01_16dpp_AAGCAGTGGTAT', 'BM01_16dpp_TAAGCAGTGGTA', 'BM01_16dpp_CTAAGCAGTGGT', 'BM01_16dpp_CGGTAAACCATT', 'BM01_16dpp_CCGAATTCACCG'], 'expression_scores': [-0.191, -0.191, 2.646, -0.191, -0.191], 'subdocument': {'cell_names': ['BM01_16dpp_AAGCAGTGGTAT', 'BM01_16dpp_TAAGCAGTGGTA', 'BM01_16dpp_CTAAGCAGTGGT', 'BM01_16dpp_CGGTAAACCATT', 'BM01_16dpp_CCGAATTCACCG'], 'expression_scores': [-0.191, -0.191, 2.646, -0.191, -0.191], 'source_file_type': 'loom'}, 'top_level_doc': {'file_id': '1234abc', 'searchable_name': 'a2m', 'name': 'A2M', 'gene_id': None, 'study_accession': 'SCP1', 'taxon_name': 'Homo Sapiens', 'taxon_common_name': 'human', 'ncbi_taxid': '9606', 'genome_assembly_accession': 'GCA_000001405.15', 'genome_annotation': 'Ensemble 94'}}