Merge pull request #298 from broadinstitute/development

Release 1.25.0

Author: jlchang

.circleci/config.yml

@@ -10,7 +10,7 @@ orbs:
 jobs:
   build:
     docker:
-      - image: circleci/python:3.7.5-stretch
+      - image: cimg/python:3.10.9
     resource_class: large
 
     working_directory: ~/scp-ingest-pipeline
@@ -21,34 +21,29 @@ jobs:
       # Download and cache dependencies
       - restore_cache:
           keys:
-            - v3-dependencies-{{ checksum "requirements.txt" }}
+            - v6-dependencies-{{ checksum "requirements.txt" }}
             # fallback to using the latest cache if no exact match is found
-            - v3-dependencies-
-
-      - run:
-          name: Install system dependencies for Genomes Pipeline
-          command: |
-            sudo apt-get install -y tabix
+            - v6-dependencies-
 
       - run:
           name: Install Python dependencies
           command: |
-            python3 -m venv venv
+            python -m venv venv
             . venv/bin/activate
             pip install --upgrade pip
             pip install -r requirements.txt
 
       - save_cache:
           paths:
             - ./venv
-          key: v3-dependencies-{{ checksum "requirements.txt" }}
+          key: v6-dependencies-{{ checksum "requirements.txt" }}
 
       - run:
           name: Run tests
           command: |
             . venv/bin/activate
             cd tests
-            pytest -k 'not test_genomes and not test_make_toy and not test_delocalize_file' --cov-report=xml --cov=../ingest/
+            pytest -k 'not test_genomes and not test_make_toy and not test_delocalize_file' -p no:warnings --cov-report=xml --cov=../ingest/
 
       - codecov/upload:
           file: tests/coverage.xml

.gitignore

@@ -1,5 +1,8 @@
 tests/issues.json
 
+# Ignore secrets
+config/
+
 # Ignore Python runtime assets
 env/
 __pycache__/

Dockerfile

@@ -10,23 +10,24 @@
 # Use a managed base image from Google.  It is continually updated for
 # security fixes (thus the "latest" tag).
 # https://github.com/GoogleContainerTools/base-images-docker/tree/master/ubuntu
-FROM marketplace.gcr.io/google/ubuntu1804:latest
+FROM marketplace.gcr.io/google/ubuntu2004:latest
 
-# RUN echo "Uncomment to clear cached layers below this statement (2020-01-07-0947)"
+# RUN echo "Uncomment to clear cached layers below this statement (2022-03-14-1441)"
 
-# Install Python 3.7
+# Install Python 3.10
 RUN apt-get -y update && \
   apt-get -y install software-properties-common && \
   add-apt-repository ppa:deadsnakes/ppa && \
   apt-get -y install python3-pip && \
-  apt-get -y install python3.7 && \
-  apt-get -y install python3.7-dev
+  apt-get -y install python3.10 && \
+  apt-get -y install python3.10-dev && \
+  apt-get -y install python3.10-distutils
 
-RUN python3.7 -m pip install --upgrade pip
+RUN curl -sS https://bootstrap.pypa.io/get-pip.py | python3.10
 
-# Set cleaner defaults (`alias` fails)
-RUN ln -s /usr/bin/python3.7 /usr/bin/python && \
-  ln -s /usr/bin/pip3 /usr/bin/pip
+# # Set cleaner defaults (`alias` fails)
+# RUN ln -s /usr/bin/python3.10 /usr/bin/python && \
+#   ln -s /usr/bin/pip3 /usr/bin/pip
 
 # Copy contents of this repo into the Docker image
 # (See .Dockerignore for omitted files)
@@ -35,7 +36,7 @@ COPY . scp-ingest-pipeline
 WORKDIR /scp-ingest-pipeline
 
 # Install Python dependencies
-RUN python3.7 -m pip install -r requirements.txt
+RUN python3.10 -m pip install -r requirements.txt
 
 WORKDIR /scp-ingest-pipeline/ingest
 CMD ["python", "ingest_pipeline.py", "--help"]

README.md

@@ -9,14 +9,16 @@ The SCP Ingest Pipeline is an ETL pipeline for single-cell RNA-seq data.
 
 # Prerequisites
 
-- Python 3.7+
+- Python 3.10
 - Google Cloud Platform project
 - Suitable service account (SA) and MongoDB VM in GCP. SA needs roles "Editor", "Genomics Pipelines Runner", and "Storage Object Admin". Broad Institute engineers: see instructions [here](https://github.com/broadinstitute/single_cell_portal_configs/tree/master/terraform-mongodb).
 - SAMtools, if using `ingest/make_toy_data.py`
 - Tabix, if using `ingest/genomes/genomes_pipeline.py`
 
 # Install
 
+### Native
+
 Fetch the code, boot your virtualenv, install dependencies:
 
 ```
@@ -25,68 +27,41 @@ cd scp-ingest-pipeline
 python3 -m venv env --copies
 source env/bin/activate
 pip install -r requirements.txt
+scripts/setup-mongo-dev.sh <PATH_TO_YOUR_VAULT_TOKEN> # E.g. ~/.github-token
 ```
 
-To use `ingest/make_toy_data.py`:
+### Docker
 
-```
-brew install samtools
-```
-
-To use `ingest/genomes/genomes_pipeline.py`:
+With Docker running and Vault active on your local machine, run:
 
 ```
-brew install tabix
+scripts/docker-compose-setup.sh -t <PATH_TO_YOUR_VAULT_TOKEN> # E.g. ~/.github-token
 ```
 
-Now get secrets from Vault to set environment variables needed to write to the database:
-(see also scripts/setup_mongo_dev.sh)
+If on Apple silicon Mac (e.g. M1), and performance seems poor, consider generating a docker image using the arm64 base. Example test image: gcr.io/broad-singlecellportal-staging/single-cell-portal:development-2.2.0-arm64, usage:
 
 ```
-export BROAD_USER="<username in your email address>"
-
-export DATABASE_NAME="single_cell_portal_development"
-
-vault login -method=github token=`~/bin/git-vault-token`
-
-# Get username and password
-vault read secret/kdux/scp/development/$BROAD_USER/mongo/user
-
-export MONGODB_USERNAME="<username from Vault>"
-export MONGODB_PASSWORD="<password from Vault>"
-
-# Get external IP address for host
-vault read secret/kdux/scp/development/$BROAD_USER/mongo/hostname
-
-export DATABASE_HOST="<ip from Vault (omit brackets)>"
+scripts/docker-compose-setup.sh -i development-2.2.0-arm64 -t <PATH_TO_YOUR_VAULT_TOKEN>
 ```
 
-For testing/development using ingest_pipeline.py on the command line, annotation file input validation and MongoDB writes can be bypassed if you set:
-(Note: this bypass does not currently apply to expression matrix ingest).
+To update dependencies when in Docker, you can pip install from within the Docker Bash shell after adjusting your requirements.txt.
+If you close your shell after that, your newly installed dependencies will be lost.  Dependencies only persist after merging your
+new requirements.txt into `development`.  TODO (SCP-4941): Add entry-point script to run `pip install`.
 
-```
-export BYPASS_MONGO_WRITES='yes'
-```
+### Optional
 
-If you are developing updates for Mixpanel logging, set the Bard host URL:
+To use `ingest/make_toy_data.py`:
 
 ```
-
-export BARD_HOST_URL="https://terra-bard-dev.appspot.com"
+brew install samtools
 ```
 
-Be sure to `unset BARD_HOST_URL` when your updates are done, so development ingest events are not always sent to Mixpanel.
-
-If you are developing updates for Sentry logging, then set the DSN:
+To use `ingest/genomes/genomes_pipeline.py`:
 
 ```
-vault read secret/kdux/scp/production/scp_config.json | grep SENTRY
-
-export SENTRY_DSN="<Sentry DSN value from Vault>"
+brew install tabix
 ```
 
-Be sure to `unset SENTRY_DSN` when your updates are done, so development logs are not always sent to Sentry.
-
 ## Git hooks
 
 After installing Ingest Pipeline, add Git hooks to help ensure code quality:
@@ -106,7 +81,7 @@ In rare cases, you might need to skip Git hooks, like so:
 
 # Test
 
-After [installing](#Install):
+After [installing](#install):
 
 ```
 source env/bin/activate
@@ -128,49 +103,83 @@ pytest test_ingest.py
 # Run all tests, show code coverage metrics
 pytest --cov=../ingest/
 ```
+
 For more, see <https://docs.pytest.org/en/stable/usage.html>.
 
-## Testing in Docker
-If you have difficulties installing and configuring `scp-ingest-pipeline` due to hardware issues (e.g. Mac M1 chips), 
-you can alternatively test locally by building the Docker image and then running any commands inside the container. 
+## Testing in Docker locally
+<!--
+Step 1 is also useful for troubleshooting when Dockerfile updates fail to build
+-->
+If you have difficulties installing and configuring `scp-ingest-pipeline` due to hardware issues (e.g. Mac M1 chips),
+you can alternatively test locally by building the Docker image and then running any commands inside the container.
 There are some extra steps required, but this sidesteps the need to install packages locally.
 
 ### 1. Build the image
-Run the following command to build the testing Docker image locally (make sure Docker is running first):
+
+Run the following command to build the testing Docker image locally (make sure Docker is running first). This build command will incorporate any changes in the local instance of your repo, committed or not:
+
 ```
 docker build -t gcr.io/broad-singlecellportal-staging/ingest-pipeline:test-candidate .
 ```
+
+Note - if this is your first time doing `docker build` you may need to configure Docker to use the Google Cloud CLI to authenticate requests to Container Registry:
+
+```
+gcloud auth configure-docker
+```
+
+Pro-Tip: For local builds, you can try adding docker build options `--progress=plain` (for more verbose build info) and/or `--no-cache` (when you want to ensure a build with NO cached layers)
+
 ### 2. Set up environment variables
+
 Run the following to pull database-specific secrets out of vault (passing in the path to your vault token):
+
 ```
-source scripts/setup_mongo_dev.sh ~/.your-vault-token
+source scripts/setup-mongo-dev.sh ~/.your-vault-token
 ```
+
 Now run `env` to make sure you've set the following values:
+
 ```
 MONGODB_USERNAME=single_cell
 DATABASE_NAME=single_cell_portal_development
 MONGODB_PASSWORD=<password>
 DATABASE_HOST=<ip address>
 ```
+
 ### 3. Print out your service account keyfile
+
 Run the following to export out your default service account JSON keyfile:
+
 ```
 vault read -format=json secret/kdux/scp/development/$(whoami)/scp_service_account.json | jq .data > /tmp/keyfile.json
 ```
+
 ### 4. Start the Docker container
+
 Run the container, passing in the proper environment variables:
+
 ```
 docker run --name scp-ingest-test -e MONGODB_USERNAME="$MONGODB_USERNAME" -e DATABASE_NAME="$DATABASE_NAME" \
            -e MONGODB_PASSWORD="$MONGODB_PASSWORD" -e DATABASE_HOST="$DATABASE_HOST" \
            -e GOOGLE_APPLICATION_CREDENTIALS=/tmp/keyfile.json --rm -it \
            gcr.io/broad-singlecellportal-staging/ingest-pipeline:test-candidate bash
 ```
+
+Note: on an M1 machine, you may see this message:
+
+> WARNING: The requested image's platform (linux/amd64) does not match the detected host platform (linux/arm64/v8) and no specific platform was requested
+
 ### 5. Copy keyfile to running container
+
 In a separate terminal window, copy the JSON keyfile from above to the expected location:
+
 ```
 docker cp /tmp/keyfile.json scp-ingest-test:/tmp
 ```
+
 You can now run any `ingest_pipeline.py` command you wish inside the container.
+
 # Use
 
 Run this every time you start a new terminal to work on this project:
@@ -184,10 +193,12 @@ See [`ingest_pipeline.py`](https://github.com/broadinstitute/scp-ingest-pipeline
 ## Troubleshooting during set up
 
 If you run into an error like: "... [SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed ... " try:
+
 - Open terminal
 - `cd` to where python is installed
 - Run the certificates command with `/Applications/Python\ < Your Version of Python Here >/Install\ Certificates.command`
 
 If you run into an error like "ModuleNotFoundError: No module named 'google'" try:
+
 - Open terminal
 - Run `pip install --upgrade google-api-python-client`

docker-compose-dev.yaml

@@ -0,0 +1,11 @@
+services:
+  ingest:
+    container_name: scp-ingest-pipeline
+    image: "${GCR_IMAGE}"
+    env_file:
+      - config/.ingest_env
+    expose:
+      - 27017 # Allow outgoing traffic to MongoDB
+    volumes:
+      - .:/scp-ingest-pipeline # Needed to mount
+

images/metadata.png

@@ -0,0 +1,62 @@
+#using https://nbviewer.org/github/theislab/diffxpy_tutorials/blob/master/diffxpy_tutorials/test/multiple_tests_per_gene.ipynb
+import anndata
+import matplotlib.pyplot as plt
+import seaborn as sns
+import logging
+import numpy as np
+import pandas as pd
+import scipy.stats
+import scanpy as sc
+import os
+
+
+import batchglm.api as glm
+
+import diffxpy.api as de
+
+# TODO (SCP-5041): Extract these to CLI arguments
+
+#Pairwise tests between groups
+#answers whether a given pair of groups shows differential expression for each gene
+
+#arguments:
+#data: Anndata object, data matrix with cells x genes
+
+#grouping: str, column in data, the column that contains cell type labels. alternatively vector containing group labels
+
+#test: type of test, default is z-test. other options: ’wald’, ’lrt’, ’t-test’, ’rank’
+
+#lazy: bool, only possible if test is ztest, if true only evaluated once the user requests pval/coefficients for a specific pair of models
+
+#noise_model: default is nb, specify NONE for wald and t test
+
+h5ad_file = "tests/data/anndata/trimmed_compliant_pbmc3K.h5ad"
+
+adata_file = sc.read_h5ad(h5ad_file)
+
+adata_file.var_names_make_unique()
+
+print(adata_file.obs["louvain"])
+
+
+test = de.test.pairwise(
+    data=adata_file,
+    grouping="louvain",
+    test="rank",
+    lazy=False,
+    noise_model=None
+ )
+
+# Accessing results
+
+#outputs shape of p values
+np.set_printoptions(precision=3)
+print("shape of p-values: %s" % str(test.pval.shape))
+
+
+#brings up pvalue
+print(test.pval[:,:,0])
+
+#brings up plot 
+test.plot_volcano()
+