Merge pull request #424 from maxplanck-ie/develop

Merge develop into master

.ci_stuff/ChIP.sample_config.yaml

@@ -0,0 +1,13 @@
+chip_dict:
+  sample1:
+    control: sample3
+    broad: False
+  sample2:
+    control: sample3
+    broad: False
+  sample4:
+    control: sample6
+    broad: False
+  sample5:
+    control: sample6
+    broad: False

.ci_stuff/organism.yaml

@@ -0,0 +1,15 @@
+genome_size: 2652783500
+genome_fasta: "/tmp/genome.fa"
+genome_index: "/tmp/genome.fa.fai"
+genome_2bit: ".ci_stuff/genome_fasta/genome.2bit"
+bowtie2_index: ".ci_stuff/BowtieIndex/genome"
+hisat2_index: ".ci_stuff/HISAT2Index/genome"
+bwa_index: ".ci_stuff/BWAindex/genome.fa"
+bwameth_index: "/tmp/genome.fa"
+known_splicesites: ".ci_stuff/gencode/m9/HISAT2/splice_sites.txt"
+star_index: ".ci_stuff/STARIndex/"
+genes_bed: ".ci_stuff/gencode/m9/genes.bed"
+genes_gtf: "/tmp/genes.gtf"
+extended_coding_regions_gtf: ".ci_stuff/gencode/m9/genes.slop.gtf"
+blacklist_bed: ".ci_stuff/DKFZ/GRCm38_General_readAttractingRegions.UseThisOne.bed"
+ignore_forNorm: "MT X Y JH584299.1 GL456233.1 JH584301.1 GL456211.1 GL456350.1 JH584293.1 GL456221.1 JH584297.1 JH584296.1 GL456354.1 JH584294.1 JH584298.1 JH584300.1 GL456219.1 GL456210.1 JH584303.1 JH584302.1 GL456212.1 JH584304.1 GL456379.1 GL456216.1 GL456393.1 GL456366.1 GL456367.1 GL456239.1 GL456213.1 GL456383.1 GL456385.1 GL456360.1 GL456378.1 GL456389.1 GL456372.1 GL456370.1 GL456381.1 GL456387.1 GL456390.1 GL456394.1 GL456392.1 GL456382.1 GL456359.1 GL456396.1 GL456368.1 JH584292.1 JH584295.1"

.ci_stuff/setup_conda.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+curl https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -o miniconda.sh
+bash miniconda.sh -b -p $HOME/miniconda
+export PATH="$HOME/miniconda/bin:$PATH"
+hash -r
+conda config --set always_yes yes --set changeps1 no
+
+conda install --yes --quiet -c conda-forge -c bioconda snakemake fuzzywuzzy mock sphinx sphinx-argparse
+pip install sphinx-rtd-theme

.ci_stuff/test_dag.sh

@@ -0,0 +1,114 @@
+#!/bin/bash
+set -ex
+if [[ ${CI:-"false"} == "true" ]]; then
+    export PATH="$HOME/miniconda/bin:$PATH"
+    hash -r
+    python -m pip install --no-deps --ignore-installed .
+fi
+
+# Needed by DNA, HiC, RNA-seq, WGBS and scRNA-seq workflows
+mkdir -p PE_input
+touch PE_input/sample1_R1.fastq.gz PE_input/sample1_R2.fastq.gz \
+      PE_input/sample2_R1.fastq.gz PE_input/sample2_R2.fastq.gz \
+      PE_input/sample3_R1.fastq.gz PE_input/sample3_R2.fastq.gz \
+      PE_input/sample4_R1.fastq.gz PE_input/sample4_R2.fastq.gz \
+      PE_input/sample5_R1.fastq.gz PE_input/sample5_R2.fastq.gz \
+      PE_input/sample6_R1.fastq.gz PE_input/sample6_R2.fastq.gz
+mkdir -p SE_input
+touch SE_input/sample1_R1.fastq.gz \
+      SE_input/sample2_R1.fastq.gz \
+      SE_input/sample3_R1.fastq.gz \
+      SE_input/sample4_R1.fastq.gz \
+      SE_input/sample5_R1.fastq.gz \
+      SE_input/sample6_R1.fastq.gz
+# Needed by ChIP and ATAC workflows
+mkdir -p BAM_input/deepTools_qc/bamPEFragmentSize BAM_input/filtered_bam BAM_input/Sambamba
+touch BAM_input/sample1.bam \
+      BAM_input/sample2.bam \
+      BAM_input/sample3.bam \
+      BAM_input/sample4.bam \
+      BAM_input/sample5.bam \
+      BAM_input/sample6.bam \
+      BAM_input/filtered_bam/sample1.filtered.bam \
+      BAM_input/filtered_bam/sample2.filtered.bam \
+      BAM_input/filtered_bam/sample3.filtered.bam \
+      BAM_input/filtered_bam/sample4.filtered.bam \
+      BAM_input/filtered_bam/sample5.filtered.bam \
+      BAM_input/filtered_bam/sample6.filtered.bam \
+      BAM_input/filtered_bam/sample1.filtered.bam.bai \
+      BAM_input/filtered_bam/sample2.filtered.bam.bai \
+      BAM_input/filtered_bam/sample3.filtered.bam.bai \
+      BAM_input/filtered_bam/sample4.filtered.bam.bai \
+      BAM_input/filtered_bam/sample5.filtered.bam.bai \
+      BAM_input/filtered_bam/sample6.filtered.bam.bai \
+      BAM_input/Sambamba/sample1.markdup.txt \
+      BAM_input/Sambamba/sample2.markdup.txt \
+      BAM_input/Sambamba/sample3.markdup.txt \
+      BAM_input/Sambamba/sample4.markdup.txt \
+      BAM_input/Sambamba/sample5.markdup.txt \
+      BAM_input/Sambamba/sample6.markdup.txt \
+      BAM_input/deepTools_qc/bamPEFragmentSize/fragmentSize.metric.tsv
+mkdir -p output
+touch /tmp/genes.gtf /tmp/genome.fa /tmp/genome.fa.fai
+
+# DNA mapping
+WC=`DNA-mapping -i PE_input -o output --tempdir /tmp .ci_stuff/organism.yaml --snakemake_options " --dryrun --conda-prefix /tmp" | tee /dev/stderr | grep -v "Conda environment" | wc -l`
+if [ ${PIPESTATUS[0]} -ne 0 ] || [ $WC -ne 734 ]; then exit 1 ; fi
+WC=`DNA-mapping -i PE_input -o output --tempdir /tmp .ci_stuff/organism.yaml --snakemake_options " --dryrun --conda-prefix /tmp" --trim --mapq 20 --dedup --properpairs | tee /dev/stderr | grep -v "Conda environment" | wc -l`
+if [ ${PIPESTATUS[0]} -ne 0 ] || [ $WC -ne 936 ]; then exit 1 ; fi
+WC=`DNA-mapping -i SE_input -o output --tempdir /tmp .ci_stuff/organism.yaml --snakemake_options " --dryrun --conda-prefix /tmp" | tee /dev/stderr | grep -v "Conda environment" | wc -l`
+if [ ${PIPESTATUS[0]} -ne 0 ] || [ $WC -ne 684 ]; then exit 1 ; fi
+WC=`DNA-mapping -i SE_input -o output --tempdir /tmp .ci_stuff/organism.yaml --snakemake_options " --dryrun --conda-prefix /tmp" --trim --mapq 20 --dedup --properpairs | tee /dev/stderr | grep -v "Conda environment" | wc -l`
+if [ ${PIPESTATUS[0]} -ne 0 ] || [ $WC -ne 820 ]; then exit 1 ; fi
+
+# ChIP-seq
+WC=`ChIP-seq -d BAM_input --tempdir /tmp --snakemake_options " --dryrun --conda-prefix /tmp" .ci_stuff/organism.yaml .ci_stuff/ChIP.sample_config.yaml | tee /dev/stderr | grep -v "Conda environment" | wc -l`
+if [ ${PIPESTATUS[0]} -ne 0 ] || [ $WC -ne 268 ]; then exit 1 ; fi
+WC=`ChIP-seq -d BAM_input --tempdir /tmp --snakemake_options " --dryrun --conda-prefix /tmp" --single-end .ci_stuff/organism.yaml .ci_stuff/ChIP.sample_config.yaml | tee /dev/stderr | grep -v "Conda environment" | wc -l`
+if [ ${PIPESTATUS[0]} -ne 0 ] || [ $WC -ne 268 ]; then exit 1 ; fi
+WC=`ChIP-seq -d BAM_input --tempdir /tmp --snakemake_options " --dryrun --conda-prefix /tmp" --bigWigType log2ratio .ci_stuff/organism.yaml .ci_stuff/ChIP.sample_config.yaml | tee /dev/stderr | grep -v "Conda environment" | wc -l`
+if [ ${PIPESTATUS[0]} -ne 0 ] || [ $WC -ne 222 ]; then exit 1 ; fi
+
+# ATAC-seq
+WC=`ATAC-seq -d BAM_input --tempdir /tmp --snakemake_options " --dryrun --conda-prefix /tmp" .ci_stuff/organism.yaml | tee /dev/stderr | grep -v "Conda environment" | wc -l`
+if [ ${PIPESTATUS[0]} -ne 0 ] || [ $WC -ne 336 ]; then exit 1 ; fi
+WC=`ATAC-seq -d BAM_input --tempdir /tmp --snakemake_options " --dryrun --conda-prefix /tmp" --fragmentSize-cutoff 120 --qval 0.1 .ci_stuff/organism.yaml | tee /dev/stderr | grep -v "Conda environment" | wc -l`
+if [ ${PIPESTATUS[0]} -ne 0 ] || [ $WC -ne 336 ]; then exit 1 ; fi
+
+# RNA-seq
+WC=`RNA-seq -i PE_input -o output --tempdir /tmp --snakemake_options " --dryrun --conda-prefix /tmp" .ci_stuff/organism.yaml | tee /dev/stderr | grep -v "Conda environment" | wc -l`
+if [ ${PIPESTATUS[0]} -ne 0 ] || [ $WC -ne 889 ]; then exit 1 ; fi
+WC=`RNA-seq -i PE_input -o output --tempdir /tmp --snakemake_options " --dryrun --conda-prefix /tmp" -m "alignment" .ci_stuff/organism.yaml | tee /dev/stderr | grep -v "Conda environment" | wc -l`
+if [ ${PIPESTATUS[0]} -ne 0 ] || [ $WC -ne 578 ]; then exit 1 ; fi
+WC=`RNA-seq -i PE_input -o output --tempdir /tmp --snakemake_options " --dryrun --conda-prefix /tmp" -m "alignment,deepTools_qc" --trim .ci_stuff/organism.yaml | tee /dev/stderr | grep -v "Conda environment" | wc -l`
+if [ ${PIPESTATUS[0]} -ne 0 ] || [ $WC -ne 1052 ]; then exit 1 ; fi
+WC=`RNA-seq -i SE_input -o output --tempdir /tmp --snakemake_options " --dryrun --conda-prefix /tmp" .ci_stuff/organism.yaml | tee /dev/stderr | grep -v "Conda environment" | wc -l`
+if [ ${PIPESTATUS[0]} -ne 0 ] || [ $WC -ne 828 ]; then exit 1 ; fi
+WC=`RNA-seq -i SE_input -o output --tempdir /tmp --snakemake_options " --dryrun --conda-prefix /tmp" -m "alignment" .ci_stuff/organism.yaml | tee /dev/stderr | grep -v "Conda environment" | wc -l`
+if [ ${PIPESTATUS[0]} -ne 0 ] || [ $WC -ne 528 ]; then exit 1 ; fi
+WC=`RNA-seq -i SE_input -o output --tempdir /tmp --snakemake_options " --dryrun --conda-prefix /tmp" -m "alignment,deepTools_qc" --trim .ci_stuff/organism.yaml | tee /dev/stderr | grep -v "Conda environment" | wc -l`
+if [ ${PIPESTATUS[0]} -ne 0 ] || [ $WC -ne 925 ]; then exit 1 ; fi
+
+# HiC - fails
+WC=`HiC -i PE_input -o output --tempdir /tmp --snakemake_options " --dryrun --conda-prefix /tmp" .ci_stuff/organism.yaml | tee /dev/stderr | grep -v "Conda environment" | wc -l`
+if [ ${PIPESTATUS[0]} -ne 0 ] || [ $WC -ne 522 ]; then exit 1 ; fi
+WC=`HiC -i PE_input -o output --tempdir /tmp --snakemake_options " --dryrun --conda-prefix /tmp" --trim .ci_stuff/organism.yaml | tee /dev/stderr | grep -v "Conda environment" | wc -l`
+if [ ${PIPESTATUS[0]} -ne 0 ] || [ $WC -ne 724 ]; then exit 1 ; fi
+WC=`HiC -i PE_input -o output --tempdir /tmp --snakemake_options " --dryrun --conda-prefix /tmp" --enzyme DpnII .ci_stuff/organism.yaml | tee /dev/stderr | grep -v "Conda environment" | wc -l`
+if [ ${PIPESTATUS[0]} -ne 0 ] || [ $WC -ne 522 ]; then exit 1 ; fi
+WC=`HiC -i PE_input -o output --tempdir /tmp --snakemake_options " --dryrun --conda-prefix /tmp" --noTAD .ci_stuff/organism.yaml | tee /dev/stderr | grep -v "Conda environment" | wc -l`
+if [ ${PIPESTATUS[0]} -ne 0 ] || [ $WC -ne 460 ]; then exit 1 ; fi
+
+# scRNA-seq
+WC=`scRNAseq -i PE_input -o output --tempdir /tmp --snakemake_options " --dryrun --conda-prefix /tmp" .ci_stuff/organism.yaml | tee /dev/stderr | grep -v "Conda environment" | wc -l`
+if [ ${PIPESTATUS[0]} -ne 0 ] || [ $WC -ne 1080 ]; then exit 1 ; fi
+WC=`scRNAseq -i PE_input -o output --tempdir /tmp --snakemake_options " --dryrun --conda-prefix /tmp" --skipRaceID --split_lib .ci_stuff/organism.yaml | tee /dev/stderr | grep -v "Conda environment" | wc -l`
+if [ ${PIPESTATUS[0]} -ne 0 ] || [ $WC -ne 1051 ]; then exit 1 ; fi
+
+# WGBS
+WC=`WGBS -i PE_input -o output --tempdir /tmp --snakemake_options " --dryrun --conda-prefix /tmp" .ci_stuff/organism.yaml | tee /dev/stderr | grep -v "Conda environment" | wc -l`
+if [ ${PIPESTATUS[0]} -ne 0 ] || [ $WC -ne 962 ]; then exit 1 ; fi
+WC=`WGBS -i PE_input -o output --tempdir /tmp --snakemake_options " --dryrun --conda-prefix /tmp" --skipDOC --skipGCbias --nextera .ci_stuff/organism.yaml | tee /dev/stderr | grep -v "Conda environment" | wc -l`
+if [ ${PIPESTATUS[0]} -ne 0 ] || [ $WC -ne 844 ]; then exit 1 ; fi
+
+rm -rf SE_input PE_input BAM_input output /tmp/genes.gtf /tmp/genome.fa /tmp/genome.fa.fai

.ci_stuff/test_docs.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+export PATH="$HOME/miniconda/bin:$PATH"
+hash -r
+python -m pip install --no-deps --ignore-installed .
+
+cd docs && make html

.readthedocs.yaml

@@ -0,0 +1,5 @@
+conda:
+  file: docs/environment.yaml
+python:
+  version: 3.7
+  pip_install: true

.travis.yml

@@ -1,9 +1,19 @@
 language: python
 python:
   - "3.6"
+env:
+  - STAGE="0"
+  - STAGE="1"
+  - STAGE="2"
+
+# setup conda
+before_install:
+  - if [[ ${STAGE:-"0"} != "0" ]] ; then bash .ci_stuff/setup_conda.sh ; fi
 
 install:
   - pip install flake8
 
 script:
-  - flake8 --ignore=E501,E722 --exclude docs/conf.py .
+  - if [[ ${STAGE:-"0"} == "0" ]] ; then flake8 --ignore=E501,E722 --exclude docs/conf.py . ; fi
+  - if [[ ${STAGE:-"0"} == "1" ]] ; then bash .ci_stuff/test_docs.sh ; fi
+  - if [[ ${STAGE:-"0"} == "2" ]] ; then bash .ci_stuff/test_dag.sh ; fi