Merge pull request #1017 from maxplanck-ie/snakepipes_pytest

Real-data pytests

.github/workflows/linux.yml

@@ -11,7 +11,7 @@ jobs:
   docs:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - uses: mamba-org/setup-micromamba@main
       with:
         environment-file: .github/snakePipesEnvCI.yml
@@ -34,7 +34,7 @@ jobs:
   flake:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - uses: mamba-org/setup-micromamba@main
       with:
         environment-file: .github/snakePipesEnvCI.yml
@@ -56,7 +56,7 @@ jobs:
   CI:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - uses: mamba-org/setup-micromamba@main
       with:
         environment-file: .github/snakePipesEnvCI.yml
@@ -75,11 +75,6 @@ jobs:
         micromamba activate snakePipes_CI
         snakePipes config --tempDir /tmp
         ./.ci_stuff/test_dag.sh
-    - name: pytest
-      run: |
-        micromamba activate snakePipes_CI
-        snakePipes config --tempDir /tmp
-        pytest --verbosity=2
   createEnvs:
     needs: CI
     strategy:

.github/workflows/osx.yml

@@ -35,7 +35,7 @@ jobs:
         ]
     runs-on: macos-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: mamba-org/setup-micromamba@main
         with:
           environment-file: .github/snakePipesEnvCI.yml

.github/workflows/pytest.yml

@@ -0,0 +1,72 @@
+name: pytest
+
+on: [pull_request, push]
+
+defaults:
+  run:
+      shell: bash -l {0}
+
+jobs:
+  test_mRNA:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+    - uses: mamba-org/setup-micromamba@main
+      with:
+        environment-file: snakePipes/shared/rules/envs/rna_seq.yaml
+        environment-name: rnaseqenv
+        condarc: |
+          channels:
+            - conda-forge
+            - bioconda
+            - defaults
+          channel_priority: 'strict'
+        cache-downloads: true
+        post-cleanup: 'none'
+    - name: create_starix
+      run: |
+        micromamba activate rnaseqenv
+        gunzip -c tests/data/genomes/genome_chr17.fa.gz > genome_chr17.fa
+        gunzip -c tests/data/genomes/genes_chr17.gtf.gz > genes_chr17.gtf
+        STAR --runThreadN 4 --runMode genomeGenerate --genomeDir tests/data/mRNA_STAR --genomeFastaFiles genome_chr17.fa --sjdbGTFfile genes_chr17.gtf --sjdbOverhang 100 --genomeSAindexNbases 12
+    - uses: mamba-org/setup-micromamba@main
+      with:
+        environment-file: .github/snakePipesEnvCI.yml
+        condarc: |
+          channels:
+            - conda-forge
+            - bioconda
+            - defaults
+          channel_priority: 'strict'
+        cache-downloads: true
+    - name: Install snakePipes
+      run: |
+        micromamba run -n snakePipes_CI python -m pip install . --no-deps --ignore-installed -vvv
+    - name: pytest
+      run: |
+        micromamba activate snakePipes_CI
+        snakePipes config --tempDir /tmp --condaEnvDir ./
+        snakePipes createEnvs --only CONDA_SHARED_ENV CONDA_RNASEQ_ENV
+        pytest --verbosity=2 -rP tests/test_mRNA.py
+  CI_jobcounts:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+    - uses: mamba-org/setup-micromamba@main
+      with:
+        environment-file: .github/snakePipesEnvCI.yml
+        condarc: |
+          channels:
+            - conda-forge
+            - bioconda
+            - defaults
+          channel_priority: 'strict'
+        cache-downloads: true
+    - name: Install snakePipes
+      run: |
+        micromamba run -n snakePipes_CI python -m pip install . --no-deps --ignore-installed -vvv
+    - name: pytest
+      run: |
+        micromamba activate snakePipes_CI
+        snakePipes config --tempDir /tmp --condaEnvDir ./
+        pytest --verbosity=2 -rP tests/test_jobcounts.py

.gitignore

@@ -30,3 +30,7 @@ snakePipes.egg-info
 
 # misc
 .vscode/
+
+# tests
+*fa
+*gtf

tests/data/cluster_config.yaml

@@ -0,0 +1 @@
+snakemake_cluster_cmd: ''

tests/test_mRNA.py

@@ -0,0 +1,87 @@
+from pathlib import Path
+import subprocess as sp
+from ruamel.yaml import YAML
+import shutil
+import gzip
+import pytest
+
+def extract_gz(i, o):
+  '''
+  extracts i into file o
+  '''
+  with gzip.open(i, 'rb') as f:
+    with open(o, 'wb') as of:
+      shutil.copyfileobj(f, of)
+
+def createTestData(fp):
+    '''
+    Sets up fasta, gtf and organism yaml in a factory (fp)
+    '''
+    # uncompress fna into tmp factory
+    fnagz = Path('tests') / 'data' / 'genomes' / 'genome_chr17.fa.gz'
+    faout = fp / 'genome.fa'
+    extract_gz(fnagz, faout)
+    # uncompress gtf into tmp factory
+    gtfgz = Path('tests') / 'data' / 'genomes' / 'genes_chr17.gtf.gz'
+    gtfout = fp / 'genes.gtf'
+    extract_gz(gtfgz, gtfout)
+    # Path to STAR index (created in action)
+    STARpath = Path('tests') / 'data' / 'mRNA_STAR'
+    STARpath = STARpath.resolve()
+
+    orgyaml = {
+      "genome_size": 94987271 , #we can also extract genome size from STARindex output
+      "genome_fasta": faout.as_posix(),
+      "star_index": STARpath.as_posix(),
+      "genes_gtf" : gtfout.as_posix(),
+      "extended_coding_regions_gtf" : "",
+      "blacklist_bed": "",
+      "ignoreForNormalization": "" 
+    }
+    # set up yaml
+    yaml = YAML()
+    yaml.boolean_representation = ['False', 'True']
+    with open(fp / 'org.yaml', 'w') as of:
+        yaml.dump(orgyaml, of)
+
+@pytest.fixture(scope='session')
+def ifs(tmp_path_factory):
+  fp = tmp_path_factory.mktemp("data")
+  createTestData(fp)
+  return fp
+
+class TestmRNAseq:
+    def test_mrna(self, ifs):
+      org = ifs / 'org.yaml'
+      clusterconfig = Path('tests') / 'data' / 'cluster_config.yaml'
+      sp.run(
+        [
+          'mRNA-seq',
+          '-i',
+          Path('tests') / 'data' / 'mRNA_mIFNB',
+          '-o',
+          'test_mrna',
+          '--clusterConfig',
+          clusterconfig,
+          org
+        ]
+      )
+      assert Path('test_mrna/mRNA-seq_snakePipes.done').is_file() == True
+
+    def test_mrna4(self, ifs):
+      org = ifs / 'org.yaml'
+      clusterconfig = Path('tests') / 'data' / 'cluster_config.yaml'
+      sp.run(
+        [
+          'mRNA-seq',
+          '-i',
+          Path('tests') / 'data' / 'mRNA_BcellPancreas',
+          '-o',
+          'test_mrna_4sample',
+          '--clusterConfig',
+          clusterconfig,
+          org
+        ]
+      )
+      assert Path('test_mrna_4sample/mRNA-seq_snakePipes.done').is_file() == True
+