Merge pull request #1041 from maxplanck-ie/develop

3.0.0

.ci_stuff/test_ignore_patterns.txt

@@ -0,0 +1,10 @@
+run involves checkpoint jobs
+conda installation
+Conda environment
+Using profile
+Execute
+sanitize_local_storage_copies
+tracemalloc
+wildcard_constraints
+SyntaxWarning: invalid escape sequence
+Would remove temporary

.github/conda_ci.yml

@@ -0,0 +1,6 @@
+name: conda_ci
+dependencies:
+  - python=3.11
+  - anaconda-client
+  - conda-build
+  - conda-verify

.github/condarc.yml

@@ -0,0 +1,6 @@
+channels:
+  - conda-forge
+  - bioconda
+  - defaults
+channel_priority: 'strict'
+show_channel_urls: True

.github/workflows/conda-build.yml

@@ -0,0 +1,19 @@
+name: conda-build
+
+on: [pull_request, push]
+
+jobs:
+  condaBuild:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: conda-incubator/setup-miniconda@v3
+        with:
+          environment-file: .github/conda_ci.yml
+          condarc-file: .github/condarc.yml
+      - name: buildSnakePipes
+        uses: uibcdf/action-build-and-upload-conda-packages@v1.3.0
+        with:
+          meta_yaml_dir: conda-recipe
+          python-version: 3.11
+          upload: false

.github/workflows/linux.yml

@@ -8,78 +8,73 @@ defaults:
 
 
 jobs:
+  pip:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ['3.11', '3.12']
+        optdeps: [".", ".[actions]", ".[docs]"]
+    steps:
+    - uses: actions/checkout@v4
+    - uses: actions/setup-python@v5
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: pip
+      run: |
+        pip install ${{ matrix.optdeps }}
   docs:
+    needs: pip
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v3
-    - uses: mamba-org/setup-micromamba@main
+    - uses: actions/checkout@v4
+    - uses: actions/setup-python@v5
       with:
-        environment-file: .github/snakePipesEnvCI.yml
-        condarc: |
-          channels:
-            - conda-forge
-            - bioconda
-            - defaults
-          channel_priority: 'strict'
-        cache-downloads: true
+        python-version: 3.11
     - name: Install snakePipes
       run: |
-        micromamba run -n snakePipes_CI python -m pip install . --no-deps --ignore-installed -vvv
+        pip install .[docs]
     - name: docs
       run: |
-        micromamba activate snakePipes_CI
         snakePipes config --tempDir /tmp
         cd docs
         make html
-  flake:
+  lint:
+    needs: pip
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v3
-    - uses: mamba-org/setup-micromamba@main
+    - uses: actions/checkout@v4
+    - uses: actions/setup-python@v5
       with:
-        environment-file: .github/snakePipesEnvCI.yml
-        condarc: |
-          channels:
-            - conda-forge
-            - bioconda
-            - defaults
-          channel_priority: 'strict'
-        cache-downloads: true
+        python-version: 3.11
     - name: Install snakePipes
       run: |
-        micromamba run -n snakePipes_CI python -m pip install . --no-deps --ignore-installed -vvv
-    - name: flake
+        pip install .[actions]
+    - name: ruff
       run: |
-        micromamba activate snakePipes_CI
-        snakePipes config --tempDir /tmp
-        flake8 --ignore=E501,E722,E402 --exclude tests,docs/conf.py,build/lib/snakePipes/shared/tools/three_prime_seq,snakePipes/shared/tools/three_prime_seq .
+        ruff check .
   CI:
+    needs: pip
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v3
-    - uses: mamba-org/setup-micromamba@main
+    - uses: actions/checkout@v4
+    - uses: conda-incubator/setup-miniconda@v3
       with:
-        environment-file: .github/snakePipesEnvCI.yml
-        condarc: |
-          channels:
-            - conda-forge
-            - bioconda
-            - defaults
-          channel_priority: 'strict'
-        cache-downloads: true
+        environment-file: .github/conda_ci.yml
+        activate-environment: conda_ci
+        condarc-file: .github/condarc.yml
     - name: Install snakePipes
       run: |
-        micromamba run -n snakePipes_CI python -m pip install . --no-deps --ignore-installed -vvv
+        pip install .
+    - name: print_conda_env_export
+      run: |
+        conda env export 
+    - name: print_pip_list
+      run: |
+        pip list 
     - name: CI
       run: |
-        micromamba activate snakePipes_CI
         snakePipes config --tempDir /tmp
         ./.ci_stuff/test_dag.sh
-    - name: pytest
-      run: |
-        micromamba activate snakePipes_CI
-        snakePipes config --tempDir /tmp
-        pytest --verbosity=2
   createEnvs:
     needs: CI
     strategy:
@@ -109,22 +104,16 @@ jobs:
         ]
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v3
-    - uses: mamba-org/setup-micromamba@main
+    - uses: actions/checkout@v4
+    - uses: conda-incubator/setup-miniconda@v3
       with:
-        environment-file: .github/snakePipesEnvCI.yml
-        condarc: |
-          channels:
-            - conda-forge
-            - bioconda
-            - defaults
-          channel_priority: 'strict'
-        cache-downloads: true
+        environment-file: .github/conda_ci.yml
+        activate-environment: conda_ci
+        condarc-file: .github/condarc.yml
     - name: install snakePipes
       run: |
-        micromamba run -n snakePipes_CI python -m pip install . --no-deps --ignore-installed -vvv
+        pip install .
     - name: createEnvs
       run: |
-        micromamba activate snakePipes_CI
         snakePipes config --tempDir /tmp
-        snakePipes createEnvs --autodetectCondaEnvDir --only ${{matrix.envs}}
+        snakePipes createEnvs --only ${{matrix.envs}}

.github/workflows/osx.yml

@@ -35,22 +35,17 @@ jobs:
         ]
     runs-on: macos-latest
     steps:
-      - uses: actions/checkout@v3
-      - uses: mamba-org/setup-micromamba@main
+      - uses: actions/checkout@v4
+      - uses: conda-incubator/setup-miniconda@v3
         with:
-          environment-file: .github/snakePipesEnvCI.yml
-          condarc: |
-            channels:
-              - conda-forge
-              - bioconda
-              - defaults
-            channel_priority: 'strict'
-          cache-downloads: true
+          environment-file: .github/conda_ci.yml
+          activate-environment: conda_ci
+          condarc-file: .github/condarc.yml
       - name: install snakePipes_OSX
         run: |
-          micromamba run -n snakePipes_CI python -m pip install . --no-deps --ignore-installed -vvv
+          pip install .
       - name: createEnvsOSX
         run: |
-          micromamba activate snakePipes_CI
-          snakePipes createEnvs --autodetectCondaEnvDir --only ${{matrix.envs}}
+          conda config --add subdirs osx-64
+          snakePipes createEnvs --only ${{matrix.envs}}
      

.github/workflows/pytest.yml

@@ -0,0 +1,52 @@
+name: pytest
+
+on: [pull_request, push]
+
+defaults:
+  run:
+      shell: bash -l {0}
+
+jobs:
+  test_mRNA:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+    - uses: conda-incubator/setup-miniconda@v3
+      with:
+        environment-file: snakePipes/shared/rules/envs/rna_seq.yaml
+        activate-environment: snakepipes_RNAseq_environment_3.0
+        condarc-file: .github/condarc.yml
+    - name: create_starix
+      run: |
+        gunzip -c tests/data/genomes/genome_chr17.fa.gz > genome_chr17.fa
+        gunzip -c tests/data/genomes/genes_chr17.gtf.gz > genes_chr17.gtf
+        STAR --runThreadN 4 --runMode genomeGenerate --genomeDir tests/data/mRNA_STAR --genomeFastaFiles genome_chr17.fa --sjdbGTFfile genes_chr17.gtf --sjdbOverhang 100 --genomeSAindexNbases 12
+    - uses: conda-incubator/setup-miniconda@v3
+      with:
+        environment-file: .github/conda_ci.yml
+        activate-environment: conda_ci
+        condarc-file: .github/condarc.yml
+    - name: Install snakePipes
+      run: |
+        pip install .[actions]
+    - name: pytest
+      run: |
+        snakePipes config --tempDir /tmp --condaEnvDir ./
+        snakePipes createEnvs --only CONDA_SHARED_ENV CONDA_RNASEQ_ENV
+        pytest --verbosity=2 -rP tests/test_mRNA.py
+  CI_jobcounts:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+    - uses: conda-incubator/setup-miniconda@v3
+      with:
+        environment-file: .github/conda_ci.yml
+        activate-environment: conda_ci
+        condarc-file: .github/condarc.yml
+    - name: Install snakePipes
+      run: |
+        pip install .[actions]
+    - name: pytest
+      run: |
+        snakePipes config --tempDir /tmp --condaEnvDir ./
+        pytest -n 4 --verbosity=2 -rP tests/test_jobcounts.py

.gitignore

@@ -30,3 +30,7 @@ snakePipes.egg-info
 
 # misc
 .vscode/
+
+# tests
+*fa
+*gtf

.readthedocs.yaml

@@ -3,7 +3,7 @@ version: 2
 build:
   os: ubuntu-22.04
   tools:
-    python: "3.7"
+    python: "3.12"
 
 sphinx:
   configuration: docs/conf.py
@@ -12,4 +12,5 @@ python:
   install:
   - method: pip
     path: .
-  - requirements: docs/requirements.txt
+    extra_requirements: 
+      - docs