diff --git a/.ci_stuff/test_dag.sh b/.ci_stuff/test_dag.sh index f0cdbf636..01708f714 100755 --- a/.ci_stuff/test_dag.sh +++ b/.ci_stuff/test_dag.sh @@ -194,11 +194,11 @@ WC=`DNAmapping -i PE_input -o output .ci_stuff/organism.yaml --snakemakeOptions if [ ${PIPESTATUS[0]} -ne 0 ] || [ $WC -ne 1388 ]; then exit 1 ; fi #allelic WC=`DNAmapping -m allelic-mapping -i PE_input -o output --snakemakeOptions " --dryrun --conda-prefix /tmp" --VCFfile allelic_input/file.vcf.gz --strains strain1,strain2 .ci_stuff/organism.yaml | tee >(cat 1>&2) | grep -v -f .ci_stuff/test_ignore_patterns.txt | sed '/^\s*$/d' | sed '/^host:/d' | wc -l ` -if [ ${PIPESTATUS[0]} -ne 0 ] || [ $WC -ne 2237 ]; then exit 1 ; fi +if [ ${PIPESTATUS[0]} -ne 0 ] || [ $WC -ne 2238 ]; then exit 1 ; fi WC=`DNAmapping -m allelic-mapping -i PE_input -o output --snakemakeOptions " --dryrun --conda-prefix /tmp" --SNPfile allelic_input/snpfile.txt --NMaskedIndex allelic_input/Ngenome .ci_stuff/organism.yaml | tee >(cat 1>&2) | grep -v -f .ci_stuff/test_ignore_patterns.txt | sed '/^\s*$/d' | sed '/^host:/d' | wc -l ` if [ ${PIPESTATUS[0]} -ne 0 ] || [ $WC -ne 2218 ]; then exit 1 ; fi WC=`DNAmapping -m allelic-mapping -i PE_input -o output --snakemakeOptions " --dryrun --conda-prefix /tmp" --VCFfile allelic_input/file.vcf.gz --strains strain1 .ci_stuff/organism.yaml | tee >(cat 1>&2) | grep -v -f .ci_stuff/test_ignore_patterns.txt | sed '/^\s*$/d' | sed '/^host:/d' | wc -l ` -if [ ${PIPESTATUS[0]} -ne 0 ] || [ $WC -ne 2237 ]; then exit 1 ; fi +if [ ${PIPESTATUS[0]} -ne 0 ] || [ $WC -ne 2238 ]; then exit 1 ; fi # ChIPseq WC=`ChIPseq -d BAM_input --snakemakeOptions " --dryrun --conda-prefix /tmp" .ci_stuff/organism.yaml .ci_stuff/ChIP.sample_config.yaml | tee >(cat 1>&2) | grep -v -f .ci_stuff/test_ignore_patterns.txt | sed '/^\s*$/d' | sed '/^host:/d' | wc -l ` @@ -289,54 +289,54 @@ if [ ${PIPESTATUS[0]} -ne 0 ] || [ $WC -ne 1081 ]; then exit 1 ; fi # mRNAseq WC=`mRNAseq -i PE_input -o output --snakemakeOptions " --dryrun --conda-prefix /tmp" .ci_stuff/organism.yaml | tee >(cat 1>&2) | grep -v -f .ci_stuff/test_ignore_patterns.txt | sed '/^\s*$/d' | sed '/^host:/d' | wc -l ` -if [ ${PIPESTATUS[0]} -ne 0 ] || [ $WC -ne 1536 ]; then exit 1 ; fi +if [ ${PIPESTATUS[0]} -ne 0 ] || [ $WC -ne 1628 ]; then exit 1 ; fi WC=`mRNAseq -i PE_input -o output --sampleSheet .ci_stuff/test_sampleSheet.tsv --snakemakeOptions " --dryrun --conda-prefix /tmp" .ci_stuff/organism.yaml | tee >(cat 1>&2) | grep -v -f .ci_stuff/test_ignore_patterns.txt | sed '/^\s*$/d' | sed '/^host:/d' | wc -l ` -if [ ${PIPESTATUS[0]} -ne 0 ] || [ $WC -ne 1547 ]; then exit 1 ; fi +if [ ${PIPESTATUS[0]} -ne 0 ] || [ $WC -ne 1639 ]; then exit 1 ; fi WC=`mRNAseq -i PE_input -o output --rMats --sampleSheet .ci_stuff/test_sampleSheet.tsv --snakemakeOptions " --dryrun --conda-prefix /tmp" .ci_stuff/organism.yaml | tee >(cat 1>&2) | grep -v -f .ci_stuff/test_ignore_patterns.txt | sed '/^\s*$/d' | sed '/^host:/d' | wc -l ` -if [ ${PIPESTATUS[0]} -ne 0 ] || [ $WC -ne 1566 ]; then exit 1 ; fi +if [ ${PIPESTATUS[0]} -ne 0 ] || [ $WC -ne 1658 ]; then exit 1 ; fi WC=`mRNAseq -i PE_input -o output --sampleSheet .ci_stuff/test_sampleSheet.tsv --snakemakeOptions " --dryrun --conda-prefix /tmp" -m "alignment" .ci_stuff/organism.yaml | tee >(cat 1>&2) | grep -v -f .ci_stuff/test_ignore_patterns.txt | sed '/^\s*$/d' | sed '/^host:/d' | wc -l ` -if [ ${PIPESTATUS[0]} -ne 0 ] || [ $WC -ne 1106 ]; then exit 1 ; fi +if [ ${PIPESTATUS[0]} -ne 0 ] || [ $WC -ne 1198 ]; then exit 1 ; fi WC=`mRNAseq -i PE_input -o output --sampleSheet .ci_stuff/test_sampleSheet.tsv --snakemakeOptions " --dryrun --conda-prefix /tmp" -m "alignment,deepTools_qc" --trim .ci_stuff/organism.yaml | tee >(cat 1>&2) | grep -v -f .ci_stuff/test_ignore_patterns.txt | sed '/^\s*$/d' | sed '/^host:/d' | wc -l ` -if [ ${PIPESTATUS[0]} -ne 0 ] || [ $WC -ne 1639 ]; then exit 1 ; fi +if [ ${PIPESTATUS[0]} -ne 0 ] || [ $WC -ne 1731 ]; then exit 1 ; fi WC=`mRNAseq -i PE_input -o output --sampleSheet .ci_stuff/test_sampleSheet.tsv --snakemakeOptions " --dryrun --conda-prefix /tmp" -m "alignment-free,deepTools_qc" .ci_stuff/organism.yaml | tee >(cat 1>&2) | grep -v -f .ci_stuff/test_ignore_patterns.txt | sed '/^\s*$/d' | sed '/^host:/d' | wc -l ` -if [ ${PIPESTATUS[0]} -ne 0 ] || [ $WC -ne 1736 ]; then exit 1 ; fi +if [ ${PIPESTATUS[0]} -ne 0 ] || [ $WC -ne 1828 ]; then exit 1 ; fi WC=`mRNAseq -i PE_input -o output --sampleSheet .ci_stuff/test_sampleSheet.tsv --snakemakeOptions " --dryrun --conda-prefix /tmp" -m "alignment,deepTools_qc" --bcExtract --trim .ci_stuff/organism.yaml | tee >(cat 1>&2) | grep -v -f .ci_stuff/test_ignore_patterns.txt | sed '/^\s*$/d' | sed '/^host:/d' | wc -l ` -if [ ${PIPESTATUS[0]} -ne 0 ] || [ $WC -ne 1565 ]; then exit 1 ; fi +if [ ${PIPESTATUS[0]} -ne 0 ] || [ $WC -ne 1657 ]; then exit 1 ; fi WC=`mRNAseq -i PE_input -o output --sampleSheet .ci_stuff/test_sampleSheet.tsv --snakemakeOptions " --dryrun --conda-prefix /tmp" -m "alignment,deepTools_qc" --bcExtract --UMIDedup --trim .ci_stuff/organism.yaml | tee >(cat 1>&2) | grep -v -f .ci_stuff/test_ignore_patterns.txt | sed '/^\s*$/d' | sed '/^host:/d' | wc -l ` -if [ ${PIPESTATUS[0]} -ne 0 ] || [ $WC -ne 1639 ]; then exit 1 ; fi +if [ ${PIPESTATUS[0]} -ne 0 ] || [ $WC -ne 1731 ]; then exit 1 ; fi WC=`mRNAseq -i SE_input -o output --sampleSheet .ci_stuff/test_sampleSheet.tsv --snakemakeOptions " --dryrun --conda-prefix /tmp" .ci_stuff/organism.yaml | tee >(cat 1>&2) | grep -v -f .ci_stuff/test_ignore_patterns.txt | sed '/^\s*$/d' | sed '/^host:/d' | wc -l ` -if [ ${PIPESTATUS[0]} -ne 0 ] || [ $WC -ne 1389 ]; then exit 1 ; fi +if [ ${PIPESTATUS[0]} -ne 0 ] || [ $WC -ne 1481 ]; then exit 1 ; fi WC=`mRNAseq -i SE_input -o output --sampleSheet .ci_stuff/test_sampleSheet.tsv --snakemakeOptions " --dryrun --conda-prefix /tmp" -m "alignment" .ci_stuff/organism.yaml | tee >(cat 1>&2) | grep -v -f .ci_stuff/test_ignore_patterns.txt | sed '/^\s*$/d' | sed '/^host:/d' | wc -l ` -if [ ${PIPESTATUS[0]} -ne 0 ] || [ $WC -ne 958 ]; then exit 1 ; fi +if [ ${PIPESTATUS[0]} -ne 0 ] || [ $WC -ne 1050 ]; then exit 1 ; fi WC=`mRNAseq -i SE_input -o output --sampleSheet .ci_stuff/test_sampleSheet.tsv --snakemakeOptions " --dryrun --conda-prefix /tmp" -m "alignment,deepTools_qc" --trim .ci_stuff/organism.yaml | tee >(cat 1>&2) | grep -v -f .ci_stuff/test_ignore_patterns.txt | sed '/^\s*$/d' | sed '/^host:/d' | wc -l ` -if [ ${PIPESTATUS[0]} -ne 0 ] || [ $WC -ne 1481 ]; then exit 1 ; fi +if [ ${PIPESTATUS[0]} -ne 0 ] || [ $WC -ne 1573 ]; then exit 1 ; fi WC=`mRNAseq -i SE_input -o output --sampleSheet .ci_stuff/test_sampleSheet.tsv --snakemakeOptions " --dryrun --conda-prefix /tmp" -m "alignment-free,deepTools_qc" .ci_stuff/organism.yaml | tee >(cat 1>&2) | grep -v -f .ci_stuff/test_ignore_patterns.txt | sed '/^\s*$/d' | sed '/^host:/d' | wc -l ` -if [ ${PIPESTATUS[0]} -ne 0 ] || [ $WC -ne 1578 ]; then exit 1 ; fi +if [ ${PIPESTATUS[0]} -ne 0 ] || [ $WC -ne 1670 ]; then exit 1 ; fi WC=`mRNAseq -i SE_input -o output --sampleSheet .ci_stuff/test_sampleSheet.tsv --snakemakeOptions " --dryrun --conda-prefix /tmp" --trim --fastqc .ci_stuff/organism.yaml | tee >(cat 1>&2) | grep -v -f .ci_stuff/test_ignore_patterns.txt | sed '/^\s*$/d' | sed '/^host:/d' | wc -l ` -if [ ${PIPESTATUS[0]} -ne 0 ] || [ $WC -ne 1665 ]; then exit 1 ; fi +if [ ${PIPESTATUS[0]} -ne 0 ] || [ $WC -ne 1757 ]; then exit 1 ; fi WC=`mRNAseq -i BAM_input/filtered_bam -o output --sampleSheet .ci_stuff/test_sampleSheet.tsv --snakemakeOptions " --dryrun --conda-prefix /tmp" --fromBAM .ci_stuff/organism.yaml | tee >(cat 1>&2) | grep -v -f .ci_stuff/test_ignore_patterns.txt | sed '/^\s*$/d' | sed '/^host:/d' | wc -l ` if [ ${PIPESTATUS[0]} -ne 0 ] || [ $WC -ne 1058 ]; then exit 1 ; fi #multiple comparison groups WC=`mRNAseq --mode alignment,alignment-free -i PE_input -o output --rMats --sampleSheet .ci_stuff/test_sampleSheet_multiComp.tsv --snakemakeOptions " --dryrun --conda-prefix /tmp" .ci_stuff/organism.yaml | tee >(cat 1>&2) | grep -v -f .ci_stuff/test_ignore_patterns.txt | sed '/^\s*$/d' | sed '/^host:/d' | wc -l ` -if [ ${PIPESTATUS[0]} -ne 0 ] || [ $WC -ne 1476 ]; then exit 1 ; fi +if [ ${PIPESTATUS[0]} -ne 0 ] || [ $WC -ne 1568 ]; then exit 1 ; fi # three prime sequencing WC=`mRNAseq -i PE_input -o output --mode three-prime-seq --sampleSheet .ci_stuff/test_sampleSheet.tsv --snakemakeOptions " --dryrun --conda-prefix /tmp" .ci_stuff/organism.yaml | tee >(cat 1>&2) | grep -v -f .ci_stuff/test_ignore_patterns.txt | sed '/^\s*$/d' | sed '/^host:/d' | wc -l ` -if [ ${PIPESTATUS[0]} -ne 0 ] || [ $WC -ne 1668 ]; then exit 1 ; fi +if [ ${PIPESTATUS[0]} -ne 0 ] || [ $WC -ne 1686 ]; then exit 1 ; fi WC=`mRNAseq -i PE_input -o output --mode three-prime-seq,deepTools_qc --sampleSheet .ci_stuff/test_sampleSheet.tsv --snakemakeOptions " --dryrun --conda-prefix /tmp" .ci_stuff/organism.yaml | tee >(cat 1>&2) | grep -v -f .ci_stuff/test_ignore_patterns.txt | sed '/^\s*$/d' | sed '/^host:/d' | wc -l ` -if [ ${PIPESTATUS[0]} -ne 0 ] || [ $WC -ne 2109 ]; then exit 1 ; fi +if [ ${PIPESTATUS[0]} -ne 0 ] || [ $WC -ne 2127 ]; then exit 1 ; fi #allelic WC=`mRNAseq -m allelic-mapping,deepTools_qc -i PE_input -o output --snakemakeOptions " --dryrun --conda-prefix /tmp" --VCFfile allelic_input/file.vcf.gz --strains strain1,strain2 .ci_stuff/organism.yaml | tee >(cat 1>&2) | grep -v -f .ci_stuff/test_ignore_patterns.txt | sed '/^\s*$/d' | sed '/^host:/d' | wc -l ` -if [ ${PIPESTATUS[0]} -ne 0 ] || [ $WC -ne 2304 ]; then exit 1 ; fi +if [ ${PIPESTATUS[0]} -ne 0 ] || [ $WC -ne 2397 ]; then exit 1 ; fi WC=`mRNAseq -m allelic-mapping,deepTools_qc -i allelic_BAM_input/filtered_bam --fromBAM --bamExt '.filtered.bam' -o output --sampleSheet .ci_stuff/test_sampleSheet.tsv --snakemakeOptions " --dryrun --conda-prefix /tmp" --SNPfile allelic_input/snpfile.txt --NMaskedIndex allelic_input/Ngenome .ci_stuff/organism.yaml | tee >(cat 1>&2) | grep -v -f .ci_stuff/test_ignore_patterns.txt | sed '/^\s*$/d' | sed '/^host:/d' | wc -l ` if [ ${PIPESTATUS[0]} -ne 0 ] || [ $WC -ne 1285 ]; then exit 1 ; fi WC=`mRNAseq -m allelic-mapping,deepTools_qc -i PE_input -o output --sampleSheet .ci_stuff/test_sampleSheet.tsv --snakemakeOptions " --dryrun --conda-prefix /tmp" --VCFfile allelic_input/file.vcf.gz --strains strain1,strain2 .ci_stuff/organism.yaml | tee >(cat 1>&2) | grep -v -f .ci_stuff/test_ignore_patterns.txt | sed '/^\s*$/d' | sed '/^host:/d' | wc -l ` -if [ ${PIPESTATUS[0]} -ne 0 ] || [ $WC -ne 2315 ]; then exit 1 ; fi +if [ ${PIPESTATUS[0]} -ne 0 ] || [ $WC -ne 2408 ]; then exit 1 ; fi WC=`mRNAseq -m allelic-mapping,deepTools_qc -i PE_input -o output --sampleSheet .ci_stuff/test_sampleSheet.tsv --snakemakeOptions " --dryrun --conda-prefix /tmp" --SNPfile allelic_input/snpfile.txt --NMaskedIndex allelic_input/Ngenome .ci_stuff/organism.yaml | tee >(cat 1>&2) | grep -v -f .ci_stuff/test_ignore_patterns.txt | sed '/^\s*$/d' | sed '/^host:/d' | wc -l ` -if [ ${PIPESTATUS[0]} -ne 0 ] || [ $WC -ne 2296 ]; then exit 1 ; fi +if [ ${PIPESTATUS[0]} -ne 0 ] || [ $WC -ne 2388 ]; then exit 1 ; fi WC=`mRNAseq -m allelic-mapping,deepTools_qc -i PE_input -o output --sampleSheet .ci_stuff/test_sampleSheet.tsv --snakemakeOptions " --dryrun --conda-prefix /tmp" --VCFfile allelic_input/file.vcf.gz --strains strain1 .ci_stuff/organism.yaml | tee >(cat 1>&2) | grep -v -f .ci_stuff/test_ignore_patterns.txt | sed '/^\s*$/d' | sed '/^host:/d' | wc -l ` -if [ ${PIPESTATUS[0]} -ne 0 ] || [ $WC -ne 2315 ]; then exit 1 ; fi +if [ ${PIPESTATUS[0]} -ne 0 ] || [ $WC -ne 2408 ]; then exit 1 ; fi WC=`mRNAseq -m allelic-mapping,deepTools_qc,alignment-free -i PE_input -o output --sampleSheet .ci_stuff/test_sampleSheet.tsv --snakemakeOptions " --dryrun --conda-prefix /tmp" --VCFfile allelic_input/file.vcf.gz --strains strain1 .ci_stuff/organism.yaml | tee >(cat 1>&2) | grep -v -f .ci_stuff/test_ignore_patterns.txt | sed '/^\s*$/d' | sed '/^host:/d' | wc -l ` -if [ ${PIPESTATUS[0]} -ne 0 ] || [ $WC -ne 3012 ]; then exit 1 ; fi +if [ ${PIPESTATUS[0]} -ne 0 ] || [ $WC -ne 3105 ]; then exit 1 ; fi WC=`mRNAseq -m allelic-counting -i allelic_BAM_input/allelic_bams --fromBAM --bamExt '.sorted.bam' -o output --sampleSheet .ci_stuff/test_sampleSheet.tsv --snakemakeOptions " --dryrun --conda-prefix /tmp" .ci_stuff/organism.yaml | tee >(cat 1>&2) | grep -v -f .ci_stuff/test_ignore_patterns.txt | sed '/^\s*$/d' | sed '/^host:/d' | wc -l ` if [ ${PIPESTATUS[0]} -ne 0 ] || [ $WC -ne 638 ]; then exit 1 ; fi WC=`mRNAseq -m allelic-counting -i allelic_BAM_input/allelic_bams --fromBAM --bamExt '.sorted.bam' -o output --sampleSheet .ci_stuff/test_sampleSheet.tsv --snakemakeOptions " --dryrun --conda-prefix /tmp" .ci_stuff/organism.yaml | tee >(cat 1>&2) | grep -v -f .ci_stuff/test_ignore_patterns.txt | sed '/^\s*$/d' | sed '/^host:/d' | wc -l ` @@ -347,7 +347,7 @@ if [ ${PIPESTATUS[0]} -ne 0 ] || [ $WC -ne 668 ]; then exit 1 ; fi WC=`mRNAseq -m allelic-mapping,deepTools_qc -i allelic_BAM_input/filtered_bam --fromBAM --bamExt '.filtered.bam' -o output --sampleSheet .ci_stuff/test_sampleSheet_multiComp.tsv --snakemakeOptions " --dryrun --conda-prefix /tmp" --SNPfile allelic_input/snpfile.txt --NMaskedIndex allelic_input/Ngenome .ci_stuff/organism.yaml | tee >(cat 1>&2) | grep -v -f .ci_stuff/test_ignore_patterns.txt | sed '/^\s*$/d' | sed '/^host:/d' | wc -l ` if [ ${PIPESTATUS[0]} -ne 0 ] || [ $WC -ne 1315 ]; then exit 1 ; fi WC=`mRNAseq -m allelic-mapping,deepTools_qc,alignment-free -i PE_input -o output --sampleSheet .ci_stuff/test_sampleSheet_multiComp.tsv --snakemakeOptions " --dryrun --conda-prefix /tmp" --VCFfile allelic_input/file.vcf.gz --strains strain1 .ci_stuff/organism.yaml | tee >(cat 1>&2) | grep -v -f .ci_stuff/test_ignore_patterns.txt | sed '/^\s*$/d' | sed '/^host:/d' | wc -l ` -if [ ${PIPESTATUS[0]} -ne 0 ] || [ $WC -ne 3021 ]; then exit 1 ; fi +if [ ${PIPESTATUS[0]} -ne 0 ] || [ $WC -ne 3114 ]; then exit 1 ; fi #ncRNAseq WC=`ncRNAseq -i PE_input -o output --snakemakeOptions " --dryrun --conda-prefix /tmp" .ci_stuff/organism.yaml | tee >(cat 1>&2) | grep -v -f .ci_stuff/test_ignore_patterns.txt | sed '/^\s*$/d' | sed '/^host:/d' | wc -l ` @@ -378,17 +378,17 @@ if [ ${PIPESTATUS[0]} -ne 0 ] || [ $WC -ne 604 ]; then exit 1 ; fi # WGBS WC=`WGBS -i PE_input -o output --snakemakeOptions " --dryrun --conda-prefix /tmp" .ci_stuff/organism.yaml | tee >(cat 1>&2) | grep -v -f .ci_stuff/test_ignore_patterns.txt | sed '/^\s*$/d' | sed '/^host:/d' | wc -l ` -if [ ${PIPESTATUS[0]} -ne 0 ] || [ $WC -ne 1300 ]; then exit 1 ; fi +if [ ${PIPESTATUS[0]} -ne 0 ] || [ $WC -ne 1327 ]; then exit 1 ; fi WC=`WGBS -i PE_input -o output --sampleSheet .ci_stuff/test_sampleSheet.tsv --snakemakeOptions " --dryrun --conda-prefix /tmp" .ci_stuff/organism.yaml | tee >(cat 1>&2) | grep -v -f .ci_stuff/test_ignore_patterns.txt | sed '/^\s*$/d' | sed '/^host:/d' | wc -l ` -if [ ${PIPESTATUS[0]} -ne 0 ] || [ $WC -ne 1343 ]; then exit 1 ; fi +if [ ${PIPESTATUS[0]} -ne 0 ] || [ $WC -ne 1370 ]; then exit 1 ; fi WC=`WGBS -i PE_input -o output --sampleSheet .ci_stuff/test_sampleSheet.tsv --aligner bwameth2 --snakemakeOptions " --dryrun --conda-prefix /tmp" .ci_stuff/organism.yaml | tee >(cat 1>&2) | grep -v -f .ci_stuff/test_ignore_patterns.txt | sed '/^\s*$/d' | sed '/^host:/d' | wc -l ` -if [ ${PIPESTATUS[0]} -ne 0 ] || [ $WC -ne 1343 ]; then exit 1 ; fi +if [ ${PIPESTATUS[0]} -ne 0 ] || [ $WC -ne 1370 ]; then exit 1 ; fi WC=`WGBS -i PE_input -o output --sampleSheet .ci_stuff/test_sampleSheet.tsv --snakemakeOptions " --dryrun --conda-prefix /tmp" --trim --GCbias .ci_stuff/organism.yaml | tee >(cat 1>&2) | grep -v -f .ci_stuff/test_ignore_patterns.txt | sed '/^\s*$/d' | sed '/^host:/d' | wc -l ` -if [ ${PIPESTATUS[0]} -ne 0 ] || [ $WC -ne 1353 ]; then exit 1 ; fi +if [ ${PIPESTATUS[0]} -ne 0 ] || [ $WC -ne 1380 ]; then exit 1 ; fi WC=`WGBS -i BAM_input/filtered_bam -o output --sampleSheet .ci_stuff/test_sampleSheet.tsv --fromBAM --snakemakeOptions " --dryrun --conda-prefix /tmp" --GCbias .ci_stuff/organism.yaml | tee >(cat 1>&2) | grep -v -f .ci_stuff/test_ignore_patterns.txt | sed '/^\s*$/d' | sed '/^host:/d' | wc -l ` -if [ ${PIPESTATUS[0]} -ne 0 ] || [ $WC -ne 974 ]; then exit 1 ; fi +if [ ${PIPESTATUS[0]} -ne 0 ] || [ $WC -ne 817 ]; then exit 1 ; fi WC=`WGBS -i BAM_input/filtered_bam -o output --sampleSheet .ci_stuff/test_sampleSheet.tsv --fromBAM --fastqc --snakemakeOptions " --dryrun --conda-prefix /tmp" --GCbias .ci_stuff/organism.yaml | tee >(cat 1>&2) | grep -v -f .ci_stuff/test_ignore_patterns.txt | sed '/^\s*$/d' | sed '/^host:/d' | wc -l ` -if [ ${PIPESTATUS[0]} -ne 0 ] || [ $WC -ne 974 ]; then exit 1 ; fi +if [ ${PIPESTATUS[0]} -ne 0 ] || [ $WC -ne 817 ]; then exit 1 ; fi WC=`WGBS -i BAM_input/filtered_bam -o output --sampleSheet .ci_stuff/test_sampleSheet.tsv --fromBAM --skipBamQC --snakemakeOptions " --dryrun --conda-prefix /tmp" --GCbias .ci_stuff/organism.yaml | tee >(cat 1>&2) | grep -v -f .ci_stuff/test_ignore_patterns.txt | sed '/^\s*$/d' | sed '/^host:/d' | wc -l ` if [ ${PIPESTATUS[0]} -ne 0 ] || [ $WC -ne 530 ]; then exit 1 ; fi diff --git a/snakePipes/shared/profiles/local/config.yaml b/snakePipes/shared/profiles/local/config.yaml index 196c513b2..f02ef804d 100644 --- a/snakePipes/shared/profiles/local/config.yaml +++ b/snakePipes/shared/profiles/local/config.yaml @@ -205,4 +205,4 @@ set-resources: velo_to_sce: mem: 30G velocyto: - mem: 20G \ No newline at end of file + mem: 20G diff --git a/snakePipes/shared/rscripts/DB_functions.R b/snakePipes/shared/rscripts/DB_functions.R index 7cf6926cd..4e0abeb9f 100644 --- a/snakePipes/shared/rscripts/DB_functions.R +++ b/snakePipes/shared/rscripts/DB_functions.R @@ -343,6 +343,8 @@ writeOutput_chip <- function(chipResultObject, outfile_prefix, fdrcutoff,lfccuto full_res[,2]<-full_res[,2]-1 full_res[,2]<-format(full_res[,2], scientific = FALSE,trim=TRUE) full_res[,3]<-format(full_res[,3], scientific = FALSE,trim=TRUE) + #write full results + write.table(full_res,file="Full.results.bed",row.names=FALSE,col.names=FALSE,sep="\t",quote=FALSE) ##filter full result for FDR and LFC and write to output full_res.filt<-subset(full_res,(FDR<=fdrcutoff)&(abs(best.logFC)>=lfccutoff)) if(nrow(full_res.filt)>0){ diff --git a/snakePipes/shared/rscripts/sleuth.R b/snakePipes/shared/rscripts/sleuth.R index 23f4c3bfa..1225c70b2 100644 --- a/snakePipes/shared/rscripts/sleuth.R +++ b/snakePipes/shared/rscripts/sleuth.R @@ -24,17 +24,43 @@ sample_info = read.table(sample_info_file, header=T)#[,1:2] cnames.sub<-unique(colnames(sample_info)[2:which(colnames(sample_info) %in% "condition")]) d<-as.formula(noquote(paste0("~",paste(cnames.sub,collapse="+")))) colnames(sample_info)[colnames(sample_info) %in% "name"] ="sample" + +#check if sample names are pure numbers +numbers_only <- function(x) !grepl("\\D", x) +if(any(numbers_only(sample_info$sample))){sample_info$sample[numbers_only(sample_info$sample)]<-paste0("X",sample_info$sample[numbers_only(sample_info$sample)])} +rownames(sample_info)<-sample_info$sample + print(sample_info) sample_info$sample -sample_id = list.dirs(file.path(indir), recursive=F, full.names=F) -sample_id = sort(sample_id[grep('[^benchmark][^SalmonIndex]', sample_id, invert=F)]) + +###MODIFY THIS PART +#sample_id = list.dirs(file.path(indir), recursive=F, full.names=F) +#sample_id = sort(sample_id[grep('[^benchmark][^SalmonIndex]', sample_id, invert=F)]) +#if(any(numbers_only(sample_id))){sample_id[numbers_only(sample_id)]<-paste0("X",sample_id[numbers_only(sample_id)])} #sample_id = intersect(sample_info$sample, sample_id) # get only those sample that are defined in the sampleInfo! -sample_id<-sample_id[match(sample_info$sample,sample_id)] +#sample_id<-sample_id[match(sample_info$sample,sample_id)] +#print(sample_id) +#if(any(is.na(sample_id))){stop("Sample names from sample sheet and from Salmon output are not matching each other.")} + +#salmon_dirs = sapply(sample_id, function(id) file.path(indir, id)) +#print(salmon_dirs) +###### +sample_id<- dir(file.path(indir),recursive=FALSE,full.names=TRUE) +sample_id<-sample_id[grep("*quant.sf",sample_id)] +sample_id<-sub(".quant.sf","",sample_id) +names(sample_id)<-basename(sample_id) +print(sample_id) + +if(any(numbers_only(names(sample_id)))){names(sample_id)[numbers_only(names(sample_id))]<-paste0("X",names(sample_id)[numbers_only(names(sample_id))])} + +sample_id<-sample_id[match(sample_info$sample,names(sample_id))] print(sample_id) +if(any(is.na(names(sample_id)))){stop("Sample names from sample sheet and from Salmon output are not matching each other.")} -salmon_dirs = sapply(sample_id, function(id) file.path(indir, id)) +salmon_dirs = sample_id print(salmon_dirs) +########## s2c = mutate(sample_info, path=salmon_dirs) ## reorder conditions (for Wald test later on: order of comparison important for fold change) diff --git a/snakePipes/shared/rules/CSAW.multiComp.snakefile b/snakePipes/shared/rules/CSAW.multiComp.snakefile index 1d9945a9a..6600d7e82 100644 --- a/snakePipes/shared/rules/CSAW.multiComp.snakefile +++ b/snakePipes/shared/rules/CSAW.multiComp.snakefile @@ -87,7 +87,8 @@ rule CSAW: output: "{}/CSAW.session_info.txt".format(get_outdir(peakCaller,os.path.splitext(os.path.basename(str(sampleSheet)))[0]+".{compGroup}.tsv")), "{}/DiffBinding_analysis.Rdata".format(get_outdir(peakCaller,os.path.splitext(os.path.basename(str(sampleSheet)))[0]+".{compGroup}.tsv")), - expand("{}".format(get_outdir(peakCaller,os.path.splitext(os.path.basename(str(sampleSheet)))[0]+".{{compGroup}}.tsv")) + "/Filtered.results.{change_dir}.bed", change_dir=change_direction) + expand("{}".format(get_outdir(peakCaller,os.path.splitext(os.path.basename(str(sampleSheet)))[0]+".{{compGroup}}.tsv")) + "/Filtered.results.{change_dir}.bed", change_dir=change_direction), + "{}".format(get_outdir(peakCaller,os.path.splitext(os.path.basename(str(sampleSheet)))[0]+".{compGroup}.tsv")) + "/Full.results.bed" benchmark: "{}/.benchmark/CSAW.benchmark".format(get_outdir(peakCaller,os.path.splitext(os.path.basename(str(sampleSheet)))[0]+".{compGroup}.tsv")) params: diff --git a/snakePipes/shared/rules/CSAW.singleComp.snakefile b/snakePipes/shared/rules/CSAW.singleComp.snakefile index 52f38656c..2d935df5c 100644 --- a/snakePipes/shared/rules/CSAW.singleComp.snakefile +++ b/snakePipes/shared/rules/CSAW.singleComp.snakefile @@ -71,7 +71,8 @@ rule CSAW: output: "CSAW_{}_{}/CSAW.session_info.txt".format(peakCaller, sample_name), "CSAW_{}_{}/DiffBinding_analysis.Rdata".format(peakCaller, sample_name), - expand("CSAW_{}_{}".format(peakCaller, sample_name) + "/Filtered.results.{change_dir}.bed", change_dir=change_direction) + expand("CSAW_{}_{}".format(peakCaller, sample_name) + "/Filtered.results.{change_dir}.bed", change_dir=change_direction), + "CSAW_{}_{}".format(peakCaller, sample_name) + "/Full.results.bed" benchmark: "CSAW_{}_{}/.benchmark/CSAW.benchmark".format(peakCaller, sample_name) params: diff --git a/snakePipes/shared/rules/DESeq2.singleComp.snakefile b/snakePipes/shared/rules/DESeq2.singleComp.snakefile index 93090fccb..5e918af98 100644 --- a/snakePipes/shared/rules/DESeq2.singleComp.snakefile +++ b/snakePipes/shared/rules/DESeq2.singleComp.snakefile @@ -42,6 +42,7 @@ rule DESeq2_Salmon_basic: "{}/.benchmark/DESeq2.Salmon.benchmark".format(get_outdir("DESeq2_Salmon",sampleSheet)) params: script=os.path.join(maindir, "shared", "rscripts", "DESeq2.R"), + sampleSheet = lambda wildcards,input: input.sampleSheet, outdir = get_outdir("DESeq2_Salmon",sampleSheet), fdr = fdr, importfunc = os.path.join(maindir, "shared", "rscripts", "DE_functions.R"), @@ -67,6 +68,7 @@ rule DESeq2_Salmon_allelic: "{}/.benchmark/DESeq2.SalmonAllelic.benchmark".format(get_outdir("DESeq2_SalmonAllelic",sampleSheet)) params: script=os.path.join(maindir, "shared", "rscripts", "DESeq2.R"), + sampleSheet = lambda wildcards,input: input.sampleSheet, outdir = get_outdir("DESeq2_SalmonAllelic",sampleSheet), fdr = fdr, importfunc = os.path.join(maindir, "shared", "rscripts", "DE_functions.R"), diff --git a/snakePipes/shared/rules/LinkBam.snakefile b/snakePipes/shared/rules/LinkBam.snakefile index fa3041efe..1098ca485 100644 --- a/snakePipes/shared/rules/LinkBam.snakefile +++ b/snakePipes/shared/rules/LinkBam.snakefile @@ -29,42 +29,43 @@ else: input: indir + "/{sample}" + bamExt output: - aligner + "/{sample}.unsorted.bam" if pipeline=="ncRNAseq" else aligner + "/{sample}.bam" + aligner + "/{sample}.unsorted.bam" if pipeline=="ncRNAseq" else aligner + "/{sample}.markdup.bam" params: input_bai = indir + "/{sample}" + bamExt + ".bai", - output_bai = aligner + "/{sample}.unsorted.bam.bai" if pipeline=="ncRNAseq" else aligner + "/{sample}.bam.bai" + output_bai = aligner + "/{sample}.unsorted.bam.bai" if pipeline=="ncRNAseq" else aligner + "/{sample}.markdup.bam.bai" run: if os.path.exists(params.input_bai) and not os.path.exists(os.path.join(outdir,params.output_bai)): os.symlink(params.input_bai,os.path.join(outdir,params.output_bai)) if not os.path.exists(os.path.join(outdir,output[0])): os.symlink(os.path.join(outdir,input[0]),os.path.join(outdir,output[0])) + if not pipeline=="ncRNAseq": rule samtools_index_external: input: - aligner + "/{sample}.bam" + aligner + "/{sample}.markdup.bam" output: - aligner + "/{sample}.bam.bai" + aligner + "/{sample}.markdup.bam.bai" conda: CONDA_SHARED_ENV shell: "if [[ ! -f {output[0]} ]]; then samtools index {input[0]}; fi" - if not pipeline=="WGBS" or pipeline=="WGBS" and skipBamQC: - rule link_bam_bai_external: - input: - bam = aligner + "/{sample}.bam", - bai = aligner + "/{sample}.bam.bai" - output: - bam_out = "filtered_bam/{sample}.filtered.bam", - bai_out = "filtered_bam/{sample}.filtered.bam.bai", - shell: """ - ln -s ../{input.bam} {output.bam_out}; - ln -s ../{input.bai} {output.bai_out} + + rule link_bam_bai_external: + input: + bam = aligner + "/{sample}.markdup.bam", + bai = aligner + "/{sample}.markdup.bam.bai" + output: + bam_out = "filtered_bam/{sample}.filtered.bam", + bai_out = "filtered_bam/{sample}.filtered.bam.bai", + shell: """ + ln -s ../{input.bam} {output.bam_out}; + ln -s ../{input.bai} {output.bai_out} """ rule sambamba_flagstat: input: - aligner + "/{sample}.bam" + aligner + "/{sample}.markdup.bam" output: "Sambamba/{sample}.markdup.txt" conda: CONDA_SAMBAMBA_ENV diff --git a/snakePipes/shared/rules/SNPsplit.snakefile b/snakePipes/shared/rules/SNPsplit.snakefile index 13acbb8a8..1c1918ffa 100644 --- a/snakePipes/shared/rules/SNPsplit.snakefile +++ b/snakePipes/shared/rules/SNPsplit.snakefile @@ -22,12 +22,12 @@ elif aligner == "STAR" or aligner == "EXTERNAL_BAM": rule snp_split: input: snp = SNPFile, - bam = aligner+"/{sample}.bam" + bam = aligner+"/{sample}.markdup.bam" output: - targetbam = expand("allelic_bams/{{sample}}.{suffix}.bam", suffix = ['allele_flagged', 'genome1', 'genome2', 'unassigned']), + targetbam = expand("allelic_bams/{{sample}}.markdup.{suffix}.bam", suffix = ['allele_flagged', 'genome1', 'genome2', 'unassigned']), #tempbam = temp(aligner+"/{sample}.sortedByName.bam"), - rep1 = "allelic_bams/{sample}.SNPsplit_report.yaml", - rep2 = "allelic_bams/{sample}.SNPsplit_sort.yaml" + rep1 = "allelic_bams/{sample}.markdup.SNPsplit_report.yaml", + rep2 = "allelic_bams/{sample}.markdup.SNPsplit_sort.yaml" params: pairedEnd = '--paired' if pairedEnd else '', outdir = "allelic_bams" @@ -62,7 +62,7 @@ elif aligner == "STAR" or aligner == "EXTERNAL_BAM": # sort them rule BAMsort_allelic: - input: "allelic_bams/{sample}.filtered.{suffix}.bam" if aligner == "Bowtie2" else "allelic_bams/{sample}.{suffix}.bam" + input: "allelic_bams/{sample}.filtered.{suffix}.bam" if aligner == "Bowtie2" else "allelic_bams/{sample}.markdup.{suffix}.bam" output: "allelic_bams/{sample}.{suffix}.sorted.bam" threads: diff --git a/snakePipes/shared/rules/Salmon.snakefile b/snakePipes/shared/rules/Salmon.snakefile index c64f1d307..402625cd4 100755 --- a/snakePipes/shared/rules/Salmon.snakefile +++ b/snakePipes/shared/rules/Salmon.snakefile @@ -57,7 +57,7 @@ if pairedEnd: gtf = genes_gtf, lib_type = getSalmon_libtype(pairedEnd, libraryType) threads: 8 - conda: CONDA_RNASEQ_ENV + conda: CONDA_SALMON_ENV shell: """ salmon quant -p {threads} --softclipOverhangs --validateMappings --numBootstraps 50 -i {params.index} -l {params.lib_type} -1 {input.r1} -2 {input.r2} -o {params.outdir} """ diff --git a/snakePipes/shared/rules/WGBS.snakefile b/snakePipes/shared/rules/WGBS.snakefile index 61ad3d24b..d0135c8ba 100755 --- a/snakePipes/shared/rules/WGBS.snakefile +++ b/snakePipes/shared/rules/WGBS.snakefile @@ -24,7 +24,7 @@ if pairedEnd and not fromBAM: r1=fastq_dir + "/{sample}" + reads[0] + ".fastq.gz", r2=fastq_dir + "/{sample}" + reads[1] + ".fastq.gz" output: - sbam=temp(aligner+"/{sample}.bam") + sbam=temp(aligner+"/{sample}.sorted.bam") params: bwameth_index=bwameth_index if aligner=="bwameth" else bwameth2_index, tempDir = tempDir @@ -43,7 +43,7 @@ elif not pairedEnd and not fromBAM: input: r1=fastq_dir + "/{sample}" + reads[0] + ".fastq.gz", output: - sbam=temp(aligner+"/{sample}.bam") + sbam=temp(aligner+"/{sample}.sorted.bam") params: bwameth_index=bwameth_index if aligner=="bwameth" else bwameth2_index, tempDir = tempDir @@ -57,59 +57,59 @@ elif not pairedEnd and not fromBAM: rm -rf "$MYTEMP" """ -if not fromBAM: - rule index_bam: - input: - aligner+"/{sample}.bam" - output: - temp(aligner+"/{sample}.bam.bai") - conda: CONDA_SHARED_ENV - shell: """ - samtools index "{input}" - """ - -if not skipBamQC: - rule markDupes: - input: - aligner+"/{sample}.bam", - aligner+"/{sample}.bam.bai" - output: - "Sambamba/{sample}.markdup.bam" - threads: lambda wildcards: 10 if 10 {output} """ @@ -189,7 +189,7 @@ rule calcCHHbias: threads: lambda wildcards: 10 if 10 {output} """ diff --git a/snakePipes/shared/rules/bam_filtering.snakefile b/snakePipes/shared/rules/bam_filtering.snakefile index b6a0616df..960be6e8d 100755 --- a/snakePipes/shared/rules/bam_filtering.snakefile +++ b/snakePipes/shared/rules/bam_filtering.snakefile @@ -7,7 +7,7 @@ bam_filter_string = "{} {} {}".format("-F 1024" if dedup else "", "-f 2" if prop rule samtools_filter: input: - aligner+"/{sample}.bam" + aligner+"/{sample}.markdup.bam" output: bam = temp("filtered_bam/{sample}.filtered.tmp.bam") params: diff --git a/snakePipes/shared/rules/deepTools_qc.snakefile b/snakePipes/shared/rules/deepTools_qc.snakefile index 99ee5d252..e0206f695 100755 --- a/snakePipes/shared/rules/deepTools_qc.snakefile +++ b/snakePipes/shared/rules/deepTools_qc.snakefile @@ -3,8 +3,8 @@ rule bamCoverage: input: - bam = aligner+"/{sample}.bam", - bai = aligner+"/{sample}.bam.bai" + bam = aligner+"/{sample}.markdup.bam", + bai = aligner+"/{sample}.markdup.bam.bai" output: "bamCoverage/{sample}.seq_depth_norm.bw" params: @@ -143,8 +143,8 @@ rule plotPCA: rule estimate_read_filtering: input: - bam = aligner+"/{sample}.bam", - bai = aligner+"/{sample}.bam.bai" + bam = aligner+"/{sample}.markdup.bam", + bai = aligner+"/{sample}.markdup.bam.bai" output: "deepTools_qc/estimateReadFiltering/{sample}_filtering_estimation.txt" conda: CONDA_SHARED_ENV diff --git a/snakePipes/shared/rules/envs/sleuth.yaml b/snakePipes/shared/rules/envs/sleuth.yaml index 6c504b2aa..194bd8cb5 100755 --- a/snakePipes/shared/rules/envs/sleuth.yaml +++ b/snakePipes/shared/rules/envs/sleuth.yaml @@ -6,3 +6,4 @@ dependencies: - r-base = 4.2.0 - r-sleuth = 0.30.1 - r-dplyr + - r-gtools diff --git a/snakePipes/shared/rules/masked_genomeIndex.snakefile b/snakePipes/shared/rules/masked_genomeIndex.snakefile index 028bc5e83..ca7326f71 100644 --- a/snakePipes/shared/rules/masked_genomeIndex.snakefile +++ b/snakePipes/shared/rules/masked_genomeIndex.snakefile @@ -20,7 +20,7 @@ def getref_fileList(dir): ## Create masked genome if allele_hybrid == 'dual': - rule create_snpgenome: + checkpoint create_snpgenome: input: genome = GENOMEDIR output: @@ -41,7 +41,7 @@ if allele_hybrid == 'dual': " --strain {params.strain1} --strain2 {params.strain2}" "&& cd ../" else: - rule create_snpgenome: + checkpoint create_snpgenome: input: genome = GENOMEDIR output: @@ -84,18 +84,18 @@ if aligner == "STAR": elif aligner == "Bowtie2": rule bowtie2_index: input: - snpgenome_dir = SNPdir + snpgenome_dir = SNPdir#, +# filelist = lambda wildcards: getref_fileList(checkpoints.create_snpgenome.get().output.snpgenome_dir) output: bowtie2_index_allelic threads: lambda wildcards: 10 if 10