aligners

.github/filters.yml

@@ -20,8 +20,8 @@ validate: &validate
   - 'test/bam/**'
   - 'harpy/utils/check_bam.py'
   - 'harpy/utils/check_fastq.py'
-  - 'harpy/report/notebooks/validate_fastq.ipynb'
-  - 'harpy/report/notebooks/validate_bam.ipynb'
+  - 'harpy/notebooks/validate_fastq.ipynb'
+  - 'harpy/notebooks/validate_bam.ipynb'
 deconvolve: &deconvolve
   - *common
   - *environments
@@ -44,15 +44,20 @@ qc: &qc
   - 'harpy/utils/bx_stats_fq.py'
   - 'harpy/validation/fastq.py'
   - 'test/fastq/**'
+  - 'harpy/notebooks/fastp_qc.ipynb'
+  - 'harpy/notebooks/qc_bx_stats.ipynb'
 align: &align
   - *common
   - *environments
+  - 'harpy/utils/standardize/standardize.go'
   - 'harpy/commands/align.py'
   - 'harpy/snakefiles/align_bwa.smk'
   - 'harpy/snakefiles/align_strobe.smk'
-  - 'harpy/report/notebooks/align_stats.ipynb'
-  - 'harpy/report/notebooks/align_lrstats.ipynb'
-  - 'harpy/report/notebooks/samtools_stats.ipynb'
+  - 'harpy/snakefiles/align_minimap.smk'
+  - 'harpy/snakefiles/align.smk'
+  - 'harpy/notebooks/align_stats.ipynb'
+  - 'harpy/notebooks/align_lrstats.ipynb'
+  - 'harpy/notebooks/samtools_stats.ipynb'
   - 'harpy/utils/bx_stats_sam.py'
   - 'harpy/utils/molecule_coverage.py'
   - 'harpy/validation/fasta.py'
@@ -64,7 +69,7 @@ snp: &snp
   - 'harpy/commands/snp.py'
   - 'harpy/snakefiles/snp_mpileup.smk'
   - 'harpy/snakefiles/snp_freebayes.smk'
-  - 'harpy/report/notebooks/bcftools_stats.ipynb'
+  - 'harpy/notebooks/bcftools_stats.ipynb'
   - 'harpy/validation/fasta.py'
   - 'harpy/validation/xam.py'
   - 'test/bam/**'
@@ -73,8 +78,8 @@ impute: &impute
   - *environments
   - 'harpy/commands/impute.py'
   - 'harpy/snakefiles/impute.smk'
-  - 'harpy/report/notebooks/impute.ipynb'
-  - 'harpy/report/notebooks/stitch_collate.ipynb'
+  - 'harpy/notebooks/impute.ipynb'
+  - 'harpy/notebooks/stitch_collate.ipynb'
   - 'harpy/validation/fasta.py'
   - 'harpy/validation/xam.py'
   - 'test/bam/**'
@@ -84,7 +89,7 @@ leviathan: &leviathan
   - *environments
   - 'harpy/commands/sv.py'
   - 'harpy/snakefiles/sv_leviathan.smk'
-  - 'harpy/report/notebooks/sv.ipynb'
+  - 'harpy/notebooks/sv.ipynb'
   - 'harpy/validation/fasta.py'
   - 'harpy/validation/xam.py'
   - 'test/bam/**'
@@ -93,7 +98,7 @@ naibr: &naibr
   - *environments
   - 'harpy/commands/sv.py'
   - 'harpy/snakefiles/sv_naibr.smk'
-  - 'harpy/report/notebooks/sv.ipynb'
+  - 'harpy/notebooks/sv.ipynb'
   - 'harpy/utils/infer_sv.py'
   - 'harpy/validation/fasta.py'
   - 'harpy/validation/xam.py'
@@ -109,7 +114,7 @@ phase: &phase
   - 'test/bam/**'
   - 'test/vcf/test.bcf'
   - 'test/vcf/test.phased.bcf'
-  - 'harpy/report/notebooks/hapcut.ipynb'
+  - 'harpy/notebooks/hapcut.ipynb'
   - 'harpy/utils/parse_phaseblocks.py'
 assembly: &assembly
   - *common

.github/workflows/tests.yml

@@ -190,6 +190,10 @@ jobs:
         run: |
           harpy align bwa --quiet 2 -x "-A 2" test/genome/genome.fasta.gz test/fastq && \
           ls -lh Align/bwa
+      - name: test minimap
+        run: |
+          harpy align minimap --quiet 2 test/genome/genome.fasta.gz test/fastq && \
+          ls -lh Align/minimap
 
   snp:
     needs: changes

CHANGELOG.md

@@ -1,6 +1,26 @@
 # New
 ## QC
 - replace multiqc report with native harpy report
+## Align
+- BWAMEM2 has been replaced with minibwa. Long live BWA!
+  - it's much faster, and takes much less time to index a reference
+- coverage depth added to aggregate report for processed alignments 
+- minimap2 added back in (with extra perks) for long-read compatability
+  - called with `harpy align minimap`
+
+## Report
+- harpy reports can be converted to less-nice but functional standalone HTML files
+  - this feature is accessed using `harpy report static`
+  - to accomodate this, `harpy report` (live report website) is now `harpy report live`
+
+# Changes
+## Align
+- `-d` (molecule distance threshold) has its default restored to 50kb since this value is used exclusively for reporting and does not alter the data
+
+## misc
+- removed FASTA format validation because it can be dreadfully slow with existing tools
+
+
 
 ## Report
 - harpy reports can be converted to less-nice but functional standalone HTML files
@@ -14,4 +34,9 @@
 ## misc
 - constrain CASAVA regex in FASTQ file validation so it doesn't trigger false positives when new CASAVA appears in unexpected places
 - [internal] notebooks no longer a submodule/subdirectory of `harpy.report`
-- utility `check_fastq.py` no longer employs globals, instead uses a sensible class system
+- utility `check_fastq.py` no longer employs globals, instead uses a sensible class system
+- add multithreading to pre-workflow VCF and XAM file validation and parsing
+- [internal-ish] the bwa, strobealign, and minimap2 workflows are nearly identical except for the reference preprocessing and alignment, so to minimize redundancy and duplication, those workflows have a single `align.smk` that imports a second snakefile `align_{aligner}.smk` that handles just the preprocessing and direct alignment for those aligners, then hands off to `align.smk` for all the downstream things (dedup, sorting, reports, etc)
+
+# Documentation
+- the pages for bwa, strobealign, and minimap have been consolidated into a single page bc they are nearly identical

docs/Commands/align/align.md

@@ -9,18 +9,18 @@ will need to align them to a reference genome before you can call variants.
 Harpy offers several aligners for this purpose:
 
 {.compact .clean .whitespace-nowrap}
-| aligner                  |     speed     |                    repository                    |                     publication                     |
-|:-------------------------|:-------------:|:------------------------------------------------:|:---------------------------------------------------:|
-| [BWA](bwa.md)            |    fast ⚡     |       [github](https://github.com/lh3/bwa)       |       [paper](http://arxiv.org/abs/1303.3997)       |
-| [strobealign](strobe.md) | super fast ⚡⚡ | [github](https://github.com/ksahlin/strobealign) | [paper](https://doi.org/10.1186/s13059-022-02831-7) |
+| command | aligner     | best for    |                                       repository |                      publication                       |
+| :------ | :---------- | :---------- | -----------------------------------------------: | :----------------------------------------------------: |
+| bwa     | minibwa     | general use |         [github](https://github.com/lh3/minibwa) |       [preprint](https://github.com/lh3/minibwa)       |
+| minimap | minimap2    | long reads  |        [github](https://github.com/lh3/minimap2) | [paper](https://doi.org/10.1093/bioinformatics/bty191) |
+| strobe  | strobealign | speed       | [github](https://github.com/ksahlin/strobealign) |  [paper](https://doi.org/10.1186/s13059-022-02831-7)   |
 
-Neither of these are linked-read aware aligners, but Harpy transfers the barcode information from the sequence headers into the alignments and will
-assign molecule identifiers (`MI:i` SAM tags) based on these barcodes and the [molecule distance threshold](../../Getting_Started/linked_read_data.md#barcode-thresholds).
+Neither of these are linked-read aware aligners, but Harpy transfers the barcode information from the sequence headers into the alignments.
 
 ## Non linked-read WGS data
 Starting with Harpy `v2.x`, you can skip the workflow
 routines that do things specific to linked reads, meaning you can comfortably use
-[!badge corners="pill" text="harpy align bwa"](bwa.md) and [!badge corners="pill" text="harpy align strobe"](strobe.md) to align your WGS sequence data. 
+[!badge corners="pill" text="harpy align bwa"](standard.md) and [!badge corners="pill" text="harpy align strobe"](standard.md) to align your WGS sequence data. 
 - version `2.0-2.7` : `--ignore-bx`
 - version `>2.7` : `--lr-type none`
 - version `>=3.0`: autodetected or forced with `--unlinked`