diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 39597f0..e80a477 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -27,7 +27,7 @@ jobs: TEST_PROFILE: ${{ matrix.test_profile }} EXEC_PROFILE: ${{ matrix.exec_profile }} - name: "CI [${{ matrix.test_profile }}] DIA-NN=1.8.1 NXF=${{ matrix.NXF_VER }}" + name: "CI [${{ matrix.test_profile }}] NXF=${{ matrix.NXF_VER }}" if: ${{ github.event_name != 'push' || (github.event_name == 'push' && github.repository == 'bigbio/quantmsdiann') }} runs-on: ubuntu-latest strategy: @@ -35,7 +35,7 @@ jobs: matrix: NXF_VER: - "25.04.0" - test_profile: ["test_dia", "test_dia_dotd"] + test_profile: ["test_dia", "test_dia_dotd", "test_dda"] exec_profile: ["docker"] # extended ci tests singularity. steps: @@ -60,11 +60,24 @@ jobs: mkdir -p $NXF_SINGULARITY_CACHEDIR mkdir -p $NXF_SINGULARITY_LIBRARYDIR + - name: Log in to GitHub Container Registry + if: matrix.test_profile == 'test_dda' + env: + GHCR_TOKEN: ${{ secrets.GHCR_TOKEN }} + GHCR_USERNAME: ${{ secrets.GHCR_USERNAME }} + run: | + if [ -z "$GHCR_TOKEN" ] || [ -z "$GHCR_USERNAME" ]; then + echo "::warning::Skipping test_dda: GHCR credentials not available (expected for fork PRs)" + echo "SKIP_DDA=true" >> $GITHUB_ENV + exit 0 + fi + echo "$GHCR_TOKEN" | docker login ghcr.io -u "$GHCR_USERNAME" --password-stdin + - name: Disk space cleanup uses: jlumbroso/free-disk-space@v1.3.1 - name: Run pipeline with test data in docker/singularity profile - if: github.event.pull_request.base.ref != 'master' + if: github.event.pull_request.base.ref != 'master' && env.SKIP_DDA != 'true' run: | nextflow run ${GITHUB_WORKSPACE} -profile $TEST_PROFILE,$EXEC_PROFILE,dev --outdir ${TEST_PROFILE}_${EXEC_PROFILE}_results diff --git a/.github/workflows/extended_ci.yml b/.github/workflows/extended_ci.yml index 42a9f48..621ab2c 100644 --- a/.github/workflows/extended_ci.yml +++ b/.github/workflows/extended_ci.yml @@ -114,7 +114,8 @@ jobs: strategy: fail-fast: false matrix: - test_profile: ["test_latest_dia", "test_dia_quantums", "test_dia_parquet"] + test_profile: + ["test_latest_dia", "test_dia_quantums", "test_dia_parquet", "test_dda", "test_dia_skip_preanalysis"] env: NXF_ANSI_LOG: false CAPSULE_LOG: none diff --git a/.gitignore b/.gitignore index 52812e6..6a54d55 100644 --- a/.gitignore +++ b/.gitignore @@ -21,3 +21,4 @@ null/ .codacy/ .github/instructions/codacy.instructions.md docs/superpowers/ +docs/plans/ diff --git a/AGENTS.md b/AGENTS.md index 2c32d31..67852f5 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -24,7 +24,7 @@ This is **non-negotiable**. All code must pass formatting and style checks befor - Built with Nextflow DSL2 - DIA-NN for peptide/protein identification and quantification -- Supports DIA-NN v1.8.1, v2.1.0, and v2.2.0 (latest) +- Supports DIA-NN v1.8.1, v2.1.0, v2.2.0, v2.3.2, and v2.5.0 (latest) - QuantUMS quantification method (DIA-NN >= 1.9.2) - Parquet-native output with decoy reporting (DIA-NN >= 2.0) - MSstats-compatible output generation (via quantms-utils conversion, no MSstats analysis) @@ -44,7 +44,7 @@ This is **non-negotiable**. All code must pass formatting and style checks befor - **nf-test**: Testing framework (config: `nf-test.config`) - **nf-core tools**: Pipeline standards and linting - **Containers**: Docker/Singularity/Apptainer/Podman (Conda deprecated) -- **DIA-NN**: Primary search engine (versions 1.8.1 through 2.2.0) +- **DIA-NN**: Primary search engine (versions 1.8.1 through 2.5.0) ### Key Configuration Files @@ -80,7 +80,7 @@ quantmsdiann/ │ ├── pmultiqc/ # QC reporting │ ├── sdrf_parsing/ # SDRF parsing │ ├── samplesheet_check/ # Input validation -│ └── utils/ # tdf2mzml, decompress, mzml stats +│ └── utils/ # decompress, mzml stats ├── conf/ │ ├── base.config # Resource definitions │ ├── modules/ # Module-specific configs @@ -97,7 +97,7 @@ quantmsdiann/ The pipeline executes the following steps: 1. **SDRF Validation & Parsing** - Validates input SDRF and extracts metadata -2. **File Preparation** - Converts RAW/mzML/.d/.dia files (ThermoRawFileParser, tdf2mzml) +2. **File Preparation** - Converts RAW/mzML/.d/.dia files (ThermoRawFileParser) 3. **Generate Config** - Creates DIA-NN config from enzyme/modifications (`quantmsutilsc dianncfg`) 4. **In-Silico Library Generation** - Predicts spectral library from FASTA (or uses provided library) 5. **Preliminary Analysis** - Per-file calibration and mass accuracy determination @@ -109,13 +109,16 @@ The pipeline executes the following steps: ### DIA-NN Version-Specific Features -| Feature | Min Version | Parameter | -| ------------------------------------------- | ----------- | ---------------------------- | -| Core workflow, library-free, .quant caching | 1.8.1 | (default) | -| QuantUMS quantification | 1.9.2 | `--quantums true` | -| Parquet output format | 2.0 | (automatic in 2.0+) | -| Decoy reporting | 2.0 | `--diann_report_decoys true` | -| Native .raw on Linux | 2.1.0 | (automatic) | +| Feature | Min Version | Parameter | +| ------------------------------------------- | ----------- | ---------------------- | +| Core workflow, library-free, .quant caching | 1.8.1 | (default) | +| QuantUMS quantification | 1.9.2 | `--quantums true` | +| Parquet output format | 2.0 | (automatic in 2.0+) | +| Decoy reporting | 2.0 | `--report_decoys true` | +| Native .raw on Linux | 2.1.0 | (automatic) | +| DDA support | 2.3.2 | `--dda true` | +| InfinDIA | 2.3.0 | `--enable_infin_dia` | +| DL model fine-tuning & selection | 2.5.0 | `--extra_args` | --- @@ -194,6 +197,8 @@ These apply on top of test profiles to override the DIA-NN container version: | `diann_v1_8_1` | `biocontainers/diann:v1.8.1_cv1` | none | | `diann_v2_1_0` | `ghcr.io/bigbio/diann:2.1.0` | GHCR | | `diann_v2_2_0` | `ghcr.io/bigbio/diann:2.2.0` | GHCR | +| `diann_v2_3_2` | `ghcr.io/bigbio/diann:2.3.2` | GHCR | +| `diann_v2_5_0` | `ghcr.io/bigbio/diann:2.5.0` | GHCR | ### CI Workflows diff --git a/CHANGELOG.md b/CHANGELOG.md index fff751a..ca35846 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,62 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [2.0.0] bigbio/quantmsdiann — Rome - 2026-04-18 + +### `Added` + +- DDA analysis mode support (requires DIA-NN >= 2.3.2), auto-detected from SDRF or set via `--dda` +- DIA-NN 2.3.2 version profile (`-profile diann_v2_3_2`) with InfinDIA and DDA support +- Scoring mode parameter (`--scoring_mode`): `generic` (default), `proteoforms` (proteogenomics/variant detection, >= 2.0), `peptidoforms` (PTM analysis) +- FDR controls: `--precursor_qvalue`, `--matrix_qvalue`, `--matrix_spec_q` replacing the misleadingly named `protein_level_fdr_cutoff` +- Channel normalization flags: `--channel_run_norm`, `--channel_spec_norm` for multiplexing workflows (plexDIA/SILAC) +- InfinDIA support: `--enable_infin_dia`, `--pre_select` for ultra-large search spaces (DIA-NN >= 2.3.0) +- Fragment-level export: `--export_quant` for parquet fragment data (DIA-NN >= 2.0) +- MS1 PTM quantification: `--site_ms1_quant` (DIA-NN >= 2.0) +- Skip preliminary analysis: `--skip_preliminary_analysis` to use a provided spectral library directly +- Centralized blocked-flags registry (`lib/BlockedFlags.groovy`) replacing duplicated per-module logic +- Version guards for all DIA-NN version-dependent features with clear error messages +- `VersionUtils.groovy` for semantic version comparison (replaces fragile string comparisons) +- Log capture via `tee` fallback in all DIA-NN modules for robust log handling across versions +- CI: `test_dda` and `test_dia_skip_preanalysis` test profiles + +### `Changed` + +- Removed `diann_` prefix from all user-facing parameters (e.g., `--diann_report_decoys` → `--report_decoys`, `--diann_normalize` → `--normalize`) +- Removed `diann_no_peptidoforms` parameter (superseded by `--scoring_mode`) +- Separated FINAL_QUANTIFICATION report output into parquet and TSV channels (DIA-NN >= 1.9 produces parquet) +- Matrix/stats outputs in FINAL_QUANTIFICATION now `optional: true` for DIA-NN 2.x compatibility +- Decoupled container engine from DIA-NN version configs (engine selected via `-profile`, not version config) +- Removed hardcoded DIA-NN container from `pride_codon_slurm.config` +- Updated Zenodo DOI to 10.5281/zenodo.19437128 + +### `Removed` + +- `tdf2mzml` module and all references (Bruker .d files handled natively by DIA-NN >= 2.0) +- `protein_level_fdr_cutoff` parameter (replaced by `precursor_qvalue`) +- `diann_no_peptidoforms` parameter (replaced by `scoring_mode`) + +### `Fixed` + +- DIA-NN log file handling for versions >= 2.x that don't produce `*.log.txt` files +- `DIANN_MSSTATS` receiving both parquet and TSV when both exist (now only one format reaches downstream) +- Missing blocked flags: `--no-prot-inf` in ASSEMBLE/INDIVIDUAL/FINAL, `--channel-run-norm`/`--channel-spec-norm` in FINAL, `--var-mod`/`--fixed-mod`/`--channels` in INSILICO +- Unnecessary `.first()` warning on `ch_sdrf` value channel +- `--out` added to in-silico library generation to produce a log file with DIA-NN 2.3.2 + +### `Dependencies` + +| Dependency | Version | +| --------------------- | --------- | +| `nextflow` | >=25.04.0 | +| `dia-nn` | 1.8.1+ | +| `thermorawfileparser` | 2.0.0.dev | +| `sdrf-pipelines` | 0.1.2 | +| `pmultiqc` | 0.0.44 | +| `quantms-utils` | 0.0.29 | + +--- + ## [1.0.0] bigbio/quantmsdiann - 2026-04-03 Initial release of the standalone DIA-NN quantitative proteomics pipeline, refactored from [bigbio/quantms](https://github.com/bigbio/quantms). diff --git a/CITATION.cff b/CITATION.cff new file mode 100644 index 0000000..07bc295 --- /dev/null +++ b/CITATION.cff @@ -0,0 +1,44 @@ +cff-version: 1.2.0 +title: "quantmsdiann" +message: "If you use quantmsdiann, please cite the quantms paper." +type: software +license: MIT +repository-code: https://github.com/bigbio/quantmsdiann +url: https://quantmsdiann.quantms.org +keywords: + - proteomics + - mass-spectrometry + - dia + - data-independent-acquisition + - dia-nn + - nextflow +preferred-citation: + type: article + title: "quantms: a cloud-based pipeline for quantitative proteomics enables the reanalysis of public proteomics data" + journal: "Nature Methods" + year: 2024 + volume: "21" + start: 1603 + end: 1607 + doi: "10.1038/s41592-024-02343-1" + authors: + - family-names: Dai + given-names: Chengxin + - family-names: Pfeuffer + given-names: Julianus + - family-names: Wang + given-names: Hong + - family-names: Zheng + given-names: Ping + - family-names: Kall + given-names: Lukas + - family-names: Sachsenberg + given-names: Timo + - family-names: Demichev + given-names: Vadim + - family-names: Bai + given-names: Mingze + - family-names: Kohlbacher + given-names: Oliver + - family-names: Perez-Riverol + given-names: Yasset diff --git a/CITATIONS.md b/CITATIONS.md index d74e0f9..8985a13 100644 --- a/CITATIONS.md +++ b/CITATIONS.md @@ -1,6 +1,6 @@ # bigbio/quantmsdiann: Citations -## [Pipeline](https://www.researchsquare.com/article/rs-3002027/v1) +## [Pipeline](https://doi.org/10.1038/s41592-024-02343-1) > Dai C, Pfeuffer J, Wang H, Zheng P, Käll L, Sachsenberg T, Demichev V, Bai M, Kohlbacher O, Perez-Riverol Y. quantms: a cloud-based pipeline for quantitative proteomics enables the reanalysis of public proteomics data. Nat Methods. 2024 Jul 4. doi: 10.1038/s41592-024-02343-1. Epub ahead of print. PMID: 38965444. diff --git a/README.md b/README.md index 67c2de6..2d0d725 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ [![GitHub Actions CI Status](https://github.com/bigbio/quantmsdiann/actions/workflows/ci.yml/badge.svg)](https://github.com/bigbio/quantmsdiann/actions/workflows/ci.yml) [![GitHub Actions Linting Status](https://github.com/bigbio/quantmsdiann/actions/workflows/linting.yml/badge.svg)](https://github.com/bigbio/quantmsdiann/actions/workflows/linting.yml) -[![Cite with Zenodo](https://zenodo.org/badge/DOI/10.5281/zenodo.15573386.svg)](https://doi.org/10.5281/zenodo.15573386) +[![Cite with Zenodo](https://zenodo.org/badge/DOI/10.5281/zenodo.19437128.svg)](https://doi.org/10.5281/zenodo.19437128) [![nf-test](https://img.shields.io/badge/unit_tests-nf--test-337ab7.svg)](https://www.nf-test.com) [![Nextflow](https://img.shields.io/badge/version-%E2%89%A525.04.0-green?style=flat&logo=nextflow&logoColor=white&color=%230DC09D&link=https%3A%2F%2Fnextflow.io)](https://www.nextflow.io/) @@ -25,8 +25,8 @@ The pipeline is built using [Nextflow](https://www.nextflow.io), a workflow tool The pipeline takes [SDRF](https://github.com/bigbio/proteomics-metadata-standard) metadata and mass spectrometry data files (`.raw`, `.mzML`, `.d`, `.dia`) as input and performs: 1. **Input validation** — SDRF parsing and validation via [sdrf-pipelines](https://github.com/bigbio/sdrf-pipelines) -2. **File preparation** — RAW to mzML conversion ([ThermoRawFileParser](https://github.com/compomics/ThermoRawFileParser)), indexing, Bruker `.d` handling ([tdf2mzml](https://github.com/bigbio/tdf2mzml)) -3. **In-silico spectral library generation** — deep learning-based prediction, or use a user-provided library (`--diann_speclib`) +2. **File preparation** — RAW to mzML conversion ([ThermoRawFileParser](https://github.com/compomics/ThermoRawFileParser)), indexing +3. **In-silico spectral library generation** — deep learning-based prediction, or use a user-provided library (`--speclib`) 4. **Preliminary analysis** — per-file calibration and mass accuracy estimation (parallelized) 5. **Empirical library assembly** — consensus library from preliminary results with RT profiling 6. **Individual analysis** — per-file search with the empirical library (parallelized) @@ -103,7 +103,7 @@ If you would like to contribute to this pipeline, please see the [contributing g If you use quantmsdiann in your research, please cite: -> Dai et al. "quantms: a cloud-based pipeline for quantitative proteomics" (2024). DOI: [10.5281/zenodo.15573386](https://doi.org/10.5281/zenodo.15573386) +> Dai et al. "quantms: a cloud-based pipeline for quantitative proteomics" (2024). DOI: [10.5281/zenodo.19437128](https://doi.org/10.5281/zenodo.19437128) An extensive list of references for the tools used by the pipeline can be found in the [CITATIONS.md](CITATIONS.md) file. diff --git a/assets/schema_input.json b/assets/schema_input.json index 7b15010..1699aad 100644 --- a/assets/schema_input.json +++ b/assets/schema_input.json @@ -11,7 +11,7 @@ "source name": { "type": "string", "pattern": "^\\S+$", - "errorMessage": "Sample name must be provided and cannot contain spaces" + "errorMessage": "Source name must be provided and cannot contain spaces" }, "comment[data file]": { "type": "string", @@ -22,7 +22,7 @@ "assay name": { "type": "string", "pattern": "^\\S+$", - "errorMessage": "Assay name must be provided and cannot contain spaces", + "errorMessage": "Assay name must be provided and cannot contain whitespace", "meta": ["assay"] } } diff --git a/conf/diann_versions/v2_1_0.config b/conf/diann_versions/v2_1_0.config index 9915726..9bd5208 100644 --- a/conf/diann_versions/v2_1_0.config +++ b/conf/diann_versions/v2_1_0.config @@ -10,5 +10,4 @@ process { } } -singularity.enabled = false -docker.enabled = true +// Container engine is selected via -profile (docker/singularity), not here diff --git a/conf/diann_versions/v2_2_0.config b/conf/diann_versions/v2_2_0.config index 93ea4ee..e72fcba 100644 --- a/conf/diann_versions/v2_2_0.config +++ b/conf/diann_versions/v2_2_0.config @@ -10,5 +10,4 @@ process { } } -singularity.enabled = false -docker.enabled = true +// Container engine is selected via -profile (docker/singularity), not here diff --git a/conf/diann_versions/v2_3_2.config b/conf/diann_versions/v2_3_2.config new file mode 100644 index 0000000..1a1dd97 --- /dev/null +++ b/conf/diann_versions/v2_3_2.config @@ -0,0 +1,13 @@ +/* + * DIA-NN 2.3.2 container override (private ghcr.io) + * Latest release with DDA support and InfinDIA. + */ +params.diann_version = '2.3.2' + +process { + withLabel: diann { + container = 'ghcr.io/bigbio/diann:2.3.2' + } +} + +// Container engine is selected via -profile (docker/singularity), not here diff --git a/conf/diann_versions/v2_5_0.config b/conf/diann_versions/v2_5_0.config new file mode 100644 index 0000000..1e584d4 --- /dev/null +++ b/conf/diann_versions/v2_5_0.config @@ -0,0 +1,14 @@ +/* + * DIA-NN 2.5.0 container override (private ghcr.io) + * Major protein ID improvements (+70%), DL model selection flags, + * new --aa-eq flag for amino acid equivalence. + */ +params.diann_version = '2.5.0' + +process { + withLabel: diann { + container = 'ghcr.io/bigbio/diann:2.5.0' + } +} + +// Container engine is selected via -profile (docker/singularity), not here diff --git a/conf/modules/dia.config b/conf/modules/dia.config index fff6e00..36708bb 100644 --- a/conf/modules/dia.config +++ b/conf/modules/dia.config @@ -2,7 +2,7 @@ ======================================================================================== DIA-NN module options — all DIA pipeline steps ======================================================================================== - Passes params.diann_extra_args to each DIA-NN step via ext.args. + Passes params.extra_args (formerly params.diann_extra_args) to each DIA-NN step via ext.args. Blocked-flag validation is performed in each module's script block, so it applies regardless of whether ext.args is overridden by the user. ---------------------------------------------------------------------------------------- @@ -11,22 +11,22 @@ process { withName: ".*:DIA:INSILICO_LIBRARY_GENERATION" { - ext.args = { params.diann_extra_args ?: '' } + ext.args = { params.extra_args ?: '' } } withName: ".*:DIA:PRELIMINARY_ANALYSIS" { - ext.args = { params.diann_extra_args ?: '' } + ext.args = { params.extra_args ?: '' } } withName: ".*:DIA:ASSEMBLE_EMPIRICAL_LIBRARY" { - ext.args = { params.diann_extra_args ?: '' } + ext.args = { params.extra_args ?: '' } } withName: ".*:DIA:INDIVIDUAL_ANALYSIS" { - ext.args = { params.diann_extra_args ?: '' } + ext.args = { params.extra_args ?: '' } } withName: ".*:DIA:FINAL_QUANTIFICATION" { - ext.args = { params.diann_extra_args ?: '' } + ext.args = { params.extra_args ?: '' } } } diff --git a/conf/pride_codon_slurm.config b/conf/pride_codon_slurm.config index 0848be1..e38b8b6 100644 --- a/conf/pride_codon_slurm.config +++ b/conf/pride_codon_slurm.config @@ -42,9 +42,8 @@ process { cpus = {Math.min((ms_files as List).size() < 200 ? 12 * task.attempt : 24 * task.attempt, 48)} } - withLabel: diann { - container = '/hps/nobackup/juan/pride/reanalysis/singularity/ghcr.io-bigbio-diann-1.9.2.sif' - } + // DIA-NN container: use -c conf/diann_versions/v.config or + // --diann_container to select the version. No default hardcoded here. } executor { diff --git a/conf/tests/test_dda.config b/conf/tests/test_dda.config new file mode 100644 index 0000000..6c81d4e --- /dev/null +++ b/conf/tests/test_dda.config @@ -0,0 +1,53 @@ +/* +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + Nextflow config file for testing DDA analysis (requires DIA-NN >= 2.3.2) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + Tests DDA mode using the PXD022287 HeLa dataset with --dda flag. + Uses ghcr.io/bigbio/diann:2.3.2. + + Use as follows: + nextflow run bigbio/quantmsdiann -profile test_dda,docker [--outdir ] + +------------------------------------------------------------------------------------------------ +*/ + +process { + resourceLimits = [ + cpus: 4, + memory: '12.GB', + time: '48.h' + ] +} + +params { + config_profile_name = 'Test profile for DDA analysis' + config_profile_description = 'DDA test using PXD022287 HeLa dataset with DIA-NN 2.3.2.' + + outdir = './results_dda' + + // Input data - PXD022287 HeLa DDA dataset + input = 'https://raw.githubusercontent.com/bigbio/quantms-test-datasets/quantms/testdata/diann_dda_ci/PXD022287.sdrf.tsv' + database = 'https://raw.githubusercontent.com/bigbio/quantms-test-datasets/quantms/testdata/diann_dda_ci/PXD022287_subset_human.fasta' + + // DDA mode + dda = true + diann_version = '2.3.2' + + // Search parameters matching PXD022287 HeLa dataset + min_peptide_length = 7 + max_peptide_length = 30 + max_precursor_charge = 3 + allowed_missed_cleavages = 1 + normalize = false + publish_dir_mode = 'symlink' + max_mods = 2 +} + +process { + withLabel: diann { + container = 'ghcr.io/bigbio/diann:2.3.2' + } +} + +singularity.enabled = false +docker.enabled = true diff --git a/conf/tests/test_dia.config b/conf/tests/test_dia.config index 2d321bc..85139de 100644 --- a/conf/tests/test_dia.config +++ b/conf/tests/test_dia.config @@ -34,7 +34,7 @@ params { max_peptide_length = 30 max_precursor_charge = 3 allowed_missed_cleavages = 1 - diann_normalize = false + normalize = false publish_dir_mode = 'symlink' max_mods = 2 } diff --git a/conf/tests/test_dia_2_2_0.config b/conf/tests/test_dia_2_2_0.config index 3772ec6..b36403d 100644 --- a/conf/tests/test_dia_2_2_0.config +++ b/conf/tests/test_dia_2_2_0.config @@ -36,7 +36,7 @@ params { max_peptide_length = 30 max_precursor_charge = 3 allowed_missed_cleavages = 1 - diann_normalize = false + normalize = false publish_dir_mode = 'symlink' max_mods = 2 } diff --git a/conf/tests/test_dia_dotd.config b/conf/tests/test_dia_dotd.config index cd151ca..8d1d8fc 100644 --- a/conf/tests/test_dia_dotd.config +++ b/conf/tests/test_dia_dotd.config @@ -37,7 +37,7 @@ params { max_peptide_length = 30 max_precursor_charge = 4 allowed_missed_cleavages = 2 - diann_normalize = false + normalize = false publish_dir_mode = 'symlink' max_mods = 2 mass_acc_automatic = false diff --git a/conf/tests/test_dia_parquet.config b/conf/tests/test_dia_parquet.config index dfea81d..ab80af0 100644 --- a/conf/tests/test_dia_parquet.config +++ b/conf/tests/test_dia_parquet.config @@ -36,12 +36,12 @@ params { max_peptide_length = 30 max_precursor_charge = 3 allowed_missed_cleavages = 1 - diann_normalize = false + normalize = false publish_dir_mode = 'symlink' max_mods = 2 // Parquet-specific: enable decoy reporting (Parquet-only feature in DIA-NN 2.0+) - diann_report_decoys = true + report_decoys = true } process { diff --git a/conf/tests/test_dia_quantums.config b/conf/tests/test_dia_quantums.config index 9f531b6..aa8e260 100644 --- a/conf/tests/test_dia_quantums.config +++ b/conf/tests/test_dia_quantums.config @@ -36,7 +36,7 @@ params { max_peptide_length = 30 max_precursor_charge = 3 allowed_missed_cleavages = 1 - diann_normalize = false + normalize = false publish_dir_mode = 'symlink' max_mods = 2 diff --git a/conf/tests/test_dia_skip_preanalysis.config b/conf/tests/test_dia_skip_preanalysis.config new file mode 100644 index 0000000..49e0409 --- /dev/null +++ b/conf/tests/test_dia_skip_preanalysis.config @@ -0,0 +1,48 @@ +/* +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + Nextflow config file for testing skip_preliminary_analysis path +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + Tests the pipeline with skip_preliminary_analysis=true, using default + mass accuracy parameters. Validates the untested code path in dia.nf. + + Use as follows: + nextflow run bigbio/quantmsdiann -profile test_dia_skip_preanalysis,docker [--outdir ] + +------------------------------------------------------------------------------------------------ +*/ + +process { + resourceLimits = [ + cpus: 4, + memory: '12.GB', + time: '48.h' + ] +} + +params { + config_profile_name = 'Test profile for skip preliminary analysis' + config_profile_description = 'Tests skip_preliminary_analysis path with default mass accuracy params.' + + outdir = './results_skip_preanalysis' + + // Input data - same as test_dia + input = 'https://raw.githubusercontent.com/bigbio/quantms-test-datasets/quantms/testdata/dia_ci/PXD026600.sdrf.tsv' + database = 'https://raw.githubusercontent.com/bigbio/quantms-test-datasets/quantms/testdata/dia_ci/REF_EColi_K12_UPS1_combined.fasta' + min_pr_mz = 350 + max_pr_mz = 950 + min_fr_mz = 500 + max_fr_mz = 1500 + min_peptide_length = 15 + max_peptide_length = 30 + max_precursor_charge = 3 + allowed_missed_cleavages = 1 + normalize = false + publish_dir_mode = 'symlink' + max_mods = 2 + + // Skip preliminary analysis - use default mass accuracy params + skip_preliminary_analysis = true + mass_acc_ms2 = 15 + mass_acc_ms1 = 15 + scan_window = 8 +} diff --git a/conf/tests/test_full_dia.config b/conf/tests/test_full_dia.config index 67b05a2..edce596 100644 --- a/conf/tests/test_full_dia.config +++ b/conf/tests/test_full_dia.config @@ -33,6 +33,6 @@ params { max_fr_mz = 1500 max_precursor_charge = 3 allowed_missed_cleavages = 1 - diann_normalize = false + normalize = false max_mods = 1 } diff --git a/conf/tests/test_latest_dia.config b/conf/tests/test_latest_dia.config index 10bd418..bd25abe 100644 --- a/conf/tests/test_latest_dia.config +++ b/conf/tests/test_latest_dia.config @@ -27,7 +27,7 @@ params { max_peptide_length = 30 max_precursor_charge = 3 allowed_missed_cleavages = 1 - diann_normalize = false + normalize = false publish_dir_mode = 'symlink' max_mods = 2 } diff --git a/docs/parameters.md b/docs/parameters.md index 0f25c4d..0844985 100644 --- a/docs/parameters.md +++ b/docs/parameters.md @@ -51,12 +51,37 @@ This document lists every pipeline parameter organised by category. Default valu ## 5. DIA-NN General -| Parameter | Type | Default | Description | -| -------------------- | ------- | ------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | -| `--diann_version` | string | `1.8.1` | DIA-NN version used by the workflow. Controls version-dependent flags (e.g. `--monitor-mod` for 1.8.x). See [DIA-NN Version Selection](usage.md#dia-nn-version-selection). | -| `--diann_debug` | integer | `3` | DIA-NN debug/verbosity level (0-4). Higher values produce more verbose logs. | -| `--diann_speclib` | string | `null` | Path to an external spectral library. If provided, the in-silico library generation step is skipped. | -| `--diann_extra_args` | string | `null` | Extra arguments appended to all DIA-NN steps. Flags incompatible with a step are automatically stripped with a warning. See [Passing Extra Arguments to DIA-NN](usage.md#passing-extra-arguments-to-dia-nn). | +| Parameter | Type | Default | Description | +| ------------------ | ------- | ------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `--diann_version` | string | `1.8.1` | DIA-NN version used by the workflow. Controls version-dependent flags (e.g. `--monitor-mod` for 1.8.x). See [DIA-NN Version Selection](usage.md#dia-nn-version-selection). | +| `--debug_level` | integer | `3` | DIA-NN debug/verbosity level (0-4). Higher values produce more verbose logs. | +| `--speclib` | string | `null` | Path to an external spectral library. If provided, the in-silico library generation step is skipped. | +| `--extra_args` | string | `null` | Extra arguments appended to all DIA-NN steps. Flags incompatible with a step are automatically stripped with a warning. See [Passing Extra Arguments to DIA-NN](usage.md#passing-extra-arguments-to-dia-nn). | +| `--dda` | boolean | `false` | Explicitly enable DDA mode. Normally auto-detected from the SDRF `comment[proteomics data acquisition method]` column. Use this flag only when the SDRF lacks the acquisition method. Requires DIA-NN >= 2.3.2. | +| `--light_models` | boolean | `false` | Enable `--light-models` for 10x faster in-silico library generation. Requires DIA-NN >= 2.0. | +| `--export_quant` | boolean | `false` | Enable `--export-quant` for fragment-level parquet data export. Requires DIA-NN >= 2.0. | +| `--site_ms1_quant` | boolean | `false` | Enable `--site-ms1-quant` to use MS1 apex intensities for PTM site quantification. Requires DIA-NN >= 2.0. | +| `--aa_eq` | boolean | `false` | Treat I&L, Q&E, N&D as equivalent amino acids during reannotation. Essential for entrapment FDR benchmarks. Maps to `--aa-eq`. | + +### DIA-NN 2.5.0 flags (via `--extra_args`) + +The following DIA-NN 2.5.0 flags are not exposed as pipeline parameters but can be passed via `--extra_args`. See [Fine-Tuning Deep Learning Models](usage.md#fine-tuning-deep-learning-models-dia-nn-20) for the complete workflow. + +| DIA-NN flag | Description | +| ------------------------ | ------------------------------------------------------------------------------------------------ | +| `--tokens ` | Tokenizer dictionary mapping modified residues to neural network token IDs (0-255). | +| `--rt-model ` | Fine-tuned retention time prediction model (`.pt` PyTorch file). | +| `--fr-model ` | Fine-tuned fragment ion prediction model (`.pt` file). Quality-sensitive — verify vs base model. | +| `--im-model ` | Fine-tuned ion mobility prediction model (`.pt` file). | +| `--tune-lib ` | Spectral library for fine-tuning (`.parquet`). Requires `--tune-rt` and/or `--tune-im`. | +| `--tune-rt` | Fine-tune the RT deep learning predictor. Requires `--tune-lib`. | +| `--tune-im` | Fine-tune the IM deep learning predictor. Requires `--tune-lib`. | +| `--tune-fr` | Fine-tune the fragmentation predictor. Requires `--tune-lib`. Use with caution. | +| `--tune-lr ` | Fine-tuning learning rate (default: 0.0005). | +| `--tune-restrict-layers` | Keep RNN layer weights fixed during fine-tuning (except cysteine embeddings). | +| `--tune-level ` | Limit fine-tuning to a specific model distillation level (0, 1, or 2). | + +> **Note:** `--parent` and `--aa-eq` are blocked from `--extra_args` — they are managed as pipeline parameters (`--aa_eq`). `--parent` is container-managed and overriding it would break model discovery. ## 6. Mass Accuracy & Calibration @@ -72,10 +97,10 @@ This document lists every pipeline parameter organised by category. Default valu ## 7. Bruker/timsTOF -| Parameter | Type | Default | Description | -| ------------------- | ------- | ------- | ---------------------------------------------------------------------------------------------------- | -| `--diann_tims_sum` | boolean | `false` | Enable `--quant-tims-sum` for slice/scanning timsTOF methods (highly recommended for Synchro-PASEF). | -| `--diann_im_window` | number | `null` | Set `--im-window` to ensure the IM extraction window is not smaller than the specified value. | +| Parameter | Type | Default | Description | +| ------------- | ------- | ------- | ---------------------------------------------------------------------------------------------------- | +| `--tims_sum` | boolean | `false` | Enable `--quant-tims-sum` for slice/scanning timsTOF methods (highly recommended for Synchro-PASEF). | +| `--im_window` | number | `null` | Set `--im-window` to ensure the IM extraction window is not smaller than the specified value. | ## 8. PTM Localization @@ -92,39 +117,62 @@ This document lists every pipeline parameter organised by category. Default valu ## 10. Preliminary Analysis -| Parameter | Type | Default | Description | -| ----------------------------- | ------- | ------- | ------------------------------------------------------------------------------------------------------------------- | -| `--skip_preliminary_analysis` | boolean | `false` | Skip preliminary analysis. Use the provided spectral library as-is instead of generating a local consensus library. | -| `--random_preanalysis` | boolean | `false` | Enable random selection of spectrum files for empirical library generation. | -| `--random_preanalysis_seed` | integer | `42` | Random seed for file selection when `--random_preanalysis` is enabled. | -| `--empirical_assembly_ms_n` | integer | `200` | Number of randomly selected spectrum files when `--random_preanalysis` is enabled. | +| Parameter | Type | Default | Description | +| ----------------------------- | ------- | ------- | ------------------------------------------------------------------------------------------------------------------------------ | +| `--skip_preliminary_analysis` | boolean | `false` | Skip preliminary analysis. Use the provided spectral library as-is instead of generating a local consensus library. | +| `--empirical_assembly_log` | string | `null` | Path to a pre-existing empirical assembly log file. Only used when `--skip_preliminary_analysis true` and `--speclib` are set. | +| `--random_preanalysis` | boolean | `false` | Enable random selection of spectrum files for empirical library generation. | +| `--random_preanalysis_seed` | integer | `42` | Random seed for file selection when `--random_preanalysis` is enabled. | +| `--empirical_assembly_ms_n` | integer | `200` | Number of randomly selected spectrum files when `--random_preanalysis` is enabled. | ## 11. Quantification & Output -| Parameter | Type | Default | Description | -| ------------------------- | ------- | ------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `--pg_level` | integer | `2` | Protein inference mode. `0` = isoforms, `1` = protein names (from FASTA), `2` = genes (default). | -| `--species_genes` | boolean | `false` | Add the organism identifier to gene names in DIA-NN output. | -| `--diann_normalize` | boolean | `true` | Enable cross-run normalisation in DIA-NN. Set to `false` to add `--no-norm`. | -| `--diann_report_decoys` | boolean | `false` | Include decoy PSMs in the main `.parquet` report (DIA-NN 2.0+ only). | -| `--diann_export_xic` | boolean | `false` | Extract MS1/fragment chromatograms for identified precursors (equivalent to the XICs option in the DIA-NN GUI). | -| `--diann_no_peptidoforms` | boolean | `false` | Disable automatic peptidoform scoring when variable modifications are declared (not recommended by DIA-NN). | -| `--diann_use_quant` | boolean | `true` | Reuse existing `.quant` files if available (`--use-quant`). | -| `--quantums` | boolean | `false` | Enable QuantUMS quantification (requires DIA-NN >= 1.9.2). When `false`, the pipeline passes `--direct-quant` to use legacy quantification (only for DIA-NN >= 1.9.2; silently skipped for 1.8.x where direct quant is the only mode). | -| `--quantums_train_runs` | string | `null` | Run index range for QuantUMS training (e.g. `0:5`). Maps to `--quant-train-runs`. Requires DIA-NN >= 1.9.2. | -| `--quantums_sel_runs` | integer | `null` | Number of automatically selected runs for QuantUMS training. Must be >= 6. Maps to `--quant-sel-runs`. Requires DIA-NN >= 1.9.2. | -| `--quantums_params` | string | `null` | Pre-calculated QuantUMS parameters. Maps to `--quant-params`. Requires DIA-NN >= 1.9.2. | - -## 12. Quality Control - -| Parameter | Type | Default | Description | -| ---------------------------- | ------- | ------- | ------------------------------------------------------------------------------------ | -| `--enable_pmultiqc` | boolean | `true` | Generate the pmultiqc QC report. | -| `--pmultiqc_idxml_skip` | boolean | `true` | Skip idXML files (do not generate search engine score plots) in the pmultiqc report. | -| `--contaminant_string` | string | `CONT` | Contaminant affix string for pmultiqc. Maps to `--contaminant_affix` in pmultiqc. | -| `--protein_level_fdr_cutoff` | number | `0.01` | Experiment-wide protein (group)-level FDR cutoff. | - -## 13. MultiQC & Reporting +| Parameter | Type | Default | Description | +| ----------------------- | ------- | --------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `--pg_level` | integer | `2` | Protein inference mode. `0` = isoforms, `1` = protein names (from FASTA), `2` = genes (default). | +| `--species_genes` | boolean | `false` | Add the organism identifier to gene names in DIA-NN output. | +| `--normalize` | boolean | `true` | Enable cross-run normalisation in DIA-NN. Set to `false` to add `--no-norm`. | +| `--report_decoys` | boolean | `false` | Include decoy PSMs in the main `.parquet` report (DIA-NN 2.0+ only). | +| `--export_xic` | boolean | `false` | Extract MS1/fragment chromatograms for identified precursors (equivalent to the XICs option in the DIA-NN GUI). | +| `--scoring_mode` | string | `generic` | DIA-NN scoring mode: `generic` (default, maximizes IDs), `proteoforms` (recommended for proteogenomics/variant detection, requires >= 2.0), `peptidoforms` (extra q-values for PTM analysis). | +| `--use_quant` | boolean | `true` | Reuse existing `.quant` files if available (`--use-quant`). | +| `--quantums` | boolean | `false` | Enable QuantUMS quantification (requires DIA-NN >= 1.9.2). When `false`, the pipeline passes `--direct-quant` to use legacy quantification (only for DIA-NN >= 1.9.2; silently skipped for 1.8.x where direct quant is the only mode). | +| `--quantums_train_runs` | string | `null` | Run index range for QuantUMS training (e.g. `0:5`). Maps to `--quant-train-runs`. Requires DIA-NN >= 1.9.2. | +| `--quantums_sel_runs` | integer | `null` | Number of automatically selected runs for QuantUMS training. Must be >= 6. Maps to `--quant-sel-runs`. Requires DIA-NN >= 1.9.2. | +| `--quantums_params` | string | `null` | Pre-calculated QuantUMS parameters. Maps to `--quant-params`. Requires DIA-NN >= 1.9.2. | + +## 12. DDA Mode + +| Parameter | Type | Default | Description | +| --------- | ------- | ------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `--dda` | boolean | `false` | Explicitly enable DDA mode when SDRF lacks the acquisition method column. Normally DDA is auto-detected from the SDRF `comment[proteomics data acquisition method]`. Requires DIA-NN >= 2.3.2 (use `-profile diann_v2_3_2`). Beta feature. | + +> **Note:** DDA mode is auto-detected from the SDRF when the `comment[proteomics data acquisition method]` +> column contains `data-dependent acquisition`. The `--dda` flag is only needed as a +> fallback when the SDRF does not include this column. DDA requires DIA-NN >= 2.3.2 +> (`-profile diann_v2_3_2`). + +## 13. InfinDIA (Experimental) + +| Parameter | Type | Default | Description | +| -------------------- | ------- | ------- | -------------------------------------------------------------------------------------- | +| `--enable_infin_dia` | boolean | `false` | Enable InfinDIA for ultra-large search spaces. Requires DIA-NN >= 2.3.0. Experimental. | +| `--pre_select` | integer | `null` | Precursor limit (`--pre-select N`) for InfinDIA pre-search. | + +> **Note:** InfinDIA requires DIA-NN >= 2.3.0 and is considered experimental. + +## 14. Quality Control + +| Parameter | Type | Default | Description | +| ----------------------- | ------- | ------- | ----------------------------------------------------------------------------------------------- | +| `--enable_pmultiqc` | boolean | `true` | Generate the pmultiqc QC report. | +| `--pmultiqc_idxml_skip` | boolean | `true` | Skip idXML files (do not generate search engine score plots) in the pmultiqc report. | +| `--contaminant_string` | string | `CONT` | Contaminant affix string for pmultiqc. Maps to `--contaminant_affix` in pmultiqc. | +| `--precursor_qvalue` | number | `0.01` | Precursor-level q-value threshold for the DIA-NN main report. Maps to `--qvalue`. | +| `--matrix_qvalue` | number | `0.01` | Q-value threshold for DIA-NN output matrices (pr_matrix, pg_matrix). Maps to `--matrix-qvalue`. | +| `--matrix_spec_q` | number | `0.05` | Run-specific protein q-value for protein/gene matrices. Maps to `--matrix-spec-q`. | + +## 15. MultiQC & Reporting | Parameter | Type | Default | Description | | ------------------------------- | ------------------ | ------- | --------------------------------------------------------------------------------- | diff --git a/docs/usage.md b/docs/usage.md index b04cd7d..2abcd2e 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -59,11 +59,82 @@ nextflow run bigbio/quantmsdiann \ -profile docker ``` -For Synchro-PASEF data, enable `--diann_tims_sum` (which adds `--quant-tims-sum` to DIA-NN). +For Synchro-PASEF data, enable `--tims_sum` (which adds `--quant-tims-sum` to DIA-NN). > [!NOTE] > The pipeline will emit a warning during PRELIMINARY_ANALYSIS if it detects `.d` files with automatic mass accuracy calibration enabled, recommending to set tolerances via SDRF or pipeline parameters. +### DDA Analysis Mode (Beta) + +DIA-NN 2.3.2+ supports DDA data analysis via the `--dda` flag. The pipeline **auto-detects DDA mode** from the SDRF `comment[proteomics data acquisition method]` column — no extra flags needed if your SDRF contains `data-dependent acquisition`: + +```bash +nextflow run bigbio/quantmsdiann \ + --input dda_sdrf.tsv \ + --database proteins.fasta \ + -profile diann_v2_3_2,docker +``` + +If your SDRF does not include the acquisition method column, you can explicitly enable DDA mode with `--dda true`: + +```bash +nextflow run bigbio/quantmsdiann \ + --input sdrf.tsv \ + --database proteins.fasta \ + --dda true \ + -profile diann_v2_3_2,docker +``` + +**Limitations (beta feature):** + +- Only trust: q-values, PEP values, RT/IM values, Ms1.Apex.Area, Normalisation.Factor +- PTM localization probabilities are **unreliable** with DDA data +- MBR requires MS2-level evidence (DIA-like, not classical DDA MBR) +- No isobaric labeling or reporter-tag quantification +- Primary use cases: legacy DDA reanalysis, spectral library creation, immunopeptidomics + +The pipeline uses the same workflow for DDA as DIA — the `--dda` flag is passed to all DIA-NN steps automatically when DDA is detected from the SDRF or enabled via `--dda`. + +### Preprocessing Options + +- `--reindex_mzml` (default: true) — Re-index mzML files before processing. Disable with `--reindex_mzml false` if files are already indexed. +- `--mzml_statistics` (default: false) — Generate mzML statistics (parquet format) for QC. +- `--mzml_features` (default: false) — Enable feature detection in mzML statistics. + +Bruker `.d` files are supported natively by the current workflow and are passed directly to DIA-NN; there is no `--convert_dotd` preprocessing option. + +### Passing Extra Arguments to DIA-NN + +Use `--extra_args` to pass additional flags to all DIA-NN steps. The pipeline validates and strips flags it manages internally to prevent conflicts. + +Managed flags (stripped with a warning if passed via extra_args): `--lib`, `--f`, `--fasta`, `--threads`, `--verbose`, `--temp`, `--out`, `--matrices`, `--use-quant`, `--gen-spec-lib`, `--mass-acc`, `--mass-acc-ms1`, `--window`, `--var-mod`, `--fixed-mod`, `--monitor-mod`, and others. + +To enable this, add `includeConfig 'conf/modules/dia.config'` to your configuration (already included by default). + +### DIA-NN Version Selection + +The default DIA-NN version is 1.8.1. To use a different version: + +| Version | Profile | Features | +| ------- | ----------------------- | ----------------------------------- | +| 1.8.1 | (default) | Core DIA analysis | +| 2.1.0 | `-profile diann_v2_1_0` | Native .raw support, reduced memory | +| 2.2.0 | `-profile diann_v2_2_0` | Speed optimizations | +| 2.3.2 | `-profile diann_v2_3_2` | DDA support, InfinDIA | +| 2.5.0 | `-profile diann_v2_5_0` | +70% protein IDs, model fine-tuning | + +Example: `nextflow run bigbio/quantmsdiann -profile test_dia,docker,diann_v2_2_0` + +### Verbose Module Output + +Use `-profile verbose_modules` to publish intermediate files from all pipeline steps: + +```bash +nextflow run bigbio/quantmsdiann -profile test_dia,docker,verbose_modules --outdir results +``` + +This publishes ThermoRawFileParser conversions, mzML indexing results, per-file DIA-NN logs, and spectral library intermediates. + ### Pipeline settings via params file Pipeline settings can be provided in a `yaml` or `json` file via `-params-file `: @@ -86,7 +157,7 @@ outdir: "./results" Specify the pipeline version when running on your data: ```bash -nextflow run bigbio/quantmsdiann -r 1.0.0 -profile docker --input sdrf.tsv --database db.fasta --outdir results +nextflow run bigbio/quantmsdiann -r 2.0.0 -profile docker --input sdrf.tsv --database db.fasta --outdir results ``` ## Core Nextflow arguments @@ -119,7 +190,7 @@ nextflow run . -profile test_dia,docker --outdir results # DIA with Bruker .d files nextflow run . -profile test_dia_dotd,docker --outdir results -# Latest DIA-NN version (2.1.0) +# Latest DIA-NN version (2.5.0) nextflow run . -profile test_latest_dia,docker --outdir results ``` @@ -173,7 +244,7 @@ These parameters apply globally across all files. They are set in `diann_config. | `--min-corr 2 --corr-diff 1 --time-corr-only` | `--performance_mode` | `true` | PRELIMINARY | High-speed, low-RAM mode | | `--pg-level` | `--pg_level` | `2` | INDIVIDUAL, FINAL | Protein grouping level | | `--species-genes` | `--species_genes` | `false` | FINAL | Use species-specific gene names | -| `--no-norm` | `--diann_normalize` | `true` | FINAL | Disable normalization when `false` | +| `--no-norm` | `--normalize` | `true` | FINAL | Disable normalization when `false` | ### PTM site localization (`--monitor-mod`) @@ -214,11 +285,11 @@ Supported modification name mappings: ## Passing Extra Arguments to DIA-NN -The `--diann_extra_args` parameter appends additional DIA-NN command-line flags to **all** DIA-NN steps (INSILICO_LIBRARY_GENERATION, PRELIMINARY_ANALYSIS, ASSEMBLE_EMPIRICAL_LIBRARY, INDIVIDUAL_ANALYSIS, FINAL_QUANTIFICATION). +The `--extra_args` parameter appends additional DIA-NN command-line flags to **all** DIA-NN steps (INSILICO_LIBRARY_GENERATION, PRELIMINARY_ANALYSIS, ASSEMBLE_EMPIRICAL_LIBRARY, INDIVIDUAL_ANALYSIS, FINAL_QUANTIFICATION). ```bash nextflow run bigbio/quantmsdiann \ - --diann_extra_args '--smart-profiling --peak-center' \ + --extra_args '--smart-profiling --peak-center' \ ... ``` @@ -249,17 +320,18 @@ process { The pipeline supports multiple DIA-NN versions via built-in Nextflow profiles. Each profile sets `params.diann_version` and overrides the container image for all `diann`-labelled processes. -| Profile | DIA-NN Version | Container | Key features | -| -------------- | -------------- | ------------------------------------------ | -------------------------------------------------------------- | -| `diann_v1_8_1` | 1.8.1 | `docker.io/biocontainers/diann:v1.8.1_cv1` | Default. Public BioContainers image. TSV output. | -| `diann_v2_1_0` | 2.1.0 | `ghcr.io/bigbio/diann:2.1.0` | Parquet output. Native .raw on Linux. QuantUMS (`--quantums`). | -| `diann_v2_2_0` | 2.2.0 | `ghcr.io/bigbio/diann:2.2.0` | Speed optimizations (up to 1.6x on HPC). Parquet output. | -| `diann_v2_3_2` | 2.3.2 | `ghcr.io/bigbio/diann:2.3.2` | DDA support (`--diann_dda`), InfinDIA, up to 9 variable mods. | +| Profile | DIA-NN Version | Container | Key features | +| -------------- | -------------- | ------------------------------------------ | --------------------------------------------------------------- | +| `diann_v1_8_1` | 1.8.1 | `docker.io/biocontainers/diann:v1.8.1_cv1` | Default. Public BioContainers image. TSV output. | +| `diann_v2_1_0` | 2.1.0 | `ghcr.io/bigbio/diann:2.1.0` | Parquet output. Native .raw on Linux. QuantUMS (`--quantums`). | +| `diann_v2_2_0` | 2.2.0 | `ghcr.io/bigbio/diann:2.2.0` | Speed optimizations (up to 1.6x on HPC). Parquet output. | +| `diann_v2_3_2` | 2.3.2 | `ghcr.io/bigbio/diann:2.3.2` | DDA support (`--dda`), InfinDIA, up to 9 variable mods. | +| `diann_v2_5_0` | 2.5.0 | `ghcr.io/bigbio/diann:2.5.0` | Up to 70% more protein IDs. DL model fine-tuning and selection. | **Version-dependent features:** Some parameters are only available with newer DIA-NN versions. The pipeline handles version compatibility automatically: - **QuantUMS** (`--quantums`): Requires >= 1.9.2. The `--direct-quant` flag is automatically skipped for DIA-NN 1.8.x where direct quantification is the only mode. -- **DDA mode** (`--diann_dda`): Requires >= 2.3.2. The pipeline will error if enabled with an older version. +- **DDA mode** (`--dda`): Requires >= 2.3.2. The pipeline will error if enabled with an older version. - **InfinDIA** (`--enable_infin_dia`): Requires >= 2.3.0. Usage: @@ -279,6 +351,86 @@ nextflow run bigbio/quantmsdiann \ > [!NOTE] > DIA-NN 2.x images are hosted on `ghcr.io/bigbio` and may require authentication for private registries. The `diann_v2_1_0` and `diann_v2_2_0` profiles force Docker mode by default; for Singularity, override with your own config. +## Fine-Tuning Deep Learning Models (DIA-NN 2.0+) + +DIA-NN uses deep learning models to predict retention time (RT), ion mobility (IM), and fragment ion intensities. For non-standard modifications, fine-tuning these models on real data can substantially improve detection. + +**When to fine-tune:** Fine-tuning is beneficial for custom chemical labels (e.g., mTRAQ, dimethyl), exotic PTMs, or unmodified cysteines. Standard modifications (Phospho, Oxidation, Acetylation, Deamidation, diGlycine) do not require fine-tuning — DIA-NN's built-in models already handle them well. + +### How fine-tuning works + +DIA-NN's neural networks encode each amino acid and modification as a "token" — an integer ID (0-255) mapped in a dictionary file (`dict.txt`). The default dictionary ships with DIA-NN and covers common modifications. When you fine-tune, DIA-NN: + +1. Reads a spectral library containing empirically observed peptides with the modifications of interest +2. Learns how those modifications affect RT, IM, and fragmentation patterns +3. Outputs new model files (`.pt` PyTorch format) and an expanded dictionary (`dict.txt`) that includes tokens for the new modifications + +The fine-tuned models are then used in place of the defaults when generating predicted spectral libraries. + +> [!NOTE] +> **`--tune-lib` cannot be combined with `--gen-spec-lib` in a single DIA-NN invocation** ([confirmed in DIA-NN #1499](https://github.com/vdemichev/DiaNN/issues/1499)). Fine-tuning and library generation are still separate DIA-NN commands, but quantmsdiann can now orchestrate them within a single pipeline run when `--enable_fine_tuning` is used. Integrated fine-tuning requires DIA-NN v2.5.0 or later. The two-run/manual approach below is only needed when integrated fine-tuning is not enabled, or when using an older DIA-NN version that does not support this workflow. + +### Manual fallback workflow (two-run fine-tuning) + +**Run 1 — Generate the tuning library:** + +Run quantmsdiann normally. The empirical library produced by the ASSEMBLE_EMPIRICAL_LIBRARY step (after preliminary analysis) serves as the tuning library. This library contains empirically observed RT, IM, and fragment intensities for peptides bearing the modifications of interest. + +```bash +# First run: standard pipeline to produce empirical library +nextflow run bigbio/quantmsdiann \ + -profile diann_v2_5_0,docker \ + --input sdrf.tsv --database db.fasta --outdir results_run1 +# Output: results_run1/library_generation/assemble_empirical_library/empirical_library.parquet +``` + +**Fine-tune models (outside the pipeline):** + +```bash +# Fine-tune RT and IM models using the empirical library +diann --tune-lib /abs/path/to/empirical_library.parquet --tune-rt --tune-im + +# Optionally also fine-tune the fragmentation model (quality-sensitive — verify vs base model) +diann --tune-lib /abs/path/to/empirical_library.parquet --tune-rt --tune-im --tune-fr +``` + +DIA-NN will output (named after the input library): + +- `empirical_library.dict.txt` — expanded tokenizer dictionary with new modification tokens +- `empirical_library.rt.d0.pt` (+ `.d1.pt`, `.d2.pt`) — fine-tuned RT models (3 distillation levels) +- `empirical_library.im.d0.pt` (+ `.d1.pt`, `.d2.pt`) — fine-tuned IM models +- `empirical_library.fr.d0.pt` (+ `.d1.pt`, `.d2.pt`) — fine-tuned fragment models (if `--tune-fr`) + +Additional tuning parameters: `--tune-lr` (learning rate, default 0.0005), `--tune-restrict-layers` (fix RNN weights), `--tune-level` (limit to a specific distillation level 0/1/2). + +**Run 2 — Re-run the pipeline with fine-tuned models:** + +```bash +# Second run: use tuned models for in-silico library generation and all downstream steps +nextflow run bigbio/quantmsdiann \ + -profile diann_v2_5_0,docker \ + --input sdrf.tsv --database db.fasta \ + --extra_args "--tokens /abs/path/to/empirical_library.dict.txt --rt-model /abs/path/to/empirical_library.rt.d0.pt --im-model /abs/path/to/empirical_library.im.d0.pt" \ + --outdir results_run2 +``` + +The `--tokens`, `--rt-model`, and `--im-model` flags are passed to all DIA-NN steps via `--extra_args`, so the in-silico library generation uses the fine-tuned models to produce better-predicted spectra for the non-standard modifications. + +> [!IMPORTANT] +> Use **absolute paths** for model files. The `--parent` flag is blocked by the pipeline (it controls the container's DIA-NN installation path). + +### Integrated fine-tuning step + +The pipeline now includes an optional integrated fine-tuning phase, which eliminates the need for two separate runs. You can enable this feature by using the `--enable_fine_tuning` flag. The integrated workflow is: + +``` +INSILICO_LIBRARY → PRELIMINARY_ANALYSIS → ASSEMBLE_EMPIRICAL_LIBRARY + → [FINE_TUNE_MODELS] → INSILICO_LIBRARY (with tuned models) + → INDIVIDUAL_ANALYSIS → FINAL_QUANTIFICATION +``` + +This would be gated by a `--enable_fine_tuning` parameter. [@vdemichev](https://github.com/vdemichev): would this approach work correctly — using the empirical library from assembly as `--tune-lib`, then regenerating the in-silico library with the tuned models before proceeding to individual analysis? Or would you recommend a different integration point? + ## Verbose Module Output By default, only final result files are published. For debugging or detailed inspection, the `verbose_modules` profile publishes all intermediate files from every DIA-NN step: diff --git a/lib/BlockedFlags.groovy b/lib/BlockedFlags.groovy new file mode 100644 index 0000000..f46bf41 --- /dev/null +++ b/lib/BlockedFlags.groovy @@ -0,0 +1,148 @@ +/** + * Centralised registry of DIA-NN flags managed by the pipeline. + * + * WHY THIS EXISTS: + * The pipeline controls certain DIA-NN flags directly (e.g. --threads, --out, --qvalue) + * based on pipeline parameters, SDRF metadata, or step-specific logic. If a user also + * passes these via --extra_args, the flag would appear twice in the DIA-NN command — + * causing silent conflicts or undefined behaviour. This registry strips managed flags + * from extra_args with a warning, so the pipeline's values always take precedence. + * + * WHY A GROOVY CLASS AND NOT CONFIG FILES: + * Blocked flags are a safety mechanism. Defining them in Nextflow config (e.g. via + * ext.blocked_flags) would allow users to accidentally override or disable them in + * custom configs. A compiled Groovy class in lib/ cannot be overridden by user configs, + * ensuring the safety net is always active. It also provides a single file to audit + * and edit when adding new managed flags. + * + * HOW TO ADD A NEW BLOCKED FLAG: + * - If the flag applies to ALL DIA-NN steps, add it to the COMMON list below. + * - If it applies to specific steps only, add it to the relevant MODULE_FLAGS entry. + * - No changes needed in the module .nf files — they call BlockedFlags.strip() which + * reads from this registry automatically. + * + * Nextflow auto-loads all classes in lib/, so this is available in all modules. + */ +class BlockedFlags { + + // Flags common to ALL DIA-NN steps. + // These are always set by the pipeline from config/SDRF, so user overrides would conflict. + // --temp/--threads/--verbose: controlled by Nextflow task resources and debug_level param + // --lib/--f/--fasta: set from workflow inputs (speclib, ms_files, database) + // --var-mod/--fixed-mod/--monitor-mod/--channels/--lib-fixed-mod/--original-mods: injected from diann_config.cfg + // --dda: auto-detected from SDRF or set via --dda param + // --proteoforms/--peptidoforms/--no-peptidoforms: controlled by scoring_mode param + // --parent: container-managed (DIA-NN model path), overriding breaks model discovery + // --aa-eq: controlled by aa_eq param + private static final List COMMON = [ + '--temp', '--threads', '--verbose', '--lib', '--f', '--fasta', + '--monitor-mod', '--var-mod', '--fixed-mod', '--dda', '--parent', '--aa-eq', + '--channels', '--lib-fixed-mod', '--original-mods', + '--proteoforms', '--peptidoforms', '--no-peptidoforms', + ] + + // Per-module additional blocked flags (on top of COMMON). + // + // Flags are blocked for two reasons: + // 1. "Pipeline-managed": the pipeline sets them explicitly from params/SDRF/metadata. + // Allowing user overrides would create duplicate/conflicting flags. + // 2. "No-effect guard": the flag has no effect in this step (e.g., protein inference + // during library assembly). Blocking prevents users from passing them and wrongly + // believing they had an effect. + // + // When reason is non-obvious, a comment explains why the flag is blocked. + private static final Map> MODULE_FLAGS = [ + INSILICO_LIBRARY_GENERATION: [ + // Pipeline-managed: set from params (library generation settings) + '--use-quant', '--no-main-report', '--matrices', '--out', + '--fasta-search', '--predictor', '--gen-spec-lib', + '--missed-cleavages', '--min-pep-len', '--max-pep-len', + '--min-pr-charge', '--max-pr-charge', '--var-mods', + '--min-pr-mz', '--max-pr-mz', '--min-fr-mz', '--max-fr-mz', + '--met-excision', '--light-models', + '--infin-dia', '--pre-select', + // Pipeline-managed when fine-tuning: tuned model files passed as process inputs + '--tokens', '--rt-model', '--im-model', '--fr-model', + ], + PRELIMINARY_ANALYSIS: [ + // Pipeline-managed: set from params and SDRF calibration metadata + '--use-quant', '--gen-spec-lib', '--out-lib', '--matrices', '--out', + '--mass-acc', '--mass-acc-ms1', '--window', + '--quick-mass-acc', '--min-corr', '--corr-diff', '--time-corr-only', + '--min-pr-mz', '--max-pr-mz', '--min-fr-mz', '--max-fr-mz', + // Pipeline-managed: preliminary step disables protein inference (--no-prot-inf) + '--no-prot-inf', + ], + ASSEMBLE_EMPIRICAL_LIBRARY: [ + // Pipeline-managed: set from params and calibration results + '--no-main-report', '--no-ifs-removal', '--matrices', '--out', + '--mass-acc', '--mass-acc-ms1', '--window', + '--individual-mass-acc', '--individual-windows', + '--out-lib', '--use-quant', '--gen-spec-lib', '--rt-profiling', + // No-effect guard: protein inference flags have no effect during library assembly + // (--gen-spec-lib produces a spectral library, not a quantified report). + // Blocked to prevent users from thinking they affect this step. + '--no-prot-inf', '--relaxed-prot-inf', '--pg-level', + ], + INDIVIDUAL_ANALYSIS: [ + // Pipeline-managed: set from params and calibrated values from assembly step + '--use-quant', '--gen-spec-lib', '--out-lib', '--matrices', '--out', '--rt-profiling', + '--mass-acc', '--mass-acc-ms1', '--window', + '--no-ifs-removal', '--no-main-report', + // Pipeline-managed: protein inference set by pipeline (--relaxed-prot-inf --pg-level) + '--relaxed-prot-inf', '--pg-level', + '--min-pr-mz', '--max-pr-mz', '--min-fr-mz', '--max-fr-mz', + '--no-prot-inf', + ], + FINAL_QUANTIFICATION: [ + // Pipeline-managed: set from params for final report generation + '--no-main-report', '--gen-spec-lib', '--out-lib', '--no-ifs-removal', + '--use-quant', '--matrices', '--out', + // Pipeline-managed: protein inference (--relaxed-prot-inf --pg-level) + '--relaxed-prot-inf', '--pg-level', + // Pipeline-managed: FDR controls (precursor_qvalue, matrix_qvalue, matrix_spec_q) + '--qvalue', '--matrix-qvalue', '--matrix-spec-q', + '--window', '--individual-windows', + '--species-genes', '--report-decoys', '--xic', '--no-norm', + '--export-quant', '--site-ms1-quant', + '--channel-run-norm', '--channel-spec-norm', + '--no-prot-inf', + ], + FINE_TUNE_MODELS: [ + // Pipeline-managed: tuning flags set from enable_fine_tuning, tune_fr, tune_lr params + '--tune-lib', '--tune-rt', '--tune-im', '--tune-fr', '--tune-lr', + '--tune-restrict-layers', '--tune-level', + ], + ] + + /** + * Get the full blocked flags list for a module (COMMON + module-specific). + */ + static List forModule(String moduleName) { + def moduleFlags = MODULE_FLAGS[moduleName] ?: [] + return (COMMON + moduleFlags).unique() + } + + /** + * Strip blocked flags from an args string, logging a warning for each. + * Returns the cleaned args string. + * + * @param moduleName e.g. 'FINAL_QUANTIFICATION' + * @param args the raw args string from task.ext.args + * @param log the Nextflow log object + * @return cleaned args string + */ + static String strip(String moduleName, String args, def log) { + if (!args) return '' + def blocked = forModule(moduleName) + // Sort by length descending so longer flags (e.g. --mass-acc-ms1) match before shorter prefixes (--mass-acc) + blocked.sort { a -> -a.length() }.each { flag -> + def flagPattern = '(?<=^|\\s)' + java.util.regex.Pattern.quote(flag) + '(?=\\s|\$)(\\s+(?!-{1,2}[a-zA-Z])\\S+)*' + if (args =~ flagPattern) { + log.warn "DIA-NN: '${flag}' is managed by the pipeline for ${moduleName} and will be stripped." + args = args.replaceAll(flagPattern, '').trim() + } + } + return args + } +} diff --git a/lib/VersionUtils.groovy b/lib/VersionUtils.groovy new file mode 100644 index 0000000..f340a61 --- /dev/null +++ b/lib/VersionUtils.groovy @@ -0,0 +1,34 @@ +/** + * Semantic version comparison utility for DIA-NN version guards. + * + * Nextflow auto-loads all classes in lib/, so these are available + * in workflows and module scripts without explicit imports. + */ +class VersionUtils { + + /** + * Compare two version strings semantically (e.g. '2.10.0' > '2.3.2'). + * Returns negative if a < b, zero if equal, positive if a > b. + */ + static int compare(String a, String b) { + def partsA = a.tokenize('.').collect { it.isInteger() ? it.toInteger() : 0 } + def partsB = b.tokenize('.').collect { it.isInteger() ? it.toInteger() : 0 } + def maxLen = Math.max(partsA.size(), partsB.size()) + for (int i = 0; i < maxLen; i++) { + int va = i < partsA.size() ? partsA[i] : 0 + int vb = i < partsB.size() ? partsB[i] : 0 + if (va != vb) return va <=> vb + } + return 0 + } + + /** True if version is strictly less than required. */ + static boolean versionLessThan(String version, String required) { + return compare(version, required) < 0 + } + + /** True if version is greater than or equal to required. */ + static boolean versionAtLeast(String version, String required) { + return compare(version, required) >= 0 + } +} diff --git a/mkdocs/images/social-card.png b/mkdocs/images/social-card.png new file mode 100644 index 0000000..7ce482a Binary files /dev/null and b/mkdocs/images/social-card.png differ diff --git a/modules/local/diann/assemble_empirical_library/main.nf b/modules/local/diann/assemble_empirical_library/main.nf index 2bc0525..a1cb2ca 100644 --- a/modules/local/diann/assemble_empirical_library/main.nf +++ b/modules/local/diann/assemble_empirical_library/main.nf @@ -2,6 +2,7 @@ process ASSEMBLE_EMPIRICAL_LIBRARY { tag "$meta.experiment_id" label 'process_low' label 'diann' + label 'error_retry' container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? 'https://containers.biocontainers.pro/s3/SingImgsRepo/diann/v1.8.1_cv1/diann_v1.8.1_cv1.img' : @@ -25,22 +26,9 @@ process ASSEMBLE_EMPIRICAL_LIBRARY { script: def args = task.ext.args ?: '' - // Strip flags that are managed by the pipeline to prevent silent conflicts - def blocked = ['--no-main-report', '--no-ifs-removal', '--matrices', '--out', - '--temp', '--threads', '--verbose', '--lib', '--f', '--fasta', - '--mass-acc', '--mass-acc-ms1', '--window', - '--individual-mass-acc', '--individual-windows', - '--out-lib', '--use-quant', '--gen-spec-lib', '--rt-profiling', - '--monitor-mod', '--var-mod', '--fixed-mod', - '--channels', '--lib-fixed-mod', '--original-mods'] - // Sort by length descending so longer flags (e.g. --mass-acc-ms1) are matched before shorter prefixes (--mass-acc) - blocked.sort { a -> -a.length() }.each { flag -> - def flagPattern = '(?<=^|\\s)' + java.util.regex.Pattern.quote(flag) + '(?=\\s|\$)(\\s+(?!-{1,2}[a-zA-Z])\\S+)*' - if (args =~ flagPattern) { - log.warn "DIA-NN: '${flag}' is managed by the pipeline for ASSEMBLE_EMPIRICAL_LIBRARY and will be stripped." - args = args.replaceAll(flagPattern, '').trim() - } - } + // Strip flags managed by the pipeline from extra_args to prevent silent conflicts. + // Blocked flags are defined centrally in lib/BlockedFlags.groovy — edit there, not here. + args = BlockedFlags.strip('ASSEMBLE_EMPIRICAL_LIBRARY', args, log) if (params.mass_acc_automatic) { mass_acc = '--individual-mass-acc' @@ -50,9 +38,12 @@ process ASSEMBLE_EMPIRICAL_LIBRARY { mass_acc = '--individual-mass-acc' } scan_window = params.scan_window_automatic ? '--individual-windows' : "--window $params.scan_window" - diann_no_peptidoforms = params.diann_no_peptidoforms ? "--no-peptidoforms" : "" - diann_tims_sum = params.diann_tims_sum ? "--quant-tims-sum" : "" - diann_im_window = params.diann_im_window ? "--im-window $params.diann_im_window" : "" + scoring_mode = params.scoring_mode == 'proteoforms' ? '--proteoforms' : + params.scoring_mode == 'peptidoforms' ? '--peptidoforms' : '' + aa_eq = params.aa_eq ? '--aa-eq' : '' + diann_tims_sum = params.tims_sum ? "--quant-tims-sum" : "" + diann_im_window = params.im_window ? "--im-window $params.im_window" : "" + diann_dda_flag = meta.acquisition_method == 'dda' ? "--dda" : "" """ # Precursor Tolerance value was: ${meta['precursormasstolerance']} @@ -69,20 +60,25 @@ process ASSEMBLE_EMPIRICAL_LIBRARY { --lib ${lib} \\ --threads ${task.cpus} \\ --out-lib empirical_library \\ - --verbose $params.diann_debug \\ + --verbose $params.debug_level \\ --rt-profiling \\ --temp ./quant/ \\ --use-quant \\ ${mass_acc} \\ ${scan_window} \\ --gen-spec-lib \\ - ${diann_no_peptidoforms} \\ + ${scoring_mode} \\ + ${aa_eq} \\ ${diann_tims_sum} \\ ${diann_im_window} \\ + ${diann_dda_flag} \\ \${mod_flags} \\ - $args + $args \\ + 2>&1 | tee assemble_empirical_library.log - cp report.log.txt assemble_empirical_library.log + if [ -f report.log.txt ]; then + cp report.log.txt assemble_empirical_library.log + fi cat <<-END_VERSIONS > versions.yml "${task.process}": diff --git a/modules/local/diann/assemble_empirical_library/meta.yml b/modules/local/diann/assemble_empirical_library/meta.yml index 7fa6585..4355482 100644 --- a/modules/local/diann/assemble_empirical_library/meta.yml +++ b/modules/local/diann/assemble_empirical_library/meta.yml @@ -29,7 +29,7 @@ output: pattern: "empirical_library.tsv" - log: type: file - description: DIA-NN log file + description: DIA-NN log file (captured via tee, overwritten with native log if available) pattern: "assemble_empirical_library.log" - versions: type: file diff --git a/modules/local/diann/diann_msstats/main.nf b/modules/local/diann/diann_msstats/main.nf index 4374b58..4ad3b0c 100644 --- a/modules/local/diann/diann_msstats/main.nf +++ b/modules/local/diann/diann_msstats/main.nf @@ -3,8 +3,8 @@ process DIANN_MSSTATS { label 'process_medium' container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/quantms-utils:0.0.28--pyh106432d_0' : - 'biocontainers/quantms-utils:0.0.28--pyh106432d_0' }" + 'https://depot.galaxyproject.org/singularity/quantms-utils:0.0.29--pyhdfd78af_0' : + 'biocontainers/quantms-utils:0.0.29--pyhdfd78af_0' }" input: path(report) @@ -18,10 +18,11 @@ process DIANN_MSSTATS { script: def args = task.ext.args ?: '' """ + set -o pipefail quantmsutilsc diann2msstats \\ --report ${report} \\ --exp_design ${exp_design} \\ - --qvalue_threshold $params.protein_level_fdr_cutoff \\ + --qvalue_threshold $params.precursor_qvalue \\ $args \\ 2>&1 | tee convert_report.log diff --git a/modules/local/diann/diann_msstats/meta.yml b/modules/local/diann/diann_msstats/meta.yml index ac1f147..a440d61 100644 --- a/modules/local/diann/diann_msstats/meta.yml +++ b/modules/local/diann/diann_msstats/meta.yml @@ -17,7 +17,7 @@ input: pattern: "*.tsv" - exp_design: type: file - description: An experimental design file including Sample and replicates column et al. + description: An experimental design file including Sample and replicates column etc. pattern: "*.tsv" - report_pr: type: file diff --git a/modules/local/diann/final_quantification/main.nf b/modules/local/diann/final_quantification/main.nf index cb7d481..fcbd4ca 100644 --- a/modules/local/diann/final_quantification/main.nf +++ b/modules/local/diann/final_quantification/main.nf @@ -2,6 +2,7 @@ process FINAL_QUANTIFICATION { tag "$meta.experiment_id" label 'process_high' label 'diann' + label 'error_retry' container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? 'https://containers.biocontainers.pro/s3/SingImgsRepo/diann/v1.8.1_cv1/diann_v1.8.1_cv1.img' : @@ -20,20 +21,23 @@ process FINAL_QUANTIFICATION { path(diann_config) output: - // DIA-NN 2.0 don't return report in tsv format - path "diann_report.{tsv,parquet}", emit: main_report, optional: true + // DIA-NN < 1.9 produces TSV; >= 1.9 produces parquet (ignores .tsv in --out) + // Both are optional since only one format will be produced per version + path "diann_report.parquet", emit: main_report_parquet, optional: true + path "diann_report.tsv", emit: main_report_tsv, optional: true path "diann_report.manifest.txt", emit: report_manifest, optional: true path "diann_report.protein_description.tsv", emit: protein_description, optional: true - path "diann_report.stats.tsv", emit: report_stats - path "diann_report.pr_matrix.tsv", emit: pr_matrix - path "diann_report.pg_matrix.tsv", emit: pg_matrix - path "diann_report.gg_matrix.tsv", emit: gg_matrix - path "diann_report.unique_genes_matrix.tsv", emit: unique_gene_matrix + path "diann_report.stats.tsv", emit: report_stats, optional: true + path "diann_report.pr_matrix.tsv", emit: pr_matrix, optional: true + path "diann_report.pg_matrix.tsv", emit: pg_matrix, optional: true + path "diann_report.gg_matrix.tsv", emit: gg_matrix, optional: true + path "diann_report.unique_genes_matrix.tsv", emit: unique_gene_matrix, optional: true path "diannsummary.log", emit: log // Different library files format are exported due to different DIA-NN versions path "empirical_library.tsv", emit: final_speclib, optional: true path "empirical_library.tsv.skyline.speclib", emit: skyline_speclib, optional: true + path "*.site_report.parquet", emit: site_report, optional: true path "versions.yml", emit: versions when: @@ -41,35 +45,29 @@ process FINAL_QUANTIFICATION { script: def args = task.ext.args ?: '' - // Strip flags that are managed by the pipeline to prevent silent conflicts - def blocked = ['--no-main-report', '--gen-spec-lib', '--out-lib', '--no-ifs-removal', - '--temp', '--threads', '--verbose', '--lib', '--f', '--fasta', - '--use-quant', '--matrices', '--out', '--relaxed-prot-inf', '--pg-level', - '--qvalue', '--window', '--individual-windows', - '--species-genes', '--report-decoys', '--xic', '--no-norm', - '--monitor-mod', '--var-mod', '--fixed-mod', - '--channels', '--lib-fixed-mod', '--original-mods'] - // Sort by length descending so longer flags (e.g. --individual-windows) are matched before shorter prefixes (--window) - blocked.sort { a -> -a.length() }.each { flag -> - def flagPattern = '(?<=^|\\s)' + java.util.regex.Pattern.quote(flag) + '(?=\\s|\$)(\\s+(?!-{1,2}[a-zA-Z])\\S+)*' - if (args =~ flagPattern) { - log.warn "DIA-NN: '${flag}' is managed by the pipeline for FINAL_QUANTIFICATION and will be stripped." - args = args.replaceAll(flagPattern, '').trim() - } - } + // Strip flags managed by the pipeline from extra_args to prevent silent conflicts. + // Blocked flags are defined centrally in lib/BlockedFlags.groovy — edit there, not here. + args = BlockedFlags.strip('FINAL_QUANTIFICATION', args, log) scan_window = params.scan_window_automatic ? "--individual-windows" : "--window $params.scan_window" species_genes = params.species_genes ? "--species-genes": "" - no_norm = params.diann_normalize ? "" : "--no-norm" - report_decoys = params.diann_report_decoys ? "--report-decoys": "" - diann_export_xic = params.diann_export_xic ? "--xic": "" - // --direct-quant only exists in DIA-NN >= 1.9.2 (QuantUMS counterpart); skip for older versions - quantums = params.quantums ? "" : (params.diann_version >= '1.9' ? "--direct-quant" : "") + no_norm = params.normalize ? "" : "--no-norm" + report_decoys = params.report_decoys ? "--report-decoys": "" + diann_export_xic = params.export_xic ? "--xic": "" + // --direct-quant exists in DIA-NN >= 1.9.2 (QuantUMS counterpart); skip for older versions + quantums = params.quantums ? "" : (VersionUtils.versionAtLeast(params.diann_version, '1.9.2') ? "--direct-quant" : "") quantums_train_runs = params.quantums_train_runs ? "--quant-train-runs $params.quantums_train_runs": "" quantums_sel_runs = params.quantums_sel_runs ? "--quant-sel-runs $params.quantums_sel_runs": "" quantums_params = params.quantums_params ? "--quant-params $params.quantums_params": "" - diann_no_peptidoforms = params.diann_no_peptidoforms ? "--no-peptidoforms" : "" - diann_use_quant = params.diann_use_quant ? "--use-quant" : "" + scoring_mode = params.scoring_mode == 'proteoforms' ? '--proteoforms' : + params.scoring_mode == 'peptidoforms' ? '--peptidoforms' : '' + aa_eq = params.aa_eq ? '--aa-eq' : '' + diann_use_quant = params.use_quant ? "--use-quant" : "" + diann_dda_flag = meta.acquisition_method == 'dda' ? "--dda" : "" + diann_export_quant = params.export_quant ? "--export-quant" : "" + diann_site_ms1_quant = params.site_ms1_quant ? "--site-ms1-quant" : "" + diann_channel_run_norm = params.channel_run_norm ? "--channel-run-norm" : "" + diann_channel_spec_norm = params.channel_spec_norm ? "--channel-spec-norm" : "" """ # Notes: if .quant files are passed, mzml/.d files are not accessed, so the name needs to be passed but files @@ -82,7 +80,7 @@ process FINAL_QUANTIFICATION { --fasta ${fasta} \\ --f ${(ms_files as List).join(' --f ')} \\ --threads ${task.cpus} \\ - --verbose $params.diann_debug \\ + --verbose $params.debug_level \\ --temp ./quant/ \\ --relaxed-prot-inf \\ --pg-level $params.pg_level \\ @@ -90,19 +88,30 @@ process FINAL_QUANTIFICATION { ${no_norm} \\ --matrices \\ --out diann_report.tsv \\ - --qvalue $params.protein_level_fdr_cutoff \\ + --qvalue $params.precursor_qvalue \\ + --matrix-qvalue $params.matrix_qvalue \\ + --matrix-spec-q $params.matrix_spec_q \\ ${report_decoys} \\ ${diann_export_xic} \\ ${quantums} \\ ${quantums_train_runs} \\ ${quantums_sel_runs} \\ ${quantums_params} \\ - ${diann_no_peptidoforms} \\ + ${scoring_mode} \\ + ${aa_eq} \\ ${diann_use_quant} \\ + ${diann_dda_flag} \\ + ${diann_export_quant} \\ + ${diann_site_ms1_quant} \\ + ${diann_channel_run_norm} \\ + ${diann_channel_spec_norm} \\ \${mod_flags} \\ - $args + $args \\ + 2>&1 | tee diannsummary.log - cp diann_report.log.txt diannsummary.log + if [ -f diann_report.log.txt ]; then + cp diann_report.log.txt diannsummary.log + fi cat <<-END_VERSIONS > versions.yml "${task.process}": diff --git a/modules/local/diann/final_quantification/meta.yml b/modules/local/diann/final_quantification/meta.yml index 23f64c7..9db0e12 100644 --- a/modules/local/diann/final_quantification/meta.yml +++ b/modules/local/diann/final_quantification/meta.yml @@ -27,10 +27,14 @@ input: description: Identification and Quantification file from DIA-NN. pattern: "*.quant" output: - - main_report: + - main_report_parquet: type: file - description: A text table containing precursor and protein IDs, as well as plenty of associated information. - pattern: "*.tsv" + description: DIA-NN main report in parquet format (DIA-NN >= 1.9) + pattern: "diann_report.parquet" + - main_report_tsv: + type: file + description: DIA-NN main report in TSV format (DIA-NN < 1.9) + pattern: "diann_report.tsv" - pr_matrix: type: file description: A text table containing normalized quantities for precursors. They are filtered at 1% FDR, using both global and run-specific q-values for precursors @@ -45,7 +49,7 @@ output: pattern: "*.tsv" - log: type: file - description: DIA-NN log file + description: DIA-NN log file (captured via tee, overwritten with native log if available) pattern: "diannsummary.log" - versions: type: file diff --git a/modules/local/diann/fine_tune_models/main.nf b/modules/local/diann/fine_tune_models/main.nf new file mode 100644 index 0000000..be798e7 --- /dev/null +++ b/modules/local/diann/fine_tune_models/main.nf @@ -0,0 +1,56 @@ +process FINE_TUNE_MODELS { + tag "fine_tune" + label 'process_medium' + label 'diann' + + container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? + 'https://containers.biocontainers.pro/s3/SingImgsRepo/diann/v1.8.1_cv1/diann_v1.8.1_cv1.img' : + 'docker.io/biocontainers/diann:v1.8.1_cv1' }" + + input: + path(tune_lib) + path(fasta) + path(diann_config) + + output: + path "*.dict.txt", emit: tokens + path "*.{tuned_rt,rt.d0}.pt", emit: rt_model, optional: true + path "*.{tuned_im,im.d0}.pt", emit: im_model, optional: true + path "*.{tuned_fr,fr.d0}.pt", emit: fr_model, optional: true + path "fine_tune.log", emit: log + path "versions.yml", emit: versions + + when: + task.ext.when == null || task.ext.when + + script: + def args = task.ext.args ?: '' + // Strip flags managed by the pipeline from extra_args to prevent silent conflicts. + // Blocked flags are defined centrally in lib/BlockedFlags.groovy — edit there, not here. + args = BlockedFlags.strip('FINE_TUNE_MODELS', args, log) + + tune_fr = params.tune_fr ? '--tune-fr' : '' + tune_lr = params.tune_lr ? "--tune-lr ${params.tune_lr}" : '' + + // Extract mod flags from diann_config.cfg so DIA-NN recognises modifications in the library + """ + mod_flags=\$(grep -oP '(--var-mod\\s+\\S+|--fixed-mod\\s+\\S+|--monitor-mod\\s+\\S+|--lib-fixed-mod\\s+\\S+|--original-mods|--channels\\s+.+)' ${diann_config} | tr '\\n' ' ') + + diann --tune-lib ${tune_lib} \\ + --tune-rt \\ + --tune-im \\ + ${tune_fr} \\ + ${tune_lr} \\ + --fasta ${fasta} \\ + --threads ${task.cpus} \\ + --verbose $params.debug_level \\ + \${mod_flags} \\ + $args \\ + 2>&1 | tee fine_tune.log + + cat <<-END_VERSIONS > versions.yml + "${task.process}": + DIA-NN: \$(diann 2>&1 | grep "DIA-NN" | grep -oP "\\d+\\.\\d+(\\.\\w+)*(\\.[\\d]+)?") + END_VERSIONS + """ +} diff --git a/modules/local/diann/fine_tune_models/meta.yml b/modules/local/diann/fine_tune_models/meta.yml new file mode 100644 index 0000000..b097909 --- /dev/null +++ b/modules/local/diann/fine_tune_models/meta.yml @@ -0,0 +1,53 @@ +name: fine_tune_models +description: Fine-tune DIA-NN deep learning prediction models (RT, IM, fragmentation) using an empirical spectral library. +keywords: + - DIA-NN + - fine-tuning + - deep learning + - prediction models +tools: + - DIA-NN: + description: | + DIA-NN - a universal software for data-independent acquisition (DIA) proteomics data processing by Demichev. + homepage: https://github.com/vdemichev/DiaNN + documentation: https://github.com/vdemichev/DiaNN +input: + - tune_lib: + type: file + description: Empirical spectral library to use for fine-tuning (typically from ASSEMBLE_EMPIRICAL_LIBRARY) + pattern: "*.{parquet,tsv}" + - fasta: + type: file + description: Protein sequence database (needed to resolve modification declarations) + pattern: "*.{fasta,fa}" + - diann_config: + type: file + description: DIA-NN config file with modification declarations + pattern: "*.cfg" +output: + - tokens: + type: file + description: Expanded tokenizer dictionary mapping modifications to neural network token IDs + pattern: "*.dict.txt" + - rt_model: + type: file + description: Fine-tuned retention time prediction model (distillation level 0) + pattern: "*.rt.d0.pt" + - im_model: + type: file + description: Fine-tuned ion mobility prediction model (optional, distillation level 0) + pattern: "*.im.d0.pt" + - fr_model: + type: file + description: Fine-tuned fragment ion prediction model (optional, distillation level 0) + pattern: "*.fr.d0.pt" + - log: + type: file + description: DIA-NN fine-tuning log + pattern: "fine_tune.log" + - versions: + type: file + description: File containing software version + pattern: "versions.yml" +authors: + - "@ypriverol" diff --git a/modules/local/diann/generate_cfg/main.nf b/modules/local/diann/generate_cfg/main.nf index 8377030..8d1ce2a 100644 --- a/modules/local/diann/generate_cfg/main.nf +++ b/modules/local/diann/generate_cfg/main.nf @@ -3,8 +3,8 @@ process GENERATE_CFG { label 'process_tiny' container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/quantms-utils:0.0.28--pyh106432d_0' : - 'biocontainers/quantms-utils:0.0.28--pyh106432d_0' }" + 'https://depot.galaxyproject.org/singularity/quantms-utils:0.0.29--pyhdfd78af_0' : + 'biocontainers/quantms-utils:0.0.29--pyhdfd78af_0' }" input: val(meta) @@ -18,6 +18,7 @@ process GENERATE_CFG { def args = task.ext.args ?: '' """ + set -o pipefail quantmsutilsc dianncfg \\ --enzyme "${meta.enzyme}" \\ --fix_mod "${meta.fixedmodifications}" \\ diff --git a/modules/local/diann/individual_analysis/main.nf b/modules/local/diann/individual_analysis/main.nf index 3bdccae..814895f 100644 --- a/modules/local/diann/individual_analysis/main.nf +++ b/modules/local/diann/individual_analysis/main.nf @@ -2,6 +2,7 @@ process INDIVIDUAL_ANALYSIS { tag "$ms_file.baseName" label 'process_high' label 'diann' + label 'error_retry' container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? 'https://containers.biocontainers.pro/s3/SingImgsRepo/diann/v1.8.1_cv1/diann_v1.8.1_cv1.img' : @@ -21,22 +22,9 @@ process INDIVIDUAL_ANALYSIS { script: def args = task.ext.args ?: '' - // Strip flags that are managed by the pipeline to prevent silent conflicts - def blocked = ['--use-quant', '--gen-spec-lib', '--out-lib', '--matrices', '--out', '--rt-profiling', - '--temp', '--threads', '--verbose', '--lib', '--f', '--fasta', - '--mass-acc', '--mass-acc-ms1', '--window', - '--no-ifs-removal', '--no-main-report', '--relaxed-prot-inf', '--pg-level', - '--min-pr-mz', '--max-pr-mz', '--min-fr-mz', '--max-fr-mz', - '--monitor-mod', '--var-mod', '--fixed-mod', - '--channels', '--lib-fixed-mod', '--original-mods'] - // Sort by length descending so longer flags (e.g. --mass-acc-ms1) are matched before shorter prefixes (--mass-acc) - blocked.sort { a -> -a.length() }.each { flag -> - def flagPattern = '(?<=^|\\s)' + java.util.regex.Pattern.quote(flag) + '(?=\\s|\$)(\\s+(?!-{1,2}[a-zA-Z])\\S+)*' - if (args =~ flagPattern) { - log.warn "DIA-NN: '${flag}' is managed by the pipeline for INDIVIDUAL_ANALYSIS and will be stripped." - args = args.replaceAll(flagPattern, '').trim() - } - } + // Strip flags managed by the pipeline from extra_args to prevent silent conflicts. + // Blocked flags are defined centrally in lib/BlockedFlags.groovy — edit there, not here. + args = BlockedFlags.strip('INDIVIDUAL_ANALYSIS', args, log) // Warn about flags that override pipeline-computed calibration values (not blocked, but may change behaviour) ['--individual-windows', '--individual-mass-acc'].each { flag -> @@ -79,9 +67,16 @@ process INDIVIDUAL_ANALYSIS { } } - diann_no_peptidoforms = params.diann_no_peptidoforms ? "--no-peptidoforms" : "" - diann_tims_sum = params.diann_tims_sum ? "--quant-tims-sum" : "" - diann_im_window = params.diann_im_window ? "--im-window $params.diann_im_window" : "" + scoring_mode = params.scoring_mode == 'proteoforms' ? '--proteoforms' : + params.scoring_mode == 'peptidoforms' ? '--peptidoforms' : '' + aa_eq = params.aa_eq ? '--aa-eq' : '' + diann_tims_sum = params.tims_sum ? "--quant-tims-sum" : "" + diann_im_window = params.im_window ? "--im-window $params.im_window" : "" + diann_dda_flag = meta.acquisition_method == 'dda' ? "--dda" : "" + + // Flags removed in DIA-NN 2.3.x — only pass for older versions + no_ifs_removal = VersionUtils.versionLessThan(params.diann_version, '2.3') ? "--no-ifs-removal" : "" + no_main_report = VersionUtils.versionLessThan(params.diann_version, '2.3') ? "--no-main-report" : "" // Per-file scan ranges from SDRF (empty = no flag, DIA-NN auto-detects) min_pr_mz = meta['ms1minmz'] ? "--min-pr-mz ${meta['ms1minmz']}" : "" @@ -97,26 +92,31 @@ process INDIVIDUAL_ANALYSIS { --f ${ms_file} \\ --fasta ${fasta} \\ --threads ${task.cpus} \\ - --verbose $params.diann_debug \\ + --verbose $params.debug_level \\ --temp ./ \\ --mass-acc ${mass_acc_ms2} \\ --mass-acc-ms1 ${mass_acc_ms1} \\ --window ${scan_window} \\ - --no-ifs-removal \\ - --no-main-report \\ + ${no_ifs_removal} \\ + ${no_main_report} \\ --relaxed-prot-inf \\ --pg-level $params.pg_level \\ ${min_pr_mz} \\ ${max_pr_mz} \\ ${min_fr_mz} \\ ${max_fr_mz} \\ - ${diann_no_peptidoforms} \\ + ${scoring_mode} \\ + ${aa_eq} \\ ${diann_tims_sum} \\ ${diann_im_window} \\ + ${diann_dda_flag} \\ \${mod_flags} \\ - $args + $args \\ + 2>&1 | tee ${ms_file.baseName}_final_diann.log - cp report.log.txt ${ms_file.baseName}_final_diann.log + if [ -f report.log.txt ]; then + cp report.log.txt ${ms_file.baseName}_final_diann.log + fi cat <<-END_VERSIONS > versions.yml "${task.process}": diff --git a/modules/local/diann/individual_analysis/meta.yml b/modules/local/diann/individual_analysis/meta.yml index 8c0c677..8550821 100644 --- a/modules/local/diann/individual_analysis/meta.yml +++ b/modules/local/diann/individual_analysis/meta.yml @@ -33,8 +33,8 @@ output: pattern: "*.tsv" - log: type: file - description: DIA-NN log file - pattern: "*_diann.log" + description: DIA-NN log file (captured via tee, overwritten with native log if available) + pattern: "*_final_diann.log" - versions: type: file description: File containing software version diff --git a/modules/local/diann/insilico_library_generation/main.nf b/modules/local/diann/insilico_library_generation/main.nf index b347483..17bf536 100644 --- a/modules/local/diann/insilico_library_generation/main.nf +++ b/modules/local/diann/insilico_library_generation/main.nf @@ -2,6 +2,7 @@ process INSILICO_LIBRARY_GENERATION { tag "$fasta.name" label 'process_medium' label 'diann' + label 'error_retry' container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? 'https://containers.biocontainers.pro/s3/SingImgsRepo/diann/v1.8.1_cv1/diann_v1.8.1_cv1.img' : @@ -10,6 +11,10 @@ process INSILICO_LIBRARY_GENERATION { input: path(fasta) path(diann_config) + val(is_dda) + path(tuned_tokens) // optional: fine-tuned tokenizer dict (pass [] when not used) + path(tuned_rt_model) // optional: fine-tuned RT model (pass [] when not used) + path(tuned_im_model) // optional: fine-tuned IM model (pass [] when not used) output: path "versions.yml", emit: versions @@ -22,34 +27,32 @@ process INSILICO_LIBRARY_GENERATION { script: def args = task.ext.args ?: '' - // Strip flags that are managed by the pipeline to prevent silent conflicts - def blocked = ['--use-quant', '--no-main-report', '--matrices', '--out', - '--temp', '--threads', '--verbose', '--lib', '--f', '--fasta', - '--fasta-search', '--predictor', '--gen-spec-lib', - '--missed-cleavages', '--min-pep-len', '--max-pep-len', - '--min-pr-charge', '--max-pr-charge', '--var-mods', - '--min-pr-mz', '--max-pr-mz', '--min-fr-mz', '--max-fr-mz', - '--met-excision', '--monitor-mod'] - // Sort by length descending so longer flags (e.g. --fasta-search) are matched before shorter prefixes (--fasta, --f) - blocked.sort { a -> -a.length() }.each { flag -> - def flagPattern = '(?<=^|\\s)' + java.util.regex.Pattern.quote(flag) + '(?=\\s|\$)(\\s+(?!-{1,2}[a-zA-Z])\\S+)*' - if (args =~ flagPattern) { - log.warn "DIA-NN: '${flag}' is managed by the pipeline for INSILICO_LIBRARY_GENERATION and will be stripped." - args = args.replaceAll(flagPattern, '').trim() - } - } + // Strip flags managed by the pipeline from extra_args to prevent silent conflicts. + // Blocked flags are defined centrally in lib/BlockedFlags.groovy — edit there, not here. + args = BlockedFlags.strip('INSILICO_LIBRARY_GENERATION', args, log) min_pr_mz = params.min_pr_mz ? "--min-pr-mz $params.min_pr_mz":"" max_pr_mz = params.max_pr_mz ? "--max-pr-mz $params.max_pr_mz":"" min_fr_mz = params.min_fr_mz ? "--min-fr-mz $params.min_fr_mz":"" max_fr_mz = params.max_fr_mz ? "--max-fr-mz $params.max_fr_mz":"" met_excision = params.met_excision ? "--met-excision" : "" - diann_no_peptidoforms = params.diann_no_peptidoforms ? "--no-peptidoforms" : "" + scoring_mode = params.scoring_mode == 'proteoforms' ? '--proteoforms' : + params.scoring_mode == 'peptidoforms' ? '--peptidoforms' : '' + aa_eq = params.aa_eq ? '--aa-eq' : '' + diann_dda_flag = is_dda ? "--dda" : "" + diann_light_models = params.light_models ? "--light-models" : "" + // Fine-tuned model flags — only set when tuned model files are provided + tuned_tokens_flag = tuned_tokens ? "--tokens ${tuned_tokens}" : '' + tuned_rt_flag = tuned_rt_model ? "--rt-model ${tuned_rt_model}" : '' + tuned_im_flag = tuned_im_model ? "--im-model ${tuned_im_model}" : '' + infin_dia_flag = params.enable_infin_dia ? "--infin-dia" : "" + pre_select_flag = (params.enable_infin_dia && params.pre_select) ? "--pre-select $params.pre_select" : "" """ diann `cat ${diann_config}` \\ --fasta ${fasta} \\ --fasta-search \\ + --out silicolibrarygeneration.tsv \\ ${min_pr_mz} \\ ${max_pr_mz} \\ ${min_fr_mz} \\ @@ -62,13 +65,24 @@ process INSILICO_LIBRARY_GENERATION { --var-mods $params.max_mods \\ --threads ${task.cpus} \\ --predictor \\ - --verbose $params.diann_debug \\ + --verbose $params.debug_level \\ --gen-spec-lib \\ - ${diann_no_peptidoforms} \\ + ${scoring_mode} \\ + ${aa_eq} \\ + ${diann_light_models} \\ + ${tuned_tokens_flag} \\ + ${tuned_rt_flag} \\ + ${tuned_im_flag} \\ + ${infin_dia_flag} \\ + ${pre_select_flag} \\ ${met_excision} \\ - ${args} + ${diann_dda_flag} \\ + ${args} \\ + 2>&1 | tee silicolibrarygeneration.log - cp *lib.log.txt silicolibrarygeneration.log + if [ -f silicolibrarygeneration.log.txt ]; then + cp silicolibrarygeneration.log.txt silicolibrarygeneration.log + fi cat <<-END_VERSIONS > versions.yml "${task.process}": diff --git a/modules/local/diann/insilico_library_generation/meta.yml b/modules/local/diann/insilico_library_generation/meta.yml index 5f9d68b..c1af0db 100644 --- a/modules/local/diann/insilico_library_generation/meta.yml +++ b/modules/local/diann/insilico_library_generation/meta.yml @@ -19,6 +19,9 @@ input: type: file description: specifies a configuration file to load options/commands from. pattern: "*.cfg" + - is_dda: + type: boolean + description: Whether DDA mode is enabled (auto-detected from SDRF or set via --dda) output: - predict_speclib: type: file @@ -26,7 +29,7 @@ output: pattern: "*.predicted.speclib" - log: type: file - description: DIA-NN log file + description: DIA-NN log file (captured via tee, overwritten with native log if available) pattern: "silicolibrarygeneration.log" - versions: type: file diff --git a/modules/local/diann/preliminary_analysis/main.nf b/modules/local/diann/preliminary_analysis/main.nf index 8bb818b..c09d41a 100644 --- a/modules/local/diann/preliminary_analysis/main.nf +++ b/modules/local/diann/preliminary_analysis/main.nf @@ -2,6 +2,7 @@ process PRELIMINARY_ANALYSIS { tag "$ms_file.baseName" label 'process_high' label 'diann' + label 'error_retry' container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? 'https://containers.biocontainers.pro/s3/SingImgsRepo/diann/v1.8.1_cv1/diann_v1.8.1_cv1.img' : @@ -21,27 +22,16 @@ process PRELIMINARY_ANALYSIS { script: def args = task.ext.args ?: '' - // Strip flags that are managed by the pipeline to prevent silent conflicts - def blocked = ['--use-quant', '--gen-spec-lib', '--out-lib', '--matrices', '--out', - '--temp', '--threads', '--verbose', '--lib', '--f', '--fasta', - '--mass-acc', '--mass-acc-ms1', '--window', - '--quick-mass-acc', '--min-corr', '--corr-diff', '--time-corr-only', - '--min-pr-mz', '--max-pr-mz', '--min-fr-mz', '--max-fr-mz', - '--monitor-mod', '--var-mod', '--fixed-mod', '--no-prot-inf', - '--channels', '--lib-fixed-mod', '--original-mods'] - // Sort by length descending so longer flags (e.g. --mass-acc-ms1) are matched before shorter prefixes (--mass-acc) - blocked.sort { a -> -a.length() }.each { flag -> - def flagPattern = '(?<=^|\\s)' + java.util.regex.Pattern.quote(flag) + '(?=\\s|\$)(\\s+(?!-{1,2}[a-zA-Z])\\S+)*' - if (args =~ flagPattern) { - log.warn "DIA-NN: '${flag}' is managed by the pipeline for PRELIMINARY_ANALYSIS and will be stripped." - args = args.replaceAll(flagPattern, '').trim() - } - } + // Strip flags managed by the pipeline from extra_args to prevent silent conflicts. + // Blocked flags are defined centrally in lib/BlockedFlags.groovy — edit there, not here. + args = BlockedFlags.strip('PRELIMINARY_ANALYSIS', args, log) // Performance flags for preliminary analysis calibration step quick_mass_acc = params.quick_mass_acc ? "--quick-mass-acc" : "" performance_flags = params.performance_mode ? "--min-corr 2 --corr-diff 1 --time-corr-only" : "" - diann_no_peptidoforms = params.diann_no_peptidoforms ? "--no-peptidoforms" : "" + scoring_mode = params.scoring_mode == 'proteoforms' ? '--proteoforms' : + params.scoring_mode == 'peptidoforms' ? '--peptidoforms' : '' + aa_eq = params.aa_eq ? '--aa-eq' : '' // I am using here the ["key"] syntax, since the preprocessed meta makes // was evaluating to null when using the dot notation. @@ -65,8 +55,9 @@ process PRELIMINARY_ANALYSIS { // Notes: Use double quotes for params, so that it is escaped in the shell. scan_window = params.scan_window_automatic ? '' : "--window $params.scan_window" - diann_tims_sum = params.diann_tims_sum ? "--quant-tims-sum" : "" - diann_im_window = params.diann_im_window ? "--im-window $params.diann_im_window" : "" + diann_tims_sum = params.tims_sum ? "--quant-tims-sum" : "" + diann_im_window = params.im_window ? "--im-window $params.im_window" : "" + diann_dda_flag = meta.acquisition_method == 'dda' ? "--dda" : "" // Per-file scan ranges from SDRF (empty = no flag, DIA-NN auto-detects) min_pr_mz = meta['ms1minmz'] ? "--min-pr-mz ${meta['ms1minmz']}" : "" @@ -88,7 +79,7 @@ process PRELIMINARY_ANALYSIS { diann --lib ${predict_library} \\ --f ${ms_file} \\ --threads ${task.cpus} \\ - --verbose $params.diann_debug \\ + --verbose $params.debug_level \\ ${scan_window} \\ --temp ./ \\ ${mass_acc} \\ @@ -98,14 +89,19 @@ process PRELIMINARY_ANALYSIS { ${max_pr_mz} \\ ${min_fr_mz} \\ ${max_fr_mz} \\ - ${diann_no_peptidoforms} \\ + ${scoring_mode} \\ + ${aa_eq} \\ ${diann_tims_sum} \\ ${diann_im_window} \\ --no-prot-inf \\ + ${diann_dda_flag} \\ \${mod_flags} \\ - $args + $args \\ + 2>&1 | tee ${ms_file.baseName}_diann.log - cp report.log.txt ${ms_file.baseName}_diann.log + if [ -f report.log.txt ]; then + cp report.log.txt ${ms_file.baseName}_diann.log + fi cat <<-END_VERSIONS > versions.yml "${task.process}": diff --git a/modules/local/diann/preliminary_analysis/meta.yml b/modules/local/diann/preliminary_analysis/meta.yml index 5eb752a..5185c32 100644 --- a/modules/local/diann/preliminary_analysis/meta.yml +++ b/modules/local/diann/preliminary_analysis/meta.yml @@ -28,7 +28,7 @@ output: pattern: "*.quant" - log: type: file - description: DIA-NN log file + description: DIA-NN log file (captured via tee, overwritten with native log if available) pattern: "*_diann.log" - version: type: file diff --git a/modules/local/pmultiqc/main.nf b/modules/local/pmultiqc/main.nf index facf4e8..2218a9f 100644 --- a/modules/local/pmultiqc/main.nf +++ b/modules/local/pmultiqc/main.nf @@ -2,8 +2,8 @@ process PMULTIQC { label 'process_high' container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/pmultiqc:0.0.43--pyhdfd78af_0' : - 'biocontainers/pmultiqc:0.0.43--pyhdfd78af_0' }" + 'https://depot.galaxyproject.org/singularity/pmultiqc:0.0.44--pyhdfd78af_0' : + 'biocontainers/pmultiqc:0.0.44--pyhdfd78af_0' }" input: path multiqc_inputs, stageAs: 'results/*' diff --git a/modules/local/pmultiqc/meta.yml b/modules/local/pmultiqc/meta.yml index adf63f2..fcca33e 100644 --- a/modules/local/pmultiqc/meta.yml +++ b/modules/local/pmultiqc/meta.yml @@ -23,7 +23,7 @@ output: pattern: "*.html" - quantmsdb: type: file - description: Sqlite3 database file stored protein psm and quantification information + description: SQLite3 database file that stores protein, PSM, and quantification information pattern: "*.db" - data: type: dir diff --git a/modules/local/samplesheet_check/main.nf b/modules/local/samplesheet_check/main.nf index f2b7112..129a646 100644 --- a/modules/local/samplesheet_check/main.nf +++ b/modules/local/samplesheet_check/main.nf @@ -4,8 +4,8 @@ process SAMPLESHEET_CHECK { label 'process_tiny' container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/quantms-utils:0.0.28--pyh106432d_0' : - 'biocontainers/quantms-utils:0.0.28--pyh106432d_0' }" + 'https://depot.galaxyproject.org/singularity/quantms-utils:0.0.29--pyhdfd78af_0' : + 'biocontainers/quantms-utils:0.0.29--pyhdfd78af_0' }" input: path input_file @@ -23,6 +23,7 @@ process SAMPLESHEET_CHECK { def string_use_ols_cache_only = params.use_ols_cache_only == true ? "--use_ols_cache_only" : "" """ + set -o pipefail # Get basename and create output filename BASENAME=\$(basename "${input_file}") # Remove .sdrf.tsv, .sdrf.csv, or .sdrf extension (in that order to match longest first) diff --git a/modules/local/sdrf_parsing/main.nf b/modules/local/sdrf_parsing/main.nf index fabc1a8..1ae0c3b 100644 --- a/modules/local/sdrf_parsing/main.nf +++ b/modules/local/sdrf_parsing/main.nf @@ -22,6 +22,7 @@ process SDRF_PARSING { def diann_version_flag = params.diann_version ? "--diann_version '${params.diann_version}'" : '' """ + set -o pipefail parse_sdrf convert-diann \\ -s ${sdrf} \\ ${mod_loc_flag} \\ diff --git a/modules/local/sdrf_parsing/meta.yml b/modules/local/sdrf_parsing/meta.yml index 7c311f4..846cfaa 100644 --- a/modules/local/sdrf_parsing/meta.yml +++ b/modules/local/sdrf_parsing/meta.yml @@ -28,7 +28,7 @@ output: type: file description: log file pattern: "*.log" - - version: + - versions: type: file description: File containing software version pattern: "versions.yml" diff --git a/modules/local/utils/decompress_dotd/meta.yml b/modules/local/utils/decompress_dotd/meta.yml index bbc7c58..55330d3 100644 --- a/modules/local/utils/decompress_dotd/meta.yml +++ b/modules/local/utils/decompress_dotd/meta.yml @@ -22,7 +22,7 @@ input: type: file description: | Bruker Raw file archived using tar - pattern: "*.{d.tar,.tar,.gz,.d.tar.gz}" + pattern: "*.{d.tar,tar,gz,d.tar.gz}" output: - meta: type: map diff --git a/modules/local/utils/mzml_statistics/main.nf b/modules/local/utils/mzml_statistics/main.nf index d6eb3e0..f493371 100644 --- a/modules/local/utils/mzml_statistics/main.nf +++ b/modules/local/utils/mzml_statistics/main.nf @@ -4,8 +4,8 @@ process MZML_STATISTICS { label 'process_single' container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/quantms-utils:0.0.28--pyh106432d_0' : - 'biocontainers/quantms-utils:0.0.28--pyh106432d_0' }" + 'https://depot.galaxyproject.org/singularity/quantms-utils:0.0.29--pyhdfd78af_0' : + 'biocontainers/quantms-utils:0.0.29--pyhdfd78af_0' }" input: tuple val(meta), path(ms_file) diff --git a/modules/local/utils/tdf2mzml/main.nf b/modules/local/utils/tdf2mzml/main.nf deleted file mode 100644 index a242935..0000000 --- a/modules/local/utils/tdf2mzml/main.nf +++ /dev/null @@ -1,38 +0,0 @@ -process TDF2MZML { - tag "$meta.id" - label 'process_single' - label 'error_retry' - - container 'quay.io/bigbio/tdf2mzml:latest' // TODO: pin to a specific version tag for reproducibility - - input: - tuple val(meta), path(rawfile) - - output: - tuple val(meta), path("*.mzML"), emit: mzmls_converted - path "versions.yml", emit: versions - path "*.log", emit: log - - script: - def args = task.ext.args ?: '' - def prefix = task.ext.prefix ?: "${meta.id}" - - """ - echo "Converting..." | tee --append ${rawfile.baseName}_conversion.log - tdf2mzml.py -i *.d $args 2>&1 | tee --append ${rawfile.baseName}_conversion.log - - # Rename .mzml to .mzML via temp file to handle case-insensitive filesystems (e.g. macOS) - mv *.mzml __tmp_converted.mzML && mv __tmp_converted.mzML ${file(rawfile.baseName).baseName}.mzML - - # Rename .d directory only if the name differs (avoid 'same file' error) - target_d="${file(rawfile.baseName).baseName}.d" - if [ ! -d "\${target_d}" ]; then - mv *.d "\${target_d}" - fi - - cat <<-END_VERSIONS > versions.yml - "${task.process}": - tdf2mzml.py: \$(tdf2mzml.py --version) - END_VERSIONS - """ -} diff --git a/modules/local/utils/tdf2mzml/meta.yml b/modules/local/utils/tdf2mzml/meta.yml deleted file mode 100644 index ebb90b8..0000000 --- a/modules/local/utils/tdf2mzml/meta.yml +++ /dev/null @@ -1,42 +0,0 @@ -name: tdf2mzml -description: convert raw bruker files to mzml files -keywords: - - raw - - mzML - - .d -tools: - - tdf2mzml: - description: | - It takes a bruker .d raw file as input and outputs indexed mzML - homepage: https://github.com/mafreitas/tdf2mzml - documentation: https://github.com/mafreitas/tdf2mzml -input: - - meta: - type: map - description: | - Groovy Map containing sample information - - rawfile: - type: file - description: | - Bruker .d raw directory - pattern: "*.d" -output: - - meta: - type: map - description: | - Groovy Map containing sample information - e.g. [ id:'MD5', enzyme:trypsin ] - - mzml: - type: file - description: indexed mzML - pattern: "*.mzML" - - log: - type: file - description: log file - pattern: "*.log" - - version: - type: file - description: File containing software version - pattern: "versions.yml" -authors: - - "@jspaezp" diff --git a/nextflow.config b/nextflow.config index 33c5c95..0d48e94 100644 --- a/nextflow.config +++ b/nextflow.config @@ -19,7 +19,9 @@ params { use_ols_cache_only = true // Use only the OLS cache for ontology validation (no network requests) // Tools flags - protein_level_fdr_cutoff = 0.01 + precursor_qvalue = 0.01 // --qvalue: precursor-level q-value threshold for main report + matrix_qvalue = 0.01 // --matrix-qvalue: q-value threshold for output matrices + matrix_spec_q = 0.05 // --matrix-spec-q: run-specific protein q-value for protein/gene matrices // Debug level pp_debug = 0 @@ -49,9 +51,25 @@ params { // DIA-NN: General diann_version = '1.8.1' // Used to control version-dependent flags (e.g. --monitor-mod for 1.8.x) - diann_debug = 3 - diann_speclib = null - diann_extra_args = null + debug_level = 3 + speclib = null + extra_args = null + scoring_mode = 'generic' // Scoring mode: 'generic' (default), 'proteoforms' (variant detection, >= 2.0), 'peptidoforms' (PTM analysis) + aa_eq = false // add '--aa-eq': treat I&L, Q&E, N&D as equivalent during reannotation (essential for entrapment FDR benchmarks) + dda = false // Fallback: explicitly enable DDA when SDRF lacks acquisition method (requires DIA-NN >= 2.3.2) + light_models = false // add '--light-models' for 10x faster library generation (DIA-NN >= 2.0) + export_quant = false // add '--export-quant' for fragment-level parquet export (DIA-NN >= 2.0) + site_ms1_quant = false // add '--site-ms1-quant' for MS1 apex PTM quantification (DIA-NN >= 2.0) + + // DIA-NN: Model fine-tuning (v2.3.2+) + enable_fine_tuning = false // Enable model fine-tuning before the main analysis + tune_n_files = 3 // Number of files to use for the tuning search (largest/best quality recommended) + tune_fr = false // Also fine-tune the fragmentation model (quality-sensitive) + tune_lr = null // Fine-tuning learning rate (default: 0.0005) + + // DIA-NN: InfinDIA (experimental, v2.3.0+) + enable_infin_dia = false // Enable InfinDIA for ultra-large search spaces + pre_select = null // --pre-select N precursor limit for InfinDIA // Optional outputs — control which intermediate files are published save_speclib_tsv = false // Save the TSV spectral library from in-silico generation @@ -68,8 +86,8 @@ params { mass_acc_automatic = true performance_mode = true // add '--min-corr 2 --corr-diff 1 --time-corr-only' quick_mass_acc = true - diann_tims_sum = false // add '--quant-tims-sum' - diann_im_window = null // add '--im-window' + tims_sum = false // add '--quant-tims-sum' + im_window = null // add '--im-window' // DIA-NN: ASSEMBLE_EMPIRICAL_LIBRARY — library assembly skip_preliminary_analysis = false @@ -84,15 +102,16 @@ params { // DIA-NN: FINAL_QUANTIFICATION — summarization & output pg_level = 2 species_genes = false - diann_normalize = true - diann_report_decoys = false - diann_export_xic = false + normalize = true + report_decoys = false + export_xic = false quantums = false quantums_train_runs = null quantums_sel_runs = null quantums_params = null - diann_no_peptidoforms = false // add '--no-peptidoforms' - diann_use_quant = true // add '--use-quant' to FINAL_QUANTIFICATION + use_quant = true // add '--use-quant' to FINAL_QUANTIFICATION + channel_run_norm = false // add '--channel-run-norm' to FINAL_QUANTIFICATION + channel_spec_norm = false // add '--channel-spec-norm' to FINAL_QUANTIFICATION // pmultiqc options enable_pmultiqc = true @@ -237,10 +256,14 @@ profiles { test_dia_2_2_0 { includeConfig 'conf/tests/test_dia_2_2_0.config' } test_latest_dia { includeConfig 'conf/tests/test_latest_dia.config' } test_full_dia { includeConfig 'conf/tests/test_full_dia.config' } + test_dda { includeConfig 'conf/tests/test_dda.config' } + test_dia_skip_preanalysis { includeConfig 'conf/tests/test_dia_skip_preanalysis.config' } // DIA-NN version overrides (used by merge_ci.yml matrix) diann_v1_8_1 { includeConfig 'conf/diann_versions/v1_8_1.config' } diann_v2_1_0 { includeConfig 'conf/diann_versions/v2_1_0.config' } diann_v2_2_0 { includeConfig 'conf/diann_versions/v2_2_0.config' } + diann_v2_3_2 { includeConfig 'conf/diann_versions/v2_3_2.config' } + diann_v2_5_0 { includeConfig 'conf/diann_versions/v2_5_0.config' } dev { includeConfig 'conf/dev.config' } pride_slurm { includeConfig 'conf/pride_codon_slurm.config' } manual_wave { includeConfig 'conf/wave.config' } @@ -362,7 +385,7 @@ manifest { mainScript = 'main.nf' defaultBranch = 'main' nextflowVersion = '!>=25.04.0' - version = '1.0.0' + version = '2.0.0' doi = '10.5281/zenodo.15573386' } diff --git a/nextflow_schema.json b/nextflow_schema.json index 8708cf7..31823d7 100644 --- a/nextflow_schema.json +++ b/nextflow_schema.json @@ -248,22 +248,6 @@ }, "fa_icon": "fas fa-star-half-alt" }, - "protein_inference": { - "title": "Protein inference", - "type": "object", - "description": "To group proteins, calculate scores on the protein (group) level and to potentially modify associations from peptides to proteins.", - "default": "", - "properties": { - "protein_level_fdr_cutoff": { - "type": "number", - "description": "The experiment-wide protein (group)-level FDR cutoff. Default: 0.01", - "default": 0.01, - "fa_icon": "fas fa-filter", - "help_text": "This can be protein level if 'strictly_unique_peptides' are used for protein quantification. See [`--protein_quant`](#protein_quant)" - } - }, - "fa_icon": "fab fa-hubspot" - }, "DIA-NN": { "title": "DIA-NN", "type": "object", @@ -337,25 +321,46 @@ "enum": [0, 1, 2], "default": 2 }, + "precursor_qvalue": { + "type": "number", + "description": "Precursor-level q-value filtering threshold for the DIA-NN main report. Maps to --qvalue.", + "default": 0.01, + "fa_icon": "fas fa-filter", + "help_text": "Controls how strictly precursor identifications are filtered in the DIA-NN main report. For proteogenomics with variant databases, the standard 0.01 (1%) is recommended." + }, + "matrix_qvalue": { + "type": "number", + "description": "Q-value threshold for DIA-NN output matrices (pr_matrix, pg_matrix, etc.). Maps to --matrix-qvalue.", + "default": 0.01, + "fa_icon": "fas fa-filter", + "help_text": "Controls the global q-value filtering applied when generating the quantification matrices. Default matches DIA-NN's built-in default of 1%." + }, + "matrix_spec_q": { + "type": "number", + "description": "Run-specific protein q-value filter for protein/gene matrices. Maps to --matrix-spec-q.", + "default": 0.05, + "fa_icon": "fas fa-filter", + "help_text": "An additional run-specific protein-level FDR filter applied to the protein and gene matrices. Default matches DIA-NN's built-in default of 5%. For proteogenomics/variant detection, consider setting to 1.0 to retain variant proteins that lack sufficient unique peptides for protein-level confidence." + }, "species_genes": { "type": "boolean", "description": "Instructs DIA-NN to add the organism identifier to the gene names", "fa_icon": "far fa-check-square" }, - "diann_speclib": { + "speclib": { "type": "string", "description": "The spectral library to use for DIA-NN", "fa_icon": "fas fa-file", "help_text": "If passed, will use that spectral library to carry out the DIA-NN search, instead of predicting one from the fasta file.", "hidden": false }, - "diann_report_decoys": { + "report_decoys": { "type": "boolean", "description": "Save decoy PSMs to the main .parquet report for DIA-NN 2.0.*", "fa_icon": "far fa-check-square", "hidden": false }, - "diann_export_xic": { + "export_xic": { "type": "boolean", "description": "instructs DIA-NN to extract MS1/fragment chromatograms for identified precursors within X seconds from the elution apex, with X set to 10s if not provided;equivalent to the 'XICs' option in the GUI", "fa_icon": "far fa-check-square", @@ -388,36 +393,58 @@ "help_text": "Sets the DIA-NN parameter --quant-params. Provide previously obtained QuantUMS parameters to use instead of training new ones.", "hidden": false }, - "diann_no_peptidoforms": { + "scoring_mode": { + "type": "string", + "default": "generic", + "enum": ["generic", "proteoforms", "peptidoforms"], + "description": "DIA-NN scoring mode. 'generic' maximizes IDs (default). 'proteoforms' validates single-residue differences — recommended for proteogenomics/variant detection (requires DIA-NN >= 2.0). 'peptidoforms' provides extra peptidoform q-values for PTM analysis.", + "fa_icon": "fas fa-bullseye", + "help_text": "Controls DIA-NN's decoy generation strategy. In 'generic' mode, decoys are fully shuffled sequences. In 'proteoforms' mode, decoys differ by a single residue — directly modelling the confusion between canonical and variant peptides (e.g., COSMIC missense mutations). In 'peptidoforms' mode, generic q-values are calculated plus extra peptidoform-level q-values for PTM localization." + }, + "aa_eq": { "type": "boolean", - "description": "Set no-peptidoforms mode: disables automatic peptidoform scoring when variable modifications are declared (DIA-NN: not recommended).", - "fa_icon": "far fa-check-square", - "default": false + "default": false, + "description": "Treat I&L, Q&E, N&D as equivalent amino acids during reannotation. Essential for entrapment FDR benchmarks.", + "fa_icon": "fas fa-equals", + "help_text": "Maps to DIA-NN's --aa-eq flag. When enabled, peptides are matched to proteins considering isoleucine/leucine, glutamine/glutamic acid, and asparagine/aspartic acid as equivalent. This is essential when benchmarking FDR using entrapment databases, as without it the assessment measures the ability to distinguish deamidated peptides rather than true FDR." }, - "diann_tims_sum": { + "tims_sum": { "type": "boolean", "description": "Enable '--quant-tims-sum' for slice/scanning timsTOF methods (highly recommended for Synchro-PASEF).", "fa_icon": "far fa-check-square", "default": false }, - "diann_im_window": { + "im_window": { "type": "number", "description": "Set '--im-window' to ensure the IM extraction window is not smaller than the specified value.", "fa_icon": "fas fa-filter" }, - "diann_use_quant": { + "use_quant": { "type": "boolean", "description": "Set '--use-quant' to reuse existing .quant files if available.", "fa_icon": "far fa-check-square", "default": true }, + + "channel_run_norm": { + "type": "boolean", + "description": "Set '--channel-run-norm'. Run-specific normalisation for multiplexing (e.g., Protein Turnover SILAC)", + "fa_icon": "far fa-check-square", + "default": false + }, + "channel_spec_norm": { + "type": "boolean", + "description": "Set '--channel-spec-norm'. Channel-specific normalisation for multiplexing (Independent Samples)", + "fa_icon": "far fa-check-square", + "default": false + }, "skip_preliminary_analysis": { "type": "boolean", "description": "Skip the preliminary analysis step, thus use the passed spectral library as-is instead of generating a local consensus library.", "fa_icon": "fas fa-forward", "hidden": false }, - "diann_debug": { + "debug_level": { "type": "integer", "description": "Debug level", "default": 3, @@ -425,7 +452,7 @@ "enum": [0, 1, 2, 3, 4], "hidden": true }, - "diann_normalize": { + "normalize": { "type": "boolean", "description": "Enable cross-run normalization between runs by diann.", "default": true, @@ -449,13 +476,72 @@ "fa_icon": "fas fa-filter", "hidden": true }, - "diann_extra_args": { + "extra_args": { "type": "string", "description": "Extra arguments appended to all DIA-NN steps. Flags incompatible with specific steps are automatically stripped with a warning.", "fa_icon": "fas fa-terminal", "hidden": false, "help_text": "Pass additional DIA-NN command-line arguments that will be appended to all DIA-NN steps (INSILICO_LIBRARY_GENERATION, PRELIMINARY_ANALYSIS, ASSEMBLE_EMPIRICAL_LIBRARY, INDIVIDUAL_ANALYSIS, FINAL_QUANTIFICATION). Flags that conflict with a specific step are automatically stripped with a warning. For step-specific overrides, use custom Nextflow config files with ext.args." }, + "dda": { + "type": "boolean", + "description": "Explicitly enable DDA mode. Normally auto-detected from the SDRF acquisition method column. Use only when SDRF lacks this column. Requires DIA-NN >= 2.3.2.", + "fa_icon": "fas fa-flask", + "default": false + }, + "light_models": { + "type": "boolean", + "description": "Enable --light-models for 10x faster in-silico library generation (DIA-NN >= 2.0).", + "fa_icon": "fas fa-bolt", + "default": false + }, + "export_quant": { + "type": "boolean", + "description": "Enable --export-quant for fragment-level parquet data export (DIA-NN >= 2.0).", + "fa_icon": "fas fa-file-export", + "default": false + }, + "site_ms1_quant": { + "type": "boolean", + "description": "Enable --site-ms1-quant to use MS1 apex intensities for PTM site quantification (DIA-NN >= 2.0).", + "fa_icon": "fas fa-crosshairs", + "default": false + }, + "enable_fine_tuning": { + "type": "boolean", + "description": "Enable DL model fine-tuning before the main analysis. Runs a tuning search on a file subset, fine-tunes RT/IM models, then runs the full pipeline with tuned models. Requires DIA-NN >= 2.3.2.", + "fa_icon": "fas fa-brain", + "default": false, + "help_text": "When enabled, Phase 0 runs before the main pipeline: (1) in-silico library generation with default models, (2) preliminary analysis + assembly on a subset of files (controlled by --tune_n_files), (3) fine-tuning RT/IM models on the resulting empirical library, (4) re-generation of the in-silico library with tuned models. The main pipeline then runs from preliminary analysis using the tuned library." + }, + "tune_n_files": { + "type": "integer", + "description": "Number of files to use for the fine-tuning search. Use the largest/best-quality files.", + "fa_icon": "fas fa-layer-group", + "default": 3 + }, + "tune_fr": { + "type": "boolean", + "description": "Also fine-tune the fragmentation model (quality-sensitive — verify results vs base model).", + "fa_icon": "fas fa-flask", + "default": false + }, + "tune_lr": { + "type": "number", + "description": "Fine-tuning learning rate (default in DIA-NN: 0.0005). Maps to --tune-lr.", + "fa_icon": "fas fa-sliders-h" + }, + "enable_infin_dia": { + "type": "boolean", + "description": "Enable InfinDIA for ultra-large search spaces (DIA-NN >= 2.3.0). Experimental.", + "fa_icon": "fas fa-infinity", + "default": false + }, + "pre_select": { + "type": "integer", + "description": "Set --pre-select N precursor limit for InfinDIA pre-search.", + "fa_icon": "fas fa-filter" + }, "save_speclib_tsv": { "type": "boolean", "default": false, @@ -674,9 +760,6 @@ { "$ref": "#/$defs/psm_re_scoring_general" }, - { - "$ref": "#/$defs/protein_inference" - }, { "$ref": "#/$defs/DIA-NN" }, diff --git a/subworkflows/local/create_input_channel/main.nf b/subworkflows/local/create_input_channel/main.nf index 588e144..1a8f144 100644 --- a/subworkflows/local/create_input_channel/main.nf +++ b/subworkflows/local/create_input_channel/main.nf @@ -18,19 +18,31 @@ workflow CREATE_INPUT_CHANNEL { ch_expdesign = SDRF_PARSING.out.ch_expdesign ch_diann_cfg = SDRF_PARSING.out.ch_diann_cfg - def enzymes = new HashSet() - def files = new HashSet() - // Extract experiment_id from the SDRF filename via the value channel - ch_sdrf_val = ch_sdrf.first() - ch_experiment_id = ch_sdrf_val.map { sdrf_file -> file(sdrf_file).baseName } + // Extract experiment_id from the SDRF filename + ch_experiment_id = ch_sdrf.map { sdrf_file -> file(sdrf_file).baseName } ch_experiment_id .combine(ch_expdesign) .splitCsv(header: true, sep: '\t') .map { experiment_id, row -> + def filestr + if (!params.root_folder) { + filestr = row.URI?.toString()?.trim() ? row.URI.toString() : row.Filename.toString() + } else { + filestr = row.Filename.toString() + filestr = params.root_folder + File.separator + filestr + filestr = (params.local_input_type + ? filestr.take(filestr.lastIndexOf('.')) + '.' + params.local_input_type + : filestr) + } + return [filestr, experiment_id, row] + } + .groupTuple(by: 0) + .map { filestr, experiment_ids, rows -> + def experiment_id = experiment_ids[0] def wrapper = [acquisition_method: "", experiment_id: experiment_id] - create_meta_channel(row, enzymes, files, wrapper) + return create_meta_channel_grouped(filestr, rows, wrapper) } .set { ch_meta_config_dia } @@ -42,66 +54,66 @@ workflow CREATE_INPUT_CHANNEL { } // Function to get list of [meta, [ spectra_files ]] -def create_meta_channel(LinkedHashMap row, enzymes, files, wrapper) { +def create_meta_channel_grouped(String filestr, List rows, Map wrapper) { def meta = [:] - def filestr - // Always use SDRF format - if (!params.root_folder) { - filestr = row.URI?.toString()?.trim() ? row.URI.toString() : row.Filename.toString() - } - else { - filestr = row.Filename.toString() - } + def base_row = rows[0] def fileName = file(filestr).name def dotIndex = fileName.lastIndexOf('.') meta.id = dotIndex > 0 ? fileName.take(dotIndex) : fileName meta.experiment_id = wrapper.experiment_id - // apply transformations given by specified root_folder and type - if (params.root_folder) { - filestr = params.root_folder + File.separator + filestr - filestr = (params.local_input_type - ? filestr.take(filestr.lastIndexOf('.')) + '.' + params.local_input_type - : filestr) - } - // existence check if (!file(filestr).exists()) { exit(1, "ERROR: Please check input file -> File Uri does not exist!\n${filestr}") } - // Validate acquisition method is DIA - // AcquisitionMethod is already extracted by convert-diann (e.g. "Data-Independent Acquisition") - def acqMethod = row.AcquisitionMethod?.toString()?.trim() ?: "" + // Detect acquisition method from SDRF or fallback to --dda param + def acqMethod = base_row.AcquisitionMethod?.toString()?.trim() ?: "" if (acqMethod.toLowerCase().contains("data-independent acquisition") || acqMethod.toLowerCase().contains("dia")) { meta.acquisition_method = "dia" - } - else if (acqMethod.isEmpty()) { - // If no acquisition method column in SDRF, assume DIA (this is a DIA-only pipeline) - meta.acquisition_method = "dia" - } - else { - log.error("This pipeline only supports Data-Independent Acquisition (DIA). Found: '${acqMethod}'. Use the quantms pipeline for DDA workflows.") + } else if (acqMethod.toLowerCase().contains("data-dependent acquisition") || acqMethod.toLowerCase().contains("dda")) { + meta.acquisition_method = "dda" + } else if (acqMethod.isEmpty()) { + meta.acquisition_method = params.dda ? "dda" : "dia" + } else { + log.error("Unsupported acquisition method: '${acqMethod}'. This pipeline supports DIA and DDA. Found in file: ${filestr}") exit(1) } - // DissociationMethod is already normalized by convert-diann (HCD, CID, ETD, ECD) - meta.dissociationmethod = row.DissociationMethod?.toString()?.trim() ?: "" - + meta.dissociationmethod = base_row.DissociationMethod?.toString()?.trim() ?: "" wrapper.acquisition_method = meta.acquisition_method - // Validate required SDRF columns - these parameters are exclusively read from SDRF (no command-line override) + def labels = rows.collect { it.Label?.toString()?.trim() }.findAll { it }.unique() + meta.labelling_type = labels.join(';') + + def is_plexdia = labels.size() > 1 || (labels.size() == 1 && !labels[0].toLowerCase().contains("label free")) + meta.plexdia = is_plexdia + + def enzymes = rows.collect { it.Enzyme?.toString()?.trim() }.findAll { it }.unique() + if (enzymes.size() > 1) { + log.error("Currently only one enzyme is supported per file. Found conflicting enzymes for ${filestr}: '${enzymes}'.") + exit(1) + } + meta.enzyme = enzymes ? enzymes[0] : null + + def fixedMods = rows.collect { it.FixedModifications?.toString()?.trim() }.findAll { it }.unique() + if (fixedMods.size() > 1) { + log.error("SDRF conflict: Multiple FixedModifications (${fixedMods.join(',')}) found for file ${meta.id}. Please fix the SDRF.") + } + meta.fixedmodifications = fixedMods ? fixedMods[0] : null + + // Validate required SDRF columns def requiredColumns = [ - 'Label': row.Label, - 'Enzyme': row.Enzyme, - 'FixedModifications': row.FixedModifications + 'Label': meta.labelling_type, + 'Enzyme': meta.enzyme, + 'FixedModifications': meta.fixedmodifications ] def missingColumns = [] requiredColumns.each { colName, colValue -> - if (colValue == null || colValue.toString().trim().isEmpty()) { + if (colValue == null || colValue.toString().isEmpty()) { missingColumns.add(colName) } } @@ -112,20 +124,13 @@ def create_meta_channel(LinkedHashMap row, enzymes, files, wrapper) { exit(1) } - // Set values from SDRF (required columns) - meta.labelling_type = row.Label - meta.fixedmodifications = row.FixedModifications - meta.enzyme = row.Enzyme - - // Set tolerance values: use SDRF if available, otherwise fall back to params def validUnits = ['ppm', 'da', 'Da', 'PPM'] - // Precursor mass tolerance - if (row.PrecursorMassTolerance != null && !row.PrecursorMassTolerance.toString().trim().isEmpty()) { + if (base_row.PrecursorMassTolerance != null && !base_row.PrecursorMassTolerance.toString().trim().isEmpty()) { try { - meta.precursormasstolerance = Double.parseDouble(row.PrecursorMassTolerance) + meta.precursormasstolerance = Double.parseDouble(base_row.PrecursorMassTolerance) } catch (NumberFormatException e) { - log.error("ERROR: Invalid PrecursorMassTolerance value '${row.PrecursorMassTolerance}' for file '${filestr}'. Must be a valid number.") + log.error("ERROR: Invalid PrecursorMassTolerance value '${base_row.PrecursorMassTolerance}' for file '${filestr}'. Must be a valid number.") exit(1) } } else { @@ -133,23 +138,21 @@ def create_meta_channel(LinkedHashMap row, enzymes, files, wrapper) { meta.precursormasstolerance = params.precursor_mass_tolerance } - // Precursor mass tolerance unit - if (row.PrecursorMassToleranceUnit != null && !row.PrecursorMassToleranceUnit.toString().trim().isEmpty()) { - if (!validUnits.any { row.PrecursorMassToleranceUnit.toString().equalsIgnoreCase(it) }) { - log.error("ERROR: Invalid PrecursorMassToleranceUnit '${row.PrecursorMassToleranceUnit}' for file '${filestr}'. Must be 'ppm' or 'Da'.") + if (base_row.PrecursorMassToleranceUnit != null && !base_row.PrecursorMassToleranceUnit.toString().trim().isEmpty()) { + if (!validUnits.any { base_row.PrecursorMassToleranceUnit.toString().equalsIgnoreCase(it) }) { + log.error("ERROR: Invalid PrecursorMassToleranceUnit '${base_row.PrecursorMassToleranceUnit}' for file '${filestr}'. Must be 'ppm' or 'Da'.") exit(1) } - meta.precursormasstoleranceunit = row.PrecursorMassToleranceUnit + meta.precursormasstoleranceunit = base_row.PrecursorMassToleranceUnit } else { meta.precursormasstoleranceunit = params.precursor_mass_tolerance_unit } - // Fragment mass tolerance - if (row.FragmentMassTolerance != null && !row.FragmentMassTolerance.toString().trim().isEmpty()) { + if (base_row.FragmentMassTolerance != null && !base_row.FragmentMassTolerance.toString().trim().isEmpty()) { try { - meta.fragmentmasstolerance = Double.parseDouble(row.FragmentMassTolerance) + meta.fragmentmasstolerance = Double.parseDouble(base_row.FragmentMassTolerance) } catch (NumberFormatException e) { - log.error("ERROR: Invalid FragmentMassTolerance value '${row.FragmentMassTolerance}' for file '${filestr}'. Must be a valid number.") + log.error("ERROR: Invalid FragmentMassTolerance value '${base_row.FragmentMassTolerance}' for file '${filestr}'. Must be a valid number.") exit(1) } } else { @@ -157,43 +160,26 @@ def create_meta_channel(LinkedHashMap row, enzymes, files, wrapper) { meta.fragmentmasstolerance = params.fragment_mass_tolerance } - // Fragment mass tolerance unit - if (row.FragmentMassToleranceUnit != null && !row.FragmentMassToleranceUnit.toString().trim().isEmpty()) { - if (!validUnits.any { row.FragmentMassToleranceUnit.toString().equalsIgnoreCase(it) }) { - log.error("ERROR: Invalid FragmentMassToleranceUnit '${row.FragmentMassToleranceUnit}' for file '${filestr}'. Must be 'ppm' or 'Da'.") + if (base_row.FragmentMassToleranceUnit != null && !base_row.FragmentMassToleranceUnit.toString().trim().isEmpty()) { + if (!validUnits.any { base_row.FragmentMassToleranceUnit.toString().equalsIgnoreCase(it) }) { + log.error("ERROR: Invalid FragmentMassToleranceUnit '${base_row.FragmentMassToleranceUnit}' for file '${filestr}'. Must be 'ppm' or 'Da'.") exit(1) } - meta.fragmentmasstoleranceunit = row.FragmentMassToleranceUnit + meta.fragmentmasstoleranceunit = base_row.FragmentMassToleranceUnit } else { meta.fragmentmasstoleranceunit = params.fragment_mass_tolerance_unit } - // Variable modifications: use SDRF if available, otherwise fall back to params - if (row.VariableModifications != null && !row.VariableModifications.toString().trim().isEmpty()) { - meta.variablemodifications = row.VariableModifications + if (base_row.VariableModifications != null && !base_row.VariableModifications.toString().trim().isEmpty()) { + meta.variablemodifications = base_row.VariableModifications } else { meta.variablemodifications = params.variable_mods } - // Per-file scan ranges (empty string = no flags passed, DIA-NN auto-detects) - meta.ms1minmz = row.MS1MinMz?.toString()?.trim() ?: "" - meta.ms1maxmz = row.MS1MaxMz?.toString()?.trim() ?: "" - meta.ms2minmz = row.MS2MinMz?.toString()?.trim() ?: "" - meta.ms2maxmz = row.MS2MaxMz?.toString()?.trim() ?: "" - - enzymes += row.Enzyme - if (enzymes.size() > 1) { - log.error("Currently only one enzyme is supported for the whole experiment. Specified was '${enzymes}'. Check or split your SDRF.") - log.error(filestr) - exit(1) - } - - // Check for duplicate files - if (filestr in files) { - log.error("Currently only one DIA-NN setting per file is supported for the whole experiment. ${filestr} has multiple entries in your SDRF. Consider splitting your design into multiple experiments.") - exit(1) - } - files += filestr + meta.ms1minmz = base_row.MS1MinMz?.toString()?.trim() ?: "" + meta.ms1maxmz = base_row.MS1MaxMz?.toString()?.trim() ?: "" + meta.ms2minmz = base_row.MS2MinMz?.toString()?.trim() ?: "" + meta.ms2maxmz = base_row.MS2MaxMz?.toString()?.trim() ?: "" return [meta, filestr] } diff --git a/subworkflows/local/file_preparation/meta.yml b/subworkflows/local/file_preparation/meta.yml index 54211c7..54d34fc 100644 --- a/subworkflows/local/file_preparation/meta.yml +++ b/subworkflows/local/file_preparation/meta.yml @@ -8,7 +8,6 @@ keywords: - proteomics components: - thermorawfileparser - - tdf2mzml - decompress - mzml/indexing - mzml/statistics diff --git a/subworkflows/local/input_check/meta.yml b/subworkflows/local/input_check/meta.yml index abe2c7f..1f2cefb 100644 --- a/subworkflows/local/input_check/meta.yml +++ b/subworkflows/local/input_check/meta.yml @@ -14,7 +14,7 @@ input: description: | Input file to be validated output: - - ch_input_file: + - input_file: type: file description: | Channel containing validated input files diff --git a/subworkflows/local/utils_nfcore_quantms_pipeline/meta.yml b/subworkflows/local/utils_nfcore_quantms_pipeline/meta.yml index 06365ae..cf1fd6d 100644 --- a/subworkflows/local/utils_nfcore_quantms_pipeline/meta.yml +++ b/subworkflows/local/utils_nfcore_quantms_pipeline/meta.yml @@ -4,7 +4,7 @@ description: Pipeline completion utilities for the nf-core quantmsdiann pipeline keywords: - utils - nf-core - - quantms + - quantmsdiann components: - completionemail - completionsummary diff --git a/workflows/dia.nf b/workflows/dia.nf index 8a1b13b..839e7e7 100644 --- a/workflows/dia.nf +++ b/workflows/dia.nf @@ -9,8 +9,12 @@ // include { DIANN_MSSTATS } from '../modules/local/diann/diann_msstats/main' include { PRELIMINARY_ANALYSIS } from '../modules/local/diann/preliminary_analysis/main' +include { PRELIMINARY_ANALYSIS as TUNE_PRELIMINARY_ANALYSIS } from '../modules/local/diann/preliminary_analysis/main' include { ASSEMBLE_EMPIRICAL_LIBRARY } from '../modules/local/diann/assemble_empirical_library/main' +include { ASSEMBLE_EMPIRICAL_LIBRARY as TUNE_ASSEMBLE_LIBRARY } from '../modules/local/diann/assemble_empirical_library/main' include { INSILICO_LIBRARY_GENERATION } from '../modules/local/diann/insilico_library_generation/main' +include { INSILICO_LIBRARY_GENERATION as TUNED_LIBRARY_GENERATION } from '../modules/local/diann/insilico_library_generation/main' +include { FINE_TUNE_MODELS } from '../modules/local/diann/fine_tune_models/main' include { INDIVIDUAL_ANALYSIS } from '../modules/local/diann/individual_analysis/main' include { FINAL_QUANTIFICATION } from '../modules/local/diann/final_quantification/main' @@ -34,7 +38,47 @@ workflow DIA { main: ch_software_versions = channel.empty() - ch_searchdb = channel.fromPath(params.database, checkIfExists: true).first() + + // Version guard for DDA mode (when explicitly set via param) + if (params.dda && VersionUtils.versionLessThan(params.diann_version, '2.3.2')) { + error("DDA mode (--dda) requires DIA-NN >= 2.3.2. Current version: ${params.diann_version}. Use -profile diann_v2_3_2") + } + + // Version guard for InfinDIA + if (params.enable_infin_dia && VersionUtils.versionLessThan(params.diann_version, '2.3.0')) { + error("InfinDIA requires DIA-NN >= 2.3.0. Current version: ${params.diann_version}. Use -profile diann_v2_3_2") + } + + // Version guard for scoring mode + if (params.scoring_mode == 'proteoforms' && VersionUtils.versionLessThan(params.diann_version, '2.0')) { + error("--proteoforms scoring mode requires DIA-NN >= 2.0. Current version: ${params.diann_version}. Use -profile diann_v2_1_0 or later") + } + + // Version guard for DIA-NN 2.0+ features + if ((params.light_models || params.export_quant || params.site_ms1_quant || params.channel_run_norm || params.channel_spec_norm) && VersionUtils.versionLessThan(params.diann_version, '2.0')) { + def enabled = [] + if (params.light_models) enabled << '--light-models' + if (params.export_quant) enabled << '--export-quant' + if (params.site_ms1_quant) enabled << '--site-ms1-quant' + if (params.channel_run_norm) enabled << '--channel-run-norm' + if (params.channel_spec_norm) enabled << '--channel-spec-norm' + error("${enabled.join(', ')} require DIA-NN >= 2.0. Current version: ${params.diann_version}. Use -profile diann_v2_1_0 or later") + } + + // Version guard for model fine-tuning + if (params.enable_fine_tuning && VersionUtils.versionLessThan(params.diann_version, '2.3.2')) { + error("Model fine-tuning requires DIA-NN >= 2.3.2. Current version: ${params.diann_version}. Use -profile diann_v2_3_2 or later") + } + + // Warn about contradictory normalization flags + if (!params.normalize && (params.channel_run_norm || params.channel_spec_norm)) { + log.warn "Both --normalize false (adds --no-norm) and channel normalization flags are set. " + + "These may conflict — --no-norm disables cross-run normalization while channel normalization requires it." + } + + ch_searchdb = channel.fromPath(params.database, checkIfExists: true) + .ifEmpty { error("No protein database found at '${params.database}'. Provide --database ") } + .first() ch_file_preparation_results.multiMap { result -> @@ -42,20 +86,106 @@ workflow DIA { ms_file:result[1] }.set { ch_result } - ch_experiment_meta = ch_result.meta.unique { m -> m.experiment_id }.first() + ch_experiment_meta = ch_result.meta.unique { m -> m.experiment_id } + .ifEmpty { error("No valid input files found after SDRF parsing. Check your SDRF file and input paths.") } + .first() + + // Determine DDA mode: true if explicitly set via param OR auto-detected from SDRF + ch_is_dda = ch_experiment_meta.map { meta -> + def dda = params.dda || meta.acquisition_method == 'dda' + if (dda && VersionUtils.versionLessThan(params.diann_version, '2.3.2')) { + error("DDA mode (detected from SDRF) requires DIA-NN >= 2.3.2. Current version: ${params.diann_version}. Use -profile diann_v2_3_2") + } + return dda + } // diann_config.cfg comes directly from SDRF_PARSING (convert-diann) // Use as value channel so it can be consumed by all per-file processes ch_diann_cfg_val = ch_diann_cfg // - // MODULE: SILICOLIBRARYGENERATION + // PHASE 0 (optional): FINE-TUNE DL MODELS // - if (params.diann_speclib != null && params.diann_speclib.toString() != "") { - speclib = channel.from(file(params.diann_speclib, checkIfExists: true)) - } else { - INSILICO_LIBRARY_GENERATION(ch_searchdb, ch_diann_cfg_val) - speclib = INSILICO_LIBRARY_GENERATION.out.predict_speclib + // Per DIA-NN author's recommendation (Vadim Demichev): + // 1. Run InfinDIA on a subset of files with RT/IM filtering set to Relaxed + // 2. Fine-tune models using the resulting empirical library + // 3. Then run the full pipeline from in-silico library generation with tuned models + // + // The tuned models feed into INSILICO_LIBRARY_GENERATION at the very start. + // + ch_tuned_tokens = Channel.empty() + ch_tuned_rt = Channel.empty() + ch_tuned_im = Channel.empty() + + if (params.enable_fine_tuning) { + // Step 0a: Generate a tuning library via InfinDIA on a subset of files + // Use a deterministic subset (or all files if small dataset) for the tuning search + tuning_files = ch_file_preparation_results + .toSortedList{ a, b -> file(a[1]).getName() <=> file(b[1]).getName() } + .flatMap() + .take(params.tune_n_files) + + // Run in-silico library generation first (with default models) for the tuning search + INSILICO_LIBRARY_GENERATION(ch_searchdb, ch_diann_cfg_val, ch_is_dda, [], [], []) + tune_speclib = INSILICO_LIBRARY_GENERATION.out.predict_speclib + + // Run preliminary analysis on the tuning subset to produce .quant files + TUNE_PRELIMINARY_ANALYSIS(tuning_files.combine(tune_speclib), ch_diann_cfg_val) + + // Assemble the tuning empirical library from the subset + tune_lib_files = tuning_files + .map { result -> result[1] } + .collect( sort: { a, b -> file(a).getName() <=> file(b).getName() } ) + + TUNE_ASSEMBLE_LIBRARY( + tune_lib_files, + ch_experiment_meta, + TUNE_PRELIMINARY_ANALYSIS.out.diann_quant.collect(), + tune_speclib, + ch_diann_cfg_val + ) + ch_software_versions = ch_software_versions + .mix(TUNE_PRELIMINARY_ANALYSIS.out.versions) + .mix(TUNE_ASSEMBLE_LIBRARY.out.versions) + + // Step 0b: Fine-tune models on the empirical library + FINE_TUNE_MODELS( + TUNE_ASSEMBLE_LIBRARY.out.empirical_library, + ch_searchdb, + ch_diann_cfg_val + ) + ch_software_versions = ch_software_versions + .mix(FINE_TUNE_MODELS.out.versions) + + ch_tuned_tokens = FINE_TUNE_MODELS.out.tokens + ch_tuned_rt = FINE_TUNE_MODELS.out.rt_model + ch_tuned_im = FINE_TUNE_MODELS.out.im_model + + // Step 0c: Re-generate in-silico library with tuned models + TUNED_LIBRARY_GENERATION( + ch_searchdb, + ch_diann_cfg_val, + ch_is_dda, + ch_tuned_tokens, + ch_tuned_rt, + ch_tuned_im + ) + ch_software_versions = ch_software_versions + .mix(TUNED_LIBRARY_GENERATION.out.versions) + + speclib = TUNED_LIBRARY_GENERATION.out.predict_speclib + } + + // + // MODULE: INSILICO_LIBRARY_GENERATION (standard, when not fine-tuning) + // + if (!params.enable_fine_tuning) { + if (params.speclib != null && params.speclib.toString() != "") { + speclib = channel.from(file(params.speclib, checkIfExists: true)) + } else { + INSILICO_LIBRARY_GENERATION(ch_searchdb, ch_diann_cfg_val, ch_is_dda, [], [], []) + speclib = INSILICO_LIBRARY_GENERATION.out.predict_speclib + } } if (params.skip_preliminary_analysis) { @@ -111,17 +241,22 @@ workflow DIA { ch_software_versions = ch_software_versions .mix(ASSEMBLE_EMPIRICAL_LIBRARY.out.versions) // Parse calibrated params from the assembly log on the head node + // Format changed in 2.5.0 ch_parsed_vals = ASSEMBLE_EMPIRICAL_LIBRARY.out.log .map { log_file -> + def ms1 = "${params.mass_acc_ms1}" + def ms2 = "${params.mass_acc_ms2}" + def sw = "${params.scan_window}" def match = log_file.text.readLines().find { it.contains("Averaged recommended settings") } if (match) { - def parts = match.trim().split(/\s+/) - def ms2 = parts.size() > 10 ? parts[10].replaceAll(/[^0-9.]/, '') : "${params.mass_acc_ms2}" - def ms1 = parts.size() > 14 ? parts[14].replaceAll(/[^0-9.]/, '') : "${params.mass_acc_ms1}" - def sw = parts.size() > 18 ? parts[18].replaceAll(/[^0-9.]/, '') : "${params.scan_window}" - return "${ms2},${ms1},${sw}" + def ms1_match = match =~ /MS1 accuracy\s*=\s*([0-9.]+)/ + if (ms1_match.find()) ms1 = ms1_match.group(1) + def ms2_match = match =~ /(?:MS2|Mass) accuracy\s*=\s*([0-9.]+)/ + if (ms2_match.find()) ms2 = ms2_match.group(1) + def sw_match = match =~ /Scan window\s*=\s*([0-9.]+)/ + if (sw_match.find()) sw = sw_match.group(1) } - return "${params.mass_acc_ms2},${params.mass_acc_ms1},${params.scan_window}" + return "${ms2},${ms1},${sw}" } indiv_fin_analysis_in = ch_file_preparation_results .combine(ch_searchdb) @@ -171,7 +306,9 @@ workflow DIA { FINAL_QUANTIFICATION.out.versions ) - diann_main_report = FINAL_QUANTIFICATION.out.main_report + // Only one format is produced per DIA-NN version: parquet (>= 1.9) or TSV (< 1.9) + diann_main_report = FINAL_QUANTIFICATION.out.main_report_parquet + .mix(FINAL_QUANTIFICATION.out.main_report_tsv) // // MODULE: DIANN_MSSTATS — Convert DIA-NN report to MSstats-compatible format diff --git a/workflows/quantmsdiann.nf b/workflows/quantmsdiann.nf index 079858a..e55a1e3 100644 --- a/workflows/quantmsdiann.nf +++ b/workflows/quantmsdiann.nf @@ -60,7 +60,7 @@ workflow QUANTMSDIANN { FILE_PREPARATION.out.results .branch { item -> - dia: item[0].acquisition_method.toLowerCase().contains("dia") + dia: item[0].acquisition_method.toLowerCase().contains("dia") || item[0].acquisition_method.toLowerCase().contains("dda") } .set { ch_fileprep_result } //