config.yaml

---
work_path: ../pipeline_work
log_level: DEBUG
pool_size: 4

scratchpad: {}

steps:
  #: DISEASE STEP :#################################################################################
  disease:
    - name: transform disease_efo
      source: input/disease/efo_otar_slim.json
      destination: output/disease/disease.parquet
      transformer: disease

    - name: transform disease_hpo
      source: input/disease/hp-full.json
      destination: output/disease_hpo/disease_hpo.parquet
      transformer: disease_hpo

    - name: transform disease_phenotype
      requires:
        - transform disease_efo
      source:
        disease_path: output/disease/disease.parquet
        phenotype_path: input/disease/phenotype.hpoa
        mondo_path: input/disease/mondo.json
      destination: output/disease_phenotype/disease_phenotype.parquet
      transformer: disease_phenotype
  ##################################################################################################

  #: GO STEP :######################################################################################
  go:
    - name: transform go
      transformer: go
      source: input/go/go.obo
      destination: output/go/go.parquet

  #: EXPRESSION STEP :##############################################################################
  expression:
    - name: unzip normal tissue
      source: input/expression/normal_tissue.tsv.zip
      inner_file: normal_ihc_data.tsv
      destination: intermediate/expression/normal_tissue.tsv
    - name: gzip normal tissue
      requires:
        - unzip normal tissue
      source: intermediate/expression/normal_tissue.tsv
      destination: intermediate/expression/normal_tissue.tsv.gz

    - name: transform expression tissue
      source: input/expression/map_with_efos.json
      destination: intermediate/expression/tissue-translation-map.parquet
      transformer: expression_tissue
  ##################################################################################################

  #: MOUSE_PHENOTYPES STEP :########################################################################
  mouse_phenotype:
    - name: pyspark generate from IMPC and validate with target
      pyspark: mouse_phenotype
      source:
        target: output/target
        # IMPC source files
        solr_gene_gene: input/impc/gene_gene.csv
        solr_mouse_model: input/impc/mouse_model.csv
        solr_disease_model_summary: input/impc/disease_model_summary.csv
        solr_ontology: input/impc/ontology.csv
        hgnc_gene_mappings: input/target/genenames/hgnc_complete_set.tsv
        mouse_gene_mappings: input/impc/MGI_Gene_Model_Coord.rpt
        mouse_pubmed_refs: input/impc/MGI_PhenoGenoMP.rpt
        mp_ontology: input/impc/mp.json
      destination:
        output: output/mouse_phenotype
        excluded: excluded/mouse_phenotype
  ##################################################################################################

  #: CHEMICAL PROBES STEP :#########################################################################
  chemical_probes:
    - name: pyspark chemical probes
      pyspark: chemical_probes
      source:
        probes_excel: input/target/chemicalprobes/probes.xlsx
        drugs_csv: input/target/chemicalprobes/probes_links.csv
        chembl_molecule: intermediate/chembl_molecule
      destination: intermediate/chemical_probes_evidence.parquet
  ##################################################################################################

  #: CODING VARIANT STEP :##########################################################################
  coding_variant:
    - name: pyspark coding_variant view
      pyspark: coding_variant
      source:
        variant: output/variant
        target: output/target
        disease: output/disease
        evidence_eva: output/evidence_eva
        evidence_eva_somatic: output/evidence_eva_somatic
        evidence_uniprot_variant: output/evidence_uniprot_variants
        evidence_gwas_credible_set: output/evidence_gwas_credible_sets
        credible_set: output/credible_set
        pharmacogenetics: output/pharmacogenomics
      destination: view/coding_variant
  ##################################################################################################

  #: OPENFDA STEP :#################################################################################
  openfda:
    - name: explode_glob transform openfda drug events
      glob: input/openfda/**/*.zip
      do:
        - name: transform ${match_path}/${match_stem}
          source: ${uri}
          destination: intermediate/openfda/${uuid}.parquet
          transformer: openfda
  ##################################################################################################

  #: SO STEP :######################################################################################
  so:
    - name: transform so
      source: input/so/so.json
      destination: output/so/so.parquet
      transformer: so
  ##################################################################################################

  #: TARGET STEP :##################################################################################
  target:
    - name: unzip subcellular location
      source: input/target/hpa/subcellular_location.tsv.zip
      destination: intermediate/target/hpa/subcellular_location.tsv
    - name: gzip subcellular location
      requires:
        - unzip subcellular location
      source: intermediate/target/hpa/subcellular_location.tsv
      destination: intermediate/target/hpa/subcellular_location.tsv.gz

    - name: csv_to_parquet subcellular location ssl
      source: input/target/hpa/subcellular_locations_ssl.tsv
      destination: intermediate/target/hpa/subcellular_locations_ssl.parquet
      separator: "\t"

    - name: unzip essentiality matrix
      source: input/target/project-scores/essentiality_matrices.zip
      inner_file: EssentialityMatrices/04_binaryDepScores.tsv
      destination: intermediate/target/project-scores/04_binaryDepScores.tsv
    - name: csv_to_parquet essentiality matrix
      requires:
        - unzip essentiality matrix
      source: intermediate/target/project-scores/04_binaryDepScores.tsv
      local_source: True
      destination: intermediate/target/project-scores/04_binaryDepScores.parquet
      separator: "\t"

    - name: csv_to_parquet gene identifier
      source: input/target/project-scores/gene_identifiers_latest.csv.gz
      destination: intermediate/target/project-scores/gene_identifiers_latest.parquet

    - name: download ensembl
      source: input/target/ensembl/homo_sapiens.json
    - name: transform ensembl
      requires:
        - download ensembl
      transformer: ensembl
      source: input/target/ensembl/homo_sapiens.json
      destination: intermediate/target/ensembl/homo_sapiens.parquet

    - name: explode_glob homology
      glob: input/target/homologue/gene_dictionary/*.json
      do:
        - name: transform ${match_stem} to parquet
          source: ${uri}
          destination: intermediate/target/homologue/gene_dictionary/${match_path}/${match_stem}.parquet
          transformer: homology
  ##################################################################################################

  #: TARGET_SAFETY STEP :############### - part of target, pyspark tasks must run standalone for now
  target_safety:
    - name: pyspark safety
      pyspark: target_safety
      source:
        adverse_events: input/target/safety/adverse_effects.tsv
        safety_risks: input/target/safety/safety_risks.tsv
        brennan: input/target/safety/secondary_pharmacology.json
        toxcast: input/target/safety/toxcast.tsv
        aopwiki: input/target/safety/aopwiki.json
        pharmacogenetics: output/pharmacogenomics
      destination: intermediate/target/safety.parquet

  #: TARGET_GENE_ESSENTIALITY STEP :#### - part of target, pyspark tasks must run standalone for now
  target_gene_essentiality:
    - name: pyspark gene_essentiality
      pyspark: gene_essentiality
      source:
        models: input/target/gene-essentiality/depmap/Model.csv
        essential_genes: input/target/gene-essentiality/depmap/CRISPRInferredCommonEssentials.csv
        gene_effects: input/target/gene-essentiality/depmap/CRISPRGeneEffect.csv
        gene_expression: input/target/gene-essentiality/depmap/OmicsExpressionProteinCodingGenesTPMLogp1.csv
        mutation_hotspots: input/target/gene-essentiality/depmap/OmicsSomaticMutationsMatrixHotspot.csv
        mutation_damaging: input/target/gene-essentiality/depmap/OmicsSomaticMutationsMatrixDamaging.csv
        depmap_tissue_mapping: input/ot_curation/mappings/biosystem/depmap_uberon_mapping.csv
      destination: intermediate/target/gene-essentiality/essentiality.parquet
      settings:
        keep_only_essentials: false
  ##################################################################################################

  #: PHARMACOGENETICS STEP :########################################################################
  pharmacogenetics:
    - name: pyspark pharmacogenetics
      pyspark: pharmacogenetics
      source:
        clinpgx: input/pharmacogenetics/clinpgx_annotation.json.gz
        phenotypes: input/pharmacogenetics/phenotypes.json
        ontoma_disease_label_lut: intermediate/ontoma/disease_label_lookup_table.parquet
        chembl_molecule: intermediate/chembl_molecule
        drug_mechanism_of_action: output/drug_mechanism_of_action
      destination:
        associations: output/pharmacogenomics
        phenotypes: intermediate/pharmacogenetics/phenotypes.json
      settings:
        openai_token_filename: /var/run/secrets/openai_token
      properties:
        'spark.jars.packages': 'com.johnsnowlabs.nlp:spark-nlp_2.12:6.1.3'
  ##################################################################################################

  #: EVIDENCE_PROJECT_SCORE STEP :##################################################################
  evidence_project_score:
    - name: pyspark project_score
      pyspark: project_score
      source:
        gene_scores: input/evidence/project_score/mapped_diseases.tsv
        cell_types: input/evidence/project_score/cell_types.tsv
        cell_passport: input/evidence/cell_passport.csv.gz
        cell_line_mapping: input/ot_curation/mappings/biosystem/depmap_uberon_mapping.csv
      destination: intermediate/evidence/project_score.parquet
  ##################################################################################################

  #: EVIDENCE_GENE_BURDEN STEP :####################################################################
  evidence_gene_burden:
    - name: pyspark gene_burden
      pyspark: gene_burden
      source:
        az_binary: input/evidence/gene_burden/astrazeneca/azphewas-com-470k-phewas-binary
        az_quantitative: input/evidence/gene_burden/astrazeneca/azphewas-com-470k-phewas-quantitative
        az_genes: input/evidence/gene_burden/astrazeneca/azphewas_com_genes_UK_Biobank_470k.csv
        az_phenotypes: input/evidence/gene_burden/astrazeneca/azphewas_com_phenotypes_UK_Biobank_470k.csv
        finngen: input/evidence/gene_burden/finngen/finngen.txt.gz
        finngen_phenotypes: input/evidence/gene_burden/finngen/phenos
        genebass: input/evidence/gene_burden/genebass/**.parquet
        cvdi: input/evidence/gene_burden/cvdi.xlsx
        curated_studies: input/ot_curation/gene_burden/curated_evidence.tsv
        ontoma_disease_label_lut: intermediate/ontoma/disease_label_lookup_table.parquet
        ontoma_disease_id_lut: intermediate/ontoma/disease_id_lookup_table.parquet
      destination: intermediate/evidence/gene_burden.parquet
      settings:
        finngen_release: R12
  ##################################################################################################

  #: EVIDENCE_CLINICAL_PRECEDENCE STEP :############################################################
  evidence_clinical_precedence:
    - name: pyspark clinical_precedence
      pyspark: clinical_precedence
      source:
        clinical_report: output/clinical_report
        drug_mechanism_of_action: output/drug_mechanism_of_action
      destination:
        output: intermediate/evidence/clinical_precedence.parquet
        excluded: excluded/clinical_precedence_failed_report
      settings:
        invalid_clinical_report_qc:
          - PHASE_IV_NOT_APPROVED
          - UNVALIDATED_INDICATION
          - INDIRECT_PRIMARY_PURPOSE
  ##################################################################################################

  #: EVIDENCE_CANCER_BIOMARKERS STEP :################################################################
  evidence_cancer_biomarkers:
    - name: pyspark cancer_biomarkers
      pyspark: cancer_biomarkers
      source:
        associations: input/evidence/cancerbiomarkers/associations.tsv
        source: input/evidence/cancerbiomarkers/sources.jsonl
        disease: input/evidence/cancerbiomarkers/diseases.jsonl
        drugs: output/drug_molecule
      destination: intermediate/evidence/cancer_biomarkers.parquet
  ##################################################################################################

  #: EVIDENCE_ORPHANET STEP :#######################################################################
  evidence_orphanet:
    - name: pyspark orphanet
      pyspark: orphanet
      source:
        evidence: input/evidence/orphanet.xml
        ontoma_disease_label_lut: intermediate/ontoma/disease_label_lookup_table.parquet
        ontoma_disease_id_lut: intermediate/ontoma/disease_id_lookup_table.parquet
      destination: intermediate/evidence/orphanet.parquet
  ##################################################################################################

  #: EVIDENCE_CLINGEN STEP :#######################################################################
  evidence_clingen:
    - name: pyspark clingen
      pyspark: clingen
      source:
        evidence: input/evidence/clingen.csv
        ontoma_disease_label_lut: intermediate/ontoma/disease_label_lookup_table.parquet
        ontoma_disease_id_lut: intermediate/ontoma/disease_id_lookup_table.parquet
      destination: intermediate/evidence/clingen.parquet
  ##################################################################################################

  #: EVIDENCE_IMPC STEP :###########################################################################
  evidence_impc:
    - name: pyspark impc evidence strings
      pyspark: impc
      source:
        solr_gene_gene: input/impc/gene_gene.csv
        solr_ontology_ontology: input/impc/ontology_ontology.csv
        solr_mouse_model: input/impc/mouse_model.csv
        solr_disease: input/impc/disease.csv
        solr_disease_model_summary: input/impc/disease_model_summary.csv
        solr_ontology: input/impc/ontology.csv
        hgnc_gene_mappings: input/target/genenames/hgnc_complete_set.tsv
        mouse_gene_mappings: input/impc/MGI_Gene_Model_Coord.rpt
        mouse_pubmed_refs: input/impc/MGI_PhenoGenoMP.rpt
        mp_ontology: input/impc/mp.json
        ontoma_disease_label_lut: intermediate/ontoma/disease_label_lookup_table.parquet
        ontoma_disease_id_lut: intermediate/ontoma/disease_id_lookup_table.parquet
      destination: intermediate/evidence/impc.parquet
      settings:
        score_cutoff: 41
  ##################################################################################################

  #: EVIDENCE PANEL APP STEP :#######################################################################
  evidence_panel_app:
    - name: pyspark panel_app
      pyspark: panel_app
      source:
        evidence: input/evidence/panelapp/panel_app.tsv
        panels: input/evidence/panelapp/panels.jsonl
        ontoma_disease_label_lut: intermediate/ontoma/disease_label_lookup_table.parquet
        ontoma_disease_id_lut: intermediate/ontoma/disease_id_lookup_table.parquet
      destination: intermediate/evidence/panel_app.parquet
  ##################################################################################################

  #: EVIDENCE_CRISPR_SCREENS STEP :#################################################################
  evidence_crispr_screens:
    - name: pyspark crispr screens
      pyspark: crispr_screens
      source:
        screens: input/evidence/crispr_screens/brain_screens.json.gz
        studies_dir: input/evidence/crispr_screens/brain_studies
        disease_mapping: input/ot_curation/mappings/disease/brain_crispr_studies.tsv
      destination: intermediate/evidence/crispr_screens.parquet
  ##################################################################################################

  #: EVIDENCE_GENE2PHENOTYPE STEP :#################################################################
  evidence_gene2phenotype:
    - name: pyspark gene2phenotype
      pyspark: gene2phenotype
      source:
        cancer_panel: input/evidence/gene2phenotype/cancer.csv
        cardiac_panel: input/evidence/gene2phenotype/cardiac.csv
        developmental_panel: input/evidence/gene2phenotype/dd.csv
        ear_panel: input/evidence/gene2phenotype/ear.csv
        eye_panel: input/evidence/gene2phenotype/eye.csv
        skeletal_panel: input/evidence/gene2phenotype/skeletal.csv
        skin_panel: input/evidence/gene2phenotype/skin.csv
        ontoma_disease_label_lut: intermediate/ontoma/disease_label_lookup_table.parquet
        ontoma_disease_id_lut: intermediate/ontoma/disease_id_lookup_table.parquet
      destination: intermediate/evidence/gene2phenotype.parquet
  ##################################################################################################

  #: EVIDENCE_INTOGEN STEP :#######################################################################
  evidence_intogen:
    - name: pyspark intogen
      pyspark: intogen
      source:
        genes: input/evidence/intogen/Compendium_Cancer_Genes.tsv
        cohorts: input/evidence/intogen/cohorts.tsv
        disease_mapping: input/ot_curation/mappings/disease/cancer2EFO_mappings.tsv
      destination: intermediate/evidence/intogen.parquet
  ##################################################################################################

  #: EVIDENCE PPP STEP :############################################################################
  evidence_ot_crispr:
    - name: pyspark ot_crispr
      pyspark: ot_crispr
      source:
        study_table: input/evidence/ot_crispr/config.tsv
        ot_crispr_data: input/evidence/ot_crispr/data
      destination: intermediate/evidence/ot_crispr.parquet
  ##################################################################################################

  #: ONTOMA STEP :##################################################################################
  ontoma:
    - name: pyspark ontoma generate lookup tables
      pyspark: ontoma_lut_generation
      source:
        disease_index: output/disease/disease.parquet
        ot_disease_curation: input/ontoma/ot_disease_curation.tsv
        eva_clinvar: input/ontoma/eva_clinvar.txt
        clinvar_xrefs: input/ontoma/clinvar_xrefs.txt
      destination:
        disease_label_lut: intermediate/ontoma/disease_label_lookup_table.parquet
        disease_id_lut: intermediate/ontoma/disease_id_lookup_table.parquet
  ##################################################################################################

  #: DRUG_WARNING STEP :############################################################################
  drug_warning:
    - name: pyspark drug_warning
      pyspark: drug_warning
      source: input/drug/chembl_drug_warning.jsonl
      destination: output/drug_warning
  ##################################################################################################

  #: DRUG_MECHANISM_OF_ACTION STEP :###############################################################
  drug_mechanism_of_action:
    - name: pyspark drug_mechanism_of_action
      pyspark: drug_mechanism_of_action
      source:
        chembl_mechanism: input/drug/chembl_mechanism.jsonl
        chembl_target: input/drug/chembl_target.jsonl
        target: intermediate/target/ensembl/homo_sapiens.parquet
      destination: output/drug_mechanism_of_action
  ##################################################################################################

  #: CHEMBL_MOLECULE STEP :#########################################################################
  chembl_molecule:
    - name: pyspark chembl_molecule
      pyspark: chembl_molecule
      source:
        chembl_molecule: input/drug/chembl_molecule.jsonl
        drugbank: input/drug/drugbank.csv.gz
      destination: intermediate/chembl_molecule
  ##################################################################################################

  #: DRUG_MOLECULE STEP :##########################################################################
  drug_molecule:
    - name: pyspark drug_molecule
      pyspark: drug_molecule
      source:
        molecule: intermediate/chembl_molecule
        chemical_probes: intermediate/chemical_probes_evidence.parquet
        mechanism_of_action: output/drug_mechanism_of_action
        clinical_report: output/clinical_report
        disease: output/disease
      destination:
        output: output/drug_molecule
        excluded: excluded/drug_molecule_failed_report
      settings:
        invalid_clinical_report_qc:
          - PHASE_IV_NOT_APPROVED
          - UNVALIDATED_INDICATION
          - INDIRECT_PRIMARY_PURPOSE
  ##################################################################################################

  #: CLINICAL_REPORT STEP :#######################################################################
  clinical_report:
    - name: pyspark clinical_report
      source:
        chembl_molecule: intermediate/chembl_molecule/*.parquet
        disease: output/disease/disease.parquet
        pmda: input/clinical_report/pmda.pdf
        ttd: input/clinical_report/ttd.txt
        ema: input/clinical_report/ema_medicines.xlsx
        aact_studies: input/clinical_report/aact/studies.parquet
        aact_interventions: input/clinical_report/aact/interventions.parquet
        aact_conditions: input/clinical_report/aact/conditions.parquet
        aact_study_references: input/clinical_report/aact/study_references.parquet
        aact_designs: input/clinical_report/aact/designs.parquet
        aact_summaries: input/clinical_report/aact/summaries.parquet
        chembl_indication: input/clinical_report/chembl/drug_indication.parquet
        chembl_indication_references: input/clinical_report/chembl/drug_indication_refs.parquet
        chembl_drug_warning: input/clinical_report/chembl/drug_warning.parquet
        chembl_drug_warning_references: input/clinical_report/chembl/drug_warning_refs.parquet
        chembl_molecule_dictionary: input/clinical_report/chembl/molecule_dictionary.parquet
        chembl_curation: input/clinical_report/chembl_mapping.parquet
        ner_cache_path: input/clinical_report/ner_cache.parquet
      destination:
        output: output/clinical_report/clinical_report.parquet
      settings:
        ner_batch_size: 256
      pyspark: clinical_report
  ##################################################################################################

  #: CLINICAL_INDICATION STEP :####################################################################
  clinical_indication:
    - name: transform clinical_indication
      source: output/clinical_report/clinical_report.parquet
      destination:
        output: output/clinical_indication/clinical_indication.parquet
        excluded: excluded/clinical_indication_failed_report/clinical_indication_failed_report.parquet
      transformer: clinical_indication
      settings:
        invalid_clinical_report_qc:
          - PHASE_IV_NOT_APPROVED
          - UNVALIDATED_INDICATION
          - INDIRECT_PRIMARY_PURPOSE
  ##################################################################################################

  #: CLINICAL_TARGET STEP :#######################################################################
  clinical_target:
    - name: transform clinical_target
      source:
        clinical_report: output/clinical_report/clinical_report.parquet
        drug_mechanism_of_action: output/drug_mechanism_of_action/*.parquet
      destination:
        output: output/clinical_target/clinical_target.parquet
        excluded: excluded/clinical_target_failed_report/clinical_target_failed_report.parquet
      transformer: clinical_target
      settings:
        invalid_clinical_report_qc:
          - PHASE_IV_NOT_APPROVED
          - UNVALIDATED_INDICATION
          - INDIRECT_PRIMARY_PURPOSE
  ##################################################################################################

  #: TARGET_PRIORITISATION STEP :###################################################################
  target_prioritisation:
    - name: pyspark target_prioritisation
      pyspark: target_prioritisation
      source:
        targets: output/target
        mouse_phenotypes: output/mouse_phenotype
        molecule: output/drug_molecule
        mechanism_of_action: output/drug_mechanism_of_action
        hpa_data: input/target_prioritisation/proteinatlas.json.gz
        uniprot_slterms: input/target_prioritisation/uniprot_locations.tsv.gz
        mouse_pheno_scores: input/target_prioritisation/mouse_pheno_scores.csv
      destination: output/target_prioritisation
      properties:
        spark.driver.memory: 16g
  ##################################################################################################

  #: ASSOCIATION STEP :##############################################################################
  association:
    - name: pyspark association
      pyspark: association
      source:
        evidence: output/evidence_*
        disease: output/disease
      destination:
        overall_direct: output/association_overall_direct
        overall_indirect: output/association_overall_indirect
        by_datasource_direct: output/association_by_datasource_direct
        by_datasource_indirect: output/association_by_datasource_indirect
        by_datatype_direct: output/association_by_datatype_direct
        by_datatype_indirect: output/association_by_datatype_indirect
        temporary: intermediate/indirect_association_intermediate
      properties:
        spark.driver.memory: '12g'
      settings:
        novelty_scale: 2 # 2 for long tailed slow decays
        novelty_shift: 3 # 3 for long tailed slow decays
        novelty_window: 10
        datasource_weights:
          - datasourceId: cancer_biomarkers
            weight: 0.5
            datatypeId: affected_pathway
            toPropagate: true
          - datasourceId: cancer_gene_census
            weight: 1.0
            datatypeId: somatic_mutation
            toPropagate: true
          - datasourceId: clingen
            weight: 1.0
            datatypeId: genetic_literature
            toPropagate: true
          - datasourceId: clinical_precedence
            weight: 1.0
            datatypeId: known_drug
            toPropagate: true
          - datasourceId: crispr
            weight: 1.0
            datatypeId: affected_pathway
            toPropagate: true
          - datasourceId: crispr_screen
            weight: 1.0
            datatypeId: affected_pathway
            toPropagate: true
          - datasourceId: encore
            weight: 0.5
            datatypeId: ot_partner
            toPropagate: true
          - datasourceId: europepmc
            weight: 0.2
            datatypeId: literature
            toPropagate: true
          - datasourceId: eva
            weight: 1.0
            datatypeId: genetic_association
            toPropagate: true
          - datasourceId: eva_somatic
            weight: 1.0
            datatypeId: somatic_mutation
            toPropagate: true
          - datasourceId: expression_atlas
            weight: 0.2
            datatypeId: rna_expression
            toPropagate: false
          - datasourceId: gene2phenotype
            weight: 1.0
            datatypeId: genetic_literature
            toPropagate: true
          - datasourceId: gene_burden
            weight: 1.0
            datatypeId: genetic_association
            toPropagate: true
          - datasourceId: genomics_england
            weight: 1.0
            datatypeId: genetic_literature
            toPropagate: true
          - datasourceId: gwas_credible_sets
            weight: 1.0
            datatypeId: genetic_association
            toPropagate: true
          - datasourceId: impc
            weight: 0.2
            datatypeId: animal_model
            toPropagate: true
          - datasourceId: intogen
            weight: 1.0
            datatypeId: somatic_mutation
            toPropagate: true
          - datasourceId: orphanet
            weight: 1.0
            datatypeId: genetic_association
            toPropagate: true
          - datasourceId: ot_crispr
            weight: 0.5
            datatypeId: ot_partner
            toPropagate: true
          - datasourceId: ot_crispr_validation
            weight: 0.5
            datatypeId: ot_validation_lab
            toPropagate: true
          - datasourceId: reactome
            weight: 1.0
            datatypeId: affected_pathway
            toPropagate: true
          - datasourceId: uniprot_literature
            weight: 1.0
            datatypeId: genetic_literature
            toPropagate: true
          - datasourceId: uniprot_variants
            weight: 1.0
            datatypeId: genetic_association
            toPropagate: true
  ##################################################################################################

  #: EVIDENCE POST PROCESS STEP :###################################################################
  evidence_postprocess_gwas_credible_sets:
    - name: pyspark evidence_postprocess
      pyspark: evidence_postprocess
      source:
        evidence_path: intermediate/evidence/l2g_evidence
        target_path: output/target
        disease_path: output/disease
        publication_date_lut: input/literature/literature_export
      destination:
        failed_evidence: excluded/evidence/gwas_credible_sets
        evidence: output/evidence_gwas_credible_sets
      settings:
        evidence_format: parquet
        score_expression: resourceScore
        datasource_id: gwas_credible_sets
        unique_fields:
          - targetId
          - targetFromSourceId
          - diseaseId
          - datasourceId
          - studyLocusId

  evidence_postprocess_expression_atlas:
    - name: pyspark evidence_postprocess
      pyspark: evidence_postprocess
      source:
        evidence_path: input/evidence/atlas.json.bz2
        target_path: output/target
        disease_path: output/disease
        publication_date_lut: input/literature/literature_export
      destination:
        failed_evidence: excluded/evidence/expression_atlas
        evidence: output/evidence_expression_atlas
      settings:
        evidence_format: json
        datasource_id: expression_atlas
        score_expression: 'array_min(array(1.0, pvalue_linear_score(resourceScore) * (abs(log2FoldChangeValue) / 10) * (log2FoldChangePercentileRank / 100)))'
        unique_fields:
          - targetId
          - targetFromSourceId
          - diseaseId
          - datasourceId
          - contrast
          - studyId
        excluded_biotypes:
          - 'IG_C_pseudogene'
          - 'IG_J_pseudogene'
          - 'IG_pseudogene'
          - 'IG_V_pseudogene'
          - 'polymorphic_pseudogene'
          - 'processed_pseudogene'
          - 'pseudogene'
          - 'rRNA'
          - 'rRNA_pseudogene'
          - 'snoRNA'
          - 'snRNA'
          - 'transcribed_processed_pseudogene'
          - 'transcribed_unitary_pseudogene'
          - 'transcribed_unprocessed_pseudogene'
          - 'TR_J_pseudogene'
          - 'TR_V_pseudogene'
          - 'unitary_pseudogene'
          - 'unprocessed_pseudogene'

  evidence_postprocess_clinvar:
    - name: pyspark evidence_postprocess
      pyspark: evidence_postprocess
      source:
        evidence_path: input/evidence/eva.json.gz
        target_path: output/target
        disease_path: output/disease
        publication_date_lut: input/literature/literature_export
      destination:
        failed_evidence: excluded/evidence/eva
        evidence: output/evidence_eva
      settings:
        evidence_format: json
        datasource_id: eva
        score_expression: |
          coalesce(
            array_max(
              transform(
                clinicalSignificances,
                x -> element_at(
                  map(
                    'association not found', 0.0,
                    'benign', 0.0,
                    'not provided', 0.0,
                    'likely benign', 0.0,
                    'evidence only', 0.0,
                    'likely risk allele', 0.3,
                    'low penetrance', 0.3,
                    'conflicting interpretations of pathogenicity', 0.3,
                    'conflicting data from submitters', 0.3,
                    'other', 0.3,
                    'uncertain significance', 0.3,
                    'uncertain risk allele', 0.3,
                    'established risk allele', 0.5,
                    'risk factor', 0.5,
                    'affects', 0.5,
                    'likely pathogenic', 0.7,
                    'confers sensitivity', 0.9,
                    'association', 0.9,
                    'drug response', 0.9,
                    'protective', 0.9,
                    'pathogenic', 0.9
                  ),
                  x
                )
              )
            ),
            0.0
          ) + coalesce(
            element_at(
              map(
                'practice guideline', 0.1,
                'reviewed by expert panel', 0.07,
                'criteria provided, multiple submitters, no conflicts', 0.05,
                'criteria provided, conflicting interpretations', 0.02,
                'criteria provided, single submitter', 0.02,
                'no assertion for the individual variant', 0.0,
                'no assertion criteria provided', 0.0,
                'no assertion provided', 0.0
              ),
              confidence
            ),
            0.0
          )
        unique_fields:
          - targetId
          - targetFromSourceId
          - diseaseId
          - datasourceId
          - studyId
          - variantId
        direction_on_trait_expression: |
          CASE WHEN CONCAT_WS('', clinicalSignificances) RLIKE '(?i)pathogenic'
          AND CONCAT_WS('', clinicalSignificances) RLIKE '(?i)protect' THEN null
          WHEN CONCAT_WS('', clinicalSignificances) RLIKE '(?i)pathogenic' THEN 'risk'
          WHEN CONCAT_WS('', clinicalSignificances) RLIKE '(?i)protect' THEN 'protect'
          ELSE null
          END
        direction_on_target_expression: |
          CASE
            WHEN variantFunctionalConsequenceId IN (
              'SO_0001589',
              'SO_0001587',
              'SO_0001574',
              'SO_0001575',
              'SO_0002012',
              'SO_0001578',
              'SO_0001893'
            ) THEN 'LoF'
            ELSE null
          END

  evidence_postprocess_clinvar_somatic:
    - name: pyspark evidence_postprocess
      pyspark: evidence_postprocess
      source:
        evidence_path: input/evidence/eva.json.gz
        target_path: output/target
        disease_path: output/disease
        publication_date_lut: input/literature/literature_export
      destination:
        failed_evidence: excluded/evidence/eva_somatic
        evidence: output/evidence_eva_somatic
      settings:
        evidence_format: json
        datasource_id: eva_somatic
        score_expression: |
          coalesce(
            array_max(
              transform(
                clinicalSignificances,
                x -> element_at(
                  map(
                    'association not found', 0.0,
                    'benign', 0.0,
                    'not provided', 0.0,
                    'likely benign', 0.0,
                    'evidence only', 0.0,
                    'likely risk allele', 0.3,
                    'conflicting interpretations of pathogenicity', 0.3,
                    'conflicting data from submitters', 0.3,
                    'other', 0.3,
                    'uncertain significance', 0.3,
                    'low penetrance', 0.3,
                    'uncertain risk allele', 0.3,
                    'established risk allele', 0.5,
                    'risk factor', 0.5,
                    'affects', 0.5,
                    'likely pathogenic', 0.7,
                    'confers sensitivity', 0.9,
                    'association', 0.9,
                    'drug response', 0.9,
                    'protective', 0.9,
                    'pathogenic', 0.9
                  ),
                  x
                )
              )
            ),
            0.0
          ) + coalesce(
            element_at(
              map(
                'practice guideline', 0.1,
                'reviewed by expert panel', 0.07,
                'criteria provided, multiple submitters, no conflicts', 0.05,
                'criteria provided, conflicting interpretations', 0.02,
                'criteria provided, single submitter', 0.02,
                'no assertion for the individual variant', 0.0,
                'no assertion criteria provided', 0.0,
                'no assertion provided', 0.0
              ),
              confidence
            ),
            0.0
          )
        unique_fields:
          - targetId
          - targetFromSourceId
          - diseaseId
          - datasourceId
          - studyId
          - variantId
        direction_on_trait_expression: |
          CASE WHEN CONCAT_WS('', clinicalSignificances) RLIKE '(?i)pathogenic'
          AND CONCAT_WS('', clinicalSignificances) RLIKE '(?i)protect' THEN null
          WHEN CONCAT_WS('', clinicalSignificances) RLIKE '(?i)pathogenic' THEN 'risk'
          WHEN CONCAT_WS('', clinicalSignificances) RLIKE '(?i)protect' THEN 'protect'
          ELSE null
          END
        direction_on_target_expression: |
          CASE
            WHEN variantFunctionalConsequenceId IN (
              'SO_0001589',
              'SO_0001587',
              'SO_0001574',
              'SO_0001575',
              'SO_0002012',
              'SO_0001578',
              'SO_0001893'
            ) THEN 'LoF'
            ELSE null
          END

  evidence_postprocess_uniprot_variants:
    - name: pyspark evidence_postprocess
      pyspark: evidence_postprocess
      source:
        evidence_path: input/evidence/uniprot_variants.json.gz
        target_path: output/target
        disease_path: output/disease
        publication_date_lut: input/literature/literature_export
      destination:
        failed_evidence: excluded/evidence/uniprot_variants
        evidence: output/evidence_uniprot_variants
      settings:
        evidence_format: json
        datasource_id: uniprot_variants
        score_expression: |
          element_at(
            map(
              'high', 1.0,
              'medium', 0.5
            ),
            confidence
          )
        unique_fields:
          - targetId
          - targetFromSourceId
          - diseaseId
          - datasourceId
          - diseaseFromSource
          - variantRsId
          - variantId
          - literature

  evidence_postprocess_uniprot_literature:
    - name: pyspark evidence_postprocess
      pyspark: evidence_postprocess
      source:
        evidence_path: input/evidence/uniprot_literature.json.gz
        target_path: output/target
        disease_path: output/disease
        publication_date_lut: input/literature/literature_export
      destination:
        failed_evidence: excluded/evidence/uniprot_literature
        evidence: output/evidence_uniprot_literature
      settings:
        evidence_format: json
        datasource_id: uniprot_literature
        score_expression: |
          element_at(
            map(
              'high', 1.0,
              'medium', 0.5
            ),
            confidence
          )
        unique_fields:
          - targetId
          - targetFromSourceId
          - diseaseId
          - datasourceId
          - diseaseFromSource

  evidence_postprocess_clingen:
    - name: pyspark evidence_postprocess
      pyspark: evidence_postprocess
      source:
        evidence_path: intermediate/evidence/clingen.parquet
        target_path: output/target
        disease_path: output/disease
        publication_date_lut: input/literature/literature_export
      destination:
        failed_evidence: excluded/evidence/clingen
        evidence: output/evidence_clingen
      settings:
        datasource_id: clingen
        evidence_format: parquet
        score_expression: |
          element_at(
            map(
              'No Known Disease Relationship', 0.01,
              'Refuted', 0.01,
              'Disputed', 0.01,
              'Limited', 0.01,
              'Moderate', 0.5,
              'Strong', 1.0,
              'Definitive', 1.0
            ),
            confidence
          )
        unique_fields:
          - targetId
          - targetFromSourceId
          - diseaseId
          - datasourceId
          - diseaseFromSource
          - studyId
          - allelicRequirements

  evidence_postprocess_cancer_biomarkers:
    - name: pyspark evidence_postprocess
      pyspark: evidence_postprocess
      source:
        evidence_path: intermediate/evidence/cancer_biomarkers.parquet
        target_path: output/target
        disease_path: output/disease
        publication_date_lut: input/literature/literature_export
      destination:
        failed_evidence: excluded/evidence/cancer_biomarkers
        evidence: output/evidence_cancer_biomarkers
      settings:
        evidence_format: parquet
        datasource_id: cancer_biomarkers
        score_expression: '1.0'
        unique_fields:
          - targetId
          - targetFromSourceId
          - diseaseId
          - datasourceId
          - biomarkerName
          - confidence
          - diseaseFromSource
          - drugFromSource
          - drugResponse

  evidence_postprocess_clinical_precedence:
    - name: pyspark evidence_postprocess
      pyspark: evidence_postprocess
      source:
        evidence_path: intermediate/evidence/clinical_precedence.parquet
        target_path: output/target
        disease_path: output/disease
        publication_date_lut: input/literature/literature_export
      destination:
        failed_evidence: excluded/evidence/clinical_precedence
        evidence: output/evidence_clinical_precedence
      properties:
        spark.driver.memory: 8g
        spark.executor.memory: 8g
        spark.sql.shuffle.partitions: '800'
      settings:
        evidence_format: parquet
        datasource_id: clinical_precedence
        score_expression: |
          element_at(
            map(
              'APPROVAL', 1.0,
              'PHASE_4', 1.0,
              'WITHDRAWAL', 1.0,
              'PREAPPROVAL', 0.8,
              'PHASE_3', 0.7,
              'PHASE_2_3', 0.5,
              'PHASE_2', 0.2,
              'PHASE_1_2', 0.15,
              'PHASE_1', 0.1,
              'EARLY_PHASE_1', 0.05,
              'IND', 0.05,
              'PRECLINICAL', 0.01,
              'UNKNOWN', 0.01
            ),
            clinicalStage
          )
          *
          case
          when trialStopReasonCategories is null then
            double(1.0)
          else
            array_min(
              transform(
                trialStopReasonCategories,
                x -> element_at(
                  map(
                    'Another_Study', double(1.0),
                    'Business_Administrative', double(1.0),
                    'Covid19', double(1.0),
                    'Ethical_Reason', double(1.0),
                    'Insufficient_Data', double(1.0),
                    'Insufficient_Enrollment', double(1.0),
                    'Interim_Analysis', double(1.0),
                    'Invalid_Reason', double(1.0),
                    'Logistics_Resources', double(1.0),
                    'Endpoint_Met', double(1.0),
                    'Negative', double(0.5),
                    'No_Context', double(1.0),
                    'Regulatory', double(1.0),
                    'Safety_Sideeffects', double(0.5),
                    'Study_Design', double(1.0),
                    'Study_Staff_Moved', double(1.0),
                    'Success', double(1.0),
                    'Uncategorised', double(1.0)
                  ),
                  x
                )
              )
            )
          end
        unique_fields:
          - targetId
          - targetFromSourceId
          - diseaseId
          - datasourceId
          - drugId
          - clinicalReportId
        direction_on_trait_expression: CASE WHEN diseaseId IS NOT NULL THEN 'protect' END
        direction_on_target_expression: "CASE
          WHEN actionType IN (
          'RNAI INHIBITOR',
          'NEGATIVE MODULATOR',
          'NEGATIVE ALLOSTERIC MODULATOR',
          'ANTAGONIST',
          'ANTISENSE INHIBITOR',
          'BLOCKER',
          'INHIBITOR',
          'DEGRADER',
          'INVERSE AGONIST',
          'ALLOSTERIC ANTAGONIST',
          'DISRUPTING AGENT'
          'GENE EDITING NEGATIVE MODULATOR'
          ) THEN 'LoF'
          WHEN actionType IN (
          'PARTIAL AGONIST',
          'ACTIVATOR',
          'POSITIVE ALLOSTERIC MODULATOR',
          'POSITIVE MODULATOR',
          'AGONIST',
          'SEQUESTERING AGENT',
          'STABILISER'
          ) THEN 'GoF'
          ELSE NULL
          END"

  evidence_postprocess_reactome:
    - name: pyspark evidence_postprocess
      pyspark: evidence_postprocess
      source:
        evidence_path: input/evidence/reactome.json.gz
        target_path: output/target
        disease_path: output/disease
        publication_date_lut: input/literature/literature_export
      destination:
        failed_evidence: excluded/evidence/reactome
        evidence: output/evidence_reactome
      settings:
        evidence_format: json
        datasource_id: reactome
        score_expression: '1.0'
        unique_fields:
          - targetId
          - targetFromSourceId
          - diseaseId
          - datasourceId
          - variantAminoacidDescriptions
          - targetModulation
          - reactionId

  evidence_postprocess_impc:
    - name: pyspark evidence_postprocess
      pyspark: evidence_postprocess
      source:
        evidence_path: intermediate/evidence/impc.parquet
        target_path: output/target
        disease_path: output/disease
        publication_date_lut: input/literature/literature_export
      destination:
        failed_evidence: excluded/evidence/impc
        evidence: output/evidence_impc
      settings:
        evidence_format: parquet
        datasource_id: impc
        score_expression: 'resourceScore / 100'
        unique_fields:
          - targetId
          - targetFromSourceId
          - diseaseId
          - datasourceId
          - biologicalModelGeneticBackground
          - targetInModel
          - biologicalModelAllelicComposition
          - diseaseFromSource
        direction_on_trait_expression: CASE WHEN diseaseId IS NOT NULL THEN 'risk' END
        direction_on_target_expression: CASE WHEN diseaseId IS NOT NULL THEN 'LoF' END

  evidence_postprocess_orphanet:
    - name: pyspark evidence_postprocess
      pyspark: evidence_postprocess
      source:
        evidence_path: intermediate/evidence/orphanet.parquet
        target_path: output/target
        disease_path: output/disease
        publication_date_lut: input/literature/literature_export
      destination:
        failed_evidence: excluded/evidence/orphanet
        evidence: output/evidence_orphanet
      settings:
        evidence_format: parquet
        datasource_id: orphanet
        score_expression: "element_at(map('Assessed', 1.0, 'Not yet assessed', 0.5), confidence)"
        unique_fields:
          - targetId
          - targetFromSourceId
          - diseaseId
          - datasourceId
          - diseaseFromSourceId
        direction_on_trait_expression: CASE WHEN diseaseId IS NOT NULL THEN 'risk' END
        direction_on_target_expression: |
          CASE
            WHEN variantFunctionalConsequenceId == 'SO_0002053' THEN 'GoF'
            WHEN variantFunctionalConsequenceId == 'SO_0002054' THEN 'LoF'
            ELSE null
          END

  evidence_postprocess_gene2phenotype:
    - name: pyspark evidence_postprocess
      pyspark: evidence_postprocess
      source:
        evidence_path: intermediate/evidence/gene2phenotype.parquet
        target_path: output/target
        disease_path: output/disease
        publication_date_lut: input/literature/literature_export
      destination:
        failed_evidence: excluded/evidence/gene2phenotype
        evidence: output/evidence_gene2phenotype
      settings:
        evidence_format: parquet
        datasource_id: gene2phenotype
        score_expression: |
          element_at(
            map(
              'definitive', 1.0,
              'both RD and IF', 1.0,
              'strong', 1.0,
              'moderate', 0.5,
              'limited', 0.01
            ),
            confidence
          )
        unique_fields:
          - targetId
          - targetFromSourceId
          - diseaseId
          - datasourceId
          - diseaseFromSource
          - allelicRequirements
          - studyId
        direction_on_trait_expression: CASE WHEN diseaseId IS NOT NULL THEN 'risk' END
        direction_on_target_expression: |
          CASE
            WHEN variantFunctionalConsequenceId == 'SO_0002315' THEN 'GoF'
            WHEN variantFunctionalConsequenceId == 'SO_0002317' THEN 'LoF'
            ELSE null
          END

  evidence_postprocess_intogen:
    - name: pyspark evidence_postprocess
      pyspark: evidence_postprocess
      source:
        evidence_path: intermediate/evidence/intogen.parquet
        target_path: output/target
        disease_path: output/disease
        publication_date_lut: input/literature/literature_export
      destination:
        failed_evidence: excluded/evidence/intogen
        evidence: output/evidence_intogen
      settings:
        evidence_format: parquet
        datasource_id: intogen
        score_expression: pvalue_linear_score(resourceScore, 0.1, 1e-10, 0.25, 1.0)
        unique_fields:
          - targetId
          - targetFromSourceId
          - diseaseId
          - datasourceId
          - cohortShortName
        direction_on_trait_expression: CASE WHEN diseaseId IS NOT NULL THEN 'risk' END
        direction_on_target_expression: |
          CASE
            WHEN mutatedSamples IS NOT NULL AND
              exists(
                transform(
                  mutatedSamples,
                  x -> x.functionalConsequenceId
                ), y -> y IN ("SO_0002054")
              )
            THEN 'LoF'
            WHEN mutatedSamples IS NOT NULL AND
              exists(
                transform(
                  mutatedSamples,
                  x -> x.functionalConsequenceId
                ), y -> y IN ("SO_0002053")
              )
            THEN 'GoF'
          END

  evidence_postprocess_gene_burden:
    - name: pyspark evidence_postprocess
      pyspark: evidence_postprocess
      source:
        evidence_path: intermediate/evidence/gene_burden.parquet
        target_path: output/target
        disease_path: output/disease
        publication_date_lut: input/literature/literature_export
      destination:
        failed_evidence: excluded/evidence/gene_burden
        evidence: output/evidence_gene_burden
      settings:
        evidence_format: parquet
        datasource_id: gene_burden
        score_expression: pvalue_linear_score(resourceScore, 1e-7, 1e-17, 0.25, 1.0)
        unique_fields:
          - targetId
          - targetFromSourceId
          - diseaseId
          - datasourceId
          - ancestryId
          - diseaseFromSource
          - statisticalMethod
          - projectId
        direction_on_trait_expression: |
          CASE WHEN oddsRatio > 1.0 THEN 'risk'
          WHEN oddsRatio < 1.0 THEN 'protect'
          WHEN beta > 0.0 THEN 'risk'
          WHEN beta < 0.0 THEN 'protect'
          END
        direction_on_target_expression: "'LoF'"

  evidence_postprocess_crispr_screens:
    - name: pyspark evidence_postprocess
      pyspark: evidence_postprocess
      source:
        evidence_path: intermediate/evidence/crispr_screens.parquet
        target_path: output/target
        disease_path: output/disease
        publication_date_lut: input/literature/literature_export
      destination:
        failed_evidence: excluded/evidence/crispr_screen
        evidence: output/evidence_crispr_screen
      settings:
        evidence_format: parquet
        datasource_id: crispr_screen
        score_expression: pvalue_linear_score(resourceScore, 0.5, 0.005, 0.0, 1.0)
        unique_fields:
          - targetId
          - targetFromSourceId
          - diseaseId
          - datasourceId
          - studyId

  evidence_postprocess_europepmc:
    - name: pyspark evidence_postprocess
      pyspark: evidence_postprocess
      source:
        evidence_path: intermediate/evidence/literature_epmc
        target_path: output/target
        disease_path: output/disease
        publication_date_lut: input/literature/literature_export
      destination:
        failed_evidence: excluded/evidence/europepmc
        evidence: output/evidence_europepmc
      settings:
        evidence_format: json
        datasource_id: europepmc
        score_expression: array_min(array(resourceScore / 100.0, 1.0))
        unique_fields:
          - targetId
          - targetFromSourceId
          - diseaseId
          - datasourceId
          - literature

  evidence_postprocess_panel_app:
    - name: pyspark evidence_postprocess
      pyspark: evidence_postprocess
      source:
        evidence_path: intermediate/evidence/panel_app.parquet
        target_path: output/target
        disease_path: output/disease
        publication_date_lut: input/literature/literature_export
      destination:
        failed_evidence: excluded/evidence/panel_app
        evidence: output/evidence_genomics_england
      settings:
        evidence_format: parquet
        datasource_id: genomics_england
        score_expression: |
          element_at(
            map(
              'amber', 0.5,
              'green', 1.0
            ),
            confidence
          )
        unique_fields:
          - targetId
          - targetFromSourceId
          - diseaseId
          - datasourceId
          - diseaseFromSource
          - studyId

  evidence_postprocess_project_score:
    - name: pyspark evidence_postprocess
      pyspark: evidence_postprocess
      source:
        evidence_path: intermediate/evidence/project_score.parquet
        target_path: output/target
        disease_path: output/disease/
        publication_date_lut: input/literature/literature_export/
      destination:
        failed_evidence: excluded/evidence/crispr
        evidence: output/evidence_crispr
      settings:
        evidence_format: parquet
        datasource_id: crispr
        score_expression: linear_rescale(resourceScore, 41.5, 100, 0.415, 1.0)
        unique_fields:
          - targetId
          - targetFromSourceId
          - diseaseId
          - datasourceId

  evidence_postprocess_cosmic:
    - name: pyspark evidence_postprocess
      pyspark: evidence_postprocess
      source:
        evidence_path: input/evidence/cosmic.json.gz
        target_path: output/target
        disease_path: output/disease
        publication_date_lut: input/literature/literature_export
      destination:
        failed_evidence: excluded/evidence/cancer_gene_census
        evidence: output/evidence_cancer_gene_census
      settings:
        evidence_format: json
        datasource_id: cancer_gene_census
        score_expression: resourceScore
        unique_fields:
          - targetId
          - targetFromSourceId
          - diseaseId
          - datasourceId
        direction_on_trait_expression: CASE WHEN diseaseId IS NOT NULL THEN 'risk' END
        direction_on_target_expression: |
          CASE
            WHEN TSorOncogene == 'oncogene' THEN 'GoF'
            WHEN TSorOncogene == 'tsg' THEN 'LoF'
            ELSE null
          END

  evidence_postprocess_ot_crispr:
    - name: pyspark evidence_postprocess
      pyspark: evidence_postprocess
      source:
        evidence_path: intermediate/evidence/ot_crispr.parquet
        target_path: output/target
        disease_path: output/disease
        publication_date_lut: input/literature/literature_export
      destination:
        failed_evidence: excluded/evidence/ot_crispr
        evidence: output/evidence_ot_crispr
      settings:
        evidence_format: parquet
        datasource_id: ot_crispr
        score_expression: '1.0'
        unique_fields:
          - targetId
          - targetFromSourceId
          - diseaseId
          - datasourceId
          - studyId
          - diseaseFromSourceMappedId
          - targetFromSourceId
          - resourceScore
          - statisticalTestTail

  evidence_postprocess_encore:
    - name: pyspark evidence_postprocess
      pyspark: evidence_postprocess
      source:
        evidence_path: input/evidence/encore.json.gz
        target_path: output/target
        disease_path: output/disease
        publication_date_lut: input/literature/literature_export
      destination:
        failed_evidence: excluded/evidence/encore
        evidence: output/evidence_encore
      settings:
        evidence_format: json
        datasource_id: encore
        score_expression: pvalue_linear_score(geneticInteractionPValue, 1.0, 0.01, 0.0, 1.0)
        unique_fields:
          - targetId
          - targetFromSourceId
          - diseaseId
          - datasourceId
          - targetFromSourceId
          - interactingTargetFromSourceId
          - diseaseFromSourceMappedId
          - diseaseCellLines

  evidence_postprocess_validation_lab:
    - name: pyspark evidence_postprocess
      pyspark: evidence_postprocess
      source:
        evidence_path: input/evidence/validation_lab.json.gz
        target_path: output/target
        disease_path: output/disease
        publication_date_lut: input/literature/literature_export
      destination:
        failed_evidence: excluded/evidence/ot_crispr_validation
        evidence: output/evidence_ot_crispr_validation
      settings:
        evidence_format: json
        datasource_id: ot_crispr_validation
        score_expression: resourceScore
        unique_fields:
          - targetId
          - targetFromSourceId
          - diseaseId
          - datasourceId
          - targetFromSourceId
          - diseaseFromSourceMappedId
          - resourceScore
          - diseaseCellLines
          - primaryProjectId
  ##################################################################################################

  #: Association on the fly STEP :##################################################################
  association_otf:
    - name: pyspark association_otf
      pyspark: association_otf
      properties:
        spark.driver.memory: '20g'
      source:
        disease: output/disease
        evidence: output/evidence_*
        target: output/target
        association_direct: output/association_overall_direct
        association_indirect: output/association_overall_indirect
      destination: view/association_otf
  ##################################################################################################

  #: VEP View generation STEP :#####################################################################
  vep_view:
    - name: pyspark generate vep view
      pyspark: vep_view
      source:
        credible_set: output/credible_set
        study_table: output/study
        l2g_table: output/l2g_prediction
        biosample: output/biosample
      destination:
        vep_data_header: intermediate/vep_view_header.tsv
        vep_data_all: intermediate/vep_view
        vep_data_gwas: intermediate/vep_view_gwas
        vep_data_qtl: intermediate/vep_view_qtl
      properties:
        spark.driver.memory: 16g
  ##################################################################################################

  #: ASSOCIATION TIMESERIES VIEW STEP :#############################################################
  association_timeseries_view:
    - name: pyspark timeseries view
      pyspark: association_timeseries_view
      source:
        association_overall_direct: output/association_overall_direct
        association_overall_indirect: output/association_overall_indirect
        association_by_datasource_direct: output/association_by_datasource_direct
        association_by_datasource_indirect: output/association_by_datasource_indirect
      destination: view/association_timeseries
      properties:
        spark.driver.memory: 12g
  ##################################################################################################

  #: RELEASE METRICS STEP :##########################################################################
  release_metrics:
    - name: transform release_metrics
      transformer: release_metrics
      source: {}
      destination:
        parquet: metrics.parquet
      settings:
        ot_release: '26.03-test4'
        metric_scopes:
          - /output/*
          - /excluded/evidence/*
        rich_dataset_whitelist:
          - /output/evidence_*
          - /excluded/evidence/*
          - /output/association_by_datasource_direct
          - /output/association_by_datasource_indirect
          - /output/disease
          - /output/target
          - /output/drug_molecule
          - /output/study
          - /output/clinical_report
        upload_to_hf_hub: false
        hf_token_filename: /var/run/secrets/hf_token
        hf_repo_id: opentargets/ot-release-metrics
        hf_data_dir: metrics
  ##################################################################################################