task-groups.yaml

task_metrics:
  mmlu: acc
  mmlu_prox_cs: exact_match
  mmlu_prox_de: exact_match
  mmlu_prox_en: exact_match
  mmlu_prox_es: exact_match
  mmlu_prox_fr: exact_match
  mmlu_prox_hu: exact_match
  mmlu_prox_it: exact_match
  mmlu_prox_pt: exact_match
  mmlu_prox_sr: exact_match
  mmlu_prox_uk: exact_match
  sib200_bul_Cyrl: acc_norm
  sib200_hrv_Latn: acc_norm
  sib200_ces_Latn: acc_norm
  sib200_dan_Latn: acc_norm
  sib200_nld_Latn: acc_norm
  sib200_eng_Latn: acc_norm
  sib200_est_Latn: acc_norm
  sib200_fin_Latn: acc_norm
  sib200_fra_Latn: acc_norm
  sib200_deu_Latn: acc_norm
  sib200_ell_Grek: acc_norm
  sib200_hun_Latn: acc_norm
  sib200_gle_Latn: acc_norm
  sib200_ita_Latn: acc_norm
  sib200_lvs_Latn: acc_norm
  sib200_lit_Latn: acc_norm
  sib200_mlt_Latn: acc_norm
  sib200_pol_Latn: acc_norm
  sib200_por_Latn: acc_norm
  sib200_ron_Latn: acc_norm
  sib200_slk_Latn: acc_norm
  sib200_slv_Latn: acc_norm
  sib200_spa_Latn: acc_norm
  sib200_swe_Latn: acc_norm
  sib200_cat_Latn: acc_norm
  sib200_eus_Latn: acc_norm
  sib200_glg_Latn: acc_norm
  sib200_bos_Latn: acc_norm
  sib200_kat_Geor: acc_norm
  sib200_mkd_Cyrl: acc_norm
  sib200_als_Latn: acc_norm
  sib200_srp_Cyrl: acc_norm
  sib200_tur_Latn: acc_norm
  sib200_ukr_Cyrl: acc_norm
  sib200_isl_Latn: acc_norm
  sib200_nob_Latn: acc_norm
  belebele_bul_Cyrl_cf: acc_norm
  belebele_hrv_Latn_cf: acc_norm
  belebele_ces_Latn_cf: acc_norm
  belebele_dan_Latn_cf: acc_norm
  belebele_nld_Latn_cf: acc_norm
  belebele_eng_Latn_cf: acc_norm
  belebele_est_Latn_cf: acc_norm
  belebele_fin_Latn_cf: acc_norm
  belebele_fra_Latn_cf: acc_norm
  belebele_deu_Latn_cf: acc_norm
  belebele_ell_Grek_cf: acc_norm
  belebele_hun_Latn_cf: acc_norm
  belebele_ita_Latn_cf: acc_norm
  belebele_lvs_Latn_cf: acc_norm
  belebele_lit_Latn_cf: acc_norm
  belebele_mlt_Latn_cf: acc_norm
  belebele_pol_Latn_cf: acc_norm
  belebele_por_Latn_cf: acc_norm
  belebele_ron_Latn_cf: acc_norm
  belebele_slk_Latn_cf: acc_norm
  belebele_slv_Latn_cf: acc_norm
  belebele_spa_Latn_cf: acc_norm
  belebele_swe_Latn_cf: acc_norm
  belebele_nob_Latn_cf: acc_norm
  belebele_eus_Latn_cf: acc_norm
  belebele_cat_Latn_cf: acc_norm
  xcsqa_deu_Latn: acc_norm
  xcsqa_eng_Latn: acc_norm
  xcsqa_spa_Latn: acc_norm
  xcsqa_fra_Latn: acc_norm
  xcsqa_ita_Latn: acc_norm
  xcsqa_nld_Latn: acc_norm
  xcsqa_pol_Latn: acc_norm
  xcsqa_por_Latn: acc_norm
  copa: acc
  lambada_openai: acc
  openbookqa: acc_norm
  winogrande: acc
  arc_challenge: acc_norm
  arc_easy: acc_norm
  arc_challenge_mt_bg: acc_norm
  arc_challenge_mt_cs: acc_norm
  arc_challenge_mt_da: acc_norm
  arc_challenge_mt_nl: acc_norm
  arc_challenge_mt_et: acc_norm
  arc_challenge_mt_fi: acc_norm
  arc_challenge_mt_fr: acc_norm
  arc_challenge_mt_de: acc_norm
  arc_challenge_mt_el: acc_norm
  arc_challenge_mt_hu: acc_norm
  arc_challenge_mt_it: acc_norm
  arc_challenge_mt_lv: acc_norm
  arc_challenge_mt_lt: acc_norm
  arc_challenge_mt_pl: acc_norm
  arc_challenge_mt_pt: acc_norm
  arc_challenge_mt_ro: acc_norm
  arc_challenge_mt_sk: acc_norm
  arc_challenge_mt_sl: acc_norm
  arc_challenge_mt_es: acc_norm
  arc_challenge_mt_sv: acc_norm
  arc_challenge_mt_is: acc_norm
  arc_challenge_mt_nb: acc_norm
  boolq: acc
  commonsense_qa: acc
  hellaswag: acc_norm
  piqa: acc_norm
  social_iqa: acc
  agieval_lsat_ar: acc
  wsc273: acc
  bigbench_language_identification_multiple_choice: acc
  squadv2: f1
  coqa: f1
  bigbench_qa_wikidata_generate_until: exact_match
  bigbench_dyck_languages_generate_until: exact_match
  bigbench_operators_generate_until: exact_match
  bigbench_repeat_copy_logic_generate_until: exact_match
  bigbench_cs_algorithms_generate_until: exact_match
  opensubtitles_multi40_en_to_bg: bleu
  opensubtitles_multi40_en_to_hr: bleu
  opensubtitles_multi40_en_to_cs: bleu
  opensubtitles_multi40_en_to_da: bleu
  opensubtitles_multi40_en_to_nl: bleu
  opensubtitles_multi40_en_to_et: bleu
  opensubtitles_multi40_en_to_fi: bleu
  opensubtitles_multi40_en_to_fr: bleu
  opensubtitles_multi40_en_to_de: bleu
  opensubtitles_multi40_en_to_el: bleu
  opensubtitles_multi40_en_to_hu: bleu
  opensubtitles_multi40_en_to_it: bleu
  opensubtitles_multi40_en_to_lv: bleu
  opensubtitles_multi40_en_to_lt: bleu
  opensubtitles_multi40_en_to_pl: bleu
  opensubtitles_multi40_en_to_pt: bleu
  opensubtitles_multi40_en_to_ro: bleu
  opensubtitles_multi40_en_to_sk: bleu
  opensubtitles_multi40_en_to_sl: bleu
  opensubtitles_multi40_en_to_es: bleu
  opensubtitles_multi40_en_to_sv: bleu
  opensubtitles_multi40_en_to_sr: bleu
  opensubtitles_multi40_en_to_tr: bleu
  opensubtitles_multi40_en_to_uk: bleu
  opensubtitles_multi40_en_to_no: bleu
  opensubtitles_multi40_bg_to_en: bleu
  opensubtitles_multi40_hr_to_en: bleu
  opensubtitles_multi40_cs_to_en: bleu
  opensubtitles_multi40_da_to_en: bleu
  opensubtitles_multi40_nl_to_en: bleu
  opensubtitles_multi40_et_to_en: bleu
  opensubtitles_multi40_fi_to_en: bleu
  opensubtitles_multi40_fr_to_en: bleu
  opensubtitles_multi40_de_to_en: bleu
  opensubtitles_multi40_el_to_en: bleu
  opensubtitles_multi40_hu_to_en: bleu
  opensubtitles_multi40_it_to_en: bleu
  opensubtitles_multi40_lv_to_en: bleu
  opensubtitles_multi40_lt_to_en: bleu
  opensubtitles_multi40_pl_to_en: bleu
  opensubtitles_multi40_pt_to_en: bleu
  opensubtitles_multi40_ro_to_en: bleu
  opensubtitles_multi40_sk_to_en: bleu
  opensubtitles_multi40_sl_to_en: bleu
  opensubtitles_multi40_es_to_en: bleu
  opensubtitles_multi40_sv_to_en: bleu
  opensubtitles_multi40_sr_to_en: bleu
  opensubtitles_multi40_tr_to_en: bleu
  opensubtitles_multi40_uk_to_en: bleu
  opensubtitles_multi40_no_to_en: bleu
  # flores200 (lighteval) translation: chrf++ is the primary metric
  flores200:bul_Cyrl-eng_Latn: chrf++
  flores200:hrv_Latn-eng_Latn: chrf++
  flores200:ces_Latn-eng_Latn: chrf++
  flores200:dan_Latn-eng_Latn: chrf++
  flores200:nld_Latn-eng_Latn: chrf++
  flores200:est_Latn-eng_Latn: chrf++
  flores200:fin_Latn-eng_Latn: chrf++
  flores200:fra_Latn-eng_Latn: chrf++
  flores200:deu_Latn-eng_Latn: chrf++
  flores200:ell_Grek-eng_Latn: chrf++
  flores200:hun_Latn-eng_Latn: chrf++
  flores200:gle_Latn-eng_Latn: chrf++
  flores200:ita_Latn-eng_Latn: chrf++
  flores200:lvs_Latn-eng_Latn: chrf++
  flores200:lit_Latn-eng_Latn: chrf++
  flores200:mlt_Latn-eng_Latn: chrf++
  flores200:pol_Latn-eng_Latn: chrf++
  flores200:por_Latn-eng_Latn: chrf++
  flores200:ron_Latn-eng_Latn: chrf++
  flores200:slk_Latn-eng_Latn: chrf++
  flores200:slv_Latn-eng_Latn: chrf++
  flores200:spa_Latn-eng_Latn: chrf++
  flores200:swe_Latn-eng_Latn: chrf++
  flores200:cat_Latn-eng_Latn: chrf++
  flores200:eus_Latn-eng_Latn: chrf++
  flores200:glg_Latn-eng_Latn: chrf++
  flores200:bos_Latn-eng_Latn: chrf++
  flores200:kat_Geor-eng_Latn: chrf++
  flores200:mkd_Cyrl-eng_Latn: chrf++
  flores200:als_Latn-eng_Latn: chrf++
  flores200:srp_Cyrl-eng_Latn: chrf++
  flores200:tur_Latn-eng_Latn: chrf++
  flores200:ukr_Cyrl-eng_Latn: chrf++
  flores200:isl_Latn-eng_Latn: chrf++
  flores200:nob_Latn-eng_Latn: chrf++
  flores200:eng_Latn-bul_Cyrl: chrf++
  flores200:eng_Latn-hrv_Latn: chrf++
  flores200:eng_Latn-ces_Latn: chrf++
  flores200:eng_Latn-dan_Latn: chrf++
  flores200:eng_Latn-nld_Latn: chrf++
  flores200:eng_Latn-est_Latn: chrf++
  flores200:eng_Latn-fin_Latn: chrf++
  flores200:eng_Latn-fra_Latn: chrf++
  flores200:eng_Latn-deu_Latn: chrf++
  flores200:eng_Latn-ell_Grek: chrf++
  flores200:eng_Latn-hun_Latn: chrf++
  flores200:eng_Latn-gle_Latn: chrf++
  flores200:eng_Latn-ita_Latn: chrf++
  flores200:eng_Latn-lvs_Latn: chrf++
  flores200:eng_Latn-lit_Latn: chrf++
  flores200:eng_Latn-mlt_Latn: chrf++
  flores200:eng_Latn-pol_Latn: chrf++
  flores200:eng_Latn-por_Latn: chrf++
  flores200:eng_Latn-ron_Latn: chrf++
  flores200:eng_Latn-slk_Latn: chrf++
  flores200:eng_Latn-slv_Latn: chrf++
  flores200:eng_Latn-spa_Latn: chrf++
  flores200:eng_Latn-swe_Latn: chrf++
  flores200:eng_Latn-cat_Latn: chrf++
  flores200:eng_Latn-eus_Latn: chrf++
  flores200:eng_Latn-glg_Latn: chrf++
  flores200:eng_Latn-bos_Latn: chrf++
  flores200:eng_Latn-kat_Geor: chrf++
  flores200:eng_Latn-mkd_Cyrl: chrf++
  flores200:eng_Latn-als_Latn: chrf++
  flores200:eng_Latn-srp_Cyrl: chrf++
  flores200:eng_Latn-tur_Latn: chrf++
  flores200:eng_Latn-ukr_Cyrl: chrf++
  flores200:eng_Latn-isl_Latn: chrf++
  flores200:eng_Latn-nob_Latn: chrf++
  global_piqa_completions_als_latn: acc_norm
  global_piqa_completions_bos_latn: acc_norm
  global_piqa_completions_bul_cyrl: acc_norm
  global_piqa_completions_cat_latn: acc_norm
  global_piqa_completions_ces_latn: acc_norm
  global_piqa_completions_deu_latn: acc_norm
  global_piqa_completions_ekk_latn: acc_norm
  global_piqa_completions_ell_grek: acc_norm
  global_piqa_completions_eng_latn: acc_norm
  global_piqa_completions_fin_latn: acc_norm
  global_piqa_completions_fra_latn_fran: acc_norm
  global_piqa_completions_glg_latn: acc_norm
  global_piqa_completions_hrv_latn: acc_norm
  global_piqa_completions_hun_latn: acc_norm
  global_piqa_completions_isl_latn: acc_norm
  global_piqa_completions_ita_latn: acc_norm
  global_piqa_completions_kat_geor: acc_norm
  global_piqa_completions_lit_latn: acc_norm
  global_piqa_completions_mkd_cyrl: acc_norm
  global_piqa_completions_nld_latn: acc_norm
  global_piqa_completions_nno_latn: acc_norm
  global_piqa_completions_nob_latn: acc_norm
  global_piqa_completions_pol_latn: acc_norm
  global_piqa_completions_por_latn_port: acc_norm
  global_piqa_completions_ron_latn: acc_norm
  global_piqa_completions_slk_latn: acc_norm
  global_piqa_completions_slv_latn: acc_norm
  global_piqa_completions_spa_latn_spai: acc_norm
  global_piqa_completions_srp_cyrl: acc_norm
  global_piqa_completions_swe_latn: acc_norm
  global_piqa_completions_tur_latn: acc_norm
  global_piqa_completions_ukr_cyrl: acc_norm
  global_piqa_prompted_als_latn: exact_match
  global_piqa_prompted_bos_latn: exact_match
  global_piqa_prompted_bul_cyrl: exact_match
  global_piqa_prompted_cat_latn: exact_match
  global_piqa_prompted_ces_latn: exact_match
  global_piqa_prompted_deu_latn: exact_match
  global_piqa_prompted_ekk_latn: exact_match
  global_piqa_prompted_ell_grek: exact_match
  global_piqa_prompted_eng_latn: exact_match
  global_piqa_prompted_fin_latn: exact_match
  global_piqa_prompted_fra_latn_fran: exact_match
  global_piqa_prompted_glg_latn: exact_match
  global_piqa_prompted_hrv_latn: exact_match
  global_piqa_prompted_hun_latn: exact_match
  global_piqa_prompted_isl_latn: exact_match
  global_piqa_prompted_ita_latn: exact_match
  global_piqa_prompted_kat_geor: exact_match
  global_piqa_prompted_lit_latn: exact_match
  global_piqa_prompted_mkd_cyrl: exact_match
  global_piqa_prompted_nld_latn: exact_match
  global_piqa_prompted_nno_latn: exact_match
  global_piqa_prompted_nob_latn: exact_match
  global_piqa_prompted_pol_latn: exact_match
  global_piqa_prompted_por_latn_port: exact_match
  global_piqa_prompted_ron_latn: exact_match
  global_piqa_prompted_slk_latn: exact_match
  global_piqa_prompted_slv_latn: exact_match
  global_piqa_prompted_spa_latn_spai: exact_match
  global_piqa_prompted_srp_cyrl: exact_match
  global_piqa_prompted_swe_latn: exact_match
  global_piqa_prompted_tur_latn: exact_match
  global_piqa_prompted_ukr_cyrl: exact_match
  polymath_de_low: exact_match
  polymath_de_medium: exact_match
  polymath_de_high: exact_match
  polymath_de_top: exact_match
  polymath_en_low: exact_match
  polymath_en_medium: exact_match
  polymath_en_high: exact_match
  polymath_en_top: exact_match
  polymath_es_low: exact_match
  polymath_es_medium: exact_match
  polymath_es_high: exact_match
  polymath_es_top: exact_match
  polymath_fr_low: exact_match
  polymath_fr_medium: exact_match
  polymath_fr_high: exact_match
  polymath_fr_top: exact_match
  polymath_it_low: exact_match
  polymath_it_medium: exact_match
  polymath_it_high: exact_match
  polymath_it_top: exact_match
  polymath_pt_low: exact_match
  polymath_pt_medium: exact_match
  polymath_pt_high: exact_match
  polymath_pt_top: exact_match

task_groups:
  sib200-eu:
    description: "SIB-200 European language topic classification tasks (0-shot)"
    suite: lm-eval-harness
    n_shots: [0]
    dataset: Davlan/sib200
    valid_langs: [bul_Cyrl, hrv_Latn, ces_Latn, dan_Latn, nld_Latn, eng_Latn, est_Latn, fin_Latn,
                  fra_Latn, deu_Latn, ell_Grek, hun_Latn, gle_Latn, ita_Latn, lvs_Latn, lit_Latn,
                  mlt_Latn, pol_Latn, por_Latn, ron_Latn, slk_Latn, slv_Latn, spa_Latn, swe_Latn,
                  cat_Latn, eus_Latn, glg_Latn, bos_Latn, kat_Geor, mkd_Cyrl, als_Latn, srp_Cyrl,
                  tur_Latn, ukr_Cyrl, isl_Latn, nob_Latn]
    tasks:
      - task: "sib200_{lang}"
        subset: "{lang}"
  open-sci-0.01:
    description: "open-sci-ref 0.01 evals"
    suite: lm-eval-harness
    tasks:
      - task: copa
        n_shots: [0]
        dataset: aps/super_glue
        subset: copa
      - task: social_iqa
        n_shots: [0]
        dataset: allenai/social_i_qa
      - task: openbookqa
        n_shots: [0]
        dataset: allenai/openbookqa
      - task: lambada_openai
        n_shots: [0]
        dataset: EleutherAI/lambada_openai
      - task: winogrande
        n_shots: [0]
        dataset: allenai/winogrande
        subset: winogrande_xl
      - task: mmlu
        n_shots: [5]
        dataset: cais/mmlu
      - task: hellaswag
        n_shots: [10]
        dataset: Rowan/hellaswag
      - task: arc_easy
        n_shots: [10]
        dataset: allenai/ai2_arc
        subset: ARC-Easy
      - task: arc_challenge
        n_shots: [10]
        dataset: allenai/ai2_arc
        subset: ARC-Challenge
      - task: commonsense_qa
        n_shots: [10]
        dataset: tau/commonsense_qa
      - task: piqa
        n_shots: [10]
        dataset: baber/piqa
      - task: boolq
        n_shots: [10]
        dataset: aps/super_glue
        subset: boolq
  belebele-eu-5-shot:
    description: "Belebele European language tasks"
    suite: lm-eval-harness
    n_shots: [5]
    dataset: facebook/belebele
    valid_langs: [bul_Cyrl, hrv_Latn, ces_Latn, dan_Latn, nld_Latn, eng_Latn, est_Latn, fin_Latn,
                  fra_Latn, deu_Latn, ell_Grek, hun_Latn, ita_Latn, lvs_Latn, lit_Latn, mlt_Latn,
                  pol_Latn, por_Latn, ron_Latn, slk_Latn, slv_Latn, spa_Latn, swe_Latn, nob_Latn]
    tasks:
      - task: "belebele_{lang}"
        subset: "{lang}"
  belebele-eu-cf:
    description: "Belebele European language tasks (cloze formulation, lighteval)"
    suite: lighteval
    n_shots: [0]
    dataset: facebook/belebele
    valid_langs: [bul_Cyrl, hrv_Latn, ces_Latn, dan_Latn, nld_Latn, eng_Latn, est_Latn, fin_Latn,
                  fra_Latn, deu_Latn, ell_Grek, hun_Latn, ita_Latn, lvs_Latn, lit_Latn, mlt_Latn,
                  pol_Latn, por_Latn, ron_Latn, slk_Latn, slv_Latn, spa_Latn, swe_Latn, nob_Latn,
                  eus_Latn, cat_Latn]
    tasks:
      - task: "belebele_{lang}_cf"
        subset: "{lang}"

  xcsqa-eu:
    description: "XCSQA EU commonsense QA (0-shot MCQ, acc_norm)"
    suite: lm-eval-harness
    n_shots: [0]
    dataset: INK-USC/xcsr
    tasks:
      - task: xcsqa_deu_Latn
        subset: X-CSQA-de
      - task: xcsqa_eng_Latn
        subset: X-CSQA-en
      - task: xcsqa_spa_Latn
        subset: X-CSQA-es
      - task: xcsqa_fra_Latn
        subset: X-CSQA-fr
      - task: xcsqa_ita_Latn
        subset: X-CSQA-it
      - task: xcsqa_nld_Latn
        subset: X-CSQA-nl
      - task: xcsqa_pol_Latn
        subset: X-CSQA-pl
      - task: xcsqa_por_Latn
        subset: X-CSQA-pt

  flores-200-eu-to-eng:
    description: "Flores 200 EU->English translation (0-shot, chrf++)."
    suite: lighteval
    n_shots: [0]
    dataset: facebook/flores
    valid_langs: [bul_Cyrl, hrv_Latn, ces_Latn, dan_Latn, nld_Latn, est_Latn,
                  fin_Latn, fra_Latn, deu_Latn, ell_Grek, hun_Latn, gle_Latn,
                  ita_Latn, lvs_Latn, lit_Latn, mlt_Latn, pol_Latn, por_Latn,
                  ron_Latn, slk_Latn, slv_Latn, spa_Latn, swe_Latn, cat_Latn,
                  eus_Latn, glg_Latn, bos_Latn, kat_Geor, mkd_Cyrl, als_Latn,
                  srp_Cyrl, tur_Latn, ukr_Cyrl, isl_Latn, nob_Latn]
    tasks:
      - task: "flores200:{lang}-eng_Latn"
  flores-200-eng-to-eu:
    description: "Flores 200 English->EU translation (0-shot, chrf++)."
    suite: lighteval
    n_shots: [0]
    dataset: facebook/flores
    valid_langs: [bul_Cyrl, hrv_Latn, ces_Latn, dan_Latn, nld_Latn, est_Latn,
                  fin_Latn, fra_Latn, deu_Latn, ell_Grek, hun_Latn, gle_Latn,
                  ita_Latn, lvs_Latn, lit_Latn, mlt_Latn, pol_Latn, por_Latn,
                  ron_Latn, slk_Latn, slv_Latn, spa_Latn, swe_Latn, cat_Latn,
                  eus_Latn, glg_Latn, bos_Latn, kat_Geor, mkd_Cyrl, als_Latn,
                  srp_Cyrl, tur_Latn, ukr_Cyrl, isl_Latn, nob_Latn]
    tasks:
      - task: "flores200:eng_Latn-{lang}"
  global-mmlu-eu:
    description: "Global MMLU EU tasks"
    suite: lm-eval-harness
    n_shots: [5]
    dataset: CohereForAI/Global-MMLU
    valid_langs: [cs, de, el, en, es, fr, it, lt, nl, pl, pt, ro, ru, sr, sv, tr, uk, he]
    tasks:
      - task: "global_mmlu_full_{lang}"
        subset: "{lang}"
  mmlu-prox-eu:
    description: "MMLU-ProX EU subset, 5-shot CoT, exact-match."
    suite: lm-eval-harness
    n_shots: [5]
    dataset: li-lab/MMLU-ProX
    valid_langs: [cs, de, en, es, fr, hu, it, pt, sr, uk]
    tasks:
      - task: "mmlu_prox_{lang}"
        subset: "{lang}"
  mgsm-eu:
    description: "EU Language GSM benchmarks in Aya Expanse"
    suite: lm-eval-harness
    n_shots: [5]
    dataset: jbross-ibm-research/mgsm
    tasks:
      - task: mgsm_native_cot_en
        subset: en
      - task: mgsm_native_cot_de
        subset: de
      - task: mgsm_native_cot_es
        subset: es
      - task: mgsm_native_cot_fr
        subset: fr

  generic-multilingual:
    description: "Generic multilingual benchmarks in Aya Expanse"
    suite: lm-eval-harness
    n_shots: [0]
    tasks:
      - task: xwinograd
        dataset: Muennighoff/xwinograd
      - task: xcopa
        dataset: cambridgeltl/xcopa
      - task: xstorycloze
        dataset: juletxara/xstory_cloze

  include:
    description: "INCLUDE benchmarks in Aya Expanse"
    suite: lm-eval-harness
    n_shots: [0]
    dataset: CohereForAI/include-base-44
    tasks:
      - task: include_base_44_albanian
        subset: Albanian
      - task: include_base_44_armenian
        subset: Armenian
      - task: include_base_44_azerbaijani
        subset: Azerbaijani
      - task: include_base_44_basque
        subset: Basque
      - task: include_base_44_belarusian
        subset: Belarusian
      - task: include_base_44_bulgarian
        subset: Bulgarian
      - task: include_base_44_croatian
        subset: Croatian
      - task: include_base_44_dutch
        subset: Dutch
      - task: include_base_44_estonian
        subset: Estonian
      - task: include_base_44_finnish
        subset: Finnish
      - task: include_base_44_french
        subset: French
      - task: include_base_44_georgian
        subset: Georgian
      - task: include_base_44_german
        subset: German
      - task: include_base_44_greek
        subset: Greek
      - task: include_base_44_hungarian
        subset: Hungarian
      - task: include_base_44_italian
        subset: Italian
      - task: include_base_44_lithuanian
        subset: Lithuanian
      - task: include_base_44_north macedonian
        subset: North Macedonian
      - task: include_base_44_polish
        subset: Polish
      - task: include_base_44_portuguese
        subset: Portuguese
      - task: include_base_44_russian
        subset: Russian
      - task: include_base_44_serbian
        subset: Serbian
      - task: include_base_44_spanish
        subset: Spanish
      - task: include_base_44_turkish
        subset: Turkish
      - task: include_base_44_ukrainian
        subset: Ukrainian

  dclm-core-22:
    description: "DCLM core 22 evaluation tasks (lm-eval-harness, matching LLM Foundry task types)"
    suite: lm-eval-harness
    tasks:
      - task: agieval_lsat_ar
        n_shots: [3]
        dataset: hails/agieval-lsat-ar
      - task: arc_easy
        n_shots: [10]
        dataset: allenai/ai2_arc
        subset: ARC-Easy
      - task: arc_challenge
        n_shots: [10]
        dataset: allenai/ai2_arc
        subset: ARC-Challenge
      - task: boolq
        n_shots: [10]
        dataset: aps/super_glue
        subset: boolq
      - task: commonsense_qa
        n_shots: [10]
        dataset: tau/commonsense_qa
      - task: copa
        n_shots: [0]
        dataset: aps/super_glue
        subset: copa
      - task: hellaswag
        n_shots: [0, 10]
        dataset: Rowan/hellaswag
      - task: openbookqa
        n_shots: [0]
        dataset: allenai/openbookqa
        subset: main
      - task: piqa
        n_shots: [10]
        dataset: baber/piqa
      - task: bigbench_language_identification_multiple_choice
        n_shots: [10]
        dataset: hails/bigbench
        subset: language_identification_zero_shot
      - task: winogrande
        n_shots: [0]
        dataset: allenai/winogrande
        subset: winogrande_xl
      - task: wsc273
        n_shots: [0]
        dataset: winograd_wsc
      - task: lambada_openai
        n_shots: [0]
        dataset: EleutherAI/lambada_openai
      - task: bigbench_qa_wikidata_generate_until
        n_shots: [10]
        dataset: hails/bigbench
        subset: qa_wikidata_zero_shot
      - task: bigbench_dyck_languages_generate_until
        n_shots: [10]
        dataset: hails/bigbench
        subset: dyck_languages_zero_shot
      - task: bigbench_operators_generate_until
        n_shots: [10]
        dataset: hails/bigbench
        subset: operators_zero_shot
      - task: bigbench_repeat_copy_logic_generate_until
        n_shots: [10]
        dataset: hails/bigbench
        subset: repeat_copy_logic_zero_shot
      - task: bigbench_cs_algorithms_generate_until
        n_shots: [10]
        dataset: hails/bigbench
        subset: cs_algorithms_zero_shot
      - task: coqa
        n_shots: [0]
        dataset: EleutherAI/coqa
      - task: squadv2
        n_shots: [10]
        dataset: rajpurkar/squad_v2
      - task: jeopardy
        n_shots: [10]
        dataset: openeurollm/jeopardy

  opensubtitles-eu-en-xx:
    description: "OpenSubtitles Multi40 English->EU translation (0-shot, BLEU)."
    suite: lm-eval-harness
    n_shots: [0]
    dataset: Helsinki-NLP/OpenSubtitles2024-40-langs-15-movies
    valid_langs: [bg, hr, cs, da, nl, et, fi, fr, de, el, hu, it, lv, lt, pl,
                  pt, ro, sk, sl, es, sv, sr, tr, uk, "no"]
    tasks:
      - task: "opensubtitles_multi40_en_to_{lang}"

  opensubtitles-eu-xx-en:
    description: "OpenSubtitles Multi40 EU->English translation (0-shot, BLEU)."
    suite: lm-eval-harness
    n_shots: [0]
    dataset: Helsinki-NLP/OpenSubtitles2024-40-langs-15-movies
    valid_langs: [bg, hr, cs, da, nl, et, fi, fr, de, el, hu, it, lv, lt, pl,
                  pt, ro, sk, sl, es, sv, sr, tr, uk, "no"]
    tasks:
      - task: "opensubtitles_multi40_{lang}_to_en"

  arc-challenge-mt-eu:
    description: "ARC Challenge multilingual (22 European languages)"
    suite: lm-eval-harness
    n_shots: [0]
    dataset: LumiOpen/arc_challenge_mt
    valid_langs: [bg, cs, da, nl, et, fi, fr, de, el, hu, it, lv, lt, pl, pt, ro, sk, sl, es, sv, nb]
    tasks:
      - task: "arc_challenge_mt_{lang}"
        subset: "{lang}"
      - task: arc_challenge_mt_is  # no subset in the source dataset

  reasoning:
    description: "Reasoning tasks: GSM8k (4-shot), IFEval (0-shot), MBPP (3-shot), GPQA-Diamond (0-shot), MATH500 (0-shot), LiveCodeBench (0-shot), HumanEval (0-shot), AIME24 (0-shot), AIME25 (0-shot), AMC23 (0-shot)"
    suite: lm-eval-harness
    tasks:
      - task: gsm8k
        n_shots: [4]
        dataset: openai/gsm8k
        subset: main
      - task: ifeval
        n_shots: [0]
        dataset: google/IFEval
      - task: mbpp
        n_shots: [3]
        dataset: google-research-datasets/mbpp
      - task: GPQADiamond
        n_shots: [0]
        suite: evalchemy
      - task: MATH500
        n_shots: [0]
        suite: evalchemy
      - task: LiveCodeBench
        n_shots: [0]
        suite: evalchemy
        dataset: livecodebench/code_generation_lite
      - task: HumanEval
        n_shots: [0]
        suite: evalchemy
      - task: AIME24
        n_shots: [0]
        suite: evalchemy
      - task: AIME25
        n_shots: [0]
        suite: evalchemy
      - task: AMC23
        n_shots: [0]
        suite: evalchemy

  global-piqa-eu-completions:
    description: "Global PIQA European languages — multiple-choice completions (0-shot, mrlbenchmarks/global-piqa-nonparallel)"
    suite: lm-eval-harness
    n_shots: [0]
    dataset: mrlbenchmarks/global-piqa-nonparallel
    valid_langs: [als_latn, bos_latn, bul_cyrl, cat_latn, ces_latn, deu_latn, ekk_latn, ell_grek,
                  eng_latn, fin_latn, fra_latn_fran, glg_latn, hrv_latn, hun_latn, isl_latn, ita_latn,
                  kat_geor, lit_latn, mkd_cyrl, nld_latn, nno_latn, nob_latn, pol_latn, por_latn_port,
                  ron_latn, slk_latn, slv_latn, spa_latn_spai, srp_cyrl, swe_latn, tur_latn, ukr_cyrl]
    tasks:
      - task: "global_piqa_completions_{lang}"
        subset: "{lang}"

  global-piqa-eu-prompted:
    description: "Global PIQA European languages — prompted generation (0-shot, mrlbenchmarks/global-piqa-nonparallel)"
    suite: lm-eval-harness
    n_shots: [0]
    dataset: mrlbenchmarks/global-piqa-nonparallel
    valid_langs: [als_latn, bos_latn, bul_cyrl, cat_latn, ces_latn, deu_latn, ekk_latn, ell_grek,
                  eng_latn, fin_latn, fra_latn_fran, glg_latn, hrv_latn, hun_latn, isl_latn, ita_latn,
                  kat_geor, lit_latn, mkd_cyrl, nld_latn, nno_latn, nob_latn, pol_latn, por_latn_port,
                  ron_latn, slk_latn, slv_latn, spa_latn_spai, srp_cyrl, swe_latn, tur_latn, ukr_cyrl]
    tasks:
      - task: "global_piqa_prompted_{lang}"
        subset: "{lang}"

  # PolyMath ships 4 difficulty tiers (low/medium/high/top); each is a
  # separate group so a tier can be run in isolation. EU languages only
  # (de/en/es/fr/it/pt); 0-shot generative, \boxed{} answer + exact_match.
  polymath-eu-low:
    description: "PolyMath EU math, low tier (0-shot, exact_match)."
    suite: lm-eval-harness
    n_shots: [0]
    dataset: Qwen/PolyMath
    valid_langs: [de, en, es, fr, it, pt]
    tasks:
      - task: "polymath_{lang}_low"
        subset: "{lang}"

  polymath-eu-medium:
    description: "PolyMath EU math, medium tier (0-shot, exact_match)."
    suite: lm-eval-harness
    n_shots: [0]
    dataset: Qwen/PolyMath
    valid_langs: [de, en, es, fr, it, pt]
    tasks:
      - task: "polymath_{lang}_medium"
        subset: "{lang}"

  polymath-eu-high:
    description: "PolyMath EU math, high tier (0-shot, exact_match)."
    suite: lm-eval-harness
    n_shots: [0]
    dataset: Qwen/PolyMath
    valid_langs: [de, en, es, fr, it, pt]
    tasks:
      - task: "polymath_{lang}_high"
        subset: "{lang}"

  polymath-eu-top:
    description: "PolyMath EU math, top/Olympiad tier (0-shot, exact_match)."
    suite: lm-eval-harness
    n_shots: [0]
    dataset: Qwen/PolyMath
    valid_langs: [de, en, es, fr, it, pt]
    tasks:
      - task: "polymath_{lang}_top"
        subset: "{lang}"

super_groups:
  oellm-multilingual:
    description: "Combined Belebele EU set plus multilingual benchmarks"
    task_groups:
      - task: flores-200-eu-to-eng
      - task: flores-200-eng-to-eu
      - task: belebele-eu-5-shot
      - task: global-mmlu-eu
      - task: mgsm-eu
      - task: generic-multilingual
      - task: include