eval_intern_s1_pro.py

# flake8: noqa

from mmengine.config import read_base

from opencompass.partitioners import NaivePartitioner, NumWorkerPartitioner
from opencompass.runners import LocalRunner
from opencompass.tasks import OpenICLEvalTask, OpenICLInferTask
from copy import deepcopy
from opencompass.utils.text_postprocessors import extract_non_reasoning_content
from opencompass.models import OpenAISDKStreaming

#######################################################################
#                          PART 0  Essential Configs                  #
#######################################################################
with read_base():
    # Datasets

    from opencompass.configs.datasets.mmlu_pro.mmlu_pro_0shot_nocot_genericllmeval_gen_08c1de import (
        mmlu_pro_datasets,
    )
    from opencompass.configs.datasets.gpqa.gpqa_cascade_eval_gen_772ea0 import (
        gpqa_datasets,
    )
    from opencompass.configs.datasets.aime2025.aime2025_cascade_eval_gen_5e9f4f import (
        aime2025_datasets,
    )
    from opencompass.configs.chatml_datasets.IMO_Bench_AnswerBench.IMO_Bench_AnswerBench_gen import (
        datasets as IMO_Bench_AnswerBench_chatml
    )
    from opencompass.configs.datasets.IFBench.IFBench_gen import (
        ifbench_datasets,
    )
    from opencompass.configs.datasets.livecodebench.livecodebench_gen_a4f90b import (
        LCBCodeGeneration_dataset,
    )
    from opencompass.configs.datasets.SmolInstruct.smolinstruct_0shot_instruct_gen import (
        smolinstruct_datasets_0shot_instruct as smolinstruct_datasets,
    )
    from opencompass.configs.datasets.matbench.matbench_llm_judge_gen_0e9276 import (
        matbench_datasets,
    )
    from opencompass.configs.datasets.biodata.biodata_task_gen import (
        biodata_task_datasets
    )
    from opencompass.configs.datasets.MolInstructions_chem.mol_instructions_chem_gen import (
        mol_gen_selfies_datasets
    )

    # Summary Groups
    from opencompass.configs.summarizers.groups.mmlu_pro import \
        mmlu_pro_summary_groups
    from opencompass.configs.summarizers.groups.biodata import (
        biodata_summary_groups,
    )

LCBCodeGeneration_v6_datasets = deepcopy(LCBCodeGeneration_dataset)
LCBCodeGeneration_v6_datasets['abbr'] = 'lcb_code_generation_v6'
LCBCodeGeneration_v6_datasets['release_version'] = 'v6'
LCBCodeGeneration_v6_datasets['eval_cfg']['evaluator'][
    'release_version'
] = 'v6'
LCBCodeGeneration_v6_datasets = [LCBCodeGeneration_v6_datasets]

#######################################################################
#                          PART 1  Datasets List                      #
#######################################################################
# datasets list for evaluation

repeated_info = [
    (gpqa_datasets, 8),
    (aime2025_datasets, 32),
]

for datasets_, num in repeated_info:
    for dataset_ in datasets_:
        dataset_['n'] = num
        dataset_['k'] = num

datasets = sum(
    (v for k, v in locals().items() if k.endswith('_datasets')),
    [],
)

chatml_datasets = sum(
    (v for k, v in locals().items() if k.endswith('_chatml')),
    [],
)

# LLM judge config: using LLM to evaluate predictions
judge_cfg = dict(
    abbr='YOUR_JUDGE_MODEL',
    type=OpenAISDKStreaming,
    path='YOUR_JUDGE_MODEL',
    key='YOUR_JUDGE_KEY',
    openai_api_base='YOUR_JUDGE_URL',
    mode='mid',
    meta_template=dict(
    round=[
        dict(role='HUMAN', api_role='HUMAN'),
        dict(role='BOT', api_role='BOT', generate=True),
    ]
),
    query_per_second=16,
    batch_size=64,
    temperature=0.001,
    max_out_len=8192,
    max_seq_len=32768,
)

for item in datasets:
    if 'judge_cfg' in item['eval_cfg']['evaluator']:
        item['eval_cfg']['evaluator']['judge_cfg'] = judge_cfg
    if 'llm_evaluator' in item['eval_cfg']['evaluator'].keys() and 'judge_cfg' in item['eval_cfg']['evaluator']['llm_evaluator']:
        item['eval_cfg']['evaluator']['llm_evaluator']['judge_cfg'] = judge_cfg

for item in chatml_datasets:
    if item['evaluator']['type'] == 'llm_evaluator':
        item['evaluator']['judge_cfg'] = judge_cfg
    if item['evaluator']['type'] == 'cascade_evaluator':
        item['evaluator']['llm_evaluator']['judge_cfg'] = judge_cfg


#######################################################################
#                       PART 2  Datset Summarizer                     #
#######################################################################

summary_groups = sum(
    [v for k, v in locals().items() if k.endswith('_summary_groups')], []
)

summarizer = dict(
    dataset_abbrs=[
        ['mmlu_pro', 'accuracy'],
        ['IFBench', 'score'],
        ['GPQA_diamond', 'accuracy (8 runs average)'],
        ['aime2025', 'accuracy (32 runs average)'],
        ['lcb_code_generation_v6', 'pass@1'],
        ['bio_data', 'naive_average'],
        ['IMO-Bench-AnswerBench', 'accuracy'],
        '',
        'Mol_Instruct',
        ['FS-selfies', 'score'],
        ['MC-selfies', 'score'],
        ['MG-selfies', 'score'],
        ['PP-selfies', 'score'],
        ['RP-selfies', 'score'],
        ['RS-selfies', 'score'],
        '',
        'SmolInstruct',
        ['NC-I2F-0shot-instruct', 'score'],
        ['NC-I2S-0shot-instruct', 'score'],
        ['NC-S2F-0shot-instruct', 'score'],
        ['NC-S2I-0shot-instruct', 'score'],
        ['PP-ESOL-0shot-instruct', 'score'],
        ['PP-Lipo-0shot-instruct', 'score'],
        ['PP-BBBP-0shot-instruct', 'accuracy'],
        ['PP-ClinTox-0shot-instruct', 'accuracy'],
        ['PP-HIV-0shot-instruct', 'accuracy'],
        ['PP-SIDER-0shot-instruct', 'accuracy'],
        ['MC-0shot-instruct', 'score'],
        ['MG-0shot-instruct', 'score'],
        ['FS-0shot-instruct', 'score'],
        ['RS-0shot-instruct', 'score'],
        '',
        ['matbench_expt_gap', 'mae'],
        ['matbench_steels', 'mae'],
        ['matbench_expt_is_metal', 'accuracy'],
        ['matbench_glass', 'accuracy'],
    ],
    summary_groups=summary_groups,
)


#######################################################################
#                        PART 3  Models                               #
#######################################################################

api_meta_template = dict(
    round=[
        dict(role='SYSTEM', api_role='SYSTEM'), # System prompt is only needed when evaluating Bio_data and Mol_instructions
        dict(role='HUMAN', api_role='HUMAN'),
        dict(role='BOT', api_role='BOT', generate=True),
    ]
)

models = [
dict(
    abbr='intern-s1-pro',
    type=OpenAISDKStreaming,
    path='intern-s1-pro',
    key='YOUR_API_KEY',
    openai_api_base='YOUR_API_BASE',
    meta_template=api_meta_template,
    query_per_second=16,
    batch_size=8,
    temperature=0.8,
    retry=10,
    max_out_len=65536,
    max_seq_len=65536,
    extra_body={
            'chat_template_kwargs': {'enable_thinking': True} # Disable thinking when evaluating scientific benchmarks
            },
    pred_postprocessor=dict(
        type=extract_non_reasoning_content,
    ),
),
]

#######################################################################
#                 PART 4  Inference/Evaluation Configuaration         #
#######################################################################

# infer with local runner
infer = dict(
    partitioner=dict(type=NumWorkerPartitioner, num_worker=8),
    runner=dict(
        type=LocalRunner,
        max_num_workers=16,
        task=dict(type=OpenICLInferTask),
    ),
)

# eval with local runner
eval = dict(
    partitioner=dict(type=NaivePartitioner, n=10),
    runner=dict(
        type=LocalRunner,
         max_num_workers=16,
        task=dict(type=OpenICLEvalTask)
    ),
)

#######################################################################
#                      PART 5  Utils Configuaration                   #
#######################################################################

work_dir = './outputs/oc_intern_s1_pro_eval'