diff --git a/benchmarks/indic/mbpp/README.md b/benchmarks/indic/mbpp/README.md new file mode 100644 index 00000000000..8a4efeca8d3 --- /dev/null +++ b/benchmarks/indic/mbpp/README.md @@ -0,0 +1,118 @@ + + + +# Indic MBPP + +Original MBPP from `ai4bharat/indic-mbpp`: 500 Python tasks in each of 14 Indic +languages. Includes an English baseline (`en`) from the same source records. +Reuses Gym's [English MBPP prompt](../../mbpp/prompts/default.yaml), `simple_agent`, +and the existing [SciCodePile assertion verifier](../../../resources_servers/scicodepile/README.md). +The `indic_mbpp_environment_server` uses Gym's `legacy_agent` relay to route +episodes to `indic_mbpp_simple_agent`, following the upstream benchmark layout. +The relay does not change the prompt or scoring contract. + +## Configuration + +Each prompt contains the translated description, task setup when present, and the +first original assertion as an example. Four spaces are replaced with a tab, as +in English MBPP preparation. Reference solutions and the other two assertions are +kept out of the prompt. Each response must provide complete Python code. + +Reward is 1 only when all three original `test_list` assertions pass. The existing +verifier runs each response in a fresh subprocess with a 120-second timeout. +`challenge_test_list` is excluded from the standard score. Default repeats: 1. +Use `pass@1/accuracy` for a single attempt; repeated runs also report pass@k and +`pass@1[avg-of-k]/accuracy`. Inspect coverage and `harness_failure` alongside scores. +Saved verifier results retain a language-qualified `task_id`, such as +`Mbpp/hi/11`, so results can be grouped by language without joining the inputs. +The top-level input `language` and `uuid` are not returned by SciCodePile. +Older prepared files use `Mbpp/11`; regenerate them to obtain qualified IDs. +This metadata change does not change prompts, assertions, or rewards. + +Gym's existing `mbpp` benchmark is MBPP+ (378 EvalPlus tasks), while this dataset +contains the 500 original MBPP test tasks. Their scores are different benchmarks: +only 224 task IDs overlap, and 115 of those have revised English descriptions. +Use `indic/mbpp` with `languages: [en]` for a comparison on the same 500 tasks, +with identical model and generation settings. The existing `mbpp` benchmark and +its EvalPlus base/plus metrics are unchanged. + +## Prepare data + +Install Gym and follow the shared verifier's execution requirements. Generated +code runs as a subprocess; use an isolated evaluation environment as described +in the verifier README. +The source repository requires approved Hugging Face access. Authenticate with +`hf auth login` (or supply `HF_TOKEN`) using an account that can read +`ai4bharat/indic-mbpp`; authentication alone does not grant dataset access. + +```bash +gym eval prepare --benchmark indic/mbpp +``` + +Defaults to `as`, `bn`, `gu`, `hi`, `kn`, `ml`, `mr`, `ne`, `or`, `pa`, `sa`, `ta`, +`te`, and `ur` (7,000 tasks). For per-language results, prepare and run one language +at a time; use `en` for the matching baseline: + +```bash +gym eval prepare --benchmark indic/mbpp \ + '+prepare_script_args={languages:[hi]}' \ + +use_cached_prepared_benchmarks=false +``` + +Use `task_ids` in `prepare_script_args` to select smoke-test tasks. Data is downloaded +from the dataset's default revision. Missing translations are rejected and generated +data remains local. +Preparation also writes an adjacent `.provenance.json` file recording the resolved +dataset commit, source parquet SHA-256, selected languages/task IDs, and row count. +The revision comes from the downloaded snapshot path, not a separate lookup of +the latest commit. Non-snapshot paths are recorded as `null`, with the checksum +still retained. No source revision is pinned. Keep this sidecar with the inputs +and results; cached prepared data does not automatically refresh upstream changes. + +## Collect rollouts + +```bash +gym eval run --benchmark indic/mbpp \ + --model-type vllm_model \ + --model MODEL_NAME \ + --model-url http://HOST:PORT/v1 \ + --model-api-key dummy \ + --split benchmark \ + --output results/indic_mbpp/rollouts.jsonl +``` + +The canonical config does not set sampling, thinking, a seed, or an output-token +limit. Specify these in the model/run configuration, and use the same settings +for each language and the English baseline. The verifier timeout is not a model +generation-token limit. + +## Paired real-model smoke + +The standalone [smoke config](smoke.yaml) selects one attempt each for two task +IDs in English, Hindi, and Kannada after preparing the inputs below. It uses +`google/gemma-4-31B-it`, thinking enabled, temperature 1.0, top-p 0.95, top-k 64, +seed 0, and at most 8,192 output tokens (including reasoning). +See the [inspected smoke results](smoke-results.md) for the source/model revisions, +server settings, outputs, and rewards. This small check is not a benchmark score. + +```bash +python -m benchmarks.indic.mbpp.prepare \ + --languages en hi kn --task-ids 11 367 \ + --output-fpath benchmarks/indic/mbpp/data/review-smoke.jsonl + +gym eval run --config benchmarks/indic/mbpp/smoke.yaml \ + --agent indic_mbpp_simple_agent \ + --model google/gemma-4-31B-it \ + --model-url http://HOST:PORT/v1 --model-api-key dummy \ + --split benchmark --concurrency 3 \ + --output results/indic_mbpp_smoke/rollouts.jsonl +``` + +Start a compatible vLLM endpoint with the `gemma4` reasoning parser. Do not append +`--model-type vllm_model` to this smoke command: the config already includes the +adapter, and reloading its defaults afterward can erase the sampling overrides. +Sampling/seed and thinking are scoped to the smoke model adapter, not global +request metadata. The smoke has one repeat; use a separate run configuration +with independent seeds when estimating multi-attempt metrics. + +The dataset declares Apache-2.0. diff --git a/benchmarks/indic/mbpp/__init__.py b/benchmarks/indic/mbpp/__init__.py new file mode 100644 index 00000000000..52a7a9daf02 --- /dev/null +++ b/benchmarks/indic/mbpp/__init__.py @@ -0,0 +1,2 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 diff --git a/benchmarks/indic/mbpp/config.yaml b/benchmarks/indic/mbpp/config.yaml new file mode 100644 index 00000000000..b3de2726898 --- /dev/null +++ b/benchmarks/indic/mbpp/config.yaml @@ -0,0 +1,29 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +config_paths: + - resources_servers/scicodepile/configs/scicodepile.yaml + +indic_mbpp_resources_server: + _inherit_from: scicodepile + +indic_mbpp_simple_agent: + _inherit_from: scicodepile_simple_agent + responses_api_agents: + simple_agent: + resources_server: + name: indic_mbpp_resources_server + datasets: + - name: indic_mbpp + type: benchmark + jsonl_fpath: benchmarks/indic/mbpp/data/mbpp_benchmark.jsonl + prepare_script: benchmarks/indic/mbpp/prepare.py + prompt_config: benchmarks/mbpp/prompts/default.yaml + num_repeats: 1 + license: Apache-2.0 + +indic_mbpp_environment_server: + _inherit_from: scicodepile_environment_server + environment_servers: + legacy_agent: + agent_server: + name: indic_mbpp_simple_agent diff --git a/benchmarks/indic/mbpp/data/.gitignore b/benchmarks/indic/mbpp/data/.gitignore new file mode 100644 index 00000000000..d6b7ef32c84 --- /dev/null +++ b/benchmarks/indic/mbpp/data/.gitignore @@ -0,0 +1,2 @@ +* +!.gitignore diff --git a/benchmarks/indic/mbpp/manifest.yaml b/benchmarks/indic/mbpp/manifest.yaml new file mode 100644 index 00000000000..8a947ba268b --- /dev/null +++ b/benchmarks/indic/mbpp/manifest.yaml @@ -0,0 +1,31 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +name: indic/mbpp +version: 0.1.0 +experimental: true +kind: benchmark +integration_profile: custom-gym-verifier +domain: coding +description: Original MBPP in 14 Indic languages, with unchanged assertion tests and a matching English baseline. +modality: text +licensing: Apache-2.0 +authors: [NVIDIA NeMo Gym contributors] +reward: + range: [0, 1] + higher_is_better: true +determinism: stochastic +resources_server: scicodepile +agent_server: simple_agent +model_server: policy_model +datasets: + - name: indic_mbpp + type: benchmark + jsonl_fpath: benchmarks/indic/mbpp/data/mbpp_benchmark.jsonl + prepare_script: benchmarks/indic/mbpp/prepare.py + prompt_config: benchmarks/mbpp/prompts/default.yaml + num_repeats: 1 +standard_prompt_config: benchmarks/mbpp/prompts/default.yaml +canonical_split: benchmark +session_model: episode +state: none +lifecycle: active diff --git a/benchmarks/indic/mbpp/prepare.py b/benchmarks/indic/mbpp/prepare.py new file mode 100644 index 00000000000..2a8cda9295a --- /dev/null +++ b/benchmarks/indic/mbpp/prepare.py @@ -0,0 +1,178 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Prepare original MBPP translations and a matching English baseline.""" + +import argparse +import ast +import hashlib +import json +import re +from collections.abc import Mapping, Sequence +from pathlib import Path +from textwrap import indent + +from datasets import load_dataset +from huggingface_hub import get_token, hf_hub_download + +from nemo_gym.global_config import HF_TOKEN_KEY_NAME, maybe_get_global_config_dict + + +BENCHMARK_DIR = Path(__file__).parent +OUTPUT_FPATH = BENCHMARK_DIR / "data" / "mbpp_benchmark.jsonl" +SOURCE_ID = "ai4bharat/indic-mbpp" +EXPECTED_ROWS = 500 +LANGUAGE_NAMES = { + "as": "Assamese", + "bn": "Bengali", + "gu": "Gujarati", + "hi": "Hindi", + "kn": "Kannada", + "ml": "Malayalam", + "mr": "Marathi", + "ne": "Nepali", + "or": "Odia", + "pa": "Punjabi", + "sa": "Sanskrit", + "ta": "Tamil", + "te": "Telugu", + "ur": "Urdu", +} +DEFAULT_LANGUAGES = tuple(LANGUAGE_NAMES) +SourceRow = Mapping[str, str | int | list[str]] + + +def _verifier_metadata(record: SourceRow) -> dict: + task_id = record.get("task_id") + tests = record.get("test_list") + setup = record.get("test_setup_code") + code = record.get("code") + if type(task_id) is not int or task_id < 1: + raise ValueError("task_id must be a positive integer") + if not isinstance(tests, list) or len(tests) != 3 or any(not isinstance(t, str) or not t.strip() for t in tests): + raise ValueError(f"Expected three nonempty original tests for task {task_id}") + if not isinstance(setup, str) or not isinstance(code, str): + raise ValueError(f"Missing setup or reference code for task {task_id}") + functions = {node.name for node in ast.parse(code).body if isinstance(node, ast.FunctionDef)} + calls = { + node.func.id + for node in ast.walk(ast.parse(tests[0])) + if isinstance(node, ast.Call) and isinstance(node.func, ast.Name) + } + entry_points = functions & calls + if len(entry_points) != 1: + raise ValueError(f"Cannot identify one tested function for task {task_id}") + # The setup can depend on classes supplied by the candidate, so run it inside check(). + # Preserve a candidate function named check before the runner installs its callback. + test = ( + 'def check(candidate, _original_check=globals().get("check")):\n' + " if _original_check is not None:\n" + ' globals()["check"] = _original_check\n' + indent(setup + "\n" + "\n".join(tests), " ") + "\n" + ) + ast.parse(test) + return {"task_id": f"Mbpp/{task_id}", "entry_point": entry_points.pop(), "test": test} + + +def build_rows( + records: Sequence[SourceRow], + *, + languages: Sequence[str] = DEFAULT_LANGUAGES, + task_ids: Sequence[int] | None = None, +) -> list[dict]: + """Build translated prompts and unchanged original MBPP assertions.""" + if ( + isinstance(languages, str) + or not languages + or any(code not in (*DEFAULT_LANGUAGES, "en") for code in languages) + ): + raise ValueError(f"languages must be a nonempty sequence from {(*DEFAULT_LANGUAGES, 'en')}") + if len(set(languages)) != len(languages): + raise ValueError("languages must be unique") + if len(records) != EXPECTED_ROWS: + raise ValueError(f"Expected {EXPECTED_ROWS} MBPP tasks, got {len(records)}") + metadata = {} + for record in records: + meta = _verifier_metadata(record) + identity = record["task_id"] + if identity in metadata: + raise ValueError(f"Duplicate task_id: {identity}") + metadata[identity] = meta + if task_ids is not None: + if not task_ids or any(type(value) is not int for value in task_ids) or len(set(task_ids)) != len(task_ids): + raise ValueError("task_ids must contain unique integer IDs") + if set(task_ids) - metadata.keys(): + raise ValueError(f"Unknown task_ids: {sorted(set(task_ids) - metadata.keys())}") + selected = set(metadata if task_ids is None else task_ids) + rows = [] + for language in languages: + column = "text" if language == "en" else f"text_{LANGUAGE_NAMES[language]}_translation" + for record in records: + identity = record["task_id"] + text = record.get(column) + if not isinstance(text, str) or not text.strip(): + raise ValueError(f"Missing problem text: {language}/{identity}") + if identity not in selected: + continue + setup = record["test_setup_code"] + example = (setup + "\n" if setup else "") + record["test_list"][0] + question = ('"""\n' + text + "\n" + example + '\n"""\n').replace(" ", "\t") + rows.append( + { + "question": question, + "verifier_metadata": {**metadata[identity], "task_id": f"Mbpp/{language}/{identity}"}, + "language": language, + "uuid": f"{SOURCE_ID}/{language}/{identity}", + } + ) + return rows + + +def prepare( + *, + languages: Sequence[str] = DEFAULT_LANGUAGES, + task_ids: Sequence[int] | None = None, + output_fpath: str | None = None, +) -> Path: + """Download the latest translations and write tasks plus source provenance.""" + config = maybe_get_global_config_dict() + token = config.get(HF_TOKEN_KEY_NAME) if config is not None else None + source = hf_hub_download( + repo_id=SOURCE_ID, + filename="test.parquet", + repo_type="dataset", + token=token or get_token(), + ) + records = load_dataset("parquet", data_files={"test": source}, split="test").to_list() + rows = build_rows(records, languages=languages, task_ids=task_ids) + # Do not resolve this symlink: the snapshot path carries the downloaded commit, + # while its target is a content-addressed blob. No second lookup of mutable HEAD. + source_path = Path(source) + revision = source_path.parent.name + provenance = { + "repo_id": SOURCE_ID, + "filename": "test.parquet", + "resolved_revision": revision + if source_path.parent.parent.name == "snapshots" and re.fullmatch(r"[0-9a-f]{40}", revision) + else None, + "sha256": hashlib.sha256(source_path.read_bytes()).hexdigest(), + "languages": list(languages), + "task_ids": list(task_ids) if task_ids is not None else None, + "rows": len(rows), + } + output = Path(output_fpath) if output_fpath else OUTPUT_FPATH + output.parent.mkdir(parents=True, exist_ok=True) + with output.open("w", encoding="utf-8") as stream: + for row in rows: + stream.write(json.dumps(row, ensure_ascii=False) + "\n") + provenance_path = output.with_suffix(".provenance.json") + provenance_path.write_text(json.dumps(provenance, indent=2) + "\n", encoding="utf-8") + print(f"Wrote {len(rows)} problems to {output}") + print(f"Dataset revision: {provenance['resolved_revision'] or 'unknown'}; provenance: {provenance_path}") + return output + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--languages", nargs="+", default=DEFAULT_LANGUAGES) + parser.add_argument("--task-ids", nargs="+", type=int) + parser.add_argument("--output-fpath") + prepare(**vars(parser.parse_args())) diff --git a/benchmarks/indic/mbpp/smoke-results.md b/benchmarks/indic/mbpp/smoke-results.md new file mode 100644 index 00000000000..c8245dd66aa --- /dev/null +++ b/benchmarks/indic/mbpp/smoke-results.md @@ -0,0 +1,216 @@ + + + +# Indic MBPP paired smoke evidence + +Run date: 2026-10-11. This is a six-example integration check, not a full +benchmark result or an estimate of multilingual model quality. +The initial run and inspected outputs are preserved below; a separate +[post-rebase check](#post-rebase-validation) records the fresh follow-up run. + +## Configuration and provenance + +Used [smoke.yaml](smoke.yaml) through `gym eval run`, the shared English MBPP +prompt, `simple_agent`, the vLLM model adapter, and the real SciCodePile verifier. +Inputs were freshly built with the updated preparation code for task IDs 11 and +367 in `en`, `hi`, and `kn`; no cached model responses or reference solutions +were used for generation. Task 11 exercises string manipulation; task 367 +exercises candidate-defined classes used by the original test setup. + +- Model: `google/gemma-4-31B-it`. +- Model snapshot: `842da3794eaa0b77d5f08bae87a17459d91ff475`. +- Source: `ai4bharat/indic-mbpp`, `test.parquet`. +- Recorded source revision: `64e7f7cecd2a6b66a0bcbc8c4c02200d40ae520c`. +- Source SHA-256: `d945fec4af126e0d00d31c9dff015ec93054ee9ba2f5f5b44b7f9ee267ba5ecb`. +- Prepared six-row JSONL SHA-256: `c7746ca6654c92e17271be0d38200dad23b2183c597435faca589a243221a6db`. +- Generation: thinking on; temperature 1.0; top-p 0.95; top-k 64; seed 0; + maximum 8,192 output tokens, including reasoning; one attempt; concurrency 3. +- Runtime: vLLM 0.21.0, two H100 GPUs, BF16 weights, FP8 KV cache, 65,536-token + context, tensor parallelism 2, eager execution, prefix caching off. +- Verification: all three original assertions, unchanged 120-second subprocess + timeout; challenge assertions excluded. Model reasoning is not executed. + +The online source refresh returned a repository/access error with the available +credentials. This run used a previously downloaded local snapshot, checking its +SHA-256 against its saved download provenance before preparing new inputs. It +does not claim to validate current upstream HEAD. The revision above records +this run only; preparation still downloads the default revision without a pin. +Authorized dataset access is required to reproduce the source download. + +Equivalent model-server launch (use an authorized copy of the model snapshot): + +```bash +vllm serve google/gemma-4-31B-it \ + --revision 842da3794eaa0b77d5f08bae87a17459d91ff475 \ + --served-model-name google/gemma-4-31B-it --host 127.0.0.1 --port 8000 \ + --tensor-parallel-size 2 --dtype bfloat16 --kv-cache-dtype fp8 \ + --max-model-len 65536 --gpu-memory-utilization 0.85 --max-num-seqs 32 \ + --enable-chunked-prefill --max-num-batched-tokens 2048 \ + --enforce-eager --no-enable-prefix-caching --reasoning-parser gemma4 \ + --limit-mm-per-prompt '{"image":0,"video":0,"audio":0}' \ + --generation-config vllm --override-generation-config '{"eos_token_id":[1,106,50]}' \ + --trust-remote-code +``` + +See the [README commands](README.md#paired-real-model-smoke) for input preparation +and rollout collection. The live Gym configuration was inspected to confirm the +sampling overrides, thinking flag, and output limit survived composition. + +## Saved results + +| Returned task ID | Reward | Verifier status | Output tokens | +| --- | ---: | --- | ---: | +| `Mbpp/en/11` | 1 | pass | 1,725 | +| `Mbpp/en/367` | 1 | pass | 2,737 | +| `Mbpp/hi/11` | 1 | pass | 1,361 | +| `Mbpp/hi/367` | 1 | pass | 1,568 | +| `Mbpp/kn/11` | 1 | pass | 1,068 | +| `Mbpp/kn/367` | 1 | pass | 2,028 | + +Coverage: 6/6 returned results, 2/2 per language. Every response had status +`completed`, a separate `reasoning` item and final `message`, a null +`failure_reason`, and reward 1. No response reached the output-token cap; +the harness-failure metric was 0 and the failed-request artifact was empty. +The saved `task_id` preserves the language without an input-file join. +Reasoning-token counts were not separately reported by this endpoint; output +tokens above include the full generated response. + +## Inspected final outputs + +The following code was extracted from the model's final answers, not its +reasoning. Blank-line whitespace is normalized for display. All six final +answers were inspected; these three illustrate the English/Indic comparison +and the candidate-defined-class setup case. + +### English, task 11: reward 1 + +Problem: remove the first and last occurrence of a character from a string. +The prompt's example is `assert remove_Occ("hello","l") == "heo"`. +The model returns `"heo"` for that example and passes both withheld assertions. + +```python +def remove_Occ(s, char): + first = s.find(char) + if first == -1: + return s + last = s.rfind(char) + if first == last: + return s[:first] + s[first+1:] + return s[:first] + s[first+1:last] + s[last+1:] +``` + +### Hindi, task 11: reward 1 + +The actual description in the saved prompt was: + +> स्ट्रिंग से दिए गए character के पहले और अंतिम occurrence को हटाने के लिए एक python फ़ंक्शन लिखें। + +The wrapper and example assertion were identical to the English task. This +answer also produces `"heo"` and passes the same three assertions. + +```python +def remove_Occ(s, char): + first = s.find(char) + last = s.rfind(char) + + if first == -1: + return s + if first == last: + # Only one occurrence exists + return s[:first] + s[first + 1:] + + # Remove first and last occurrences + return s[:first] + s[first + 1:last] + s[last + 1:] +``` + +### Kannada, task 367: reward 1 + +The actual description in the saved prompt was: + +> ಒಂದು ಬೈನರಿ ಟ್ರೀ ಬ್ಯಾಲೆನ್ಸ್ಡ್ ಆಗಿದೆಯೇ ಅಥವಾ ಇಲ್ಲವೇ ಎಂದು ಪರಿಶೀಲಿಸಲು ಒಂದು ಫಂಕ್ಷನ್ ಅನ್ನು ಬರೆಯಿರಿ. + +The prompt also included the original tree setup and first assertion, +`assert is_tree_balanced(root) == False`. The generated `Node` class was available +when the verifier ran that setup. All three original assertions passed. + +```python +class Node: + def __init__(self, data): + self.data = data + self.left = None + self.right = None + +def is_tree_balanced(root): + def check_height(node): + # Base case: an empty node has height 0 + if node is None: + return 0 + + # Recursively find the height of the left subtree + left_height = check_height(node.left) + if left_height == -1: + return -1 # Left subtree is not balanced + + # Recursively find the height of the right subtree + right_height = check_height(node.right) + if right_height == -1: + return -1 # Right subtree is not balanced + + # If the difference between heights is greater than 1, it's unbalanced + if abs(left_height - right_height) > 1: + return -1 + + # Return the height of the current node + return max(left_height, right_height) + 1 + + # If check_height returns -1, the tree is unbalanced + return check_height(root) != -1 +``` + +## Additional verifier checks + +All 500 reference solutions from the recorded source snapshot passed through +the updated prepared English rows and the real verifier. An intentionally +incorrect task-11 solution (`def remove_Occ(s, ch): return s`) received reward 0 +with `AssertionError`, not a harness failure. Preparing all 15 languages yielded +7,500 unique qualified task IDs with unchanged tests and entry points. + +The CI-collected preparation tests cover provenance and config composition. +The SciCodePile response-roundtrip tests cover both reward 1 and reward 0 for +English/Hindi IDs; restoring the old language-less IDs makes those tests fail. + +## Post-rebase validation + +Repeated on 2026-10-11 after rebasing onto upstream `main` at +`4a9236e48c82f0dc23365a4c0088b1e54d00d3a8` and adding the explicit +`indic_mbpp_environment_server` relay, matching the current benchmark layout. +Used the same six-row input, model snapshot, and generation settings recorded +above, with fresh model requests rather than reused responses. The live Gym +configuration confirmed the sampling overrides, thinking flag, 8,192-token +limit, and relay reference to `indic_mbpp_simple_agent`, with +`error_on_agent_without_environment_server: true`. + +| Returned task ID | Reward | Verifier status | Output tokens | +| --- | ---: | --- | ---: | +| `Mbpp/en/11` | 1 | pass | 1,454 | +| `Mbpp/en/367` | 1 | pass | 3,104 | +| `Mbpp/hi/11` | 1 | pass | 1,393 | +| `Mbpp/hi/367` | 1 | pass | 2,859 | +| `Mbpp/kn/11` | 1 | pass | 1,070 | +| `Mbpp/kn/367` | 1 | pass | 2,975 | + +All six responses completed below the token cap, retained their language-qualified +IDs, and had null `failure_reason` values. All final answers were inspected: +the task-11 answers remove the first/last matching character with string slices; +the task-367 answers define `Node` and use recursive subtree-height checks. +All original assertions passed. The aggregate harness-failure metric was 0, +the failed-request artifact was empty, and the evaluation process exited 0. +The optional telemetry-based rollout-health report classified all six records +as `unobserved` because no captures were recorded; this is not a rollout-health +check pass. + +Also reran the 500 reference solutions and incorrect-solution control described +above, all 114 preparation/SciCodePile tests, and all five `legacy_agent` tests. +Both config-composition tests now check the named relay and require strict +environment wiring. All-files pre-commit checks passed. The full repository CI +suite was not run locally. diff --git a/benchmarks/indic/mbpp/smoke.yaml b/benchmarks/indic/mbpp/smoke.yaml new file mode 100644 index 00000000000..8b416dd2ccd --- /dev/null +++ b/benchmarks/indic/mbpp/smoke.yaml @@ -0,0 +1,34 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# Standalone, model-specific smoke config; not canonical benchmark defaults. +config_paths: + - responses_api_models/vllm_model/configs/vllm_model.yaml + - benchmarks/indic/mbpp/config.yaml + +responses_create_params: + max_output_tokens: 8192 +resume_from_cache: false + +policy_model: + responses_api_models: + vllm_model: + uses_interleaved_reasoning: false + chat_template_kwargs: + enable_thinking: true + sampling_overrides: + temperature: 1.0 + top_p: 0.95 + top_k: 64 + seed: 0 + +indic_mbpp_simple_agent: + responses_api_agents: + simple_agent: + datasets: + - name: indic_mbpp_smoke + type: benchmark + jsonl_fpath: benchmarks/indic/mbpp/data/review-smoke.jsonl + prepare_script: benchmarks/indic/mbpp/prepare.py + prompt_config: benchmarks/mbpp/prompts/default.yaml + num_repeats: 1 + license: Apache-2.0 diff --git a/resources_servers/scicodepile/tests/test_app.py b/resources_servers/scicodepile/tests/test_app.py index 11454a3261c..7499e96d1cc 100644 --- a/resources_servers/scicodepile/tests/test_app.py +++ b/resources_servers/scicodepile/tests/test_app.py @@ -353,6 +353,43 @@ def short_timeout_client(): yield c +@pytest.mark.parametrize("code,reward", [("def add_one(x): return x + 1", 1.0), ("def add_one(x): return 2", 0.0)]) +def test_indic_mbpp_language_ids_survive_verify_roundtrip(short_timeout_client, monkeypatch, code, reward): + from app import SciCodePileVerifyRequest, SciCodePileVerifyResponse + + from benchmarks.indic.mbpp import prepare as mbpp + + monkeypatch.setattr(mbpp, "EXPECTED_ROWS", 1) + record = { + "task_id": 11, + "text": "Add one.", + "text_Hindi_translation": "एक जोड़ें।", + "code": "def add_one(x): return x + 1", + "test_list": ["assert add_one(1) == 2", "assert add_one(2) == 3", "assert add_one(3) == 4"], + "test_setup_code": "", + } + rows = mbpp.build_rows([record], languages=["en", "hi"]) + assert rows[0]["verifier_metadata"] is not rows[1]["verifier_metadata"] + for row in rows: + body = SciCodePileVerifyRequest.model_validate( + { + **row, + "responses_create_params": {"input": [{"role": "user", "content": row["question"]}]}, + "response": _make_response(f"```python\n{code}\n```"), + } + ) + response = short_timeout_client.post("/verify", json=body.model_dump()) + assert response.status_code == 200 + saved = SciCodePileVerifyResponse.model_validate_json(response.content) + assert saved.task_id == f"Mbpp/{row['language']}/11" + assert saved.reward == reward + assert saved.status == ("pass" if reward else "fail") + assert saved.failure_reason is None + # Grouping must work from the serialized result alone, without an input join. + assert "language" not in response.json() + assert "verifier_metadata" not in response.json() + + class TestScratchDirectoryLifecycle: """The scratch CWD is created and removed by the parent, never by the runner. diff --git a/tests/unit_tests/test_indic_mbpp_prepare.py b/tests/unit_tests/test_indic_mbpp_prepare.py new file mode 100644 index 00000000000..8bf87b4f2ba --- /dev/null +++ b/tests/unit_tests/test_indic_mbpp_prepare.py @@ -0,0 +1,323 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +import ast +import hashlib +import json +import subprocess +import sys + +import pytest +from datasets import Dataset +from omegaconf import OmegaConf + +from benchmarks.indic.mbpp import prepare as module +from nemo_gym.benchmarks import BenchmarkConfig +from nemo_gym.environment.manifest import load_manifest +from nemo_gym.global_config import GlobalConfigDictParser, GlobalConfigDictParserConfig +from nemo_gym.prompt import apply_prompt_to_row, load_prompt_config + + +@pytest.fixture +def records(monkeypatch): + monkeypatch.setattr(module, "EXPECTED_ROWS", 2) + return [ + { + "task_id": task_id, + "text": " Add one to x.\n Preserve the description. ", + "code": "def add_one(x):\n return x + 1 # HIDDEN_REFERENCE\n", + "test_list": ["assert add_one(1) == 2", "assert add_one(2) == 3", "assert add_one(3) == 4"], + "test_setup_code": "", + "challenge_test_list": ["assert add_one(4) == 5"], + **{ + f"text_{name}_translation": f" {code}: x में एक जोड़ें.\n {{x}} " + for code, name in module.LANGUAGE_NAMES.items() + }, + } + for task_id in (11, 12) + ] + + +def test_shared_prompt_and_identical_english_indic_tests(records): + rows = module.build_rows(records, languages=["en", "hi"]) + prompt = load_prompt_config("benchmarks/mbpp/prompts/default.yaml") + english, hindi = rows[0], rows[2] + assert english["verifier_metadata"]["task_id"] == "Mbpp/en/11" + assert hindi["verifier_metadata"]["task_id"] == "Mbpp/hi/11" + assert english["verifier_metadata"] is not hindi["verifier_metadata"] + for field in ("test", "entry_point"): + assert english["verifier_metadata"][field] == hindi["verifier_metadata"][field] + en_input = apply_prompt_to_row(english, prompt)["responses_create_params"]["input"] + hi_input = apply_prompt_to_row(hindi, prompt)["responses_create_params"]["input"] + assert hi_input == [ + { + "role": "user", + "content": en_input[0]["content"].replace( + records[0]["text"].replace(" ", "\t"), + records[0]["text_Hindi_translation"].replace(" ", "\t"), + ), + } + ] + assert "assert add_one(1) == 2" in hi_input[0]["content"] + assert "assert add_one(2) == 3" not in hi_input[0]["content"] + assert "assert add_one(3) == 4" not in hi_input[0]["content"] + assert "HIDDEN_REFERENCE" not in json.dumps(hindi) + assert "assert add_one(4) == 5" not in json.dumps(hindi) + check = ast.parse(hindi["verifier_metadata"]["test"]).body[0] + assert [ast.dump(node) for node in check.body[1:]] == [ + ast.dump(node) for node in ast.parse("\n".join(records[0]["test_list"])).body + ] + + +def test_full_coverage_stable_ids_and_selection(records): + rows = module.build_rows(records) + assert len(rows) == 28 + assert len({row["uuid"] for row in rows}) == 28 + assert {row["language"] for row in rows} == set(module.DEFAULT_LANGUAGES) + selected = module.build_rows(records, languages=["hi"], task_ids=[12]) + assert selected == [ + row for row in rows if row["language"] == "hi" and row["verifier_metadata"]["task_id"] == "Mbpp/hi/12" + ] + + +@pytest.mark.parametrize("value", [None, "", " "]) +def test_missing_translation_rejected_without_fallback(records, value): + records[0]["text_Hindi_translation"] = value + with pytest.raises(ValueError, match="Missing problem text: hi/11"): + module.build_rows(records, languages=["hi"], task_ids=[12]) + + +@pytest.mark.parametrize( + "kwargs,error", + [ + ({"languages": "hi"}, "nonempty sequence"), + ({"languages": []}, "nonempty sequence"), + ({"languages": ["xx"]}, "nonempty sequence"), + ({"languages": ["hi", "hi"]}, "unique"), + ({"task_ids": []}, "unique integer"), + ({"task_ids": [True]}, "unique integer"), + ({"task_ids": [11, 11]}, "unique integer"), + ({"task_ids": ["11"]}, "unique integer"), + ({"task_ids": [999]}, "Unknown task_ids"), + ], +) +def test_bad_selections(records, kwargs, error): + with pytest.raises(ValueError, match=error): + module.build_rows(records, **kwargs) + + +def test_missing_and_duplicate_tasks(records): + with pytest.raises(ValueError, match="Expected 2"): + module.build_rows(records[:1]) + with pytest.raises(ValueError, match="Duplicate task_id"): + module.build_rows([records[0], records[0]]) + + +@pytest.mark.parametrize( + "change,error", + [ + ({"task_id": True}, "positive integer"), + ({"test_list": []}, "three nonempty"), + ({"test_list": ["", "assert True", "assert True"]}, "three nonempty"), + ({"code": None}, "reference code"), + ({"test_setup_code": None}, "setup"), + ({"code": "def unrelated(): return 0"}, "one tested function"), + ], +) +def test_malformed_test_contract(records, change, error): + records[0].update(change) + with pytest.raises(ValueError, match=error): + module.build_rows(records) + + +def run_native(meta, code, tmp_path): + runner = module.BENCHMARK_DIR.parents[2] / "resources_servers/scicodepile/scp_runner.py" + result = subprocess.run( + [sys.executable, str(runner)], + input=json.dumps({**meta, "code": code, "max_as_limit": 0, "workdir": str(tmp_path)}), + capture_output=True, + text=True, + timeout=10, + check=True, + ) + return json.loads(result.stdout) + + +def test_native_runner_accepts_correct_and_rejects_hidden_test_failure(records, tmp_path): + meta = module.build_rows(records, languages=["hi"])[0]["verifier_metadata"] + assert run_native(meta, records[0]["code"], tmp_path)["status"] == "pass" + # This satisfies the visible example but fails the two remaining original assertions. + assert run_native(meta, "def add_one(x): return 2", tmp_path)["status"] == "fail" + + +def test_candidate_named_check_and_recursive_calls_keep_original_binding(records, tmp_path): + records[0].update( + { + "code": "def check(n): return n if n <= 1 else check(n - 1) + 1", + "test_list": ["assert check(0) == 0", "assert check(2) == 2", "assert check(3) == 3"], + } + ) + meta = module.build_rows(records, languages=["hi"])[0]["verifier_metadata"] + assert run_native(meta, records[0]["code"], tmp_path)["status"] == "pass" + assert run_native(meta, "def check(n): return 0", tmp_path)["status"] == "fail" + + +def test_setup_runs_after_candidate_classes(records, tmp_path): + records[0].update( + { + "code": "class Node:\n def __init__(self, x): self.x = x\ndef value(node): return node.x\n", + "test_setup_code": "root = Node(3)", + "test_list": ["assert value(root) == 3", "assert value(Node(4)) == 4", "assert value(Node(5)) == 5"], + } + ) + meta = module.build_rows(records, languages=["en"])[0]["verifier_metadata"] + assert run_native(meta, records[0]["code"], tmp_path)["status"] == "pass" + + +def test_prepare_downloads_source_and_preserves_output_on_failure(records, monkeypatch, tmp_path): + calls = [] + revision = "a" * 40 + source = tmp_path / "snapshots" / revision / "test.parquet" + source.parent.mkdir(parents=True) + blob = tmp_path / "blob" + blob.write_bytes(b"test parquet contents") + source.symlink_to(blob) + monkeypatch.setattr(module, "maybe_get_global_config_dict", lambda: None) + monkeypatch.setattr(module, "get_token", lambda: "test-token") + monkeypatch.setattr(module, "hf_hub_download", lambda **kwargs: calls.append(kwargs) or str(source)) + monkeypatch.setattr(module, "load_dataset", lambda *args, **kwargs: Dataset.from_list(records)) + output = tmp_path / "tasks.jsonl" + assert module.prepare(languages=["hi"], output_fpath=str(output)) == output + assert calls[0] == { + "repo_id": module.SOURCE_ID, + "filename": "test.parquet", + "repo_type": "dataset", + "token": "test-token", + } + original = output.read_bytes() + provenance_path = output.with_suffix(".provenance.json") + provenance_bytes = provenance_path.read_bytes() + assert json.loads(provenance_bytes) == { + "repo_id": module.SOURCE_ID, + "filename": "test.parquet", + "resolved_revision": revision, + "sha256": hashlib.sha256(blob.read_bytes()).hexdigest(), + "languages": ["hi"], + "task_ids": None, + "rows": 2, + } + assert [json.loads(line) for line in output.read_text().splitlines()] == module.build_rows( + records, languages=["hi"] + ) + records[0]["test_list"] = [] + with pytest.raises(ValueError): + module.prepare(languages=["hi"], output_fpath=str(output)) + assert output.read_bytes() == original + assert provenance_path.read_bytes() == provenance_bytes + + +def test_prepare_reports_unknown_revision_without_guessing(records, monkeypatch, tmp_path, capsys): + source = tmp_path / "test.parquet" + source.write_bytes(b"uncached source") + monkeypatch.setattr(module, "maybe_get_global_config_dict", lambda: None) + monkeypatch.setattr(module, "get_token", lambda: None) + monkeypatch.setattr(module, "hf_hub_download", lambda **kwargs: str(source)) + monkeypatch.setattr(module, "load_dataset", lambda *args, **kwargs: Dataset.from_list(records)) + output = module.prepare(languages=["en", "hi"], task_ids=[11], output_fpath=str(tmp_path / "tasks.jsonl")) + provenance = json.loads(output.with_suffix(".provenance.json").read_text()) + assert provenance["resolved_revision"] is None + assert provenance["sha256"] == hashlib.sha256(source.read_bytes()).hexdigest() + assert provenance["task_ids"] == [11] + assert provenance["languages"] == ["en", "hi"] + assert provenance["rows"] == 2 + assert "Dataset revision: unknown" in capsys.readouterr().out + + +def test_native_config_reuses_prompt_agent_and_verifier(): + parser = GlobalConfigDictParser() + configs = [] + for path in ("benchmarks/mbpp/config.yaml", "benchmarks/indic/mbpp/config.yaml"): + configs.append( + parser.parse( + GlobalConfigDictParserConfig( + initial_global_config_dict=OmegaConf.create( + { + "config_paths": ["responses_api_models/vllm_model/configs/vllm_model.yaml", path], + "policy_base_url": "http://unused/v1", + "policy_api_key": "dummy", + "policy_model_name": "test", + "error_on_agent_without_environment_server": True, + } + ), + skip_load_from_cli=True, + skip_load_from_dotenv=True, + offline=True, + ) + ) + ) + en, indic = configs + assert "responses_create_params" not in OmegaConf.load(module.BENCHMARK_DIR / "config.yaml") + assert indic.policy_model.responses_api_models.vllm_model.sampling_overrides is None + assert indic.policy_model.responses_api_models.vllm_model.chat_template_kwargs is None + original = OmegaConf.to_container(en.mbpp_evalplus_simple_agent.responses_api_agents.simple_agent, resolve=True) + translated = OmegaConf.to_container(indic.indic_mbpp_simple_agent.responses_api_agents.simple_agent, resolve=True) + assert original["datasets"][0]["prompt_config"] == translated["datasets"][0]["prompt_config"] + original["datasets"] = translated["datasets"] + original["resources_server"]["name"] = "indic_mbpp_resources_server" + assert original == translated + assert indic.indic_mbpp_resources_server.resources_servers.scicodepile.subprocess_timeout == 120 + relay = indic.indic_mbpp_environment_server.environment_servers.legacy_agent + assert relay.agent_server == {"type": "responses_api_agents", "name": "indic_mbpp_simple_agent"} + assert {server.name for server in parser.filter_for_server_instance_configs(indic)} == { + "policy_model", + "indic_mbpp_resources_server", + "indic_mbpp_simple_agent", + "indic_mbpp_environment_server", + } + benchmark = BenchmarkConfig.from_config_path(module.BENCHMARK_DIR / "config.yaml") + assert benchmark.agent_name == "indic_mbpp_simple_agent" + manifest = load_manifest(module.BENCHMARK_DIR / "manifest.yaml") + assert manifest.canonical_split == "benchmark" + assert manifest.resources_server == "scicodepile" + assert manifest.agent_server == "simple_agent" + assert manifest.datasets[0].model_dump(exclude_none=True) == { + key: value for key, value in translated["datasets"][0].items() if key != "license" + } + + +def test_standalone_smoke_config_preserves_model_settings(): + parser = GlobalConfigDictParser() + config = parser.parse( + GlobalConfigDictParserConfig( + initial_global_config_dict=OmegaConf.create( + { + "config_paths": ["benchmarks/indic/mbpp/smoke.yaml"], + "policy_base_url": "http://unused/v1", + "policy_api_key": "dummy", + "policy_model_name": "google/gemma-4-31B-it", + "error_on_agent_without_environment_server": True, + } + ), + skip_load_from_cli=True, + skip_load_from_dotenv=True, + offline=True, + ) + ) + model = config.policy_model.responses_api_models.vllm_model + assert model.sampling_overrides == {"temperature": 1.0, "top_p": 0.95, "top_k": 64, "seed": 0} + assert model.chat_template_kwargs == {"enable_thinking": True} + assert model.uses_reasoning_parser and not model.uses_interleaved_reasoning + assert config.responses_create_params == {"max_output_tokens": 8192} + assert config.resume_from_cache is False + dataset = config.indic_mbpp_simple_agent.responses_api_agents.simple_agent.datasets[0] + assert dataset.num_repeats == 1 + assert dataset.type == "benchmark" + assert dataset.jsonl_fpath == "benchmarks/indic/mbpp/data/review-smoke.jsonl" + relay = config.indic_mbpp_environment_server.environment_servers.legacy_agent + assert relay.agent_server == {"type": "responses_api_agents", "name": "indic_mbpp_simple_agent"} + assert {server.name for server in parser.filter_for_server_instance_configs(config)} == { + "policy_model", + "indic_mbpp_resources_server", + "indic_mbpp_simple_agent", + "indic_mbpp_environment_server", + }