From 902b98510364175dedf0d63e034357a177ab8605 Mon Sep 17 00:00:00 2001 From: Brian Thompson Date: Thu, 30 Jul 2026 13:19:08 -0700 Subject: [PATCH 1/2] feat(translation): overhaul WMT24++ and FLORES evaluation (#2251) Expand the translation benchmarks and standardize scoring around chrF, FLORES-200 spBLEU, COMET, and target-language consistency. - expand WMT24++ from five defaults to all 55 target locales - expand FLORES from the six-language pair matrix to 219 English-to-X devtest pairs across 220 supported FLORES+ configurations - store generated language names and FLORES devtest availability in checked-in metadata instead of querying or hardcoding them at runtime - preserve complete FLORES language and script codes in prepared rows - reject unknown or devtest-incompatible FLORES configurations - deduplicate WMT target locales while preserving their requested order - exclude WMT24++ rows marked as bad source data - publish WMT preparation output atomically after model prefetch succeeds - add CLI controls for COMET, spBLEU, and GlotLID prefetching Replace language-specific SacreBLEU tokenization with NFC-normalized chrF and FLORES-200 SentencePiece spBLEU scoring. Use sentence chrF as the dense RL reward and report sentence and corpus chrF/spBLEU separately. Emit a startup warning because the new spBLEU values are not comparable with the previous BLEU metrics. Add configurable target-language consistency scoring: - use CLD2 for WMT24++ locale-to-language validation - use GlotLID probabilities for FLORES languages and dialect mappings - load language-detection backends lazily and reuse them across requests - report per-rollout scores and per-pair, source, target, and global aggregate metrics - warn when a language pair falls below the configured consistency threshold - document CLD2 limitations for regional language variants Prefetch the shared SentencePiece, COMET, and GlotLID assets for offline verification. Update translation dependencies, configs, documentation, and example rollouts for the new scoring contract. Add metadata generation and mapping-validation utilities plus unit coverage for preparation, scoring, aggregation, backend loading, and language-code handling. BREAKING CHANGE: The wmt_translation per-rollout reward now uses sentence- level chrF instead of sentence-level BLEU; spBLEU remains a separately reported metric. The sentence_bleu field and */bleu aggregate keys are replaced by sentence_chrf, sentence_spbleu, */chrF, and */spBLEU. --------- Signed-off-by: Brian Thompson <3534106+thompsonb@users.noreply.github.com> Co-authored-by: Brian Thompson <3534106+thompsonb@users.noreply.github.com> --- README.md | 2 +- benchmarks/flores200/README.md | 45 +- benchmarks/flores200/config.yaml | 3 + .../flores200/data/example_rollouts.jsonl | 10 +- .../data/flores_devtest_languages.json | 222 +++++++++ .../flores200/data/flores_language_names.json | 232 +++++++++ benchmarks/flores200/prepare.py | 208 ++++---- benchmarks/flores200/tests/test_prepare.py | 150 ++++++ benchmarks/wmt24pp/README.md | 31 +- benchmarks/wmt24pp/config.yaml | 3 + .../wmt24pp/data/wmt24pp_language_names.json | 57 +++ benchmarks/wmt24pp/prepare.py | 217 ++++---- .../wmt24pp/tests/test_wmt24pp_prepare.py | 357 +++++++++++++ resources_servers/wmt_translation/README.md | 86 +++- resources_servers/wmt_translation/app.py | 370 +++++++++++--- .../configs/wmt_translation.yaml | 8 +- .../data/example_rollouts.jsonl | 10 +- .../flores_glotlid_language_consistency.py | 91 ++++ .../wmt_translation/language_consistency.py | 55 ++ .../wmt_translation/requirements.txt | 12 +- .../generate_flores_language_metadata.py | 182 +++++++ .../generate_flores_to_glotlid_mapping.py | 71 +++ .../generate_wmt24pp_language_names.py | 75 +++ ...cld2_and_wmt24pp_language_code_mappings.py | 127 +++++ ...otlid_and_flores_language_code_mappings.py | 62 +++ .../wmt_translation/tests/test_app.py | 468 ++++++++++++++++-- ...est_flores_glotlid_language_consistency.py | 191 +++++++ .../tests/test_language_consistency.py | 104 ++++ .../test_wmt24pp_cld2_language_consistency.py | 209 ++++++++ .../wmt24pp_cld2_language_consistency.py | 129 +++++ 30 files changed, 3404 insertions(+), 383 deletions(-) create mode 100644 benchmarks/flores200/data/flores_devtest_languages.json create mode 100644 benchmarks/flores200/data/flores_language_names.json create mode 100644 benchmarks/flores200/tests/test_prepare.py create mode 100644 benchmarks/wmt24pp/data/wmt24pp_language_names.json create mode 100644 benchmarks/wmt24pp/tests/test_wmt24pp_prepare.py create mode 100644 resources_servers/wmt_translation/flores_glotlid_language_consistency.py create mode 100644 resources_servers/wmt_translation/language_consistency.py create mode 100644 resources_servers/wmt_translation/scripts/generate_flores_language_metadata.py create mode 100644 resources_servers/wmt_translation/scripts/generate_flores_to_glotlid_mapping.py create mode 100644 resources_servers/wmt_translation/scripts/generate_wmt24pp_language_names.py create mode 100644 resources_servers/wmt_translation/scripts/validate_cld2_and_wmt24pp_language_code_mappings.py create mode 100644 resources_servers/wmt_translation/scripts/validate_glotlid_and_flores_language_code_mappings.py create mode 100644 resources_servers/wmt_translation/tests/test_flores_glotlid_language_consistency.py create mode 100644 resources_servers/wmt_translation/tests/test_language_consistency.py create mode 100644 resources_servers/wmt_translation/tests/test_wmt24pp_cld2_language_consistency.py create mode 100644 resources_servers/wmt_translation/wmt24pp_cld2_language_consistency.py diff --git a/README.md b/README.md index 11f78db7a61..b46f5d1d74c 100644 --- a/README.md +++ b/README.md @@ -350,7 +350,7 @@ The Dataset column links to publicly available datasets (e.g., on HuggingFace). | Vlm Eval Kit | other | - | Measure VLM capabilities | - | ✓ | - | MMBench_DEV_EN_V11.yaml | - | | Vlm Eval Kit | other | - | Measure VLM capabilities | - | ✓ | - | OCRBench.yaml | - | | Vlm Eval Kit | other | Run all supported VLMEvalKit benchmarks. | Measure VLM capabilities | - | ✓ | - | vlm_eval_kit.yaml | - | -| Wmt Translation | other | Machine-translation verifier computing corpus-level BLEU per language pair plus optional xCOMET-XXL neural QE via a Ray GPU actor. | Improves multilingual translation quality across BLEU and COMET-family metrics. | - | - | - | wmt_translation.yaml | - | +| Wmt Translation | other | Machine-translation verifier computing corpus-level spBLEU and ChrF per language pair plus optional xCOMET-XXL neural QE via a Ray GPU actor. | Improves multilingual translation quality across spBLEU, ChrF, and COMET-family metrics. | - | - | - | wmt_translation.yaml | - | | Workplace Assistant | agent | Workplace assistant multi-step tool-using environment | Improve multi-step tool use capability | ✓ | ✓ | Apache 2.0 | workplace_assistant.yaml | Nemotron-RL-agent-workplace_assistant | | Xlam Fc | agent | Salesforce xlam-function-calling-60k tool calling tasks | Improve tool-calling capabilities | ✓ | ✓ | Apache 2.0 | xlam_fc.yaml | - | | Xstest | safety | XSTest safety benchmark - exaggerated safety (over-refusal) evaluation | Evaluate model safety calibration between helpfulness and harmlessness | - | - | - | xstest.yaml | - | diff --git a/benchmarks/flores200/README.md b/benchmarks/flores200/README.md index df3ef4872bf..9de4b8c8f61 100644 --- a/benchmarks/flores200/README.md +++ b/benchmarks/flores200/README.md @@ -1,16 +1,19 @@ # FLORES-200 Translation Benchmark -Multilingual segment-level translation across the default six-language set -(`en`, `de`, `es`, `fr`, `it`, `ja`) from the +English-to-X segment-level translation from `eng_Latn` into every FLORES+ +configuration in the bundled metadata that provides the [`openlanguagedata/flores_plus`](https://huggingface.co/datasets/openlanguagedata/flores_plus) -`devtest` split. The benchmark interleaves all 30 directed pairs (6 sources × -6 targets, minus self-pairs) yielding 30 360 rows -(30 × 1012 sentences). +`devtest` split. The current metadata contains 220 such configurations, so the +benchmark interleaves 219 directed pairs and yields 221 628 rows +(219 × 1012 sentences). Verification reuses the shared `wmt_translation` resource server: deterministic -corpus-level BLEU (sacrebleu) per language pair plus the cross-pair -aggregations `xx->xx`, `->xx`, `xx->{tgt}`, optionally augmented by -xCOMET-XXL neural QE when `compute_comet: true` is set on the server. +corpus-level spBLEU and chrF (sacrebleu) per language pair plus the cross-pair +aggregations `xx->xx`, `->xx`, `xx->{tgt}`. spBLEU consistently uses +the `flores200` SentencePiece tokenizer. Metrics are optionally augmented by +xCOMET-XXL neural QE when `compute_comet: true` is set on the server. Each +rollout also reports the probability GlotLID assigns to its expected target +language, aggregated under `/language_consistency` metric keys. See `resources_servers/wmt_translation/README.md` for verifier details and the Ray GPU-scheduled COMET path. @@ -21,24 +24,28 @@ the Ray GPU-scheduled COMET path. gym eval prepare --benchmark flores200 ``` -Writes `data/flores200_devtest_benchmark.jsonl` and pre-fetches the -xCOMET-XXL checkpoint plus its xlm-roberta-xxl tokenizer into `HF_HOME` -(when `unbabel-comet` is installed in the active env). That keeps the -resource server's Ray actors fully offline at runtime — no HF Hub calls -during `verify()`, no rate-limit retries. +Writes `data/flores200_devtest_benchmark.jsonl` and pre-fetches SacreBLEU's +FLORES-200 SentencePiece model into its cache, the xCOMET-XXL checkpoint plus +its xlm-roberta-xxl tokenizer into `HF_HOME` (when `unbabel-comet` is installed +in the active env), and the GlotLID model used for language-consistency +scoring. That keeps verification fully offline at runtime. -The default downloads six FLORES+ language configs (`eng_Latn`, `deu_Latn`, -`spa_Latn`, `fra_Latn`, `ita_Latn`, `jpn_Jpan`) and produces a JSONL whose -order, fields, and contents are byte-equivalent to the reference -[NeMo-Skills](https://github.com/NVIDIA-NeMo/Skills) prepare script -(`nemo_skills/dataset/flores200/prepare.py`) under the same defaults. +Each row stores complete FLORES+ codes in `source_language` and +`target_language`, plus their English display names in `source_lang_name` and +`target_lang_name`. + +The checked-in `data/flores_language_names.json` and +`data/flores_devtest_languages.json` files are generated by +`resources_servers/wmt_translation/scripts/generate_flores_language_metadata.py`. +Preparation reads these files locally, so it does not make per-language +metadata requests. Regenerate them deliberately when updating FLORES+. ## Running servers The xCOMET-XXL actor pool requires the `extra_gpu` Ray resource which is only advertised on multi-node SLURM deployments via NeMo-Skills' `get_ray_server_cmd`. Local / single-node runs disable COMET via Hydra -override and rely on corpus-BLEU only: +override and rely on local spBLEU and chrF: ```bash gym env start \ diff --git a/benchmarks/flores200/config.yaml b/benchmarks/flores200/config.yaml index d4430eed58e..098be9c24f1 100644 --- a/benchmarks/flores200/config.yaml +++ b/benchmarks/flores200/config.yaml @@ -3,6 +3,9 @@ config_paths: flores200_wmt_translation_resources_server: _inherit_from: wmt_translation + resources_servers: + wmt_translation: + language_consistency_backend: flores_glotlid flores200_wmt_translation_simple_agent: _inherit_from: wmt_translation_simple_agent diff --git a/benchmarks/flores200/data/example_rollouts.jsonl b/benchmarks/flores200/data/example_rollouts.jsonl index e7e20eff764..50b1d6a1f87 100644 --- a/benchmarks/flores200/data/example_rollouts.jsonl +++ b/benchmarks/flores200/data/example_rollouts.jsonl @@ -1,5 +1,5 @@ -{"responses_create_params":{"background":null,"include":null,"input":[{"content":"Translate the following segment into German, without additional explanation.\n\n\"We now have 4-month-old mice that are non-diabetic that used to be diabetic,\" he added.","role":"user","type":"message"}],"instructions":null,"max_output_tokens":2048,"max_tool_calls":null,"metadata":null,"model":null,"parallel_tool_calls":true,"previous_response_id":null,"prompt":null,"reasoning":null,"service_tier":null,"store":null,"temperature":null,"text":null,"tool_choice":"auto","tools":[],"top_logprobs":null,"top_p":null,"truncation":null,"user":null,"stream":null},"response":{"id":"resp_9ca21326530c4c5aa1b54920c42dce5f","created_at":1777660821.0,"error":null,"incomplete_details":null,"instructions":null,"metadata":null,"model":"nvidia/nvidia-nemotron-nano-9b-v2","object":"response","output":[{"id":"rs_178fa151039b4c19a1023c19161f51ce","summary":[{"text":"Okay, let's tackle this translation. The user wants the segment translated into German without any extra explanation. The original sentence is: \"We now have 4-month-old mice that are non-diabetic that used to be diabetic,\" he added.\n\nFirst, I need to make sure I understand the sentence correctly. The key points are: 4-month-old mice, previously diabetic but now non-diabetic, and this is reported by someone (\"he added\").\n\nStarting with \"We now have\" – in German, that's \"Wir haben jetzt\". Then \"4-month-old mice\" would be \"4-monatige Mäuse\". The next part is a bit tricky: \"that are non-diabetic that used to be diabetic\". Here, \"non-diabetic\" in German is \"nicht-diabetisch\" and \"used to be diabetic\" is \"habeen diabetic\" or more naturally \"waren diabetic\" (plural). Since the mice are plural, I should use \"waren diabetic\". \n\nPutting it together: \"diese sind nicht-diabetisch gewesen, die formerly diabetic waren\". Wait, but \"die formerly diabetic\" might not be the smoothest. Alternatively, \"die formerly diabetic sind\" might be clearer. But \"were diabetic\" is better as \"waren diabetic\".\n\nSo the sentence would be: \"Wir haben jetzt 4-monatige Mäuse, die nicht-diabetisch sind, dieformerly diabetic waren, sagte er dazu.\" Hmm, \"dazu\" is appropriate for \"added\" in this context. \n\nWait, \"that used to be diabetic\" is a relative clause modifying the mice. So the structure in German should be: \"Mäuse, die nicht-diabetisch sind, die formerly diabetic waren\". But in German, after the second relative clause, you might need to adjust. Alternatively, combining them: \"Mäuse, dieformerly diabetic, aber jetzt nicht-diabetisch sind\". That might flow better. \n\nSo maybe: \"Wir haben jetzt 4-monatige Mäuse, die formerly diabetic, aber jetzt nicht-diabetisch sind, sagte er dazu.\" That sounds more natural. The original uses \"that used to be\", so \"formerly diabetic\" is a good translation here. \n\nLet me check if \"formerly diabetic\" is the standard way. Yes, that's commonly used. \"diabetic\" in German is the same when referring to the condition. \n\nSo final translation: \"Wir haben jetzt 4-monatige Mäuse, die formerly diabetic, aber jetzt nicht-diabetisch sind, sagte er dazu.\" \n\nWait, but in the original, it's \"that are non-diabetic that used to be diabetic\". So both clauses are attributive. In German, using two relative clauses is acceptable. Alternatively, \"Mäuse, dieformerly diabetic sind, aber jetzt nicht-diabetisch sind\". \n\nAnother way: \"Mäuse, dieformerly diabetic, sind jetzt nicht-diabetisch\". But the original structure is \"are non-diabetic that used to be\". So the active form might be better. \n\nAlternatively, \"Mäuse, die formerly diabetic waren, sind jetzt nicht-diabetisch\". \n\nBut in German, the past tense is not always necessary. However, to make it clear that they were diabetic in the past, \"waren diabetic\" is better. \n\nSo \"Mäuse, dieformerly diabetic waren, sind jetzt nicht-diabetisch\". \n\nPutting it all together: \"Wir haben jetzt 4-monatige Mäuse, die formerly diabetic waren, sind jetzt nicht-diabetisch, sagte er dazu.\" \n\nWait, \"sind jetzt nicht-diabetisch\" is redundant with \"sein\" and \"jetzt\". Maybe \"sind nicht-diabetisch geworden\" would be better: \"sind jetzt nicht-diabetisch geworden\". But the original doesn't specify the exact time, just that they used to be diabetic. \n\nAlternatively, \"haben jetzt 4-monatige Mäuse, dieformerly diabetic, aber jetzt nicht-diabetisch sind\". \n\nYes, \"aber jetzt\" contrasts the past and present. \n\nSo the final translation should be: \"Wir haben jetzt 4-monatige Mäuse, dieformerly diabetic, aber jetzt nicht-diabetisch sind, sagte er dazu.\" \n\nChecking for any errors. \"Además\" is not in the original, so no need. The name \"he\" is translated as \"er\" or \"er sagte dazu\". \"Dazu\" is correct for \"added\". \n\nYes, this should be accurate and natural in German.\n","type":"summary_text"}],"type":"reasoning","encrypted_content":null},{"id":"msg_ba81286209e544dc85f5f257d7c12f34","content":[{"annotations":[],"text":"\n\n\"Wir haben jetzt 4-monatige Mäuse, die formerly diabetic, aber jetzt nicht-diabetisch sind, sagte er dazu.\"\n","type":"output_text","logprobs":null}],"role":"assistant","status":"completed","type":"message"}],"parallel_tool_calls":true,"temperature":null,"tool_choice":"auto","tools":[],"top_p":null,"background":null,"conversation":null,"max_output_tokens":2048,"max_tool_calls":null,"previous_response_id":null,"prompt":null,"prompt_cache_key":null,"reasoning":null,"safety_identifier":null,"service_tier":null,"status":null,"text":null,"top_logprobs":null,"truncation":null,"usage":{"input_tokens":45,"input_tokens_details":{"cached_tokens":0},"output_tokens":991,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":1036},"user":null},"reward":0.1171291646655074,"text":"\"We now have 4-month-old mice that are non-diabetic that used to be diabetic,\" he added.","translation":"„Wir haben jetzt 4 Monate alte Mäuse, die Diabetes hatten und jetzt keinen mehr haben“, fügte er hinzu.","source_language":"en","target_language":"de","source_lang_name":"English","target_lang_name":"German","generation":"\"Wir haben jetzt 4-monatige Mäuse, die formerly diabetic, aber jetzt nicht-diabetisch sind, sagte er dazu.\"","sentence_bleu":11.71291646655074,"comet_score":null,"_ng_task_index":0,"_ng_rollout_index":0,"agent_ref":{"type":"responses_api_agents","name":"flores200_wmt_translation_simple_agent"}} -{"responses_create_params":{"background":null,"include":null,"input":[{"content":"Translate the following segment into German, without additional explanation.\n\nDr. Ehud Ur, professor of medicine at Dalhousie University in Halifax, Nova Scotia and chair of the clinical and scientific division of the Canadian Diabetes Association cautioned that the research is still in its early days.","role":"user","type":"message"}],"instructions":null,"max_output_tokens":2048,"max_tool_calls":null,"metadata":null,"model":null,"parallel_tool_calls":true,"previous_response_id":null,"prompt":null,"reasoning":null,"service_tier":null,"store":null,"temperature":null,"text":null,"tool_choice":"auto","tools":[],"top_logprobs":null,"top_p":null,"truncation":null,"user":null,"stream":null},"response":{"id":"resp_aba3f55756314e17a41bf40171eff413","created_at":1777660822.0,"error":null,"incomplete_details":null,"instructions":null,"metadata":null,"model":"nvidia/nvidia-nemotron-nano-9b-v2","object":"response","output":[{"id":"rs_8cdbc07fd2324ee3ac2e62727c0853a3","summary":[{"text":"Okay, I need to translate the given English sentence into German. Let me start by reading the original text carefully to understand the context and key terms.\n\nThe sentence is: \"Dr. Ehud Ur, professor of medicine at Dalhousie University in Halifax, Nova Scotia and chair of the clinical and scientific division of the Canadian Diabetes Association cautioned that the research is still in its early days.\"\n\nFirst, I'll identify the proper nouns and their German equivalents. \"Dr. Ehud Ur\" – names usually stay the same in German unless there's a specific adaptation. So it remains \"Dr. Ehud Ur.\"\n\nNext, \"professor of medicine at Dalhousie University in Halifax, Nova Scotia.\" The university name \"Dalhousie University\" would typically be kept in all caps as in English, but sometimes in German it might be written in lowercase. However, since it's a proper noun, I'll keep it as \"Dalhousie University.\" \"Halifax, Nova Scotia\" – Halifax is a city in Nova Scotia, Canada. In German, the structure is \"Halifax, Neuschottland.\" But sometimes \"Neuschottland\" might be replaced with \"Kanada\" to specify the country, but since it's part of the location, \"Halifax, Neuschottland\" is correct.\n\nThen, \"chair of the clinical and scientific division of the Canadian Diabetes Association.\" \"Chair\" here is a title, so in German, it would be \"Präsident\" or \"Präsidentin,\" but \"chair\" in an organizational context is often translated as \"Präsident.\" However, sometimes \" Vorsitzenden\" is used if it's a board position. Wait, \"chair\" as a title might be \"Präsident\" here. \"Clinical and scientific division\" – \"klinische und wissenschaftliche Abteilung\" or \"Stelle.\" \"Canadian Diabetes Association\" is a proper noun, so \"Canadian Diabetes Association\" in German is usually kept as is, but sometimes translated to \"Kanadische Diabetes-Assoziation.\" However, since it's an official name, it's better to keep it in the original: \"Canadian Diabetes Association.\"\n\nNow, the verb \"cautioned\" – in German, \"warnt\" or \"+deutet an\" might work. \"Cautioned that\" would be \"warnte, dass\" or \"+deutete an, dass.\" \n\n\"The research is still in its early days\" – \"die Forschung ist noch in ihren frühen Tagen\" or \"noch in den Anfängen.\" \"Early days\" can be \"Anfängen\" or \"frühen Tagen.\"\n\nPutting it all together, let me check each part:\n\n- \"professor of medicine\" – \"Lehrstuhlhalter für Medizin\" or \"Professor für Medizin.\" Since he's a professor, \"Professor für Medizin an der Dalhousie University in Halifax, Neuschottland\" is better.\n\n- \"chair of the clinical and scientific division\" – \"Präsident des klinischen und wissenschaftlichen Bereichs\" or \"Vorsitzende der klinischen und wissenschaftlichen Abteilung.\" Since it's a division, \"Abteilung\" is more appropriate than \"Bereich.\"\n\n- \"cautioned that\" – \"warnte, dass\" or \"+deutete an, dass.\" \"Deuten auf\" is more like \"indicated,\" but \"warnte\" is a direct translation of \"cautioned.\"\n\n- \"research is still in its early days\" – \"die Forschung ist noch in ihren Anfängen\" or \"noch in den Anfängen.\"\n\nSo, putting it all together:\n\n\"Dr. Ehud Ur, Professor für Medizin an der Dalhousie University in Halifax, Neuschottland und Präsident der klinischen und wissenschaftlichen Abteilung der Canadian Diabetes Association warnte, dass die Forschung noch in ihren Anfängen ist.\"\n\nWait, \"chair of the division\" – is \"Präsident der Abteilung\" correct? Sometimes \"Präsident\" might refer to the head of an organization, not a division. Maybe \"Vorsitzender der Abteilung\" would be more accurate if it's a division's chair. But \"chair\" as a position in an association's division might be \"Präsident.\" Alternatively, \"Präsident des klinischen und wissenschaftlichen Bereichs der Canadian Diabetes Association.\" \n\nAlternatively, \"Chair\" could be translated as \"Präsident\" here. Let me check standard translations. For example, the American Diabetes Association has a president, so in German, \"Präsident\" is appropriate.\n\nSo final translation:\n\n\"Dr. Ehud Ur, Professor für Medizin an der Dalhousie University in Halifax, Neuschottland und Präsident der klinischen und wissenschaftlichen Abteilung der Canadian Diabetes Association warnte, dass die Forschung noch in ihren Anfängen ist.\"\n\nI should check if \" Михаил\" is needed, but since it's a proper name, no. Also, \"Michaiл\" is Cyrillic, but the name here is Ehud Ur, so no issues.\n\nDouble-checking prepositions and articles. \"An der Dalhousie University\" is correct. \"Präsident der Abteilung\" – yes. \"Warnte, dass\" – correct. \"Anfängen\" is the right term for \"early days\" in this context.\n\nAlternatively, \"noch in den Anfängen\" without \"seiner\" might be better. \"Die Forschung ist noch in den Anfängen.\" Since \"Anfängen\" can be plural, \"die Forschung\" is singular, so \"in den Anfängen\" is correct.\n\nYes, that seems accurate.\n","type":"summary_text"}],"type":"reasoning","encrypted_content":null},{"id":"msg_31f943d913e249928ea71fd7be452373","content":[{"annotations":[],"text":"\n\nDr. Ehud Ur, Professor für Medizin an der Dalhousie University in Halifax, Neuschottland und Präsident der klinischen und wissenschaftlichen Abteilung der Canadian Diabetes Association warnte, dass die Forschung noch in den Anfängen ist.\n","type":"output_text","logprobs":null}],"role":"assistant","status":"completed","type":"message"}],"parallel_tool_calls":true,"temperature":null,"tool_choice":"auto","tools":[],"top_p":null,"background":null,"conversation":null,"max_output_tokens":2048,"max_tool_calls":null,"previous_response_id":null,"prompt":null,"prompt_cache_key":null,"reasoning":null,"safety_identifier":null,"service_tier":null,"status":null,"text":null,"top_logprobs":null,"truncation":null,"usage":{"input_tokens":68,"input_tokens_details":{"cached_tokens":0},"output_tokens":1198,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":1266},"user":null},"reward":0.4203155083891092,"text":"Dr. Ehud Ur, professor of medicine at Dalhousie University in Halifax, Nova Scotia and chair of the clinical and scientific division of the Canadian Diabetes Association cautioned that the research is still in its early days.","translation":"Dr. Ehud Ur, Professor für Medizin an der Dalhousie University in Halifax, Nova Scotia, und Vorsitzender der Abteilung für Klinik und Wissenschaft des Kanadischen Diabetesverbands gab zu bedenken, dass die Forschungsarbeit noch in den Kinderschuhen stecke.","source_language":"en","target_language":"de","source_lang_name":"English","target_lang_name":"German","generation":"Dr. Ehud Ur, Professor für Medizin an der Dalhousie University in Halifax, Neuschottland und Präsident der klinischen und wissenschaftlichen Abteilung der Canadian Diabetes Association warnte, dass die Forschung noch in den Anfängen ist.","sentence_bleu":42.03155083891092,"comet_score":null,"_ng_task_index":1,"_ng_rollout_index":0,"agent_ref":{"type":"responses_api_agents","name":"flores200_wmt_translation_simple_agent"}} -{"responses_create_params":{"background":null,"include":null,"input":[{"content":"Translate the following segment into German, without additional explanation.\n\nLike some other experts, he is skeptical about whether diabetes can be cured, noting that these findings have no relevance to people who already have Type 1 diabetes.","role":"user","type":"message"}],"instructions":null,"max_output_tokens":2048,"max_tool_calls":null,"metadata":null,"model":null,"parallel_tool_calls":true,"previous_response_id":null,"prompt":null,"reasoning":null,"service_tier":null,"store":null,"temperature":null,"text":null,"tool_choice":"auto","tools":[],"top_logprobs":null,"top_p":null,"truncation":null,"user":null,"stream":null},"response":{"id":"resp_d584ceec15f44ef2ad3d9ccf4f47c698","created_at":1777660835.0,"error":null,"incomplete_details":null,"instructions":null,"metadata":null,"model":"nvidia/nvidia-nemotron-nano-9b-v2","object":"response","output":[{"id":"rs_b5746be784a84d6190db6802b7ebcaae","summary":[{"text":"Okay, I need to translate this sentence into German. Let me start by reading the original text carefully.\n\nThe sentence is: \"Like some other experts, he is skeptical about whether diabetes can be cured, noting that these findings have no relevance to people who already have Type 1 diabetes.\"\n\nFirst, I'll break it down. The first part is \"Like some other experts, he is skeptical about whether diabetes can be cured.\" The key here is to convey the comparison to other experts and his skepticism.\n\n\"Like some other experts\" – in German, \"wie einige andere Experten\" is a direct translation. \"He is skeptical\" would be \"er ist skeptisch\". The question \"whether diabetes can be cured\" is \"ob Diabetes geheilt werden kann\".\n\nSo the first part could be: \"Wie einige andere Experten ist er skeptisch, ob Diabetes geheilt werden kann.\"\n\nNext part: \"noting that these findings have no relevance to people who already have Type 1 diabetes.\" \"Noting\" here is \"mit der Hinweis\" or \"quiry\" but \"mit der Hinweis\" might be more natural. \"These findings\" is \"diese Ergebnisse\". \"Have no relevance\" is \"-linking zu\" or \"relevant\" – \"haben keine Relevanz zu\". \n\n\"People who already have Type 1 diabetes\" is \"Menschen, die bereits an Typ-1-Diabetes leiden\".\n\nPutting it together: \"mit der Hinweis, dass diese Ergebnisse für diejenigen nicht relevant sind, die bereits an Typ-1-Diabetes leiden.\"\n\nWait, maybe \"mit der Hinweis\" is a bit off. In German, \"mit der Bemerkung\" or \"mit der Hinweis\" – but \"noting\" here is more like \"pointing out\" or \"noting\" as in making a remark. So \"mit der Bemerkung\" might be better. Alternatively, \"da hinweisend\" but that's more like \"while noting\". \n\nAlternatively, maybe \"und betont, dass diese Ergebnisse für diejenigen nicht relevant sind...\" but the original says \"noting\", which is more like a remark. So \"mit der Bemerkung\" is appropriate.\n\nSo the full translation would be: \"Wie einige andere Experten ist er skeptisch, ob Diabetes geheilt werden kann, mit der Bemerkung, dass diese Ergebnisse für diejenigen nicht relevant sind, die bereits an Typ-1-Diabetes leiden.\"\n\nLet me check the grammar. \"Mit der Bemerkung\" is correct. \"Für diejenigen nicht relevant\" – yes. \"Verlieran\" is correct for \"haben\". \n\nAnother thing: \"Type 1 diabetes\" in German is \"Typ-1-Diabetes\". The hyphen is correct. \n\nI think that's accurate. The structure is a bit long, but that's how the original is. No additional explanations needed, just the translation.\n","type":"summary_text"}],"type":"reasoning","encrypted_content":null},{"id":"msg_2921452710b34f60971f47591a19b7f8","content":[{"annotations":[],"text":"\n\nWie einige andere Experten ist er skeptisch, ob Diabetes geheilt werden kann, mit der Bemerkung, dass diese Ergebnisse für diejenigen nicht relevant sind, die bereits an Typ-1-Diabetes leiden.\n","type":"output_text","logprobs":null}],"role":"assistant","status":"completed","type":"message"}],"parallel_tool_calls":true,"temperature":null,"tool_choice":"auto","tools":[],"top_p":null,"background":null,"conversation":null,"max_output_tokens":2048,"max_tool_calls":null,"previous_response_id":null,"prompt":null,"prompt_cache_key":null,"reasoning":null,"safety_identifier":null,"service_tier":null,"status":null,"text":null,"top_logprobs":null,"truncation":null,"usage":{"input_tokens":55,"input_tokens_details":{"cached_tokens":0},"output_tokens":651,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":706},"user":null},"reward":0.14149685839851064,"text":"Like some other experts, he is skeptical about whether diabetes can be cured, noting that these findings have no relevance to people who already have Type 1 diabetes.","translation":"Wie einige andere Experten zeigte er sich skeptisch, ob es möglich sei, Diabetes zu heilen, und wies darauf hin, dass die Befunde für Menschen, die bereits unter Typ-1-Diabetes litten, keine Bedeutung hätten.","source_language":"en","target_language":"de","source_lang_name":"English","target_lang_name":"German","generation":"Wie einige andere Experten ist er skeptisch, ob Diabetes geheilt werden kann, mit der Bemerkung, dass diese Ergebnisse für diejenigen nicht relevant sind, die bereits an Typ-1-Diabetes leiden.","sentence_bleu":14.149685839851065,"comet_score":null,"_ng_task_index":2,"_ng_rollout_index":0,"agent_ref":{"type":"responses_api_agents","name":"flores200_wmt_translation_simple_agent"}} -{"responses_create_params":{"background":null,"include":null,"input":[{"content":"Translate the following segment into German, without additional explanation.\n\nDanius said, \"Right now we are doing nothing. I have called and sent emails to his closest collaborator and received very friendly replies. For now, that is certainly enough.\"","role":"user","type":"message"}],"instructions":null,"max_output_tokens":2048,"max_tool_calls":null,"metadata":null,"model":null,"parallel_tool_calls":true,"previous_response_id":null,"prompt":null,"reasoning":null,"service_tier":null,"store":null,"temperature":null,"text":null,"tool_choice":"auto","tools":[],"top_logprobs":null,"top_p":null,"truncation":null,"user":null,"stream":null},"response":{"id":"resp_1f9263ff3586420aa0a51da26fbf1558","created_at":1777660837.0,"error":null,"incomplete_details":null,"instructions":null,"metadata":null,"model":"nvidia/nvidia-nemotron-nano-9b-v2","object":"response","output":[{"id":"rs_507d7e4ba6ce45e8a52e289592334e52","summary":[{"text":"Okay, I need to translate this segment into German without any additional explanation. Let me start by reading the original text carefully.\n\n\"Danius said, 'Right now we are doing nothing. I have called and sent emails to his closest collaborator and received very friendly replies. For now, that is certainly enough.'\" \n\nFirst, I'll break it down sentence by sentence. \n\n\"Danius said,\" – In German, that's \"Danius sagte,\".\n\n\"Right now we are doing nothing.\" – The phrase \"Right now\" can be \"Jetzt\" or \"In diesem Moment\". \"We are doing nothing\" is \"machen nichts\" or \"etwas tun\". So maybe \"Jetzt machen wir nichts.\"\n\nNext sentence: \"I have called and sent emails to his closest collaborator...\" \n\n\"I have called\" – \"Ich habe angerufen\" or \"Ich habe angerufen hat\". But \"angerrufen\" is correct. \n\n\"sent emails\" – \"E-Mails absend\" or \"E-Mails gesendet\". \"Gesendet\" is more common. \n\n\"to his closest collaborator\" – \"zu seinem nächsten Kollaborateur\" or \"zusammenarbeitspartner\". \"Kollaborateur\" is a bit formal, but it's acceptable. \n\n\"and received very friendly replies.\" – \"und habe sehr freundliche Rückmeldungen erhalten.\" Or \"sehr freundliche Antworten erhalten\". \"Rückmeldungen\" might be better here for replies. \n\n\"For now, that is certainly enough.\" – \"Für den Moment ist das sicherlich ausreichend.\" Or \"Das ist für den Moment definitiv genug.\" But \"sicherlich\" is more natural here. \n\nPutting it all together:\n\n\"Danius sagte: 'Jetzt machen wir nichts. Ich habe angerufen und E-Mails an seinen nächsten Kollaborateur geschickt und habe sehr freundliche Rückmeldungen erhalten. Für den Moment ist das sicherlich ausreichend.'\"\n\nI should check for any grammatical errors. \"Gegeben\" is correct for \"sent\". \"Nächsten Kollaborateur\" – \"nächster\" is correct. \"Rückmeldungen\" is plural, so \"angefangen\" is correct. \n\nAlternatively, \"I have called and sent emails\" could be \"Ich habe angerufen und E-Mails gesendet\". But \"geschickt\" is also correct. \n\nAnother point: \"closest collaborator\" – \"nächster Kollaborateur\" or \"engster Unterstützer\"? \"Kollaborateur\" is more about collaboration, which fits. \n\n\"Received very friendly replies\" – \"sehr freundliche Antworten\" or \"freundliche Antworten\". \"Rückmeldungen\" is more about feedback, but in this context, \"Antworten\" might be better. \n\nSo maybe: \"und habe sehr freundliche Antworten erhalten.\"\n\nFinal check: The structure is correct. The quotes are properly opened and closed. The verbs are in the right tense. \n\nI think that's a solid translation. Let me make sure there are no typos or spelling mistakes. \"Kollaborateur\" is spelled correctly. \"Geschickt\" or \"gesendet\" – both are acceptable. \n\nYes, this should be the correct translation.\n","type":"summary_text"}],"type":"reasoning","encrypted_content":null},{"id":"msg_f0996f1f2b1b4e62818710bee1e18e59","content":[{"annotations":[],"text":"\n\nDanius sagte: \"Jetzt machen wir nichts. Ich habe angerufen und E-Mails an seinen nächsten Kollaborateur geschickt und habe sehr freundliche Rückmeldungen erhalten. Für den Moment ist das sicherlich ausreichend.\"\n","type":"output_text","logprobs":null}],"role":"assistant","status":"completed","type":"message"}],"parallel_tool_calls":true,"temperature":null,"tool_choice":"auto","tools":[],"top_p":null,"background":null,"conversation":null,"max_output_tokens":2048,"max_tool_calls":null,"previous_response_id":null,"prompt":null,"prompt_cache_key":null,"reasoning":null,"safety_identifier":null,"service_tier":null,"status":null,"text":null,"top_logprobs":null,"truncation":null,"usage":{"input_tokens":60,"input_tokens_details":{"cached_tokens":0},"output_tokens":744,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":804},"user":null},"reward":0.19121616355935117,"text":"Danius said, \"Right now we are doing nothing. I have called and sent emails to his closest collaborator and received very friendly replies. For now, that is certainly enough.\"","translation":"Danius sagte: „Im Moment unternehmen wir nichts. Ich habe seinen engsten Mitarbeiter angerufen und ihm E-Mails geschickt und sehr freundliche Antworten erhalten. Das ist vorläufig genug.“","source_language":"en","target_language":"de","source_lang_name":"English","target_lang_name":"German","generation":"Danius sagte: \"Jetzt machen wir nichts. Ich habe angerufen und E-Mails an seinen nächsten Kollaborateur geschickt und habe sehr freundliche Rückmeldungen erhalten. Für den Moment ist das sicherlich ausreichend.\"","sentence_bleu":19.121616355935117,"comet_score":null,"_ng_task_index":4,"_ng_rollout_index":0,"agent_ref":{"type":"responses_api_agents","name":"flores200_wmt_translation_simple_agent"}} -{"responses_create_params":{"background":null,"include":null,"input":[{"content":"Translate the following segment into German, without additional explanation.\n\nOn Monday, Sara Danius, permanent secretary of the Nobel Committee for Literature at the Swedish Academy, publicly announced during a radio program on Sveriges Radio in Sweden the committee, unable to reach Bob Dylan directly about winning the 2016 Nobel Prize in Literature, had abandoned its efforts to reach him.","role":"user","type":"message"}],"instructions":null,"max_output_tokens":2048,"max_tool_calls":null,"metadata":null,"model":null,"parallel_tool_calls":true,"previous_response_id":null,"prompt":null,"reasoning":null,"service_tier":null,"store":null,"temperature":null,"text":null,"tool_choice":"auto","tools":[],"top_logprobs":null,"top_p":null,"truncation":null,"user":null,"stream":null},"response":{"id":"resp_341824482a2d4f1fb401790964600984","created_at":1777660854.0,"error":null,"incomplete_details":null,"instructions":null,"metadata":null,"model":"nvidia/nvidia-nemotron-nano-9b-v2","object":"response","output":[{"id":"rs_bacaa51f597a4ce1bbf7a9ef03fec4e1","summary":[{"text":"Okay, let's tackle this translation. First, I need to make sure I understand the original sentence correctly. The key points are: Sara Danius is the permanent secretary of the Nobel Committee for Literature. She announced on Monday in a radio program on Sveriges Radio that the committee couldn't contact Bob Dylan directly about winning the 2016 Nobel Prize in Literature, so they stopped trying.\n\nNow, translating into German. Let's start with the names and titles. \"Sara Danius\" stays the same. \"Permanent secretary of the Nobel Committee for Literature at the Swedish Academy\" – in German, \"Permanente Sekretärin des Nobel-Komitees für Literatur am Schwedischen Academia\". I should check if \"Academy\" is capitalized here. Yes, since it's a specific institution.\n\nNext, \"publicly announced during a radio program on Sveriges Radio in Sweden\". \"Öffentlich bekannt gegeben während eines Rundfunkprogramms auf Sveriges Radio in Schweden\". \"Sveriges Radio\" is the correct name, so no change there.\n\nThe main clause: \"the committee, unable to reach Bob Dylan directly about winning the 2016 Nobel Prize in Literature, had abandoned its efforts to reach him.\" \n\n\"Die Kommission, die Bob Dylan nicht direkt zu einer Nominierung als Gewinner des Nobelpreises für Literatur 2016 erreichen konnte, hatte ihre Versuche, ihn zu kontaktieren, aufgegeben.\"\n\nBreaking it down: \"unable to reach\" is \"nicht direkt erreichen konnten\". \"About winning\" – \"zu einer Nominierung als Gewinner\". Wait, \"winning\" here is the context of the prize, so \"Nominierung\" might be more accurate than \"Sieg\" since the Nobel is awarded after nomination. \"2016\" stays the same. \"Had abandoned efforts to reach him\" – \"ihre Versuche, ihn zu kontaktieren, aufgegeben.\"\n\nPutting it all together. Let me check for any possible errors. \"Nobel-Komitee\" is correct. \"Gewinner\" is okay, but sometimes \"Preisträger\" is used. However, \"Gewinner\" is commonly used for Nobel winners. \"Nominierung\" vs \"Gewinn\" – since the prize is awarded, \"Gewinn\" might be better. Wait, the original says \"about winning\", so maybe \"zum Gewinn des Preises\". But the structure in German might be \"zu einem Gewinn des Nobelpreises...\". Alternatively, \"zur Verleihung\" could be another option. Hmm. The original is about reaching Dylan regarding the prize win. So maybe \"zum Gewinn des Nobelpreises für Literatur 2016\". \n\nWait, the original says \"about winning the 2016 Nobel Prize in Literature\". So the committee was trying to reach Dylan about his winning the prize. So \"zu einem Gewinn des Nobelpreises für Literatur 2016\". That makes sense. So adjusting that part: \"die Bob Dylan nicht direkt zu einem Gewinn des Nobelpreises für Literatur 2016 erreichen konnte\".\n\nAlso, \"had abandoned its efforts to reach him\" – \"war ihre Ansprache zu Dylan aufgegeben\". Or \"ihre Versuche, ihn zu kontaktieren, aufgaben\". \"Versuche\" is plural, so \"ihre Versuche, ihn zu kontaktieren, hatten aufgegeben\". \n\nPutting it all together:\n\nAm Montag bekanntgab während eines Rundfunkprogramms auf Sveriges Radio in Schweden die Permane Sekretärin des Nobel-Komitees für Literatur am Schwedischen Academia, Sara Danius, dass die Kommission, die Bob Dylan nicht direkt zu einem Gewinn des Nobelpreises für Literatur 2016 erreichen konnte, ihre Versuche, ihn zu kontaktieren, aufgab.\n\nWait, \"had abandoned\" is \"had given up\", so \"hatten aufgeben\" in past perfect. Since the main verb is \"kennenshaftgab\" (Berlinte), which is in the past, the past perfect is correct. \n\nAnother point: \"publicly announced\" is \"bekanntgab\". Correct. \"Publicly\" is implied by \"publicly announced\", so no need for \"öffentlich\" unless specified. \n\nDouble-checking all names and titles. \"Schwedischen Academia\" – \"im Schwedischen Academia\" might be better, but \"Schwedischen Academia\" is acceptable as the academy is a proper noun. \n\nYes, this seems accurate. Let me read it once more to ensure flow and correctness.\n","type":"summary_text"}],"type":"reasoning","encrypted_content":null},{"id":"msg_2c59e9d45f8e44acb4b4963b263f7d15","content":[{"annotations":[],"text":"\n\nAm Montag bekanntgab während eines Rundfunkprogramms auf Sveriges Radio in Schweden die Permane Sekretärin des Nobel-Komitees für Literatur am Schwedischen Academia, Sara Danius, dass die Kommission, die Bob Dylan nicht direkt zu einem Gewinn des Nobelpreises für Literatur 2016 erreichen konnte, ihre Versuche, ihn zu kontaktieren, aufgab.\n","type":"output_text","logprobs":null}],"role":"assistant","status":"completed","type":"message"}],"parallel_tool_calls":true,"temperature":null,"tool_choice":"auto","tools":[],"top_p":null,"background":null,"conversation":null,"max_output_tokens":2048,"max_tool_calls":null,"previous_response_id":null,"prompt":null,"prompt_cache_key":null,"reasoning":null,"safety_identifier":null,"service_tier":null,"status":null,"text":null,"top_logprobs":null,"truncation":null,"usage":{"input_tokens":84,"input_tokens_details":{"cached_tokens":0},"output_tokens":1015,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":1099},"user":null},"reward":0.13057448793891352,"text":"On Monday, Sara Danius, permanent secretary of the Nobel Committee for Literature at the Swedish Academy, publicly announced during a radio program on Sveriges Radio in Sweden the committee, unable to reach Bob Dylan directly about winning the 2016 Nobel Prize in Literature, had abandoned its efforts to reach him.","translation":"Am Montag gab Sara Danius, Ständige Sekretärin des Literaturnobelpreiskomitees der Schwedischen Akademie in einer Radiosendung auf Sveriges Radio in Schweden bekannt, dass das Komitee seine Bemühungen eingestellt habe, Bob Dylan über seine Ehrung mit dem Literaturnobelpreis für Literator zu informieren, nachdem es nicht gelungen war, ihn persönlich zu erreichen.","source_language":"en","target_language":"de","source_lang_name":"English","target_lang_name":"German","generation":"Am Montag bekanntgab während eines Rundfunkprogramms auf Sveriges Radio in Schweden die Permane Sekretärin des Nobel-Komitees für Literatur am Schwedischen Academia, Sara Danius, dass die Kommission, die Bob Dylan nicht direkt zu einem Gewinn des Nobelpreises für Literatur 2016 erreichen konnte, ihre Versuche, ihn zu kontaktieren, aufgab.","sentence_bleu":13.057448793891352,"comet_score":null,"_ng_task_index":3,"_ng_rollout_index":0,"agent_ref":{"type":"responses_api_agents","name":"flores200_wmt_translation_simple_agent"}} +{"responses_create_params":{"background":null,"include":null,"input":[{"content":"Translate the following segment into German, without additional explanation.\n\n\"We now have 4-month-old mice that are non-diabetic that used to be diabetic,\" he added.","role":"user","type":"message"}],"instructions":null,"max_output_tokens":2048,"max_tool_calls":null,"metadata":null,"model":null,"parallel_tool_calls":true,"previous_response_id":null,"prompt":null,"reasoning":null,"service_tier":null,"store":null,"temperature":null,"text":null,"tool_choice":"auto","tools":[],"top_logprobs":null,"top_p":null,"truncation":null,"user":null,"stream":null},"response":{"id":"resp_9ca21326530c4c5aa1b54920c42dce5f","created_at":1777660821.0,"error":null,"incomplete_details":null,"instructions":null,"metadata":null,"model":"nvidia/nvidia-nemotron-nano-9b-v2","object":"response","output":[{"id":"rs_178fa151039b4c19a1023c19161f51ce","summary":[{"text":"Okay, let's tackle this translation. The user wants the segment translated into German without any extra explanation. The original sentence is: \"We now have 4-month-old mice that are non-diabetic that used to be diabetic,\" he added.\n\nFirst, I need to make sure I understand the sentence correctly. The key points are: 4-month-old mice, previously diabetic but now non-diabetic, and this is reported by someone (\"he added\").\n\nStarting with \"We now have\" – in German, that's \"Wir haben jetzt\". Then \"4-month-old mice\" would be \"4-monatige Mäuse\". The next part is a bit tricky: \"that are non-diabetic that used to be diabetic\". Here, \"non-diabetic\" in German is \"nicht-diabetisch\" and \"used to be diabetic\" is \"habeen diabetic\" or more naturally \"waren diabetic\" (plural). Since the mice are plural, I should use \"waren diabetic\". \n\nPutting it together: \"diese sind nicht-diabetisch gewesen, die formerly diabetic waren\". Wait, but \"die formerly diabetic\" might not be the smoothest. Alternatively, \"die formerly diabetic sind\" might be clearer. But \"were diabetic\" is better as \"waren diabetic\".\n\nSo the sentence would be: \"Wir haben jetzt 4-monatige Mäuse, die nicht-diabetisch sind, dieformerly diabetic waren, sagte er dazu.\" Hmm, \"dazu\" is appropriate for \"added\" in this context. \n\nWait, \"that used to be diabetic\" is a relative clause modifying the mice. So the structure in German should be: \"Mäuse, die nicht-diabetisch sind, die formerly diabetic waren\". But in German, after the second relative clause, you might need to adjust. Alternatively, combining them: \"Mäuse, dieformerly diabetic, aber jetzt nicht-diabetisch sind\". That might flow better. \n\nSo maybe: \"Wir haben jetzt 4-monatige Mäuse, die formerly diabetic, aber jetzt nicht-diabetisch sind, sagte er dazu.\" That sounds more natural. The original uses \"that used to be\", so \"formerly diabetic\" is a good translation here. \n\nLet me check if \"formerly diabetic\" is the standard way. Yes, that's commonly used. \"diabetic\" in German is the same when referring to the condition. \n\nSo final translation: \"Wir haben jetzt 4-monatige Mäuse, die formerly diabetic, aber jetzt nicht-diabetisch sind, sagte er dazu.\" \n\nWait, but in the original, it's \"that are non-diabetic that used to be diabetic\". So both clauses are attributive. In German, using two relative clauses is acceptable. Alternatively, \"Mäuse, dieformerly diabetic sind, aber jetzt nicht-diabetisch sind\". \n\nAnother way: \"Mäuse, dieformerly diabetic, sind jetzt nicht-diabetisch\". But the original structure is \"are non-diabetic that used to be\". So the active form might be better. \n\nAlternatively, \"Mäuse, die formerly diabetic waren, sind jetzt nicht-diabetisch\". \n\nBut in German, the past tense is not always necessary. However, to make it clear that they were diabetic in the past, \"waren diabetic\" is better. \n\nSo \"Mäuse, dieformerly diabetic waren, sind jetzt nicht-diabetisch\". \n\nPutting it all together: \"Wir haben jetzt 4-monatige Mäuse, die formerly diabetic waren, sind jetzt nicht-diabetisch, sagte er dazu.\" \n\nWait, \"sind jetzt nicht-diabetisch\" is redundant with \"sein\" and \"jetzt\". Maybe \"sind nicht-diabetisch geworden\" would be better: \"sind jetzt nicht-diabetisch geworden\". But the original doesn't specify the exact time, just that they used to be diabetic. \n\nAlternatively, \"haben jetzt 4-monatige Mäuse, dieformerly diabetic, aber jetzt nicht-diabetisch sind\". \n\nYes, \"aber jetzt\" contrasts the past and present. \n\nSo the final translation should be: \"Wir haben jetzt 4-monatige Mäuse, dieformerly diabetic, aber jetzt nicht-diabetisch sind, sagte er dazu.\" \n\nChecking for any errors. \"Además\" is not in the original, so no need. The name \"he\" is translated as \"er\" or \"er sagte dazu\". \"Dazu\" is correct for \"added\". \n\nYes, this should be accurate and natural in German.\n","type":"summary_text"}],"type":"reasoning","encrypted_content":null},{"id":"msg_ba81286209e544dc85f5f257d7c12f34","content":[{"annotations":[],"text":"\n\n\"Wir haben jetzt 4-monatige Mäuse, die formerly diabetic, aber jetzt nicht-diabetisch sind, sagte er dazu.\"\n","type":"output_text","logprobs":null}],"role":"assistant","status":"completed","type":"message"}],"parallel_tool_calls":true,"temperature":null,"tool_choice":"auto","tools":[],"top_p":null,"background":null,"conversation":null,"max_output_tokens":2048,"max_tool_calls":null,"previous_response_id":null,"prompt":null,"prompt_cache_key":null,"reasoning":null,"safety_identifier":null,"service_tier":null,"status":null,"text":null,"top_logprobs":null,"truncation":null,"usage":{"input_tokens":45,"input_tokens_details":{"cached_tokens":0},"output_tokens":991,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":1036},"user":null},"reward":0.3946063684687171,"text":"\"We now have 4-month-old mice that are non-diabetic that used to be diabetic,\" he added.","translation":"„Wir haben jetzt 4 Monate alte Mäuse, die Diabetes hatten und jetzt keinen mehr haben“, fügte er hinzu.","source_language":"eng_Latn","target_language":"deu_Latn","source_lang_name":"English","target_lang_name":"German","generation":"\"Wir haben jetzt 4-monatige Mäuse, die formerly diabetic, aber jetzt nicht-diabetisch sind, sagte er dazu.\"","comet_score":null,"_ng_task_index":0,"_ng_rollout_index":0,"agent_ref":{"type":"responses_api_agents","name":"flores200_wmt_translation_simple_agent"},"sentence_chrf":39.46063684687171,"sentence_spbleu":16.073034972341446,"language_consistency_score":0.99893422729493753} +{"responses_create_params":{"background":null,"include":null,"input":[{"content":"Translate the following segment into German, without additional explanation.\n\nDr. Ehud Ur, professor of medicine at Dalhousie University in Halifax, Nova Scotia and chair of the clinical and scientific division of the Canadian Diabetes Association cautioned that the research is still in its early days.","role":"user","type":"message"}],"instructions":null,"max_output_tokens":2048,"max_tool_calls":null,"metadata":null,"model":null,"parallel_tool_calls":true,"previous_response_id":null,"prompt":null,"reasoning":null,"service_tier":null,"store":null,"temperature":null,"text":null,"tool_choice":"auto","tools":[],"top_logprobs":null,"top_p":null,"truncation":null,"user":null,"stream":null},"response":{"id":"resp_aba3f55756314e17a41bf40171eff413","created_at":1777660822.0,"error":null,"incomplete_details":null,"instructions":null,"metadata":null,"model":"nvidia/nvidia-nemotron-nano-9b-v2","object":"response","output":[{"id":"rs_8cdbc07fd2324ee3ac2e62727c0853a3","summary":[{"text":"Okay, I need to translate the given English sentence into German. Let me start by reading the original text carefully to understand the context and key terms.\n\nThe sentence is: \"Dr. Ehud Ur, professor of medicine at Dalhousie University in Halifax, Nova Scotia and chair of the clinical and scientific division of the Canadian Diabetes Association cautioned that the research is still in its early days.\"\n\nFirst, I'll identify the proper nouns and their German equivalents. \"Dr. Ehud Ur\" – names usually stay the same in German unless there's a specific adaptation. So it remains \"Dr. Ehud Ur.\"\n\nNext, \"professor of medicine at Dalhousie University in Halifax, Nova Scotia.\" The university name \"Dalhousie University\" would typically be kept in all caps as in English, but sometimes in German it might be written in lowercase. However, since it's a proper noun, I'll keep it as \"Dalhousie University.\" \"Halifax, Nova Scotia\" – Halifax is a city in Nova Scotia, Canada. In German, the structure is \"Halifax, Neuschottland.\" But sometimes \"Neuschottland\" might be replaced with \"Kanada\" to specify the country, but since it's part of the location, \"Halifax, Neuschottland\" is correct.\n\nThen, \"chair of the clinical and scientific division of the Canadian Diabetes Association.\" \"Chair\" here is a title, so in German, it would be \"Präsident\" or \"Präsidentin,\" but \"chair\" in an organizational context is often translated as \"Präsident.\" However, sometimes \" Vorsitzenden\" is used if it's a board position. Wait, \"chair\" as a title might be \"Präsident\" here. \"Clinical and scientific division\" – \"klinische und wissenschaftliche Abteilung\" or \"Stelle.\" \"Canadian Diabetes Association\" is a proper noun, so \"Canadian Diabetes Association\" in German is usually kept as is, but sometimes translated to \"Kanadische Diabetes-Assoziation.\" However, since it's an official name, it's better to keep it in the original: \"Canadian Diabetes Association.\"\n\nNow, the verb \"cautioned\" – in German, \"warnt\" or \"+deutet an\" might work. \"Cautioned that\" would be \"warnte, dass\" or \"+deutete an, dass.\" \n\n\"The research is still in its early days\" – \"die Forschung ist noch in ihren frühen Tagen\" or \"noch in den Anfängen.\" \"Early days\" can be \"Anfängen\" or \"frühen Tagen.\"\n\nPutting it all together, let me check each part:\n\n- \"professor of medicine\" – \"Lehrstuhlhalter für Medizin\" or \"Professor für Medizin.\" Since he's a professor, \"Professor für Medizin an der Dalhousie University in Halifax, Neuschottland\" is better.\n\n- \"chair of the clinical and scientific division\" – \"Präsident des klinischen und wissenschaftlichen Bereichs\" or \"Vorsitzende der klinischen und wissenschaftlichen Abteilung.\" Since it's a division, \"Abteilung\" is more appropriate than \"Bereich.\"\n\n- \"cautioned that\" – \"warnte, dass\" or \"+deutete an, dass.\" \"Deuten auf\" is more like \"indicated,\" but \"warnte\" is a direct translation of \"cautioned.\"\n\n- \"research is still in its early days\" – \"die Forschung ist noch in ihren Anfängen\" or \"noch in den Anfängen.\"\n\nSo, putting it all together:\n\n\"Dr. Ehud Ur, Professor für Medizin an der Dalhousie University in Halifax, Neuschottland und Präsident der klinischen und wissenschaftlichen Abteilung der Canadian Diabetes Association warnte, dass die Forschung noch in ihren Anfängen ist.\"\n\nWait, \"chair of the division\" – is \"Präsident der Abteilung\" correct? Sometimes \"Präsident\" might refer to the head of an organization, not a division. Maybe \"Vorsitzender der Abteilung\" would be more accurate if it's a division's chair. But \"chair\" as a position in an association's division might be \"Präsident.\" Alternatively, \"Präsident des klinischen und wissenschaftlichen Bereichs der Canadian Diabetes Association.\" \n\nAlternatively, \"Chair\" could be translated as \"Präsident\" here. Let me check standard translations. For example, the American Diabetes Association has a president, so in German, \"Präsident\" is appropriate.\n\nSo final translation:\n\n\"Dr. Ehud Ur, Professor für Medizin an der Dalhousie University in Halifax, Neuschottland und Präsident der klinischen und wissenschaftlichen Abteilung der Canadian Diabetes Association warnte, dass die Forschung noch in ihren Anfängen ist.\"\n\nI should check if \" Михаил\" is needed, but since it's a proper name, no. Also, \"Michaiл\" is Cyrillic, but the name here is Ehud Ur, so no issues.\n\nDouble-checking prepositions and articles. \"An der Dalhousie University\" is correct. \"Präsident der Abteilung\" – yes. \"Warnte, dass\" – correct. \"Anfängen\" is the right term for \"early days\" in this context.\n\nAlternatively, \"noch in den Anfängen\" without \"seiner\" might be better. \"Die Forschung ist noch in den Anfängen.\" Since \"Anfängen\" can be plural, \"die Forschung\" is singular, so \"in den Anfängen\" is correct.\n\nYes, that seems accurate.\n","type":"summary_text"}],"type":"reasoning","encrypted_content":null},{"id":"msg_31f943d913e249928ea71fd7be452373","content":[{"annotations":[],"text":"\n\nDr. Ehud Ur, Professor für Medizin an der Dalhousie University in Halifax, Neuschottland und Präsident der klinischen und wissenschaftlichen Abteilung der Canadian Diabetes Association warnte, dass die Forschung noch in den Anfängen ist.\n","type":"output_text","logprobs":null}],"role":"assistant","status":"completed","type":"message"}],"parallel_tool_calls":true,"temperature":null,"tool_choice":"auto","tools":[],"top_p":null,"background":null,"conversation":null,"max_output_tokens":2048,"max_tool_calls":null,"previous_response_id":null,"prompt":null,"prompt_cache_key":null,"reasoning":null,"safety_identifier":null,"service_tier":null,"status":null,"text":null,"top_logprobs":null,"truncation":null,"usage":{"input_tokens":68,"input_tokens_details":{"cached_tokens":0},"output_tokens":1198,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":1266},"user":null},"reward":0.5863888666289676,"text":"Dr. Ehud Ur, professor of medicine at Dalhousie University in Halifax, Nova Scotia and chair of the clinical and scientific division of the Canadian Diabetes Association cautioned that the research is still in its early days.","translation":"Dr. Ehud Ur, Professor für Medizin an der Dalhousie University in Halifax, Nova Scotia, und Vorsitzender der Abteilung für Klinik und Wissenschaft des Kanadischen Diabetesverbands gab zu bedenken, dass die Forschungsarbeit noch in den Kinderschuhen stecke.","source_language":"eng_Latn","target_language":"deu_Latn","source_lang_name":"English","target_lang_name":"German","generation":"Dr. Ehud Ur, Professor für Medizin an der Dalhousie University in Halifax, Neuschottland und Präsident der klinischen und wissenschaftlichen Abteilung der Canadian Diabetes Association warnte, dass die Forschung noch in den Anfängen ist.","comet_score":null,"_ng_task_index":1,"_ng_rollout_index":0,"agent_ref":{"type":"responses_api_agents","name":"flores200_wmt_translation_simple_agent"},"sentence_chrf":58.63888666289676,"sentence_spbleu":36.80247113063898,"language_consistency_score":0.98418651173189287} +{"responses_create_params":{"background":null,"include":null,"input":[{"content":"Translate the following segment into German, without additional explanation.\n\nLike some other experts, he is skeptical about whether diabetes can be cured, noting that these findings have no relevance to people who already have Type 1 diabetes.","role":"user","type":"message"}],"instructions":null,"max_output_tokens":2048,"max_tool_calls":null,"metadata":null,"model":null,"parallel_tool_calls":true,"previous_response_id":null,"prompt":null,"reasoning":null,"service_tier":null,"store":null,"temperature":null,"text":null,"tool_choice":"auto","tools":[],"top_logprobs":null,"top_p":null,"truncation":null,"user":null,"stream":null},"response":{"id":"resp_d584ceec15f44ef2ad3d9ccf4f47c698","created_at":1777660835.0,"error":null,"incomplete_details":null,"instructions":null,"metadata":null,"model":"nvidia/nvidia-nemotron-nano-9b-v2","object":"response","output":[{"id":"rs_b5746be784a84d6190db6802b7ebcaae","summary":[{"text":"Okay, I need to translate this sentence into German. Let me start by reading the original text carefully.\n\nThe sentence is: \"Like some other experts, he is skeptical about whether diabetes can be cured, noting that these findings have no relevance to people who already have Type 1 diabetes.\"\n\nFirst, I'll break it down. The first part is \"Like some other experts, he is skeptical about whether diabetes can be cured.\" The key here is to convey the comparison to other experts and his skepticism.\n\n\"Like some other experts\" – in German, \"wie einige andere Experten\" is a direct translation. \"He is skeptical\" would be \"er ist skeptisch\". The question \"whether diabetes can be cured\" is \"ob Diabetes geheilt werden kann\".\n\nSo the first part could be: \"Wie einige andere Experten ist er skeptisch, ob Diabetes geheilt werden kann.\"\n\nNext part: \"noting that these findings have no relevance to people who already have Type 1 diabetes.\" \"Noting\" here is \"mit der Hinweis\" or \"quiry\" but \"mit der Hinweis\" might be more natural. \"These findings\" is \"diese Ergebnisse\". \"Have no relevance\" is \"-linking zu\" or \"relevant\" – \"haben keine Relevanz zu\". \n\n\"People who already have Type 1 diabetes\" is \"Menschen, die bereits an Typ-1-Diabetes leiden\".\n\nPutting it together: \"mit der Hinweis, dass diese Ergebnisse für diejenigen nicht relevant sind, die bereits an Typ-1-Diabetes leiden.\"\n\nWait, maybe \"mit der Hinweis\" is a bit off. In German, \"mit der Bemerkung\" or \"mit der Hinweis\" – but \"noting\" here is more like \"pointing out\" or \"noting\" as in making a remark. So \"mit der Bemerkung\" might be better. Alternatively, \"da hinweisend\" but that's more like \"while noting\". \n\nAlternatively, maybe \"und betont, dass diese Ergebnisse für diejenigen nicht relevant sind...\" but the original says \"noting\", which is more like a remark. So \"mit der Bemerkung\" is appropriate.\n\nSo the full translation would be: \"Wie einige andere Experten ist er skeptisch, ob Diabetes geheilt werden kann, mit der Bemerkung, dass diese Ergebnisse für diejenigen nicht relevant sind, die bereits an Typ-1-Diabetes leiden.\"\n\nLet me check the grammar. \"Mit der Bemerkung\" is correct. \"Für diejenigen nicht relevant\" – yes. \"Verlieran\" is correct for \"haben\". \n\nAnother thing: \"Type 1 diabetes\" in German is \"Typ-1-Diabetes\". The hyphen is correct. \n\nI think that's accurate. The structure is a bit long, but that's how the original is. No additional explanations needed, just the translation.\n","type":"summary_text"}],"type":"reasoning","encrypted_content":null},{"id":"msg_2921452710b34f60971f47591a19b7f8","content":[{"annotations":[],"text":"\n\nWie einige andere Experten ist er skeptisch, ob Diabetes geheilt werden kann, mit der Bemerkung, dass diese Ergebnisse für diejenigen nicht relevant sind, die bereits an Typ-1-Diabetes leiden.\n","type":"output_text","logprobs":null}],"role":"assistant","status":"completed","type":"message"}],"parallel_tool_calls":true,"temperature":null,"tool_choice":"auto","tools":[],"top_p":null,"background":null,"conversation":null,"max_output_tokens":2048,"max_tool_calls":null,"previous_response_id":null,"prompt":null,"prompt_cache_key":null,"reasoning":null,"safety_identifier":null,"service_tier":null,"status":null,"text":null,"top_logprobs":null,"truncation":null,"usage":{"input_tokens":55,"input_tokens_details":{"cached_tokens":0},"output_tokens":651,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":706},"user":null},"reward":0.4755332744530898,"text":"Like some other experts, he is skeptical about whether diabetes can be cured, noting that these findings have no relevance to people who already have Type 1 diabetes.","translation":"Wie einige andere Experten zeigte er sich skeptisch, ob es möglich sei, Diabetes zu heilen, und wies darauf hin, dass die Befunde für Menschen, die bereits unter Typ-1-Diabetes litten, keine Bedeutung hätten.","source_language":"eng_Latn","target_language":"deu_Latn","source_lang_name":"English","target_lang_name":"German","generation":"Wie einige andere Experten ist er skeptisch, ob Diabetes geheilt werden kann, mit der Bemerkung, dass diese Ergebnisse für diejenigen nicht relevant sind, die bereits an Typ-1-Diabetes leiden.","comet_score":null,"_ng_task_index":2,"_ng_rollout_index":0,"agent_ref":{"type":"responses_api_agents","name":"flores200_wmt_translation_simple_agent"},"sentence_chrf":47.55332744530898,"sentence_spbleu":23.346481873874154,"language_consistency_score":0.99730848182932608} +{"responses_create_params":{"background":null,"include":null,"input":[{"content":"Translate the following segment into German, without additional explanation.\n\nDanius said, \"Right now we are doing nothing. I have called and sent emails to his closest collaborator and received very friendly replies. For now, that is certainly enough.\"","role":"user","type":"message"}],"instructions":null,"max_output_tokens":2048,"max_tool_calls":null,"metadata":null,"model":null,"parallel_tool_calls":true,"previous_response_id":null,"prompt":null,"reasoning":null,"service_tier":null,"store":null,"temperature":null,"text":null,"tool_choice":"auto","tools":[],"top_logprobs":null,"top_p":null,"truncation":null,"user":null,"stream":null},"response":{"id":"resp_1f9263ff3586420aa0a51da26fbf1558","created_at":1777660837.0,"error":null,"incomplete_details":null,"instructions":null,"metadata":null,"model":"nvidia/nvidia-nemotron-nano-9b-v2","object":"response","output":[{"id":"rs_507d7e4ba6ce45e8a52e289592334e52","summary":[{"text":"Okay, I need to translate this segment into German without any additional explanation. Let me start by reading the original text carefully.\n\n\"Danius said, 'Right now we are doing nothing. I have called and sent emails to his closest collaborator and received very friendly replies. For now, that is certainly enough.'\" \n\nFirst, I'll break it down sentence by sentence. \n\n\"Danius said,\" – In German, that's \"Danius sagte,\".\n\n\"Right now we are doing nothing.\" – The phrase \"Right now\" can be \"Jetzt\" or \"In diesem Moment\". \"We are doing nothing\" is \"machen nichts\" or \"etwas tun\". So maybe \"Jetzt machen wir nichts.\"\n\nNext sentence: \"I have called and sent emails to his closest collaborator...\" \n\n\"I have called\" – \"Ich habe angerufen\" or \"Ich habe angerufen hat\". But \"angerrufen\" is correct. \n\n\"sent emails\" – \"E-Mails absend\" or \"E-Mails gesendet\". \"Gesendet\" is more common. \n\n\"to his closest collaborator\" – \"zu seinem nächsten Kollaborateur\" or \"zusammenarbeitspartner\". \"Kollaborateur\" is a bit formal, but it's acceptable. \n\n\"and received very friendly replies.\" – \"und habe sehr freundliche Rückmeldungen erhalten.\" Or \"sehr freundliche Antworten erhalten\". \"Rückmeldungen\" might be better here for replies. \n\n\"For now, that is certainly enough.\" – \"Für den Moment ist das sicherlich ausreichend.\" Or \"Das ist für den Moment definitiv genug.\" But \"sicherlich\" is more natural here. \n\nPutting it all together:\n\n\"Danius sagte: 'Jetzt machen wir nichts. Ich habe angerufen und E-Mails an seinen nächsten Kollaborateur geschickt und habe sehr freundliche Rückmeldungen erhalten. Für den Moment ist das sicherlich ausreichend.'\"\n\nI should check for any grammatical errors. \"Gegeben\" is correct for \"sent\". \"Nächsten Kollaborateur\" – \"nächster\" is correct. \"Rückmeldungen\" is plural, so \"angefangen\" is correct. \n\nAlternatively, \"I have called and sent emails\" could be \"Ich habe angerufen und E-Mails gesendet\". But \"geschickt\" is also correct. \n\nAnother point: \"closest collaborator\" – \"nächster Kollaborateur\" or \"engster Unterstützer\"? \"Kollaborateur\" is more about collaboration, which fits. \n\n\"Received very friendly replies\" – \"sehr freundliche Antworten\" or \"freundliche Antworten\". \"Rückmeldungen\" is more about feedback, but in this context, \"Antworten\" might be better. \n\nSo maybe: \"und habe sehr freundliche Antworten erhalten.\"\n\nFinal check: The structure is correct. The quotes are properly opened and closed. The verbs are in the right tense. \n\nI think that's a solid translation. Let me make sure there are no typos or spelling mistakes. \"Kollaborateur\" is spelled correctly. \"Geschickt\" or \"gesendet\" – both are acceptable. \n\nYes, this should be the correct translation.\n","type":"summary_text"}],"type":"reasoning","encrypted_content":null},{"id":"msg_f0996f1f2b1b4e62818710bee1e18e59","content":[{"annotations":[],"text":"\n\nDanius sagte: \"Jetzt machen wir nichts. Ich habe angerufen und E-Mails an seinen nächsten Kollaborateur geschickt und habe sehr freundliche Rückmeldungen erhalten. Für den Moment ist das sicherlich ausreichend.\"\n","type":"output_text","logprobs":null}],"role":"assistant","status":"completed","type":"message"}],"parallel_tool_calls":true,"temperature":null,"tool_choice":"auto","tools":[],"top_p":null,"background":null,"conversation":null,"max_output_tokens":2048,"max_tool_calls":null,"previous_response_id":null,"prompt":null,"prompt_cache_key":null,"reasoning":null,"safety_identifier":null,"service_tier":null,"status":null,"text":null,"top_logprobs":null,"truncation":null,"usage":{"input_tokens":60,"input_tokens_details":{"cached_tokens":0},"output_tokens":744,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":804},"user":null},"reward":0.5540812850260183,"text":"Danius said, \"Right now we are doing nothing. I have called and sent emails to his closest collaborator and received very friendly replies. For now, that is certainly enough.\"","translation":"Danius sagte: „Im Moment unternehmen wir nichts. Ich habe seinen engsten Mitarbeiter angerufen und ihm E-Mails geschickt und sehr freundliche Antworten erhalten. Das ist vorläufig genug.“","source_language":"eng_Latn","target_language":"deu_Latn","source_lang_name":"English","target_lang_name":"German","generation":"Danius sagte: \"Jetzt machen wir nichts. Ich habe angerufen und E-Mails an seinen nächsten Kollaborateur geschickt und habe sehr freundliche Rückmeldungen erhalten. Für den Moment ist das sicherlich ausreichend.\"","comet_score":null,"_ng_task_index":4,"_ng_rollout_index":0,"agent_ref":{"type":"responses_api_agents","name":"flores200_wmt_translation_simple_agent"},"sentence_chrf":55.408128502601826,"sentence_spbleu":28.191419038905007,"language_consistency_score":0.99985034931825678} +{"responses_create_params":{"background":null,"include":null,"input":[{"content":"Translate the following segment into German, without additional explanation.\n\nOn Monday, Sara Danius, permanent secretary of the Nobel Committee for Literature at the Swedish Academy, publicly announced during a radio program on Sveriges Radio in Sweden the committee, unable to reach Bob Dylan directly about winning the 2016 Nobel Prize in Literature, had abandoned its efforts to reach him.","role":"user","type":"message"}],"instructions":null,"max_output_tokens":2048,"max_tool_calls":null,"metadata":null,"model":null,"parallel_tool_calls":true,"previous_response_id":null,"prompt":null,"reasoning":null,"service_tier":null,"store":null,"temperature":null,"text":null,"tool_choice":"auto","tools":[],"top_logprobs":null,"top_p":null,"truncation":null,"user":null,"stream":null},"response":{"id":"resp_341824482a2d4f1fb401790964600984","created_at":1777660854.0,"error":null,"incomplete_details":null,"instructions":null,"metadata":null,"model":"nvidia/nvidia-nemotron-nano-9b-v2","object":"response","output":[{"id":"rs_bacaa51f597a4ce1bbf7a9ef03fec4e1","summary":[{"text":"Okay, let's tackle this translation. First, I need to make sure I understand the original sentence correctly. The key points are: Sara Danius is the permanent secretary of the Nobel Committee for Literature. She announced on Monday in a radio program on Sveriges Radio that the committee couldn't contact Bob Dylan directly about winning the 2016 Nobel Prize in Literature, so they stopped trying.\n\nNow, translating into German. Let's start with the names and titles. \"Sara Danius\" stays the same. \"Permanent secretary of the Nobel Committee for Literature at the Swedish Academy\" – in German, \"Permanente Sekretärin des Nobel-Komitees für Literatur am Schwedischen Academia\". I should check if \"Academy\" is capitalized here. Yes, since it's a specific institution.\n\nNext, \"publicly announced during a radio program on Sveriges Radio in Sweden\". \"Öffentlich bekannt gegeben während eines Rundfunkprogramms auf Sveriges Radio in Schweden\". \"Sveriges Radio\" is the correct name, so no change there.\n\nThe main clause: \"the committee, unable to reach Bob Dylan directly about winning the 2016 Nobel Prize in Literature, had abandoned its efforts to reach him.\" \n\n\"Die Kommission, die Bob Dylan nicht direkt zu einer Nominierung als Gewinner des Nobelpreises für Literatur 2016 erreichen konnte, hatte ihre Versuche, ihn zu kontaktieren, aufgegeben.\"\n\nBreaking it down: \"unable to reach\" is \"nicht direkt erreichen konnten\". \"About winning\" – \"zu einer Nominierung als Gewinner\". Wait, \"winning\" here is the context of the prize, so \"Nominierung\" might be more accurate than \"Sieg\" since the Nobel is awarded after nomination. \"2016\" stays the same. \"Had abandoned efforts to reach him\" – \"ihre Versuche, ihn zu kontaktieren, aufgegeben.\"\n\nPutting it all together. Let me check for any possible errors. \"Nobel-Komitee\" is correct. \"Gewinner\" is okay, but sometimes \"Preisträger\" is used. However, \"Gewinner\" is commonly used for Nobel winners. \"Nominierung\" vs \"Gewinn\" – since the prize is awarded, \"Gewinn\" might be better. Wait, the original says \"about winning\", so maybe \"zum Gewinn des Preises\". But the structure in German might be \"zu einem Gewinn des Nobelpreises...\". Alternatively, \"zur Verleihung\" could be another option. Hmm. The original is about reaching Dylan regarding the prize win. So maybe \"zum Gewinn des Nobelpreises für Literatur 2016\". \n\nWait, the original says \"about winning the 2016 Nobel Prize in Literature\". So the committee was trying to reach Dylan about his winning the prize. So \"zu einem Gewinn des Nobelpreises für Literatur 2016\". That makes sense. So adjusting that part: \"die Bob Dylan nicht direkt zu einem Gewinn des Nobelpreises für Literatur 2016 erreichen konnte\".\n\nAlso, \"had abandoned its efforts to reach him\" – \"war ihre Ansprache zu Dylan aufgegeben\". Or \"ihre Versuche, ihn zu kontaktieren, aufgaben\". \"Versuche\" is plural, so \"ihre Versuche, ihn zu kontaktieren, hatten aufgegeben\". \n\nPutting it all together:\n\nAm Montag bekanntgab während eines Rundfunkprogramms auf Sveriges Radio in Schweden die Permane Sekretärin des Nobel-Komitees für Literatur am Schwedischen Academia, Sara Danius, dass die Kommission, die Bob Dylan nicht direkt zu einem Gewinn des Nobelpreises für Literatur 2016 erreichen konnte, ihre Versuche, ihn zu kontaktieren, aufgab.\n\nWait, \"had abandoned\" is \"had given up\", so \"hatten aufgeben\" in past perfect. Since the main verb is \"kennenshaftgab\" (Berlinte), which is in the past, the past perfect is correct. \n\nAnother point: \"publicly announced\" is \"bekanntgab\". Correct. \"Publicly\" is implied by \"publicly announced\", so no need for \"öffentlich\" unless specified. \n\nDouble-checking all names and titles. \"Schwedischen Academia\" – \"im Schwedischen Academia\" might be better, but \"Schwedischen Academia\" is acceptable as the academy is a proper noun. \n\nYes, this seems accurate. Let me read it once more to ensure flow and correctness.\n","type":"summary_text"}],"type":"reasoning","encrypted_content":null},{"id":"msg_2c59e9d45f8e44acb4b4963b263f7d15","content":[{"annotations":[],"text":"\n\nAm Montag bekanntgab während eines Rundfunkprogramms auf Sveriges Radio in Schweden die Permane Sekretärin des Nobel-Komitees für Literatur am Schwedischen Academia, Sara Danius, dass die Kommission, die Bob Dylan nicht direkt zu einem Gewinn des Nobelpreises für Literatur 2016 erreichen konnte, ihre Versuche, ihn zu kontaktieren, aufgab.\n","type":"output_text","logprobs":null}],"role":"assistant","status":"completed","type":"message"}],"parallel_tool_calls":true,"temperature":null,"tool_choice":"auto","tools":[],"top_p":null,"background":null,"conversation":null,"max_output_tokens":2048,"max_tool_calls":null,"previous_response_id":null,"prompt":null,"prompt_cache_key":null,"reasoning":null,"safety_identifier":null,"service_tier":null,"status":null,"text":null,"top_logprobs":null,"truncation":null,"usage":{"input_tokens":84,"input_tokens_details":{"cached_tokens":0},"output_tokens":1015,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":1099},"user":null},"reward":0.479294678361032,"text":"On Monday, Sara Danius, permanent secretary of the Nobel Committee for Literature at the Swedish Academy, publicly announced during a radio program on Sveriges Radio in Sweden the committee, unable to reach Bob Dylan directly about winning the 2016 Nobel Prize in Literature, had abandoned its efforts to reach him.","translation":"Am Montag gab Sara Danius, Ständige Sekretärin des Literaturnobelpreiskomitees der Schwedischen Akademie in einer Radiosendung auf Sveriges Radio in Schweden bekannt, dass das Komitee seine Bemühungen eingestellt habe, Bob Dylan über seine Ehrung mit dem Literaturnobelpreis für Literator zu informieren, nachdem es nicht gelungen war, ihn persönlich zu erreichen.","source_language":"eng_Latn","target_language":"deu_Latn","source_lang_name":"English","target_lang_name":"German","generation":"Am Montag bekanntgab während eines Rundfunkprogramms auf Sveriges Radio in Schweden die Permane Sekretärin des Nobel-Komitees für Literatur am Schwedischen Academia, Sara Danius, dass die Kommission, die Bob Dylan nicht direkt zu einem Gewinn des Nobelpreises für Literatur 2016 erreichen konnte, ihre Versuche, ihn zu kontaktieren, aufgab.","comet_score":null,"_ng_task_index":3,"_ng_rollout_index":0,"agent_ref":{"type":"responses_api_agents","name":"flores200_wmt_translation_simple_agent"},"sentence_chrf":47.9294678361032,"sentence_spbleu":17.161140484297107,"language_consistency_score":0.9941285573866423} diff --git a/benchmarks/flores200/data/flores_devtest_languages.json b/benchmarks/flores200/data/flores_devtest_languages.json new file mode 100644 index 00000000000..ff9df71a83f --- /dev/null +++ b/benchmarks/flores200/data/flores_devtest_languages.json @@ -0,0 +1,222 @@ +[ + "ace_Arab", + "ace_Latn", + "acm_Arab", + "acq_Arab", + "aeb_Arab", + "afr_Latn", + "als_Latn", + "amh_Ethi", + "apc_Arab_nort3139", + "apc_Arab_sout3123", + "apd_Arab", + "arb_Arab", + "arb_Latn", + "arg_Latn", + "ars_Arab", + "ary_Arab", + "arz_Arab", + "asm_Beng", + "ast_Latn", + "awa_Deva", + "ayr_Latn", + "azb_Arab", + "azj_Latn", + "bak_Cyrl", + "bam_Latn", + "ban_Latn", + "bel_Cyrl", + "bem_Latn", + "ben_Beng", + "bho_Deva", + "bjn_Arab", + "bjn_Latn", + "bod_Tibt", + "bos_Latn", + "bug_Latn", + "bul_Cyrl", + "cat_Latn", + "cat_Latn_vale1252", + "ceb_Latn", + "ces_Latn", + "chv_Cyrl", + "cjk_Latn", + "ckb_Arab", + "cmn_Hans", + "cmn_Hant", + "crh_Latn", + "cym_Latn", + "dan_Latn", + "deu_Latn", + "dik_Latn", + "dyu_Latn", + "dzo_Tibt", + "ekk_Latn", + "ell_Grek", + "eng_Latn", + "epo_Latn", + "eus_Latn", + "ewe_Latn", + "fao_Latn", + "fij_Latn", + "fil_Latn", + "fin_Latn", + "fon_Latn", + "fra_Latn", + "fur_Latn", + "fuv_Latn", + "gaz_Latn", + "gla_Latn", + "gle_Latn", + "glg_Latn", + "gug_Latn", + "guj_Gujr", + "hat_Latn", + "hau_Latn", + "heb_Hebr", + "hin_Deva", + "hne_Deva", + "hrv_Latn", + "hun_Latn", + "hye_Armn", + "ibo_Latn", + "ilo_Latn", + "ind_Latn", + "isl_Latn", + "ita_Latn", + "jav_Latn", + "jpn_Jpan", + "kaa_Latn", + "kab_Latn", + "kac_Latn", + "kam_Latn", + "kan_Knda", + "kas_Arab", + "kas_Deva", + "kat_Geor", + "kaz_Cyrl", + "kbp_Latn", + "kea_Latn", + "khk_Cyrl", + "khk_Mong", + "khm_Khmr", + "kik_Latn", + "kin_Latn", + "kir_Cyrl", + "kjh_Cyrl", + "kmb_Latn", + "kmr_Latn", + "knc_Arab", + "knc_Latn", + "kor_Hang", + "ktu_Latn", + "lao_Laoo", + "lij_Latn", + "lim_Latn", + "lin_Latn", + "lit_Latn", + "lld_Latn", + "lmo_Latn", + "ltg_Latn", + "ltz_Latn", + "lua_Latn", + "lug_Latn", + "luo_Latn", + "lus_Latn", + "lvs_Latn", + "mag_Deva", + "mai_Deva", + "mal_Mlym", + "mar_Deva", + "mfe_Latn", + "mhr_Cyrl", + "min_Arab", + "min_Latn", + "mkd_Cyrl", + "mlt_Latn", + "mni_Beng", + "mos_Latn", + "mri_Latn", + "mya_Mymr", + "myv_Cyrl", + "nld_Latn", + "nno_Latn", + "nob_Latn", + "nob_Latn_radical", + "npi_Deva", + "nqo_Nkoo", + "nso_Latn", + "nus_Latn", + "nya_Latn", + "oci_Latn", + "oci_Latn_aran1260", + "ory_Orya", + "pag_Latn", + "pan_Guru", + "pap_Latn", + "pbt_Arab", + "pes_Arab", + "plt_Latn", + "pol_Latn", + "por_Latn", + "prs_Arab", + "quy_Latn", + "ron_Latn", + "run_Latn", + "rus_Cyrl", + "sag_Latn", + "san_Deva", + "sat_Olck", + "scn_Latn", + "shn_Mymr", + "sin_Sinh", + "slk_Latn", + "slv_Latn", + "smo_Latn", + "sna_Latn", + "snd_Arab", + "som_Latn", + "sot_Latn", + "spa_Latn", + "srd_Latn", + "srp_Cyrl", + "ssw_Latn", + "sun_Latn", + "swe_Latn", + "swh_Latn", + "szl_Latn", + "tam_Taml", + "taq_Latn", + "taq_Tfng", + "tat_Cyrl", + "tel_Telu", + "tgk_Cyrl", + "tha_Thai", + "tir_Ethi", + "tpi_Latn", + "tsn_Latn", + "tso_Latn", + "tuk_Latn", + "tum_Latn", + "tur_Latn", + "twi_Latn_akua1239", + "twi_Latn_asan1239", + "tyv_Cyrl", + "uig_Arab", + "ukr_Cyrl", + "umb_Latn", + "urd_Arab", + "uzn_Latn", + "vec_Latn", + "vie_Latn", + "vmw_Latn", + "war_Latn", + "wol_Latn", + "xho_Latn", + "ydd_Hebr", + "yor_Latn", + "yue_Hant", + "zgh_Tfng", + "zsm_Latn", + "zul_Latn" +] diff --git a/benchmarks/flores200/data/flores_language_names.json b/benchmarks/flores200/data/flores_language_names.json new file mode 100644 index 00000000000..d125dde8492 --- /dev/null +++ b/benchmarks/flores200/data/flores_language_names.json @@ -0,0 +1,232 @@ +{ + "ace_Arab": "Acehnese (Jawi script)", + "ace_Latn": "Acehnese (Latin script)", + "acm_Arab": "Mesopotamian Arabic", + "acq_Arab": "Taʽizzi-Adeni Arabic", + "aeb_Arab": "Tunisian Arabic", + "afr_Latn": "Afrikaans", + "als_Latn": "Albanian (Tosk)", + "amh_Ethi": "Amharic", + "apc_Arab_nort3139": "Levantine Arabic (North)", + "apc_Arab_sout3123": "Levantine Arabic (South)", + "apd_Arab": "Sudanese Arabic", + "arb_Arab": "Modern Standard Arabic", + "arb_Latn": "Modern Standard Arabic (Romanized)", + "arg_Latn": "Aragonese", + "ars_Arab": "Najdi Arabic", + "ary_Arab": "Moroccan Arabic", + "arz_Arab": "Egyptian Arabic", + "asm_Beng": "Assamese", + "ast_Latn": "Asturian", + "awa_Deva": "Awadhi", + "ayr_Latn": "Central Aymara", + "azb_Arab": "South Azerbaijani", + "azj_Latn": "North Azerbaijani", + "bak_Cyrl": "Bashkir", + "bam_Latn": "Bambara", + "ban_Latn": "Balinese", + "bel_Cyrl": "Belarusian", + "bem_Latn": "Bemba", + "ben_Beng": "Bengali", + "bho_Deva": "Bhojpuri", + "bjn_Arab": "Banjar (Jawi script)", + "bjn_Latn": "Banjar (Latin script)", + "bod_Tibt": "Lhasa Tibetan", + "bos_Latn": "Bosnian", + "brx_Deva": "Bodo", + "bug_Latn": "Buginese", + "bul_Cyrl": "Bulgarian", + "cat_Latn": "Catalan", + "cat_Latn_vale1252": "Valencian", + "ceb_Latn": "Cebuano", + "ces_Latn": "Czech", + "chv_Cyrl": "Chuvash", + "cjk_Latn": "Chokwe", + "ckb_Arab": "Central Kurdish", + "cmn_Hans": "Mandarin Chinese (Standard Beijing)", + "cmn_Hant": "Mandarin Chinese (Taiwanese)", + "crh_Latn": "Crimean Tatar", + "cym_Latn": "Welsh", + "dan_Latn": "Danish", + "dar_Cyrl": "Dargwa", + "deu_Latn": "German", + "dgo_Deva": "Dogri", + "dik_Latn": "Southwestern Dinka", + "dyu_Latn": "Dyula", + "dzo_Tibt": "Dzongkha", + "ekk_Latn": "Estonian", + "ell_Grek": "Greek", + "eng_Latn": "English", + "epo_Latn": "Esperanto", + "eus_Latn": "Basque", + "ewe_Latn": "Ewe", + "fao_Latn": "Faroese", + "fij_Latn": "Fijian", + "fil_Latn": "Filipino", + "fin_Latn": "Finnish", + "fon_Latn": "Fon", + "fra_Latn": "French", + "fur_Latn": "Friulian", + "fuv_Latn": "Nigerian Fulfulde", + "gaz_Latn": "West Central Oromo", + "gla_Latn": "Scottish Gaelic", + "gle_Latn": "Irish", + "glg_Latn": "Galician", + "gom_Deva": "Goan Konkani", + "gug_Latn": "Paraguayan Guaraní", + "guj_Gujr": "Gujarati", + "hat_Latn": "Haitian Creole", + "hau_Latn": "Hausa", + "heb_Hebr": "Hebrew", + "hin_Deva": "Hindi", + "hne_Deva": "Chhattisgarhi", + "hrv_Latn": "Croatian", + "hun_Latn": "Hungarian", + "hye_Armn": "Armenian", + "ibo_Latn": "Igbo", + "ilo_Latn": "Ilocano", + "ind_Latn": "Indonesian", + "isl_Latn": "Icelandic", + "ita_Latn": "Italian", + "jav_Latn": "Javanese", + "jpn_Jpan": "Japanese", + "kaa_Latn": "Karakalpak", + "kab_Latn": "Kabyle", + "kac_Latn": "Jingpho", + "kam_Latn": "Kamba", + "kan_Knda": "Kannada", + "kas_Arab": "Kashmiri (Arabic script)", + "kas_Deva": "Kashmiri (Devanagari script)", + "kat_Geor": "Georgian", + "kaz_Cyrl": "Kazakh", + "kbp_Latn": "Kabiyè", + "kea_Latn": "Kabuverdianu", + "khk_Cyrl": "Halh Mongolian (Cyrillic script)", + "khk_Mong": "Halh Mongolian (traditional Mongolian script)", + "khm_Khmr": "Khmer (Central)", + "kik_Latn": "Kikuyu", + "kin_Latn": "Kinyarwanda", + "kir_Cyrl": "Kyrgyz", + "kjh_Cyrl": "Khakas", + "kmb_Latn": "Kimbundu", + "kmr_Latn": "Northern Kurdish", + "knc_Arab": "Central Kanuri (Arabic script)", + "knc_Latn": "Central Kanuri (Latin script)", + "kor_Hang": "Korean", + "ktu_Latn": "Kituba (DRC)", + "lao_Laoo": "Lao", + "lij_Latn": "Ligurian (Genoese)", + "lim_Latn": "Limburgish", + "lin_Latn": "Lingala", + "lit_Latn": "Lithuanian", + "lld_Latn": "Ladin", + "lld_Latn_gard1241": "Ladin (Gherdëina)", + "lmo_Latn": "Lombard", + "ltg_Latn": "Latgalian", + "ltz_Latn": "Luxembourgish", + "lua_Latn": "Luba-Kasai", + "lug_Latn": "Ganda", + "luo_Latn": "Luo", + "lus_Latn": "Mizo", + "lvs_Latn": "Standard Latvian", + "mag_Deva": "Magahi", + "mai_Deva": "Maithili", + "mal_Mlym": "Malayalam", + "mar_Deva": "Marathi", + "mfe_Latn": "Mauritian Creole", + "mhr_Cyrl": "Meadow Mari", + "min_Arab": "Minangkabau (Jawi script)", + "min_Latn": "Minangkabau (Latin script)", + "mkd_Cyrl": "Macedonian", + "mlt_Latn": "Maltese", + "mni_Beng": "Meitei (Manipuri, Bengali script)", + "mni_Mtei": "Meitei (Manipuri, Meitei script)", + "mos_Latn": "Mossi", + "mri_Latn": "Maori", + "mya_Mymr": "Burmese", + "myv_Cyrl": "Erzya", + "nld_Latn": "Dutch", + "nno_Latn": "Norwegian Nynorsk", + "nob_Latn": "Norwegian Bokmål, moderate variety", + "nob_Latn_radical": "Norwegian Bokmål, radical variety", + "npi_Deva": "Nepali", + "nqo_Nkoo": "Nko", + "nso_Latn": "Northern Sotho", + "nus_Latn": "Nuer", + "nya_Latn": "Nyanja", + "oci_Latn": "Occitan", + "oci_Latn_aran1260": "Aranese", + "ory_Orya": "Odia", + "pag_Latn": "Pangasinan", + "pan_Guru": "Eastern Panjabi", + "pap_Latn": "Papiamento", + "pbt_Arab": "Southern Pashto", + "pes_Arab": "Western Persian", + "plt_Latn": "Plateau Malagasy", + "pol_Latn": "Polish", + "por_Latn": "Portuguese (Brazilian)", + "prs_Arab": "Dari", + "quy_Latn": "Ayacucho Quechua", + "ron_Latn": "Romanian", + "run_Latn": "Rundi", + "rus_Cyrl": "Russian", + "sag_Latn": "Sango", + "san_Deva": "Sanskrit", + "sat_Olck": "Santali", + "scn_Latn": "Sicilian", + "shn_Mymr": "Shan", + "sin_Sinh": "Sinhala", + "slk_Latn": "Slovak", + "slv_Latn": "Slovenian", + "smo_Latn": "Samoan", + "sna_Latn": "Shona", + "snd_Arab": "Sindhi (Arabic script)", + "snd_Deva": "Sindhi (Devanagari script)", + "som_Latn": "Somali", + "sot_Latn": "Southern Sotho", + "spa_Latn": "Spanish (Latin American)", + "srd_Latn": "Sardinian", + "srp_Cyrl": "Serbian", + "ssw_Latn": "Swati", + "sun_Latn": "Sundanese", + "swe_Latn": "Swedish", + "swh_Latn": "Swahili", + "szl_Latn": "Silesian", + "tam_Taml": "Tamil", + "taq_Latn": "Tamasheq (Latin script)", + "taq_Tfng": "Tamasheq (Tifinagh script)", + "tat_Cyrl": "Tatar", + "tel_Telu": "Telugu", + "tgk_Cyrl": "Tajik", + "tha_Thai": "Thai", + "tir_Ethi": "Tigrinya", + "tpi_Latn": "Tok Pisin", + "tsn_Latn": "Tswana", + "tso_Latn": "Tsonga", + "tuk_Latn": "Turkmen", + "tum_Latn": "Tumbuka", + "tur_Latn": "Turkish", + "twi_Latn_akua1239": "Akuapem Twi", + "twi_Latn_asan1239": "Asante Twi", + "tyv_Cyrl": "Tuvan", + "udm_Cyrl": "Udmurt", + "uig_Arab": "Uyghur", + "ukr_Cyrl": "Ukrainian", + "umb_Latn": "Umbundu", + "urd_Arab": "Urdu", + "uzn_Latn": "Northern Uzbek", + "uzs_Arab": "Southern Uzbek", + "vec_Latn": "Venetian", + "vie_Latn": "Vietnamese", + "vmw_Latn": "Emakhuwa (Central)", + "war_Latn": "Waray", + "wol_Latn": "Wolof", + "wuu_Hans": "Wu Chinese", + "xho_Latn": "Xhosa", + "ydd_Hebr": "Eastern Yiddish", + "yor_Latn": "Yoruba", + "yue_Hant": "Yue Chinese (Hong Kong Cantonese)", + "zgh_Tfng": "Standard Moroccan Tamazight", + "zsm_Latn": "Standard Malay", + "zul_Latn": "Zulu" +} diff --git a/benchmarks/flores200/prepare.py b/benchmarks/flores200/prepare.py index 6e5fd7c6442..1e40df18cb8 100644 --- a/benchmarks/flores200/prepare.py +++ b/benchmarks/flores200/prepare.py @@ -14,12 +14,13 @@ # limitations under the License. """Prepare FLORES-200 benchmark data. -Two steps: +Three steps: 1. Download the FLORES+ dataset from ``openlanguagedata/flores_plus`` and write - the interleaved benchmark JSONL covering all directed pairs of the default - six-language set (excluding self-pairs). -2. Pre-fetch the xCOMET-XXL checkpoint AND its underlying xlm-roberta-xxl + the interleaved benchmark JSONL covering English-to-X for every target + represented in the bundled FLORES+ devtest allowlist. +2. Pre-fetch the FLORES-200 SentencePiece model used for spBLEU. +3. Pre-fetch the xCOMET-XXL checkpoint AND its underlying xlm-roberta-xxl tokenizer into ``HF_HOME`` so the ``wmt_translation`` resource server can run fully offline. Without this, every fresh ``CometActor`` resolves the tokenizer from HF Hub on startup, which hits the rate limiter when N @@ -30,9 +31,9 @@ resource server: ``text``, ``translation``, ``source_language``, ``target_language``, ``source_lang_name``, ``target_lang_name``. -Mirrors NeMo-Skills' ``nemo_skills/dataset/flores200/prepare.py`` row order -and field set so the output is byte-equivalent under the same default -arguments and split. +Uses the same row ordering as NeMo-Skills' +``nemo_skills/dataset/flores200/prepare.py`` under the same default arguments +and split. """ import argparse @@ -41,102 +42,85 @@ from datasets import load_dataset +from benchmarks.wmt24pp.prepare import prefetch_translation_models + DATA_DIR = Path(__file__).parent / "data" HF_REPO_ID = "openlanguagedata/flores_plus" -# Default COMET model used by the wmt_translation resource server. -# Pre-downloading it (plus its xlm-roberta-xxl backbone tokenizer) -# populates HF_HOME so the server can run with HF_HUB_OFFLINE=1 — no -# online tokenizer resolution at verify() time and no per-actor HF -# rate-limit retries. -COMET_MODEL = "Unbabel/XCOMET-XXL" - -# Same default language set as NeMo-Skills' flores200/prepare.py. Order is -# preserved so the interleaved JSONL is byte-equivalent to Skills' output -# under identical args. -DEFAULT_LANGUAGES = ["en", "de", "es", "fr", "it", "ja"] +GLOTLID_REPO_ID = "cis-lmu/glotlid" +GLOTLID_MODEL_FILENAME = "model_v3.bin" DEFAULT_SPLIT = "devtest" -# FLORES+ config name (``iso639-3_iso15924``) for each supported 2-letter -# code. Skills' prepare derives these dynamically via -# ``langcodes.Language``; we hardcode the default six so the prepare step -# adds no runtime dependency to the host Gym venv. -_FLORES_CONFIG = { - "en": "eng_Latn", - "de": "deu_Latn", - "es": "spa_Latn", - "fr": "fra_Latn", - "it": "ita_Latn", - "ja": "jpn_Jpan", -} - -# Display names for each supported code, byte-equivalent to -# ``langcodes.Language(code).display_name()`` for the defaults above. -_LANG_DISPLAY_NAMES = { - "en": "English", - "de": "German", - "es": "Spanish", - "fr": "French", - "it": "Italian", - "ja": "Japanese", -} +# Language metadata generated by resources_servers/wmt_translation/scripts/ +# generate_flores_language_metadata.py. +LANGUAGE_NAMES_PATH = DATA_DIR / "flores_language_names.json" +DEVTEST_LANGUAGES_PATH = DATA_DIR / "flores_devtest_languages.json" + +with LANGUAGE_NAMES_PATH.open(encoding="utf-8") as metadata_file: + _FLORES_LANG_MAP: dict[str, str] = json.load(metadata_file) +with DEVTEST_LANGUAGES_PATH.open(encoding="utf-8") as metadata_file: + _FLORES_DEVTEST_LANGUAGES = frozenset(json.load(metadata_file)) + +assert _FLORES_DEVTEST_LANGUAGES <= _FLORES_LANG_MAP.keys() +assert "eng_Latn" in _FLORES_DEVTEST_LANGUAGES + +# Translate from English into every other devtest-capable FLORES+ +# configuration by default. Dict insertion order keeps benchmark generation +# deterministic. +DEFAULT_SOURCE_LANGUAGES = ["eng_Latn"] +DEFAULT_TARGET_LANGUAGES = [ + code for code in _FLORES_LANG_MAP if code != "eng_Latn" and code in _FLORES_DEVTEST_LANGUAGES +] def _flores_lang_code(lang: str) -> str: - """Map a 2-letter code (e.g. ``en``) to FLORES' ``iso639-3_iso15924`` config name. + """Validate and return a complete FLORES+ configuration code.""" + if lang in _FLORES_LANG_MAP: + return lang + raise ValueError( + f"Unknown FLORES+ configuration code '{lang}'. Use a complete code such as 'eng_Latn' or 'deu_Latn'." + ) - Mirrors NeMo-Skills' use of - ``f"{Language(lang).to_alpha3()}_{Language(lang).maximize().script}"`` - for the default-six codes; raises a clear error on unsupported inputs so - users can extend ``_FLORES_CONFIG`` / ``_LANG_DISPLAY_NAMES`` together. - """ + +def _display_name(flores_code: str) -> str: try: - return _FLORES_CONFIG[lang] + return _FLORES_LANG_MAP[flores_code] except KeyError as exc: raise ValueError( - f"Unknown language code '{lang}'. Supported defaults: {sorted(_FLORES_CONFIG)}. " - "To add a new code, extend _FLORES_CONFIG and _LANG_DISPLAY_NAMES in " - "benchmarks/flores200/prepare.py." + f"No English name registered for FLORES code '{flores_code}'. Regenerate " + "the JSON metadata with " + "resources_servers/wmt_translation/scripts/generate_flores_language_metadata.py." ) from exc -def _display_name(lang: str) -> str: - try: - return _LANG_DISPLAY_NAMES[lang] - except KeyError as exc: - raise ValueError( - f"No display name registered for '{lang}'. Add it to _LANG_DISPLAY_NAMES in " - "benchmarks/flores200/prepare.py (must match the value of " - "``langcodes.Language().display_name()`` to keep parity with " - "NeMo-Skills' flores200 prepare output)." - ) from exc +def _validate_split_languages(languages: list[str], split: str) -> None: + if split != "devtest": + return + unsupported = [language for language in languages if language not in _FLORES_DEVTEST_LANGUAGES] + if unsupported: + raise ValueError(f"These FLORES+ configurations do not provide devtest: {', '.join(unsupported)}") -def _prefetch_comet_model(model_name: str = COMET_MODEL) -> None: - """Pre-download xCOMET-XXL + its xlm-roberta-xxl tokenizer to HF_HOME. - ``download_model`` fetches just the COMET checkpoint files; the actual - tokenizer + transformer backbone are pulled when ``load_from_checkpoint`` - instantiates the model. We do both here so the cache is fully primed - before any actor starts. Skipped silently if ``unbabel-comet`` is not - installed in the active Python env (e.g. when ``prepare()`` is invoked - from a venv that doesn't carry the server's heavy deps — the resource - server's own venv will fetch on first use in that case). - """ +def _prefetch_glotlid_model() -> None: + """Pre-download the GlotLID model used by the FLORES verifier.""" try: - from comet import download_model, load_from_checkpoint - except ImportError: - print(f"unbabel-comet not installed; skipping {model_name} prefetch") + from huggingface_hub import hf_hub_download + except ModuleNotFoundError as exc: + if exc.name != "huggingface_hub": + raise + print("huggingface-hub not installed; skipping GlotLID prefetch") return - print(f"Pre-fetching {model_name} checkpoint...") - ckpt_path = download_model(model_name) - print(f"Loading {model_name} once to populate xlm-roberta-xxl cache...") - load_from_checkpoint(ckpt_path) - print(f"{model_name} + tokenizer cached") + print(f"Pre-fetching {GLOTLID_REPO_ID}/{GLOTLID_MODEL_FILENAME}...") + hf_hub_download( + repo_id=GLOTLID_REPO_ID, + filename=GLOTLID_MODEL_FILENAME, + ) + print("GlotLID model cached") def prepare( @@ -145,6 +129,8 @@ def prepare( target_languages: list[str] | None = None, split: str = DEFAULT_SPLIT, prefetch_comet: bool = True, + prefetch_glotlid: bool = True, + prefetch_spbleu: bool = True, ) -> Path: """Download and interleave FLORES+ pairs. Returns the output JSONL path. @@ -153,27 +139,37 @@ def prepare( Pass ``source_languages`` / ``target_languages`` explicitly to override one direction. - If ``prefetch_comet`` is True (the default), also pre-downloads - xCOMET-XXL and its tokenizer into HF_HOME so the server can run offline. + If ``prefetch_spbleu`` is True (the default), caches SacreBLEU's + FLORES-200 SentencePiece tokenizer model. If ``prefetch_comet`` is True, + also pre-downloads xCOMET-XXL and its tokenizer into HF_HOME so the server + can run offline. Pass ``prefetch_comet=False`` for benchmark prep on a machine without GPU / unbabel-comet (the cache will be populated on first server use - instead). + instead). ``prefetch_glotlid`` similarly controls prefetching the GlotLID + model used for FLORES language-consistency scoring. """ if source_languages is None: - source_languages = list(languages) if languages is not None else list(DEFAULT_LANGUAGES) + source_languages = list(languages) if languages is not None else list(DEFAULT_SOURCE_LANGUAGES) if target_languages is None: - target_languages = list(languages) if languages is not None else list(DEFAULT_LANGUAGES) + target_languages = list(languages) if languages is not None else list(DEFAULT_TARGET_LANGUAGES) + + for language in source_languages: + _flores_lang_code(language) + for language in target_languages: + _flores_lang_code(language) + _validate_split_languages(source_languages, split) + _validate_split_languages(target_languages, split) DATA_DIR.mkdir(parents=True, exist_ok=True) output_fpath = DATA_DIR / f"flores200_{split}_benchmark.jsonl" all_languages = sorted(set(source_languages) | set(target_languages)) + print(f"Preparing {len(all_languages)} FLORES+ configurations for split {split!r}") datasets: dict = {} for lang in all_languages: - lang_code = _flores_lang_code(lang) - print(f"Loading {HF_REPO_ID} config {lang_code} (split={split})...") - datasets[lang] = load_dataset(HF_REPO_ID, lang_code, split=split)["text"] + print(f"Loading {HF_REPO_ID} config {lang} (split={split})...") + datasets[lang] = load_dataset(HF_REPO_ID, lang, split=split)["text"] count = 0 with output_fpath.open("w", encoding="utf-8") as fout: @@ -182,22 +178,28 @@ def prepare( for tgt_lang in target_languages: if src_lang == tgt_lang: continue + src_flores_code = _flores_lang_code(src_lang) + tgt_flores_code = _flores_lang_code(tgt_lang) for src, tgt in zip(datasets[src_lang], datasets[tgt_lang], strict=True): row = { "text": src, "translation": tgt, "source_language": src_lang, - "target_language": tgt_lang, - "source_lang_name": _display_name(src_lang), - "target_lang_name": _display_name(tgt_lang), + "target_language": tgt_flores_code, + "source_lang_name": _display_name(src_flores_code), + "target_lang_name": _display_name(tgt_flores_code), } fout.write(json.dumps(row) + "\n") count += 1 print(f"Wrote {count} rows to {output_fpath}") - if prefetch_comet: - _prefetch_comet_model() + prefetch_translation_models( + prefetch_comet=prefetch_comet, + prefetch_spbleu=prefetch_spbleu, + ) + if prefetch_glotlid: + _prefetch_glotlid_model() return output_fpath @@ -212,15 +214,21 @@ def _parse_args() -> argparse.Namespace: ) parser.add_argument( "--source_languages", - default=DEFAULT_LANGUAGES, + default=DEFAULT_SOURCE_LANGUAGES, nargs="+", - help="2-letter language codes to translate from.", + help="Complete FLORES+ configuration codes to translate from.", ) parser.add_argument( "--target_languages", - default=DEFAULT_LANGUAGES, + default=DEFAULT_TARGET_LANGUAGES, nargs="+", - help="2-letter language codes to translate to.", + help="Complete FLORES+ configuration codes to translate to.", + ) + parser.add_argument( + "--no-prefetch-spbleu", + dest="prefetch_spbleu", + action="store_false", + help="Skip the FLORES-200 SentencePiece tokenizer prefetch used by spBLEU.", ) parser.add_argument( "--no-prefetch-comet", @@ -228,6 +236,12 @@ def _parse_args() -> argparse.Namespace: action="store_false", help="Skip xCOMET-XXL prefetch (e.g. when unbabel-comet is unavailable).", ) + parser.add_argument( + "--no-prefetch-glotlid", + dest="prefetch_glotlid", + action="store_false", + help="Skip GlotLID model prefetch.", + ) return parser.parse_args() @@ -237,5 +251,7 @@ def _parse_args() -> argparse.Namespace: source_languages=args.source_languages, target_languages=args.target_languages, split=args.split, + prefetch_spbleu=args.prefetch_spbleu, prefetch_comet=args.prefetch_comet, + prefetch_glotlid=args.prefetch_glotlid, ) diff --git a/benchmarks/flores200/tests/test_prepare.py b/benchmarks/flores200/tests/test_prepare.py new file mode 100644 index 00000000000..7901bd81352 --- /dev/null +++ b/benchmarks/flores200/tests/test_prepare.py @@ -0,0 +1,150 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import builtins +import json + +import pytest + +from benchmarks.flores200 import prepare as prepare_module + + +def _fail_import(monkeypatch, module_name: str, exc: BaseException) -> None: + real_import = builtins.__import__ + + def fake_import(name, *args, **kwargs): + if name == module_name: + raise exc + return real_import(name, *args, **kwargs) + + monkeypatch.setattr(builtins, "__import__", fake_import) + + +def test_example_rollouts_follow_current_language_and_score_contract() -> None: + example_path = prepare_module.DATA_DIR / "example_rollouts.jsonl" + rows = [json.loads(line) for line in example_path.read_text(encoding="utf-8").splitlines()] + + assert rows + for row in rows: + assert row["source_language"] == "eng_Latn" + assert row["target_language"] in prepare_module._FLORES_LANG_MAP + assert row["language_consistency_score"] is not None + assert 0.0 <= row["language_consistency_score"] <= 1.0 + + +def test_glotlid_prefetch_skips_when_huggingface_hub_is_absent(monkeypatch, capsys) -> None: + _fail_import( + monkeypatch, + "huggingface_hub", + ModuleNotFoundError("No module named 'huggingface_hub'", name="huggingface_hub"), + ) + + prepare_module._prefetch_glotlid_model() + + assert "huggingface-hub not installed" in capsys.readouterr().out + + +def test_glotlid_prefetch_propagates_missing_transitive_dependency(monkeypatch) -> None: + _fail_import( + monkeypatch, + "huggingface_hub", + ModuleNotFoundError("No module named 'requests'", name="requests"), + ) + + with pytest.raises(ModuleNotFoundError) as exc_info: + prepare_module._prefetch_glotlid_model() + + assert exc_info.value.name == "requests" + + +def test_prepare_uses_full_flores_target_codes_and_english_names(monkeypatch, tmp_path) -> None: + texts_by_config = { + "eng_Latn": ["English text"], + "spa_Latn": ["Texto en español"], + } + + def fake_load_dataset(_repo_id, config, *, split): + assert split == "devtest" + return {"text": texts_by_config[config]} + + monkeypatch.setattr(prepare_module, "DATA_DIR", tmp_path) + monkeypatch.setattr(prepare_module, "load_dataset", fake_load_dataset) + prefetch_calls = [] + monkeypatch.setattr( + prepare_module, + "prefetch_translation_models", + lambda **kwargs: prefetch_calls.append(kwargs), + ) + + output_path = prepare_module.prepare( + languages=["eng_Latn", "spa_Latn"], + prefetch_spbleu=False, + prefetch_comet=False, + prefetch_glotlid=False, + ) + rows = [json.loads(line) for line in output_path.read_text().splitlines()] + + assert prefetch_calls == [{"prefetch_comet": False, "prefetch_spbleu": False}] + assert rows == [ + { + "text": "English text", + "translation": "Texto en español", + "source_language": "eng_Latn", + "target_language": "spa_Latn", + "source_lang_name": "English", + "target_lang_name": "Spanish (Latin American)", + }, + { + "text": "Texto en español", + "translation": "English text", + "source_language": "spa_Latn", + "target_language": "eng_Latn", + "source_lang_name": "Spanish (Latin American)", + "target_lang_name": "English", + }, + ] + + +def test_flores_names_distinguish_dialects() -> None: + assert prepare_module._FLORES_LANG_MAP["apc_Arab_nort3139"] == "Levantine Arabic (North)" + assert prepare_module._FLORES_LANG_MAP["apc_Arab_sout3123"] == "Levantine Arabic (South)" + assert prepare_module._flores_lang_code("apc_Arab_sout3123") == "apc_Arab_sout3123" + + +def test_two_character_language_aliases_are_rejected() -> None: + with pytest.raises(ValueError, match="complete code"): + prepare_module._flores_lang_code("en") + + +def test_explicit_dev_only_language_is_rejected_before_download(monkeypatch, tmp_path) -> None: + monkeypatch.setattr(prepare_module, "DATA_DIR", tmp_path) + monkeypatch.setattr( + prepare_module, + "load_dataset", + lambda *_args, **_kwargs: pytest.fail("load_dataset should not be called"), + ) + + with pytest.raises(ValueError, match="do not provide devtest: brx_Deva"): + prepare_module.prepare( + source_languages=["eng_Latn"], + target_languages=["brx_Deva"], + prefetch_spbleu=False, + prefetch_comet=False, + prefetch_glotlid=False, + ) + + +def test_dev_split_accepts_language_without_devtest() -> None: + prepare_module._validate_split_languages(["brx_Deva"], "dev") diff --git a/benchmarks/wmt24pp/README.md b/benchmarks/wmt24pp/README.md index 6d0f3e622f5..ee9a1e2fd92 100644 --- a/benchmarks/wmt24pp/README.md +++ b/benchmarks/wmt24pp/README.md @@ -1,12 +1,17 @@ # WMT24++ Translation Benchmark -English to {de_DE, es_MX, fr_FR, it_IT, ja_JP} segment-level translation -from [`google/wmt24pp`](https://huggingface.co/datasets/google/wmt24pp). +English-to-X segment-level translation across all 55 target locales in +[`google/wmt24pp`](https://huggingface.co/datasets/google/wmt24pp). -Verification is deterministic corpus-level BLEU (sacrebleu) per language -pair, with cross-pair aggregations `en->xx`, `xx->xx`, and `xx->{tgt}`. +Verification reports deterministic corpus-level spBLEU and chrF (sacrebleu) per language +pair, with spBLEU consistently using the `flores200` SentencePiece tokenizer +and cross-pair aggregations `en->xx`, `xx->xx`, and `xx->{tgt}`. Optionally augments with xCOMET-XXL neural QE scores when `compute_comet: true` is set on the wmt_translation server. +Each rollout also reports CLD2 target-language consistency. CLD2 cannot +distinguish the regional varieties in the `ar_EG`/`ar_SA`, +`fr_CA`/`fr_FR`, `pt_BR`/`pt_PT`, and `sw_KE`/`sw_TZ` pairs, so these scores +validate the major language only and cannot detect the wrong regional variety. See `resources_servers/wmt_translation/README.md` for the verifier details and the Ray GPU-scheduled COMET path. @@ -18,17 +23,24 @@ gym eval prepare --benchmark wmt24pp ``` In addition to writing `data/wmt24pp_benchmark.jsonl`, the prepare step -pre-fetches the xCOMET-XXL checkpoint and its xlm-roberta-xxl tokenizer -into `HF_HOME` (when `unbabel-comet` is installed in the active env). +pre-fetches SacreBLEU's FLORES-200 SentencePiece model into its cache, plus +the xCOMET-XXL checkpoint and its xlm-roberta-xxl tokenizer into `HF_HOME` +(when `unbabel-comet` is installed in the active env). That keeps the resource server's Ray actors fully offline at runtime — no HF Hub calls during `verify()`, no rate-limit retries. +Target-locale display names are loaded from the checked-in +`data/wmt24pp_language_names.json` file. Regenerate it deliberately with +`resources_servers/wmt_translation/scripts/generate_wmt24pp_language_names.py` +when updating the WMT24++ dataset; preparation does not query metadata at +runtime. + ## Running servers The xCOMET-XXL actor pool requires the `extra_gpu` Ray resource, which is only advertised on multi-node SLURM deployments via NeMo-Skills' `get_ray_server_cmd` (see the SLURM block below). Local / single-node -runs disable COMET via Hydra override and rely on corpus-BLEU only; +runs disable COMET via Hydra override and rely on local spBLEU and chrF; xCOMET scoring still works end-to-end on the cluster path: ```bash @@ -92,7 +104,8 @@ The two container fields that aren't trivial: — cross-container Ray-cluster joins fail with `ConnectionError: Could not read 'temp_dir' from GCS` on protocol mismatch. - **`nemo-gym`**: any image where `pip install -e [dev]` resolves - cleanly AND has `unbabel-comet`, `torch>=2.5`, `sacrebleu` baked in. + cleanly AND has `unbabel-comet`, `torch>=2.5`, `sacrebleu`, and + `sentencepiece` baked in. The lazy-install path in `resources_servers/wmt_translation/.venv` works as a fallback but adds 2–3 min to first-job startup. @@ -126,7 +139,7 @@ and switch to a batch partition for larger evaluations. ```bash # Pick a translation-capable policy model accessible from your cluster. -# The PR's parity numbers come from nvidia/Nemotron-3-Nano-30B-A3B-BF16. +# Example translation-capable policy model. MODEL="nvidia/Nemotron-3-Nano-30B-A3B-BF16" ns nemo_gym_rollouts \ diff --git a/benchmarks/wmt24pp/config.yaml b/benchmarks/wmt24pp/config.yaml index e683207e46a..d6ed249e21b 100644 --- a/benchmarks/wmt24pp/config.yaml +++ b/benchmarks/wmt24pp/config.yaml @@ -3,6 +3,9 @@ config_paths: wmt24pp_wmt_translation_resources_server: _inherit_from: wmt_translation + resources_servers: + wmt_translation: + language_consistency_backend: wmt24pp_cld2 wmt24pp_wmt_translation_simple_agent: _inherit_from: wmt_translation_simple_agent diff --git a/benchmarks/wmt24pp/data/wmt24pp_language_names.json b/benchmarks/wmt24pp/data/wmt24pp_language_names.json new file mode 100644 index 00000000000..fad07b7eca7 --- /dev/null +++ b/benchmarks/wmt24pp/data/wmt24pp_language_names.json @@ -0,0 +1,57 @@ +{ + "ar_EG": "Arabic (Egypt)", + "ar_SA": "Arabic (Saudi Arabia)", + "bg_BG": "Bulgarian (Bulgaria)", + "bn_IN": "Bangla (India)", + "ca_ES": "Catalan (Spain)", + "cs_CZ": "Czech (Czechia)", + "da_DK": "Danish (Denmark)", + "de_DE": "German (Germany)", + "el_GR": "Greek (Greece)", + "es_MX": "Spanish (Mexico)", + "et_EE": "Estonian (Estonia)", + "fa_IR": "Persian (Iran)", + "fi_FI": "Finnish (Finland)", + "fil_PH": "Filipino (Philippines)", + "fr_CA": "French (Canada)", + "fr_FR": "French (France)", + "gu_IN": "Gujarati (India)", + "he_IL": "Hebrew (Israel)", + "hi_IN": "Hindi (India)", + "hr_HR": "Croatian (Croatia)", + "hu_HU": "Hungarian (Hungary)", + "id_ID": "Indonesian (Indonesia)", + "is_IS": "Icelandic (Iceland)", + "it_IT": "Italian (Italy)", + "ja_JP": "Japanese (Japan)", + "kn_IN": "Kannada (India)", + "ko_KR": "Korean (South Korea)", + "lt_LT": "Lithuanian (Lithuania)", + "lv_LV": "Latvian (Latvia)", + "ml_IN": "Malayalam (India)", + "mr_IN": "Marathi (India)", + "nl_NL": "Dutch (Netherlands)", + "no_NO": "Norwegian (Norway)", + "pa_IN": "Punjabi (India)", + "pl_PL": "Polish (Poland)", + "pt_BR": "Portuguese (Brazil)", + "pt_PT": "Portuguese (Portugal)", + "ro_RO": "Romanian (Romania)", + "ru_RU": "Russian (Russia)", + "sk_SK": "Slovak (Slovakia)", + "sl_SI": "Slovenian (Slovenia)", + "sr_RS": "Serbian (Serbia)", + "sv_SE": "Swedish (Sweden)", + "sw_KE": "Swahili (Kenya)", + "sw_TZ": "Swahili (Tanzania)", + "ta_IN": "Tamil (India)", + "te_IN": "Telugu (India)", + "th_TH": "Thai (Thailand)", + "tr_TR": "Turkish (Türkiye)", + "uk_UA": "Ukrainian (Ukraine)", + "ur_PK": "Urdu (Pakistan)", + "vi_VN": "Vietnamese (Vietnam)", + "zh_CN": "Chinese (China)", + "zh_TW": "Chinese (Taiwan)", + "zu_ZA": "Zulu (South Africa)" +} diff --git a/benchmarks/wmt24pp/prepare.py b/benchmarks/wmt24pp/prepare.py index 781afc2196a..eb2b035519f 100644 --- a/benchmarks/wmt24pp/prepare.py +++ b/benchmarks/wmt24pp/prepare.py @@ -14,11 +14,12 @@ # limitations under the License. """Prepare WMT24++ benchmark data. -Two steps: +Three steps: 1. Download the WMT24++ dataset from ``google/wmt24pp`` and write the interleaved benchmark JSONL (one config per en- pair). -2. Pre-fetch the xCOMET-XXL checkpoint AND its underlying +2. Pre-fetch the FLORES-200 SentencePiece model used for spBLEU. +3. Pre-fetch the xCOMET-XXL checkpoint AND its underlying xlm-roberta-xxl tokenizer into HF_HOME so the ``wmt_translation`` resource server can run fully offline. Without this, every fresh ``CometActor`` resolves the tokenizer from HF Hub on startup, which @@ -35,7 +36,11 @@ ``target_lang_name`` to () (e.g. "Spanish (Mexico)") """ +import argparse +import importlib.util import json +import os +import tempfile from pathlib import Path from datasets import load_dataset @@ -53,77 +58,18 @@ # online tokenizer resolution at verify() time and no per-actor HF # rate-limit retries. COMET_MODEL = "Unbabel/XCOMET-XXL" +SPBLEU_TOKENIZER = "flores200" -# Same five targets + same order as Skills' default. Keeping the order -# stable is what makes the interleaved JSONL byte-comparable. -DEFAULT_TARGET_LANGUAGES = ["de_DE", "es_MX", "fr_FR", "it_IT", "ja_JP"] - -# Hardcoding to avoid both dependency on https://github.com/rspeer/langcodes -# and network call in prepare.py -# import langcodes #uv pip install langcodes[data] -# from datasets import get_dataset_config_names #uv pip install datasets -# lang_pairs = get_dataset_config_names("google/wmt24pp") -# _WMT24PP_LANG_MAP = {} -# for lang_pair in lang_pairs: -# _, tgt = lang_pair.split('-') -# _WMT24PP_LANG_MAP[tgt] = langcodes.Language.get(tgt).display_name() -_WMT24PP_LANG_MAP = { - "ar_EG": "Arabic (Egypt)", - "ar_SA": "Arabic (Saudi Arabia)", - "bg_BG": "Bulgarian (Bulgaria)", - "bn_IN": "Bangla (India)", - "ca_ES": "Catalan (Spain)", - "cs_CZ": "Czech (Czechia)", - "da_DK": "Danish (Denmark)", - "de_DE": "German (Germany)", - "el_GR": "Greek (Greece)", - "es_MX": "Spanish (Mexico)", - "et_EE": "Estonian (Estonia)", - "fa_IR": "Persian (Iran)", - "fi_FI": "Finnish (Finland)", - "fil_PH": "Filipino (Philippines)", - "fr_CA": "French (Canada)", - "fr_FR": "French (France)", - "gu_IN": "Gujarati (India)", - "he_IL": "Hebrew (Israel)", - "hi_IN": "Hindi (India)", - "hr_HR": "Croatian (Croatia)", - "hu_HU": "Hungarian (Hungary)", - "id_ID": "Indonesian (Indonesia)", - "is_IS": "Icelandic (Iceland)", - "it_IT": "Italian (Italy)", - "ja_JP": "Japanese (Japan)", - "kn_IN": "Kannada (India)", - "ko_KR": "Korean (South Korea)", - "lt_LT": "Lithuanian (Lithuania)", - "lv_LV": "Latvian (Latvia)", - "ml_IN": "Malayalam (India)", - "mr_IN": "Marathi (India)", - "nl_NL": "Dutch (Netherlands)", - "no_NO": "Norwegian (Norway)", - "pa_IN": "Punjabi (India)", - "pl_PL": "Polish (Poland)", - "pt_BR": "Portuguese (Brazil)", - "pt_PT": "Portuguese (Portugal)", - "ro_RO": "Romanian (Romania)", - "ru_RU": "Russian (Russia)", - "sk_SK": "Slovak (Slovakia)", - "sl_SI": "Slovenian (Slovenia)", - "sr_RS": "Serbian (Serbia)", - "sv_SE": "Swedish (Sweden)", - "sw_KE": "Swahili (Kenya)", - "sw_TZ": "Swahili (Tanzania)", - "ta_IN": "Tamil (India)", - "te_IN": "Telugu (India)", - "th_TH": "Thai (Thailand)", - "tr_TR": "Turkish (Türkiye)", - "uk_UA": "Ukrainian (Ukraine)", - "ur_PK": "Urdu (Pakistan)", - "vi_VN": "Vietnamese (Vietnam)", - "zh_CN": "Chinese (China)", - "zh_TW": "Chinese (Taiwan)", - "zu_ZA": "Zulu (South Africa)", -} +# Language names generated by resources_servers/wmt_translation/scripts/ +# generate_wmt24pp_language_names.py. +LANGUAGE_NAMES_PATH = DATA_DIR / "wmt24pp_language_names.json" + +with LANGUAGE_NAMES_PATH.open(encoding="utf-8") as metadata_file: + _WMT24PP_LANG_MAP: dict[str, str] = json.load(metadata_file) + +# Use every supported English-to-target pair by default. Dict insertion order +# keeps the generated interleaved JSONL deterministic. +DEFAULT_TARGET_LANGUAGES = list(_WMT24PP_LANG_MAP) def _prefetch_comet_model(model_name: str = COMET_MODEL) -> None: @@ -140,7 +86,9 @@ def _prefetch_comet_model(model_name: str = COMET_MODEL) -> None: """ try: from comet import download_model, load_from_checkpoint - except ImportError: + except ModuleNotFoundError as exc: + if exc.name != "comet": + raise print(f"unbabel-comet not installed; skipping {model_name} prefetch") return @@ -154,18 +102,51 @@ def _prefetch_comet_model(model_name: str = COMET_MODEL) -> None: print(f"{model_name} + tokenizer cached") -def prepare(target_languages: list[str] | None = None, prefetch_comet: bool = True) -> Path: +def _prefetch_spbleu_tokenizer() -> None: + """Cache SacreBLEU's FLORES-200 SentencePiece model for offline scoring.""" + try: + from sacrebleu.metrics import BLEU + except ModuleNotFoundError as exc: + if exc.name != "sacrebleu": + raise + print("sacrebleu not installed; skipping FLORES-200 spBLEU tokenizer prefetch") + return + + if importlib.util.find_spec("sentencepiece") is None: + print("sentencepiece not installed; skipping FLORES-200 spBLEU tokenizer prefetch") + return + + print("Pre-fetching FLORES-200 SentencePiece tokenizer for spBLEU...") + BLEU(tokenize=SPBLEU_TOKENIZER) + print("FLORES-200 spBLEU tokenizer cached") + + +def prefetch_translation_models(*, prefetch_comet: bool = True, prefetch_spbleu: bool = True) -> None: + """Cache the models shared by WMT-style translation benchmarks.""" + if prefetch_spbleu: + _prefetch_spbleu_tokenizer() + if prefetch_comet: + _prefetch_comet_model() + + +def prepare( + target_languages: list[str] | None = None, + prefetch_comet: bool = True, + prefetch_spbleu: bool = True, +) -> Path: """Download and interleave WMT24++ en- pairs. Returns the output file path. - If ``prefetch_comet`` is True (the default), also pre-downloads - xCOMET-XXL and its tokenizer into HF_HOME so the server can run - offline. Pass ``prefetch_comet=False`` for benchmark prep on a - machine without GPU / unbabel-comet (the cache will be populated on - first server use instead). + If ``prefetch_spbleu`` is True (the default), caches SacreBLEU's + FLORES-200 SentencePiece tokenizer model. If ``prefetch_comet`` is True, + also pre-downloads xCOMET-XXL and its tokenizer into HF_HOME so the server + can run offline. Pass ``prefetch_comet=False`` for benchmark prep on a + machine without GPU / unbabel-comet (the cache will be populated on first + server use instead). """ if target_languages is None: target_languages = DEFAULT_TARGET_LANGUAGES else: + target_languages = list(dict.fromkeys(target_languages)) # check user passed in langs unknown = [lang for lang in target_languages if lang not in _WMT24PP_LANG_MAP] if unknown: @@ -181,32 +162,66 @@ def prepare(target_languages: list[str] | None = None, prefetch_comet: bool = Tr print(f"Loading {HF_REPO_ID} config en-{lang}...") datasets[lang] = load_dataset(HF_REPO_ID, f"en-{lang}")["train"] - count = 0 - with OUTPUT_FPATH.open("w", encoding="utf-8") as fout: - for tgt_lang in target_languages: - for src, tgt in zip( - datasets[tgt_lang]["source"], - datasets[tgt_lang]["target"], - strict=True, - ): - row = { - "text": src, - "translation": tgt, - "source_language": "en", - "target_language": tgt_lang, - "source_lang_name": "English", - "target_lang_name": _WMT24PP_LANG_MAP[tgt_lang], - } - fout.write(json.dumps(row) + "\n") - count += 1 + file_descriptor, temp_name = tempfile.mkstemp(dir=DATA_DIR, prefix=f".{OUTPUT_FPATH.name}.") + temp_path = Path(temp_name) + try: + count = 0 + with os.fdopen(file_descriptor, "w", encoding="utf-8") as fout: + for tgt_lang in target_languages: + for dataset_row in datasets[tgt_lang]: + if dataset_row["is_bad_source"]: + continue + row = { + "text": dataset_row["source"], + "translation": dataset_row["target"], + "source_language": "en", + "target_language": tgt_lang, + "source_lang_name": "English", + "target_lang_name": _WMT24PP_LANG_MAP[tgt_lang], + } + fout.write(json.dumps(row) + "\n") + count += 1 + + prefetch_translation_models( + prefetch_comet=prefetch_comet, + prefetch_spbleu=prefetch_spbleu, + ) + os.replace(temp_path, OUTPUT_FPATH) + finally: + temp_path.unlink(missing_ok=True) print(f"Wrote {count} rows to {OUTPUT_FPATH}") - if prefetch_comet: - _prefetch_comet_model() - return OUTPUT_FPATH +def _parse_args(argv: list[str] | None = None) -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--target_languages", + nargs="+", + default=None, + help="Target language codes (default: all supported WMT24++ locales).", + ) + parser.add_argument( + "--no-prefetch-comet", + dest="prefetch_comet", + action="store_false", + help="Skip xCOMET-XXL checkpoint and tokenizer prefetch.", + ) + parser.add_argument( + "--no-prefetch-spbleu", + dest="prefetch_spbleu", + action="store_false", + help="Skip the FLORES-200 SentencePiece tokenizer prefetch.", + ) + return parser.parse_args(argv) + + if __name__ == "__main__": - prepare() + args = _parse_args() + prepare( + target_languages=args.target_languages, + prefetch_comet=args.prefetch_comet, + prefetch_spbleu=args.prefetch_spbleu, + ) diff --git a/benchmarks/wmt24pp/tests/test_wmt24pp_prepare.py b/benchmarks/wmt24pp/tests/test_wmt24pp_prepare.py new file mode 100644 index 00000000000..73164a2ee0d --- /dev/null +++ b/benchmarks/wmt24pp/tests/test_wmt24pp_prepare.py @@ -0,0 +1,357 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import builtins +import json + +import pytest + +from benchmarks.wmt24pp import prepare as prepare_module + + +def _fail_import(monkeypatch, module_name: str, exc: BaseException) -> None: + real_import = builtins.__import__ + + def fake_import(name, *args, **kwargs): + if name == module_name: + raise exc + return real_import(name, *args, **kwargs) + + monkeypatch.setattr(builtins, "__import__", fake_import) + + +def test_language_names_are_loaded_from_json() -> None: + assert prepare_module._WMT24PP_LANG_MAP["es_MX"] == "Spanish (Mexico)" + assert prepare_module._WMT24PP_LANG_MAP["pt_BR"] == "Portuguese (Brazil)" + assert prepare_module.DEFAULT_TARGET_LANGUAGES == list(prepare_module._WMT24PP_LANG_MAP) + + +def test_parse_args_defaults() -> None: + args = prepare_module._parse_args([]) + + assert args.target_languages is None + assert args.prefetch_comet is True + assert args.prefetch_spbleu is True + + +def test_parse_args_accepts_prepare_options() -> None: + args = prepare_module._parse_args( + [ + "--target_languages", + "de_DE", + "es_MX", + "--no-prefetch-comet", + "--no-prefetch-spbleu", + ] + ) + + assert args.target_languages == ["de_DE", "es_MX"] + assert args.prefetch_comet is False + assert args.prefetch_spbleu is False + + +@pytest.mark.parametrize( + ("argv", "expected_exit_code"), + [ + (["--help"], 0), + (["--definitely-invalid"], 2), + ], +) +def test_parse_args_exits_before_download(monkeypatch, argv, expected_exit_code) -> None: + monkeypatch.setattr( + prepare_module, + "load_dataset", + lambda *_args, **_kwargs: pytest.fail("load_dataset should not be called"), + ) + + with pytest.raises(SystemExit) as exc_info: + prepare_module._parse_args(argv) + + assert exc_info.value.code == expected_exit_code + + +def test_comet_prefetch_skips_when_comet_is_absent(monkeypatch, capsys) -> None: + _fail_import( + monkeypatch, + "comet", + ModuleNotFoundError("No module named 'comet'", name="comet"), + ) + + prepare_module._prefetch_comet_model() + + assert "unbabel-comet not installed" in capsys.readouterr().out + + +def test_comet_prefetch_propagates_missing_transitive_dependency(monkeypatch) -> None: + _fail_import( + monkeypatch, + "comet", + ModuleNotFoundError("No module named 'torch'", name="torch"), + ) + + with pytest.raises(ModuleNotFoundError) as exc_info: + prepare_module._prefetch_comet_model() + + assert exc_info.value.name == "torch" + + +def test_comet_prefetch_propagates_incompatible_api(monkeypatch) -> None: + _fail_import( + monkeypatch, + "comet", + ImportError("cannot import name 'load_from_checkpoint' from 'comet'"), + ) + + with pytest.raises(ImportError, match="load_from_checkpoint"): + prepare_module._prefetch_comet_model() + + +def test_spbleu_prefetch_skips_when_sacrebleu_is_absent(monkeypatch, capsys) -> None: + _fail_import( + monkeypatch, + "sacrebleu.metrics", + ModuleNotFoundError("No module named 'sacrebleu'", name="sacrebleu"), + ) + + prepare_module._prefetch_spbleu_tokenizer() + + assert "sacrebleu not installed" in capsys.readouterr().out + + +def test_spbleu_prefetch_propagates_missing_transitive_dependency(monkeypatch) -> None: + _fail_import( + monkeypatch, + "sacrebleu.metrics", + ModuleNotFoundError("No module named 'regex'", name="regex"), + ) + + with pytest.raises(ModuleNotFoundError) as exc_info: + prepare_module._prefetch_spbleu_tokenizer() + + assert exc_info.value.name == "regex" + + +def test_spbleu_prefetch_skips_when_sentencepiece_is_absent(monkeypatch, capsys) -> None: + monkeypatch.setattr(prepare_module.importlib.util, "find_spec", lambda name: None) + + prepare_module._prefetch_spbleu_tokenizer() + + assert "sentencepiece not installed" in capsys.readouterr().out + + +def test_spbleu_prefetch_propagates_broken_sentencepiece(monkeypatch) -> None: + from sacrebleu import metrics + + class BrokenBLEU: + def __init__(self, **_kwargs): + raise ImportError("sentencepiece native library failed to load") + + monkeypatch.setattr(prepare_module.importlib.util, "find_spec", lambda name: object()) + monkeypatch.setattr(metrics, "BLEU", BrokenBLEU) + + with pytest.raises(ImportError, match="native library failed"): + prepare_module._prefetch_spbleu_tokenizer() + + +def test_prepare_uses_json_language_name(monkeypatch, tmp_path) -> None: + def fake_load_dataset(repo_id, config): + assert repo_id == prepare_module.HF_REPO_ID + assert config == "en-es_MX" + return { + "train": [ + { + "source": "Pass the salt.", + "target": "Pasa la sal.", + "is_bad_source": False, + } + ] + } + + output_path = tmp_path / "wmt24pp_benchmark.jsonl" + monkeypatch.setattr(prepare_module, "DATA_DIR", tmp_path) + monkeypatch.setattr(prepare_module, "OUTPUT_FPATH", output_path) + monkeypatch.setattr(prepare_module, "load_dataset", fake_load_dataset) + monkeypatch.setattr(prepare_module, "prefetch_translation_models", lambda **_kwargs: None) + + result = prepare_module.prepare( + target_languages=["es_MX"], + prefetch_comet=False, + prefetch_spbleu=False, + ) + + assert result == output_path + assert [json.loads(line) for line in output_path.read_text().splitlines()] == [ + { + "text": "Pass the salt.", + "translation": "Pasa la sal.", + "source_language": "en", + "target_language": "es_MX", + "source_lang_name": "English", + "target_lang_name": "Spanish (Mexico)", + } + ] + + +def test_prepare_deduplicates_target_languages_without_changing_order(monkeypatch, tmp_path) -> None: + load_calls = [] + + def fake_load_dataset(repo_id, config): + load_calls.append((repo_id, config)) + return { + "train": [ + { + "source": "Pass the salt.", + "target": "Pasa la sal.", + "is_bad_source": False, + } + ] + } + + output_path = tmp_path / "wmt24pp_benchmark.jsonl" + monkeypatch.setattr(prepare_module, "DATA_DIR", tmp_path) + monkeypatch.setattr(prepare_module, "OUTPUT_FPATH", output_path) + monkeypatch.setattr(prepare_module, "load_dataset", fake_load_dataset) + monkeypatch.setattr(prepare_module, "prefetch_translation_models", lambda **_kwargs: None) + + prepare_module.prepare( + target_languages=["fr_FR", "es_MX", "fr_FR", "de_DE"], + prefetch_comet=False, + prefetch_spbleu=False, + ) + + assert load_calls == [ + (prepare_module.HF_REPO_ID, "en-fr_FR"), + (prepare_module.HF_REPO_ID, "en-es_MX"), + (prepare_module.HF_REPO_ID, "en-de_DE"), + ] + assert [json.loads(line)["target_language"] for line in output_path.read_text().splitlines()] == [ + "fr_FR", + "es_MX", + "de_DE", + ] + + +def test_prepare_does_not_publish_output_when_prefetch_fails(monkeypatch, tmp_path) -> None: + output_path = tmp_path / "wmt24pp_benchmark.jsonl" + output_path.write_text("existing output\n") + + monkeypatch.setattr(prepare_module, "DATA_DIR", tmp_path) + monkeypatch.setattr(prepare_module, "OUTPUT_FPATH", output_path) + monkeypatch.setattr( + prepare_module, + "load_dataset", + lambda *_args, **_kwargs: { + "train": [ + { + "source": "Pass the salt.", + "target": "Pasa la sal.", + "is_bad_source": False, + } + ] + }, + ) + monkeypatch.setattr( + prepare_module, + "prefetch_translation_models", + lambda **_kwargs: (_ for _ in ()).throw(RuntimeError("prefetch failed")), + ) + + with pytest.raises(RuntimeError, match="prefetch failed"): + prepare_module.prepare(target_languages=["es_MX"]) + + assert output_path.read_text() == "existing output\n" + assert list(tmp_path.glob(f".{output_path.name}.*")) == [] + + +def test_prepare_omits_bad_sources_and_preserves_good_row_order(monkeypatch, tmp_path) -> None: + output_path = tmp_path / "wmt24pp_benchmark.jsonl" + monkeypatch.setattr(prepare_module, "DATA_DIR", tmp_path) + monkeypatch.setattr(prepare_module, "OUTPUT_FPATH", output_path) + monkeypatch.setattr( + prepare_module, + "load_dataset", + lambda *_args, **_kwargs: { + "train": [ + { + "source": "First good source.", + "target": "Primera fuente buena.", + "is_bad_source": False, + }, + { + "source": "CANARY GUID", + "target": "CANARY GUID", + "is_bad_source": True, + }, + { + "source": "Second good source.", + "target": "Segunda fuente buena.", + "is_bad_source": False, + }, + ] + }, + ) + monkeypatch.setattr(prepare_module, "prefetch_translation_models", lambda **_kwargs: None) + + prepare_module.prepare( + target_languages=["es_MX"], + prefetch_comet=False, + prefetch_spbleu=False, + ) + + rows = [json.loads(line) for line in output_path.read_text().splitlines()] + assert [(row["text"], row["translation"]) for row in rows] == [ + ("First good source.", "Primera fuente buena."), + ("Second good source.", "Segunda fuente buena."), + ] + + +def test_prepare_requires_is_bad_source_field(monkeypatch, tmp_path) -> None: + monkeypatch.setattr(prepare_module, "DATA_DIR", tmp_path) + monkeypatch.setattr(prepare_module, "OUTPUT_FPATH", tmp_path / "wmt24pp_benchmark.jsonl") + monkeypatch.setattr( + prepare_module, + "load_dataset", + lambda *_args, **_kwargs: { + "train": [ + { + "source": "Pass the salt.", + "target": "Pasa la sal.", + } + ] + }, + ) + monkeypatch.setattr(prepare_module, "prefetch_translation_models", lambda **_kwargs: None) + + with pytest.raises(KeyError, match="is_bad_source"): + prepare_module.prepare( + target_languages=["es_MX"], + prefetch_comet=False, + prefetch_spbleu=False, + ) + + +def test_prepare_rejects_unknown_language_before_download(monkeypatch) -> None: + monkeypatch.setattr( + prepare_module, + "load_dataset", + lambda *_args, **_kwargs: pytest.fail("load_dataset should not be called"), + ) + + with pytest.raises(ValueError, match="xx_XX"): + prepare_module.prepare( + target_languages=["xx_XX"], + prefetch_comet=False, + prefetch_spbleu=False, + ) diff --git a/resources_servers/wmt_translation/README.md b/resources_servers/wmt_translation/README.md index a99c31cc375..dbf46aa5cbe 100644 --- a/resources_servers/wmt_translation/README.md +++ b/resources_servers/wmt_translation/README.md @@ -1,40 +1,61 @@ # wmt_translation Generic machine-translation verifier for WMT-style benchmarks. Computes -corpus-level BLEU per `(source_language, target_language)` pair via -sacrebleu, with language-specific tokenizers (`13a` default; `ja-mecab`, -`ko-mecab`, `zh` as appropriate). Optionally augments with xCOMET-XXL +corpus-level spBLEU and chrF per `(source_language, target_language)` pair via +sacrebleu. spBLEU always uses SacreBLEU's `flores200` SentencePiece tokenizer, +regardless of the target language. Optionally augments them with xCOMET-XXL neural QE scores, served by a persistent Ray actor pool that loads the -checkpoint once per actor and stays resident for the whole run. - -Ported from NeMo-Skills' `TranslationMetrics` -(`nemo_skills/evaluation/metrics/translation_metrics.py`) plus the -xCOMET-XXL judge script at `nemo_skills/evaluation/evaluator/comet.py`. +checkpoint once per actor and stays resident for the whole run, plus optional +language-consistency scoring through a configured backend. + +> **Warning:** We switched from the default SacreBLEU tokenizer to the +> FLORES-200 SentencePiece tokenizer for computing BLEU. The new BLEU scores +> are reported as **spBLEU**—do **NOT** compare these spBLEU scores to prior +> BLEU scores reported by this benchmark in the past; they are not comparable. +> Also be very careful when comparing them to BLEU scores reported in papers or +> elsewhere: the tokenizer must be the same for the scores to be comparable. +> In general, we recommend using **chrF** instead of BLEU because it performs at +> least as well and requires no tokenization, so there is no chance of tokenizer +> mismatch. ## Metric outputs -`compute_metrics()` emits Skills-equivalent keys: +Human-facing text and aggregate JSON metric keys use the canonical display +names **spBLEU** and **chrF**. Per-sentence Python and JSON fields remain +snake_case: `sentence_spbleu` and `sentence_chrf`. + +Before sentence- or corpus-level chrF and spBLEU are computed, both the model +generation and reference translation are normalized to Unicode NFC. The +original generation text stored in rollout output is not modified. -- Per-pair: `->/bleu`, `->/bleu_std_dev_across_runs`, - `->/comet`, `->/comet_std_dev_across_runs` -- Aggregated: `xx->xx/bleu`, `->xx/bleu`, `xx->/bleu` - (and matching `/comet` keys when `compute_comet: true`) +`compute_metrics()` emits: + +- Per-pair: `->/spBLEU`, `->/spBLEU_std_dev_across_runs`, + `->/chrF`, `->/chrF_std_dev_across_runs`, + `->/comet`, `->/comet_std_dev_across_runs`, + `->/language_consistency`, + `->/language_consistency_std_dev_across_runs` +- Aggregated: `xx->xx/{spBLEU,chrF}`, `->xx/{spBLEU,chrF}`, + `xx->/{spBLEU,chrF}` + (and matching `/comet` keys when `compute_comet: true`, and + `/language_consistency` keys when a language-consistency backend is configured) `get_key_metrics()` returns the headline aggregates -(`xx->xx/bleu`, `xx->xx/comet`, `en->xx/bleu`, `en->xx/comet`). +(`xx->xx/spBLEU`, `xx->xx/chrF`, `xx->xx/comet`, +`xx->xx/language_consistency`, `en->xx/spBLEU`, `en->xx/chrF`, +`en->xx/comet`, `en->xx/language_consistency`). -> **Note:** `compute_metrics()` emits corpus-level BLEU/COMET keyed by +> **Note:** `compute_metrics()` emits corpus-level spBLEU/chrF/COMET keyed by > language pair, not the `pass@k/{name}` pattern produced by > `compute_pass_majority_metrics()`. This is intentional — translation > quality is a corpus-level score, not a per-task correctness probability, > so the Tier 1 pass@k template in `migrate-benchmark` doesn't apply. -> Parity with NeMo-Skills is on the corpus aggregates. ## Per-sample reward + per-row COMET -`verify()` returns `sentence_bleu(generation, [reference]) / 100` as the -`reward` field. This is a useful dense RL signal but it is NOT the -parity target — corpus-level BLEU in `compute_metrics()` is. +`verify()` reports both `sentence_spbleu` and `sentence_chrf`, and returns +`sentence_chrf(generation, [reference]) / 100` as the `reward` field. +`compute_metrics()` reports both metrics at corpus level. When `compute_comet: true`, `verify()` also dispatches a per-row score request to the persistent xCOMET-XXL actor pool and awaits the result @@ -43,6 +64,25 @@ its own `comet_score` (or `None` when the model produced an empty generation), and `compute_metrics()` aggregates those per-row scores into per-pair / cross-pair means and std-dev keys. +When `language_consistency_backend` is configured, `verify()` also computes a +per-row `language_consistency_score` in-process. WMT24++ selects the +`wmt24pp_cld2` backend, which reports the fraction (0–1) of the generation that +CLD2 attributes to the target language (`0.0` for empty or wrong-language +output). This is a language-level check, not a regional-variety check: +`ar_EG`/`ar_SA`, `fr_CA`/`fr_FR`, `pt_BR`/`pt_PT`, and `sw_KE`/`sw_TZ` +each collapse to one CLD2 language code. Consequently, this metric cannot +detect a translation written in the wrong regional variety within one of +those pairs. FLORES selects `flores_glotlid`, which reports the summed GlotLID +probability for the expected full FLORES target code and any configured +equivalent codes. Both run on CPU without Ray/GPU dependencies, and +`compute_metrics()` aggregates their output into `/language_consistency` keys +alongside spBLEU, chrF, and COMET. + +After aggregation, the server emits one warning listing every source-target +pair whose mean language-consistency score is below +`language_consistency_warning_threshold` (default: `50.0` on the 0–100 metric +scale). Each pair is printed with its score, ordered from lowest to highest. + ## COMET actor pool When `compute_comet: true`, `_ensure_comet_actors()` lazily spawns @@ -64,12 +104,12 @@ startup. The xCOMET-XXL actor pool requires the `extra_gpu` Ray resource, which is only advertised on multi-node SLURM deployments. Local / single-node -runs disable COMET via Hydra override and rely on corpus-BLEU only. +runs disable COMET via Hydra override and rely on local spBLEU and chrF. For an end-to-end SLURM run with COMET enabled, see the [`ns nemo_gym_rollouts` block in benchmarks/wmt24pp/README.md](../../benchmarks/wmt24pp/README.md#end-to-end-reproduction-on-a-slurm-cluster-via-nemo-skills). ```bash -# Running servers (BLEU-only locally; flip compute_comet=true on cluster) +# Running servers (spBLEU + chrF locally; flip compute_comet=true on cluster) gym env start \ --model-type vllm_model \ --resources-server wmt_translation \ @@ -96,10 +136,12 @@ COMET actor pool) and launches Gym in one shot, see the | `comet_model` | `Unbabel/XCOMET-XXL` | HF repo passed to `comet.download_model` | | `comet_batch_size` | `16` | Batch size for `model.predict` | | `comet_num_shards` | `8` | Number of CometActors in the pool; cap at the extra node's GPU count | +| `language_consistency_warning_threshold` | `50.0` | Warn for source-target pairs below this mean 0–100 language-consistency score | | `strip_reasoning` | `true` | Drop a `...` preamble before scoring | ## Licensing - Code: Apache 2.0 - `Unbabel/XCOMET-XXL`: check model card (CC-BY-NC 4.0 at time of writing) -- Dependencies: `sacrebleu` (Apache 2.0), `unbabel-comet` (Apache 2.0) +- Dependencies: `sacrebleu` (Apache 2.0), `sentencepiece` (Apache 2.0), + `unbabel-comet` (Apache 2.0) diff --git a/resources_servers/wmt_translation/app.py b/resources_servers/wmt_translation/app.py index 052f55d02eb..a5d34d563d9 100644 --- a/resources_servers/wmt_translation/app.py +++ b/resources_servers/wmt_translation/app.py @@ -16,32 +16,40 @@ Two scoring layers: - * ``verify()`` returns a per-sample sentence-BLEU reward (useful as an RL - signal) plus a per-rollout xCOMET-XXL ``comet_score``. The COMET score - is computed by a persistent Ray actor pool (one actor per GPU on the - extra_gpu node) that keeps the xCOMET-XXL checkpoint resident; each - verify() call awaits its own future before returning, so every - rollout in ``rollouts.jsonl`` carries a finite ``comet_score`` whenever - the model produced a non-empty translation. + * ``verify()`` returns per-sample sentence-chrF and sentence-spBLEU, using + chrF as the RL reward, plus a per-rollout xCOMET-XXL ``comet_score`` and a per-rollout + language-consistency ``language_consistency_score`` (fraction of the generation + detected as the target language). The COMET score is computed by a + persistent Ray actor pool (one actor per GPU on the extra_gpu node) that + keeps the xCOMET-XXL checkpoint resident; each verify() call awaits its + own future before returning, so every rollout in ``rollouts.jsonl`` + carries a finite ``comet_score`` whenever the model produced a non-empty + translation. Language-consistency backends are currently assumed to run + synchronously in-process on CPU. * ``compute_metrics(tasks)`` groups rollouts by ``(source_language, target_language, rollout_index)``, computes - corpus-BLEU with the language-specific sacrebleu tokenizer - (``13a`` default, ``ja-mecab``/``ko-mecab``/``zh`` as needed), and - aggregates the per-row COMET scores into per-pair + cross-pair means - (``xx->xx``, ``->xx``, ``xx->{tgt}``) with ``std_dev_across_runs``. + corpus-chrF and corpus-spBLEU, and + aggregates the per-row COMET and language-consistency scores into per-pair + + cross-pair + means (``xx->xx``, ``->xx``, ``xx->{tgt}``) with + ``std_dev_across_runs``. """ from __future__ import annotations import logging import threading +import unicodedata from collections import defaultdict from typing import Any, Dict, List, Optional, Tuple import ray from fastapi import FastAPI -from pydantic import PrivateAttr -from sacrebleu import corpus_bleu, sentence_bleu +from language_consistency import LanguageConsistencyBackend, get_language_consistency_backend +from pydantic import Field, PrivateAttr +from sacrebleu import corpus_bleu as corpus_spbleu +from sacrebleu import corpus_chrf, sentence_chrf +from sacrebleu.metrics import BLEU from nemo_gym.base_resources_server import ( BaseResourcesServerConfig, @@ -55,18 +63,21 @@ LOG = logging.getLogger(__name__) -# --- Tokenizer selection ------------------------------------------------------ -# ``13a`` is sacrebleu's default; ``ja-mecab`` / ``ko-mecab`` need sacrebleu's -# [ja]/[ko] extras installed; ``zh`` is built in. -_TOKENIZER_BY_LANG_PREFIX = { - "ja": "ja-mecab", - "ko": "ko-mecab", - "zh": "zh", -} +SPBLEU_TOKENIZER = "flores200" +BLEU_MIGRATION_WARNING = ( + "We switched from the default SacreBLEU tokenizer to the FLORES-200 SentencePiece tokenizer " + 'for computing BLEU. The new BLEU scores are reported as "spBLEU"—do NOT compare these spBLEU ' + "scores to prior BLEU scores reported by this benchmark in the past; they are not comparable. " + "Also be very careful when comparing them to BLEU scores reported in papers or elsewhere: the " + "tokenizer must be the same for the scores to be comparable. In general, we recommend using " + "chrF instead of BLEU because it performs at least as well and requires no tokenization, so " + "there is no chance of tokenizer mismatch." +) -def _tokenizer_for(target_language: str) -> str: - return _TOKENIZER_BY_LANG_PREFIX.get(target_language[:2], "13a") +def _normalize_for_scoring(text: str) -> str: + """Canonicalize equivalent Unicode sequences before metric comparison.""" + return unicodedata.normalize("NFC", text) # --- Thinking-preamble handling --------------------------------------------- @@ -74,8 +85,8 @@ def _tokenizer_for(target_language: str) -> str: # .... vLLM's reasoning parser strips the opening # tag but keeps the closing , so the raw response looks like # "We need to translate ... \nProlog" -# We must drop the preamble before scoring or corpus BLEU is computed -# against the reasoning text and collapses (~3x lower BLEU). +# We must drop the preamble before scoring so the reasoning text does not +# contaminate the translation metric. def _strip_reasoning_preamble(text: str) -> str: @@ -108,7 +119,7 @@ class WmtTranslationResourcesServerConfig(BaseResourcesServerConfig): Attributes: compute_comet: Run xCOMET-XXL inside ``verify()`` and aggregate in ``compute_metrics``. Default True. Turn off for smoke tests or - RL training runs where only BLEU is needed. + RL training runs where only local spBLEU/chrF scoring is needed. comet_model: HuggingFace repo for the COMET checkpoint. Resolved via ``comet.download_model`` (cached under HF_HOME). comet_batch_size: Batch size passed to ``model.predict``. @@ -116,6 +127,12 @@ class WmtTranslationResourcesServerConfig(BaseResourcesServerConfig): xCOMET-XXL once and serves score requests from the persistent actor pool. Each actor requests one ``extra_gpu`` Ray resource, so the upper limit is the extra node(s)' GPU count. + language_consistency_backend: Backend used to compute a per-rollout + language-consistency score in ``verify()``. ``None`` disables + language-consistency scoring. + language_consistency_warning_threshold: Emit an aggregation-time + warning for each source-target pair whose mean language-consistency + score is below this 0-100 threshold. strip_reasoning: When True, drop a ``...`` preamble before scoring. Required for reasoning models; safe to leave on for instruction-tuned models that don't emit reasoning traces. @@ -125,6 +142,8 @@ class WmtTranslationResourcesServerConfig(BaseResourcesServerConfig): comet_model: str = "Unbabel/XCOMET-XXL" comet_batch_size: int = 16 comet_num_shards: int = 8 + language_consistency_backend: Optional[str] = None + language_consistency_warning_threshold: float = Field(default=50.0, ge=0.0, le=100.0) strip_reasoning: bool = True @@ -144,12 +163,19 @@ class WmtTranslationVerifyRequest(WmtTranslationRunRequest, BaseVerifyRequest): class WmtTranslationVerifyResponse(WmtTranslationVerifyRequest, BaseVerifyResponse): # Model's translation, post-strip-reasoning if enabled. generation: str - # Per-sample sentence-BLEU, useful as a dense RL reward. - sentence_bleu: float + # Per-sample sentence-chrF, useful as a dense RL reward. + sentence_chrf: float + # Per-sample sentence-spBLEU, reported alongside chrF for comparison. + sentence_spbleu: float # Per-rollout xCOMET-XXL score (0–1). None for empty generations or # actor-pool failures; aggregate corpus COMET is computed in # compute_metrics(). comet_score: Optional[float] = None + # Per-rollout language-consistency score (0–1): fraction of the + # generation attributed to the target language. 0.0 for empty or + # wrong-language output; None when no language-consistency backend is + # configured. Aggregated in compute_metrics(). + language_consistency_score: Optional[float] = None # --- Ray COMET scoring -------------------------------------------------------- @@ -267,7 +293,7 @@ def __init__(self, gpu_idx: int, model_name: str): # Both download_model() and load_from_checkpoint() resolve # from the HF cache populated by the benchmark prepare step - # (see benchmarks/wmt24pp/prepare.py:_prefetch_comet_model). + # (see benchmarks/wmt24pp/prepare.py:prefetch_translation_models). # If the cache is missing, this falls back to fetching from # HF Hub at startup, subject to HF_HUB_OFFLINE. LOG.info("CometActor[%d]: loading %s on %s", gpu_idx, model_name, self._device) @@ -301,8 +327,16 @@ class WmtTranslationResourcesServer(SimpleResourcesServer): _comet_state_lock: Any = PrivateAttr(default=None) _comet_actor_idx: int = PrivateAttr(default=0) _comet_init_attempted: bool = PrivateAttr(default=False) + # Constructing the FLORES-200 tokenizer loads its SentencePiece model from + # disk. Keep one metric instance per server instead of repeating that work + # for every verify() call. + _sentence_spbleu_metric: Optional[BLEU] = PrivateAttr(default=None) + # Backend loading may import heavyweight dependencies such as GlotLID. + # Resolve the configured backend once per server and reuse it. + _language_consistency_backend: Optional[LanguageConsistencyBackend] = PrivateAttr(default=None) def setup_webserver(self) -> FastAPI: + LOG.warning(BLEU_MIGRATION_WARNING) return super().setup_webserver() def _ensure_comet_actors(self) -> None: @@ -375,8 +409,26 @@ def _dispatch_comet_score(self, src_text: str, generation: str, reference: str) LOG.exception("COMET actor.score.remote dispatch failed") return None + def _score_sentence_spbleu(self, generation: str, reference: str) -> float: + """Score one sentence while reusing the FLORES-200 tokenizer.""" + if self._sentence_spbleu_metric is None: + self._sentence_spbleu_metric = BLEU( + tokenize=SPBLEU_TOKENIZER, + effective_order=True, + ) + return self._sentence_spbleu_metric.sentence_score(generation, [reference]).score + + def _get_language_consistency_backend(self) -> Optional[LanguageConsistencyBackend]: + """Resolve and cache the configured language-consistency backend.""" + backend_name = self.config.language_consistency_backend + if backend_name is None: + return None + if self._language_consistency_backend is None: + self._language_consistency_backend = get_language_consistency_backend(backend_name) + return self._language_consistency_backend + async def verify(self, body: WmtTranslationVerifyRequest) -> WmtTranslationVerifyResponse: - """Return per-sample sentence-BLEU as the RL reward + per-row COMET. + """Return sentence spBLEU/chrF, with chrF as reward, plus per-row auxiliary scores. Dispatches a per-rollout COMET score on the persistent actor pool and awaits the future before returning, so each row in @@ -389,9 +441,10 @@ async def verify(self, body: WmtTranslationVerifyRequest) -> WmtTranslationVerif if self.config.compute_comet: self._ensure_comet_actors() + language_consistency_backend = self._get_language_consistency_backend() + raw = body.response.output_text or "" - # Drop the reasoning preamble before scoring so BLEU is computed - # against the actual translation only. + # Drop the reasoning preamble before scoring the actual translation. if self.config.strip_reasoning: raw = _strip_reasoning_preamble(raw) generation = raw.strip() @@ -400,14 +453,31 @@ async def verify(self, body: WmtTranslationVerifyRequest) -> WmtTranslationVerif **body.model_dump(), reward=0.0, generation="", - sentence_bleu=0.0, + sentence_chrf=0.0, + sentence_spbleu=0.0, + # Empty output is genuinely 0% target language. + language_consistency_score=(0.0 if language_consistency_backend is not None else None), ) - tokenize = _tokenizer_for(body.target_language) - # sentence_bleu returns a BLEUScore; .score is 0-100. - sent_score = sentence_bleu(generation, [body.translation], tokenize=tokenize).score + normalized_generation = _normalize_for_scoring(generation) + normalized_reference = _normalize_for_scoring(body.translation) + + # sentence_chrf returns a CHRFScore; .score is 0-100. + sentence_chrf_score = sentence_chrf(normalized_generation, [normalized_reference]).score + # Sentence spBLEU uses the FLORES-200 SentencePiece tokenizer. + sentence_spbleu_score = self._score_sentence_spbleu( + normalized_generation, + normalized_reference, + ) # Normalize to [0, 1] so the "reward" field stays conventional. - reward = sent_score / 100.0 + reward = sentence_chrf_score / 100.0 + + language_consistency_score: Optional[float] = None + if language_consistency_backend is not None: + language_consistency_score = language_consistency_backend( + generation, + body.target_language, + ) comet_score: Optional[float] = None if self.config.compute_comet: @@ -428,8 +498,10 @@ async def verify(self, body: WmtTranslationVerifyRequest) -> WmtTranslationVerif **body.model_dump(), reward=reward, generation=generation, - sentence_bleu=sent_score, + sentence_chrf=sentence_chrf_score, + sentence_spbleu=sentence_spbleu_score, comet_score=comet_score, + language_consistency_score=language_consistency_score, ) # --- COMET aggregation ---------------------------------------------------- @@ -459,19 +531,48 @@ def _collect_per_row_comet( continue comet_per_pair[(src, tgt)][k].append(float(score)) + def _collect_per_row_language_consistency( + self, + tasks: List[List[Dict[str, Any]]], + max_k: int, + language_consistency_per_pair: Dict[Tuple[str, str], List[List[float]]], + ) -> None: + """Read per-row ``language_consistency_score`` from rollout dicts and bucket by pair/k. + + verify() computes language_consistency_score in-process and stores it on each rollout + response, so by compute_metrics() the scores are already in ``tasks``. + This method just buckets them. + """ + for task_rollouts in tasks: + for k, rollout in enumerate(task_rollouts): + if k >= max_k: + break + score = rollout.get("language_consistency_score") + if score is None: + continue + src = rollout.get("source_language") + tgt = rollout.get("target_language") + if not src or not tgt: + continue + language_consistency_per_pair[(src, tgt)][k].append(float(score)) + # --- Aggregate metrics --------------------------------------------------- def compute_metrics(self, tasks: List[List[Dict[str, Any]]]) -> Dict[str, Any]: - """Compute corpus BLEU + (optional) COMET metrics. + """Compute corpus spBLEU and chrF plus optional auxiliary metrics. Output keys: - ->/bleu (mean across rollouts) - ->/bleu_std_dev_across_runs + ->/chrF (mean across rollouts) + ->/chrF_std_dev_across_runs + ->/spBLEU (mean across rollouts) + ->/spBLEU_std_dev_across_runs ->/comet (mean across rollouts) ->/comet_std_dev_across_runs - ->xx/bleu xx->xx/bleu xx->/bleu (aggregations) - ... same with /comet + ->/language_consistency (mean across rollouts) + ->/language_consistency_std_dev_across_runs + ->xx/chrF xx->xx/chrF xx->/chrF (aggregations) + ... same with /spBLEU, /comet, and /language_consistency """ if not tasks: return {} @@ -487,6 +588,7 @@ def compute_metrics(self, tasks: List[List[Dict[str, Any]]]) -> Dict[str, Any]: ) any_comet_rows = False + any_language_consistency_rows = False for task_rollouts in tasks: for k, rollout in enumerate(task_rollouts): if k >= max_k: @@ -500,19 +602,27 @@ def compute_metrics(self, tasks: List[List[Dict[str, Any]]]) -> Dict[str, Any]: per_pair_runs[(src, tgt)][k].append((mt, ref)) if self.config.compute_comet and rollout.get("comet_score") is not None: any_comet_rows = True - - # 2. Per-(src, tgt) corpus BLEU per rollout. - bleu_per_pair: Dict[Tuple[str, str], List[float]] = {} + if ( + self.config.language_consistency_backend is not None + and rollout.get("language_consistency_score") is not None + ): + any_language_consistency_rows = True + + # 2. Per-(src, tgt) corpus chrF and spBLEU per rollout. + chrf_per_pair: Dict[Tuple[str, str], List[float]] = {} + spbleu_per_pair: Dict[Tuple[str, str], List[float]] = {} for (src, tgt), runs in per_pair_runs.items(): - tokenize = _tokenizer_for(tgt) - per_run = [] + chrf_per_run = [] + spbleu_per_run = [] for run in runs: if not run: continue - preds = [mt for mt, _ in run] - refs = [ref for _, ref in run] - per_run.append(corpus_bleu(preds, [refs], tokenize=tokenize).score) - bleu_per_pair[(src, tgt)] = per_run + preds = [_normalize_for_scoring(mt) for mt, _ in run] + refs = [_normalize_for_scoring(ref) for _, ref in run] + chrf_per_run.append(corpus_chrf(preds, [refs]).score) + spbleu_per_run.append(corpus_spbleu(preds, [refs], tokenize=SPBLEU_TOKENIZER).score) + chrf_per_pair[(src, tgt)] = chrf_per_run + spbleu_per_pair[(src, tgt)] = spbleu_per_run # 3. COMET aggregation: bucket the per-row comet_score values that # verify() already populated. @@ -529,6 +639,27 @@ def compute_metrics(self, tasks: List[List[Dict[str, Any]]]) -> Dict[str, Any]: means.append(100.0 * sum(run_scores) / len(run_scores)) comet_mean_per_pair[pair_key] = means + # 3b. Language-consistency aggregation: bucket the per-row + # language_consistency_score values that verify() populated in-process. + language_consistency_per_pair: Dict[Tuple[str, str], List[List[float]]] = defaultdict( + lambda: [list() for _ in range(max_k)] + ) + if self.config.language_consistency_backend is not None and any_language_consistency_rows: + self._collect_per_row_language_consistency( + tasks=tasks, + max_k=max_k, + language_consistency_per_pair=language_consistency_per_pair, + ) + + # Per-rollout-index mean language-consistency score per (pair, k), then averaged across k. + language_consistency_mean_per_pair: Dict[Tuple[str, str], List[float]] = {} + for pair_key, per_run in language_consistency_per_pair.items(): + means = [] + for run_scores in per_run: + if run_scores: + means.append(100.0 * sum(run_scores) / len(run_scores)) + language_consistency_mean_per_pair[pair_key] = means + # 4. Build output dict with per-pair + cross-pair aggregations. metrics: Dict[str, Any] = {} all_pairs = sorted(per_pair_runs.keys()) @@ -544,12 +675,18 @@ def _mean_std(values: List[float]) -> Tuple[float, float]: return (mean, var**0.5) # Per-pair + low_language_consistency_pairs: List[Tuple[float, str, str]] = [] for src, tgt in all_pairs: pair_label = f"{src}->{tgt}" - bleu_runs = bleu_per_pair.get((src, tgt), []) - m, s = _mean_std(bleu_runs) - metrics[f"{pair_label}/bleu"] = m - metrics[f"{pair_label}/bleu_std_dev_across_runs"] = s + chrf_runs = chrf_per_pair.get((src, tgt), []) + m, s = _mean_std(chrf_runs) + metrics[f"{pair_label}/chrF"] = m + metrics[f"{pair_label}/chrF_std_dev_across_runs"] = s + + spbleu_runs = spbleu_per_pair.get((src, tgt), []) + m, s = _mean_std(spbleu_runs) + metrics[f"{pair_label}/spBLEU"] = m + metrics[f"{pair_label}/spBLEU_std_dev_across_runs"] = s if self.config.compute_comet: comet_runs = comet_mean_per_pair.get((src, tgt), []) @@ -558,22 +695,50 @@ def _mean_std(values: List[float]) -> Tuple[float, float]: metrics[f"{pair_label}/comet"] = cm metrics[f"{pair_label}/comet_std_dev_across_runs"] = cs + if self.config.language_consistency_backend is not None: + language_consistency_runs = language_consistency_mean_per_pair.get((src, tgt), []) + if language_consistency_runs: + lm, ls = _mean_std(language_consistency_runs) + metrics[f"{pair_label}/language_consistency"] = lm + metrics[f"{pair_label}/language_consistency_std_dev_across_runs"] = ls + if lm < self.config.language_consistency_warning_threshold: + low_language_consistency_pairs.append((lm, src, tgt)) + + if low_language_consistency_pairs: + threshold = self.config.language_consistency_warning_threshold + pair_scores = "\n".join( + f" {src} -> {tgt}: {score:.1f}" for score, src, tgt in sorted(low_language_consistency_pairs) + ) + LOG.warning( + "Warning - the following language pairs had language consistency scores < %.1f, " + "indicating the model is likely generating in the wrong language:\n%s", + threshold, + pair_scores, + ) + # Aggregations: xx->xx, ->xx, xx->{tgt}. For each, average per-run - # BLEU across the contributing pairs first (per-run mean of per-pair - # BLEU), then average across runs. + # metric across the contributing pairs first, then average across runs. def _aggregate(pair_filter) -> Dict[str, List[float]]: - """Return per-run aggregated BLEU/COMET across filtered pairs.""" + """Return per-run spBLEU/chrF/COMET/language-consistency aggregates.""" filtered_pairs = [p for p in all_pairs if pair_filter(p)] if not filtered_pairs: - return {"bleu": [], "comet": []} + return {"spbleu": [], "chrf": [], "comet": [], "language_consistency": []} # Align rollout-index across pairs: take the min number of rollouts # present across the pairs so we don't average over missing runs. - min_runs = min(len(bleu_per_pair.get(p, [])) for p in filtered_pairs) - bleu_runs = [] + min_runs = min(len(chrf_per_pair.get(p, [])) for p in filtered_pairs) + chrf_runs = [] for k in range(min_runs): - per_pair_k = [bleu_per_pair[p][k] for p in filtered_pairs if k < len(bleu_per_pair[p])] + per_pair_k = [chrf_per_pair[p][k] for p in filtered_pairs if k < len(chrf_per_pair[p])] + if per_pair_k: + chrf_runs.append(sum(per_pair_k) / len(per_pair_k)) + + spbleu_min = min(len(spbleu_per_pair.get(p, [])) for p in filtered_pairs) + spbleu_runs = [] + for k in range(spbleu_min): + per_pair_k = [spbleu_per_pair[p][k] for p in filtered_pairs if k < len(spbleu_per_pair[p])] if per_pair_k: - bleu_runs.append(sum(per_pair_k) / len(per_pair_k)) + spbleu_runs.append(sum(per_pair_k) / len(per_pair_k)) + comet_runs: List[float] = [] if self.config.compute_comet: comet_min = min( @@ -586,46 +751,101 @@ def _aggregate(pair_filter) -> Dict[str, List[float]]: ] if per_pair_k: comet_runs.append(sum(per_pair_k) / len(per_pair_k)) - return {"bleu": bleu_runs, "comet": comet_runs} + + language_consistency_runs: List[float] = [] + if self.config.language_consistency_backend is not None: + language_consistency_min = min( + (len(language_consistency_mean_per_pair.get(p, [])) for p in filtered_pairs), + default=0, + ) + for k in range(language_consistency_min): + per_pair_k = [ + language_consistency_mean_per_pair[p][k] + for p in filtered_pairs + if k < len(language_consistency_mean_per_pair.get(p, [])) + ] + if per_pair_k: + language_consistency_runs.append(sum(per_pair_k) / len(per_pair_k)) + + return { + "spbleu": spbleu_runs, + "chrf": chrf_runs, + "comet": comet_runs, + "language_consistency": language_consistency_runs, + } src_langs = sorted({p[0] for p in all_pairs}) tgt_langs = sorted({p[1] for p in all_pairs}) # xx->xx (global) agg = _aggregate(lambda p: True) - m, s = _mean_std(agg["bleu"]) - metrics["xx->xx/bleu"] = m - metrics["xx->xx/bleu_std_dev_across_runs"] = s + m, s = _mean_std(agg["chrf"]) + metrics["xx->xx/chrF"] = m + metrics["xx->xx/chrF_std_dev_across_runs"] = s + m, s = _mean_std(agg["spbleu"]) + metrics["xx->xx/spBLEU"] = m + metrics["xx->xx/spBLEU_std_dev_across_runs"] = s if agg["comet"]: m, s = _mean_std(agg["comet"]) metrics["xx->xx/comet"] = m metrics["xx->xx/comet_std_dev_across_runs"] = s + if agg["language_consistency"]: + m, s = _mean_std(agg["language_consistency"]) + metrics["xx->xx/language_consistency"] = m + metrics["xx->xx/language_consistency_std_dev_across_runs"] = s # ->xx and xx-> for src in src_langs: agg = _aggregate(lambda p, _s=src: p[0] == _s) - m, s = _mean_std(agg["bleu"]) - metrics[f"{src}->xx/bleu"] = m - metrics[f"{src}->xx/bleu_std_dev_across_runs"] = s + m, s = _mean_std(agg["chrf"]) + metrics[f"{src}->xx/chrF"] = m + metrics[f"{src}->xx/chrF_std_dev_across_runs"] = s + m, s = _mean_std(agg["spbleu"]) + metrics[f"{src}->xx/spBLEU"] = m + metrics[f"{src}->xx/spBLEU_std_dev_across_runs"] = s if agg["comet"]: m, s = _mean_std(agg["comet"]) metrics[f"{src}->xx/comet"] = m metrics[f"{src}->xx/comet_std_dev_across_runs"] = s + if agg["language_consistency"]: + m, s = _mean_std(agg["language_consistency"]) + metrics[f"{src}->xx/language_consistency"] = m + metrics[f"{src}->xx/language_consistency_std_dev_across_runs"] = s for tgt in tgt_langs: agg = _aggregate(lambda p, _t=tgt: p[1] == _t) - m, s = _mean_std(agg["bleu"]) - metrics[f"xx->{tgt}/bleu"] = m - metrics[f"xx->{tgt}/bleu_std_dev_across_runs"] = s + m, s = _mean_std(agg["chrf"]) + metrics[f"xx->{tgt}/chrF"] = m + metrics[f"xx->{tgt}/chrF_std_dev_across_runs"] = s + m, s = _mean_std(agg["spbleu"]) + metrics[f"xx->{tgt}/spBLEU"] = m + metrics[f"xx->{tgt}/spBLEU_std_dev_across_runs"] = s if agg["comet"]: m, s = _mean_std(agg["comet"]) metrics[f"xx->{tgt}/comet"] = m metrics[f"xx->{tgt}/comet_std_dev_across_runs"] = s + if agg["language_consistency"]: + m, s = _mean_std(agg["language_consistency"]) + metrics[f"xx->{tgt}/language_consistency"] = m + metrics[f"xx->{tgt}/language_consistency_std_dev_across_runs"] = s return metrics def get_key_metrics(self, agent_metrics: Dict[str, Any]) -> Dict[str, Any]: """Headline metrics: global + per-source aggregations.""" - keys_of_interest = ("xx->xx/bleu", "xx->xx/comet", "en->xx/bleu", "en->xx/comet") + keys_of_interest = ( + "xx->xx/chrF", + "xx->xx/spBLEU", + "xx->xx/comet", + "xx->xx/language_consistency", + "en->xx/chrF", + "en->xx/spBLEU", + "en->xx/comet", + "en->xx/language_consistency", + "eng_Latn->xx/chrF", + "eng_Latn->xx/spBLEU", + "eng_Latn->xx/comet", + "eng_Latn->xx/language_consistency", + ) return {k: agent_metrics[k] for k in keys_of_interest if k in agent_metrics} diff --git a/resources_servers/wmt_translation/configs/wmt_translation.yaml b/resources_servers/wmt_translation/configs/wmt_translation.yaml index cb13747dff1..c49c4fc4d31 100644 --- a/resources_servers/wmt_translation/configs/wmt_translation.yaml +++ b/resources_servers/wmt_translation/configs/wmt_translation.yaml @@ -4,13 +4,17 @@ wmt_translation: entrypoint: app.py domain: other verified: false - description: Machine-translation verifier computing corpus-level BLEU per language pair plus optional xCOMET-XXL neural QE via a Ray GPU actor. - value: Improves multilingual translation quality across BLEU and COMET-family metrics. + description: Machine-translation verifier computing corpus-level spBLEU and ChrF per language pair plus optional xCOMET-XXL neural QE via a Ray GPU actor. + value: Improves multilingual translation quality across spBLEU, ChrF, and COMET-family metrics. compute_comet: true comet_model: Unbabel/XCOMET-XXL comet_batch_size: 16 # Listed explicitly so OmegaConf struct mode accepts `++…comet_num_shards=N` CLI overrides. comet_num_shards: 8 + # Optional per-rollout language-consistency backend. + language_consistency_backend: null + # Warn once per aggregation for source-target pairs below this 0-100 score. + language_consistency_warning_threshold: 50.0 wmt_translation_simple_agent: responses_api_agents: diff --git a/resources_servers/wmt_translation/data/example_rollouts.jsonl b/resources_servers/wmt_translation/data/example_rollouts.jsonl index 027b56fcd50..1221769cd89 100644 --- a/resources_servers/wmt_translation/data/example_rollouts.jsonl +++ b/resources_servers/wmt_translation/data/example_rollouts.jsonl @@ -1,5 +1,5 @@ -{"responses_create_params":{"background":null,"include":null,"input":[{"content":"Translate the following segment into Japanese, without additional explanation.\n\nGood morning, how are you?","role":"user","type":"message"}],"instructions":null,"max_output_tokens":2048,"max_tool_calls":null,"metadata":null,"model":null,"parallel_tool_calls":true,"previous_response_id":null,"prompt":null,"reasoning":null,"service_tier":null,"store":null,"temperature":1.0,"text":null,"tool_choice":"auto","tools":[],"top_logprobs":null,"top_p":0.95,"truncation":null,"user":null,"stream":null},"response":{"id":"resp_e4258ff036114cb18d939a0f05e2b94b","created_at":1777574294.0,"error":null,"incomplete_details":null,"instructions":null,"metadata":null,"model":"/hf_models/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16","object":"response","output":[{"id":"rs_126f2b24e18e453c80d6a324e14504f9","summary":[{"text":"The user wants translation of \"Good morning, how are you?\" into Japanese, without additional explanation. So just output the Japanese translation. Probably \"おはようございます、どうですか?\" or \"おはようございます、調子はどうですか?\" The phrase \"how are you?\" could be \"お元気ですか?\" Usually \"Good morning, how are you?\" => \"おはようございます。お元気ですか?\" But they want a single translation likely: \"おはようございます。調子はどうですか?\" The instruction: \"without additional explanation\". So just output the translation. Probably \"おはようございます、調子はどうですか?\". Let's output that.\n","type":"summary_text"}],"type":"reasoning","encrypted_content":null},{"id":"msg_6d654e8e1a954076a3628202c8658bb3","content":[{"annotations":[],"text":"\nおはようございます、調子はどうですか?","type":"output_text","logprobs":null}],"role":"assistant","status":"completed","type":"message"}],"parallel_tool_calls":true,"temperature":1.0,"tool_choice":"auto","tools":[],"top_p":0.95,"background":null,"conversation":null,"max_output_tokens":2048,"max_tool_calls":null,"previous_response_id":null,"prompt":null,"prompt_cache_key":null,"reasoning":null,"safety_identifier":null,"service_tier":null,"status":null,"text":null,"top_logprobs":null,"truncation":null,"usage":{"input_tokens":34,"input_tokens_details":{"cached_tokens":0},"output_tokens":145,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":179},"user":null},"reward":0.3799178428257963,"text":"Good morning, how are you?","translation":"おはようございます、お元気ですか?","source_language":"en","target_language":"ja_JP","source_lang_name":"English","target_lang_name":"Japanese","generation":"おはようございます、調子はどうですか?","sentence_bleu":37.99178428257963,"comet_score":0.9493920803070068,"_ng_task_index":4,"_ng_rollout_index":0,"agent_ref":{"type":"responses_api_agents","name":"wmt_translation_simple_agent"}} -{"responses_create_params":{"background":null,"include":null,"input":[{"content":"Translate the following segment into French, without additional explanation.\n\nI would like a cup of coffee.","role":"user","type":"message"}],"instructions":null,"max_output_tokens":2048,"max_tool_calls":null,"metadata":null,"model":null,"parallel_tool_calls":true,"previous_response_id":null,"prompt":null,"reasoning":null,"service_tier":null,"store":null,"temperature":1.0,"text":null,"tool_choice":"auto","tools":[],"top_logprobs":null,"top_p":0.95,"truncation":null,"user":null,"stream":null},"response":{"id":"resp_37e2a90e6d80412fbc65ddac62ed5eab","created_at":1777574290.0,"error":null,"incomplete_details":null,"instructions":null,"metadata":null,"model":"/hf_models/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16","object":"response","output":[{"id":"rs_144a7ed952ee4dbb90336c913fec07f7","summary":[{"text":"We need to translate to French, no extra explanation. Just translation. \"I would like a cup of coffee.\" French: \"Je voudrais une tasse de café.\" Could also be \"Je voudrais une tasse de café.\" That's fine. Provide only translation.\n","type":"summary_text"}],"type":"reasoning","encrypted_content":null},{"id":"msg_24dd4237131c4b7b94a9e5304e94aacc","content":[{"annotations":[],"text":"\nJe voudrais une tasse de café.","type":"output_text","logprobs":null}],"role":"assistant","status":"completed","type":"message"}],"parallel_tool_calls":true,"temperature":1.0,"tool_choice":"auto","tools":[],"top_p":0.95,"background":null,"conversation":null,"max_output_tokens":2048,"max_tool_calls":null,"previous_response_id":null,"prompt":null,"prompt_cache_key":null,"reasoning":null,"safety_identifier":null,"service_tier":null,"status":null,"text":null,"top_logprobs":null,"truncation":null,"usage":{"input_tokens":35,"input_tokens_details":{"cached_tokens":0},"output_tokens":62,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":97},"user":null},"reward":1.0000000000000004,"text":"I would like a cup of coffee.","translation":"Je voudrais une tasse de café.","source_language":"en","target_language":"fr_FR","source_lang_name":"English","target_lang_name":"French","generation":"Je voudrais une tasse de café.","sentence_bleu":100.00000000000004,"comet_score":1.0,"_ng_task_index":2,"_ng_rollout_index":0,"agent_ref":{"type":"responses_api_agents","name":"wmt_translation_simple_agent"}} -{"responses_create_params":{"background":null,"include":null,"input":[{"content":"Translate the following segment into German, without additional explanation.\n\nThe weather is lovely today.","role":"user","type":"message"}],"instructions":null,"max_output_tokens":2048,"max_tool_calls":null,"metadata":null,"model":null,"parallel_tool_calls":true,"previous_response_id":null,"prompt":null,"reasoning":null,"service_tier":null,"store":null,"temperature":1.0,"text":null,"tool_choice":"auto","tools":[],"top_logprobs":null,"top_p":0.95,"truncation":null,"user":null,"stream":null},"response":{"id":"resp_fce025ad94ba4e228b80a5b46c7aa17c","created_at":1777574290.0,"error":null,"incomplete_details":null,"instructions":null,"metadata":null,"model":"/hf_models/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16","object":"response","output":[{"id":"rs_d2210c2350364346bde02517c05777ef","summary":[{"text":"User wants translation into German, no additional explanation. Provide German translation. Probably \"Das Wetter ist heute toll.\" Or \"Das Wetter ist heute schön.\" Both acceptable. They said \"wunderbar\" maybe \"wunderbar\" but translation \"Das Wetter ist heute schön.\" Provide just that.\n","type":"summary_text"}],"type":"reasoning","encrypted_content":null},{"id":"msg_213266b664c547779e8c7d2130067fa9","content":[{"annotations":[],"text":"\nDas Wetter ist heute schön.","type":"output_text","logprobs":null}],"role":"assistant","status":"completed","type":"message"}],"parallel_tool_calls":true,"temperature":1.0,"tool_choice":"auto","tools":[],"top_p":0.95,"background":null,"conversation":null,"max_output_tokens":2048,"max_tool_calls":null,"previous_response_id":null,"prompt":null,"prompt_cache_key":null,"reasoning":null,"safety_identifier":null,"service_tier":null,"status":null,"text":null,"top_logprobs":null,"truncation":null,"usage":{"input_tokens":33,"input_tokens_details":{"cached_tokens":0},"output_tokens":68,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":101},"user":null},"reward":1.0000000000000004,"text":"The weather is lovely today.","translation":"Das Wetter ist heute schön.","source_language":"en","target_language":"de_DE","source_lang_name":"English","target_lang_name":"German","generation":"Das Wetter ist heute schön.","sentence_bleu":100.00000000000004,"comet_score":1.0,"_ng_task_index":0,"_ng_rollout_index":0,"agent_ref":{"type":"responses_api_agents","name":"wmt_translation_simple_agent"}} -{"responses_create_params":{"background":null,"include":null,"input":[{"content":"Translate the following segment into Spanish, without additional explanation.\n\nPlease pass the salt.","role":"user","type":"message"}],"instructions":null,"max_output_tokens":2048,"max_tool_calls":null,"metadata":null,"model":null,"parallel_tool_calls":true,"previous_response_id":null,"prompt":null,"reasoning":null,"service_tier":null,"store":null,"temperature":1.0,"text":null,"tool_choice":"auto","tools":[],"top_logprobs":null,"top_p":0.95,"truncation":null,"user":null,"stream":null},"response":{"id":"resp_4391155de9d14b3fa44fdfb09a00e34f","created_at":1777574290.0,"error":null,"incomplete_details":null,"instructions":null,"metadata":null,"model":"/hf_models/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16","object":"response","output":[{"id":"rs_4ef9c2c8d4fe42b59441cf4874ab9161","summary":[{"text":"User wants translation to Spanish, no additional explanation. So just output Spanish translation. \"Please pass the salt.\" => \"Por favor, pasa la sal.\" Could also be \"Por favor, pásame la sal.\" Both are okay. Probably \"Por favor, pasa la sal.\" I'll give that.\n","type":"summary_text"}],"type":"reasoning","encrypted_content":null},{"id":"msg_4452c03a9ae741aa99cb618ba7e6a448","content":[{"annotations":[],"text":"\nPor favor, pasa la sal.","type":"output_text","logprobs":null}],"role":"assistant","status":"completed","type":"message"}],"parallel_tool_calls":true,"temperature":1.0,"tool_choice":"auto","tools":[],"top_p":0.95,"background":null,"conversation":null,"max_output_tokens":2048,"max_tool_calls":null,"previous_response_id":null,"prompt":null,"prompt_cache_key":null,"reasoning":null,"safety_identifier":null,"service_tier":null,"status":null,"text":null,"top_logprobs":null,"truncation":null,"usage":{"input_tokens":32,"input_tokens_details":{"cached_tokens":0},"output_tokens":72,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":104},"user":null},"reward":0.4111336169005198,"text":"Please pass the salt.","translation":"Por favor, pásame la sal.","source_language":"en","target_language":"es_MX","source_lang_name":"English","target_lang_name":"Spanish","generation":"Por favor, pasa la sal.","sentence_bleu":41.11336169005198,"comet_score":0.8624756932258606,"_ng_task_index":1,"_ng_rollout_index":0,"agent_ref":{"type":"responses_api_agents","name":"wmt_translation_simple_agent"}} -{"responses_create_params":{"background":null,"include":null,"input":[{"content":"Translate the following segment into Italian, without additional explanation.\n\nThe train arrives at nine.","role":"user","type":"message"}],"instructions":null,"max_output_tokens":2048,"max_tool_calls":null,"metadata":null,"model":null,"parallel_tool_calls":true,"previous_response_id":null,"prompt":null,"reasoning":null,"service_tier":null,"store":null,"temperature":1.0,"text":null,"tool_choice":"auto","tools":[],"top_logprobs":null,"top_p":0.95,"truncation":null,"user":null,"stream":null},"response":{"id":"resp_747a0bcadaae4ef6a50cfb4841e4fd1e","created_at":1777574290.0,"error":null,"incomplete_details":null,"instructions":null,"metadata":null,"model":"/hf_models/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16","object":"response","output":[{"id":"rs_fc0bb648d30f499bb21d2fa5026224e6","summary":[{"text":"We need to translate \"The train arrives at nine.\" into Italian, without additional explanation. So just the Italian translation. Probably \"Il treno arriva alle nove.\" or \"Il treno arriva alle 9.\" Use Italian words. Probably \"Il treno arriva alle nove.\" Provide just that.\n","type":"summary_text"}],"type":"reasoning","encrypted_content":null},{"id":"msg_f775caf514ff402d879efd64a5bf3c7d","content":[{"annotations":[],"text":"\nIl treno arriva alle nove.","type":"output_text","logprobs":null}],"role":"assistant","status":"completed","type":"message"}],"parallel_tool_calls":true,"temperature":1.0,"tool_choice":"auto","tools":[],"top_p":0.95,"background":null,"conversation":null,"max_output_tokens":2048,"max_tool_calls":null,"previous_response_id":null,"prompt":null,"prompt_cache_key":null,"reasoning":null,"safety_identifier":null,"service_tier":null,"status":null,"text":null,"top_logprobs":null,"truncation":null,"usage":{"input_tokens":33,"input_tokens_details":{"cached_tokens":0},"output_tokens":70,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":103},"user":null},"reward":1.0000000000000004,"text":"The train arrives at nine.","translation":"Il treno arriva alle nove.","source_language":"en","target_language":"it_IT","source_lang_name":"English","target_lang_name":"Italian","generation":"Il treno arriva alle nove.","sentence_bleu":100.00000000000004,"comet_score":1.0,"_ng_task_index":3,"_ng_rollout_index":0,"agent_ref":{"type":"responses_api_agents","name":"wmt_translation_simple_agent"}} +{"responses_create_params":{"background":null,"include":null,"input":[{"content":"Translate the following segment into Japanese, without additional explanation.\n\nGood morning, how are you?","role":"user","type":"message"}],"instructions":null,"max_output_tokens":2048,"max_tool_calls":null,"metadata":null,"model":null,"parallel_tool_calls":true,"previous_response_id":null,"prompt":null,"reasoning":null,"service_tier":null,"store":null,"temperature":1.0,"text":null,"tool_choice":"auto","tools":[],"top_logprobs":null,"top_p":0.95,"truncation":null,"user":null,"stream":null},"response":{"id":"resp_e4258ff036114cb18d939a0f05e2b94b","created_at":1777574294.0,"error":null,"incomplete_details":null,"instructions":null,"metadata":null,"model":"/hf_models/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16","object":"response","output":[{"id":"rs_126f2b24e18e453c80d6a324e14504f9","summary":[{"text":"The user wants translation of \"Good morning, how are you?\" into Japanese, without additional explanation. So just output the Japanese translation. Probably \"おはようございます、どうですか?\" or \"おはようございます、調子はどうですか?\" The phrase \"how are you?\" could be \"お元気ですか?\" Usually \"Good morning, how are you?\" => \"おはようございます。お元気ですか?\" But they want a single translation likely: \"おはようございます。調子はどうですか?\" The instruction: \"without additional explanation\". So just output the translation. Probably \"おはようございます、調子はどうですか?\". Let's output that.\n","type":"summary_text"}],"type":"reasoning","encrypted_content":null},{"id":"msg_6d654e8e1a954076a3628202c8658bb3","content":[{"annotations":[],"text":"\nおはようございます、調子はどうですか?","type":"output_text","logprobs":null}],"role":"assistant","status":"completed","type":"message"}],"parallel_tool_calls":true,"temperature":1.0,"tool_choice":"auto","tools":[],"top_p":0.95,"background":null,"conversation":null,"max_output_tokens":2048,"max_tool_calls":null,"previous_response_id":null,"prompt":null,"prompt_cache_key":null,"reasoning":null,"safety_identifier":null,"service_tier":null,"status":null,"text":null,"top_logprobs":null,"truncation":null,"usage":{"input_tokens":34,"input_tokens_details":{"cached_tokens":0},"output_tokens":145,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":179},"user":null},"reward":0.5987198339641038,"text":"Good morning, how are you?","translation":"おはようございます、お元気ですか?","source_language":"en","target_language":"ja_JP","source_lang_name":"English","target_lang_name":"Japanese","generation":"おはようございます、調子はどうですか?","comet_score":0.9493920803070068,"_ng_task_index":4,"_ng_rollout_index":0,"agent_ref":{"type":"responses_api_agents","name":"wmt_translation_simple_agent"},"sentence_chrf":59.87198339641038,"sentence_spbleu":45.180100180492246} +{"responses_create_params":{"background":null,"include":null,"input":[{"content":"Translate the following segment into French, without additional explanation.\n\nI would like a cup of coffee.","role":"user","type":"message"}],"instructions":null,"max_output_tokens":2048,"max_tool_calls":null,"metadata":null,"model":null,"parallel_tool_calls":true,"previous_response_id":null,"prompt":null,"reasoning":null,"service_tier":null,"store":null,"temperature":1.0,"text":null,"tool_choice":"auto","tools":[],"top_logprobs":null,"top_p":0.95,"truncation":null,"user":null,"stream":null},"response":{"id":"resp_37e2a90e6d80412fbc65ddac62ed5eab","created_at":1777574290.0,"error":null,"incomplete_details":null,"instructions":null,"metadata":null,"model":"/hf_models/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16","object":"response","output":[{"id":"rs_144a7ed952ee4dbb90336c913fec07f7","summary":[{"text":"We need to translate to French, no extra explanation. Just translation. \"I would like a cup of coffee.\" French: \"Je voudrais une tasse de café.\" Could also be \"Je voudrais une tasse de café.\" That's fine. Provide only translation.\n","type":"summary_text"}],"type":"reasoning","encrypted_content":null},{"id":"msg_24dd4237131c4b7b94a9e5304e94aacc","content":[{"annotations":[],"text":"\nJe voudrais une tasse de café.","type":"output_text","logprobs":null}],"role":"assistant","status":"completed","type":"message"}],"parallel_tool_calls":true,"temperature":1.0,"tool_choice":"auto","tools":[],"top_p":0.95,"background":null,"conversation":null,"max_output_tokens":2048,"max_tool_calls":null,"previous_response_id":null,"prompt":null,"prompt_cache_key":null,"reasoning":null,"safety_identifier":null,"service_tier":null,"status":null,"text":null,"top_logprobs":null,"truncation":null,"usage":{"input_tokens":35,"input_tokens_details":{"cached_tokens":0},"output_tokens":62,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":97},"user":null},"reward":1.0,"text":"I would like a cup of coffee.","translation":"Je voudrais une tasse de café.","source_language":"en","target_language":"fr_FR","source_lang_name":"English","target_lang_name":"French","generation":"Je voudrais une tasse de café.","comet_score":1.0,"_ng_task_index":2,"_ng_rollout_index":0,"agent_ref":{"type":"responses_api_agents","name":"wmt_translation_simple_agent"},"sentence_chrf":100.0,"sentence_spbleu":100.00000000000004} +{"responses_create_params":{"background":null,"include":null,"input":[{"content":"Translate the following segment into German, without additional explanation.\n\nThe weather is lovely today.","role":"user","type":"message"}],"instructions":null,"max_output_tokens":2048,"max_tool_calls":null,"metadata":null,"model":null,"parallel_tool_calls":true,"previous_response_id":null,"prompt":null,"reasoning":null,"service_tier":null,"store":null,"temperature":1.0,"text":null,"tool_choice":"auto","tools":[],"top_logprobs":null,"top_p":0.95,"truncation":null,"user":null,"stream":null},"response":{"id":"resp_fce025ad94ba4e228b80a5b46c7aa17c","created_at":1777574290.0,"error":null,"incomplete_details":null,"instructions":null,"metadata":null,"model":"/hf_models/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16","object":"response","output":[{"id":"rs_d2210c2350364346bde02517c05777ef","summary":[{"text":"User wants translation into German, no additional explanation. Provide German translation. Probably \"Das Wetter ist heute toll.\" Or \"Das Wetter ist heute schön.\" Both acceptable. They said \"wunderbar\" maybe \"wunderbar\" but translation \"Das Wetter ist heute schön.\" Provide just that.\n","type":"summary_text"}],"type":"reasoning","encrypted_content":null},{"id":"msg_213266b664c547779e8c7d2130067fa9","content":[{"annotations":[],"text":"\nDas Wetter ist heute schön.","type":"output_text","logprobs":null}],"role":"assistant","status":"completed","type":"message"}],"parallel_tool_calls":true,"temperature":1.0,"tool_choice":"auto","tools":[],"top_p":0.95,"background":null,"conversation":null,"max_output_tokens":2048,"max_tool_calls":null,"previous_response_id":null,"prompt":null,"prompt_cache_key":null,"reasoning":null,"safety_identifier":null,"service_tier":null,"status":null,"text":null,"top_logprobs":null,"truncation":null,"usage":{"input_tokens":33,"input_tokens_details":{"cached_tokens":0},"output_tokens":68,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":101},"user":null},"reward":1.0,"text":"The weather is lovely today.","translation":"Das Wetter ist heute schön.","source_language":"en","target_language":"de_DE","source_lang_name":"English","target_lang_name":"German","generation":"Das Wetter ist heute schön.","comet_score":1.0,"_ng_task_index":0,"_ng_rollout_index":0,"agent_ref":{"type":"responses_api_agents","name":"wmt_translation_simple_agent"},"sentence_chrf":100.0,"sentence_spbleu":100.00000000000004} +{"responses_create_params":{"background":null,"include":null,"input":[{"content":"Translate the following segment into Spanish, without additional explanation.\n\nPlease pass the salt.","role":"user","type":"message"}],"instructions":null,"max_output_tokens":2048,"max_tool_calls":null,"metadata":null,"model":null,"parallel_tool_calls":true,"previous_response_id":null,"prompt":null,"reasoning":null,"service_tier":null,"store":null,"temperature":1.0,"text":null,"tool_choice":"auto","tools":[],"top_logprobs":null,"top_p":0.95,"truncation":null,"user":null,"stream":null},"response":{"id":"resp_4391155de9d14b3fa44fdfb09a00e34f","created_at":1777574290.0,"error":null,"incomplete_details":null,"instructions":null,"metadata":null,"model":"/hf_models/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16","object":"response","output":[{"id":"rs_4ef9c2c8d4fe42b59441cf4874ab9161","summary":[{"text":"User wants translation to Spanish, no additional explanation. So just output Spanish translation. \"Please pass the salt.\" => \"Por favor, pasa la sal.\" Could also be \"Por favor, pásame la sal.\" Both are okay. Probably \"Por favor, pasa la sal.\" I'll give that.\n","type":"summary_text"}],"type":"reasoning","encrypted_content":null},{"id":"msg_4452c03a9ae741aa99cb618ba7e6a448","content":[{"annotations":[],"text":"\nPor favor, pasa la sal.","type":"output_text","logprobs":null}],"role":"assistant","status":"completed","type":"message"}],"parallel_tool_calls":true,"temperature":1.0,"tool_choice":"auto","tools":[],"top_p":0.95,"background":null,"conversation":null,"max_output_tokens":2048,"max_tool_calls":null,"previous_response_id":null,"prompt":null,"prompt_cache_key":null,"reasoning":null,"safety_identifier":null,"service_tier":null,"status":null,"text":null,"top_logprobs":null,"truncation":null,"usage":{"input_tokens":32,"input_tokens_details":{"cached_tokens":0},"output_tokens":72,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":104},"user":null},"reward":0.61984068577194,"text":"Please pass the salt.","translation":"Por favor, pásame la sal.","source_language":"en","target_language":"es_MX","source_lang_name":"English","target_lang_name":"Spanish","generation":"Por favor, pasa la sal.","comet_score":0.8624756932258606,"_ng_task_index":1,"_ng_rollout_index":0,"agent_ref":{"type":"responses_api_agents","name":"wmt_translation_simple_agent"},"sentence_chrf":61.984068577193995,"sentence_spbleu":35.64026463354184} +{"responses_create_params":{"background":null,"include":null,"input":[{"content":"Translate the following segment into Italian, without additional explanation.\n\nThe train arrives at nine.","role":"user","type":"message"}],"instructions":null,"max_output_tokens":2048,"max_tool_calls":null,"metadata":null,"model":null,"parallel_tool_calls":true,"previous_response_id":null,"prompt":null,"reasoning":null,"service_tier":null,"store":null,"temperature":1.0,"text":null,"tool_choice":"auto","tools":[],"top_logprobs":null,"top_p":0.95,"truncation":null,"user":null,"stream":null},"response":{"id":"resp_747a0bcadaae4ef6a50cfb4841e4fd1e","created_at":1777574290.0,"error":null,"incomplete_details":null,"instructions":null,"metadata":null,"model":"/hf_models/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16","object":"response","output":[{"id":"rs_fc0bb648d30f499bb21d2fa5026224e6","summary":[{"text":"We need to translate \"The train arrives at nine.\" into Italian, without additional explanation. So just the Italian translation. Probably \"Il treno arriva alle nove.\" or \"Il treno arriva alle 9.\" Use Italian words. Probably \"Il treno arriva alle nove.\" Provide just that.\n","type":"summary_text"}],"type":"reasoning","encrypted_content":null},{"id":"msg_f775caf514ff402d879efd64a5bf3c7d","content":[{"annotations":[],"text":"\nIl treno arriva alle nove.","type":"output_text","logprobs":null}],"role":"assistant","status":"completed","type":"message"}],"parallel_tool_calls":true,"temperature":1.0,"tool_choice":"auto","tools":[],"top_p":0.95,"background":null,"conversation":null,"max_output_tokens":2048,"max_tool_calls":null,"previous_response_id":null,"prompt":null,"prompt_cache_key":null,"reasoning":null,"safety_identifier":null,"service_tier":null,"status":null,"text":null,"top_logprobs":null,"truncation":null,"usage":{"input_tokens":33,"input_tokens_details":{"cached_tokens":0},"output_tokens":70,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":103},"user":null},"reward":1.0,"text":"The train arrives at nine.","translation":"Il treno arriva alle nove.","source_language":"en","target_language":"it_IT","source_lang_name":"English","target_lang_name":"Italian","generation":"Il treno arriva alle nove.","comet_score":1.0,"_ng_task_index":3,"_ng_rollout_index":0,"agent_ref":{"type":"responses_api_agents","name":"wmt_translation_simple_agent"},"sentence_chrf":100.0,"sentence_spbleu":100.00000000000004} diff --git a/resources_servers/wmt_translation/flores_glotlid_language_consistency.py b/resources_servers/wmt_translation/flores_glotlid_language_consistency.py new file mode 100644 index 00000000000..2dc30569f2e --- /dev/null +++ b/resources_servers/wmt_translation/flores_glotlid_language_consistency.py @@ -0,0 +1,91 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import fasttext +import numpy as np +from huggingface_hub import hf_hub_download + + +GLOTLID_REPO_ID = "cis-lmu/glotlid" +GLOTLID_MODEL_FILENAME = "model_v3.bin" + + +_MODEL_PATH = hf_hub_download(repo_id=GLOTLID_REPO_ID, filename=GLOTLID_MODEL_FILENAME) +_MODEL = fasttext.load_model(_MODEL_PATH) +_OUTPUT_MATRIX = _MODEL.get_output_matrix() +_LABEL_INDICES = {label: index for index, label in enumerate(_MODEL.get_labels())} + + +_LANG_CODE_OVERRIDES = { + "acm_Arab": ["acm_Arab", "arb_Arab", "ary_Arab"], + "acq_Arab": ["arb_Arab", "ars_Arab", "arz_Arab"], + "aeb_Arab": ["aeb_Arab", "arb_Arab"], + "apc_Arab": ["arb_Arab", "apc_Arab", "arz_Arab", "ajp_Arab"], + "apd_Arab": ["arz_Arab", "arb_Arab", "ajp_Arab", "apc_Arab"], + "ars_Arab": ["arb_Arab"], + "arz_Arab": ["arz_Arab", "arb_Arab"], + "bos_Latn": ["hrv_Latn", "bos_Latn"], + "cmn_Hans": ["cmn_Hani"], + "cmn_Hant": ["cmn_Hani"], + "dgo_Deva": ["doi_Deva"], + "dyu_Latn": ["dyu_Latn", "bam_Latn"], + "kaa_Latn": ["kaa_Latn", "crh_Latn", "uig_Latn"], + "khk_Mong": ["und_Mong"], + "ktu_Latn": ["kng_Latn"], + "lld_Latn": ["lld_Latn", "lmo_Latn"], + "pes_Arab": ["fas_Arab"], + "prs_Arab": ["fas_Arab"], + "quy_Latn": ["quy_Latn", "quz_Latn"], + "wuu_Hans": ["wuu_Hani"], + "yue_Hant": ["yue_Hani"], + "zgh_Tfng": ["zgh_Tfng", "taq_Tfng"], +} + + +def _predict_all_language_probabilities(text: str) -> np.ndarray: + """ + Return normalized GlotLID probabilities aligned with `_LABEL_INDICES`. + + This follows GlotLID's documented all-language inference path to produce + properly normalized output. + """ + sentence_vector = _MODEL.get_sentence_vector(text) + logits = _OUTPUT_MATRIX @ sentence_vector + unnormalized_probabilities = np.exp(logits.astype(np.float64) - np.max(logits)) + return unnormalized_probabilities / unnormalized_probabilities.sum() + + +def flores_glotlid_language_consistency_score(text: str, expected_language_code: str) -> float: + """ + Return the probability GlotLID assigns to the expected FLORES+ language. + + FLORES+ dialect suffixes are removed because GlotLID does not represent + them. For known ambiguous or differently named languages, probabilities + for all configured GlotLID codes are summed. + """ + text = " ".join(text.split()) + if not text: + return 0.0 + + flores_code = "_".join(expected_language_code.split("_")[:2]) + glotlid_codes = _LANG_CODE_OVERRIDES.get(flores_code, [flores_code]) + expected_indices = [ + _LABEL_INDICES[label] for code in glotlid_codes if (label := f"__label__{code}") in _LABEL_INDICES + ] + if not expected_indices: + return 0.0 + + probabilities = _predict_all_language_probabilities(text) + return float(probabilities[expected_indices].sum()) diff --git a/resources_servers/wmt_translation/language_consistency.py b/resources_servers/wmt_translation/language_consistency.py new file mode 100644 index 00000000000..208fa1a273c --- /dev/null +++ b/resources_servers/wmt_translation/language_consistency.py @@ -0,0 +1,55 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from typing import Callable, Protocol + + +class LanguageConsistencyBackend(Protocol): + def __call__(self, text: str, expected_language_code: str) -> float: + """Return a language-consistency score between 0.0 and 1.0.""" + + +def _load_wmt24pp_cld2() -> LanguageConsistencyBackend: + # Imported lazily so `pycld2` (a compiled extension without wheels on every + # Python version) is only required when this backend is actually selected — + # not merely by importing the wmt_translation server with the feature off. + from wmt24pp_cld2_language_consistency import wmt24pp_cld2_language_consistency_score + + return wmt24pp_cld2_language_consistency_score + + +def _load_flores_glotlid() -> LanguageConsistencyBackend: + # GlotLID loads a large model at module import, so defer importing it until + # this backend is explicitly selected. + from flores_glotlid_language_consistency import flores_glotlid_language_consistency_score + + return flores_glotlid_language_consistency_score + + +# name -> zero-arg loader that imports and returns the backend on demand. +_LANGUAGE_CONSISTENCY_BACKEND_LOADERS: dict[str, Callable[[], LanguageConsistencyBackend]] = { + "flores_glotlid": _load_flores_glotlid, + "wmt24pp_cld2": _load_wmt24pp_cld2, +} + + +def get_language_consistency_backend(name: str) -> LanguageConsistencyBackend: + try: + loader = _LANGUAGE_CONSISTENCY_BACKEND_LOADERS[name] + except KeyError as exc: + available_backends = ", ".join(sorted(_LANGUAGE_CONSISTENCY_BACKEND_LOADERS)) + raise ValueError( + f"Unknown language-consistency backend {name!r}; available backends: {available_backends}" + ) from exc + return loader() diff --git a/resources_servers/wmt_translation/requirements.txt b/resources_servers/wmt_translation/requirements.txt index 33207c172a6..45a16849347 100644 --- a/resources_servers/wmt_translation/requirements.txt +++ b/resources_servers/wmt_translation/requirements.txt @@ -5,9 +5,19 @@ setuptools>=70 torch==2.12.0 torchvision==0.27.0 -sacrebleu[ja,ko]>=2.4 +sacrebleu>=2.4 +sentencepiece unbabel-comet>=2.2 +# GlotLID scoring dependencies. +numpy +fasttext-wheel==0.9.2 +huggingface-hub # scipy is a transitive dep of unbabel-comet (via comet.models.metrics → # scipy.stats) but uv's resolution doesn't pull it automatically here — # explicit pin avoids ModuleNotFoundError in the COMET Ray task. scipy +# Compact Language Detector v2 (Google CLD2) Python binding. Pure C++ +# extension with no Python deps. Prebuilt wheels exist through cp313, so on +# the repo-pinned 3.12 (and 3.13) it installs from a wheel. On 3.14 there is +# no wheel yet but it could be built from CLD2 source (needs a C++ compiler). +pycld2>=0.42 diff --git a/resources_servers/wmt_translation/scripts/generate_flores_language_metadata.py b/resources_servers/wmt_translation/scripts/generate_flores_language_metadata.py new file mode 100644 index 00000000000..5adccbd0de1 --- /dev/null +++ b/resources_servers/wmt_translation/scripts/generate_flores_language_metadata.py @@ -0,0 +1,182 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +uv run --isolated --no-project \ + --with datasets \ + --with huggingface-hub \ + --with 'langcodes[data]' \ + python generate_flores_language_metadata.py + +Generate the checked-in FLORES+ language-name and devtest-availability JSON +files consumed by benchmarks/flores200/prepare.py. +""" + +import argparse +import json +import re +from collections import defaultdict +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path + +import langcodes +from datasets import get_dataset_config_names, get_dataset_split_names +from huggingface_hub import hf_hub_download + + +FLORES_REPO_ID = "openlanguagedata/flores_plus" +REPO_ROOT = Path(__file__).resolve().parents[3] +DEFAULT_OUTPUT_DIR = REPO_ROOT / "benchmarks" / "flores200" / "data" +LANGUAGE_NAMES_FILENAME = "flores_language_names.json" +DEVTEST_LANGUAGES_FILENAME = "flores_devtest_languages.json" +DEFAULT_WORKERS = 16 + + +def _plain_text(markdown: str) -> str: + markdown = re.sub(r"\[([^\]]+)\]\([^)]+\)", r"\1", markdown) + return markdown.replace("`", "").strip() + + +def _read_names_by_code_and_script(revision: str | None) -> dict[tuple[str, str], list[tuple[str, str | None, str]]]: + readme_path = hf_hub_download( + repo_id=FLORES_REPO_ID, + filename="README.md", + repo_type="dataset", + revision=revision, + ) + + names_by_code_and_script = defaultdict(list) + with open(readme_path, encoding="utf-8") as readme: + for line in readme: + cells = [cell.strip() for cell in line.strip().strip("|").split("|")] + if len(cells) < 4 or not cells[0].startswith("`"): + continue + + language_code = _plain_text(cells[0]) + script_code = _plain_text(cells[1]) + glottocode = _plain_text(cells[2]).split()[0] + english_name = _plain_text(cells[3]) + variant_match = re.search(r"variant=([a-zA-Z0-9_-]+)", cells[2]) + variant = variant_match.group(1) if variant_match else None + names_by_code_and_script[(language_code, script_code)].append((glottocode, variant, english_name)) + return names_by_code_and_script + + +def _resolve_english_name( + flores_code: str, + names_by_code_and_script: dict[tuple[str, str], list[tuple[str, str | None, str]]], +) -> str: + language_code, script_code, *dialect = flores_code.split("_") + candidates = names_by_code_and_script[(language_code, script_code)] + dialect_code = "_".join(dialect) or None + + if dialect_code is None: + matches = [name for _glottocode, variant, name in candidates if variant is None] + if len(matches) > 1: + canonical_name = langcodes.Language.get(language_code).display_name() + matching_canonical_names = [ + name for name in matches if name == canonical_name or name.startswith(f"{canonical_name} (") + ] + matches = matching_canonical_names if len(matching_canonical_names) == 1 else [canonical_name] + else: + matches = [name for glottocode, variant, name in candidates if dialect_code in (glottocode, variant)] + + if len(matches) != 1: + raise ValueError(f"Could not uniquely resolve an English name for {flores_code}: {candidates}") + return matches[0] + + +def _provides_devtest(flores_code: str, revision: str | None) -> bool: + return "devtest" in get_dataset_split_names( + FLORES_REPO_ID, + flores_code, + revision=revision, + ) + + +def generate_metadata( + output_dir: Path, + revision: str | None = None, + workers: int = DEFAULT_WORKERS, +) -> tuple[Path, Path]: + names_by_code_and_script = _read_names_by_code_and_script(revision) + flores_configs = [ + config for config in sorted(get_dataset_config_names(FLORES_REPO_ID, revision=revision)) if config != "default" + ] + + language_names = { + flores_code: _resolve_english_name( + flores_code, + names_by_code_and_script, + ) + for flores_code in flores_configs + } + with ThreadPoolExecutor(max_workers=workers) as executor: + provides_devtest = executor.map( + lambda flores_code: _provides_devtest(flores_code, revision), + flores_configs, + ) + devtest_languages = [ + flores_code + for flores_code, is_supported in zip( + flores_configs, + provides_devtest, + strict=True, + ) + if is_supported + ] + + output_dir.mkdir(parents=True, exist_ok=True) + language_names_path = output_dir / LANGUAGE_NAMES_FILENAME + devtest_languages_path = output_dir / DEVTEST_LANGUAGES_FILENAME + with language_names_path.open("w", encoding="utf-8") as output: + json.dump(language_names, output, ensure_ascii=False, indent=2, sort_keys=True) + output.write("\n") + with devtest_languages_path.open("w", encoding="utf-8") as output: + json.dump(devtest_languages, output, ensure_ascii=False, indent=2) + output.write("\n") + + print(f"Wrote {len(language_names)} language names to {language_names_path}") + print(f"Wrote {len(devtest_languages)} devtest language codes to {devtest_languages_path}") + return language_names_path, devtest_languages_path + + +def _parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--output-dir", + type=Path, + default=DEFAULT_OUTPUT_DIR, + help="Directory in which to write the generated JSON metadata.", + ) + parser.add_argument( + "--revision", + help="Optional FLORES+ Hugging Face revision or commit to query.", + ) + parser.add_argument( + "--workers", + type=int, + default=DEFAULT_WORKERS, + help="Number of concurrent split-availability queries.", + ) + return parser.parse_args() + + +if __name__ == "__main__": + args = _parse_args() + generate_metadata( + args.output_dir, + revision=args.revision, + workers=args.workers, + ) diff --git a/resources_servers/wmt_translation/scripts/generate_flores_to_glotlid_mapping.py b/resources_servers/wmt_translation/scripts/generate_flores_to_glotlid_mapping.py new file mode 100644 index 00000000000..77d0151c0b8 --- /dev/null +++ b/resources_servers/wmt_translation/scripts/generate_flores_to_glotlid_mapping.py @@ -0,0 +1,71 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Build a mapping from FLORES+ to GlotLID language codes, +using a data-driven approach to overcome both inconsistencies +in language-code naming and languages/dialects that GlotLID is not +able to distinguish accurately. +""" + +from collections import Counter +from pprint import pprint + +import fasttext +from datasets import get_dataset_config_names, load_dataset +from huggingface_hub import hf_hub_download + + +FLORES_REPO_ID = "openlanguagedata/flores_plus" +GLOTLID_REPO_ID = "cis-lmu/glotlid" +GLOTLID_MODEL_FILENAME = "model_v3.bin" +MASS_THRESHOLD = 0.80 + + +model_path = hf_hub_download(repo_id=GLOTLID_REPO_ID, filename=GLOTLID_MODEL_FILENAME) +model = fasttext.load_model(model_path) +counts_by_flores_code: dict[str, Counter[str]] = {} + +for flores_config in sorted(get_dataset_config_names(FLORES_REPO_ID)): + # "default" combines every language dataset and is not itself a language code. + if flores_config == "default": + continue + + flores_code = "_".join(flores_config.split("_")[:2]) # drop optional dialect + label_counts = counts_by_flores_code.setdefault(flores_code, Counter()) + splits = load_dataset(FLORES_REPO_ID, flores_config) + dataset = splits["devtest" if "devtest" in splits else "dev"] # naming is inconsistent + + for text in dataset["text"]: + text = " ".join(text.split()) + # The internal binding avoids fasttext-wheel's NumPy 2-incompatible public wrapper. + _probability, label = model.f.predict(text + "\n", 1, 0.0, "strict")[0] + label_counts[label.removeprefix("__label__")] += 1 + +exceptions = {} +for flores_code, label_counts in sorted(counts_by_flores_code.items()): + # Here mass is the fraction of sentence-level top-1 labels, not model confidence. + top_codes = [] + cumulative_count = 0 + for glotlid_code, count in label_counts.most_common(): + top_codes.append(glotlid_code) + cumulative_count += count + if cumulative_count / label_counts.total() > MASS_THRESHOLD: + break + + if top_codes != [flores_code]: # only track exceptions + exceptions[flores_code] = top_codes + +pprint(exceptions, sort_dicts=True) diff --git a/resources_servers/wmt_translation/scripts/generate_wmt24pp_language_names.py b/resources_servers/wmt_translation/scripts/generate_wmt24pp_language_names.py new file mode 100644 index 00000000000..9e2b80999a7 --- /dev/null +++ b/resources_servers/wmt_translation/scripts/generate_wmt24pp_language_names.py @@ -0,0 +1,75 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +uv run --isolated --no-project \ + --with datasets \ + --with 'langcodes[data]' \ + python generate_wmt24pp_language_names.py + +Generate the checked-in WMT24++ target-language-name JSON file consumed by +benchmarks/wmt24pp/prepare.py. +""" + +import argparse +import json +from pathlib import Path + +import langcodes +from datasets import get_dataset_config_names + + +HF_REPO_ID = "google/wmt24pp" +REPO_ROOT = Path(__file__).resolve().parents[3] +DEFAULT_OUTPUT_PATH = REPO_ROOT / "benchmarks" / "wmt24pp" / "data" / "wmt24pp_language_names.json" + + +def generate_language_names( + output_path: Path, + revision: str | None = None, +) -> Path: + language_names = {} + for language_pair in sorted(get_dataset_config_names(HF_REPO_ID, revision=revision)): + source_language, separator, target_language = language_pair.partition("-") + if source_language != "en" or not separator or not target_language: + raise ValueError(f"Unexpected WMT24++ configuration name: {language_pair}") + language_names[target_language] = langcodes.Language.get(target_language).display_name() + + output_path.parent.mkdir(parents=True, exist_ok=True) + with output_path.open("w", encoding="utf-8") as output: + json.dump(language_names, output, ensure_ascii=False, indent=2, sort_keys=True) + output.write("\n") + + print(f"Wrote {len(language_names)} language names to {output_path}") + return output_path + + +def _parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--output", + type=Path, + default=DEFAULT_OUTPUT_PATH, + help="Path at which to write the generated JSON metadata.", + ) + parser.add_argument( + "--revision", + help="Optional WMT24++ Hugging Face revision or commit to query.", + ) + return parser.parse_args() + + +if __name__ == "__main__": + args = _parse_args() + generate_language_names(args.output, revision=args.revision) diff --git a/resources_servers/wmt_translation/scripts/validate_cld2_and_wmt24pp_language_code_mappings.py b/resources_servers/wmt_translation/scripts/validate_cld2_and_wmt24pp_language_code_mappings.py new file mode 100644 index 00000000000..a9a59bda064 --- /dev/null +++ b/resources_servers/wmt_translation/scripts/validate_cld2_and_wmt24pp_language_code_mappings.py @@ -0,0 +1,127 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +This script: +1) downloads the wmt24pp data and makes sure each wmt24pp + language is present in _WMT24PP_TO_CLD2_CODES. + +2) For each language: + 2.a) Run CLD2 on the entire wmt24pp human reference concatenated + together and raise an error if it is not classified + as the expected language + 2.b) Run CLD2 on each sentence in the wmt24pp human reference + and raise an error if the average per-sentence score is + below an allowed minimum. + +This confirms that the mappings in _WMT24PP_TO_CLD2_CODES, +in combination with CLD2, are correct. + +These checks are too expensive to include in unit tests, +so they are provided instead as a standalone script. + +""" + +import sys +from pathlib import Path + +import pycld2 as cld2 +from datasets import get_dataset_config_names, load_dataset + + +_WMT_TRANSLATION_DIR = Path(__file__).resolve().parents[1] +if str(_WMT_TRANSLATION_DIR) not in sys.path: + sys.path.insert(0, str(_WMT_TRANSLATION_DIR)) + +from wmt24pp_cld2_language_consistency import ( + _WMT24PP_TO_CLD2_CODES, + _sanitize_text, + wmt24pp_cld2_language_consistency_score, +) + + +def _cld2_predict(text: str) -> str: + """ + Return most likely CLD2 language code for given text + """ + text = _sanitize_text(text) # pycld2 rejects some control chars + _, _, details = cld2.detect(text, isPlainText=True) + cld2_lang_code = details[0][1] # top (name, code, percent, score) -> code + return cld2_lang_code + + +def _validate_language_mappings() -> None: + # worst observed average per-sentence language-consistency score was 0.64012 for Croatian (hr_HR) + minimum_average_per_sentence_language_consistency_score = 0.60 + minimum_fraction_sents_in_expected_language = 0.60 + + # Verify _WMT24PP_TO_CLD2_CODES has full coverage of the wmt24pp languages + available_wmt_targets = { + config_name.removeprefix("en-") + for config_name in get_dataset_config_names("google/wmt24pp") + if config_name.startswith("en-") + } + mapped_wmt_targets = set(_WMT24PP_TO_CLD2_CODES) + if available_wmt_targets != mapped_wmt_targets: + missing_mappings = sorted(available_wmt_targets - mapped_wmt_targets) + unknown_mappings = sorted(mapped_wmt_targets - available_wmt_targets) + raise AssertionError( + f"WMT24++/CLD2 mapping mismatch: missing mappings={missing_mappings}, unknown mappings={unknown_mappings}" + ) + else: + print("_WMT24PP_TO_CLD2_CODES mapping coverage: pass") + + for wmt_code, expected_cld2_code in _WMT24PP_TO_CLD2_CODES.items(): + per_sent_scores = [] + + rows = load_dataset("google/wmt24pp", f"en-{wmt_code}", split="train") + text = " ".join(r["target"] for r in rows if not r["is_bad_source"]) + for r in rows: + if not r["is_bad_source"]: + sent = r["target"] + score = wmt24pp_cld2_language_consistency_score(sent, wmt_code) + per_sent_scores.append(score) + + if not per_sent_scores: + raise AssertionError(f"{wmt_code}: no valid target sentences found") + + # 2) Verify all valid rows in the train split (concatenated together) return the expected language + actual_cld2_code = _cld2_predict(text) + if actual_cld2_code != expected_cld2_code: + raise AssertionError( + f"{wmt_code}: expected CLD2 {expected_cld2_code!r}, CLD2 detected {actual_cld2_code!r}" + ) + + # 3) Verify the average per-sentence language-consistency score is >= allowed minimum + average_score = sum(per_sent_scores) / len(per_sent_scores) + if average_score < minimum_average_per_sentence_language_consistency_score: + raise AssertionError( + f"{wmt_code}: average per-sentence language-consistency score {average_score:.5f} " + f"is below minimum {minimum_average_per_sentence_language_consistency_score:.2f}" + ) + + fraction_sents_in_expected_language = sum(score > 0.5 for score in per_sent_scores) / len(per_sent_scores) + if fraction_sents_in_expected_language <= minimum_fraction_sents_in_expected_language: + raise AssertionError( + f"{wmt_code}: fraction of sentences in the expected language " + f"{fraction_sents_in_expected_language:.5f} is not greater than minimum " + f"{minimum_fraction_sents_in_expected_language:.2f}" + ) + + print(f"{wmt_code} -> {expected_cld2_code} mapping: pass") + + +if __name__ == "__main__": + _validate_language_mappings() diff --git a/resources_servers/wmt_translation/scripts/validate_glotlid_and_flores_language_code_mappings.py b/resources_servers/wmt_translation/scripts/validate_glotlid_and_flores_language_code_mappings.py new file mode 100644 index 00000000000..74dbd56489a --- /dev/null +++ b/resources_servers/wmt_translation/scripts/validate_glotlid_and_flores_language_code_mappings.py @@ -0,0 +1,62 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Validate the FLORES+ GlotLID language-consistency scorer against FLORES+ text. + +For every FLORES+ language config, score each sentence in devtest when +available, otherwise dev, and raise an error if the average score is too low. +This check is too expensive for the default unit-test suite. +""" + +import sys +from pathlib import Path + +from datasets import get_dataset_config_names, load_dataset + + +_WMT_TRANSLATION_DIR = Path(__file__).resolve().parents[1] +if str(_WMT_TRANSLATION_DIR) not in sys.path: + sys.path.insert(0, str(_WMT_TRANSLATION_DIR)) + +from flores_glotlid_language_consistency import flores_glotlid_language_consistency_score + + +FLORES_REPO_ID = "openlanguagedata/flores_plus" +MINIMUM_AVERAGE_LANGUAGE_CONSISTENCY_SCORE = 0.60 + + +def _validate_language_consistency_scores() -> None: + flores_configs = [config for config in sorted(get_dataset_config_names(FLORES_REPO_ID)) if config != "default"] + + for flores_config in flores_configs: + splits = load_dataset(FLORES_REPO_ID, flores_config) + split = "devtest" if "devtest" in splits else "dev" + scores = [flores_glotlid_language_consistency_score(text, flores_config) for text in splits[split]["text"]] + if not scores: + raise AssertionError(f"{flores_config}: no sentences found in {split}") + + average_score = sum(scores) / len(scores) + if average_score < MINIMUM_AVERAGE_LANGUAGE_CONSISTENCY_SCORE: + raise AssertionError( + f"{flores_config}: average language-consistency score {average_score:.5f} " + f"is below minimum {MINIMUM_AVERAGE_LANGUAGE_CONSISTENCY_SCORE:.2f}" + ) + + print(f"{flores_config} ({split}): pass ({average_score:.5f})") + + +if __name__ == "__main__": + _validate_language_consistency_scores() diff --git a/resources_servers/wmt_translation/tests/test_app.py b/resources_servers/wmt_translation/tests/test_app.py index cc202c5092d..7f74a229da3 100644 --- a/resources_servers/wmt_translation/tests/test_app.py +++ b/resources_servers/wmt_translation/tests/test_app.py @@ -12,18 +12,22 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import logging import sys from pathlib import Path +from typing import Optional from unittest.mock import MagicMock, patch import pytest from app import ( + BLEU_MIGRATION_WARNING, + SPBLEU_TOKENIZER, WmtTranslationResourcesServer, WmtTranslationResourcesServerConfig, WmtTranslationVerifyRequest, _build_comet_actor_class, + _normalize_for_scoring, _strip_reasoning_preamble, - _tokenizer_for, ) from nemo_gym.openai_utils import NeMoGymResponse @@ -52,7 +56,11 @@ def _make_response(text: str) -> NeMoGymResponse: def _make_server( - compute_comet: bool = False, strip_reasoning: bool = False, comet_num_shards: int = 8 + compute_comet: bool = False, + strip_reasoning: bool = False, + comet_num_shards: int = 8, + language_consistency_backend: Optional[str] = "wmt24pp_cld2", + language_consistency_warning_threshold: float = 50.0, ) -> WmtTranslationResourcesServer: # Tests default strip_reasoning=False so plain-text generations score # against the reference directly. Production default is True (drops the @@ -65,6 +73,8 @@ def _make_server( compute_comet=compute_comet, strip_reasoning=strip_reasoning, comet_num_shards=comet_num_shards, + language_consistency_backend=language_consistency_backend, + language_consistency_warning_threshold=language_consistency_warning_threshold, ) return WmtTranslationResourcesServer(config=config, server_client=MagicMock(spec=ServerClient)) @@ -115,23 +125,60 @@ def test_empty_input(self) -> None: assert _strip_reasoning_preamble("") == "" -class TestTokenizer: - def test_default_tokenizer(self) -> None: - assert _tokenizer_for("de_DE") == "13a" - assert _tokenizer_for("fr_FR") == "13a" - assert _tokenizer_for("en") == "13a" +class TestScoringHelpers: + def test_normalizes_canonically_equivalent_text_to_nfc(self) -> None: + assert _normalize_for_scoring("Cafe\u0301") == "Café" - def test_japanese_tokenizer(self) -> None: - assert _tokenizer_for("ja_JP") == "ja-mecab" + def test_spbleu_always_uses_flores200_tokenizer(self) -> None: + assert SPBLEU_TOKENIZER == "flores200" - def test_korean_tokenizer(self) -> None: - assert _tokenizer_for("ko_KR") == "ko-mecab" - def test_chinese_tokenizer(self) -> None: - assert _tokenizer_for("zh_CN") == "zh" +class TestServerSetup: + def test_warns_that_spbleu_is_not_comparable_to_prior_bleu(self, caplog) -> None: + server = _make_server() + + with caplog.at_level(logging.WARNING, logger="app"): + server.setup_webserver() + + assert BLEU_MIGRATION_WARNING in caplog.messages class TestVerify: + @pytest.mark.parametrize("target_language", ["de_DE", "ja_JP", "zho_Hans"]) + async def test_spbleu_uses_flores200_for_every_target_language(self, target_language: str) -> None: + server = _make_server(language_consistency_backend=None) + request = _make_request( + text="Hello world.", + translation="Hallo Welt.", + generation="Hallo Welt.", + target_language=target_language, + ) + with patch("app.BLEU") as mock_bleu: + mock_bleu.return_value.sentence_score.return_value.score = 100.0 + result = await server.verify(request) + + assert result.sentence_spbleu == 100.0 + mock_bleu.assert_called_once_with(tokenize="flores200", effective_order=True) + mock_bleu.return_value.sentence_score.assert_called_once_with("Hallo Welt.", ["Hallo Welt."]) + + async def test_spbleu_metric_is_reused_across_verify_calls(self) -> None: + server = _make_server(language_consistency_backend=None) + request = _make_request( + text="Hello world.", + translation="Hallo Welt.", + generation="Hallo Welt.", + target_language="de_DE", + ) + with patch("app.BLEU") as mock_bleu: + mock_bleu.return_value.sentence_score.return_value.score = 100.0 + first_result = await server.verify(request) + second_result = await server.verify(request) + + assert first_result.sentence_spbleu == 100.0 + assert second_result.sentence_spbleu == 100.0 + mock_bleu.assert_called_once_with(tokenize="flores200", effective_order=True) + assert mock_bleu.return_value.sentence_score.call_count == 2 + async def test_empty_generation_scores_zero(self) -> None: server = _make_server() request = _make_request( @@ -142,8 +189,120 @@ async def test_empty_generation_scores_zero(self) -> None: ) result = await server.verify(request) assert result.reward == 0.0 - assert result.sentence_bleu == 0.0 + assert result.sentence_chrf == 0.0 + assert result.sentence_spbleu == 0.0 assert result.generation == "" + # Empty output is 0% target language. + assert result.language_consistency_score == 0.0 + + async def test_language_consistency_high_for_correct_language(self) -> None: + server = _make_server() + ref = "Der schnelle braune Fuchs springt über den faulen Hund im schönen Garten." + request = _make_request( + text="The quick brown fox jumps over the lazy dog in the beautiful garden.", + translation=ref, + generation=ref, + target_language="de_DE", + ) + result = await server.verify(request) + assert result.language_consistency_score is not None + assert result.language_consistency_score > 0.5 + + async def test_language_consistency_low_for_wrong_language(self) -> None: + server = _make_server() + request = _make_request( + text="The quick brown fox jumps over the lazy dog.", + translation="Der schnelle braune Fuchs springt über den faulen Hund.", + generation="This is clearly an English sentence, not German at all.", + target_language="de_DE", + ) + result = await server.verify(request) + assert result.language_consistency_score is not None + assert result.language_consistency_score < 0.5 + + @patch("app.get_language_consistency_backend") + async def test_language_consistency_backend_receives_full_flores_code( + self, mock_get_language_consistency_backend + ) -> None: + scorer = MagicMock(return_value=0.75) + mock_get_language_consistency_backend.return_value = scorer + server = _make_server(language_consistency_backend="flores_glotlid") + request = _make_request( + text="Hello.", + translation="مرحبًا.", + generation="مرحبًا.", + target_language="apc_Arab_sout3123", + ) + + result = await server.verify(request) + + mock_get_language_consistency_backend.assert_called_once_with("flores_glotlid") + scorer.assert_called_once_with("مرحبًا.", "apc_Arab_sout3123") + assert result.language_consistency_score == 0.75 + + @patch("app.get_language_consistency_backend") + async def test_language_consistency_backend_is_reused_across_verify_calls( + self, mock_get_language_consistency_backend + ) -> None: + scorer = MagicMock(return_value=0.75) + mock_get_language_consistency_backend.return_value = scorer + server = _make_server(language_consistency_backend="flores_glotlid") + request = _make_request( + text="Hello.", + translation="مرحبًا.", + generation="مرحبًا.", + target_language="apc_Arab_sout3123", + ) + + with patch.object(WmtTranslationResourcesServer, "_score_sentence_spbleu", return_value=100.0): + first_result = await server.verify(request) + second_result = await server.verify(request) + + mock_get_language_consistency_backend.assert_called_once_with("flores_glotlid") + assert scorer.call_args_list == [ + (("مرحبًا.", "apc_Arab_sout3123"),), + (("مرحبًا.", "apc_Arab_sout3123"),), + ] + assert first_result.language_consistency_score == 0.75 + assert second_result.language_consistency_score == 0.75 + + async def test_language_consistency_none_when_disabled(self) -> None: + server = _make_server(language_consistency_backend=None) + ref = "Der schnelle braune Fuchs springt über den faulen Hund im schönen Garten." + request = _make_request( + text="The quick brown fox jumps over the lazy dog in the beautiful garden.", + translation=ref, + generation=ref, + target_language="de_DE", + ) + result = await server.verify(request) + assert result.language_consistency_score is None + + @patch("app.get_language_consistency_backend") + async def test_language_consistency_none_when_no_backend_is_configured( + self, mock_get_language_consistency_backend + ) -> None: + server = _make_server(language_consistency_backend=None) + ref = "Der schnelle braune Fuchs springt über den faulen Hund im schönen Garten." + request = _make_request( + text="The quick brown fox jumps over the lazy dog in the beautiful garden.", + translation=ref, + generation=ref, + target_language="de_DE", + ) + result = await server.verify(request) + assert result.language_consistency_score is None + + empty_request = _make_request( + text="The quick brown fox jumps over the lazy dog in the beautiful garden.", + translation=ref, + generation="", + target_language="de_DE", + ) + empty_result = await server.verify(empty_request) + assert empty_result.language_consistency_score is None + + mock_get_language_consistency_backend.assert_not_called() async def test_perfect_generation_high_reward(self) -> None: server = _make_server() @@ -156,10 +315,25 @@ async def test_perfect_generation_high_reward(self) -> None: target_language="de_DE", ) result = await server.verify(request) - assert result.sentence_bleu > 50.0 - assert result.reward == result.sentence_bleu / 100.0 + assert result.sentence_chrf > 50.0 + assert result.sentence_spbleu > 50.0 + assert result.reward == result.sentence_chrf / 100.0 assert result.generation == ref + async def test_canonically_equivalent_unicode_scores_perfectly(self) -> None: + server = _make_server() + request = _make_request( + text="Coffee.", + translation="Café", + generation="Cafe\u0301", + target_language="fr_FR", + ) + result = await server.verify(request) + assert result.sentence_chrf == pytest.approx(100.0) + assert result.sentence_spbleu == pytest.approx(100.0) + assert result.reward == pytest.approx(1.0) + assert result.generation == "Cafe\u0301" + async def test_bad_generation_low_reward(self) -> None: server = _make_server() request = _make_request( @@ -169,7 +343,7 @@ async def test_bad_generation_low_reward(self) -> None: target_language="de_DE", ) result = await server.verify(request) - assert result.reward < 0.1 + assert result.reward < 0.2 async def test_strip_reasoning_recovers_score(self) -> None: """With strip_reasoning=True, a reasoning preamble must not @@ -187,9 +361,10 @@ async def test_strip_reasoning_recovers_score(self) -> None: target_language="de_DE", ) result = await server.verify(request) - # Exactly the reference should land in generation, giving near-perfect BLEU. + # Exactly the reference should produce a near-perfect score. assert result.generation == ref - assert result.sentence_bleu > 50.0 + assert result.sentence_chrf > 50.0 + assert result.sentence_spbleu > 50.0 async def test_strip_reasoning_empty_when_truncated_mid_reasoning(self) -> None: """If opens but never closes, verify() emits no generation.""" @@ -221,23 +396,59 @@ async def test_strip_reasoning_keeps_text_without_reasoning_tags(self) -> None: ) result = await server.verify(request) assert result.generation == ref - assert result.sentence_bleu > 50.0 + assert result.sentence_chrf > 50.0 + assert result.sentence_spbleu > 50.0 class TestComputeMetrics: + def test_corpus_spbleu_uses_flores200_for_every_language_pair(self) -> None: + server = _make_server(compute_comet=False) + tasks = [ + [ + { + "translation": "Hallo Welt.", + "generation": "Hallo Welt.", + "source_language": "en", + "target_language": target_language, + } + ] + for target_language in ("de_DE", "ja_JP", "zho_Hans") + ] + with patch("app.corpus_spbleu") as mock_spbleu: + mock_spbleu.return_value.score = 100.0 + server.compute_metrics(tasks) + + assert len(mock_spbleu.call_args_list) == 3 + assert all(call.kwargs["tokenize"] == "flores200" for call in mock_spbleu.call_args_list) + def test_empty_tasks(self) -> None: server = _make_server() assert server.compute_metrics([]) == {} - def test_bleu_per_pair_and_aggregations(self) -> None: - """Feed two language pairs x two rollouts each; expect per-pair BLEU, + def test_corpus_chrf_normalizes_canonically_equivalent_unicode(self) -> None: + server = _make_server(compute_comet=False) + metrics = server.compute_metrics( + [ + [ + { + "translation": "Café au lait chaud", + "generation": "Cafe\u0301 au lait chaud", + "source_language": "en", + "target_language": "fr_FR", + } + ] + ] + ) + assert metrics["en->fr_FR/chrF"] == pytest.approx(100.0) + assert metrics["en->fr_FR/spBLEU"] == pytest.approx(100.0) + + def test_spbleu_and_chrf_per_pair_and_aggregations(self) -> None: + """Feed two language pairs x two rollouts each; expect spBLEU and chrF cross-pair aggregates, and std-dev keys without COMET fields.""" server = _make_server(compute_comet=False) - # Sentences long enough that 4-gram matches exist (sacrebleu's - # corpus_bleu uses the default 4-gram geometric mean, which is 0 - # when any n-gram precision is 0). Two "tasks", 2 rollouts each: - # rollout 0 is perfect, rollout 1 is a slight variant, so std_dev - # across runs is non-zero but BLEU > 0 on both. + # Two tasks with two rollouts each: rollout 0 is perfect and rollout 1 + # is a slight variant, so the standard deviation across runs is + # non-zero while both runs have positive chrF. de_ref = "Der schnelle braune Fuchs springt \u00fcber den faulen Hund in dem sch\u00f6nen Garten." de_perfect = de_ref de_variant = "Der schnelle braune Fuchs springt \u00fcber den faulen Hund im sch\u00f6nen Garten." @@ -282,20 +493,30 @@ def test_bleu_per_pair_and_aggregations(self) -> None: ] m = server.compute_metrics(tasks) - # Per-pair BLEU - assert "en->de_DE/bleu" in m - assert "en->fr_FR/bleu" in m - assert m["en->de_DE/bleu"] > 0 - assert m["en->fr_FR/bleu"] > 0 + # Per-pair chrF + assert "en->de_DE/chrF" in m + assert "en->fr_FR/chrF" in m + assert m["en->de_DE/chrF"] > 0 + assert m["en->fr_FR/chrF"] > 0 # Std-dev keys exist for per-pair - assert "en->de_DE/bleu_std_dev_across_runs" in m - assert "en->fr_FR/bleu_std_dev_across_runs" in m + assert "en->de_DE/chrF_std_dev_across_runs" in m + assert "en->fr_FR/chrF_std_dev_across_runs" in m + + # Per-pair spBLEU + assert m["en->de_DE/spBLEU"] > 0 + assert m["en->fr_FR/spBLEU"] > 0 + assert "en->de_DE/spBLEU_std_dev_across_runs" in m + assert "en->fr_FR/spBLEU_std_dev_across_runs" in m # Cross-pair aggregations - assert "xx->xx/bleu" in m - assert "en->xx/bleu" in m - assert "xx->de_DE/bleu" in m - assert "xx->fr_FR/bleu" in m + assert "xx->xx/chrF" in m + assert "en->xx/chrF" in m + assert "xx->de_DE/chrF" in m + assert "xx->fr_FR/chrF" in m + assert "xx->xx/spBLEU" in m + assert "en->xx/spBLEU" in m + assert "xx->de_DE/spBLEU" in m + assert "xx->fr_FR/spBLEU" in m # No COMET when disabled assert not any(k.endswith("/comet") for k in m) @@ -316,23 +537,41 @@ def test_comet_disabled_does_not_call_ray(self) -> None: ] ] m = server.compute_metrics(tasks) - # BLEU is emitted; /comet keys are not. - assert "en->de_DE/bleu" in m + # spBLEU and chrF are emitted; /comet keys are not. + assert "en->de_DE/chrF" in m + assert "en->de_DE/spBLEU" in m for k in m: assert "/comet" not in k def test_get_key_metrics_filters(self) -> None: server = _make_server() agent = { - "xx->xx/bleu": 35.0, + "xx->xx/chrF": 35.0, + "xx->xx/spBLEU": 30.0, "xx->xx/comet": 78.0, - "en->xx/bleu": 32.0, + "en->xx/chrF": 32.0, + "en->xx/spBLEU": 27.0, "en->xx/comet": 77.0, - "en->de_DE/bleu": 30.0, # not in key metrics + "eng_Latn->xx/chrF": 31.0, + "eng_Latn->xx/spBLEU": 26.0, + "eng_Latn->xx/comet": 76.0, + "eng_Latn->xx/language_consistency": 96.0, + "en->de_DE/chrF": 30.0, # not in key metrics "mean/reward": 0.45, # not in key metrics } key = server.get_key_metrics(agent) - assert set(key.keys()) == {"xx->xx/bleu", "xx->xx/comet", "en->xx/bleu", "en->xx/comet"} + assert set(key.keys()) == { + "xx->xx/chrF", + "xx->xx/spBLEU", + "xx->xx/comet", + "en->xx/chrF", + "en->xx/spBLEU", + "en->xx/comet", + "eng_Latn->xx/chrF", + "eng_Latn->xx/spBLEU", + "eng_Latn->xx/comet", + "eng_Latn->xx/language_consistency", + } def test_per_row_comet_scores_emit_aggregate_metrics(self) -> None: """When rollouts carry comet_score (from verify()'s actor pool await), @@ -384,9 +623,143 @@ def test_per_row_comet_scores_emit_aggregate_metrics(self) -> None: assert m["xx->de_DE/comet"] == pytest.approx(90.0) assert m["xx->fr_FR/comet"] == pytest.approx(90.0) - def test_no_comet_rows_emits_bleu_only(self) -> None: + def test_per_row_language_consistency_scores_emit_aggregate_metrics(self) -> None: + """When rollouts carry language_consistency_score (computed in verify()), compute_metrics + buckets those values into per-pair and cross-pair aggregates.""" + server = _make_server( + compute_comet=False, + language_consistency_backend="wmt24pp_cld2", + ) + de_ref = "Der schnelle braune Fuchs springt über den faulen Hund im schönen Garten." + fr_ref = "Le renard brun rapide saute par dessus le chien paresseux dans le beau jardin." + tasks = [ + [ + { + "text": "T1", + "translation": de_ref, + "generation": de_ref, + "source_language": "en", + "target_language": "de_DE", + "language_consistency_score": 0.80, + }, + ], + [ + { + "text": "T2", + "translation": de_ref, + "generation": de_ref, + "source_language": "en", + "target_language": "de_DE", + "language_consistency_score": 1.00, + }, + ], + [ + { + "text": "T3", + "translation": fr_ref, + "generation": fr_ref, + "source_language": "en", + "target_language": "fr_FR", + "language_consistency_score": 0.90, + }, + ], + ] + m = server.compute_metrics(tasks) + # Per-pair: en->de_DE = mean(0.80, 1.00) × 100 = 90.0; en->fr_FR = 90.0 + assert m["en->de_DE/language_consistency"] == pytest.approx(90.0) + assert m["en->fr_FR/language_consistency"] == pytest.approx(90.0) + # Cross-pair aggregations. + assert m["xx->xx/language_consistency"] == pytest.approx(90.0) + assert m["en->xx/language_consistency"] == pytest.approx(90.0) + assert m["xx->de_DE/language_consistency"] == pytest.approx(90.0) + assert m["xx->fr_FR/language_consistency"] == pytest.approx(90.0) + # spBLEU and chrF are present; COMET is absent (disabled). + assert "en->de_DE/chrF" in m + assert "en->de_DE/spBLEU" in m + assert not any(k.endswith("/comet") for k in m) + + def test_no_language_consistency_rows_emits_no_language_consistency_keys(self) -> None: + """Rollouts without language_consistency_score yield no /language_consistency keys.""" + server = _make_server( + compute_comet=False, + language_consistency_backend="wmt24pp_cld2", + ) + tasks = [ + [ + { + "text": "The quick brown fox jumps over the lazy dog in the beautiful garden.", + "translation": "Der schnelle braune Fuchs springt über den faulen Hund im schönen Garten.", + "generation": "Der schnelle braune Fuchs springt über den faulen Hund im schönen Garten.", + "source_language": "en", + "target_language": "de_DE", + # No language_consistency_score field. + } + ] + ] + m = server.compute_metrics(tasks) + assert "en->de_DE/chrF" in m + assert "en->de_DE/spBLEU" in m + assert not any("/language_consistency" in k for k in m) + + def test_warns_for_low_language_consistency_pairs_with_scores(self, caplog) -> None: + server = _make_server( + compute_comet=False, + language_consistency_backend="flores_glotlid", + language_consistency_warning_threshold=50.0, + ) + tasks = [ + [ + { + "translation": "reference text", + "generation": "generated text", + "source_language": src, + "target_language": tgt, + "language_consistency_score": score, + } + ] + for src, tgt, score in ( + ("en", "jpn_Jpan", 0.40), + ("de", "jpn_Jpan", 0.20), + ("en", "fra_Latn", 0.50), + ) + ] + + with caplog.at_level(logging.WARNING, logger="app"): + server.compute_metrics(tasks) + + warning = next( + message for message in caplog.messages if message.startswith("Warning - the following language pairs") + ) + assert "scores < 50.0" in warning + assert " de -> jpn_Jpan: 20.0\n en -> jpn_Jpan: 40.0" in warning + assert "en -> fra_Latn" not in warning + + def test_does_not_warn_when_language_consistency_pairs_meet_threshold(self, caplog) -> None: + server = _make_server( + compute_comet=False, + language_consistency_backend="wmt24pp_cld2", + language_consistency_warning_threshold=50.0, + ) + tasks = [ + [ + { + "translation": "reference text", + "generation": "generated text", + "source_language": "en", + "target_language": "ja_JP", + "language_consistency_score": 0.50, + } + ] + ] + + with caplog.at_level(logging.WARNING, logger="app"): + server.compute_metrics(tasks) + + assert not any(message.startswith("Warning - the following language pairs") for message in caplog.messages) + + def test_no_comet_rows_emits_local_metrics_only(self) -> None: """Rollouts without comet_score (compute_comet disabled mid-run, or - actor pool unavailable) yield BLEU metrics with no /comet keys.""" + actor pool unavailable) yield spBLEU/chrF metrics with no /comet keys.""" server = _make_server(compute_comet=True) tasks = [ [ @@ -401,7 +774,8 @@ def test_no_comet_rows_emits_bleu_only(self) -> None: ] ] m = server.compute_metrics(tasks) - assert "en->de_DE/bleu" in m + assert "en->de_DE/chrF" in m + assert "en->de_DE/spBLEU" in m assert not any(k.endswith("/comet") for k in m) diff --git a/resources_servers/wmt_translation/tests/test_flores_glotlid_language_consistency.py b/resources_servers/wmt_translation/tests/test_flores_glotlid_language_consistency.py new file mode 100644 index 00000000000..1a42c0a53e1 --- /dev/null +++ b/resources_servers/wmt_translation/tests/test_flores_glotlid_language_consistency.py @@ -0,0 +1,191 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import importlib.util +import sys +from pathlib import Path +from types import ModuleType +from unittest.mock import MagicMock, patch + +import numpy as np +import pytest + + +def _load_module_with_fake_model(): + fake_model = MagicMock() + fake_model.get_output_matrix.return_value = np.zeros((6, 2)) + fake_model.get_labels.return_value = [ + "__label__eng_Latn", + "__label__deu_Latn", + "__label__acm_Arab", + "__label__arb_Arab", + "__label__ary_Arab", + "__label__cmn_Hani", + ] + + fake_fasttext = ModuleType("fasttext") + fake_fasttext.load_model = MagicMock(return_value=fake_model) + fake_huggingface_hub = ModuleType("huggingface_hub") + fake_huggingface_hub.hf_hub_download = MagicMock(return_value="model_v3.bin") + + module_path = Path(__file__).parents[1] / "flores_glotlid_language_consistency.py" + spec = importlib.util.spec_from_file_location("_flores_glotlid_language_consistency_for_test", module_path) + assert spec is not None + assert spec.loader is not None + module = importlib.util.module_from_spec(spec) + with patch.dict( + sys.modules, + { + "fasttext": fake_fasttext, + "huggingface_hub": fake_huggingface_hub, + }, + ): + spec.loader.exec_module(module) + return module + + +@pytest.fixture(scope="module") +def lid_module(): + return _load_module_with_fake_model() + + +@pytest.mark.parametrize("text", ["", " \n\t "]) +def test_empty_text_returns_zero_without_inference(lid_module, monkeypatch, text: str) -> None: + predict = MagicMock() + monkeypatch.setattr(lid_module, "_predict_all_language_probabilities", predict) + + assert lid_module.flores_glotlid_language_consistency_score(text, "eng_Latn") == 0.0 + predict.assert_not_called() + + +def test_text_is_collapsed_to_one_line_before_inference(lid_module, monkeypatch) -> None: + predict = MagicMock(return_value=np.array([0.8, 0.2, 0.0, 0.0, 0.0, 0.0])) + monkeypatch.setattr(lid_module, "_predict_all_language_probabilities", predict) + + score = lid_module.flores_glotlid_language_consistency_score(" Hello\n world\t", "eng_Latn") + + assert score == pytest.approx(0.8) + predict.assert_called_once_with("Hello world") + + +def test_unlisted_flores_code_uses_matching_glotlid_code(lid_module, monkeypatch) -> None: + monkeypatch.setattr( + lid_module, + "_predict_all_language_probabilities", + MagicMock(return_value=np.array([0.7, 0.3, 0.0, 0.0, 0.0, 0.0])), + ) + + assert lid_module.flores_glotlid_language_consistency_score("English text", "eng_Latn") == pytest.approx(0.7) + + +def test_dialect_suffix_is_removed(lid_module, monkeypatch) -> None: + monkeypatch.setattr( + lid_module, + "_predict_all_language_probabilities", + MagicMock(return_value=np.array([0.65, 0.35, 0.0, 0.0, 0.0, 0.0])), + ) + + assert lid_module.flores_glotlid_language_consistency_score( + "English text", + "eng_Latn_dial1234", + ) == pytest.approx(0.65) + + +def test_override_probabilities_are_summed(lid_module, monkeypatch) -> None: + monkeypatch.setattr( + lid_module, + "_predict_all_language_probabilities", + MagicMock(return_value=np.array([0.1, 0.1, 0.2, 0.3, 0.25, 0.05])), + ) + + assert lid_module.flores_glotlid_language_consistency_score( + "Arabic text", + "acm_Arab", + ) == pytest.approx(0.75) + + +def test_renamed_override_uses_replacement_code(lid_module, monkeypatch) -> None: + monkeypatch.setattr( + lid_module, + "_predict_all_language_probabilities", + MagicMock(return_value=np.array([0.01, 0.01, 0.01, 0.01, 0.01, 0.95])), + ) + + assert lid_module.flores_glotlid_language_consistency_score( + "Chinese text", + "cmn_Hans", + ) == pytest.approx(0.95) + + +def test_missing_glotlid_label_returns_zero_without_inference(lid_module, monkeypatch) -> None: + predict = MagicMock() + monkeypatch.setattr(lid_module, "_predict_all_language_probabilities", predict) + + assert lid_module.flores_glotlid_language_consistency_score("text", "xxx_Latn") == 0.0 + predict.assert_not_called() + + +def test_partially_missing_override_sums_available_labels(lid_module, monkeypatch) -> None: + monkeypatch.delitem(lid_module._LABEL_INDICES, "__label__ary_Arab") + monkeypatch.setattr( + lid_module, + "_predict_all_language_probabilities", + MagicMock(return_value=np.array([0.1, 0.1, 0.2, 0.3, 0.25, 0.05])), + ) + + assert lid_module.flores_glotlid_language_consistency_score( + "Arabic text", + "acm_Arab", + ) == pytest.approx(0.5) + + +def test_all_language_probabilities_match_reference_softmax(lid_module, monkeypatch) -> None: + monkeypatch.setattr( + lid_module, + "_OUTPUT_MATRIX", + np.array([[1.0, 0.0], [0.0, 1.0], [-1.0, 0.0]]), + ) + lid_module._MODEL.get_sentence_vector.return_value = np.array([2.0, 1.0]) + + probabilities = lid_module._predict_all_language_probabilities("text") + expected = np.exp(np.array([2.0, 1.0, -2.0])) + expected /= expected.sum() + + assert probabilities == pytest.approx(expected) + assert probabilities.sum() == pytest.approx(1.0) + + +def test_all_language_softmax_is_stable_for_large_logits(lid_module, monkeypatch) -> None: + monkeypatch.setattr( + lid_module, + "_OUTPUT_MATRIX", + np.array([[10_000.0], [9_999.0], [-10_000.0]]), + ) + lid_module._MODEL.get_sentence_vector.return_value = np.array([1.0]) + + probabilities = lid_module._predict_all_language_probabilities("text") + + assert np.all(np.isfinite(probabilities)) + assert np.all(probabilities >= 0.0) + assert np.all(probabilities <= 1.0) + assert probabilities.sum() == pytest.approx(1.0) + + +def test_override_table_has_well_formed_unique_codes(lid_module) -> None: + for flores_code, glotlid_codes in lid_module._LANG_CODE_OVERRIDES.items(): + assert len(flores_code.split("_")) == 2 + assert glotlid_codes + assert len(glotlid_codes) == len(set(glotlid_codes)) + assert all(len(code.split("_")) == 2 for code in glotlid_codes) diff --git a/resources_servers/wmt_translation/tests/test_language_consistency.py b/resources_servers/wmt_translation/tests/test_language_consistency.py new file mode 100644 index 00000000000..11eb258357b --- /dev/null +++ b/resources_servers/wmt_translation/tests/test_language_consistency.py @@ -0,0 +1,104 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import builtins +import importlib.util +import sys +from pathlib import Path +from types import ModuleType +from unittest.mock import MagicMock, patch + +import pytest +from language_consistency import get_language_consistency_backend + + +def test_wmt24pp_cld2_backend_scores_wmt24pp_language_codes() -> None: + backend = get_language_consistency_backend("wmt24pp_cld2") + + german = "Der schnelle braune Fuchs springt über den faulen Hund." + french = "Le renard brun rapide saute par-dessus le chien paresseux." + + assert backend(german, "de_DE") > 0.5 + assert backend(french, "fr_FR") > 0.5 + assert backend(german, "fr_FR") < 0.5 + + +def test_unknown_backend_name_raises_with_available_backends_listed() -> None: + with pytest.raises(ValueError) as exc_info: + get_language_consistency_backend("does_not_exist") + + message = str(exc_info.value) + assert "flores_glotlid" in message + assert "wmt24pp_cld2" in message + + +def test_unknown_backend_name_does_not_invoke_any_loader() -> None: + mock_loaders = { + "flores_glotlid": MagicMock(), + "wmt24pp_cld2": MagicMock(), + } + with patch.dict( + "language_consistency._LANGUAGE_CONSISTENCY_BACKEND_LOADERS", + mock_loaders, + clear=True, + ): + with pytest.raises(ValueError): + get_language_consistency_backend("does_not_exist") + + for mock_loader in mock_loaders.values(): + mock_loader.assert_not_called() + + +def test_known_backend_name_invokes_its_lazy_loader_once() -> None: + mock_loader = MagicMock() + + with patch.dict( + "language_consistency._LANGUAGE_CONSISTENCY_BACKEND_LOADERS", + {"wmt24pp_cld2": mock_loader}, + ): + backend = get_language_consistency_backend("wmt24pp_cld2") + + mock_loader.assert_called_once_with() + assert backend is mock_loader.return_value + + +def test_flores_glotlid_backend_is_imported_lazily() -> None: + expected_backend = MagicMock() + fake_module = ModuleType("flores_glotlid_language_consistency") + fake_module.flores_glotlid_language_consistency_score = expected_backend + + with patch.dict(sys.modules, {"flores_glotlid_language_consistency": fake_module}): + backend = get_language_consistency_backend("flores_glotlid") + + assert backend is expected_backend + + +def test_importing_registry_does_not_import_glotlid_dependencies() -> None: + module_path = Path(__file__).parents[1] / "language_consistency.py" + spec = importlib.util.spec_from_file_location("_language_consistency_import_safety_test", module_path) + assert spec is not None + assert spec.loader is not None + module = importlib.util.module_from_spec(spec) + real_import = builtins.__import__ + + def guarded_import(name, *args, **kwargs): + assert name not in { + "fasttext", + "flores_glotlid_language_consistency", + "huggingface_hub", + } + return real_import(name, *args, **kwargs) + + with patch("builtins.__import__", new=guarded_import): + spec.loader.exec_module(module) diff --git a/resources_servers/wmt_translation/tests/test_wmt24pp_cld2_language_consistency.py b/resources_servers/wmt_translation/tests/test_wmt24pp_cld2_language_consistency.py new file mode 100644 index 00000000000..c78bd7f49be --- /dev/null +++ b/resources_servers/wmt_translation/tests/test_wmt24pp_cld2_language_consistency.py @@ -0,0 +1,209 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import logging +from unittest.mock import patch + +import pycld2 as cld2 +import pytest +from wmt24pp_cld2_language_consistency import ( + _sanitize_text, + wmt24pp_cld2_language_consistency_score, +) + + +def _detection_result(*details): + return True, 100, details + + +def test_sanitize_text_removes_unsupported_control_characters() -> None: + assert _sanitize_text("a\x00b\x07c\x1bd") == "abcd" + + +def test_sanitize_text_preserves_printable_unicode_and_whitespace() -> None: + text = "Zażółć 中文\n\t\r\u2003" + assert _sanitize_text(text) == text + + +@patch("wmt24pp_cld2_language_consistency.cld2.detect") +def test_empty_input_returns_zero_without_calling_cld2(mock_detect) -> None: + assert wmt24pp_cld2_language_consistency_score("", "de_DE") == 0.0 + mock_detect.assert_not_called() + + +@patch("wmt24pp_cld2_language_consistency.cld2.detect") +def test_control_characters_only_returns_zero_without_calling_cld2(mock_detect) -> None: + assert wmt24pp_cld2_language_consistency_score("\x00\x07\x1b", "de_DE") == 0.0 + mock_detect.assert_not_called() + + +@patch("wmt24pp_cld2_language_consistency.cld2.detect") +def test_expected_language_can_be_found_when_not_top_prediction(mock_detect) -> None: + mock_detect.return_value = _detection_result( + ("ENGLISH", "en", 70, 1000), + ("GERMAN", "de", 30, 500), + ) + assert wmt24pp_cld2_language_consistency_score("mixed text", "de_DE") == pytest.approx(0.30) + + +@patch("wmt24pp_cld2_language_consistency.cld2.detect") +def test_missing_expected_language_returns_zero(mock_detect) -> None: + mock_detect.return_value = _detection_result(("ENGLISH", "en", 100, 1000)) + + assert wmt24pp_cld2_language_consistency_score("English text", "de_DE") == 0.0 + + +@patch("wmt24pp_cld2_language_consistency.cld2.detect") +def test_empty_detection_details_returns_zero(mock_detect) -> None: + mock_detect.return_value = _detection_result() + + assert wmt24pp_cld2_language_consistency_score("Hallo Welt", "de_DE") == 0.0 + + +@patch("wmt24pp_cld2_language_consistency.cld2.detect") +def test_unknown_wmt_language_code_raises_before_calling_cld2(mock_detect) -> None: + with pytest.raises(ValueError, match=r"Unknown WMT24\+\+ language code"): + wmt24pp_cld2_language_consistency_score("text", "unknown_CODE") + + mock_detect.assert_not_called() + + +@patch("wmt24pp_cld2_language_consistency.cld2.detect") +def test_cld2_receives_sanitized_text_and_detection_options(mock_detect) -> None: + mock_detect.return_value = _detection_result(("GERMAN", "de", 100, 1000)) + + wmt24pp_cld2_language_consistency_score("Hallo\x00\nWelt", "de_DE") + + mock_detect.assert_called_once_with( + "Hallo\nWelt", + isPlainText=True, + bestEffort=True, + ) + + +@patch("wmt24pp_cld2_language_consistency.cld2.detect") +def test_cld2_error_is_logged_and_returns_zero(mock_detect, caplog) -> None: + mock_detect.side_effect = cld2.error("detection failed") + + with caplog.at_level(logging.ERROR, logger="wmt24pp_cld2_language_consistency"): + score = wmt24pp_cld2_language_consistency_score("Hallo Welt", "de_DE") + + assert score == 0.0 + error_records = [record for record in caplog.records if record.name == "wmt24pp_cld2_language_consistency"] + assert len(error_records) == 1 + assert "CLD2 detection failed for target language" in error_records[0].message + assert error_records[0].exc_info is not None + + +@pytest.mark.parametrize( + ("wmt_code", "cld2_code"), + [ + ("fil_PH", "tl"), + ("he_IL", "iw"), + ("zh_TW", "zh-Hant"), + ("ar_EG", "ar"), + ("ar_SA", "ar"), + ("sw_KE", "sw"), + ("sw_TZ", "sw"), + ], +) +@patch("wmt24pp_cld2_language_consistency.cld2.detect") +def test_exceptional_language_code_mappings(mock_detect, wmt_code: str, cld2_code: str) -> None: + mock_detect.return_value = _detection_result(("EXPECTED", cld2_code, 80, 1000)) + + assert wmt24pp_cld2_language_consistency_score("target-language text", wmt_code) == pytest.approx(0.80) + + +@pytest.mark.parametrize( + ("percent", "expected_fraction"), + [ + (0, 0.0), + (100, 1.0), + ], +) +@patch("wmt24pp_cld2_language_consistency.cld2.detect") +def test_boundary_percentages(mock_detect, percent: int, expected_fraction: float) -> None: + mock_detect.return_value = _detection_result(("GERMAN", "de", percent, 1000)) + + assert wmt24pp_cld2_language_consistency_score("Hallo Welt", "de_DE") == expected_fraction + + +@pytest.mark.parametrize("reliability_score", [-100, 0, 1_000_000]) +@patch("wmt24pp_cld2_language_consistency.cld2.detect") +def test_cld2_reliability_score_is_ignored(mock_detect, reliability_score: int) -> None: + mock_detect.return_value = _detection_result(("GERMAN", "de", 42, reliability_score)) + + assert wmt24pp_cld2_language_consistency_score("Hallo Welt", "de_DE") == pytest.approx(0.42) + + +@pytest.mark.parametrize( + ("wmt_code", "text"), + [ + ( + "de_DE", + "Der schnelle braune Fuchs springt über den faulen Hund. " + "Heute scheint die Sonne und die Kinder spielen im Garten.", + ), + ( + "fr_FR", + "Le renard brun rapide saute par-dessus le chien paresseux. " + "Aujourd'hui, le soleil brille et les enfants jouent dans le jardin.", + ), + ( + "es_MX", + "El rápido zorro marrón salta sobre el perro perezoso. Hoy brilla el sol y los niños juegan en el jardín.", + ), + ( + "ja_JP", + "素早い茶色のキツネが怠け者の犬を飛び越えます。今日は天気がよく、子どもたちは庭で遊んでいます。", + ), + ( + "zh_CN", + "快速的棕色狐狸跳过了那只懒狗。今天天气很好,孩子们正在花园里快乐地玩耍。", + ), + ( + "zh_TW", + "快速的棕色狐狸跳過了那隻懶狗。今天天氣很好,孩子們正在花園裡快樂地玩耍。", + ), + ], +) +def test_real_cld2_gives_expected_language_a_high_consistency_score(wmt_code: str, text: str) -> None: + assert wmt24pp_cld2_language_consistency_score(text, wmt_code) > 0.5 + + +@pytest.mark.parametrize( + ("wmt_code", "text"), + [ + ( + "fr_FR", # Text actually in German + "Der schnelle braune Fuchs springt über den faulen Hund. " + "Heute scheint die Sonne und die Kinder spielen im Garten.", + ), + ( + "de_DE", # Text actually in French + "Le renard brun rapide saute par-dessus le chien paresseux. " + "Aujourd'hui, le soleil brille et les enfants jouent dans le jardin.", + ), + ( + "ja_JP", # Text actually in Spanish + "El rápido zorro marrón salta sobre el perro perezoso. Hoy brilla el sol y los niños juegan en el jardín.", + ), + ( + "zh_TW", # Text actually in Simplified Chinese + "快速的棕色狐狸跳过了那只懒狗。今天天气很好,孩子们正在花园里快乐地玩耍。", + ), + ], +) +def test_real_cld2_gives_wrong_language_a_low_consistency_score(wmt_code: str, text: str) -> None: + assert wmt24pp_cld2_language_consistency_score(text, wmt_code) < 0.5 diff --git a/resources_servers/wmt_translation/wmt24pp_cld2_language_consistency.py b/resources_servers/wmt_translation/wmt24pp_cld2_language_consistency.py new file mode 100644 index 00000000000..905dbd79ff9 --- /dev/null +++ b/resources_servers/wmt_translation/wmt24pp_cld2_language_consistency.py @@ -0,0 +1,129 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import logging + +import pycld2 as cld2 + + +LOG = logging.getLogger(__name__) + + +# WMT24++ distinguishes regional varieties that CLD2 cannot. Arabic, French, +# Portuguese, and Swahili each have two WMT24++ locales that map to one CLD2 +# language code. +# This means that _WMT24PP_TO_CLD2_CODES[wmt24pp_language_code] == cld2_code +# is a **lenient** language-level comparison for those locales, not a check of +# the requested regional variety. +_WMT24PP_TO_CLD2_CODES = { + "ar_EG": "ar", # Egyptian Arabic -> Arabic + "ar_SA": "ar", # Saudi Arabic -> Arabic + "bg_BG": "bg", + "bn_IN": "bn", + "ca_ES": "ca", + "cs_CZ": "cs", + "da_DK": "da", + "de_DE": "de", + "el_GR": "el", + "es_MX": "es", + "et_EE": "et", + "fa_IR": "fa", + "fi_FI": "fi", + "fil_PH": "tl", + "fr_CA": "fr", + "fr_FR": "fr", + "gu_IN": "gu", + "he_IL": "iw", + "hi_IN": "hi", + "hr_HR": "hr", + "hu_HU": "hu", + "id_ID": "id", + "is_IS": "is", + "it_IT": "it", + "ja_JP": "ja", + "kn_IN": "kn", + "ko_KR": "ko", + "lt_LT": "lt", + "lv_LV": "lv", + "ml_IN": "ml", + "mr_IN": "mr", + "nl_NL": "nl", + "no_NO": "no", + "pa_IN": "pa", + "pl_PL": "pl", + "pt_BR": "pt", + "pt_PT": "pt", + "ro_RO": "ro", + "ru_RU": "ru", + "sk_SK": "sk", + "sl_SI": "sl", + "sr_RS": "sr", + "sv_SE": "sv", + "sw_KE": "sw", # Kenyan Swahili -> Swahili + "sw_TZ": "sw", # Tanzanian Swahili -> Swahili + "ta_IN": "ta", + "te_IN": "te", + "th_TH": "th", + "tr_TR": "tr", + "uk_UA": "uk", + "ur_PK": "ur", + "vi_VN": "vi", + "zh_CN": "zh", + "zh_TW": "zh-Hant", + "zu_ZA": "zu", +} + + +def _sanitize_text(text: str) -> str: + return "".join(c for c in text if c.isprintable() or c.isspace()) + + +def wmt24pp_cld2_language_consistency_score(text: str, expected_language_code: str) -> float: + """ + Return the fraction of `text` (0.0-1.0) that CLD2 attributes to the + expected language for `expected_language_code`. + + CLD2 has no "score language X" function: it always reports its own top guesses, + so we detect and then read off the percentage for the language we wanted. + Returns 0.0 when that language is not among the detections (e.g. the model + translated into the wrong language, or produced empty output). + + WMT24++ has multiple regional varieties of Arabic, French, Portuguese, and + Swahili that CLD2 cannot distinguish. Checks for those locales validate the + major language only and cannot detect a wrong regional variety. + """ + text = _sanitize_text(text) # pycld2 rejects some control chars + if not text: + return 0.0 + + try: + expected_code = _WMT24PP_TO_CLD2_CODES[expected_language_code] + except KeyError as exc: + raise ValueError(f"Unknown WMT24++ language code: {expected_language_code!r}") from exc + + # details is up to three (name, code, percent, score) tuples; `percent` is + # the share of the text detected as that language. A code appears at most + # once, so find the matching entry (if any) and return its percentage. + try: + _, _, details = cld2.detect(text, isPlainText=True, bestEffort=True) + except cld2.error: + LOG.exception( + "CLD2 detection failed for target language %r; returning 0.0", + expected_language_code, + ) + return 0.0 + for _name, code, percent, _score in details: + if code == expected_code: + return percent / 100.0 + return 0.0 From fae15a3500b20ae5ee74c0b395a55dea88c1b61c Mon Sep 17 00:00:00 2001 From: plaszkiewicz Date: Fri, 31 Jul 2026 12:16:45 +0200 Subject: [PATCH 2/2] feat(pinchbench): add the benchmark config for the full 147-task suite PinchBench shipped as an agent server (responses_api_agents/pinchbench) but had no entry under benchmarks/, so it was neither discoverable via `gym list benchmarks` nor preparable via `gym eval prepare`. Add the Layer-3 benchmark definition: config.yaml chains to the agent config and narrows it to the benchmark dataset via `_inherit_from` (the tau2 shape, the existing precedent for a benchmark chaining to a responses_api_agents config), and prepare.py clones the skill at v2.0.0 -- the ref Dockerfile.benchmark bakes into the per-task image -- to write all 147 manifest tasks. The eight run-time knobs the agent config interpolates are declared with `oc.env:...,null` defaults so `gym list` and `gym eval prepare` resolve without them; a null still reaching the agent fails fast, since the fields are required `str` in PinchBenchAgentConfig. CLI overrides keep working because Hydra parses them against an empty base before config_paths loads and merges them last. Preparation fails loudly if the manifest is not 147 tasks, or if the regenerated rows do not byte-match the committed example.jsonl, so upstream drift and divergence from dataset_preprocess.py cannot pass silently. Also declare domain/description/value on the agent config: listing metadata has to live on the base node, since adding it to the inheriting benchmark node raises `Key 'domain' is not in struct`. The README table row is regenerated by the update-readme-table hook. Signed-off-by: plaszkiewicz (cherry picked from commit 8efb8612875bee55f26a8c2f909145cf2ddcbe62) --- README.md | 2 +- benchmarks/pinchbench/README.md | 64 +++++++ benchmarks/pinchbench/__init__.py | 14 ++ benchmarks/pinchbench/config.yaml | 44 +++++ benchmarks/pinchbench/prepare.py | 159 ++++++++++++++++++ .../pinchbench/configs/pinchbench.yaml | 3 + 6 files changed, 285 insertions(+), 1 deletion(-) create mode 100644 benchmarks/pinchbench/README.md create mode 100644 benchmarks/pinchbench/__init__.py create mode 100644 benchmarks/pinchbench/config.yaml create mode 100644 benchmarks/pinchbench/prepare.py diff --git a/README.md b/README.md index b46f5d1d74c..1fbc5898fa7 100644 --- a/README.md +++ b/README.md @@ -291,7 +291,7 @@ The Dataset column links to publicly available datasets (e.g., on HuggingFace). | Openenv | games | Maze navigation environment via OpenEnv. Agent navigates an 8x8 grid to find the exit. | - | - | - | - | openenv_maze.yaml | - | | Over Refusal Detection | | - | - | ✓ | - | TBD | over_refusal_detection.yaml | - | | Physics Judge | math | Physics QA verified by NeMo Skills' physics judge LLM with math-verify symbolic fallback | Score open-ended physics benchmarks (e.g. PHYSICS) where the judge emits [Correct] / [Incorrect] verdicts | - | - | - | physics_judge.yaml | - | -| Pinchbench | | - | - | - | - | - | pinchbench.yaml | - | +| Pinchbench | agent | PinchBench benchmark integration | Evaluate a model as the brain of an OpenClaw agent on real-world tasks. | - | - | - | pinchbench.yaml | - | | Polymath | math | PolyMath multilingual math benchmark with weighted (difficulty) and per-language metrics | Improve multilingual math reasoning across 18 languages and 4 difficulty tiers | - | - | - | polymath.yaml | - | | Proof Genselect | math | Pairwise proof selection with binary correctness reward | - | - | - | - | proof_genselect.yaml | - | | Proof Judge | math | Theorem proving with verifier + meta-verifier judge (combined env) | - | - | - | - | proof_judge.yaml | - | diff --git a/benchmarks/pinchbench/README.md b/benchmarks/pinchbench/README.md new file mode 100644 index 00000000000..52a9549420c --- /dev/null +++ b/benchmarks/pinchbench/README.md @@ -0,0 +1,64 @@ +# PinchBench + +All 147 tasks of [PinchBench](https://github.com/pinchbench/skill) — calendar, email triage, +CSV/log analysis, coding, research, writing — measuring how well a model performs as the brain +of an [OpenClaw](https://github.com/openclaw/openclaw) agent. + +This benchmark config is a thin narrowing of the `pinchbench` agent server, which owns the whole +integration (one sandbox per task, in-sandbox OpenClaw gateway, PinchBench's own grading harness). +Read [`responses_api_agents/pinchbench/README.md`](../../responses_api_agents/pinchbench/README.md) +first — architecture, the per-task image build, config knobs, and the parity validation against +vanilla standalone PinchBench all live there. + +## Verification + +Grading is PinchBench's own, run inside the sandbox: deterministic checks (`automated`), an LLM +judge (`llm_judge`), or a weighted mix of both (`hybrid`). Reward is **continuous** in `[0,1]`, +not pass/fail — threshold it if you need a binary signal. Each rollout also returns +`grading_type`, `grading_breakdown`, `grading_notes` and `raw_rollout`. + +`num_repeats` is 3: temperature 1.0 plus live web search make a single pass noisy, and 3 is what +the agent-side parity validation used. Trust aggregates, not individual runs. + +## Data preparation + +``` +gym eval prepare --benchmark pinchbench +``` + +The PinchBench skill is not vendored. `prepare.py` clones it at `v2.0.0` — the same ref +`responses_api_agents/pinchbench/Dockerfile.benchmark` bakes into the per-task image — and writes +`data/pinchbench_benchmark.jsonl`, one row per manifest task: + +```json +{"responses_create_params": {"input": [{"role": "user", "content": ""}]}, + "verifier_metadata": {"task_id": "task_sanity"}} +``` + +`task_id` is the authoritative selector: at run time `run_task.sh` passes it to +`benchmark.py --suite`, which loads the full task (prompt + assets + grading) from the skill inside +the sandbox. `input` carries the prompt for readability only. **Subset by dropping rows** — no +config change needed. + +Set `PINCHBENCH_SKILL_DIR` to an existing `v2.0.0` checkout to skip the clone. Preparation fails +loudly if the manifest is not 147 tasks or if the regenerated rows do not byte-match the committed +`responses_api_agents/pinchbench/data/example.jsonl`, so upstream drift can't slip through. + +## Quickstart + +Build the per-task image first (`setup_scripts/build_image.sh --apptainer`), then: + +``` +gym eval run --benchmark pinchbench --model-type vllm_model \ + +sandbox_image= \ + +model_base_url= +model_api_key= +model_name= \ + +judge_model= +judge_base_url= +judge_api_key= \ + +brave_api_key= +``` + +Those eight keys are declared in `config.yaml` with `null` defaults so `gym list` and +`gym eval prepare` resolve without them; each also reads a `PINCHBENCH_*` env var. A `null` still +reaching the agent fails fast — the fields are required in `PinchBenchAgentConfig`. + +For slow or highly concurrent endpoints, raise `openclaw_provider_timeout_seconds` **and** +`task_timeout_s` together; see the agent README's "OpenClaw LLM idle timeout" section. diff --git a/benchmarks/pinchbench/__init__.py b/benchmarks/pinchbench/__init__.py new file mode 100644 index 00000000000..467079831e1 --- /dev/null +++ b/benchmarks/pinchbench/__init__.py @@ -0,0 +1,14 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/benchmarks/pinchbench/config.yaml b/benchmarks/pinchbench/config.yaml new file mode 100644 index 00000000000..dfe29eb4007 --- /dev/null +++ b/benchmarks/pinchbench/config.yaml @@ -0,0 +1,44 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# PinchBench — 147 real-world OpenClaw agent tasks. Run: +# gym eval prepare --benchmark pinchbench +# gym eval run --benchmark pinchbench --model-type vllm_model \ +# +sandbox_image= \ +# +model_base_url= +model_api_key= +model_name= \ +# +judge_model= +judge_base_url= +judge_api_key= \ +# +brave_api_key= + +# Chain to existing agent config +config_paths: + - responses_api_agents/pinchbench/configs/pinchbench.yaml + +# Run-time knobs the agent config interpolates. Declared here so `gym list` and +# `gym eval prepare` resolve without them; a null reaching the agent fails fast in +# PinchBenchAgentConfig (the fields are required `str`). Supply real values on the +# command line (`+model_base_url=...`, which merges last) or via these env vars. +sandbox_image: ${oc.env:PINCHBENCH_SANDBOX_IMAGE,null} +model_base_url: ${oc.env:PINCHBENCH_MODEL_BASE_URL,null} +model_api_key: ${oc.env:PINCHBENCH_MODEL_API_KEY,null} +model_name: ${oc.env:PINCHBENCH_MODEL_NAME,null} +judge_model: ${oc.env:PINCHBENCH_JUDGE_MODEL,null} +judge_base_url: ${oc.env:PINCHBENCH_JUDGE_BASE_URL,null} +judge_api_key: ${oc.env:PINCHBENCH_JUDGE_API_KEY,null} +brave_api_key: ${oc.env:PINCHBENCH_BRAVE_API_KEY,null} + +# `_inherit_from` keeps this benchmark isolated from the generic agent instance +# above, so overriding the dataset list here does not mutate `pinchbench_agent`. +pinchbench_benchmark_agent: + _inherit_from: pinchbench_agent + responses_api_agents: + pinchbench: + datasets: + - name: pinchbench + type: benchmark + jsonl_fpath: benchmarks/pinchbench/data/pinchbench_benchmark.jsonl + prompt_config: null + prepare_script: benchmarks/pinchbench/prepare.py + # Reward is continuous and temperature 1.0 + live web search make a single + # pass noisy; the parity validation in the agent README used n=3. + num_repeats: 3 + license: MIT diff --git a/benchmarks/pinchbench/prepare.py b/benchmarks/pinchbench/prepare.py new file mode 100644 index 00000000000..b3fa3fa6137 --- /dev/null +++ b/benchmarks/pinchbench/prepare.py @@ -0,0 +1,159 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Prepare the full 147-task PinchBench benchmark dataset. + +The PinchBench skill is not vendored (same convention as `harbor_agent` / `mini_swe_agent`: +pin an upstream ref, don't copy task files). This clones it at the SAME ref the per-task +image bakes in — `Dockerfile.benchmark`'s `PINCHBENCH_SKILL_REF` — so the prompts written +here describe the tasks `benchmark.py` actually loads inside the sandbox. + +Row shape mirrors `responses_api_agents/pinchbench/dataset_preprocess.py` (which generates +the committed 5-task `data/example.jsonl`): the task's human-readable `## Prompt` section in +`input`, plus `verifier_metadata.task_id`. `task_id` is the authoritative selector — at run +time `run_task.sh` passes it to `benchmark.py --suite`, which loads the full task (prompt + +assets + grading) from the skill. `_assert_matches_example_jsonl` re-derives the 5 example +rows and compares them byte-for-byte against that committed file, so the two generators +cannot silently diverge and upstream prompt drift is caught here rather than mid-run. + +Set `PINCHBENCH_SKILL_DIR` to an existing checkout at `SKILL_REF` to skip the clone. +""" + +from __future__ import annotations + +import json +import os +import re +import subprocess +import sys +import tempfile +from pathlib import Path + +import yaml + + +if __package__ in (None, ""): + sys.path.append(str(Path(__file__).resolve().parents[2])) + + +BENCHMARK_DIR = Path(__file__).resolve().parent +DATA_DIR = BENCHMARK_DIR / "data" +OUTPUT_FPATH = DATA_DIR / "pinchbench_benchmark.jsonl" + +AGENT_DIR = BENCHMARK_DIR.parents[1] / "responses_api_agents" / "pinchbench" +EXAMPLE_JSONL_FPATH = AGENT_DIR / "data" / "example.jsonl" + +SKILL_REPO_URL = "https://github.com/pinchbench/skill" +# Keep in lockstep with `PINCHBENCH_SKILL_REF` in +# responses_api_agents/pinchbench/Dockerfile.benchmark. +SKILL_REF = "v2.0.0" +# The v2.0.0 manifest's task count; upstream drift surfaces here rather than as a +# silently shorter run. +EXPECTED_TASK_COUNT = 147 + +_PROMPT_SECTION_RE = re.compile(r"##\s*Prompt\s*\n(.*?)(?:\n##\s|\Z)", re.S) + + +def _prompt_for(skill_dir: Path, task_id: str) -> str: + """The task's human-readable prompt = the `## Prompt` section of its skill `.md`.""" + md = (skill_dir / "tasks" / f"{task_id}.md").read_text() + match = _PROMPT_SECTION_RE.search(md) + return (match.group(1).strip() if match else "").strip() + + +def _record_line(skill_dir: Path, task_id: str) -> str: + record = { + "responses_create_params": {"input": [{"role": "user", "content": _prompt_for(skill_dir, task_id)}]}, + "verifier_metadata": {"task_id": task_id}, + } + return json.dumps(record, separators=(",", ":")) + "\n" + + +def _all_task_ids(skill_dir: Path) -> list[str]: + """Every task in the skill's manifest, in manifest order (`run_first` first).""" + manifest = yaml.safe_load((skill_dir / "tasks" / "manifest.yaml").read_text()) + task_ids: list[str] = list(manifest.get("run_first", [])) + for category_task_ids in (manifest.get("categories") or {}).values(): + for task_id in category_task_ids or []: + if task_id not in task_ids: + task_ids.append(task_id) + return task_ids + + +def _assert_matches_example_jsonl(skill_dir: Path) -> None: + """Re-derive the committed example rows from this checkout and require an exact match. + + Guards both directions: a drifted skill checkout, and this script diverging from the + agent-side `dataset_preprocess.py` that produced `example.jsonl`. + """ + expected_lines = EXAMPLE_JSONL_FPATH.read_text().splitlines(keepends=True) + for line_number, expected_line in enumerate(expected_lines, 1): + task_id = json.loads(expected_line)["verifier_metadata"]["task_id"] + actual_line = _record_line(skill_dir, task_id) + if actual_line != expected_line: + raise ValueError( + f"Regenerated row for '{task_id}' does not match {EXAMPLE_JSONL_FPATH} line {line_number}. " + f"The skill checkout is not at {SKILL_REF}, or the row format has diverged from " + "responses_api_agents/pinchbench/dataset_preprocess.py." + ) + + +def _write_from_skill(skill_dir: Path) -> None: + _assert_matches_example_jsonl(skill_dir) + + task_ids = _all_task_ids(skill_dir) + if len(task_ids) != EXPECTED_TASK_COUNT: + raise ValueError( + f"Expected {EXPECTED_TASK_COUNT} PinchBench tasks in the {SKILL_REF} manifest, " + f"found {len(task_ids)}; the skill checkout may not be at {SKILL_REF}." + ) + + DATA_DIR.mkdir(parents=True, exist_ok=True) + with OUTPUT_FPATH.open("w") as out: + for task_id in task_ids: + out.write(_record_line(skill_dir, task_id)) + print(f"Wrote {len(task_ids)} PinchBench tasks to {OUTPUT_FPATH}") + + +def _clone_skill(target_dir: Path) -> None: + print(f"Cloning {SKILL_REPO_URL} at {SKILL_REF}...") + subprocess.run( + ["git", "clone", "--depth", "1", "--branch", SKILL_REF, SKILL_REPO_URL, str(target_dir)], + check=True, + ) + + +def prepare() -> Path: + """Clone (or reuse) the pinned skill, write every manifest task. Returns the JSONL path.""" + preexisting_skill_dir = os.environ.get("PINCHBENCH_SKILL_DIR") + if preexisting_skill_dir: + skill_dir = Path(preexisting_skill_dir) + if not (skill_dir / "tasks" / "manifest.yaml").is_file(): + raise FileNotFoundError( + f"PINCHBENCH_SKILL_DIR={skill_dir} is not a PinchBench skill checkout " + "(tasks/manifest.yaml is missing)." + ) + _write_from_skill(skill_dir) + return OUTPUT_FPATH + + with tempfile.TemporaryDirectory() as tmpdir: + skill_dir = Path(tmpdir) / "pinchbench-skill" + _clone_skill(skill_dir) + _write_from_skill(skill_dir) + + return OUTPUT_FPATH + + +if __name__ == "__main__": + prepare() diff --git a/responses_api_agents/pinchbench/configs/pinchbench.yaml b/responses_api_agents/pinchbench/configs/pinchbench.yaml index e5946591b14..cd5480ce28f 100644 --- a/responses_api_agents/pinchbench/configs/pinchbench.yaml +++ b/responses_api_agents/pinchbench/configs/pinchbench.yaml @@ -2,6 +2,9 @@ pinchbench_agent: responses_api_agents: pinchbench: entrypoint: app.py + domain: agent + description: PinchBench benchmark integration + value: Evaluate a model as the brain of an OpenClaw agent on real-world tasks. # One sandbox per task (the isolation boundary, mirrors swe_agents/harbor). Each # sandbox runs its OWN in-sandbox OpenClaw gateway — the only mode (at openclaw # 2026.6.5 `openclaw agent` needs a gateway to persist transcripts).