Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
60 changes: 60 additions & 0 deletions resources_servers/aalcr/task_data.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,60 @@
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
"""Task-data schema for the aalcr server.

There is no verifier_metadata: all nine task fields ride as REQUIRED top-level row fields on
``AALCRVerifyRequest`` (app.py), so all nine stay required here even though verify() only reads
``question``, ``answer``, and ``input_tokens_band`` — the other six exist to be echoed into
``AALCRVerifyResponse``. ``input_tokens_band`` is narrowed to the five-band Literal: verify()'s
``match`` statement has no default case, so any other value crashes with an UnboundLocalError,
making the enum the server's real contract even though the wire types it as a bare ``str``.
"""

from typing import Literal

from pydantic import BaseModel, ConfigDict, Field


class TaskData(BaseModel):
model_config = ConfigDict(extra="allow")

document_category: str = Field(
description="Category of the source document set; echoed into the verify response, never graded on.",
json_schema_extra={"consumed_by": ["provenance"]},
)
document_set_id: str = Field(
description="Identifier of the document set this question is drawn from; echo-only.",
json_schema_extra={"consumed_by": ["provenance"]},
)
question_id: int = Field(
description="Numeric question identifier within the document set; echo-only.",
json_schema_extra={"consumed_by": ["provenance"]},
)
question: str = Field(
description="The long-context question; interpolated into the LLM judge prompt for reference.",
json_schema_extra={"consumed_by": ["verify"]},
)
answer: str = Field(
description="The official answer the judge compares the candidate answer against.",
json_schema_extra={"consumed_by": ["verify"]},
)
data_source_filenames: str = Field(
description="Source document filenames (single string, not a JSON list); echo-only.",
json_schema_extra={"consumed_by": ["provenance"]},
)
data_source_urls: str = Field(
description="Source document URLs (single string, not a JSON list); echo-only.",
json_schema_extra={"consumed_by": ["provenance"]},
)
input_tokens: int = Field(
description="Prompt length in tokens used to derive input_tokens_band; echo-only.",
json_schema_extra={"consumed_by": ["provenance"]},
)
input_tokens_band: Literal["<80k", "80k-100k", "100k-110k", "110k-128k", "128k+"] = Field(
description=(
"Context-length band selecting which per-band reward field (reward_lt_80k .. reward_128k_plus) "
"the row's reward is mirrored into. Wire-typed str, but verify()'s match has no default case, "
"so these five values are the de-facto enum."
),
json_schema_extra={"consumed_by": ["verify", "metrics"]},
)
28 changes: 28 additions & 0 deletions resources_servers/abstention/task_data.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
"""Task-data schema for the abstention server.

Judge-QA family heir: extends simpleqa's ``JudgeQATaskDataCore`` (optional id + question) with
the ground truth under the name ``answer`` instead of the family's ``expected_answer``. The LLM
judge grades the extracted ``\\boxed{}`` answer against it; an abstention-token match
short-circuits the judge. Required-ness mirrors ``AbstentionRunRequest`` (app.py): every task
field is Optional on the wire (verify() reads ``body.answer or ""`` / ``body.question or ""``)
with ``extra="allow"``.
"""

from typing import Optional

from pydantic import Field

from resources_servers.simpleqa.task_data import JudgeQATaskDataCore


class TaskData(JudgeQATaskDataCore):
answer: Optional[str] = Field(
default=None,
description=(
"Ground-truth answer the LLM judge grades the extracted \\boxed{} answer against; verify() "
"falls back to '' when absent. Family twin of simpleqa's expected_answer under another name."
),
json_schema_extra={"consumed_by": ["verify"]},
)
39 changes: 39 additions & 0 deletions resources_servers/arc_agi/task_data.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
"""Task-data schema for the arc_agi server.

Task fields ride at the row top level (no verifier_metadata). All four fields carry permissive
defaults on the wire (ARCAGIRunRequest), so none is required here. verify() reads only
``expected_output`` (exact grid equality against the parsed ``\\boxed{[[...]]}`` answer);
``train``/``test_input`` exist because the same fields seed the prompt. nvarc's TaskData
subclasses this model.
"""

from typing import Any, Dict, List, Optional

from pydantic import BaseModel, ConfigDict, Field


class TaskData(BaseModel):
model_config = ConfigDict(extra="allow")

train: List[Dict[str, Any]] = Field(
default_factory=list,
description="Demonstration pairs [{'input': grid, 'output': grid}, ...] used to build the prompt.",
json_schema_extra={"consumed_by": ["prompt"]},
)
test_input: List[List[int]] = Field(
default_factory=list,
description="Test grid the model must transform; prompt-side only for this server.",
json_schema_extra={"consumed_by": ["prompt"]},
)
expected_output: List[List[int]] = Field(
default_factory=list,
description="Ground-truth output grid; verify() checks exact equality against the extracted grid.",
json_schema_extra={"consumed_by": ["verify"]},
)
task_id: Optional[str] = Field(
default=None,
description="Upstream ARC-AGI task identifier.",
json_schema_extra={"consumed_by": ["provenance"]},
)
45 changes: 45 additions & 0 deletions resources_servers/arena_judge/task_data.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
"""Task-data schema for the arena_judge server.

Rows produced by ``benchmarks/arena_hard_v2/prepare.py`` carry the task fields at the top level;
there is no verifier_metadata. Mirrors ``ArenaJudgeRunRequest`` (app.py): every field is
Optional[str] with ``extra="allow"``, and verify() reads all of them defensively
(``body.question or ""``, ``body.category or config.default_category``). ``category`` is a
config-driven enum in practice (the keys of ``judge_prompt_paths``, e.g. ``hard_prompt`` /
``creative_writing``) with a logged fallback to the default for unknown values, so it stays a
plain ``str`` here.
"""

from typing import Optional

from pydantic import BaseModel, ConfigDict, Field


class TaskData(BaseModel):
model_config = ConfigDict(extra="allow")

question: Optional[str] = Field(
default=None,
description="The arena prompt both answers respond to; interpolated into the pairwise judge prompts.",
json_schema_extra={"consumed_by": ["verify"]},
)
baseline_answer: Optional[str] = Field(
default=None,
description="The frozen baseline model's answer the candidate is judged against (both A/B orderings).",
json_schema_extra={"consumed_by": ["verify"]},
)
category: Optional[str] = Field(
default=None,
description=(
"Arena-hard-v2 category selecting the judge prompt (config judge_prompt_paths key, e.g. "
"'hard_prompt', 'creative_writing'); unknown or missing values fall back to config.default_category. "
"Also grouped on by compute_metrics for per-category Elo."
),
json_schema_extra={"consumed_by": ["verify", "metrics"]},
)
uid: Optional[str] = Field(
default=None,
description="Stable task identifier from the source benchmark; never read by verify().",
json_schema_extra={"consumed_by": ["provenance"]},
)
66 changes: 66 additions & 0 deletions resources_servers/asr_with_pc/task_data.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,66 @@
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
"""Task-data schema for the asr_with_pc server.

Task fields ride at the top level; there is no verifier_metadata. Mirrors
``ASRWithPCVerifyRequest`` (app.py): one model covers all four task families because the
``task_type`` discriminator is itself Optional (a missing value falls back to the server
config's default), so a discriminated union cannot dispatch on it.

Deliberately OPEN: ``ASR_LEADERBOARD`` rows carry DYNAMIC sibling reference columns named by
``reference_fields`` (e.g. ``text_tn``, ``text_itn``) that verify() fetches via
``body.model_dump()`` and hard-fails on if missing, and compute_metrics() reads the matching
``text_<suffix>`` keys back off rollout dicts. That column set cannot be closed by a schema —
the extra columns flow through ``extra="allow"`` and are consumed by verify/metrics despite
being undeclared here.
"""

from typing import List, Literal, Optional

from pydantic import BaseModel, ConfigDict, Field


class TaskData(BaseModel):
model_config = ConfigDict(extra="allow")

expected_answer: str = Field(
default="",
description="Reference transcript the model transcription is scored against (WER/PER).",
json_schema_extra={"consumed_by": ["verify"]},
)
sample_id: Optional[str] = Field(
default=None,
description="Source-dataset sample identifier; never read by verify().",
json_schema_extra={"consumed_by": ["provenance"]},
)
split: Optional[str] = Field(
default=None,
description="Source-dataset split name (e.g. 'test'); never read by verify().",
json_schema_extra={"consumed_by": ["provenance"]},
)
task_type: Optional[Literal["ASR-PC", "ASR", "Hallucination", "ASR_LEADERBOARD"]] = Field(
default=None,
description=(
"Per-row scoring-family override; when absent the server config's task_type (default 'ASR-PC') "
"applies. 'Hallucination' rows need audio_duration; 'ASR_LEADERBOARD' rows need reference_fields."
),
json_schema_extra={"consumed_by": ["verify"]},
)
audio_duration: Optional[float] = Field(
default=None,
description=(
"Audio clip length in seconds, used by 'Hallucination' rows to compute characters-per-minute; "
"a Hallucination row without it is scored as error='missing_audio_duration'."
),
json_schema_extra={"consumed_by": ["verify"]},
)
reference_fields: Optional[List[str]] = Field(
default=None,
description=(
"For 'ASR_LEADERBOARD' rows: names of sibling row columns holding alternate reference "
"normalizations (e.g. ['text_tn', 'text_itn']). Each named column MUST exist on the row "
"(verify() raises ValueError if absent) and yields per-suffix wer_<suffix>/is_correct_<suffix> "
"scores aggregated by compute_metrics()."
),
json_schema_extra={"consumed_by": ["verify", "metrics"]},
)
24 changes: 24 additions & 0 deletions resources_servers/aviary/task_data.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
"""Task-data schema for the aviary server family.

Stateful environment: the row is a pointer whose only task datum is ``task_idx``, an integer
index into an aviary ``TaskDataset`` supplied OUT of the row by the concrete subclass server
(the aviary_bbh / aviary_bixbench / aviary_gsm8k / aviary_hotpotqa environments). Reward
accumulates server-side across /step calls keyed by env_id; verify() reads that state, never the
row. toolsandbox's TaskData subclasses this model.
"""

from pydantic import BaseModel, ConfigDict, Field


class TaskData(BaseModel):
model_config = ConfigDict(extra="allow")

task_idx: int = Field(
description=(
"Index into the server's task registry. Wire-required by AviarySeedSessionRequest and consumed "
"by /seed_session to instantiate the episode (initial observations + tools); not read by verify()."
),
json_schema_extra={"consumed_by": ["prompt"]},
)
Loading
Loading